Add era file support (#3883)

* Add support for era files

* update era comments

* spelling

* era: clean up switch statement

---------

Co-authored-by: ScottyPoi <[email protected]>

config/cspell-ts.json

@@ -24,6 +24,7 @@
     }
   ],
   "words": [
+    "bytestring",
     "binarytree",
     "merkelize",
     "kaust",
@@ -634,4 +635,4 @@
     "bytevector",
     "blobschedule"
   ]
-}
+}

packages/era/src/e2store.ts

@@ -3,7 +3,7 @@ import { bigInt64ToBytes, bytesToHex, concatBytes, equalsBytes } from '@ethereum
 import { uint256 } from 'micro-eth-signer/ssz'
 
 import { compressData, decompressData } from './snappy.js'
-import { Era1Types } from './types.js'
+import { Era1Types, EraTypes } from './types.js'
 
 import type { e2StoreEntry } from './types.js'
 
@@ -14,19 +14,18 @@ export async function parseEntry(entry: e2StoreEntry) {
   const decompressed = await decompressData(entry.data)
   let data
   switch (bytesToHex(entry.type)) {
-    case bytesToHex(Era1Types.CompressedHeader):
-      data = RLP.decode(decompressed)
-      break
     case bytesToHex(Era1Types.CompressedBody): {
       const [txs, uncles, withdrawals] = RLP.decode(decompressed)
       data = { txs, uncles, withdrawals }
       break
     }
+    case bytesToHex(Era1Types.CompressedHeader):
     case bytesToHex(Era1Types.CompressedReceipts):
-      data = decompressed
       data = RLP.decode(decompressed)
       break
     case bytesToHex(Era1Types.AccumulatorRoot):
+    case bytesToHex(EraTypes.CompressedBeaconState):
+    case bytesToHex(EraTypes.CompressedSignedBeaconBlockType):
       data = decompressed
       break
     default:

packages/era/src/era.ts

@@ -0,0 +1,115 @@
+import { equalsBytes } from '@ethereumjs/util'
+import * as ssz from 'micro-eth-signer/ssz'
+
+import { EraTypes, parseEntry, readEntry } from './index.js'
+
+import type { SlotIndex } from './index.js'
+
+/**
+ * Reads a Slot Index from the end of a bytestring representing an era file
+ * @param bytes a Uint8Array bytestring representing a {@link SlotIndex} plus any arbitrary prefixed data
+ * @returns a deserialized {@link SlotIndex}
+ */
+export const readSlotIndex = (bytes: Uint8Array): SlotIndex => {
+  const recordEnd = bytes.length
+  const countBytes = bytes.slice(recordEnd - 8)
+  const count = Number(new DataView(countBytes.buffer).getBigInt64(0, true))
+  const recordStart = recordEnd - (8 * count + 24)
+  const slotIndexEntry = readEntry(bytes.subarray(recordStart, recordEnd))
+  if (equalsBytes(slotIndexEntry.type, EraTypes.SlotIndex) === false) {
+    throw new Error(`expected SlotIndex type, got ${slotIndexEntry.type}`)
+  }
+
+  const startSlot = Number(
+    new DataView(slotIndexEntry.data.slice(0, 8).buffer).getBigInt64(0, true),
+  )
+  const slotOffsets = []
+
+  for (let i = 0; i < count; i++) {
+    const slotEntry = slotIndexEntry.data.subarray((i + 1) * 8, (i + 2) * 8)
+    const slotOffset = Number(new DataView(slotEntry.slice(0, 8).buffer).getBigInt64(0, true))
+    slotOffsets.push(slotOffset)
+  }
+  return {
+    startSlot,
+    recordStart,
+    slotOffsets,
+  }
+}
+
+/**
+ * Reads a an era file and extracts the State and Block slot indices
+ * @param eraContents a bytestring representing a serialized era file
+ * @returns a dictionary containing the State and Block Slot Indices (if present)
+ */
+export const getEraIndexes = (
+  eraContents: Uint8Array,
+): { stateSlotIndex: SlotIndex; blockSlotIndex: SlotIndex | undefined } => {
+  const stateSlotIndex = readSlotIndex(eraContents)
+  let blockSlotIndex = undefined
+  if (stateSlotIndex.startSlot > 0) {
+    blockSlotIndex = readSlotIndex(eraContents.slice(0, stateSlotIndex.recordStart))
+  }
+  return { stateSlotIndex, blockSlotIndex }
+}
+
+/**
+ *
+ * @param eraData a bytestring representing an era file
+ * @returns a BeaconState object of the same time as returned by {@link ssz.ETH2_TYPES.BeaconState}
+ * @throws if BeaconState cannot be found
+ */
+export const readBeaconState = async (eraData: Uint8Array) => {
+  const indices = getEraIndexes(eraData)
+  const stateEntry = readEntry(
+    eraData.slice(indices.stateSlotIndex.recordStart + indices.stateSlotIndex.slotOffsets[0]),
+  )
+  const data = await parseEntry(stateEntry)
+  if (equalsBytes(stateEntry.type, EraTypes.CompressedBeaconState) === false) {
+    throw new Error(`expected CompressedBeaconState type, got ${stateEntry.type}`)
+  }
+  return ssz.ETH2_TYPES.BeaconState.decode(data.data as Uint8Array)
+}
+
+/**
+ *
+ * @param eraData a bytestring representing an era file
+ * @returns a decompressed SignedBeaconBlock object of the same time as returned by {@link ssz.ETH2_TYPES.SignedBeaconBlock}
+ * @throws if SignedBeaconBlock cannot be found
+ */
+export const readBeaconBlock = async (eraData: Uint8Array, offset: number) => {
+  const indices = getEraIndexes(eraData)
+  const blockEntry = readEntry(
+    eraData.slice(
+      indices.blockSlotIndex!.recordStart + indices.blockSlotIndex!.slotOffsets[offset],
+    ),
+  )
+  const data = await parseEntry(blockEntry)
+  if (equalsBytes(blockEntry.type, EraTypes.CompressedSignedBeaconBlockType) === false) {
+    throw new Error(`expected CompressedSignedBeaconBlockType type, got ${blockEntry.type}`)
+  }
+  return ssz.ETH2_TYPES.SignedBeaconBlock.decode(data.data as Uint8Array)
+}
+
+/**
+ * Reads a an era file and yields a stream of decompressed SignedBeaconBlocks
+ * @param eraFile Uint8Array a serialized era file
+ * @returns a stream of decompressed SignedBeaconBlocks or undefined if no blocks are present
+ */
+export async function* readBlocksFromEra(eraFile: Uint8Array) {
+  const indices = getEraIndexes(eraFile)
+  const maxBlocks = indices.blockSlotIndex?.slotOffsets.length
+  if (maxBlocks === undefined) {
+    // Return early if no blocks are present
+    return
+  }
+
+  for (let x = 0; x < maxBlocks; x++) {
+    try {
+      const block = await readBeaconBlock(eraFile, x)
+      yield block
+    } catch {
+      // noop - we skip empty slots
+    }
+  }
+}

packages/era/src/types.ts

@@ -5,6 +5,7 @@ export type e2StoreEntry = {
   data: Uint8Array
 }
 
+/** Era 1 Type Identifiers */
 export const Era1Types = {
   Version: new Uint8Array([0x65, 0x32]),
   CompressedHeader: new Uint8Array([0x03, 0x00]),
@@ -20,9 +21,24 @@ export const VERSION = {
   data: new Uint8Array([]),
 }
 
+/** Era1 SSZ containers */
 export const HeaderRecord = ssz.container({
   blockHash: ssz.bytevector(32),
   totalDifficulty: ssz.uint256,
 })
 
 export const EpochAccumulator = ssz.list(8192, HeaderRecord)
+
+/** Era Type Identifiers */
+export const EraTypes = {
+  CompressedSignedBeaconBlockType: new Uint8Array([0x01, 0x00]),
+  CompressedBeaconState: new Uint8Array([0x02, 0x00]),
+  Empty: new Uint8Array([0x00, 0x00]),
+  SlotIndex: new Uint8Array([0x69, 0x32]),
+}
+
+export type SlotIndex = {
+  startSlot: number
+  recordStart: number
+  slotOffsets: number[]
+}

packages/era/test/era.spec.ts

@@ -0,0 +1,40 @@
+import { readFileSync } from 'fs'
+import { assert, beforeAll, describe, it } from 'vitest'
+
+import { readBeaconBlock, readBeaconState, readBlocksFromEra, readSlotIndex } from '../src/era.js'
+import { readBinaryFile } from '../src/index.js'
+
+// To test this, download mainnet-01339-75d1c621.era from https://mainnet.era.nimbus.team/mainnet-01339-75d1c621.era
+// This era file is around 500mb in size so don't commit it to the repo
+describe.skip('it should be able to extract beacon objects from an era file', () => {
+  let data: Uint8Array
+  beforeAll(() => {
+    data = readBinaryFile(__dirname + '/mainnet-01339-75d1c621.era')
+  })
+  it('should read a slot index from the era file', async () => {
+    const slotIndex = readSlotIndex(data)
+    assert.equal(slotIndex.startSlot, 10969088)
+  })
+  it('should extract the beacon state', async () => {
+    const state = await readBeaconState(data)
+    assert.equal(Number(state.slot), 10969088)
+  }, 30000)
+  it('should read a block from the era file and decompress it', async () => {
+    const block = await readBeaconBlock(data, 0)
+    assert.equal(Number(block.message.slot), 10960896)
+  })
+  it('read blocks from an era file', async () => {
+    let count = 0
+    for await (const block of readBlocksFromEra(data)) {
+      assert.exists(block.message.slot)
+      count++
+      if (count > 10) break
+    }
+  }, 30000)
+  it('reads no blocks from the genesis era file', async () => {
+    const data = new Uint8Array(readFileSync(__dirname + '/mainnet-00000-4b363db9.era'))
+    for await (const block of readBlocksFromEra(data)) {
+      assert.equal(block, undefined)
+    }
+  })
+})