From 2b3d8708648d4ed2fafbd3a9d3d7b6c0f4b5e264 Mon Sep 17 00:00:00 2001 From: harkamal Date: Mon, 29 May 2023 13:30:01 +0530 Subject: [PATCH 1/2] feat: move req/resp to blob_sidecars_by_range/root methods --- .../src/api/impl/beacon/blocks/index.ts | 24 ++-- .../beacon-node/src/chain/blocks/types.ts | 21 ++++ packages/beacon-node/src/network/interface.ts | 10 +- packages/beacon-node/src/network/network.ts | 20 +-- .../src/network/reqresp/ReqRespBeaconNode.ts | 7 +- .../reqresp/beaconBlocksMaybeBlobsByRange.ts | 119 ++++++++++-------- .../reqresp/beaconBlocksMaybeBlobsByRoot.ts | 89 ++++--------- .../beaconBlockAndBlobsSidecarByRoot.ts | 78 ------------ .../reqresp/handlers/beaconBlocksByRange.ts | 10 +- .../reqresp/handlers/blobSidecarsByRange.ts | 108 ++++++++++++++++ .../reqresp/handlers/blobSidecarsByRoot.ts | 59 +++++++++ .../reqresp/handlers/blobsSidecarsByRange.ts | 18 --- .../src/network/reqresp/handlers/index.ts | 16 +-- .../src/network/reqresp/protocols.ts | 16 +-- .../src/network/reqresp/rateLimit.ts | 8 +- .../beacon-node/src/network/reqresp/types.ts | 20 +-- .../test/e2e/network/deneb.test.ts | 29 ----- 17 files changed, 340 insertions(+), 312 deletions(-) delete mode 100644 packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts create mode 100644 packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts create mode 100644 packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts delete mode 100644 packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts delete mode 100644 packages/beacon-node/test/e2e/network/deneb.test.ts diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 49273a96981a..7700a7dfa73e 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -10,14 +10,19 @@ import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; import {allForks, deneb} from "@lodestar/types"; import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js"; +import { + BlockSource, + getBlockInput, + ImportBlockOpts, + BlockInput, + blobSidecarsToBlobsSidecar, +} from "../../../../chain/blocks/types.js"; import {promiseAllMaybeAsync} from "../../../../util/promises.js"; import {isOptimisticBlock} from "../../../../util/forkChoice.js"; import {BlockError, BlockErrorCode} from "../../../../chain/errors/index.js"; import {OpSource} from "../../../../metrics/validatorMonitor.js"; import {NetworkEvent} from "../../../../network/index.js"; import {ApiModules} from "../../types.js"; -import {ckzg} from "../../../../util/kzg.js"; import {resolveBlockId, toBeaconHeaderResponse} from "./utils.js"; /** @@ -213,21 +218,18 @@ export function getBeaconBlockApi({ if (isSignedBlockContents(signedBlockOrContents)) { // Build a blockInput for post deneb, signedBlobs will be be used in followup PRs ({signedBlock, signedBlobSidecars: signedBlobs} = signedBlockOrContents as SignedBlockContents); - const beaconBlockSlot = signedBlock.message.slot; - const beaconBlockRoot = config.getForkTypes(beaconBlockSlot).BeaconBlock.hashTreeRoot(signedBlock.message); - const blobs = signedBlobs.map((sblob) => sblob.message.blob); + const blobsSidecar = blobSidecarsToBlobsSidecar( + config, + signedBlock, + signedBlobs.map(({message}) => message) + ); blockForImport = getBlockInput.postDeneb( config, signedBlock, BlockSource.api, // The blobsSidecar will be replaced in the followup PRs with just blobs - { - beaconBlockRoot, - beaconBlockSlot, - blobs, - kzgAggregatedProof: ckzg.computeAggregateKzgProof(blobs), - } + blobsSidecar ); } else { signedBlock = signedBlockOrContents as allForks.SignedBeaconBlock; diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 1663ae6c3d18..2b1c521000ef 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -4,6 +4,8 @@ import {allForks, deneb, Slot, WithOptionalBytes} from "@lodestar/types"; import {ForkSeq, MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; +import {ckzg} from "../../util/kzg.js"; + export enum BlockInputType { preDeneb = "preDeneb", postDeneb = "postDeneb", @@ -29,6 +31,25 @@ export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clo ); } +// TODO DENEB: a helper function to convert blobSidecars to blobsSidecar, to be cleanup on BlockInput +// migration +export function blobSidecarsToBlobsSidecar( + config: ChainForkConfig, + signedBlock: allForks.SignedBeaconBlock, + blobSidecars: deneb.BlobSidecars +): deneb.BlobsSidecar { + const beaconBlockSlot = signedBlock.message.slot; + const beaconBlockRoot = config.getForkTypes(beaconBlockSlot).BeaconBlock.hashTreeRoot(signedBlock.message); + const blobs = blobSidecars.map(({blob}) => blob); + const blobsSidecar = { + beaconBlockRoot, + beaconBlockSlot, + blobs, + kzgAggregatedProof: ckzg.computeAggregateKzgProof(blobs), + }; + return blobsSidecar; +} + export const getBlockInput = { preDeneb(config: ChainForkConfig, block: allForks.SignedBeaconBlock, source: BlockSource): BlockInput { if (config.getForkSeq(block.message.slot) >= ForkSeq.deneb) { diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index be232adea788..0522526c3350 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -39,14 +39,8 @@ export interface INetwork extends INetworkCorePublic { peerId: PeerIdStr, request: phase0.BeaconBlocksByRootRequest ): Promise; - sendBlobsSidecarsByRange( - peerId: PeerIdStr, - request: deneb.BlobsSidecarsByRangeRequest - ): Promise; - sendBeaconBlockAndBlobsSidecarByRoot( - peerId: PeerIdStr, - request: deneb.BeaconBlockAndBlobsSidecarByRootRequest - ): Promise; + sendBlobSidecarsByRange(peerId: PeerIdStr, request: deneb.BlobSidecarsByRangeRequest): Promise; + sendBlobSidecarsByRoot(peerId: PeerIdStr, request: deneb.BlobSidecarsByRootRequest): Promise; // Gossip publishBeaconBlockMaybeBlobs(blockInput: BlockInput): Promise; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index c2191fd7f1d3..5eb04b9b29ed 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -466,25 +466,25 @@ export class Network implements INetwork { ); } - async sendBlobsSidecarsByRange( + async sendBlobSidecarsByRange( peerId: PeerIdStr, - request: deneb.BlobsSidecarsByRangeRequest - ): Promise { + request: deneb.BlobSidecarsByRangeRequest + ): Promise { return collectMaxResponseTyped( - this.sendReqRespRequest(peerId, ReqRespMethod.BlobsSidecarsByRange, [Version.V1], request), + this.sendReqRespRequest(peerId, ReqRespMethod.BlobSidecarsByRange, [Version.V1], request), request.count, - responseSszTypeByMethod[ReqRespMethod.BlobsSidecarsByRange] + responseSszTypeByMethod[ReqRespMethod.BlobSidecarsByRange] ); } - async sendBeaconBlockAndBlobsSidecarByRoot( + async sendBlobSidecarsByRoot( peerId: PeerIdStr, - request: deneb.BeaconBlockAndBlobsSidecarByRootRequest - ): Promise { + request: deneb.BlobSidecarsByRootRequest + ): Promise { return collectMaxResponseTyped( - this.sendReqRespRequest(peerId, ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot, [Version.V1], request), + this.sendReqRespRequest(peerId, ReqRespMethod.BlobSidecarsByRoot, [Version.V1], request), request.length, - responseSszTypeByMethod[ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot] + responseSszTypeByMethod[ReqRespMethod.BlobSidecarsByRoot] ); } diff --git a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts index b547c49e0838..bf45c1239bf6 100644 --- a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts +++ b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts @@ -247,11 +247,8 @@ export class ReqRespBeaconNode extends ReqResp { if (ForkSeq[fork] >= ForkSeq.deneb) { protocolsAtFork.push( - [ - protocols.BeaconBlockAndBlobsSidecarByRoot(this.config), - this.getHandler(ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot), - ], - [protocols.BlobsSidecarsByRange(this.config), this.getHandler(ReqRespMethod.BlobsSidecarsByRange)] + [protocols.BlobSidecarsByRoot(this.config), this.getHandler(ReqRespMethod.BlobSidecarsByRoot)], + [protocols.BlobSidecarsByRange(this.config), this.getHandler(ReqRespMethod.BlobSidecarsByRange)] ); } diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts index eaf2b46dce41..c254127f5a98 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts @@ -1,15 +1,14 @@ -import {BeaconConfig} from "@lodestar/config"; -import {deneb, Epoch, phase0} from "@lodestar/types"; +import {ChainForkConfig} from "@lodestar/config"; +import {deneb, Epoch, phase0, allForks, Slot} from "@lodestar/types"; import {ForkSeq, MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS} from "@lodestar/params"; import {computeEpochAtSlot} from "@lodestar/state-transition"; -import {BlockInput, BlockSource, getBlockInput} from "../../chain/blocks/types.js"; -import {getEmptyBlobsSidecar} from "../../util/blobs.js"; +import {BlockInput, BlockSource, getBlockInput, blobSidecarsToBlobsSidecar} from "../../chain/blocks/types.js"; import {PeerIdStr} from "../../util/peerId.js"; import {INetwork} from "../interface.js"; export async function beaconBlocksMaybeBlobsByRange( - config: BeaconConfig, + config: ChainForkConfig, network: INetwork, peerId: PeerIdStr, request: phase0.BeaconBlocksByRangeRequest, @@ -39,63 +38,81 @@ export async function beaconBlocksMaybeBlobsByRange( // Only request blobs if they are recent enough else if (computeEpochAtSlot(startSlot) >= currentEpoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS) { - const [blocks, blobsSidecars] = await Promise.all([ + const [allBlocks, allBlobSidecars] = await Promise.all([ network.sendBeaconBlocksByRange(peerId, request), - network.sendBlobsSidecarsByRange(peerId, request), + network.sendBlobSidecarsByRange(peerId, request), ]); - const blockInputs: BlockInput[] = []; - let blobSideCarIndex = 0; - let lastMatchedSlot = -1; + return matchBlockWithBlobs(config, allBlocks, allBlobSidecars, endSlot, BlockSource.byRange); + } + + // Post Deneb but old blobs + else { + throw Error("Cannot sync blobs outside of blobs prune window"); + } +} + +// Assumes that the blobs are in the same sequence as blocks, doesn't require block to be sorted +export function matchBlockWithBlobs( + config: ChainForkConfig, + allBlocks: allForks.SignedBeaconBlock[], + allBlobSidecars: deneb.BlobSidecar[], + endSlot: Slot, + blockSource: BlockSource +): BlockInput[] { + const blockInputs: BlockInput[] = []; + let blobSideCarIndex = 0; + let lastMatchedSlot = -1; - // Match blobSideCar with the block as some blocks would have no blobs and hence - // would be omitted from the response. If there are any inconsitencies in the - // response, the validations during import will reject the block and hence this - // entire segment. - // - // Assuming that the blocks and blobs will come in same sorted order - for (let i = 0; i < blocks.length; i++) { - const block = blocks[i]; - let blobsSidecar: deneb.BlobsSidecar; + // Match blobSideCar with the block as some blocks would have no blobs and hence + // would be omitted from the response. If there are any inconsitencies in the + // response, the validations during import will reject the block and hence this + // entire segment. + // + // Assuming that the blocks and blobs will come in same sorted order + for (let i = 0; i < allBlocks.length; i++) { + const block = allBlocks[i]; + if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) { + blockInputs.push(getBlockInput.preDeneb(config, block, blockSource)); + } else { + const blobSidecars: deneb.BlobSidecar[] = []; - if (blobsSidecars[blobSideCarIndex]?.beaconBlockSlot === block.message.slot) { - blobsSidecar = blobsSidecars[blobSideCarIndex]; + let blobSidecar: deneb.BlobSidecar; + while ((blobSidecar = allBlobSidecars[blobSideCarIndex])?.slot === block.message.slot) { + blobSidecars.push(blobSidecar); lastMatchedSlot = block.message.slot; blobSideCarIndex++; - } else { - // Quick inspect if the blobsSidecar was expected - const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; - if (blobKzgCommitmentsLen !== 0) { - throw Error( - `Missing blobsSidecar for blockSlot=${block.message.slot} with blobKzgCommitmentsLen=${blobKzgCommitmentsLen}` - ); - } - blobsSidecar = getEmptyBlobsSidecar(config, block as deneb.SignedBeaconBlock); } - blockInputs.push(getBlockInput.postDeneb(config, block, BlockSource.byRange, blobsSidecar)); - } - // If there are still unconsumed blobs this means that the response was inconsistent - // and matching was wrong and hence we should throw error - if ( - blobsSidecars[blobSideCarIndex] !== undefined && - // If there are no blobs, the blobs request can give 1 block outside the requested range - blobsSidecars[blobSideCarIndex].beaconBlockSlot <= endSlot - ) { - throw Error( - `Unmatched blobsSidecars, blocks=${blocks.length}, blobs=${ - blobsSidecars.length - } lastMatchedSlot=${lastMatchedSlot}, pending blobsSidecars slots=${blobsSidecars - .slice(blobSideCarIndex) - .map((blb) => blb.beaconBlockSlot) - .join(",")}` - ); + // Quick inspect how many blobSidecars was expected + const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; + if (blobKzgCommitmentsLen !== blobSidecars.length) { + throw Error( + `Missing blobSidecars for blockSlot=${block.message.slot} with blobKzgCommitmentsLen=${blobKzgCommitmentsLen} blobSidecars=${blobSidecars.length}` + ); + } + + // TODO DENEB: cleanup blobSidecars to blobsSidecar conversion on migration of blockInput + const blobsSidecar = blobSidecarsToBlobsSidecar(config, block, blobSidecars); + blockInputs.push(getBlockInput.postDeneb(config, block, blockSource, blobsSidecar)); } - return blockInputs; } - // Post Deneb but old blobs - else { - throw Error("Cannot sync blobs outside of blobs prune window"); + // If there are still unconsumed blobs this means that the response was inconsistent + // and matching was wrong and hence we should throw error + if ( + allBlobSidecars[blobSideCarIndex] !== undefined && + // If there are no blobs, the blobs request can give 1 block outside the requested range + allBlobSidecars[blobSideCarIndex].slot <= endSlot + ) { + throw Error( + `Unmatched blobSidecars, blocks=${allBlocks.length}, blobs=${ + allBlobSidecars.length + } lastMatchedSlot=${lastMatchedSlot}, pending blobSidecars slots=${allBlobSidecars + .slice(blobSideCarIndex) + .map((blb) => blb.slot) + .join(",")}` + ); } + return blockInputs; } diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts index 93b0fc62fa4f..962ca7bb48dc 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts @@ -1,83 +1,38 @@ import {ChainForkConfig} from "@lodestar/config"; -import {RequestError, RequestErrorCode} from "@lodestar/reqresp"; -import {Epoch, phase0, Root, Slot} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; -import {ForkSeq} from "@lodestar/params"; -import {BlockInput, BlockSource, getBlockInput} from "../../chain/blocks/types.js"; -import {wrapError} from "../../util/wrapError.js"; +import {Epoch, phase0, deneb, Slot} from "@lodestar/types"; +import {BlockInput, BlockSource} from "../../chain/blocks/types.js"; import {PeerIdStr} from "../../util/peerId.js"; import {INetwork} from "../interface.js"; +import {matchBlockWithBlobs} from "./beaconBlocksMaybeBlobsByRange.js"; export async function beaconBlocksMaybeBlobsByRoot( config: ChainForkConfig, network: INetwork, peerId: PeerIdStr, request: phase0.BeaconBlocksByRootRequest, - currentSlot: Epoch, - finalizedSlot: Slot + // TODO DENEB: Some validations can be done to see if this is deneb block, ignoring below two for now + _currentSlot: Epoch, + _finalizedSlot: Slot ): Promise { - // Assume all requests are post Deneb - if (config.getForkSeq(finalizedSlot) >= ForkSeq.deneb) { - const blocksAndBlobs = await network.sendBeaconBlockAndBlobsSidecarByRoot(peerId, request); - return blocksAndBlobs.map(({beaconBlock, blobsSidecar}) => - getBlockInput.postDeneb(config, beaconBlock, BlockSource.byRoot, blobsSidecar) - ); - } - - // Assume all request are pre EIP-4844 - else if (config.getForkSeq(currentSlot) < ForkSeq.deneb) { - const blocks = await network.sendBeaconBlocksByRoot(peerId, request); - return blocks.map((block) => getBlockInput.preDeneb(config, block, BlockSource.byRoot)); - } - - // We don't know if a requested root is after the deneb fork or not. - // Thus some sort of retry is necessary while deneb is not finalized - else { - return Promise.all( - request.map(async (beaconBlockRoot) => - beaconBlockAndBlobsSidecarByRootFallback(config, network, peerId, beaconBlockRoot) - ) - ); - } -} - -async function beaconBlockAndBlobsSidecarByRootFallback( - config: ChainForkConfig, - network: INetwork, - peerId: PeerIdStr, - beaconBlockRoot: Root -): Promise { - const resBlockBlobs = await wrapError(network.sendBeaconBlockAndBlobsSidecarByRoot(peerId, [beaconBlockRoot])); - - if (resBlockBlobs.err) { - // From the spec, if the block is from before the fork, errors with 3: ResourceUnavailable - // > Clients MUST support requesting blocks and sidecars since minimum_request_epoch, where - // minimum_request_epoch = max(finalized_epoch, current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH). - // If any root in the request content references a block earlier than minimum_request_epoch, - // peers SHOULD respond with error code 3: ResourceUnavailable. - // Ref: https://github.com/ethereum/consensus-specs/blob/aede132f4999ed54b98d35e27aca9451042a1ee9/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1 - if ( - resBlockBlobs.err instanceof RequestError && - resBlockBlobs.err.type.code === RequestErrorCode.RESOURCE_UNAVAILABLE - ) { - // retry with blocks - } else { - // Unexpected error, throw - throw resBlockBlobs.err; + const allBlocks = await network.sendBeaconBlocksByRoot(peerId, request); + const blobIdentifiers: deneb.BlobIdentifier[] = []; + + for (const block of allBlocks) { + const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); + const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments?.length ?? 0; + for (let index = 0; index < blobKzgCommitmentsLen; index++) { + blobIdentifiers.push({blockRoot, index}); } - } else { - if (resBlockBlobs.result.length < 1) { - throw Error(`beaconBlockAndBlobsSidecarByRoot return empty for block root ${toHex(beaconBlockRoot)}`); - } - - const {beaconBlock, blobsSidecar} = resBlockBlobs.result[0]; - return getBlockInput.postDeneb(config, beaconBlock, BlockSource.byRoot, blobsSidecar); } - const resBlocks = await network.sendBeaconBlocksByRoot(peerId, [beaconBlockRoot]); - if (resBlocks.length < 1) { - throw Error(`beaconBlocksByRoot return empty for block root ${toHex(beaconBlockRoot)}`); + let allBlobSidecars: deneb.BlobSidecar[]; + if (blobIdentifiers.length > 0) { + allBlobSidecars = await network.sendBlobSidecarsByRoot(peerId, blobIdentifiers); + } else { + allBlobSidecars = []; } - return getBlockInput.preDeneb(config, resBlocks[0], BlockSource.byRoot); + // The last arg is to provide slot to which all blobs should be exausted in matching + // and here it should be infinity since all bobs should match + return matchBlockWithBlobs(config, allBlocks, allBlobSidecars, Infinity, BlockSource.byRoot); } diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts deleted file mode 100644 index c5038c5e2a82..000000000000 --- a/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts +++ /dev/null @@ -1,78 +0,0 @@ -import {ResponseOutgoing} from "@lodestar/reqresp"; -import {deneb} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; -import {IBeaconChain} from "../../../chain/index.js"; -import {IBeaconDb} from "../../../db/index.js"; -import {getSlotFromSignedBeaconBlockSerialized} from "../../../util/sszBytes.js"; - -export async function* onBeaconBlockAndBlobsSidecarByRoot( - requestBody: deneb.BeaconBlockAndBlobsSidecarByRootRequest, - chain: IBeaconChain, - db: IBeaconDb -): AsyncIterable { - const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot; - - for (const blockRoot of requestBody) { - const blockRootHex = toHex(blockRoot); - const summary = chain.forkChoice.getBlockHex(blockRootHex); - - // NOTE: Only support non-finalized blocks. - // SPEC: Clients MUST support requesting blocks and sidecars since the latest finalized epoch. - // https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1 - if (!summary || summary.slot <= finalizedSlot) { - // TODO: Should accept the finalized block? Is the finalized block in the archive DB or hot DB? - continue; - } - - // finalized block has summary in forkchoice but it stays in blockArchive db - const blockBytes = await db.block.getBinary(blockRoot); - if (!blockBytes) { - throw Error(`Inconsistent state, block known to fork-choice not in db ${blockRootHex}`); - } - - const blobsSidecarBytes = await db.blobsSidecar.getBinary(blockRoot); - if (!blobsSidecarBytes) { - throw Error(`Inconsistent state, blobsSidecar known to fork-choice not in db ${blockRootHex}`); - } - - const forkSlot = getSlotFromSignedBeaconBlockSerialized(blockBytes); - if (forkSlot === null) { - throw Error(`Invalid block bytes for block ${blockRootHex}`); - } - - yield { - data: signedBeaconBlockAndBlobsSidecarFromBytes(blockBytes, blobsSidecarBytes), - fork: chain.config.getForkName(forkSlot), - }; - } -} - -/** - * Construct a valid SSZ serialized container from its properties also serialized. - * ``` - * class SignedBeaconBlockAndBlobsSidecar(Container): - * beacon_block: SignedBeaconBlock - * blobs_sidecar: BlobsSidecar - * ``` - */ -export function signedBeaconBlockAndBlobsSidecarFromBytes( - blockBytes: Uint8Array, - blobsSidecarBytes: Uint8Array -): Uint8Array { - const totalLen = 4 + 4 + blockBytes.length + blobsSidecarBytes.length; - const arrayBuffer = new ArrayBuffer(totalLen); - const dataView = new DataView(arrayBuffer); - const uint8Array = new Uint8Array(arrayBuffer); - - const blockOffset = 8; - const blobsOffset = 8 + blockBytes.length; - - // Write offsets - dataView.setUint32(0, blockOffset, true); - dataView.setUint32(4, blobsOffset, true); - - uint8Array.set(blockBytes, blockOffset); - uint8Array.set(blobsSidecarBytes, blobsOffset); - - return uint8Array; -} diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts index fd5580d59021..9e621054e2fb 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts @@ -12,14 +12,14 @@ export function onBeaconBlocksByRange( chain: IBeaconChain, db: IBeaconDb ): AsyncIterable { - return onBlocksOrBlobsSidecarsByRange(request, chain, { + return onBlocksOrBlobSidecarsByRange(request, chain, { finalized: db.blockArchive, unfinalized: db.block, }); } -export async function* onBlocksOrBlobsSidecarsByRange( - request: deneb.BlobsSidecarsByRangeRequest, +export async function* onBlocksOrBlobSidecarsByRange( + request: phase0.BeaconBlocksByRangeRequest, chain: IBeaconChain, db: { finalized: Pick; @@ -88,8 +88,8 @@ export async function* onBlocksOrBlobsSidecarsByRange( } export function validateBeaconBlocksByRangeRequest( - request: deneb.BlobsSidecarsByRangeRequest -): deneb.BlobsSidecarsByRangeRequest { + request: deneb.BlobSidecarsByRangeRequest +): deneb.BlobSidecarsByRangeRequest { const {startSlot} = request; let {count} = request; diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts new file mode 100644 index 000000000000..633b0ad42d6a --- /dev/null +++ b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts @@ -0,0 +1,108 @@ +import {GENESIS_SLOT, MAX_REQUEST_BLOCKS_DENEB} from "@lodestar/params"; +import {ResponseError, ResponseOutgoing, RespStatus} from "@lodestar/reqresp"; +import {deneb, Slot} from "@lodestar/types"; +import {fromHex} from "@lodestar/utils"; +import {IBeaconChain} from "../../../chain/index.js"; +import {IBeaconDb} from "../../../db/index.js"; +import {BLOB_SIDECARS_IN_WRAPPER_INDEX, BLOBSIDECAR_FIXED_SIZE} from "../../../db/repositories/blobSidecars.js"; + +export async function* onBlobSidecarsByRange( + request: deneb.BlobSidecarsByRangeRequest, + chain: IBeaconChain, + db: IBeaconDb +): AsyncIterable { + // Non-finalized range of blobs + const {startSlot, count} = validateBlobSidecarsByRangeRequest(request); + const endSlot = startSlot + count; + const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot; + + // Finalized range of blobs + + if (startSlot <= finalizedSlot) { + // Chain of blobs won't change + for await (const {key, value: blobSideCarsBytesWrapped} of db.blobSidecarsArchive.binaryEntriesStream({ + gte: startSlot, + lt: endSlot, + })) { + yield* iterateBlobBytesFromWrapper(chain, blobSideCarsBytesWrapped, db.blobSidecarsArchive.decodeKey(key)); + } + } + + if (endSlot > finalizedSlot) { + const headRoot = chain.forkChoice.getHeadRoot(); + // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot); + + // Iterate head chain with ascending block numbers + for (let i = headChain.length - 1; i >= 0; i--) { + const block = headChain[i]; + + // Must include only blobs in the range requested + if (block.slot >= startSlot && block.slot < endSlot) { + // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the cannonical chain + // at the time of the start of the request. Spec is clear the chain of blobs must be consistent, but on + // re-org there's no need to abort the request + // Spec: https://github.com/ethereum/consensus-specs/blob/a1e46d1ae47dd9d097725801575b46907c12a1f8/specs/eip4844/p2p-interface.md#blobssidecarsbyrange-v1 + + const blobSideCarsBytesWrapped = await db.blobSidecars.getBinary(fromHex(block.blockRoot)); + if (!blobSideCarsBytesWrapped) { + // Handle the same to onBeaconBlocksByRange + throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`); + } + yield* iterateBlobBytesFromWrapper(chain, blobSideCarsBytesWrapped, block.slot); + } + + // If block is after endSlot, stop iterating + else if (block.slot >= endSlot) { + break; + } + } + } +} + +export function* iterateBlobBytesFromWrapper( + chain: IBeaconChain, + blobSideCarsBytesWrapped: Uint8Array, + blockSlot: Slot +): Iterable { + const blobSideCarsBytes = blobSideCarsBytesWrapped.slice(BLOB_SIDECARS_IN_WRAPPER_INDEX); + const blobsLen = blobSideCarsBytes.length / BLOBSIDECAR_FIXED_SIZE; + + for (let index = 0; index < blobsLen; index++) { + const blobSideCarBytes = blobSideCarsBytes.slice( + index * BLOBSIDECAR_FIXED_SIZE, + (index + 1) * BLOBSIDECAR_FIXED_SIZE + ); + if (blobSideCarBytes.length !== BLOBSIDECAR_FIXED_SIZE) { + throw new ResponseError( + RespStatus.SERVER_ERROR, + `Invalid blobSidecar index=${index} bytes length=${blobSideCarBytes.length} expected=${BLOBSIDECAR_FIXED_SIZE} for slot ${blockSlot} blobsLen=${blobsLen}` + ); + } + yield { + data: blobSideCarsBytes, + fork: chain.config.getForkName(blockSlot), + }; + } +} + +export function validateBlobSidecarsByRangeRequest( + request: deneb.BlobSidecarsByRangeRequest +): deneb.BlobSidecarsByRangeRequest { + const {startSlot} = request; + let {count} = request; + + if (count < 1) { + throw new ResponseError(RespStatus.INVALID_REQUEST, "count < 1"); + } + // TODO: validate against MIN_EPOCHS_FOR_BLOCK_REQUESTS + if (startSlot < GENESIS_SLOT) { + throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis"); + } + + if (count > MAX_REQUEST_BLOCKS_DENEB) { + count = MAX_REQUEST_BLOCKS_DENEB; + } + + return {startSlot, count}; +} diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts new file mode 100644 index 000000000000..3bb162d019e3 --- /dev/null +++ b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts @@ -0,0 +1,59 @@ +import {ResponseError, ResponseOutgoing, RespStatus} from "@lodestar/reqresp"; +import {deneb, RootHex} from "@lodestar/types"; +import {toHex, fromHex} from "@lodestar/utils"; +import {IBeaconChain} from "../../../chain/index.js"; +import {IBeaconDb} from "../../../db/index.js"; +import {BLOB_SIDECARS_IN_WRAPPER_INDEX, BLOBSIDECAR_FIXED_SIZE} from "../../../db/repositories/blobSidecars.js"; + +export async function* onBlobSidecarsByRoot( + requestBody: deneb.BlobSidecarsByRootRequest, + chain: IBeaconChain, + db: IBeaconDb +): AsyncIterable { + const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot; + + // In sidecars by root request, it can be expected that sidecar requests will be come + // clustured by blockroots, and this helps us save db lookups once we load sidecars + // for a root + let lastFetchedSideCars: {blockRoot: RootHex; bytes: Uint8Array} | null = null; + + for (const blobIdentifier of requestBody) { + const {blockRoot, index} = blobIdentifier; + const blockRootHex = toHex(blockRoot); + const block = chain.forkChoice.getBlockHex(blockRootHex); + + // NOTE: Only support non-finalized blocks. + // SPEC: Clients MUST support requesting blocks and sidecars since the latest finalized epoch. + // https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1 + if (!block || block.slot <= finalizedSlot) { + continue; + } + + // Check if we need to load sidecars for a new block root + if (lastFetchedSideCars === null || lastFetchedSideCars.blockRoot !== blockRootHex) { + const blobSideCarsBytesWrapped = await db.blobSidecars.getBinary(fromHex(block.blockRoot)); + if (!blobSideCarsBytesWrapped) { + // Handle the same to onBeaconBlocksByRange + throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`); + } + const blobSideCarsBytes = blobSideCarsBytesWrapped.slice(BLOB_SIDECARS_IN_WRAPPER_INDEX); + + lastFetchedSideCars = {blockRoot: blockRootHex, bytes: blobSideCarsBytes}; + } + + const blobSidecarBytes = lastFetchedSideCars.bytes.slice( + index * BLOBSIDECAR_FIXED_SIZE, + (index + 1) * BLOBSIDECAR_FIXED_SIZE + ); + if (blobSidecarBytes.length !== BLOBSIDECAR_FIXED_SIZE) { + throw Error( + `Inconsistent state, blobSidecar blockRoot=${blockRootHex} index=${index} blobSidecarBytes=${blobSidecarBytes.length} expected=${BLOBSIDECAR_FIXED_SIZE}` + ); + } + + yield { + data: blobSidecarBytes, + fork: chain.config.getForkName(block.slot), + }; + } +} diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts deleted file mode 100644 index 1cdfafdc09e6..000000000000 --- a/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts +++ /dev/null @@ -1,18 +0,0 @@ -import {deneb} from "@lodestar/types"; -import {ResponseOutgoing} from "@lodestar/reqresp"; -import {IBeaconChain} from "../../../chain/index.js"; -import {IBeaconDb} from "../../../db/index.js"; -import {onBlocksOrBlobsSidecarsByRange} from "./beaconBlocksByRange.js"; - -// TODO DENEB: Unit test - -export function onBlobsSidecarsByRange( - request: deneb.BlobsSidecarsByRangeRequest, - chain: IBeaconChain, - db: IBeaconDb -): AsyncIterable { - return onBlocksOrBlobsSidecarsByRange(request, chain, { - finalized: db.blobsSidecarArchive, - unfinalized: db.blobsSidecar, - }); -} diff --git a/packages/beacon-node/src/network/reqresp/handlers/index.ts b/packages/beacon-node/src/network/reqresp/handlers/index.ts index 8646ee02340e..50b8cc870844 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/index.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/index.ts @@ -5,8 +5,8 @@ import {IBeaconDb} from "../../../db/index.js"; import {GetReqRespHandlerFn, ReqRespMethod} from "../types.js"; import {onBeaconBlocksByRange} from "./beaconBlocksByRange.js"; import {onBeaconBlocksByRoot} from "./beaconBlocksByRoot.js"; -import {onBeaconBlockAndBlobsSidecarByRoot} from "./beaconBlockAndBlobsSidecarByRoot.js"; -import {onBlobsSidecarsByRange} from "./blobsSidecarsByRange.js"; +import {onBlobSidecarsByRoot} from "./blobSidecarsByRoot.js"; +import {onBlobSidecarsByRange} from "./blobSidecarsByRange.js"; import {onLightClientBootstrap} from "./lightClientBootstrap.js"; import {onLightClientFinalityUpdate} from "./lightClientFinalityUpdate.js"; import {onLightClientOptimisticUpdate} from "./lightClientOptimisticUpdate.js"; @@ -36,13 +36,13 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh const body = ssz.phase0.BeaconBlocksByRootRequest.deserialize(req.data); return onBeaconBlocksByRoot(body, chain, db); }, - [ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot]: (req) => { - const body = ssz.deneb.BeaconBlockAndBlobsSidecarByRootRequest.deserialize(req.data); - return onBeaconBlockAndBlobsSidecarByRoot(body, chain, db); + [ReqRespMethod.BlobSidecarsByRoot]: (req) => { + const body = ssz.deneb.BlobSidecarsByRootRequest.deserialize(req.data); + return onBlobSidecarsByRoot(body, chain, db); }, - [ReqRespMethod.BlobsSidecarsByRange]: (req) => { - const body = ssz.deneb.BlobsSidecarsByRangeRequest.deserialize(req.data); - return onBlobsSidecarsByRange(body, chain, db); + [ReqRespMethod.BlobSidecarsByRange]: (req) => { + const body = ssz.deneb.BlobSidecarsByRangeRequest.deserialize(req.data); + return onBlobSidecarsByRange(body, chain, db); }, [ReqRespMethod.LightClientBootstrap]: (req) => { const body = ssz.Root.deserialize(req.data); diff --git a/packages/beacon-node/src/network/reqresp/protocols.ts b/packages/beacon-node/src/network/reqresp/protocols.ts index c77ce11aa129..a0fa9576c93c 100644 --- a/packages/beacon-node/src/network/reqresp/protocols.ts +++ b/packages/beacon-node/src/network/reqresp/protocols.ts @@ -35,12 +35,6 @@ export const Status = toProtocol({ contextBytesType: ContextBytesType.Empty, }); -export const BeaconBlockAndBlobsSidecarByRoot = toProtocol({ - method: ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot, - version: Version.V1, - contextBytesType: ContextBytesType.ForkDigest, -}); - export const BeaconBlocksByRange = toProtocol({ method: ReqRespMethod.BeaconBlocksByRange, version: Version.V1, @@ -65,8 +59,14 @@ export const BeaconBlocksByRootV2 = toProtocol({ contextBytesType: ContextBytesType.ForkDigest, }); -export const BlobsSidecarsByRange = toProtocol({ - method: ReqRespMethod.BlobsSidecarsByRange, +export const BlobSidecarsByRange = toProtocol({ + method: ReqRespMethod.BlobSidecarsByRange, + version: Version.V1, + contextBytesType: ContextBytesType.ForkDigest, +}); + +export const BlobSidecarsByRoot = toProtocol({ + method: ReqRespMethod.BlobSidecarsByRoot, version: Version.V1, contextBytesType: ContextBytesType.ForkDigest, }); diff --git a/packages/beacon-node/src/network/reqresp/rateLimit.ts b/packages/beacon-node/src/network/reqresp/rateLimit.ts index 3df0eafe1d93..6d2ec66777dd 100644 --- a/packages/beacon-node/src/network/reqresp/rateLimit.ts +++ b/packages/beacon-node/src/network/reqresp/rateLimit.ts @@ -31,15 +31,15 @@ export const rateLimitQuotas: Record = { byPeer: {quota: 128, quotaTimeMs: 10_000}, getRequestCount: getRequestCountFn(ReqRespMethod.BeaconBlocksByRoot, (req) => req.length), }, - [ReqRespMethod.BlobsSidecarsByRange]: { + [ReqRespMethod.BlobSidecarsByRange]: { // TODO DENEB: For now same value as BeaconBlocksByRange https://github.com/sigp/lighthouse/blob/bf533c8e42cc73c35730e285c21df8add0195369/beacon_node/lighthouse_network/src/rpc/mod.rs#L118-L130 byPeer: {quota: MAX_REQUEST_BLOCKS, quotaTimeMs: 10_000}, - getRequestCount: getRequestCountFn(ReqRespMethod.BlobsSidecarsByRange, (req) => req.count), + getRequestCount: getRequestCountFn(ReqRespMethod.BlobSidecarsByRange, (req) => req.count), }, - [ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot]: { + [ReqRespMethod.BlobSidecarsByRoot]: { // TODO DENEB: For now same value as BeaconBlocksByRoot https://github.com/sigp/lighthouse/blob/bf533c8e42cc73c35730e285c21df8add0195369/beacon_node/lighthouse_network/src/rpc/mod.rs#L118-L130 byPeer: {quota: 128, quotaTimeMs: 10_000}, - getRequestCount: getRequestCountFn(ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot, (req) => req.length), + getRequestCount: getRequestCountFn(ReqRespMethod.BlobSidecarsByRoot, (req) => req.length), }, [ReqRespMethod.LightClientBootstrap]: { // As similar in the nature of `Status` protocol so we use the same rate limits. diff --git a/packages/beacon-node/src/network/reqresp/types.ts b/packages/beacon-node/src/network/reqresp/types.ts index d1c5db025be9..f690d282307f 100644 --- a/packages/beacon-node/src/network/reqresp/types.ts +++ b/packages/beacon-node/src/network/reqresp/types.ts @@ -14,8 +14,8 @@ export enum ReqRespMethod { Metadata = "metadata", BeaconBlocksByRange = "beacon_blocks_by_range", BeaconBlocksByRoot = "beacon_blocks_by_root", - BlobsSidecarsByRange = "blobs_sidecars_by_range", - BeaconBlockAndBlobsSidecarByRoot = "beacon_block_and_blobs_sidecar_by_root", + BlobSidecarsByRange = "blob_sidecars_by_range", + BlobSidecarsByRoot = "blob_sidecars_by_root", LightClientBootstrap = "light_client_bootstrap", LightClientUpdatesByRange = "light_client_updates_by_range", LightClientFinalityUpdate = "light_client_finality_update", @@ -30,8 +30,8 @@ export type RequestBodyByMethod = { [ReqRespMethod.Metadata]: null; [ReqRespMethod.BeaconBlocksByRange]: phase0.BeaconBlocksByRangeRequest; [ReqRespMethod.BeaconBlocksByRoot]: phase0.BeaconBlocksByRootRequest; - [ReqRespMethod.BlobsSidecarsByRange]: deneb.BlobsSidecarsByRangeRequest; - [ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot]: deneb.BeaconBlockAndBlobsSidecarByRootRequest; + [ReqRespMethod.BlobSidecarsByRange]: deneb.BlobSidecarsByRangeRequest; + [ReqRespMethod.BlobSidecarsByRoot]: deneb.BlobSidecarsByRootRequest; [ReqRespMethod.LightClientBootstrap]: Root; [ReqRespMethod.LightClientUpdatesByRange]: altair.LightClientUpdatesByRange; [ReqRespMethod.LightClientFinalityUpdate]: null; @@ -46,8 +46,8 @@ type ResponseBodyByMethod = { // Do not matter [ReqRespMethod.BeaconBlocksByRange]: allForks.SignedBeaconBlock; [ReqRespMethod.BeaconBlocksByRoot]: allForks.SignedBeaconBlock; - [ReqRespMethod.BlobsSidecarsByRange]: deneb.BlobsSidecar; - [ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot]: deneb.SignedBeaconBlockAndBlobsSidecar; + [ReqRespMethod.BlobSidecarsByRange]: deneb.BlobSidecar; + [ReqRespMethod.BlobSidecarsByRoot]: deneb.BlobSidecar; [ReqRespMethod.LightClientBootstrap]: altair.LightClientBootstrap; [ReqRespMethod.LightClientUpdatesByRange]: altair.LightClientUpdate; [ReqRespMethod.LightClientFinalityUpdate]: altair.LightClientFinalityUpdate; @@ -64,8 +64,8 @@ export const requestSszTypeByMethod: { [ReqRespMethod.Metadata]: null, [ReqRespMethod.BeaconBlocksByRange]: ssz.phase0.BeaconBlocksByRangeRequest, [ReqRespMethod.BeaconBlocksByRoot]: ssz.phase0.BeaconBlocksByRootRequest, - [ReqRespMethod.BlobsSidecarsByRange]: ssz.deneb.BlobsSidecarsByRangeRequest, - [ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot]: ssz.deneb.BeaconBlockAndBlobsSidecarByRootRequest, + [ReqRespMethod.BlobSidecarsByRange]: ssz.deneb.BlobSidecarsByRangeRequest, + [ReqRespMethod.BlobSidecarsByRoot]: ssz.deneb.BlobSidecarsByRootRequest, [ReqRespMethod.LightClientBootstrap]: ssz.Root, [ReqRespMethod.LightClientUpdatesByRange]: ssz.altair.LightClientUpdatesByRange, [ReqRespMethod.LightClientFinalityUpdate]: null, @@ -89,8 +89,8 @@ export const responseSszTypeByMethod: {[K in ReqRespMethod]: ResponseTypeGetter< [ReqRespMethod.Metadata]: (_, version) => (version == Version.V1 ? ssz.phase0.Metadata : ssz.altair.Metadata), [ReqRespMethod.BeaconBlocksByRange]: blocksResponseType, [ReqRespMethod.BeaconBlocksByRoot]: blocksResponseType, - [ReqRespMethod.BlobsSidecarsByRange]: () => ssz.deneb.BlobsSidecar, - [ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot]: () => ssz.deneb.SignedBeaconBlockAndBlobsSidecar, + [ReqRespMethod.BlobSidecarsByRange]: () => ssz.deneb.BlobSidecar, + [ReqRespMethod.BlobSidecarsByRoot]: () => ssz.deneb.BlobSidecar, [ReqRespMethod.LightClientBootstrap]: (fork) => ssz.allForksLightClient[onlyLightclientFork(fork)].LightClientBootstrap, [ReqRespMethod.LightClientUpdatesByRange]: (fork) => diff --git a/packages/beacon-node/test/e2e/network/deneb.test.ts b/packages/beacon-node/test/e2e/network/deneb.test.ts deleted file mode 100644 index af99e916c3de..000000000000 --- a/packages/beacon-node/test/e2e/network/deneb.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {expect} from "chai"; -import {deneb, ssz} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; -import {signedBeaconBlockAndBlobsSidecarFromBytes} from "../../../src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.js"; - -describe("signedBeaconBlockAndBlobsSidecarFromBytes", () => { - it("signedBeaconBlockAndBlobsSidecarFromBytes", () => { - const beaconBlock = ssz.deneb.SignedBeaconBlock.defaultValue(); - const blobsSidecar = ssz.deneb.BlobsSidecar.defaultValue(); - - const signedBeaconBlockAndBlobsSidecarBytes = signedBeaconBlockAndBlobsSidecarFromBytes( - ssz.deneb.SignedBeaconBlock.serialize(beaconBlock), - ssz.deneb.BlobsSidecar.serialize(blobsSidecar) - ); - - const signedBeaconBlockAndBlobsSidecar: deneb.SignedBeaconBlockAndBlobsSidecar = { - beaconBlock, - blobsSidecar, - }; - - expect(toHex(signedBeaconBlockAndBlobsSidecarBytes)).equals( - toHex(ssz.deneb.SignedBeaconBlockAndBlobsSidecar.serialize(signedBeaconBlockAndBlobsSidecar)), - "Wrong signedBeaconBlockAndBlobsSidecarBytes" - ); - - // Ensure deserialize does not throw - ssz.deneb.SignedBeaconBlockAndBlobsSidecar.deserialize(signedBeaconBlockAndBlobsSidecarBytes); - }); -}); From 5e24f9b635520b2e136eaa5005c8e81b0566185a Mon Sep 17 00:00:00 2001 From: harkamal Date: Mon, 29 May 2023 14:31:14 +0530 Subject: [PATCH 2/2] fix unit test --- .../beaconBlocksMaybeBlobsByRange.test.ts | 93 +++++++++++++------ 1 file changed, 66 insertions(+), 27 deletions(-) diff --git a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts index 5a2900b2c059..4193553303e7 100644 --- a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts +++ b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts @@ -1,10 +1,11 @@ import {expect} from "chai"; import {ssz, deneb} from "@lodestar/types"; import {createBeaconConfig, createChainForkConfig, defaultChainConfig} from "@lodestar/config"; +import {BYTES_PER_FIELD_ELEMENT} from "@lodestar/params"; import {beaconBlocksMaybeBlobsByRange} from "../../../src/network/reqresp/index.js"; -import {BlockInputType, BlockSource} from "../../../src/chain/blocks/types.js"; -import {ckzg, initCKZG, loadEthereumTrustedSetup} from "../../../src/util/kzg.js"; +import {BlockInputType, BlockSource, blobSidecarsToBlobsSidecar} from "../../../src/chain/blocks/types.js"; +import {ckzg, initCKZG, loadEthereumTrustedSetup, FIELD_ELEMENTS_PER_BLOB_MAINNET} from "../../../src/util/kzg.js"; import {INetwork} from "../../../src/network/interface.js"; describe("beaconBlocksMaybeBlobsByRange", () => { @@ -31,54 +32,83 @@ describe("beaconBlocksMaybeBlobsByRange", () => { const block1 = ssz.deneb.SignedBeaconBlock.defaultValue(); block1.message.slot = 1; + block1.message.body.blobKzgCommitments.push(ssz.deneb.KZGCommitment.defaultValue()); + const blobSidecar1 = ssz.deneb.BlobSidecar.defaultValue(); + blobSidecar1.slot = 1; + const block2 = ssz.deneb.SignedBeaconBlock.defaultValue(); block2.message.slot = 2; + block2.message.body.blobKzgCommitments.push(ssz.deneb.KZGCommitment.defaultValue()); + const blobSidecar2 = ssz.deneb.BlobSidecar.defaultValue(); + blobSidecar2.slot = 2; + + const block3 = ssz.deneb.SignedBeaconBlock.defaultValue(); + block3.message.slot = 3; + // no blobsidecar for block3 - const blobsSidecar1 = ssz.deneb.BlobsSidecar.defaultValue(); - blobsSidecar1.beaconBlockSlot = 1; - const blobsSidecar2 = ssz.deneb.BlobsSidecar.defaultValue(); - blobsSidecar2.beaconBlockSlot = 2; + const block4 = ssz.deneb.SignedBeaconBlock.defaultValue(); + block4.message.slot = 4; + // two blobsidecars + block4.message.body.blobKzgCommitments.push(ssz.deneb.KZGCommitment.defaultValue()); + block4.message.body.blobKzgCommitments.push(ssz.deneb.KZGCommitment.defaultValue()); + const blobSidecar41 = ssz.deneb.BlobSidecar.defaultValue(); + blobSidecar41.slot = 4; + const blobSidecar42 = ssz.deneb.BlobSidecar.defaultValue(); + blobSidecar42.slot = 4; + blobSidecar42.index = 1; // Array of testcases which are array of matched blocks with/without (if empty) sidecars - const testCases: [string, [deneb.SignedBeaconBlock, deneb.BlobsSidecar | undefined][]][] = [ - ["one block with sidecar", [[block1, blobsSidecar1]]], + const testCases: [string, [deneb.SignedBeaconBlock, deneb.BlobSidecar[] | undefined][]][] = [ + ["one block with sidecar", [[block1, [blobSidecar1]]]], [ "two blocks with sidecar", [ - [block1, blobsSidecar1], - [block2, blobsSidecar2], + [block1, [blobSidecar1]], + [block2, [blobSidecar2]], + ], + ], + ["block with skipped sidecar", [[block3, undefined]]], + ["multiple blob sidecars per block", [[block4, [blobSidecar41, blobSidecar42]]]], + [ + "all blocks together", + [ + [block1, [blobSidecar1]], + [block2, [blobSidecar2]], + [block3, undefined], + [block4, [blobSidecar41, blobSidecar42]], ], ], - ["block with skipped sidecar", [[block1, undefined]]], ]; testCases.map(([testName, blocksWithBlobs]) => { it(testName, async () => { const blocks = blocksWithBlobs.map(([block, _blobs]) => block as deneb.SignedBeaconBlock); - const blobsSidecars = blocksWithBlobs - .map(([_block, blobs]) => blobs as deneb.BlobsSidecar) - .filter((blobs) => blobs !== undefined); - const emptyKzgAggregatedProof = ckzg.computeAggregateKzgProof([]); - const expectedResponse = blocksWithBlobs.map(([block, blobsSidecar]) => { - const blobs = - blobsSidecar !== undefined - ? blobsSidecar - : { - beaconBlockRoot: ssz.deneb.BeaconBlock.hashTreeRoot(block.message), - beaconBlockSlot: block.message.slot, - blobs: [], - kzgAggregatedProof: emptyKzgAggregatedProof, - }; + + const blobSidecars = blocksWithBlobs + .map(([_block, blobs]) => blobs as deneb.BlobSidecars) + .filter((blobs) => blobs !== undefined) + .reduce((acc, elem) => acc.concat(elem), []); + + const expectedResponse = blocksWithBlobs.map(([block, blobSidecars]) => { + const blobs = (blobSidecars !== undefined ? blobSidecars : []).map((bscar) => { + // TODO DENEB: cleanup the following generation as its not required to generate + // proper field elements for the aggregate proofs compute + bscar.blob = generateRandomBlob(); + (bscar.kzgCommitment = ckzg.blobToKzgCommitment(bscar.blob)), + (bscar.kzgProof = ckzg.computeAggregateKzgProof([bscar.blob])); + return bscar; + }); return { type: BlockInputType.postDeneb, block, source: BlockSource.byRange, - blobs, + // TODO DENEB: Cleanup the conversion once migration complete + blobs: blobSidecarsToBlobsSidecar(chainConfig, block, blobs), }; }); const network = { sendBeaconBlocksByRange: async () => blocks, - sendBlobsSidecarsByRange: async () => blobsSidecars, + sendBlobSidecarsByRange: async () => blobSidecars, } as Partial as INetwork; const response = await beaconBlocksMaybeBlobsByRange(config, network, peerId, rangeRequest, 0); @@ -86,3 +116,12 @@ describe("beaconBlocksMaybeBlobsByRange", () => { }); }); }); + +function generateRandomBlob(): deneb.Blob { + const blob = new Uint8Array(FIELD_ELEMENTS_PER_BLOB_MAINNET * BYTES_PER_FIELD_ELEMENT); + const dv = new DataView(blob.buffer, blob.byteOffset, blob.byteLength); + for (let i = 0; i < FIELD_ELEMENTS_PER_BLOB_MAINNET; i++) { + dv.setUint32(i * BYTES_PER_FIELD_ELEMENT, i); + } + return blob; +}