From 63cff370db5c35f16d53c247b2bb0b697bac1afb Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 28 Jun 2024 16:36:23 +0100 Subject: [PATCH 1/2] refactor(evm): set prune modes optionally for the batch executor --- bin/reth/src/commands/debug_cmd/merkle.rs | 10 +++--- crates/ethereum/evm/src/execute.rs | 34 +++++++++----------- crates/evm/src/either.rs | 13 ++++++-- crates/evm/src/execute.rs | 16 ++++++--- crates/evm/src/noop.rs | 4 ++- crates/evm/src/test_utils.rs | 4 ++- crates/optimism/evm/src/execute.rs | 14 ++++---- crates/stages/stages/src/stages/execution.rs | 3 +- 8 files changed, 56 insertions(+), 42 deletions(-) diff --git a/bin/reth/src/commands/debug_cmd/merkle.rs b/bin/reth/src/commands/debug_cmd/merkle.rs index bd8f690b9a59..96da3bb736ae 100644 --- a/bin/reth/src/commands/debug_cmd/merkle.rs +++ b/bin/reth/src/commands/debug_cmd/merkle.rs @@ -23,7 +23,6 @@ use reth_provider::{ BlockNumReader, BlockWriter, ChainSpecProvider, HeaderProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, ProviderFactory, StateWriter, }; -use reth_prune::PruneModes; use reth_revm::database::StateProviderDatabase; use reth_stages::{ stages::{AccountHashingStage, MerkleStage, StorageHashingStage}, @@ -148,13 +147,12 @@ impl Command { provider_rw.insert_block(sealed_block.clone(), None)?; td += sealed_block.difficulty; - let mut executor = executor_provider.batch_executor( - StateProviderDatabase::new(LatestStateProviderRef::new( + let mut executor = executor_provider.batch_executor(StateProviderDatabase::new( + LatestStateProviderRef::new( provider_rw.tx_ref(), provider_rw.static_file_provider().clone(), - )), - PruneModes::none(), - ); + ), + )); executor.execute_and_verify_one((&sealed_block.clone().unseal(), td).into())?; executor.finalize().write_to_storage( provider_rw.tx_ref(), diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index 76ec292396df..084bb412b6b9 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -93,14 +93,14 @@ where self.eth_executor(db) } - fn batch_executor(&self, db: DB, prune_modes: PruneModes) -> Self::BatchExecutor + fn batch_executor(&self, db: DB) -> Self::BatchExecutor where DB: Database + Display>, { let executor = self.eth_executor(db); EthBatchExecutor { executor, - batch_record: BlockBatchRecord::new(prune_modes), + batch_record: BlockBatchRecord::default(), stats: BlockExecutorStats::default(), } } @@ -451,6 +451,10 @@ where self.batch_record.set_tip(tip); } + fn set_prune_modes(&mut self, prune_modes: PruneModes) { + self.batch_record.set_prune_modes(prune_modes); + } + fn size_hint(&self) -> Option { Some(self.executor.state.bundle_state.size_hint()) } @@ -634,7 +638,7 @@ mod tests { // attempt to execute an empty block with parent beacon block root, this should not fail provider - .batch_executor(StateProviderDatabase::new(&db), PruneModes::none()) + .batch_executor(StateProviderDatabase::new(&db)) .execute_and_verify_one( ( &BlockWithSenders { @@ -684,8 +688,7 @@ mod tests { ..Header::default() }; - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); // attempt to execute an empty block with parent beacon block root, this should not fail executor @@ -728,8 +731,7 @@ mod tests { let mut header = chain_spec.genesis_header(); let provider = executor_provider(chain_spec); - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); // attempt to execute the genesis block with non-zero parent beacon block root, expect err header.parent_beacon_block_root = Some(B256::with_last_byte(0x69)); @@ -816,8 +818,7 @@ mod tests { let provider = executor_provider(chain_spec); // execute header - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); // Now execute a block with the fixed header, ensure that it does not fail executor @@ -884,8 +885,7 @@ mod tests { ); let provider = executor_provider(chain_spec); - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); // construct the header for block one let header = Header { timestamp: 1, number: 1, ..Header::default() }; @@ -938,8 +938,7 @@ mod tests { let header = chain_spec.genesis_header(); let provider = executor_provider(chain_spec); - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); // attempt to execute genesis block, this should not fail executor @@ -996,8 +995,7 @@ mod tests { ..Header::default() }; let provider = executor_provider(chain_spec); - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); // attempt to execute the fork activation block, this should not fail executor @@ -1052,8 +1050,7 @@ mod tests { ); let provider = executor_provider(chain_spec); - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); let header = Header { parent_hash: B256::random(), @@ -1115,8 +1112,7 @@ mod tests { let header_hash = header.hash_slow(); let provider = executor_provider(chain_spec); - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); // attempt to execute the genesis block, this should not fail executor diff --git a/crates/evm/src/either.rs b/crates/evm/src/either.rs index 2f55f1668923..f6af36d2eb63 100644 --- a/crates/evm/src/either.rs +++ b/crates/evm/src/either.rs @@ -36,13 +36,13 @@ where } } - fn batch_executor(&self, db: DB, prune_modes: PruneModes) -> Self::BatchExecutor + fn batch_executor(&self, db: DB) -> Self::BatchExecutor where DB: Database + Display>, { match self { - Self::Left(a) => Either::Left(a.batch_executor(db, prune_modes)), - Self::Right(b) => Either::Right(b.batch_executor(db, prune_modes)), + Self::Left(a) => Either::Left(a.batch_executor(db)), + Self::Right(b) => Either::Right(b.batch_executor(db)), } } } @@ -116,6 +116,13 @@ where } } + fn set_prune_modes(&mut self, prune_modes: PruneModes) { + match self { + Self::Left(a) => a.set_prune_modes(prune_modes), + Self::Right(b) => b.set_prune_modes(prune_modes), + } + } + fn size_hint(&self) -> Option { match self { Self::Left(a) => a.size_hint(), diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 6d076fd45303..586fed53d997 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -85,6 +85,11 @@ pub trait BatchExecutor { /// This can be used to optimize state pruning during execution. fn set_tip(&mut self, tip: BlockNumber); + /// Set the prune modes. + /// + /// They are used to determine which parts of the state should be kept during execution. + fn set_prune_modes(&mut self, prune_modes: PruneModes); + /// The size hint of the batch's tracked state size. /// /// This is used to optimize DB commits depending on the size of the state. @@ -169,10 +174,7 @@ pub trait BlockExecutorProvider: Send + Sync + Clone + Unpin + 'static { /// /// Batch executor is used to execute multiple blocks in sequence and keep track of the state /// during historical sync which involves executing multiple blocks in sequence. - /// - /// The pruning modes are used to determine which parts of the state should be kept during - /// execution. - fn batch_executor(&self, db: DB, prune_modes: PruneModes) -> Self::BatchExecutor + fn batch_executor(&self, db: DB) -> Self::BatchExecutor where DB: Database + Display>; } @@ -198,7 +200,7 @@ mod tests { TestExecutor(PhantomData) } - fn batch_executor(&self, _db: DB, _prune_modes: PruneModes) -> Self::BatchExecutor + fn batch_executor(&self, _db: DB) -> Self::BatchExecutor where DB: Database + Display>, { @@ -235,6 +237,10 @@ mod tests { todo!() } + fn set_prune_modes(&mut self, _prune_modes: PruneModes) { + todo!() + } + fn size_hint(&self) -> Option { None } diff --git a/crates/evm/src/noop.rs b/crates/evm/src/noop.rs index d393f66d566d..80a2b76de834 100644 --- a/crates/evm/src/noop.rs +++ b/crates/evm/src/noop.rs @@ -32,7 +32,7 @@ impl BlockExecutorProvider for NoopBlockExecutorProvider { Self } - fn batch_executor(&self, _: DB, _: PruneModes) -> Self::BatchExecutor + fn batch_executor(&self, _: DB) -> Self::BatchExecutor where DB: Database + Display>, { @@ -65,6 +65,8 @@ impl BatchExecutor for NoopBlockExecutorProvider { fn set_tip(&mut self, _: BlockNumber) {} + fn set_prune_modes(&mut self, _: PruneModes) {} + fn size_hint(&self) -> Option { None } diff --git a/crates/evm/src/test_utils.rs b/crates/evm/src/test_utils.rs index a4d098f0b3aa..c3aa34a56a45 100644 --- a/crates/evm/src/test_utils.rs +++ b/crates/evm/src/test_utils.rs @@ -37,7 +37,7 @@ impl BlockExecutorProvider for MockExecutorProvider { self.clone() } - fn batch_executor(&self, _: DB, _: PruneModes) -> Self::BatchExecutor + fn batch_executor(&self, _: DB) -> Self::BatchExecutor where DB: Database + Display>, { @@ -77,6 +77,8 @@ impl BatchExecutor for MockExecutorProvider { fn set_tip(&mut self, _: BlockNumber) {} + fn set_prune_modes(&mut self, _: PruneModes) {} + fn size_hint(&self) -> Option { None } diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 1f873d234a46..de4fc071e0cf 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -79,14 +79,14 @@ where self.op_executor(db) } - fn batch_executor(&self, db: DB, prune_modes: PruneModes) -> Self::BatchExecutor + fn batch_executor(&self, db: DB) -> Self::BatchExecutor where DB: Database + std::fmt::Display>, { let executor = self.op_executor(db); OpBatchExecutor { executor, - batch_record: BlockBatchRecord::new(prune_modes), + batch_record: BlockBatchRecord::default(), stats: BlockExecutorStats::default(), } } @@ -435,6 +435,10 @@ where self.batch_record.set_tip(tip); } + fn set_prune_modes(&mut self, prune_modes: PruneModes) { + self.batch_record.set_prune_modes(prune_modes); + } + fn size_hint(&self) -> Option { Some(self.executor.state.bundle_state.size_hint()) } @@ -528,8 +532,7 @@ mod tests { ); let provider = executor_provider(chain_spec); - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); executor.state_mut().load_cache_account(L1_BLOCK_CONTRACT).unwrap(); @@ -610,8 +613,7 @@ mod tests { ); let provider = executor_provider(chain_spec); - let mut executor = - provider.batch_executor(StateProviderDatabase::new(&db), PruneModes::none()); + let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); executor.state_mut().load_cache_account(L1_BLOCK_CONTRACT).unwrap(); diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index a37c081b4da7..f745e053c3e2 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -224,8 +224,9 @@ where provider.tx_ref(), provider.static_file_provider().clone(), )); - let mut executor = self.executor_provider.batch_executor(db, prune_modes); + let mut executor = self.executor_provider.batch_executor(db); executor.set_tip(max_block); + executor.set_prune_modes(prune_modes); // Progress tracking let mut stage_progress = start_block; From 486d3f24e11b3ff34cb5a01cf8a731d06a995012 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 28 Jun 2024 16:41:20 +0100 Subject: [PATCH 2/2] optional prune modes for exex backfill --- crates/exex/exex/src/backfill.rs | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/crates/exex/exex/src/backfill.rs b/crates/exex/exex/src/backfill.rs index f46c82d24101..aecb498b8cf6 100644 --- a/crates/exex/exex/src/backfill.rs +++ b/crates/exex/exex/src/backfill.rs @@ -24,8 +24,19 @@ pub struct BackfillJobFactory { impl BackfillJobFactory { /// Creates a new [`BackfillJobFactory`]. - pub fn new(executor: E, provider: P, prune_modes: PruneModes) -> Self { - Self { executor, provider, prune_modes, thresholds: ExecutionStageThresholds::default() } + pub fn new(executor: E, provider: P) -> Self { + Self { + executor, + provider, + prune_modes: PruneModes::none(), + thresholds: ExecutionStageThresholds::default(), + } + } + + /// Sets the prune modes + pub fn with_prune_modes(mut self, prune_modes: PruneModes) -> Self { + self.prune_modes = prune_modes; + self } /// Sets the thresholds @@ -53,12 +64,10 @@ impl BackfillJobFactory<(), ()> { /// Creates a new [`BackfillJobFactory`] from [`FullNodeComponents`]. pub fn new_from_components( components: Node, - prune_modes: PruneModes, ) -> BackfillJobFactory { BackfillJobFactory::<_, _>::new( components.block_executor().clone(), components.provider().clone(), - prune_modes, ) } } @@ -72,8 +81,8 @@ pub struct BackfillJob { executor: E, provider: P, prune_modes: PruneModes, - range: RangeInclusive, thresholds: ExecutionStageThresholds, + range: RangeInclusive, _db: PhantomData, } @@ -101,12 +110,10 @@ where P: FullProvider, { fn execute_range(&mut self) -> Result { - let mut executor = self.executor.batch_executor( - StateProviderDatabase::new( - self.provider.history_by_block_number(self.range.start().saturating_sub(1))?, - ), - self.prune_modes.clone(), - ); + let mut executor = self.executor.batch_executor(StateProviderDatabase::new( + self.provider.history_by_block_number(self.range.start().saturating_sub(1))?, + )); + executor.set_prune_modes(self.prune_modes.clone()); let mut fetch_block_duration = Duration::default(); let mut execution_duration = Duration::default(); @@ -204,7 +211,6 @@ mod tests { providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, LatestStateProviderRef, }; - use reth_prune_types::PruneModes; use reth_revm::database::StateProviderDatabase; use reth_testing_utils::generators::{self, sign_tx_with_key_pair}; use secp256k1::Keypair; @@ -288,24 +294,18 @@ mod tests { let provider = provider_factory.provider()?; // Execute only the first block on top of genesis state let mut outcome_single = EthExecutorProvider::ethereum(chain_spec.clone()) - .batch_executor( - StateProviderDatabase::new(LatestStateProviderRef::new( - provider.tx_ref(), - provider.static_file_provider().clone(), - )), - PruneModes::none(), - ) + .batch_executor(StateProviderDatabase::new(LatestStateProviderRef::new( + provider.tx_ref(), + provider.static_file_provider().clone(), + ))) .execute_and_verify_batch([(&block1, U256::ZERO).into()])?; outcome_single.bundle.reverts.sort(); // Execute both blocks on top of the genesis state let outcome_batch = EthExecutorProvider::ethereum(chain_spec) - .batch_executor( - StateProviderDatabase::new(LatestStateProviderRef::new( - provider.tx_ref(), - provider.static_file_provider().clone(), - )), - PruneModes::none(), - ) + .batch_executor(StateProviderDatabase::new(LatestStateProviderRef::new( + provider.tx_ref(), + provider.static_file_provider().clone(), + ))) .execute_and_verify_batch([ (&block1, U256::ZERO).into(), (&block2, U256::ZERO).into(), @@ -327,7 +327,7 @@ mod tests { provider_rw.commit()?; // Backfill the first block - let factory = BackfillJobFactory::new(executor, blockchain_db, PruneModes::none()); + let factory = BackfillJobFactory::new(executor, blockchain_db); let job = factory.backfill(1..=1); let chains = job.collect::, _>>()?;