Merge branch 'main' into devnet-0

Cargo.toml

@@ -239,6 +239,7 @@ reth-etl = { path = "crates/etl" }
 reth-evm = { path = "crates/evm" }
 reth-evm-ethereum = { path = "crates/ethereum/evm" }
 reth-exex = { path = "crates/exex" }
+reth-fs-util = { path = "crates/fs-util" }
 reth-optimism-payload-builder = { path = "crates/optimism/payload" }
 reth-interfaces = { path = "crates/interfaces" }
 reth-ipc = { path = "crates/rpc/ipc" }
@@ -283,7 +284,7 @@ reth-testing-utils = { path = "testing/testing-utils" }
 revm = { version = "9.0.0", features = ["std", "secp256k1", "blst"], default-features = false }
 revm-primitives = { version = "4.0.0", features = ["std"], default-features = false }
 revm-interpreter = { version = "5.0.0", features = ["std"], default-features = false }
-revm-inspectors = { git = "https://github.com/paradigmxyz/evm-inspectors", rev = "fb5d095" }
+revm-inspectors = { git = "https://github.com/paradigmxyz/evm-inspectors", rev = "5a4fd5e" }
 
 # eth
 alloy-chains = "0.1.15"
@@ -292,21 +293,21 @@ alloy-dyn-abi = "0.7.2"
 alloy-sol-types = "0.7.2"
 alloy-rlp = "0.3.4"
 alloy-trie = "0.4.0"
-alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-genesis = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-node-bindings = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-provider = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf", default-features = false, features = [
+alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-genesis = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-node-bindings = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-provider = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b", default-features = false, features = [
     "reqwest",
 ] }
-alloy-eips = { git = "https://github.com/alloy-rs/alloy", default-features = false, rev = "07611cf" }
-alloy-signer = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-signer-wallet = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-network = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
-alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "07611cf" }
+alloy-eips = { git = "https://github.com/alloy-rs/alloy", default-features = false, rev = "64feb9b" }
+alloy-signer = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-signer-wallet = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-network = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
+alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "64feb9b" }
 
 # misc
 auto_impl = "1"
@@ -382,7 +383,10 @@ secp256k1 = { version = "0.28", default-features = false, features = [
     "recovery",
 ] }
 # TODO: Remove `k256` feature: https://github.com/sigp/enr/pull/74
-enr = { version = "0.12.0", default-features = false, features = ["k256", "rust-secp256k1"] }
+enr = { version = "0.12.0", default-features = false, features = [
+    "k256",
+    "rust-secp256k1",
+] }
 
 # for eip-4844
 c-kzg = "1.0.0"

Makefile

@@ -59,11 +59,11 @@ install-op: ## Build and install the op-reth binary under `~/.cargo/bin`.
 
 .PHONY: build
 build: ## Build the reth binary into `target` directory.
-	$(MAKE) build-native-$(shell rustc -Vv | grep host | cut -d ' ' -f2)
+	cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)"
 
 .PHONY: build-op
 build-op: ## Build the op-reth binary into `target` directory.
-	$(MAKE) op-build-native-$(shell rustc -Vv | grep host | cut -d ' ' -f2)
+	cargo build --bin op-reth --features "optimism,$(FEATURES)" --profile "$(PROFILE)"
 
 # Builds the reth binary natively.
 build-native-%:
@@ -303,8 +303,7 @@ db-tools: ## Compile MDBX debugging tools.
 	@echo "Run \"$(DB_TOOLS_DIR)/mdbx_chk\" for the MDBX db file integrity check."
 
 .PHONY: update-book-cli
-update-book-cli: ## Update book cli documentation.
-	cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)"
+update-book-cli: build ## Update book cli documentation.
 	@echo "Updating book cli doc..."
 	@./book/cli/update.sh $(BUILD_PATH)/$(PROFILE)/reth
 

bin/reth/Cargo.toml

@@ -16,6 +16,7 @@ workspace = true
 # reth
 reth-config.workspace = true
 reth-primitives = { workspace = true, features = ["arbitrary", "clap"] }
+reth-fs-util.workspace = true
 reth-db = { workspace = true, features = ["mdbx"] }
 reth-exex.workspace = true
 reth-provider = { workspace = true }
@@ -43,6 +44,7 @@ reth-payload-builder.workspace = true
 reth-payload-validator.workspace = true
 reth-basic-payload-builder.workspace = true
 reth-discv4.workspace = true
+reth-discv5.workspace = true
 reth-static-file = { workspace = true }
 reth-trie = { workspace = true, features = ["metrics"] }
 reth-nippy-jar.workspace = true

bin/reth/src/commands/db/stats.rs

@@ -15,6 +15,7 @@ use reth_db::{
     StoragesTrie, Tables, TransactionBlocks, TransactionHashNumbers, TransactionSenders,
     Transactions, VersionHistory,
 };
+use reth_fs_util as fs;
 use reth_node_core::dirs::{ChainPath, DataDirPath};
 use reth_primitives::static_file::{find_fixed_range, SegmentRangeInclusive};
 use reth_provider::providers::StaticFileProvider;
@@ -203,16 +204,16 @@ impl Command {
                 let columns = jar_provider.columns();
                 let rows = jar_provider.rows();
 
-                let data_size = reth_primitives::fs::metadata(jar_provider.data_path())
+                let data_size = fs::metadata(jar_provider.data_path())
                     .map(|metadata| metadata.len())
                     .unwrap_or_default();
-                let index_size = reth_primitives::fs::metadata(jar_provider.index_path())
+                let index_size = fs::metadata(jar_provider.index_path())
                     .map(|metadata| metadata.len())
                     .unwrap_or_default();
-                let offsets_size = reth_primitives::fs::metadata(jar_provider.offsets_path())
+                let offsets_size = fs::metadata(jar_provider.offsets_path())
                     .map(|metadata| metadata.len())
                     .unwrap_or_default();
-                let config_size = reth_primitives::fs::metadata(jar_provider.config_path())
+                let config_size = fs::metadata(jar_provider.config_path())
                     .map(|metadata| metadata.len())
                     .unwrap_or_default();
 

bin/reth/src/commands/debug_cmd/build_block.rs

@@ -22,12 +22,12 @@ use reth_cli_runner::CliContext;
 use reth_consensus::Consensus;
 use reth_db::{init_db, DatabaseEnv};
 use reth_evm::execute::{BlockExecutionOutput, BlockExecutorProvider, Executor};
+use reth_fs_util as fs;
 use reth_interfaces::RethResult;
 use reth_node_api::PayloadBuilderAttributes;
 use reth_payload_builder::database::CachedReads;
 use reth_primitives::{
     constants::eip4844::{LoadKzgSettingsError, MAINNET_KZG_TRUSTED_SETUP},
-    fs,
     revm_primitives::KzgSettings,
     stage::StageId,
     Address, BlobTransaction, BlobTransactionSidecar, Bytes, ChainSpec, PooledTransactionsElement,

bin/reth/src/commands/debug_cmd/execution.rs

@@ -22,12 +22,13 @@ use reth_downloaders::{
     headers::reverse_headers::ReverseHeadersDownloaderBuilder,
 };
 use reth_exex::ExExManagerHandle;
+use reth_fs_util as fs;
 use reth_interfaces::p2p::{bodies::client::BodiesClient, headers::client::HeadersClient};
 use reth_network::{NetworkEvents, NetworkHandle};
 use reth_network_api::NetworkInfo;
 use reth_node_core::init::init_genesis;
 use reth_primitives::{
-    fs, stage::StageId, BlockHashOrNumber, BlockNumber, ChainSpec, PruneModes, B256,
+    stage::StageId, BlockHashOrNumber, BlockNumber, ChainSpec, PruneModes, B256,
 };
 use reth_provider::{
     BlockExecutionWriter, HeaderSyncMode, ProviderFactory, StageCheckpointReader,

bin/reth/src/commands/debug_cmd/in_memory_merkle.rs

@@ -16,10 +16,11 @@ use reth_cli_runner::CliContext;
 use reth_config::Config;
 use reth_db::{init_db, DatabaseEnv};
 use reth_evm::execute::{BlockExecutionOutput, BlockExecutorProvider, Executor};
+use reth_fs_util as fs;
 use reth_interfaces::executor::BlockValidationError;
 use reth_network::NetworkHandle;
 use reth_network_api::NetworkInfo;
-use reth_primitives::{fs, stage::StageId, BlockHashOrNumber, ChainSpec, Receipts};
+use reth_primitives::{stage::StageId, BlockHashOrNumber, ChainSpec, Receipts};
 use reth_provider::{
     AccountExtReader, BundleStateWithReceipts, HashingWriter, HeaderProvider,
     LatestStateProviderRef, OriginalValuesKnown, ProviderFactory, StageCheckpointReader,

bin/reth/src/commands/debug_cmd/merkle.rs

@@ -18,10 +18,11 @@ use reth_config::Config;
 use reth_consensus::Consensus;
 use reth_db::{cursor::DbCursorRO, init_db, tables, transaction::DbTx, DatabaseEnv};
 use reth_evm::execute::{BatchBlockExecutionOutput, BatchExecutor, BlockExecutorProvider};
+use reth_fs_util as fs;
 use reth_interfaces::p2p::full_block::FullBlockClient;
 use reth_network::NetworkHandle;
 use reth_network_api::NetworkInfo;
-use reth_primitives::{fs, stage::StageCheckpoint, BlockHashOrNumber, ChainSpec, PruneModes};
+use reth_primitives::{stage::StageCheckpoint, BlockHashOrNumber, ChainSpec, PruneModes};
 use reth_provider::{
     BlockNumReader, BlockWriter, BundleStateWithReceipts, HeaderProvider, LatestStateProviderRef,
     OriginalValuesKnown, ProviderError, ProviderFactory, StateWriter,

bin/reth/src/commands/debug_cmd/replay_engine.rs

@@ -18,11 +18,12 @@ use reth_cli_runner::CliContext;
 use reth_config::Config;
 use reth_consensus::Consensus;
 use reth_db::{init_db, DatabaseEnv};
+use reth_fs_util as fs;
 use reth_network::NetworkHandle;
 use reth_network_api::NetworkInfo;
 use reth_node_core::engine::engine_store::{EngineMessageStore, StoredEngineApiMessage};
 use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService};
-use reth_primitives::{fs, ChainSpec, PruneModes};
+use reth_primitives::{ChainSpec, PruneModes};
 use reth_provider::{
     providers::BlockchainProvider, CanonStateSubscriptions, ProviderFactory,
     StaticFileProviderFactory,

bin/reth/src/commands/import_op.rs

@@ -121,7 +121,7 @@ impl ImportOpCommand {
             info!(target: "reth::cli", "Chain file chunk read");
 
             total_decoded_blocks += file_client.headers_len();
-            total_decoded_txns += file_client.bodies_len();
+            total_decoded_txns += file_client.total_transactions();
 
             for (block_number, body) in file_client.bodies_iter_mut() {
                 body.transactions.retain(|tx| {
@@ -172,16 +172,17 @@ impl ImportOpCommand {
 
         let provider = provider_factory.provider()?;
 
-        let total_imported_blocks = provider.tx_ref().entries::<tables::Headers>()?;
+        let total_imported_blocks = provider.tx_ref().entries::<tables::HeaderNumbers>()?;
         let total_imported_txns = provider.tx_ref().entries::<tables::TransactionHashNumbers>()?;
 
         if total_decoded_blocks != total_imported_blocks ||
-            total_decoded_txns != total_imported_txns
+            total_decoded_txns != total_imported_txns + total_filtered_out_dup_txns
         {
             error!(target: "reth::cli",
                 total_decoded_blocks,
                 total_imported_blocks,
                 total_decoded_txns,
+                total_filtered_out_dup_txns,
                 total_imported_txns,
                 "Chain was partially imported"
             );

bin/reth/src/commands/import_receipts.rs

@@ -19,7 +19,7 @@ use reth_provider::{
     BundleStateWithReceipts, OriginalValuesKnown, ProviderFactory, StageCheckpointReader,
     StateWriter, StaticFileProviderFactory, StaticFileWriter,
 };
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, trace};
 
 use std::{path::PathBuf, sync::Arc};
 
@@ -89,7 +89,7 @@ impl ImportReceiptsCommand {
 
         for stage in StageId::ALL {
             let checkpoint = provider.get_stage_checkpoint(stage)?;
-            debug!(target: "reth::cli",
+            trace!(target: "reth::cli",
                 ?stage,
                 ?checkpoint,
                 "Read stage checkpoints from db"