sync with upstream reth release v1.0.1

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +1,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: GitHub Discussions
-    url: https://github.com/foundry-rs/reth/discussions
+    url: https://github.com/paradigmxyz/reth/discussions
     about: Please ask and answer questions here to keep the issue tracker clean.
-  - name: Security
-    url: mailto:[email protected]
-    about: Please report security vulnerabilities here.

.github/assets/check_no_std.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# TODO
+no_std_packages=(
+#   reth-codecs
+#   reth-consensus
+#   reth-db
+#   reth-errors
+#   reth-ethereum-forks
+#   reth-evm
+#   reth-evm-ethereum
+#   reth-network-peers
+#   reth-primitives
+#   reth-primitives-traits
+#   reth-revm
+)
+
+for package in "${no_std_packages[@]}"; do
+  cmd="cargo +stable build -p $package --target riscv32imac-unknown-none-elf --no-default-features"
+
+  if [ -n "$CI" ]; then
+    echo "::group::$cmd"
+  else
+    printf "\n%s:\n  %s\n" "$package" "$cmd"
+  fi
+
+  $cmd
+
+  if [ -n "$CI" ]; then
+    echo "::endgroup::"
+  fi
+done

.github/assets/hive/Dockerfile

@@ -0,0 +1,8 @@
+FROM ubuntu
+
+COPY dist/reth /usr/local/bin
+
+COPY LICENSE-* ./
+
+EXPOSE 30303 30303/udp 9001 8545 8546
+ENTRYPOINT ["/usr/local/bin/reth"]

.github/assets/hive/build_simulators.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# Create the hive_assets directory
+mkdir hive_assets/
+
+cd hivetests
+go build .
+
+./hive -client reth # first builds and caches the client
+
+# Run each hive command in the background for each simulator and wait
+echo "Building images"
+./hive -client reth --sim "pyspec" -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
+./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &
+./hive -client reth --sim "smoke/genesis" -sim.timelimit 1s || true &
+./hive -client reth --sim "smoke/network" -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/sync" -sim.timelimit 1s || true &
+wait
+
+# Run docker save in parallel and wait
+echo "Saving images"
+docker save hive/hiveproxy:latest -o ../hive_assets/hiveproxy.tar &
+docker save hive/simulators/devp2p:latest -o ../hive_assets/devp2p.tar &
+docker save hive/simulators/ethereum/engine:latest -o ../hive_assets/engine.tar &
+docker save hive/simulators/ethereum/rpc-compat:latest -o ../hive_assets/rpc_compat.tar &
+docker save hive/simulators/ethereum/pyspec:latest -o ../hive_assets/pyspec.tar &
+docker save hive/simulators/smoke/genesis:latest -o ../hive_assets/smoke_genesis.tar &
+docker save hive/simulators/smoke/network:latest -o ../hive_assets/smoke_network.tar &
+docker save hive/simulators/ethereum/sync:latest -o ../hive_assets/ethereum_sync.tar &
+wait
+
+# Make sure we don't rebuild images on the CI jobs
+git apply ../.github/assets/hive/no_sim_build.diff
+go build .
+mv ./hive ../hive_assets/

.github/assets/hive/expected_failures.yaml

@@ -0,0 +1,120 @@
+# https://github.com/paradigmxyz/reth/issues/7015
+# https://github.com/paradigmxyz/reth/issues/6332
+rpc-compat:
+  - debug_getRawBlock/get-invalid-number (reth)
+  - debug_getRawHeader/get-invalid-number (reth)
+  - debug_getRawReceipts/get-invalid-number (reth)
+  - debug_getRawTransaction/get-invalid-hash (reth)
+
+  - eth_call/call-callenv (reth)
+  - eth_createAccessList/create-al-contract-eip1559 (reth)
+  - eth_createAccessList/create-al-contract (reth)
+  - eth_feeHistory/fee-history (reth)
+  - eth_getStorageAt/get-storage-invalid-key-too-large (reth)
+  - eth_getStorageAt/get-storage-invalid-key (reth)
+  - eth_getTransactionReceipt/get-access-list (reth)
+  - eth_getTransactionReceipt/get-blob-tx (reth)
+  - eth_getTransactionReceipt/get-dynamic-fee (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8732
+engine-withdrawals:
+  - Withdrawals Fork On Genesis (Paris) (reth)
+  - Withdrawals Fork on Block 1 (Paris) (reth)
+  - Withdrawals Fork on Block 2 (Paris) (reth)
+  - Withdrawals Fork on Block 3 (Paris) (reth)
+  - Withdraw to a single account (Paris) (reth)
+  - Withdraw to two accounts (Paris) (reth)
+  - Withdraw many accounts (Paris) (reth)
+  - Withdraw zero amount (Paris) (reth)
+  - Empty Withdrawals (Paris) (reth)
+  - Corrupted Block Hash Payload (INVALID) (Paris) (reth)
+  - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)
+  - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth)
+  - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)
+  - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8305
+# https://github.com/paradigmxyz/reth/issues/6217
+engine-api:
+  - Inconsistent Head in ForkchoiceState (Paris) (reth)
+  - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)
+  - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)
+  - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+
+  # Hive issue
+  # https://github.com/ethereum/hive/issues/1135
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8305
+# https://github.com/paradigmxyz/reth/issues/6217
+# https://github.com/paradigmxyz/reth/issues/8306
+# https://github.com/paradigmxyz/reth/issues/7144
+engine-cancun:
+  - Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
+  - Inconsistent Head in ForkchoiceState (Cancun) (reth)
+  - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
+  - Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+
+  # Hive issue
+  # https://github.com/ethereum/hive/issues/1135
+  - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8579
+sync:
+  - sync reth -> reth

.github/assets/hive/load_images.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# List of tar files to load
+IMAGES=(
+    "/tmp/hiveproxy.tar"
+    "/tmp/devp2p.tar"
+    "/tmp/engine.tar"
+    "/tmp/rpc_compat.tar"
+    "/tmp/pyspec.tar"
+    "/tmp/smoke_genesis.tar"
+    "/tmp/smoke_network.tar"
+    "/tmp/ethereum_sync.tar"
+    "/tmp/reth_image.tar"
+)
+
+# Loop through the images and load them
+for IMAGE_TAR in "${IMAGES[@]}"; do
+    echo "Loading image $IMAGE_TAR..."
+    docker load -i "$IMAGE_TAR" &
+done
+
+wait
+
+docker image ls -a

.github/assets/hive/no_sim_build.diff

@@ -0,0 +1,53 @@
+diff --git a/internal/libdocker/builder.go b/internal/libdocker/builder.go
+index 4731c9d..d717f52 100644
+--- a/internal/libdocker/builder.go
++++ b/internal/libdocker/builder.go
+@@ -7,9 +7,7 @@ import (
+ 	"fmt"
+ 	"io"
+ 	"io/fs"
+-	"os"
+ 	"path/filepath"
+-	"strings"
+
+ 	"github.com/ethereum/hive/internal/libhive"
+ 	docker "github.com/fsouza/go-dockerclient"
+@@ -53,24 +51,8 @@ func (b *Builder) BuildClientImage(ctx context.Context, client libhive.ClientDes
+
+ // BuildSimulatorImage builds a docker image of a simulator.
+ func (b *Builder) BuildSimulatorImage(ctx context.Context, name string) (string, error) {
+-	dir := b.config.Inventory.SimulatorDirectory(name)
+-	buildContextPath := dir
+-	buildDockerfile := "Dockerfile"
+-	// build context dir of simulator can be overridden with "hive_context.txt" file containing the desired build path
+-	if contextPathBytes, err := os.ReadFile(filepath.Join(filepath.FromSlash(dir), "hive_context.txt")); err == nil {
+-		buildContextPath = filepath.Join(dir, strings.TrimSpace(string(contextPathBytes)))
+-		if strings.HasPrefix(buildContextPath, "../") {
+-			return "", fmt.Errorf("cannot access build directory outside of Hive root: %q", buildContextPath)
+-		}
+-		if p, err := filepath.Rel(buildContextPath, filepath.Join(filepath.FromSlash(dir), "Dockerfile")); err != nil {
+-			return "", fmt.Errorf("failed to derive relative simulator Dockerfile path: %v", err)
+-		} else {
+-			buildDockerfile = p
+-		}
+-	}
+ 	tag := fmt.Sprintf("hive/simulators/%s:latest", name)
+-	err := b.buildImage(ctx, buildContextPath, buildDockerfile, tag, nil)
+-	return tag, err
++	return tag, nil
+ }
+
+ // BuildImage creates a container by archiving the given file system,
+diff --git a/internal/libdocker/proxy.go b/internal/libdocker/proxy.go
+index a53e5af..0bb2ea9 100644
+--- a/internal/libdocker/proxy.go
++++ b/internal/libdocker/proxy.go
+@@ -16,7 +16,7 @@ const hiveproxyTag = "hive/hiveproxy"
+
+ // Build builds the hiveproxy image.
+ func (cb *ContainerBackend) Build(ctx context.Context, b libhive.Builder) error {
+-	return b.BuildImage(ctx, hiveproxyTag, hiveproxy.Source)
++	return nil
+ }
+
+ // ServeAPI starts the API server.

.github/assets/hive/parse.py

@@ -0,0 +1,43 @@
+import json
+import yaml
+import sys
+import argparse
+
+# Argument parser setup
+parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.")
+parser.add_argument("report_json", help="Path to the hive report JSON file.")
+parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.")
+args = parser.parse_args()
+
+# Load hive JSON
+with open(args.report_json, 'r') as file:
+    report = json.load(file)
+
+# Load exclusion YAML
+with open(args.exclusion, 'r') as file:
+    exclusion_data = yaml.safe_load(file)
+    exclusions = exclusion_data.get(report['name'], [])
+
+# Collect unexpected failures and passes
+unexpected_failures = []
+unexpected_passes = []
+
+for test in report['testCases'].values():
+    test_name = test['name']
+    test_pass = test['summaryResult']['pass']
+    if test_name in exclusions:
+        if test_pass:
+            unexpected_passes.append(test_name)
+    else:
+        if not test_pass:
+            unexpected_failures.append(test_name)
+
+# Check if there are any unexpected failures or passes and exit with error
+if unexpected_failures or unexpected_passes:
+    if unexpected_failures:
+        print("Unexpected Failures:", unexpected_failures)
+    if unexpected_passes:
+        print("Unexpected Passes:", unexpected_passes)
+    sys.exit(1)
+
+print("Success.")

.github/assets/hive/run_simulator.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# set -x
+
+cd hivetests/
+
+sim="${1}"
+limit="${2}"
+
+run_hive() {
+    hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism 4 --client reth 2>&1 | tee /tmp/log || true
+}
+
+check_log() {
+    tail -n 1 /tmp/log | sed -r 's/\x1B\[[0-9;]*[mK]//g'
+}
+
+attempt=0
+max_attempts=5
+
+while [ $attempt -lt $max_attempts ]; do
+    run_hive
+
+    # Check if no tests were run. sed removes ansi colors
+    if check_log | grep -q "suites=0"; then
+        echo "no tests were run, retrying in 5 seconds"
+        sleep 5
+        attempt=$((attempt + 1))
+        continue
+    fi
+
+    # Check the last line of the log for "finished", "tests failed", or "test failed"
+    if check_log | grep -Eq "(finished|tests? failed)"; then
+        exit 0
+    else
+        exit 1
+    fi
+done
+exit 1

.github/workflows/assertoor.yml

@@ -33,7 +33,7 @@ jobs:
 
         cat etc/assertoor/assertoor-template.yaml | envsubst > etc/assertoor/assertoor.yaml
 
-        kurtosis run github.com/kurtosis-tech/ethereum-package --enclave assertoor-${{ github.run_id }} --args-file etc/assertoor/assertoor.yaml
+        kurtosis run github.com/ethpandaops/ethereum-package --enclave assertoor-${{ github.run_id }} --args-file etc/assertoor/assertoor.yaml
 
         enclave_dump=$(kurtosis enclave inspect assertoor-${{ github.run_id }})