From 085e12b8c7f5a41dcffcc8ed9b912ef1ce7b5b32 Mon Sep 17 00:00:00 2001 From: rahul Date: Thu, 7 Nov 2024 05:52:05 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20feat:=20Migrate=20to=20state=20?= =?UTF-8?q?test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/writing_tests/types_of_tests.md | 27 +++++++++++++++++-- docs/writing_tests/writing_a_new_test.md | 2 +- src/cli/gentest/cli.py | 4 +-- .../blockchain_test/transaction.py.j2 | 13 +++------ src/cli/gentest/test_context_providers.py | 4 +-- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/docs/writing_tests/types_of_tests.md b/docs/writing_tests/types_of_tests.md index 38572790d3c..14a80ece5a8 100644 --- a/docs/writing_tests/types_of_tests.md +++ b/docs/writing_tests/types_of_tests.md @@ -7,16 +7,28 @@ There are currently two types of tests that can be produced by a test spec: ## State Tests -State tests span a single block and, ideally, a single transaction. For example: +### Purpose + +Tests the effects of individual transactions (ideally a single one) that span a single block in a controlled environment. + +### Use cases - Test a single opcode behavior. - Verify opcode gas costs. - Test interactions between multiple smart contracts. - Test creation of smart contracts. +!!! info + + The fill function will automatically generate a `blockchain_test` fixture from `state_tests`, consisting of one block and one transaction. + ## Blockchain Tests -Blockchain tests span multiple blocks which may or may not contain transactions and mainly focus on the block to block effects to the Ethereum state. For example: +### Purpose + +Blockchain tests span multiple blocks which may or may not contain transactions and mainly focus on the block to block effects to the Ethereum state. + +### Use cases - Verify system-level operations such as coinbase balance updates or withdrawals. - Verify fork transitions. @@ -38,3 +50,14 @@ def test_blob_type_tx_pre_fork( Reject blocks with blobs before blobs fork """ ``` + +## Deciding on a test type + +### Prefer `state_test` for single transactions + +Whenever possible, use `state_test` to examine individual transactions. This method is more straightforward and less prone to external +influences that can occur during block building. + +This provides more targeted testing since it does not invoke the client's block-building machinery. This reduces the risk of +encountering false positives, particularly in exception scenarios (e.g., see issue +[#343: "Zero max_fee_per_blob_gas test is ineffective"](https://github.com/ethereum/execution-spec-tests/issues/343)). diff --git a/docs/writing_tests/writing_a_new_test.md b/docs/writing_tests/writing_a_new_test.md index e5525a32695..9cb3f6123f1 100644 --- a/docs/writing_tests/writing_a_new_test.md +++ b/docs/writing_tests/writing_a_new_test.md @@ -2,7 +2,7 @@ ## Test Functions -Every test case is defined as a python function that defines a single `StateTest` or `BlockchainTest` by using one of the `state_test` or `blockchain_test` objects made available by the framework. Test cases, respectively test modules, must fulfill the following requirements: +Every test case is defined as a Python function that implements a single `StateTest` or `BlockchainTest` using the `state_test` or `blockchain_test` objects made available by the framework ([learn how to decide on a test type](./types_of_tests.md#deciding-on-a-test-type)). Test cases, and the respective test modules, must fulfill the following requirements: | Requirement | When | | -----------------------------------------------------------------------|---------------------------------------------| diff --git a/src/cli/gentest/cli.py b/src/cli/gentest/cli.py index db2e00c8ae8..5c7a7efd68c 100644 --- a/src/cli/gentest/cli.py +++ b/src/cli/gentest/cli.py @@ -13,7 +13,7 @@ from ethereum_test_base_types import Hash from .source_code_generator import get_test_source -from .test_context_providers import BlockchainTestContextProvider +from .test_context_providers import StateTestProvider @click.command() @@ -27,7 +27,7 @@ def generate(transaction_hash: str, output_file: TextIO): OUTPUT_FILE is the path to the output python script. """ - provider = BlockchainTestContextProvider(transaction_hash=Hash(transaction_hash)) + provider = StateTestProvider(transaction_hash=Hash(transaction_hash)) source = get_test_source(provider=provider, template_path="blockchain_test/transaction.py.j2") output_file.write(source) diff --git a/src/cli/gentest/templates/blockchain_test/transaction.py.j2 b/src/cli/gentest/templates/blockchain_test/transaction.py.j2 index f5a034d6686..e6a232ade7d 100644 --- a/src/cli/gentest/templates/blockchain_test/transaction.py.j2 +++ b/src/cli/gentest/templates/blockchain_test/transaction.py.j2 @@ -8,14 +8,7 @@ from typing import Dict import pytest -from ethereum_test_tools import ( - Account, - Block, - BlockchainTestFiller, - Environment, - Storage, - Transaction, -) +from ethereum_test_tools import Account, Environment, StateTestFiller, Storage, Transaction REFERENCE_SPEC_GIT_PATH = "N/A" REFERENCE_SPEC_VERSION = "N/A" @@ -28,7 +21,7 @@ def env(): # noqa: D103 @pytest.mark.valid_from("Paris") def test_transaction_{{ tx_hash }}( # noqa: SC200, E501 env: Environment, - blockchain_test: BlockchainTestFiller, + state_test: StateTestFiller, ): """ Gentest autogenerated test for tx.hash: @@ -40,4 +33,4 @@ def test_transaction_{{ tx_hash }}( # noqa: SC200, E501 tx = {{ transaction | stringify }} - blockchain_test(genesis_environment=env, pre=pre, post=post, blocks=[Block(txs=[tx])]) + state_test(env=env, pre=pre, post=post, tx=tx) \ No newline at end of file diff --git a/src/cli/gentest/test_context_providers.py b/src/cli/gentest/test_context_providers.py index 7e8cdfd5952..1315cd08801 100644 --- a/src/cli/gentest/test_context_providers.py +++ b/src/cli/gentest/test_context_providers.py @@ -37,9 +37,9 @@ def get_context(self) -> Dict: pass -class BlockchainTestContextProvider(Provider): +class StateTestProvider(Provider): """ - Provides context required to generate a `blockchain_test` using pytest. + Provides context required to generate a `state_test` using pytest. """ transaction_hash: Hash From fb5b78942ba8f574d2fd9262fcfe619bfba35b2b Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 7 Nov 2024 08:48:55 +0100 Subject: [PATCH 2/3] Update docs/writing_tests/types_of_tests.md --- docs/writing_tests/types_of_tests.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/writing_tests/types_of_tests.md b/docs/writing_tests/types_of_tests.md index 14a80ece5a8..9d330170a57 100644 --- a/docs/writing_tests/types_of_tests.md +++ b/docs/writing_tests/types_of_tests.md @@ -55,8 +55,7 @@ def test_blob_type_tx_pre_fork( ### Prefer `state_test` for single transactions -Whenever possible, use `state_test` to examine individual transactions. This method is more straightforward and less prone to external -influences that can occur during block building. +Whenever possible, use `state_test` to examine individual transactions. This method is more straightforward and less prone to external influences that can occur during block building. This provides more targeted testing since it does not invoke the client's block-building machinery. This reduces the risk of encountering false positives, particularly in exception scenarios (e.g., see issue From ce603cb8545f5ed35ee04259a22eb9a449d7aaaf Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 7 Nov 2024 08:49:23 +0100 Subject: [PATCH 3/3] Update docs/writing_tests/types_of_tests.md --- docs/writing_tests/types_of_tests.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/writing_tests/types_of_tests.md b/docs/writing_tests/types_of_tests.md index 9d330170a57..4fedb814db7 100644 --- a/docs/writing_tests/types_of_tests.md +++ b/docs/writing_tests/types_of_tests.md @@ -57,6 +57,4 @@ def test_blob_type_tx_pre_fork( Whenever possible, use `state_test` to examine individual transactions. This method is more straightforward and less prone to external influences that can occur during block building. -This provides more targeted testing since it does not invoke the client's block-building machinery. This reduces the risk of -encountering false positives, particularly in exception scenarios (e.g., see issue -[#343: "Zero max_fee_per_blob_gas test is ineffective"](https://github.com/ethereum/execution-spec-tests/issues/343)). +This provides more targeted testing since it does not invoke the client's block-building machinery. This reduces the risk of encountering false positives, particularly in exception scenarios (e.g., see issue [#343: "Zero max_fee_per_blob_gas test is ineffective"](https://github.com/ethereum/execution-spec-tests/issues/343)).