From 0f3d78eb8252c6bafbb64c3dc64638ff87c673e8 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 17 Jan 2024 18:07:35 +0000 Subject: [PATCH 01/12] Improve JumpDestAnalysis --- .../CodeAnalysis/CodeInfoTests.cs | 77 +++++++++++++++++ .../Nethermind.Evm/CodeAnalysis/CodeInfo.cs | 21 ++--- .../CodeAnalysis/JumpDestinationAnalyzer.cs | 86 ++++++++----------- .../Nethermind.Evm/IVirtualMachine.cs | 2 + .../TransactionProcessor.cs | 6 +- .../Nethermind.Evm/VirtualMachine.cs | 22 +++-- .../Nethermind.State/IWorldState.cs | 2 + .../Nethermind.State/StateProvider.cs | 3 +- src/Nethermind/Nethermind.State/WorldState.cs | 8 +- 9 files changed, 148 insertions(+), 79 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeInfoTests.cs b/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeInfoTests.cs index cf2c6e68472..a76c67e2eb6 100644 --- a/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeInfoTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeInfoTests.cs @@ -3,6 +3,8 @@ using System.Linq; using System.Reflection; +using System.Runtime.Intrinsics; + using FluentAssertions; using Nethermind.Evm.CodeAnalysis; using NUnit.Framework; @@ -165,5 +167,80 @@ public void Push1Jumpdest_Over10k() codeInfo.ValidateJump(10, false).Should().BeFalse(); codeInfo.ValidateJump(11, false).Should().BeFalse(); // 0x5b but not JUMPDEST but data } + + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + [TestCase(4)] + [TestCase(5)] + [TestCase(6)] + [TestCase(7)] + [TestCase(8)] + [TestCase(9)] + [TestCase(10)] + [TestCase(11)] + [TestCase(12)] + [TestCase(13)] + [TestCase(14)] + [TestCase(15)] + [TestCase(16)] + [TestCase(17)] + [TestCase(18)] + [TestCase(19)] + [TestCase(20)] + [TestCase(21)] + [TestCase(22)] + [TestCase(23)] + [TestCase(24)] + [TestCase(25)] + [TestCase(26)] + [TestCase(27)] + [TestCase(28)] + [TestCase(29)] + [TestCase(30)] + [TestCase(31)] + [TestCase(32)] + public void PushNJumpdest_Over10k(int n) + { + byte[] code = new byte[10_001]; + + // One vector (aligned), half vector to unalign + int i; + for (i = 0; i < Vector256.Count * 2 + Vector128.Count; i++) + { + code[i] = (byte)0x5b; + } + for (; i < Vector256.Count * 3; i++) + { + // + } + var triggerPushes = false; + for (; i < code.Length; i++) + { + if (i % (n + 1) == 0) + { + triggerPushes = true; + } + if (triggerPushes) + { + code[i] = i % (n + 1) == 0 ? (byte)(0x60 + n - 1) : (byte)0x5b; + } + } + + CodeInfo codeInfo = new(code); + + for (i = 0; i < Vector256.Count * 2 + Vector128.Count; i++) + { + codeInfo.ValidateJump(i, false).Should().BeTrue(); + } + for (; i < Vector256.Count * 3; i++) + { + codeInfo.ValidateJump(i, false).Should().BeFalse(); + } + for (; i < code.Length; i++) + { + codeInfo.ValidateJump(i, false).Should().BeFalse(); // Are 0x5b but not JUMPDEST but data + } + } } } diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs index 73d517ad78d..e13b1d6db0a 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Runtime.CompilerServices; using Nethermind.Evm.Precompiles; @@ -12,11 +11,13 @@ public class CodeInfo { public byte[] MachineCode { get; set; } public IPrecompile? Precompile { get; set; } - private JumpDestinationAnalyzer? _analyzer; + private readonly JumpDestinationAnalyzer _analyzer; + private static readonly JumpDestinationAnalyzer _emptyAnalyzer = new(Array.Empty()); public CodeInfo(byte[] code) { MachineCode = code; + _analyzer = code.Length == 0 ? _emptyAnalyzer : new JumpDestinationAnalyzer(code); } public bool IsPrecompile => Precompile is not null; @@ -25,24 +26,12 @@ public CodeInfo(IPrecompile precompile) { Precompile = precompile; MachineCode = Array.Empty(); + _analyzer = _emptyAnalyzer; } public bool ValidateJump(int destination, bool isSubroutine) { - JumpDestinationAnalyzer analyzer = _analyzer; - analyzer ??= CreateAnalyzer(); - - return analyzer.ValidateJump(destination, isSubroutine); - } - - /// - /// Do sampling to choose an algo when the code is big enough. - /// When the code size is small we can use the default analyzer. - /// - [MethodImpl(MethodImplOptions.NoInlining)] - private JumpDestinationAnalyzer CreateAnalyzer() - { - return _analyzer = new JumpDestinationAnalyzer(MachineCode); + return _analyzer.ValidateJump(destination, isSubroutine); } } } diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 8451a422ac4..abdfa617309 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -1,33 +1,21 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Threading; namespace Nethermind.Evm.CodeAnalysis { - public sealed class JumpDestinationAnalyzer : IThreadPoolWorkItem + public sealed class JumpDestinationAnalyzer(byte[] code) { private const int PUSH1 = 0x60; private const int PUSH32 = 0x7f; private const int JUMPDEST = 0x5b; private const int BEGINSUB = 0x5c; - private const int BitShiftPerInt32 = 5; + private const int BitShiftPerInt64 = 6; - private int[]? _jumpDestBitmap; - public byte[] MachineCode { get; set; } - - public JumpDestinationAnalyzer(byte[] code) - { - // Store the code refence as the JumpDest analysis is lazy - // and not performed until first jump. - MachineCode = code; - - // Start generating the JumpDestinationBitmap in background. - ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: false); - } + private long[]? _jumpDestBitmap; + public byte[] MachineCode { get; } = code; public bool ValidateJump(int destination, bool isSubroutine) { @@ -59,13 +47,13 @@ public bool ValidateJump(int destination, bool isSubroutine) /// /// Used for conversion between different representations of bit array. - /// Returns (n + (32 - 1)) / 32, rearranged to avoid arithmetic overflow. + /// Returns (n + (64 - 1)) / 64, rearranged to avoid arithmetic overflow. /// For example, in the bit to int case, the straightforward calc would - /// be (n + 31) / 32, but that would cause overflow. So instead it's - /// rearranged to ((n - 1) / 32) + 1. + /// be (n + 63) / 64, but that would cause overflow. So instead it's + /// rearranged to ((n - 1) / 64) + 1. /// Due to sign extension, we don't need to special case for n == 0, if we use - /// bitwise operations (since ((n - 1) >> 5) + 1 = 0). - /// This doesn't hold true for ((n - 1) / 32) + 1, which equals 1. + /// bitwise operations (since ((n - 1) >> 6) + 1 = 0). + /// This doesn't hold true for ((n - 1) / 64) + 1, which equals 1. /// /// Usage: /// GetInt32ArrayLengthFromBitLength(77): returns how many ints must be @@ -73,20 +61,21 @@ public bool ValidateJump(int destination, bool isSubroutine) /// /// /// how many ints are required to store n bytes - private static int GetInt32ArrayLengthFromBitLength(int n) + private static int GetInt64ArrayLengthFromBitLength(int n) { - return (int)((uint)(n - 1 + (1 << BitShiftPerInt32)) >> BitShiftPerInt32); + return (int)((uint)(n - 1 + (1 << BitShiftPerInt64)) >> BitShiftPerInt64); } /// /// Collects data locations in code. /// An unset bit means the byte is an opcode, a set bit means it's data. /// - private static int[] CreateJumpDestinationBitmap(byte[] code) + private static long[] CreateJumpDestinationBitmap(byte[] code) { - int[] jumpDestBitmap = new int[GetInt32ArrayLengthFromBitLength(code.Length)]; + long[] jumpDestBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)]; int pc = 0; + long flags = 0; while (true) { // Since we are using a non-standard for loop here; @@ -97,20 +86,29 @@ private static int[] CreateJumpDestinationBitmap(byte[] code) // Grab the instruction from the code. int op = code[pc]; - if (op >= PUSH1 && op <= PUSH32) + int move = 1; + if ((uint)op - JUMPDEST <= BEGINSUB - JUMPDEST) + { + // Accumulate JumpDest to register + flags |= 1L << pc; + } + else if ((uint)op - PUSH1 <= PUSH32 - PUSH1) { // Skip forward amount of data the push represents // don't need to analyse data for JumpDests - pc += op - PUSH1 + 1; + move = op - PUSH1 + 2; } - else if (op == JUMPDEST || op == BEGINSUB) + + int next = pc + move; + if ((pc & 0x3F) + move > 0x3f || next >= code.Length) { - // Exact type will be checked again by ValidateJump - MarkAsJumpDestination(jumpDestBitmap, pc); + // Moving to next array element (or finishing) assign to array. + MarkJumpDestinations(jumpDestBitmap, pc, flags); + flags = 0; } // Next instruction - pc++; + pc = next; } return jumpDestBitmap; @@ -119,33 +117,21 @@ private static int[] CreateJumpDestinationBitmap(byte[] code) /// /// Checks if the position is in a code segment. /// - private static bool IsJumpDestination(int[] bitvec, int pos) + private static bool IsJumpDestination(long[] bitvec, int pos) { - int vecIndex = pos >> BitShiftPerInt32; + int vecIndex = pos >> BitShiftPerInt64; // Check if in bounds, Jit will add slightly more expensive exception throwing check if we don't if ((uint)vecIndex >= (uint)bitvec.Length) return false; - return (bitvec[vecIndex] & (1 << pos)) != 0; + return (bitvec[vecIndex] & (1L << pos)) != 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void MarkAsJumpDestination(int[] bitvec, int pos) - { - int vecIndex = pos >> BitShiftPerInt32; - Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bitvec), vecIndex) - |= 1 << pos; - } - - void IThreadPoolWorkItem.Execute() + private static void MarkJumpDestinations(long[] bitvec, int pos, long flags) { - if (_jumpDestBitmap is null) - { - var jumpDestBitmap = CreateJumpDestinationBitmap(MachineCode); - // Atomically assign if still null. Aren't really any thread safety issues here, - // as will be same result. Just keep first one we created; as Evm will have started - // using it if already created and let this one be Gen0 GC'd. - Interlocked.CompareExchange(ref _jumpDestBitmap, jumpDestBitmap, null); - } + uint offset = (uint)pos >> BitShiftPerInt64; + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bitvec), offset) + |= flags; } } } diff --git a/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs b/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs index 2a8188828b3..2aaf01f4e22 100644 --- a/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Evm.CodeAnalysis; using Nethermind.Evm.Tracing; @@ -17,5 +18,6 @@ TransactionSubstate Run(EvmState state, IWorldState worldState, where TTracingActions : struct, IIsTracing; CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec spec); + void CacheCodeInfo(Hash256 codeHash, CodeInfo cachedCodeInfo); } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs index 90d039ff7eb..502acb61bcf 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs @@ -552,7 +552,11 @@ protected virtual bool ExecuteEvmCall( if (unspentGas >= codeDepositGasCost) { - WorldState.InsertCode(env.ExecutingAccount, substate.Output, spec); + var code = substate.Output.ToArray(); + Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.AsSpan()); + WorldState.InsertCode(env.ExecutingAccount, codeHash, code, spec); + VirtualMachine.CacheCodeInfo(codeHash, new CodeInfo(code)); + unspentGas -= codeDepositGasCost; } } diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index d388f58a775..c20b043797c 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -59,6 +59,9 @@ public VirtualMachine( public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec spec) => _evm.GetCachedCodeInfo(worldState, codeSource, spec); + public void CacheCodeInfo(Hash256 codeHash, CodeInfo cachedCodeInfo) + => _evm.CacheCodeInfo(codeHash, cachedCodeInfo); + public TransactionSubstate Run(EvmState state, IWorldState worldState, ITxTracer txTracer) where TTracingActions : struct, IIsTracing => _evm.Run(state, worldState, txTracer); @@ -354,7 +357,12 @@ public TransactionSubstate Run(EvmState state, IWorldState worl bool invalidCode = CodeDepositHandler.CodeIsInvalid(spec, callResult.Output); if (gasAvailableForCodeDeposit >= codeDepositGasCost && !invalidCode) { - _state.InsertCode(callCodeOwner, callResult.Output, spec); + var code = callResult.Output; + + Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.AsSpan()); + _state.InsertCode(callCodeOwner, codeHash, callResult.Output, spec); + _codeCache.Set(codeHash, new CodeInfo(code)); + currentState.GasAvailable -= codeDepositGasCost; if (typeof(TTracingActions) == typeof(IsTracing)) @@ -475,6 +483,9 @@ private void RevertParityTouchBugAccount(IReleaseSpec spec) } } + public void CacheCodeInfo(Hash256 codeHash, CodeInfo cachedCodeInfo) + => _codeCache.Set(codeHash, cachedCodeInfo); + public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec vmSpec) { if (codeSource.IsPrecompile(vmSpec)) @@ -2524,14 +2535,7 @@ private EvmExceptionType InstructionSelfDestruct(EvmState vmState, ref _state.SubtractFromBalance(env.ExecutingAccount, value, spec); - ValueHash256 codeHash = ValueKeccak.Compute(initCode); - // Prefer code from code cache (e.g. if create from a factory contract or copypasta) - if (!_codeCache.TryGet(codeHash, out CodeInfo codeInfo)) - { - codeInfo = new(initCode.ToArray()); - // Prime the code cache as likely to be used by more txs - _codeCache.Set(codeHash, codeInfo); - } + CodeInfo codeInfo = new(initCode.ToArray()); ExecutionEnvironment callEnv = new ( diff --git a/src/Nethermind/Nethermind.State/IWorldState.cs b/src/Nethermind/Nethermind.State/IWorldState.cs index 98c85d14503..b74386936f8 100644 --- a/src/Nethermind/Nethermind.State/IWorldState.cs +++ b/src/Nethermind/Nethermind.State/IWorldState.cs @@ -85,6 +85,8 @@ public interface IWorldState : IJournal, IReadOnlyStateProvider void InsertCode(Address address, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false); + void InsertCode(Address address, Hash256 codeHash, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false); + void AddToBalance(Address address, in UInt256 balanceChange, IReleaseSpec spec); void AddToBalanceAndCreateIfNotExists(Address address, in UInt256 balanceChange, IReleaseSpec spec); diff --git a/src/Nethermind/Nethermind.State/StateProvider.cs b/src/Nethermind/Nethermind.State/StateProvider.cs index ea04bc1dbc1..5fd453c0ed9 100644 --- a/src/Nethermind/Nethermind.State/StateProvider.cs +++ b/src/Nethermind/Nethermind.State/StateProvider.cs @@ -146,10 +146,9 @@ public UInt256 GetBalance(Address address) return account?.Balance ?? UInt256.Zero; } - public void InsertCode(Address address, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false) + public void InsertCode(Address address, Hash256 codeHash, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false) { _needsStateRootUpdate = true; - Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.Span); // Don't reinsert if already inserted. This can be the case when the same // code is used by multiple deployments. Either from factory contracts (e.g. LPs) diff --git a/src/Nethermind/Nethermind.State/WorldState.cs b/src/Nethermind/Nethermind.State/WorldState.cs index aaeaf4bab1d..abd3d358afb 100644 --- a/src/Nethermind/Nethermind.State/WorldState.cs +++ b/src/Nethermind/Nethermind.State/WorldState.cs @@ -108,8 +108,14 @@ public void CreateAccount(Address address, in UInt256 balance, in UInt256 nonce } public void InsertCode(Address address, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false) { - _stateProvider.InsertCode(address, code, spec, isGenesis); + Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.Span); + InsertCode(address, codeHash, code, spec, isGenesis); } + public void InsertCode(Address address, Hash256 codeHash, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false) + { + _stateProvider.InsertCode(address, codeHash, code, spec, isGenesis); + } + public void AddToBalance(Address address, in UInt256 balanceChange, IReleaseSpec spec) { _stateProvider.AddToBalance(address, balanceChange, spec); From 3c9450a6f35f49b32a8d1ab251c20a2bdc82c262 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 17 Jan 2024 21:40:05 +0000 Subject: [PATCH 02/12] Vectorize --- .../Nethermind.Evm/CodeAnalysis/CodeInfo.cs | 1 + .../CodeAnalysis/JumpDestinationAnalyzer.cs | 28 +++++++++++++++++-- .../Nethermind.Evm/VirtualMachine.cs | 8 +++++- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs index e13b1d6db0a..da6f2333e0c 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs @@ -13,6 +13,7 @@ public class CodeInfo public IPrecompile? Precompile { get; set; } private readonly JumpDestinationAnalyzer _analyzer; private static readonly JumpDestinationAnalyzer _emptyAnalyzer = new(Array.Empty()); + public static CodeInfo Empty { get; } = new CodeInfo(Array.Empty()); public CodeInfo(byte[] code) { diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index abdfa617309..97864e66756 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -1,19 +1,25 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace Nethermind.Evm.CodeAnalysis { public sealed class JumpDestinationAnalyzer(byte[] code) { private const int PUSH1 = 0x60; + private const int PUSHx = PUSH1 - 1; private const int PUSH32 = 0x7f; private const int JUMPDEST = 0x5b; private const int BEGINSUB = 0x5c; private const int BitShiftPerInt64 = 6; + private readonly static long[]? _emptyJumpDestBitmap = new long[1]; + private long[]? _jumpDestBitmap; public byte[] MachineCode { get; } = code; @@ -72,21 +78,37 @@ private static int GetInt64ArrayLengthFromBitLength(int n) /// private static long[] CreateJumpDestinationBitmap(byte[] code) { + if (code.Length == 0) return _emptyJumpDestBitmap; + long[] jumpDestBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)]; int pc = 0; long flags = 0; while (true) { + int move = 1; + if (Avx2.IsSupported && (pc & 0x1f) == 0 && pc < code.Length - Vector256.Count) + { + Vector256 data = Unsafe.As>(ref Unsafe.AddByteOffset(ref MemoryMarshal.GetArrayDataReference(code), pc)); + Vector256 compare = Avx2.CompareGreaterThan(data, Vector256.Create((sbyte)PUSHx)); + if (compare == default) + { + Vector256 dest = Avx2.CompareEqual(data, Vector256.Create((sbyte)JUMPDEST)); + Vector256 sub = Avx2.CompareEqual(data, Vector256.Create((sbyte)BEGINSUB)); + Vector256 combined = Avx2.Or(dest, sub); + flags |= (long)Avx2.MoveMask(combined) << (pc & 0x20); + move = Vector256.Count; + goto Next; + } + } + // Since we are using a non-standard for loop here; // changing to while(true) plus below if check elides // the bounds check from the following code array access. if ((uint)pc >= (uint)code.Length) break; - // Grab the instruction from the code. int op = code[pc]; - int move = 1; if ((uint)op - JUMPDEST <= BEGINSUB - JUMPDEST) { // Accumulate JumpDest to register @@ -98,7 +120,7 @@ private static long[] CreateJumpDestinationBitmap(byte[] code) // don't need to analyse data for JumpDests move = op - PUSH1 + 2; } - + Next: int next = pc + move; if ((pc & 0x3F) + move > 0x3f || next >= code.Length) { diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index c20b043797c..4c102dff2ec 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -498,8 +498,14 @@ public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IR return _precompiles[codeSource]; } + CodeInfo cachedCodeInfo = null; Hash256 codeHash = worldState.GetCodeHash(codeSource); - CodeInfo cachedCodeInfo = _codeCache.Get(codeHash); + if (ReferenceEquals(codeHash, Keccak.OfAnEmptyString)) + { + cachedCodeInfo = CodeInfo.Empty; + } + + cachedCodeInfo ??= _codeCache.Get(codeHash); if (cachedCodeInfo is null) { byte[] code = worldState.GetCode(codeHash); From f2f0ff8998ddf8f6805c9d5b3e3e110a89011a80 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 18 Jan 2024 05:53:06 +0000 Subject: [PATCH 03/12] Process fresh code in background --- .../Nethermind.Evm/CodeAnalysis/CodeInfo.cs | 8 ++++- .../CodeAnalysis/JumpDestinationAnalyzer.cs | 33 ++++++++++++------- .../TransactionProcessor.cs | 8 ++++- .../Nethermind.Evm/VirtualMachine.cs | 7 +++- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs index da6f2333e0c..a7e939a05dd 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs @@ -2,12 +2,13 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Threading; using Nethermind.Evm.Precompiles; namespace Nethermind.Evm.CodeAnalysis { - public class CodeInfo + public class CodeInfo : IThreadPoolWorkItem { public byte[] MachineCode { get; set; } public IPrecompile? Precompile { get; set; } @@ -34,5 +35,10 @@ public bool ValidateJump(int destination, bool isSubroutine) { return _analyzer.ValidateJump(destination, isSubroutine); } + + void IThreadPoolWorkItem.Execute() + { + _analyzer.Execute(); + } } } diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 97864e66756..367a008d475 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -19,8 +19,8 @@ public sealed class JumpDestinationAnalyzer(byte[] code) private const int BitShiftPerInt64 = 6; private readonly static long[]? _emptyJumpDestBitmap = new long[1]; + private long[]? _jumpDestBitmap = code.Length == 0 ? _emptyJumpDestBitmap : null; - private long[]? _jumpDestBitmap; public byte[] MachineCode { get; } = code; public bool ValidateJump(int destination, bool isSubroutine) @@ -78,10 +78,9 @@ private static int GetInt64ArrayLengthFromBitLength(int n) /// private static long[] CreateJumpDestinationBitmap(byte[] code) { - if (code.Length == 0) return _emptyJumpDestBitmap; - long[] jumpDestBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)]; + bool exit = false; int pc = 0; long flags = 0; while (true) @@ -102,12 +101,8 @@ private static long[] CreateJumpDestinationBitmap(byte[] code) } } - // Since we are using a non-standard for loop here; - // changing to while(true) plus below if check elides - // the bounds check from the following code array access. - if ((uint)pc >= (uint)code.Length) break; // Grab the instruction from the code. - int op = code[pc]; + int op = Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(code), pc); if ((uint)op - JUMPDEST <= BEGINSUB - JUMPDEST) { @@ -122,11 +117,20 @@ private static long[] CreateJumpDestinationBitmap(byte[] code) } Next: int next = pc + move; - if ((pc & 0x3F) + move > 0x3f || next >= code.Length) + exit = next >= code.Length; + if ((pc & 0x3F) + move > 0x3f || exit) { - // Moving to next array element (or finishing) assign to array. - MarkJumpDestinations(jumpDestBitmap, pc, flags); - flags = 0; + if (flags != 0) + { + // Moving to next array element (or finishing) assign to array. + MarkJumpDestinations(jumpDestBitmap, pc, flags); + flags = 0; + } + } + + if (exit) + { + break; } // Next instruction @@ -155,5 +159,10 @@ private static void MarkJumpDestinations(long[] bitvec, int pos, long flags) Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bitvec), offset) |= flags; } + + public void Execute() + { + _jumpDestBitmap ??= CreateJumpDestinationBitmap(MachineCode); + } } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs index 502acb61bcf..7e0a8c78c74 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs @@ -5,6 +5,8 @@ using System.IO; using System.Linq; using System.Runtime.CompilerServices; +using System.Threading; + using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; @@ -553,9 +555,13 @@ protected virtual bool ExecuteEvmCall( if (unspentGas >= codeDepositGasCost) { var code = substate.Output.ToArray(); + var codeInfo = new CodeInfo(code); + // Start generating the JumpDestinationBitmap in background. + ThreadPool.UnsafeQueueUserWorkItem(codeInfo, preferLocal: false); + Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.AsSpan()); WorldState.InsertCode(env.ExecutingAccount, codeHash, code, spec); - VirtualMachine.CacheCodeInfo(codeHash, new CodeInfo(code)); + VirtualMachine.CacheCodeInfo(codeHash, codeInfo); unspentGas -= codeDepositGasCost; } diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index 4c102dff2ec..fe50b0ea75c 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -32,6 +32,8 @@ namespace Nethermind.Evm; using System.Linq; +using System.Threading; + using Int256; public class VirtualMachine : IVirtualMachine @@ -358,10 +360,13 @@ public TransactionSubstate Run(EvmState state, IWorldState worl if (gasAvailableForCodeDeposit >= codeDepositGasCost && !invalidCode) { var code = callResult.Output; + var codeInfo = new CodeInfo(code); + // Start generating the JumpDestinationBitmap in background. + ThreadPool.UnsafeQueueUserWorkItem(codeInfo, preferLocal: false); Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.AsSpan()); _state.InsertCode(callCodeOwner, codeHash, callResult.Output, spec); - _codeCache.Set(codeHash, new CodeInfo(code)); + _codeCache.Set(codeHash, codeInfo); currentState.GasAvailable -= codeDepositGasCost; From 39064aff2207e22c78aa5d1d5b9b7039c895bfab Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 18 Jan 2024 07:44:21 +0000 Subject: [PATCH 04/12] Operate on initcode inplace rather than take copy --- .../Nethermind.Evm/ByteArrayExtensions.cs | 10 +++++ .../Nethermind.Evm/CodeAnalysis/CodeInfo.cs | 8 +++- .../CodeAnalysis/JumpDestinationAnalyzer.cs | 37 ++++++++++--------- src/Nethermind/Nethermind.Evm/EvmStack.cs | 2 +- .../Nethermind.Evm/VirtualMachine.cs | 16 ++++---- 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs b/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs index ec6368ac113..b4ce8e60d6c 100644 --- a/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs +++ b/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs @@ -44,6 +44,16 @@ public static ZeroPaddedSpan SliceWithZeroPadding(this Span span, scoped i return SliceWithZeroPadding(span, (int)startIndex, length, padDirection); } + public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlySpan span, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) + { + if (startIndex >= span.Length || startIndex > int.MaxValue) + { + return new ZeroPaddedSpan(default, length, PadDirection.Right); + } + + return SliceWithZeroPadding(span, (int)startIndex, length, padDirection); + } + public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlyMemory bytes, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) { if (startIndex >= bytes.Length || startIndex > int.MaxValue) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs index a7e939a05dd..6d167abac09 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs @@ -10,7 +10,7 @@ namespace Nethermind.Evm.CodeAnalysis { public class CodeInfo : IThreadPoolWorkItem { - public byte[] MachineCode { get; set; } + public ReadOnlyMemory MachineCode { get; } public IPrecompile? Precompile { get; set; } private readonly JumpDestinationAnalyzer _analyzer; private static readonly JumpDestinationAnalyzer _emptyAnalyzer = new(Array.Empty()); @@ -22,6 +22,12 @@ public CodeInfo(byte[] code) _analyzer = code.Length == 0 ? _emptyAnalyzer : new JumpDestinationAnalyzer(code); } + public CodeInfo(ReadOnlyMemory code) + { + MachineCode = code; + _analyzer = code.Length == 0 ? _emptyAnalyzer : new JumpDestinationAnalyzer(code); + } + public bool IsPrecompile => Precompile is not null; public CodeInfo(IPrecompile precompile) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 367a008d475..86f7e18259d 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Numerics; +using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -9,7 +9,7 @@ namespace Nethermind.Evm.CodeAnalysis { - public sealed class JumpDestinationAnalyzer(byte[] code) + public sealed class JumpDestinationAnalyzer(ReadOnlyMemory code) { private const int PUSH1 = 0x60; private const int PUSHx = PUSH1 - 1; @@ -21,30 +21,32 @@ public sealed class JumpDestinationAnalyzer(byte[] code) private readonly static long[]? _emptyJumpDestBitmap = new long[1]; private long[]? _jumpDestBitmap = code.Length == 0 ? _emptyJumpDestBitmap : null; - public byte[] MachineCode { get; } = code; + public ReadOnlyMemory MachineCode { get; } = code; public bool ValidateJump(int destination, bool isSubroutine) { // Take array ref to local so Jit knows its size won't change in the method. - byte[] machineCode = MachineCode; + ReadOnlySpan machineCode = MachineCode.Span; _jumpDestBitmap ??= CreateJumpDestinationBitmap(machineCode); var result = false; // Cast to uint to change negative numbers to very int high numbers // Then do length check, this both reduces check by 1 and eliminates the bounds // check from accessing the array. - if ((uint)destination < (uint)machineCode.Length && - IsJumpDestination(_jumpDestBitmap, destination)) + if ((uint)destination < (uint)machineCode.Length) { // Store byte to int, as less expensive operations at word size int codeByte = machineCode[destination]; - if (isSubroutine) + if (IsJumpDestination(_jumpDestBitmap, destination)) { - result = codeByte == BEGINSUB; - } - else - { - result = codeByte == JUMPDEST; + if (isSubroutine) + { + result = codeByte == BEGINSUB; + } + else + { + result = codeByte == JUMPDEST; + } } } @@ -76,11 +78,10 @@ private static int GetInt64ArrayLengthFromBitLength(int n) /// Collects data locations in code. /// An unset bit means the byte is an opcode, a set bit means it's data. /// - private static long[] CreateJumpDestinationBitmap(byte[] code) + private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) { long[] jumpDestBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)]; - bool exit = false; int pc = 0; long flags = 0; while (true) @@ -88,7 +89,7 @@ private static long[] CreateJumpDestinationBitmap(byte[] code) int move = 1; if (Avx2.IsSupported && (pc & 0x1f) == 0 && pc < code.Length - Vector256.Count) { - Vector256 data = Unsafe.As>(ref Unsafe.AddByteOffset(ref MemoryMarshal.GetArrayDataReference(code), pc)); + Vector256 data = Unsafe.As>(ref Unsafe.AddByteOffset(ref MemoryMarshal.GetReference(code), pc)); Vector256 compare = Avx2.CompareGreaterThan(data, Vector256.Create((sbyte)PUSHx)); if (compare == default) { @@ -102,7 +103,7 @@ private static long[] CreateJumpDestinationBitmap(byte[] code) } // Grab the instruction from the code. - int op = Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(code), pc); + int op = Unsafe.Add(ref MemoryMarshal.GetReference(code), pc); if ((uint)op - JUMPDEST <= BEGINSUB - JUMPDEST) { @@ -117,7 +118,7 @@ private static long[] CreateJumpDestinationBitmap(byte[] code) } Next: int next = pc + move; - exit = next >= code.Length; + bool exit = next >= code.Length; if ((pc & 0x3F) + move > 0x3f || exit) { if (flags != 0) @@ -162,7 +163,7 @@ private static void MarkJumpDestinations(long[] bitvec, int pos, long flags) public void Execute() { - _jumpDestBitmap ??= CreateJumpDestinationBitmap(MachineCode); + _jumpDestBitmap ??= CreateJumpDestinationBitmap(MachineCode.Span); } } } diff --git a/src/Nethermind/Nethermind.Evm/EvmStack.cs b/src/Nethermind/Nethermind.Evm/EvmStack.cs index b59af410489..0256eda50e2 100644 --- a/src/Nethermind/Nethermind.Evm/EvmStack.cs +++ b/src/Nethermind/Nethermind.Evm/EvmStack.cs @@ -333,7 +333,7 @@ public byte PopByte() return _bytes[Head * WordSize + WordSize - sizeof(byte)]; } - public void PushLeftPaddedBytes(Span value, int paddingLength) + public void PushLeftPaddedBytes(ReadOnlySpan value, int paddingLength) { if (typeof(TTracing) == typeof(IsTracing)) _tracer.ReportStackPush(value); diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index fe50b0ea75c..b5d98a0a3ec 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -234,7 +234,7 @@ public TransactionSubstate Run(EvmState state, IWorldState worl if (typeof(TTracingActions) == typeof(IsTracing) && !currentState.IsContinuation) { _txTracer.ReportAction(currentState.GasAvailable, currentState.Env.Value, currentState.From, currentState.To, currentState.ExecutionType.IsAnyCreate() ? currentState.Env.CodeInfo.MachineCode : currentState.Env.InputData, currentState.ExecutionType); - if (_txTracer.IsTracingCode) _txTracer.ReportByteCode(currentState.Env.CodeInfo.MachineCode); + if (_txTracer.IsTracingCode) _txTracer.ReportByteCode(currentState.Env.CodeInfo.MachineCode.ToArray()); } if (!_txTracer.IsTracingInstructions) @@ -796,7 +796,7 @@ private CallResult ExecuteCode code = env.CodeInfo.MachineCode.AsSpan(); + ReadOnlySpan code = env.CodeInfo.MachineCode.Span; EvmExceptionType exceptionType = EvmExceptionType.None; bool isRevert = false; #if DEBUG @@ -1422,7 +1422,7 @@ private CallResult ExecuteCode externalCode = GetCachedCodeInfo(_worldState, address, spec).MachineCode; slice = externalCode.SliceWithZeroPadding(b, (int)result); vmState.Memory.Save(in a, in slice); if (typeof(TTracingInstructions) == typeof(IsTracing)) @@ -2192,8 +2192,8 @@ private CallResult ExecuteCode(Address address, ref EvmStack stack, IReleaseSpec spec) where TTracingInstructions : struct, IIsTracing { - byte[] accountCode = GetCachedCodeInfo(_worldState, address, spec).MachineCode; - UInt256 result = (UInt256)accountCode.Length; + int codeLength = GetCachedCodeInfo(_worldState, address, spec).MachineCode.Span.Length; + UInt256 result = (UInt256)codeLength; stack.PushUInt256(in result); } @@ -2485,7 +2485,7 @@ private EvmExceptionType InstructionSelfDestruct(EvmState vmState, ref return (EvmExceptionType.None, null); } - Span initCode = vmState.Memory.LoadSpan(in memoryPositionOfInitCode, initCodeLength); + ReadOnlyMemory initCode = vmState.Memory.Load(in memoryPositionOfInitCode, initCodeLength); UInt256 balance = _state.GetBalance(env.ExecutingAccount); if (value > balance) @@ -2512,7 +2512,7 @@ private EvmExceptionType InstructionSelfDestruct(EvmState vmState, ref Address contractAddress = instruction == Instruction.CREATE ? ContractAddress.From(env.ExecutingAccount, _state.GetNonce(env.ExecutingAccount)) - : ContractAddress.From(env.ExecutingAccount, salt, initCode); + : ContractAddress.From(env.ExecutingAccount, salt, initCode.Span); if (spec.UseHotAndColdStorage) { @@ -2546,7 +2546,7 @@ private EvmExceptionType InstructionSelfDestruct(EvmState vmState, ref _state.SubtractFromBalance(env.ExecutingAccount, value, spec); - CodeInfo codeInfo = new(initCode.ToArray()); + CodeInfo codeInfo = new(initCode); ExecutionEnvironment callEnv = new ( From c88129a456e219db41410a35b2dc858113b7f204 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 18 Jan 2024 08:29:42 +0000 Subject: [PATCH 05/12] Add comment --- src/Nethermind/Nethermind.Evm/VirtualMachine.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index b5d98a0a3ec..525eadc43d8 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -2546,6 +2546,9 @@ private EvmExceptionType InstructionSelfDestruct(EvmState vmState, ref _state.SubtractFromBalance(env.ExecutingAccount, value, spec); + // Do not add the initCode to the cache as it is + // pointing to data in this tx and will become invalid + // for another tx as returned to pool. CodeInfo codeInfo = new(initCode); ExecutionEnvironment callEnv = new From 191031186d4e233ecb32bac0546bd0c448ffbe0c Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 18 Jan 2024 09:39:20 +0000 Subject: [PATCH 06/12] Feedback --- .../Nethermind.Evm/ByteArrayExtensions.cs | 27 ++--- .../CodeAnalysis/JumpDestinationAnalyzer.cs | 102 +++++++++++------- 2 files changed, 70 insertions(+), 59 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs b/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs index b4ce8e60d6c..18a52c5c32b 100644 --- a/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs +++ b/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs @@ -36,32 +36,23 @@ private static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlySpan span, public static ZeroPaddedSpan SliceWithZeroPadding(this Span span, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) { - if (startIndex >= span.Length || startIndex > int.MaxValue) - { - return new ZeroPaddedSpan(default, length, PadDirection.Right); - } - - return SliceWithZeroPadding(span, (int)startIndex, length, padDirection); + return startIndex >= span.Length || startIndex > int.MaxValue + ? new ZeroPaddedSpan(default, length, PadDirection.Right) + : SliceWithZeroPadding(span, (int)startIndex, length, padDirection); } public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlySpan span, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) { - if (startIndex >= span.Length || startIndex > int.MaxValue) - { - return new ZeroPaddedSpan(default, length, PadDirection.Right); - } - - return SliceWithZeroPadding(span, (int)startIndex, length, padDirection); + return startIndex >= span.Length || startIndex > int.MaxValue + ? new ZeroPaddedSpan(default, length, PadDirection.Right) + : SliceWithZeroPadding(span, (int)startIndex, length, padDirection); } public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlyMemory bytes, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) { - if (startIndex >= bytes.Length || startIndex > int.MaxValue) - { - return new ZeroPaddedSpan(default, length, PadDirection.Right); - } - - return SliceWithZeroPadding(bytes.Span, (int)startIndex, length, padDirection); + return startIndex >= bytes.Length || startIndex > int.MaxValue + ? new ZeroPaddedSpan(default, length, PadDirection.Right) + : SliceWithZeroPadding(bytes.Span, (int)startIndex, length, padDirection); } public static ZeroPaddedSpan SliceWithZeroPadding(this byte[] bytes, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) => diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 86f7e18259d..d86ed0d1be3 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -13,40 +13,33 @@ public sealed class JumpDestinationAnalyzer(ReadOnlyMemory code) { private const int PUSH1 = 0x60; private const int PUSHx = PUSH1 - 1; - private const int PUSH32 = 0x7f; private const int JUMPDEST = 0x5b; private const int BEGINSUB = 0x5c; private const int BitShiftPerInt64 = 6; - private readonly static long[]? _emptyJumpDestBitmap = new long[1]; - private long[]? _jumpDestBitmap = code.Length == 0 ? _emptyJumpDestBitmap : null; + private readonly static long[]? _emptyJumpDestinationBitmap = new long[1]; + private long[]? _jumpDestinationBitmap = code.Length == 0 ? _emptyJumpDestinationBitmap : null; public ReadOnlyMemory MachineCode { get; } = code; public bool ValidateJump(int destination, bool isSubroutine) { - // Take array ref to local so Jit knows its size won't change in the method. ReadOnlySpan machineCode = MachineCode.Span; - _jumpDestBitmap ??= CreateJumpDestinationBitmap(machineCode); + _jumpDestinationBitmap ??= CreateJumpDestinationBitmap(machineCode); var result = false; // Cast to uint to change negative numbers to very int high numbers // Then do length check, this both reduces check by 1 and eliminates the bounds - // check from accessing the array. + // check from accessing the span. if ((uint)destination < (uint)machineCode.Length) { // Store byte to int, as less expensive operations at word size int codeByte = machineCode[destination]; - if (IsJumpDestination(_jumpDestBitmap, destination)) + if (IsJumpDestination(_jumpDestinationBitmap, destination)) { - if (isSubroutine) - { - result = codeByte == BEGINSUB; - } - else - { - result = codeByte == JUMPDEST; - } + result = isSubroutine ? + codeByte == BEGINSUB : + codeByte == JUMPDEST; } } @@ -80,65 +73,91 @@ private static int GetInt64ArrayLengthFromBitLength(int n) /// private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) { - long[] jumpDestBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)]; + long[] jumpDestinationBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)]; - int pc = 0; - long flags = 0; + int programCounter = 0; + // We accumulate each array segment to a register and then flush to memory when we move to next. + long currentFlags = 0; while (true) { + // Set default programCounter increment to 1 for default case when don't vectorize or read a PUSH. int move = 1; - if (Avx2.IsSupported && (pc & 0x1f) == 0 && pc < code.Length - Vector256.Count) + if (Avx2.IsSupported && + // Check not going to read passed end of code. + programCounter <= code.Length - Vector256.Count && + // Are we on an int stride, one or other half of the long flags? + (programCounter & 31) == 0) { - Vector256 data = Unsafe.As>(ref Unsafe.AddByteOffset(ref MemoryMarshal.GetReference(code), pc)); + Vector256 data = Unsafe.As>(ref Unsafe.AddByteOffset(ref MemoryMarshal.GetReference(code), programCounter)); + // Pushes are 0x60 to 0x7f; converting to signed bytes any instruction higher than PUSH32 + // becomes negative so we can just do a single greater than test to see if any present. Vector256 compare = Avx2.CompareGreaterThan(data, Vector256.Create((sbyte)PUSHx)); if (compare == default) { + // Check the bytes for any JUMPDESTs. Vector256 dest = Avx2.CompareEqual(data, Vector256.Create((sbyte)JUMPDEST)); + // Check the bytes for any BEGINSUBs. Vector256 sub = Avx2.CompareEqual(data, Vector256.Create((sbyte)BEGINSUB)); + // Merge the two results. Vector256 combined = Avx2.Or(dest, sub); - flags |= (long)Avx2.MoveMask(combined) << (pc & 0x20); + // Extract the checks as a set of int flags. + int flags = Avx2.MoveMask(combined); + // Shift up flags by depending which side of long we are on, and merge to current set. + currentFlags |= (long)flags << (programCounter & 32); + // Forward programCounter by Vector256 stride. move = Vector256.Count; goto Next; } } - // Grab the instruction from the code. - int op = Unsafe.Add(ref MemoryMarshal.GetReference(code), pc); + // Grab the instruction from the code; zero length code + // doesn't enter this method and we check at end of loop if + // hit the last element and should exit, so skip bounds check + // access here. + int op = Unsafe.Add(ref MemoryMarshal.GetReference(code), programCounter); if ((uint)op - JUMPDEST <= BEGINSUB - JUMPDEST) { - // Accumulate JumpDest to register - flags |= 1L << pc; + // Accumulate Jump Destinations to register, shift will wrap and single bit + // so can shift by the whole programCounter. + currentFlags |= 1L << programCounter; } - else if ((uint)op - PUSH1 <= PUSH32 - PUSH1) + else if ((sbyte)op > (sbyte)PUSHx) { - // Skip forward amount of data the push represents - // don't need to analyse data for JumpDests + // Fast forward programCounter by the amount of data the push + // represents as don't need to analyse data for Jump Destinations. move = op - PUSH1 + 2; } + Next: - int next = pc + move; - bool exit = next >= code.Length; - if ((pc & 0x3F) + move > 0x3f || exit) + int nextCounter = programCounter + move; + // Check if read last item of code; we want to write this out also even if not + // at a boundary and then we will return the results. + bool exit = nextCounter >= code.Length; + // Does the move mean we are moving to new segment of the long array? + if ((programCounter & 63) + move > 63 || exit) { - if (flags != 0) + // If so write out the flags (if any are set) + if (currentFlags != 0) { // Moving to next array element (or finishing) assign to array. - MarkJumpDestinations(jumpDestBitmap, pc, flags); - flags = 0; + MarkJumpDestinations(jumpDestinationBitmap, programCounter, currentFlags); + // Clear the flags in preparation for the next array segment. + currentFlags = 0; } } if (exit) { + // End of code. break; } - // Next instruction - pc = next; + // Move to next instruction. + programCounter = nextCounter; } - return jumpDestBitmap; + return jumpDestinationBitmap; } /// @@ -147,23 +166,24 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) private static bool IsJumpDestination(long[] bitvec, int pos) { int vecIndex = pos >> BitShiftPerInt64; - // Check if in bounds, Jit will add slightly more expensive exception throwing check if we don't + // Check if in bounds, Jit will add slightly more expensive exception throwing check if we don't. if ((uint)vecIndex >= (uint)bitvec.Length) return false; return (bitvec[vecIndex] & (1L << pos)) != 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void MarkJumpDestinations(long[] bitvec, int pos, long flags) + private static void MarkJumpDestinations(long[] jumpDestinationBitmap, int pos, long flags) { uint offset = (uint)pos >> BitShiftPerInt64; - Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bitvec), offset) + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(jumpDestinationBitmap), offset) |= flags; } public void Execute() { - _jumpDestBitmap ??= CreateJumpDestinationBitmap(MachineCode.Span); + // This is to support background thread preparation of the bitmap. + _jumpDestinationBitmap ??= CreateJumpDestinationBitmap(MachineCode.Span); } } } From 8a4806cceecb4445019eb6d5047d7d6b3ff9a080 Mon Sep 17 00:00:00 2001 From: "lukasz.rozmej" Date: Thu, 18 Jan 2024 10:41:14 +0100 Subject: [PATCH 07/12] refactors --- .../AccountAbstractionRpcModuleTests.cs | 3 +- .../UserOperationTracerTests.cs | 1 + .../Nethermind.Evm.Test/Eip1014Tests.cs | 1 + .../Nethermind.Evm.Test/Eip1052Tests.cs | 1 + .../Nethermind.Evm.Test/Eip1153Tests.cs | 1 + .../Nethermind.Evm.Test/Eip1884Tests.cs | 1 + .../Nethermind.Evm.Test/Eip3860Tests.cs | 1 + .../StorageAndSelfDestructTests.cs | 1 + .../Tracing/GethLikeJavaScriptTracerTests.cs | 2 +- .../Tracing/GethLikeTxMemoryTracerTests.cs | 1 + .../Tracing/ParityLikeTxTracerTests.cs | 1 + .../Nethermind.Evm.Test/VmCodeDepositTests.cs | 1 + .../Nethermind.Evm/ByteArrayExtensions.cs | 13 +++----- .../CodeAnalysis/JumpDestinationAnalyzer.cs | 14 ++------ .../Nethermind.Evm/IVirtualMachine.cs | 2 +- .../Tracing/AlwaysCancelTxTracer.cs | 2 +- .../Tracing/BlockReceiptsTracer.cs | 2 +- .../Tracing/CancellationTxTracer.cs | 2 +- .../Tracing/CompositeTxTracer.cs | 2 +- .../Tracing/Debugger/DebugTracer.cs | 2 +- .../Nethermind.Evm/Tracing/ITxTracer.cs | 4 +-- .../Nethermind.Evm/Tracing/NullTxTracer.cs | 2 +- .../Tracing/ParityStyle/ParityLikeTxTracer.cs | 5 +-- .../Nethermind.Evm/Tracing/TxTracer.cs | 2 +- .../TransactionProcessor.cs | 8 +---- .../Nethermind.Evm/VirtualMachine.cs | 33 ++++++++++--------- .../Eth/EthRpcModuleTests.EstimateGas.cs | 1 + .../Modules/Eth/EthRpcModuleTests.EthCall.cs | 1 + .../Nethermind.State/IWorldState.cs | 2 -- .../Nethermind.State/IWorldStateExtensions.cs | 14 +++++--- src/Nethermind/Nethermind.State/WorldState.cs | 6 +--- 31 files changed, 62 insertions(+), 70 deletions(-) diff --git a/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.cs b/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.cs index b9b1e0947dd..026dfdb5063 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.cs @@ -22,8 +22,7 @@ using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Test.Modules; using Nethermind.Logging; - -using Newtonsoft.Json.Linq; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.AccountAbstraction.Test; diff --git a/src/Nethermind/Nethermind.AccountAbstraction.Test/UserOperationTracerTests.cs b/src/Nethermind/Nethermind.AccountAbstraction.Test/UserOperationTracerTests.cs index e6246baf185..4e546819c4b 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction.Test/UserOperationTracerTests.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction.Test/UserOperationTracerTests.cs @@ -11,6 +11,7 @@ using Nethermind.Evm.Test; using Nethermind.Logging; using Nethermind.Specs; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.AccountAbstraction.Test diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip1014Tests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip1014Tests.cs index ca30797a92f..3ec2002df2b 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip1014Tests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip1014Tests.cs @@ -7,6 +7,7 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Specs; +using Nethermind.State; using Nethermind.Core.Test.Builders; using Nethermind.Evm.Tracing.GethStyle; using Nethermind.Trie; diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip1052Tests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip1052Tests.cs index c9ed0fa5c82..ae825118bac 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip1052Tests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip1052Tests.cs @@ -11,6 +11,7 @@ using Nethermind.Evm.Precompiles; using Nethermind.Specs.Forks; using Nethermind.Specs.Test; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.Evm.Test diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip1153Tests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip1153Tests.cs index f908d1638a2..7e21148f754 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip1153Tests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip1153Tests.cs @@ -3,6 +3,7 @@ using Nethermind.Core.Extensions; using Nethermind.Specs; +using Nethermind.State; using Nethermind.Core.Test.Builders; using NUnit.Framework; using System.Diagnostics; diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip1884Tests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip1884Tests.cs index aed97068526..9e9f11bded1 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip1884Tests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip1884Tests.cs @@ -5,6 +5,7 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Specs; +using Nethermind.State; using Nethermind.Core.Test.Builders; using Nethermind.Int256; using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip3860Tests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip3860Tests.cs index 377a40460f1..946bafcb754 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip3860Tests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip3860Tests.cs @@ -3,6 +3,7 @@ using Nethermind.Core.Extensions; using Nethermind.Specs; +using Nethermind.State; using Nethermind.Core.Test.Builders; using NUnit.Framework; using Nethermind.Core; diff --git a/src/Nethermind/Nethermind.Evm.Test/StorageAndSelfDestructTests.cs b/src/Nethermind/Nethermind.Evm.Test/StorageAndSelfDestructTests.cs index 842bbc2f2b8..9477089d2db 100644 --- a/src/Nethermind/Nethermind.Evm.Test/StorageAndSelfDestructTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/StorageAndSelfDestructTests.cs @@ -11,6 +11,7 @@ using Nethermind.Logging; using Nethermind.Specs; using Nethermind.Specs.Forks; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.Evm.Test diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeJavaScriptTracerTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeJavaScriptTracerTests.cs index 45009fe185e..0b935557f3d 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeJavaScriptTracerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeJavaScriptTracerTests.cs @@ -12,9 +12,9 @@ using Nethermind.Core.Test.Builders; using Nethermind.Evm.Tracing.GethStyle.JavaScript; using Nethermind.Int256; -using Nethermind.JsonRpc.Modules.DebugModule; using Nethermind.Serialization.Json; using Nethermind.Specs.Forks; +using Nethermind.State; namespace Nethermind.Evm.Test.Tracing; diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeTxMemoryTracerTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeTxMemoryTracerTests.cs index 09d19f396a7..ae5ceab5c4b 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeTxMemoryTracerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeTxMemoryTracerTests.cs @@ -6,6 +6,7 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; using Nethermind.Evm.Tracing.GethStyle; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.Evm.Test.Tracing; diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs index 8867edb930b..dabc8b9ae63 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/ParityLikeTxTracerTests.cs @@ -11,6 +11,7 @@ using Nethermind.Int256; using Nethermind.Evm.Precompiles; using Nethermind.Evm.Tracing.ParityStyle; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.Evm.Test.Tracing diff --git a/src/Nethermind/Nethermind.Evm.Test/VmCodeDepositTests.cs b/src/Nethermind/Nethermind.Evm.Test/VmCodeDepositTests.cs index 4554f5a319c..99441287fa9 100644 --- a/src/Nethermind/Nethermind.Evm.Test/VmCodeDepositTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/VmCodeDepositTests.cs @@ -4,6 +4,7 @@ using Nethermind.Core; using Nethermind.Core.Extensions; using Nethermind.Specs; +using Nethermind.State; using Nethermind.Core.Test.Builders; using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs b/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs index b4ce8e60d6c..a012d4208ec 100644 --- a/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs +++ b/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs @@ -54,15 +54,10 @@ public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlySpan span, return SliceWithZeroPadding(span, (int)startIndex, length, padDirection); } - public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlyMemory bytes, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) - { - if (startIndex >= bytes.Length || startIndex > int.MaxValue) - { - return new ZeroPaddedSpan(default, length, PadDirection.Right); - } - - return SliceWithZeroPadding(bytes.Span, (int)startIndex, length, padDirection); - } + public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlyMemory bytes, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) => + startIndex >= bytes.Length || startIndex > int.MaxValue + ? new ZeroPaddedSpan(default, length, PadDirection.Right) + : SliceWithZeroPadding(bytes.Span, (int)startIndex, length, padDirection); public static ZeroPaddedSpan SliceWithZeroPadding(this byte[] bytes, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) => bytes.AsSpan().SliceWithZeroPadding(startIndex, length, padDirection); diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 86f7e18259d..6b98273662e 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -33,21 +33,11 @@ public bool ValidateJump(int destination, bool isSubroutine) // Cast to uint to change negative numbers to very int high numbers // Then do length check, this both reduces check by 1 and eliminates the bounds // check from accessing the array. - if ((uint)destination < (uint)machineCode.Length) + if ((uint)destination < (uint)machineCode.Length && IsJumpDestination(_jumpDestBitmap, destination)) { // Store byte to int, as less expensive operations at word size int codeByte = machineCode[destination]; - if (IsJumpDestination(_jumpDestBitmap, destination)) - { - if (isSubroutine) - { - result = codeByte == BEGINSUB; - } - else - { - result = codeByte == JUMPDEST; - } - } + result = isSubroutine ? codeByte == BEGINSUB : codeByte == JUMPDEST; } return result; diff --git a/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs b/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs index 2aaf01f4e22..ffe5b0a613f 100644 --- a/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs @@ -18,6 +18,6 @@ TransactionSubstate Run(EvmState state, IWorldState worldState, where TTracingActions : struct, IIsTracing; CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec spec); - void CacheCodeInfo(Hash256 codeHash, CodeInfo cachedCodeInfo); + void InsertCode(byte[] code, Address codeOwner, IReleaseSpec spec); } } diff --git a/src/Nethermind/Nethermind.Evm/Tracing/AlwaysCancelTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/AlwaysCancelTxTracer.cs index a3ddb2a9ca1..52c917694e5 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/AlwaysCancelTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/AlwaysCancelTxTracer.cs @@ -89,7 +89,7 @@ public static AlwaysCancelTxTracer Instance public void ReportActionEnd(long gas, Address deploymentAddress, ReadOnlyMemory deployedCode) => throw new OperationCanceledException(ErrorMessage); public void ReportBlockHash(Hash256 blockHash) => throw new OperationCanceledException(ErrorMessage); - public void ReportByteCode(byte[] byteCode) => throw new OperationCanceledException(ErrorMessage); + public void ReportByteCode(ReadOnlyMemory byteCode) => throw new OperationCanceledException(ErrorMessage); public void ReportGasUpdateForVmTrace(long refund, long gasAvailable) => throw new OperationCanceledException(ErrorMessage); public void ReportRefund(long refund) => throw new OperationCanceledException(ErrorMessage); public void ReportExtraGasPressure(long extraGasPressure) => throw new OperationCanceledException(ErrorMessage); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs index c43469bd91c..9ade3798ea3 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs @@ -157,7 +157,7 @@ public void ReportActionRevert(long gasLeft, byte[] output) => public void ReportActionEnd(long gas, Address deploymentAddress, ReadOnlyMemory deployedCode) => _currentTxTracer.ReportActionEnd(gas, deploymentAddress, deployedCode); - public void ReportByteCode(byte[] byteCode) => + public void ReportByteCode(ReadOnlyMemory byteCode) => _currentTxTracer.ReportByteCode(byteCode); public void ReportGasUpdateForVmTrace(long refund, long gasAvailable) => diff --git a/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs index 31759e88649..af2fed7b7c2 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs @@ -384,7 +384,7 @@ public void ReportBlockHash(Hash256 blockHash) } } - public void ReportByteCode(byte[] byteCode) + public void ReportByteCode(ReadOnlyMemory byteCode) { _token.ThrowIfCancellationRequested(); if (_innerTracer.IsTracingCode) diff --git a/src/Nethermind/Nethermind.Evm/Tracing/CompositeTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/CompositeTxTracer.cs index c4ad9abfd32..c6b8277a6b6 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/CompositeTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/CompositeTxTracer.cs @@ -413,7 +413,7 @@ public void ReportBlockHash(Hash256 blockHash) } } - public void ReportByteCode(byte[] byteCode) + public void ReportByteCode(ReadOnlyMemory byteCode) { for (int index = 0; index < _txTracers.Count; index++) { diff --git a/src/Nethermind/Nethermind.Evm/Tracing/Debugger/DebugTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/Debugger/DebugTracer.cs index 5014ff4808c..02cf430b759 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/Debugger/DebugTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/Debugger/DebugTracer.cs @@ -239,7 +239,7 @@ public void ReportActionEnd(long gas, Address deploymentAddress, ReadOnlyMemory< public void ReportBlockHash(Hash256 blockHash) => InnerTracer.ReportBlockHash(blockHash); - public void ReportByteCode(byte[] byteCode) + public void ReportByteCode(ReadOnlyMemory byteCode) => InnerTracer.ReportByteCode(byteCode); public void ReportGasUpdateForVmTrace(long refund, long gasAvailable) diff --git a/src/Nethermind/Nethermind.Evm/Tracing/ITxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/ITxTracer.cs index ca99d17265e..26057c97f89 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/ITxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/ITxTracer.cs @@ -375,11 +375,11 @@ void LoadOperationTransientStorage(Address storageCellAddress, UInt256 storageIn void ReportBlockHash(Hash256 blockHash); /// - /// + /// /// /// /// Depends on - void ReportByteCode(byte[] byteCode); + void ReportByteCode(ReadOnlyMemory byteCode); /// /// Special case for VM trace in Parity but we consider removing support for it diff --git a/src/Nethermind/Nethermind.Evm/Tracing/NullTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/NullTxTracer.cs index ee1c0bb456a..0daaf63efc1 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/NullTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/NullTxTracer.cs @@ -93,7 +93,7 @@ public override void ReportActionEnd(long gas, Address deploymentAddress, ReadOn => ThrowInvalidOperationException(); public override void ReportBlockHash(Hash256 blockHash) => ThrowInvalidOperationException(); - public override void ReportByteCode(byte[] byteCode) + public override void ReportByteCode(ReadOnlyMemory byteCode) => ThrowInvalidOperationException(); public override void ReportGasUpdateForVmTrace(long refund, long gasAvailable) diff --git a/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs index 908a90527d8..3bddd02d4d5 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs @@ -468,9 +468,10 @@ public override void ReportActionEnd(long gas, Address deploymentAddress, ReadOn PopAction(); } - public override void ReportByteCode(byte[] byteCode) + public override void ReportByteCode(ReadOnlyMemory byteCode) { - _currentVmTrace.VmTrace.Code = byteCode; + // TODO: use memory pool? + _currentVmTrace.VmTrace.Code = byteCode.ToArray(); } public override void ReportGasUpdateForVmTrace(long refund, long gasAvailable) diff --git a/src/Nethermind/Nethermind.Evm/Tracing/TxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/TxTracer.cs index 14343567c4c..1b196a33507 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/TxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/TxTracer.cs @@ -66,7 +66,7 @@ public virtual void ReportActionEnd(long gas, ReadOnlyMemory output) { } public virtual void ReportActionError(EvmExceptionType evmExceptionType) { } public virtual void ReportActionEnd(long gas, Address deploymentAddress, ReadOnlyMemory deployedCode) { } public virtual void ReportBlockHash(Hash256 blockHash) { } - public virtual void ReportByteCode(byte[] byteCode) { } + public virtual void ReportByteCode(ReadOnlyMemory byteCode) { } public virtual void ReportGasUpdateForVmTrace(long refund, long gasAvailable) { } public virtual void ReportRefund(long refund) { } public virtual void ReportExtraGasPressure(long extraGasPressure) { } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs index 7e0a8c78c74..8dcbb876024 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs @@ -555,13 +555,7 @@ protected virtual bool ExecuteEvmCall( if (unspentGas >= codeDepositGasCost) { var code = substate.Output.ToArray(); - var codeInfo = new CodeInfo(code); - // Start generating the JumpDestinationBitmap in background. - ThreadPool.UnsafeQueueUserWorkItem(codeInfo, preferLocal: false); - - Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.AsSpan()); - WorldState.InsertCode(env.ExecutingAccount, codeHash, code, spec); - VirtualMachine.CacheCodeInfo(codeHash, codeInfo); + VirtualMachine.InsertCode(code, env.ExecutingAccount, spec); unspentGas -= codeDepositGasCost; } diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index 525eadc43d8..5e01871d832 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -61,8 +61,10 @@ public VirtualMachine( public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec spec) => _evm.GetCachedCodeInfo(worldState, codeSource, spec); - public void CacheCodeInfo(Hash256 codeHash, CodeInfo cachedCodeInfo) - => _evm.CacheCodeInfo(codeHash, cachedCodeInfo); + public void InsertCode(byte[] code, Address codeOwner, IReleaseSpec spec) + { + _evm.InsertCode(code, codeOwner, spec); + } public TransactionSubstate Run(EvmState state, IWorldState worldState, ITxTracer txTracer) where TTracingActions : struct, IIsTracing @@ -131,8 +133,7 @@ public interface IIsTracing { } public readonly struct IsTracing : IIsTracing { } } -internal sealed class VirtualMachine : IVirtualMachine - where TLogger : struct, IIsTracing +internal sealed class VirtualMachine : IVirtualMachine where TLogger : struct, IIsTracing { private readonly UInt256 P255Int = (UInt256)System.Numerics.BigInteger.Pow(2, 255); private UInt256 P255 => P255Int; @@ -234,7 +235,7 @@ public TransactionSubstate Run(EvmState state, IWorldState worl if (typeof(TTracingActions) == typeof(IsTracing) && !currentState.IsContinuation) { _txTracer.ReportAction(currentState.GasAvailable, currentState.Env.Value, currentState.From, currentState.To, currentState.ExecutionType.IsAnyCreate() ? currentState.Env.CodeInfo.MachineCode : currentState.Env.InputData, currentState.ExecutionType); - if (_txTracer.IsTracingCode) _txTracer.ReportByteCode(currentState.Env.CodeInfo.MachineCode.ToArray()); + if (_txTracer.IsTracingCode) _txTracer.ReportByteCode(currentState.Env.CodeInfo.MachineCode); } if (!_txTracer.IsTracingInstructions) @@ -360,13 +361,7 @@ public TransactionSubstate Run(EvmState state, IWorldState worl if (gasAvailableForCodeDeposit >= codeDepositGasCost && !invalidCode) { var code = callResult.Output; - var codeInfo = new CodeInfo(code); - // Start generating the JumpDestinationBitmap in background. - ThreadPool.UnsafeQueueUserWorkItem(codeInfo, preferLocal: false); - - Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.AsSpan()); - _state.InsertCode(callCodeOwner, codeHash, callResult.Output, spec); - _codeCache.Set(codeHash, codeInfo); + InsertCode(code, callCodeOwner, spec); currentState.GasAvailable -= codeDepositGasCost; @@ -475,6 +470,17 @@ public TransactionSubstate Run(EvmState state, IWorldState worl } } + public void InsertCode(byte[] code, Address callCodeOwner, IReleaseSpec spec) + { + var codeInfo = new CodeInfo(code); + // Start generating the JumpDestinationBitmap in background. + ThreadPool.UnsafeQueueUserWorkItem(codeInfo, preferLocal: false); + + Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.AsSpan()); + _state.InsertCode(callCodeOwner, codeHash, code, spec); + _codeCache.Set(codeHash, codeInfo); + } + private void RevertParityTouchBugAccount(IReleaseSpec spec) { if (_parityTouchBugAccount.ShouldDelete) @@ -488,9 +494,6 @@ private void RevertParityTouchBugAccount(IReleaseSpec spec) } } - public void CacheCodeInfo(Hash256 codeHash, CodeInfo cachedCodeInfo) - => _codeCache.Set(codeHash, cachedCodeInfo); - public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec vmSpec) { if (codeSource.IsPrecompile(vmSpec)) diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs index c0448532a2f..95154d4d1a6 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs @@ -14,6 +14,7 @@ using Nethermind.Specs; using Nethermind.Specs.Forks; using Nethermind.Specs.Test; +using Nethermind.State; using Newtonsoft.Json.Linq; using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs index ceb8024cbbb..73791ea3169 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs @@ -12,6 +12,7 @@ using Nethermind.Specs; using Nethermind.Specs.Forks; using Nethermind.Specs.Test; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.JsonRpc.Test.Modules.Eth; diff --git a/src/Nethermind/Nethermind.State/IWorldState.cs b/src/Nethermind/Nethermind.State/IWorldState.cs index b74386936f8..b25ef558f67 100644 --- a/src/Nethermind/Nethermind.State/IWorldState.cs +++ b/src/Nethermind/Nethermind.State/IWorldState.cs @@ -83,8 +83,6 @@ public interface IWorldState : IJournal, IReadOnlyStateProvider void CreateAccount(Address address, in UInt256 balance, in UInt256 nonce = default); void CreateAccountIfNotExists(Address address, in UInt256 balance, in UInt256 nonce = default); - void InsertCode(Address address, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false); - void InsertCode(Address address, Hash256 codeHash, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false); void AddToBalance(Address address, in UInt256 balanceChange, IReleaseSpec spec); diff --git a/src/Nethermind/Nethermind.State/IWorldStateExtensions.cs b/src/Nethermind/Nethermind.State/IWorldStateExtensions.cs index 468763e770b..b5e7dc0a99f 100644 --- a/src/Nethermind/Nethermind.State/IWorldStateExtensions.cs +++ b/src/Nethermind/Nethermind.State/IWorldStateExtensions.cs @@ -3,7 +3,9 @@ using System; using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; using Nethermind.Logging; using Nethermind.Trie; @@ -14,11 +16,13 @@ public static class WorldStateExtensions public static byte[] GetCode(this IWorldState stateProvider, Address address) { Account account = stateProvider.GetAccount(address); - if (!account.HasCode) - { - return Array.Empty(); - } - return stateProvider.GetCode(account.CodeHash); + return !account.HasCode ? Array.Empty() : stateProvider.GetCode(account.CodeHash); + } + + public static void InsertCode(this IWorldState worldState, Address address, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false) + { + Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.Span); + worldState.InsertCode(address, codeHash, code, spec, isGenesis); } public static string DumpState(this IWorldState stateProvider) diff --git a/src/Nethermind/Nethermind.State/WorldState.cs b/src/Nethermind/Nethermind.State/WorldState.cs index abd3d358afb..4d93ab22831 100644 --- a/src/Nethermind/Nethermind.State/WorldState.cs +++ b/src/Nethermind/Nethermind.State/WorldState.cs @@ -106,11 +106,7 @@ public void CreateAccount(Address address, in UInt256 balance, in UInt256 nonce { _stateProvider.CreateAccount(address, balance, nonce); } - public void InsertCode(Address address, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false) - { - Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.Span); - InsertCode(address, codeHash, code, spec, isGenesis); - } + public void InsertCode(Address address, Hash256 codeHash, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false) { _stateProvider.InsertCode(address, codeHash, code, spec, isGenesis); From 40c5d94cf2b916051a831aa025bcb822c3cc9ba1 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 18 Jan 2024 09:48:53 +0000 Subject: [PATCH 08/12] Merge conflicts --- src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs | 10 ++-------- .../CodeAnalysis/JumpDestinationAnalyzer.cs | 6 ++++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs b/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs index 08aa8363d80..57159706c74 100644 --- a/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs +++ b/src/Nethermind/Nethermind.Evm/ByteArrayExtensions.cs @@ -34,19 +34,13 @@ private static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlySpan span, return new ZeroPaddedSpan(span.Slice(startIndex, copiedLength), length - copiedLength, padDirection); } - public static ZeroPaddedSpan SliceWithZeroPadding(this Span span, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) - { - return startIndex >= span.Length || startIndex > int.MaxValue + public static ZeroPaddedSpan SliceWithZeroPadding(this Span span, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) => startIndex >= span.Length || startIndex > int.MaxValue ? new ZeroPaddedSpan(default, length, PadDirection.Right) : SliceWithZeroPadding(span, (int)startIndex, length, padDirection); - } - public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlySpan span, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) - { - return startIndex >= span.Length || startIndex > int.MaxValue + public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlySpan span, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) => startIndex >= span.Length || startIndex > int.MaxValue ? new ZeroPaddedSpan(default, length, PadDirection.Right) : SliceWithZeroPadding(span, (int)startIndex, length, padDirection); - } public static ZeroPaddedSpan SliceWithZeroPadding(this ReadOnlyMemory bytes, scoped in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) => startIndex >= bytes.Length || startIndex > int.MaxValue diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 32e842a0380..40625a69e7b 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -31,7 +31,7 @@ public bool ValidateJump(int destination, bool isSubroutine) // Cast to uint to change negative numbers to very int high numbers // Then do length check, this both reduces check by 1 and eliminates the bounds // check from accessing the span. - if ((uint)destination < (uint)machineCode.Length && IsJumpDestination(_jumpDestBitmap, destination)) + if ((uint)destination < (uint)machineCode.Length && IsJumpDestination(_jumpDestinationBitmap, destination)) { // Store byte to int, as less expensive operations at word size int codeByte = machineCode[destination]; @@ -130,7 +130,9 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) // at a boundary and then we will return the results. bool exit = nextCounter >= code.Length; // Does the move mean we are moving to new segment of the long array? - if ((programCounter & 63) + move > 63 || exit) + // If we take the current index in flags, and add the move, are we at + // a new long segment, i.e. a larger than 64 position move. + if ((programCounter & 63) + move >= 64 || exit) { // If so write out the flags (if any are set) if (currentFlags != 0) From 5712f52b440f4e949dd9fdb7088a35f8de0b95ac Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 18 Jan 2024 09:55:03 +0000 Subject: [PATCH 09/12] Add missing implementation --- src/Nethermind/Nethermind.Test.Runner/StateTestTxTracer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Test.Runner/StateTestTxTracer.cs b/src/Nethermind/Nethermind.Test.Runner/StateTestTxTracer.cs index 322ea016b81..31707568258 100644 --- a/src/Nethermind/Nethermind.Test.Runner/StateTestTxTracer.cs +++ b/src/Nethermind/Nethermind.Test.Runner/StateTestTxTracer.cs @@ -220,7 +220,7 @@ public void ReportBlockHash(Hash256 blockHash) throw new NotImplementedException(); } - public void ReportByteCode(byte[] byteCode) + public void ReportByteCode(ReadOnlyMemory byteCode) { throw new NotSupportedException(); } From 64dda0f1d16b44ace8a3887ddf7b8854a058b4c3 Mon Sep 17 00:00:00 2001 From: "lukasz.rozmej" Date: Thu, 18 Jan 2024 11:20:24 +0100 Subject: [PATCH 10/12] small refactor --- .../CodeAnalysis/JumpDestinationAnalyzer.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 40625a69e7b..732a2255504 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -17,10 +17,10 @@ public sealed class JumpDestinationAnalyzer(ReadOnlyMemory code) private const int BEGINSUB = 0x5c; private const int BitShiftPerInt64 = 6; - private readonly static long[]? _emptyJumpDestinationBitmap = new long[1]; + private static readonly long[]? _emptyJumpDestinationBitmap = new long[1]; private long[]? _jumpDestinationBitmap = code.Length == 0 ? _emptyJumpDestinationBitmap : null; - public ReadOnlyMemory MachineCode { get; } = code; + private ReadOnlyMemory MachineCode { get; } = code; public bool ValidateJump(int destination, bool isSubroutine) { @@ -57,10 +57,8 @@ public bool ValidateJump(int destination, bool isSubroutine) /// /// /// how many ints are required to store n bytes - private static int GetInt64ArrayLengthFromBitLength(int n) - { - return (int)((uint)(n - 1 + (1 << BitShiftPerInt64)) >> BitShiftPerInt64); - } + private static int GetInt64ArrayLengthFromBitLength(int n) => + (n - 1 + (1 << BitShiftPerInt64)) >>> BitShiftPerInt64; /// /// Collects data locations in code. @@ -117,14 +115,14 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) // so can shift by the whole programCounter. currentFlags |= 1L << programCounter; } - else if ((sbyte)op > (sbyte)PUSHx) + else if ((sbyte)op > PUSHx) { // Fast forward programCounter by the amount of data the push // represents as don't need to analyse data for Jump Destinations. move = op - PUSH1 + 2; } - Next: + Next: int nextCounter = programCounter + move; // Check if read last item of code; we want to write this out also even if not // at a boundary and then we will return the results. @@ -173,8 +171,7 @@ private static bool IsJumpDestination(long[] bitvec, int pos) private static void MarkJumpDestinations(long[] jumpDestinationBitmap, int pos, long flags) { uint offset = (uint)pos >> BitShiftPerInt64; - Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(jumpDestinationBitmap), offset) - |= flags; + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(jumpDestinationBitmap), offset) |= flags; } public void Execute() From e9b9d9b64ed2949559c9c3a387b7e19eda97a761 Mon Sep 17 00:00:00 2001 From: "lukasz.rozmej" Date: Thu, 18 Jan 2024 11:31:24 +0100 Subject: [PATCH 11/12] whitespace fix --- .../Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 732a2255504..42235e33e69 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -122,7 +122,7 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) move = op - PUSH1 + 2; } - Next: + Next: int nextCounter = programCounter + move; // Check if read last item of code; we want to write this out also even if not // at a boundary and then we will return the results. From 0673b9c41175709174dc0da24c88cf520e378353 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 18 Jan 2024 11:20:24 +0000 Subject: [PATCH 12/12] Use Vector128 instead --- .../CodeAnalysis/JumpDestinationAnalyzer.cs | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 42235e33e69..8e944c18101 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -67,7 +67,6 @@ private static int GetInt64ArrayLengthFromBitLength(int n) => private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) { long[] jumpDestinationBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)]; - int programCounter = 0; // We accumulate each array segment to a register and then flush to memory when we move to next. long currentFlags = 0; @@ -75,30 +74,32 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) { // Set default programCounter increment to 1 for default case when don't vectorize or read a PUSH. int move = 1; - if (Avx2.IsSupported && + // We use Sse rather than Avx or Avx-512 as is optimization for stretch of code without PUSHes. + // As the vector size increases the chance of there being a PUSH increases which will disable this optimization. + if (Sse2.IsSupported && // Check not going to read passed end of code. - programCounter <= code.Length - Vector256.Count && - // Are we on an int stride, one or other half of the long flags? - (programCounter & 31) == 0) + programCounter <= code.Length - Vector128.Count && + // Are we on an short stride, one quarter of the long flags? + (programCounter & 15) == 0) { - Vector256 data = Unsafe.As>(ref Unsafe.AddByteOffset(ref MemoryMarshal.GetReference(code), programCounter)); + Vector128 data = Unsafe.As>(ref Unsafe.AddByteOffset(ref MemoryMarshal.GetReference(code), programCounter)); // Pushes are 0x60 to 0x7f; converting to signed bytes any instruction higher than PUSH32 // becomes negative so we can just do a single greater than test to see if any present. - Vector256 compare = Avx2.CompareGreaterThan(data, Vector256.Create((sbyte)PUSHx)); + Vector128 compare = Sse2.CompareGreaterThan(data, Vector128.Create((sbyte)PUSHx)); if (compare == default) { // Check the bytes for any JUMPDESTs. - Vector256 dest = Avx2.CompareEqual(data, Vector256.Create((sbyte)JUMPDEST)); + Vector128 dest = Sse2.CompareEqual(data, Vector128.Create((sbyte)JUMPDEST)); // Check the bytes for any BEGINSUBs. - Vector256 sub = Avx2.CompareEqual(data, Vector256.Create((sbyte)BEGINSUB)); + Vector128 sub = Sse2.CompareEqual(data, Vector128.Create((sbyte)BEGINSUB)); // Merge the two results. - Vector256 combined = Avx2.Or(dest, sub); + Vector128 combined = Sse2.Or(dest, sub); // Extract the checks as a set of int flags. - int flags = Avx2.MoveMask(combined); + int flags = Sse2.MoveMask(combined); // Shift up flags by depending which side of long we are on, and merge to current set. - currentFlags |= (long)flags << (programCounter & 32); - // Forward programCounter by Vector256 stride. - move = Vector256.Count; + currentFlags |= (long)flags << (programCounter & (32 + 16)); + // Forward programCounter by Vector128 stride. + move = Vector128.Count; goto Next; } }