Merge pull request #510 from 0xPolygon/thiago/nonce-gap

add account nonce fix-gap command

README.md

@@ -49,6 +49,8 @@ Note: Do not modify this section! It is auto-generated by `cobra` using `make ge
 
 - [polycli enr](doc/polycli_enr.md) - Convert between ENR and Enode format
 
+- [polycli fix-nonce-gap](doc/polycli_fix-nonce-gap.md) - Send txs to fix the nonce gap for a specific account
+
 - [polycli fork](doc/polycli_fork.md) - Take a forked block and walk up the chain to do analysis.
 
 - [polycli fund](doc/polycli_fund.md) - Bulk fund crypto wallets automatically.

cmd/fixnoncegap/FixNonceGapUsage.md

@@ -0,0 +1,27 @@
+This command will check the account current nonce against the max nonce found in the pool. In case of a nonce gap is found, txs will be sent to fill those gaps.
+
+To fix a nonce gap, we can use a command like this:
+
+```bash
+polycli fix-nonce-gap \
+    --rpc-url https://sepolia.drpc.org
+    --private-key 0x32430699cd4f46ab2422f1df4ad6546811be20c9725544e99253a887e971f92b
+```
+
+In case the RPC doesn't provide the `txpool_content` endpoint, the flag `--max-nonce` can be set to define the max nonce. The command will generate TXs from the current nonce up to the max nonce set.
+
+```bash
+polycli fix-nonce-gap \
+    --rpc-url https://sepolia.drpc.org
+    --private-key 0x32430699cd4f46ab2422f1df4ad6546811be20c9725544e99253a887e971f92b
+    --max-nonce
+```
+
+By default, the command will skip TXs found in the pool, for example, let's assume the current nonce is 10, there is a TX with nonce 15 and 20 in the pool. When sending TXs to fill the gaps, the TXs 15 and 20 will be skipped. IN case you want to force these TXs to be replaced, you must provide the flag `--replace`.
+
+```bash
+polycli fix-nonce-gap \
+    --rpc-url https://sepolia.drpc.org
+    --private-key 0x32430699cd4f46ab2422f1df4ad6546811be20c9725544e99253a887e971f92b
+    --replace
+```

cmd/fixnoncegap/fixnoncegap.go

@@ -0,0 +1,309 @@
+package fixnoncegap
+
+import (
+	"context"
+	_ "embed"
+	"errors"
+	"fmt"
+	"math/big"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var FixNonceGapCmd = &cobra.Command{
+	Use:          "fix-nonce-gap",
+	Short:        "Send txs to fix the nonce gap for a specific account",
+	Long:         fixNonceGapUsage,
+	Args:         cobra.NoArgs,
+	PreRunE:      prepareRpcClient,
+	RunE:         fixNonceGap,
+	SilenceUsage: true,
+}
+
+var (
+	rpcClient *ethclient.Client
+)
+
+type fixNonceGapArgs struct {
+	rpcURL     *string
+	privateKey *string
+	replace    *bool
+	maxNonce   *uint64
+}
+
+var inputFixNonceGapArgs = fixNonceGapArgs{}
+
+const (
+	ArgPrivateKey = "private-key"
+	ArgRpcURL     = "rpc-url"
+	ArgReplace    = "replace"
+	ArgMaxNonce   = "max-nonce"
+)
+
+//go:embed FixNonceGapUsage.md
+var fixNonceGapUsage string
+
+func prepareRpcClient(cmd *cobra.Command, args []string) error {
+	var err error
+	rpcURL := *inputFixNonceGapArgs.rpcURL
+
+	rpcClient, err = ethclient.Dial(rpcURL)
+	if err != nil {
+		log.Error().Err(err).Msgf("Unable to Dial RPC %s", rpcURL)
+		return err
+	}
+
+	if _, err = rpcClient.BlockNumber(cmd.Context()); err != nil {
+		log.Error().Err(err).Msg("Unable to get block number")
+		return err
+	}
+
+	return nil
+}
+
+func fixNonceGap(cmd *cobra.Command, args []string) error {
+	replace := *inputFixNonceGapArgs.replace
+	pvtKey := strings.TrimPrefix(*inputFixNonceGapArgs.privateKey, "0x")
+	pk, err := crypto.HexToECDSA(pvtKey)
+	if err != nil {
+		log.Error().Err(err).Msg("Invalid private key")
+		return err
+	}
+
+	chainID, err := rpcClient.ChainID(cmd.Context())
+	if err != nil {
+		log.Error().Err(err).Msg("Cannot get chain ID")
+		return err
+	}
+
+	opts, err := bind.NewKeyedTransactorWithChainID(pk, chainID)
+	if err != nil {
+		log.Error().Err(err).Msg("Cannot generate transactionOpts")
+		return err
+	}
+
+	addr := opts.From
+
+	currentNonce, err := rpcClient.NonceAt(cmd.Context(), addr, nil)
+	if err != nil {
+		log.Error().Err(err).Msg("Unable to get current nonce")
+		return err
+	}
+	log.Info().Stringer("addr", addr).Msgf("Current nonce: %d", currentNonce)
+
+	var maxNonce uint64
+	if *inputFixNonceGapArgs.maxNonce != 0 {
+		maxNonce = *inputFixNonceGapArgs.maxNonce
+	} else {
+		maxNonce, err = getMaxNonceFromTxPool(addr)
+		if err != nil {
+			if strings.Contains(err.Error(), "the method txpool_content does not exist/is not available") {
+				log.Error().Err(err).Msg("The RPC doesn't provide access to txpool_content, please check --help for more information about --max-nonce")
+				return nil
+			}
+			log.Error().Err(err).Msg("Unable to get max nonce from txpool")
+			return err
+		}
+	}
+
+	// check if there is a nonce gap
+	if maxNonce == 0 || currentNonce >= maxNonce {
+		log.Info().Stringer("addr", addr).Msg("There is no nonce gap.")
+		return nil
+	}
+	log.Info().Stringer("addr", addr).Msgf("Nonce gap found. Max nonce: %d", maxNonce)
+
+	gasPrice, err := rpcClient.SuggestGasPrice(cmd.Context())
+	if err != nil {
+		log.Error().Err(err).Msg("Unable to get suggested gas price")
+		return err
+	}
+
+	to := &common.Address{}
+
+	gas, err := rpcClient.EstimateGas(cmd.Context(), ethereum.CallMsg{
+		From:     addr,
+		To:       to,
+		GasPrice: gasPrice,
+		Value:    big.NewInt(1),
+	})
+	if err != nil {
+		log.Error().Err(err).Msg("Unable to estimate gas")
+		return err
+	}
+
+	txTemplate := &types.LegacyTx{
+		To:       to,
+		Gas:      gas,
+		GasPrice: gasPrice,
+		Value:    big.NewInt(1),
+	}
+
+	var lastTx *types.Transaction
+	for i := currentNonce; i < maxNonce; i++ {
+		txTemplate.Nonce = i
+		tx := types.NewTx(txTemplate)
+	out:
+		for {
+			signedTx, err := opts.Signer(opts.From, tx)
+			if err != nil {
+				log.Error().Err(err).Msg("Unable to sign tx")
+				return err
+			}
+			log.Info().Stringer("hash", signedTx.Hash()).Msgf("sending tx with nonce %d", txTemplate.Nonce)
+
+			err = rpcClient.SendTransaction(cmd.Context(), signedTx)
+			if err != nil {
+				if strings.Contains(err.Error(), "nonce too low") {
+					log.Info().Stringer("hash", signedTx.Hash()).Msgf("another tx with nonce %d was mined while trying to increase the fee, skipping it", txTemplate.Nonce)
+					break out
+				} else if strings.Contains(err.Error(), "already known") {
+					log.Info().Stringer("hash", signedTx.Hash()).Msgf("same tx with nonce %d already exists, skipping it", txTemplate.Nonce)
+					break out
+				} else if strings.Contains(err.Error(), "replacement transaction underpriced") ||
+					strings.Contains(err.Error(), "INTERNAL_ERROR: could not replace existing tx") {
+					if replace {
+						txTemplateCopy := *txTemplate
+						oldGasPrice := txTemplate.GasPrice
+						// increase TX gas price by 10% and retry
+						txTemplateCopy.GasPrice = new(big.Int).Mul(txTemplate.GasPrice, big.NewInt(11))
+						txTemplateCopy.GasPrice = new(big.Int).Div(txTemplateCopy.GasPrice, big.NewInt(10))
+						tx = types.NewTx(&txTemplateCopy)
+						log.Info().Stringer("hash", signedTx.Hash()).Msgf("tx with nonce %d is underpriced, increasing fee by 10%%. From %d To %d", txTemplate.Nonce, oldGasPrice, txTemplateCopy.GasPrice)
+						time.Sleep(time.Second)
+						continue
+					} else {
+						log.Info().Stringer("hash", signedTx.Hash()).Msgf("another tx with nonce %d already exists, skipping it", txTemplate.Nonce)
+						break out
+					}
+				}
+				log.Error().Err(err).Msg("Unable to send tx")
+				return err
+			}
+
+			// if we get here, just break the infinite loop and move to the next
+			lastTx = signedTx
+			break
+		}
+	}
+
+	if lastTx != nil {
+		log.Info().Stringer("hash", lastTx.Hash()).Msg("waiting for the last tx to get mined")
+		err := WaitMineTransaction(cmd.Context(), rpcClient, lastTx, 600)
+		if err != nil {
+			log.Error().Err(err).Msg("Unable to wait for last tx to get mined")
+			return err
+		}
+		log.Info().Stringer("addr", addr).Msg("Nonce gap fixed successfully")
+		currentNonce, err = rpcClient.NonceAt(cmd.Context(), addr, nil)
+		if err != nil {
+			log.Error().Err(err).Msg("Unable to get current nonce")
+			return err
+		}
+		log.Info().Stringer("addr", addr).Msgf("Current nonce: %d", currentNonce)
+		return nil
+	}
+
+	return nil
+}
+
+func init() {
+	inputFixNonceGapArgs.rpcURL = FixNonceGapCmd.PersistentFlags().StringP(ArgRpcURL, "r", "http://localhost:8545", "The RPC endpoint url")
+	inputFixNonceGapArgs.privateKey = FixNonceGapCmd.PersistentFlags().String(ArgPrivateKey, "", "the private key to be used when sending the txs to fix the nonce gap")
+	inputFixNonceGapArgs.replace = FixNonceGapCmd.PersistentFlags().Bool(ArgReplace, false, "replace the existing txs in the pool")
+	inputFixNonceGapArgs.maxNonce = FixNonceGapCmd.PersistentFlags().Uint64(ArgMaxNonce, 0, "when set, the max nonce will be this value instead of trying to get it from the pool")
+	fatalIfError(FixNonceGapCmd.MarkPersistentFlagRequired(ArgPrivateKey))
+}
+
+// Wait for the transaction to be mined
+func WaitMineTransaction(ctx context.Context, client *ethclient.Client, tx *types.Transaction, txTimeout uint64) error {
+	timeout := time.NewTimer(time.Duration(txTimeout) * time.Second)
+	defer timeout.Stop()
+	for {
+		select {
+		case <-timeout.C:
+			err := fmt.Errorf("timeout waiting for transaction to be mined")
+			return err
+		default:
+			r, err := client.TransactionReceipt(ctx, tx.Hash())
+			if err != nil {
+				if !errors.Is(err, ethereum.NotFound) {
+					log.Error().Err(err)
+					return err
+				}
+				time.Sleep(1 * time.Second)
+				continue
+			}
+			if r.Status != 0 {
+				log.Info().Stringer("hash", r.TxHash).Msg("transaction successful")
+				return nil
+			} else if r.Status == 0 {
+				log.Error().Stringer("hash", r.TxHash).Msg("transaction failed")
+				return nil
+			}
+			time.Sleep(1 * time.Second)
+		}
+	}
+}
+
+func fatalIfError(err error) {
+	if err == nil {
+		return
+	}
+	log.Fatal().Err(err).Msg("Unexpected error occurred")
+}
+
+func getMaxNonceFromTxPool(addr common.Address) (uint64, error) {
+	var result PoolContent
+	err := rpcClient.Client().Call(&result, "txpool_content")
+	if err != nil {
+		return 0, err
+	}
+
+	txCollections := []PoolContentTxs{
+		result.BaseFee,
+		result.Pending,
+		result.Queued,
+	}
+
+	maxNonceFound := uint64(0)
+	for _, txCollection := range txCollections {
+		// get only txs from the address we are looking for
+		txs, found := txCollection[addr.String()]
+		if !found {
+			continue
+		}
+
+		// iterate over the transactions and get the nonce
+		for nonce := range txs {
+			nonceInt, ok := new(big.Int).SetString(nonce, 10)
+			if !ok {
+				err = fmt.Errorf("invalid nonce found: %s", nonce)
+				return 0, err
+			}
+
+			if nonceInt.Uint64() > maxNonceFound {
+				maxNonceFound = nonceInt.Uint64()
+			}
+		}
+	}
+
+	return maxNonceFound, nil
+}
+
+type PoolContent struct {
+	BaseFee PoolContentTxs
+	Pending PoolContentTxs
+	Queued  PoolContentTxs
+}
+
+type PoolContentTxs map[string]map[string]any

cmd/root.go

@@ -2,9 +2,11 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
+	"github.com/0xPolygon/polygon-cli/cmd/fixnoncegap"
 	"github.com/0xPolygon/polygon-cli/cmd/retest"
 	"github.com/0xPolygon/polygon-cli/cmd/ulxly"
-	"os"
 
 	"github.com/0xPolygon/polygon-cli/cmd/fork"
 	"github.com/0xPolygon/polygon-cli/cmd/p2p"
@@ -109,13 +111,14 @@ func NewPolycliCommand() *cobra.Command {
 	// Define commands.
 	cmd.AddCommand(
 		abi.ABICmd,
+		dbbench.DBBenchCmd,
 		dumpblocks.DumpblocksCmd,
 		ecrecover.EcRecoverCmd,
+		enr.ENRCmd,
+		fixnoncegap.FixNonceGapCmd,
 		fork.ForkCmd,
 		fund.FundCmd,
 		hash.HashCmd,
-		enr.ENRCmd,
-		dbbench.DBBenchCmd,
 		loadtest.LoadtestCmd,
 		metricsToDash.MetricsToDashCmd,
 		mnemonic.MnemonicCmd,

doc/polycli.md

@@ -45,6 +45,8 @@ Polycli is a collection of tools that are meant to be useful while building, tes
 
 - [polycli enr](polycli_enr.md) - Convert between ENR and Enode format
 
+- [polycli fix-nonce-gap](polycli_fix-nonce-gap.md) - Send txs to fix the nonce gap for a specific account
+
 - [polycli fork](polycli_fork.md) - Take a forked block and walk up the chain to do analysis.
 
 - [polycli fund](polycli_fund.md) - Bulk fund crypto wallets automatically.