Stop server elegantly instead of panic when db closed

blockchain/blockchain.go

@@ -37,6 +37,7 @@ var (
 	ErrInvalidStateRoot     = errors.New("invalid block state root")
 	ErrInvalidGasUsed       = errors.New("invalid block gas used")
 	ErrInvalidReceiptsRoot  = errors.New("invalid block receipts root")
+	ErrClosed               = errors.New("blockchain is closed")
 )
 
 // Blockchain is a blockchain reference
@@ -46,6 +47,7 @@ type Blockchain struct {
 	db        storage.Storage // The Storage object (database)
 	consensus Verifier
 	executor  Executor
+	stopped   uint32 // used in executor halting
 
 	config  *chain.Chain // Config containing chain information
 	genesis types.Hash   // The hash of the genesis block
@@ -93,6 +95,7 @@ type Verifier interface {
 
 type Executor interface {
 	ProcessBlock(parentRoot types.Hash, block *types.Block, blockCreator types.Address) (*state.Transition, error)
+	Stop()
 }
 
 type BlockResult struct {
@@ -862,6 +865,11 @@ func (b *Blockchain) executeBlockTransactions(block *types.Block) (*BlockResult,
 		b.logger,
 	)
 
+	if b.isStopped() {
+		// execute stop, should not commit
+		return nil, ErrClosed
+	}
+
 	_, root := txn.Commit()
 
 	// Append the receipts to the receipts cache
@@ -1303,5 +1311,17 @@ func (b *Blockchain) GetBlockByNumber(blockNumber uint64, full bool) (*types.Blo
 
 // Close closes the DB connection
 func (b *Blockchain) Close() error {
+	b.executor.Stop()
+	b.stop()
+
+	// close db at last
 	return b.db.Close()
 }
+
+func (b *Blockchain) stop() {
+	atomic.StoreUint32(&b.stopped, 1)
+}
+
+func (b *Blockchain) isStopped() bool {
+	return atomic.LoadUint32(&b.stopped) > 0
+}

blockchain/testing.go

@@ -298,6 +298,10 @@ func (m *mockExecutor) ProcessBlock(
 	return nil, nil
 }
 
+func (m *mockExecutor) Stop() {
+	// do nothing
+}
+
 func (m *mockExecutor) HookProcessBlock(fn processBlockDelegate) {
 	m.processBlockFn = fn
 }

e2e/framework/config.go

@@ -36,21 +36,23 @@ type TestServerConfig struct {
 	PremineAccts            []*SrvAccount // Accounts with existing balances (genesis accounts)
 	GenesisValidatorBalance *big.Int      // Genesis the balance for the validators
 	// List of initial staking addresses for the ValidatorSet SC with dev consensus
-	DevStakers     []types.Address
-	Consensus      ConsensusType        // Consensus MechanismType
-	Bootnodes      []string             // Bootnode Addresses
-	PriceLimit     *uint64              // Minimum gas price limit to enforce for acceptance into the pool
-	DevInterval    int                  // Dev consensus update interval [s]
-	EpochSize      uint64               // The epoch size in blocks for the IBFT layer
-	BlockGasLimit  uint64               // Block gas limit
-	BlockGasTarget uint64               // Gas target for new blocks
-	ShowsLog       bool                 // Flag specifying if logs are shown
-	IsPos          bool                 // Specifies the mechanism used for IBFT (PoA / PoS)
-	Signer         *crypto.EIP155Signer // Signer used for transactions
-	BridgeOwner    types.Address        // bridge contract owner
-	BridgeSigners  []types.Address      // bridge contract signers
-	IsWSEnable     bool                 // enable websocket or not
-	RestoreFile    string               // blockchain restore file
+	DevStakers        []types.Address
+	Consensus         ConsensusType        // Consensus MechanismType
+	Bootnodes         []string             // Bootnode Addresses
+	PriceLimit        *uint64              // Minimum gas price limit to enforce for acceptance into the pool
+	DevInterval       int                  // Dev consensus update interval [s]
+	EpochSize         uint64               // The epoch size in blocks for the IBFT layer
+	BlockGasLimit     uint64               // Block gas limit
+	BlockGasTarget    uint64               // Gas target for new blocks
+	ShowsLog          bool                 // Flag specifying if logs are shown
+	IsPos             bool                 // Specifies the mechanism used for IBFT (PoA / PoS)
+	Signer            *crypto.EIP155Signer // Signer used for transactions
+	ValidatorSetOwner types.Address        // validatorset contract owner
+	BridgeOwner       types.Address        // bridge contract owner
+	BridgeSigners     []types.Address      // bridge contract signers
+	IsWSEnable        bool                 // enable websocket or not
+	RestoreFile       string               // blockchain restore file
+	BlockTime         uint64               // minimum block generation time (in s)
 }
 
 // DataDir returns path of data directory server uses
@@ -160,6 +162,10 @@ func (t *TestServerConfig) SetEpochSize(epochSize uint64) {
 	t.EpochSize = epochSize
 }
 
+func (t *TestServerConfig) SetValidatorSetOwner(owner types.Address) {
+	t.ValidatorSetOwner = owner
+}
+
 func (t *TestServerConfig) SetBridgeOwner(owner types.Address) {
 	t.BridgeOwner = owner
 }
@@ -175,3 +181,7 @@ func (t *TestServerConfig) EnableWebSocket() {
 func (t *TestServerConfig) SetRestoreFile(path string) {
 	t.RestoreFile = path
 }
+
+func (t *TestServerConfig) SetBlockTime(blockTime uint64) {
+	t.BlockTime = blockTime
+}

e2e/framework/helper.go

@@ -30,6 +30,10 @@ import (
 	empty "google.golang.org/protobuf/types/known/emptypb"
 )
 
+var (
+	DefaultTimeout = time.Minute
+)
+
 func EthToWei(ethValue int64) *big.Int {
 	return EthToWeiPrecise(ethValue, 18)
 }

e2e/framework/ibft.go

@@ -83,14 +83,16 @@ func NewIBFTServersManager(
 }
 
 func (m *IBFTServersManager) StartServers(ctx context.Context) {
-	for _, srv := range m.servers {
+	for i, srv := range m.servers {
 		if err := srv.Start(ctx); err != nil {
+			m.t.Logf("server %d failed to start: %+v", i, err)
 			m.t.Fatal(err)
 		}
 	}
 
-	for _, srv := range m.servers {
+	for i, srv := range m.servers {
 		if err := srv.WaitForReady(ctx); err != nil {
+			m.t.Logf("server %d couldn't advance block: %+v", i, err)
 			m.t.Fatal(err)
 		}
 	}

e2e/framework/testserver.go

@@ -302,6 +302,11 @@ func (t *TestServer) GenerateGenesis() error {
 	blockGasLimit := strconv.FormatUint(t.Config.BlockGasLimit, 10)
 	args = append(args, "--block-gas-limit", blockGasLimit)
 
+	// add validatorset contract owner
+	if t.Config.ValidatorSetOwner != types.ZeroAddress {
+		args = append(args, "--validatorset-owner", t.Config.ValidatorSetOwner.String())
+	}
+
 	// add bridge contract owner
 	if t.Config.BridgeOwner != types.ZeroAddress {
 		args = append(args, "--bridge-owner", t.Config.BridgeOwner.String())
@@ -371,6 +376,11 @@ func (t *TestServer) Start(ctx context.Context) error {
 		args = append(args, "--block-gas-target", *types.EncodeUint64(t.Config.BlockGasTarget))
 	}
 
+	// block generation time, not dev consesus
+	if t.Config.BlockTime > 0 {
+		args = append(args, "--block-time", strconv.FormatUint(t.Config.BlockTime, 10))
+	}
+
 	t.ReleaseReservedPorts()
 
 	// Start the server

e2e/transaction_test.go

@@ -188,88 +188,87 @@ func TestEthTransfer(t *testing.T) {
 	rpcClient := srv.JSONRPC()
 
 	for _, testCase := range testTable {
-		t.Run(testCase.name, func(t *testing.T) {
-			// Fetch the balances before sending
-			balanceSender, err := rpcClient.Eth().GetBalance(
-				web3.Address(testCase.sender),
-				web3.Latest,
-			)
-			assert.NoError(t, err)
+		// Fetch the balances before sending
+		balanceSender, err := rpcClient.Eth().GetBalance(
+			web3.Address(testCase.sender),
+			web3.Latest,
+		)
+		assert.NoError(t, err, testCase.name)
+
+		balanceReceiver, err := rpcClient.Eth().GetBalance(
+			web3.Address(testCase.recipient),
+			web3.Latest,
+		)
+		assert.NoError(t, err, testCase.name)
+
+		// Set the preSend balances
+		previousSenderBalance := balanceSender
+		previousReceiverBalance := balanceReceiver
+
+		// Do the transfer
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+
+		txn := &framework.PreparedTransaction{
+			From:     testCase.sender,
+			To:       &testCase.recipient,
+			GasPrice: big.NewInt(1048576),
+			Gas:      1000000,
+			Value:    testCase.amount,
+		}
 
-			balanceReceiver, err := rpcClient.Eth().GetBalance(
-				web3.Address(testCase.recipient),
-				web3.Latest,
+		receipt, err := srv.SendRawTx(ctx, txn, testCase.senderKey)
+
+		if testCase.shouldSucceed {
+			assert.NoError(t, err, testCase.name)
+			assert.NotNil(t, receipt, testCase.name)
+		} else { // When an invalid transaction is supplied, there should be no receipt.
+			assert.Error(t, err, testCase.name)
+			assert.Nil(t, receipt, testCase.name)
+		}
+
+		// Fetch the balances after sending
+		balanceSender, err = rpcClient.Eth().GetBalance(
+			web3.Address(testCase.sender),
+			web3.Latest,
+		)
+		assert.NoError(t, err, testCase.name)
+
+		balanceReceiver, err = rpcClient.Eth().GetBalance(
+			web3.Address(testCase.recipient),
+			web3.Latest,
+		)
+		assert.NoError(t, err, testCase.name)
+
+		expectedSenderBalance := previousSenderBalance
+		expectedReceiverBalance := previousReceiverBalance
+
+		if testCase.shouldSucceed {
+			fee := new(big.Int).Mul(
+				big.NewInt(int64(receipt.GasUsed)),
+				txn.GasPrice,
 			)
-			assert.NoError(t, err)
 
-			// Set the preSend balances
-			previousSenderBalance := balanceSender
-			previousReceiverBalance := balanceReceiver
-
-			// Do the transfer
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-
-			txn := &framework.PreparedTransaction{
-				From:     testCase.sender,
-				To:       &testCase.recipient,
-				GasPrice: big.NewInt(1048576),
-				Gas:      1000000,
-				Value:    testCase.amount,
-			}
-
-			receipt, err := srv.SendRawTx(ctx, txn, testCase.senderKey)
-
-			if testCase.shouldSucceed {
-				assert.NoError(t, err)
-				assert.NotNil(t, receipt)
-			} else { // When an invalid transaction is supplied, there should be no receipt.
-				assert.Error(t, err)
-				assert.Nil(t, receipt)
-			}
-
-			// Fetch the balances after sending
-			balanceSender, err = rpcClient.Eth().GetBalance(
-				web3.Address(testCase.sender),
-				web3.Latest,
+			expectedSenderBalance = previousSenderBalance.Sub(
+				previousSenderBalance,
+				new(big.Int).Add(testCase.amount, fee),
 			)
-			assert.NoError(t, err)
 
-			balanceReceiver, err = rpcClient.Eth().GetBalance(
-				web3.Address(testCase.recipient),
-				web3.Latest,
+			expectedReceiverBalance = previousReceiverBalance.Add(
+				previousReceiverBalance,
+				testCase.amount,
 			)
-			assert.NoError(t, err)
+		}
 
-			expectedSenderBalance := previousSenderBalance
-			expectedReceiverBalance := previousReceiverBalance
-			if testCase.shouldSucceed {
-				fee := new(big.Int).Mul(
-					big.NewInt(int64(receipt.GasUsed)),
-					txn.GasPrice,
-				)
-
-				expectedSenderBalance = previousSenderBalance.Sub(
-					previousSenderBalance,
-					new(big.Int).Add(testCase.amount, fee),
-				)
-
-				expectedReceiverBalance = previousReceiverBalance.Add(
-					previousReceiverBalance,
-					testCase.amount,
-				)
-			}
-
-			// Check the balances
-			assert.Equalf(t,
-				expectedSenderBalance,
-				balanceSender,
-				"Sender balance incorrect")
-			assert.Equalf(t,
-				expectedReceiverBalance,
-				balanceReceiver,
-				"Receiver balance incorrect")
-		})
+		// Check the balances
+		assert.Equalf(t,
+			expectedSenderBalance,
+			balanceSender,
+			testCase.name+": Sender balance incorrect")
+		assert.Equalf(t,
+			expectedReceiverBalance,
+			balanceReceiver,
+			testCase.name+": Receiver balance incorrect")
 	}
 }