#365 Retry sending transaction if it failed because of blocked accounts

proxy/common_neon/solana_interactor.py

@@ -257,3 +257,41 @@ def check_if_continue_returned(result):
                         return tx_info['transaction']['signatures'][0]
 
     return None
+
+
+def get_logs_from_reciept(receipt):
+    log_from_reciept = get_from_dict(receipt, 'result', 'meta', 'logMessages')
+    if log_from_reciept is not None:
+        return log_from_reciept
+
+    log_from_reciept_result = get_from_dict(receipt, 'meta', 'logMessages')
+    if log_from_reciept_result is not None:
+        return log_from_reciept_result
+
+    log_from_reciept_result_meta = get_from_dict(receipt, 'logMessages')
+    if log_from_reciept_result_meta is not None:
+        return log_from_reciept_result_meta
+
+    log_from_send_trx_error = get_from_dict(receipt, 'data', 'logs')
+    if log_from_send_trx_error is not None:
+        return log_from_send_trx_error
+
+    log_from_prepared_receipt = get_from_dict(receipt, 'logs')
+    if log_from_prepared_receipt is not None:
+        return log_from_prepared_receipt
+
+    return None
+
+
+def check_if_accounts_blocked(receipt):
+    logs = get_logs_from_reciept(receipt)
+    if logs is None:
+        logger.error("Can't get logs")
+        logger.info("Failed result: %s"%json.dumps(receipt, indent=3))
+
+    ro_blocked = "trying to execute transaction on ro locked account"
+    rw_blocked = "trying to execute transaction on rw locked account"
+    for log in logs:
+        if log.find(ro_blocked) >= 0 or log.find(rw_blocked) >= 0:
+            return True
+    return False

proxy/common_neon/transaction_sender.py

@@ -19,9 +19,9 @@
 from .emulator_interactor import call_emulated
 from .layouts import ACCOUNT_INFO_LAYOUT
 from .neon_instruction import NeonInstruction
-from .solana_interactor import SolanaInteractor, check_if_continue_returned, \
-    check_if_program_exceeded_instructions, check_for_errors
-from ..environment import EVM_LOADER_ID
+from .solana_interactor import SolanaInteractor, check_if_continue_returned, check_for_errors,\
+    check_if_program_exceeded_instructions, check_if_accounts_blocked
+from ..environment import EVM_LOADER_ID, RETRY_ON_BLOCKED
 from ..plugin.eth_proto import Trx as EthTrx
 
 
@@ -57,7 +57,10 @@ def execute(self):
                 errStr = str(err)
                 if "Program failed to complete" in errStr or "Computational budget exceeded" in errStr:
                     logger.debug("Program exceeded instructions")
-                    call_iterative = True
+                    if self.steps_emulated / self.steps > self.steps / 2:
+                        call_from_holder = True
+                    else:
+                        call_iterative = True
                 elif str(err).startswith("transaction too large:"):
                     logger.debug("Transaction too large, call call_signed_with_holder_acc():")
                     call_from_holder = True
@@ -316,14 +319,27 @@ def call_signed_noniterative(self):
         if len(self.create_acc_trx.instructions) > 0:
             call_txs_05.add(self.create_acc_trx)
         call_txs_05.add(self.instruction.make_noniterative_call_transaction(len(call_txs_05.instructions)))
-        result = self.sender.send_measured_transaction(call_txs_05, self.eth_trx, 'CallFromRawEthereumTX')
 
-        if check_for_errors(result):
-            if check_if_program_exceeded_instructions(result):
-                raise Exception("Program failed to complete")
-            raise Exception(json.dumps(result['result']['meta']))
+        for _i in range(RETRY_ON_BLOCKED):
+            try:
+                result = self.sender.send_measured_transaction(call_txs_05, self.eth_trx, 'CallFromRawEthereumTX')
+            except Exception as err:
+                if check_if_accounts_blocked(err.result):
+                    time.sleep(0.5)
+                    continue
+                else:
+                    raise
 
-        return result['result']['transaction']['signatures'][0]
+            if check_for_errors(result):
+                if check_if_program_exceeded_instructions(result):
+                    raise Exception("Program failed to complete")
+                elif check_if_accounts_blocked(result):
+                    time.sleep(0.5)
+                    continue
+                else:
+                    raise Exception(json.dumps(result['result']['meta']))
+            else:
+                return result['result']['transaction']['signatures'][0]
 
 
 class IterativeTransactionSender:
@@ -385,21 +401,38 @@ def write_trx_to_holder_account(self):
 
 
     def call_continue(self):
+        retry_unblock = False
         return_result = None
         try:
             return_result = self.call_continue_bucked()
         except Exception as err:
             logger.debug("call_continue_bucked_combined exception: {}".format(str(err)))
             if str(err).startswith("transaction too large:"):
                 raise
+            elif check_if_accounts_blocked(err.result):
+                retry_unblock = True
 
-        if return_result is not None:
-            return return_result
+        if retry_unblock:
+            for _i in range(RETRY_ON_BLOCKED):
+                try:
+                    self.call_continue_step()
+                except Exception as err:
+                    if check_if_accounts_blocked(err.result):
+                        pass
+                    else:
+                        raise
+                else:
+                    break
+                time.sleep(0.5)
 
-        return self.call_continue_iterative()
+            try:
+                return_result = self.call_continue_bucked()
+            except Exception as err:
+                logger.debug("call_continue_bucked_combined exception: {}".format(str(err)))
 
+        if return_result is not None:
+            return return_result
 
-    def call_continue_iterative(self):
         try:
             return self.call_continue_step_by_step()
         except Exception as err:

proxy/environment.py

@@ -19,6 +19,7 @@
 LOG_SENDING_SOLANA_TRANSACTION = os.environ.get("LOG_SENDING_SOLANA_TRANSACTION", "NO") == "YES"
 LOG_NEON_CLI_DEBUG = os.environ.get("LOG_NEON_CLI_DEBUG", "NO") == "YES"
 WRITE_TRANSACTION_COST_IN_DB = os.environ.get("WRITE_TRANSACTION_COST_IN_DB", "NO") == "YES"
+RETRY_ON_BLOCKED = int(os.environ.get("RETRY_ON_BLOCKED", "32"))
 
 class solana_cli:
     def call(self, *args):

proxy/run-proxy.sh

@@ -12,13 +12,15 @@ if [ "$CONFIG" == "ci" ]; then
   [[ -z "$MINIMAL_GAS_PRICE"            ]] && export MINIMAL_GAS_PRICE=0
   [[ -z "$POSTGRES_HOST"                ]] && export POSTGRES_HOST="postgres"
   [[ -z "$CANCEL_TIMEOUT"               ]] && export CANCEL_TIMEOUT=10
+  [[ -z "$RETRY_ON_BLOCKED"             ]] && export RETRY_ON_BLOCKED=32
 elif [ "$CONFIG" == "local" ]; then
   [[ -z "$SOLANA_URL"                   ]] && export SOLANA_URL="http://localhost:8899"
   [[ -z "$EXTRA_GAS"                    ]] && export EXTRA_GAS=0
   [[ -z "$NEON_CLI_TIMEOUT"             ]] && export NEON_CLI_TIMEOUT="0.9"
   [[ -z "$MINIMAL_GAS_PRICE"            ]] && export MINIMAL_GAS_PRICE=0
   [[ -z "$POSTGRES_HOST"                ]] && export POSTGRES_HOST="localhost"
   [[ -z "$CANCEL_TIMEOUT"               ]] && export CANCEL_TIMEOUT=10
+  [[ -z "$RETRY_ON_BLOCKED"             ]] && export RETRY_ON_BLOCKED=32
 elif [ "$CONFIG" == "devnet" ]; then
   [[ -z "$SOLANA_URL"                   ]] && export SOLANA_URL="https://api.devnet.solana.com"
   [[ -z "$EVM_LOADER"                   ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU
@@ -27,6 +29,7 @@ elif [ "$CONFIG" == "devnet" ]; then
   [[ -z "$MINIMAL_GAS_PRICE"            ]] && export MINIMAL_GAS_PRICE=1
   [[ -z "$POSTGRES_HOST"                ]] && export POSTGRES_HOST="localhost"
   [[ -z "$CANCEL_TIMEOUT"               ]] && export CANCEL_TIMEOUT=60
+  [[ -z "$RETRY_ON_BLOCKED"             ]] && export RETRY_ON_BLOCKED=32
 elif [ "$CONFIG" == "testnet" ]; then
   [[ -z "$SOLANA_URL"                   ]] && export SOLANA_URL="https://api.testnet.solana.com"
   [[ -z "$EVM_LOADER"                   ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU
@@ -35,6 +38,7 @@ elif [ "$CONFIG" == "testnet" ]; then
   [[ -z "$MINIMAL_GAS_PRICE"            ]] && export MINIMAL_GAS_PRICE="1"
   [[ -z "$POSTGRES_HOST"                ]] && export POSTGRES_HOST="localhost"
   [[ -z "$CANCEL_TIMEOUT"               ]] && export CANCEL_TIMEOUT=60
+  [[ -z "$RETRY_ON_BLOCKED"             ]] && export RETRY_ON_BLOCKED=32
 elif [ "$CONFIG" != "custom" ]; then
   exit 1
 fi