feat: Add RSK support

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Features
 
 - [#1739](https://github.com/poanetwork/blockscout/pull/1739) - highlight decompiled source code
+- [#1742](https://github.com/poanetwork/blockscout/pull/1742) - Support RSK
 
 ### Fixes
 

README.md

@@ -119,7 +119,7 @@ The [development stack page](https://github.com/poanetwork/blockscout/wiki/Devel
   `cd apps/explorer && npm install; cd -`
 
   7. Update your JSON RPC Variant in `apps/explorer/config/dev.exs` and `apps/indexer/config/dev.exs`.
-  For `variant`, enter `ganache`, `geth`, or `parity`
+  For `variant`, enter `ganache`, `geth`, `parity`, or `rsk`
 
   8. Update your JSON RPC Endpoint in `apps/explorer/config/dev/` and `apps/indexer/config/dev/`
   For the `variant` chosen in step 7, enter the correct information for the corresponding JSON RPC Endpoint in `parity.exs`, `geth.exs`, or `ganache.exs`

apps/ethereum_jsonrpc/config/config.exs

@@ -9,6 +9,14 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   wait_per_timeout: :timer.seconds(20),
   max_jitter: :timer.seconds(2)
 
+# Add this configuration to add global RPC request throttling.
+# throttle_rate_limit: 250,
+# throttle_rolling_window_opts: [
+#   window_count: 4,
+#   duration: :timer.seconds(15),
+#   table: EthereumJSONRPC.RequestCoordinator.ThrottleCounter
+# ]
+
 config :ethereum_jsonrpc, EthereumJSONRPC.Tracer,
   service: :ethereum_jsonrpc,
   adapter: SpandexDatadog.Adapter,

apps/ethereum_jsonrpc/config/test.exs

@@ -7,7 +7,14 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
     table: EthereumJSONRPC.RequestCoordinator.TimeoutCounter
   ],
   wait_per_timeout: 2,
-  max_jitter: 1
+  max_jitter: 1,
+  # This should not actually limit anything in tests, but it is here to enable the relevant code for testing
+  throttle_rate_limit: 10_000,
+  throttle_rolling_window_opts: [
+    window_count: 4,
+    duration: :timer.seconds(1),
+    table: EthereumJSONRPC.RequestCoordinator.ThrottleCounter
+  ]
 
 config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, disabled?: false
 

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex

@@ -326,11 +326,18 @@ defmodule EthereumJSONRPC do
   @doc """
   Converts `t:quantity/0` to `t:non_neg_integer/0`.
   """
-  @spec quantity_to_integer(quantity) :: non_neg_integer()
+  @spec quantity_to_integer(quantity) :: non_neg_integer() | :error
   def quantity_to_integer("0x" <> hexadecimal_digits) do
     String.to_integer(hexadecimal_digits, 16)
   end
 
+  def quantity_to_integer(string) do
+    case Integer.parse(string) do
+      {integer, ""} -> integer
+      _ -> :error
+    end
+  end
+
   @doc """
   Converts `t:non_neg_integer/0` to `t:quantity/0`
   """
@@ -398,9 +405,13 @@ defmodule EthereumJSONRPC do
   Converts `t:timestamp/0` to `t:DateTime.t/0`
   """
   def timestamp_to_datetime(timestamp) do
-    timestamp
-    |> quantity_to_integer()
-    |> Timex.from_unix()
+    case quantity_to_integer(timestamp) do
+      :error ->
+        nil
+
+      quantity ->
+        Timex.from_unix(quantity)
+    end
   end
 
   defp fetch_blocks_by_params(params, request, json_rpc_named_arguments)

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex

@@ -9,16 +9,32 @@ defmodule EthereumJSONRPC.Application do
 
   @impl Application
   def start(_type, _args) do
-    rolling_window_opts =
-      :ethereum_jsonrpc
-      |> Application.fetch_env!(RequestCoordinator)
-      |> Keyword.fetch!(:rolling_window_opts)
+    config = Application.fetch_env!(:ethereum_jsonrpc, RequestCoordinator)
 
-    children = [
+    rolling_window_opts = Keyword.fetch!(config, :rolling_window_opts)
+
+    [
       :hackney_pool.child_spec(:ethereum_jsonrpc, recv_timeout: 60_000, timeout: 60_000, max_connections: 1000),
-      {RollingWindow, [rolling_window_opts]}
+      Supervisor.child_spec({RollingWindow, [rolling_window_opts]}, id: RollingWindow.ErrorThrottle)
     ]
+    |> add_throttle_rolling_window(config)
+    |> Supervisor.start_link(strategy: :one_for_one, name: EthereumJSONRPC.Supervisor)
+  end
+
+  defp add_throttle_rolling_window(children, config) do
+    if config[:throttle_rate_limit] do
+      case Keyword.fetch(config, :throttle_rolling_window_opts) do
+        {:ok, throttle_rolling_window_opts} ->
+          child =
+            Supervisor.child_spec({RollingWindow, [throttle_rolling_window_opts]}, id: RollingWindow.ThrottleRateLimit)
+
+          [child | children]
 
-    Supervisor.start_link(children, strategy: :one_for_one, name: EthereumJSONRPC.Supervisor)
+        :error ->
+          raise "If you have configured `:throttle_rate_limit` you must also configure `:throttle_rolling_window_opts`"
+      end
+    else
+      children
+    end
   end
 end

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex

@@ -431,7 +431,8 @@ defmodule EthereumJSONRPC.Block do
     Enum.into(block, %{}, &entry_to_elixir/1)
   end
 
-  defp entry_to_elixir({key, quantity}) when key in ~w(difficulty gasLimit gasUsed number size totalDifficulty) do
+  defp entry_to_elixir({key, quantity})
+       when key in ~w(difficulty gasLimit gasUsed minimumGasPrice number size totalDifficulty) do
     {key, quantity_to_integer(quantity)}
   end
 

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipt.ex

@@ -274,10 +274,10 @@ defmodule EthereumJSONRPC.Receipt do
 
   defp entry_to_elixir({"status" = key, status}) do
     case status do
-      "0x0" ->
+      zero when zero in ["0x0", "0x00"] ->
         {:ok, {key, :error}}
 
-      "0x1" ->
+      one when one in ["0x1", "0x01"] ->
         {:ok, {key, :ok}}
 
       # pre-Byzantium / Ethereum Classic on Parity

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex

@@ -18,9 +18,14 @@ defmodule EthereumJSONRPC.RequestCoordinator do
     the tracked window
   * `:max_jitter` - Maximimum amount of time in milliseconds to be added to each
     wait before multiplied by timeout count
+  * `:throttle_rolling_window_opts` - Options for the process tracking all requests
+    * `:window_count` - Number of windows
+    * `:duration` - Total amount of time to coumt events in milliseconds
+    * `:table` - name of the ets table to store the data in
+  * `:throttle_rate_limit` - The total number of requests allowed in the all windows.
 
   See the docs for `EthereumJSONRPC.RollingWindow` for more documentation for
-  `:rolling_window_opts`.
+  `:rolling_window_opts` and `:throttle_rolling_window_opts`.
 
   This is how the wait time for each request is calculated:
 
@@ -33,20 +38,28 @@ defmodule EthereumJSONRPC.RequestCoordinator do
       config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
         rolling_window_opts: [
           window_count: 6,
-          duration: :timer.seconds(10),
+          duration: :timer.minutes(1),
           table: EthereumJSONRPC.RequestCoordinator.TimeoutCounter
         ],
         wait_per_timeout: :timer.seconds(10),
         max_jitter: :timer.seconds(1)
+        throttle_rate_limit: 60,
+        throttle_rolling_window_opts: [
+          window_count: 3,
+          duration: :timer.seconds(10),
+          table: EthereumJSONRPC.RequestCoordinator.RequestCounter
+        ]
 
   With this configuration, timeouts are tracked for 6 windows of 10 seconds for a total of 1 minute.
+  Requests are tracked for 3 windows of 10 seconds, for a total of 30 seconds, and
   """
 
   require EthereumJSONRPC.Tracer
 
   alias EthereumJSONRPC.{RollingWindow, Tracer, Transport}
 
   @error_key :throttleable_error_count
+  @throttle_key :throttle_requests_count
 
   @doc """
   Performs a JSON RPC request and adds necessary backoff.
@@ -64,12 +77,19 @@ defmodule EthereumJSONRPC.RequestCoordinator do
 
     if sleep_time <= throttle_timeout do
       :timer.sleep(sleep_time)
-
-      trace_request(request, fn ->
-        request
-        |> transport.json_rpc(transport_options)
-        |> handle_transport_response()
-      end)
+      remaining_wait_time = throttle_timeout - sleep_time
+
+      case throttle_request(remaining_wait_time) do
+        :ok ->
+          trace_request(request, fn ->
+            request
+            |> transport.json_rpc(transport_options)
+            |> handle_transport_response()
+          end)
+
+        :error ->
+          {:error, :timeout}
+      end
     else
       :timer.sleep(throttle_timeout)
 
@@ -91,33 +111,78 @@ defmodule EthereumJSONRPC.RequestCoordinator do
 
   defp handle_transport_response({:error, {:bad_gateway, _}} = error) do
     RollingWindow.inc(table(), @error_key)
+    inc_throttle_table()
     error
   end
 
   defp handle_transport_response({:error, :timeout} = error) do
     RollingWindow.inc(table(), @error_key)
+    inc_throttle_table()
     error
   end
 
-  defp handle_transport_response(response), do: response
+  defp handle_transport_response(response) do
+    inc_throttle_table()
+    response
+  end
+
+  defp inc_throttle_table do
+    if config(:throttle_rolling_window_opts) do
+      RollingWindow.inc(throttle_table(), @throttle_key)
+    end
+  end
+
+  defp throttle_request(
+         remaining_time,
+         rate_limit \\ config(:throttle_rate_limit),
+         opts \\ config(:throttle_rolling_window_opts)
+       ) do
+    if opts[:throttle_rate_limit] && RollingWindow.count(throttle_table(), @throttle_key) >= rate_limit do
+      if opts[:duration] >= remaining_time do
+        :timer.sleep(remaining_time)
+
+        :error
+      else
+        new_remaining_time = remaining_time - opts[:duration]
+        :timer.sleep(opts[:duration])
+
+        throttle_request(new_remaining_time, rate_limit, opts)
+      end
+    else
+      :ok
+    end
+  end
 
   defp sleep_time do
     wait_coefficient = RollingWindow.count(table(), @error_key)
-    jitter = :rand.uniform(config(:max_jitter))
-    wait_per_timeout = config(:wait_per_timeout)
+    jitter = :rand.uniform(config!(:max_jitter))
+    wait_per_timeout = config!(:wait_per_timeout)
 
     wait_coefficient * (wait_per_timeout + jitter)
   end
 
   defp table do
     :rolling_window_opts
-    |> config()
+    |> config!()
     |> Keyword.fetch!(:table)
   end
 
-  defp config(key) do
+  defp throttle_table do
+    case config(:throttle_rolling_window_opts) do
+      nil -> :ignore
+      keyword -> Keyword.fetch!(keyword, :table)
+    end
+  end
+
+  defp config!(key) do
     :ethereum_jsonrpc
     |> Application.get_env(__MODULE__)
     |> Keyword.fetch!(key)
   end
+
+  defp config(key) do
+    :ethereum_jsonrpc
+    |> Application.get_env(__MODULE__)
+    |> Keyword.get(key)
+  end
 end

apps/ethereum_jsonrpc/lib/rsk.ex

@@ -0,0 +1,12 @@
+defmodule EthereumJSONRPC.RSK do
+  @moduledoc """
+  Ethereum JSONRPC methods that are/are not supported by [RSK](https://www.rsk.co/).
+  """
+
+  @behaviour EthereumJSONRPC.Variant
+
+  def fetch_internal_transactions(_, _), do: :ignore
+  def fetch_pending_transactions(_), do: :ignore
+  def fetch_block_internal_transactions(_block_numbers, _json_rpc_named_arguments), do: :ignore
+  def fetch_beneficiaries(_, _), do: :ignore
+end

apps/ethereum_jsonrpc/test/ethereum_jsonrpc/request_coordinator_test.exs

@@ -11,38 +11,50 @@ defmodule EthereumJSONRPC.RequestCoordinatorTest do
   setup :verify_on_exit!
 
   setup do
-    table = Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator)[:rolling_window_opts][:table]
+    timeout_table =
+      Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator)[:rolling_window_opts][:table]
 
-    :ets.delete_all_objects(table)
+    throttle_table =
+      Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator)[:throttle_rolling_window_opts][:table]
 
-    %{table: table}
+    :ets.delete_all_objects(timeout_table)
+    :ets.delete_all_objects(throttle_table)
+
+    %{timeout_table: timeout_table, throttle_table: throttle_table}
   end
 
   describe "perform/4" do
-    test "forwards result whenever a request doesn't timeout", %{table: table} do
+    test "forwards result whenever a request doesn't timeout", %{timeout_table: timeout_table} do
       expect(EthereumJSONRPC.Mox, :json_rpc, fn _, _ -> {:ok, %{}} end)
-      assert RollingWindow.count(table, :throttleable_error_count) == 0
+      assert RollingWindow.count(timeout_table, :throttleable_error_count) == 0
       assert {:ok, %{}} == RequestCoordinator.perform(%{}, EthereumJSONRPC.Mox, [], :timer.minutes(60))
-      assert RollingWindow.count(table, :throttleable_error_count) == 0
+      assert RollingWindow.count(timeout_table, :throttleable_error_count) == 0
     end
 
-    test "increments counter on certain errors", %{table: table} do
+    test "increments counter on certain errors", %{timeout_table: timeout_table} do
       expect(EthereumJSONRPC.Mox, :json_rpc, fn :timeout, _ -> {:error, :timeout} end)
       expect(EthereumJSONRPC.Mox, :json_rpc, fn :bad_gateway, _ -> {:error, {:bad_gateway, "message"}} end)
 
       assert {:error, :timeout} == RequestCoordinator.perform(:timeout, EthereumJSONRPC.Mox, [], :timer.minutes(60))
-      assert RollingWindow.count(table, :throttleable_error_count) == 1
+      assert RollingWindow.count(timeout_table, :throttleable_error_count) == 1
 
       assert {:error, {:bad_gateway, "message"}} ==
                RequestCoordinator.perform(:bad_gateway, EthereumJSONRPC.Mox, [], :timer.minutes(60))
 
-      assert RollingWindow.count(table, :throttleable_error_count) == 2
+      assert RollingWindow.count(timeout_table, :throttleable_error_count) == 2
     end
 
-    test "returns timeout error if sleep time will exceed max timeout", %{table: table} do
+    test "returns timeout error if sleep time will exceed max timeout", %{timeout_table: timeout_table} do
       expect(EthereumJSONRPC.Mox, :json_rpc, 0, fn _, _ -> :ok end)
-      RollingWindow.inc(table, :throttleable_error_count)
+      RollingWindow.inc(timeout_table, :throttleable_error_count)
       assert {:error, :timeout} == RequestCoordinator.perform(%{}, EthereumJSONRPC.Mox, [], 1)
     end
+
+    test "increments throttle_table even when not an error", %{throttle_table: throttle_table} do
+      expect(EthereumJSONRPC.Mox, :json_rpc, fn _, _ -> {:ok, %{}} end)
+      assert RollingWindow.count(throttle_table, :throttle_requests_count) == 0
+      assert {:ok, %{}} == RequestCoordinator.perform(%{}, EthereumJSONRPC.Mox, [], :timer.minutes(60))
+      assert RollingWindow.count(throttle_table, :throttle_requests_count) == 1
+    end
   end
 end