diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index e3901a0d4e..572b9906ec 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -167,6 +167,7 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.network_threads, defaults.node.network_threads); ASSERT_EQ (conf.node.secondary_work_peers, defaults.node.secondary_work_peers); ASSERT_EQ (conf.node.online_weight_minimum, defaults.node.online_weight_minimum); + ASSERT_EQ (conf.node.rep_crawler_weight_minimum, defaults.node.rep_crawler_weight_minimum); ASSERT_EQ (conf.node.election_hint_weight_percent, defaults.node.election_hint_weight_percent); ASSERT_EQ (conf.node.password_fanout, defaults.node.password_fanout); ASSERT_EQ (conf.node.peering_port, defaults.node.peering_port); @@ -213,6 +214,7 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.logging.active_update_value, defaults.node.logging.active_update_value); ASSERT_EQ (conf.node.logging.upnp_details_logging_value, defaults.node.logging.upnp_details_logging_value); ASSERT_EQ (conf.node.logging.vote_logging_value, defaults.node.logging.vote_logging_value); + ASSERT_EQ (conf.node.logging.rep_crawler_logging_value, defaults.node.logging.rep_crawler_logging_value); ASSERT_EQ (conf.node.logging.work_generation_time_value, defaults.node.logging.work_generation_time_value); ASSERT_EQ (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); @@ -404,6 +406,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) lmdb_max_dbs = 999 network_threads = 999 online_weight_minimum = "999" + rep_crawler_weight_minimum = "999" election_hint_weight_percent = 19 password_fanout = 999 peering_port = 999 @@ -479,6 +482,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) active_update = true upnp_details = true vote = true + rep_crawler = true work_generation_time = false [node.statistics.log] @@ -570,6 +574,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.max_pruning_age, defaults.node.max_pruning_age); ASSERT_NE (conf.node.max_pruning_depth, defaults.node.max_pruning_depth); ASSERT_NE (conf.node.online_weight_minimum, defaults.node.online_weight_minimum); + ASSERT_NE (conf.node.rep_crawler_weight_minimum, defaults.node.rep_crawler_weight_minimum); ASSERT_NE (conf.node.election_hint_weight_percent, defaults.node.election_hint_weight_percent); ASSERT_NE (conf.node.password_fanout, defaults.node.password_fanout); ASSERT_NE (conf.node.peering_port, defaults.node.peering_port); @@ -616,6 +621,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.logging.active_update_value, defaults.node.logging.active_update_value); ASSERT_NE (conf.node.logging.upnp_details_logging_value, defaults.node.logging.upnp_details_logging_value); ASSERT_NE (conf.node.logging.vote_logging_value, defaults.node.logging.vote_logging_value); + ASSERT_NE (conf.node.logging.rep_crawler_logging_value, defaults.node.logging.rep_crawler_logging_value); ASSERT_NE (conf.node.logging.work_generation_time_value, defaults.node.logging.work_generation_time_value); ASSERT_NE (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); diff --git a/nano/node/logging.cpp b/nano/node/logging.cpp index 2b7d007fa4..fddb14dd18 100644 --- a/nano/node/logging.cpp +++ b/nano/node/logging.cpp @@ -146,6 +146,7 @@ nano::error nano::logging::serialize_toml (nano::tomlconfig & toml) const toml.put ("ledger_duplicate", ledger_duplicate_logging_value, "Log when a duplicate block is attempted inserted into the ledger.\ntype:bool"); toml.put ("ledger_rollback", election_fork_tally_logging_value, "Log when a block is replaced in the ledger.\ntype:bool"); toml.put ("vote", vote_logging_value, "Vote logging. Enabling this option leads to a high volume.\nof log messages which may affect node performance.\ntype:bool"); + toml.put ("rep_crawler", rep_crawler_logging_value, "Rep crawler logging. Enabling this option leads to a high volume.\nof log messages which may affect node performance.\ntype:bool"); toml.put ("election_expiration", election_expiration_tally_logging_value, "Log election tally on expiration.\ntype:bool"); toml.put ("election_fork", election_fork_tally_logging_value, "Log election tally when more than one block is seen.\ntype:bool"); toml.put ("network", network_logging_value, "Log network related messages.\ntype:bool"); @@ -182,6 +183,7 @@ nano::error nano::logging::deserialize_toml (nano::tomlconfig & toml) toml.get ("ledger_duplicate", ledger_duplicate_logging_value); toml.get ("ledger_rollback", ledger_rollback_logging_value); toml.get ("vote", vote_logging_value); + toml.get ("rep_crawler", rep_crawler_logging_value); toml.get ("election_expiration", election_expiration_tally_logging_value); toml.get ("election_fork", election_fork_tally_logging_value); toml.get ("network", network_logging_value); @@ -220,6 +222,7 @@ nano::error nano::logging::serialize_json (nano::jsonconfig & json) const json.put ("ledger", ledger_logging_value); json.put ("ledger_duplicate", ledger_duplicate_logging_value); json.put ("vote", vote_logging_value); + json.put ("rep_crawler", rep_crawler_logging_value); json.put ("network", network_logging_value); json.put ("network_timeout", network_timeout_logging_value); json.put ("network_message", network_message_logging_value); @@ -276,6 +279,7 @@ nano::error nano::logging::deserialize_json (bool & upgraded_a, nano::jsonconfig json.get ("ledger", ledger_logging_value); json.get ("ledger_duplicate", ledger_duplicate_logging_value); json.get ("vote", vote_logging_value); + json.get ("rep_crawler", rep_crawler_logging_value); json.get ("network", network_logging_value); json.get ("network_timeout", network_timeout_logging_value); json.get ("network_message", network_message_logging_value); @@ -321,6 +325,11 @@ bool nano::logging::vote_logging () const return vote_logging_value; } +bool nano::logging::rep_crawler_logging () const +{ + return rep_crawler_logging_value; +} + bool nano::logging::election_expiration_tally_logging () const { return election_expiration_tally_logging_value; diff --git a/nano/node/logging.hpp b/nano/node/logging.hpp index a347fb52d4..1692195ad9 100644 --- a/nano/node/logging.hpp +++ b/nano/node/logging.hpp @@ -46,6 +46,7 @@ class logging final bool ledger_duplicate_logging () const; bool ledger_rollback_logging () const; bool vote_logging () const; + bool rep_crawler_logging () const; bool election_fork_tally_logging () const; bool election_expiration_tally_logging () const; bool network_logging () const; @@ -74,6 +75,7 @@ class logging final bool ledger_duplicate_logging_value{ false }; bool ledger_rollback_logging_value{ false }; bool vote_logging_value{ false }; + bool rep_crawler_logging_value{ false }; bool election_fork_tally_logging_value{ false }; bool election_expiration_tally_logging_value{ false }; bool network_logging_value{ true }; diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index 5bfc9db5e5..5603775168 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -113,6 +113,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const toml.put ("frontiers_confirmation", serialize_frontiers_confirmation (frontiers_confirmation), "Mode controlling frontier confirmation rate.\ntype:string,{auto,always,disabled}"); toml.put ("max_queued_requests", max_queued_requests, "Limit for number of queued confirmation requests for one channel, after which new requests are dropped until the queue drops below this value.\ntype:uint32"); toml.put ("confirm_req_batches_max", confirm_req_batches_max, "Limit for the number of confirmation requests for one channel per request attempt\ntype:uint32"); + toml.put ("rep_crawler_weight_minimum", rep_crawler_weight_minimum.to_string_dec (), "Rep crawler minimum weight, if this is less than minimum principal weight then this is taken as the minimum weight a rep must have to be tracked. If you want to track all reps set this to 0. If you do not want this to influence anything then set it to max value. This is only useful for debugging or for people who really know what they are doing.\ntype:string,amount,raw"); auto work_peers_l (toml.create_array ("work_peers", "A list of \"address:port\" entries to identify work peers.")); for (auto i (work_peers.begin ()), n (work_peers.end ()); i != n; ++i) @@ -356,6 +357,16 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) toml.get ("max_queued_requests", max_queued_requests); toml.get ("confirm_req_batches_max", confirm_req_batches_max); + auto rep_crawler_weight_minimum_l (rep_crawler_weight_minimum.to_string_dec ()); + if (toml.has_key ("rep_crawler_weight_minimum")) + { + rep_crawler_weight_minimum_l = toml.get ("rep_crawler_weight_minimum"); + } + if (rep_crawler_weight_minimum.decode_dec (rep_crawler_weight_minimum_l)) + { + toml.get_error ().set ("rep_crawler_weight_minimum contains an invalid decimal amount"); + } + if (toml.has_key ("frontiers_confirmation")) { auto frontiers_confirmation_l (toml.get ("frontiers_confirmation")); @@ -491,6 +502,7 @@ nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const json.put ("external_port", external_port); json.put ("tcp_incoming_connections_max", tcp_incoming_connections_max); json.put ("use_memory_pools", use_memory_pools); + json.put ("rep_crawler_weight_minimum", online_weight_minimum.to_string_dec ()); nano::jsonconfig websocket_l; websocket_config.serialize_json (websocket_l); json.put_child ("websocket", websocket_l); @@ -613,6 +625,12 @@ nano::error nano::node_config::deserialize_json (bool & upgraded_a, nano::jsonco json.get_error ().set ("online_weight_minimum contains an invalid decimal amount"); } + auto rep_crawler_weight_minimum_l (json.get ("rep_crawler_weight_minimum")); + if (rep_crawler_weight_minimum.decode_dec (rep_crawler_weight_minimum_l)) + { + json.get_error ().set ("rep_crawler_weight_minimum contains an invalid decimal amount"); + } + auto vote_minimum_l (json.get ("vote_minimum")); if (vote_minimum.decode_dec (vote_minimum_l)) { diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index 13d5b74029..8dab45086b 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -52,6 +52,7 @@ class node_config unsigned bootstrap_fraction_numerator{ 1 }; nano::amount receive_minimum{ nano::xrb_ratio }; nano::amount vote_minimum{ nano::Gxrb_ratio }; + nano::amount rep_crawler_weight_minimum{ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" }; std::chrono::milliseconds vote_generator_delay{ std::chrono::milliseconds (100) }; unsigned vote_generator_threshold{ 3 }; nano::amount online_weight_minimum{ 60000 * nano::Gxrb_ratio }; diff --git a/nano/node/repcrawler.cpp b/nano/node/repcrawler.cpp index f60251297b..d8e307fbd6 100644 --- a/nano/node/repcrawler.cpp +++ b/nano/node/repcrawler.cpp @@ -32,46 +32,76 @@ void nano::rep_crawler::validate () nano::lock_guard lock (active_mutex); responses_l.swap (responses); } - auto minimum = node.minimum_principal_weight (); + + // normally the rep_crawler only tracks principal reps but it can be made to track + // reps with less weight by setting rep_crawler_weight_minimum to a low value + auto minimum = std::min (node.minimum_principal_weight (), node.config.rep_crawler_weight_minimum.number ()); + for (auto const & i : responses_l) { auto & vote = i.second; auto & channel = i.first; debug_assert (channel != nullptr); - if (channel->get_type () != nano::transport::transport_type::loopback) + + if (channel->get_type () == nano::transport::transport_type::loopback) { - nano::uint128_t rep_weight = node.ledger.weight (vote->account); - if (rep_weight > minimum) + if (node.config.logging.rep_crawler_logging ()) { - auto updated_or_inserted = false; - nano::unique_lock lock (probable_reps_mutex); - auto existing (probable_reps.find (vote->account)); - if (existing != probable_reps.end ()) - { - probable_reps.modify (existing, [rep_weight, &updated_or_inserted, &vote, &channel] (nano::representative & info) { - info.last_response = std::chrono::steady_clock::now (); - - // Update if representative channel was changed - if (info.channel->get_endpoint () != channel->get_endpoint ()) - { - debug_assert (info.account == vote->account); - updated_or_inserted = true; - info.weight = rep_weight; - info.channel = channel; - } - }); - } - else - { - probable_reps.emplace (nano::representative (vote->account, rep_weight, channel)); - updated_or_inserted = true; - } - lock.unlock (); - if (updated_or_inserted) + node.logger.try_log (boost::str (boost::format ("rep_crawler ignoring vote from loopback channel %1%") % channel->to_string ())); + } + continue; + } + + nano::uint128_t rep_weight = node.ledger.weight (vote->account); + if (rep_weight < minimum) + { + if (node.config.logging.rep_crawler_logging ()) + { + node.logger.try_log (boost::str (boost::format ("rep_crawler ignoring vote from account %1% with too little voting weight %2%") % vote->account.to_account () % rep_weight)); + } + continue; + } + + // temporary data used for logging after dropping the lock + auto inserted = false; + auto updated = false; + std::shared_ptr prev_channel; + + nano::unique_lock lock (probable_reps_mutex); + + auto existing (probable_reps.find (vote->account)); + if (existing != probable_reps.end ()) + { + probable_reps.modify (existing, [rep_weight, &updated, &vote, &channel, &prev_channel] (nano::representative & info) { + info.last_response = std::chrono::steady_clock::now (); + + // Update if representative channel was changed + if (info.channel->get_endpoint () != channel->get_endpoint ()) { - node.logger.try_log (boost::str (boost::format ("Found a representative at %1%") % channel->to_string ())); + debug_assert (info.account == vote->account); + updated = true; + info.weight = rep_weight; + prev_channel = info.channel; + info.channel = channel; } - } + }); + } + else + { + probable_reps.emplace (nano::representative (vote->account, rep_weight, channel)); + inserted = true; + } + + lock.unlock (); + + if (inserted) + { + node.logger.try_log (boost::str (boost::format ("Found representative %1% at %2%") % vote->account.to_account () % channel->to_string ())); + } + + if (updated) + { + node.logger.try_log (boost::str (boost::format ("Updated representative %1% at %2% (was at: %3%)") % vote->account.to_account () % channel->to_string () % prev_channel->to_string ())); } } }