Rep crawler weight minimum

nano/core_test/toml.cpp

@@ -167,6 +167,7 @@ TEST (toml, daemon_config_deserialize_defaults)
 	ASSERT_EQ (conf.node.network_threads, defaults.node.network_threads);
 	ASSERT_EQ (conf.node.secondary_work_peers, defaults.node.secondary_work_peers);
 	ASSERT_EQ (conf.node.online_weight_minimum, defaults.node.online_weight_minimum);
+	ASSERT_EQ (conf.node.rep_crawler_weight_minimum, defaults.node.rep_crawler_weight_minimum);
 	ASSERT_EQ (conf.node.election_hint_weight_percent, defaults.node.election_hint_weight_percent);
 	ASSERT_EQ (conf.node.password_fanout, defaults.node.password_fanout);
 	ASSERT_EQ (conf.node.peering_port, defaults.node.peering_port);
@@ -213,6 +214,7 @@ TEST (toml, daemon_config_deserialize_defaults)
 	ASSERT_EQ (conf.node.logging.active_update_value, defaults.node.logging.active_update_value);
 	ASSERT_EQ (conf.node.logging.upnp_details_logging_value, defaults.node.logging.upnp_details_logging_value);
 	ASSERT_EQ (conf.node.logging.vote_logging_value, defaults.node.logging.vote_logging_value);
+	ASSERT_EQ (conf.node.logging.rep_crawler_logging_value, defaults.node.logging.rep_crawler_logging_value);
 	ASSERT_EQ (conf.node.logging.work_generation_time_value, defaults.node.logging.work_generation_time_value);
 
 	ASSERT_EQ (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled);
@@ -404,6 +406,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	lmdb_max_dbs = 999
 	network_threads = 999
 	online_weight_minimum = "999"
+	rep_crawler_weight_minimum = "999"
 	election_hint_weight_percent = 19
 	password_fanout = 999
 	peering_port = 999
@@ -479,6 +482,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	active_update = true
 	upnp_details = true
 	vote = true
+	rep_crawler = true
 	work_generation_time = false
 
 	[node.statistics.log]
@@ -570,6 +574,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_NE (conf.node.max_pruning_age, defaults.node.max_pruning_age);
 	ASSERT_NE (conf.node.max_pruning_depth, defaults.node.max_pruning_depth);
 	ASSERT_NE (conf.node.online_weight_minimum, defaults.node.online_weight_minimum);
+	ASSERT_NE (conf.node.rep_crawler_weight_minimum, defaults.node.rep_crawler_weight_minimum);
 	ASSERT_NE (conf.node.election_hint_weight_percent, defaults.node.election_hint_weight_percent);
 	ASSERT_NE (conf.node.password_fanout, defaults.node.password_fanout);
 	ASSERT_NE (conf.node.peering_port, defaults.node.peering_port);
@@ -616,6 +621,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_NE (conf.node.logging.active_update_value, defaults.node.logging.active_update_value);
 	ASSERT_NE (conf.node.logging.upnp_details_logging_value, defaults.node.logging.upnp_details_logging_value);
 	ASSERT_NE (conf.node.logging.vote_logging_value, defaults.node.logging.vote_logging_value);
+	ASSERT_NE (conf.node.logging.rep_crawler_logging_value, defaults.node.logging.rep_crawler_logging_value);
 	ASSERT_NE (conf.node.logging.work_generation_time_value, defaults.node.logging.work_generation_time_value);
 
 	ASSERT_NE (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled);

nano/node/logging.cpp

@@ -146,6 +146,7 @@ nano::error nano::logging::serialize_toml (nano::tomlconfig & toml) const
 	toml.put ("ledger_duplicate", ledger_duplicate_logging_value, "Log when a duplicate block is attempted inserted into the ledger.\ntype:bool");
 	toml.put ("ledger_rollback", election_fork_tally_logging_value, "Log when a block is replaced in the ledger.\ntype:bool");
 	toml.put ("vote", vote_logging_value, "Vote logging. Enabling this option leads to a high volume.\nof log messages which may affect node performance.\ntype:bool");
+	toml.put ("rep_crawler", rep_crawler_logging_value, "Rep crawler logging. Enabling this option leads to a high volume.\nof log messages which may affect node performance.\ntype:bool");
 	toml.put ("election_expiration", election_expiration_tally_logging_value, "Log election tally on expiration.\ntype:bool");
 	toml.put ("election_fork", election_fork_tally_logging_value, "Log election tally when more than one block is seen.\ntype:bool");
 	toml.put ("network", network_logging_value, "Log network related messages.\ntype:bool");
@@ -182,6 +183,7 @@ nano::error nano::logging::deserialize_toml (nano::tomlconfig & toml)
 	toml.get<bool> ("ledger_duplicate", ledger_duplicate_logging_value);
 	toml.get<bool> ("ledger_rollback", ledger_rollback_logging_value);
 	toml.get<bool> ("vote", vote_logging_value);
+	toml.get<bool> ("rep_crawler", rep_crawler_logging_value);
 	toml.get<bool> ("election_expiration", election_expiration_tally_logging_value);
 	toml.get<bool> ("election_fork", election_fork_tally_logging_value);
 	toml.get<bool> ("network", network_logging_value);
@@ -220,6 +222,7 @@ nano::error nano::logging::serialize_json (nano::jsonconfig & json) const
 	json.put ("ledger", ledger_logging_value);
 	json.put ("ledger_duplicate", ledger_duplicate_logging_value);
 	json.put ("vote", vote_logging_value);
+	json.put ("rep_crawler", rep_crawler_logging_value);
 	json.put ("network", network_logging_value);
 	json.put ("network_timeout", network_timeout_logging_value);
 	json.put ("network_message", network_message_logging_value);
@@ -276,6 +279,7 @@ nano::error nano::logging::deserialize_json (bool & upgraded_a, nano::jsonconfig
 	json.get<bool> ("ledger", ledger_logging_value);
 	json.get<bool> ("ledger_duplicate", ledger_duplicate_logging_value);
 	json.get<bool> ("vote", vote_logging_value);
+	json.get<bool> ("rep_crawler", rep_crawler_logging_value);
 	json.get<bool> ("network", network_logging_value);
 	json.get<bool> ("network_timeout", network_timeout_logging_value);
 	json.get<bool> ("network_message", network_message_logging_value);
@@ -321,6 +325,11 @@ bool nano::logging::vote_logging () const
 	return vote_logging_value;
 }
 
+bool nano::logging::rep_crawler_logging () const
+{
+	return rep_crawler_logging_value;
+}
+
 bool nano::logging::election_expiration_tally_logging () const
 {
 	return election_expiration_tally_logging_value;

nano/node/logging.hpp

@@ -46,6 +46,7 @@ class logging final
 	bool ledger_duplicate_logging () const;
 	bool ledger_rollback_logging () const;
 	bool vote_logging () const;
+	bool rep_crawler_logging () const;
 	bool election_fork_tally_logging () const;
 	bool election_expiration_tally_logging () const;
 	bool network_logging () const;
@@ -74,6 +75,7 @@ class logging final
 	bool ledger_duplicate_logging_value{ false };
 	bool ledger_rollback_logging_value{ false };
 	bool vote_logging_value{ false };
+	bool rep_crawler_logging_value{ false };
 	bool election_fork_tally_logging_value{ false };
 	bool election_expiration_tally_logging_value{ false };
 	bool network_logging_value{ true };

nano/node/nodeconfig.cpp

@@ -113,6 +113,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	toml.put ("frontiers_confirmation", serialize_frontiers_confirmation (frontiers_confirmation), "Mode controlling frontier confirmation rate.\ntype:string,{auto,always,disabled}");
 	toml.put ("max_queued_requests", max_queued_requests, "Limit for number of queued confirmation requests for one channel, after which new requests are dropped until the queue drops below this value.\ntype:uint32");
 	toml.put ("confirm_req_batches_max", confirm_req_batches_max, "Limit for the number of confirmation requests for one channel per request attempt\ntype:uint32");
+	toml.put ("rep_crawler_weight_minimum", rep_crawler_weight_minimum.to_string_dec (), "Rep crawler minimum weight, if this is less than minimum principal weight then this is taken as the minimum weight a rep must have to be tracked. If you want to track all reps set this to 0. If you do not want this to influence anything then set it to max value. This is only useful for debugging or for people who really know what they are doing.\ntype:string,amount,raw");
 
 	auto work_peers_l (toml.create_array ("work_peers", "A list of \"address:port\" entries to identify work peers."));
 	for (auto i (work_peers.begin ()), n (work_peers.end ()); i != n; ++i)
@@ -356,6 +357,16 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 		toml.get<uint32_t> ("max_queued_requests", max_queued_requests);
 		toml.get<uint32_t> ("confirm_req_batches_max", confirm_req_batches_max);
 
+		auto rep_crawler_weight_minimum_l (rep_crawler_weight_minimum.to_string_dec ());
+		if (toml.has_key ("rep_crawler_weight_minimum"))
+		{
+			rep_crawler_weight_minimum_l = toml.get<std::string> ("rep_crawler_weight_minimum");
+		}
+		if (rep_crawler_weight_minimum.decode_dec (rep_crawler_weight_minimum_l))
+		{
+			toml.get_error ().set ("rep_crawler_weight_minimum contains an invalid decimal amount");
+		}
+
 		if (toml.has_key ("frontiers_confirmation"))
 		{
 			auto frontiers_confirmation_l (toml.get<std::string> ("frontiers_confirmation"));
@@ -491,6 +502,7 @@ nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const
 	json.put ("external_port", external_port);
 	json.put ("tcp_incoming_connections_max", tcp_incoming_connections_max);
 	json.put ("use_memory_pools", use_memory_pools);
+	json.put ("rep_crawler_weight_minimum", online_weight_minimum.to_string_dec ());
 	nano::jsonconfig websocket_l;
 	websocket_config.serialize_json (websocket_l);
 	json.put_child ("websocket", websocket_l);
@@ -613,6 +625,12 @@ nano::error nano::node_config::deserialize_json (bool & upgraded_a, nano::jsonco
 			json.get_error ().set ("online_weight_minimum contains an invalid decimal amount");
 		}
 
+		auto rep_crawler_weight_minimum_l (json.get<std::string> ("rep_crawler_weight_minimum"));
+		if (rep_crawler_weight_minimum.decode_dec (rep_crawler_weight_minimum_l))
+		{
+			json.get_error ().set ("rep_crawler_weight_minimum contains an invalid decimal amount");
+		}
+
 		auto vote_minimum_l (json.get<std::string> ("vote_minimum"));
 		if (vote_minimum.decode_dec (vote_minimum_l))
 		{

nano/node/nodeconfig.hpp

@@ -52,6 +52,7 @@ class node_config
 	unsigned bootstrap_fraction_numerator{ 1 };
 	nano::amount receive_minimum{ nano::xrb_ratio };
 	nano::amount vote_minimum{ nano::Gxrb_ratio };
+	nano::amount rep_crawler_weight_minimum{ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" };
 	std::chrono::milliseconds vote_generator_delay{ std::chrono::milliseconds (100) };
 	unsigned vote_generator_threshold{ 3 };
 	nano::amount online_weight_minimum{ 60000 * nano::Gxrb_ratio };

nano/node/repcrawler.cpp

@@ -32,46 +32,76 @@ void nano::rep_crawler::validate ()
 		nano::lock_guard<nano::mutex> lock (active_mutex);
 		responses_l.swap (responses);
 	}
-	auto minimum = node.minimum_principal_weight ();
+
+	// normally the rep_crawler only tracks principal reps but it can be made to track
+	// reps with less weight by setting rep_crawler_weight_minimum to a low value
+	auto minimum = std::min (node.minimum_principal_weight (), node.config.rep_crawler_weight_minimum.number ());
+
 	for (auto const & i : responses_l)
 	{
 		auto & vote = i.second;
 		auto & channel = i.first;
 		debug_assert (channel != nullptr);
-		if (channel->get_type () != nano::transport::transport_type::loopback)
+
+		if (channel->get_type () == nano::transport::transport_type::loopback)
 		{
-			nano::uint128_t rep_weight = node.ledger.weight (vote->account);
-			if (rep_weight > minimum)
+			if (node.config.logging.rep_crawler_logging ())
 			{
-				auto updated_or_inserted = false;
-				nano::unique_lock<nano::mutex> lock (probable_reps_mutex);
-				auto existing (probable_reps.find (vote->account));
-				if (existing != probable_reps.end ())
-				{
-					probable_reps.modify (existing, [rep_weight, &updated_or_inserted, &vote, &channel] (nano::representative & info) {
-						info.last_response = std::chrono::steady_clock::now ();
-
-						// Update if representative channel was changed
-						if (info.channel->get_endpoint () != channel->get_endpoint ())
-						{
-							debug_assert (info.account == vote->account);
-							updated_or_inserted = true;
-							info.weight = rep_weight;
-							info.channel = channel;
-						}
-					});
-				}
-				else
-				{
-					probable_reps.emplace (nano::representative (vote->account, rep_weight, channel));
-					updated_or_inserted = true;
-				}
-				lock.unlock ();
-				if (updated_or_inserted)
+				node.logger.try_log (boost::str (boost::format ("rep_crawler ignoring vote from loopback channel %1%") % channel->to_string ()));
+			}
+			continue;
+		}
+
+		nano::uint128_t rep_weight = node.ledger.weight (vote->account);
+		if (rep_weight < minimum)
+		{
+			if (node.config.logging.rep_crawler_logging ())
+			{
+				node.logger.try_log (boost::str (boost::format ("rep_crawler ignoring vote from account %1% with too little voting weight %2%") % vote->account.to_account () % rep_weight));
+			}
+			continue;
+		}
+
+		// temporary data used for logging after dropping the lock
+		auto inserted = false;
+		auto updated = false;
+		std::shared_ptr<nano::transport::channel> prev_channel;
+
+		nano::unique_lock<nano::mutex> lock (probable_reps_mutex);
+
+		auto existing (probable_reps.find (vote->account));
+		if (existing != probable_reps.end ())
+		{
+			probable_reps.modify (existing, [rep_weight, &updated, &vote, &channel, &prev_channel] (nano::representative & info) {
+				info.last_response = std::chrono::steady_clock::now ();
+
+				// Update if representative channel was changed
+				if (info.channel->get_endpoint () != channel->get_endpoint ())
 				{
-					node.logger.try_log (boost::str (boost::format ("Found a representative at %1%") % channel->to_string ()));
+					debug_assert (info.account == vote->account);
+					updated = true;
+					info.weight = rep_weight;
+					prev_channel = info.channel;
+					info.channel = channel;
 				}
-			}
+			});
+		}
+		else
+		{
+			probable_reps.emplace (nano::representative (vote->account, rep_weight, channel));
+			inserted = true;
+		}
+
+		lock.unlock ();
+
+		if (inserted)
+		{
+			node.logger.try_log (boost::str (boost::format ("Found representative %1% at %2%") % vote->account.to_account () % channel->to_string ()));
+		}
+
+		if (updated)
+		{
+			node.logger.try_log (boost::str (boost::format ("Updated representative %1% at %2% (was at: %3%)") % vote->account.to_account () % channel->to_string () % prev_channel->to_string ()));
 		}
 	}
 }