Allow block_processor variables tuning with launch flags (#1842)

* Allow block_processor variables tuning with launch flags

--block_processor_batch_size
--block_processor_full_size
--block_processor_verification_size

Can be useful for fast nodes and database profiling
#1660

* Define fast_bootstrap values

* Apply changes from reviews

* Update --debug_profile_bootstrap flags

* Make separate function update_flags

* Removing default node_flags constructor

* Formatting

* Typo

nano/nano_node/entry.cpp

@@ -11,6 +11,47 @@
 #include <boost/lexical_cast.hpp>
 #include <boost/program_options.hpp>
 
+namespace
+{
+void update_flags (nano::node_flags & flags_a, boost::program_options::variables_map const & vm)
+{
+	auto batch_size_it = vm.find ("batch_size");
+	if (batch_size_it != vm.end ())
+	{
+		flags_a.sideband_batch_size = batch_size_it->second.as<size_t> ();
+	}
+	flags_a.disable_backup = (vm.count ("disable_backup") > 0);
+	flags_a.disable_lazy_bootstrap = (vm.count ("disable_lazy_bootstrap") > 0);
+	flags_a.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
+	flags_a.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
+	flags_a.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
+	flags_a.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0);
+	flags_a.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
+	flags_a.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
+	if (flags_a.fast_bootstrap)
+	{
+		flags_a.block_processor_batch_size = 256 * 1024;
+		flags_a.block_processor_full_size = 1024 * 1024;
+		flags_a.block_processor_verification_size = std::numeric_limits<size_t>::max ();
+	}
+	auto block_processor_batch_size_it = vm.find ("block_processor_batch_size");
+	if (block_processor_batch_size_it != vm.end ())
+	{
+		flags_a.block_processor_batch_size = block_processor_batch_size_it->second.as<size_t> ();
+	}
+	auto block_processor_full_size_it = vm.find ("block_processor_full_size");
+	if (block_processor_full_size_it != vm.end ())
+	{
+		flags_a.block_processor_full_size = block_processor_full_size_it->second.as<size_t> ();
+	}
+	auto block_processor_verification_size_it = vm.find ("block_processor_verification_size");
+	if (block_processor_verification_size_it != vm.end ())
+	{
+		flags_a.block_processor_verification_size = block_processor_verification_size_it->second.as<size_t> ();
+	}
+}
+}
+
 int main (int argc, char * const * argv)
 {
 	nano::set_umask ();
@@ -32,6 +73,9 @@ int main (int argc, char * const * argv)
 		("disable_unchecked_drop", "Disables drop of unchecked table at startup")
 		("fast_bootstrap", "Increase bootstrap speed for high end nodes with higher limits")
 		("batch_size",boost::program_options::value<std::size_t> (), "Increase sideband batch size, default 512")
+		("block_processor_batch_size",boost::program_options::value<std::size_t> (), "Increase block processor transaction batch write size, default 0 (limited by config block_processor_batch_max_time), 256k for fast_bootstrap")
+		("block_processor_full_size",boost::program_options::value<std::size_t> (), "Increase block processor allowed blocks queue size before dropping live network packets and holding bootstrap download, default 65536, 1 million for fast_bootstrap")
+		("block_processor_verification_size",boost::program_options::value<std::size_t> (), "Increase batch signature verification size in block processor, default 0 (limited by config signature_checker_threads), unlimited for fast_bootstrap")
 		("debug_block_count", "Display the number of block")
 		("debug_bootstrap_generate", "Generate bootstrap sequence of blocks")
 		("debug_dump_online_weight", "Dump online_weights table")
@@ -102,19 +146,7 @@ int main (int argc, char * const * argv)
 		{
 			nano_daemon::daemon daemon;
 			nano::node_flags flags;
-			auto batch_size_it = vm.find ("batch_size");
-			if (batch_size_it != vm.end ())
-			{
-				flags.sideband_batch_size = batch_size_it->second.as<size_t> ();
-			}
-			flags.disable_backup = (vm.count ("disable_backup") > 0);
-			flags.disable_lazy_bootstrap = (vm.count ("disable_lazy_bootstrap") > 0);
-			flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
-			flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
-			flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
-			flags.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0);
-			flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
-			flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
+			update_flags (flags, vm);
 			daemon.run (data_path, flags);
 		}
 		else if (vm.count ("debug_block_count"))
@@ -856,7 +888,7 @@ int main (int argc, char * const * argv)
 		else if (vm.count ("debug_profile_bootstrap"))
 		{
 			nano::inactive_node node2 (nano::unique_path (), 24001);
-			node2.node->flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
+			update_flags (node2.node->flags, vm);
 			nano::genesis genesis;
 			auto begin (std::chrono::high_resolution_clock::now ());
 			uint64_t block_count (0);

nano/nano_wallet/entry.cpp

@@ -211,7 +211,7 @@ bool update_config (qt_wallet_config & config_a, boost::filesystem::path const &
 }
 }
 
-int run_wallet (QApplication & application, int argc, char * const * argv, boost::filesystem::path const & data_path, nano::node_flags const & flags)
+int run_wallet (QApplication & application, int argc, char * const * argv, boost::filesystem::path const & data_path)
 {
 	nano_qt::eventloop_processor processor;
 	boost::system::error_code error_chmod;
@@ -243,6 +243,7 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost
 		                                                       : std::function<boost::optional<uint64_t> (nano::uint256_union const &, uint64_t)> (nullptr));
 		nano::alarm alarm (io_ctx);
 		nano::node_init init;
+		nano::node_flags flags;
 		node = std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, work, flags);
 		if (!init.error ())
 		{
@@ -369,21 +370,7 @@ int main (int argc, char * const * argv)
 					{
 						data_path = nano::working_path ();
 					}
-					nano::node_flags flags;
-					auto batch_size_it = vm.find ("batch_size");
-					if (batch_size_it != vm.end ())
-					{
-						flags.sideband_batch_size = batch_size_it->second.as<size_t> ();
-					}
-					flags.disable_backup = (vm.count ("disable_backup") > 0);
-					flags.disable_lazy_bootstrap = (vm.count ("disable_lazy_bootstrap") > 0);
-					flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
-					flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
-					flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
-					flags.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0);
-					flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
-					flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
-					result = run_wallet (application, argc, argv, data_path, flags);
+					result = run_wallet (application, argc, argv, data_path);
 				}
 				catch (std::exception const & e)
 				{

nano/node/blockprocessor.cpp

@@ -42,9 +42,8 @@ void nano::block_processor::flush ()
 
 bool nano::block_processor::full ()
 {
-	size_t full_size (node.flags.fast_bootstrap ? 1024 * 1024 : 65536);
 	std::unique_lock<std::mutex> lock (mutex);
-	return (blocks.size () + state_blocks.size ()) > full_size;
+	return (blocks.size () + state_blocks.size ()) > node.flags.block_processor_full_size;
 }
 
 void nano::block_processor::add (std::shared_ptr<nano::block> block_a, uint64_t origination)
@@ -231,7 +230,7 @@ void nano::block_processor::process_batch (std::unique_lock<std::mutex> & lock_a
 	lock_a.lock ();
 	timer_l.start ();
 	// Limit state blocks verification time
-	size_t max_verification_batch (node.flags.fast_bootstrap ? std::numeric_limits<size_t>::max () : 2048 * (node.config.signature_checker_threads + 1));
+	size_t max_verification_batch (node.flags.block_processor_verification_size != 0 ? node.flags.block_processor_verification_size : 2048 * (node.config.signature_checker_threads + 1));
 	if (!state_blocks.empty ())
 	{
 		auto transaction (node.store.tx_begin_read ());
@@ -247,7 +246,7 @@ void nano::block_processor::process_batch (std::unique_lock<std::mutex> & lock_a
 	// Processing blocks
 	auto first_time (true);
 	unsigned number_of_blocks_processed (0), number_of_forced_processed (0);
-	while ((!blocks.empty () || !forced.empty ()) && (timer_l.before_deadline (node.config.block_processor_batch_max_time) || (node.flags.fast_bootstrap && number_of_blocks_processed < 256 * 1024)))
+	while ((!blocks.empty () || !forced.empty ()) && (timer_l.before_deadline (node.config.block_processor_batch_max_time) || (number_of_blocks_processed < node.flags.block_processor_batch_size)))
 	{
 		auto log_this_record (false);
 		if (node.config.logging.timing_logging ())

nano/node/nodeconfig.cpp

@@ -419,16 +419,3 @@ nano::account nano::node_config::random_representative ()
 	auto result (preconfigured_representatives[index]);
 	return result;
 }
-
-nano::node_flags::node_flags () :
-disable_backup (false),
-disable_lazy_bootstrap (false),
-disable_legacy_bootstrap (false),
-disable_wallet_bootstrap (false),
-disable_bootstrap_listener (false),
-disable_unchecked_cleanup (false),
-disable_unchecked_drop (true),
-fast_bootstrap (false),
-sideband_batch_size (512)
-{
-}

nano/node/nodeconfig.hpp

@@ -68,15 +68,17 @@ class node_config
 class node_flags
 {
 public:
-	node_flags ();
-	bool disable_backup;
-	bool disable_lazy_bootstrap;
-	bool disable_legacy_bootstrap;
-	bool disable_wallet_bootstrap;
-	bool disable_bootstrap_listener;
-	bool disable_unchecked_cleanup;
-	bool disable_unchecked_drop;
-	bool fast_bootstrap;
-	size_t sideband_batch_size;
+	bool disable_backup{ false };
+	bool disable_lazy_bootstrap{ false };
+	bool disable_legacy_bootstrap{ false };
+	bool disable_wallet_bootstrap{ false };
+	bool disable_bootstrap_listener{ false };
+	bool disable_unchecked_cleanup{ false };
+	bool disable_unchecked_drop{ true };
+	bool fast_bootstrap{ false };
+	size_t sideband_batch_size{ 512 };
+	size_t block_processor_batch_size{ 0 };
+	size_t block_processor_full_size{ 65536 };
+	size_t block_processor_verification_size{ 0 };
 };
 }