Memory stats in RPC (#1632)

* Changes without many tests

* Fix compiler error with gcc/clang

* Add votes cache

* Hopefully fixed formatting

nano/core_test/rpc.cpp

@@ -44,7 +44,7 @@ class test_response
 									boost::property_tree::read_json (body, json);
 									status = 200;
 								}
-								catch (std::exception & e)
+								catch (std::exception &)
 								{
 									status = 500;
 								}
@@ -4243,7 +4243,7 @@ TEST (rpc, wallet_history)
 	system.deadline_set (5s);
 	while (response.status == 0)
 	{
-		system.poll ();
+		ASSERT_NO_ERROR (system.poll ());
 	}
 	ASSERT_EQ (200, response.status);
 	std::vector<std::tuple<std::string, std::string, std::string, std::string, std::string, std::string>> history_l;
@@ -4345,3 +4345,31 @@ TEST (rpc, sign_block)
 	ASSERT_NE (block->block_signature (), send.block_signature ());
 	ASSERT_EQ (block->hash (), send.hash ());
 }
+
+TEST (rpc, memory_stats)
+{
+	nano::system system (24000, 1);
+	auto node = system.nodes.front ();
+	nano::rpc rpc (system.io_ctx, *node, nano::rpc_config (true));
+
+	// Preliminary test adding to the vote uniquer and checking json output is correct
+	nano::keypair key;
+	auto block (std::make_shared<nano::state_block> (0, 0, 0, 0, 0, key.prv, key.pub, 0));
+	std::vector<nano::block_hash> hashes;
+	hashes.push_back (block->hash ());
+	auto vote (std::make_shared<nano::vote> (key.pub, key.prv, 0, hashes));
+	node->vote_uniquer.unique (vote);
+	rpc.start ();
+	boost::property_tree::ptree request;
+	request.put ("action", "stats");
+	request.put ("type", "objects");
+	test_response response (request, rpc, system.io_ctx);
+	system.deadline_set (5s);
+	while (response.status == 0)
+	{
+		ASSERT_NO_ERROR (system.poll ());
+	}
+	ASSERT_EQ (200, response.status);
+
+	ASSERT_EQ (response.json.get_child ("node").get_child ("vote_uniquer").get_child ("votes").get<std::string> ("count"), "1");
+}

nano/lib/blocks.cpp

@@ -1607,3 +1607,15 @@ size_t nano::block_uniquer::size ()
 	std::lock_guard<std::mutex> lock (mutex);
 	return blocks.size ();
 }
+
+namespace nano
+{
+std::unique_ptr<seq_con_info_component> collect_seq_con_info (block_uniquer & block_uniquer, const std::string & name)
+{
+	auto count = block_uniquer.size ();
+	auto sizeof_element = sizeof (block_uniquer::value_type);
+	auto composite = std::make_unique<seq_con_info_composite> (name);
+	composite->add_component (std::make_unique<seq_con_info_leaf> (seq_con_info{ "blocks", count, sizeof_element }));
+	return composite;
+}
+}

nano/lib/blocks.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <nano/lib/numbers.hpp>
+#include <nano/lib/utility.hpp>
 
 #include <boost/property_tree/json_parser.hpp>
 #include <cassert>
@@ -322,14 +323,19 @@ class block_visitor
 class block_uniquer
 {
 public:
+	using value_type = std::pair<const nano::uint256_union, std::weak_ptr<nano::block>>;
+
 	std::shared_ptr<nano::block> unique (std::shared_ptr<nano::block>);
 	size_t size ();
 
 private:
 	std::mutex mutex;
-	std::unordered_map<nano::uint256_union, std::weak_ptr<nano::block>> blocks;
+	std::unordered_map<std::remove_const_t<value_type::first_type>, value_type::second_type> blocks;
 	static unsigned constexpr cleanup_count = 2;
 };
+
+std::unique_ptr<seq_con_info_component> collect_seq_con_info (block_uniquer & block_uniquer, const std::string & name);
+
 std::shared_ptr<nano::block> deserialize_block (nano::stream &, nano::block_uniquer * = nullptr);
 std::shared_ptr<nano::block> deserialize_block (nano::stream &, nano::block_type, nano::block_uniquer * = nullptr);
 std::shared_ptr<nano::block> deserialize_block_json (boost::property_tree::ptree const &, nano::block_uniquer * = nullptr);

nano/lib/utility.cpp

@@ -3,6 +3,44 @@
 
 namespace nano
 {
+seq_con_info_composite::seq_con_info_composite (const std::string & name) :
+name (name)
+{
+}
+
+bool seq_con_info_composite::is_composite () const
+{
+	return true;
+}
+
+void seq_con_info_composite::add_component (std::unique_ptr<seq_con_info_component> child)
+{
+	children.push_back (std::move (child));
+}
+
+const std::vector<std::unique_ptr<seq_con_info_component>> & seq_con_info_composite::get_children () const
+{
+	return children;
+}
+
+const std::string & seq_con_info_composite::get_name () const
+{
+	return name;
+}
+
+seq_con_info_leaf::seq_con_info_leaf (const seq_con_info & info) :
+info (info)
+{
+}
+bool seq_con_info_leaf::is_composite () const
+{
+	return false;
+}
+const seq_con_info & seq_con_info_leaf::get_info () const
+{
+	return info;
+}
+
 namespace thread_role
 {
 	/*

nano/lib/utility.hpp

@@ -11,6 +11,49 @@
 
 namespace nano
 {
+/* These containers are used to collect information about sequence containers.
+ * It makes use of the composite design pattern to collect information
+ * from sequence containers and sequence containers inside member variables.
+ */
+struct seq_con_info
+{
+	std::string name;
+	size_t count;
+	size_t sizeof_element;
+};
+
+class seq_con_info_component
+{
+public:
+	virtual ~seq_con_info_component () = default;
+	virtual bool is_composite () const = 0;
+};
+
+class seq_con_info_composite : public seq_con_info_component
+{
+public:
+	seq_con_info_composite (const std::string & name);
+	bool is_composite () const override;
+	void add_component (std::unique_ptr<seq_con_info_component> child);
+	const std::vector<std::unique_ptr<seq_con_info_component>> & get_children () const;
+	const std::string & get_name () const;
+
+private:
+	std::string name;
+	std::vector<std::unique_ptr<seq_con_info_component>> children;
+};
+
+class seq_con_info_leaf : public seq_con_info_component
+{
+public:
+	seq_con_info_leaf (const seq_con_info & info);
+	bool is_composite () const override;
+	const seq_con_info & get_info () const;
+
+private:
+	seq_con_info info;
+};
+
 // Lower priority of calling work generating thread
 void work_thread_reprioritize ();
 
@@ -86,6 +129,21 @@ class observer_set
 	std::mutex mutex;
 	std::vector<std::function<void(T...)>> observers;
 };
+
+template <typename... T>
+inline std::unique_ptr<seq_con_info_component> collect_seq_con_info (observer_set<T...> & observer_set, const std::string & name)
+{
+	size_t count = 0;
+	{
+		std::lock_guard<std::mutex> lock (observer_set.mutex);
+		count = observer_set.observers.size ();
+	}
+
+	auto sizeof_element = sizeof (typename decltype (observer_set.observers)::value_type);
+	auto composite = std::make_unique<seq_con_info_composite> (name);
+	composite->add_component (std::make_unique<seq_con_info_leaf> (seq_con_info{ "observers", count, sizeof_element }));
+	return composite;
+}
 }
 
 void release_assert_internal (bool check, const char * check_expr, const char * file, unsigned int line);

nano/lib/work.cpp

@@ -193,3 +193,21 @@ uint64_t nano::work_pool::generate (nano::uint256_union const & hash_a, uint64_t
 	auto result (work.get_future ().get ());
 	return result.value ();
 }
+
+namespace nano
+{
+std::unique_ptr<seq_con_info_component> collect_seq_con_info (work_pool & work_pool, const std::string & name)
+{
+	auto composite = std::make_unique<seq_con_info_composite> (name);
+
+	size_t count = 0;
+	{
+		std::lock_guard<std::mutex> (work_pool.mutex);
+		count = work_pool.pending.size ();
+	}
+	auto sizeof_element = sizeof (decltype (work_pool.pending)::value_type);
+	composite->add_component (std::make_unique<seq_con_info_leaf> (seq_con_info{ "pending", count, sizeof_element }));
+	composite->add_component (collect_seq_con_info (work_pool.work_observers, "work_observers"));
+	return composite;
+}
+}

nano/lib/work.hpp

@@ -48,4 +48,6 @@ class work_pool
 	static uint64_t const publish_full_threshold = 0xffffffc000000000;
 	static uint64_t const publish_threshold = nano::nano_network == nano::nano_networks::nano_test_network ? publish_test_threshold : publish_full_threshold;
 };
+
+std::unique_ptr<seq_con_info_component> collect_seq_con_info (work_pool & work_pool, const std::string & name);
 }

nano/node/bootstrap.cpp

@@ -1903,6 +1903,23 @@ void nano::bootstrap_initiator::notify_listeners (bool in_progress_a)
 	}
 }
 
+namespace nano
+{
+std::unique_ptr<seq_con_info_component> collect_seq_con_info (bootstrap_initiator & bootstrap_initiator, const std::string & name)
+{
+	size_t count = 0;
+	{
+		std::lock_guard<std::mutex> guard (bootstrap_initiator.mutex);
+		count = bootstrap_initiator.observers.size ();
+	}
+
+	auto sizeof_element = sizeof (decltype (bootstrap_initiator.observers)::value_type);
+	auto composite = std::make_unique<seq_con_info_composite> (name);
+	composite->add_component (std::make_unique<seq_con_info_leaf> (seq_con_info{ "observers", count, sizeof_element }));
+	return composite;
+}
+}
+
 nano::bootstrap_listener::bootstrap_listener (boost::asio::io_context & io_ctx_a, uint16_t port_a, nano::node & node_a) :
 acceptor (io_ctx_a),
 defer_acceptor (io_ctx_a),
@@ -2005,6 +2022,23 @@ boost::asio::ip::tcp::endpoint nano::bootstrap_listener::endpoint ()
 	return boost::asio::ip::tcp::endpoint (boost::asio::ip::address_v6::loopback (), local.port ());
 }
 
+namespace nano
+{
+std::unique_ptr<seq_con_info_component> collect_seq_con_info (bootstrap_listener & bootstrap_listener, const std::string & name)
+{
+	size_t count = 0;
+	{
+		std::lock_guard<std::mutex> guard (bootstrap_listener.mutex);
+		count = bootstrap_listener.connections.size ();
+	}
+
+	auto sizeof_element = sizeof (decltype (bootstrap_listener.connections)::value_type);
+	auto composite = std::make_unique<seq_con_info_composite> (name);
+	composite->add_component (std::make_unique<seq_con_info_leaf> (seq_con_info{ "connections", count, sizeof_element }));
+	return composite;
+}
+}
+
 nano::bootstrap_server::~bootstrap_server ()
 {
 	if (node->config.logging.bulk_pull_logging ())

nano/node/bootstrap.hpp

@@ -243,7 +243,12 @@ class bootstrap_initiator
 	std::condition_variable condition;
 	std::vector<std::function<void(bool)>> observers;
 	boost::thread thread;
+
+	friend std::unique_ptr<seq_con_info_component> collect_seq_con_info (bootstrap_initiator & bootstrap_initiator, const std::string & name);
 };
+
+std::unique_ptr<seq_con_info_component> collect_seq_con_info (bootstrap_initiator & bootstrap_initiator, const std::string & name);
+
 class bootstrap_server;
 class bootstrap_listener
 {
@@ -265,6 +270,9 @@ class bootstrap_listener
 private:
 	boost::asio::steady_timer defer_acceptor;
 };
+
+std::unique_ptr<seq_con_info_component> collect_seq_con_info (bootstrap_listener & bootstrap_listener, const std::string & name);
+
 class message;
 class bootstrap_server : public std::enable_shared_from_this<nano::bootstrap_server>
 {