From 9df932098ca0956a59ac2ac9811eb20351435efb Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Fri, 26 Jun 2020 19:58:04 +0100 Subject: [PATCH 1/6] Merge block databases --- nano/core_test/block_store.cpp | 148 +++++++++- nano/nano_node/entry.cpp | 10 +- nano/node/blockprocessor.cpp | 2 +- nano/node/confirmation_height_bounded.cpp | 2 +- nano/node/confirmation_height_unbounded.cpp | 2 +- nano/node/json_handler.cpp | 13 - nano/node/json_handler.hpp | 1 - nano/node/lmdb/lmdb.cpp | 305 ++++++++++++++++++-- nano/node/lmdb/lmdb.hpp | 24 +- nano/node/node.cpp | 10 +- nano/node/rocksdb/rocksdb.cpp | 22 +- nano/node/rocksdb/rocksdb.hpp | 7 - nano/node/testing.cpp | 7 +- nano/rpc_test/rpc.cpp | 34 --- nano/secure/blockstore.hpp | 49 ++-- nano/secure/blockstore_partial.hpp | 264 +++-------------- nano/secure/ledger.cpp | 39 ++- nano/secure/ledger.hpp | 1 - nano/slow_test/node.cpp | 2 +- 19 files changed, 537 insertions(+), 405 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 4fcffd04c1..9d88765905 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -31,6 +31,7 @@ void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction cons void modify_confirmation_height_to_v15 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height); void write_sideband_v14 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a, MDB_dbi db_a); void write_sideband_v15 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a); +void write_block_w_sideband_v18 (nano::mdb_store & store_a, MDB_dbi database, nano::write_transaction & transaction_a, nano::block const & block_a); } TEST (block_store, construction) @@ -126,7 +127,7 @@ TEST (block_store, add_item) ASSERT_EQ (block, *latest2); ASSERT_TRUE (store->block_exists (transaction, hash1)); ASSERT_FALSE (store->block_exists (transaction, hash1.number () - 1)); - store->block_del (transaction, hash1, block.type ()); + store->block_del (transaction, hash1); auto latest3 (store->block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest3); } @@ -684,7 +685,9 @@ TEST (mdb_block_store, supported_version_upgrades) store.version_put (transaction, store.minimum_version); store.confirmation_height_del (transaction, nano::genesis_account); ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, &store.accounts_v1)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); modify_account_info_to_v14 (store, transaction, nano::genesis_account, 1, nano::genesis_hash); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *nano::genesis ().open); } // Upgrade should work @@ -823,14 +826,14 @@ TEST (block_store, block_count) ASSERT_TRUE (!store->init_error ()); { auto transaction (store->tx_begin_write ()); - ASSERT_EQ (0, store->block_count (transaction).sum ()); + ASSERT_EQ (0, store->block_count (transaction)); nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0); block.sideband_set ({}); auto hash1 (block.hash ()); store->block_put (transaction, hash1, block); } auto transaction (store->tx_begin_read ()); - ASSERT_EQ (1, store->block_count (transaction).sum ()); + ASSERT_EQ (1, store->block_count (transaction)); } TEST (block_store, account_count) @@ -1012,13 +1015,13 @@ TEST (block_store, state_block) { auto transaction (store->tx_begin_write ()); auto count (store->block_count (transaction)); - ASSERT_EQ (1, count.state); - store->block_del (transaction, block1.hash (), block1.type ()); + ASSERT_EQ (2, count); + store->block_del (transaction, block1.hash ()); ASSERT_FALSE (store->block_exists (transaction, block1.hash ())); } auto transaction (store->tx_begin_read ()); auto count2 (store->block_count (transaction)); - ASSERT_EQ (0, count2.state); + ASSERT_EQ (1, count2); } TEST (mdb_block_store, sideband_height) @@ -1235,6 +1238,9 @@ TEST (mdb_block_store, upgrade_v14_v15) ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1)); ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, &store.accounts_v1)); ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "pending_v1", MDB_CREATE, &store.pending_v1)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "send", MDB_CREATE, &store.send_blocks)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, &store.state_blocks)); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send).code); @@ -1247,10 +1253,12 @@ TEST (mdb_block_store, upgrade_v14_v15) write_sideband_v14 (store, transaction, state_send, store.state_blocks_v1); write_sideband_v14 (store, transaction, epoch, store.state_blocks_v1); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); + write_block_w_sideband_v18 (store, store.send_blocks, transaction, send); - // Remove from state table - store.block_del (transaction, state_send.hash (), state_send.type ()); - store.block_del (transaction, epoch.hash (), epoch.type ()); + // Remove from blocks table + store.block_del (transaction, state_send.hash ()); + store.block_del (transaction, epoch.hash ()); // Turn pending into v14 ASSERT_FALSE (mdb_put (store.env.tx (transaction), store.pending_v0, nano::mdb_val (nano::pending_key (nano::test_genesis_key.pub, send.hash ())), nano::mdb_val (nano::pending_info_v14 (nano::genesis_account, nano::Gxrb_ratio, nano::epoch::epoch_0)), 0)); @@ -1328,6 +1336,8 @@ TEST (mdb_block_store, upgrade_v15_v16) ASSERT_FALSE (mdb_dbi_open (txn, "representation", MDB_CREATE, &store.representation)); auto weight = ledger.cache.rep_weights.representation_get (nano::genesis_account); ASSERT_EQ (MDB_SUCCESS, mdb_put (txn, store.representation, nano::mdb_val (nano::genesis_account), nano::mdb_val (nano::uint128_union (weight)), 0)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); // Lower the database to the previous version store.version_put (transaction, 15); // Confirm the rep weight exists in the database @@ -1374,6 +1384,13 @@ TEST (mdb_block_store, upgrade_v16_v17) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block3).code); modify_confirmation_height_to_v15 (store, transaction, nano::genesis_account, confirmation_height); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, &store.state_blocks)); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, block1); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, block2); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, block3); + // Lower the database to the previous version store.version_put (transaction, 16); } @@ -1443,22 +1460,37 @@ TEST (mdb_block_store, upgrade_v17_v18) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_open).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send_epoch_link).code); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "send", MDB_CREATE, &store.send_blocks)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, &store.state_blocks)); + // Downgrade the store store.version_put (transaction, 17); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, state_receive); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, epoch_first); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, state_send2); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, state_send_epoch_link); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); + write_block_w_sideband_v18 (store, store.send_blocks, transaction, send_zero); + // Replace with the previous sideband version for state blocks // The upgrade can resume after upgrading some blocks, test this by only downgrading some of them write_sideband_v15 (store, transaction, state_receive_zero); write_sideband_v15 (store, transaction, epoch); write_sideband_v15 (store, transaction, state_send); - // DISABLED write_sideband_v15 (store, transaction, state_receive); write_sideband_v15 (store, transaction, state_change); write_sideband_v15 (store, transaction, state_send_change); - // DISABLED write_sideband_v15 (store, transaction, epoch_first); write_sideband_v15 (store, transaction, state_receive2); - // DISABLED write_sideband_v15 (store, transaction, state_send2); write_sideband_v15 (store, transaction, state_open); - // DISABLED write_sideband_v15 (store, transaction, state_send_epoch_link); + + store.block_del (transaction, state_receive_zero.hash ()); + store.block_del (transaction, epoch.hash ()); + store.block_del (transaction, state_send.hash ()); + store.block_del (transaction, state_change.hash ()); + store.block_del (transaction, state_send_change.hash ()); + store.block_del (transaction, state_receive2.hash ()); + store.block_del (transaction, state_open.hash ()); } // Now do the upgrade @@ -1469,8 +1501,8 @@ TEST (mdb_block_store, upgrade_v17_v18) // Size of state block should equal that set in db (no change) nano::mdb_val value; - ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.state_blocks, nano::mdb_val (state_send.hash ()), value)); - ASSERT_EQ (value.size (), nano::state_block::size + nano::block_sideband::size (nano::block_type::state)); + ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.blocks, nano::mdb_val (state_send.hash ()), value)); + ASSERT_EQ (value.size (), sizeof (nano::block_type) + nano::state_block::size + nano::block_sideband::size (nano::block_type::state)); // Check that sidebands are correctly populated { @@ -1586,6 +1618,74 @@ TEST (mdb_block_store, upgrade_v17_v18) ASSERT_LT (17, store.version_get (transaction)); } +TEST (mdb_block_store, upgrade_v18_v19) +{ + auto path (nano::unique_path ()); + nano::keypair key1; + nano::work_pool pool (std::numeric_limits::max ()); + nano::send_block send (nano::genesis_hash, nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (nano::genesis_hash)); + nano::receive_block receive (send.hash (), send.hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (send.hash ())); + nano::change_block change (receive.hash (), 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (receive.hash ())); + nano::state_block state (nano::test_genesis_key.pub, change.hash (), 0, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (change.hash ())); + + { + nano::genesis genesis; + nano::logger_mt logger; + nano::mdb_store store (logger, path); + nano::stat stats; + nano::ledger ledger (store, stats); + auto transaction (store.tx_begin_write ()); + store.initialize (transaction, genesis, ledger.cache); + + // Put the genesis block back into open blocks, and clear blocks. // Upgrade the genesis block + auto txn = store.env.tx (transaction); + ASSERT_FALSE (mdb_dbi_open (txn, "open", MDB_CREATE, &store.open_blocks)); + ASSERT_FALSE (mdb_dbi_open (txn, "receive", MDB_CREATE, &store.receive_blocks)); + ASSERT_FALSE (mdb_dbi_open (txn, "send", MDB_CREATE, &store.send_blocks)); + ASSERT_FALSE (mdb_dbi_open (txn, "change", MDB_CREATE, &store.change_blocks)); + ASSERT_FALSE (mdb_dbi_open (txn, "state_blocks", MDB_CREATE, &store.state_blocks)); + + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, change).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state).code); + + // Modify blocks back to the old tables + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); + write_block_w_sideband_v18 (store, store.send_blocks, transaction, send); + write_block_w_sideband_v18 (store, store.receive_blocks, transaction, receive); + write_block_w_sideband_v18 (store, store.change_blocks, transaction, change); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, state); + + store.version_put (transaction, 18); + } + + // Now do the upgrade + nano::logger_mt logger; + nano::mdb_store store (logger, path); + ASSERT_FALSE (store.init_error ()); + auto transaction (store.tx_begin_read ()); + + // These tables should be deleted + ASSERT_EQ (store.send_blocks, 0); + ASSERT_EQ (store.receive_blocks, 0); + ASSERT_EQ (store.change_blocks, 0); + ASSERT_EQ (store.open_blocks, 0); + ASSERT_EQ (store.state_blocks, 0); + + // Confirm these blocks all exist after the upgrade + ASSERT_TRUE (store.block_get (transaction, send.hash ())); + ASSERT_TRUE (store.block_get (transaction, receive.hash ())); + ASSERT_TRUE (store.block_get (transaction, change.hash ())); + ASSERT_TRUE (store.block_get (transaction, nano::genesis_hash)); + ASSERT_TRUE (store.block_get (transaction, state.hash ())); + + ASSERT_EQ (5, store.count (transaction, store.blocks)); + + // Version should be correct + ASSERT_LT (18, store.version_get (transaction)); +} + TEST (mdb_block_store, upgrade_backup) { auto dir (nano::unique_path ()); @@ -1791,6 +1891,24 @@ void write_sideband_v15 (nano::mdb_store & store_a, nano::transaction & transact ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), store_a.state_blocks, nano::mdb_val (block_a.hash ()), &val, 0)); } +void write_block_w_sideband_v18 (nano::mdb_store & store_a, MDB_dbi database, nano::write_transaction & transaction_a, nano::block const & block_a) +{ + auto block = store_a.block_get (transaction_a, block_a.hash ()); + ASSERT_NE (block, nullptr); + + // Simulated by writing 0 on every of the most significant bits, leaving out epoch only, as if pre-upgrade + std::vector data; + { + nano::vectorstream stream (data); + block->serialize (stream); + block->sideband ().serialize (stream, block->type ()); + } + + MDB_val val{ data.size (), data.data () }; + ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), database, nano::mdb_val (block_a.hash ()), &val, 0)); + store_a.del (transaction_a, nano::tables::blocks, nano::mdb_val (block_a.hash ())); +} + void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height, nano::block_hash const & rep_block) { nano::account_info info; diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 8eca92a1ed..967852fcfe 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -312,7 +312,7 @@ int main (int argc, char * const * argv) auto inactive_node = nano::default_inactive_node (data_path, vm); auto node = inactive_node->node; auto transaction (node->store.tx_begin_read ()); - std::cout << boost::str (boost::format ("Block count: %1%\n") % node->store.block_count (transaction).sum ()); + std::cout << boost::str (boost::format ("Block count: %1%\n") % node->store.block_count (transaction)); } else if (vm.count ("debug_bootstrap_generate")) { @@ -1012,7 +1012,7 @@ int main (int argc, char * const * argv) { std::this_thread::sleep_for (std::chrono::milliseconds (10)); auto transaction (node->store.tx_begin_read ()); - block_count = node->store.block_count (transaction).sum (); + block_count = node->store.block_count (transaction); } auto end (std::chrono::high_resolution_clock::now ()); auto time (std::chrono::duration_cast (end - begin).count ()); @@ -1530,7 +1530,7 @@ int main (int argc, char * const * argv) { // State receive block_details_error = !sideband.details.is_receive || sideband.details.is_send || sideband.details.is_epoch; - block_details_error |= !node->store.source_exists (transaction, block->link ()); + block_details_error |= !node->store.block_exists (transaction, block->link ()); } } } @@ -1628,7 +1628,7 @@ int main (int argc, char * const * argv) } // Validate total block count - auto ledger_block_count (node->store.block_count (transaction).sum ()); + auto ledger_block_count (node->store.block_count (transaction)); if (block_count != ledger_block_count) { print_error_message (boost::str (boost::format ("Incorrect total block count. Blocks validated %1%. Block count in database: %2%\n") % block_count % ledger_block_count)); @@ -1787,7 +1787,7 @@ int main (int argc, char * const * argv) { std::this_thread::sleep_for (std::chrono::milliseconds (50)); auto transaction_2 (node.node->store.tx_begin_read ()); - block_count_2 = node.node->store.block_count (transaction_2).sum (); + block_count_2 = node.node->store.block_count (transaction_2); } auto end (std::chrono::high_resolution_clock::now ()); auto time (std::chrono::duration_cast (end - begin).count ()); diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index 32bf0613a8..794703ccf3 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -214,7 +214,7 @@ void nano::block_processor::process_batch (nano::unique_lock & lock_ { auto scoped_write_guard = write_database_queue.wait (nano::writer::process_batch); block_post_events post_events; - auto transaction (node.store.tx_begin_write ({ tables::accounts, nano::tables::cached_counts, nano::tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks, tables::unchecked }, { tables::confirmation_height })); + auto transaction (node.store.tx_begin_write ({ tables::accounts, tables::blocks, tables::cached_counts, tables::frontiers, tables::pending, tables::unchecked }, { tables::confirmation_height })); nano::timer timer_l; lock_a.lock (); timer_l.start (); diff --git a/nano/node/confirmation_height_bounded.cpp b/nano/node/confirmation_height_bounded.cpp index 05928da0ad..e2d4405f46 100644 --- a/nano/node/confirmation_height_bounded.cpp +++ b/nano/node/confirmation_height_bounded.cpp @@ -244,7 +244,7 @@ bool nano::confirmation_height_bounded::iterate (nano::read_transaction const & source = block->link (); } - if (!source.is_zero () && !ledger.is_epoch_link (source) && ledger.store.source_exists (transaction_a, source)) + if (!source.is_zero () && !ledger.is_epoch_link (source) && ledger.store.block_exists (transaction_a, source)) { hit_receive = true; reached_target = true; diff --git a/nano/node/confirmation_height_unbounded.cpp b/nano/node/confirmation_height_unbounded.cpp index 9cd29f02ea..cfed229c6a 100644 --- a/nano/node/confirmation_height_unbounded.cpp +++ b/nano/node/confirmation_height_unbounded.cpp @@ -194,7 +194,7 @@ void nano::confirmation_height_unbounded::collect_unconfirmed_receive_and_source source = block->link (); } - if (!source.is_zero () && !ledger.is_epoch_link (source) && ledger.store.source_exists (transaction_a, source)) + if (!source.is_zero () && !ledger.is_epoch_link (source) && ledger.store.block_exists (transaction_a, source)) { if (!hit_receive && !block_callback_data_a.empty ()) { diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index bbec8f5ca5..419ceffd20 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1247,18 +1247,6 @@ void nano::json_handler::block_count () response_errors (); } -void nano::json_handler::block_count_type () -{ - auto transaction (node.store.tx_begin_read ()); - nano::block_counts count (node.store.block_count (transaction)); - response_l.put ("send", std::to_string (count.send)); - response_l.put ("receive", std::to_string (count.receive)); - response_l.put ("open", std::to_string (count.open)); - response_l.put ("change", std::to_string (count.change)); - response_l.put ("state", std::to_string (count.state)); - response_errors (); -} - void nano::json_handler::block_create () { std::string type (request.get ("type")); @@ -5048,7 +5036,6 @@ ipc_json_handler_no_arg_func_map create_ipc_json_handler_no_arg_func_map () no_arg_funcs.emplace ("blocks_info", &nano::json_handler::blocks_info); no_arg_funcs.emplace ("block_account", &nano::json_handler::block_account); no_arg_funcs.emplace ("block_count", &nano::json_handler::block_count); - no_arg_funcs.emplace ("block_count_type", &nano::json_handler::block_count_type); no_arg_funcs.emplace ("block_create", &nano::json_handler::block_create); no_arg_funcs.emplace ("block_hash", &nano::json_handler::block_hash); no_arg_funcs.emplace ("bootstrap", &nano::json_handler::bootstrap); diff --git a/nano/node/json_handler.hpp b/nano/node/json_handler.hpp index f739ab6eca..bcfd85297b 100644 --- a/nano/node/json_handler.hpp +++ b/nano/node/json_handler.hpp @@ -51,7 +51,6 @@ class json_handler : public std::enable_shared_from_this void blocks_info (); void block_account (); void block_count (); - void block_count_type (); void block_create (); void block_hash (); void bootstrap (); diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index e4d06e673b..074f83ec70 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -176,10 +176,6 @@ nano::mdb_txn_callbacks nano::mdb_store::create_txn_callbacks () void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags) { error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "send", flags, &send_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "receive", flags, &receive_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "open", flags, &open_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "change", flags, &change_blocks) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "vote", flags, &vote) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0; @@ -191,13 +187,32 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_v0) != 0; pending = pending_v0; - if (version_get (transaction_a) < 16) + auto version_l = version_get (transaction_a); + if (version_l < 19) + { + // These legacy (and state) block databases are no longer used, but need opening so they can be deleted during an upgrade + error_a |= mdb_dbi_open (env.tx (transaction_a), "send", flags, &send_blocks) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "receive", flags, &receive_blocks) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "open", flags, &open_blocks) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "change", flags, &change_blocks) != 0; + if (version_l >= 15) + { + error_a |= mdb_dbi_open (env.tx (transaction_a), "state_blocks", flags, &state_blocks) != 0; + state_blocks_v0 = state_blocks; + } + } + else + { + error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &blocks) != 0; + } + + if (version_l < 16) { // The representation database is no longer used, but needs opening so that it can be deleted during an upgrade error_a |= mdb_dbi_open (env.tx (transaction_a), "representation", flags, &representation) != 0; } - if (version_get (transaction_a) < 15) + if (version_l < 15) { // These databases are no longer used, but need opening so they can be deleted during an upgrade error_a |= mdb_dbi_open (env.tx (transaction_a), "state", flags, &state_blocks_v0) != 0; @@ -206,11 +221,6 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & error_a |= mdb_dbi_open (env.tx (transaction_a), "pending_v1", flags, &pending_v1) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "state_v1", flags, &state_blocks_v1) != 0; } - else - { - error_a |= mdb_dbi_open (env.tx (transaction_a), "state_blocks", flags, &state_blocks) != 0; - state_blocks_v0 = state_blocks; - } } bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool & needs_vacuuming) @@ -247,6 +257,9 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool upgrade_v17_to_v18 (transaction_a); needs_vacuuming = true; case 18: + upgrade_v18_to_v19 (transaction_a); + needs_vacuuming = true; // Make this optional via config + case 19: break; default: logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); @@ -416,13 +429,13 @@ void nano::mdb_store::upgrade_v16_to_v17 (nano::write_transaction const & transa if (account_info_i->second.block_count / 2 >= confirmation_height) { // The confirmation height of the account is closer to the bottom of the chain, so start there and work up - auto block = block_get (transaction_a, account_info.open_block); + auto block = block_get_v18 (transaction_a, account_info.open_block); debug_assert (block); auto height = 1; while (height != confirmation_height) { - block = block_get (transaction_a, block->sideband ().successor); + block = block_get_v18 (transaction_a, block->sideband ().successor); debug_assert (block); ++height; } @@ -433,11 +446,11 @@ void nano::mdb_store::upgrade_v16_to_v17 (nano::write_transaction const & transa else { // The confirmation height of the account is closer to the top of the chain so start there and work down - auto block = block_get (transaction_a, account_info.head); + auto block = block_get_v18 (transaction_a, account_info.head); auto height = block->sideband ().height; while (height != confirmation_height) { - block = block_get (transaction_a, block->previous ()); + block = block_get_v18 (transaction_a, block->previous ()); debug_assert (block); --height; } @@ -473,11 +486,11 @@ void nano::mdb_store::upgrade_v17_to_v18 (nano::write_transaction const & transa auto count_pre (count (transaction_a, state_blocks)); auto num = 0u; - for (nano::mdb_iterator state_i (transaction_a, state_blocks), state_n{}; state_i != state_n; ++state_i, ++num) + for (nano::mdb_iterator> state_i (transaction_a, state_blocks), state_n{}; state_i != state_n; ++state_i, ++num) { - nano::state_block_w_sideband block_sideband (state_i->second); - auto & block (block_sideband.state_block); - auto & sideband (block_sideband.sideband); + nano::block_w_sideband_v18 block_w_sideband (state_i->second); + auto & block (block_w_sideband.block); + auto & sideband (block_w_sideband.sideband); bool is_send{ false }; bool is_receive{ false }; @@ -486,7 +499,7 @@ void nano::mdb_store::upgrade_v17_to_v18 (nano::write_transaction const & transa nano::amount prev_balance (0); if (!block->hashables.previous.is_zero ()) { - prev_balance = block_balance (transaction_a, block->hashables.previous); + prev_balance = block_balance_v18 (transaction_a, block->hashables.previous); } if (block->hashables.balance == prev_balance && network_params.ledger.epochs.is_epoch_link (block->hashables.link)) { @@ -528,6 +541,153 @@ void nano::mdb_store::upgrade_v17_to_v18 (nano::write_transaction const & transa logger.always_log ("Finished upgrading the sideband"); } +void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transaction_a) +{ + logger.always_log ("Preparing v18 to v19 database upgrade..."); + auto count_pre (count (transaction_a, state_blocks) + count (transaction_a, send_blocks) + count (transaction_a, receive_blocks) + count (transaction_a, change_blocks) + count (transaction_a, open_blocks)); + + // Combine in order of likeliness of counts + std::map legacy_open_receive_change_blocks; + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, change_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ std::dynamic_pointer_cast (i->second.block), i->second.sideband } }; + } + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, open_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ std::dynamic_pointer_cast (i->second.block), i->second.sideband } }; + } + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, receive_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ std::dynamic_pointer_cast (i->second.block), i->second.sideband } }; + } + + release_assert (!mdb_drop (env.tx (transaction_a), receive_blocks, 1)); + receive_blocks = 0; + release_assert (!mdb_drop (env.tx (transaction_a), open_blocks, 1)); + open_blocks = 0; + release_assert (!mdb_drop (env.tx (transaction_a), change_blocks, 1)); + change_blocks = 0; + + logger.always_log ("Write legacy open/receive/change to new format"); + + MDB_dbi temp_legacy_open_receive_change_blocks; + { + mdb_dbi_open (env.tx (transaction_a), "temp_legacy_open_receive_change_blocks", MDB_CREATE, &temp_legacy_open_receive_change_blocks); + + for (auto const & legacy_block : legacy_open_receive_change_blocks) + { + std::vector data; + { + nano::vectorstream stream (data); + nano::serialize_block (stream, *legacy_block.second.block); + legacy_block.second.sideband.serialize (stream, legacy_block.second.block->type ()); + } + + nano::mdb_val value{ data.size (), (void *)data.data () }; + auto s = mdb_put (env.tx (transaction_a), temp_legacy_open_receive_change_blocks, nano::mdb_val (legacy_block.first), value, MDB_APPEND); + release_assert (success (s)); + } + } + + logger.always_log ("Write legacy send to new format"); + + // Write send blocks to a new table (this was not done in memory as it would push us above memory requirements) + MDB_dbi temp_legacy_send_blocks; + { + mdb_dbi_open (env.tx (transaction_a), "temp_legacy_send_blocks", MDB_CREATE, &temp_legacy_send_blocks); + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, send_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + auto const & block_w_sideband_v18 (i->second); + + std::vector data; + { + nano::vectorstream stream (data); + nano::serialize_block (stream, *block_w_sideband_v18.block); + block_w_sideband_v18.sideband.serialize (stream, nano::block_type::send); + } + + nano::mdb_val value{ data.size (), (void *)data.data () }; + auto s = mdb_put (env.tx (transaction_a), temp_legacy_send_blocks, nano::mdb_val (i->first), value, MDB_APPEND); + release_assert (success (s)); + } + } + + release_assert (!mdb_drop (env.tx (transaction_a), send_blocks, 1)); + send_blocks = 0; + + logger.always_log ("Merge legacy open/receive/change with legacy send blocks"); + + MDB_dbi temp_legacy_send_open_receive_change_blocks; + { + mdb_dbi_open (env.tx (transaction_a), "temp_legacy_send_open_receive_change_blocks", MDB_CREATE, &temp_legacy_send_open_receive_change_blocks); + + nano::mdb_merge_iterator i (transaction_a, temp_legacy_open_receive_change_blocks, temp_legacy_send_blocks); + nano::mdb_merge_iterator n{}; + for (; i != n; ++i) + { + auto s = mdb_put (env.tx (transaction_a), temp_legacy_send_open_receive_change_blocks, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND); + release_assert (success (s)); + } + + // Delete tables + mdb_drop (env.tx (transaction_a), temp_legacy_send_blocks, 1); + mdb_drop (env.tx (transaction_a), temp_legacy_open_receive_change_blocks, 1); + } + + logger.always_log ("Write state blocks to new format"); + + // Write state blocks to a new table (this was not done in memory as it would push us above memory requirements) + MDB_dbi temp_state_blocks; + { + mdb_dbi_open (env.tx (transaction_a), "temp_state_blocks", MDB_CREATE, &temp_state_blocks); + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, state_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + auto const & block_w_sideband_v18 (i->second); + + std::vector data; + { + nano::vectorstream stream (data); + nano::serialize_block (stream, *block_w_sideband_v18.block); + block_w_sideband_v18.sideband.serialize (stream, nano::block_type::state); + } + + nano::mdb_val value{ data.size (), (void *)data.data () }; + auto s = mdb_put (env.tx (transaction_a), temp_state_blocks, nano::mdb_val (i->first), value, MDB_APPEND); + release_assert (success (s)); + } + } + + release_assert (!mdb_drop (env.tx (transaction_a), state_blocks, 1)); + state_blocks = 0; + + logger.always_log ("Merging all legacy blocks with state blocks"); + + // Merge all legacy blocks with state blocks into the final table + nano::mdb_merge_iterator i (transaction_a, temp_legacy_send_open_receive_change_blocks, temp_state_blocks); + nano::mdb_merge_iterator n{}; + mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &blocks); + for (; i != n; ++i) + { + auto s = mdb_put (env.tx (transaction_a), blocks, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND); + release_assert (success (s)); + } + + // Delete tables + mdb_drop (env.tx (transaction_a), temp_legacy_send_open_receive_change_blocks, 1); + mdb_drop (env.tx (transaction_a), temp_state_blocks, 1); + + auto count_post (count (transaction_a, blocks)); + release_assert (count_pre == count_post); + + version_put (transaction_a, 19); + logger.always_log ("Finished upgrading all blocks to new blocks database"); +} + /** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a) { @@ -621,16 +781,8 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const return frontiers; case tables::accounts: return accounts; - case tables::send_blocks: - return send_blocks; - case tables::receive_blocks: - return receive_blocks; - case tables::open_blocks: - return open_blocks; - case tables::change_blocks: - return change_blocks; - case tables::state_blocks: - return state_blocks; + case tables::blocks: + return blocks; case tables::pending: return pending; case tables::unchecked: @@ -726,6 +878,99 @@ bool nano::mdb_store::init_error () const return error; } +std::shared_ptr nano::mdb_store::block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + nano::block_type type; + auto value (block_raw_get_v18 (transaction_a, hash_a, type)); + std::shared_ptr result; + if (value.size () != 0) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = nano::deserialize_block (stream, type); + release_assert (result != nullptr); + nano::block_sideband sideband; + auto error = (sideband.deserialize (stream, type)); + release_assert (!error); + result->sideband_set (sideband); + } + return result; +} + +nano::mdb_val nano::mdb_store::block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const +{ + nano::mdb_val result; + // Table lookups are ordered by match probability + nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; + for (auto current_type : block_types) + { + auto db_val (block_raw_get_by_type_v18 (transaction_a, hash_a, current_type)); + if (db_val.is_initialized ()) + { + type_a = current_type; + result = db_val.get (); + break; + } + } + + return result; +} + +boost::optional nano::mdb_store::block_raw_get_by_type_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const +{ + nano::mdb_val value; + nano::mdb_val hash (hash_a); + int status = status_code_not_found (); + switch (type_a) + { + case nano::block_type::send: + { + status = mdb_get (env.tx (transaction_a), send_blocks, hash, value); + break; + } + case nano::block_type::receive: + { + status = mdb_get (env.tx (transaction_a), receive_blocks, hash, value); + break; + } + case nano::block_type::open: + { + status = mdb_get (env.tx (transaction_a), open_blocks, hash, value); + break; + } + case nano::block_type::change: + { + status = mdb_get (env.tx (transaction_a), change_blocks, hash, value); + break; + } + case nano::block_type::state: + { + status = mdb_get (env.tx (transaction_a), state_blocks, hash, value); + break; + } + case nano::block_type::invalid: + case nano::block_type::not_a_block: + { + break; + } + } + + release_assert (success (status) || not_found (status)); + boost::optional result; + if (success (status)) + { + result = value; + } + return result; +} + +nano::uint128_t nano::mdb_store::block_balance_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + auto block (block_get_v18 (transaction_a, hash_a)); + release_assert (block); + nano::uint128_t result (block_balance_calculated (block)); + return result; +} + // All the v14 functions below are only needed during upgrades size_t nano::mdb_store::block_successor_offset_v14 (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const { diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 483e2af2e1..53d215550a 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -81,25 +81,25 @@ class mdb_store : public block_store_partial MDB_dbi accounts{ 0 }; /** - * Maps block hash to send block. + * Maps block hash to send block. (Removed) * nano::block_hash -> nano::send_block */ MDB_dbi send_blocks{ 0 }; /** - * Maps block hash to receive block. + * Maps block hash to receive block. (Removed) * nano::block_hash -> nano::receive_block */ MDB_dbi receive_blocks{ 0 }; /** - * Maps block hash to open block. + * Maps block hash to open block. (Removed) * nano::block_hash -> nano::open_block */ MDB_dbi open_blocks{ 0 }; /** - * Maps block hash to change block. + * Maps block hash to change block. (Removed) * nano::block_hash -> nano::change_block */ MDB_dbi change_blocks{ 0 }; @@ -117,7 +117,7 @@ class mdb_store : public block_store_partial MDB_dbi state_blocks_v1{ 0 }; /** - * Maps block hash to state block. + * Maps block hash to state block. (Removed) * nano::block_hash -> nano::state_block */ MDB_dbi state_blocks{ 0 }; @@ -182,6 +182,12 @@ class mdb_store : public block_store_partial */ MDB_dbi confirmation_height{ 0 }; + /* + * Contains block_sideband and block for all block types (legacy send/change/open/receive & state blocks) + * nano::block_hash -> nano::block_sideband, nano::block + */ + MDB_dbi blocks{ 0 }; + bool exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const; int get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const; @@ -208,7 +214,7 @@ class mdb_store : public block_store_partial size_t count (nano::transaction const &, MDB_dbi) const; // These are only use in the upgrade process. - std::shared_ptr block_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband_v14 * sideband_a = nullptr, bool * is_state_v1 = nullptr) const override; + std::shared_ptr block_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband_v14 * sideband_a = nullptr, bool * is_state_v1 = nullptr) const; size_t block_successor_offset_v14 (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; nano::block_hash block_successor_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; nano::mdb_val block_raw_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1 = nullptr) const; @@ -220,6 +226,12 @@ class mdb_store : public block_store_partial void upgrade_v15_to_v16 (nano::write_transaction const &); void upgrade_v16_to_v17 (nano::write_transaction const &); void upgrade_v17_to_v18 (nano::write_transaction const &); + void upgrade_v18_to_v19 (nano::write_transaction const & transaction_a); + + std::shared_ptr block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; + nano::mdb_val block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const; + boost::optional block_raw_get_by_type_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const; + nano::uint128_t block_balance_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; void open_databases (bool &, nano::transaction const &, unsigned); diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 80b16e3ef2..3a0dcf8a50 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -365,7 +365,7 @@ node_seq (seq) if (!is_initialized) { release_assert (!flags.read_only); - auto transaction (store.tx_begin_write ({ tables::accounts, tables::cached_counts, tables::confirmation_height, tables::frontiers, tables::open_blocks })); + auto transaction (store.tx_begin_write ({ tables::accounts, tables::blocks, tables::cached_counts, tables::confirmation_height, tables::frontiers })); // Store was empty meaning we just created it, add the genesis block store.initialize (transaction, genesis, ledger.cache); } @@ -521,7 +521,7 @@ bool nano::node::copy_with_compaction (boost::filesystem::path const & destinati void nano::node::process_fork (nano::transaction const & transaction_a, std::shared_ptr block_a) { auto root (block_a->root ()); - if (!store.block_exists (transaction_a, block_a->type (), block_a->hash ()) && store.root_exists (transaction_a, block_a->root ())) + if (!store.block_exists (transaction_a, block_a->hash ()) && store.root_exists (transaction_a, block_a->root ())) { std::shared_ptr ledger_block (ledger.forked_block (transaction_a, *block_a)); if (ledger_block && !block_confirmed_or_being_confirmed (transaction_a, ledger_block->hash ()) && ledger.can_vote (transaction_a, *ledger_block)) @@ -596,7 +596,7 @@ void nano::node::process_active (std::shared_ptr incoming) nano::process_return nano::node::process (nano::block & block_a) { - auto transaction (store.tx_begin_write ({ tables::accounts, tables::cached_counts, tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks }, { tables::confirmation_height })); + auto transaction (store.tx_begin_write ({ tables::accounts, tables::blocks, tables::cached_counts, tables::frontiers, tables::pending }, { tables::confirmation_height })); auto result (ledger.process (transaction, block_a)); return result; } @@ -611,7 +611,7 @@ nano::process_return nano::node::process_local (std::shared_ptr blo block_processor.wait_write (); // Process block block_post_events events; - auto transaction (store.tx_begin_write ({ tables::accounts, tables::cached_counts, tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks }, { tables::confirmation_height })); + auto transaction (store.tx_begin_write ({ tables::accounts, tables::blocks, tables::cached_counts, tables::frontiers, tables::pending }, { tables::confirmation_height })); return block_processor.process_one (transaction, events, info, work_watcher_a, nano::block_origin::local); } @@ -1263,7 +1263,7 @@ void nano::node::process_confirmed (nano::election_status const & status_a, uint auto block_a (status_a.winner); auto hash (block_a->hash ()); const auto num_iters = (config.block_processor_batch_max_time / network_params.node.process_confirmed_interval) * 4; - if (ledger.block_exists (block_a->type (), hash)) + if (ledger.block_exists (hash)) { confirmation_height_processor.add (hash); } diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index 2d77da3011..8e955a8857 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -159,20 +159,10 @@ rocksdb::ColumnFamilyHandle * nano::rocksdb_store::table_to_column_family (table return get_handle ("frontiers"); case tables::accounts: return get_handle ("accounts"); - case tables::send_blocks: - return get_handle ("send"); - case tables::receive_blocks: - return get_handle ("receive"); - case tables::open_blocks: - return get_handle ("open"); - case tables::change_blocks: - return get_handle ("change"); - case tables::state_blocks: - return get_handle ("state_blocks"); + case tables::blocks: + return get_handle ("blocks"); case tables::pending: return get_handle ("pending"); - case tables::representation: - return get_handle ("representation"); case tables::unchecked: return get_handle ("unchecked"); case tables::vote: @@ -269,11 +259,7 @@ bool nano::rocksdb_store::is_caching_counts (nano::tables table_a) const { switch (table_a) { - case tables::send_blocks: - case tables::receive_blocks: - case tables::open_blocks: - case tables::change_blocks: - case tables::state_blocks: + case tables::blocks: return true; default: return false; @@ -539,7 +525,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options () const std::vector nano::rocksdb_store::all_tables () const { - return std::vector{ tables::accounts, tables::cached_counts, tables::change_blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::open_blocks, tables::peers, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks, tables::unchecked, tables::vote }; + return std::vector{ tables::accounts, tables::blocks, tables::cached_counts, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::unchecked, tables::vote }; } bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_path) diff --git a/nano/node/rocksdb/rocksdb.hpp b/nano/node/rocksdb/rocksdb.hpp index 723bfd54d0..6c953c87cc 100644 --- a/nano/node/rocksdb/rocksdb.hpp +++ b/nano/node/rocksdb/rocksdb.hpp @@ -46,13 +46,6 @@ class rocksdb_store : public block_store_partial // Do nothing } - std::shared_ptr block_get_v14 (nano::transaction const &, nano::block_hash const &, nano::block_sideband_v14 * = nullptr, bool * = nullptr) const override - { - // Should not be called as RocksDB has no such upgrade path - release_assert (false); - return nullptr; - } - bool copy_db (boost::filesystem::path const & destination) override; void rebuild_db (nano::write_transaction const & transaction_a) override; diff --git a/nano/node/testing.cpp b/nano/node/testing.cpp index d185779fe0..0b1da52b20 100644 --- a/nano/node/testing.cpp +++ b/nano/node/testing.cpp @@ -476,14 +476,11 @@ void nano::system::generate_mass_activity (uint32_t count_a, nano::node & node_a auto now (std::chrono::steady_clock::now ()); auto us (std::chrono::duration_cast (now - previous).count ()); uint64_t count (0); - uint64_t state (0); { auto transaction (node_a.store.tx_begin_read ()); - auto block_counts (node_a.store.block_count (transaction)); - count = block_counts.sum (); - state = block_counts.state; + count = node_a.store.block_count (transaction); } - std::cerr << boost::str (boost::format ("Mass activity iteration %1% us %2% us/t %3% state: %4% old: %5%\n") % i % us % (us / 256) % state % (count - state)); + std::cerr << boost::str (boost::format ("Mass activity iteration %1% us %2% us/t %3% block count: %5%\n") % i % us % (us / 256) % count); previous = now; } generate_activity (node_a, accounts); diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index aa6e9ee858..9f909e90cf 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -5100,40 +5100,6 @@ TEST (rpc, work_peers_all) ASSERT_EQ (0, peers_node.size ()); } -TEST (rpc, block_count_type) -{ - nano::system system; - auto node = add_ipc_enabled_node (system); - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - auto send (system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, node->config.receive_minimum.number ())); - ASSERT_NE (nullptr, send); - auto receive (system.wallet (0)->receive_action (*send, nano::test_genesis_key.pub, node->config.receive_minimum.number ())); - ASSERT_NE (nullptr, receive); - scoped_io_thread_name_change scoped_thread_name_io; - nano::node_rpc_config node_rpc_config; - nano::ipc::ipc_server ipc_server (*node, node_rpc_config); - nano::rpc_config rpc_config (nano::get_available_port (), true); - rpc_config.rpc_process.ipc_port = node->config.ipc_config.transport_tcp.port; - nano::ipc_rpc_processor ipc_rpc_processor (system.io_ctx, rpc_config); - nano::rpc rpc (system.io_ctx, rpc_config, ipc_rpc_processor); - rpc.start (); - boost::property_tree::ptree request; - request.put ("action", "block_count_type"); - test_response response (request, rpc.config.port, system.io_ctx); - ASSERT_TIMELY (5s, response.status != 0); - ASSERT_EQ (200, response.status); - std::string send_count (response.json.get ("send")); - ASSERT_EQ ("0", send_count); - std::string receive_count (response.json.get ("receive")); - ASSERT_EQ ("0", receive_count); - std::string open_count (response.json.get ("open")); - ASSERT_EQ ("1", open_count); - std::string change_count (response.json.get ("change")); - ASSERT_EQ ("0", change_count); - std::string state_count (response.json.get ("state")); - ASSERT_EQ ("2", state_count); -} - TEST (rpc, ledger) { nano::system system; diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index 623a1b728a..e783fe9796 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -18,10 +18,18 @@ namespace nano { // Move to versioning with a specific version if required for a future upgrade -class state_block_w_sideband +template +class block_w_sideband_v18 { public: - std::shared_ptr state_block; + std::shared_ptr block; + nano::block_sideband sideband; +}; + +class block_w_sideband +{ +public: + std::shared_ptr block; nano::block_sideband sideband; }; @@ -260,17 +268,28 @@ class db_val return result; } - explicit operator state_block_w_sideband () const + template + explicit operator block_w_sideband_v18 () const { nano::bufferstream stream (reinterpret_cast (data ()), size ()); auto error (false); - nano::state_block_w_sideband block_w_sideband; - block_w_sideband.state_block = std::make_shared (error, stream); - debug_assert (!error); + block_w_sideband_v18 block_w_sideband; + block_w_sideband.block = std::make_shared (error, stream); + release_assert (!error); - error = block_w_sideband.sideband.deserialize (stream, nano::block_type::state); - debug_assert (!error); + error = block_w_sideband.sideband.deserialize (stream, block_w_sideband.block->type ()); + release_assert (!error); + + return block_w_sideband; + } + explicit operator block_w_sideband () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + nano::block_w_sideband block_w_sideband; + block_w_sideband.block = (nano::deserialize_block (stream)); + auto error = block_w_sideband.sideband.deserialize (stream, block_w_sideband.block->type ()); + release_assert (!error); return block_w_sideband; } @@ -483,19 +502,14 @@ class store_iterator final enum class tables { accounts, + blocks, cached_counts, // RocksDB only - change_blocks, confirmation_height, frontiers, meta, online_weight, - open_blocks, peers, pending, - receive_blocks, - representation, - send_blocks, - state_blocks, unchecked, vote }; @@ -578,14 +592,11 @@ class block_store virtual void block_successor_clear (nano::write_transaction const &, nano::block_hash const &) = 0; virtual std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &) const = 0; virtual std::shared_ptr block_get_no_sideband (nano::transaction const &, nano::block_hash const &) const = 0; - virtual std::shared_ptr block_get_v14 (nano::transaction const &, nano::block_hash const &, nano::block_sideband_v14 * = nullptr, bool * = nullptr) const = 0; virtual std::shared_ptr block_random (nano::transaction const &) = 0; - virtual void block_del (nano::write_transaction const &, nano::block_hash const &, nano::block_type) = 0; + virtual void block_del (nano::write_transaction const &, nano::block_hash const &) = 0; virtual bool block_exists (nano::transaction const &, nano::block_hash const &) = 0; - virtual bool block_exists (nano::transaction const &, nano::block_type, nano::block_hash const &) = 0; - virtual nano::block_counts block_count (nano::transaction const &) = 0; + virtual uint64_t block_count (nano::transaction const &) = 0; virtual bool root_exists (nano::transaction const &, nano::root const &) = 0; - virtual bool source_exists (nano::transaction const &, nano::block_hash const &) = 0; virtual nano::account block_account (nano::transaction const &, nano::block_hash const &) const = 0; virtual nano::account block_account_calculated (nano::block const &) const = 0; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index d967842f1a..1f67a2bcc0 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -109,10 +109,10 @@ class block_store_partial : public block_store std::vector vector; { nano::vectorstream stream (vector); - block_a.serialize (stream); + nano::serialize_block (stream, block_a); block_a.sideband ().serialize (stream, block_a.type ()); } - block_raw_put (transaction_a, vector, block_a.type (), hash_a); + block_raw_put (transaction_a, vector, hash_a); nano::block_predecessor_set predecessor (transaction_a, *this); block_a.visit (predecessor); debug_assert (block_a.previous ().is_zero () || block_successor (transaction_a, block_a.previous ()) == hash_a); @@ -128,66 +128,48 @@ class block_store_partial : public block_store std::shared_ptr block_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override { - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); + auto value (block_raw_get (transaction_a, hash_a)); std::shared_ptr result; if (value.size () != 0) { nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + nano::block_type type; + auto error (try_read (stream, type)); + release_assert (!error); result = nano::deserialize_block (stream, type); - debug_assert (result != nullptr); + release_assert (result != nullptr); nano::block_sideband sideband; - auto error (sideband.deserialize (stream, type)); - (void)error; - debug_assert (!error); + error = (sideband.deserialize (stream, type)); + release_assert (!error); result->sideband_set (sideband); } return result; } + bool block_exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override + { + auto junk = block_raw_get (transaction_a, hash_a); + return junk.size () != 0; + } + std::shared_ptr block_get_no_sideband (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override { - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); + auto value (block_raw_get (transaction_a, hash_a)); std::shared_ptr result; if (value.size () != 0) { nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = nano::deserialize_block (stream, type); + result = nano::deserialize_block (stream); debug_assert (result != nullptr); } return result; } - bool block_exists (nano::transaction const & transaction_a, nano::block_type type, nano::block_hash const & hash_a) override - { - auto junk = block_raw_get_by_type (transaction_a, hash_a, type); - return junk.is_initialized (); - } - - bool block_exists (nano::transaction const & tx_a, nano::block_hash const & hash_a) override - { - // Table lookups are ordered by match probability - // clang-format off - return - block_exists (tx_a, nano::block_type::state, hash_a) || - block_exists (tx_a, nano::block_type::send, hash_a) || - block_exists (tx_a, nano::block_type::receive, hash_a) || - block_exists (tx_a, nano::block_type::open, hash_a) || - block_exists (tx_a, nano::block_type::change, hash_a); - // clang-format on - } - bool root_exists (nano::transaction const & transaction_a, nano::root const & root_a) override { return block_exists (transaction_a, root_a) || account_exists (transaction_a, root_a); } - bool source_exists (nano::transaction const & transaction_a, nano::block_hash const & source_a) override - { - return block_exists (transaction_a, nano::block_type::state, source_a) || block_exists (transaction_a, nano::block_type::send, source_a); - } - nano::account block_account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override { auto block (block_get (transaction_a, hash_a)); @@ -233,12 +215,12 @@ class block_store_partial : public block_store nano::block_hash block_successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override { - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); + auto value (block_raw_get (transaction_a, hash_a)); nano::block_hash result; if (value.size () != 0) { debug_assert (value.size () >= result.bytes.size ()); + auto type = static_cast ((reinterpret_cast (value.data ()))[0]); nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); auto error (nano::try_read (stream, result.bytes)); (void)error; @@ -253,12 +235,12 @@ class block_store_partial : public block_store void block_successor_clear (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override { - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); + auto value (block_raw_get (transaction_a, hash_a)); debug_assert (value.size () != 0); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); + auto type = static_cast (data[0]); std::fill_n (data.begin () + block_successor_offset (transaction_a, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); - block_raw_put (transaction_a, data, type, hash_a); + block_raw_put (transaction_a, data, hash_a); } void unchecked_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr const & block_a) override @@ -366,31 +348,9 @@ class block_store_partial : public block_store return cache_mutex; } - void block_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type block_type_a) override + void block_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override { - auto table = tables::state_blocks; - switch (block_type_a) - { - case nano::block_type::open: - table = tables::open_blocks; - break; - case nano::block_type::receive: - table = tables::receive_blocks; - break; - case nano::block_type::send: - table = tables::send_blocks; - break; - case nano::block_type::change: - table = tables::change_blocks; - break; - case nano::block_type::state: - table = tables::state_blocks; - break; - default: - debug_assert (false); - } - - auto status = del (transaction_a, table, hash_a); + auto status = del (transaction_a, tables::blocks, hash_a); release_assert (success (status)); } @@ -421,11 +381,10 @@ class block_store_partial : public block_store return nano::epoch::epoch_0; } - void block_raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::block_hash const & hash_a) + void block_raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) { - auto database_a = block_database (block_type_a); nano::db_val value{ data.size (), (void *)data.data () }; - auto status = put (transaction_a, database_a, hash_a, value); + auto status = put (transaction_a, tables::blocks, hash_a, value); release_assert (success (status)); } @@ -624,15 +583,9 @@ class block_store_partial : public block_store return static_cast (*this).exists (transaction_a, table_a, key_a); } - nano::block_counts block_count (nano::transaction const & transaction_a) override + uint64_t block_count (nano::transaction const & transaction_a) override { - nano::block_counts result; - result.send = count (transaction_a, tables::send_blocks); - result.receive = count (transaction_a, tables::receive_blocks); - result.open = count (transaction_a, tables::open_blocks); - result.change = count (transaction_a, tables::change_blocks); - result.state = count (transaction_a, tables::state_blocks); - return result; + return count (transaction_a, tables::blocks); } size_t account_count (nano::transaction const & transaction_a) override @@ -642,45 +595,17 @@ class block_store_partial : public block_store std::shared_ptr block_random (nano::transaction const & transaction_a) override { - auto count (block_count (transaction_a)); - release_assert (std::numeric_limits::max () > count.sum ()); - auto region = static_cast (nano::random_pool::generate_word32 (0, static_cast (count.sum () - 1))); - std::shared_ptr result; - auto & derived_store = static_cast (*this); - if (region < count.send) - { - result = derived_store.template block_random (transaction_a, tables::send_blocks); - } - else + nano::block_hash hash; + nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); + auto existing = make_iterator> (transaction_a, tables::blocks, nano::db_val (hash)); + auto end (nano::store_iterator> (nullptr)); + if (existing == end) { - region -= count.send; - if (region < count.receive) - { - result = derived_store.template block_random (transaction_a, tables::receive_blocks); - } - else - { - region -= count.receive; - if (region < count.open) - { - result = derived_store.template block_random (transaction_a, tables::open_blocks); - } - else - { - region -= count.open; - if (region < count.change) - { - result = derived_store.template block_random (transaction_a, tables::change_blocks); - } - else - { - result = derived_store.template block_random (transaction_a, tables::state_blocks); - } - } - } + existing = make_iterator> (transaction_a, tables::blocks); } - debug_assert (result != nullptr); - return result; + debug_assert (existing != end); + return block_get (transaction_a, nano::block_hash (existing->first)); + // TODO: Why can't this be used? return existing->second.block; } uint64_t confirmation_height_count (nano::transaction const & transaction_a) override @@ -786,22 +711,7 @@ class block_store_partial : public block_store nano::network_params network_params; std::unordered_map> vote_cache_l1; std::unordered_map> vote_cache_l2; - int const version{ 18 }; - - template - std::shared_ptr block_random (nano::transaction const & transaction_a, tables table_a) - { - nano::block_hash hash; - nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); - auto existing = make_iterator> (transaction_a, table_a, nano::db_val (hash)); - if (existing == nano::store_iterator> (nullptr)) - { - existing = make_iterator> (transaction_a, table_a); - } - auto end (nano::store_iterator> (nullptr)); - debug_assert (existing != end); - return block_get (transaction_a, nano::block_hash (existing->first)); - } + int const version{ 19 }; template nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a) const @@ -815,22 +725,11 @@ class block_store_partial : public block_store return static_cast (*this).template make_iterator (transaction_a, table_a, key); } - nano::db_val block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const + nano::db_val block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const { nano::db_val result; - // Table lookups are ordered by match probability - nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; - for (auto current_type : block_types) - { - auto db_val (block_raw_get_by_type (transaction_a, hash_a, current_type)); - if (db_val.is_initialized ()) - { - type_a = current_type; - result = db_val.get (); - break; - } - } - + auto status = get (transaction_a, tables::blocks, hash_a, result); + release_assert (success (status) || not_found (status)); return result; } @@ -839,81 +738,6 @@ class block_store_partial : public block_store return entry_size_a - nano::block_sideband::size (type_a); } - boost::optional> block_raw_get_by_type (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const - { - nano::db_val value; - nano::db_val hash (hash_a); - int status = status_code_not_found (); - switch (type_a) - { - case nano::block_type::send: - { - status = get (transaction_a, tables::send_blocks, hash, value); - break; - } - case nano::block_type::receive: - { - status = get (transaction_a, tables::receive_blocks, hash, value); - break; - } - case nano::block_type::open: - { - status = get (transaction_a, tables::open_blocks, hash, value); - break; - } - case nano::block_type::change: - { - status = get (transaction_a, tables::change_blocks, hash, value); - break; - } - case nano::block_type::state: - { - status = get (transaction_a, tables::state_blocks, hash, value); - break; - } - case nano::block_type::invalid: - case nano::block_type::not_a_block: - { - break; - } - } - - release_assert (success (status) || not_found (status)); - boost::optional> result; - if (success (status)) - { - result = value; - } - return result; - } - - tables block_database (nano::block_type type_a) - { - tables result = tables::frontiers; - switch (type_a) - { - case nano::block_type::send: - result = tables::send_blocks; - break; - case nano::block_type::receive: - result = tables::receive_blocks; - break; - case nano::block_type::open: - result = tables::open_blocks; - break; - case nano::block_type::change: - result = tables::change_blocks; - break; - case nano::block_type::state: - result = tables::state_blocks; - break; - default: - debug_assert (false); - break; - } - return result; - } - size_t count (nano::transaction const & transaction_a, std::initializer_list dbs_a) const { size_t total_count = 0; @@ -962,12 +786,12 @@ class block_predecessor_set : public nano::block_visitor void fill_value (nano::block const & block_a) { auto hash (block_a.hash ()); - nano::block_type type; - auto value (store.block_raw_get (transaction, block_a.previous (), type)); + auto value (store.block_raw_get (transaction, block_a.previous ())); debug_assert (value.size () != 0); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); + auto type = static_cast (data[0]); std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + store.block_successor_offset (transaction, value.size (), type)); - store.block_raw_put (transaction, data, type, block_a.previous ()); + store.block_raw_put (transaction, data, block_a.previous ()); } void send_block (nano::send_block const & block_a) override { diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index bbf39361b2..73df183cbb 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -40,7 +40,7 @@ class rollback_visitor : public nano::block_visitor ledger.cache.rep_weights.representation_add (info.representative, pending.amount.number ()); nano::account_info new_info (block_a.hashables.previous, info.representative, info.open_block, ledger.balance (transaction, block_a.hashables.previous), nano::seconds_since_epoch (), info.block_count - 1, nano::epoch::epoch_0); ledger.change_latest (transaction, pending.source, info, new_info); - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, pending.source); ledger.store.block_successor_clear (transaction, block_a.hashables.previous); @@ -60,7 +60,7 @@ class rollback_visitor : public nano::block_visitor ledger.cache.rep_weights.representation_add (info.representative, 0 - amount); nano::account_info new_info (block_a.hashables.previous, info.representative, info.open_block, ledger.balance (transaction, block_a.hashables.previous), nano::seconds_since_epoch (), info.block_count - 1, nano::epoch::epoch_0); ledger.change_latest (transaction, destination_account, info, new_info); - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); ledger.store.pending_put (transaction, nano::pending_key (destination_account, block_a.hashables.source), { source_account, amount, nano::epoch::epoch_0 }); ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, destination_account); @@ -76,7 +76,7 @@ class rollback_visitor : public nano::block_visitor ledger.cache.rep_weights.representation_add (block_a.representative (), 0 - amount); nano::account_info new_info; ledger.change_latest (transaction, destination_account, new_info, new_info); - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); ledger.store.pending_put (transaction, nano::pending_key (destination_account, block_a.hashables.source), { source_account, amount, nano::epoch::epoch_0 }); ledger.store.frontier_del (transaction, hash); ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::open); @@ -96,7 +96,7 @@ class rollback_visitor : public nano::block_visitor auto representative = block->representative (); ledger.cache.rep_weights.representation_add (block_a.representative (), 0 - balance); ledger.cache.rep_weights.representation_add (representative, balance); - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); nano::account_info new_info (block_a.hashables.previous, representative, info.open_block, info.balance, nano::seconds_since_epoch (), info.block_count - 1, nano::epoch::epoch_0); ledger.change_latest (transaction, account, info, new_info); ledger.store.frontier_del (transaction, hash); @@ -165,7 +165,7 @@ class rollback_visitor : public nano::block_visitor { ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::open); } - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); } nano::write_transaction const & transaction; nano::ledger & ledger; @@ -257,7 +257,7 @@ void ledger_processor::state_block (nano::state_block & block_a) void ledger_processor::state_block_impl (nano::state_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block before? (Unambiguous) if (result.code == nano::process_result::progress) { @@ -314,7 +314,7 @@ void ledger_processor::state_block_impl (nano::state_block & block_a) { if (!block_a.hashables.link.is_zero ()) { - result.code = ledger.store.source_exists (transaction, block_a.hashables.link) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block already? (Harmless) + result.code = ledger.store.block_exists (transaction, block_a.hashables.link) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block already? (Harmless) if (result.code == nano::process_result::progress) { nano::pending_key key (block_a.hashables.account, block_a.hashables.link); @@ -381,7 +381,7 @@ void ledger_processor::state_block_impl (nano::state_block & block_a) void ledger_processor::epoch_block_impl (nano::state_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block before? (Unambiguous) if (result.code == nano::process_result::progress) { @@ -473,7 +473,7 @@ void ledger_processor::epoch_block_impl (nano::state_block & block_a) void ledger_processor::change_block (nano::change_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block before? (Harmless) if (result.code == nano::process_result::progress) { @@ -530,7 +530,7 @@ void ledger_processor::change_block (nano::change_block & block_a) void ledger_processor::send_block (nano::send_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block before? (Harmless) if (result.code == nano::process_result::progress) { @@ -592,7 +592,7 @@ void ledger_processor::send_block (nano::send_block & block_a) void ledger_processor::receive_block (nano::receive_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block already? (Harmless) if (result.code == nano::process_result::progress) { @@ -616,7 +616,7 @@ void ledger_processor::receive_block (nano::receive_block & block_a) { debug_assert (!validate_message (account, hash, block_a.signature)); result.verified = nano::signature_verification::valid; - result.code = ledger.store.source_exists (transaction, block_a.hashables.source) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block already? (Harmless) + result.code = ledger.store.block_exists (transaction, block_a.hashables.source) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block already? (Harmless) if (result.code == nano::process_result::progress) { nano::account_info info; @@ -672,7 +672,7 @@ void ledger_processor::receive_block (nano::receive_block & block_a) void ledger_processor::open_block (nano::open_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block already? (Harmless) if (result.code == nano::process_result::progress) { @@ -685,7 +685,7 @@ void ledger_processor::open_block (nano::open_block & block_a) { debug_assert (!validate_message (block_a.hashables.account, hash, block_a.signature)); result.verified = nano::signature_verification::valid; - result.code = ledger.store.source_exists (transaction, block_a.hashables.source) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block? (Harmless) + result.code = ledger.store.block_exists (transaction, block_a.hashables.source) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block? (Harmless) if (result.code == nano::process_result::progress) { nano::account_info info; @@ -776,7 +776,7 @@ epoch_2_started_cb (epoch_2_started_cb_a) cache.unchecked_count = store.unchecked_count (transaction); } - cache.block_count = store.block_count (transaction).sum (); + cache.block_count = store.block_count (transaction); } } @@ -842,11 +842,6 @@ bool nano::ledger::block_exists (nano::block_hash const & hash_a) return store.block_exists (store.tx_begin_read (), hash_a); } -bool nano::ledger::block_exists (nano::block_type type, nano::block_hash const & hash_a) -{ - return store.block_exists (store.tx_begin_read (), type, hash_a); -} - std::string nano::ledger::block_text (char const * hash_a) { return block_text (nano::block_hash (hash_a)); @@ -1204,7 +1199,7 @@ std::shared_ptr nano::ledger::successor (nano::transaction const & std::shared_ptr nano::ledger::forked_block (nano::transaction const & transaction_a, nano::block const & block_a) { - debug_assert (!store.block_exists (transaction_a, block_a.type (), block_a.hash ())); + debug_assert (!store.block_exists (transaction_a, block_a.hash ())); auto root (block_a.root ()); debug_assert (store.block_exists (transaction_a, root) || store.account_exists (transaction_a, root)); auto result (store.block_get (transaction_a, store.block_successor (transaction_a, root))); @@ -1251,7 +1246,7 @@ bool nano::ledger::block_not_confirmed_or_not_exists (nano::block const & block_ bool result (true); auto hash (block_a.hash ()); auto transaction (store.tx_begin_read ()); - if (store.block_exists (transaction, block_a.type (), hash)) + if (store.block_exists (transaction, hash)) { result = !block_confirmed (transaction, hash); } diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index a246f0433c..cd7deeb9e9 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -33,7 +33,6 @@ class ledger final nano::block_hash representative (nano::transaction const &, nano::block_hash const &); nano::block_hash representative_calculated (nano::transaction const &, nano::block_hash const &); bool block_exists (nano::block_hash const &); - bool block_exists (nano::block_type, nano::block_hash const &); std::string block_text (char const *); std::string block_text (nano::block_hash const &); bool is_send (nano::transaction const &, nano::state_block const &) const; diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index a11604c961..fb717b3919 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -1623,7 +1623,7 @@ TEST (node, mass_epoch_upgrader) // Check upgrade { auto transaction (node.store.tx_begin_read ()); - ASSERT_EQ (expected_blocks, node.store.block_count (transaction).sum ()); + ASSERT_EQ (expected_blocks, node.store.block_count (transaction)); for (auto i (node.store.latest_begin (transaction)); i != node.store.latest_end (); ++i) { nano::account_info info (i->second); From b3af678cd38cd3d80154aabfc3f07748b42ef524 Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Sat, 27 Jun 2020 12:03:59 +0100 Subject: [PATCH 2/6] Some code cleanup --- nano/core_test/block_store.cpp | 13 ++++++------- nano/node/lmdb/lmdb.cpp | 6 +++--- nano/node/testing.cpp | 2 +- nano/secure/blockstore_partial.hpp | 15 ++++++++++----- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 9d88765905..52726c35ab 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1637,7 +1637,12 @@ TEST (mdb_block_store, upgrade_v18_v19) auto transaction (store.tx_begin_write ()); store.initialize (transaction, genesis, ledger.cache); - // Put the genesis block back into open blocks, and clear blocks. // Upgrade the genesis block + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, change).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state).code); + + // These tables need to be re-opened and populated so that an upgrade can be done auto txn = store.env.tx (transaction); ASSERT_FALSE (mdb_dbi_open (txn, "open", MDB_CREATE, &store.open_blocks)); ASSERT_FALSE (mdb_dbi_open (txn, "receive", MDB_CREATE, &store.receive_blocks)); @@ -1645,11 +1650,6 @@ TEST (mdb_block_store, upgrade_v18_v19) ASSERT_FALSE (mdb_dbi_open (txn, "change", MDB_CREATE, &store.change_blocks)); ASSERT_FALSE (mdb_dbi_open (txn, "state_blocks", MDB_CREATE, &store.state_blocks)); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, change).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state).code); - // Modify blocks back to the old tables write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); write_block_w_sideband_v18 (store, store.send_blocks, transaction, send); @@ -1896,7 +1896,6 @@ void write_block_w_sideband_v18 (nano::mdb_store & store_a, MDB_dbi database, na auto block = store_a.block_get (transaction_a, block_a.hash ()); ASSERT_NE (block, nullptr); - // Simulated by writing 0 on every of the most significant bits, leaving out epoch only, as if pre-upgrade std::vector data; { nano::vectorstream stream (data); diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 074f83ec70..9d2b9e411e 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -551,17 +551,17 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, change_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) { - legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ std::dynamic_pointer_cast (i->second.block), i->second.sideband } }; + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, i->second.sideband } }; } for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, open_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) { - legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ std::dynamic_pointer_cast (i->second.block), i->second.sideband } }; + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, i->second.sideband } }; } for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, receive_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) { - legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ std::dynamic_pointer_cast (i->second.block), i->second.sideband } }; + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, i->second.sideband } }; } release_assert (!mdb_drop (env.tx (transaction_a), receive_blocks, 1)); diff --git a/nano/node/testing.cpp b/nano/node/testing.cpp index 0b1da52b20..a8fd4477a5 100644 --- a/nano/node/testing.cpp +++ b/nano/node/testing.cpp @@ -480,7 +480,7 @@ void nano::system::generate_mass_activity (uint32_t count_a, nano::node & node_a auto transaction (node_a.store.tx_begin_read ()); count = node_a.store.block_count (transaction); } - std::cerr << boost::str (boost::format ("Mass activity iteration %1% us %2% us/t %3% block count: %5%\n") % i % us % (us / 256) % count); + std::cerr << boost::str (boost::format ("Mass activity iteration %1% us %2% us/t %3% block count: %4%\n") % i % us % (us / 256) % count); previous = now; } generate_activity (node_a, accounts); diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index 1f67a2bcc0..0f98bc0134 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -220,7 +220,7 @@ class block_store_partial : public block_store if (value.size () != 0) { debug_assert (value.size () >= result.bytes.size ()); - auto type = static_cast ((reinterpret_cast (value.data ()))[0]); + auto type = block_type_from_raw (value.data ()); nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); auto error (nano::try_read (stream, result.bytes)); (void)error; @@ -237,8 +237,8 @@ class block_store_partial : public block_store { auto value (block_raw_get (transaction_a, hash_a)); debug_assert (value.size () != 0); + auto type = block_type_from_raw (value.data ()); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - auto type = static_cast (data[0]); std::fill_n (data.begin () + block_successor_offset (transaction_a, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); block_raw_put (transaction_a, data, hash_a); } @@ -604,8 +604,7 @@ class block_store_partial : public block_store existing = make_iterator> (transaction_a, tables::blocks); } debug_assert (existing != end); - return block_get (transaction_a, nano::block_hash (existing->first)); - // TODO: Why can't this be used? return existing->second.block; + return existing->second; } uint64_t confirmation_height_count (nano::transaction const & transaction_a) override @@ -738,6 +737,12 @@ class block_store_partial : public block_store return entry_size_a - nano::block_sideband::size (type_a); } + static nano::block_type block_type_from_raw (void * data_a) + { + // The block type is the first byte + return static_cast ((reinterpret_cast (data_a))[0]); + } + size_t count (nano::transaction const & transaction_a, std::initializer_list dbs_a) const { size_t total_count = 0; @@ -788,8 +793,8 @@ class block_predecessor_set : public nano::block_visitor auto hash (block_a.hash ()); auto value (store.block_raw_get (transaction, block_a.previous ())); debug_assert (value.size () != 0); + auto type = store.block_type_from_raw (value.data ()); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - auto type = static_cast (data[0]); std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + store.block_successor_offset (transaction, value.size (), type)); store.block_raw_put (transaction, data, block_a.previous ()); } From edb246c5b0814947f34fb035037e0cd435e68745 Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Tue, 30 Jun 2020 13:11:42 +0100 Subject: [PATCH 3/6] Open new blocks database in RocksDB --- nano/node/rocksdb/rocksdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index 8e955a8857..288778584a 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -74,7 +74,7 @@ nano::rocksdb_store::~rocksdb_store () void nano::rocksdb_store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a) { - std::initializer_list names{ rocksdb::kDefaultColumnFamilyName.c_str (), "frontiers", "accounts", "send", "receive", "open", "change", "state_blocks", "pending", "representation", "unchecked", "vote", "online_weight", "meta", "peers", "cached_counts", "confirmation_height" }; + std::initializer_list names{ rocksdb::kDefaultColumnFamilyName.c_str (), "frontiers", "accounts", "blocks", "pending", "representation", "unchecked", "vote", "online_weight", "meta", "peers", "cached_counts", "confirmation_height" }; std::vector column_families; for (const auto & cf_name : names) { From 0b18c83a6ad17d69d72d936d28db4a54fdcc3ad0 Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Mon, 13 Jul 2020 22:22:48 +0100 Subject: [PATCH 4/6] Removed comment --- nano/node/lmdb/lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 9d2b9e411e..bf677651e3 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -258,7 +258,7 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool needs_vacuuming = true; case 18: upgrade_v18_to_v19 (transaction_a); - needs_vacuuming = true; // Make this optional via config + needs_vacuuming = true; case 19: break; default: From 35e38cd03d6666971a707b9050e3eab364e6e26b Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Tue, 14 Jul 2020 10:25:45 +0100 Subject: [PATCH 5/6] Use new blocks database with --rebuild --- nano/node/lmdb/lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index bf677651e3..a49d878e8a 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -826,7 +826,7 @@ bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file) void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a) { // Tables with uint256_union key - std::vector tables = { accounts, send_blocks, receive_blocks, open_blocks, change_blocks, state_blocks, vote, confirmation_height }; + std::vector tables = { accounts, blocks, vote, confirmation_height }; for (auto const & table : tables) { MDB_dbi temp; From 89aea938358f8a744ada15ae782412d4916e0a43 Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Fri, 24 Jul 2020 18:21:21 +0100 Subject: [PATCH 6/6] Remove block_counts (Serg review) --- nano/secure/common.cpp | 5 ----- nano/secure/common.hpp | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index 69ec26d124..6f554b08b7 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -252,11 +252,6 @@ nano::epoch nano::account_info::epoch () const return epoch_m; } -size_t nano::block_counts::sum () const -{ - return send + receive + open + change + state; -} - nano::pending_info::pending_info (nano::account const & source_a, nano::amount const & amount_a, nano::epoch epoch_a) : source (source_a), amount (amount_a), diff --git a/nano/secure/common.hpp b/nano/secure/common.hpp index 96bfae445b..bbfe9db039 100644 --- a/nano/secure/common.hpp +++ b/nano/secure/common.hpp @@ -208,16 +208,6 @@ class block_info final nano::account account{ 0 }; nano::amount balance{ 0 }; }; -class block_counts final -{ -public: - size_t sum () const; - size_t send{ 0 }; - size_t receive{ 0 }; - size_t open{ 0 }; - size_t change{ 0 }; - size_t state{ 0 }; -}; class confirmation_height_info final {