diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index 27838e67f6..933a7e15e3 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -42,6 +42,7 @@ add_executable( toml.cpp timer.cpp uint256_union.cpp + unchecked_map.cpp utility.cpp vote_processor.cpp voting.cpp diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 1820f0fe09..ee6811b92d 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -22,6 +22,8 @@ #include +using namespace std::chrono_literals; + namespace { void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height, nano::block_hash const & rep_block); @@ -347,109 +349,152 @@ TEST (block_store, genesis) ASSERT_EQ (nano::dev::genesis->account (), nano::dev::genesis_key.pub); } -TEST (bootstrap, simple) +// This test checks for basic operations in the unchecked table such as putting a new block, retrieving it, and +// deleting it from the database +TEST (unchecked, simple) { - nano::logger_mt logger; + nano::system system{}; + nano::logger_mt logger{}; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); + nano::unchecked_map unchecked{ *store, false }; ASSERT_TRUE (!store->init_error ()); - std::shared_ptr block1 = std::make_shared (0, 1, 2, nano::keypair ().prv, 4, 5); - auto transaction (store->tx_begin_write ()); - auto block2 (store->unchecked.get (transaction, block1->previous ())); - ASSERT_TRUE (block2.empty ()); - store->unchecked.put (transaction, block1->previous (), block1); - auto block3 (store->unchecked.get (transaction, block1->previous ())); - ASSERT_FALSE (block3.empty ()); - ASSERT_EQ (*block1, *(block3[0].block)); - store->unchecked.del (transaction, nano::unchecked_key (block1->previous (), block1->hash ())); - auto block4 (store->unchecked.get (transaction, block1->previous ())); - ASSERT_TRUE (block4.empty ()); -} - + std::shared_ptr block = std::make_shared (0, 1, 2, nano::keypair ().prv, 4, 5); + // Asserts the block wasn't added yet to the unchecked table + auto block_listing1 = unchecked.get (store->tx_begin_read (), block->previous ()); + ASSERT_TRUE (block_listing1.empty ()); + // Enqueues a block to be saved on the unchecked table + unchecked.put (block->previous (), block); + // Waits for the block to get written in the database + auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + return unchecked.get (transaction_a, block_hash_a).size () > 0; + }; + ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->previous ())); + auto transaction = store->tx_begin_write (); + // Retrieves the block from the database + auto block_listing2 = unchecked.get (transaction, block->previous ()); + ASSERT_FALSE (block_listing2.empty ()); + // Asserts the added block is equal to the retrieved one + ASSERT_EQ (*block, *(block_listing2[0].block)); + // Deletes the block from the database + unchecked.del (transaction, nano::unchecked_key (block->previous (), block->hash ())); + // Asserts the block is deleted + auto block_listing3 = unchecked.get (transaction, block->previous ()); + ASSERT_TRUE (block_listing3.empty ()); +} + +// This test ensures the unchecked table is able to receive more than one block TEST (unchecked, multiple) { + nano::system system{}; if (nano::rocksdb_config::using_rocksdb_in_tests ()) { // Don't test this in rocksdb mode return; } - nano::logger_mt logger; + nano::logger_mt logger{}; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); + nano::unchecked_map unchecked{ *store, false }; ASSERT_TRUE (!store->init_error ()); - std::shared_ptr block1 = std::make_shared (4, 1, 2, nano::keypair ().prv, 4, 5); - auto transaction (store->tx_begin_write ()); - auto block2 (store->unchecked.get (transaction, block1->previous ())); - ASSERT_TRUE (block2.empty ()); - store->unchecked.put (transaction, block1->previous (), block1); - store->unchecked.put (transaction, block1->source (), block1); - auto block3 (store->unchecked.get (transaction, block1->previous ())); - ASSERT_FALSE (block3.empty ()); - auto block4 (store->unchecked.get (transaction, block1->source ())); - ASSERT_FALSE (block4.empty ()); + std::shared_ptr block = std::make_shared (4, 1, 2, nano::keypair ().prv, 4, 5); + // Asserts the block wasn't added yet to the unchecked table + auto block_listing1 = unchecked.get (store->tx_begin_read (), block->previous ()); + ASSERT_TRUE (block_listing1.empty ()); + // Enqueues the first block + unchecked.put (block->previous (), block); + // Enqueues a second block + unchecked.put (block->source (), block); + auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + return unchecked.get (transaction_a, block_hash_a).size () > 0; + }; + // Waits for and asserts the first block gets saved in the database + ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->previous ())); + // Waits for and asserts the second block gets saved in the database + ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->source ())); } +// This test ensures that a block can't occur twice in the unchecked table. TEST (unchecked, double_put) { - nano::logger_mt logger; + nano::system system{}; + nano::logger_mt logger{}; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); + nano::unchecked_map unchecked{ *store, false }; ASSERT_TRUE (!store->init_error ()); - std::shared_ptr block1 = std::make_shared (4, 1, 2, nano::keypair ().prv, 4, 5); - auto transaction (store->tx_begin_write ()); - auto block2 (store->unchecked.get (transaction, block1->previous ())); - ASSERT_TRUE (block2.empty ()); - store->unchecked.put (transaction, block1->previous (), block1); - store->unchecked.put (transaction, block1->previous (), block1); - auto block3 (store->unchecked.get (transaction, block1->previous ())); - ASSERT_EQ (block3.size (), 1); + std::shared_ptr block = std::make_shared (4, 1, 2, nano::keypair ().prv, 4, 5); + // Asserts the block wasn't added yet to the unchecked table + auto block_listing1 = unchecked.get (store->tx_begin_read (), block->previous ()); + ASSERT_TRUE (block_listing1.empty ()); + // Enqueues the block to be saved in the unchecked table + unchecked.put (block->previous (), block); + // Enqueues the block again in an attempt to have it there twice + unchecked.put (block->previous (), block); + auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + return unchecked.get (transaction_a, block_hash_a).size () > 0; + }; + // Waits for and asserts the block was added at least once + ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->previous ())); + // Asserts the block was added at most once -- this is objective of this test. + auto block_listing2 = unchecked.get (store->tx_begin_read (), block->previous ()); + ASSERT_EQ (block_listing2.size (), 1); } +// Tests that recurrent get calls return the correct values TEST (unchecked, multiple_get) { - nano::logger_mt logger; + nano::system system{}; + nano::logger_mt logger{}; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); + nano::unchecked_map unchecked{ *store, false }; ASSERT_TRUE (!store->init_error ()); + // Instantiates three blocks std::shared_ptr block1 = std::make_shared (4, 1, 2, nano::keypair ().prv, 4, 5); std::shared_ptr block2 = std::make_shared (3, 1, 2, nano::keypair ().prv, 4, 5); std::shared_ptr block3 = std::make_shared (5, 1, 2, nano::keypair ().prv, 4, 5); - { - auto transaction (store->tx_begin_write ()); - store->unchecked.put (transaction, block1->previous (), block1); // unchecked1 - store->unchecked.put (transaction, block1->hash (), block1); // unchecked2 - store->unchecked.put (transaction, block2->previous (), block2); // unchecked3 - store->unchecked.put (transaction, block1->previous (), block2); // unchecked1 - store->unchecked.put (transaction, block1->hash (), block2); // unchecked2 - store->unchecked.put (transaction, block3->previous (), block3); - store->unchecked.put (transaction, block3->hash (), block3); // unchecked4 - store->unchecked.put (transaction, block1->previous (), block3); // unchecked1 - } - auto transaction (store->tx_begin_read ()); - auto unchecked_count (store->unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 8); + // Add the blocks' info to the unchecked table + unchecked.put (block1->previous (), block1); // unchecked1 + unchecked.put (block1->hash (), block1); // unchecked2 + unchecked.put (block2->previous (), block2); // unchecked3 + unchecked.put (block1->previous (), block2); // unchecked1 + unchecked.put (block1->hash (), block2); // unchecked2 + unchecked.put (block3->previous (), block3); + unchecked.put (block3->hash (), block3); // unchecked4 + unchecked.put (block1->previous (), block3); // unchecked1 + // Waits for the blocks to get saved in the database + ASSERT_TIMELY (5s, 8 == unchecked.count (store->tx_begin_read ())); std::vector unchecked1; - auto unchecked1_blocks (store->unchecked.get (transaction, block1->previous ())); + // Asserts the entries will be found for the provided key + auto transaction = store->tx_begin_read (); + auto unchecked1_blocks = unchecked.get (transaction, block1->previous ()); ASSERT_EQ (unchecked1_blocks.size (), 3); for (auto & i : unchecked1_blocks) { unchecked1.push_back (i.block->hash ()); } + // Asserts the payloads where correclty saved ASSERT_TRUE (std::find (unchecked1.begin (), unchecked1.end (), block1->hash ()) != unchecked1.end ()); ASSERT_TRUE (std::find (unchecked1.begin (), unchecked1.end (), block2->hash ()) != unchecked1.end ()); ASSERT_TRUE (std::find (unchecked1.begin (), unchecked1.end (), block3->hash ()) != unchecked1.end ()); std::vector unchecked2; - auto unchecked2_blocks (store->unchecked.get (transaction, block1->hash ())); + // Asserts the entries will be found for the provided key + auto unchecked2_blocks = unchecked.get (transaction, block1->hash ()); ASSERT_EQ (unchecked2_blocks.size (), 2); for (auto & i : unchecked2_blocks) { unchecked2.push_back (i.block->hash ()); } + // Asserts the payloads where correctly saved ASSERT_TRUE (std::find (unchecked2.begin (), unchecked2.end (), block1->hash ()) != unchecked2.end ()); ASSERT_TRUE (std::find (unchecked2.begin (), unchecked2.end (), block2->hash ()) != unchecked2.end ()); - auto unchecked3 (store->unchecked.get (transaction, block2->previous ())); + // Asserts the entry is found by the key and the payload is saved + auto unchecked3 = unchecked.get (transaction, block2->previous ()); ASSERT_EQ (unchecked3.size (), 1); ASSERT_EQ (unchecked3[0].block->hash (), block2->hash ()); - auto unchecked4 (store->unchecked.get (transaction, block3->hash ())); + // Asserts the entry is found by the key and the payload is saved + auto unchecked4 = unchecked.get (transaction, block3->hash ()); ASSERT_EQ (unchecked4.size (), 1); ASSERT_EQ (unchecked4[0].block->hash (), block3->hash ()); - auto unchecked5 (store->unchecked.get (transaction, block2->hash ())); + // Asserts no entry is found for a block that wasn't added + auto unchecked5 = unchecked.get (transaction, block2->hash ()); ASSERT_EQ (unchecked5.size (), 0); } @@ -480,30 +525,10 @@ TEST (block_store, empty_bootstrap) { nano::logger_mt logger; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); + nano::unchecked_map unchecked{ *store, false }; ASSERT_TRUE (!store->init_error ()); auto transaction (store->tx_begin_read ()); - auto [begin, end] = store->unchecked.full_range (transaction); - ASSERT_EQ (end, begin); -} - -TEST (block_store, one_bootstrap) -{ - nano::logger_mt logger; - auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); - ASSERT_TRUE (!store->init_error ()); - std::shared_ptr block1 = std::make_shared (0, 1, 2, nano::keypair ().prv, 4, 5); - auto transaction (store->tx_begin_write ()); - store->unchecked.put (transaction, block1->hash (), block1); - auto begin (store->unchecked.begin (transaction)); - auto end (store->unchecked.end ()); - ASSERT_NE (end, begin); - auto hash1 (begin->first.key ()); - ASSERT_EQ (block1->hash (), hash1); - auto blocks (store->unchecked.get (transaction, hash1)); - ASSERT_EQ (1, blocks.size ()); - auto block2 (blocks[0].block); - ASSERT_EQ (*block1, *block2); - ++begin; + auto [begin, end] = unchecked.full_range (transaction); ASSERT_EQ (end, begin); } @@ -919,42 +944,26 @@ TEST (block_store, pruned_random) TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort table { auto path (nano::unique_path ()); - nano::logger_mt logger; - nano::mdb_store store (logger, path, nano::dev::constants); + nano::logger_mt logger{}; + nano::mdb_store store{ logger, path, nano::dev::constants }; + nano::unchecked_map unchecked{ store, false }; auto transaction (store.tx_begin_write ()); ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 1)); ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE, &store.unchecked_handle)); std::shared_ptr send1 = std::make_shared (0, 0, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); std::shared_ptr send2 = std::make_shared (1, 0, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); ASSERT_NE (send1->hash (), send2->hash ()); - store.unchecked.put (transaction, send1->hash (), send1); - store.unchecked.put (transaction, send1->hash (), send2); - { - auto iterator1 (store.unchecked.begin (transaction)); - ++iterator1; - ASSERT_EQ (store.unchecked.end (), iterator1); - } + unchecked.put (send1->hash (), send1); + unchecked.put (send1->hash (), send2); ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 0)); mdb_dbi_close (store.env, store.unchecked_handle); ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked_handle)); - store.unchecked.put (transaction, send1->hash (), send1); - store.unchecked.put (transaction, send1->hash (), send2); - { - auto iterator1 (store.unchecked.begin (transaction)); - ++iterator1; - ASSERT_EQ (store.unchecked.end (), iterator1); - } + unchecked.put (send1->hash (), send1); + unchecked.put (send1->hash (), send2); ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 1)); ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked_handle)); - store.unchecked.put (transaction, send1->hash (), send1); - store.unchecked.put (transaction, send1->hash (), send2); - { - auto iterator1 (store.unchecked.begin (transaction)); - ++iterator1; - ASSERT_NE (store.unchecked.end (), iterator1); - ++iterator1; - ASSERT_EQ (store.unchecked.end (), iterator1); - } + unchecked.put (send1->hash (), send1); + unchecked.put (send1->hash (), send2); } TEST (block_store, state_block) @@ -2017,18 +2026,28 @@ TEST (block_store, rocksdb_force_test_env_variable) namespace nano { +// This thest ensures the tombstone_count is increased when there is a delete. The tombstone_count is part of a flush +// logic bound to the way RocksDB is used by the node. TEST (rocksdb_block_store, tombstone_count) { if (nano::rocksdb_config::using_rocksdb_in_tests ()) { - nano::logger_mt logger; + nano::system system{}; + nano::logger_mt logger{}; auto store = std::make_unique (logger, nano::unique_path (), nano::dev::constants); + nano::unchecked_map unchecked{ *store, false }; ASSERT_TRUE (!store->init_error ()); - auto transaction = store->tx_begin_write (); - std::shared_ptr block1 = std::make_shared (0, 1, 2, nano::keypair ().prv, 4, 5); - store->unchecked.put (transaction, block1->previous (), block1); + std::shared_ptr block = std::make_shared (0, 1, 2, nano::keypair ().prv, 4, 5); + // Enqueues a block to be saved in the database + unchecked.put (block->previous (), block); + auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + return unchecked.get (transaction_a, block_hash_a).size () > 0; + }; + // Waits for the block to get saved + ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->previous ())); ASSERT_EQ (store->tombstone_map.at (nano::tables::unchecked).num_since_last_flush.load (), 0); - store->unchecked.del (transaction, nano::unchecked_key (block1->previous (), block1->hash ())); + // Perorms a delete and checks for the tombstone counter + unchecked.del (store->tx_begin_write (), nano::unchecked_key (block->previous (), block->hash ())); ASSERT_EQ (store->tombstone_map.at (nano::tables::unchecked).num_since_last_flush.load (), 1); } } diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index 30bf889ae1..7a752d0de8 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -924,11 +924,10 @@ TEST (bootstrap_processor, DISABLED_lazy_unclear_state_link) node2->bootstrap_initiator.bootstrap_lazy (receive->hash ()); // Check processed blocks ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ()); - node2->block_processor.flush (); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ())); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ())); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ())); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (receive->hash ())); + ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send1->hash ())); + ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send2->hash ())); + ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (open->hash ())); + ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (receive->hash ())); ASSERT_EQ (0, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in)); } @@ -976,10 +975,9 @@ TEST (bootstrap_processor, lazy_unclear_state_link_not_existing) node2->bootstrap_initiator.bootstrap_lazy (send2->hash ()); // Check processed blocks ASSERT_TIMELY (15s, !node2->bootstrap_initiator.in_progress ()); - node2->block_processor.flush (); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ())); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ())); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ())); + ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send1->hash ())); + ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (open->hash ())); + ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send2->hash ())); ASSERT_EQ (1, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in)); } @@ -1038,7 +1036,6 @@ TEST (bootstrap_processor, DISABLED_lazy_destinations) node2->bootstrap_initiator.bootstrap_lazy (send2->hash ()); // Check processed blocks ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ()); - node2->block_processor.flush (); ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ())); ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ())); ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ())); @@ -1127,7 +1124,7 @@ TEST (bootstrap_processor, lazy_pruning_missing_block) ASSERT_FALSE (node2->ledger.block_or_pruned_exists (state_open->hash ())); { auto transaction (node2->store.tx_begin_read ()); - ASSERT_TRUE (node2->store.unchecked.exists (transaction, nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ()))); + ASSERT_TRUE (node2->unchecked.exists (transaction, nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ()))); } // Insert missing block node2->process_active (send1); @@ -1838,7 +1835,7 @@ TEST (bulk, DISABLED_genesis_pruning) ASSERT_EQ (1, node2->ledger.cache.block_count); { auto transaction (node2->store.tx_begin_write ()); - node2->store.unchecked.clear (transaction); + node2->unchecked.clear (transaction); } // Insert pruned blocks node2->process_active (send1); diff --git a/nano/core_test/confirmation_height.cpp b/nano/core_test/confirmation_height.cpp index 10d9681104..e3df14af73 100644 --- a/nano/core_test/confirmation_height.cpp +++ b/nano/core_test/confirmation_height.cpp @@ -219,24 +219,24 @@ TEST (confirmation_height, multiple_accounts) TEST (confirmation_height, gap_bootstrap) { auto test_mode = [] (nano::confirmation_height_mode mode_a) { - nano::system system; - nano::node_flags node_flags; + nano::system system{}; + nano::node_flags node_flags{}; node_flags.confirmation_height_processor_mode = mode_a; auto & node1 = *system.add_node (node_flags); - nano::keypair destination; - auto send1 (std::make_shared (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); + nano::keypair destination{}; + auto send1 = std::make_shared (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); node1.work_generate_blocking (*send1); - auto send2 (std::make_shared (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); + auto send2 = std::make_shared (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); node1.work_generate_blocking (*send2); - auto send3 (std::make_shared (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); + auto send3 = std::make_shared (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); node1.work_generate_blocking (*send3); - auto open1 (std::make_shared (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0)); + auto open1 = std::make_shared (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0); node1.work_generate_blocking (*open1); // Receive - auto receive1 (std::make_shared (open1->hash (), send2->hash (), destination.prv, destination.pub, 0)); + auto receive1 = std::make_shared (open1->hash (), send2->hash (), destination.prv, destination.pub, 0); node1.work_generate_blocking (*receive1); - auto receive2 (std::make_shared (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0)); + auto receive2 = std::make_shared (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0); node1.work_generate_blocking (*receive2); node1.block_processor.add (send1); @@ -249,12 +249,16 @@ TEST (confirmation_height, gap_bootstrap) // Receive 2 comes in on the live network, however the chain has not been finished so it gets added to unchecked node1.process_active (receive2); - node1.block_processor.flush (); + // Waits for the unchecked_map to process the 4 blocks added to the block_processor, saving them in the unchecked table + auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + return !node1.unchecked.get (transaction_a, block_hash_a).empty (); + }; + ASSERT_TIMELY (15s, check_block_is_listed (node1.store.tx_begin_read (), receive2->previous ())); // Confirmation heights should not be updated { auto transaction (node1.store.tx_begin_read ()); - auto unchecked_count (node1.store.unchecked.count (transaction)); + auto unchecked_count (node1.unchecked.count (transaction)); ASSERT_EQ (unchecked_count, 2); nano::confirmation_height_info confirmation_height_info; @@ -265,14 +269,11 @@ TEST (confirmation_height, gap_bootstrap) // Now complete the chain where the block comes in on the bootstrap network. node1.block_processor.add (open1); - node1.block_processor.flush (); + ASSERT_TIMELY (10s, node1.unchecked.count (node1.store.tx_begin_read ()) == 0); // Confirmation height should be unchanged and unchecked should now be 0 { - auto transaction (node1.store.tx_begin_read ()); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 0); - + auto transaction = node1.store.tx_begin_read (); nano::confirmation_height_info confirmation_height_info; ASSERT_FALSE (node1.store.confirmation_height.get (transaction, nano::dev::genesis_key.pub, confirmation_height_info)); ASSERT_EQ (1, confirmation_height_info.height); @@ -296,10 +297,10 @@ TEST (confirmation_height, gap_bootstrap) TEST (confirmation_height, gap_live) { auto test_mode = [] (nano::confirmation_height_mode mode_a) { - nano::system system; - nano::node_flags node_flags; + nano::system system{}; + nano::node_flags node_flags{}; node_flags.confirmation_height_processor_mode = mode_a; - nano::node_config node_config (nano::get_available_port (), system.logging); + nano::node_config node_config{ nano::get_available_port (), system.logging }; node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; auto node = system.add_node (node_config, node_flags); node_config.peering_port = nano::get_available_port (); @@ -309,18 +310,18 @@ TEST (confirmation_height, gap_live) system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); system.wallet (1)->insert_adhoc (destination.prv); - auto send1 (std::make_shared (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 1, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); + auto send1 = std::make_shared (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 1, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); node->work_generate_blocking (*send1); - auto send2 (std::make_shared (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); + auto send2 = std::make_shared (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); node->work_generate_blocking (*send2); - auto send3 (std::make_shared (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); + auto send3 = std::make_shared (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); node->work_generate_blocking (*send3); - auto open1 (std::make_shared (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0)); + auto open1 = std::make_shared (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0); node->work_generate_blocking (*open1); - auto receive1 (std::make_shared (open1->hash (), send2->hash (), destination.prv, destination.pub, 0)); + auto receive1 = std::make_shared (open1->hash (), send2->hash (), destination.prv, destination.pub, 0); node->work_generate_blocking (*receive1); - auto receive2 (std::make_shared (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0)); + auto receive2 = std::make_shared (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0); node->work_generate_blocking (*receive2); node->block_processor.add (send1); @@ -355,11 +356,11 @@ TEST (confirmation_height, gap_live) ASSERT_TIMELY (10s, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out) == 6); // This should confirm the open block and the source of the receive blocks - auto transaction (node->store.tx_begin_read ()); - auto unchecked_count (node->store.unchecked.count (transaction)); + auto transaction = node->store.tx_begin_read (); + auto unchecked_count = node->unchecked.count (transaction); ASSERT_EQ (unchecked_count, 0); - nano::confirmation_height_info confirmation_height_info; + nano::confirmation_height_info confirmation_height_info{}; ASSERT_TRUE (node->ledger.block_confirmed (transaction, receive2->hash ())); ASSERT_FALSE (node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub, confirmation_height_info)); ASSERT_EQ (4, confirmation_height_info.height); diff --git a/nano/core_test/gap_cache.cpp b/nano/core_test/gap_cache.cpp index 3bd70273e8..bbbd1ceec5 100644 --- a/nano/core_test/gap_cache.cpp +++ b/nano/core_test/gap_cache.cpp @@ -103,7 +103,7 @@ TEST (gap_cache, two_dependencies) ASSERT_EQ (2, node1.gap_cache.size ()); node1.block_processor.add (send1, nano::seconds_since_epoch ()); node1.block_processor.flush (); - ASSERT_EQ (0, node1.gap_cache.size ()); + ASSERT_TIMELY (5s, node1.gap_cache.size () == 0); auto transaction (node1.store.tx_begin_read ()); ASSERT_TRUE (node1.store.block.exists (transaction, send1->hash ())); ASSERT_TRUE (node1.store.block.exists (transaction, send2->hash ())); diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index e0bb7dab1d..4134ce77ea 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -2454,11 +2454,11 @@ TEST (ledger, successor_epoch) TEST (ledger, epoch_open_pending) { - nano::block_builder builder; - nano::system system (1); - auto & node1 (*system.nodes[0]); + nano::block_builder builder{}; + nano::system system{ 1 }; + auto & node1 = *system.nodes[0]; nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; - nano::keypair key1; + nano::keypair key1{}; auto epoch_open = builder.state () .account (key1.pub) .previous (0) @@ -2468,14 +2468,15 @@ TEST (ledger, epoch_open_pending) .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) .work (*pool.generate (key1.pub)) .build_shared (); - auto process_result (node1.ledger.process (node1.store.tx_begin_write (), *epoch_open)); + auto process_result = node1.ledger.process (node1.store.tx_begin_write (), *epoch_open); ASSERT_EQ (nano::process_result::gap_epoch_open_pending, process_result.code); ASSERT_EQ (nano::signature_verification::valid_epoch, process_result.verified); node1.block_processor.add (epoch_open); - node1.block_processor.flush (); + // Waits for the block to get saved in the database + ASSERT_TIMELY (10s, 1 == node1.unchecked.count (node1.store.tx_begin_read ())); ASSERT_FALSE (node1.ledger.block_or_pruned_exists (epoch_open->hash ())); // Open block should be inserted into unchecked - auto blocks (node1.store.unchecked.get (node1.store.tx_begin_read (), nano::hash_or_account (epoch_open->account ()).hash)); + auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), nano::hash_or_account (epoch_open->account ()).hash); ASSERT_EQ (blocks.size (), 1); ASSERT_EQ (blocks[0].block->full_hash (), epoch_open->full_hash ()); ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid_epoch); @@ -2490,8 +2491,7 @@ TEST (ledger, epoch_open_pending) .work (*pool.generate (nano::dev::genesis->hash ())) .build_shared (); node1.block_processor.add (send1); - node1.block_processor.flush (); - ASSERT_TRUE (node1.ledger.block_or_pruned_exists (epoch_open->hash ())); + ASSERT_TIMELY (10s, node1.ledger.block_or_pruned_exists (epoch_open->hash ())); } TEST (ledger, block_hash_account_conflict) @@ -2657,27 +2657,21 @@ TEST (ledger, unchecked_epoch) auto epoch1 (std::make_shared (destination.pub, open1->hash (), destination.pub, nano::Gxrb_ratio, node1.ledger.epoch_link (nano::epoch::epoch_1), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); node1.work_generate_blocking (*epoch1); node1.block_processor.add (epoch1); - node1.block_processor.flush (); { - auto transaction (node1.store.tx_begin_read ()); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 1); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - auto blocks (node1.store.unchecked.get (transaction, epoch1->previous ())); + // Waits for the epoch1 block to pass through block_processor and unchecked.put queues + ASSERT_TIMELY (10s, 1 == node1.unchecked.count (node1.store.tx_begin_read ())); + auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), epoch1->previous ()); ASSERT_EQ (blocks.size (), 1); ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid_epoch); } node1.block_processor.add (send1); node1.block_processor.add (open1); - node1.block_processor.flush (); + ASSERT_TIMELY (5s, node1.store.block.exists (node1.store.tx_begin_read (), epoch1->hash ())); { - auto transaction (node1.store.tx_begin_read ()); - ASSERT_TRUE (node1.store.block.exists (transaction, epoch1->hash ())); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 0); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - nano::account_info info; - ASSERT_FALSE (node1.store.account.get (transaction, destination.pub, info)); + // Waits for the last blocks to pass through block_processor and unchecked.put queues + ASSERT_TIMELY (10s, 0 == node1.unchecked.count (node1.store.tx_begin_read ())); + nano::account_info info{}; + ASSERT_FALSE (node1.store.account.get (node1.store.tx_begin_read (), destination.pub, info)); ASSERT_EQ (info.epoch (), nano::epoch::epoch_1); } } @@ -2701,32 +2695,29 @@ TEST (ledger, unchecked_epoch_invalid) node1.work_generate_blocking (*epoch2); node1.block_processor.add (epoch1); node1.block_processor.add (epoch2); - node1.block_processor.flush (); { - auto transaction (node1.store.tx_begin_read ()); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 2); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - auto blocks (node1.store.unchecked.get (transaction, epoch1->previous ())); + // Waits for the last blocks to pass through block_processor and unchecked.put queues + ASSERT_TIMELY (10s, 2 == node1.unchecked.count (node1.store.tx_begin_read ())); + auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), epoch1->previous ()); ASSERT_EQ (blocks.size (), 2); ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid); ASSERT_EQ (blocks[1].verified, nano::signature_verification::valid); } node1.block_processor.add (send1); node1.block_processor.add (open1); - node1.block_processor.flush (); + // Waits for the last blocks to pass through block_processor and unchecked.put queues + ASSERT_TIMELY (10s, node1.store.block.exists (node1.store.tx_begin_read (), epoch2->hash ())); { - auto transaction (node1.store.tx_begin_read ()); + auto transaction = node1.store.tx_begin_read (); ASSERT_FALSE (node1.store.block.exists (transaction, epoch1->hash ())); - ASSERT_TRUE (node1.store.block.exists (transaction, epoch2->hash ())); ASSERT_TRUE (node1.active.empty ()); - auto unchecked_count (node1.store.unchecked.count (transaction)); + auto unchecked_count = node1.unchecked.count (transaction); ASSERT_EQ (unchecked_count, 0); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - nano::account_info info; + ASSERT_EQ (unchecked_count, node1.unchecked.count (transaction)); + nano::account_info info{}; ASSERT_FALSE (node1.store.account.get (transaction, destination.pub, info)); ASSERT_NE (info.epoch (), nano::epoch::epoch_1); - auto epoch2_store (node1.store.block.get (transaction, epoch2->hash ())); + auto epoch2_store = node1.store.block.get (transaction, epoch2->hash ()); ASSERT_NE (nullptr, epoch2_store); ASSERT_EQ (nano::epoch::epoch_0, epoch2_store->sideband ().details.epoch); ASSERT_TRUE (epoch2_store->sideband ().details.is_send); @@ -2750,74 +2741,58 @@ TEST (ledger, unchecked_open) open2->signature.bytes[0] ^= 1; node1.block_processor.add (open1); node1.block_processor.add (open2); - node1.block_processor.flush (); { - auto transaction (node1.store.tx_begin_read ()); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 1); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - auto blocks (node1.store.unchecked.get (transaction, open1->source ())); + // Waits for the last blocks to pass through block_processor and unchecked.put queues + ASSERT_TIMELY (10s, 1 == node1.unchecked.count (node1.store.tx_begin_read ())); + auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), open1->source ()); ASSERT_EQ (blocks.size (), 1); ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid); } node1.block_processor.add (send1); - node1.block_processor.flush (); - { - auto transaction (node1.store.tx_begin_read ()); - ASSERT_TRUE (node1.store.block.exists (transaction, open1->hash ())); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 0); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - } + // Waits for the send1 block to pass through block_processor and unchecked.put queues + ASSERT_TIMELY (10s, node1.store.block.exists (node1.store.tx_begin_read (), open1->hash ())); + ASSERT_EQ (0, node1.unchecked.count (node1.store.tx_begin_read ())); } TEST (ledger, unchecked_receive) { - nano::system system (1); - auto & node1 (*system.nodes[0]); - nano::keypair destination; - auto send1 (std::make_shared (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); + nano::system system{ 1 }; + auto & node1 = *system.nodes[0]; + nano::keypair destination{}; + auto send1 = std::make_shared (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); node1.work_generate_blocking (*send1); - auto send2 (std::make_shared (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0)); + auto send2 = std::make_shared (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); node1.work_generate_blocking (*send2); - auto open1 (std::make_shared (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0)); + auto open1 = std::make_shared (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0); node1.work_generate_blocking (*open1); - auto receive1 (std::make_shared (open1->hash (), send2->hash (), destination.prv, destination.pub, 0)); + auto receive1 = std::make_shared (open1->hash (), send2->hash (), destination.prv, destination.pub, 0); node1.work_generate_blocking (*receive1); node1.block_processor.add (send1); node1.block_processor.add (receive1); - node1.block_processor.flush (); + auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + return !node1.unchecked.get (transaction_a, block_hash_a).empty (); + }; // Previous block for receive1 is unknown, signature cannot be validated { - auto transaction (node1.store.tx_begin_read ()); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 1); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - auto blocks (node1.store.unchecked.get (transaction, receive1->previous ())); + // Waits for the last blocks to pass through block_processor and unchecked.put queues + ASSERT_TIMELY (15s, check_block_is_listed (node1.store.tx_begin_read (), receive1->previous ())); + auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), receive1->previous ()); ASSERT_EQ (blocks.size (), 1); ASSERT_EQ (blocks[0].verified, nano::signature_verification::unknown); } + // Waits for the open1 block to pass through block_processor and unchecked.put queues node1.block_processor.add (open1); - node1.block_processor.flush (); + ASSERT_TIMELY (15s, check_block_is_listed (node1.store.tx_begin_read (), receive1->source ())); // Previous block for receive1 is known, signature was validated { - auto transaction (node1.store.tx_begin_read ()); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 1); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - auto blocks (node1.store.unchecked.get (transaction, receive1->source ())); + auto transaction = node1.store.tx_begin_read (); + auto blocks (node1.unchecked.get (transaction, receive1->source ())); ASSERT_EQ (blocks.size (), 1); ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid); } node1.block_processor.add (send2); - node1.block_processor.flush (); - { - auto transaction (node1.store.tx_begin_read ()); - ASSERT_TRUE (node1.store.block.exists (transaction, receive1->hash ())); - auto unchecked_count (node1.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 0); - ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction)); - } + ASSERT_TIMELY (10s, node1.store.block.exists (node1.store.tx_begin_read (), receive1->hash ())); + ASSERT_EQ (0, node1.unchecked.count (node1.store.tx_begin_read ())); } TEST (ledger, confirmation_height_not_updated) @@ -3621,13 +3596,14 @@ TEST (ledger, hash_root_random) TEST (ledger, migrate_lmdb_to_rocksdb) { - auto path (nano::unique_path ()); - nano::logger_mt logger; + auto path = nano::unique_path (); + nano::logger_mt logger{}; boost::asio::ip::address_v6 address (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1")); uint16_t port = 100; - nano::mdb_store store (logger, path / "data.ldb", nano::dev::constants); - nano::stat stats; - nano::ledger ledger (store, stats, nano::dev::constants); + nano::mdb_store store{ logger, path / "data.ldb", nano::dev::constants }; + nano::unchecked_map unchecked{ store, false }; + nano::stat stats{}; + nano::ledger ledger{ store, stats, nano::dev::constants }; nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; std::shared_ptr send = nano::state_block_builder () @@ -3644,7 +3620,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb) auto version = 99; { - auto transaction (store.tx_begin_write ()); + auto transaction = store.tx_begin_write (); store.initialize (transaction, ledger.cache); ASSERT_FALSE (store.init_error ()); @@ -3657,7 +3633,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb) store.pending.put (transaction, nano::pending_key (nano::dev::genesis->account (), send->hash ()), nano::pending_info (nano::dev::genesis->account (), 100, nano::epoch::epoch_0)); store.pruned.put (transaction, send->hash ()); - store.unchecked.put (transaction, nano::dev::genesis->hash (), send); store.version.put (transaction, version); send->sideband_set ({}); store.block.put (transaction, send->hash (), *send); @@ -3667,10 +3642,11 @@ TEST (ledger, migrate_lmdb_to_rocksdb) auto error = ledger.migrate_lmdb_to_rocksdb (path); ASSERT_FALSE (error); - nano::rocksdb_store rocksdb_store (logger, path / "rocksdb", nano::dev::constants); + nano::rocksdb_store rocksdb_store{ logger, path / "rocksdb", nano::dev::constants }; + nano::unchecked_map rocksdb_unchecked{ rocksdb_store, false }; auto rocksdb_transaction (rocksdb_store.tx_begin_read ()); - nano::pending_info pending_info; + nano::pending_info pending_info{}; ASSERT_FALSE (rocksdb_store.pending.get (rocksdb_transaction, nano::pending_key (nano::dev::genesis->account (), send->hash ()), pending_info)); for (auto i = rocksdb_store.online_weight.begin (rocksdb_transaction); i != rocksdb_store.online_weight.end (); ++i) @@ -3693,11 +3669,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb) ASSERT_EQ (confirmation_height_info.frontier, send->hash ()); ASSERT_TRUE (rocksdb_store.final_vote.get (rocksdb_transaction, nano::root (send->previous ())).size () == 1); ASSERT_EQ (rocksdb_store.final_vote.get (rocksdb_transaction, nano::root (send->previous ()))[0], nano::block_hash (2)); - - auto unchecked_infos = rocksdb_store.unchecked.get (rocksdb_transaction, nano::dev::genesis->hash ()); - ASSERT_EQ (unchecked_infos.size (), 1); - ASSERT_EQ (unchecked_infos.front ().account, nano::dev::genesis->account ()); - ASSERT_EQ (*unchecked_infos.front ().block, *send); } TEST (ledger, unconfirmed_frontiers) diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index eda3112ca4..f89daf48c5 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -2961,7 +2961,7 @@ TEST (node, block_processor_signatures) // Invalid signature to unchecked { auto transaction (node1.store.tx_begin_write ()); - node1.store.unchecked.put (transaction, send5->previous (), send5); + node1.unchecked.put (send5->previous (), send5); } auto receive1 = builder.make_block () .account (key1.pub) @@ -3289,11 +3289,11 @@ TEST (node, peer_cache_restart) TEST (node, unchecked_cleanup) { - nano::system system; - nano::node_flags node_flags; + nano::system system{}; + nano::node_flags node_flags{}; node_flags.disable_unchecked_cleanup = true; - nano::keypair key; - auto & node (*system.add_node (node_flags)); + nano::keypair key{}; + auto & node = *system.add_node (node_flags); auto open = nano::state_block_builder () .account (key.pub) .previous (0) @@ -3312,32 +3312,18 @@ TEST (node, unchecked_cleanup) // Should be cleared after unchecked cleanup ASSERT_FALSE (node.network.publish_filter.apply (bytes.data (), bytes.size ())); node.process_active (open); - node.block_processor.flush (); + // Waits for the open block to get saved in the database + ASSERT_TIMELY (15s, 1 == node.unchecked.count (node.store.tx_begin_read ())); node.config.unchecked_cutoff_time = std::chrono::seconds (2); - { - auto transaction (node.store.tx_begin_read ()); - auto unchecked_count (node.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 1); - ASSERT_EQ (unchecked_count, node.store.unchecked.count (transaction)); - } + ASSERT_EQ (1, node.unchecked.count (node.store.tx_begin_read ())); std::this_thread::sleep_for (std::chrono::seconds (1)); node.unchecked_cleanup (); ASSERT_TRUE (node.network.publish_filter.apply (bytes.data (), bytes.size ())); - { - auto transaction (node.store.tx_begin_read ()); - auto unchecked_count (node.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 1); - ASSERT_EQ (unchecked_count, node.store.unchecked.count (transaction)); - } + ASSERT_EQ (1, node.unchecked.count (node.store.tx_begin_read ())); std::this_thread::sleep_for (std::chrono::seconds (2)); node.unchecked_cleanup (); ASSERT_FALSE (node.network.publish_filter.apply (bytes.data (), bytes.size ())); - { - auto transaction (node.store.tx_begin_read ()); - auto unchecked_count (node.store.unchecked.count (transaction)); - ASSERT_EQ (unchecked_count, 0); - ASSERT_EQ (unchecked_count, node.store.unchecked.count (transaction)); - } + ASSERT_EQ (0, node.unchecked.count (node.store.tx_begin_read ())); } /** This checks that a node can be opened (without being blocked) when a write lock is held elsewhere */ @@ -3415,7 +3401,6 @@ TEST (node, bidirectional_tcp) .work (*node1->work_generate_blocking (nano::dev::genesis->hash ())) .build_shared (); node1->process_active (send1); - node1->block_processor.flush (); ASSERT_TIMELY (10s, node1->ledger.block_or_pruned_exists (send1->hash ()) && node2->ledger.block_or_pruned_exists (send1->hash ())); // Test block confirmation from node 1 (add representative to node 1) system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); diff --git a/nano/core_test/telemetry.cpp b/nano/core_test/telemetry.cpp index 8156bcef0d..fbe4d84917 100644 --- a/nano/core_test/telemetry.cpp +++ b/nano/core_test/telemetry.cpp @@ -686,7 +686,7 @@ TEST (telemetry, remove_peer_invalid_signature) // (Implementation detail) So that messages are not just discarded when requests were not sent. node->telemetry->recent_or_initial_request_telemetry_data.emplace (channel->get_endpoint (), nano::telemetry_data (), std::chrono::steady_clock::now (), true); - auto telemetry_data = nano::local_telemetry_data (node->ledger, node->network, node->config.bandwidth_limit, node->network_params, node->startup_time, node->default_difficulty (nano::work_version::work_1), node->node_id); + auto telemetry_data = nano::local_telemetry_data (node->ledger, node->network, node->unchecked, node->config.bandwidth_limit, node->network_params, node->startup_time, node->default_difficulty (nano::work_version::work_1), node->node_id); // Change anything so that the signed message is incorrect telemetry_data.block_count = 0; auto telemetry_ack = nano::telemetry_ack{ nano::dev::network_params.network, telemetry_data }; diff --git a/nano/core_test/unchecked_map.cpp b/nano/core_test/unchecked_map.cpp new file mode 100644 index 0000000000..c92d779d93 --- /dev/null +++ b/nano/core_test/unchecked_map.cpp @@ -0,0 +1,81 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +using namespace std::chrono_literals; + +namespace +{ +class context +{ +public: + context () : + store{ nano::make_store (logger, nano::unique_path (), nano::dev::constants) }, + unchecked{ *store, false } + { + } + nano::logger_mt logger; + std::unique_ptr store; + nano::unchecked_map unchecked; +}; +std::shared_ptr block () +{ + nano::block_builder builder; + return builder.state () + .account (nano::dev::genesis_key.pub) + .previous (nano::dev::genesis->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - 1) + .link (nano::dev::genesis_key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (0) + .build_shared (); +} +} + +TEST (unchecked_map, construction) +{ + context context; +} + +TEST (unchecked_map, put_one) +{ + context context; + nano::unchecked_info info{ block (), nano::dev::genesis_key.pub, nano::seconds_since_epoch () }; + context.unchecked.put (info.block->previous (), info); +} + +TEST (block_store, one_bootstrap) +{ + nano::system system{}; + nano::logger_mt logger{}; + auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); + nano::unchecked_map unchecked{ *store, false }; + ASSERT_TRUE (!store->init_error ()); + auto block1 = std::make_shared (0, 1, 2, nano::keypair ().prv, 4, 5); + unchecked.put (block1->hash (), nano::unchecked_info{ block1 }); + auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + return unchecked.get (transaction_a, block_hash_a).size () > 0; + }; + // Waits for the block1 to get saved in the database + ASSERT_TIMELY (10s, check_block_is_listed (store->tx_begin_read (), block1->hash ())); + auto transaction = store->tx_begin_read (); + auto [begin, end] = unchecked.full_range (transaction); + ASSERT_NE (end, begin); + auto hash1 = begin->first.key (); + ASSERT_EQ (block1->hash (), hash1); + auto blocks = unchecked.get (transaction, hash1); + ASSERT_EQ (1, blocks.size ()); + auto block2 = blocks[0].block; + ASSERT_EQ (*block1, *block2); + ++begin; + ASSERT_EQ (end, begin); +} diff --git a/nano/lib/threading.cpp b/nano/lib/threading.cpp index 809f0009c2..b0b705764a 100644 --- a/nano/lib/threading.cpp +++ b/nano/lib/threading.cpp @@ -86,6 +86,8 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) break; case nano::thread_role::name::election_scheduler: thread_role_name_string = "Election Sched"; + case nano::thread_role::name::unchecked: + thread_role_name_string = "Unchecked"; } /* diff --git a/nano/lib/threading.hpp b/nano/lib/threading.hpp index 5180880e98..142bf627ba 100644 --- a/nano/lib/threading.hpp +++ b/nano/lib/threading.hpp @@ -40,7 +40,8 @@ namespace thread_role state_block_signature_verification, epoch_upgrader, db_parallel_traversal, - election_scheduler + election_scheduler, + unchecked, }; /* * Get/Set the identifier for the current thread diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index abccd77212..522c87f57c 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -438,7 +438,7 @@ int main (int argc, char * const * argv) } // Check all unchecked keys for matching frontier hashes. Indicates an issue with process_batch algorithm - for (auto [i, n] = node->store.unchecked.full_range (transaction); i != n; ++i) + for (auto [i, n] = node->unchecked.full_range (transaction); i != n; ++i) { auto it = frontier_hashes.find (i->first.key ()); if (it != frontier_hashes.cend ()) @@ -1004,7 +1004,7 @@ int main (int argc, char * const * argv) if (timer_l.after_deadline (std::chrono::seconds (15))) { timer_l.restart (); - std::cout << boost::str (boost::format ("%1% (%2%) blocks processed (unchecked), %3% remaining") % node->ledger.cache.block_count % node->store.unchecked.count (node->store.tx_begin_read ()) % node->block_processor.size ()) << std::endl; + std::cout << boost::str (boost::format ("%1% (%2%) blocks processed (unchecked), %3% remaining") % node->ledger.cache.block_count % node->unchecked.count (node->store.tx_begin_read ()) % node->block_processor.size ()) << std::endl; } } @@ -1853,7 +1853,7 @@ int main (int argc, char * const * argv) if (timer_l.after_deadline (std::chrono::seconds (60))) { timer_l.restart (); - std::cout << boost::str (boost::format ("%1% (%2%) blocks processed (unchecked)") % node.node->ledger.cache.block_count % node.node->store.unchecked.count (node.node->store.tx_begin_read ())) << std::endl; + std::cout << boost::str (boost::format ("%1% (%2%) blocks processed (unchecked)") % node.node->ledger.cache.block_count % node.node->unchecked.count (node.node->store.tx_begin_read ())) << std::endl; } } diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index 6a332b5eec..36e1024bf9 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -136,6 +136,8 @@ add_library( transport/transport.cpp transport/udp.hpp transport/udp.cpp + unchecked_map.cpp + unchecked_map.hpp vote_processor.hpp vote_processor.cpp voting.hpp diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index fdcf67f09b..e20bbd455d 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -380,10 +380,8 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction { info_a.modified = nano::seconds_since_epoch (); } - - node.store.unchecked.put (transaction_a, block->previous (), info_a); + node.unchecked.put (block->previous (), info_a); events_a.events.emplace_back ([this, hash] (nano::transaction const & /* unused */) { this->node.gap_cache.add (hash); }); - node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_previous); break; } @@ -398,10 +396,8 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction { info_a.modified = nano::seconds_since_epoch (); } - - node.store.unchecked.put (transaction_a, node.ledger.block_source (transaction_a, *(block)), info_a); + node.unchecked.put (node.ledger.block_source (transaction_a, *(block)), info_a); events_a.events.emplace_back ([this, hash] (nano::transaction const & /* unused */) { this->node.gap_cache.add (hash); }); - node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_source); break; } @@ -416,8 +412,7 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction { info_a.modified = nano::seconds_since_epoch (); } - - node.store.unchecked.put (transaction_a, block->account (), info_a); // Specific unchecked key starting with epoch open block account public key + node.unchecked.put (block->account (), info_a); // Specific unchecked key starting with epoch open block account public key node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_source); break; } @@ -515,15 +510,7 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction void nano::block_processor::queue_unchecked (nano::write_transaction const & transaction_a, nano::hash_or_account const & hash_or_account_a) { - auto unchecked_blocks (node.store.unchecked.get (transaction_a, hash_or_account_a.hash)); - for (auto & info : unchecked_blocks) - { - if (!node.flags.disable_block_processor_unchecked_deletion) - { - node.store.unchecked.del (transaction_a, nano::unchecked_key (hash_or_account_a, info.block->hash ())); - } - add (info); - } + node.unchecked.trigger (hash_or_account_a); node.gap_cache.erase (hash_or_account_a.hash); } diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index 8c928345be..79b2fc1205 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -230,7 +230,7 @@ bool copy_database (boost::filesystem::path const & data_path, boost::program_op auto & store (node.node->store); if (vm.count ("unchecked_clear")) { - node.node->store.unchecked.clear (store.tx_begin_write ()); + node.node->unchecked.clear (store.tx_begin_write ()); } if (vm.count ("clear_send_ids")) { @@ -508,7 +508,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map if (!node.node->init_error ()) { auto transaction (node.node->store.tx_begin_write ()); - node.node->store.unchecked.clear (transaction); + node.node->unchecked.clear (transaction); std::cout << "Unchecked blocks deleted" << std::endl; } else diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 9addf691ed..d7ed1a84ef 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1358,7 +1358,7 @@ void nano::json_handler::block_account () void nano::json_handler::block_count () { response_l.put ("count", std::to_string (node.ledger.cache.block_count)); - response_l.put ("unchecked", std::to_string (node.store.unchecked.count (node.store.tx_begin_read ()))); + response_l.put ("unchecked", std::to_string (node.unchecked.count (node.store.tx_begin_read ()))); response_l.put ("cemented", std::to_string (node.ledger.cache.cemented_count)); if (node.flags.enable_pruning) { @@ -3878,7 +3878,7 @@ void nano::json_handler::telemetry () if (address.is_loopback () && port == rpc_l->node.network.endpoint ().port ()) { // Requesting telemetry metrics locally - auto telemetry_data = nano::local_telemetry_data (rpc_l->node.ledger, rpc_l->node.network, rpc_l->node.config.bandwidth_limit, rpc_l->node.network_params, rpc_l->node.startup_time, rpc_l->node.default_difficulty (nano::work_version::work_1), rpc_l->node.node_id); + auto telemetry_data = nano::local_telemetry_data (rpc_l->node.ledger, rpc_l->node.network, rpc_l->node.unchecked, rpc_l->node.config.bandwidth_limit, rpc_l->node.network_params, rpc_l->node.startup_time, rpc_l->node.default_difficulty (nano::work_version::work_1), rpc_l->node.node_id); nano::jsonconfig config_l; auto const should_ignore_identification_metrics = false; @@ -4026,7 +4026,7 @@ void nano::json_handler::unchecked () { boost::property_tree::ptree unchecked; auto transaction (node.store.tx_begin_read ()); - for (auto [i, n] = node.store.unchecked.full_range (transaction); i != n && unchecked.size () < count; ++i) + for (auto [i, n] = node.unchecked.full_range (transaction); i != n && unchecked.size () < count; ++i) { nano::unchecked_info const & info (i->second); if (json_block_l) @@ -4051,7 +4051,7 @@ void nano::json_handler::unchecked_clear () { node.workers.push_task (create_worker_task ([] (std::shared_ptr const & rpc_l) { auto transaction (rpc_l->node.store.tx_begin_write ({ tables::unchecked })); - rpc_l->node.store.unchecked.clear (transaction); + rpc_l->node.unchecked.clear (transaction); rpc_l->response_l.put ("success", ""); rpc_l->response_errors (); })); @@ -4064,7 +4064,7 @@ void nano::json_handler::unchecked_get () if (!ec) { auto transaction (node.store.tx_begin_read ()); - for (auto [i, n] = node.store.unchecked.full_range (transaction); i != n; ++i) + for (auto [i, n] = node.unchecked.full_range (transaction); i != n; ++i) { nano::unchecked_key const & key (i->first); if (key.hash == hash) @@ -4112,7 +4112,7 @@ void nano::json_handler::unchecked_keys () { boost::property_tree::ptree unchecked; auto transaction (node.store.tx_begin_read ()); - for (auto [i, n] = node.store.unchecked.equal_range (transaction, key); i != n && unchecked.size () < count; ++i) + for (auto [i, n] = node.unchecked.equal_range (transaction, key); i != n && unchecked.size () < count; ++i) { boost::property_tree::ptree entry; nano::unchecked_info const & info (i->second); diff --git a/nano/node/network.cpp b/nano/node/network.cpp index 99aa8626b6..64c6ec6730 100644 --- a/nano/node/network.cpp +++ b/nano/node/network.cpp @@ -538,7 +538,7 @@ class network_message_visitor : public nano::message_visitor nano::telemetry_ack telemetry_ack{ node.network_params.network }; if (!node.flags.disable_providing_telemetry_metrics) { - auto telemetry_data = nano::local_telemetry_data (node.ledger, node.network, node.config.bandwidth_limit, node.network_params, node.startup_time, node.default_difficulty (nano::work_version::work_1), node.node_id); + auto telemetry_data = nano::local_telemetry_data (node.ledger, node.network, node.unchecked, node.config.bandwidth_limit, node.network_params, node.startup_time, node.default_difficulty (nano::work_version::work_1), node.node_id); telemetry_ack = nano::telemetry_ack{ node.network_params.network, telemetry_data }; } channel->send (telemetry_ack, nullptr, nano::buffer_drop_policy::no_socket_drop); diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 99f9571c98..5cc5b31b6c 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -93,6 +93,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co logger (config_a.logging.min_time_between_log_output), store_impl (nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade)), store (*store_impl), + unchecked{ store, flags.disable_block_processor_unchecked_deletion }, wallets_store_impl (std::make_unique (application_path_a / "wallets.ldb", config_a.lmdb_config)), wallets_store (*wallets_store_impl), gap_cache (*this), @@ -129,6 +130,9 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co startup_time (std::chrono::steady_clock::now ()), node_seq (seq) { + unchecked.satisfied = [this] (nano::unchecked_info const & info) { + this->block_processor.add (info); + }; if (!init_error ()) { telemetry->start (); @@ -413,7 +417,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co if (!flags.disable_unchecked_drop && !use_bootstrap_weight && !flags.read_only) { auto const transaction (store.tx_begin_write ({ tables::unchecked })); - store.unchecked.clear (transaction); + unchecked.clear (transaction); logger.always_log ("Dropping unchecked blocks"); } } @@ -677,6 +681,7 @@ void nano::node::stop () // Cancels ongoing work generation tasks, which may be blocking other threads // No tasks may wait for work generation in I/O threads, or termination signal capturing will be unable to call node::stop() distributed_work.stop (); + unchecked.stop (); block_processor.stop (); aggregator.stop (); vote_processor.stop (); @@ -941,7 +946,7 @@ void nano::node::unchecked_cleanup () auto const now (nano::seconds_since_epoch ()); auto const transaction (store.tx_begin_read ()); // Max 1M records to clean, max 2 minutes reading to prevent slow i/o systems issues - for (auto [i, n] = store.unchecked.full_range (transaction); i != n && cleaning_list.size () < 1024 * 1024 && nano::seconds_since_epoch () - now < 120; ++i) + for (auto [i, n] = unchecked.full_range (transaction); i != n && cleaning_list.size () < 1024 * 1024 && nano::seconds_since_epoch () - now < 120; ++i) { nano::unchecked_key const & key (i->first); nano::unchecked_info const & info (i->second); @@ -965,9 +970,9 @@ void nano::node::unchecked_cleanup () { auto key (cleaning_list.front ()); cleaning_list.pop_front (); - if (store.unchecked.exists (transaction, key)) + if (unchecked.exists (transaction, key)) { - store.unchecked.del (transaction, key); + unchecked.del (transaction, key); } } } diff --git a/nano/node/node.hpp b/nano/node/node.hpp index a188940d43..d6efeee69d 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -165,6 +166,7 @@ class node final : public std::enable_shared_from_this nano::logger_mt logger; std::unique_ptr store_impl; nano::store & store; + nano::unchecked_map unchecked; std::unique_ptr wallets_store_impl; nano::wallets_store & wallets_store; nano::gap_cache gap_cache; diff --git a/nano/node/telemetry.cpp b/nano/node/telemetry.cpp index ba553cb0e1..23d50de152 100644 --- a/nano/node/telemetry.cpp +++ b/nano/node/telemetry.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -628,7 +629,7 @@ nano::telemetry_data nano::consolidate_telemetry_data (std::vector (std::chrono::steady_clock::now () - statup_time_a).count (); - telemetry_data.unchecked_count = ledger_a.store.unchecked.count (ledger_a.store.tx_begin_read ()); + telemetry_data.unchecked_count = unchecked.count (ledger_a.store.tx_begin_read ()); telemetry_data.genesis_block = network_params_a.ledger.genesis->hash (); telemetry_data.peer_count = nano::narrow_cast (network_a.size ()); telemetry_data.account_count = ledger_a.cache.account_count; diff --git a/nano/node/telemetry.hpp b/nano/node/telemetry.hpp index cd902cac40..44fa53916c 100644 --- a/nano/node/telemetry.hpp +++ b/nano/node/telemetry.hpp @@ -20,6 +20,7 @@ class network; class stat; class ledger; class thread_pool; +class unchecked_map; namespace transport { class channel; @@ -149,5 +150,5 @@ class telemetry : public std::enable_shared_from_this std::unique_ptr collect_container_info (telemetry & telemetry, std::string const & name); nano::telemetry_data consolidate_telemetry_data (std::vector const & telemetry_data); -nano::telemetry_data local_telemetry_data (nano::ledger const & ledger_a, nano::network &, uint64_t, nano::network_params const &, std::chrono::steady_clock::time_point, uint64_t, nano::keypair const &); +nano::telemetry_data local_telemetry_data (nano::ledger const & ledger_a, nano::network &, nano::unchecked_map const &, uint64_t, nano::network_params const &, std::chrono::steady_clock::time_point, uint64_t, nano::keypair const &); } diff --git a/nano/node/unchecked_map.cpp b/nano/node/unchecked_map.cpp new file mode 100644 index 0000000000..e73d4546ac --- /dev/null +++ b/nano/node/unchecked_map.cpp @@ -0,0 +1,154 @@ +#include +#include +#include +#include + +#include + +nano::unchecked_map::unchecked_map (nano::store & store, bool const & disable_delete) : + store{ store }, + disable_delete{ disable_delete }, + thread{ [this] () { run (); } } +{ +} + +nano::unchecked_map::~unchecked_map () +{ + stop (); + thread.join (); +} + +void nano::unchecked_map::put (nano::hash_or_account const & dependency, nano::unchecked_info const & info) +{ + nano::unique_lock lock{ mutex }; + buffer.push_back (std::make_pair (dependency, info)); + lock.unlock (); + condition.notify_all (); // Notify run () +} + +auto nano::unchecked_map::equal_range (nano::transaction const & transaction, nano::block_hash const & dependency) -> std::pair +{ + return store.unchecked.equal_range (transaction, dependency); +} + +auto nano::unchecked_map::full_range (nano::transaction const & transaction) -> std::pair +{ + return store.unchecked.full_range (transaction); +} + +std::vector nano::unchecked_map::get (nano::transaction const & transaction, nano::block_hash const & hash) +{ + return store.unchecked.get (transaction, hash); +} + +bool nano::unchecked_map::exists (nano::transaction const & transaction, nano::unchecked_key const & key) const +{ + return store.unchecked.exists (transaction, key); +} + +void nano::unchecked_map::del (nano::write_transaction const & transaction, nano::unchecked_key const & key) +{ + store.unchecked.del (transaction, key); +} + +void nano::unchecked_map::clear (nano::write_transaction const & transaction) +{ + store.unchecked.clear (transaction); +} + +size_t nano::unchecked_map::count (nano::transaction const & transaction) const +{ + return store.unchecked.count (transaction); +} + +void nano::unchecked_map::stop () +{ + if (!stopped.exchange (true)) + { + condition.notify_all (); // Notify flush (), run () + } +} + +void nano::unchecked_map::flush () +{ + nano::unique_lock lock{ mutex }; + condition.wait (lock, [this] () { + return stopped || (buffer.empty () && back_buffer.empty () && !writing_back_buffer); + }); +} + +void nano::unchecked_map::trigger (nano::hash_or_account const & dependency) +{ + nano::unique_lock lock{ mutex }; + buffer.push_back (dependency); + debug_assert (buffer.back ().which () == 1); // which stands for "query". + lock.unlock (); + condition.notify_all (); // Notify run () +} + +nano::unchecked_map::item_visitor::item_visitor (unchecked_map & unchecked, nano::write_transaction const & transaction) : + unchecked{ unchecked }, + transaction{ transaction } +{ +} +void nano::unchecked_map::item_visitor::operator() (insert const & item) +{ + auto const & [dependency, info] = item; + unchecked.store.unchecked.put (transaction, dependency, info); +} + +void nano::unchecked_map::item_visitor::operator() (query const & item) +{ + auto [i, n] = unchecked.store.unchecked.equal_range (transaction, item.hash); + std::deque delete_queue; + for (; i != n; ++i) + { + auto const & key = i->first; + auto const & info = i->second; + delete_queue.push_back (key); + unchecked.satisfied (info); + } + if (!unchecked.disable_delete) + { + for (auto const & key : delete_queue) + { + unchecked.del (transaction, key); + } + } +} + +void nano::unchecked_map::write_buffer (decltype (buffer) const & back_buffer) +{ + auto transaction = store.tx_begin_write (); + item_visitor visitor{ *this, transaction }; + for (auto const & item : back_buffer) + { + boost::apply_visitor (visitor, item); + } +} + +void nano::unchecked_map::run () +{ + nano::thread_role::set (nano::thread_role::name::unchecked); + nano::unique_lock lock{ mutex }; + while (!stopped) + { + if (!buffer.empty ()) + { + back_buffer.swap (buffer); + writing_back_buffer = true; + lock.unlock (); + write_buffer (back_buffer); + lock.lock (); + writing_back_buffer = false; + back_buffer.clear (); + } + else + { + condition.notify_all (); // Notify flush () + condition.wait (lock, [this] () { + return stopped || !buffer.empty (); + }); + } + } +} diff --git a/nano/node/unchecked_map.hpp b/nano/node/unchecked_map.hpp new file mode 100644 index 0000000000..a6523498f6 --- /dev/null +++ b/nano/node/unchecked_map.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace nano +{ +class store; +class transaction; +class unchecked_info; +class unchecked_key; +class write_transaction; +class unchecked_map +{ +public: + using iterator = nano::unchecked_store::iterator; + +public: + unchecked_map (nano::store & store, bool const & do_delete); + ~unchecked_map (); + void put (nano::hash_or_account const & dependency, nano::unchecked_info const & info); + std::pair equal_range (nano::transaction const & transaction, nano::block_hash const & dependency); + std::pair full_range (nano::transaction const & transaction); + std::vector get (nano::transaction const &, nano::block_hash const &); + bool exists (nano::transaction const & transaction, nano::unchecked_key const & key) const; + void del (nano::write_transaction const & transaction, nano::unchecked_key const & key); + void clear (nano::write_transaction const & transaction); + size_t count (nano::transaction const & transaction) const; + void stop (); + void flush (); + +public: // Trigger requested dependencies + void trigger (nano::hash_or_account const & dependency); + std::function satisfied{ [] (nano::unchecked_info const &) {} }; + +private: + using insert = std::pair; + using query = nano::hash_or_account; + class item_visitor : boost::static_visitor<> + { + public: + item_visitor (unchecked_map & unchecked, nano::write_transaction const & transaction); + void operator() (insert const & item); + void operator() (query const & item); + unchecked_map & unchecked; + nano::write_transaction const & transaction; + }; + void run (); + nano::store & store; + bool const & disable_delete; + std::deque> buffer; + std::deque> back_buffer; + bool writing_back_buffer{ false }; + std::atomic stopped{ false }; + nano::condition_variable condition; + nano::mutex mutex; + std::thread thread; + void write_buffer (decltype (buffer) const & back_buffer); +}; +} diff --git a/nano/qt/qt.cpp b/nano/qt/qt.cpp index a0fee48842..307a090f23 100644 --- a/nano/qt/qt.cpp +++ b/nano/qt/qt.cpp @@ -923,7 +923,7 @@ std::string nano_qt::status::text () std::string count_string; { auto size (wallet.wallet_m->wallets.node.ledger.cache.block_count.load ()); - unchecked = wallet.wallet_m->wallets.node.store.unchecked.count (wallet.wallet_m->wallets.node.store.tx_begin_read ()); + unchecked = wallet.wallet_m->wallets.node.unchecked.count (wallet.wallet_m->wallets.node.store.tx_begin_read ()); cemented = wallet.wallet_m->wallets.node.ledger.cache.cemented_count.load (); count_string = std::to_string (size); } @@ -959,7 +959,7 @@ std::string nano_qt::status::text () result += ", Blocks: "; if (unchecked != 0 && wallet.node.bootstrap_initiator.in_progress ()) { - count_string += "\nUnchecked: " + std::to_string (unchecked) + ", Cemented: " + std::to_string (cemented); + count_string += ", Queued: " + std::to_string (unchecked); } if (wallet.node.flags.enable_pruning) diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 51d4f13c07..14b93e105b 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -5010,17 +5010,19 @@ TEST (rpc, stats_clear) ASSERT_LE (node->stats.last_reset ().count (), 5); } +// Tests the RPC command returns the correct data for the unchecked blocks TEST (rpc, unchecked) { - nano::system system; + nano::system system{}; auto node = add_ipc_enabled_node (system); auto const rpc_ctx = add_rpc (system, node); - nano::keypair key; - auto open (std::make_shared (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); - auto open2 (std::make_shared (key.pub, 0, key.pub, 2, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); + nano::keypair key{}; + auto open = std::make_shared (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub)); + auto open2 = std::make_shared (key.pub, 0, key.pub, 2, key.pub, key.prv, key.pub, *system.work.generate (key.pub)); node->process_active (open); node->process_active (open2); - node->block_processor.flush (); + // Waits for the last block of the queue to get saved in the database + ASSERT_TIMELY (10s, 2 == node->unchecked.count (node->store.tx_begin_read ())); boost::property_tree::ptree request; request.put ("action", "unchecked"); request.put ("count", 2); @@ -5041,16 +5043,18 @@ TEST (rpc, unchecked) } } +// Tests the RPC command returns the correct data for the unchecked blocks TEST (rpc, unchecked_get) { - nano::system system; + nano::system system{}; auto node = add_ipc_enabled_node (system); auto const rpc_ctx = add_rpc (system, node); - nano::keypair key; - auto open (std::make_shared (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); + nano::keypair key{}; + auto open = std::make_shared (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub)); node->process_active (open); - node->block_processor.flush (); - boost::property_tree::ptree request; + // Waits for the open block to get saved in the database + ASSERT_TIMELY (10s, 1 == node->unchecked.count (node->store.tx_begin_read ())); + boost::property_tree::ptree request{}; request.put ("action", "unchecked_get"); request.put ("hash", open->hash ().to_string ()); { @@ -5071,21 +5075,20 @@ TEST (rpc, unchecked_get) TEST (rpc, unchecked_clear) { - nano::system system; + nano::system system{}; auto node = add_ipc_enabled_node (system); auto const rpc_ctx = add_rpc (system, node); - nano::keypair key; - auto open (std::make_shared (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); + nano::keypair key{}; + auto open = std::make_shared (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub)); node->process_active (open); - node->block_processor.flush (); - boost::property_tree::ptree request; - { - ASSERT_EQ (node->store.unchecked.count (node->store.tx_begin_read ()), 1); - } + boost::property_tree::ptree request{}; + // Waits for the open block to get saved in the database + ASSERT_TIMELY (10s, 1 == node->unchecked.count (node->store.tx_begin_read ())); request.put ("action", "unchecked_clear"); - auto response (wait_response (system, rpc_ctx, request)); + auto response = wait_response (system, rpc_ctx, request); - ASSERT_TIMELY (10s, node->store.unchecked.count (node->store.tx_begin_read ()) == 0); + // Waits for the open block to get saved in the database + ASSERT_TIMELY (10s, 0 == node->unchecked.count (node->store.tx_begin_read ())); } TEST (rpc, unopened) @@ -5807,19 +5810,19 @@ TEST (rpc, epoch_upgrade_multithreaded) TEST (rpc, account_lazy_start) { - nano::system system; - nano::node_flags node_flags; + nano::system system{}; + nano::node_flags node_flags{}; node_flags.disable_legacy_bootstrap = true; auto node1 = system.add_node (node_flags); - nano::keypair key; + nano::keypair key{}; // Generating test chain - auto send1 (std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::dev::constants.genesis_amount - nano::Gxrb_ratio, key.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system.work.generate (nano::dev::genesis->hash ()))); + auto send1 = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::dev::constants.genesis_amount - nano::Gxrb_ratio, key.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system.work.generate (nano::dev::genesis->hash ())); ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); - auto open (std::make_shared (send1->hash (), key.pub, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); + auto open = std::make_shared (send1->hash (), key.pub, key.pub, key.prv, key.pub, *system.work.generate (key.pub)); ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); // Start lazy bootstrap with account - nano::node_config node_config (nano::get_available_port (), system.logging); + nano::node_config node_config{ nano::get_available_port (), system.logging }; node_config.ipc_config.transport_tcp.enabled = true; node_config.ipc_config.transport_tcp.port = nano::get_available_port (); auto node2 = system.add_node (node_config, node_flags); @@ -5828,15 +5831,17 @@ TEST (rpc, account_lazy_start) boost::property_tree::ptree request; request.put ("action", "account_info"); request.put ("account", key.pub.to_account ()); - auto response (wait_response (system, rpc_ctx, request)); - boost::optional account_error (response.get_optional ("error")); + auto response = wait_response (system, rpc_ctx, request); + boost::optional account_error{ response.get_optional ("error") }; ASSERT_TRUE (account_error.is_initialized ()); // Check processed blocks ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ()); - node2->block_processor.flush (); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ())); - ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ())); + + // needs timed assert because the writing (put) operation is done by a different + // thread, it might not get done before DB get operation. + ASSERT_TIMELY (10s, node2->ledger.block_or_pruned_exists (send1->hash ())); + ASSERT_TIMELY (10s, node2->ledger.block_or_pruned_exists (open->hash ())); } TEST (rpc, receive) diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 675c2fa989..df2039f1ba 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -1438,15 +1438,6 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data } }); - store.unchecked.for_each_par ( - [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { - for (; i != n; ++i) - { - auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::unchecked })); - rocksdb_store->unchecked.put (rocksdb_transaction, i->first.previous, i->second); - } - }); - store.pending.for_each_par ( [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) @@ -1517,7 +1508,6 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data } // Compare counts - error |= store.unchecked.count (lmdb_transaction) != rocksdb_store->unchecked.count (rocksdb_transaction); error |= store.peer.count (lmdb_transaction) != rocksdb_store->peer.count (rocksdb_transaction); error |= store.pruned.count (lmdb_transaction) != rocksdb_store->pruned.count (rocksdb_transaction); error |= store.final_vote.count (lmdb_transaction) != rocksdb_store->final_vote.count (rocksdb_transaction); diff --git a/nano/secure/store.hpp b/nano/secure/store.hpp index ac724c0422..edea5acb16 100644 --- a/nano/secure/store.hpp +++ b/nano/secure/store.hpp @@ -815,6 +815,7 @@ class block_store virtual uint64_t account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; }; +class unchecked_map; /** * Store manager */ @@ -844,7 +845,11 @@ class store frontier_store & frontier; account_store & account; pending_store & pending; + +private: unchecked_store & unchecked; + +public: online_weight_store & online_weight; pruned_store & pruned; peer_store & peer; @@ -870,6 +875,8 @@ class store virtual nano::read_transaction tx_begin_read () const = 0; virtual std::string vendor_get () const = 0; + + friend class unchecked_map; }; std::unique_ptr make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = false, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false); diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index 65de9c38c7..7fd6f1ac79 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -420,17 +421,16 @@ TEST (peer_container, random_set) // Can take up to 2 hours TEST (store, unchecked_load) { - nano::system system (1); - auto & node (*system.nodes[0]); + nano::system system{ 1 }; + auto & node = *system.nodes[0]; std::shared_ptr block = std::make_shared (0, 0, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); - constexpr auto num_unchecked = 1000000; + constexpr auto num_unchecked = 1'000'000; for (auto i (0); i < num_unchecked; ++i) { - auto transaction (node.store.tx_begin_write ()); - node.store.unchecked.put (transaction, i, block); + node.unchecked.put (i, block); } - auto transaction (node.store.tx_begin_read ()); - ASSERT_EQ (num_unchecked, node.store.unchecked.count (transaction)); + // Waits for all the blocks to get saved in the database + ASSERT_TIMELY (8000s, num_unchecked == node.unchecked.count (node.store.tx_begin_read ())); } TEST (store, vote_load)