Skip to content

Commit

Permalink
Create nano::unchecked_map ADT which is the interface to the unchecke…
Browse files Browse the repository at this point in the history 
…d table. Rather than directly making modifications to the unchecked table, this ADT abstracts away the details of where this information is stored. nano::unchecked_map will queue write/trigger operations for processing in a background thread. This means nano::unchecked_map::put no longer requires a database transaction to call. This also changes the semantics of the unchecked put operations as they're no longer blocking, fix up many tests that made this assumption. (nanocurrency#3553)

Co-authored-by: clemahieu <[email protected]>
  • Loading branch information
@thsfs @clemahieu
thsfs and clemahieu authored Feb 3, 2022
1 parent 275a545 commit 8b42872
Show file tree
Hide file tree
Showing 28 changed files with 629 additions and 352 deletions.
1 change: 1 addition & 0 deletions nano/core_test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ add_executable(
toml.cpp
timer.cpp
uint256_union.cpp
unchecked_map.cpp
utility.cpp
vote_processor.cpp
voting.cpp
Expand Down
233 changes: 126 additions & 107 deletions nano/core_test/block_store.cpp
View file

Large diffs are not rendered by default.

21 changes: 9 additions & 12 deletions nano/core_test/bootstrap.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -924,11 +924,10 @@ TEST (bootstrap_processor, DISABLED_lazy_unclear_state_link)
node2->bootstrap_initiator.bootstrap_lazy (receive->hash ());
// Check processed blocks
ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ());
node2->block_processor.flush ();
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (receive->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (open->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (receive->hash ()));
ASSERT_EQ (0, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in));
}

Expand Down Expand Up @@ -976,10 +975,9 @@ TEST (bootstrap_processor, lazy_unclear_state_link_not_existing)
node2->bootstrap_initiator.bootstrap_lazy (send2->hash ());
// Check processed blocks
ASSERT_TIMELY (15s, !node2->bootstrap_initiator.in_progress ());
node2->block_processor.flush ();
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (open->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_EQ (1, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in));
}

Expand Down Expand Up @@ -1038,7 +1036,6 @@ TEST (bootstrap_processor, DISABLED_lazy_destinations)
node2->bootstrap_initiator.bootstrap_lazy (send2->hash ());
// Check processed blocks
ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ());
node2->block_processor.flush ();
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ()));
Expand Down Expand Up @@ -1127,7 +1124,7 @@ TEST (bootstrap_processor, lazy_pruning_missing_block)
ASSERT_FALSE (node2->ledger.block_or_pruned_exists (state_open->hash ()));
{
auto transaction (node2->store.tx_begin_read ());
ASSERT_TRUE (node2->store.unchecked.exists (transaction, nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ())));
ASSERT_TRUE (node2->unchecked.exists (transaction, nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ())));
}
// Insert missing block
node2->process_active (send1);
Expand Down Expand Up @@ -1838,7 +1835,7 @@ TEST (bulk, DISABLED_genesis_pruning)
ASSERT_EQ (1, node2->ledger.cache.block_count);
{
auto transaction (node2->store.tx_begin_write ());
node2->store.unchecked.clear (transaction);
node2->unchecked.clear (transaction);
}
// Insert pruned blocks
node2->process_active (send1);
Expand Down
57 changes: 29 additions & 28 deletions nano/core_test/confirmation_height.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,24 +219,24 @@ TEST (confirmation_height, multiple_accounts)
TEST (confirmation_height, gap_bootstrap)
{
auto test_mode = [] (nano::confirmation_height_mode mode_a) {
nano::system system;
nano::node_flags node_flags;
nano::system system{};
nano::node_flags node_flags{};
node_flags.confirmation_height_processor_mode = mode_a;
auto & node1 = *system.add_node (node_flags);
nano::keypair destination;
auto send1 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
nano::keypair destination{};
auto send1 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node1.work_generate_blocking (*send1);
auto send2 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send2 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node1.work_generate_blocking (*send2);
auto send3 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send3 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node1.work_generate_blocking (*send3);
auto open1 (std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0));
auto open1 = std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0);
node1.work_generate_blocking (*open1);

// Receive
auto receive1 (std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0));
auto receive1 = std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0);
node1.work_generate_blocking (*receive1);
auto receive2 (std::make_shared<nano::receive_block> (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0));
auto receive2 = std::make_shared<nano::receive_block> (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0);
node1.work_generate_blocking (*receive2);

node1.block_processor.add (send1);
Expand All @@ -249,12 +249,16 @@ TEST (confirmation_height, gap_bootstrap)

// Receive 2 comes in on the live network, however the chain has not been finished so it gets added to unchecked
node1.process_active (receive2);
node1.block_processor.flush ();
// Waits for the unchecked_map to process the 4 blocks added to the block_processor, saving them in the unchecked table
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return !node1.unchecked.get (transaction_a, block_hash_a).empty ();
};
ASSERT_TIMELY (15s, check_block_is_listed (node1.store.tx_begin_read (), receive2->previous ()));

// Confirmation heights should not be updated
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 2);

nano::confirmation_height_info confirmation_height_info;
Expand All @@ -265,14 +269,11 @@ TEST (confirmation_height, gap_bootstrap)

// Now complete the chain where the block comes in on the bootstrap network.
node1.block_processor.add (open1);
node1.block_processor.flush ();

ASSERT_TIMELY (10s, node1.unchecked.count (node1.store.tx_begin_read ()) == 0);
// Confirmation height should be unchanged and unchecked should now be 0
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);

auto transaction = node1.store.tx_begin_read ();
nano::confirmation_height_info confirmation_height_info;
ASSERT_FALSE (node1.store.confirmation_height.get (transaction, nano::dev::genesis_key.pub, confirmation_height_info));
ASSERT_EQ (1, confirmation_height_info.height);
Expand All @@ -296,10 +297,10 @@ TEST (confirmation_height, gap_bootstrap)
TEST (confirmation_height, gap_live)
{
auto test_mode = [] (nano::confirmation_height_mode mode_a) {
nano::system system;
nano::node_flags node_flags;
nano::system system{};
nano::node_flags node_flags{};
node_flags.confirmation_height_processor_mode = mode_a;
nano::node_config node_config (nano::get_available_port (), system.logging);
nano::node_config node_config{ nano::get_available_port (), system.logging };
node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
auto node = system.add_node (node_config, node_flags);
node_config.peering_port = nano::get_available_port ();
Expand All @@ -309,18 +310,18 @@ TEST (confirmation_height, gap_live)
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
system.wallet (1)->insert_adhoc (destination.prv);

auto send1 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 1, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send1 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 1, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node->work_generate_blocking (*send1);
auto send2 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send2 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node->work_generate_blocking (*send2);
auto send3 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send3 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node->work_generate_blocking (*send3);

auto open1 (std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0));
auto open1 = std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0);
node->work_generate_blocking (*open1);
auto receive1 (std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0));
auto receive1 = std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0);
node->work_generate_blocking (*receive1);
auto receive2 (std::make_shared<nano::receive_block> (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0));
auto receive2 = std::make_shared<nano::receive_block> (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0);
node->work_generate_blocking (*receive2);

node->block_processor.add (send1);
Expand Down Expand Up @@ -355,11 +356,11 @@ TEST (confirmation_height, gap_live)
ASSERT_TIMELY (10s, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out) == 6);

// This should confirm the open block and the source of the receive blocks
auto transaction (node->store.tx_begin_read ());
auto unchecked_count (node->store.unchecked.count (transaction));
auto transaction = node->store.tx_begin_read ();
auto unchecked_count = node->unchecked.count (transaction);
ASSERT_EQ (unchecked_count, 0);

nano::confirmation_height_info confirmation_height_info;
nano::confirmation_height_info confirmation_height_info{};
ASSERT_TRUE (node->ledger.block_confirmed (transaction, receive2->hash ()));
ASSERT_FALSE (node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub, confirmation_height_info));
ASSERT_EQ (4, confirmation_height_info.height);
Expand Down
2 changes: 1 addition & 1 deletion nano/core_test/gap_cache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ TEST (gap_cache, two_dependencies)
ASSERT_EQ (2, node1.gap_cache.size ());
node1.block_processor.add (send1, nano::seconds_since_epoch ());
node1.block_processor.flush ();
ASSERT_EQ (0, node1.gap_cache.size ());
ASSERT_TIMELY (5s, node1.gap_cache.size () == 0);
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, send1->hash ()));
ASSERT_TRUE (node1.store.block.exists (transaction, send2->hash ()));
Expand Down
Loading

0 comments on commit 8b42872

Please sign in to comment.