Skip to content

Commit

Permalink
Fleshing out unchecked_map buffer processing thread.
Browse files Browse the repository at this point in the history
  • Loading branch information
@clemahieu
clemahieu authored and Thiago Silva committed Nov 22, 2021
1 parent b087575 commit 0221ec3
Show file tree
Hide file tree
Showing 16 changed files with 268 additions and 133 deletions.
98 changes: 38 additions & 60 deletions nano/core_test/block_store.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,13 +351,14 @@ TEST (unchecked, simple)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store };
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto block1 (std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5));
auto transaction (store->tx_begin_write ());
auto block2 (unchecked.get (transaction, block1->previous ()));
auto block2 (unchecked.get (store->tx_begin_write (), block1->previous ()));
ASSERT_TRUE (block2.empty ());
unchecked.put (transaction, block1->previous (), nano::unchecked_info{ block1 });
unchecked.put (block1->previous (), nano::unchecked_info{ block1 });
unchecked.flush ();
auto transaction = store->tx_begin_write ();
auto block3 (unchecked.get (transaction, block1->previous ()));
ASSERT_FALSE (block3.empty ());
ASSERT_EQ (*block1, *(block3[0].block));
Expand All @@ -375,56 +376,54 @@ TEST (unchecked, multiple)
}
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store };
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto block1 (std::make_shared<nano::send_block> (4, 1, 2, nano::keypair ().prv, 4, 5));
auto transaction (store->tx_begin_write ());
auto block2 (unchecked.get (transaction, block1->previous ()));
auto block2 (unchecked.get (store->tx_begin_read (), block1->previous ()));
ASSERT_TRUE (block2.empty ());
unchecked.put (transaction, block1->previous (), nano::unchecked_info{ block1 });
unchecked.put (transaction, block1->source (), nano::unchecked_info{ block1 });
auto block3 (unchecked.get (transaction, block1->previous ()));
unchecked.put (block1->previous (), nano::unchecked_info{ block1 });
unchecked.put (block1->source (), nano::unchecked_info{ block1 });
unchecked.flush ();
auto block3 (unchecked.get (store->tx_begin_read (), block1->previous ()));
ASSERT_FALSE (block3.empty ());
auto block4 (unchecked.get (transaction, block1->source ()));
auto block4 (unchecked.get (store->tx_begin_read (), block1->source ()));
ASSERT_FALSE (block4.empty ());
}

TEST (unchecked, double_put)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store };
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto block1 (std::make_shared<nano::send_block> (4, 1, 2, nano::keypair ().prv, 4, 5));
auto transaction (store->tx_begin_write ());
auto block2 (unchecked.get (transaction, block1->previous ()));
auto block2 (unchecked.get (store->tx_begin_read (), block1->previous ()));
ASSERT_TRUE (block2.empty ());
unchecked.put (transaction, block1->previous (), nano::unchecked_info{ block1 });
unchecked.put (transaction, block1->previous (), nano::unchecked_info{ block1 });
auto block3 (unchecked.get (transaction, block1->previous ()));
unchecked.put (block1->previous (), nano::unchecked_info{ block1 });
unchecked.put (block1->previous (), nano::unchecked_info{ block1 });
unchecked.flush ();
auto block3 (unchecked.get (store->tx_begin_read (), block1->previous ()));
ASSERT_EQ (block3.size (), 1);
}

TEST (unchecked, multiple_get)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store };
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto block1 (std::make_shared<nano::send_block> (4, 1, 2, nano::keypair ().prv, 4, 5));
auto block2 (std::make_shared<nano::send_block> (3, 1, 2, nano::keypair ().prv, 4, 5));
auto block3 (std::make_shared<nano::send_block> (5, 1, 2, nano::keypair ().prv, 4, 5));
{
auto transaction (store->tx_begin_write ());
unchecked.put (transaction, block1->previous (), nano::unchecked_info{ block1 }); // unchecked1
unchecked.put (transaction, block1->hash (), nano::unchecked_info{ block1 }); // unchecked2
unchecked.put (transaction, block2->previous (), nano::unchecked_info{ block2 }); // unchecked3
unchecked.put (transaction, block1->previous (), nano::unchecked_info{ block2 }); // unchecked1
unchecked.put (transaction, block1->hash (), nano::unchecked_info{ block2 }); // unchecked2
unchecked.put (transaction, block3->previous (), nano::unchecked_info{ block3 });
unchecked.put (transaction, block3->hash (), nano::unchecked_info{ block3 }); // unchecked4
unchecked.put (transaction, block1->previous (), nano::unchecked_info{ block3 }); // unchecked1
}
unchecked.put (block1->previous (), nano::unchecked_info{ block1 }); // unchecked1
unchecked.put (block1->hash (), nano::unchecked_info{ block1 }); // unchecked2
unchecked.put (block2->previous (), nano::unchecked_info{ block2 }); // unchecked3
unchecked.put (block1->previous (), nano::unchecked_info{ block2 }); // unchecked1
unchecked.put (block1->hash (), nano::unchecked_info{ block2 }); // unchecked2
unchecked.put (block3->previous (), nano::unchecked_info{ block3 });
unchecked.put (block3->hash (), nano::unchecked_info{ block3 }); // unchecked4
unchecked.put (block1->previous (), nano::unchecked_info{ block3 }); // unchecked1
unchecked.flush ();
auto transaction (store->tx_begin_read ());
auto unchecked_count (unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 8);
Expand Down Expand Up @@ -484,34 +483,13 @@ TEST (block_store, empty_bootstrap)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store };
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto transaction (store->tx_begin_read ());
auto [begin, end] = unchecked.full_range (transaction);
ASSERT_EQ (end, begin);
}

TEST (block_store, one_bootstrap)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store };
ASSERT_TRUE (!store->init_error ());
auto block1 (std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5));
auto transaction (store->tx_begin_write ());
unchecked.put (transaction, block1->hash (), nano::unchecked_info{ block1 });
auto [begin, end] = unchecked.full_range (transaction);
ASSERT_NE (end, begin);
auto hash1 (begin->first.key ());
ASSERT_EQ (block1->hash (), hash1);
auto blocks (unchecked.get (transaction, hash1));
ASSERT_EQ (1, blocks.size ());
auto block2 (blocks[0].block);
ASSERT_EQ (*block1, *block2);
++begin;
ASSERT_EQ (end, begin);
}

TEST (block_store, unchecked_begin_search)
{
nano::logger_mt logger;
Expand Down Expand Up @@ -926,24 +904,24 @@ TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort ta
auto path (nano::unique_path ());
nano::logger_mt logger;
nano::mdb_store store (logger, path, nano::dev::constants);
nano::unchecked_map unchecked{ store };
nano::unchecked_map unchecked{ store, false };
auto transaction (store.tx_begin_write ());
ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 1));
ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE, &store.unchecked_handle));
auto send1 (std::make_shared<nano::send_block> (0, 0, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send2 (std::make_shared<nano::send_block> (1, 0, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
ASSERT_NE (send1->hash (), send2->hash ());
unchecked.put (transaction, send1->hash (), nano::unchecked_info{ send1 });
unchecked.put (transaction, send1->hash (), nano::unchecked_info{ send2 });
unchecked.put (send1->hash (), nano::unchecked_info{ send1 });
unchecked.put (send1->hash (), nano::unchecked_info{ send2 });
ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 0));
mdb_dbi_close (store.env, store.unchecked_handle);
ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked_handle));
unchecked.put (transaction, send1->hash (), nano::unchecked_info{ send1 });
unchecked.put (transaction, send1->hash (), nano::unchecked_info{ send2 });
unchecked.put (send1->hash (), nano::unchecked_info{ send1 });
unchecked.put (send1->hash (), nano::unchecked_info{ send2 });
ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 1));
ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked_handle));
unchecked.put (transaction, send1->hash (), nano::unchecked_info{ send1 });
unchecked.put (transaction, send1->hash (), nano::unchecked_info{ send2 });
unchecked.put (send1->hash (), nano::unchecked_info{ send1 });
unchecked.put (send1->hash (), nano::unchecked_info{ send2 });
}

TEST (block_store, state_block)
Expand Down Expand Up @@ -2012,11 +1990,11 @@ TEST (rocksdb_block_store, tombstone_count)
{
nano::logger_mt logger;
auto store = std::make_unique<nano::rocksdb_store> (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store };
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto transaction = store->tx_begin_write ();
auto block1 (std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5));
unchecked.put (transaction, block1->previous (), nano::unchecked_info{ block1 });
unchecked.put (block1->previous (), nano::unchecked_info{ block1 });
ASSERT_EQ (store->tombstone_map.at (nano::tables::unchecked).num_since_last_flush.load (), 0);
unchecked.del (transaction, nano::unchecked_key (block1->previous (), block1->hash ()));
ASSERT_EQ (store->tombstone_map.at (nano::tables::unchecked).num_since_last_flush.load (), 1);
Expand Down
7 changes: 3 additions & 4 deletions nano/core_test/confirmation_height.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,7 @@ TEST (confirmation_height, gap_bootstrap)
// Receive 2 comes in on the live network, however the chain has not been finished so it gets added to unchecked
node1.process_active (receive2);
node1.block_processor.flush ();
node1.unchecked.flush ();

// Confirmation heights should not be updated
{
Expand All @@ -267,12 +268,10 @@ TEST (confirmation_height, gap_bootstrap)
node1.block_processor.add (open1);
node1.block_processor.flush ();

ASSERT_TIMELY (5s, node1.unchecked.count (node1.store.tx_begin_read ()) == 0);
// Confirmation height should be unchanged and unchecked should now be 0
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);

auto transaction = node1.store.tx_begin_read ();
nano::confirmation_height_info confirmation_height_info;
ASSERT_FALSE (node1.store.confirmation_height.get (transaction, nano::dev::genesis_key.pub, confirmation_height_info));
ASSERT_EQ (1, confirmation_height_info.height);
Expand Down
2 changes: 1 addition & 1 deletion nano/core_test/gap_cache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ TEST (gap_cache, two_dependencies)
ASSERT_EQ (2, node1.gap_cache.size ());
node1.block_processor.add (send1, nano::seconds_since_epoch ());
node1.block_processor.flush ();
ASSERT_EQ (0, node1.gap_cache.size ());
ASSERT_TIMELY (5s, node1.gap_cache.size () == 0);
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, send1->hash ()));
ASSERT_TRUE (node1.store.block.exists (transaction, send2->hash ()));
Expand Down
48 changes: 20 additions & 28 deletions nano/core_test/ledger.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2470,6 +2470,7 @@ TEST (ledger, epoch_open_pending)
ASSERT_EQ (nano::signature_verification::valid_epoch, process_result.verified);
node1.block_processor.add (epoch_open);
node1.block_processor.flush ();
node1.unchecked.flush ();
ASSERT_FALSE (node1.ledger.block_or_pruned_exists (epoch_open->hash ()));
// Open block should be inserted into unchecked
auto blocks (node1.unchecked.get (node1.store.tx_begin_read (), nano::hash_or_account (epoch_open->account ()).hash));
Expand All @@ -2488,7 +2489,7 @@ TEST (ledger, epoch_open_pending)
.build_shared ();
node1.block_processor.add (send1);
node1.block_processor.flush ();
ASSERT_TRUE (node1.ledger.block_or_pruned_exists (epoch_open->hash ()));
ASSERT_TIMELY (5s, node1.ledger.block_or_pruned_exists (epoch_open->hash ()));
}

TEST (ledger, block_hash_account_conflict)
Expand Down Expand Up @@ -2655,6 +2656,7 @@ TEST (ledger, unchecked_epoch)
node1.work_generate_blocking (*epoch1);
node1.block_processor.add (epoch1);
node1.block_processor.flush ();
node1.unchecked.flush ();
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.unchecked.count (transaction));
Expand All @@ -2667,9 +2669,9 @@ TEST (ledger, unchecked_epoch)
node1.block_processor.add (send1);
node1.block_processor.add (open1);
node1.block_processor.flush ();
ASSERT_TIMELY (5s, node1.store.block.exists (node1.store.tx_begin_read (), epoch1->hash ()));
{
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, epoch1->hash ()));
auto transaction = node1.store.tx_begin_read ();
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node1.unchecked.count (transaction));
Expand Down Expand Up @@ -2699,6 +2701,7 @@ TEST (ledger, unchecked_epoch_invalid)
node1.block_processor.add (epoch1);
node1.block_processor.add (epoch2);
node1.block_processor.flush ();
node1.unchecked.flush ();
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.unchecked.count (transaction));
Expand All @@ -2712,10 +2715,10 @@ TEST (ledger, unchecked_epoch_invalid)
node1.block_processor.add (send1);
node1.block_processor.add (open1);
node1.block_processor.flush ();
ASSERT_TIMELY (5s, node1.store.block.exists (node1.store.tx_begin_read (), epoch2->hash ()));
{
auto transaction (node1.store.tx_begin_read ());
auto transaction = node1.store.tx_begin_read ();
ASSERT_FALSE (node1.store.block.exists (transaction, epoch1->hash ()));
ASSERT_TRUE (node1.store.block.exists (transaction, epoch2->hash ()));
ASSERT_TRUE (node1.active.empty ());
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
Expand Down Expand Up @@ -2748,6 +2751,7 @@ TEST (ledger, unchecked_open)
node1.block_processor.add (open1);
node1.block_processor.add (open2);
node1.block_processor.flush ();
node1.unchecked.flush ();
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.unchecked.count (transaction));
Expand All @@ -2759,13 +2763,8 @@ TEST (ledger, unchecked_open)
}
node1.block_processor.add (send1);
node1.block_processor.flush ();
{
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, open1->hash ()));
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node1.unchecked.count (transaction));
}
ASSERT_TIMELY (5s, node1.store.block.exists (node1.store.tx_begin_read (), open1->hash ()));
ASSERT_EQ (0, node1.unchecked.count (node1.store.tx_begin_read ()));
}

TEST (ledger, unchecked_receive)
Expand All @@ -2784,37 +2783,30 @@ TEST (ledger, unchecked_receive)
node1.block_processor.add (send1);
node1.block_processor.add (receive1);
node1.block_processor.flush ();
node1.unchecked.flush ();
// Previous block for receive1 is unknown, signature cannot be validated
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 1);
ASSERT_EQ (unchecked_count, node1.unchecked.count (transaction));
ASSERT_EQ (1, node1.unchecked.count (transaction));
auto blocks (node1.unchecked.get (transaction, receive1->previous ()));
ASSERT_EQ (blocks.size (), 1);
ASSERT_EQ (blocks[0].verified, nano::signature_verification::unknown);
}
node1.block_processor.add (open1);
node1.block_processor.flush ();
node1.unchecked.flush ();
ASSERT_TIMELY (5s, node1.unchecked.count (node1.store.tx_begin_read ()) == 1);
// Previous block for receive1 is known, signature was validated
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 1);
ASSERT_EQ (unchecked_count, node1.unchecked.count (transaction));
auto transaction = node1.store.tx_begin_read ();
auto blocks (node1.unchecked.get (transaction, receive1->source ()));
ASSERT_EQ (blocks.size (), 1);
ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid);
}
node1.block_processor.add (send2);
node1.block_processor.flush ();
{
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, receive1->hash ()));
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node1.unchecked.count (transaction));
}
ASSERT_TIMELY (5s, node1.store.block.exists (node1.store.tx_begin_read (), receive1->hash ()));
ASSERT_EQ (0, node1.unchecked.count (node1.store.tx_begin_read ()));
}

TEST (ledger, confirmation_height_not_updated)
Expand Down Expand Up @@ -3623,7 +3615,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
boost::asio::ip::address_v6 address (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1"));
uint16_t port = 100;
nano::mdb_store store (logger, path / "data.ldb", nano::dev::constants);
nano::unchecked_map unchecked{ store };
nano::unchecked_map unchecked{ store, false };
nano::stat stats;
nano::ledger ledger (store, stats, nano::dev::constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
Expand Down Expand Up @@ -3665,7 +3657,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
ASSERT_FALSE (error);

nano::rocksdb_store rocksdb_store (logger, path / "rocksdb", nano::dev::constants);
nano::unchecked_map rocksdb_unchecked{ rocksdb_store };
nano::unchecked_map rocksdb_unchecked{ rocksdb_store, false };
auto rocksdb_transaction (rocksdb_store.tx_begin_read ());

nano::pending_info pending_info;
Expand Down
3 changes: 2 additions & 1 deletion nano/core_test/node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3317,7 +3317,7 @@ TEST (node, block_processor_signatures)
// Invalid signature to unchecked
{
auto transaction (node1.store.tx_begin_write ());
node1.unchecked.put (transaction, send5->previous (), nano::unchecked_info{ send5 });
node1.unchecked.put (send5->previous (), nano::unchecked_info{ send5 });
}
auto receive1 = builder.make_block ()
.account (key1.pub)
Expand Down Expand Up @@ -3669,6 +3669,7 @@ TEST (node, unchecked_cleanup)
ASSERT_FALSE (node.network.publish_filter.apply (bytes.data (), bytes.size ()));
node.process_active (open);
node.block_processor.flush ();
node.unchecked.flush ();
node.config.unchecked_cutoff_time = std::chrono::seconds (2);
{
auto transaction (node.store.tx_begin_read ());
Expand Down
Loading

0 comments on commit 0221ec3

Please sign in to comment.