diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 6a488f083f..a607833f5b 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -876,6 +876,24 @@ TEST (block_store, cemented_count_cache) ASSERT_EQ (1, ledger_cache.cemented_count); } +TEST (block_store, pruned_count) +{ + nano::logger_mt logger; + auto store = nano::make_store (logger, nano::unique_path ()); + ASSERT_TRUE (!store->init_error ()); + { + auto transaction (store->tx_begin_write ()); + nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0); + block.sideband_set ({}); + auto hash1 (block.hash ()); + store->block_put (transaction, hash1, block); + store->pruned_put (transaction, hash1); + } + auto transaction (store->tx_begin_read ()); + ASSERT_EQ (1, store->pruned_count (transaction)); + ASSERT_EQ (1, store->block_count (transaction)); +} + TEST (block_store, sequence_increment) { nano::logger_mt logger; @@ -922,6 +940,26 @@ TEST (block_store, block_random) ASSERT_EQ (*block, *genesis.open); } +TEST (block_store, pruned_random) +{ + nano::logger_mt logger; + auto store = nano::make_store (logger, nano::unique_path ()); + ASSERT_TRUE (!store->init_error ()); + nano::genesis genesis; + nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0); + block.sideband_set ({}); + auto hash1 (block.hash ()); + { + nano::ledger_cache ledger_cache; + auto transaction (store->tx_begin_write ()); + store->initialize (transaction, genesis, ledger_cache); + store->pruned_put (transaction, hash1); + } + auto transaction (store->tx_begin_read ()); + auto random_hash (store->pruned_random (transaction)); + ASSERT_EQ (hash1, random_hash); +} + // Databases need to be dropped in order to convert to dupsort compatible TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort table { @@ -1228,6 +1266,69 @@ TEST (block_store, online_weight) ASSERT_EQ (store->online_weight_end (), store->online_weight_begin (transaction)); } +TEST (block_store, pruned_blocks) +{ + nano::logger_mt logger; + auto store = nano::make_store (logger, nano::unique_path ()); + ASSERT_TRUE (!store->init_error ()); + + nano::keypair key1; + nano::open_block block1 (0, 1, key1.pub, key1.prv, key1.pub, 0); + auto hash1 (block1.hash ()); + { + auto transaction (store->tx_begin_write ()); + + // Confirm that the store is empty + ASSERT_FALSE (store->pruned_exists (transaction, hash1)); + ASSERT_EQ (store->pruned_count (transaction), 0); + + // Add one + store->pruned_put (transaction, hash1); + ASSERT_TRUE (store->pruned_exists (transaction, hash1)); + } + + // Confirm that it can be found + ASSERT_EQ (store->pruned_count (store->tx_begin_read ()), 1); + + // Add another one and check that it (and the existing one) can be found + nano::open_block block2 (1, 2, key1.pub, key1.prv, key1.pub, 0); + block2.sideband_set ({}); + auto hash2 (block2.hash ()); + { + auto transaction (store->tx_begin_write ()); + store->pruned_put (transaction, hash2); + ASSERT_TRUE (store->pruned_exists (transaction, hash2)); // Check new pruned hash is here + ASSERT_TRUE (store->block_or_pruned_exists (transaction, hash2)); + ASSERT_TRUE (store->pruned_exists (transaction, hash1)); // Check first pruned hash is still here + ASSERT_TRUE (store->block_or_pruned_exists (transaction, hash1)); + } + + ASSERT_EQ (store->pruned_count (store->tx_begin_read ()), 2); + + // Delete the first one + { + auto transaction (store->tx_begin_write ()); + store->pruned_del (transaction, hash2); + ASSERT_FALSE (store->pruned_exists (transaction, hash2)); // Confirm it no longer exists + ASSERT_FALSE (store->block_or_pruned_exists (transaction, hash2)); + store->block_put (transaction, hash2, block2); // Add corresponding block + ASSERT_TRUE (store->block_or_pruned_exists (transaction, hash2)); + ASSERT_TRUE (store->pruned_exists (transaction, hash1)); // Check first pruned hash is still here + ASSERT_TRUE (store->block_or_pruned_exists (transaction, hash1)); + } + + ASSERT_EQ (store->pruned_count (store->tx_begin_read ()), 1); + + // Delete original one + { + auto transaction (store->tx_begin_write ()); + store->pruned_del (transaction, hash1); + ASSERT_FALSE (store->pruned_exists (transaction, hash1)); + } + + ASSERT_EQ (store->pruned_count (store->tx_begin_read ()), 0); +} + TEST (mdb_block_store, upgrade_v14_v15) { if (nano::using_rocksdb_in_tests ()) @@ -1748,6 +1849,36 @@ TEST (mdb_block_store, upgrade_v18_v19) ASSERT_LT (18, store.version_get (transaction)); } +TEST (mdb_block_store, upgrade_v19_v20) +{ + if (nano::using_rocksdb_in_tests ()) + { + // Don't test this in rocksdb mode + return; + } + auto path (nano::unique_path ()); + nano::genesis genesis; + nano::logger_mt logger; + nano::stat stats; + { + nano::mdb_store store (logger, path); + nano::ledger ledger (store, stats); + auto transaction (store.tx_begin_write ()); + store.initialize (transaction, genesis, ledger.cache); + // Delete pruned table + ASSERT_FALSE (mdb_drop (store.env.tx (transaction), store.pruned, 1)); + store.version_put (transaction, 19); + } + // Upgrading should create the table + nano::mdb_store store (logger, path); + ASSERT_FALSE (store.init_error ()); + ASSERT_NE (store.pruned, 0); + + // Version should be correct + auto transaction (store.tx_begin_read ()); + ASSERT_LT (19, store.version_get (transaction)); +} + TEST (mdb_block_store, upgrade_backup) { if (nano::using_rocksdb_in_tests ()) diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 90e63b4b83..3fa0c2901c 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -3305,12 +3305,14 @@ TEST (ledger, cache) auto block_count = 1 + 2 * (i + 1) - 2; auto cemented_count = 1 + 2 * (i + 1) - 2; auto genesis_weight = nano::genesis_amount - i; + auto pruned_count = i; auto cache_check = [&, i](nano::ledger_cache const & cache_a) { ASSERT_EQ (account_count, cache_a.account_count); ASSERT_EQ (block_count, cache_a.block_count); ASSERT_EQ (cemented_count, cache_a.cemented_count); ASSERT_EQ (genesis_weight, cache_a.rep_weights.representation_get (nano::genesis_account)); + ASSERT_EQ (pruned_count, cache_a.pruned_count); }; nano::keypair key; @@ -3382,5 +3384,14 @@ TEST (ledger, cache) ++cemented_count; cache_check (ledger.cache); cache_check (nano::ledger (*store, stats).cache); + + { + auto transaction (store->tx_begin_write ()); + ledger.store.pruned_put (transaction, open->hash ()); + ++ledger.cache.pruned_count; + } + ++pruned_count; + cache_check (ledger.cache); + cache_check (nano::ledger (*store, stats).cache); } } diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 2398421254..d7effcb6fc 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -194,6 +194,7 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &meta) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peers) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "pruned", flags, &pruned) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "confirmation_height", flags, &confirmation_height) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts", flags, &accounts_v0) != 0; accounts = accounts_v0; @@ -273,6 +274,8 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool upgrade_v18_to_v19 (transaction_a); needs_vacuuming = true; case 19: + upgrade_v19_to_v20 (transaction_a); + case 20: break; default: logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); @@ -726,6 +729,14 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa logger.always_log ("Finished upgrading all blocks to new blocks database"); } +void nano::mdb_store::upgrade_v19_to_v20 (nano::write_transaction const & transaction_a) +{ + logger.always_log ("Preparing v19 to v20 database upgrade..."); + mdb_dbi_open (env.tx (transaction_a), "pruned", MDB_CREATE, &pruned); + version_put (transaction_a, 20); + logger.always_log ("Finished creating new pruned table"); +} + /** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a) { @@ -844,6 +855,8 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const return meta; case tables::peers: return peers; + case tables::pruned: + return pruned; case tables::confirmation_height: return confirmation_height; default: @@ -875,7 +888,7 @@ bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file) void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a) { // Tables with uint256_union key - std::vector tables = { accounts, blocks, vote, confirmation_height }; + std::vector tables = { accounts, blocks, vote, pruned, confirmation_height }; for (auto const & table : tables) { MDB_dbi temp; diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 47f2ed354d..b2c71f8521 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -174,6 +174,12 @@ class mdb_store : public block_store_partial */ MDB_dbi meta{ 0 }; + /** + * Pruned blocks hashes + * nano::block_hash -> none + */ + MDB_dbi pruned{ 0 }; + /* * Endpoints for peers * nano::endpoint_key -> no_value @@ -231,7 +237,8 @@ class mdb_store : public block_store_partial void upgrade_v15_to_v16 (nano::write_transaction const &); void upgrade_v16_to_v17 (nano::write_transaction const &); void upgrade_v17_to_v18 (nano::write_transaction const &); - void upgrade_v18_to_v19 (nano::write_transaction const & transaction_a); + void upgrade_v18_to_v19 (nano::write_transaction const &); + void upgrade_v19_to_v20 (nano::write_transaction const &); std::shared_ptr block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; nano::mdb_val block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const; diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index b95e9cff42..97376bb8ab 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -98,7 +98,8 @@ std::unordered_map nano::rocksdb_store::create_cf_na { "online_weight", tables::online_weight }, { "meta", tables::meta }, { "peers", tables::peers }, - { "confirmation_height", tables::confirmation_height } }; + { "confirmation_height", tables::confirmation_height }, + { "pruned", tables::pruned } }; debug_assert (map.size () == all_tables ().size () + 1); return map; @@ -257,6 +258,11 @@ rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options (std::string co std::shared_ptr table_factory (rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 2))); cf_options = get_active_cf_options (table_factory, memtable_size_bytes); } + else if (cf_name_a == "pruned") + { + std::shared_ptr table_factory (rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 2))); + cf_options = get_active_cf_options (table_factory, memtable_size_bytes); + } else if (cf_name_a == rocksdb::kDefaultColumnFamilyName) { // Do nothing. @@ -341,6 +347,8 @@ rocksdb::ColumnFamilyHandle * nano::rocksdb_store::table_to_column_family (table return get_handle ("meta"); case tables::peers: return get_handle ("peers"); + case tables::pruned: + return get_handle ("pruned"); case tables::confirmation_height: return get_handle ("confirmation_height"); default: @@ -480,6 +488,11 @@ uint64_t nano::rocksdb_store::count (nano::transaction const & transaction_a, ta { db->GetIntProperty (table_to_column_family (table_a), "rocksdb.estimate-num-keys", &sum); } + // This should be correct at node start, later only cache should be used + else if (table_a == tables::pruned) + { + db->GetIntProperty (table_to_column_family (table_a), "rocksdb.estimate-num-keys", &sum); + } // These should only be used in tests to check database consistency else if (table_a == tables::accounts) { @@ -700,7 +713,7 @@ void nano::rocksdb_store::on_flush (rocksdb::FlushJobInfo const & flush_job_info std::vector nano::rocksdb_store::all_tables () const { - return std::vector{ tables::accounts, tables::blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::unchecked, tables::vote }; + return std::vector{ tables::accounts, tables::blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::unchecked, tables::vote }; } bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_path) diff --git a/nano/node/write_database_queue.hpp b/nano/node/write_database_queue.hpp index 72f5fa30b9..bdd20250d9 100644 --- a/nano/node/write_database_queue.hpp +++ b/nano/node/write_database_queue.hpp @@ -14,6 +14,7 @@ enum class writer { confirmation_height, process_batch, + pruning, testing // Used in tests to emulate a write lock }; diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index f2b3027a28..5a445df545 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -515,6 +515,7 @@ enum class tables online_weight, peers, pending, + pruned, unchecked, vote }; @@ -667,6 +668,14 @@ class block_store virtual void version_put (nano::write_transaction const &, int) = 0; virtual int version_get (nano::transaction const &) const = 0; + virtual void pruned_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) = 0; + virtual void pruned_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) = 0; + virtual bool pruned_exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; + virtual bool block_or_pruned_exists (nano::transaction const &, nano::block_hash const &) = 0; + virtual nano::block_hash pruned_random (nano::transaction const & transaction_a) = 0; + virtual size_t pruned_count (nano::transaction const & transaction_a) const = 0; + virtual void pruned_clear (nano::write_transaction const &) = 0; + virtual void peer_put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; virtual void peer_del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; virtual bool peer_exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const = 0; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index 97e4130b1f..c376c74b53 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -553,6 +553,39 @@ class block_store_partial : public block_store release_assert (success (status)); } + void pruned_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override + { + auto status = put_key (transaction_a, tables::pruned, hash_a); + release_assert (success (status)); + } + + void pruned_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override + { + auto status = del (transaction_a, tables::pruned, hash_a); + release_assert (success (status)); + } + + bool pruned_exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override + { + return exists (transaction_a, tables::pruned, nano::db_val (hash_a)); + } + + bool block_or_pruned_exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override + { + return block_exists (transaction_a, hash_a) || pruned_exists (transaction_a, hash_a); + } + + size_t pruned_count (nano::transaction const & transaction_a) const override + { + return count (transaction_a, tables::pruned); + } + + void pruned_clear (nano::write_transaction const & transaction_a) override + { + auto status = drop (transaction_a, tables::pruned); + release_assert (success (status)); + } + void peer_put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override { auto status = put_key (transaction_a, tables::peers, endpoint_a); @@ -610,6 +643,19 @@ class block_store_partial : public block_store return existing->second; } + nano::block_hash pruned_random (nano::transaction const & transaction_a) override + { + nano::block_hash random_hash; + nano::random_pool::generate_block (random_hash.bytes.data (), random_hash.bytes.size ()); + auto existing = make_iterator> (transaction_a, tables::pruned, nano::db_val (random_hash)); + auto end (nano::store_iterator> (nullptr)); + if (existing == end) + { + existing = make_iterator> (transaction_a, tables::pruned); + } + return existing != end ? existing->first : 0; + } + uint64_t confirmation_height_count (nano::transaction const & transaction_a) override { return count (transaction_a, tables::confirmation_height); @@ -736,7 +782,7 @@ class block_store_partial : public block_store nano::network_params network_params; std::unordered_map> vote_cache_l1; std::unordered_map> vote_cache_l2; - int const version{ 19 }; + int const version{ 20 }; template nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a) const diff --git a/nano/secure/common.hpp b/nano/secure/common.hpp index 214db04302..617509a152 100644 --- a/nano/secure/common.hpp +++ b/nano/secure/common.hpp @@ -505,6 +505,7 @@ class ledger_cache nano::rep_weights rep_weights; std::atomic cemented_count{ 0 }; std::atomic block_count{ 0 }; + std::atomic pruned_count{ 0 }; std::atomic account_count{ 0 }; std::atomic epoch_2_started{ false }; }; diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index e5d53a2a5b..b3b4b8bdd3 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -747,8 +747,6 @@ epoch_2_started_cb (epoch_2_started_cb_a) void nano::ledger::initialize (nano::generate_cache const & generate_cache_a) { - auto transaction = store.tx_begin_read (); - if (generate_cache_a.reps || generate_cache_a.account_count || generate_cache_a.epoch_2 || generate_cache_a.block_count) { store.latest_for_each_par ( @@ -787,6 +785,9 @@ void nano::ledger::initialize (nano::generate_cache const & generate_cache_a) this->cache.cemented_count += cemented_count_l; }); } + + auto transaction (store.tx_begin_read ()); + cache.pruned_count = store.pruned_count (transaction); } // Balance for account containing hash