diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 6d224bf491..16076f918d 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -17,9 +17,11 @@ namespace { -void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction const & transaction_a, nano::account const & account_a); -void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height); +void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction const & transaction_a, nano::account const & account_a, nano::block_hash const & rep_block); +void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height, nano::block_hash const & rep_block); void modify_genesis_account_info_to_v5 (nano::mdb_store & store, nano::transaction const & transaction_a); +void write_sideband_v12 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block & block_a, nano::block_hash const & successor_a, MDB_dbi db_a); +void write_sideband_v14 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a, MDB_dbi db_a); } TEST (block_store, construction) @@ -65,7 +67,7 @@ TEST (block_store, add_item) auto latest1 (store->block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); ASSERT_FALSE (store->block_exists (transaction, hash1)); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, hash1, block, sideband); auto latest2 (store->block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); @@ -84,7 +86,7 @@ TEST (block_store, clear_successor) ASSERT_TRUE (!store->init_error ()); nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, block1.hash (), block1, sideband); nano::open_block block2 (0, 2, 0, nano::keypair ().prv, 0, 0); store->block_put (transaction, block2.hash (), block2, sideband); @@ -111,7 +113,7 @@ TEST (block_store, add_nonempty_block) auto transaction (store->tx_begin_write ()); auto latest1 (store->block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, hash1, block, sideband); auto latest2 (store->block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); @@ -136,9 +138,9 @@ TEST (block_store, add_two_items) block2.signature = nano::sign_message (key1.prv, key1.pub, hash2); auto latest2 (store->block_get (transaction, hash2)); ASSERT_EQ (nullptr, latest2); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, hash1, block, sideband); - nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, hash2, block2, sideband2); auto latest3 (store->block_get (transaction, hash1)); ASSERT_NE (nullptr, latest3); @@ -158,13 +160,13 @@ TEST (block_store, add_receive) nano::keypair key2; nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, block1.hash (), block1, sideband1); nano::receive_block block (block1.hash (), 1, nano::keypair ().prv, 2, 3); nano::block_hash hash1 (block.hash ()); auto latest1 (store->block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); - nano::block_sideband sideband (nano::block_type::receive, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::receive, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, hash1, block, sideband); auto latest2 (store->block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); @@ -407,7 +409,7 @@ TEST (block_store, one_block) ASSERT_TRUE (!store->init_error ()); nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, block1.hash (), block1, sideband); ASSERT_TRUE (store->block_exists (transaction, block1.hash ())); } @@ -508,12 +510,12 @@ TEST (block_store, two_block) hashes.push_back (block1.hash ()); blocks.push_back (block1); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, hashes[0], block1, sideband1); nano::open_block block2 (0, 1, 2, nano::keypair ().prv, 0, 0); hashes.push_back (block2.hash ()); blocks.push_back (block2); - nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, hashes[1], block2, sideband2); ASSERT_TRUE (store->block_exists (transaction, block1.hash ())); ASSERT_TRUE (store->block_exists (transaction, block2.hash ())); @@ -584,14 +586,12 @@ TEST (block_store, latest_find) ASSERT_EQ (second, find3); } -#if !NANO_ROCKSDB -TEST (block_store, bad_path) +TEST (mdb_block_store, bad_path) { nano::logger_mt logger; - auto store = nano::make_store (logger, boost::filesystem::path ("///")); - ASSERT_TRUE (store->init_error ()); + nano::mdb_store store (logger, boost::filesystem::path ("///")); + ASSERT_TRUE (store.init_error ()); } -#endif TEST (block_store, DISABLED_already_open) // File can be shared { @@ -699,9 +699,9 @@ TEST (block_store, block_replace) nano::send_block send1 (0, 0, 0, nano::keypair ().prv, 0, 1); nano::send_block send2 (0, 0, 0, nano::keypair ().prv, 0, 2); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband1 (nano::block_type::send, 0, 0, 0, 0, 0); + nano::block_sideband sideband1 (nano::block_type::send, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, 0, send1, sideband1); - nano::block_sideband sideband2 (nano::block_type::send, 0, 0, 0, 0, 0); + nano::block_sideband sideband2 (nano::block_type::send, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, 0, send2, sideband2); auto block3 (store->block_get (transaction, 0)); ASSERT_NE (nullptr, block3); @@ -718,7 +718,7 @@ TEST (block_store, block_count) ASSERT_EQ (0, store->block_count (transaction).sum ()); nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0); auto hash1 (block.hash ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, hash1, block, sideband); } auto transaction (store->tx_begin_read ()); @@ -893,6 +893,9 @@ TEST (mdb_block_store, upgrade_v4_v5) store.block_successor_clear (transaction, info.head); ASSERT_TRUE (store.block_successor (transaction, genesis_hash).is_zero ()); modify_genesis_account_info_to_v5 (store, transaction); + // The pending send needs to be the correct version + auto status (mdb_put (store.env.tx (transaction), store.pending_v0, nano::mdb_val (nano::pending_key (key0.pub, block0.hash ())), nano::mdb_val (nano::pending_info_v14 (nano::genesis_account, nano::Gxrb_ratio, nano::epoch::epoch_0)), 0)); + ASSERT_EQ (status, MDB_SUCCESS); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -959,7 +962,7 @@ TEST (mdb_block_store, upgrade_v6_v7) std::atomic block_count_cache{ 0 }; store.initialize (transaction, genesis, rep_weights, cemented_count, block_count_cache); store.version_put (transaction, 6); - modify_account_info_to_v13 (store, transaction, nano::genesis_account); + modify_account_info_to_v13 (store, transaction, nano::genesis_account, genesis.open->hash ()); auto send1 (std::make_shared (0, 0, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0)); store.unchecked_put (transaction, send1->hash (), send1); store.flush (transaction); @@ -1123,7 +1126,7 @@ TEST (block_store, state_block) std::atomic block_count_cache{ 0 }; store->initialize (transaction, genesis, rep_weights, cemented_count, block_count_cache); ASSERT_EQ (nano::block_type::state, block1.type ()); - nano::block_sideband sideband1 (nano::block_type::state, 0, 0, 0, 0, 0); + nano::block_sideband sideband1 (nano::block_type::state, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (transaction, block1.hash (), block1, sideband1); ASSERT_TRUE (store->block_exists (transaction, block1.hash ())); auto block2 (store->block_get (transaction, block1.hash ())); @@ -1133,36 +1136,13 @@ TEST (block_store, state_block) { auto transaction (store->tx_begin_write ()); auto count (store->block_count (transaction)); - ASSERT_EQ (1, count.state_v0); - ASSERT_EQ (0, count.state_v1); + ASSERT_EQ (1, count.state); store->block_del (transaction, block1.hash ()); ASSERT_FALSE (store->block_exists (transaction, block1.hash ())); } auto transaction (store->tx_begin_read ()); auto count2 (store->block_count (transaction)); - ASSERT_EQ (0, count2.state_v0); - ASSERT_EQ (0, count2.state_v1); -} - -namespace -{ -void write_legacy_sideband (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block & block_a, nano::block_hash const & successor_a, MDB_dbi db_a) -{ - std::vector vector; - { - nano::vectorstream stream (vector); - block_a.serialize (stream); - nano::write (stream, successor_a); - } - MDB_val val{ vector.size (), vector.data () }; - auto hash (block_a.hash ()); - auto status2 (mdb_put (store_a.env.tx (transaction_a), db_a, nano::mdb_val (hash), &val, 0)); - ASSERT_EQ (0, status2); - nano::block_sideband sideband; - auto block2 (store_a.block_get (transaction_a, block_a.hash (), &sideband)); - ASSERT_NE (nullptr, block2); - ASSERT_EQ (0, sideband.height); -}; + ASSERT_EQ (0, count2.state); } TEST (mdb_block_store, upgrade_sideband_genesis) @@ -1179,15 +1159,17 @@ TEST (mdb_block_store, upgrade_sideband_genesis) std::atomic cemented_count{ 0 }; std::atomic block_count_cache{ 0 }; store.initialize (transaction, genesis, rep_weights, cemented_count, block_count_cache); - modify_account_info_to_v13 (store, transaction, nano::genesis_account); + modify_account_info_to_v13 (store, transaction, nano::genesis_account, genesis.open->hash ()); nano::block_sideband sideband; auto genesis_block (store.block_get (transaction, genesis.hash (), &sideband)); ASSERT_NE (nullptr, genesis_block); ASSERT_EQ (1, sideband.height); - write_legacy_sideband (store, transaction, *genesis_block, 0, store.open_blocks); - auto genesis_block2 (store.block_get (transaction, genesis.hash (), &sideband)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1)); + write_sideband_v12 (store, transaction, *genesis_block, 0, store.open_blocks); + nano::block_sideband_v14 sideband1; + auto genesis_block2 (store.block_get_v14 (transaction, genesis.hash (), &sideband1)); ASSERT_NE (nullptr, genesis_block); - ASSERT_EQ (0, sideband.height); + ASSERT_EQ (0, sideband1.height); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1218,9 +1200,14 @@ TEST (mdb_block_store, upgrade_sideband_two_blocks) nano::state_block block (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (genesis.hash ())); hash2 = block.hash (); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block).code); - write_legacy_sideband (store, transaction, *genesis.open, hash2, store.open_blocks); - write_legacy_sideband (store, transaction, block, 0, store.state_blocks_v0); - modify_account_info_to_v13 (store, transaction, nano::genesis_account); + store.block_del (transaction, hash2); + mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1); + mdb_dbi_open (store.env.tx (transaction), "state", MDB_CREATE, &store.state_blocks_v0); + write_sideband_v12 (store, transaction, *genesis.open, hash2, store.open_blocks); + write_sideband_v12 (store, transaction, block, 0, store.state_blocks_v0); + modify_account_info_to_v13 (store, transaction, nano::genesis_account, hash2); + auto status (mdb_put (store.env.tx (transaction), store.pending_v0, nano::mdb_val (nano::pending_key (nano::test_genesis_key.pub, block.hash ())), nano::mdb_val (nano::pending_info_v14 (nano::genesis_account, nano::Gxrb_ratio, nano::epoch::epoch_0)), 0)); + ASSERT_EQ (status, MDB_SUCCESS); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1259,11 +1246,15 @@ TEST (mdb_block_store, upgrade_sideband_two_accounts) nano::state_block block2 (key.pub, 0, nano::test_genesis_key.pub, nano::Gxrb_ratio, hash2, key.prv, key.pub, pool.generate (key.pub)); hash3 = block2.hash (); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block2).code); - write_legacy_sideband (store, transaction, *genesis.open, hash2, store.open_blocks); - write_legacy_sideband (store, transaction, block1, 0, store.state_blocks_v0); - write_legacy_sideband (store, transaction, block2, 0, store.state_blocks_v0); - modify_account_info_to_v13 (store, transaction, nano::genesis_account); - modify_account_info_to_v13 (store, transaction, block2.account ()); + store.block_del (transaction, hash2); + store.block_del (transaction, hash3); + mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1); + mdb_dbi_open (store.env.tx (transaction), "state", MDB_CREATE, &store.state_blocks_v0); + write_sideband_v12 (store, transaction, *genesis.open, hash2, store.open_blocks); + write_sideband_v12 (store, transaction, block1, 0, store.state_blocks_v0); + write_sideband_v12 (store, transaction, block2, 0, store.state_blocks_v0); + modify_account_info_to_v13 (store, transaction, nano::genesis_account, hash2); + modify_account_info_to_v13 (store, transaction, block2.account (), hash3); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1295,7 +1286,8 @@ TEST (mdb_block_store, insert_after_legacy) auto transaction (store.tx_begin_write ()); store.version_put (transaction, 11); store.initialize (transaction, genesis, ledger.rep_weights, ledger.cemented_count, ledger.block_count_cache); - write_legacy_sideband (store, transaction, *genesis.open, 0, store.open_blocks); + mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1); + write_sideband_v12 (store, transaction, *genesis.open, 0, store.open_blocks); nano::work_pool pool (std::numeric_limits::max ()); nano::state_block block (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (genesis.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block).code); @@ -1313,7 +1305,8 @@ TEST (mdb_block_store, legacy_account_computed) auto transaction (store.tx_begin_write ()); store.initialize (transaction, genesis, ledger.rep_weights, ledger.cemented_count, ledger.block_count_cache); store.version_put (transaction, 11); - write_legacy_sideband (store, transaction, *genesis.open, 0, store.open_blocks); + mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1); + write_sideband_v12 (store, transaction, *genesis.open, 0, store.open_blocks); ASSERT_EQ (nano::genesis_account, ledger.account (transaction, genesis.hash ())); } @@ -1335,11 +1328,21 @@ TEST (mdb_block_store, upgrade_sideband_epoch) store.initialize (transaction, genesis, ledger.rep_weights, ledger.cemented_count, ledger.block_count_cache); nano::state_block block1 (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount, ledger.link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (genesis.hash ())); hash2 = block1.hash (); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1)); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block1).code); ASSERT_EQ (nano::epoch::epoch_1, store.block_version (transaction, hash2)); - write_legacy_sideband (store, transaction, *genesis.open, hash2, store.open_blocks); - write_legacy_sideband (store, transaction, block1, 0, store.state_blocks_v1); - modify_account_info_to_v13 (store, transaction, nano::genesis_account); + store.block_del (transaction, hash2); + store.block_del (transaction, genesis.open->hash ()); + write_sideband_v12 (store, transaction, *genesis.open, hash2, store.open_blocks); + write_sideband_v12 (store, transaction, block1, 0, store.state_blocks_v1); + + nano::mdb_val value; + ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.state_blocks_v1, nano::mdb_val (hash2), value)); + ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.open_blocks, nano::mdb_val (genesis.open->hash ()), value)); + + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, &store.accounts_v1)); + modify_account_info_to_v13 (store, transaction, nano::genesis_account, hash2); + store.account_del (transaction, nano::genesis_account); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1575,7 +1578,7 @@ TEST (mdb_block_store, upgrade_v13_v14) ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); ASSERT_EQ (confirmation_height, 1); store.version_put (transaction, 13); - modify_account_info_to_v13 (store, transaction, nano::genesis_account); + modify_account_info_to_v13 (store, transaction, nano::genesis_account, genesis.open->hash ()); // This should fail as sizes are no longer correct for account_info_v14 nano::mdb_val value; @@ -1614,31 +1617,55 @@ TEST (mdb_block_store, upgrade_v14_v15) { // Extract confirmation height to a separate database auto path (nano::unique_path ()); + nano::genesis genesis; + nano::network_params network_params; + nano::work_pool pool (std::numeric_limits::max ()); + nano::send_block send (genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (genesis.hash ())); + nano::state_block epoch (nano::test_genesis_key.pub, send.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, network_params.ledger.epochs.link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (send.hash ())); + nano::state_block state_send (nano::test_genesis_key.pub, epoch.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio * 2, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (epoch.hash ())); { nano::logger_mt logger; - nano::genesis genesis; nano::mdb_store store (logger, path); + nano::stat stats; + nano::ledger ledger (store, stats); auto transaction (store.tx_begin_write ()); - nano::rep_weights rep_weights; - std::atomic cemented_count{ 0 }; - std::atomic block_count_cache{ 0 }; - store.initialize (transaction, genesis, rep_weights, cemented_count, block_count_cache); + store.initialize (transaction, genesis, ledger.rep_weights, ledger.cemented_count, ledger.block_count_cache); nano::account_info account_info; ASSERT_FALSE (store.account_get (transaction, nano::genesis_account, account_info)); uint64_t confirmation_height; ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); ASSERT_EQ (confirmation_height, 1); - + // These databases get remove after an upgrade, so readd them + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, &store.accounts_v1)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "pending_v1", MDB_CREATE, &store.pending_v1)); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send).code); // Lower the database to the previous version store.version_put (transaction, 14); store.confirmation_height_del (transaction, nano::genesis_account); - modify_account_info_to_v14 (store, transaction, nano::genesis_account, confirmation_height); + modify_account_info_to_v14 (store, transaction, nano::genesis_account, confirmation_height, state_send.hash ()); + + store.pending_del (transaction, nano::pending_key (nano::genesis_account, state_send.hash ())); + + write_sideband_v14 (store, transaction, state_send, store.state_blocks_v1); + write_sideband_v14 (store, transaction, epoch, store.state_blocks_v1); + + // Remove from state table + store.block_del (transaction, state_send.hash ()); + store.block_del (transaction, epoch.hash ()); + + // Turn pending into v14 + ASSERT_FALSE (mdb_put (store.env.tx (transaction), store.pending_v0, nano::mdb_val (nano::pending_key (nano::test_genesis_key.pub, send.hash ())), nano::mdb_val (nano::pending_info_v14 (nano::genesis_account, nano::Gxrb_ratio, nano::epoch::epoch_0)), 0)); + ASSERT_FALSE (mdb_put (store.env.tx (transaction), store.pending_v1, nano::mdb_val (nano::pending_key (nano::test_genesis_key.pub, state_send.hash ())), nano::mdb_val (nano::pending_info_v14 (nano::genesis_account, nano::Gxrb_ratio, nano::epoch::epoch_1)), 0)); // This should fail as sizes are no longer correct for account_info nano::mdb_val value; - ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (nano::genesis_account), value)); + ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.accounts_v1, nano::mdb_val (nano::genesis_account), value)); nano::account_info info; ASSERT_NE (value.size (), info.db_size ()); + store.account_del (transaction, nano::genesis_account); // Confirmation height for the account should be deleted ASSERT_TRUE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); @@ -1653,8 +1680,8 @@ TEST (mdb_block_store, upgrade_v14_v15) // Size of account_info should now equal that set in db nano::mdb_val value; - ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (nano::genesis_account), value)); - nano::account_info info; + ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.accounts, nano::mdb_val (nano::genesis_account), value)); + nano::account_info info (value); ASSERT_EQ (value.size (), info.db_size ()); // Confirmation height should exist @@ -1667,6 +1694,30 @@ TEST (mdb_block_store, upgrade_v14_v15) ASSERT_NE (error_get_representation, MDB_SUCCESS); ASSERT_EQ (store.representation, 0); + // accounts_v1, state_blocks_v1 & pending_v1 tables should be deleted + auto error_get_accounts_v1 (mdb_get (store.env.tx (transaction), store.accounts_v1, nano::mdb_val (nano::genesis_account), value)); + ASSERT_NE (error_get_accounts_v1, MDB_SUCCESS); + auto error_get_pending_v1 (mdb_get (store.env.tx (transaction), store.pending_v1, nano::mdb_val (nano::pending_key (nano::test_genesis_key.pub, state_send.hash ())), value)); + ASSERT_NE (error_get_pending_v1, MDB_SUCCESS); + auto error_get_state_v1 (mdb_get (store.env.tx (transaction), store.state_blocks_v1, nano::mdb_val (state_send.hash ()), value)); + ASSERT_NE (error_get_state_v1, MDB_SUCCESS); + + // Check that the epochs are set correctly for the sideband, accounts and pending entries + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_send.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.epoch, nano::epoch::epoch_1); + block = store.block_get (transaction, send.hash (), &sideband); + ASSERT_NE (block, nullptr); + nano::block_sideband sideband1; + ASSERT_EQ (sideband1.epoch, nano::epoch::epoch_0); + ASSERT_EQ (info.epoch (), nano::epoch::epoch_1); + nano::pending_info pending_info; + store.pending_get (transaction, nano::pending_key (nano::test_genesis_key.pub, send.hash ()), pending_info); + ASSERT_EQ (pending_info.epoch, nano::epoch::epoch_0); + store.pending_get (transaction, nano::pending_key (nano::test_genesis_key.pub, state_send.hash ()), pending_info); + ASSERT_EQ (pending_info.epoch, nano::epoch::epoch_1); + // Version should be correct ASSERT_LT (14, store.version_get (transaction)); } @@ -1748,7 +1799,7 @@ TEST (block_store, confirmation_height) } // Upgrade many accounts and check they all have a confirmation height of 0 (except genesis which should have 1) -TEST (block_store, upgrade_confirmation_height_many) +TEST (mdb_block_store, upgrade_confirmation_height_many) { auto error (false); nano::genesis genesis; @@ -1765,21 +1816,21 @@ TEST (block_store, upgrade_confirmation_height_many) std::atomic cemented_count{ 0 }; std::atomic block_count_cache{ 0 }; store.initialize (transaction, genesis, rep_weights, cemented_count, block_count_cache); - modify_account_info_to_v13 (store, transaction, nano::genesis_account); + modify_account_info_to_v13 (store, transaction, nano::genesis_account, genesis.open->hash ()); // Add many accounts for (auto i = 0; i < total_num_accounts - 1; ++i) { nano::account account (i); nano::open_block open (1, nano::genesis_account, 3, nullptr); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store.block_put (transaction, open.hash (), open, sideband); - nano::account_info_v13 account_info_v13 (open.hash (), open.hash (), open.hash (), 3, 4, 1, nano::epoch::epoch_1); - auto status (mdb_put (store.env.tx (transaction), store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0)); + nano::account_info_v13 account_info_v13 (open.hash (), open.hash (), open.hash (), 3, 4, 1, nano::epoch::epoch_0); + auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0)); ASSERT_EQ (status, 0); } - ASSERT_EQ (store.account_count (transaction), total_num_accounts); + ASSERT_EQ (store.count (transaction, store.accounts_v0), total_num_accounts); } // Loop over them all and confirm they all have the correct confirmation heights @@ -1841,7 +1892,7 @@ TEST (block_store, reset_renew_existing_transaction) // Write the block { auto write_transaction (store->tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store->block_put (write_transaction, hash1, block, sideband); } @@ -1881,27 +1932,59 @@ TEST (block_store, rocksdb_force_test_env_variable) namespace { +void write_sideband_v12 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block & block_a, nano::block_hash const & successor_a, MDB_dbi db_a) +{ + std::vector vector; + { + nano::vectorstream stream (vector); + block_a.serialize (stream); + nano::write (stream, successor_a); + } + MDB_val val{ vector.size (), vector.data () }; + auto hash (block_a.hash ()); + auto status (mdb_put (store_a.env.tx (transaction_a), db_a, nano::mdb_val (hash), &val, 0)); + ASSERT_EQ (0, status); + nano::block_sideband_v14 sideband_v14; + auto block (store_a.block_get_v14 (transaction_a, hash, &sideband_v14)); + ASSERT_NE (nullptr, block); + ASSERT_EQ (0, sideband_v14.height); +}; + +void write_sideband_v14 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a, MDB_dbi db_a) +{ + nano::block_sideband sideband; + auto block = store_a.block_get (transaction_a, block_a.hash (), &sideband); + ASSERT_NE (block, nullptr); + + nano::block_sideband_v14 sideband_v14 (sideband.type, sideband.account, sideband.successor, sideband.balance, sideband.timestamp, sideband.height); + std::vector data; + { + nano::vectorstream stream (data); + block_a.serialize (stream); + sideband_v14.serialize (stream); + } + + MDB_val val{ data.size (), data.data () }; + ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), sideband.epoch == nano::epoch::epoch_0 ? store_a.state_blocks_v0 : store_a.state_blocks_v1, nano::mdb_val (block_a.hash ()), &val, 0)); +} + // These functions take the latest account_info and create a legacy one so that upgrade tests can be emulated more easily. -void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account) +void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, nano::block_hash const & rep_block) { nano::account_info info; ASSERT_FALSE (store.account_get (transaction, account, info)); - nano::representative_visitor visitor (transaction, store); - visitor.compute (info.head); - nano::account_info_v13 account_info_v13 (info.head, visitor.result, info.open_block, info.balance, info.modified, info.block_count, info.epoch ()); - auto status (mdb_put (store.env.tx (transaction), store.get_account_db (info.epoch ()) == nano::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0)); + nano::account_info_v13 account_info_v13 (info.head, rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch ()); + auto status (mdb_put (store.env.tx (transaction), (info.epoch () == nano::epoch::epoch_0) ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0)); (void)status; assert (status == 0); } -void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height) +void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height, nano::block_hash const & rep_block) { nano::account_info info; ASSERT_FALSE (store.account_get (transaction, account, info)); - nano::representative_visitor visitor (transaction, store); - visitor.compute (info.head); - nano::account_info_v14 account_info_v14 (info.head, visitor.result, info.open_block, info.balance, info.modified, info.block_count, confirmation_height, info.epoch ()); - auto status (mdb_put (store.env.tx (transaction), store.get_account_db (info.epoch ()) == nano::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0)); + nano::account_info_v14 account_info_v14 (info.head, rep_block, info.open_block, info.balance, info.modified, info.block_count, confirmation_height, info.epoch ()); + auto status (mdb_put (store.env.tx (transaction), info.epoch () == nano::epoch::epoch_0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0)); (void)status; assert (status == 0); } diff --git a/nano/core_test/epochs.cpp b/nano/core_test/epochs.cpp index 5e96d517f9..96a9c3292d 100644 --- a/nano/core_test/epochs.cpp +++ b/nano/core_test/epochs.cpp @@ -6,11 +6,37 @@ TEST (epochs, is_epoch_link) { nano::epochs epochs; + // Test epoch 1 nano::keypair key1; + auto link1 = 42; + auto link2 = 43; + ASSERT_FALSE (epochs.is_epoch_link (link1)); + ASSERT_FALSE (epochs.is_epoch_link (link2)); + epochs.add (nano::epoch::epoch_1, key1.pub, link1); + ASSERT_TRUE (epochs.is_epoch_link (link1)); + ASSERT_FALSE (epochs.is_epoch_link (link2)); + ASSERT_EQ (key1.pub, epochs.signer (nano::epoch::epoch_1)); + ASSERT_EQ (epochs.epoch (link1), nano::epoch::epoch_1); + + // Test epoch 2 nano::keypair key2; - ASSERT_FALSE (epochs.is_epoch_link (42)); - ASSERT_FALSE (epochs.is_epoch_link (43)); - epochs.add (nano::epoch::epoch_1, key1.pub, 42); - ASSERT_TRUE (epochs.is_epoch_link (42)); - ASSERT_FALSE (epochs.is_epoch_link (43)); + epochs.add (nano::epoch::epoch_2, key2.pub, link2); + ASSERT_TRUE (epochs.is_epoch_link (link2)); + ASSERT_EQ (key2.pub, epochs.signer (nano::epoch::epoch_2)); + ASSERT_EQ (nano::uint256_union (link1), epochs.link (nano::epoch::epoch_1)); + ASSERT_EQ (nano::uint256_union (link2), epochs.link (nano::epoch::epoch_2)); + ASSERT_EQ (epochs.epoch (link2), nano::epoch::epoch_2); +} + +TEST (epochs, is_sequential) +{ + ASSERT_TRUE (nano::epochs::is_sequential (nano::epoch::epoch_0, nano::epoch::epoch_1)); + ASSERT_TRUE (nano::epochs::is_sequential (nano::epoch::epoch_1, nano::epoch::epoch_2)); + + ASSERT_FALSE (nano::epochs::is_sequential (nano::epoch::epoch_0, nano::epoch::epoch_2)); + ASSERT_FALSE (nano::epochs::is_sequential (nano::epoch::epoch_0, nano::epoch::invalid)); + ASSERT_FALSE (nano::epochs::is_sequential (nano::epoch::unspecified, nano::epoch::epoch_1)); + ASSERT_FALSE (nano::epochs::is_sequential (nano::epoch::epoch_1, nano::epoch::epoch_0)); + ASSERT_FALSE (nano::epochs::is_sequential (nano::epoch::epoch_2, nano::epoch::epoch_0)); + ASSERT_FALSE (nano::epochs::is_sequential (nano::epoch::epoch_2, nano::epoch::epoch_2)); } diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 5723fdf4a2..9b3bcdc968 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -8,17 +8,15 @@ using namespace std::chrono_literals; -#if !NANO_ROCKSDB // Init returns an error if it can't open files at the path TEST (ledger, store_error) { nano::logger_mt logger; - auto store = nano::make_store (logger, boost::filesystem::path ("///")); - ASSERT_TRUE (store->init_error ()); + nano::mdb_store store (logger, boost::filesystem::path ("///")); + ASSERT_TRUE (store.init_error ()); nano::stat stats; - nano::ledger ledger (*store, stats); + nano::ledger ledger (store, stats); } -#endif // Ledger can be initialized and returns a basic query for an empty account TEST (ledger, empty) @@ -2286,7 +2284,7 @@ TEST (ledger, state_receive_change_rollback) ASSERT_EQ (0, ledger.weight (rep.pub)); } -TEST (ledger, epoch_blocks_general) +TEST (ledger, epoch_blocks_v1_general) { nano::logger_mt logger; auto store = nano::make_store (logger, nano::unique_path ()); @@ -2332,6 +2330,60 @@ TEST (ledger, epoch_blocks_general) ASSERT_EQ (nano::Gxrb_ratio, ledger.weight (destination.pub)); } +TEST (ledger, epoch_blocks_v2_general) +{ + nano::logger_mt logger; + auto store = nano::make_store (logger, nano::unique_path ()); + ASSERT_TRUE (!store->init_error ()); + nano::stat stats; + nano::ledger ledger (*store, stats); + nano::genesis genesis; + auto transaction (store->tx_begin_write ()); + store->initialize (transaction, genesis, ledger.rep_weights, ledger.cemented_count, ledger.block_count_cache); + nano::work_pool pool (std::numeric_limits::max ()); + nano::keypair destination; + nano::state_block epoch1 (nano::genesis_account, genesis.hash (), nano::genesis_account, nano::genesis_amount, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (genesis.hash ())); + // Trying to upgrade from epoch 0 to epoch 2. It is a requirement epoch upgrades are sequential unless the account is unopened + ASSERT_EQ (nano::process_result::block_position, ledger.process (transaction, epoch1).code); + // Set it to the first epoch and it should now succeed + epoch1 = nano::state_block (nano::genesis_account, genesis.hash (), nano::genesis_account, nano::genesis_amount, ledger.link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, epoch1.work); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch1).code); + nano::state_block epoch2 (nano::genesis_account, epoch1.hash (), nano::genesis_account, nano::genesis_amount, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (epoch1.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch2).code); + nano::state_block epoch3 (nano::genesis_account, epoch2.hash (), nano::genesis_account, nano::genesis_amount, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (epoch2.hash ())); + ASSERT_EQ (nano::process_result::block_position, ledger.process (transaction, epoch3).code); + nano::account_info genesis_info; + ASSERT_FALSE (ledger.store.account_get (transaction, nano::genesis_account, genesis_info)); + ASSERT_EQ (genesis_info.epoch (), nano::epoch::epoch_2); + ASSERT_FALSE (ledger.rollback (transaction, epoch1.hash ())); + ASSERT_FALSE (ledger.store.account_get (transaction, nano::genesis_account, genesis_info)); + ASSERT_EQ (genesis_info.epoch (), nano::epoch::epoch_0); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch1).code); + ASSERT_FALSE (ledger.store.account_get (transaction, nano::genesis_account, genesis_info)); + ASSERT_EQ (genesis_info.epoch (), nano::epoch::epoch_1); + nano::change_block change1 (epoch1.hash (), nano::genesis_account, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (epoch1.hash ())); + ASSERT_EQ (nano::process_result::block_position, ledger.process (transaction, change1).code); + nano::state_block send1 (nano::genesis_account, epoch1.hash (), nano::genesis_account, nano::genesis_amount - nano::Gxrb_ratio, destination.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (epoch1.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send1).code); + nano::open_block open1 (send1.hash (), nano::genesis_account, destination.pub, destination.prv, destination.pub, pool.generate (destination.pub)); + ASSERT_EQ (nano::process_result::unreceivable, ledger.process (transaction, open1).code); + nano::state_block epoch4 (destination.pub, 0, 0, 0, ledger.link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (destination.pub)); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch4).code); + nano::state_block epoch5 (destination.pub, epoch4.hash (), nano::genesis_account, 0, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (epoch4.hash ())); + ASSERT_EQ (nano::process_result::representative_mismatch, ledger.process (transaction, epoch5).code); + nano::state_block epoch6 (destination.pub, epoch4.hash (), 0, 0, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (epoch4.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch6).code); + nano::receive_block receive1 (epoch6.hash (), send1.hash (), destination.prv, destination.pub, pool.generate (epoch6.hash ())); + ASSERT_EQ (nano::process_result::block_position, ledger.process (transaction, receive1).code); + nano::state_block receive2 (destination.pub, epoch6.hash (), destination.pub, nano::Gxrb_ratio, send1.hash (), destination.prv, destination.pub, pool.generate (epoch6.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive2).code); + ASSERT_EQ (0, ledger.balance (transaction, epoch6.hash ())); + ASSERT_EQ (nano::Gxrb_ratio, ledger.balance (transaction, receive2.hash ())); + ASSERT_EQ (nano::Gxrb_ratio, ledger.amount (transaction, receive2.hash ())); + ASSERT_EQ (nano::genesis_amount - nano::Gxrb_ratio, ledger.weight (nano::genesis_account)); + ASSERT_EQ (nano::Gxrb_ratio, ledger.weight (destination.pub)); +} + TEST (ledger, epoch_blocks_receive_upgrade) { nano::logger_mt logger; @@ -2370,6 +2422,29 @@ TEST (ledger, epoch_blocks_receive_upgrade) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send3).code); nano::open_block open2 (send3.hash (), destination2.pub, destination2.pub, destination2.prv, destination2.pub, pool.generate (destination2.pub)); ASSERT_EQ (nano::process_result::unreceivable, ledger.process (transaction, open2).code); + // Upgrade to epoch 2 and send to destination. Try to create an open block from an epoch 2 source block. + nano::keypair destination3; + nano::state_block epoch2 (nano::genesis_account, send2.hash (), nano::genesis_account, nano::genesis_amount - nano::Gxrb_ratio * 2, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (send2.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch2).code); + nano::state_block send4 (nano::genesis_account, epoch2.hash (), nano::genesis_account, nano::genesis_amount - nano::Gxrb_ratio * 3, destination3.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (epoch2.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send4).code); + nano::open_block open3 (send4.hash (), destination3.pub, destination3.pub, destination3.prv, destination3.pub, pool.generate (destination3.pub)); + ASSERT_EQ (nano::process_result::unreceivable, ledger.process (transaction, open3).code); + // Send it to an epoch 1 account + nano::state_block send5 (nano::genesis_account, send4.hash (), nano::genesis_account, nano::genesis_amount - nano::Gxrb_ratio * 4, destination.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (send4.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send5).code); + ASSERT_FALSE (ledger.store.account_get (transaction, destination.pub, destination_info)); + ASSERT_EQ (destination_info.epoch (), nano::epoch::epoch_1); + nano::state_block receive3 (destination.pub, send3.hash (), destination.pub, nano::Gxrb_ratio * 2, send5.hash (), destination.prv, destination.pub, pool.generate (send3.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive3).code); + ASSERT_FALSE (ledger.store.account_get (transaction, destination.pub, destination_info)); + ASSERT_EQ (destination_info.epoch (), nano::epoch::epoch_2); + // Upgrade an unopened account straight to epoch 2 + nano::keypair destination4; + nano::state_block send6 (nano::genesis_account, send5.hash (), nano::genesis_account, nano::genesis_amount - nano::Gxrb_ratio * 5, destination4.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (send5.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send6).code); + nano::state_block epoch4 (destination4.pub, 0, 0, 0, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (destination4.pub)); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch4).code); } TEST (ledger, epoch_blocks_fork) @@ -2388,6 +2463,12 @@ TEST (ledger, epoch_blocks_fork) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send1).code); nano::state_block epoch1 (nano::genesis_account, genesis.hash (), nano::genesis_account, nano::genesis_amount, ledger.link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (genesis.hash ())); ASSERT_EQ (nano::process_result::fork, ledger.process (transaction, epoch1).code); + nano::state_block epoch2 (nano::genesis_account, genesis.hash (), nano::genesis_account, nano::genesis_amount, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (genesis.hash ())); + ASSERT_EQ (nano::process_result::fork, ledger.process (transaction, epoch2).code); + nano::state_block epoch3 (nano::genesis_account, send1.hash (), nano::genesis_account, nano::genesis_amount, ledger.link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (send1.hash ())); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch3).code); + nano::state_block epoch4 (nano::genesis_account, send1.hash (), nano::genesis_account, nano::genesis_amount, ledger.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (send1.hash ())); + ASSERT_EQ (nano::process_result::fork, ledger.process (transaction, epoch2).code); } TEST (ledger, successor_epoch) diff --git a/nano/core_test/versioning.cpp b/nano/core_test/versioning.cpp index 293c3b3033..8503ca11b7 100644 --- a/nano/core_test/versioning.cpp +++ b/nano/core_test/versioning.cpp @@ -16,7 +16,7 @@ TEST (versioning, account_info_v1) nano::mdb_store store (logger, file); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store.block_put (transaction, open.hash (), open, sideband); auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (sizeof (v1), &v1), 0)); ASSERT_EQ (0, status); @@ -52,7 +52,7 @@ TEST (versioning, account_info_v5) nano::mdb_store store (logger, file); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store.block_put (transaction, open.hash (), open, sideband); auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (sizeof (v5), &v5), 0)); ASSERT_EQ (0, status); @@ -88,7 +88,7 @@ TEST (versioning, account_info_v13) nano::mdb_store store (logger, file); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); store.block_put (transaction, open.hash (), open, sideband); auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (v13), 0)); ASSERT_EQ (0, status); diff --git a/nano/core_test/wallets.cpp b/nano/core_test/wallets.cpp index a153b2a0be..b9caae24a9 100644 --- a/nano/core_test/wallets.cpp +++ b/nano/core_test/wallets.cpp @@ -76,14 +76,26 @@ TEST (wallets, remove) } } -#if !NANO_ROCKSDB TEST (wallets, upgrade) { - nano::system system (24000, 1); + // Don't test this in rocksdb mode + static nano::network_constants network_constants; + auto use_rocksdb_str = std::getenv ("TEST_USE_ROCKSDB"); + if (use_rocksdb_str && boost::lexical_cast (use_rocksdb_str) == 1) + { + return; + } + + nano::system system; + nano::node_config node_config (24000, system.logging); + node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; + system.add_node (node_config); auto path (nano::unique_path ()); auto id = nano::random_wallet_id (); + nano::node_config node_config1 (24001, system.logging); + node_config1.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; { - auto node1 (std::make_shared (system.io_ctx, 24001, path, system.alarm, system.logging, system.work)); + auto node1 (std::make_shared (system.io_ctx, path, system.alarm, node_config1, system.work)); ASSERT_FALSE (node1->init_error ()); bool error (false); nano::wallets wallets (error, *node1); @@ -100,11 +112,11 @@ TEST (wallets, upgrade) ASSERT_FALSE (mdb_store.account_get (transaction_destination, nano::genesis_account, info)); auto rep_block = node1->rep_block (nano::genesis_account); nano::account_info_v13 account_info_v13 (info.head, rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch ()); - auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch ()) == nano::tables::accounts_v0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0)); + auto status (mdb_put (mdb_store.env.tx (transaction_destination), info.epoch () == nano::epoch::epoch_0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0)); (void)status; assert (status == 0); } - auto node1 (std::make_shared (system.io_ctx, 24001, path, system.alarm, system.logging, system.work)); + auto node1 (std::make_shared (system.io_ctx, path, system.alarm, node_config1, system.work)); ASSERT_EQ (1, node1->wallets.items.size ()); ASSERT_EQ (id, node1->wallets.items.begin ()->first); auto transaction_new (node1->wallets.env.tx_begin_write ()); @@ -116,7 +128,6 @@ TEST (wallets, upgrade) MDB_dbi new_handle; ASSERT_EQ (0, mdb_dbi_open (tx_new, id.to_string ().c_str (), 0, &new_handle)); } -#endif // Keeps breaking whenever we add new DBs TEST (wallets, DISABLED_wallet_create_max) diff --git a/nano/lib/blocks.hpp b/nano/lib/blocks.hpp index 318044c77c..b4af2962e3 100644 --- a/nano/lib/blocks.hpp +++ b/nano/lib/blocks.hpp @@ -15,7 +15,7 @@ namespace nano { // We operate on streams of uint8_t by convention using stream = std::basic_streambuf; -// Read a raw byte stream the size of `T' and fill value. +// Read a raw byte stream the size of `T' and fill value. Returns true if there was an error, false otherwise template bool try_read (nano::stream & stream_a, T & value) { diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index fbfbdc2660..b7fa65019d 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -260,7 +260,7 @@ void nano::block_processor::process_batch (nano::unique_lock & lock_ } lock_a.unlock (); auto scoped_write_guard = write_database_queue.wait (nano::writer::process_batch); - auto transaction (node.store.tx_begin_write ({ nano::tables::accounts_v0, nano::tables::accounts_v1, nano::tables::cached_counts, nano::tables::change_blocks, nano::tables::frontiers, nano::tables::open_blocks, nano::tables::pending_v0, nano::tables::pending_v1, nano::tables::receive_blocks, nano::tables::representation, nano::tables::send_blocks, nano::tables::state_blocks_v0, nano::tables::state_blocks_v1, nano::tables::unchecked }, { nano::tables::confirmation_height })); + auto transaction (node.store.tx_begin_write ({ nano::tables::accounts, nano::tables::cached_counts, nano::tables::change_blocks, nano::tables::frontiers, nano::tables::open_blocks, nano::tables::pending, nano::tables::receive_blocks, nano::tables::representation, nano::tables::send_blocks, nano::tables::state_blocks, nano::tables::unchecked }, { nano::tables::confirmation_height })); timer_l.restart (); lock_a.lock (); // Processing blocks diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 9ca85da6a5..f54fc5dbc7 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -31,6 +31,7 @@ using ipc_json_handler_no_arg_func_map = std::unordered_map const & response_a, std::function stop_callback_a) : @@ -534,7 +535,7 @@ void nano::json_handler::account_info () response_l.put ("balance", balance); response_l.put ("modified_timestamp", std::to_string (info.modified)); response_l.put ("block_count", std::to_string (info.block_count)); - response_l.put ("account_version", info.epoch () == nano::epoch::epoch_1 ? "1" : "0"); + response_l.put ("account_version", epoch_as_string (info.epoch ())); response_l.put ("confirmation_height", std::to_string (confirmation_height)); if (representative) { @@ -1214,9 +1215,7 @@ void nano::json_handler::block_count_type () response_l.put ("receive", std::to_string (count.receive)); response_l.put ("open", std::to_string (count.open)); response_l.put ("change", std::to_string (count.change)); - response_l.put ("state_v0", std::to_string (count.state_v0)); - response_l.put ("state_v1", std::to_string (count.state_v1)); - response_l.put ("state", std::to_string (count.state_v0 + count.state_v1)); + response_l.put ("state", std::to_string (count.state)); response_errors (); } @@ -2694,7 +2693,7 @@ void nano::json_handler::pending () } if (min_version) { - pending_tree.put ("min_version", info.epoch == nano::epoch::epoch_1 ? "1" : "0"); + pending_tree.put ("min_version", epoch_as_string (info.epoch)); } peers_l.add_child (key.hash.to_string (), pending_tree); } @@ -4322,7 +4321,7 @@ void nano::json_handler::wallet_pending () } if (min_version) { - pending_tree.put ("min_version", info.epoch == nano::epoch::epoch_1 ? "1" : "0"); + pending_tree.put ("min_version", epoch_as_string (info.epoch)); } peers_l.add_child (key.hash.to_string (), pending_tree); } @@ -4818,4 +4817,17 @@ bool block_confirmed (nano::node & node, nano::transaction & transaction, nano:: return is_confirmed; } + +const char * epoch_as_string (nano::epoch epoch) +{ + switch (epoch) + { + case nano::epoch::epoch_2: + return "2"; + case nano::epoch::epoch_1: + return "1"; + default: + return "0"; + } +} } diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 598117a65d..6b7c2019af 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -26,9 +26,8 @@ size_t mdb_val::size () const } template <> -mdb_val::db_val (size_t size_a, void * data_a, nano::epoch epoch_a) : -value ({ size_a, data_a }), -epoch (epoch_a) +mdb_val::db_val (size_t size_a, void * data_a) : +value ({ size_a, data_a }) { } @@ -117,16 +116,10 @@ nano::mdb_txn_callbacks nano::mdb_store::create_txn_callbacks () void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags) { error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts", flags, &accounts_v0) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts_v1", flags, &accounts_v1) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "send", flags, &send_blocks) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "receive", flags, &receive_blocks) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "open", flags, &open_blocks) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "change", flags, &change_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "state", flags, &state_blocks_v0) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "state_v1", flags, &state_blocks_v1) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_v0) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "pending_v1", flags, &pending_v1) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "vote", flags, &vote) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0; @@ -137,6 +130,24 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & { error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks_info", flags, &blocks_info) != 0; } + error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts", flags, &accounts_v0) != 0; + accounts = accounts_v0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_v0) != 0; + pending = pending_v0; + + if (version_get (transaction_a) < 15) + { + error_a |= mdb_dbi_open (env.tx (transaction_a), "state", flags, &state_blocks_v0) != 0; + state_blocks = state_blocks_v0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts_v1", flags, &accounts_v1) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "pending_v1", flags, &pending_v1) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "state_v1", flags, &state_blocks_v1) != 0; + } + else + { + error_a |= mdb_dbi_open (env.tx (transaction_a), "state_blocks", flags, &state_blocks) != 0; + state_blocks_v0 = state_blocks; + } } bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, size_t batch_size) @@ -190,7 +201,7 @@ void nano::mdb_store::upgrade_v1_to_v2 (nano::write_transaction const & transact { nano::mdb_iterator i (transaction_a, accounts_v0, nano::mdb_val (account)); std::cerr << std::hex; - if (i != nano::mdb_iterator (nullptr)) + if (i != nano::mdb_iterator{}) { account = nano::account (i->first); nano::account_info_v1 v1 (i->second); @@ -220,7 +231,7 @@ void nano::mdb_store::upgrade_v2_to_v3 (nano::write_transaction const & transact { version_put (transaction_a, 3); mdb_drop (env.tx (transaction_a), representation, 0); - for (auto i (std::make_unique> (transaction_a, accounts_v0)), n (std::make_unique> (nullptr)); *i != *n; ++(*i)) + for (auto i (std::make_unique> (transaction_a, accounts_v0)), n (std::make_unique> ()); *i != *n; ++(*i)) { nano::account account_l ((*i)->first); nano::account_info_v5 info ((*i)->second); @@ -236,17 +247,18 @@ void nano::mdb_store::upgrade_v2_to_v3 (nano::write_transaction const & transact void nano::mdb_store::upgrade_v3_to_v4 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 4); - std::queue> items; + std::queue> items; for (auto i (nano::store_iterator (std::make_unique> (transaction_a, pending_v0))), n (nano::store_iterator (nullptr)); i != n; ++i) { nano::block_hash const & hash (i->first); nano::pending_info_v3 const & info (i->second); - items.push (std::make_pair (nano::pending_key (info.destination, hash), nano::pending_info (info.source, info.amount, nano::epoch::epoch_0))); + items.push (std::make_pair (nano::pending_key (info.destination, hash), nano::pending_info_v14 (info.source, info.amount, nano::epoch::epoch_0))); } mdb_drop (env.tx (transaction_a), pending_v0, 0); while (!items.empty ()) { - pending_put (transaction_a, items.front ().first, items.front ().second); + auto status (mdb_put (env.tx (transaction_a), pending, nano::mdb_val (items.front ().first), nano::mdb_val (items.front ().second), 0)); + assert (success (status)); items.pop (); } } @@ -270,16 +282,15 @@ void nano::mdb_store::upgrade_v4_to_v5 (nano::write_transaction const & transact block->serialize (stream); nano::write (stream, successor.bytes); } - block_raw_put (transaction_a, vector, block->type (), nano::epoch::epoch_0, hash); + block_raw_put (transaction_a, vector, block->type (), hash); if (!block->previous ().is_zero ()) { nano::block_type type; auto value (block_raw_get (transaction_a, block->previous (), type)); - auto version (block_version (transaction_a, block->previous ())); assert (value.size () != 0); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); std::copy (hash.bytes.begin (), hash.bytes.end (), data.end () - nano::block_sideband::size (type)); - block_raw_put (transaction_a, data, type, version, block->previous ()); + block_raw_put (transaction_a, data, type, block->previous ()); } } successor = hash; @@ -335,7 +346,7 @@ void nano::mdb_store::upgrade_v8_to_v9 (nano::write_transaction const & transact nano::genesis genesis; std::shared_ptr block (std::move (genesis.open)); nano::keypair junk; - for (nano::mdb_iterator i (transaction_a, sequence), n (nano::mdb_iterator (nullptr)); i != n; ++i) + for (nano::mdb_iterator i (transaction_a, sequence), n (nano::mdb_iterator{}); i != n; ++i) { nano::bufferstream stream (reinterpret_cast (i->second.data ()), i->second.size ()); uint64_t sequence; @@ -383,19 +394,19 @@ void nano::mdb_store::upgrade_v12_to_v13 (nano::write_transaction & transaction_ nano::account first (0); nano::account_info_v13 second; { - nano::store_iterator current (std::make_unique> (transaction_a, accounts_v0, accounts_v1, nano::mdb_val (account))); - nano::store_iterator end (nullptr); + nano::mdb_merge_iterator current (transaction_a, accounts_v0, accounts_v1, nano::mdb_val (account)); + nano::mdb_merge_iterator end{}; if (current != end) { - first = current->first; - second = current->second; + first = nano::account (current->first); + second = nano::account_info_v13 (current->second); } } if (!first.is_zero ()) { auto hash (second.open_block); uint64_t height (1); - nano::block_sideband sideband; + nano::block_sideband_v14 sideband; while (!hash.is_zero ()) { if (cost >= batch_size) @@ -406,12 +417,31 @@ void nano::mdb_store::upgrade_v12_to_v13 (nano::write_transaction & transaction_ transaction_a.renew (); cost = 0; } - auto block (block_get (transaction_a, hash, &sideband)); + + bool is_state_block_v1 = false; + auto block = block_get_v14 (transaction_a, hash, &sideband, &is_state_block_v1); + assert (block != nullptr); if (sideband.height == 0) { sideband.height = height; - block_put (transaction_a, hash, *block, sideband, block_version (transaction_a, hash)); + + std::vector vector; + { + nano::vectorstream stream (vector); + block->serialize (stream); + sideband.serialize (stream); + } + + nano::mdb_val value{ vector.size (), (void *)vector.data () }; + MDB_dbi database = is_state_block_v1 ? state_blocks_v1 : table_to_dbi (block_database (sideband.type)); + + auto status = mdb_put (env.tx (transaction_a), database, nano::mdb_val (hash), value, 0); + release_assert (success (status)); + + nano::block_predecessor_set predecessor (transaction_a, *this); + block->visit (predecessor); + assert (block->previous ().is_zero () || block_successor (transaction_a, block->previous ()) == hash); cost += 16; } else @@ -439,25 +469,26 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::write_transaction const & transa { // Upgrade all accounts to have a confirmation of 0 (except genesis which should have 1) version_put (transaction_a, 14); - nano::store_iterator i (std::make_unique> (transaction_a, accounts_v0, accounts_v1)); - nano::store_iterator n (nullptr); - + nano::mdb_merge_iterator i (transaction_a, accounts_v0, accounts_v1); + nano::mdb_merge_iterator n{}; std::vector> account_infos; - account_infos.reserve (account_count (transaction_a)); + account_infos.reserve (count (transaction_a, accounts_v0) + count (transaction_a, accounts_v1)); for (; i != n; ++i) { - nano::account_info_v13 const & account_info_v13 (i->second); + nano::account account (i->first); + nano::account_info_v13 account_info_v13 (i->second); + uint64_t confirmation_height = 0; - if (i->first == network_params.ledger.genesis_account) + if (account == network_params.ledger.genesis_account) { confirmation_height = 1; } - account_infos.emplace_back (i->first, nano::account_info_v14{ account_info_v13.head, account_info_v13.rep_block, account_info_v13.open_block, account_info_v13.balance, account_info_v13.modified, account_info_v13.block_count, confirmation_height, account_info_v13.epoch }); + account_infos.emplace_back (account, nano::account_info_v14{ account_info_v13.head, account_info_v13.rep_block, account_info_v13.open_block, account_info_v13.balance, account_info_v13.modified, account_info_v13.block_count, confirmation_height, i.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); } for (auto const & account_info : account_infos) { - auto status1 (mdb_put (env.tx (transaction_a), table_to_dbi (get_account_db (account_info.second.epoch)), nano::mdb_val (account_info.first), nano::mdb_val (account_info.second), 0)); + auto status1 (mdb_put (env.tx (transaction_a), account_info.second.epoch == nano::epoch::epoch_0 ? accounts_v0 : accounts_v1, nano::mdb_val (account_info.first), nano::mdb_val (account_info.second), 0)); release_assert (status1 == 0); } @@ -470,28 +501,101 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::write_transaction const & transa void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transaction_a) { - version_put (transaction_a, 15); - // Move confirmation height from account_info database to its own table std::vector> account_infos; - account_infos.reserve (account_count (transaction_a)); + account_infos.reserve (count (transaction_a, accounts_v0) + count (transaction_a, accounts_v1)); - nano::store_iterator i (std::make_unique> (transaction_a, accounts_v0, accounts_v1)); - nano::store_iterator n (nullptr); + nano::mdb_merge_iterator i (transaction_a, accounts_v0, accounts_v1); + nano::mdb_merge_iterator n{}; for (; i != n; ++i) { - auto const & account_info_v14 (i->second); + nano::account account (i->first); + nano::account_info_v14 account_info_v14 (i->second); // Upgrade rep block to representative account - auto rep_block = block_get (transaction_a, account_info_v14.rep_block); + auto rep_block = block_get_v14 (transaction_a, account_info_v14.rep_block); release_assert (rep_block != nullptr); - account_infos.emplace_back (i->first, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, account_info_v14.epoch }); - confirmation_height_put (transaction_a, i->first, i->second.confirmation_height); + account_infos.emplace_back (account, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, i.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); + confirmation_height_put (transaction_a, account, account_info_v14.confirmation_height); } - for (auto const & account_info : account_infos) + // No longer need accounts_v1, keep v0 but clear it + mdb_drop (env.tx (transaction_a), accounts_v1, 1); + mdb_drop (env.tx (transaction_a), accounts_v0, 0); + + for (auto const & account_account_info_pair : account_infos) + { + auto const & account_info (account_account_info_pair.second); + mdb_put (env.tx (transaction_a), accounts, nano::mdb_val (account_account_info_pair.first), nano::mdb_val (account_info), MDB_APPEND); + } + + logger.always_log ("Epoch merge upgrade. Finished accounts, now doing state blocks"); + + account_infos.clear (); + + // Have to create a new database as we are iterating over the existing ones and want to use MDB_APPEND for quick insertion + MDB_dbi state_blocks_new; + mdb_dbi_open (env.tx (transaction_a), "state_blocks", MDB_CREATE, &state_blocks_new); + + nano::mdb_merge_iterator i_state (transaction_a, state_blocks_v0, state_blocks_v1); + nano::mdb_merge_iterator n_state{}; + auto num = 0u; + for (; i_state != n_state; ++i_state, ++num) + { + nano::block_hash hash (i_state->first); + nano::state_block_w_sideband_v14 state_block_w_sideband_v14 (i_state->second); + auto & sideband_v14 = state_block_w_sideband_v14.sideband; + + nano::block_sideband sideband{ sideband_v14.type, sideband_v14.account, sideband_v14.successor, sideband_v14.balance, sideband_v14.height, sideband_v14.timestamp, i_state.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }; + + // Write these out + std::vector data; + { + nano::vectorstream stream (data); + state_block_w_sideband_v14.state_block->serialize (stream); + sideband.serialize (stream); + } + + nano::mdb_val value{ data.size (), (void *)data.data () }; + auto s = mdb_put (env.tx (transaction_a), state_blocks_new, nano::mdb_val (hash), value, MDB_APPEND); + release_assert (success (s)); + + // Every so often output to the log to indicate progress + constexpr auto output_cutoff = 1000000; + if (num % output_cutoff == 0) + { + logger.always_log (boost::str (boost::format ("Database epoch merge upgrade %1% million state blocks upgraded") % (num / output_cutoff))); + } + } + + logger.always_log ("Epoch merge upgrade. Finished state blocks, now doing pending blocks"); + + state_blocks = state_blocks_new; + + // No longer need states v0/v1 databases + mdb_drop (env.tx (transaction_a), state_blocks_v1, 1); + mdb_drop (env.tx (transaction_a), state_blocks_v0, 1); + + state_blocks_v0 = state_blocks; + + std::vector> pending_infos; + pending_infos.reserve (count (transaction_a, pending_v0) + count (transaction_a, pending_v1)); + + nano::mdb_merge_iterator i_pending (transaction_a, pending_v0, pending_v1); + nano::mdb_merge_iterator n_pending{}; + for (; i_pending != n_pending; ++i_pending) + { + nano::pending_info_v14 info (i_pending->second); + pending_infos.emplace_back (nano::pending_key (i_pending->first), nano::pending_info{ info.source, info.amount, i_pending.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); + } + + // No longer need the pending v1 table + mdb_drop (env.tx (transaction_a), pending_v1, 1); + mdb_drop (env.tx (transaction_a), pending_v0, 0); + + for (auto const & pending_key_pending_info_pair : pending_infos) { - account_put (transaction_a, account_info.first, account_info.second); + mdb_put (env.tx (transaction_a), pending, nano::mdb_val (pending_key_pending_info_pair.first), nano::mdb_val (pending_key_pending_info_pair.second), MDB_APPEND); } // Representation table is no longer used @@ -501,6 +605,8 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa release_assert (status == MDB_SUCCESS); representation = 0; } + version_put (transaction_a, 15); + logger.always_log ("Finished epoch merge upgrade"); } /** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ @@ -627,10 +733,8 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const { case tables::frontiers: return frontiers; - case tables::accounts_v0: - return accounts_v0; - case tables::accounts_v1: - return accounts_v1; + case tables::accounts: + return accounts; case tables::send_blocks: return send_blocks; case tables::receive_blocks: @@ -639,14 +743,10 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const return open_blocks; case tables::change_blocks: return change_blocks; - case tables::state_blocks_v0: - return state_blocks_v0; - case tables::state_blocks_v1: - return state_blocks_v1; - case tables::pending_v0: - return pending_v0; - case tables::pending_v1: - return pending_v1; + case tables::state_blocks: + return state_blocks; + case tables::pending: + return pending; case tables::blocks_info: return blocks_info; case tables::unchecked: @@ -691,3 +791,216 @@ bool nano::mdb_store::init_error () const { return error; } + +// All the v14 functions below are only needed during upgrades +bool nano::mdb_store::entry_has_sideband_v14 (size_t entry_size_a, nano::block_type type_a) const +{ + return (entry_size_a == nano::block::size (type_a) + nano::block_sideband_v14::size (type_a)); +} + +size_t nano::mdb_store::block_successor_offset_v14 (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const +{ + size_t result; + if (full_sideband (transaction_a) || entry_has_sideband_v14 (entry_size_a, type_a)) + { + result = entry_size_a - nano::block_sideband_v14::size (type_a); + } + else + { + // Read old successor-only sideband + assert (entry_size_a == nano::block::size (type_a) + sizeof (nano::uint256_union)); + result = entry_size_a - sizeof (nano::uint256_union); + } + return result; +} + +nano::block_hash nano::mdb_store::block_successor_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + nano::block_type type; + auto value (block_raw_get_v14 (transaction_a, hash_a, type)); + nano::block_hash result; + if (value.size () != 0) + { + assert (value.size () >= result.bytes.size ()); + nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset_v14 (transaction_a, value.size (), type), result.bytes.size ()); + auto error (nano::try_read (stream, result.bytes)); + (void)error; + assert (!error); + } + else + { + result.clear (); + } + return result; +} + +nano::mdb_val nano::mdb_store::block_raw_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1) const +{ + nano::mdb_val result; + // Table lookups are ordered by match probability + nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; + for (auto current_type : block_types) + { + auto db_val (block_raw_get_by_type_v14 (transaction_a, hash_a, current_type, is_state_v1)); + if (db_val.is_initialized ()) + { + type_a = current_type; + result = db_val.get (); + break; + } + } + + return result; +} + +boost::optional nano::mdb_store::block_raw_get_by_type_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1) const +{ + nano::mdb_val value; + nano::mdb_val hash (hash_a); + int status = status_code_not_found (); + switch (type_a) + { + case nano::block_type::send: + { + status = mdb_get (env.tx (transaction_a), send_blocks, hash, value); + break; + } + case nano::block_type::receive: + { + status = mdb_get (env.tx (transaction_a), receive_blocks, hash, value); + break; + } + case nano::block_type::open: + { + status = mdb_get (env.tx (transaction_a), open_blocks, hash, value); + break; + } + case nano::block_type::change: + { + status = mdb_get (env.tx (transaction_a), change_blocks, hash, value); + break; + } + case nano::block_type::state: + { + status = mdb_get (env.tx (transaction_a), state_blocks_v1, hash, value); + if (is_state_v1 != nullptr) + { + *is_state_v1 = success (status); + } + if (not_found (status)) + { + status = mdb_get (env.tx (transaction_a), state_blocks_v0, hash, value); + } + break; + } + case nano::block_type::invalid: + case nano::block_type::not_a_block: + { + break; + } + } + + release_assert (success (status) || not_found (status)); + boost::optional result; + if (success (status)) + { + result = value; + } + return result; +} + +nano::account nano::mdb_store::block_account_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + nano::block_sideband_v14 sideband; + auto block (block_get_v14 (transaction_a, hash_a, &sideband)); + nano::account result (block->account ()); + if (result.is_zero ()) + { + result = sideband.account; + } + assert (!result.is_zero ()); + return result; +} + +// Return account containing hash +nano::account nano::mdb_store::block_account_computed_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + assert (!full_sideband (transaction_a)); + nano::account result (0); + auto hash (hash_a); + while (result.is_zero ()) + { + auto block (block_get_v14 (transaction_a, hash)); + assert (block); + result = block->account (); + if (result.is_zero ()) + { + auto type (nano::block_type::invalid); + auto value (block_raw_get_v14 (transaction_a, block->previous (), type)); + if (entry_has_sideband_v14 (value.size (), type)) + { + result = block_account_v14 (transaction_a, block->previous ()); + } + else + { + nano::block_info block_info; + if (!block_info_get (transaction_a, hash, block_info)) + { + result = block_info.account; + } + else + { + result = frontier_get (transaction_a, hash); + if (result.is_zero ()) + { + auto successor (block_successor_v14 (transaction_a, hash)); + assert (!successor.is_zero ()); + hash = successor; + } + } + } + } + } + assert (!result.is_zero ()); + return result; +} + +nano::uint128_t nano::mdb_store::block_balance_computed_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + assert (!full_sideband (transaction_a)); + summation_visitor visitor (transaction_a, *this, true); + return visitor.compute_balance (hash_a); +} + +std::shared_ptr nano::mdb_store::block_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband_v14 * sideband_a, bool * is_state_v1) const +{ + nano::block_type type; + auto value (block_raw_get_v14 (transaction_a, hash_a, type, is_state_v1)); + std::shared_ptr result; + if (value.size () != 0) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = nano::deserialize_block (stream, type); + assert (result != nullptr); + if (sideband_a) + { + sideband_a->type = type; + if (full_sideband (transaction_a) || entry_has_sideband_v14 (value.size (), type)) + { + bool error = sideband_a->deserialize (stream); + (void)error; + assert (!error); + } + else + { + // Reconstruct sideband data for block. + sideband_a->account = block_account_computed_v14 (transaction_a, hash_a); + sideband_a->balance = block_balance_computed_v14 (transaction_a, hash_a); + sideband_a->successor = block_successor_v14 (transaction_a, hash_a); + sideband_a->height = 0; + sideband_a->timestamp = 0; + } + } + } + return result; +} diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 08f66afd3d..7221f53d73 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -58,17 +58,23 @@ class mdb_store : public block_store_partial MDB_dbi frontiers{ 0 }; /** - * Maps account v1 to account information, head, rep, open, balance, timestamp and block count. + * Maps account v1 to account information, head, rep, open, balance, timestamp and block count. (Removed) * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t */ MDB_dbi accounts_v0{ 0 }; /** - * Maps account v0 to account information, head, rep, open, balance, timestamp and block count. + * Maps account v0 to account information, head, rep, open, balance, timestamp and block count. (Removed) * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t */ MDB_dbi accounts_v1{ 0 }; + /** + * Maps account v0 to account information, head, rep, open, balance, timestamp, block count and epoch. (Removed) + * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t, nano::epoch + */ + MDB_dbi accounts{ 0 }; + /** * Maps block hash to send block. * nano::block_hash -> nano::send_block @@ -94,29 +100,41 @@ class mdb_store : public block_store_partial MDB_dbi change_blocks{ 0 }; /** - * Maps block hash to v0 state block. + * Maps block hash to v0 state block. (Removed) * nano::block_hash -> nano::state_block */ MDB_dbi state_blocks_v0{ 0 }; /** - * Maps block hash to v1 state block. + * Maps block hash to v1 state block. (Removed) * nano::block_hash -> nano::state_block */ MDB_dbi state_blocks_v1{ 0 }; /** - * Maps min_version 0 (destination account, pending block) to (source account, amount). + * Maps block hash to state block. + * nano::block_hash -> nano::state_block + */ + MDB_dbi state_blocks{ 0 }; + + /** + * Maps min_version 0 (destination account, pending block) to (source account, amount). (Removed) * nano::account, nano::block_hash -> nano::account, nano::amount */ MDB_dbi pending_v0{ 0 }; /** - * Maps min_version 1 (destination account, pending block) to (source account, amount). + * Maps min_version 1 (destination account, pending block) to (source account, amount). (Removed) * nano::account, nano::block_hash -> nano::account, nano::amount */ MDB_dbi pending_v1{ 0 }; + /** + * Maps (destination account, pending block) to (source account, amount, version). (Removed) + * nano::account, nano::block_hash -> nano::account, nano::amount, nano::epoch + */ + MDB_dbi pending{ 0 }; + /** * Maps block hash to account and balance. (Removed) * block_hash -> nano::account, nano::amount @@ -185,19 +203,20 @@ class mdb_store : public block_store_partial return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table_a), key)); } - template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::mdb_val const & key) const - { - return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table1_a), table_to_dbi (table2_a), key)); - } + bool init_error () const override; - template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const - { - return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table1_a), table_to_dbi (table2_a))); - } + size_t count (nano::transaction const &, MDB_dbi) const; - bool init_error () const override; + // These are only use in the upgrade process. + std::shared_ptr block_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband_v14 * sideband_a = nullptr, bool * is_state_v1 = nullptr) const override; + bool entry_has_sideband_v14 (size_t entry_size_a, nano::block_type type_a) const; + size_t block_successor_offset_v14 (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; + nano::block_hash block_successor_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; + nano::mdb_val block_raw_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1 = nullptr) const; + boost::optional block_raw_get_by_type_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1) const; + nano::account block_account_computed_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; + nano::account block_account_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; + nano::uint128_t block_balance_computed_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; private: bool do_upgrades (nano::write_transaction &, size_t); @@ -230,7 +249,6 @@ class mdb_store : public block_store_partial bool txn_tracking_enabled; size_t count (nano::transaction const & transaction_a, tables table_a) const override; - size_t count (nano::transaction const &, MDB_dbi) const; }; template <> @@ -238,7 +256,7 @@ void * mdb_val::data () const; template <> size_t mdb_val::size () const; template <> -mdb_val::db_val (size_t size_a, void * data_a, nano::epoch epoch_a); +mdb_val::db_val (size_t size_a, void * data_a); template <> void mdb_val::convert_buffer_to_value (); } diff --git a/nano/node/lmdb/lmdb_iterator.hpp b/nano/node/lmdb/lmdb_iterator.hpp index c36e182746..88e3bf19b1 100644 --- a/nano/node/lmdb/lmdb_iterator.hpp +++ b/nano/node/lmdb/lmdb_iterator.hpp @@ -10,11 +10,8 @@ template class mdb_iterator : public store_iterator_impl { public: - mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, nano::epoch epoch_a = nano::epoch::unspecified) : - cursor (nullptr) + mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a) { - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; auto status (mdb_cursor_open (tx (transaction_a), db_a, &cursor)); release_assert (status == 0); auto status2 (mdb_cursor_get (cursor, ¤t.first.value, ¤t.second.value, MDB_FIRST)); @@ -34,18 +31,10 @@ class mdb_iterator : public store_iterator_impl } } - mdb_iterator (std::nullptr_t, nano::epoch epoch_a = nano::epoch::unspecified) : - cursor (nullptr) - { - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; - } + mdb_iterator () = default; - mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a, nano::epoch epoch_a = nano::epoch::unspecified) : - cursor (nullptr) + mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a) { - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; auto status (mdb_cursor_open (tx (transaction_a), db_a, &cursor)); release_assert (status == 0); current.first = val_a; @@ -139,8 +128,8 @@ class mdb_iterator : public store_iterator_impl } void clear () { - current.first = nano::db_val (current.first.epoch); - current.second = nano::db_val (current.second.epoch); + current.first = nano::db_val (); + current.second = nano::db_val (); assert (is_end_sentinal ()); } @@ -158,7 +147,7 @@ class mdb_iterator : public store_iterator_impl } nano::store_iterator_impl & operator= (nano::store_iterator_impl const &) = delete; - MDB_cursor * cursor; + MDB_cursor * cursor{ nullptr }; std::pair, nano::db_val> current; private: @@ -176,20 +165,20 @@ class mdb_merge_iterator : public store_iterator_impl { public: mdb_merge_iterator (nano::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a) : - impl1 (std::make_unique> (transaction_a, db1_a, nano::epoch::epoch_0)), - impl2 (std::make_unique> (transaction_a, db2_a, nano::epoch::epoch_1)) + impl1 (std::make_unique> (transaction_a, db1_a)), + impl2 (std::make_unique> (transaction_a, db2_a)) { } - mdb_merge_iterator (std::nullptr_t) : - impl1 (std::make_unique> (nullptr, nano::epoch::epoch_0)), - impl2 (std::make_unique> (nullptr, nano::epoch::epoch_1)) + mdb_merge_iterator () : + impl1 (std::make_unique> ()), + impl2 (std::make_unique> ()) { } mdb_merge_iterator (nano::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a, MDB_val const & val_a) : - impl1 (std::make_unique> (transaction_a, db1_a, val_a, nano::epoch::epoch_0)), - impl2 (std::make_unique> (transaction_a, db2_a, val_a, nano::epoch::epoch_1)) + impl1 (std::make_unique> (transaction_a, db1_a, val_a)), + impl2 (std::make_unique> (transaction_a, db2_a, val_a)) { } @@ -247,6 +236,8 @@ class mdb_merge_iterator : public store_iterator_impl nano::mdb_merge_iterator & operator= (nano::mdb_merge_iterator &&) = default; nano::mdb_merge_iterator & operator= (nano::mdb_merge_iterator const &) = delete; + mutable bool from_first_database{ false }; + private: nano::mdb_iterator & least_iterator () const { @@ -254,10 +245,12 @@ class mdb_merge_iterator : public store_iterator_impl if (impl1->is_end_sentinal ()) { result = impl2.get (); + from_first_database = false; } else if (impl2->is_end_sentinal ()) { result = impl1.get (); + from_first_database = true; } else { @@ -266,6 +259,7 @@ class mdb_merge_iterator : public store_iterator_impl if (key_cmp < 0) { result = impl1.get (); + from_first_database = true; } else if (key_cmp > 0) { @@ -275,6 +269,7 @@ class mdb_merge_iterator : public store_iterator_impl { auto val_cmp (mdb_cmp (mdb_cursor_txn (impl1->cursor), mdb_cursor_dbi (impl1->cursor), impl1->current.second, impl2->current.second)); result = val_cmp < 0 ? impl1.get () : impl2.get (); + from_first_database = (result == impl1.get ()); } } return *result; @@ -283,4 +278,4 @@ class mdb_merge_iterator : public store_iterator_impl std::unique_ptr> impl1; std::unique_ptr> impl2; }; -} \ No newline at end of file +} diff --git a/nano/node/node.cpp b/nano/node/node.cpp index cf952c5920..bcdd638614 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -624,7 +624,7 @@ void nano::node::process_active (std::shared_ptr incoming) nano::process_return nano::node::process (nano::block const & block_a) { - auto transaction (store.tx_begin_write ({ tables::accounts_v0, tables::accounts_v1, tables::cached_counts, tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending_v0, tables::pending_v1, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks_v0, tables::state_blocks_v1 }, { tables::confirmation_height })); + auto transaction (store.tx_begin_write ({ tables::accounts, tables::cached_counts, tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks }, { tables::confirmation_height })); auto result (ledger.process (transaction, block_a)); return result; } diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index 235ad01564..c3a6b8d38b 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -27,9 +27,8 @@ size_t rocksdb_val::size () const } template <> -rocksdb_val::db_val (size_t size_a, void * data_a, nano::epoch epoch_a) : -value (static_cast (data_a), size_a), -epoch (epoch_a) +rocksdb_val::db_val (size_t size_a, void * data_a) : +value (static_cast (data_a), size_a) { } @@ -72,7 +71,7 @@ nano::rocksdb_store::~rocksdb_store () void nano::rocksdb_store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a) { - std::initializer_list names{ rocksdb::kDefaultColumnFamilyName.c_str (), "frontiers", "accounts", "accounts_v1", "send", "receive", "open", "change", "state", "state_v1", "pending", "pending_v1", "representation", "unchecked", "vote", "online_weight", "meta", "peers", "cached_counts", "confirmation_height" }; + std::initializer_list names{ rocksdb::kDefaultColumnFamilyName.c_str (), "frontiers", "accounts", "send", "receive", "open", "change", "state_blocks", "pending", "representation", "unchecked", "vote", "online_weight", "meta", "peers", "cached_counts", "confirmation_height" }; std::vector column_families; for (const auto & cf_name : names) { @@ -150,10 +149,8 @@ rocksdb::ColumnFamilyHandle * nano::rocksdb_store::table_to_column_family (table { case tables::frontiers: return get_handle ("frontiers"); - case tables::accounts_v0: + case tables::accounts: return get_handle ("accounts"); - case tables::accounts_v1: - return get_handle ("accounts_v1"); case tables::send_blocks: return get_handle ("send"); case tables::receive_blocks: @@ -162,14 +159,10 @@ rocksdb::ColumnFamilyHandle * nano::rocksdb_store::table_to_column_family (table return get_handle ("open"); case tables::change_blocks: return get_handle ("change"); - case tables::state_blocks_v0: - return get_handle ("state"); - case tables::state_blocks_v1: - return get_handle ("state_v1"); - case tables::pending_v0: + case tables::state_blocks: + return get_handle ("state_blocks"); + case tables::pending: return get_handle ("pending"); - case tables::pending_v1: - return get_handle ("pending_v1"); case tables::blocks_info: assert (false); case tables::representation: @@ -281,15 +274,13 @@ bool nano::rocksdb_store::is_caching_counts (nano::tables table_a) const { switch (table_a) { - case tables::accounts_v0: - case tables::accounts_v1: + case tables::accounts: case tables::unchecked: case tables::send_blocks: case tables::receive_blocks: case tables::open_blocks: case tables::change_blocks: - case tables::state_blocks_v0: - case tables::state_blocks_v1: + case tables::state_blocks: return true; default: return false; @@ -537,7 +528,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options () const std::vector nano::rocksdb_store::all_tables () const { - return std::vector{ tables::accounts_v0, tables::accounts_v1, tables::cached_counts, tables::change_blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::open_blocks, tables::peers, tables::pending_v0, tables::pending_v1, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks_v0, tables::state_blocks_v1, tables::unchecked, tables::vote }; + return std::vector{ tables::accounts, tables::cached_counts, tables::change_blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::open_blocks, tables::peers, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks, tables::unchecked, tables::vote }; } bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_path) diff --git a/nano/node/rocksdb/rocksdb.hpp b/nano/node/rocksdb/rocksdb.hpp index 9086bbf203..256e4e7cc6 100644 --- a/nano/node/rocksdb/rocksdb.hpp +++ b/nano/node/rocksdb/rocksdb.hpp @@ -43,6 +43,13 @@ class rocksdb_store : public block_store_partial // Do nothing } + std::shared_ptr block_get_v14 (nano::transaction const &, nano::block_hash const &, nano::block_sideband_v14 * = nullptr, bool * = nullptr) const override + { + // Should not be called as RocksDB has no such upgrade path + release_assert (false); + return nullptr; + } + bool copy_db (boost::filesystem::path const & destination) override; template @@ -57,18 +64,6 @@ class rocksdb_store : public block_store_partial return nano::store_iterator (std::make_unique> (db, transaction_a, table_to_column_family (table_a), key)); } - template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, rocksdb_val const & key) const - { - return nano::store_iterator (std::make_unique> (db, transaction_a, table_to_column_family (table1_a), table_to_column_family (table2_a), key)); - } - - template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const - { - return nano::store_iterator (std::make_unique> (db, transaction_a, table_to_column_family (table1_a), table_to_column_family (table2_a))); - } - bool init_error () const override; private: diff --git a/nano/node/rocksdb/rocksdb_iterator.hpp b/nano/node/rocksdb/rocksdb_iterator.hpp index 78e9f74394..0110066a77 100644 --- a/nano/node/rocksdb/rocksdb_iterator.hpp +++ b/nano/node/rocksdb/rocksdb_iterator.hpp @@ -31,12 +31,8 @@ template class rocksdb_iterator : public store_iterator_impl { public: - rocksdb_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle_a, nano::epoch epoch_a = nano::epoch::unspecified) : - cursor (nullptr) + rocksdb_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle_a) { - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; - rocksdb::Iterator * iter; if (is_read (transaction_a)) { @@ -63,19 +59,10 @@ class rocksdb_iterator : public store_iterator_impl } } - rocksdb_iterator (std::nullptr_t, nano::epoch epoch_a = nano::epoch::unspecified) : - cursor (nullptr) - { - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; - } + rocksdb_iterator () = default; - rocksdb_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle_a, rocksdb_val const & val_a, nano::epoch epoch_a = nano::epoch::unspecified) : - cursor (nullptr) + rocksdb_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle_a, rocksdb_val const & val_a) { - current.first.epoch = epoch_a; - current.second.epoch = epoch_a; - rocksdb::Iterator * iter; if (is_read (transaction_a)) { @@ -183,8 +170,8 @@ class rocksdb_iterator : public store_iterator_impl } void clear () { - current.first = nano::rocksdb_val (current.first.epoch); - current.second = nano::rocksdb_val (current.second.epoch); + current.first = nano::rocksdb_val{}; + current.second = nano::rocksdb_val{}; assert (is_end_sentinal ()); } nano::rocksdb_iterator & operator= (nano::rocksdb_iterator && other_a) @@ -204,110 +191,4 @@ class rocksdb_iterator : public store_iterator_impl return static_cast (transaction_a.get_handle ()); } }; - -/** - * Iterates the key/value pairs of two stores merged together - */ - -template -class rocksdb_merge_iterator : public store_iterator_impl -{ -public: - rocksdb_merge_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * db1_a, rocksdb::ColumnFamilyHandle * db2_a) : - impl1 (std::make_unique> (db, transaction_a, db1_a, nano::epoch::epoch_0)), - impl2 (std::make_unique> (db, transaction_a, db2_a, nano::epoch::epoch_1)) - { - } - rocksdb_merge_iterator (std::nullptr_t) : - impl1 (std::make_unique> (nullptr, nano::epoch::epoch_0)), - impl2 (std::make_unique> (nullptr, nano::epoch::epoch_1)) - { - } - - rocksdb_merge_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * db1_a, rocksdb::ColumnFamilyHandle * db2_a, rocksdb::Slice const & val_a) : - impl1 (std::make_unique> (db, transaction_a, db1_a, val_a, nano::epoch::epoch_0)), - impl2 (std::make_unique> (db, transaction_a, db2_a, val_a, nano::epoch::epoch_1)) - { - } - rocksdb_merge_iterator (nano::rocksdb_merge_iterator && other_a) - { - impl1 = std::move (other_a.impl1); - impl2 = std::move (other_a.impl2); - } - - rocksdb_merge_iterator (nano::rocksdb_merge_iterator const &) = delete; - nano::store_iterator_impl & operator++ () override - { - ++least_iterator (); - return *this; - } - std::pair * operator-> (); - bool operator== (nano::store_iterator_impl const & base_a) const override - { - assert ((dynamic_cast const *> (&base_a) != nullptr) && "Incompatible iterator comparison"); - auto & other (static_cast const &> (base_a)); - return *impl1 == *other.impl1 && *impl2 == *other.impl2; - } - - bool is_end_sentinal () const override - { - return least_iterator ().is_end_sentinal (); - } - - void fill (std::pair & value_a) const override - { - auto & current (least_iterator ()); - if (current->first.size () != 0) - { - value_a.first = static_cast (current->first); - } - else - { - value_a.first = T (); - } - if (current->second.size () != 0) - { - value_a.second = static_cast (current->second); - } - else - { - value_a.second = U (); - } - } - nano::rocksdb_merge_iterator & operator= (nano::rocksdb_merge_iterator &&) = default; - nano::rocksdb_merge_iterator & operator= (nano::rocksdb_merge_iterator const &) = delete; - -private: - nano::rocksdb_iterator & least_iterator () const - { - nano::rocksdb_iterator * result; - if (impl1->is_end_sentinal ()) - { - result = impl2.get (); - } - else if (impl2->is_end_sentinal ()) - { - result = impl1.get (); - } - else - { - if (impl1->current.first.value.compare (impl2->current.first.value) < 0) - { - result = impl1.get (); - } - else if (impl1->current.first.value.compare (impl2->current.first.value) > 0) - { - result = impl2.get (); - } - else - { - result = impl1->current.second.value.compare (impl2->current.second.value) < 0 ? impl1.get () : impl2.get (); - } - } - return *result; - } - - std::unique_ptr> impl1; - std::unique_ptr> impl2; -}; } diff --git a/nano/node/testing.cpp b/nano/node/testing.cpp index 1839040895..e4c1fe0152 100644 --- a/nano/node/testing.cpp +++ b/nano/node/testing.cpp @@ -406,7 +406,7 @@ void nano::system::generate_mass_activity (uint32_t count_a, nano::node & node_a auto transaction (node_a.store.tx_begin_read ()); auto block_counts (node_a.store.block_count (transaction)); count = block_counts.sum (); - state = block_counts.state_v0 + block_counts.state_v1; + state = block_counts.state; } std::cerr << boost::str (boost::format ("Mass activity iteration %1% us %2% us/t %3% state: %4% old: %5%\n") % i % us % (us / 256) % state % (count - state)); previous = now; diff --git a/nano/secure/blockstore.cpp b/nano/secure/blockstore.cpp index 9b3874cec2..eec994937b 100644 --- a/nano/secure/blockstore.cpp +++ b/nano/secure/blockstore.cpp @@ -3,13 +3,14 @@ #include #include -nano::block_sideband::block_sideband (nano::block_type type_a, nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a) : +nano::block_sideband::block_sideband (nano::block_type type_a, nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a, nano::epoch epoch_a) : type (type_a), successor (successor_a), account (account_a), balance (balance_a), height (height_a), -timestamp (timestamp_a) +timestamp (timestamp_a), +epoch (epoch_a) { } @@ -30,6 +31,10 @@ size_t nano::block_sideband::size (nano::block_type type_a) result += sizeof (balance); } result += sizeof (timestamp); + if (type_a == nano::block_type::state) + { + result += sizeof (epoch); + } return result; } @@ -49,6 +54,10 @@ void nano::block_sideband::serialize (nano::stream & stream_a) const nano::write (stream_a, balance.bytes); } nano::write (stream_a, boost::endian::native_to_big (timestamp)); + if (type == nano::block_type::state) + { + nano::write (stream_a, epoch); + } } bool nano::block_sideband::deserialize (nano::stream & stream_a) @@ -76,6 +85,10 @@ bool nano::block_sideband::deserialize (nano::stream & stream_a) } nano::read (stream_a, timestamp); boost::endian::big_to_native_inplace (timestamp); + if (type == nano::block_type::state) + { + nano::read (stream_a, epoch); + } } catch (std::runtime_error &) { @@ -85,9 +98,10 @@ bool nano::block_sideband::deserialize (nano::stream & stream_a) return result; } -nano::summation_visitor::summation_visitor (nano::transaction const & transaction_a, nano::block_store const & store_a) : +nano::summation_visitor::summation_visitor (nano::transaction const & transaction_a, nano::block_store const & store_a, bool is_v14_upgrade_a) : transaction (transaction_a), -store (store_a) +store (store_a), +is_v14_upgrade (is_v14_upgrade_a) { } @@ -244,7 +258,7 @@ nano::uint128_t nano::summation_visitor::compute_internal (nano::summation_visit } else { - auto block (store.block_get (transaction, current->balance_hash)); + auto block (block_get (transaction, current->balance_hash)); assert (block != nullptr); block->visit (*this); } @@ -264,7 +278,7 @@ nano::uint128_t nano::summation_visitor::compute_internal (nano::summation_visit { if (!current->amount_hash.is_zero ()) { - auto block (store.block_get (transaction, current->amount_hash)); + auto block = block_get (transaction, current->amount_hash); if (block != nullptr) { block->visit (*this); @@ -322,6 +336,11 @@ nano::uint128_t nano::summation_visitor::compute_balance (nano::block_hash const return compute_internal (summation_type::balance, block_hash); } +std::shared_ptr nano::summation_visitor::block_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + return is_v14_upgrade ? store.block_get_v14 (transaction, hash_a) : store.block_get (transaction, hash_a); +} + nano::representative_visitor::representative_visitor (nano::transaction const & transaction_a, nano::block_store & store_a) : transaction (transaction_a), store (store_a), diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index e4cad3724d..415dd65a09 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -16,20 +16,19 @@ namespace nano { /** - * Encapsulates database specific container and provides uint256_union conversion of the data. + * Encapsulates database specific container */ template class db_val { public: - db_val (Val const & value_a, nano::epoch epoch_a = nano::epoch::unspecified) : - value (value_a), - epoch (epoch_a) + db_val (Val const & value_a) : + value (value_a) { } - db_val (nano::epoch epoch_a = nano::epoch::unspecified) : - db_val (0, nullptr, epoch_a) + db_val () : + db_val (0, nullptr) { } @@ -59,11 +58,17 @@ class db_val } db_val (nano::pending_info const & val_a) : - db_val (sizeof (val_a.source) + sizeof (val_a.amount), const_cast (&val_a)) + db_val (val_a.db_size (), const_cast (&val_a)) { static_assert (std::is_standard_layout::value, "Standard layout is required"); } + db_val (nano::pending_info_v14 const & val_a) : + db_val (val_a.db_size (), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + db_val (nano::pending_key const & val_a) : db_val (sizeof (val_a), const_cast (&val_a)) { @@ -122,7 +127,6 @@ class db_val explicit operator nano::account_info () const { nano::account_info result; - result.epoch_m = epoch; assert (size () == result.db_size ()); std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); return result; @@ -131,7 +135,6 @@ class db_val explicit operator nano::account_info_v13 () const { nano::account_info_v13 result; - result.epoch = epoch; assert (size () == result.db_size ()); std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); return result; @@ -140,7 +143,6 @@ class db_val explicit operator nano::account_info_v14 () const { nano::account_info_v14 result; - result.epoch = epoch; assert (size () == result.db_size ()); std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); return result; @@ -155,11 +157,19 @@ class db_val return result; } + explicit operator nano::pending_info_v14 () const + { + nano::pending_info_v14 result; + assert (size () == result.db_size ()); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); + return result; + } + explicit operator nano::pending_info () const { nano::pending_info result; - result.epoch = epoch; - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (nano::pending_info::source) + sizeof (nano::pending_info::amount), reinterpret_cast (&result)); + assert (size () == result.db_size ()); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); return result; } @@ -233,6 +243,21 @@ class db_val return result; } + explicit operator nano::state_block_w_sideband_v14 () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + auto error (false); + nano::state_block_w_sideband_v14 state_block_w_sideband_v14; + state_block_w_sideband_v14.state_block = std::make_shared (error, stream); + assert (!error); + + state_block_w_sideband_v14.sideband.type = nano::block_type::state; + error = state_block_w_sideband_v14.sideband.deserialize (stream); + assert (!error); + + return state_block_w_sideband_v14; + } + explicit operator nano::no_value () const { return no_value::dummy; @@ -314,12 +339,11 @@ class db_val // Must be specialized void * data () const; size_t size () const; - db_val (size_t size_a, void * data_a, nano::epoch epoch_a = nano::epoch::unspecified); + db_val (size_t size_a, void * data_a); void convert_buffer_to_value (); Val value; std::shared_ptr> buffer; - nano::epoch epoch{ nano::epoch::unspecified }; private: template @@ -336,7 +360,7 @@ class block_sideband final { public: block_sideband () = default; - block_sideband (nano::block_type, nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t); + block_sideband (nano::block_type, nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, nano::epoch); void serialize (nano::stream &) const; bool deserialize (nano::stream &); static size_t size (nano::block_type); @@ -346,6 +370,7 @@ class block_sideband final nano::amount balance{ 0 }; uint64_t height{ 0 }; uint64_t timestamp{ 0 }; + nano::epoch epoch{ nano::epoch::epoch_0 }; }; class transaction; class block_store; @@ -389,7 +414,7 @@ class summation_visitor final : public nano::block_visitor }; public: - summation_visitor (nano::transaction const &, nano::block_store const &); + summation_visitor (nano::transaction const &, nano::block_store const &, bool is_v14_upgrade = false); virtual ~summation_visitor () = default; /** Computes the balance as of \p block_hash */ nano::uint128_t compute_balance (nano::block_hash const & block_hash); @@ -420,6 +445,10 @@ class summation_visitor final : public nano::block_visitor void open_block (nano::open_block const &) override; void change_block (nano::change_block const &) override; void state_block (nano::state_block const &) override; + +private: + bool is_v14_upgrade; + std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &) const; }; /** @@ -514,8 +543,7 @@ class store_iterator final // Keep this in alphabetical order enum class tables { - accounts_v0, - accounts_v1, + accounts, blocks_info, // LMDB only cached_counts, // RocksDB only change_blocks, @@ -525,13 +553,11 @@ enum class tables online_weight, open_blocks, peers, - pending_v0, - pending_v1, + pending, receive_blocks, representation, send_blocks, - state_blocks_v0, - state_blocks_v1, + state_blocks, unchecked, vote }; @@ -609,10 +635,11 @@ class block_store public: virtual ~block_store () = default; virtual void initialize (nano::write_transaction const &, nano::genesis const &, nano::rep_weights &, std::atomic &, std::atomic &) = 0; - virtual void block_put (nano::write_transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) = 0; + virtual void block_put (nano::write_transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &) = 0; virtual nano::block_hash block_successor (nano::transaction const &, nano::block_hash const &) const = 0; virtual void block_successor_clear (nano::write_transaction const &, nano::block_hash const &) = 0; virtual std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &, nano::block_sideband * = nullptr) const = 0; + virtual std::shared_ptr block_get_v14 (nano::transaction const &, nano::block_hash const &, nano::block_sideband_v14 * = nullptr, bool * = nullptr) const = 0; virtual std::shared_ptr block_random (nano::transaction const &) = 0; virtual void block_del (nano::write_transaction const &, nano::block_hash const &) = 0; virtual bool block_exists (nano::transaction const &, nano::block_hash const &) = 0; @@ -633,12 +660,6 @@ class block_store virtual size_t account_count (nano::transaction const &) = 0; virtual void confirmation_height_clear (nano::write_transaction const &, nano::account const & account, uint64_t existing_confirmation_height) = 0; virtual void confirmation_height_clear (nano::write_transaction const &) = 0; - virtual nano::store_iterator latest_v0_begin (nano::transaction const &, nano::account const &) = 0; - virtual nano::store_iterator latest_v0_begin (nano::transaction const &) = 0; - virtual nano::store_iterator latest_v0_end () = 0; - virtual nano::store_iterator latest_v1_begin (nano::transaction const &, nano::account const &) = 0; - virtual nano::store_iterator latest_v1_begin (nano::transaction const &) = 0; - virtual nano::store_iterator latest_v1_end () = 0; virtual nano::store_iterator latest_begin (nano::transaction const &, nano::account const &) = 0; virtual nano::store_iterator latest_begin (nano::transaction const &) = 0; virtual nano::store_iterator latest_end () = 0; @@ -647,12 +668,6 @@ class block_store virtual void pending_del (nano::write_transaction const &, nano::pending_key const &) = 0; virtual bool pending_get (nano::transaction const &, nano::pending_key const &, nano::pending_info &) = 0; virtual bool pending_exists (nano::transaction const &, nano::pending_key const &) = 0; - virtual nano::store_iterator pending_v0_begin (nano::transaction const &, nano::pending_key const &) = 0; - virtual nano::store_iterator pending_v0_begin (nano::transaction const &) = 0; - virtual nano::store_iterator pending_v0_end () = 0; - virtual nano::store_iterator pending_v1_begin (nano::transaction const &, nano::pending_key const &) = 0; - virtual nano::store_iterator pending_v1_begin (nano::transaction const &) = 0; - virtual nano::store_iterator pending_v1_end () = 0; virtual nano::store_iterator pending_begin (nano::transaction const &, nano::pending_key const &) = 0; virtual nano::store_iterator pending_begin (nano::transaction const &) = 0; virtual nano::store_iterator pending_end () = 0; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index a5114f5831..3c1bf275c3 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -27,9 +27,8 @@ class block_store_partial : public block_store void initialize (nano::write_transaction const & transaction_a, nano::genesis const & genesis_a, nano::rep_weights & rep_weights, std::atomic & cemented_count, std::atomic & block_count_cache) override { auto hash_l (genesis_a.hash ()); - assert (latest_v0_begin (transaction_a) == latest_v0_end ()); - assert (latest_v1_begin (transaction_a) == latest_v1_end ()); - nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch ()); + assert (latest_begin (transaction_a) == latest_end ()); + nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); block_put (transaction_a, hash_l, *genesis_a.open, sideband); ++block_count_cache; confirmation_height_put (transaction_a, network_params.ledger.genesis_account, 1); @@ -86,7 +85,7 @@ class block_store_partial : public block_store return result; } - void block_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a = nano::epoch::epoch_0) override + void block_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a) override { assert (block_a.type () == sideband_a.type); assert (sideband_a.successor.is_zero () || block_exists (transaction_a, sideband_a.successor)); @@ -96,7 +95,7 @@ class block_store_partial : public block_store block_a.serialize (stream); sideband_a.serialize (stream); } - block_raw_put (transaction_a, vector, block_a.type (), epoch_a, hash_a); + block_raw_put (transaction_a, vector, block_a.type (), hash_a); nano::block_predecessor_set predecessor (transaction_a, *this); block_a.visit (predecessor); assert (block_a.previous ().is_zero () || block_successor (transaction_a, block_a.previous ()) == hash_a); @@ -239,11 +238,10 @@ class block_store_partial : public block_store { nano::block_type type; auto value (block_raw_get (transaction_a, hash_a, type)); - auto version (block_version (transaction_a, hash_a)); assert (value.size () != 0); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); std::fill_n (data.begin () + block_successor_offset (transaction_a, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); - block_raw_put (transaction_a, data, type, version, hash_a); + block_raw_put (transaction_a, data, type, hash_a); } void unchecked_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr const & block_a) override @@ -331,16 +329,6 @@ class block_store_partial : public block_store return nano::store_iterator (nullptr); } - nano::store_iterator pending_v0_end () override - { - return nano::store_iterator (nullptr); - } - - nano::store_iterator pending_v1_end () override - { - return nano::store_iterator (nullptr); - } - nano::store_iterator online_weight_end () const override { return nano::store_iterator (nullptr); @@ -351,16 +339,6 @@ class block_store_partial : public block_store return nano::store_iterator (nullptr); } - nano::store_iterator latest_v0_end () override - { - return nano::store_iterator (nullptr); - } - - nano::store_iterator latest_v1_end () override - { - return nano::store_iterator (nullptr); - } - nano::store_iterator confirmation_height_end () override { return nano::store_iterator (nullptr); @@ -373,29 +351,24 @@ class block_store_partial : public block_store void block_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override { - auto status = del (transaction_a, tables::state_blocks_v1, hash_a); + auto status = del (transaction_a, tables::state_blocks, hash_a); release_assert (success (status) || not_found (status)); if (!success (status)) { - auto status = del (transaction_a, tables::state_blocks_v0, hash_a); + auto status = del (transaction_a, tables::send_blocks, hash_a); release_assert (success (status) || not_found (status)); if (!success (status)) { - auto status = del (transaction_a, tables::send_blocks, hash_a); + auto status = del (transaction_a, tables::receive_blocks, hash_a); release_assert (success (status) || not_found (status)); if (!success (status)) { - auto status = del (transaction_a, tables::receive_blocks, hash_a); + auto status = del (transaction_a, tables::open_blocks, hash_a); release_assert (success (status) || not_found (status)); if (!success (status)) { - auto status = del (transaction_a, tables::open_blocks, hash_a); - release_assert (success (status) || not_found (status)); - if (!success (status)) - { - auto status = del (transaction_a, tables::change_blocks, hash_a); - release_assert (success (status)); - } + auto status = del (transaction_a, tables::change_blocks, hash_a); + release_assert (success (status)); } } } @@ -420,66 +393,47 @@ class block_store_partial : public block_store nano::epoch block_version (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override { nano::db_val value; - auto status = get (transaction_a, tables::state_blocks_v1, nano::db_val (hash_a), value); - release_assert (success (status) || not_found (status)); - return status == 0 ? nano::epoch::epoch_1 : nano::epoch::epoch_0; + nano::block_sideband sideband; + auto block = block_get (transaction_a, hash_a, &sideband); + if (sideband.type == nano::block_type::state) + { + return sideband.epoch; + } + + return nano::epoch::epoch_0; } - void block_raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) + void block_raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::block_hash const & hash_a) { - auto database_a = block_database (block_type_a, epoch_a); + auto database_a = block_database (block_type_a); nano::db_val value{ data.size (), (void *)data.data () }; auto status = put (transaction_a, database_a, hash_a, value); release_assert (success (status)); } - void pending_put (nano::write_transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_a) override + void pending_put (nano::write_transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_info_a) override { - nano::db_val pending (pending_a); - auto status = put (transaction_a, get_pending_db (pending_a.epoch), key_a, pending); + nano::db_val pending (pending_info_a); + auto status = put (transaction_a, tables::pending, key_a, pending); release_assert (success (status)); } void pending_del (nano::write_transaction const & transaction_a, nano::pending_key const & key_a) override { - auto status1 = del (transaction_a, tables::pending_v1, key_a); - if (!success (status1)) - { - release_assert (not_found (status1)); - auto status2 = del (transaction_a, tables::pending_v0, key_a); - release_assert (success (status2)); - } + auto status1 = del (transaction_a, tables::pending, key_a); + release_assert (success (status1)); } bool pending_get (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info & pending_a) override { nano::db_val value; nano::db_val key (key_a); - auto status1 = get (transaction_a, tables::pending_v1, key, value); + auto status1 = get (transaction_a, tables::pending, key, value); release_assert (success (status1) || not_found (status1)); - bool result (false); - nano::epoch epoch; + bool result (true); if (success (status1)) - { - epoch = nano::epoch::epoch_1; - } - else - { - auto status2 = get (transaction_a, tables::pending_v0, key, value); - release_assert (success (status2) || not_found (status2)); - if (success (status2)) - { - epoch = nano::epoch::epoch_0; - } - else - { - result = true; - } - } - if (!result) { nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - pending_a.epoch = epoch; result = pending_a.deserialize (stream); } return result; @@ -571,73 +525,31 @@ class block_store_partial : public block_store release_assert (success (status)); } - tables get_account_db (nano::epoch epoch_a) const - { - tables db; - switch (epoch_a) - { - case nano::epoch::invalid: - case nano::epoch::unspecified: - assert (false); - case nano::epoch::epoch_0: - db = tables::accounts_v0; - break; - case nano::epoch::epoch_1: - db = tables::accounts_v1; - break; - } - return db; - } - void account_put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::account_info const & info_a) override { // Check we are still in sync with other tables assert (confirmation_height_exists (transaction_a, account_a)); nano::db_val info (info_a); - auto status = put (transaction_a, get_account_db (info_a.epoch ()), account_a, info); + auto status = put (transaction_a, tables::accounts, account_a, info); release_assert (success (status)); } void account_del (nano::write_transaction const & transaction_a, nano::account const & account_a) override { - auto status1 = del (transaction_a, tables::accounts_v1, account_a); - if (!success (status1)) - { - release_assert (not_found (status1)); - auto status2 (del (transaction_a, tables::accounts_v0, account_a)); - release_assert (success (status2)); - } + auto status1 = del (transaction_a, tables::accounts, account_a); + release_assert (success (status1)); } bool account_get (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info & info_a) override { nano::db_val value; nano::db_val account (account_a); - auto status1 (get (transaction_a, tables::accounts_v1, account, value)); + auto status1 (get (transaction_a, tables::accounts, account, value)); release_assert (success (status1) || not_found (status1)); - bool result (false); - nano::epoch epoch; + bool result (true); if (success (status1)) - { - epoch = nano::epoch::epoch_1; - } - else - { - auto status2 (get (transaction_a, tables::accounts_v0, account, value)); - release_assert (success (status2) || not_found (status2)); - if (success (status2)) - { - epoch = nano::epoch::epoch_0; - } - else - { - result = true; - } - } - if (!result) { nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - info_a.epoch_m = epoch; result = info_a.deserialize (stream); } return result; @@ -701,14 +613,13 @@ class block_store_partial : public block_store result.receive = count (transaction_a, tables::receive_blocks); result.open = count (transaction_a, tables::open_blocks); result.change = count (transaction_a, tables::change_blocks); - result.state_v0 = count (transaction_a, tables::state_blocks_v0); - result.state_v1 = count (transaction_a, tables::state_blocks_v1); + result.state = count (transaction_a, tables::state_blocks); return result; } size_t account_count (nano::transaction const & transaction_a) override { - return count (transaction_a, { tables::accounts_v0, tables::accounts_v1 }); + return count (transaction_a, tables::accounts); } std::shared_ptr block_random (nano::transaction const & transaction_a) override @@ -745,15 +656,7 @@ class block_store_partial : public block_store } else { - region -= count.change; - if (region < count.state_v0) - { - result = derived_store.template block_random (transaction_a, tables::state_blocks_v0); - } - else - { - result = derived_store.template block_random (transaction_a, tables::state_blocks_v1); - } + result = derived_store.template block_random (transaction_a, tables::state_blocks); } } } @@ -800,62 +703,22 @@ class block_store_partial : public block_store nano::store_iterator latest_begin (nano::transaction const & transaction_a, nano::account const & account_a) override { - return make_merge_iterator (transaction_a, tables::accounts_v0, tables::accounts_v1, nano::db_val (account_a)); + return make_iterator (transaction_a, tables::accounts, nano::db_val (account_a)); } nano::store_iterator latest_begin (nano::transaction const & transaction_a) override { - return make_merge_iterator (transaction_a, tables::accounts_v0, tables::accounts_v1); - } - - nano::store_iterator latest_v0_begin (nano::transaction const & transaction_a, nano::account const & account_a) override - { - return make_iterator (transaction_a, tables::accounts_v0, nano::db_val (account_a)); - } - - nano::store_iterator latest_v0_begin (nano::transaction const & transaction_a) override - { - return make_iterator (transaction_a, tables::accounts_v0); - } - - nano::store_iterator latest_v1_begin (nano::transaction const & transaction_a, nano::account const & account_a) override - { - return make_iterator (transaction_a, tables::accounts_v1, nano::db_val (account_a)); - } - - nano::store_iterator latest_v1_begin (nano::transaction const & transaction_a) override - { - return make_iterator (transaction_a, tables::accounts_v1); - } - - nano::store_iterator pending_v0_begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) override - { - return make_iterator (transaction_a, tables::pending_v0, nano::db_val (key_a)); - } - - nano::store_iterator pending_v0_begin (nano::transaction const & transaction_a) override - { - return make_iterator (transaction_a, tables::pending_v0); - } - - nano::store_iterator pending_v1_begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) override - { - return make_iterator (transaction_a, tables::pending_v1, nano::db_val (key_a)); - } - - nano::store_iterator pending_v1_begin (nano::transaction const & transaction_a) override - { - return make_iterator (transaction_a, tables::pending_v1); + return make_iterator (transaction_a, tables::accounts); } nano::store_iterator pending_begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) override { - return make_merge_iterator (transaction_a, tables::pending_v0, tables::pending_v1, nano::db_val (key_a)); + return make_iterator (transaction_a, tables::pending, nano::db_val (key_a)); } nano::store_iterator pending_begin (nano::transaction const & transaction_a) override { - return make_merge_iterator (transaction_a, tables::pending_v0, tables::pending_v1); + return make_iterator (transaction_a, tables::pending); } nano::store_iterator unchecked_begin (nano::transaction const & transaction_a) override @@ -931,18 +794,6 @@ class block_store_partial : public block_store return static_cast (*this).template make_iterator (transaction_a, table_a, key); } - template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const - { - return static_cast (*this).template make_merge_iterator (transaction_a, table1_a, table2_a); - } - - template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::db_val const & key) const - { - return static_cast (*this).template make_merge_iterator (transaction_a, table1_a, table2_a, key); - } - bool entry_has_sideband (size_t entry_size_a, nano::block_type type_a) const { return entry_size_a == nano::block::size (type_a) + nano::block_sideband::size (type_a); @@ -1062,11 +913,7 @@ class block_store_partial : public block_store } case nano::block_type::state: { - status = get (transaction_a, tables::state_blocks_v1, hash, value); - if (!success (status)) - { - status = get (transaction_a, tables::state_blocks_v0, hash, value); - } + status = get (transaction_a, tables::state_blocks, hash, value); break; } case nano::block_type::invalid: @@ -1085,16 +932,8 @@ class block_store_partial : public block_store return result; } - tables block_database (nano::block_type type_a, nano::epoch epoch_a) + tables block_database (nano::block_type type_a) { - if (type_a == nano::block_type::state) - { - assert (epoch_a == nano::epoch::epoch_0 || epoch_a == nano::epoch::epoch_1); - } - else - { - assert (epoch_a == nano::epoch::epoch_0); - } tables result = tables::frontiers; switch (type_a) { @@ -1111,17 +950,7 @@ class block_store_partial : public block_store result = tables::change_blocks; break; case nano::block_type::state: - switch (epoch_a) - { - case nano::epoch::epoch_0: - result = tables::state_blocks_v0; - break; - case nano::epoch::epoch_1: - result = tables::state_blocks_v1; - break; - default: - assert (false); - } + result = tables::state_blocks; break; default: assert (false); @@ -1130,24 +959,6 @@ class block_store_partial : public block_store return result; } - tables get_pending_db (nano::epoch epoch_a) const - { - tables db; - switch (epoch_a) - { - case nano::epoch::invalid: - case nano::epoch::unspecified: - assert (false); - case nano::epoch::epoch_0: - db = tables::pending_v0; - break; - case nano::epoch::epoch_1: - db = tables::pending_v1; - break; - } - return db; - } - size_t count (nano::transaction const & transaction_a, std::initializer_list dbs_a) const { size_t total_count = 0; @@ -1198,11 +1009,10 @@ class block_predecessor_set : public nano::block_visitor auto hash (block_a.hash ()); nano::block_type type; auto value (store.block_raw_get (transaction, block_a.previous (), type)); - auto version (store.block_version (transaction, block_a.previous ())); assert (value.size () != 0); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + store.block_successor_offset (transaction, value.size (), type)); - store.block_raw_put (transaction, data, type, version, block_a.previous ()); + store.block_raw_put (transaction, data, type, block_a.previous ()); } void send_block (nano::send_block const & block_a) override { diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index de25b90352..095c3b0bad 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -93,10 +93,17 @@ genesis_block (network_a == nano::nano_networks::nano_test_network ? nano_test_g genesis_amount (std::numeric_limits::max ()), burn_account (0) { - nano::link epoch_link; - const char * epoch_message ("epoch v1 block"); - strncpy ((char *)epoch_link.bytes.data (), epoch_message, epoch_link.bytes.size ()); - epochs.add (nano::epoch::epoch_1, genesis_account, epoch_link); + nano::link epoch_link_v1; + const char * epoch_message_v1 ("epoch v1 block"); + strncpy ((char *)epoch_link_v1.bytes.data (), epoch_message_v1, epoch_link_v1.bytes.size ()); + epochs.add (nano::epoch::epoch_1, genesis_account, epoch_link_v1); + + nano::link epoch_link_v2; + auto nano_live_epoch_v2_signer = genesis_account; + auto epoch_v2_signer (network_a == nano::nano_networks::nano_test_network ? nano_test_account : network_a == nano::nano_networks::nano_beta_network ? nano_beta_account : nano_live_epoch_v2_signer); + const char * epoch_message_v2 ("epoch v2 block"); + strncpy ((char *)epoch_link_v2.bytes.data (), epoch_message_v2, epoch_link_v2.bytes.size ()); + epochs.add (nano::epoch::epoch_2, epoch_v2_signer, epoch_link_v2); } nano::random_constants::random_constants () @@ -204,6 +211,7 @@ bool nano::account_info::deserialize (nano::stream & stream_a) nano::read (stream_a, balance.bytes); nano::read (stream_a, modified); nano::read (stream_a, block_count); + nano::read (stream_a, epoch_m); } catch (std::runtime_error const &) { @@ -231,7 +239,8 @@ size_t nano::account_info::db_size () const assert (reinterpret_cast (&open_block) + sizeof (open_block) == reinterpret_cast (&balance)); assert (reinterpret_cast (&balance) + sizeof (balance) == reinterpret_cast (&modified)); assert (reinterpret_cast (&modified) + sizeof (modified) == reinterpret_cast (&block_count)); - return sizeof (head) + sizeof (representative) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count); + assert (reinterpret_cast (&block_count) + sizeof (block_count) == reinterpret_cast (&epoch_m)); + return sizeof (head) + sizeof (representative) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count) + sizeof (epoch_m); } nano::epoch nano::account_info::epoch () const @@ -241,7 +250,7 @@ nano::epoch nano::account_info::epoch () const size_t nano::block_counts::sum () const { - return send + receive + open + change + state_v0 + state_v1; + return send + receive + open + change + state; } nano::pending_info::pending_info (nano::account const & source_a, nano::amount const & amount_a, nano::epoch epoch_a) : @@ -258,6 +267,7 @@ bool nano::pending_info::deserialize (nano::stream & stream_a) { nano::read (stream_a, source.bytes); nano::read (stream_a, amount.bytes); + nano::read (stream_a, epoch); } catch (std::runtime_error const &) { @@ -267,6 +277,11 @@ bool nano::pending_info::deserialize (nano::stream & stream_a) return error; } +size_t nano::pending_info::db_size () const +{ + return sizeof (source) + sizeof (amount) + sizeof (epoch); +} + bool nano::pending_info::operator== (nano::pending_info const & other_a) const { return source == other_a.source && amount == other_a.amount && epoch == other_a.epoch; diff --git a/nano/secure/common.hpp b/nano/secure/common.hpp index 567991fda8..8baf622baf 100644 --- a/nano/secure/common.hpp +++ b/nano/secure/common.hpp @@ -106,7 +106,8 @@ class pending_info final { public: pending_info () = default; - pending_info (nano::account const &, nano::amount const &, epoch); + pending_info (nano::account const &, nano::amount const &, nano::epoch); + size_t db_size () const; bool deserialize (nano::stream &); bool operator== (nano::pending_info const &) const; nano::account source{ 0 }; @@ -213,8 +214,7 @@ class block_counts final size_t receive{ 0 }; size_t open{ 0 }; size_t change{ 0 }; - size_t state_v0{ 0 }; - size_t state_v1{ 0 }; + size_t state{ 0 }; }; using vote_blocks_vec_iter = std::vector, nano::block_hash>>::const_iterator; class iterate_vote_blocks_as_hash final diff --git a/nano/secure/epoch.cpp b/nano/secure/epoch.cpp index b1f5cc1f6b..9947912a52 100644 --- a/nano/secure/epoch.cpp +++ b/nano/secure/epoch.cpp @@ -27,3 +27,10 @@ void nano::epochs::add (nano::epoch epoch_a, nano::public_key const & signer_a, assert (epochs_m.find (epoch_a) == epochs_m.end ()); epochs_m[epoch_a] = { signer_a, link_a }; } + +bool nano::epochs::is_sequential (nano::epoch epoch_a, nano::epoch new_epoch_a) +{ + auto head_epoch = std::underlying_type_t (epoch_a); + bool is_valid_epoch (head_epoch >= std::underlying_type_t (nano::epoch::epoch_0)); + return is_valid_epoch && (std::underlying_type_t (new_epoch_a) == (head_epoch + 1)); +} diff --git a/nano/secure/epoch.hpp b/nano/secure/epoch.hpp index 96a70b8322..f3132b9238 100644 --- a/nano/secure/epoch.hpp +++ b/nano/secure/epoch.hpp @@ -16,7 +16,8 @@ enum class epoch : uint8_t unspecified = 1, epoch_begin = 2, epoch_0 = 2, - epoch_1 = 3 + epoch_1 = 3, + epoch_2 = 4 }; } namespace std @@ -26,8 +27,8 @@ struct hash<::nano::epoch> { std::size_t operator() (::nano::epoch const & epoch_a) const { - std::hash::type> hash; - return hash (static_cast::type> (epoch_a)); + std::hash> hash; + return hash (static_cast> (epoch_a)); } }; } @@ -47,6 +48,8 @@ class epochs nano::public_key const & signer (nano::epoch epoch_a) const; nano::epoch epoch (nano::link const & link_a) const; void add (nano::epoch epoch_a, nano::public_key const & signer_a, nano::link const & link_a); + /** Checks that new_epoch is 1 version higher than epoch */ + static bool is_sequential (nano::epoch epoch_a, nano::epoch new_epoch_a); private: std::unordered_map epochs_m; diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 37f4fafae0..e73b0f3e13 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -188,50 +188,58 @@ class ledger_processor : public nano::block_visitor nano::write_transaction const & transaction; nano::signature_verification verification; nano::process_return result; + +private: + bool validate_epoch_block (nano::state_block const & block_a); }; -void ledger_processor::state_block (nano::state_block const & block_a) +// Returns true if this block which has an epoch link is correctly formed. +bool ledger_processor::validate_epoch_block (nano::state_block const & block_a) { - result.code = nano::process_result::progress; - auto is_epoch_block (false); - // Check if this is an epoch block - if (ledger.is_epoch_link (block_a.hashables.link)) + assert (ledger.is_epoch_link (block_a.hashables.link)); + nano::amount prev_balance (0); + if (!block_a.hashables.previous.is_zero ()) { - nano::amount prev_balance (0); - if (!block_a.hashables.previous.is_zero ()) + result.code = ledger.store.block_exists (transaction, block_a.hashables.previous) ? nano::process_result::progress : nano::process_result::gap_previous; + if (result.code == nano::process_result::progress) { - result.code = ledger.store.block_exists (transaction, block_a.hashables.previous) ? nano::process_result::progress : nano::process_result::gap_previous; - if (result.code == nano::process_result::progress) - { - prev_balance = ledger.balance (transaction, block_a.hashables.previous); - } - else if (result.verified == nano::signature_verification::unknown) + prev_balance = ledger.balance (transaction, block_a.hashables.previous); + } + else if (result.verified == nano::signature_verification::unknown) + { + // Check for possible regular state blocks with epoch link (send subtype) + if (validate_message (block_a.hashables.account, block_a.hash (), block_a.signature)) { - // Check for possible regular state blocks with epoch link (send subtype) - if (validate_message (block_a.hashables.account, block_a.hash (), block_a.signature)) + // Is epoch block signed correctly + if (validate_message (ledger.signer (block_a.link ()), block_a.hash (), block_a.signature)) { - // Is epoch block signed correctly - if (validate_message (ledger.signer (block_a.link ()), block_a.hash (), block_a.signature)) - { - result.verified = nano::signature_verification::invalid; - result.code = nano::process_result::bad_signature; - } - else - { - result.verified = nano::signature_verification::valid_epoch; - } + result.verified = nano::signature_verification::invalid; + result.code = nano::process_result::bad_signature; } else { - result.verified = nano::signature_verification::valid; + result.verified = nano::signature_verification::valid_epoch; } } + else + { + result.verified = nano::signature_verification::valid; + } } - if (block_a.hashables.balance == prev_balance) - { - is_epoch_block = true; - } } + return (block_a.hashables.balance == prev_balance); +} + +void ledger_processor::state_block (nano::state_block const & block_a) +{ + result.code = nano::process_result::progress; + auto is_epoch_block = false; + if (ledger.is_epoch_link (block_a.hashables.link)) + { + // This function also modifies the result variable if epoch is mal-formed + is_epoch_block = validate_epoch_block (block_a); + } + if (result.code == nano::process_result::progress) { if (is_epoch_block) @@ -324,8 +332,8 @@ void ledger_processor::state_block_impl (nano::state_block const & block_a) { ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::state_block); result.state_is_send = is_send; - nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch ()); - ledger.store.block_put (transaction, hash, block_a, sideband, epoch); + nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), epoch); + ledger.store.block_put (transaction, hash, block_a, sideband); if (!info.head.is_zero ()) { @@ -370,11 +378,11 @@ void ledger_processor::epoch_block_impl (nano::state_block const & block_a) // Validate block if not verified outside of ledger if (result.verified != nano::signature_verification::valid_epoch) { - result.code = validate_message (ledger.signer (block_a.link ()), hash, block_a.signature) ? nano::process_result::bad_signature : nano::process_result::progress; // Is this block signed correctly (Unambiguous) + result.code = validate_message (ledger.signer (block_a.hashables.link), hash, block_a.signature) ? nano::process_result::bad_signature : nano::process_result::progress; // Is this block signed correctly (Unambiguous) } if (result.code == nano::process_result::progress) { - assert (!validate_message (ledger.signer (block_a.link ()), hash, block_a.signature)); + assert (!validate_message (ledger.signer (block_a.hashables.link), hash, block_a.signature)); result.verified = nano::signature_verification::valid_epoch; result.code = block_a.hashables.account.is_zero () ? nano::process_result::opened_burn_account : nano::process_result::progress; // Is this for the burn account? (Unambiguous) if (result.code == nano::process_result::progress) @@ -400,7 +408,10 @@ void ledger_processor::epoch_block_impl (nano::state_block const & block_a) } if (result.code == nano::process_result::progress) { - result.code = info.epoch () == nano::epoch::epoch_0 ? nano::process_result::progress : nano::process_result::block_position; + auto epoch = ledger.network_params.ledger.epochs.epoch (block_a.hashables.link); + // Must be an epoch for an unopened account or the epoch upgrade must be sequential + auto is_valid_epoch_upgrade = account_error ? static_cast> (epoch) > 0 : nano::epochs::is_sequential (info.epoch (), epoch); + result.code = is_valid_epoch_upgrade ? nano::process_result::progress : nano::process_result::block_position; if (result.code == nano::process_result::progress) { result.code = block_a.hashables.balance == info.balance ? nano::process_result::progress : nano::process_result::balance_mismatch; @@ -409,9 +420,9 @@ void ledger_processor::epoch_block_impl (nano::state_block const & block_a) ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::epoch_block); result.account = block_a.hashables.account; result.amount = 0; - nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch ()); - ledger.store.block_put (transaction, hash, block_a, sideband, nano::epoch::epoch_1); - nano::account_info new_info (hash, block_a.representative (), info.open_block.is_zero () ? hash : info.open_block, info.balance, nano::seconds_since_epoch (), info.block_count + 1, nano::epoch::epoch_1); + nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), epoch); + ledger.store.block_put (transaction, hash, block_a, sideband); + nano::account_info new_info (hash, block_a.representative (), info.open_block.is_zero () ? hash : info.open_block, info.balance, nano::seconds_since_epoch (), info.block_count + 1, epoch); ledger.change_latest (transaction, block_a.hashables.account, info, new_info); if (!ledger.store.frontier_get (transaction, info.head).is_zero ()) { @@ -457,7 +468,7 @@ void ledger_processor::change_block (nano::change_block const & block_a) { assert (!validate_message (account, hash, block_a.signature)); result.verified = nano::signature_verification::valid; - nano::block_sideband sideband (nano::block_type::change, account, 0, info.balance, info.block_count + 1, nano::seconds_since_epoch ()); + nano::block_sideband sideband (nano::block_type::change, account, 0, info.balance, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); ledger.store.block_put (transaction, hash, block_a, sideband); auto balance (ledger.balance (transaction, block_a.hashables.previous)); ledger.rep_weights.representation_add (block_a.representative (), balance); @@ -513,7 +524,7 @@ void ledger_processor::send_block (nano::send_block const & block_a) { auto amount (info.balance.number () - block_a.hashables.balance.number ()); ledger.rep_weights.representation_add (info.representative, 0 - amount); - nano::block_sideband sideband (nano::block_type::send, account, 0, block_a.hashables.balance /* unused */, info.block_count + 1, nano::seconds_since_epoch ()); + nano::block_sideband sideband (nano::block_type::send, account, 0, block_a.hashables.balance /* unused */, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); ledger.store.block_put (transaction, hash, block_a, sideband); nano::account_info new_info (hash, info.representative, info.open_block, block_a.hashables.balance, nano::seconds_since_epoch (), info.block_count + 1, nano::epoch::epoch_0); ledger.change_latest (transaction, account, info, new_info); @@ -581,7 +592,7 @@ void ledger_processor::receive_block (nano::receive_block const & block_a) (void)error; assert (!error); ledger.store.pending_del (transaction, key); - nano::block_sideband sideband (nano::block_type::receive, account, 0, new_balance, info.block_count + 1, nano::seconds_since_epoch ()); + nano::block_sideband sideband (nano::block_type::receive, account, 0, new_balance, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); ledger.store.block_put (transaction, hash, block_a, sideband); nano::account_info new_info (hash, info.representative, info.open_block, new_balance, nano::seconds_since_epoch (), info.block_count + 1, nano::epoch::epoch_0); ledger.change_latest (transaction, account, info, new_info); @@ -645,7 +656,7 @@ void ledger_processor::open_block (nano::open_block const & block_a) (void)error; assert (!error); ledger.store.pending_del (transaction, key); - nano::block_sideband sideband (nano::block_type::open, block_a.hashables.account, 0, pending.amount, 1, nano::seconds_since_epoch ()); + nano::block_sideband sideband (nano::block_type::open, block_a.hashables.account, 0, pending.amount, 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); ledger.store.block_put (transaction, hash, block_a, sideband); nano::account_info new_info (hash, block_a.representative (), hash, pending.amount.number (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0); ledger.change_latest (transaction, block_a.hashables.account, info, new_info); @@ -725,12 +736,7 @@ nano::uint128_t nano::ledger::account_pending (nano::transaction const & transac { nano::uint128_t result (0); nano::account end (account_a.number () + 1); - for (auto i (store.pending_v0_begin (transaction_a, nano::pending_key (account_a, 0))), n (store.pending_v0_begin (transaction_a, nano::pending_key (end, 0))); i != n; ++i) - { - nano::pending_info const & info (i->second); - result += info.amount.number (); - } - for (auto i (store.pending_v1_begin (transaction_a, nano::pending_key (account_a, 0))), n (store.pending_v1_begin (transaction_a, nano::pending_key (end, 0))); i != n; ++i) + for (auto i (store.pending_begin (transaction_a, nano::pending_key (account_a, 0))), n (store.pending_begin (transaction_a, nano::pending_key (end, 0))); i != n; ++i) { nano::pending_info const & info (i->second); result += info.amount.number (); @@ -1022,7 +1028,7 @@ nano::account const & nano::ledger::signer (nano::link const & link_a) const nano::link const & nano::ledger::link (nano::epoch epoch_a) const { - return network_params.ledger.epochs.link (nano::epoch::epoch_1); + return network_params.ledger.epochs.link (epoch_a); } void nano::ledger::change_latest (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::account_info const & old_a, nano::account_info const & new_a) diff --git a/nano/secure/versioning.cpp b/nano/secure/versioning.cpp index 6ab77518f7..c05bdfd323 100644 --- a/nano/secure/versioning.cpp +++ b/nano/secure/versioning.cpp @@ -2,6 +2,8 @@ #include +#include + nano::account_info_v1::account_info_v1 (MDB_val const & val_a) { assert (val_a.mv_size == sizeof (*this)); @@ -31,6 +33,39 @@ destination (destination_a) { } +nano::pending_info_v14::pending_info_v14 (nano::account const & source_a, nano::amount const & amount_a, nano::epoch epoch_a) : +source (source_a), +amount (amount_a), +epoch (epoch_a) +{ +} + +bool nano::pending_info_v14::deserialize (nano::stream & stream_a) +{ + auto error (false); + try + { + nano::read (stream_a, source.bytes); + nano::read (stream_a, amount.bytes); + } + catch (std::runtime_error const &) + { + error = true; + } + + return error; +} + +size_t nano::pending_info_v14::db_size () const +{ + return sizeof (source) + sizeof (amount); +} + +bool nano::pending_info_v14::operator== (nano::pending_info_v14 const & other_a) const +{ + return source == other_a.source && amount == other_a.amount && epoch == other_a.epoch; +} + nano::account_info_v5::account_info_v5 (MDB_val const & val_a) { assert (val_a.mv_size == sizeof (*this)); @@ -92,3 +127,85 @@ size_t nano::account_info_v14::db_size () const assert (reinterpret_cast (&block_count) + sizeof (block_count) == reinterpret_cast (&confirmation_height)); return sizeof (head) + sizeof (rep_block) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count) + sizeof (confirmation_height); } + +nano::block_sideband_v14::block_sideband_v14 (nano::block_type type_a, nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a) : +type (type_a), +successor (successor_a), +account (account_a), +balance (balance_a), +height (height_a), +timestamp (timestamp_a) +{ +} + +size_t nano::block_sideband_v14::size (nano::block_type type_a) +{ + size_t result (0); + result += sizeof (successor); + if (type_a != nano::block_type::state && type_a != nano::block_type::open) + { + result += sizeof (account); + } + if (type_a != nano::block_type::open) + { + result += sizeof (height); + } + if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open) + { + result += sizeof (balance); + } + result += sizeof (timestamp); + return result; +} + +void nano::block_sideband_v14::serialize (nano::stream & stream_a) const +{ + nano::write (stream_a, successor.bytes); + if (type != nano::block_type::state && type != nano::block_type::open) + { + nano::write (stream_a, account.bytes); + } + if (type != nano::block_type::open) + { + nano::write (stream_a, boost::endian::native_to_big (height)); + } + if (type == nano::block_type::receive || type == nano::block_type::change || type == nano::block_type::open) + { + nano::write (stream_a, balance.bytes); + } + nano::write (stream_a, boost::endian::native_to_big (timestamp)); +} + +bool nano::block_sideband_v14::deserialize (nano::stream & stream_a) +{ + bool result (false); + try + { + nano::read (stream_a, successor.bytes); + if (type != nano::block_type::state && type != nano::block_type::open) + { + nano::read (stream_a, account.bytes); + } + if (type != nano::block_type::open) + { + nano::read (stream_a, height); + boost::endian::big_to_native_inplace (height); + } + else + { + height = 1; + } + if (type == nano::block_type::receive || type == nano::block_type::change || type == nano::block_type::open) + { + nano::read (stream_a, balance.bytes); + } + nano::read (stream_a, timestamp); + boost::endian::big_to_native_inplace (timestamp); + } + catch (std::runtime_error &) + { + result = true; + } + + return result; +} diff --git a/nano/secure/versioning.hpp b/nano/secure/versioning.hpp index 68221b5dba..f9346cb857 100644 --- a/nano/secure/versioning.hpp +++ b/nano/secure/versioning.hpp @@ -29,6 +29,18 @@ class pending_info_v3 final nano::amount amount{ 0 }; nano::account destination{ 0 }; }; +class pending_info_v14 final +{ +public: + pending_info_v14 () = default; + pending_info_v14 (nano::account const &, nano::amount const &, nano::epoch); + size_t db_size () const; + bool deserialize (nano::stream &); + bool operator== (nano::pending_info_v14 const &) const; + nano::account source{ 0 }; + nano::amount amount{ 0 }; + nano::epoch epoch{ nano::epoch::epoch_0 }; +}; class account_info_v5 final { public: @@ -70,4 +82,25 @@ class account_info_v14 final uint64_t confirmation_height{ 0 }; nano::epoch epoch{ nano::epoch::epoch_0 }; }; +class block_sideband_v14 final +{ +public: + block_sideband_v14 () = default; + block_sideband_v14 (nano::block_type, nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t); + void serialize (nano::stream &) const; + bool deserialize (nano::stream &); + static size_t size (nano::block_type); + nano::block_type type{ nano::block_type::invalid }; + nano::block_hash successor{ 0 }; + nano::account account{ 0 }; + nano::amount balance{ 0 }; + uint64_t height{ 0 }; + uint64_t timestamp{ 0 }; +}; +class state_block_w_sideband_v14 +{ +public: + std::shared_ptr state_block; + nano::block_sideband_v14 sideband; +}; }