From a60ec344e69f1866f3e45286cac314cea49c5bb2 Mon Sep 17 00:00:00 2001 From: SergiySW Date: Fri, 8 Mar 2019 00:08:01 +0300 Subject: [PATCH 1/6] Use adjusted difficulty for trees in active transactions * add adjusted difficulty calculation * make list of dependent blocks in election --- nano/core_test/conflicts.cpp | 96 +++++++++++++++++++++++++++++ nano/node/node.cpp | 113 +++++++++++++++++++++++++++++++++-- nano/node/node.hpp | 6 +- 3 files changed, 210 insertions(+), 5 deletions(-) diff --git a/nano/core_test/conflicts.cpp b/nano/core_test/conflicts.cpp index 29f750ad66..8c179932c5 100644 --- a/nano/core_test/conflicts.cpp +++ b/nano/core_test/conflicts.cpp @@ -186,3 +186,99 @@ TEST (conflicts, reprioritize) ASSERT_EQ (difficulty2, existing2->difficulty); } } + +TEST (conflicts, dependency) +{ + nano::system system (24000, 1); + auto & node1 (*system.nodes[0]); + nano::genesis genesis; + nano::keypair key1; + auto send1 (std::make_shared (genesis.hash (), key1.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0)); + node1.work_generate_blocking (*send1); + ASSERT_EQ (nano::process_result::progress, node1.process (*send1).code); + ASSERT_EQ (0, node1.active.size ()); + node1.active.start (genesis.open); + node1.active.start (send1); + ASSERT_EQ (2, node1.active.size ()); + // Check dependecy for genesis block + { + std::lock_guard guard (node1.active.mutex); + auto existing1 (node1.active.roots.find (nano::uint512_union (genesis.open->previous (), genesis.open->root ()))); + ASSERT_NE (node1.active.roots.end (), existing1); + auto election1 (existing1->election); + ASSERT_NE (nullptr, election1); + ASSERT_EQ (1, election1->dependent_blocks.size ()); + ASSERT_NE (election1->dependent_blocks.end (), election1->dependent_blocks.find (send1->hash ())); + } +} + +TEST (conflicts, adjusted_difficulty) +{ + nano::system system (24000, 1); + auto & node1 (*system.nodes[0]); + nano::genesis genesis; + nano::keypair key1; + nano::keypair key2; + nano::keypair key3; + ASSERT_EQ (0, node1.active.size ()); + node1.active.start (genesis.open); + auto send1 (std::make_shared (genesis.hash (), key1.pub, nano::genesis_amount - 2 * nano::xrb_ratio, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (genesis.hash ()))); + node1.process_active (send1); + auto send2 (std::make_shared (send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - 3 * nano::xrb_ratio, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + node1.process_active (send2); + auto receive1 (std::make_shared (send2->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send2->hash ()))); + node1.process_active (receive1); + auto open1 (std::make_shared (send1->hash (), key1.pub, key1.pub, key1.prv, key1.pub, system.work.generate (key1.pub))); + node1.process_active (open1); + auto send3 (std::make_shared (key1.pub, open1->hash (), key1.pub, nano::xrb_ratio, key2.pub, key1.prv, key1.pub, system.work.generate (open1->hash ()))); + node1.process_active (send3); + auto send4 (std::make_shared (key1.pub, send3->hash (), key1.pub, 0, key3.pub, key1.prv, key1.pub, system.work.generate (send3->hash ()))); + node1.process_active (send4); + ASSERT_EQ (node1.ledger.epoch_signer, nano::test_genesis_key.pub); + auto open_epoch1 (std::make_shared (key2.pub, 0, 0, 0, node1.ledger.epoch_link, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (key2.pub))); + node1.process_active (open_epoch1); + auto receive2 (std::make_shared (key2.pub, open_epoch1->hash (), 0, nano::xrb_ratio, send3->hash (), key2.prv, key2.pub, system.work.generate (open_epoch1->hash ()))); + node1.process_active (receive2); + auto open2 (std::make_shared (key3.pub, 0, key3.pub, nano::xrb_ratio, send4->hash (), key3.prv, key3.pub, system.work.generate (key3.pub))); + node1.process_active (open2); + auto change1 (std::make_shared (key3.pub, open2->hash (), nano::test_genesis_key.pub, nano::xrb_ratio, 0, key3.prv, key3.pub, system.work.generate (open2->hash ()))); + node1.process_active (change1); + node1.block_processor.flush (); + ASSERT_EQ (11, node1.active.size ()); + std::unordered_map adjusted_difficulties; + { + std::lock_guard guard (node1.active.mutex); + ASSERT_EQ (node1.active.roots.get<1> ().begin ()->election->status.winner->hash (), genesis.hash ()); + for (auto i (node1.active.roots.get<1> ().begin ()), n (node1.active.roots.get<1> ().end ()); i != n; ++i) + { + adjusted_difficulties.insert (std::make_pair (i->election->status.winner->hash (), i->adjusted_difficulty)); + } + } + // genesis + ASSERT_GT (adjusted_difficulties.find (genesis.hash ())->second, adjusted_difficulties.find (send1->hash ())->second); + ASSERT_GT (adjusted_difficulties.find (send1->hash ())->second, adjusted_difficulties.find (send2->hash ())->second); + ASSERT_GT (adjusted_difficulties.find (send2->hash ())->second, adjusted_difficulties.find (receive1->hash ())->second); + // key1 + ASSERT_GT (adjusted_difficulties.find (send1->hash ())->second, adjusted_difficulties.find (open1->hash ())->second); + ASSERT_GT (adjusted_difficulties.find (open1->hash ())->second, adjusted_difficulties.find (send3->hash ())->second); + ASSERT_GT (adjusted_difficulties.find (send3->hash ())->second, adjusted_difficulties.find (send4->hash ())->second); + //key2 + ASSERT_GT (adjusted_difficulties.find (send3->hash ())->second, adjusted_difficulties.find (receive2->hash ())->second); + ASSERT_GT (adjusted_difficulties.find (open_epoch1->hash ())->second, adjusted_difficulties.find (receive2->hash ())->second); + // key3 + ASSERT_GT (adjusted_difficulties.find (send4->hash ())->second, adjusted_difficulties.find (open2->hash ())->second); + ASSERT_GT (adjusted_difficulties.find (open2->hash ())->second, adjusted_difficulties.find (change1->hash ())->second); + // Independent elections can have higher difficulty than adjusted tree + nano::keypair key4; + auto open_epoch2 (std::make_shared (key4.pub, 0, 0, 0, node1.ledger.epoch_link, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (key4.pub, adjusted_difficulties.find (genesis.hash ())->second))); + uint64_t difficulty; + ASSERT_FALSE (nano::work_validate (*open_epoch2, &difficulty)); + ASSERT_GT (difficulty, adjusted_difficulties.find (genesis.hash ())->second); + node1.process_active (open_epoch2); + node1.block_processor.flush (); + ASSERT_EQ (12, node1.active.size ()); + { + std::lock_guard guard (node1.active.mutex); + ASSERT_EQ (node1.active.roots.get<1> ().begin ()->election->status.winner->hash (), open_epoch2->hash ()); + } +} diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 34dfc4a39a..b303ca1426 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -2955,6 +2955,7 @@ announcements (0) { last_votes.insert (std::make_pair (nano::not_an_account (), nano::vote_info{ std::chrono::steady_clock::now (), 0, block_a->hash () })); blocks.insert (std::make_pair (block_a->hash (), block_a)); + update_dependent (); } void nano::election::compute_rep_votes (nano::transaction const & transaction_a) @@ -3067,6 +3068,8 @@ void nano::election::confirm_if_quorum (nano::transaction const & transaction_a) auto node_l (node.shared ()); node_l->block_processor.force (block_l); status.winner = block_l; + update_dependent (); + node_l->active.adjust_difficulty (block_l->hash ()); } if (have_quorum (tally_l, sum)) { @@ -3230,6 +3233,39 @@ size_t nano::election::last_votes_size () return last_votes.size (); } +void nano::election::update_dependent () +{ + assert (!node.active.mutex.try_lock ()); + std::vector blocks_search; + auto hash (status.winner->hash ()); + auto previous (status.winner->previous ()); + if (!previous.is_zero ()) + { + blocks_search.push_back (previous); + } + auto source (status.winner->source ()); + if (!source.is_zero () && source != previous) + { + blocks_search.push_back (source); + } + auto link (status.winner->link ()); + if (!link.is_zero () && !node.ledger.is_epoch_link (link) && link != previous) + { + blocks_search.push_back (link); + } + for (auto & block_search : blocks_search) + { + auto existing (node.active.blocks.find (block_search)); + if (existing != node.active.blocks.end () && !existing->second->confirmed && !existing->second->stopped) + { + if (existing->second->dependent_blocks.find (hash) == existing->second->dependent_blocks.end ()) + { + existing->second->dependent_blocks.insert (hash); + } + } + } +} + void nano::active_transactions::request_confirm (std::unique_lock & lock_a) { std::unordered_set inactive; @@ -3298,6 +3334,7 @@ void nano::active_transactions::request_confirm (std::unique_lock & } } } + election_l->update_dependent (); } } if (election_l->announcements < announcement_long || election_l->announcements % announcement_long == 1) @@ -3493,8 +3530,9 @@ bool nano::active_transactions::add (std::shared_ptr block_a, std:: uint64_t difficulty (0); auto error (nano::work_validate (*block_a, &difficulty)); release_assert (!error); - roots.insert (nano::conflict_info{ root, difficulty, election }); + roots.insert (nano::conflict_info{ root, difficulty, difficulty, election }); blocks.insert (std::make_pair (block_a->hash (), election)); + adjust_difficulty (block_a->hash ()); } error = existing != roots.end (); } @@ -3560,9 +3598,76 @@ void nano::active_transactions::update_difficulty (nano::block const & block_a) uint64_t difficulty; auto error (nano::work_validate (block_a, &difficulty)); assert (!error); - roots.modify (existing, [difficulty](nano::conflict_info & info_a) { - info_a.difficulty = difficulty; - }); + if (difficulty > existing->difficulty) + { + roots.modify (existing, [difficulty](nano::conflict_info & info_a) { + info_a.difficulty = difficulty; + }); + adjust_difficulty (block_a.hash ()); + } + } +} + +void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash_a) +{ + assert (!mutex.try_lock ()); + std::deque> remaining_blocks; + remaining_blocks.push_back (std::make_pair (hash_a, 0)); + std::unordered_set processed_blocks; + std::vector> elections_list; + uint128_t sum (0); + while (!remaining_blocks.empty ()) + { + auto item (remaining_blocks.front ()); + remaining_blocks.pop_front (); + auto hash (item.first); + // item.second is level + if (processed_blocks.find (hash) == processed_blocks.end ()) + { + auto existing (blocks.find (hash)); + if (existing != blocks.end () && !existing->second->confirmed && !existing->second->stopped && existing->second->status.winner->hash () == hash) + { + auto previous (existing->second->status.winner->previous ()); + if (!previous.is_zero ()) + { + remaining_blocks.push_back (std::make_pair (previous, item.second + 1)); + } + auto source (existing->second->status.winner->source ()); + if (!source.is_zero () && source != previous) + { + remaining_blocks.push_back (std::make_pair (source, item.second + 1)); + } + auto link (existing->second->status.winner->link ()); + if (!link.is_zero () && !node.ledger.is_epoch_link (link) && link != previous) + { + remaining_blocks.push_back (std::make_pair (link, item.second + 1)); + } + for (auto & dependent_block : existing->second->dependent_blocks) + { + remaining_blocks.push_back (std::make_pair (dependent_block, item.second - 1)); + } + processed_blocks.insert (hash); + nano::uint512_union root (previous, existing->second->status.winner->root ()); + auto existing_root (roots.find (root)); + if (existing_root != roots.end ()) + { + sum += existing_root->difficulty; + elections_list.push_back (std::make_pair (root, item.second)); + } + } + } + } + if (elections_list.size () > 1) + { + uint64_t average (static_cast (sum / elections_list.size ())); + for (auto & item : elections_list) + { + auto existing_root (roots.find (item.first)); + uint64_t difficulty_a (average + item.second); + roots.modify (existing_root, [difficulty_a](nano::conflict_info & info_a) { + info_a.adjusted_difficulty = difficulty_a; + }); + } } } diff --git a/nano/node/node.hpp b/nano/node/node.hpp index a6d09ada7d..c5a6df91be 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -81,6 +81,7 @@ class election : public std::enable_shared_from_this void log_votes (nano::tally_t const &); bool publish (std::shared_ptr block_a); size_t last_votes_size (); + void update_dependent (); void stop (); nano::node & node; std::unordered_map last_votes; @@ -91,12 +92,14 @@ class election : public std::enable_shared_from_this bool stopped; std::unordered_map last_tally; unsigned announcements; + std::unordered_set dependent_blocks; }; class conflict_info { public: nano::uint512_union root; uint64_t difficulty; + uint64_t adjusted_difficulty; std::shared_ptr election; }; // Core class for determining consensus @@ -117,6 +120,7 @@ class active_transactions // Is the root of this block in the roots container bool active (nano::block const &); void update_difficulty (nano::block const &); + void adjust_difficulty (nano::block_hash const &); std::deque> list_blocks (bool = false); void erase (nano::block const &); bool empty (); @@ -129,7 +133,7 @@ class active_transactions boost::multi_index::hashed_unique< boost::multi_index::member>, boost::multi_index::ordered_non_unique< - boost::multi_index::member, + boost::multi_index::member, std::greater>>> roots; std::unordered_map> blocks; From 4840a63b608db75d763866c590766aaf356be46c Mon Sep 17 00:00:00 2001 From: SergiySW Date: Fri, 8 Mar 2019 00:32:14 +0300 Subject: [PATCH 2/6] Adjust difficulty for dependent on deleted blocks elections --- nano/node/node.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nano/node/node.cpp b/nano/node/node.cpp index b303ca1426..0b816b0311 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -3468,6 +3468,10 @@ void nano::active_transactions::request_confirm (std::unique_lock & (void)erased; assert (erased == 1); } + for (auto & dependent_block : root_it->election->dependent_blocks) + { + adjust_difficulty (dependent_block); + } roots.erase (*i); } if (unconfirmed_count > 0) @@ -3669,6 +3673,17 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash }); } } + // Return difficulty instead of adjusted value + else if (elections_list.size () == 1) + { + auto existing_root (roots.find (elections_list.begin ()->first)); + if (existing_root->difficulty != existing_root->adjusted_difficulty) + { + roots.modify (existing_root, [](nano::conflict_info & info_a) { + info_a.adjusted_difficulty = info_a.difficulty; + }); + } + } } // List of active blocks in elections From 8876550edf34c11a4d0ca35fe79342a733e02b0e Mon Sep 17 00:00:00 2001 From: SergiySW Date: Fri, 15 Mar 2019 22:40:56 +0300 Subject: [PATCH 3/6] Potential overflow check --- nano/node/node.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 3dfeae6ac3..7a3a7d2722 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -3576,10 +3576,17 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash if (elections_list.size () > 1) { uint64_t average (static_cast (sum / elections_list.size ())); + // Potential overflow check + uint64_t divider (1); + if (elections_list.size () > 1000000 && (average - node.network_params.publish_threshold) > elections_list.size ()) + { + divider = ((average - node.network_params.publish_threshold) / elections_list.size ()) + 1; + } + // Set adjusted difficulty for (auto & item : elections_list) { auto existing_root (roots.find (item.first)); - uint64_t difficulty_a (average + item.second); + uint64_t difficulty_a (average + (item.second / divider)); roots.modify (existing_root, [difficulty_a](nano::conflict_info & info_a) { info_a.adjusted_difficulty = difficulty_a; }); From fd450151d1bef534b10c6db0cbf0e7966744a110 Mon Sep 17 00:00:00 2001 From: SergiySW Date: Thu, 21 Mar 2019 13:34:53 +0300 Subject: [PATCH 4/6] Using reference to items in remaining_blocks --- nano/node/node.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 79d0431dbf..30ca6543ff 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -3123,10 +3123,9 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash uint128_t sum (0); while (!remaining_blocks.empty ()) { - auto item (remaining_blocks.front ()); - remaining_blocks.pop_front (); + auto const & item (remaining_blocks.front ()); auto hash (item.first); - // item.second is level + auto level (item.second); if (processed_blocks.find (hash) == processed_blocks.end ()) { auto existing (blocks.find (hash)); @@ -3135,21 +3134,21 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash auto previous (existing->second->status.winner->previous ()); if (!previous.is_zero ()) { - remaining_blocks.push_back (std::make_pair (previous, item.second + 1)); + remaining_blocks.push_back (std::make_pair (previous, level + 1)); } auto source (existing->second->status.winner->source ()); if (!source.is_zero () && source != previous) { - remaining_blocks.push_back (std::make_pair (source, item.second + 1)); + remaining_blocks.push_back (std::make_pair (source, level + 1)); } auto link (existing->second->status.winner->link ()); if (!link.is_zero () && !node.ledger.is_epoch_link (link) && link != previous) { - remaining_blocks.push_back (std::make_pair (link, item.second + 1)); + remaining_blocks.push_back (std::make_pair (link, level + 1)); } for (auto & dependent_block : existing->second->dependent_blocks) { - remaining_blocks.push_back (std::make_pair (dependent_block, item.second - 1)); + remaining_blocks.push_back (std::make_pair (dependent_block, level - 1)); } processed_blocks.insert (hash); nano::uint512_union root (previous, existing->second->status.winner->root ()); @@ -3157,10 +3156,11 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash if (existing_root != roots.end ()) { sum += existing_root->difficulty; - elections_list.push_back (std::make_pair (root, item.second)); + elections_list.push_back (std::make_pair (root, level)); } } } + remaining_blocks.pop_front (); } if (elections_list.size () > 1) { From 31f18118301ed46cc754399f1205ac1ed257f74a Mon Sep 17 00:00:00 2001 From: SergiySW Date: Thu, 21 Mar 2019 13:36:43 +0300 Subject: [PATCH 5/6] correct words --- nano/node/node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 30ca6543ff..ded03205dc 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -3181,7 +3181,7 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash }); } } - // Return difficulty instead of adjusted value + // Set adjusted difficulty equals to difficulty else if (elections_list.size () == 1) { auto existing_root (roots.find (elections_list.begin ()->first)); From b7f119e045f498820fe7325e0954df6cf2f45a90 Mon Sep 17 00:00:00 2001 From: SergiySW Date: Thu, 21 Mar 2019 13:40:38 +0300 Subject: [PATCH 6/6] Replace push_back with emplace_back for deque of pairs --- nano/node/node.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nano/node/node.cpp b/nano/node/node.cpp index ded03205dc..d55c16689d 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -3117,7 +3117,7 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash { assert (!mutex.try_lock ()); std::deque> remaining_blocks; - remaining_blocks.push_back (std::make_pair (hash_a, 0)); + remaining_blocks.emplace_back (hash_a, 0); std::unordered_set processed_blocks; std::vector> elections_list; uint128_t sum (0); @@ -3134,21 +3134,21 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash auto previous (existing->second->status.winner->previous ()); if (!previous.is_zero ()) { - remaining_blocks.push_back (std::make_pair (previous, level + 1)); + remaining_blocks.emplace_back (previous, level + 1); } auto source (existing->second->status.winner->source ()); if (!source.is_zero () && source != previous) { - remaining_blocks.push_back (std::make_pair (source, level + 1)); + remaining_blocks.emplace_back (source, level + 1); } auto link (existing->second->status.winner->link ()); if (!link.is_zero () && !node.ledger.is_epoch_link (link) && link != previous) { - remaining_blocks.push_back (std::make_pair (link, level + 1)); + remaining_blocks.emplace_back (link, level + 1); } for (auto & dependent_block : existing->second->dependent_blocks) { - remaining_blocks.push_back (std::make_pair (dependent_block, level - 1)); + remaining_blocks.emplace_back (dependent_block, level - 1); } processed_blocks.insert (hash); nano::uint512_union root (previous, existing->second->status.winner->root ()); @@ -3156,7 +3156,7 @@ void nano::active_transactions::adjust_difficulty (nano::block_hash const & hash if (existing_root != roots.end ()) { sum += existing_root->difficulty; - elections_list.push_back (std::make_pair (root, level)); + elections_list.emplace_back (root, level); } } }