crypto-com · bors · Apr 14, 2020 · Apr 13, 2020
@@ -10,6 +10,7 @@
 * *chain-abci* [1100](https://github.com/crypto-com/chain/pull/1100): account nonce increased after reward distribution
 * *chain-abci* [1292](https://github.com/crypto-com/chain/pull/1292): implements new punishment and staking state transition specification and new nonce logic
 * *chain-storage* [1424](https://github.com/crypto-com/chain/pull/1424): add two columnes to the database
+* *chain-storage* [1428](https://github.com/crypto-com/chain/pull/1428): change merkle tree of staking states to jellyfish-merkle
 * *chain-core* [1162](https://github.com/crypto-com/chain/pull/1162): transaction witness contains BIP-340-compatible Schnorr signatures (instead of the previous WIP scheme)
 * *chain-core* [1325](https://github.com/crypto-com/chain/pull/1325): blake3 for message digest (txid) + tree hashing
 * *chain-core* [1222](https://github.com/crypto-com/chain/pull/1258): Add common outer type for public(non-enclave) tx, tx serialization is changed.

@@ -22,10 +22,6 @@ enclave-protocol = { path = "../enclave-protocol" }
 log = "0.4.8"
 env_logger = "0.7.1"
 bit-vec = { version = "0.6.1", features = ["serde_no_std"] }
-kvdb = "0.5"
-kvdb-rocksdb = "0.7"
-kvdb-memorydb = "0.5"
-starling = "3.1.4"
 byteorder = "1.3.4"
 serde = "1.0"
 serde_derive = "1.0"
@@ -56,6 +52,8 @@ quickcheck = "0.9"
 digest = "0.8"
 sha3 = "0.8"
 base64 = "0.11"
+kvdb = "0.5"
+kvdb-memorydb = "0.5"
 test-common = { path = "../test-common" }
 
 # TODO: currently not maintained benchmarks

@@ -21,17 +21,15 @@ use chain_core::init::coin::Coin;
 use chain_core::init::config::InitConfig;
 use chain_core::init::config::NetworkParameters;
 use chain_core::state::account::StakedStateDestination;
-use chain_core::state::account::{CouncilNode, StakedState, StakedStateAddress};
+use chain_core::state::account::{CouncilNode, StakedStateAddress};
 use chain_core::state::tendermint::{BlockHeight, TendermintVotePower};
 use chain_core::state::{ChainState, RewardsPoolState};
 use chain_core::tx::TxAux;
 use chain_core::ChainInfo;
-use chain_storage::account::AccountWrapper;
-use chain_storage::account::StarlingFixedKey;
-use chain_storage::account::{pure_account_storage, AccountStorage};
 use chain_storage::buffer::{
-    flush_storage, GetStaking, KVBuffer, StakingBuffer, StakingGetter, StoreKV, StoreStaking,
+    flush_storage, GetStaking, KVBuffer, StakingBuffer, StoreKV, StoreStaking,
 };
+use chain_storage::jellyfish::{compute_staking_root, StakingGetter, Version};
 use chain_storage::{Storage, StoredChainState};
 
 /// ABCI app state snapshot
@@ -50,6 +48,8 @@ pub struct ChainNodeState {
     pub staking_table: StakingTable,
     /// genesis time
     pub genesis_time: Timespec,
+    /// Version number of staking merkle tree
+    pub staking_version: Version,
 
     /// The parts of states which involved in computing app_hash
     pub top_level: ChainState,
@@ -73,7 +73,7 @@ impl ChainNodeState {
     pub fn genesis(
         genesis_apphash: H256,
         genesis_time: Timespec,
-        account_root: StarlingFixedKey,
+        account_root: H256,
         rewards_pool: RewardsPoolState,
         network_params: NetworkParameters,
         staking_table: StakingTable,
@@ -85,6 +85,7 @@ impl ChainNodeState {
             block_height: BlockHeight::genesis(),
             staking_table,
             genesis_time,
+            staking_version: 0,
             top_level: ChainState {
                 account_root,
                 rewards_pool,
@@ -104,8 +105,6 @@ pub enum BufferType {
 pub struct ChainNodeApp<T: EnclaveProxy> {
     /// the underlying key-value storage (+ possibly some info in the future)
     pub storage: Storage,
-    /// account trie storage
-    pub accounts: AccountStorage,
     /// valid transactions after DeliverTx before EndBlock/Commit
     pub delivered_txs: Vec<TxAux>,
     /// a reference to genesis (used when there is no committed state)
@@ -207,25 +206,14 @@ fn get_voting_power(
     }
 }
 
-pub fn compute_accounts_root(
-    account_storage: &mut AccountStorage,
-    accounts: &[StakedState],
-) -> H256 {
-    let mut keys: Vec<_> = accounts.iter().map(StakedState::key).collect();
-    let wrapped: Vec<_> = accounts.iter().cloned().map(AccountWrapper).collect();
-    account_storage
-        .insert(None, &mut keys, &wrapped)
-        .expect("insert failed")
-}
-
 pub fn init_app_hash(conf: &InitConfig, genesis_time: Timespec) -> H256 {
     let (accounts, rp, _nodes) = conf
         .validate_config_get_genesis(genesis_time)
         .expect("distribution validation error");
 
     compute_app_hash(
         &MerkleTree::empty(),
-        &compute_accounts_root(&mut pure_account_storage(20).unwrap(), &accounts),
+        &compute_staking_root(&accounts),
         &rp,
         &NetworkParameters::Genesis(conf.network_params.clone()),
     )
@@ -238,7 +226,6 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
         genesis_app_hash: [u8; HASH_SIZE_256],
         chain_id: &str,
         storage: Storage,
-        accounts: AccountStorage,
         tx_query_address: Option<String>,
     ) -> Self {
         let stored_genesis = storage.get_genesis_app_hash();
@@ -261,7 +248,6 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
             .expect("failed to decode two last hex digits in chain ID")[0];
         ChainNodeApp {
             storage,
-            accounts,
             delivered_txs: Vec::new(),
             chain_hex_id,
             genesis_app_hash,
@@ -287,15 +273,13 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
     /// * `gah` - hex-encoded genesis app hash
     /// * `chain_id` - the chain ID set in Tendermint genesis.json (our name convention is that the last two characters should be hex digits)
     /// * `storage` - underlying storage to be used (in-mem or persistent)
-    /// * `accounts` - underlying storage for account tries to be used (in-mem or persistent)
     /// * `tx_query_address` -  address of tx query enclave to supply to clients (if any)
     /// * `enclave_server` -  connection string which ZeroMQ server wrapper around the transaction validation enclave will listen on
     pub fn new_with_storage(
         tx_validator: T,
         gah: &str,
         chain_id: &str,
         mut storage: Storage,
-        accounts: AccountStorage,
         tx_query_address: Option<String>,
         enclave_server: Option<String>,
     ) -> Self {
@@ -344,7 +328,7 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
 
             // populate the indexing structures in staking table.
             last_state.staking_table.initialize(
-                &StakingGetter::new(&accounts, Some(last_state.top_level.account_root)),
+                &StakingGetter::new(&storage, last_state.staking_version),
                 last_state
                     .top_level
                     .network_params
@@ -356,7 +340,6 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
                 genesis_app_hash,
                 chain_id,
                 storage,
-                accounts,
                 tx_query_address,
             )
         } else {
@@ -374,7 +357,6 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
             storage.write_genesis_chain_id(&genesis_app_hash, chain_id);
             ChainNodeApp {
                 storage,
-                accounts,
                 delivered_txs: Vec::new(),
                 chain_hex_id,
                 genesis_app_hash,
@@ -420,7 +402,7 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
         }
 
         let network_params = NetworkParameters::Genesis(conf.network_params);
-        let new_account_root = compute_accounts_root(&mut self.accounts, &accounts);
+        let new_account_root = self.storage.put_stakings(0, &accounts);
         let genesis_app_hash = compute_app_hash(
             &MerkleTree::empty(),
             &new_account_root,
@@ -448,7 +430,7 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
 
         let val_addresses = nodes.iter().map(|(addr, _)| *addr).collect::<Vec<_>>();
         let staking_table = StakingTable::from_genesis(
-            &StakingGetter::new(&self.accounts, Some(new_account_root)),
+            &staking_getter!(self, 0),
             network_params.get_required_council_node_stake(),
             network_params.get_max_validators(),
             &val_addresses,
@@ -475,27 +457,30 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
     }
 
     pub fn staking_store(&mut self, buffer_type: BufferType) -> impl StoreStaking + '_ {
-        let root = self
+        let version = self
             .last_state
             .as_ref()
-            .map(|state| state.top_level.account_root);
-        staking_store!(self, root, buffer_type)
+            .map(|state| state.staking_version)
+            .unwrap_or(0);
+        staking_store!(self, version, buffer_type)
     }
 
     pub fn staking_getter(&self, buffer_type: BufferType) -> impl GetStaking + '_ {
-        let root = self
+        let version = self
             .last_state
             .as_ref()
-            .map(|state| state.top_level.account_root);
-        staking_getter!(self, root, buffer_type)
+            .map(|state| state.staking_version)
+            .unwrap_or(0);
+        staking_getter!(self, version, buffer_type)
     }
 
-    pub fn staking_getter_committed(&self) -> StakingGetter<'_> {
+    pub fn staking_getter_committed(&self) -> StakingGetter<'_, Storage> {
         StakingGetter::new(
-            &self.accounts,
+            &self.storage,
             self.last_state
                 .as_ref()
-                .map(|state| state.top_level.account_root),
+                .map(|state| state.staking_version)
+                .unwrap_or(0),
         )
     }
 

@@ -8,7 +8,8 @@ use chain_core::compute_app_hash;
 use chain_core::tx::data::input::{TxoPointer, TxoSize};
 use chain_core::tx::data::TxId;
 use chain_core::tx::{TxAux, TxEnclaveAux, TxPublicAux};
-use chain_storage::buffer::{flush_staking_storage, flush_storage, StoreKV};
+use chain_storage::buffer::{flush_storage, StoreKV};
+use chain_storage::jellyfish::flush_stakings;
 use parity_scale_codec::Encode;
 
 /// Given a db and a DB transaction, it will go through TX inputs and mark them as spent
@@ -91,14 +92,20 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
             top_level.rewards_pool.last_block_height = new_state.last_block_height;
             self.rewards_pool_updated = false;
         }
+
         // flush staking storage
-        top_level.account_root = flush_staking_storage(
-            &mut self.accounts,
-            Some(top_level.account_root),
-            mem::take(&mut self.staking_buffer),
-        )
-        .expect("merkle trie io error")
-        .expect("merkle trie update should return Some(root)");
+        if !self.staking_buffer.is_empty() {
+            new_state.staking_version = new_state
+                .staking_version
+                .checked_add(1)
+                .expect("staking version overflow, no way to recover");
+            top_level.account_root = flush_stakings(
+                &mut kv_store!(self),
+                new_state.staking_version,
+                mem::take(&mut self.staking_buffer),
+            )
+            .expect("merkle trie io error");
+        }
 
         let app_hash = compute_app_hash(
             &tree,

@@ -35,7 +35,7 @@ impl<T: EnclaveProxy> ChainNodeApp<T> {
         // TODO: skipchain-based validator changes?
         let state = self.last_state.as_mut().expect("executing end block, but no app state stored (i.e. no initchain or recovery was executed)");
         let val_updates = state.staking_table.end_block(
-            &staking_getter!(self, Some(state.top_level.account_root)),
+            &staking_getter!(self, state.staking_version),
             state.top_level.network_params.get_max_validators(),
         );
 

@@ -2,15 +2,15 @@
 //! Ideally these should be methods of `ChainNodeApp`, but that borrow check don't allow that,
 //! because methods will need to retain the whole reference of `self` rather than individual fields.
 macro_rules! staking_store {
-    ($app:expr, $root:expr) => {
-        chain_storage::buffer::StakingBufferStore::new(
-            chain_storage::buffer::StakingGetter::new(&$app.accounts, $root),
+    ($app:expr, $version:expr) => {
+        chain_storage::jellyfish::StakingBufferStore::new(
+            chain_storage::jellyfish::StakingGetter::new(&$app.storage, $version),
             &mut $app.staking_buffer,
         )
     };
-    ($app:expr, $root:expr, $buffer_type:expr) => {
-        chain_storage::buffer::StakingBufferStore::new(
-            chain_storage::buffer::StakingGetter::new(&$app.accounts, $root),
+    ($app:expr, $version:expr, $buffer_type:expr) => {
+        chain_storage::jellyfish::StakingBufferStore::new(
+            chain_storage::jellyfish::StakingGetter::new(&$app.storage, $version),
             match $buffer_type {
                 crate::app::app_init::BufferType::Consensus => &mut $app.staking_buffer,
                 crate::app::app_init::BufferType::Mempool => &mut $app.mempool_staking_buffer,
@@ -20,15 +20,15 @@ macro_rules! staking_store {
 }
 
 macro_rules! staking_getter {
-    ($app:expr, $root:expr) => {
-        chain_storage::buffer::StakingBufferGetter::new(
-            chain_storage::buffer::StakingGetter::new(&$app.accounts, $root),
+    ($app:expr, $version:expr) => {
+        chain_storage::jellyfish::StakingBufferGetter::new(
+            chain_storage::jellyfish::StakingGetter::new(&$app.storage, $version),
             &$app.staking_buffer,
         )
     };
-    ($app:expr, $root:expr, $buffer_type:expr) => {
-        chain_storage::buffer::StakingBufferGetter::new(
-            chain_storage::buffer::StakingGetter::new(&$app.accounts, $root),
+    ($app:expr, $version:expr, $buffer_type:expr) => {
+        chain_storage::jellyfish::StakingBufferGetter::new(
+            chain_storage::jellyfish::StakingGetter::new(&$app.storage, $version),
             match $buffer_type {
                 crate::app::app_init::BufferType::Consensus => &$app.staking_buffer,
                 crate::app::app_init::BufferType::Mempool => &$app.mempool_staking_buffer,

@@ -14,8 +14,7 @@ use log::info;
 #[cfg(fuzzing)]
 pub use self::app_init::check_validators;
 pub use self::app_init::{
-    compute_accounts_root, get_validator_key, init_app_hash, BufferType, ChainNodeApp,
-    ChainNodeState,
+    get_validator_key, init_app_hash, BufferType, ChainNodeApp, ChainNodeState,
 };
 use crate::app::validate_tx::ResponseWithCodeAndLog;
 use crate::enclave_bridge::EnclaveProxy;
@@ -141,13 +140,11 @@ impl<T: EnclaveProxy> abci::Application for ChainNodeApp<T> {
             .last_state
             .as_mut()
             .expect("executing begin block, but no app state stored (i.e. no initchain or recovery was executed)");
-        let account_root = Some(last_state.top_level.account_root);
-
         last_state.block_time = block_time;
         last_state.block_height = block_height;
 
         let slashes = last_state.staking_table.begin_block(
-            &mut staking_store!(self, account_root),
+            &mut staking_store!(self, last_state.staking_version),
             &last_state.top_level.network_params,
             block_time,
             block_height,
@@ -193,9 +190,10 @@ impl<T: EnclaveProxy> abci::Application for ChainNodeApp<T> {
 
         // FIXME: record based on votes
         if let Ok(proposer_address) = &proposer_address {
-            last_state
-                .staking_table
-                .reward_record(&staking_getter!(self, account_root), proposer_address);
+            last_state.staking_table.reward_record(
+                &staking_getter!(self, last_state.staking_version),
+                proposer_address,
+            );
         } else {
             log::error!("invalid proposer address");
         }