feat: persistent storage (#462)

* feat: persistent storage with leveldb

* tests: leveldb store tests

* chore: temporary add ci on base feat/new-storage-scheme

* chore: update typescript

* use rmdirSync on node < v14.14.0

* chore: remove node version 8 and 10 from ci

* chore: remove v16 from ci

* tests(integration): persistent storage

* chore: review changes

* feat: await rmdir

* chore: remove blank line

* chore: review changes

* chore: remove unused generic db access

* chore: add htr to tokens on constructor

* chore: split nft creation fee from mint deposit

* chore: await async methods

* feat: process history using the storage util

* feat: save address and history count on storage instead of calculating every call

* chore: add delays

* feat: only process walletData if there is something to process

* chore: integration test delay funds

* feat: actually increment counter when adding addresses

* feat: save the index size on memory

* chore: remove changes to force CI

* chore: rename editToken and editAddress methods for leveldb store

.github/workflows/main.yml

@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 40  # default is 360
     strategy:
       matrix:
-        node-version: [8.x, 10.x, 12.x, 14.x]
+        node-version: [12.x, 14.x]
     steps:
     - uses: actions/checkout@v2
     - name: Use Node.js ${{ matrix.node-version }}

.gitignore

@@ -8,3 +8,6 @@ lib
 *.swo
 
 tsconfig.tsbuildinfo
+
+// Generated test database files
+*.leveldb

__tests__/integration/hathorwallet_facade.test.js

@@ -1561,7 +1561,7 @@ describe('createNewToken', () => {
       }
     );
     await waitForTxReceived(hWallet, tokenUid);
-
+    await delay(500);
     // Validating the tokens are on the correct addresses
     const { utxos: utxosTokens } = await hWallet.getUtxos({ token: tokenUid });
     expect(utxosTokens).toContainEqual(
@@ -1747,6 +1747,7 @@ describe('meltTokens', () => {
     // Setting up scenario
     const hWallet = await generateWalletHelper();
     await GenesisWalletHelper.injectFunds(await hWallet.getAddressAtIndex(0), 20);
+    await delay(500);
     const { hash: tokenUid } = await createTokenHelper(
       hWallet,
       'Token to Melt',

__tests__/integration/hathorwallet_others.test.js

@@ -41,6 +41,7 @@ describe('getAddressInfo', () => {
 
     // Validating address after 1 transaction
     await GenesisWalletHelper.injectFunds(addr0, 10);
+    await delay(500);
     await expect(hWallet.getAddressInfo(addr0)).resolves.toMatchObject({
       total_amount_received: 10,
       total_amount_sent: 0,
@@ -92,6 +93,7 @@ describe('getAddressInfo', () => {
     expect((await hWallet.getAddressInfo(addr2)).total_amount_received).toStrictEqual(0);
     expect((await hWallet.getAddressInfo(addr3)).total_amount_received).toStrictEqual(0);
 
+    await delay(500);
     // Move all the wallet's funds to addr2
     let tx = await hWallet.sendTransaction(addr2, 10);
     await waitForTxReceived(hWallet, tx.hash);

__tests__/integration/helpers/genesis-wallet.helper.js

@@ -10,6 +10,7 @@ import Connection from '../../../src/new/connection';
 import HathorWallet from '../../../src/new/wallet';
 import { waitForTxReceived, waitForWalletReady, waitUntilNextTimestamp } from './wallet.helper';
 import { loggers } from '../utils/logger.util';
+import { delay } from '../utils/core.util';
 
 /**
  * @type {?GenesisWalletHelper}
@@ -98,6 +99,7 @@ export class GenesisWalletHelper {
 
     const hWallet = new GenesisWalletHelper();
     await hWallet.start();
+    await delay(500);
 
     singleton = hWallet;
     return singleton;

__tests__/integration/storage/leveldb.test.js

@@ -0,0 +1,82 @@
+import { precalculationHelpers } from '../helpers/wallet-precalculation.helper';
+import { GenesisWalletHelper } from '../helpers/genesis-wallet.helper';
+import {
+  createTokenHelper,
+  DEFAULT_PASSWORD,
+  DEFAULT_PIN_CODE,
+  generateConnection,
+  generateMultisigWalletHelper,
+  generateWalletHelper,
+  stopAllWallets,
+  waitForTxReceived,
+  waitForWalletReady,
+  waitUntilNextTimestamp
+} from '../helpers/wallet.helper';
+import { delay } from '../utils/core.util';
+
+import { LevelDBStore, Storage } from '../../../src/storage';
+import walletUtils from '../../../src/utils/wallet';
+import HathorWallet from '../../../src/new/wallet';
+import { loggers } from '../utils/logger.util';
+
+const startedWallets = [];
+
+/**
+ * Helper to stop wallets started manually in this test file.
+ */
+async function stopWallets() {
+  let hWallet;
+  while (hWallet = startedWallets.pop()) {
+    try {
+      await hWallet.stop({ cleanStorage: true, cleanAddresses: true });
+    } catch (e) {
+      loggers.test.error(e.stack);
+    }
+  }
+}
+
+describe('LevelDB persistent store', () => {
+  afterEach(async () => {
+    await stopWallets();
+    await stopAllWallets();
+    await GenesisWalletHelper.clearListeners();
+  });
+
+  it('should receive and send tokens', async () => {
+    const DATA_DIR = './testdata.leveldb';
+    const walletData = precalculationHelpers.test.getPrecalculatedWallet();
+    const xpubkey = walletUtils.getXPubKeyFromSeed(walletData.words, { accountDerivationIndex: '0\'/0' });
+    const store = new LevelDBStore(DATA_DIR, xpubkey);
+    const storage = new Storage(store);
+
+    // Start the wallet
+    const walletConfig = {
+      seed: walletData.words,
+      connection: generateConnection(),
+      password: DEFAULT_PASSWORD,
+      pinCode: DEFAULT_PIN_CODE,
+      preCalculatedAddresses: walletData.addresses,
+      storage,
+    };
+    const hWallet = new HathorWallet(walletConfig);
+    await hWallet.start();
+    startedWallets.push(hWallet);
+    await waitForWalletReady(hWallet);
+
+    // Expect to have an empty list for the full history
+    expect(Object.keys(hWallet.getFullHistory())).toHaveLength(0);
+
+    // Injecting some funds on this wallet
+    await GenesisWalletHelper.injectFunds(await hWallet.getAddressAtIndex(1), 10);
+
+    await delay(100);
+    // Validating the full history increased in one
+    expect(Object.keys(await hWallet.getFullHistory())).toHaveLength(1);
+
+    const tx1 = await hWallet.sendTransaction(await hWallet.getAddressAtIndex(3), 5);
+    await waitForTxReceived(hWallet, tx1.hash);
+    expect(Object.keys(await hWallet.getFullHistory())).toHaveLength(2);
+
+    await expect(store.getTx(tx1.hash)).resolves.not.toBeNull();
+  });
+});