Merge pull request #5488 from Agoric/mhofman/5031-update-lmdb

fix(swingset): update lmdb

packages/SwingSet/docs/run-policy.md

@@ -88,7 +88,7 @@ while(1) {
   processInboundIO();
   const policy = make100CrankPolicy();
   await controller.run(policy);
-  commit();
+  await commit();
   processOutboundIO();
 }
 ```

packages/SwingSet/misc-tools/db-delete.js

@@ -26,7 +26,7 @@ function fail(message, printUsage) {
   process.exit(1);
 }
 
-function run() {
+async function run() {
   const argv = process.argv.slice(2);
 
   let range = false;
@@ -56,7 +56,10 @@ function run() {
   } else {
     kvStore.delete(key);
   }
-  commit();
+  await commit();
 }
 
-run();
+run().then(
+  () => 0,
+  e => console.error(`${e}`, e),
+);

packages/SwingSet/misc-tools/db-set.js

@@ -23,7 +23,7 @@ function fail(message, printUsage) {
   process.exit(1);
 }
 
-function run() {
+async function run() {
   const argv = process.argv.slice(2);
 
   if (argv.length !== 3) {
@@ -36,7 +36,10 @@ function run() {
   const { kvStore, commit } = openSwingStore(stateDBDir);
 
   kvStore.set(key, value);
-  commit();
+  await commit();
 }
 
-run();
+run().then(
+  () => 0,
+  e => console.error(`${e}`, e),
+);

packages/SwingSet/misc-tools/extract-transcript-from-kerneldb.js

@@ -1,6 +1,6 @@
 // XXX this is wrong; it needs to use the swingstore instead of opening the LMDB
 // file directly, then use stream store reads to get the transcript entries.
-import lmdb from 'node-lmdb';
+import lmdb from 'lmdb';
 import process from 'process';
 import fs from 'fs';
 

packages/SwingSet/misc-tools/rekernelize.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 import '@endo/init/pre-bundle-source.js';
-import 'node-lmdb';
+import 'lmdb';
 import '@endo/init';
 
 import fs from 'fs';
@@ -65,8 +65,8 @@ async function main() {
   );
 
   kvStore.set('kernelBundle', JSON.stringify(kernelBundle));
-  swingStore.commit();
-  swingStore.close();
+  await swingStore.commit();
+  await swingStore.close();
 }
 
 main().then(

packages/SwingSet/misc-tools/replace-bundle.js

@@ -59,7 +59,7 @@ async function run() {
   const newBundleStr = JSON.stringify(bundle);
   log(`new bundle is ${newBundleStr.length} bytes`);
   kvStore.set(bundleName, newBundleStr);
-  commit();
+  await commit();
   log(`bundle ${bundleName} replaced`);
 }
 

packages/SwingSet/package.json

@@ -45,7 +45,7 @@
     "@endo/zip": "^0.2.25",
     "anylogger": "^0.21.0",
     "import-meta-resolve": "^1.1.1",
-    "node-lmdb": "^0.9.5",
+    "lmdb": "^2.4.5",
     "semver": "^6.3.0"
   },
   "peerDependencies": {

packages/SwingSet/src/kernel/kernelSyscall.js

@@ -19,6 +19,7 @@ export function makeKernelSyscallHandler(tools) {
     deviceHooks,
   } = tools;
 
+  /** @type {{kvStore: KVStore}} */
   const { kvStore } = kernelKeeper;
 
   function send(target, msg) {
@@ -46,13 +47,25 @@ export function makeKernelSyscallHandler(tools) {
   let workingPriorKey;
   let workingLowerBound;
   let workingUpperBound;
+  /** @type {IterableIterator<string> | undefined} */
   let workingKeyIterator;
 
-  function clearVatStoreIteration() {
-    workingPriorKey = undefined;
-    workingLowerBound = undefined;
-    workingUpperBound = undefined;
-    workingKeyIterator = undefined;
+  /** @param {boolean} [done] */
+  function clearVatStoreIteration(done = false) {
+    try {
+      if (
+        !done &&
+        workingKeyIterator &&
+        typeof workingKeyIterator.return === 'function'
+      ) {
+        workingKeyIterator.return();
+      }
+    } finally {
+      workingKeyIterator = undefined;
+      workingPriorKey = undefined;
+      workingLowerBound = undefined;
+      workingUpperBound = undefined;
+    }
   }
 
   /**
@@ -80,8 +93,8 @@ export function makeKernelSyscallHandler(tools) {
     const actualKey = vatstoreKeyKey(vatID, key);
     kernelKeeper.incStat('syscalls');
     kernelKeeper.incStat('syscallVatstoreSet');
-    kvStore.set(actualKey, value);
     clearVatStoreIteration();
+    kvStore.set(actualKey, value);
     return OKNULL;
   }
 
@@ -160,6 +173,7 @@ export function makeKernelSyscallHandler(tools) {
     }
     kernelKeeper.incStat('syscalls');
     kernelKeeper.incStat('syscallVatstoreGetAfter');
+    /** @type {IteratorResult<string>} */
     let nextIter;
     // Note that the working key iterator will be invalidated if the parameters
     // to `vatstoreGetAfter` don't correspond to the working key iterator's
@@ -180,6 +194,7 @@ export function makeKernelSyscallHandler(tools) {
     ) {
       nextIter = workingKeyIterator.next();
     } else {
+      clearVatStoreIteration();
       let startKey;
       if (priorKey === '') {
         startKey = actualLowerBound;
@@ -198,7 +213,7 @@ export function makeKernelSyscallHandler(tools) {
       }
     }
     if (nextIter.done) {
-      clearVatStoreIteration();
+      clearVatStoreIteration(true);
       return harden(['ok', [undefined, undefined]]);
     } else {
       const nextKey = nextIter.value;
@@ -220,8 +235,8 @@ export function makeKernelSyscallHandler(tools) {
     const actualKey = vatstoreKeyKey(vatID, key);
     kernelKeeper.incStat('syscalls');
     kernelKeeper.incStat('syscallVatstoreDelete');
-    kvStore.delete(actualKey);
     clearVatStoreIteration();
+    kvStore.delete(actualKey);
     return OKNULL;
   }
 

packages/SwingSet/src/kernel/state/storageWrapper.js

@@ -1,4 +1,6 @@
-import { assert } from '@agoric/assert';
+// @ts-check
+
+import { assert, details as X } from '@agoric/assert';
 import { insistStorageAPI } from '../../lib/storageAPI.js';
 
 // We wrap a provided object implementing StorageAPI methods { has, getKeys,
@@ -24,30 +26,74 @@ import { insistStorageAPI } from '../../lib/storageAPI.js';
  * @yields any
  */
 function* mergeUtf16SortedIterators(it1, it2) {
-  let v1 = it1.next();
-  let v2 = it2.next();
-  while (!v1.done && !v2.done) {
-    if (v1.value < v2.value) {
-      const result = v1.value;
-      v1 = it1.next();
-      yield result;
-    } else if (v1.value === v2.value) {
-      const result = v1.value;
-      v1 = it1.next();
-      v2 = it2.next();
-      yield result;
-    } else {
-      const result = v2.value;
-      v2 = it2.next();
+  /** @type {IteratorResult<any> | null} */
+  let v1 = null;
+  /** @type {IteratorResult<any> | null} */
+  let v2 = null;
+  /** @type {IteratorResult<any> | null} */
+  let vrest = null;
+  /** @type {Iterator<any> | null} */
+  let itrest = null;
+
+  try {
+    v1 = it1.next();
+    v2 = it2.next();
+    while (!v1.done && !v2.done) {
+      if (v1.value < v2.value) {
+        const result = v1.value;
+        v1 = it1.next();
+        yield result;
+      } else if (v1.value === v2.value) {
+        const result = v1.value;
+        v1 = it1.next();
+        v2 = it2.next();
+        yield result;
+      } else {
+        const result = v2.value;
+        v2 = it2.next();
+        yield result;
+      }
+    }
+
+    itrest = v1.done ? it2 : it1;
+    vrest = v1.done ? v2 : v1;
+    v1 = null;
+    v2 = null;
+
+    while (!vrest.done) {
+      const result = vrest.value;
+      vrest = itrest.next();
       yield result;
     }
-  }
-  const itrest = v1.done ? it2 : it1;
-  let v = v1.done ? v2 : v1;
-  while (!v.done) {
-    const result = v.value;
-    v = itrest.next();
-    yield result;
+  } finally {
+    const errors = [];
+    try {
+      if (vrest && !vrest.done && itrest && itrest.return) {
+        itrest.return();
+      }
+    } catch (e) {
+      errors.push(e);
+    }
+    try {
+      if (v1 && !v1.done && it1.return) {
+        it1.return();
+      }
+    } catch (e) {
+      errors.push(e);
+    }
+    try {
+      if (v2 && !v2.done && it2.return) {
+        it2.return();
+      }
+    } catch (e) {
+      errors.push(e);
+    }
+    if (errors.length) {
+      const err = assert.error(X`Merged iterator failed to close cleanly`);
+      for (const e of errors) {
+        assert.note(err, X`Caused by ${e}`);
+      }
+    }
   }
 }
 
@@ -56,7 +102,7 @@ function* mergeUtf16SortedIterators(it1, it2) {
  * that buffers any mutations until told to commit them.
  *
  * @param {KVStore} kvStore  The StorageAPI object that this crank buffer will be based on.
- * @param {CreateSHA256}  createSHA256
+ * @param {import('../../lib-nodejs/hasher.js').CreateSHA256}  createSHA256
  * @param { (key: string) => 'consensus' | 'local' | 'invalid' } getKeyType
  * @returns {*} an object {
  * crankBuffer,  // crank buffer as described, wrapping `kvStore`

packages/SwingSet/src/lib-nodejs/hasher.js

@@ -3,7 +3,7 @@ import { assert } from '@agoric/assert';
 import { createHash } from 'crypto';
 
 /**
- * @typedef { (initial: string?) => {
+ * @typedef { (initial?: string) => {
  *             add: (more: string) => void,
  *             finish: () => string,
  *            }

packages/SwingSet/src/liveslots/collectionManager.js

@@ -17,8 +17,11 @@ import { Far, passStyleOf } from '@endo/marshal';
 import { decodeToJustin } from '@endo/marshal/src/marshal-justin.js';
 import { parseVatSlot } from '../lib/parseVatSlots.js';
 
-// The maximum length of an LMDB key is 254 characters, which puts an upper
-// bound on the post-encoding size of keys than can be used to index entries in
+// The maximum length of an LMDB key used to be 511 bytes which corresponded to
+// 254 UTF-16 code units when using JS strings directly as keys. While we now
+// have larger key sizes and use a different unicode encoding, the prior
+// collections key constraints remain the same. They put an upper bound on the
+// post-encoding (stringified) size of keys than can be used to index entries in
 // collections.  In addition to the encoding of the collection entry key, the
 // storage key will also be prefixed with additional indexing information that
 // includes the collection ID (an integer that will grow over time as more

packages/SwingSet/test/test-state.js

@@ -36,7 +36,7 @@ function checkState(t, getState, expected) {
   t.deepEqual(got.sort(compareStrings), expected.sort(compareStrings));
 }
 
-function testStorage(t, s, getState, commit) {
+async function testStorage(t, s, getState, commit) {
   t.falsy(s.has('missing'));
   t.is(s.get('missing'), undefined);
 
@@ -60,7 +60,7 @@ function testStorage(t, s, getState, commit) {
 
   if (commit) {
     checkState(t, getState, []);
-    commit();
+    await commit();
   }
   checkState(t, getState, [
     ['foo', 'f'],
@@ -69,18 +69,23 @@ function testStorage(t, s, getState, commit) {
   ]);
 }
 
-test('storageInMemory', t => {
+test('storageInMemory', async t => {
   const store = initSwingStore(null);
-  testStorage(t, store.kvStore, () => getAllState(store).kvStuff, null);
+  await testStorage(t, store.kvStore, () => getAllState(store).kvStuff, null);
 });
 
-test('crankBuffer fulfills storage API', t => {
+test('crankBuffer fulfills storage API', async t => {
   const store = initSwingStore(null);
   const { crankBuffer, commitCrank } = buildCrankBuffer(
     store.kvStore,
     createSHA256,
   );
-  testStorage(t, crankBuffer, () => getAllState(store).kvStuff, commitCrank);
+  await testStorage(
+    t,
+    crankBuffer,
+    () => getAllState(store).kvStuff,
+    commitCrank,
+  );
 });
 
 test('crankBuffer handles key iteration properly even with intra-crank data changes', t => {

packages/SwingSet/test/vat-warehouse/test-warehouse.js

@@ -1,5 +1,10 @@
 // @ts-check
 
+import '@endo/init/pre-bundle-source.js';
+
+// import lmdb early to work around SES incompatibility
+import 'lmdb';
+
 // eslint-disable-next-line import/order
 import { test } from '../../tools/prepare-test-env-ava.js';
 import fs from 'fs';
@@ -124,7 +129,7 @@ test('snapshot after deliveries', async t => {
   t.teardown(c.shutdown);
 
   await runSteps(c, t);
-  commit();
+  await commit();
 
   const { inUse, onDisk, extra } = unusedSnapshotsOnDisk(
     kvStore,

packages/access-token/package.json

@@ -17,7 +17,8 @@
   },
   "dependencies": {
     "@agoric/assert": "^0.4.0",
-    "n-readlines": "^1.0.0"
+    "n-readlines": "^1.0.0",
+    "tmp": "^0.2.1"
   },
   "devDependencies": {
     "ava": "^3.12.1",