From 9d88a939a1d35b315b790e71f6c467753a05e37c Mon Sep 17 00:00:00 2001 From: Emma Casolin Date: Wed, 25 May 2022 14:02:46 +1000 Subject: [PATCH 1/4] Updating `keyPathToKey` to escape key parts --- src/utils.ts | 11 +++++------ tests/utils.test.ts | 40 ++++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 7cba47e5..c1f940c1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -18,8 +18,7 @@ const esc = Buffer.from([92]); * Converts KeyPath to key buffer * e.g. ['A', 'B'] => !A!B (where ! is the sep) * An empty key path is converted to `['']` - * Level parts must not contain the separator - * Key actual part is allowed to contain the separator + * Level and key parts must not contain the separator */ function keyPathToKey(keyPath: KeyPath): Buffer { if (keyPath.length < 1) { @@ -29,7 +28,7 @@ function keyPathToKey(keyPath: KeyPath): Buffer { const levelPath = keyPath.slice(0, -1); return Buffer.concat([ levelPathToKey(levelPath), - typeof keyPart === 'string' ? Buffer.from(keyPart, 'utf-8') : keyPart, + escapePart(typeof keyPart === 'string' ? Buffer.from(keyPart, 'utf-8') : keyPart), ]); } @@ -42,7 +41,7 @@ function levelPathToKey(levelPath: LevelPath): Buffer { return Buffer.concat( levelPath.map((p) => { p = typeof p === 'string' ? Buffer.from(p, 'utf-8') : p; - p = escapeLevel(p); + p = escapePart(p); return Buffer.concat([sep, p, sep]); }), ); @@ -51,7 +50,7 @@ function levelPathToKey(levelPath: LevelPath): Buffer { /** * Escapes the level part for escape and separator */ -function escapeLevel(buf: Buffer): Buffer { +function escapePart(buf: Buffer): Buffer { const bytes: Array = []; for (let i = 0; i < buf.byteLength; i++) { const b = buf[i]; @@ -250,7 +249,7 @@ function fromArrayBuffer( export { sep, esc, - escapeLevel, + escapePart, unescapeLevel, keyPathToKey, levelPathToKey, diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 2bfeeeb1..2ebd7ae2 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -4,28 +4,32 @@ import * as utils from '@/utils'; describe('utils', () => { test('parse key paths', () => { const keyPaths: Array = [ - // Separator can be used in key part - ['foo', 'bar', Buffer.concat([utils.sep, Buffer.from('key'), utils.sep])], - [utils.sep], - [Buffer.concat([utils.sep, Buffer.from('foobar')])], - [Buffer.concat([Buffer.from('foobar'), utils.sep])], - [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep])], - // Escape can be used in key part - [utils.esc], - [Buffer.concat([utils.esc, Buffer.from('foobar')])], - [Buffer.concat([Buffer.from('foobar'), utils.esc])], - [Buffer.concat([utils.esc, Buffer.from('foobar'), utils.esc])], - // Separator can be used in level parts - [Buffer.concat([utils.sep, Buffer.from('foobar')]), 'key'], - [Buffer.concat([Buffer.from('foobar'), utils.sep]), 'key'], - [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep]), 'key'], - // Escape can be used in level parts - [Buffer.concat([utils.sep, utils.esc, utils.sep]), 'key'], - [Buffer.concat([utils.esc, utils.esc, utils.esc]), 'key'], + // // Separator can be used in key part + // ['foo', 'bar', Buffer.concat([utils.sep, Buffer.from('key'), utils.sep])], + // [utils.sep], + // [Buffer.concat([utils.sep, Buffer.from('foobar')])], + // [Buffer.concat([Buffer.from('foobar'), utils.sep])], + // [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep])], + [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep, Buffer.from('foobar')])], + // // Escape can be used in key part + // [utils.esc], + // [Buffer.concat([utils.esc, Buffer.from('foobar')])], + // [Buffer.concat([Buffer.from('foobar'), utils.esc])], + // [Buffer.concat([utils.esc, Buffer.from('foobar'), utils.esc])], + // // Separator can be used in level parts + // [Buffer.concat([utils.sep, Buffer.from('foobar')]), 'key'], + // [Buffer.concat([Buffer.from('foobar'), utils.sep]), 'key'], + // [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep]), 'key'], + // // Escape can be used in level parts + // [Buffer.concat([utils.sep, utils.esc, utils.sep]), 'key'], + // [Buffer.concat([utils.esc, utils.esc, utils.esc]), 'key'], ]; for (const keyPath of keyPaths) { + console.log('BEFORE', keyPath); const key = utils.keyPathToKey(keyPath); + console.log('KEY', key); const keyPath_ = utils.parseKey(key); + console.log('AFTER', keyPath_); expect(keyPath.map((b) => b.toString())).toStrictEqual( keyPath_.map((b) => b.toString()), ); From 3e74a6699540c9dc9ae6b23d2f1cb2c48f6adbc1 Mon Sep 17 00:00:00 2001 From: Emma Casolin Date: Wed, 25 May 2022 14:38:00 +1000 Subject: [PATCH 2/4] Updating `parseKey` to unescape key parts --- src/utils.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index c1f940c1..ee81bc3d 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -28,7 +28,9 @@ function keyPathToKey(keyPath: KeyPath): Buffer { const levelPath = keyPath.slice(0, -1); return Buffer.concat([ levelPathToKey(levelPath), - escapePart(typeof keyPart === 'string' ? Buffer.from(keyPart, 'utf-8') : keyPart), + escapePart( + typeof keyPart === 'string' ? Buffer.from(keyPart, 'utf-8') : keyPart, + ), ]); } @@ -48,7 +50,7 @@ function levelPathToKey(levelPath: LevelPath): Buffer { } /** - * Escapes the level part for escape and separator + * Escapes level and key parts for escape and separator */ function escapePart(buf: Buffer): Buffer { const bytes: Array = []; @@ -66,9 +68,9 @@ function escapePart(buf: Buffer): Buffer { } /** - * Unescapes the level part of escape and separator + * Unescapes level and key parts of escape and separator */ -function unescapeLevel(buf: Buffer): Buffer { +function unescapePart(buf: Buffer): Buffer { const bytes: Array = []; for (let i = 0; i < buf.byteLength; i++) { const b = buf[i]; @@ -107,8 +109,8 @@ function parseKey(key: Buffer): KeyPath { if (bufs.length < 1) { throw new TypeError('Buffer is not a key'); } - for (let i = 0; i < bufs.length - 1; i++) { - bufs[i] = unescapeLevel(bufs[i]); + for (let i = 0; i < bufs.length; i++) { + bufs[i] = unescapePart(bufs[i]); } return bufs; } @@ -250,7 +252,7 @@ export { sep, esc, escapePart, - unescapeLevel, + unescapePart, keyPathToKey, levelPathToKey, parseKey, From f51490b451866ac2ed7f3e6a8e539dad38c2709c Mon Sep 17 00:00:00 2001 From: Emma Casolin Date: Wed, 25 May 2022 14:38:19 +1000 Subject: [PATCH 3/4] Additional keypath parsing tests --- tests/utils.test.ts | 90 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 2ebd7ae2..e02acb9d 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -4,32 +4,78 @@ import * as utils from '@/utils'; describe('utils', () => { test('parse key paths', () => { const keyPaths: Array = [ - // // Separator can be used in key part - // ['foo', 'bar', Buffer.concat([utils.sep, Buffer.from('key'), utils.sep])], - // [utils.sep], - // [Buffer.concat([utils.sep, Buffer.from('foobar')])], - // [Buffer.concat([Buffer.from('foobar'), utils.sep])], - // [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep])], - [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep, Buffer.from('foobar')])], - // // Escape can be used in key part - // [utils.esc], - // [Buffer.concat([utils.esc, Buffer.from('foobar')])], - // [Buffer.concat([Buffer.from('foobar'), utils.esc])], - // [Buffer.concat([utils.esc, Buffer.from('foobar'), utils.esc])], - // // Separator can be used in level parts - // [Buffer.concat([utils.sep, Buffer.from('foobar')]), 'key'], - // [Buffer.concat([Buffer.from('foobar'), utils.sep]), 'key'], - // [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep]), 'key'], - // // Escape can be used in level parts - // [Buffer.concat([utils.sep, utils.esc, utils.sep]), 'key'], - // [Buffer.concat([utils.esc, utils.esc, utils.esc]), 'key'], + // Separator can be used in key part + ['foo', 'bar', Buffer.concat([utils.sep, Buffer.from('key'), utils.sep])], + [utils.sep], + [Buffer.concat([utils.sep, Buffer.from('foobar')])], + [Buffer.concat([Buffer.from('foobar'), utils.sep])], + [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep])], + [ + Buffer.concat([ + utils.sep, + Buffer.from('foobar'), + utils.sep, + Buffer.from('foobar'), + ]), + ], + [ + Buffer.concat([ + Buffer.from('foobar'), + utils.sep, + Buffer.from('foobar'), + utils.sep, + ]), + ], + // Escape can be used in key part + [utils.esc], + [Buffer.concat([utils.esc, Buffer.from('foobar')])], + [Buffer.concat([Buffer.from('foobar'), utils.esc])], + [Buffer.concat([utils.esc, Buffer.from('foobar'), utils.esc])], + [ + Buffer.concat([ + utils.esc, + Buffer.from('foobar'), + utils.esc, + Buffer.from('foobar'), + ]), + ], + [ + Buffer.concat([ + Buffer.from('foobar'), + utils.esc, + Buffer.from('foobar'), + utils.esc, + ]), + ], + // Separator can be used in level parts + [Buffer.concat([utils.sep, Buffer.from('foobar')]), 'key'], + [Buffer.concat([Buffer.from('foobar'), utils.sep]), 'key'], + [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep]), 'key'], + [ + Buffer.concat([ + utils.sep, + Buffer.from('foobar'), + utils.sep, + Buffer.from('foobar'), + ]), + 'key', + ], + [ + Buffer.concat([ + Buffer.from('foobar'), + utils.sep, + Buffer.from('foobar'), + utils.sep, + ]), + 'key', + ], + // Escape can be used in level parts + [Buffer.concat([utils.sep, utils.esc, utils.sep]), 'key'], + [Buffer.concat([utils.esc, utils.esc, utils.esc]), 'key'], ]; for (const keyPath of keyPaths) { - console.log('BEFORE', keyPath); const key = utils.keyPathToKey(keyPath); - console.log('KEY', key); const keyPath_ = utils.parseKey(key); - console.log('AFTER', keyPath_); expect(keyPath.map((b) => b.toString())).toStrictEqual( keyPath_.map((b) => b.toString()), ); From cde3e6b8ca308cf986f2e908620d805fb8419543 Mon Sep 17 00:00:00 2001 From: Emma Casolin Date: Wed, 25 May 2022 16:28:06 +1000 Subject: [PATCH 4/4] WIP --- src/DB.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/DB.ts b/src/DB.ts index f540bd3a..66cda993 100644 --- a/src/DB.ts +++ b/src/DB.ts @@ -321,6 +321,7 @@ class DB { * @internal */ public async _del(keyPath: KeyPath): Promise { + console.log('DEL', keyPath); return this._db.del(utils.keyPathToKey(keyPath)); } @@ -458,15 +459,15 @@ class DB { if (options.gt != null) { options.gt = Buffer.concat([ levelKeyStart, - typeof options.gt === 'string' ? Buffer.from(options.gt) : options.gt, + utils.escapePart(typeof options.gt === 'string' ? Buffer.from(options.gt) : options.gt), ]); } if (options.gte != null) { options.gte = Buffer.concat([ levelKeyStart, - typeof options.gte === 'string' + utils.escapePart(typeof options.gte === 'string' ? Buffer.from(options.gte) - : options.gte, + : options.gte), ]); } if (options.gt == null && options.gte == null) { @@ -475,15 +476,15 @@ class DB { if (options?.lt != null) { options.lt = Buffer.concat([ levelKeyStart, - typeof options.lt === 'string' ? Buffer.from(options.lt) : options.lt, + utils.escapePart(typeof options.lt === 'string' ? Buffer.from(options.lt) : options.lt), ]); } if (options?.lte != null) { options.lte = Buffer.concat([ levelKeyStart, - typeof options.lte === 'string' + utils.escapePart(typeof options.lte === 'string' ? Buffer.from(options.lte) - : options.lte, + : options.lte), ]); } if (options.lt == null && options.lte == null) { @@ -503,11 +504,15 @@ class DB { const kv = await next(); // If kv is undefined, we have reached the end of iteration if (kv != null) { + + console.log('K', kv[0]); // Handle keys: false if (kv[0] != null) { // Truncate level path so the returned key is relative to the level path const keyPath = utils.parseKey(kv[0]).slice(levelPath.length); + console.log('KEYPATH REMAINING', keyPath); kv[0] = utils.keyPathToKey(keyPath); + console.log('KEYPATH NEW', kv[0]); } // Handle values: false if (kv[1] != null) { @@ -535,7 +540,7 @@ class DB { */ public async _clear(levelPath: LevelPath = []): Promise { for await (const [k] of this._iterator({ values: false }, levelPath)) { - await this._del([...levelPath, k]); + await this._del([...levelPath, ...utils.parseKey(k)]); } }