From ea19655fffc4056a95eec0a3def8532be4507f24 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 4 Oct 2024 13:38:59 +0200 Subject: [PATCH 01/13] remove unused method --- src/lib/proof-system/zkprogram.ts | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/lib/proof-system/zkprogram.ts b/src/lib/proof-system/zkprogram.ts index 544cb53eb6..46991dff7b 100644 --- a/src/lib/proof-system/zkprogram.ts +++ b/src/lib/proof-system/zkprogram.ts @@ -1,9 +1,8 @@ import { EmptyUndefined, EmptyVoid } from '../../bindings/lib/generic.js'; import { Snarky, initializeBindings, withThreadPool } from '../../snarky.js'; import { Pickles, Gate } from '../../snarky.js'; -import { Field, Bool } from '../provable/wrapped.js'; +import { Field } from '../provable/wrapped.js'; import { - FlexibleProvable, FlexibleProvablePure, InferProvable, ProvablePureExtended, @@ -12,7 +11,6 @@ import { import { InferProvableType, provable, - provablePure, } from '../provable/types/provable-derivers.js'; import { Provable } from '../provable/provable.js'; import { assert, prettifyStacktracePromise } from '../util/errors.js'; @@ -528,15 +526,6 @@ function isProvable(type: unknown): type is ProvableType { ); } -function isProofType(type: unknown): type is typeof ProofBase { - // the third case covers subclasses - return ( - type === Proof || - type === DynamicProof || - (typeof type === 'function' && type.prototype instanceof ProofBase) - ); -} - function isDynamicProof( type: Subclass ): type is Subclass { From fdcb8b6270fd90338a41c6ee1727e5b69ebec0e1 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 10 Oct 2024 12:48:39 +0200 Subject: [PATCH 02/13] export a few utility types --- src/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index c6e6f78f04..f01e9350fe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,7 +37,12 @@ export type { FlexibleProvablePure, InferProvable, } from './lib/provable/types/struct.js'; -export { From } from './bindings/lib/provable-generic.js'; +export { + From, + InferValue, + InferJson, + IsPure, +} from './bindings/lib/provable-generic.js'; export { ProvableType } from './lib/provable/types/provable-intf.js'; export { provable, From 80fcc08768f6586474da9b04f383c83a53c9851c Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 10 Oct 2024 13:03:52 +0200 Subject: [PATCH 03/13] expose arrayGet --- src/lib/provable/gadgets/basic.ts | 7 ++++++- src/lib/provable/gadgets/gadgets.ts | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/lib/provable/gadgets/basic.ts b/src/lib/provable/gadgets/basic.ts index d7756d951e..efc91e66e7 100644 --- a/src/lib/provable/gadgets/basic.ts +++ b/src/lib/provable/gadgets/basic.ts @@ -74,9 +74,14 @@ function assertMul( * * Assumes that index is in [0, n), returns an unconstrained result otherwise. * - * Note: This saves 0.5*n constraints compared to equals() + switch() + * Note: This saves 0.5*n constraints compared to equals() + switch() even if equals() were implemented optimally. */ function arrayGet(array: Field[], index: Field) { + // if index is constant, we can return the value directly + if (index.isConstant()) { + return array[Number(index.toBigInt())]; + } + let i = toVar(index); // witness result diff --git a/src/lib/provable/gadgets/gadgets.ts b/src/lib/provable/gadgets/gadgets.ts index 205a0990a6..bab2ca7287 100644 --- a/src/lib/provable/gadgets/gadgets.ts +++ b/src/lib/provable/gadgets/gadgets.ts @@ -30,10 +30,29 @@ import { import { divMod32, addMod32 } from './arithmetic.js'; import { SHA256 } from './sha256.js'; import { rangeCheck3x12 } from './lookup.js'; +import { arrayGet } from './basic.js'; export { Gadgets, Field3, ForeignFieldSum }; const Gadgets = { + /** + * Get value from array at a Field element index, in O(n) constraints, where n is the array length. + * + * **Warning**: This gadget assumes that the index is within the array bounds `[0, n)`, + * and returns an unconstrained result otherwise. + * To use it with an index that is not already guaranteed to be within the array bounds, you should add a suitable range check. + * + * ```ts + * let array = Provable.witnessFields(3, () => [1n, 2n, 3n]); + * let index = Provable.witness(Field, () => 1n); + * + * let value = Gadgets.arrayGet(array, index); + * ``` + * + * **Note**: This saves n constraints compared to `Provable.switch(array.map((_, i) => index.equals(i)), type, array)`. + */ + arrayGet, + /** * Asserts that the input value is in the range [0, 2^64). * From 2635545ae89bbc670895f4d20d030beb60a17cdb Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 10 Oct 2024 14:13:14 +0200 Subject: [PATCH 04/13] expose bytes base class --- src/lib/provable/wrapped-classes.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/provable/wrapped-classes.ts b/src/lib/provable/wrapped-classes.ts index ad11e446b0..79121c5b64 100644 --- a/src/lib/provable/wrapped-classes.ts +++ b/src/lib/provable/wrapped-classes.ts @@ -19,3 +19,6 @@ function Bytes(size: number) { Bytes.from = InternalBytes.from; Bytes.fromHex = InternalBytes.fromHex; Bytes.fromString = InternalBytes.fromString; + +// expore base class so that we can detect Bytes with `instanceof` +Bytes.Base = InternalBytes; From a27a6cadcdcb664a9626fe36e87578df60264d95 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 10 Oct 2024 21:19:01 +0200 Subject: [PATCH 05/13] improve packed --- src/lib/provable/packed.ts | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/lib/provable/packed.ts b/src/lib/provable/packed.ts index e1b4cfe682..45e7b6bb6e 100644 --- a/src/lib/provable/packed.ts +++ b/src/lib/provable/packed.ts @@ -51,9 +51,14 @@ class Packed { * Create a packed representation of `type`. You can then use `PackedType.pack(x)` to pack a value. */ static create( - type: WithProvable> + type: WithProvable> ): typeof Packed & { provable: ProvableHashable>; + + /** + * Pack a value. + */ + pack(x: T): Packed; } { let provable = ProvableType.get(type); // compute size of packed representation @@ -67,6 +72,15 @@ class Packed { value: Unconstrained, }) satisfies ProvableHashable> as ProvableHashable>; + static pack(x: T): Packed { + let input = provable.toInput(x); + let packed = packToFields(input); + let unconstrained = Unconstrained.witness(() => + Provable.toConstant(provable, x) + ); + return new Packed_(packed, unconstrained); + } + static empty(): Packed { return Packed_.pack(provable.empty()); } @@ -83,19 +97,6 @@ class Packed { this.value = value; } - /** - * Pack a value. - */ - static pack(x: T): Packed { - let type = this.innerProvable; - let input = type.toInput(x); - let packed = packToFields(input); - let unconstrained = Unconstrained.witness(() => - Provable.toConstant(type, x) - ); - return new this(packed, unconstrained); - } - /** * Unpack a value. */ @@ -120,13 +121,13 @@ class Packed { // dynamic subclassing infra static _provable: ProvableHashable> | undefined; - static _innerProvable: ProvableExtended | undefined; + static _innerProvable: ProvableHashable | undefined; get Constructor(): typeof Packed { return this.constructor as typeof Packed; } - static get innerProvable(): ProvableExtended { + static get innerProvable(): ProvableHashable { assert(this._innerProvable !== undefined, 'Packed not initialized'); return this._innerProvable; } From 6bcbdf7facf028c4f10b933ae830f7ce9d028433 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 10 Oct 2024 21:37:04 +0200 Subject: [PATCH 06/13] mapValue --- src/lib/provable/types/provable-derivers.ts | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/lib/provable/types/provable-derivers.ts b/src/lib/provable/types/provable-derivers.ts index 4cb5178d47..2ce7cbbbf1 100644 --- a/src/lib/provable/types/provable-derivers.ts +++ b/src/lib/provable/types/provable-derivers.ts @@ -45,6 +45,7 @@ export { InferredProvable, IsPure, NestedProvable, + mapValue, }; type ProvableExtension = { @@ -169,3 +170,39 @@ function provableExtends< }, } satisfies ProvableHashable>; } + +function mapValue, V extends InferValue, W>( + provable: A, + there: (x: V) => W, + back: (x: W) => V +): ProvableHashable, W> { + return { + sizeInFields() { + return provable.sizeInFields(); + }, + toFields(value) { + return provable.toFields(value); + }, + toAuxiliary(value) { + return provable.toAuxiliary(value); + }, + fromFields(fields, aux) { + return provable.fromFields(fields, aux); + }, + check(value) { + provable.check(value); + }, + toValue(value) { + return there(provable.toValue(value)); + }, + fromValue(value) { + return provable.fromValue(back(value)); + }, + empty() { + return provable.empty(); + }, + toInput(value) { + return provable.toInput(value); + }, + }; +} From 9e539b5f004fad93f80c9d28f184db9ea8b26930 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 10 Oct 2024 21:37:19 +0200 Subject: [PATCH 07/13] nice value type for packed --- src/lib/provable/packed.ts | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/lib/provable/packed.ts b/src/lib/provable/packed.ts index 45e7b6bb6e..5d8d8abbd8 100644 --- a/src/lib/provable/packed.ts +++ b/src/lib/provable/packed.ts @@ -1,4 +1,4 @@ -import { provableFromClass } from './types/provable-derivers.js'; +import { mapValue, provableFromClass } from './types/provable-derivers.js'; import { HashInput, ProvableExtended } from './types/struct.js'; import { Unconstrained } from './types/unconstrained.js'; import { Field } from './field.js'; @@ -50,10 +50,10 @@ class Packed { /** * Create a packed representation of `type`. You can then use `PackedType.pack(x)` to pack a value. */ - static create( - type: WithProvable> + static create( + type: WithProvable> ): typeof Packed & { - provable: ProvableHashable>; + provable: ProvableHashable, V>; /** * Pack a value. @@ -64,13 +64,28 @@ class Packed { // compute size of packed representation let input = provable.toInput(provable.empty()); let packedSize = countFields(input); + let packedFields = fields(packedSize); return class Packed_ extends Packed { static _innerProvable = provable; - static _provable = provableFromClass(Packed_, { - packed: fields(packedSize), - value: Unconstrained, - }) satisfies ProvableHashable> as ProvableHashable>; + static _provable = mapValue( + provableFromClass(Packed_, { + packed: packedFields, + value: Unconstrained, + }), + ({ value }: { value: Unconstrained }) => + provable.toValue(value.get()), + (x: V) => { + let { packed, value } = Packed_.pack(provable.fromValue(x)); + return { + packed: packedFields.toValue(packed), + value: Unconstrained.from(value), + }; + } + ) satisfies ProvableHashable, V> as ProvableHashable< + Packed, + V + >; static pack(x: T): Packed { let input = provable.toInput(x); From 90a882241234df29205059e0428d847a277875af Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 11 Oct 2024 08:31:52 +0200 Subject: [PATCH 08/13] fixup --- src/lib/provable/packed.ts | 3 ++- src/lib/provable/types/provable-derivers.ts | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lib/provable/packed.ts b/src/lib/provable/packed.ts index 5d8d8abbd8..23ba41f944 100644 --- a/src/lib/provable/packed.ts +++ b/src/lib/provable/packed.ts @@ -75,7 +75,8 @@ class Packed { }), ({ value }: { value: Unconstrained }) => provable.toValue(value.get()), - (x: V) => { + (x) => { + if (x instanceof Packed) return x; let { packed, value } = Packed_.pack(provable.fromValue(x)); return { packed: packedFields.toValue(packed), diff --git a/src/lib/provable/types/provable-derivers.ts b/src/lib/provable/types/provable-derivers.ts index 2ce7cbbbf1..8353c2474a 100644 --- a/src/lib/provable/types/provable-derivers.ts +++ b/src/lib/provable/types/provable-derivers.ts @@ -171,11 +171,16 @@ function provableExtends< } satisfies ProvableHashable>; } -function mapValue, V extends InferValue, W>( +function mapValue< + A extends ProvableHashable, + V extends InferValue, + W, + T extends InferProvable +>( provable: A, there: (x: V) => W, - back: (x: W) => V -): ProvableHashable, W> { + back: (x: W | T) => V | T +): ProvableHashable { return { sizeInFields() { return provable.sizeInFields(); From 7b62ad1ee960a4d255fc039882f8bf4f14a32f16 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 11 Oct 2024 13:29:03 +0200 Subject: [PATCH 09/13] expose sha2 padding and fix input mutation in compression --- src/lib/provable/gadgets/sha256.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/provable/gadgets/sha256.ts b/src/lib/provable/gadgets/sha256.ts index ee7529805f..4d92c2e8fe 100644 --- a/src/lib/provable/gadgets/sha256.ts +++ b/src/lib/provable/gadgets/sha256.ts @@ -102,6 +102,7 @@ const SHA256 = { }, compression: sha256Compression, createMessageSchedule, + padding, get initialState() { return SHA256Constants.H.map((x) => UInt32.from(x)); }, @@ -239,7 +240,7 @@ function sigma(u: UInt32, bits: TupleN, firstShifted = false) { * * @returns The updated intermediate hash values after compression. */ -function sha256Compression(H: UInt32[], W: UInt32[]) { +function sha256Compression([...H]: UInt32[], W: UInt32[]) { // initialize working variables let a = H[0]; let b = H[1]; From 0384736cf2e47cd60bfbd6292e0d515773a09dee Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 11 Oct 2024 13:29:19 +0200 Subject: [PATCH 10/13] export tupleN --- src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.ts b/src/index.ts index f01e9350fe..5b4e9d2f64 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ +export { TupleN } from './lib/util/types.js'; export type { ProvablePure } from './lib/provable/types/provable-intf.js'; export { Ledger, initializeBindings } from './snarky.js'; export { Field, Bool, Group, Scalar } from './lib/provable/wrapped.js'; From 8e3ac5e6071c721c7370bec92c8d648b7cf2e911 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 11 Oct 2024 16:13:04 +0200 Subject: [PATCH 11/13] uint32 to./from bytes --- src/lib/provable/int.ts | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index 15c64c6249..e26c2cca07 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -18,6 +18,7 @@ import { lessThanOrEqualGeneric, } from './gadgets/comparison.js'; import { assert } from '../util/assert.js'; +import { TupleN } from '../util/types.js'; // external API export { UInt8, UInt32, UInt64, Int64, Sign }; @@ -956,6 +957,47 @@ class UInt32 extends CircuitValue { ): InstanceType { return UInt32.from(x) as any; } + + /** + * Split a UInt32 into 4 UInt8s, in little-endian order. + */ + toBytes(): TupleN { + // witness the bytes + let bytes = Provable.witness(Provable.Array(UInt8, 4), () => { + let x = this.value.toBigInt(); + return [0, 1, 2, 3].map((i) => UInt8.from((x >> BigInt(8 * i)) & 0xffn)); + }); + // prove that bytes are correct + UInt32.fromBytes(bytes).assertEquals(this); + return TupleN.fromArray(4, bytes); + } + + /** + * Split a UInt32 into 4 UInt8s, in big-endian order. + */ + toBytesBE(): TupleN { + return TupleN.fromArray(4, this.toBytes().reverse()); + } + + /** + * Combine 4 UInt8s into a UInt32, in little-endian order. + */ + static fromBytes(bytes: UInt8[]): UInt32 { + assert(bytes.length === 4, '4 bytes needed to create a uint32'); + + let word = Field(0); + bytes.forEach(({ value }, i) => { + word = word.add(value.mul(1n << BigInt(8 * i))); + }); + return UInt32.Unsafe.fromField(word); + } + + /** + * Combine 4 UInt8s into a UInt32, in big-endian order. + */ + static fromBytesBE(bytes: UInt8[]): UInt32 { + return UInt32.fromBytes([...bytes].reverse()); + } } class Sign extends CircuitValue { From ecc4b03ab6890f5a8464a73d3d056e5c835d7846 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 11 Oct 2024 16:35:05 +0200 Subject: [PATCH 12/13] remove duplication --- src/lib/provable/gadgets/sha256.ts | 10 ++-------- src/lib/provable/int.ts | 23 ++++++----------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/src/lib/provable/gadgets/sha256.ts b/src/lib/provable/gadgets/sha256.ts index 4d92c2e8fe..60fc64ae01 100644 --- a/src/lib/provable/gadgets/sha256.ts +++ b/src/lib/provable/gadgets/sha256.ts @@ -8,7 +8,6 @@ import { Bytes } from '../wrapped-classes.js'; import { chunk } from '../../util/arrays.js'; import { TupleN } from '../../util/types.js'; import { divMod32 } from './arithmetic.js'; -import { bytesToWord, wordToBytes } from './bit-slices.js'; import { bitSlice } from './common.js'; import { rangeCheck16 } from './range-check.js'; @@ -70,11 +69,7 @@ function padding(data: FlexibleBytes): UInt32[][] { for (let i = 0; i < paddedMessage.length; i += 4) { // chunk 4 bytes into one UInt32, as expected by SHA256 // bytesToWord expects little endian, so we reverse the bytes - chunks.push( - UInt32.Unsafe.fromField( - bytesToWord(paddedMessage.slice(i, i + 4).reverse()) - ) - ); + chunks.push(UInt32.fromBytesBE(paddedMessage.slice(i, i + 4))); } // split message into 16 element sized message blocks @@ -97,8 +92,7 @@ const SHA256 = { } // the working variables H[i] are 32bit, however we want to decompose them into bytes to be more compatible - // wordToBytes expects little endian, so we reverse the bytes - return Bytes.from(H.map((x) => wordToBytes(x.value, 4).reverse()).flat()); + return Bytes.from(H.map((x) => x.toBytesBE()).flat()); }, compression: sha256Compression, createMessageSchedule, diff --git a/src/lib/provable/int.ts b/src/lib/provable/int.ts index e26c2cca07..1f4dee2512 100644 --- a/src/lib/provable/int.ts +++ b/src/lib/provable/int.ts @@ -19,6 +19,7 @@ import { } from './gadgets/comparison.js'; import { assert } from '../util/assert.js'; import { TupleN } from '../util/types.js'; +import { bytesToWord, wordToBytes } from './gadgets/bit-slices.js'; // external API export { UInt8, UInt32, UInt64, Int64, Sign }; @@ -961,22 +962,15 @@ class UInt32 extends CircuitValue { /** * Split a UInt32 into 4 UInt8s, in little-endian order. */ - toBytes(): TupleN { - // witness the bytes - let bytes = Provable.witness(Provable.Array(UInt8, 4), () => { - let x = this.value.toBigInt(); - return [0, 1, 2, 3].map((i) => UInt8.from((x >> BigInt(8 * i)) & 0xffn)); - }); - // prove that bytes are correct - UInt32.fromBytes(bytes).assertEquals(this); - return TupleN.fromArray(4, bytes); + toBytes() { + return TupleN.fromArray(4, wordToBytes(this.value, 4)); } /** * Split a UInt32 into 4 UInt8s, in big-endian order. */ - toBytesBE(): TupleN { - return TupleN.fromArray(4, this.toBytes().reverse()); + toBytesBE() { + return TupleN.fromArray(4, wordToBytes(this.value, 4).reverse()); } /** @@ -984,12 +978,7 @@ class UInt32 extends CircuitValue { */ static fromBytes(bytes: UInt8[]): UInt32 { assert(bytes.length === 4, '4 bytes needed to create a uint32'); - - let word = Field(0); - bytes.forEach(({ value }, i) => { - word = word.add(value.mul(1n << BigInt(8 * i))); - }); - return UInt32.Unsafe.fromField(word); + return UInt32.Unsafe.fromField(bytesToWord(bytes)); } /** From ee527961c427d43ea709fce9a0fc3b56c0db3d72 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 11 Oct 2024 17:05:32 +0200 Subject: [PATCH 13/13] handle out of bounds --- src/lib/provable/gadgets/basic.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/provable/gadgets/basic.ts b/src/lib/provable/gadgets/basic.ts index efc91e66e7..aaf8935917 100644 --- a/src/lib/provable/gadgets/basic.ts +++ b/src/lib/provable/gadgets/basic.ts @@ -79,13 +79,13 @@ function assertMul( function arrayGet(array: Field[], index: Field) { // if index is constant, we can return the value directly if (index.isConstant()) { - return array[Number(index.toBigInt())]; + return array[Number(index.toBigInt())] ?? createField(0n); } let i = toVar(index); // witness result - let a = existsOne(() => array[Number(i.toBigInt())].toBigInt()); + let a = existsOne(() => array[Number(i.toBigInt())].toBigInt() ?? 0n); // we prove a === array[j] + z[j]*(i - j) for some z[j], for all j. // setting j = i, this implies a === array[i]