From 4c37a426d84ac82109c20e2973c7a912771de6f1 Mon Sep 17 00:00:00 2001 From: Jeongho Nam Date: Mon, 16 Dec 2024 12:16:28 +0900 Subject: [PATCH] Enhance `ChatGptTypeChecker` and `LlmTypeCheckerV3`. `ChatGptTypeChecker` had not considered `additionalProperties` case, and `LlmTypeCheckerV3` had not defined the visitor function. --- package.json | 2 +- src/utils/ChatGptTypeChecker.ts | 30 ++- src/utils/LlmTypeCheckerV3.ts | 168 ++++++++++++++ .../validate_llm_type_checker_cover_any.ts | 63 ++++++ .../validate_llm_type_checker_cover_array.ts | 211 ++++++++++++++++++ 5 files changed, 469 insertions(+), 5 deletions(-) create mode 100644 test/features/llm/validate_llm_type_checker_cover_any.ts create mode 100644 test/features/llm/validate_llm_type_checker_cover_array.ts diff --git a/package.json b/package.json index 36a305e..b244c2a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@samchon/openapi", - "version": "2.2.0", + "version": "2.2.1", "description": "OpenAPI definitions and converters for 'typia' and 'nestia'.", "main": "./lib/index.js", "module": "./lib/index.mjs", diff --git a/src/utils/ChatGptTypeChecker.ts b/src/utils/ChatGptTypeChecker.ts index 9391e20..719dbf1 100644 --- a/src/utils/ChatGptTypeChecker.ts +++ b/src/utils/ChatGptTypeChecker.ts @@ -153,10 +153,15 @@ export namespace ChatGptTypeChecker { if (found !== undefined) next(found, `${refAccessor}[${key}]`); } else if (ChatGptTypeChecker.isAnyOf(schema)) schema.anyOf.forEach((s, i) => next(s, `${accessor}.anyOf[${i}]`)); - else if (ChatGptTypeChecker.isObject(schema)) + else if (ChatGptTypeChecker.isObject(schema)) { for (const [key, value] of Object.entries(schema.properties)) next(value, `${accessor}.properties[${JSON.stringify(key)}]`); - else if (ChatGptTypeChecker.isArray(schema)) + if ( + typeof schema.additionalProperties === "object" && + schema.additionalProperties !== null + ) + next(schema.additionalProperties, `${accessor}.additionalProperties`); + } else if (ChatGptTypeChecker.isArray(schema)) next(schema.items, `${accessor}.items`); }; next(props.schema, props.accessor ?? "$input.schemas"); @@ -288,8 +293,24 @@ export namespace ChatGptTypeChecker { visited: Map>; x: IChatGptSchema.IObject; y: IChatGptSchema.IObject; - }): boolean => - Object.entries(p.y.properties ?? {}).every(([key, b]) => { + }): boolean => { + if (!p.x.additionalProperties && !!p.y.additionalProperties) return false; + else if ( + !!p.x.additionalProperties && + !!p.y.additionalProperties && + ((typeof p.x.additionalProperties === "object" && + p.y.additionalProperties === true) || + (typeof p.x.additionalProperties === "object" && + typeof p.y.additionalProperties === "object" && + !coverStation({ + $defs: p.$defs, + visited: p.visited, + x: p.x.additionalProperties, + y: p.y.additionalProperties, + }))) + ) + return false; + return Object.entries(p.y.properties ?? {}).every(([key, b]) => { const a: IChatGptSchema | undefined = p.x.properties?.[key]; if (a === undefined) return false; else if ( @@ -304,6 +325,7 @@ export namespace ChatGptTypeChecker { y: b, }); }); + }; const coverBoolean = ( x: IChatGptSchema.IBoolean, diff --git a/src/utils/LlmTypeCheckerV3.ts b/src/utils/LlmTypeCheckerV3.ts index 3290989..2230dcd 100644 --- a/src/utils/LlmTypeCheckerV3.ts +++ b/src/utils/LlmTypeCheckerV3.ts @@ -63,6 +63,174 @@ export namespace LlmTypeCheckerV3 { }); }; + export const covers = (x: ILlmSchemaV3, y: ILlmSchemaV3): boolean => { + const alpha: ILlmSchemaV3[] = flatSchema(x); + const beta: ILlmSchemaV3[] = flatSchema(y); + if (alpha.some((x) => isUnknown(x))) return true; + else if (beta.some((x) => isUnknown(x))) return false; + return beta.every((b) => + alpha.some((a) => { + // CHECK EQUALITY + if (a === b) return true; + else if (isUnknown(a)) return true; + else if (isUnknown(b)) return false; + else if (isNullOnly(a)) return isNullOnly(b); + else if (isNullOnly(b)) return isNullable(a); + else if (isNullable(a) && !isNullable(b)) return false; + // ATOMIC CASE + else if (isBoolean(a)) return isBoolean(b) && coverBoolean(a, b); + else if (isInteger(a)) return isInteger(b) && coverInteger(a, b); + else if (isNumber(a)) + return (isNumber(b) || isInteger(b)) && coverNumber(a, b); + else if (isString(a)) return isString(b) && covertString(a, b); + // INSTANCE CASE + else if (isArray(a)) return isArray(b) && coverArray(a, b); + else if (isObject(a)) return isObject(b) && coverObject(a, b); + else if (isOneOf(a)) return false; + }), + ); + }; + + /** + * @internal + */ + const coverBoolean = ( + x: ILlmSchemaV3.IBoolean, + y: ILlmSchemaV3.IBoolean, + ): boolean => + x.enum === undefined || + (y.enum !== undefined && x.enum.every((v) => y.enum!.includes(v))); + + /** + * @internal + */ + const coverInteger = ( + x: ILlmSchemaV3.IInteger, + y: ILlmSchemaV3.IInteger, + ): boolean => { + if (x.enum !== undefined) + return y.enum !== undefined && x.enum.every((v) => y.enum!.includes(v)); + return [ + x.type === y.type, + x.minimum === undefined || + (y.minimum !== undefined && x.minimum <= y.minimum), + x.maximum === undefined || + (y.maximum !== undefined && x.maximum >= y.maximum), + x.exclusiveMinimum !== true || + x.minimum === undefined || + (y.minimum !== undefined && + (y.exclusiveMinimum === true || x.minimum < y.minimum)), + x.exclusiveMaximum !== true || + x.maximum === undefined || + (y.maximum !== undefined && + (y.exclusiveMaximum === true || x.maximum > y.maximum)), + x.multipleOf === undefined || + (y.multipleOf !== undefined && + y.multipleOf / x.multipleOf === + Math.floor(y.multipleOf / x.multipleOf)), + ].every((v) => v); + }; + + /** + * @internal + */ + const coverNumber = ( + x: ILlmSchemaV3.INumber, + y: ILlmSchemaV3.INumber | ILlmSchemaV3.IInteger, + ): boolean => { + if (x.enum !== undefined) + return y.enum !== undefined && x.enum.every((v) => y.enum!.includes(v)); + return [ + x.type === y.type || (x.type === "number" && y.type === "integer"), + x.minimum === undefined || + (y.minimum !== undefined && x.minimum <= y.minimum), + x.maximum === undefined || + (y.maximum !== undefined && x.maximum >= y.maximum), + x.exclusiveMinimum !== true || + x.minimum === undefined || + (y.minimum !== undefined && + (y.exclusiveMinimum === true || x.minimum < y.minimum)), + x.exclusiveMaximum !== true || + x.maximum === undefined || + (y.maximum !== undefined && + (y.exclusiveMaximum === true || x.maximum > y.maximum)), + x.multipleOf === undefined || + (y.multipleOf !== undefined && + y.multipleOf / x.multipleOf === + Math.floor(y.multipleOf / x.multipleOf)), + ].every((v) => v); + }; + + /** + * @internal + */ + const covertString = ( + x: ILlmSchemaV3.IString, + y: ILlmSchemaV3.IString, + ): boolean => { + if (x.enum !== undefined) + return y.enum !== undefined && x.enum.every((v) => y.enum!.includes(v)); + return [ + x.type === y.type, + x.format === undefined || + (y.format !== undefined && coverFormat(x.format, y.format)), + x.pattern === undefined || x.pattern === y.pattern, + x.minLength === undefined || + (y.minLength !== undefined && x.minLength <= y.minLength), + x.maxLength === undefined || + (y.maxLength !== undefined && x.maxLength >= y.maxLength), + ].every((v) => v); + }; + + const coverFormat = ( + x: Required["format"], + y: Required["format"], + ): boolean => + x === y || + (x === "idn-email" && y === "email") || + (x === "idn-hostname" && y === "hostname") || + (["uri", "iri"].includes(x) && y === "url") || + (x === "iri" && y === "uri") || + (x === "iri-reference" && y === "uri-reference"); + + /** + * @internal + */ + const coverArray = ( + x: ILlmSchemaV3.IArray, + y: ILlmSchemaV3.IArray, + ): boolean => covers(x.items, y.items); + + const coverObject = ( + x: ILlmSchemaV3.IObject, + y: ILlmSchemaV3.IObject, + ): boolean => { + if (!x.additionalProperties && !!y.additionalProperties) return false; + else if ( + (!!x.additionalProperties && + !!y.additionalProperties && + typeof x.additionalProperties === "object" && + y.additionalProperties === true) || + (typeof x.additionalProperties === "object" && + typeof y.additionalProperties === "object" && + !covers(x.additionalProperties, y.additionalProperties)) + ) + return false; + return Object.entries(y.properties ?? {}).every(([key, b]) => { + const a: ILlmSchemaV3 | undefined = x.properties?.[key]; + if (a === undefined) return false; + else if ( + (x.required?.includes(key) ?? false) === true && + (y.required?.includes(key) ?? false) === false + ) + return false; + return covers(a, b); + }); + }; + + const flatSchema = (schema: ILlmSchemaV3): ILlmSchemaV3[] => + isOneOf(schema) ? schema.oneOf.flatMap(flatSchema) : [schema]; + /* ----------------------------------------------------------- TYPE CHECKERS ----------------------------------------------------------- */ diff --git a/test/features/llm/validate_llm_type_checker_cover_any.ts b/test/features/llm/validate_llm_type_checker_cover_any.ts new file mode 100644 index 0000000..d898f69 --- /dev/null +++ b/test/features/llm/validate_llm_type_checker_cover_any.ts @@ -0,0 +1,63 @@ +import { TestValidator } from "@nestia/e2e"; +import { ILlmSchema, OpenApi } from "@samchon/openapi"; +import { LlmSchemaComposer } from "@samchon/openapi/lib/composers/LlmSchemaComposer"; +import typia, { IJsonSchemaCollection } from "typia"; + +export const test_chatgpt_type_checker_cover_any = (): void => + validate_llm_type_checker_cover_any("chatgpt"); + +export const test_claude_type_checker_cover_any = (): void => + validate_llm_type_checker_cover_any("claude"); + +export const test_gemini_type_checker_cover_any = (): void => + validate_llm_type_checker_cover_any("gemini"); + +export const test_llama_type_checker_cover_any = (): void => + validate_llm_type_checker_cover_any("llama"); + +export const test_llm_v30_type_checker_cover_any = (): void => + validate_llm_type_checker_cover_any("3.0"); + +export const test_llm_v31_type_checker_cover_any = (): void => + validate_llm_type_checker_cover_any("3.1"); + +const validate_llm_type_checker_cover_any = ( + model: Model, +) => { + const collection: IJsonSchemaCollection = typia.json.schemas<[IBasic]>(); + const result = LlmSchemaComposer.parameters(model)({ + config: LlmSchemaComposer.defaultConfig(model) as any, + components: collection.components, + schema: collection.schemas[0] as OpenApi.IJsonSchema.IReference, + }); + if (result.success === false) + throw new Error(`Failed to compose ${model} parameters.`); + + const parameters = result.value; + const check = (x: ILlmSchema, y: ILlmSchema): boolean => + model === "3.0" || model === "gemini" + ? (LlmSchemaComposer.typeChecker(model).covers as any)(x, y) + : (LlmSchemaComposer.typeChecker(model).covers as any)({ + x, + y, + $defs: (parameters as any).$defs, + }); + TestValidator.equals("any covers (string | null)")(true)( + check( + parameters.properties.any as ILlmSchema, + parameters.properties.string_or_null as ILlmSchema, + ), + ); + TestValidator.equals("any covers (string | undefined)")(true)( + check( + parameters.properties.any as ILlmSchema, + parameters.properties.string_or_undefined as ILlmSchema, + ), + ); +}; + +interface IBasic { + any: any; + string_or_null: null | string; + string_or_undefined: string | undefined; +} diff --git a/test/features/llm/validate_llm_type_checker_cover_array.ts b/test/features/llm/validate_llm_type_checker_cover_array.ts new file mode 100644 index 0000000..b1a64bf --- /dev/null +++ b/test/features/llm/validate_llm_type_checker_cover_array.ts @@ -0,0 +1,211 @@ +import { TestValidator } from "@nestia/e2e"; +import { ILlmSchema, OpenApi } from "@samchon/openapi"; +import { LlmSchemaComposer } from "@samchon/openapi/lib/composers/LlmSchemaComposer"; +import typia, { IJsonSchemaCollection } from "typia"; + +export const test_chatgpt_type_checker_cover_array = (): void => + validate_llm_type_checker_cover_array("chatgpt"); + +export const test_claude_type_checker_cover_array = (): void => + validate_llm_type_checker_cover_array("claude"); + +export const test_gemini_type_checker_cover_array = (): void => + validate_llm_type_checker_cover_array("gemini"); + +export const test_llama_type_checker_cover_array = (): void => + validate_llm_type_checker_cover_array("llama"); + +export const test_llm_v30_type_checker_cover_array = (): void => + validate_llm_type_checker_cover_array("3.0"); + +export const test_llm_v31_type_checker_cover_array = (): void => + validate_llm_type_checker_cover_array("3.1"); + +const validate_llm_type_checker_cover_array = ( + model: Model, +) => { + const collection: IJsonSchemaCollection = + typia.json.schemas<[Plan2D, Plan3D, Box2D, Box3D]>(); + const components: OpenApi.IComponents = collection.components as any; + const plan2D: OpenApi.IJsonSchema = components.schemas!.Plan2D; + const plan3D: OpenApi.IJsonSchema = components.schemas!.Plan3D; + const box2D: OpenApi.IJsonSchema = components.schemas!.Box2D; + const box3D: OpenApi.IJsonSchema = components.schemas!.Box3D; + + const $defs = {}; + const check = (x: OpenApi.IJsonSchema, y: OpenApi.IJsonSchema): boolean => { + const [a, b] = [x, y].map((schema) => { + const result = LlmSchemaComposer.schema(model)({ + config: LlmSchemaComposer.defaultConfig(model) as any, + components: collection.components, + schema: schema, + $defs, + }); + if (result.success === false) + throw new Error(`Failed to compose ${model} schema.`); + return result.value; + }); + return model === "3.0" || model === "gemini" + ? (LlmSchemaComposer.typeChecker(model).covers as any)(a, b) + : (LlmSchemaComposer.typeChecker(model).covers as any)({ + x: a, + y: b, + $defs, + }); + }; + + TestValidator.equals(model + " Plan3D[] covers Plan2D[]")(true)( + check({ type: "array", items: plan3D }, { type: "array", items: plan2D }), + ); + TestValidator.equals(model + " Box3D[] covers Box2D[]")(true)( + check({ type: "array", items: box3D }, { type: "array", items: box2D }), + ); + if (model !== "gemini") + TestValidator.equals( + model + " Array covers Array", + )(true)( + check( + { + type: "array", + items: { + oneOf: [plan3D, box3D], + }, + }, + { + type: "array", + items: { + oneOf: [plan2D, box2D], + }, + }, + ), + ); + if (model !== "gemini") + TestValidator.equals(model + " (Plan3D|Box3D)[] covers (Plan2D|Box2D)[]")( + true, + )( + check( + { + oneOf: [ + { type: "array", items: plan3D }, + { type: "array", items: box3D }, + ], + }, + { + oneOf: [ + { type: "array", items: plan2D }, + { type: "array", items: box2D }, + ], + }, + ), + ); + + TestValidator.equals(model + " Plan2D[] can't cover Plan3D[]")(false)( + check({ type: "array", items: plan2D }, { type: "array", items: plan3D }), + ); + TestValidator.equals(model + " Box2D[] can't cover Box3D[]")(false)( + check({ type: "array", items: box2D }, { type: "array", items: box3D }), + ); + if (model !== "gemini") + if (model !== "gemini") + TestValidator.equals( + "Array can't cover Array", + )(false)( + check( + { + type: "array", + items: { + oneOf: [plan2D, box2D], + }, + }, + { + type: "array", + items: { + oneOf: [plan3D, box3D], + }, + }, + ), + ); + if (model !== "gemini") + TestValidator.equals( + model + " (Plan2D[]|Box2D[]) can't cover (Plan3D[]|Box3D[])", + )(false)( + check( + { + oneOf: [ + { type: "array", items: plan2D }, + { type: "array", items: box2D }, + ], + }, + { + oneOf: [ + { type: "array", items: plan3D }, + { type: "array", items: box3D }, + ], + }, + ), + ); + if (model !== "gemini") + TestValidator.equals(model + " Plan3D[] can't cover (Plan2D|Box2D)[]")( + false, + )( + check( + { type: "array", items: plan3D }, + { + oneOf: [ + { type: "array", items: plan2D }, + { type: "array", items: box2D }, + ], + }, + ), + ); + if (model !== "gemini") + TestValidator.equals(model + " Box3D[] can't cover Array")( + false, + )( + check( + { type: "array", items: box3D }, + { + type: "array", + items: { + oneOf: [plan2D, box2D], + }, + }, + ), + ); +}; + +type Plan2D = { + center: Point2D; + size: Point2D; + geometries: Geometry2D[]; +}; +type Plan3D = { + center: Point3D; + size: Point3D; + geometries: Geometry3D[]; +}; +type Geometry3D = { + position: Point3D; + scale: Point3D; +}; +type Geometry2D = { + position: Point2D; + scale: Point2D; +}; +type Point2D = { + x: number; + y: number; +}; +type Point3D = { + x: number; + y: number; + z: number; +}; +type Box2D = { + size: Point2D; + nested: Point2D[]; +}; +type Box3D = { + size: Point3D; + nested: Point3D[]; +};