From 28cd0ccb37267cd6f224f8fadcacf8e20ceed6fa Mon Sep 17 00:00:00 2001 From: Alex McKinney Date: Mon, 12 Aug 2024 16:21:12 -0400 Subject: [PATCH 1/5] (feature, cli): Upload source files to S3 --- packages/cli/docs-preview/package.json | 2 +- packages/cli/docs-resolver/package.json | 2 +- .../src/SourceUploader.ts | 141 ++++++++++++++++++ .../src/runRemoteGenerationForGenerator.ts | 39 ++++- packages/cli/register/package.json | 2 +- pnpm-lock.yaml | 27 ++-- 6 files changed, 191 insertions(+), 22 deletions(-) create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts diff --git a/packages/cli/docs-preview/package.json b/packages/cli/docs-preview/package.json index f4bb7c56310..4f93f479e7f 100644 --- a/packages/cli/docs-preview/package.json +++ b/packages/cli/docs-preview/package.json @@ -28,7 +28,7 @@ }, "dependencies": { "@fern-api/docs-resolver": "workspace:*", - "@fern-api/fdr-sdk": "0.98.20-33071ab6e", + "@fern-api/fdr-sdk": "0.98.20-86eba53a1", "@fern-api/fs-utils": "workspace:*", "@fern-api/ir-sdk": "workspace:*", "@fern-api/logger": "workspace:*", diff --git a/packages/cli/docs-resolver/package.json b/packages/cli/docs-resolver/package.json index f849b493986..d4f5b407706 100644 --- a/packages/cli/docs-resolver/package.json +++ b/packages/cli/docs-resolver/package.json @@ -30,7 +30,7 @@ "@fern-api/configuration": "workspace:*", "@fern-api/core-utils": "workspace:*", "@fern-api/docs-markdown-utils": "workspace:*", - "@fern-api/fdr-sdk": "0.98.20-33071ab6e", + "@fern-api/fdr-sdk": "0.98.20-86eba53a1", "@fern-api/fs-utils": "workspace:*", "@fern-api/ir-generator": "workspace:*", "@fern-api/ir-sdk": "workspace:*", diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts new file mode 100644 index 00000000000..b878fd8e728 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts @@ -0,0 +1,141 @@ +import { AbsoluteFilePath, join, RelativeFilePath } from "@fern-api/fs-utils"; +import { ApiDefinitionSource, SourceConfig } from "@fern-api/ir-sdk"; +import { loggingExeca } from "@fern-api/logging-execa"; +import { InteractiveTaskContext } from "@fern-api/task-context"; +import { FernWorkspace, IdentifiableSource } from "@fern-api/workspace-loader"; +import { FernRegistry as FdrAPI } from "@fern-fern/fdr-cjs-sdk"; +import { readFile, unlink } from "fs/promises"; +import tmp from "tmp-promise"; + +const PROTOBUF_ZIP_FILENAME = "proto.zip"; + +export type SourceType = "asyncapi" | "openapi" | "protobuf"; + +export class SourceUploader { + private context: InteractiveTaskContext; + private sources: Record; + private sourceTypes: Set; + + constructor(context: InteractiveTaskContext, workspace: FernWorkspace) { + this.context = context; + this.sources = Object.fromEntries(workspace.getSources().map((source) => [source.id, source])); + this.sourceTypes = new Set(Object.values(this.sources).map((source) => source.type)); + } + + public getSourceTypes(): Set { + return this.sourceTypes; + } + + public getFdrApiDefinitionSources(): + | Record + | undefined { + if (Object.keys(this.sources).length === 0) { + return undefined; + } + return Object.fromEntries( + Object.entries(this.sources).map(([id, source]) => [ + id, + { + type: source.type === "protobuf" ? "proto" : source.type + } + ]) + ); + } + + public convertFdrSourceUploadsToSourceConfig( + sources: Record + ): SourceConfig { + const apiDefinitionSources: ApiDefinitionSource[] = []; + for (const [id, sourceUpload] of Object.entries(sources)) { + const identifiableSource = this.getSourceOrThrow(id); + switch (identifiableSource.type) { + case "protobuf": + apiDefinitionSources.push( + ApiDefinitionSource.proto({ + id, + protoRootUrl: sourceUpload.downloadUrl + }) + ); + continue; + case "openapi": + apiDefinitionSources.push(ApiDefinitionSource.openapi()); + continue; + case "asyncapi": + // AsyncAPI sources aren't modeled in the IR yet. + continue; + } + } + return { + sources: apiDefinitionSources + }; + } + + public async uploadSources( + sources: Record + ): Promise { + for (const [id, source] of Object.entries(sources)) { + const identifiableSource = this.getSourceOrThrow(id); + await this.uploadSource(identifiableSource, source.uploadUrl); + } + } + + private async uploadSource(source: IdentifiableSource, uploadURL: string): Promise { + const uploadCommand = await this.getUploadCommand(source); + const fileData = await readFile(uploadCommand.absoluteFilePath); + const response = await fetch(uploadURL, { + method: "PUT", + body: fileData, + headers: { + "Content-Type": "application/octet-stream" + } + }); + await uploadCommand.cleanup(); + if (!response.ok) { + this.context.failAndThrow( + `Failed to upload source file: ${source.absoluteFilePath}. Status: ${response.status}, ${response.statusText}` + ); + } + } + + private async getUploadCommand( + source: IdentifiableSource + ): Promise<{ absoluteFilePath: AbsoluteFilePath; cleanup: () => Promise }> { + if (source.type === "protobuf") { + const absoluteFilePath = await this.zipSource(source.absoluteFilePath); + return { + absoluteFilePath, + cleanup: async () => { + this.context.logger.debug(`Removing ${absoluteFilePath}`); + await unlink(absoluteFilePath); + } + }; + } + return { + absoluteFilePath: source.absoluteFilePath, + cleanup: async () => {} + }; + } + + private async zipSource(absolutePathToSource: AbsoluteFilePath): Promise { + const tmpDir = await tmp.dir(); + const destination = join(AbsoluteFilePath.of(tmpDir.path), RelativeFilePath.of(PROTOBUF_ZIP_FILENAME)); + + this.context.logger.debug(`Zipping source ${absolutePathToSource} into ${destination}`); + await loggingExeca(this.context.logger, "zip", ["-r", destination, "."], { + cwd: absolutePathToSource, + doNotPipeOutput: true + }); + + return destination; + } + + private getSourceOrThrow(id: string): IdentifiableSource { + const source = this.sources[id]; + if (source == null) { + this.context.failAndThrow( + `Internal error; server responded with source id "${id}" which does not exist in the workspace.` + ); + } + return source; + } +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts index 15cd9ab70fa..c7a0c6ba642 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts @@ -3,6 +3,7 @@ import { Audiences, fernConfigJson, generatorsYml } from "@fern-api/configuratio import { createFdrService } from "@fern-api/core"; import { AbsoluteFilePath } from "@fern-api/fs-utils"; import { generateIntermediateRepresentation } from "@fern-api/ir-generator"; +import { SourceConfig } from "@fern-api/ir-sdk"; import { convertIrToFdrApi } from "@fern-api/register"; import { InteractiveTaskContext } from "@fern-api/task-context"; import { FernWorkspace } from "@fern-api/workspace-loader"; @@ -11,6 +12,7 @@ import { FernFiddle } from "@fern-fern/fiddle-sdk"; import { createAndStartJob } from "./createAndStartJob"; import { pollJobAndReportStatus } from "./pollJobAndReportStatus"; import { RemoteTaskHandler } from "./RemoteTaskHandler"; +import { SourceUploader } from "./SourceUploader"; export async function runRemoteGenerationForGenerator({ projectConfig, @@ -57,15 +59,29 @@ export async function runRemoteGenerationForGenerator({ version: version ?? (await computeSemanticVersion({ fdr, packageName, generatorInvocation })) }); + const sourceUploader = new SourceUploader(interactiveTaskContext, workspace); + const sources = sourceUploader.getFdrApiDefinitionSources(); + const apiDefinition = convertIrToFdrApi({ ir, snippetsConfig: {} }); const response = await fdr.api.v1.register.registerApiDefinition({ orgId: organization, apiId: ir.apiName.originalName, - definition: apiDefinition + definition: apiDefinition, + sources }); + let fdrApiDefinitionId; + let sourceUploads; if (response.ok) { fdrApiDefinitionId = response.body.apiDefinitionId; + sourceUploads = response.body.sources; + } + + if (sourceUploads != null) { + const sourceConfig = await uploadSources({ context: interactiveTaskContext, sourceUploader, sourceUploads }); + + interactiveTaskContext.logger.debug(`Setting IR source configuration ...`); + ir.sourceConfig = sourceConfig; } const job = await createAndStartJob({ @@ -161,3 +177,24 @@ async function computeSemanticVersion({ } return response.body.version; } + +async function uploadSources({ + context, + sourceUploader, + sourceUploads +}: { + context: InteractiveTaskContext; + sourceUploader: SourceUploader; + sourceUploads: Record; +}): Promise { + if (sourceUploads == null && sourceUploader.getSourceTypes().has("protobuf")) { + // We only fail hard if we need to upload Protobuf source files. Unlike OpenAPI, these + // files are required for successful code generation. + context.failAndThrow("Did not successfully upload Protobuf source files."); + } + + context.logger.debug(`Uploading source files ...`); + await sourceUploader.uploadSources(sourceUploads); + + return sourceUploader.convertFdrSourceUploadsToSourceConfig(sourceUploads); +} diff --git a/packages/cli/register/package.json b/packages/cli/register/package.json index efc9983951e..89c72f16d00 100644 --- a/packages/cli/register/package.json +++ b/packages/cli/register/package.json @@ -31,7 +31,7 @@ "@fern-api/configuration": "workspace:*", "@fern-api/core": "workspace:*", "@fern-api/core-utils": "workspace:*", - "@fern-api/fdr-sdk": "0.98.20-33071ab6e", + "@fern-api/fdr-sdk": "0.98.20-86eba53a1", "@fern-api/ir-generator": "workspace:*", "@fern-api/ir-sdk": "workspace:*", "@fern-api/task-context": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 87862ad2822..7d5064b135c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3179,15 +3179,6 @@ importers: packages/cli/cli/dist/dev: {} - packages/cli/cli/dist/local: - devDependencies: - globals: - specifier: link:@types/vitest/globals - version: link:@types/vitest/globals - vitest: - specifier: ^2.0.5 - version: 2.0.5(@types/node@18.7.18)(jsdom@20.0.3)(sass@1.72.0)(terser@5.31.5) - packages/cli/cli/dist/prod: {} packages/cli/configuration: @@ -3327,8 +3318,8 @@ importers: specifier: workspace:* version: link:../docs-resolver '@fern-api/fdr-sdk': - specifier: 0.98.20-33071ab6e - version: 0.98.20-33071ab6e(typescript@4.6.4) + specifier: 0.98.20-86eba53a1 + version: 0.98.20-86eba53a1(typescript@4.6.4) '@fern-api/fs-utils': specifier: workspace:* version: link:../../commons/fs-utils @@ -3427,8 +3418,8 @@ importers: specifier: workspace:* version: link:../docs-markdown-utils '@fern-api/fdr-sdk': - specifier: 0.98.20-33071ab6e - version: 0.98.20-33071ab6e(typescript@4.6.4) + specifier: 0.98.20-86eba53a1 + version: 0.98.20-86eba53a1(typescript@4.6.4) '@fern-api/fs-utils': specifier: workspace:* version: link:../../commons/fs-utils @@ -4633,8 +4624,8 @@ importers: specifier: workspace:* version: link:../../commons/core-utils '@fern-api/fdr-sdk': - specifier: 0.98.20-33071ab6e - version: 0.98.20-33071ab6e(typescript@4.6.4) + specifier: 0.98.20-86eba53a1 + version: 0.98.20-86eba53a1(typescript@4.6.4) '@fern-api/ir-generator': specifier: workspace:* version: link:../generation/ir-generator @@ -6689,8 +6680,8 @@ packages: '@exodus/schemasafe@1.0.0': resolution: {integrity: sha512-2cyupPIZI69HQxEAPllLXBjQp4njDKkOjYRCYxvMZe3/LY9pp9fBM3Tb1wiFAdP6Emo4v3OEbCLGj6u73Q5KLw==} - '@fern-api/fdr-sdk@0.98.20-33071ab6e': - resolution: {integrity: sha512-dvyh7sDjWw1Zs8LRlLVxm2SKChH5ZY0wbybe4c2+I64i7Voy3/YQ9J03nx9IrMtsrbifiTDDtj6IxSJycsUeVA==} + '@fern-api/fdr-sdk@0.98.20-86eba53a1': + resolution: {integrity: sha512-gQsTxrmG69hE/W68e3RpS0Jiq+hD1OPwwEgPi7EjFHcUy0Ru9WlSDOGJ0Uv1WsS84AHAz84OrUs+KEQwUwDeyA==} '@fern-api/venus-api-sdk@0.0.38': resolution: {integrity: sha512-1JjuctZwyPu4jN51bBqjIy+uCBPGa/ROcDfqN0UnGeVEW7NyztkGh/qOFwnN6K07VMMql1AIF4Zov+MVqTPwmw==} @@ -13712,7 +13703,7 @@ snapshots: '@exodus/schemasafe@1.0.0': {} - '@fern-api/fdr-sdk@0.98.20-33071ab6e(typescript@4.6.4)': + '@fern-api/fdr-sdk@0.98.20-86eba53a1(typescript@4.6.4)': dependencies: dayjs: 1.11.11 fast-deep-equal: 3.1.3 From 131116c1b546fad36507866ef651e6e99d271a62 Mon Sep 17 00:00:00 2001 From: Alex McKinney Date: Mon, 12 Aug 2024 17:04:12 -0400 Subject: [PATCH 2/5] Simplify with a variety of refactors --- .../src/SourceUploader.ts | 87 ++++++++----------- .../src/runRemoteGenerationForGenerator.ts | 26 ++++-- 2 files changed, 51 insertions(+), 62 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts index b878fd8e728..7d6ede97fa9 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts @@ -2,7 +2,7 @@ import { AbsoluteFilePath, join, RelativeFilePath } from "@fern-api/fs-utils"; import { ApiDefinitionSource, SourceConfig } from "@fern-api/ir-sdk"; import { loggingExeca } from "@fern-api/logging-execa"; import { InteractiveTaskContext } from "@fern-api/task-context"; -import { FernWorkspace, IdentifiableSource } from "@fern-api/workspace-loader"; +import { IdentifiableSource } from "@fern-api/workspace-loader"; import { FernRegistry as FdrAPI } from "@fern-fern/fdr-cjs-sdk"; import { readFile, unlink } from "fs/promises"; import tmp from "tmp-promise"; @@ -12,71 +12,24 @@ const PROTOBUF_ZIP_FILENAME = "proto.zip"; export type SourceType = "asyncapi" | "openapi" | "protobuf"; export class SourceUploader { + public sourceTypes: Set; private context: InteractiveTaskContext; private sources: Record; - private sourceTypes: Set; - constructor(context: InteractiveTaskContext, workspace: FernWorkspace) { + constructor(context: InteractiveTaskContext, sources: IdentifiableSource[]) { this.context = context; - this.sources = Object.fromEntries(workspace.getSources().map((source) => [source.id, source])); + this.sources = Object.fromEntries(sources.map((source) => [source.id, source])); this.sourceTypes = new Set(Object.values(this.sources).map((source) => source.type)); } - public getSourceTypes(): Set { - return this.sourceTypes; - } - - public getFdrApiDefinitionSources(): - | Record - | undefined { - if (Object.keys(this.sources).length === 0) { - return undefined; - } - return Object.fromEntries( - Object.entries(this.sources).map(([id, source]) => [ - id, - { - type: source.type === "protobuf" ? "proto" : source.type - } - ]) - ); - } - - public convertFdrSourceUploadsToSourceConfig( - sources: Record - ): SourceConfig { - const apiDefinitionSources: ApiDefinitionSource[] = []; - for (const [id, sourceUpload] of Object.entries(sources)) { - const identifiableSource = this.getSourceOrThrow(id); - switch (identifiableSource.type) { - case "protobuf": - apiDefinitionSources.push( - ApiDefinitionSource.proto({ - id, - protoRootUrl: sourceUpload.downloadUrl - }) - ); - continue; - case "openapi": - apiDefinitionSources.push(ApiDefinitionSource.openapi()); - continue; - case "asyncapi": - // AsyncAPI sources aren't modeled in the IR yet. - continue; - } - } - return { - sources: apiDefinitionSources - }; - } - public async uploadSources( sources: Record - ): Promise { + ): Promise { for (const [id, source] of Object.entries(sources)) { const identifiableSource = this.getSourceOrThrow(id); await this.uploadSource(identifiableSource, source.uploadUrl); } + return this.convertFdrSourceUploadsToSourceConfig(sources); } private async uploadSource(source: IdentifiableSource, uploadURL: string): Promise { @@ -129,6 +82,34 @@ export class SourceUploader { return destination; } + private convertFdrSourceUploadsToSourceConfig( + sources: Record + ): SourceConfig { + const apiDefinitionSources: ApiDefinitionSource[] = []; + for (const [id, sourceUpload] of Object.entries(sources)) { + const identifiableSource = this.getSourceOrThrow(id); + switch (identifiableSource.type) { + case "protobuf": + apiDefinitionSources.push( + ApiDefinitionSource.proto({ + id, + protoRootUrl: sourceUpload.downloadUrl + }) + ); + continue; + case "openapi": + apiDefinitionSources.push(ApiDefinitionSource.openapi()); + continue; + case "asyncapi": + // AsyncAPI sources aren't modeled in the IR yet. + continue; + } + } + return { + sources: apiDefinitionSources + }; + } + private getSourceOrThrow(id: string): IdentifiableSource { const source = this.sources[id]; if (source == null) { diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts index c7a0c6ba642..7b93aa0563f 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts @@ -6,7 +6,7 @@ import { generateIntermediateRepresentation } from "@fern-api/ir-generator"; import { SourceConfig } from "@fern-api/ir-sdk"; import { convertIrToFdrApi } from "@fern-api/register"; import { InteractiveTaskContext } from "@fern-api/task-context"; -import { FernWorkspace } from "@fern-api/workspace-loader"; +import { FernWorkspace, IdentifiableSource } from "@fern-api/workspace-loader"; import { FernRegistry as FdrAPI, FernRegistryClient as FdrClient } from "@fern-fern/fdr-cjs-sdk"; import { FernFiddle } from "@fern-fern/fiddle-sdk"; import { createAndStartJob } from "./createAndStartJob"; @@ -59,15 +59,13 @@ export async function runRemoteGenerationForGenerator({ version: version ?? (await computeSemanticVersion({ fdr, packageName, generatorInvocation })) }); - const sourceUploader = new SourceUploader(interactiveTaskContext, workspace); - const sources = sourceUploader.getFdrApiDefinitionSources(); - + const sources = workspace.getSources(); const apiDefinition = convertIrToFdrApi({ ir, snippetsConfig: {} }); const response = await fdr.api.v1.register.registerApiDefinition({ orgId: organization, apiId: ir.apiName.originalName, definition: apiDefinition, - sources + sources: sources.length > 0 ? convertToFdrApiDefinitionSources(sources) : undefined }); let fdrApiDefinitionId; @@ -78,6 +76,7 @@ export async function runRemoteGenerationForGenerator({ } if (sourceUploads != null) { + const sourceUploader = new SourceUploader(interactiveTaskContext, sources); const sourceConfig = await uploadSources({ context: interactiveTaskContext, sourceUploader, sourceUploads }); interactiveTaskContext.logger.debug(`Setting IR source configuration ...`); @@ -187,14 +186,23 @@ async function uploadSources({ sourceUploader: SourceUploader; sourceUploads: Record; }): Promise { - if (sourceUploads == null && sourceUploader.getSourceTypes().has("protobuf")) { + if (sourceUploader.sourceTypes.has("protobuf")) { // We only fail hard if we need to upload Protobuf source files. Unlike OpenAPI, these // files are required for successful code generation. context.failAndThrow("Did not successfully upload Protobuf source files."); } context.logger.debug(`Uploading source files ...`); - await sourceUploader.uploadSources(sourceUploads); - - return sourceUploader.convertFdrSourceUploadsToSourceConfig(sourceUploads); + return await sourceUploader.uploadSources(sourceUploads); } + +function convertToFdrApiDefinitionSources(sources: IdentifiableSource[]): Record { + return Object.fromEntries( + Object.values(sources).map((source) => [ + source.id, + { + type: source.type === "protobuf" ? "proto" : source.type + } + ]) + ); +} \ No newline at end of file From 2ca0ad4c26a5f83c882dda147b2dfb773eea7371 Mon Sep 17 00:00:00 2001 From: Alex McKinney Date: Mon, 12 Aug 2024 17:21:08 -0400 Subject: [PATCH 3/5] Fix lint --- .../remote-workspace-runner/src/SourceUploader.ts | 4 +++- .../src/runRemoteGenerationForGenerator.ts | 15 +++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts index 7d6ede97fa9..f5bf9c1f3d5 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts @@ -65,7 +65,9 @@ export class SourceUploader { } return { absoluteFilePath: source.absoluteFilePath, - cleanup: async () => {} + cleanup: async () => { + // Do nothing. + } }; } diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts index 7b93aa0563f..ce03fcac9a9 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts @@ -75,11 +75,18 @@ export async function runRemoteGenerationForGenerator({ sourceUploads = response.body.sources; } + const sourceUploader = new SourceUploader(interactiveTaskContext, sources); + if (sourceUploads == null && sourceUploader.sourceTypes.has("protobuf")) { + // We only fail hard if we need to upload Protobuf source files. Unlike OpenAPI, these + // files are required for successful code generation. + interactiveTaskContext.failAndThrow("Did not successfully upload Protobuf source files."); + } + if (sourceUploads != null) { - const sourceUploader = new SourceUploader(interactiveTaskContext, sources); - const sourceConfig = await uploadSources({ context: interactiveTaskContext, sourceUploader, sourceUploads }); + interactiveTaskContext.logger.debug("Uploading source files ..."); + const sourceConfig = await sourceUploader.uploadSources(sourceUploads); - interactiveTaskContext.logger.debug(`Setting IR source configuration ...`); + interactiveTaskContext.logger.debug("Setting IR source configuration ..."); ir.sourceConfig = sourceConfig; } @@ -186,7 +193,7 @@ async function uploadSources({ sourceUploader: SourceUploader; sourceUploads: Record; }): Promise { - if (sourceUploader.sourceTypes.has("protobuf")) { + if (Object.keys(sourceUploads).length == 0 && sourceUploader.sourceTypes.has("protobuf")) { // We only fail hard if we need to upload Protobuf source files. Unlike OpenAPI, these // files are required for successful code generation. context.failAndThrow("Did not successfully upload Protobuf source files."); From 2f89afcefa80dd4d9551d42a78e142af8910e41b Mon Sep 17 00:00:00 2001 From: Alex McKinney Date: Mon, 12 Aug 2024 17:32:40 -0400 Subject: [PATCH 4/5] Fix depcheck and lint --- .../remote-workspace-runner/package.json | 1 + .../src/runRemoteGenerationForGenerator.ts | 20 ------------------- pnpm-lock.yaml | 7 +++---- 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/package.json b/packages/cli/generation/remote-generation/remote-workspace-runner/package.json index 8200c465271..f59565ee06c 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/package.json +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/package.json @@ -32,6 +32,7 @@ "@fern-api/core": "workspace:*", "@fern-api/core-utils": "workspace:*", "@fern-api/docs-resolver": "workspace:*", + "@fern-api/logging-execa": "workspace:*", "@fern-fern/fdr-cjs-sdk": "0.1.0", "@fern-api/fs-utils": "workspace:*", "@fern-api/ir-generator": "workspace:*", diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts index ce03fcac9a9..b34dbc6edc2 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts @@ -3,7 +3,6 @@ import { Audiences, fernConfigJson, generatorsYml } from "@fern-api/configuratio import { createFdrService } from "@fern-api/core"; import { AbsoluteFilePath } from "@fern-api/fs-utils"; import { generateIntermediateRepresentation } from "@fern-api/ir-generator"; -import { SourceConfig } from "@fern-api/ir-sdk"; import { convertIrToFdrApi } from "@fern-api/register"; import { InteractiveTaskContext } from "@fern-api/task-context"; import { FernWorkspace, IdentifiableSource } from "@fern-api/workspace-loader"; @@ -184,25 +183,6 @@ async function computeSemanticVersion({ return response.body.version; } -async function uploadSources({ - context, - sourceUploader, - sourceUploads -}: { - context: InteractiveTaskContext; - sourceUploader: SourceUploader; - sourceUploads: Record; -}): Promise { - if (Object.keys(sourceUploads).length == 0 && sourceUploader.sourceTypes.has("protobuf")) { - // We only fail hard if we need to upload Protobuf source files. Unlike OpenAPI, these - // files are required for successful code generation. - context.failAndThrow("Did not successfully upload Protobuf source files."); - } - - context.logger.debug(`Uploading source files ...`); - return await sourceUploader.uploadSources(sourceUploads); -} - function convertToFdrApiDefinitionSources(sources: IdentifiableSource[]): Record { return Object.fromEntries( Object.values(sources).map((source) => [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7d5064b135c..de4b4de4394 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3177,10 +3177,6 @@ importers: specifier: 4.6.4 version: 4.6.4 - packages/cli/cli/dist/dev: {} - - packages/cli/cli/dist/prod: {} - packages/cli/configuration: dependencies: '@fern-api/core-utils': @@ -4008,6 +4004,9 @@ importers: '@fern-api/logger': specifier: workspace:* version: link:../../../logger + '@fern-api/logging-execa': + specifier: workspace:* + version: link:../../../../commons/logging-execa '@fern-api/register': specifier: workspace:* version: link:../../../register From c46700cca48c037c5f6198bc7869a4f6aefe754d Mon Sep 17 00:00:00 2001 From: Alex McKinney Date: Mon, 12 Aug 2024 17:36:21 -0400 Subject: [PATCH 5/5] Format files --- .../src/runRemoteGenerationForGenerator.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts index b34dbc6edc2..34de2691d73 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts @@ -183,7 +183,9 @@ async function computeSemanticVersion({ return response.body.version; } -function convertToFdrApiDefinitionSources(sources: IdentifiableSource[]): Record { +function convertToFdrApiDefinitionSources( + sources: IdentifiableSource[] +): Record { return Object.fromEntries( Object.values(sources).map((source) => [ source.id, @@ -192,4 +194,4 @@ function convertToFdrApiDefinitionSources(sources: IdentifiableSource[]): Record } ]) ); -} \ No newline at end of file +}