(feature, cli): Upload source files to S3

packages/cli/docs-preview/package.json

@@ -28,7 +28,7 @@
   },
   "dependencies": {
     "@fern-api/docs-resolver": "workspace:*",
-    "@fern-api/fdr-sdk": "0.98.20-33071ab6e",
+    "@fern-api/fdr-sdk": "0.98.20-86eba53a1",
     "@fern-api/fs-utils": "workspace:*",
     "@fern-api/ir-sdk": "workspace:*",
     "@fern-api/logger": "workspace:*",

packages/cli/docs-resolver/package.json

@@ -30,7 +30,7 @@
     "@fern-api/configuration": "workspace:*",
     "@fern-api/core-utils": "workspace:*",
     "@fern-api/docs-markdown-utils": "workspace:*",
-    "@fern-api/fdr-sdk": "0.98.20-33071ab6e",
+    "@fern-api/fdr-sdk": "0.98.20-86eba53a1",
     "@fern-api/fs-utils": "workspace:*",
     "@fern-api/ir-generator": "workspace:*",
     "@fern-api/ir-sdk": "workspace:*",

packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts

@@ -0,0 +1,124 @@
+import { AbsoluteFilePath, join, RelativeFilePath } from "@fern-api/fs-utils";
+import { ApiDefinitionSource, SourceConfig } from "@fern-api/ir-sdk";
+import { loggingExeca } from "@fern-api/logging-execa";
+import { InteractiveTaskContext } from "@fern-api/task-context";
+import { IdentifiableSource } from "@fern-api/workspace-loader";
+import { FernRegistry as FdrAPI } from "@fern-fern/fdr-cjs-sdk";
+import { readFile, unlink } from "fs/promises";
+import tmp from "tmp-promise";
+
+const PROTOBUF_ZIP_FILENAME = "proto.zip";
+
+export type SourceType = "asyncapi" | "openapi" | "protobuf";
+
+export class SourceUploader {
+    public sourceTypes: Set<SourceType>;
+    private context: InteractiveTaskContext;
+    private sources: Record<string, IdentifiableSource>;
+
+    constructor(context: InteractiveTaskContext, sources: IdentifiableSource[]) {
+        this.context = context;
+        this.sources = Object.fromEntries(sources.map((source) => [source.id, source]));
+        this.sourceTypes = new Set<SourceType>(Object.values(this.sources).map((source) => source.type));
+    }
+
+    public async uploadSources(
+        sources: Record<FdrAPI.api.v1.register.SourceId, FdrAPI.api.v1.register.SourceUpload>
+    ): Promise<SourceConfig> {
+        for (const [id, source] of Object.entries(sources)) {
+            const identifiableSource = this.getSourceOrThrow(id);
+            await this.uploadSource(identifiableSource, source.uploadUrl);
+        }
+        return this.convertFdrSourceUploadsToSourceConfig(sources);
+    }
+
+    private async uploadSource(source: IdentifiableSource, uploadURL: string): Promise<void> {
+        const uploadCommand = await this.getUploadCommand(source);
+        const fileData = await readFile(uploadCommand.absoluteFilePath);
+        const response = await fetch(uploadURL, {
+            method: "PUT",
+            body: fileData,
+            headers: {
+                "Content-Type": "application/octet-stream"
+            }
+        });
+        await uploadCommand.cleanup();
+        if (!response.ok) {
+            this.context.failAndThrow(
+                `Failed to upload source file: ${source.absoluteFilePath}. Status: ${response.status}, ${response.statusText}`
+            );
+        }
+    }
+
+    private async getUploadCommand(
+        source: IdentifiableSource
+    ): Promise<{ absoluteFilePath: AbsoluteFilePath; cleanup: () => Promise<void> }> {
+        if (source.type === "protobuf") {
+            const absoluteFilePath = await this.zipSource(source.absoluteFilePath);
+            return {
+                absoluteFilePath,
+                cleanup: async () => {
+                    this.context.logger.debug(`Removing ${absoluteFilePath}`);
+                    await unlink(absoluteFilePath);
+                }
+            };
+        }
+        return {
+            absoluteFilePath: source.absoluteFilePath,
+            cleanup: async () => {
+                // Do nothing.
+            }
+        };
+    }
+
+    private async zipSource(absolutePathToSource: AbsoluteFilePath): Promise<AbsoluteFilePath> {
+        const tmpDir = await tmp.dir();
+        const destination = join(AbsoluteFilePath.of(tmpDir.path), RelativeFilePath.of(PROTOBUF_ZIP_FILENAME));
+
+        this.context.logger.debug(`Zipping source ${absolutePathToSource} into ${destination}`);
+        await loggingExeca(this.context.logger, "zip", ["-r", destination, "."], {
+            cwd: absolutePathToSource,
+            doNotPipeOutput: true
+        });
+
+        return destination;
+    }
+
+    private convertFdrSourceUploadsToSourceConfig(
+        sources: Record<FdrAPI.api.v1.register.SourceId, FdrAPI.api.v1.register.SourceUpload>
+    ): SourceConfig {
+        const apiDefinitionSources: ApiDefinitionSource[] = [];
+        for (const [id, sourceUpload] of Object.entries(sources)) {
+            const identifiableSource = this.getSourceOrThrow(id);
+            switch (identifiableSource.type) {
+                case "protobuf":
+                    apiDefinitionSources.push(
+                        ApiDefinitionSource.proto({
+                            id,
+                            protoRootUrl: sourceUpload.downloadUrl
+                        })
+                    );
+                    continue;
+                case "openapi":
+                    apiDefinitionSources.push(ApiDefinitionSource.openapi());
+                    continue;
+                case "asyncapi":
+                    // AsyncAPI sources aren't modeled in the IR yet.
+                    continue;
+            }
+        }
+        return {
+            sources: apiDefinitionSources
+        };
+    }
+
+    private getSourceOrThrow(id: string): IdentifiableSource {
+        const source = this.sources[id];
+        if (source == null) {
+            this.context.failAndThrow(
+                `Internal error; server responded with source id "${id}" which does not exist in the workspace.`
+            );
+        }
+        return source;
+    }
+}

packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts

@@ -3,14 +3,16 @@
 import { createFdrService } from "@fern-api/core";
 import { AbsoluteFilePath } from "@fern-api/fs-utils";
 import { generateIntermediateRepresentation } from "@fern-api/ir-generator";
+import { SourceConfig } from "@fern-api/ir-sdk";
 import { convertIrToFdrApi } from "@fern-api/register";
 import { InteractiveTaskContext } from "@fern-api/task-context";
-import { FernWorkspace } from "@fern-api/workspace-loader";
+import { FernWorkspace, IdentifiableSource } from "@fern-api/workspace-loader";
 import { FernRegistry as FdrAPI, FernRegistryClient as FdrClient } from "@fern-fern/fdr-cjs-sdk";
 import { FernFiddle } from "@fern-fern/fiddle-sdk";
 import { createAndStartJob } from "./createAndStartJob";
 import { pollJobAndReportStatus } from "./pollJobAndReportStatus";
 import { RemoteTaskHandler } from "./RemoteTaskHandler";
+import { SourceUploader } from "./SourceUploader";
 
 export async function runRemoteGenerationForGenerator({
     projectConfig,
@@ -57,15 +59,35 @@
         version: version ?? (await computeSemanticVersion({ fdr, packageName, generatorInvocation }))
     });
 
+    const sources = workspace.getSources();
     const apiDefinition = convertIrToFdrApi({ ir, snippetsConfig: {} });
     const response = await fdr.api.v1.register.registerApiDefinition({
         orgId: organization,
         apiId: ir.apiName.originalName,
-        definition: apiDefinition
+        definition: apiDefinition,
+        sources: sources.length > 0 ? convertToFdrApiDefinitionSources(sources) : undefined
     });
+
     let fdrApiDefinitionId;
+    let sourceUploads;
     if (response.ok) {
         fdrApiDefinitionId = response.body.apiDefinitionId;
+        sourceUploads = response.body.sources;
+    }
+
+    const sourceUploader = new SourceUploader(interactiveTaskContext, sources);
+    if (sourceUploads == null && sourceUploader.sourceTypes.has("protobuf")) {
+        // We only fail hard if we need to upload Protobuf source files. Unlike OpenAPI, these
+        // files are required for successful code generation.
+        interactiveTaskContext.failAndThrow("Did not successfully upload Protobuf source files.");
+    }
+
+    if (sourceUploads != null) {
+        interactiveTaskContext.logger.debug("Uploading source files ...");
+        const sourceConfig = await sourceUploader.uploadSources(sourceUploads);
+
+        interactiveTaskContext.logger.debug("Setting IR source configuration ...");
+        ir.sourceConfig = sourceConfig;
     }
 
     const job = await createAndStartJob({
@@ -161,3 +183,33 @@
     }
     return response.body.version;
 }
+
+async function uploadSources({
+    context,
+    sourceUploader,
+    sourceUploads
+}: {
+    context: InteractiveTaskContext;
+    sourceUploader: SourceUploader;
+    sourceUploads: Record<FdrAPI.api.v1.register.SourceId, FdrAPI.api.v1.register.SourceUpload>;
+}): Promise<SourceConfig> {
+    if (Object.keys(sourceUploads).length == 0 && sourceUploader.sourceTypes.has("protobuf")) {
+        // We only fail hard if we need to upload Protobuf source files. Unlike OpenAPI, these
+        // files are required for successful code generation.
+        context.failAndThrow("Did not successfully upload Protobuf source files.");
+    }
+
+    context.logger.debug(`Uploading source files ...`);
+    return await sourceUploader.uploadSources(sourceUploads);
+}
+
+function convertToFdrApiDefinitionSources(sources: IdentifiableSource[]): Record<FdrAPI.api.v1.register.SourceId, FdrAPI.api.v1.register.Source> {
+    return Object.fromEntries(
+        Object.values(sources).map((source) => [
+            source.id,
+            {
+                type: source.type === "protobuf" ? "proto" : source.type
+            }
+        ])
+    );
+}

packages/cli/register/package.json

@@ -31,7 +31,7 @@
     "@fern-api/configuration": "workspace:*",
     "@fern-api/core": "workspace:*",
     "@fern-api/core-utils": "workspace:*",
-    "@fern-api/fdr-sdk": "0.98.20-33071ab6e",
+    "@fern-api/fdr-sdk": "0.98.20-86eba53a1",
     "@fern-api/ir-generator": "workspace:*",
     "@fern-api/ir-sdk": "workspace:*",
     "@fern-api/task-context": "workspace:*",