From ad768f2e08a6fed5ccd80042354a61b20564c0b2 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Fri, 1 Nov 2024 20:59:23 -0700 Subject: [PATCH] duckdb aliases, platforms config --- src/config.ts | 23 +++++++++++----- src/duckdb.ts | 62 ++++++++++++++++++++++++------------------ test/config-test.ts | 8 ++++-- test/libraries-test.ts | 2 +- 4 files changed, 59 insertions(+), 36 deletions(-) diff --git a/src/config.ts b/src/config.ts index e6f5cf865..bf063bc46 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,7 +8,7 @@ import {pathToFileURL} from "node:url"; import he from "he"; import type MarkdownIt from "markdown-it"; import wrapAnsi from "wrap-ansi"; -import {DUCKDB_CORE_EXTENSIONS, DUCKDB_PLATFORMS} from "./duckdb.js"; +import {DUCKDB_CORE_ALIASES, DUCKDB_CORE_EXTENSIONS} from "./duckdb.js"; import {visitFiles} from "./files.js"; import {formatIsoDate, formatLocaleDate} from "./format.js"; import type {FrontMatter} from "./frontMatter.js"; @@ -78,7 +78,7 @@ export interface SearchConfigSpec { } export interface DuckDBConfig { - platforms: Record; + platforms: {[name: string]: true}; extensions: {[name: string]: DuckDBExtensionConfig}; } @@ -522,21 +522,22 @@ export function stringOrNull(spec: unknown): string | null { return spec == null || spec === false ? null : String(spec); } -// TODO configure platforms? function normalizeDuckDB(spec: unknown): DuckDBConfig { + const {mvp = true, eh = true} = spec?.["platforms"] ?? {}; const extensions: {[name: string]: DuckDBExtensionConfig} = {}; let extspec: Record = spec?.["extensions"] ?? {}; if (Array.isArray(extspec)) extspec = Object.fromEntries(extspec.map((name) => [name, {}])); if (extspec.json === undefined) extspec = {...extspec, json: false}; if (extspec.parquet === undefined) extspec = {...extspec, parquet: false}; - for (const name in extspec) { + for (let name in extspec) { if (!/^\w+$/.test(name)) throw new Error(`invalid extension: ${name}`); const vspec = extspec[name]; if (vspec == null) continue; + name = DUCKDB_CORE_ALIASES[name] ?? name; const { - source = DUCKDB_CORE_EXTENSIONS.some(([n]) => n === name) ? "core" : "community", + source = name in DUCKDB_CORE_EXTENSIONS ? "core" : "community", install = true, - load = !DUCKDB_CORE_EXTENSIONS.find(([n]) => n === name)?.[1] + load = !DUCKDB_CORE_EXTENSIONS[name] } = typeof vspec === "boolean" ? {load: vspec} : typeof vspec === "string" @@ -548,7 +549,15 @@ function normalizeDuckDB(spec: unknown): DuckDBConfig { load: Boolean(load) }; } - return {platforms: DUCKDB_PLATFORMS, extensions}; + return { + platforms: Object.fromEntries( + [ + ["mvp", mvp], + ["eh", eh] + ].filter(([, enabled]) => enabled) + ), + extensions + }; } function normalizeDuckDBSource(source: string): string { diff --git a/src/duckdb.ts b/src/duckdb.ts index b06ac1a5a..bc4360a50 100644 --- a/src/duckdb.ts +++ b/src/duckdb.ts @@ -8,40 +8,50 @@ const downloadRequests = new Map>(); export const DUCKDB_WASM_VERSION = "1.29.0"; export const DUCKDB_VERSION = "1.1.1"; -export const DUCKDB_PLATFORMS: DuckDBConfig["platforms"] = {eh: true, mvp: true}; // https://duckdb.org/docs/extensions/core_extensions.html -export const DUCKDB_CORE_EXTENSIONS: [name: string, autoload: boolean][] = [ - ["arrow", false], - ["autocomplete", true], - ["aws", true], - ["azure", true], - ["delta", true], - ["excel", true], - ["fts", true], - ["httpfs", true], - ["iceberg", false], - ["icu", true], - ["inet", true], - ["jemalloc", false], - ["json", true], - ["mysql", false], - ["parquet", true], - ["postgres", true], - ["spatial", false], - ["sqlite", true], - ["substrait", false], - ["tpcds", true], - ["tpch", true], - ["vss", false] -]; +export const DUCKDB_CORE_ALIASES: Record = { + sqlite: "sqlite_scanner", + sqlite3: "sqlite_scanner", + postgres_scanner: "postgres", + http: "httpfs", + https: "httpfs", + s3: "httpfs" +} as const; + +// https://duckdb.org/docs/extensions/core_extensions.html +// https://duckdb.org/docs/api/wasm/extensions.html#list-of-officially-available-extensions +export const DUCKDB_CORE_EXTENSIONS = { + arrow: false, + autocomplete: true, + aws: true, + azure: true, + delta: true, + excel: true, + fts: true, + httpfs: true, + iceberg: false, + icu: true, + inet: true, + jemalloc: false, + json: true, + mysql: false, + parquet: true, + postgres: true, + spatial: false, + sqlite_scanner: true, + substrait: false, + tpcds: true, + tpch: true, + vss: false +} as const; export async function getDuckDBManifest( {platforms, extensions}: DuckDBConfig, {root, aliases}: {root: string; aliases?: Map} ) { return { - platforms, + platforms: {mvp: "mvp" in platforms, eh: "eh" in platforms}, extensions: Object.fromEntries( await Promise.all( Object.entries(extensions).map(([name, {install, load, source}]) => diff --git a/test/config-test.ts b/test/config-test.ts index 7be173cdc..32360f734 100644 --- a/test/config-test.ts +++ b/test/config-test.ts @@ -1,11 +1,15 @@ import assert from "node:assert"; import {resolve} from "node:path"; import MarkdownIt from "markdown-it"; +import type {DuckDBConfig} from "../src/config.js"; import {normalizeConfig as config, mergeToc, readConfig, setCurrentDate} from "../src/config.js"; import {LoaderResolver} from "../src/loader.js"; -const DUCKDB_DEFAULTS = { - bundles: ["eh", "mvp"], +const DUCKDB_DEFAULTS: DuckDBConfig = { + platforms: { + eh: true, + mvp: true + }, extensions: { json: { source: "https://extensions.duckdb.org/", diff --git a/test/libraries-test.ts b/test/libraries-test.ts index 8ecbeeee5..12cba2e46 100644 --- a/test/libraries-test.ts +++ b/test/libraries-test.ts @@ -53,7 +53,7 @@ describe("getImplicitStylesheets(imports)", () => { describe("getImplicitDownloads(imports)", () => { it("supports known imports", () => { assert.deepStrictEqual( - getImplicitDownloads(["npm:@observablehq/duckdb"]), + getImplicitDownloads(["npm:@observablehq/duckdb"], {extensions: {}, platforms: {mvp: true, eh: true}}), new Set([ "npm:@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm", "npm:@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js",