[wasm-mt] Use asset loading for dotnet.worker.js; update WasmAppBuild…

…er (#73697)

Enables using asset loading to get the `dotnet.worker.js` file that provides the emscripten pthread worker code.
Also allows specifying the number of pre-allocated workers that will be created at startup using MSBuild properties.

Fixes #68509 and fixes #68397 and fixes #72606

- Override Emscripten `PThread.allocateUnusedWorker`
   We want to use our own allocateUnusedWorker because we want to load
   `dotnet.worker.js` using our asset loading machinery.

   Unfortunately, Emscripten first calls allocateUnusedWorker very early (from
   `PThread.init`) to pre-allocate the pthread worker pool.

   So we set Emscripten's own pthread worker pool to size 0 and make our own.  This requires calling `loadWasmModuleToWorker` during our startup because Emscripten deletes their code that normally does it (in "receiveInstance" in "createWasm" in "emscripten/src/preamble.js") when the pthread pool size is 0.

   Also added a pthreadPoolSize field to MonoConfig to allow specifying the initial pthread pool size in mono-config.json

- Add `IncludeThreadsWorker` and `PThreadPoolSize` props to WasmAppBuilder

   `IncludeThreadsWorker` adds the `"js-module-threads"` asset to the `mono-config.json`

   `PThreadPoolSize` can be -1 or >=0 to specify the number of workers that will be pre-allocated at startup for the pthread worker pool.  -1 means use the default compiled into `dotnet.js`

- Reorganize the pthreads TS code by moving `Internals` (access API that digs through Emscripten's pthreads implementation) to its own module. And add types.

- Replace emscripten's `allocateUnusedWorker` function with our own that goes through the asset loading API.

- Update samples

- Set up console proxying for the workers.
   This is done by sending a message from the main thread to the pthread workers with the current `MonoConfig` on our
    dedicated channel.  (This means the proxying is setup asynchronously, so if the worker is busy before it receives the message, it may not start redirecting messages right away).

---

* [wasm-mt] Override Emscripten PThread.allocateUnusedWorker

We want to use our own allocateUnusedWorker because we want to load `dotnet.worker.js` using our asset loading machinery.

Unfortunately, Emscripten first calls allocateUnusedWorker very early (from `PThread.init`) to pre-allocate the pthread worker pool.

So we set Emscripten's own pthread worker pool to size 0 and make our own.  This requires calling `loadWasmModuleToWorker` during our startup because Emscripten deletes their code that normally does
it (in "receiveInstance" in "createWasm" in "emscripten/src/preamble.js") when the pthread pool size is 0.

Also added a pthreadPoolSize field to MonoConfig to allow specifying the initial pthread pool size in mono-config.json

* Add IncludeThreadsWorker and PThreadPoolSize props to WasmAppBuilder

IncludeThreadsWorker adds the js-module-threads asset to the mono-config

PThreadPoolSize can be -1 or >=0 to specify the number of workers that will be pre-allocated at startup for the pthread worker pool.  -1 means use the default compiled into dotnet.js

* Move emscripten PThread internals access to a separate module

   and add types

* Load js-module-threads asset in replacement allocateUnusedWorker

* Update samples to explicitly enable threading / perftracing

   Makes the WasmAppBuilder include the threads worker module

* tighten up Internals types

* apply review feedback

* fix import

* Apply suggestions from code review

* proxy pthread worker messages to websocket, if enabled

use a new MonoThreadMessageApplyMonoConfig message to send the MonoConfig from the main thread to each worker when the workers set up the communication channel to the main thread.

then if the diagnosticTracing property is true, redirect the worker console logging to a websocket.

Fixes #72606

Co-authored-by: Marek Fišera <[email protected]>
Co-authored-by: Ankit Jain <[email protected]>

src/mono/sample/wasm/browser-eventpipe/Makefile

@@ -2,6 +2,8 @@ TOP=../../../../..
 
 include ../wasm.mk
 
+override MSBUILD_ARGS+=/p:WasmEnablePerfTracing=true
+
 ifneq ($(AOT),)
 override MSBUILD_ARGS+=/p:RunAOTCompilation=true
 endif

src/mono/sample/wasm/browser-eventpipe/Wasm.Browser.EventPipe.Sample.csproj

@@ -13,6 +13,10 @@
     <NoWarn>CA2007</NoWarn> <!-- consider ConfigureAwait() -->
   </PropertyGroup>
 
+  <Target Name="CheckThreadsEnabled" BeforeTargets="Compile" >
+    <Warning Condition="'$(WasmEnableThreads)' != 'true' and '$(WasmEnablePerfTracing)' != 'true'" Text="This sample requires perftracing or threading" />
+  </Target>
+
   <PropertyGroup>
     <MonoDiagnosticsMock Condition="('$(MonoDiagnosticsMock)' == '') and ('$(Configuration)' == 'Debug')">true</MonoDiagnosticsMock>
   </PropertyGroup>

src/mono/sample/wasm/browser-threads/Makefile

@@ -2,6 +2,8 @@ TOP=../../../../..
 
 include ../wasm.mk
 
+override MSBUILD_ARGS+=/p:WasmEnableThreads=true
+
 ifneq ($(AOT),)
 override MSBUILD_ARGS+=/p:RunAOTCompilation=true
 endif

src/mono/sample/wasm/browser-threads/Wasm.Browser.Threads.Sample.csproj

@@ -16,6 +16,10 @@
   </PropertyGroup>
   <Target Name="RunSample" DependsOnTargets="RunSampleWithBrowser" />
 
+  <Target Name="CheckThreadsEnabled" BeforeTargets="Compile" >
+    <Warning Condition="'$(WasmEnableThreads)' != 'true'" Text="This sample requires threading" />
+  </Target>
+
   <!-- set the condition to false and you will get a CA1416 error about the call to Thread.Start from a browser-wasm project -->
   <ItemGroup Condition="true">
     <!-- TODO: some .props file that automates this.  Unfortunately just adding a ProjectReference to Microsoft.NET.WebAssembly.Threading.proj doesn't work - it ends up bundling the ref assemblies into the publish directory and breaking the app. -->

src/mono/wasm/build/WasmApp.targets

@@ -316,6 +316,12 @@
           DependsOnTargets="_WasmGenerateRuntimeConfig;_GetWasmGenerateAppBundleDependencies">
     <Error Condition="'$(WasmMainJSPath)' == ''" Text="%24(WasmMainJSPath) property needs to be set" />
 
+    <PropertyGroup>
+      <_WasmAppIncludeThreadsWorker Condition="'$(WasmEnableThreads)' == 'true' or '$(WasmEnablePerfTracing)' == 'true'">true</_WasmAppIncludeThreadsWorker>
+      <!-- TODO: set this from some user-facing property?  -1 means use the default baked into dotnet.js -->
+      <_WasmPThreadPoolSize Condition="'$(_WasmPThreadPoolSize)' == '' and ('$(WasmEnableThreads)' == 'true' or '$(WasmEnablePerfTracing)' == 'true')">-1</_WasmPThreadPoolSize>
+    </PropertyGroup>
+
     <RemoveDir Directories="$(WasmAppDir)" />
     <WasmAppBuilder
       AppDir="$(WasmAppDir)"
@@ -334,7 +340,10 @@
       ExtraFilesToDeploy="@(WasmExtraFilesToDeploy)"
       ExtraConfig="@(WasmExtraConfig)"
       NativeAssets="@(WasmNativeAsset)"
-      DebugLevel="$(WasmDebugLevel)">
+      DebugLevel="$(WasmDebugLevel)"
+      IncludeThreadsWorker="$(_WasmAppIncludeThreadsWorker)"
+      PThreadPoolSize="$(_WasmPThreadPoolSize)"
+      >
       <Output TaskParameter="FileWrites" ItemName="FileWrites" />
     </WasmAppBuilder>
 

src/mono/wasm/runtime/dotnet.d.ts

@@ -81,6 +81,7 @@ declare type MonoConfig = {
     coverageProfilerOptions?: CoverageProfilerOptions;
     ignorePdbLoadErrors?: boolean;
     waitForDebugger?: number;
+    pthreadPoolSize?: number;
 };
 interface ResourceRequest {
     name: string;

src/mono/wasm/runtime/es6/dotnet.es6.lib.js

@@ -26,6 +26,7 @@ let __dotnet_replacement_PThread = ${usePThreads} ? {} : undefined;
 if (${usePThreads}) {
     __dotnet_replacement_PThread.loadWasmModuleToWorker = PThread.loadWasmModuleToWorker;
     __dotnet_replacement_PThread.threadInitTLS = PThread.threadInitTLS;
+    __dotnet_replacement_PThread.allocateUnusedWorker = PThread.allocateUnusedWorker;
 }
 let __dotnet_replacements = {scriptUrl: import.meta.url, fetch: globalThis.fetch, require, updateGlobalBufferAndViews, pthreadReplacements: __dotnet_replacement_PThread};
 if (ENVIRONMENT_IS_NODE) {
@@ -47,6 +48,7 @@ var noExitRuntime = __dotnet_replacements.noExitRuntime;
 if (${usePThreads}) {
     PThread.loadWasmModuleToWorker = __dotnet_replacements.pthreadReplacements.loadWasmModuleToWorker;
     PThread.threadInitTLS = __dotnet_replacements.pthreadReplacements.threadInitTLS;
+    PThread.allocateUnusedWorker = __dotnet_replacements.pthreadReplacements.allocateUnusedWorker;
 }
 `,
 };

src/mono/wasm/runtime/polyfills.ts

@@ -2,8 +2,7 @@ import BuildConfiguration from "consts:configuration";
 import MonoWasmThreads from "consts:monoWasmThreads";
 import { ENVIRONMENT_IS_NODE, ENVIRONMENT_IS_SHELL, ENVIRONMENT_IS_WEB, ENVIRONMENT_IS_WORKER, INTERNAL, Module, runtimeHelpers } from "./imports";
 import { afterUpdateGlobalBufferAndViews } from "./memory";
-import { afterLoadWasmModuleToWorker } from "./pthreads/browser";
-import { afterThreadInitTLS } from "./pthreads/worker";
+import { replaceEmscriptenPThreadLibrary } from "./pthreads/shared/emscripten-replacements";
 import { DotnetModuleConfigImports, EarlyReplacements } from "./types";
 
 let node_fs: any | undefined = undefined;
@@ -173,16 +172,7 @@ export function init_polyfills(replacements: EarlyReplacements): void {
     // threads
     if (MonoWasmThreads) {
         if (replacements.pthreadReplacements) {
-            const originalLoadWasmModuleToWorker = replacements.pthreadReplacements.loadWasmModuleToWorker;
-            replacements.pthreadReplacements.loadWasmModuleToWorker = (worker: Worker, onFinishedLoading: Function): void => {
-                originalLoadWasmModuleToWorker(worker, onFinishedLoading);
-                afterLoadWasmModuleToWorker(worker);
-            };
-            const originalThreadInitTLS = replacements.pthreadReplacements.threadInitTLS;
-            replacements.pthreadReplacements.threadInitTLS = (): void => {
-                originalThreadInitTLS();
-                afterThreadInitTLS();
-            };
+            replaceEmscriptenPThreadLibrary(replacements.pthreadReplacements);
         }
     }
 
@@ -297,4 +287,4 @@ function isPathAbsolute(path: string): boolean {
     // windows file:///C:/x.json
     // windows http://C:/x.json
     return protocolRx.test(path);
-}
+}

src/mono/wasm/runtime/pthreads/browser/index.ts

@@ -1,10 +1,13 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-import { Module } from "../../imports";
-import { pthread_ptr, MonoWorkerMessageChannelCreated, isMonoWorkerMessageChannelCreated, monoSymbol } from "../shared";
+import { MonoWorkerMessageChannelCreated, isMonoWorkerMessageChannelCreated, monoSymbol, makeMonoThreadMessageApplyMonoConfig } from "../shared";
+import { pthread_ptr } from "../shared/types";
 import { MonoThreadMessage } from "../shared";
 import { PromiseController, createPromiseController } from "../../promise-controller";
+import { MonoConfig, mono_assert } from "../../types";
+import Internals from "../shared/emscripten-internals";
+import { runtimeHelpers } from "../../imports";
 
 const threads: Map<pthread_ptr, Thread> = new Map();
 
@@ -92,6 +95,7 @@ function monoWorkerMessageHandler(worker: Worker, ev: MessageEvent<MonoWorkerMes
         const thread = addThread(pthread_id, worker, port);
         port.addEventListener("message", (ev) => monoDedicatedChannelMessageFromWorkerToMain(ev, thread));
         port.start();
+        port.postMessage(makeMonoThreadMessageApplyMonoConfig(runtimeHelpers.config));
         resolvePromises(pthread_id, thread);
     }
 }
@@ -103,21 +107,40 @@ export function afterLoadWasmModuleToWorker(worker: Worker): void {
     console.debug("MONO_WASM: afterLoadWasmModuleToWorker added message event handler", worker);
 }
 
-/// These utility functions dig into Emscripten internals
-const Internals = {
-    getWorker: (pthread_ptr: pthread_ptr): Worker => {
-        // see https://github.com/emscripten-core/emscripten/pull/16239
-        return (<any>Module).PThread.pthreads[pthread_ptr].worker;
-    },
-    getThreadId: (worker: Worker): pthread_ptr | undefined => {
-        /// See library_pthread.js in Emscripten.
-        /// They hang a "pthread" object from the worker if the worker is running a thread, and remove it when the thread stops by doing `pthread_exit` or when it's joined using `pthread_join`.
-        const emscriptenThreadInfo = (<any>worker)["pthread"];
-        if (emscriptenThreadInfo === undefined) {
-            return undefined;
-        }
-        return emscriptenThreadInfo.threadInfoStruct;
+/// We call on the main thread this during startup to pre-allocate a pool of pthread workers.
+/// At this point asset resolution needs to be working (ie we loaded MonoConfig).
+/// This is used instead of the Emscripten PThread.initMainThread because we call it later.
+export function preAllocatePThreadWorkerPool(defaultPthreadPoolSize: number, config: MonoConfig): void {
+    const poolSizeSpec = config?.pthreadPoolSize;
+    let n: number;
+    if (poolSizeSpec === undefined) {
+        n = defaultPthreadPoolSize;
+    } else {
+        mono_assert(typeof poolSizeSpec === "number", "pthreadPoolSize must be a number");
+        if (poolSizeSpec < 0)
+            n = defaultPthreadPoolSize;
+        else
+            n = poolSizeSpec;
     }
-};
-
+    for (let i = 0; i < n; i++) {
+        Internals.allocateUnusedWorker();
+    }
+}
 
+/// We call this on the main thread during startup once we fetched WasmModule.
+/// This sends a message to each pre-allocated worker to load the WasmModule and dotnet.js and to set up
+/// message handling.
+/// This is used instead of the Emscripten "receiveInstance" in "createWasm" because that code is
+/// conditioned on a non-zero PTHREAD_POOL_SIZE (but we set it to 0 to avoid early worker allocation).
+export async function instantiateWasmPThreadWorkerPool(): Promise<void> {
+    // this is largely copied from emscripten's "receiveInstance" in "createWasm" in "src/preamble.js"
+    const workers = Internals.getUnusedWorkerPool();
+    const allLoaded = createPromiseController<void>();
+    let leftToLoad = workers.length;
+    workers.forEach((w) => {
+        Internals.loadWasmModuleToWorker(w, function () {
+            if (!--leftToLoad) allLoaded.promise_control.resolve();
+        });
+    });
+    await allLoaded.promise;
+}

src/mono/wasm/runtime/pthreads/shared/emscripten-internals.ts

@@ -0,0 +1,76 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+import { Module } from "../../imports";
+import { pthread_ptr } from "./types";
+
+/** @module emscripten-internals accessors to the functions in the emscripten PThreads library, including
+ * the low-level representations of {@linkcode pthread_ptr} thread info structs, etc.
+ * Additionally, note that some of these functions are replaced by {@linkcode file://./emscripten-replacements.ts}.
+ * These have a hard dependency on the version of Emscripten that we are using and may need to be kept in sync with
+ *    {@linkcode file://./../../../emsdk/upstream/emscripten/src/library_pthread.js}
+ */
+
+// This is what we know about the Emscripten PThread library
+interface PThreadLibrary {
+    unusedWorkers: Worker[];
+    pthreads: PThreadInfoMap;
+    allocateUnusedWorker: () => void;
+    loadWasmModuleToWorker: (worker: Worker, onFinishedLoading?: (worker: Worker) => void) => void;
+}
+
+interface EmscriptenPThreadInfo {
+    threadInfoStruct: pthread_ptr;
+}
+
+/// N.B. emscripten deletes the `pthread` property from the worker when it is not actively running a pthread
+interface PThreadWorker extends Worker {
+    pthread: EmscriptenPThreadInfo;
+}
+
+interface PThreadObject {
+    worker: PThreadWorker;
+}
+
+interface PThreadInfoMap {
+    [key: pthread_ptr]: PThreadObject | undefined;
+}
+
+
+function isRunningPThreadWorker(w: Worker): w is PThreadWorker {
+    return (<any>w).pthread !== undefined;
+}
+
+/// These utility functions dig into Emscripten internals
+const Internals = {
+    get modulePThread(): PThreadLibrary {
+        return (<any>Module).PThread as PThreadLibrary;
+    },
+    getWorker: (pthread_ptr: pthread_ptr): PThreadWorker | undefined => {
+        // see https://github.com/emscripten-core/emscripten/pull/16239
+        return Internals.modulePThread.pthreads[pthread_ptr]?.worker;
+    },
+    getThreadId: (worker: Worker): pthread_ptr | undefined => {
+        /// See library_pthread.js in Emscripten.
+        /// They hang a "pthread" object from the worker if the worker is running a thread, and remove it when the thread stops by doing `pthread_exit` or when it's joined using `pthread_join`.
+        if (!isRunningPThreadWorker(worker))
+            return undefined;
+        const emscriptenThreadInfo = worker.pthread;
+        return emscriptenThreadInfo.threadInfoStruct;
+    },
+    allocateUnusedWorker: (): void => {
+        /// See library_pthread.js in Emscripten.
+        /// This function allocates a new worker and adds it to the pool of workers.
+        /// It's called when the pool of workers is empty and a new thread is created.
+        Internals.modulePThread.allocateUnusedWorker();
+    },
+    getUnusedWorkerPool: (): Worker[] => {
+        return Internals.modulePThread.unusedWorkers;
+    },
+    loadWasmModuleToWorker: (worker: Worker, onFinishedLoading: () => void): void => {
+        Internals.modulePThread.loadWasmModuleToWorker(worker, onFinishedLoading);
+    }
+};
+
+
+export default Internals;

src/mono/wasm/runtime/pthreads/shared/emscripten-replacements.ts

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+import MonoWasmThreads from "consts:monoWasmThreads";
+import { PThreadReplacements } from "../../types";
+import { afterLoadWasmModuleToWorker } from "../browser";
+import { afterThreadInitTLS } from "../worker";
+import Internals from "./emscripten-internals";
+import { resolve_asset_path } from "../../assets";
+import { mono_assert } from "../../types";
+import { runtimeHelpers }  from "../../imports";
+
+/** @module emscripten-replacements Replacements for individual functions in the emscripten PThreads library.
+ * These have a hard dependency on the version of Emscripten that we are using and may need to be kept in sync with
+ *    {@linkcode file://./../../../emsdk/upstream/emscripten/src/library_pthread.js}
+ */
+
+export function replaceEmscriptenPThreadLibrary(replacements: PThreadReplacements): void {
+    if (MonoWasmThreads) {
+        const originalLoadWasmModuleToWorker = replacements.loadWasmModuleToWorker;
+        replacements.loadWasmModuleToWorker = (worker: Worker, onFinishedLoading?: (worker: Worker) => void): void => {
+            originalLoadWasmModuleToWorker(worker, onFinishedLoading);
+            afterLoadWasmModuleToWorker(worker);
+        };
+        const originalThreadInitTLS = replacements.threadInitTLS;
+        replacements.threadInitTLS = (): void => {
+            originalThreadInitTLS();
+            afterThreadInitTLS();
+        };
+        // const originalAllocateUnusedWorker = replacements.allocateUnusedWorker;
+        replacements.allocateUnusedWorker = replacementAllocateUnusedWorker;
+    }
+}
+
+/// We replace Module["PThreads"].allocateUnusedWorker with this version that knows about assets
+function replacementAllocateUnusedWorker(): void {
+    if (runtimeHelpers.diagnosticTracing)
+        console.debug("MONO_WASM: replacementAllocateUnusedWorker");
+    const asset = resolve_asset_path("js-module-threads");
+    const uri = asset.resolvedUrl;
+    mono_assert(uri !== undefined, "could not resolve the uri for the js-module-threads asset");
+    const worker = new Worker(uri);
+    Internals.getUnusedWorkerPool().push(worker);
+}

src/mono/wasm/runtime/pthreads/shared/index.ts

@@ -2,9 +2,8 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 import { Module } from "../../imports";
-
-/// pthread_t in C
-export type pthread_ptr = number;
+import { MonoConfig } from "../../types";
+import { pthread_ptr } from "./types";
 
 export interface PThreadInfo {
     readonly pthread_id: pthread_ptr;
@@ -44,6 +43,29 @@ export function isMonoThreadMessage(x: unknown): x is MonoThreadMessage {
     return typeof (xmsg.type) === "string" && typeof (xmsg.cmd) === "string";
 }
 
+// message from the main thread to the pthread worker that passes the MonoConfig to the worker
+export interface MonoThreadMessageApplyMonoConfig extends MonoThreadMessage {
+    type: "pthread";
+    cmd: "apply_mono_config";
+    config: string;
+}
+
+export function isMonoThreadMessageApplyMonoConfig(x: unknown): x is MonoThreadMessageApplyMonoConfig {
+    if (!isMonoThreadMessage(x)) {
+        return false;
+    }
+    const xmsg = x as MonoThreadMessageApplyMonoConfig;
+    return xmsg.type === "pthread" && xmsg.cmd === "apply_mono_config" && typeof (xmsg.config) === "string";
+}
+
+export function makeMonoThreadMessageApplyMonoConfig(config: MonoConfig): MonoThreadMessageApplyMonoConfig {
+    return {
+        type: "pthread",
+        cmd: "apply_mono_config",
+        config: JSON.stringify(config)
+    };
+}
+
 /// Messages sent using the worker object's postMessage() method ///
 
 /// a symbol that we use as a key on messages on the global worker-to-main channel to identify our own messages

src/mono/wasm/runtime/pthreads/shared/tsconfig.json

@@ -1,3 +1,8 @@
 {
-    "extends": "../../tsconfig.shared.json"
+    "extends": "../../tsconfig.shared.json",
+    "include": [
+        "../../**/*.ts",
+        "../../**/*.d.ts"
+    ]
+
 }

src/mono/wasm/runtime/pthreads/shared/types.ts

@@ -0,0 +1,5 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+/// pthread_t in C
+export type pthread_ptr = number;

src/mono/wasm/runtime/pthreads/worker/events.ts

@@ -1,5 +1,6 @@
 import MonoWasmThreads from "consts:monoWasmThreads";
-import type { pthread_ptr, PThreadInfo, MonoThreadMessage } from "../shared";
+import type { pthread_ptr } from "../shared/types";
+import type { PThreadInfo, MonoThreadMessage } from "../shared";
 
 /// Identification of the current thread executing on a worker
 export interface PThreadSelf extends PThreadInfo {