Skip to content

Code-first video & image, rendered natively on WebGPU.

feat(vision): prefer the GPU for headless vision, fall back to CPU

okanasl124f92ef5046
diff --git a/.changeset/vision-node-webgpu.md b/.changeset/vision-node-webgpu.md
new file mode 100644
index 00000000..b3b51c65
--- /dev/null
+++ b/.changeset/vision-node-webgpu.md
@@ -0,0 +1,5 @@
+---
+"@framefields/vision": minor
+---
+
+Headless (Node) vision now prefers the GPU: when no provider is passed, runners try onnxruntime-web's WebGPU execution provider on the process's WebGPU device (the Dawn device the renderer already creates) and fall back to the CPU `onnxruntime-node` provider when a WebGPU session can't be created. The new `AutoSessionProvider` makes that choice, and `NodeWebGPUProvider` exposes the WebGPU-only path. On example 19's track scene this lifts the render from 2.37 to 3.49 fps with pixel-identical output (PSNR ≈ 103 dB vs CPU).
diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.ts
index 9a1734c8..873d78b6 100644
--- a/packages/vision/src/index.ts
+++ b/packages/vision/src/index.ts
@@ -10,6 +10,7 @@ export * from "./model/registry.js";
export * from "./pose/keypoints.js";
export * from "./runner/canonical-baselines.js";
export * from "./runner/vision-runner.js";
+export * from "./runtime/node-webgpu-provider.js";
export * from "./runtime/session-provider.js";
export * from "./segmentation/subject.js";
export * from "./signals/vision-bundle.js";
diff --git a/packages/vision/src/runtime/node-webgpu-provider.ts b/packages/vision/src/runtime/node-webgpu-provider.ts
new file mode 100644
index 00000000..6dead1ea
--- /dev/null
+++ b/packages/vision/src/runtime/node-webgpu-provider.ts
@@ -0,0 +1,98 @@
+/**
+ * Headless (Node) WebGPU session provider.
+ *
+ * Runs `onnxruntime-web`'s WebGPU execution provider against the WebGPU device
+ * the process already has — the Dawn device the renderer creates, or one this
+ * ensures. `AutoSessionProvider` uses it when no provider is passed, falling
+ * back to the CPU `NodeSessionProvider` if a WebGPU session can't be created.
+ *
+ * `onnxruntime-web` and `webgpu` are imported lazily behind non-analyzable
+ * specifiers, so the browser entry never pulls this module's runtime imports.
+ */
+
+import type { SessionProvider, VisionSession } from "./session-provider.js";
+import {
+ type OrtWebModule,
+ WebGPUProvider,
+ type WebGPUProviderOptions,
+} from "./webgpu-provider.js";
+
+const ORT_WEBGPU_SPECIFIER = "onnxruntime-web/webgpu";
+const WEBGPU_SPECIFIER = "webgpu";
+
+let nodeWebGPUReady: Promise<void> | undefined;
+
+/**
+ * Ensure `globalThis.navigator.gpu` exists. In the renderer this is already the
+ * Dawn device; otherwise imports the `webgpu` (Dawn) package and installs its
+ * globals plus an adapter, which is what onnxruntime-web's WebGPU EP needs.
+ */
+export function ensureNodeWebGPU(): Promise<void> {
+ if (!nodeWebGPUReady) {
+ nodeWebGPUReady = (async () => {
+ const g = globalThis as { navigator?: { gpu?: unknown } };
+ if (g.navigator?.gpu) return;
+ const { create, globals } = (await import(
+ /* @vite-ignore */ WEBGPU_SPECIFIER
+ )) as {
+ create: (flags: string[]) => unknown;
+ globals: Record<string, unknown>;
+ };
+ Object.assign(globalThis, globals);
+ const nav = (g.navigator ??= {});
+ Object.defineProperty(nav, "gpu", {
+ value: create([]),
+ configurable: true,
+ writable: true,
+ });
+ })();
+ nodeWebGPUReady.catch(() => {
+ nodeWebGPUReady = undefined;
+ });
+ }
+ return nodeWebGPUReady;
+}
+
+/** Options mirror the browser provider; the loader is overridden for Node. */
+export type NodeWebGPUProviderOptions = WebGPUProviderOptions;
+
+export class NodeWebGPUProvider implements SessionProvider {
+ public readonly kind = "webgpu" as const;
+
+ private readonly _inner: WebGPUProvider;
+
+ constructor(options: NodeWebGPUProviderOptions = {}) {
+ const loader =
+ options.loader ??
+ (async (): Promise<OrtWebModule> => {
+ await ensureNodeWebGPU();
+ const mod = (await import(
+ /* @vite-ignore */ ORT_WEBGPU_SPECIFIER
+ )) as OrtWebModule & {
+ env?: { wasm?: { numThreads?: number } };
+ };
+ // Single-threaded WASM: Node has no cross-origin isolation, and the
+ // WASM path is only a fallback behind the WebGPU EP.
+ if (mod.env?.wasm) mod.env.wasm.numThreads = 1;
+ return mod;
+ });
+ // No WASM fallback inside the WebGPU provider: when WebGPU isn't possible
+ // the caller (AutoSessionProvider) falls back to the node CPU provider,
+ // which is faster than wasm and already installed.
+ this._inner = new WebGPUProvider({
+ wasmFallback: false,
+ ...options,
+ loader,
+ });
+ }
+
+ public createSession(modelBytes: Uint8Array): Promise<VisionSession> {
+ return this._inner.createSession(modelBytes);
+ }
+}
+
+export function createNodeWebGPUProvider(
+ options?: NodeWebGPUProviderOptions,
+): NodeWebGPUProvider {
+ return new NodeWebGPUProvider(options);
+}
diff --git a/packages/vision/src/runtime/session-provider.ts b/packages/vision/src/runtime/session-provider.ts
index db613870..fcf0d42b 100644
--- a/packages/vision/src/runtime/session-provider.ts
+++ b/packages/vision/src/runtime/session-provider.ts
@@ -1,3 +1,5 @@
+import { NodeWebGPUProvider } from "./node-webgpu-provider.js";
+
/**
* Minimal ONNX Runtime surface — isolates the ORT API to exactly two files so the decode
* and signals layers never touch it. Node path uses onnxruntime-node (CPU EP); the web
@@ -41,9 +43,41 @@ export function setDefaultSessionProvider(
defaultProvider = factory;
}
-/** The provider for a runner created without one. */
+/**
+ * Prefers the GPU (onnxruntime-web's WebGPU EP on the process's WebGPU device —
+ * the Dawn device the renderer already creates, or one this ensures) and falls
+ * back to the CPU provider (onnxruntime-node) when a WebGPU session cannot be
+ * created: no device, no Dawn, or the EP unavailable. The choice is made on the
+ * first session and reused for the runner's lifetime.
+ */
+export class AutoSessionProvider implements SessionProvider {
+ private _resolved?: SessionProvider;
+
+ public get kind(): "node" | "webgpu" {
+ return this._resolved?.kind ?? "webgpu";
+ }
+
+ public async createSession(modelBytes: Uint8Array): Promise<VisionSession> {
+ if (!this._resolved) {
+ const webgpu = new NodeWebGPUProvider({ wasmFallback: false });
+ try {
+ const session = await webgpu.createSession(modelBytes);
+ this._resolved = webgpu;
+ return session;
+ } catch {
+ // WebGPU isn't possible here — use CPU from now on.
+ this._resolved = new NodeSessionProvider();
+ }
+ }
+ return this._resolved.createSession(modelBytes);
+ }
+}
+
+/**
+ * The provider for a runner created without one: GPU if possible, otherwise CPU.
+ */
export function createDefaultSessionProvider(): SessionProvider {
- return defaultProvider?.() ?? new NodeSessionProvider();
+ return defaultProvider?.() ?? new AutoSessionProvider();
}
/** onnxruntime-node provider. The native module is imported lazily on first session. */