Code-first video & image, rendered natively on WebGPU.
feat(vision): prefer the GPU for headless vision, fall back to CPU
okanasl124f92ef5046
diff --git a/.changeset/vision-node-webgpu.md b/.changeset/vision-node-webgpu.mdnew file mode 100644index 00000000..b3b51c65--- /dev/null+++ b/.changeset/vision-node-webgpu.md@@ -0,0 +1,5 @@+---+"@framefields/vision": minor+---++Headless (Node) vision now prefers the GPU: when no provider is passed, runners try onnxruntime-web's WebGPU execution provider on the process's WebGPU device (the Dawn device the renderer already creates) and fall back to the CPU `onnxruntime-node` provider when a WebGPU session can't be created. The new `AutoSessionProvider` makes that choice, and `NodeWebGPUProvider` exposes the WebGPU-only path. On example 19's track scene this lifts the render from 2.37 to 3.49 fps with pixel-identical output (PSNR ≈ 103 dB vs CPU).diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.tsindex 9a1734c8..873d78b6 100644--- a/packages/vision/src/index.ts+++ b/packages/vision/src/index.ts@@ -10,6 +10,7 @@ export * from "./model/registry.js";export * from "./pose/keypoints.js";export * from "./runner/canonical-baselines.js";export * from "./runner/vision-runner.js";+export * from "./runtime/node-webgpu-provider.js";export * from "./runtime/session-provider.js";export * from "./segmentation/subject.js";export * from "./signals/vision-bundle.js";diff --git a/packages/vision/src/runtime/node-webgpu-provider.ts b/packages/vision/src/runtime/node-webgpu-provider.tsnew file mode 100644index 00000000..6dead1ea--- /dev/null+++ b/packages/vision/src/runtime/node-webgpu-provider.ts@@ -0,0 +1,98 @@+/**+ * Headless (Node) WebGPU session provider.+ *+ * Runs `onnxruntime-web`'s WebGPU execution provider against the WebGPU device+ * the process already has — the Dawn device the renderer creates, or one this+ * ensures. `AutoSessionProvider` uses it when no provider is passed, falling+ * back to the CPU `NodeSessionProvider` if a WebGPU session can't be created.+ *+ * `onnxruntime-web` and `webgpu` are imported lazily behind non-analyzable+ * specifiers, so the browser entry never pulls this module's runtime imports.+ */++import type { SessionProvider, VisionSession } from "./session-provider.js";+import {+ type OrtWebModule,+ WebGPUProvider,+ type WebGPUProviderOptions,+} from "./webgpu-provider.js";++const ORT_WEBGPU_SPECIFIER = "onnxruntime-web/webgpu";+const WEBGPU_SPECIFIER = "webgpu";++let nodeWebGPUReady: Promise<void> | undefined;++/**+ * Ensure `globalThis.navigator.gpu` exists. In the renderer this is already the+ * Dawn device; otherwise imports the `webgpu` (Dawn) package and installs its+ * globals plus an adapter, which is what onnxruntime-web's WebGPU EP needs.+ */+export function ensureNodeWebGPU(): Promise<void> {+ if (!nodeWebGPUReady) {+ nodeWebGPUReady = (async () => {+ const g = globalThis as { navigator?: { gpu?: unknown } };+ if (g.navigator?.gpu) return;+ const { create, globals } = (await import(+ /* @vite-ignore */ WEBGPU_SPECIFIER+ )) as {+ create: (flags: string[]) => unknown;+ globals: Record<string, unknown>;+ };+ Object.assign(globalThis, globals);+ const nav = (g.navigator ??= {});+ Object.defineProperty(nav, "gpu", {+ value: create([]),+ configurable: true,+ writable: true,+ });+ })();+ nodeWebGPUReady.catch(() => {+ nodeWebGPUReady = undefined;+ });+ }+ return nodeWebGPUReady;+}++/** Options mirror the browser provider; the loader is overridden for Node. */+export type NodeWebGPUProviderOptions = WebGPUProviderOptions;++export class NodeWebGPUProvider implements SessionProvider {+ public readonly kind = "webgpu" as const;++ private readonly _inner: WebGPUProvider;++ constructor(options: NodeWebGPUProviderOptions = {}) {+ const loader =+ options.loader ??+ (async (): Promise<OrtWebModule> => {+ await ensureNodeWebGPU();+ const mod = (await import(+ /* @vite-ignore */ ORT_WEBGPU_SPECIFIER+ )) as OrtWebModule & {+ env?: { wasm?: { numThreads?: number } };+ };+ // Single-threaded WASM: Node has no cross-origin isolation, and the+ // WASM path is only a fallback behind the WebGPU EP.+ if (mod.env?.wasm) mod.env.wasm.numThreads = 1;+ return mod;+ });+ // No WASM fallback inside the WebGPU provider: when WebGPU isn't possible+ // the caller (AutoSessionProvider) falls back to the node CPU provider,+ // which is faster than wasm and already installed.+ this._inner = new WebGPUProvider({+ wasmFallback: false,+ ...options,+ loader,+ });+ }++ public createSession(modelBytes: Uint8Array): Promise<VisionSession> {+ return this._inner.createSession(modelBytes);+ }+}++export function createNodeWebGPUProvider(+ options?: NodeWebGPUProviderOptions,+): NodeWebGPUProvider {+ return new NodeWebGPUProvider(options);+}diff --git a/packages/vision/src/runtime/session-provider.ts b/packages/vision/src/runtime/session-provider.tsindex db613870..fcf0d42b 100644--- a/packages/vision/src/runtime/session-provider.ts+++ b/packages/vision/src/runtime/session-provider.ts@@ -1,3 +1,5 @@+import { NodeWebGPUProvider } from "./node-webgpu-provider.js";+/*** Minimal ONNX Runtime surface — isolates the ORT API to exactly two files so the decode* and signals layers never touch it. Node path uses onnxruntime-node (CPU EP); the web@@ -41,9 +43,41 @@ export function setDefaultSessionProvider(defaultProvider = factory;}-/** The provider for a runner created without one. */+/**+ * Prefers the GPU (onnxruntime-web's WebGPU EP on the process's WebGPU device —+ * the Dawn device the renderer already creates, or one this ensures) and falls+ * back to the CPU provider (onnxruntime-node) when a WebGPU session cannot be+ * created: no device, no Dawn, or the EP unavailable. The choice is made on the+ * first session and reused for the runner's lifetime.+ */+export class AutoSessionProvider implements SessionProvider {+ private _resolved?: SessionProvider;++ public get kind(): "node" | "webgpu" {+ return this._resolved?.kind ?? "webgpu";+ }++ public async createSession(modelBytes: Uint8Array): Promise<VisionSession> {+ if (!this._resolved) {+ const webgpu = new NodeWebGPUProvider({ wasmFallback: false });+ try {+ const session = await webgpu.createSession(modelBytes);+ this._resolved = webgpu;+ return session;+ } catch {+ // WebGPU isn't possible here — use CPU from now on.+ this._resolved = new NodeSessionProvider();+ }+ }+ return this._resolved.createSession(modelBytes);+ }+}++/**+ * The provider for a runner created without one: GPU if possible, otherwise CPU.+ */export function createDefaultSessionProvider(): SessionProvider {- return defaultProvider?.() ?? new NodeSessionProvider();+ return defaultProvider?.() ?? new AutoSessionProvider();}/** onnxruntime-node provider. The native module is imported lazily on first session. */