Code-first video & image, rendered natively on WebGPU.
feat(vision): log the inference backend; NCHW only where it's needed
okanaslf847dd1c03cd
diff --git a/.changeset/vision-webgpu-nchw.md b/.changeset/vision-webgpu-nchw.mdindex 1e3999d1..c84ec0f7 100644--- a/.changeset/vision-webgpu-nchw.md+++ b/.changeset/vision-webgpu-nchw.md@@ -2,4 +2,6 @@"@framefields/vision": patch----Fix duplicate pose skeletons on the GPU path: the WebGPU execution provider now runs in NCHW layout. In its default NHWC layout onnxruntime-web (1.30) computes grouped convolutions wrong when the channels per group aren't a multiple of 4, which corrupts RTMO's head (group-8 convs over 104 channels) and turned one person into several detections. Outputs now match the CPU provider on every pinned model.+Fix duplicate pose skeletons on the GPU path. In its default NHWC layout, onnxruntime-web (1.30)'s WebGPU execution provider computes grouped convolutions wrong when a group has more than one channel and the count isn't a multiple of 4. That corrupts RTMO's head (18 channels per group) and turned one person into several detections. Models with such a convolution (RTMO) now run in NCHW layout; the others (RTMDet-Ins, Selfie, whose grouped convolutions are depthwise) keep NHWC, which is about 30% faster. Outputs match the CPU provider on every pinned model.++Vision now reports its inference backend once per process, and the renderer logs it: `[vision] Inference on the GPU (WebGPU)` with `sharedDevice`, or `[vision] Inference on the CPU` with the reason.diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.tsindex 54aeabb1..dc304891 100644--- a/packages/renderer/src/headless-webgpu-renderer.ts+++ b/packages/renderer/src/headless-webgpu-renderer.ts@@ -50,6 +50,20 @@ import { preloadFonts } from "./asset-preloader.js";import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";import { rendererLogger } from "./logger.js";++// Vision reports its inference backend once per process (see+// VISION_BACKEND_LISTENER in @framefields/vision).+(globalThis as Record<symbol, unknown>)[+ Symbol.for("framefields.vision.backend")+] = (info: { backend: string; sharedDevice: boolean; reason?: string }) => {+ rendererLogger.info(+ info,+ info.backend === "webgpu"+ ? "[vision] Inference on the GPU (WebGPU)"+ : `[vision] Inference on the ${info.backend.toUpperCase()}`,+ );+};+import {type AudioQaStats,analyzeAudio,diff --git a/packages/vision/src/model/conv-layout.ts b/packages/vision/src/model/conv-layout.tsnew file mode 100644index 00000000..8f111e7b--- /dev/null+++ b/packages/vision/src/model/conv-layout.ts@@ -0,0 +1,108 @@+/**+ * onnxruntime-web 1.30's WebGPU EP computes grouped convolutions wrong in its+ * default NHWC layout when a group has more than one channel and the count+ * isn't a multiple of 4 (RTMO's head: 18 per group → duplicate skeletons).+ * NCHW is correct but ~30% slower, so only models with such a convolution get+ * it. Depthwise convolutions (one channel per group: RTMDet-Ins, Selfie) are+ * fine in NHWC.+ */++import {+ fields,+ firstString,+ GRAPH_INITIALIZER,+ GRAPH_NODE,+ MODEL_GRAPH,+ NODE_INPUT,+ readVarint,+ strings,+ TENSOR_NAME,+ WIRE_LEN,+ WIRE_VARINT,+} from "./onnx-proto.js";++const NODE_OP_TYPE = 4;+const NODE_ATTRIBUTE = 5;+const NODE_OUTPUT = 2;+const ATTR_NAME = 1;+const ATTR_I = 3;+const ATTR_T = 5;+const TENSOR_DIMS = 1;++function varintValues(field: {+ wire: number;+ raw: Uint8Array;+ body: Uint8Array;+}): number[] {+ const values: number[] = [];+ if (field.wire === WIRE_LEN) {+ let pos = 0;+ while (pos < field.body.length) {+ const [value, next] = readVarint(field.body, pos);+ values.push(value);+ pos = next;+ }+ } else if (field.wire === WIRE_VARINT) {+ let pos = 0;+ pos = readVarint(field.raw, pos)[1]; // tag+ values.push(readVarint(field.raw, pos)[0]);+ }+ return values;+}++function tensorDims(tensor: Uint8Array): number[] {+ return fields(tensor)+ .filter((f) => f.number === TENSOR_DIMS)+ .flatMap(varintValues);+}++/**+ * True when the model has a grouped convolution the NHWC kernels get wrong,+ * and when it can't be read (NCHW is always correct).+ */+export function needsNchwLayout(model: Uint8Array): boolean {+ let graph: ReturnType<typeof fields>[number] | undefined;+ try {+ graph = fields(model).find(+ (f) => f.number === MODEL_GRAPH && f.wire === WIRE_LEN,+ );+ } catch {+ return true;+ }+ if (!graph) return true;+ const dims = new Map<string, number[]>();+ const convs: { weight: string; group: number }[] = [];+ for (const f of fields(graph.body)) {+ if (f.number === GRAPH_INITIALIZER) {+ const name = firstString(f.body, TENSOR_NAME);+ if (name !== undefined) dims.set(name, tensorDims(f.body));+ } else if (f.number === GRAPH_NODE) {+ const op = firstString(f.body, NODE_OP_TYPE);+ if (op !== "Conv" && op !== "Constant") continue;+ const attributes = fields(f.body)+ .filter((a) => a.number === NODE_ATTRIBUTE)+ .map((a) => a.body);+ if (op === "Constant") {+ const value = attributes.find(+ (a) => firstString(a, ATTR_NAME) === "value",+ );+ const tensor = value && fields(value).find((a) => a.number === ATTR_T);+ const output = strings(f.body, NODE_OUTPUT)[0];+ if (tensor && output) dims.set(output, tensorDims(tensor.body));+ continue;+ }+ const groupAttr = attributes.find(+ (a) => firstString(a, ATTR_NAME) === "group",+ );+ const groupField =+ groupAttr && fields(groupAttr).find((a) => a.number === ATTR_I);+ const group = groupField ? (varintValues(groupField)[0] ?? 1) : 1;+ const weight = strings(f.body, NODE_INPUT)[1];+ if (group > 1 && weight) convs.push({ weight, group });+ }+ }+ return convs.some(({ weight }) => {+ const perGroup = dims.get(weight)?.[1];+ return perGroup !== undefined && perGroup > 1 && perGroup % 4 !== 0;+ });+}diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.tsindex e7d9b1eb..03eb50b6 100644--- a/packages/vision/src/models.e2e.test.ts+++ b/packages/vision/src/models.e2e.test.ts@@ -15,6 +15,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";import { imageToTensor, preprocessSpec } from "./decode/preprocess.js";import { decodeRtmdetIns } from "./decode/rtmdet-ins.js";import { FrameTensorPipeline } from "./gpu/frame-tensor.js";+import { needsNchwLayout } from "./model/conv-layout.js";import { VisionModelStore } from "./model/model-store.js";import {COCO_CLASSES,@@ -73,6 +74,13 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {await session.release();}, 600_000);+ it("runs only RTMO in NCHW (its grouped convs break the NHWC kernels)", async () => {+ const store = new VisionModelStore();+ expect(needsNchwLayout(await store.ensure("rtmo-s"))).toBe(true);+ expect(needsNchwLayout(await store.ensure("rtmdet-ins-s"))).toBe(false);+ expect(needsNchwLayout(await store.ensure("selfie-landscape"))).toBe(false);+ }, 600_000);+describe("GPU frames", () => {let device: GPUDevice;// Held for the test's lifetime: Dawn crashes if its instance is collecteddiff --git a/packages/vision/src/runtime/session-provider.test.ts b/packages/vision/src/runtime/session-provider.test.tsindex ebfa282c..88363b0d 100644--- a/packages/vision/src/runtime/session-provider.test.ts+++ b/packages/vision/src/runtime/session-provider.test.ts@@ -1,10 +1,13 @@import { afterEach, describe, expect, it, vi } from "vitest";import {+ AutoSessionProvider,availableCpus,cpuSessionThreads,createDefaultSessionProvider,requestedVisionBackend,setDefaultSessionProvider,+ VISION_BACKEND_LISTENER,+ type VisionBackendInfo,} from "./session-provider.js";afterEach(() => {@@ -21,6 +24,29 @@ describe("session provider selection", () => {expect(requestedVisionBackend()).toBe("auto");});+ it("reports the backend it picks to a registered listener", async () => {+ const reports: VisionBackendInfo[] = [];+ const g = globalThis as Record<symbol, unknown>;+ g[VISION_BACKEND_LISTENER] = (info: VisionBackendInfo) =>+ reports.push(info);+ try {+ vi.stubEnv("FRAMEFIELDS_VISION_PROVIDER", "cpu");+ // Not a model: the session fails, but the backend is decided first.+ await expect(+ new AutoSessionProvider().createSession(new Uint8Array([1, 2, 3])),+ ).rejects.toThrow();+ expect(reports).toEqual([+ {+ backend: "cpu",+ sharedDevice: false,+ reason: "FRAMEFIELDS_VISION_PROVIDER=cpu",+ },+ ]);+ } finally {+ delete g[VISION_BACKEND_LISTENER];+ }+ });+it("shares one default provider per process", () => {expect(createDefaultSessionProvider()).toBe(createDefaultSessionProvider());});diff --git a/packages/vision/src/runtime/session-provider.ts b/packages/vision/src/runtime/session-provider.tsindex 84bde92a..e6aee9f9 100644--- a/packages/vision/src/runtime/session-provider.ts+++ b/packages/vision/src/runtime/session-provider.ts@@ -59,6 +59,34 @@ export function setDefaultSessionProvider(defaultProvider = factory;}+/** Which backend the process's default provider chose, reported once. */+export interface VisionBackendInfo {+ readonly backend: "webgpu" | "cpu" | "cuda";+ /** WebGPU on the renderer's device: frames reach the models without leaving the GPU. */+ readonly sharedDevice: boolean;+ /** Why WebGPU wasn't used, when it fell back. */+ readonly reason?: string;+}++/**+ * Hosts (the renderer) register a listener under this symbol to log the+ * backend; a global keeps it working however the packages are bundled.+ */+export const VISION_BACKEND_LISTENER = Symbol.for("framefields.vision.backend");++function reportBackend(info: VisionBackendInfo): void {+ const listener = (globalThis as Record<symbol, unknown>)[+ VISION_BACKEND_LISTENER+ ];+ if (typeof listener === "function") {+ try {+ (listener as (info: VisionBackendInfo) => void)(info);+ } catch {+ // Logging must never break inference.+ }+ }+}+/** Inference backend: `FRAMEFIELDS_VISION_PROVIDER`, default "auto". */export type VisionBackend = "auto" | "webgpu" | "cuda" | "cpu";@@ -118,8 +146,16 @@ export class AutoSessionProvider implements SessionProvider {modelBytes: Uint8Array,): Promise<{ provider: SessionProvider; first?: VisionSession }> {const backend = requestedVisionBackend();- if (backend === "cpu") return { provider: new NodeSessionProvider() };+ if (backend === "cpu") {+ reportBackend({+ backend: "cpu",+ sharedDevice: false,+ reason: "FRAMEFIELDS_VISION_PROVIDER=cpu",+ });+ return { provider: new NodeSessionProvider() };+ }if (backend === "cuda") {+ reportBackend({ backend: "cuda", sharedDevice: false });return {provider: new NodeSessionProvider({executionProviders: ["cuda", "cpu"],@@ -131,13 +167,17 @@ export class AutoSessionProvider implements SessionProvider {device: visionDevice,});try {- return {- provider: webgpu,- first: await webgpu.createSession(modelBytes),- };+ const first = await webgpu.createSession(modelBytes);+ reportBackend({ backend: "webgpu", sharedDevice: !!visionDevice });+ return { provider: webgpu, first };} catch (err) {if (backend === "webgpu") throw err;// WebGPU isn't possible here — use CPU from now on.+ reportBackend({+ backend: "cpu",+ sharedDevice: false,+ reason: err instanceof Error ? err.message : String(err),+ });return { provider: new NodeSessionProvider() };}}diff --git a/packages/vision/src/runtime/webgpu-provider.ts b/packages/vision/src/runtime/webgpu-provider.tsindex ffd415c3..2c0cdd12 100644--- a/packages/vision/src/runtime/webgpu-provider.ts+++ b/packages/vision/src/runtime/webgpu-provider.ts@@ -10,6 +10,7 @@* provide it; the main `@framefields/vision` entry never touches it.*/+import { needsNchwLayout } from "../model/conv-layout.js";import type {SessionProvider,VisionSession,@@ -118,9 +119,10 @@ export class WebGPUProvider implements SessionProvider {let session: OrtWebSession;try {const device = this.device;+ const layout = needsNchwLayout(modelBytes) ? "NCHW" : "NHWC";session = await ort.InferenceSession.create(modelBytes, {executionProviders: this._providers.map((name) =>- withProviderOptions(name, device),+ withProviderOptions(name, device, layout),),graphOptimizationLevel: "all",// Errors only: node-placement notes print as console errors otherwise.@@ -189,9 +191,8 @@ export function hasWebGPU(): boolean {function withProviderOptions(name: string,device: GPUDevice | undefined,+ preferredLayout: "NCHW" | "NHWC",): string | OrtWebExecutionProvider {if (name !== "webgpu") return name;- return device- ? { name, preferredLayout: "NCHW", device }- : { name, preferredLayout: "NCHW" };+ return device ? { name, preferredLayout, device } : { name, preferredLayout };}