Skip to content

Code-first video & image, rendered natively on WebGPU.

feat(vision): log the inference backend; NCHW only where it's needed

okanaslf847dd1c03cd
diff --git a/.changeset/vision-webgpu-nchw.md b/.changeset/vision-webgpu-nchw.md
index 1e3999d1..c84ec0f7 100644
--- a/.changeset/vision-webgpu-nchw.md
+++ b/.changeset/vision-webgpu-nchw.md
@@ -2,4 +2,6 @@
"@framefields/vision": patch
---
-Fix duplicate pose skeletons on the GPU path: the WebGPU execution provider now runs in NCHW layout. In its default NHWC layout onnxruntime-web (1.30) computes grouped convolutions wrong when the channels per group aren't a multiple of 4, which corrupts RTMO's head (group-8 convs over 104 channels) and turned one person into several detections. Outputs now match the CPU provider on every pinned model.
+Fix duplicate pose skeletons on the GPU path. In its default NHWC layout, onnxruntime-web (1.30)'s WebGPU execution provider computes grouped convolutions wrong when a group has more than one channel and the count isn't a multiple of 4. That corrupts RTMO's head (18 channels per group) and turned one person into several detections. Models with such a convolution (RTMO) now run in NCHW layout; the others (RTMDet-Ins, Selfie, whose grouped convolutions are depthwise) keep NHWC, which is about 30% faster. Outputs match the CPU provider on every pinned model.
+
+Vision now reports its inference backend once per process, and the renderer logs it: `[vision] Inference on the GPU (WebGPU)` with `sharedDevice`, or `[vision] Inference on the CPU` with the reason.
diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.ts
index 54aeabb1..dc304891 100644
--- a/packages/renderer/src/headless-webgpu-renderer.ts
+++ b/packages/renderer/src/headless-webgpu-renderer.ts
@@ -50,6 +50,20 @@ import { preloadFonts } from "./asset-preloader.js";
import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";
import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";
import { rendererLogger } from "./logger.js";
+
+// Vision reports its inference backend once per process (see
+// VISION_BACKEND_LISTENER in @framefields/vision).
+(globalThis as Record<symbol, unknown>)[
+ Symbol.for("framefields.vision.backend")
+] = (info: { backend: string; sharedDevice: boolean; reason?: string }) => {
+ rendererLogger.info(
+ info,
+ info.backend === "webgpu"
+ ? "[vision] Inference on the GPU (WebGPU)"
+ : `[vision] Inference on the ${info.backend.toUpperCase()}`,
+ );
+};
+
import {
type AudioQaStats,
analyzeAudio,
diff --git a/packages/vision/src/model/conv-layout.ts b/packages/vision/src/model/conv-layout.ts
new file mode 100644
index 00000000..8f111e7b
--- /dev/null
+++ b/packages/vision/src/model/conv-layout.ts
@@ -0,0 +1,108 @@
+/**
+ * onnxruntime-web 1.30's WebGPU EP computes grouped convolutions wrong in its
+ * default NHWC layout when a group has more than one channel and the count
+ * isn't a multiple of 4 (RTMO's head: 18 per group → duplicate skeletons).
+ * NCHW is correct but ~30% slower, so only models with such a convolution get
+ * it. Depthwise convolutions (one channel per group: RTMDet-Ins, Selfie) are
+ * fine in NHWC.
+ */
+
+import {
+ fields,
+ firstString,
+ GRAPH_INITIALIZER,
+ GRAPH_NODE,
+ MODEL_GRAPH,
+ NODE_INPUT,
+ readVarint,
+ strings,
+ TENSOR_NAME,
+ WIRE_LEN,
+ WIRE_VARINT,
+} from "./onnx-proto.js";
+
+const NODE_OP_TYPE = 4;
+const NODE_ATTRIBUTE = 5;
+const NODE_OUTPUT = 2;
+const ATTR_NAME = 1;
+const ATTR_I = 3;
+const ATTR_T = 5;
+const TENSOR_DIMS = 1;
+
+function varintValues(field: {
+ wire: number;
+ raw: Uint8Array;
+ body: Uint8Array;
+}): number[] {
+ const values: number[] = [];
+ if (field.wire === WIRE_LEN) {
+ let pos = 0;
+ while (pos < field.body.length) {
+ const [value, next] = readVarint(field.body, pos);
+ values.push(value);
+ pos = next;
+ }
+ } else if (field.wire === WIRE_VARINT) {
+ let pos = 0;
+ pos = readVarint(field.raw, pos)[1]; // tag
+ values.push(readVarint(field.raw, pos)[0]);
+ }
+ return values;
+}
+
+function tensorDims(tensor: Uint8Array): number[] {
+ return fields(tensor)
+ .filter((f) => f.number === TENSOR_DIMS)
+ .flatMap(varintValues);
+}
+
+/**
+ * True when the model has a grouped convolution the NHWC kernels get wrong,
+ * and when it can't be read (NCHW is always correct).
+ */
+export function needsNchwLayout(model: Uint8Array): boolean {
+ let graph: ReturnType<typeof fields>[number] | undefined;
+ try {
+ graph = fields(model).find(
+ (f) => f.number === MODEL_GRAPH && f.wire === WIRE_LEN,
+ );
+ } catch {
+ return true;
+ }
+ if (!graph) return true;
+ const dims = new Map<string, number[]>();
+ const convs: { weight: string; group: number }[] = [];
+ for (const f of fields(graph.body)) {
+ if (f.number === GRAPH_INITIALIZER) {
+ const name = firstString(f.body, TENSOR_NAME);
+ if (name !== undefined) dims.set(name, tensorDims(f.body));
+ } else if (f.number === GRAPH_NODE) {
+ const op = firstString(f.body, NODE_OP_TYPE);
+ if (op !== "Conv" && op !== "Constant") continue;
+ const attributes = fields(f.body)
+ .filter((a) => a.number === NODE_ATTRIBUTE)
+ .map((a) => a.body);
+ if (op === "Constant") {
+ const value = attributes.find(
+ (a) => firstString(a, ATTR_NAME) === "value",
+ );
+ const tensor = value && fields(value).find((a) => a.number === ATTR_T);
+ const output = strings(f.body, NODE_OUTPUT)[0];
+ if (tensor && output) dims.set(output, tensorDims(tensor.body));
+ continue;
+ }
+ const groupAttr = attributes.find(
+ (a) => firstString(a, ATTR_NAME) === "group",
+ );
+ const groupField =
+ groupAttr && fields(groupAttr).find((a) => a.number === ATTR_I);
+ const group = groupField ? (varintValues(groupField)[0] ?? 1) : 1;
+ const weight = strings(f.body, NODE_INPUT)[1];
+ if (group > 1 && weight) convs.push({ weight, group });
+ }
+ }
+ return convs.some(({ weight }) => {
+ const perGroup = dims.get(weight)?.[1];
+ return perGroup !== undefined && perGroup > 1 && perGroup % 4 !== 0;
+ });
+}
diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.ts
index e7d9b1eb..03eb50b6 100644
--- a/packages/vision/src/models.e2e.test.ts
+++ b/packages/vision/src/models.e2e.test.ts
@@ -15,6 +15,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { imageToTensor, preprocessSpec } from "./decode/preprocess.js";
import { decodeRtmdetIns } from "./decode/rtmdet-ins.js";
import { FrameTensorPipeline } from "./gpu/frame-tensor.js";
+import { needsNchwLayout } from "./model/conv-layout.js";
import { VisionModelStore } from "./model/model-store.js";
import {
COCO_CLASSES,
@@ -73,6 +74,13 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {
await session.release();
}, 600_000);
+ it("runs only RTMO in NCHW (its grouped convs break the NHWC kernels)", async () => {
+ const store = new VisionModelStore();
+ expect(needsNchwLayout(await store.ensure("rtmo-s"))).toBe(true);
+ expect(needsNchwLayout(await store.ensure("rtmdet-ins-s"))).toBe(false);
+ expect(needsNchwLayout(await store.ensure("selfie-landscape"))).toBe(false);
+ }, 600_000);
+
describe("GPU frames", () => {
let device: GPUDevice;
// Held for the test's lifetime: Dawn crashes if its instance is collected
diff --git a/packages/vision/src/runtime/session-provider.test.ts b/packages/vision/src/runtime/session-provider.test.ts
index ebfa282c..88363b0d 100644
--- a/packages/vision/src/runtime/session-provider.test.ts
+++ b/packages/vision/src/runtime/session-provider.test.ts
@@ -1,10 +1,13 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
+ AutoSessionProvider,
availableCpus,
cpuSessionThreads,
createDefaultSessionProvider,
requestedVisionBackend,
setDefaultSessionProvider,
+ VISION_BACKEND_LISTENER,
+ type VisionBackendInfo,
} from "./session-provider.js";
afterEach(() => {
@@ -21,6 +24,29 @@ describe("session provider selection", () => {
expect(requestedVisionBackend()).toBe("auto");
});
+ it("reports the backend it picks to a registered listener", async () => {
+ const reports: VisionBackendInfo[] = [];
+ const g = globalThis as Record<symbol, unknown>;
+ g[VISION_BACKEND_LISTENER] = (info: VisionBackendInfo) =>
+ reports.push(info);
+ try {
+ vi.stubEnv("FRAMEFIELDS_VISION_PROVIDER", "cpu");
+ // Not a model: the session fails, but the backend is decided first.
+ await expect(
+ new AutoSessionProvider().createSession(new Uint8Array([1, 2, 3])),
+ ).rejects.toThrow();
+ expect(reports).toEqual([
+ {
+ backend: "cpu",
+ sharedDevice: false,
+ reason: "FRAMEFIELDS_VISION_PROVIDER=cpu",
+ },
+ ]);
+ } finally {
+ delete g[VISION_BACKEND_LISTENER];
+ }
+ });
+
it("shares one default provider per process", () => {
expect(createDefaultSessionProvider()).toBe(createDefaultSessionProvider());
});
diff --git a/packages/vision/src/runtime/session-provider.ts b/packages/vision/src/runtime/session-provider.ts
index 84bde92a..e6aee9f9 100644
--- a/packages/vision/src/runtime/session-provider.ts
+++ b/packages/vision/src/runtime/session-provider.ts
@@ -59,6 +59,34 @@ export function setDefaultSessionProvider(
defaultProvider = factory;
}
+/** Which backend the process's default provider chose, reported once. */
+export interface VisionBackendInfo {
+ readonly backend: "webgpu" | "cpu" | "cuda";
+ /** WebGPU on the renderer's device: frames reach the models without leaving the GPU. */
+ readonly sharedDevice: boolean;
+ /** Why WebGPU wasn't used, when it fell back. */
+ readonly reason?: string;
+}
+
+/**
+ * Hosts (the renderer) register a listener under this symbol to log the
+ * backend; a global keeps it working however the packages are bundled.
+ */
+export const VISION_BACKEND_LISTENER = Symbol.for("framefields.vision.backend");
+
+function reportBackend(info: VisionBackendInfo): void {
+ const listener = (globalThis as Record<symbol, unknown>)[
+ VISION_BACKEND_LISTENER
+ ];
+ if (typeof listener === "function") {
+ try {
+ (listener as (info: VisionBackendInfo) => void)(info);
+ } catch {
+ // Logging must never break inference.
+ }
+ }
+}
+
/** Inference backend: `FRAMEFIELDS_VISION_PROVIDER`, default "auto". */
export type VisionBackend = "auto" | "webgpu" | "cuda" | "cpu";
@@ -118,8 +146,16 @@ export class AutoSessionProvider implements SessionProvider {
modelBytes: Uint8Array,
): Promise<{ provider: SessionProvider; first?: VisionSession }> {
const backend = requestedVisionBackend();
- if (backend === "cpu") return { provider: new NodeSessionProvider() };
+ if (backend === "cpu") {
+ reportBackend({
+ backend: "cpu",
+ sharedDevice: false,
+ reason: "FRAMEFIELDS_VISION_PROVIDER=cpu",
+ });
+ return { provider: new NodeSessionProvider() };
+ }
if (backend === "cuda") {
+ reportBackend({ backend: "cuda", sharedDevice: false });
return {
provider: new NodeSessionProvider({
executionProviders: ["cuda", "cpu"],
@@ -131,13 +167,17 @@ export class AutoSessionProvider implements SessionProvider {
device: visionDevice,
});
try {
- return {
- provider: webgpu,
- first: await webgpu.createSession(modelBytes),
- };
+ const first = await webgpu.createSession(modelBytes);
+ reportBackend({ backend: "webgpu", sharedDevice: !!visionDevice });
+ return { provider: webgpu, first };
} catch (err) {
if (backend === "webgpu") throw err;
// WebGPU isn't possible here — use CPU from now on.
+ reportBackend({
+ backend: "cpu",
+ sharedDevice: false,
+ reason: err instanceof Error ? err.message : String(err),
+ });
return { provider: new NodeSessionProvider() };
}
}
diff --git a/packages/vision/src/runtime/webgpu-provider.ts b/packages/vision/src/runtime/webgpu-provider.ts
index ffd415c3..2c0cdd12 100644
--- a/packages/vision/src/runtime/webgpu-provider.ts
+++ b/packages/vision/src/runtime/webgpu-provider.ts
@@ -10,6 +10,7 @@
* provide it; the main `@framefields/vision` entry never touches it.
*/
+import { needsNchwLayout } from "../model/conv-layout.js";
import type {
SessionProvider,
VisionSession,
@@ -118,9 +119,10 @@ export class WebGPUProvider implements SessionProvider {
let session: OrtWebSession;
try {
const device = this.device;
+ const layout = needsNchwLayout(modelBytes) ? "NCHW" : "NHWC";
session = await ort.InferenceSession.create(modelBytes, {
executionProviders: this._providers.map((name) =>
- withProviderOptions(name, device),
+ withProviderOptions(name, device, layout),
),
graphOptimizationLevel: "all",
// Errors only: node-placement notes print as console errors otherwise.
@@ -189,9 +191,8 @@ export function hasWebGPU(): boolean {
function withProviderOptions(
name: string,
device: GPUDevice | undefined,
+ preferredLayout: "NCHW" | "NHWC",
): string | OrtWebExecutionProvider {
if (name !== "webgpu") return name;
- return device
- ? { name, preferredLayout: "NCHW", device }
- : { name, preferredLayout: "NCHW" };
+ return device ? { name, preferredLayout, device } : { name, preferredLayout };
}