Code-first video & image, rendered natively on WebGPU.
fix(vision): run the WebGPU EP in NCHW layout
okanasl34f497a2647b
diff --git a/.changeset/vision-webgpu-nchw.md b/.changeset/vision-webgpu-nchw.mdnew file mode 100644index 00000000..1e3999d1--- /dev/null+++ b/.changeset/vision-webgpu-nchw.md@@ -0,0 +1,5 @@+---+"@framefields/vision": patch+---++Fix duplicate pose skeletons on the GPU path: the WebGPU execution provider now runs in NCHW layout. In its default NHWC layout onnxruntime-web (1.30) computes grouped convolutions wrong when the channels per group aren't a multiple of 4, which corrupts RTMO's head (group-8 convs over 104 channels) and turned one person into several detections. Outputs now match the CPU provider on every pinned model.diff --git a/packages/vision/src/runtime/webgpu-provider.test.ts b/packages/vision/src/runtime/webgpu-provider.test.tsindex 7a4620a2..6520d54a 100644--- a/packages/vision/src/runtime/webgpu-provider.test.ts+++ b/packages/vision/src/runtime/webgpu-provider.test.ts@@ -17,12 +17,18 @@ class FakeOrtSession implements OrtWebSession {}function fakeModule(opts: { failWebGPU?: boolean } = {}) {- const calls: { executionProviders: string[] }[] = [];+ const calls: { executionProviders: string[]; raw: unknown[] }[] = [];+ const name = (ep: string | { name: string }) =>+ typeof ep === "string" ? ep : ep.name;const module: OrtWebModule = {InferenceSession: {async create(_bytes, options) {- calls.push({ executionProviders: [...options.executionProviders] });- if (opts.failWebGPU && options.executionProviders.includes("webgpu")) {+ const providers = options.executionProviders.map(name);+ calls.push({+ executionProviders: providers,+ raw: [...options.executionProviders],+ });+ if (opts.failWebGPU && providers.includes("webgpu")) {throw new Error("webgpu unavailable");}return new FakeOrtSession();@@ -60,6 +66,11 @@ describe("WebGPUProvider", () => {expect(outputs.output0.dims).toEqual([1, 1, 1]);expect(outputs.output0.data[0]).toBeCloseTo(0.5, 6);expect(calls[0].executionProviders).toEqual(["webgpu", "wasm"]);+ // NHWC miscomputes grouped convs with channels/group not a multiple of 4 (RTMO)+ expect(calls[0].raw[0]).toEqual({+ name: "webgpu",+ preferredLayout: "NCHW",+ });// loader is memoized across sessionsawait provider.createSession(new Uint8Array([1]));diff --git a/packages/vision/src/runtime/webgpu-provider.ts b/packages/vision/src/runtime/webgpu-provider.tsindex 62b6a2ad..aa8b9ef8 100644--- a/packages/vision/src/runtime/webgpu-provider.ts+++ b/packages/vision/src/runtime/webgpu-provider.ts@@ -17,6 +17,12 @@ import type {VisionTensorOutputs,} from "./session-provider.js";+/** An execution provider with options, e.g. `{ name: "webgpu", preferredLayout: "NCHW" }`. */+export interface OrtWebExecutionProvider {+ readonly name: string;+ readonly preferredLayout?: "NCHW" | "NHWC";+}+/** Minimal structural view of the `onnxruntime-web` module — isolates the ORT API surface. */export interface OrtWebTensor {readonly data: Float32Array | BigInt64Array | Int32Array;@@ -34,7 +40,7 @@ export interface OrtWebModule {create(modelBytes: Uint8Array,options: {- executionProviders: string[];+ executionProviders: (string | OrtWebExecutionProvider)[];graphOptimizationLevel?: "disabled" | "basic" | "extended" | "all";/** 0 verbose … 4 fatal. */logSeverityLevel?: 0 | 1 | 2 | 3 | 4;@@ -98,7 +104,7 @@ export class WebGPUProvider implements SessionProvider {let session: OrtWebSession;try {session = await ort.InferenceSession.create(modelBytes, {- executionProviders: [...this._providers],+ executionProviders: this._providers.map(withProviderOptions),graphOptimizationLevel: "all",// Errors only: node-placement notes print as console errors otherwise.logSeverityLevel: 3,@@ -151,3 +157,13 @@ export function createWebGPUProvider(export function hasWebGPU(): boolean {return typeof navigator !== "undefined" && "gpu" in navigator;}++/**+ * The WebGPU EP runs in NCHW. Its default NHWC layout computes grouped convolutions wrong+ * when the channels per group aren't a multiple of 4 (onnxruntime-web 1.30) — RTMO's head+ * has group=8 convs over 104 channels (13 per group), which turned one dancer into several+ * skeletons. NCHW matches the CPU EP on every pinned model.+ */+function withProviderOptions(name: string): string | OrtWebExecutionProvider {+ return name === "webgpu" ? { name, preferredLayout: "NCHW" } : name;+}