Skip to content

Code-first video & image, rendered natively on WebGPU.

fix(vision): run the WebGPU EP in NCHW layout

okanasl34f497a2647b
diff --git a/.changeset/vision-webgpu-nchw.md b/.changeset/vision-webgpu-nchw.md
new file mode 100644
index 00000000..1e3999d1
--- /dev/null
+++ b/.changeset/vision-webgpu-nchw.md
@@ -0,0 +1,5 @@
+---
+"@framefields/vision": patch
+---
+
+Fix duplicate pose skeletons on the GPU path: the WebGPU execution provider now runs in NCHW layout. In its default NHWC layout onnxruntime-web (1.30) computes grouped convolutions wrong when the channels per group aren't a multiple of 4, which corrupts RTMO's head (group-8 convs over 104 channels) and turned one person into several detections. Outputs now match the CPU provider on every pinned model.
diff --git a/packages/vision/src/runtime/webgpu-provider.test.ts b/packages/vision/src/runtime/webgpu-provider.test.ts
index 7a4620a2..6520d54a 100644
--- a/packages/vision/src/runtime/webgpu-provider.test.ts
+++ b/packages/vision/src/runtime/webgpu-provider.test.ts
@@ -17,12 +17,18 @@ class FakeOrtSession implements OrtWebSession {
}
function fakeModule(opts: { failWebGPU?: boolean } = {}) {
- const calls: { executionProviders: string[] }[] = [];
+ const calls: { executionProviders: string[]; raw: unknown[] }[] = [];
+ const name = (ep: string | { name: string }) =>
+ typeof ep === "string" ? ep : ep.name;
const module: OrtWebModule = {
InferenceSession: {
async create(_bytes, options) {
- calls.push({ executionProviders: [...options.executionProviders] });
- if (opts.failWebGPU && options.executionProviders.includes("webgpu")) {
+ const providers = options.executionProviders.map(name);
+ calls.push({
+ executionProviders: providers,
+ raw: [...options.executionProviders],
+ });
+ if (opts.failWebGPU && providers.includes("webgpu")) {
throw new Error("webgpu unavailable");
}
return new FakeOrtSession();
@@ -60,6 +66,11 @@ describe("WebGPUProvider", () => {
expect(outputs.output0.dims).toEqual([1, 1, 1]);
expect(outputs.output0.data[0]).toBeCloseTo(0.5, 6);
expect(calls[0].executionProviders).toEqual(["webgpu", "wasm"]);
+ // NHWC miscomputes grouped convs with channels/group not a multiple of 4 (RTMO)
+ expect(calls[0].raw[0]).toEqual({
+ name: "webgpu",
+ preferredLayout: "NCHW",
+ });
// loader is memoized across sessions
await provider.createSession(new Uint8Array([1]));
diff --git a/packages/vision/src/runtime/webgpu-provider.ts b/packages/vision/src/runtime/webgpu-provider.ts
index 62b6a2ad..aa8b9ef8 100644
--- a/packages/vision/src/runtime/webgpu-provider.ts
+++ b/packages/vision/src/runtime/webgpu-provider.ts
@@ -17,6 +17,12 @@ import type {
VisionTensorOutputs,
} from "./session-provider.js";
+/** An execution provider with options, e.g. `{ name: "webgpu", preferredLayout: "NCHW" }`. */
+export interface OrtWebExecutionProvider {
+ readonly name: string;
+ readonly preferredLayout?: "NCHW" | "NHWC";
+}
+
/** Minimal structural view of the `onnxruntime-web` module — isolates the ORT API surface. */
export interface OrtWebTensor {
readonly data: Float32Array | BigInt64Array | Int32Array;
@@ -34,7 +40,7 @@ export interface OrtWebModule {
create(
modelBytes: Uint8Array,
options: {
- executionProviders: string[];
+ executionProviders: (string | OrtWebExecutionProvider)[];
graphOptimizationLevel?: "disabled" | "basic" | "extended" | "all";
/** 0 verbose … 4 fatal. */
logSeverityLevel?: 0 | 1 | 2 | 3 | 4;
@@ -98,7 +104,7 @@ export class WebGPUProvider implements SessionProvider {
let session: OrtWebSession;
try {
session = await ort.InferenceSession.create(modelBytes, {
- executionProviders: [...this._providers],
+ executionProviders: this._providers.map(withProviderOptions),
graphOptimizationLevel: "all",
// Errors only: node-placement notes print as console errors otherwise.
logSeverityLevel: 3,
@@ -151,3 +157,13 @@ export function createWebGPUProvider(
export function hasWebGPU(): boolean {
return typeof navigator !== "undefined" && "gpu" in navigator;
}
+
+/**
+ * The WebGPU EP runs in NCHW. Its default NHWC layout computes grouped convolutions wrong
+ * when the channels per group aren't a multiple of 4 (onnxruntime-web 1.30) — RTMO's head
+ * has group=8 convs over 104 channels (13 per group), which turned one dancer into several
+ * skeletons. NCHW matches the CPU EP on every pinned model.
+ */
+function withProviderOptions(name: string): string | OrtWebExecutionProvider {
+ return name === "webgpu" ? { name, preferredLayout: "NCHW" } : name;
+}