Skip to content

Code-first video & image, rendered natively on WebGPU.

Merge branch 'perf/vision-gpu-pipeline' into feat/renderer-self-deploy

okanasl9a19b681c087
diff --cc packages/vision/src/runtime/webgpu-provider.test.ts
index 6520d54a,fd209a76..6505b8f8
--- a/packages/vision/src/runtime/webgpu-provider.test.ts
+++ b/packages/vision/src/runtime/webgpu-provider.test.ts
@@@ -17,18 -17,21 +17,26 @@@ class FakeOrtSession implements OrtWebS
}
function fakeModule(opts: { failWebGPU?: boolean } = {}) {
- const calls: { executionProviders: string[]; raw: unknown[] }[] = [];
- const calls: { executionProviders: string[]; devices: unknown[] }[] = [];
++ const calls: {
++ executionProviders: string[];
++ raw: unknown[];
++ devices: unknown[];
++ }[] = [];
+ const fromGpuBuffer: unknown[] = [];
+ const name = (ep: string | { name: string }) =>
+ typeof ep === "string" ? ep : ep.name;
const module: OrtWebModule = {
InferenceSession: {
async create(_bytes, options) {
- const names = options.executionProviders.map((p) =>
- typeof p === "string" ? p : p.name,
- );
+ const providers = options.executionProviders.map(name);
calls.push({
- executionProviders: names,
+ executionProviders: providers,
+ raw: [...options.executionProviders],
+ devices: options.executionProviders.map((p) =>
+ typeof p === "string" ? undefined : p.device,
+ ),
});
- if (opts.failWebGPU && names.includes("webgpu")) {
+ if (opts.failWebGPU && providers.includes("webgpu")) {
throw new Error("webgpu unavailable");
}
return new FakeOrtSession();
diff --cc packages/vision/src/runtime/webgpu-provider.ts
index aa8b9ef8,29e66706..ffd415c3
--- a/packages/vision/src/runtime/webgpu-provider.ts
+++ b/packages/vision/src/runtime/webgpu-provider.ts
@@@ -17,12 -17,6 +17,14 @@@ import type
VisionTensorOutputs,
} from "./session-provider.js";
+/** An execution provider with options, e.g. `{ name: "webgpu", preferredLayout: "NCHW" }`. */
+export interface OrtWebExecutionProvider {
+ readonly name: string;
+ readonly preferredLayout?: "NCHW" | "NHWC";
++ /** WebGPU: run on this device instead of one onnxruntime-web creates. */
++ readonly device?: GPUDevice;
+}
+
/** Minimal structural view of the `onnxruntime-web` module — isolates the ORT API surface. */
export interface OrtWebTensor {
readonly data: Float32Array | BigInt64Array | Int32Array;
@@@ -103,8 -109,11 +117,11 @@@ export class WebGPUProvider implements
let session: OrtWebSession;
try {
+ const device = this.device;
session = await ort.InferenceSession.create(modelBytes, {
- executionProviders: this._providers.map(withProviderOptions),
+ executionProviders: this._providers.map((name) =>
- name === "webgpu" && device ? { name, device } : name,
++ withProviderOptions(name, device),
+ ),
graphOptimizationLevel: "all",
// Errors only: node-placement notes print as console errors otherwise.
logSeverityLevel: 3,
@@@ -157,13 -171,3 +179,19 @@@ export function createWebGPUProvider
export function hasWebGPU(): boolean {
return typeof navigator !== "undefined" && "gpu" in navigator;
}
+
+/**
+ * The WebGPU EP runs in NCHW. Its default NHWC layout computes grouped convolutions wrong
+ * when the channels per group aren't a multiple of 4 (onnxruntime-web 1.30) — RTMO's head
+ * has group=8 convs over 104 channels (13 per group), which turned one dancer into several
+ * skeletons. NCHW matches the CPU EP on every pinned model.
+ */
- function withProviderOptions(name: string): string | OrtWebExecutionProvider {
- return name === "webgpu" ? { name, preferredLayout: "NCHW" } : name;
++function withProviderOptions(
++ name: string,
++ device: GPUDevice | undefined,
++): string | OrtWebExecutionProvider {
++ if (name !== "webgpu") return name;
++ return device
++ ? { name, preferredLayout: "NCHW", device }
++ : { name, preferredLayout: "NCHW" };
+}