Code-first video & image, rendered natively on WebGPU.
Merge branch 'perf/vision-gpu-pipeline' into feat/renderer-self-deploy
okanasl9a19b681c087
diff --cc packages/vision/src/runtime/webgpu-provider.test.tsindex 6520d54a,fd209a76..6505b8f8--- a/packages/vision/src/runtime/webgpu-provider.test.ts+++ b/packages/vision/src/runtime/webgpu-provider.test.ts@@@ -17,18 -17,21 +17,26 @@@ class FakeOrtSession implements OrtWebS}function fakeModule(opts: { failWebGPU?: boolean } = {}) {- const calls: { executionProviders: string[]; raw: unknown[] }[] = [];- const calls: { executionProviders: string[]; devices: unknown[] }[] = [];++ const calls: {++ executionProviders: string[];++ raw: unknown[];++ devices: unknown[];++ }[] = [];+ const fromGpuBuffer: unknown[] = [];+ const name = (ep: string | { name: string }) =>+ typeof ep === "string" ? ep : ep.name;const module: OrtWebModule = {InferenceSession: {async create(_bytes, options) {- const names = options.executionProviders.map((p) =>- typeof p === "string" ? p : p.name,- );+ const providers = options.executionProviders.map(name);calls.push({- executionProviders: names,+ executionProviders: providers,+ raw: [...options.executionProviders],+ devices: options.executionProviders.map((p) =>+ typeof p === "string" ? undefined : p.device,+ ),});- if (opts.failWebGPU && names.includes("webgpu")) {+ if (opts.failWebGPU && providers.includes("webgpu")) {throw new Error("webgpu unavailable");}return new FakeOrtSession();diff --cc packages/vision/src/runtime/webgpu-provider.tsindex aa8b9ef8,29e66706..ffd415c3--- a/packages/vision/src/runtime/webgpu-provider.ts+++ b/packages/vision/src/runtime/webgpu-provider.ts@@@ -17,12 -17,6 +17,14 @@@ import typeVisionTensorOutputs,} from "./session-provider.js";+/** An execution provider with options, e.g. `{ name: "webgpu", preferredLayout: "NCHW" }`. */+export interface OrtWebExecutionProvider {+ readonly name: string;+ readonly preferredLayout?: "NCHW" | "NHWC";++ /** WebGPU: run on this device instead of one onnxruntime-web creates. */++ readonly device?: GPUDevice;+}+/** Minimal structural view of the `onnxruntime-web` module — isolates the ORT API surface. */export interface OrtWebTensor {readonly data: Float32Array | BigInt64Array | Int32Array;@@@ -103,8 -109,11 +117,11 @@@ export class WebGPUProvider implementslet session: OrtWebSession;try {+ const device = this.device;session = await ort.InferenceSession.create(modelBytes, {- executionProviders: this._providers.map(withProviderOptions),+ executionProviders: this._providers.map((name) =>- name === "webgpu" && device ? { name, device } : name,++ withProviderOptions(name, device),+ ),graphOptimizationLevel: "all",// Errors only: node-placement notes print as console errors otherwise.logSeverityLevel: 3,@@@ -157,13 -171,3 +179,19 @@@ export function createWebGPUProviderexport function hasWebGPU(): boolean {return typeof navigator !== "undefined" && "gpu" in navigator;}++/**+ * The WebGPU EP runs in NCHW. Its default NHWC layout computes grouped convolutions wrong+ * when the channels per group aren't a multiple of 4 (onnxruntime-web 1.30) — RTMO's head+ * has group=8 convs over 104 channels (13 per group), which turned one dancer into several+ * skeletons. NCHW matches the CPU EP on every pinned model.+ */- function withProviderOptions(name: string): string | OrtWebExecutionProvider {- return name === "webgpu" ? { name, preferredLayout: "NCHW" } : name;++function withProviderOptions(++ name: string,++ device: GPUDevice | undefined,++): string | OrtWebExecutionProvider {++ if (name !== "webgpu") return name;++ return device++ ? { name, preferredLayout: "NCHW", device }++ : { name, preferredLayout: "NCHW" };+}