Code-first video & image, rendered natively on WebGPU.
perf(vision): frames stay on the GPU
okanasl5c55a7b8348a
diff --git a/.changeset/vision-gpu-frames.md b/.changeset/vision-gpu-frames.mdnew file mode 100644index 00000000..5f8847f2--- /dev/null+++ b/.changeset/vision-gpu-frames.md@@ -0,0 +1,9 @@+---+"framefields": patch+"@framefields/vision": minor+"@framefields/node-vision": patch+---++Vision layers no longer copy frames to the CPU. ONNX Runtime's WebGPU sessions now run on the renderer's own device (`setVisionDevice`), and a compute shader turns the layer's texture straight into each model's input tensor. It does the same resize, letterbox, channel order and normalisation as `imageToTensor`, within float32 rounding. A small GPU checksum replaces hashing the pixels on the CPU, so layers showing the same frame still share a pass. Frames come back to the CPU only for the opt-in background key (`keyBackground`).++`VisionRunner` methods accept a `VisionGpuImage` (`{ texture, device, width, height }`) as well as RGBA pixels. When its sessions run on another device, or on the CPU, the runner reads the texture back itself. `WebGPUProvider` takes a `device` option and feeds `gpuBuffer` inputs with `Tensor.fromGpuBuffer`.diff --git a/nodes/node-vision/src/renderers/webgpu-renderer.ts b/nodes/node-vision/src/renderers/webgpu-renderer.tsindex 1624d6c6..9173dff6 100644--- a/nodes/node-vision/src/renderers/webgpu-renderer.ts+++ b/nodes/node-vision/src/renderers/webgpu-renderer.ts@@ -8,9 +8,12 @@ import {type PersonMatte,type PoseResult,PoseSkeletonRenderer,+ readTexturePixels,+ setVisionDevice,TemporalObjectTracker,type TrackedObject,type VisionBundle,+ type VisionGpuImage,VisionRunner,} from "@framefields/vision";import type { VisionOperation } from "../shared/config.js";@@ -38,8 +41,9 @@ let sharedCompositorFormat: GPUTextureFormat | null = null;const frameCache = new VisionFrameCache();/** The frame each node's child texture holds (drawn on its last call). */const heldFrameByNode = new Map<string, number>();-/** Per node: a copy of the held frame, and the composited matte. */+/** Per node: a copy of the held frame, the frame being analysed, and the composited matte. */const heldTextures = new Map<string, GPUTexture>();+const analysisTextures = new Map<string, GPUTexture>();const matteTextures = new Map<string, GPUTexture>();/** One tracker per vision node — track ids must not mix across nodes/sources. */const objectTrackers = new Map<string, TemporalObjectTracker>();@@ -106,6 +110,18 @@ const heldTexture = (device: GPUDevice, key: string, like: GPUTexture) =>"vision_held_frame",);+const analysisTexture = (device: GPUDevice, key: string, like: GPUTexture) =>+ nodeTexture(+ analysisTextures,+ device,+ key,+ like,+ GPUTextureUsage.COPY_DST |+ GPUTextureUsage.TEXTURE_BINDING |+ GPUTextureUsage.COPY_SRC,+ "vision_analysis_frame",+ );+const matteTexture = (device: GPUDevice, key: string, like: GPUTexture) =>nodeTexture(matteTextures,@@ -392,7 +408,7 @@ function growMaskIntoForeground(mask: Uint8Array,width: number,height: number,- pixels: Uint8ClampedArray,+ pixels: Uint8ClampedArray | Uint8Array,threshold: number,featherRadius?: number,): Uint8Array {@@ -517,6 +533,9 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {} = args;const { virtualMedia } = props;+ // Vision sessions run on the renderer's device, so frames reach the models+ // without leaving the GPU.+ setVisionDevice(ctx.device);const rawOp = virtualMedia?.operation as| VisionOperation| Record<string, unknown>@@ -634,48 +653,31 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {]);}- // 3. Read back the held frame's pixels, only when it still needs inference.- let framePixels: Uint8ClampedArray | null = null;+ // 3. The held frame, when it still needs inference: copied now (submitted+ // before this frame redraws the child) and analysed on the GPU. Pixels+ // come back to the CPU only for the opt-in background key.+ let frame: VisionGpuImage | undefined;+ let framePixels: Uint8ClampedArray | Uint8Array | null = null;if (hasPreviousFrame && !cached) {- const bytesPerPixel = 4;- const unalignedBytesPerRow = targetWidth * bytesPerPixel;- const bytesPerRow = Math.ceil(unalignedBytesPerRow / 256) * 256;- const bufferSize = bytesPerRow * targetHeight;-- const stagingBuffer = ctx.device.createBuffer({- size: bufferSize,- usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,- label: "vision_frame_staging",- });-- const readbackEncoder = ctx.device.createCommandEncoder({- label: "vision_readback_encoder",+ const analysis = analysisTexture(ctx.device, nodeKeyStr, childTex);+ const copy = ctx.device.createCommandEncoder({+ label: "vision_analysis_copy",});- readbackEncoder.copyTextureToBuffer(- { texture: childTex },- { buffer: stagingBuffer, bytesPerRow, rowsPerImage: targetHeight },- [targetWidth, targetHeight, 1],- );- ctx.device.queue.submit([readbackEncoder.finish()]);-- await stagingBuffer.mapAsync(GPUMapMode.READ);- const mappedBytes = new Uint8Array(stagingBuffer.getMappedRange());- framePixels = new Uint8ClampedArray(targetWidth * targetHeight * 4);- if (bytesPerRow === unalignedBytesPerRow) {- framePixels.set(mappedBytes.subarray(0, framePixels.length));- } else {- for (let y = 0; y < targetHeight; y++) {- framePixels.set(- mappedBytes.subarray(- y * bytesPerRow,- y * bytesPerRow + unalignedBytesPerRow,- ),- y * unalignedBytesPerRow,- );- }+ copy.copyTextureToTexture({ texture: childTex }, { texture: analysis }, [+ targetWidth,+ targetHeight,+ 1,+ ]);+ ctx.device.queue.submit([copy.finish()]);+ frame = {+ texture: analysis,+ device: ctx.device,+ width: targetWidth,+ height: targetHeight,+ };+ if (isMatteMode && op.keyBackground === true) {+ framePixels = (await readTexturePixels(frame)).data;}- stagingBuffer.unmap();- stagingBuffer.destroy();}// 4. Draw THIS frame's child into the persistent texture (recorded into the@@ -725,13 +727,9 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {subject = fromCachedMask(cached.subject.mask);lastSubjectByNode.set(nodeKeyStr, { mask: subject, frame: frameIdx });}- } else if (framePixels) {+ } else if (frame) {const runner = getSharedRunner(op);- const image = {- data: framePixels,- width: targetWidth,- height: targetHeight,- };+ const image = frame;// detect + segment share one RTMDet-Ins pass per frame inside the runner.// Tracked objects are kept locally (like masks below) so `boxes` /@@ -791,7 +789,7 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {// Opt-in: grow the subject into connected pixels that stand out from// the backdrop (catches edges the model's soft mask leaves behind).const threshold = keyThresholdAt(heldFrame ?? frameIdx);- if (subject && threshold !== undefined) {+ if (subject && threshold !== undefined && framePixels) {subject = {...subject,mask: growMaskIntoForeground(diff --git a/packages/vision/src/gpu/frame-tensor.ts b/packages/vision/src/gpu/frame-tensor.tsnew file mode 100644index 00000000..b3e0b498--- /dev/null+++ b/packages/vision/src/gpu/frame-tensor.ts@@ -0,0 +1,293 @@+/**+ * Frame preprocessing on the GPU: a texture becomes a model's NCHW float input+ * in a storage buffer, so a frame drawn by the renderer reaches the model+ * without a round trip through the CPU. The math is `imageToTensor`'s+ * (pixel-centre bilinear sampling, letterbox padding, channel order,+ * normalisation); see decode/preprocess.ts for the per-model contracts.+ *+ * Also: a content hash of a texture (two order-independent 32-bit sums, read+ * back as 8 bytes), so layers showing the same pixels can share a pass, and a+ * plain RGBA readback for paths that need pixels on the CPU.+ */++import {+ computeInputTransform,+ type InputTransform,+ type PreprocessSpec,+} from "../decode/preprocess.js";+import type { VisionGpuImage, VisionImageInput } from "../types.js";++const TENSOR_WGSL = /* wgsl */ `+struct Params {+ src: vec2u,+ dst: vec2u,+ scale: vec2f,+ offset: vec2f,+ lo: vec2i,+ hi: vec2i,+ mean: vec4f,+ invStd: vec4f,+ pad: vec4f,+ order: vec4u,+};++@group(0) @binding(0) var<uniform> p: Params;+@group(0) @binding(1) var src: texture_2d<f32>;+@group(0) @binding(2) var<storage, read_write> dst: array<f32>;++@compute @workgroup_size(16, 16)+fn main(@builtin(global_invocation_id) id: vec3u) {+ if (id.x >= p.dst.x || id.y >= p.dst.y) { return; }+ let x = i32(id.x);+ let y = i32(id.y);+ let plane = p.dst.x * p.dst.y;+ let o = id.y * p.dst.x + id.x;+ var v = p.pad.xyz;+ if (x >= p.lo.x && x <= p.hi.x && y >= p.lo.y && y <= p.hi.y) {+ let maxX = i32(p.src.x) - 1;+ let maxY = i32(p.src.y) - 1;+ let sy = max(0.0, (f32(y) + 0.5 - p.offset.y) / p.scale.y - 0.5);+ let iy0 = min(maxY, i32(floor(sy)));+ let iy1 = min(maxY, iy0 + 1);+ let fy = sy - f32(iy0);+ let sx = max(0.0, (f32(x) + 0.5 - p.offset.x) / p.scale.x - 0.5);+ let ix0 = min(maxX, i32(floor(sx)));+ let ix1 = min(maxX, ix0 + 1);+ let fx = sx - f32(ix0);+ var rgba =+ textureLoad(src, vec2i(ix0, iy0), 0) * ((1.0 - fx) * (1.0 - fy)) ++ textureLoad(src, vec2i(ix1, iy0), 0) * (fx * (1.0 - fy)) ++ textureLoad(src, vec2i(ix0, iy1), 0) * ((1.0 - fx) * fy) ++ textureLoad(src, vec2i(ix1, iy1), 0) * (fx * fy);+ rgba = rgba * 255.0;+ let s = vec3f(rgba[p.order.x], rgba[p.order.y], rgba[p.order.z]);+ v = (s - p.mean.xyz) * p.invStd.xyz;+ }+ dst[o] = v.x;+ dst[plane + o] = v.y;+ dst[2u * plane + o] = v.z;+}+`;++const HASH_WGSL = /* wgsl */ `+@group(0) @binding(0) var src: texture_2d<f32>;+@group(0) @binding(1) var<storage, read_write> acc: array<atomic<u32>, 2>;++var<workgroup> local: array<atomic<u32>, 2>;++@compute @workgroup_size(16, 16)+fn main(+ @builtin(global_invocation_id) id: vec3u,+ @builtin(local_invocation_index) li: u32,+) {+ let dim = textureDimensions(src);+ if (id.x < dim.x && id.y < dim.y) {+ let c = pack4x8unorm(textureLoad(src, vec2i(id.xy), 0));+ let i = id.y * dim.x + id.x;+ let a = (c ^ (i * 0x9e3779b9u)) * 0x85ebca6bu;+ let b = (c + i * 0xc2b2ae35u) * 0x27d4eb2fu;+ atomicAdd(&local[0], a ^ (a >> 15u));+ atomicAdd(&local[1], b ^ (b >> 13u));+ }+ workgroupBarrier();+ if (li == 0u) {+ atomicAdd(&acc[0], atomicLoad(&local[0]));+ atomicAdd(&acc[1], atomicLoad(&local[1]));+ }+}+`;++const PARAMS_BYTES = 112;++const pipelines = new WeakMap<GPUDevice, FrameTensorPipeline>();++export class FrameTensorPipeline {+ private readonly tensorPipeline: GPUComputePipeline;+ private readonly hashPipeline: GPUComputePipeline;+ private readonly params: GPUBuffer;++ /** One per device. */+ public static for(device: GPUDevice): FrameTensorPipeline {+ let pipeline = pipelines.get(device);+ if (!pipeline) {+ pipeline = new FrameTensorPipeline(device);+ pipelines.set(device, pipeline);+ }+ return pipeline;+ }++ private constructor(private readonly device: GPUDevice) {+ this.tensorPipeline = device.createComputePipeline({+ label: "vision_frame_tensor",+ layout: "auto",+ compute: {+ module: device.createShaderModule({ code: TENSOR_WGSL }),+ entryPoint: "main",+ },+ });+ this.hashPipeline = device.createComputePipeline({+ label: "vision_frame_hash",+ layout: "auto",+ compute: {+ module: device.createShaderModule({ code: HASH_WGSL }),+ entryPoint: "main",+ },+ });+ this.params = device.createBuffer({+ label: "vision_frame_tensor_params",+ size: PARAMS_BYTES,+ usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,+ });+ }++ /** Bytes of the NCHW float tensor for `spec`. */+ public static tensorBytes(spec: PreprocessSpec): number {+ return 3 * spec.width * spec.height * 4;+ }++ /**+ * Writes `image` as the model input described by `spec` into `out` (at least+ * `tensorBytes(spec)`, STORAGE usage). Submits its own command buffer, so it+ * runs before anything submitted later (e.g. the inference reading `out`).+ */+ public toTensor(+ image: VisionGpuImage,+ spec: PreprocessSpec,+ out: GPUBuffer,+ ): InputTransform {+ const transform = computeInputTransform(image.width, image.height, spec);+ const { scaleX, scaleY, offsetX, offsetY } = transform;+ const params = new ArrayBuffer(PARAMS_BYTES);+ const u32 = new Uint32Array(params);+ const f32 = new Float32Array(params);+ const i32 = new Int32Array(params);+ u32.set([image.width, image.height, spec.width, spec.height], 0);+ f32.set([scaleX, scaleY, offsetX, offsetY], 4);+ i32.set(+ [+ Math.ceil(offsetX),+ Math.ceil(offsetY),+ Math.min(spec.width, Math.floor(offsetX + image.width * scaleX)) - 1,+ Math.min(spec.height, Math.floor(offsetY + image.height * scaleY)) - 1,+ ],+ 8,+ );+ const inv = spec.std.map((s) => 1 / s);+ f32.set([...spec.mean, 0], 12);+ f32.set([...inv, 0], 16);+ f32.set(+ [0, 1, 2].map(+ (c) => (spec.padValue - (spec.mean[c] as number)) * (inv[c] as number),+ ),+ 20,+ );+ u32.set(spec.channels === "bgr" ? [2, 1, 0, 3] : [0, 1, 2, 3], 24);+ this.device.queue.writeBuffer(this.params, 0, params);++ const encoder = this.device.createCommandEncoder({+ label: "vision_frame_tensor",+ });+ const pass = encoder.beginComputePass();+ pass.setPipeline(this.tensorPipeline);+ pass.setBindGroup(+ 0,+ this.device.createBindGroup({+ layout: this.tensorPipeline.getBindGroupLayout(0),+ entries: [+ { binding: 0, resource: { buffer: this.params } },+ { binding: 1, resource: image.texture.createView() },+ { binding: 2, resource: { buffer: out } },+ ],+ }),+ );+ pass.dispatchWorkgroups(+ Math.ceil(spec.width / 16),+ Math.ceil(spec.height / 16),+ );+ pass.end();+ this.device.queue.submit([encoder.finish()]);+ return transform;+ }++ /** A key that is equal for textures with equal size and pixels. */+ public async contentKey(image: VisionGpuImage): Promise<string> {+ const acc = this.device.createBuffer({+ size: 8,+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,+ });+ const staging = this.device.createBuffer({+ size: 8,+ usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,+ });+ const encoder = this.device.createCommandEncoder({+ label: "vision_frame_hash",+ });+ const pass = encoder.beginComputePass();+ pass.setPipeline(this.hashPipeline);+ pass.setBindGroup(+ 0,+ this.device.createBindGroup({+ layout: this.hashPipeline.getBindGroupLayout(0),+ entries: [+ { binding: 0, resource: image.texture.createView() },+ { binding: 1, resource: { buffer: acc } },+ ],+ }),+ );+ pass.dispatchWorkgroups(+ Math.ceil(image.width / 16),+ Math.ceil(image.height / 16),+ );+ pass.end();+ encoder.copyBufferToBuffer(acc, 0, staging, 0, 8);+ this.device.queue.submit([encoder.finish()]);+ await staging.mapAsync(GPUMapMode.READ);+ const [a, b] = new Uint32Array(staging.getMappedRange().slice(0));+ staging.unmap();+ staging.destroy();+ acc.destroy();+ return `${image.width}x${image.height}:gpu:${(a as number).toString(36)}:${(b as number).toString(36)}`;+ }+}++/** Reads an 8-bit RGBA/BGRA texture back as RGBA pixels. */+export async function readTexturePixels(+ image: VisionGpuImage,+): Promise<VisionImageInput> {+ const { device, texture, width, height } = image;+ const unaligned = width * 4;+ const bytesPerRow = Math.ceil(unaligned / 256) * 256;+ const staging = device.createBuffer({+ size: bytesPerRow * height,+ usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,+ label: "vision_frame_readback",+ });+ const encoder = device.createCommandEncoder({+ label: "vision_frame_readback",+ });+ encoder.copyTextureToBuffer(+ { texture },+ { buffer: staging, bytesPerRow, rowsPerImage: height },+ [width, height, 1],+ );+ device.queue.submit([encoder.finish()]);+ await staging.mapAsync(GPUMapMode.READ);+ const mapped = new Uint8Array(staging.getMappedRange());+ const data = new Uint8ClampedArray(width * height * 4);+ for (let y = 0; y < height; y++) {+ data.set(+ mapped.subarray(y * bytesPerRow, y * bytesPerRow + unaligned),+ y * unaligned,+ );+ }+ staging.unmap();+ staging.destroy();+ if (texture.format.startsWith("bgra")) {+ for (let i = 0; i < data.length; i += 4) {+ const b = data[i] as number;+ data[i] = data[i + 2] as number;+ data[i + 2] = b;+ }+ }+ return { data, width, height };+}diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.tsindex 873d78b6..dcd330d8 100644--- a/packages/vision/src/index.ts+++ b/packages/vision/src/index.ts@@ -3,6 +3,7 @@ export * from "./decode/preprocess.js";export * from "./decode/rtmdet-ins.js";export * from "./decode/rtmo.js";export * from "./decode/selfie.js";+export * from "./gpu/frame-tensor.js";export * from "./gpu/pose-skeleton-renderer.js";export * from "./gpu/segmentation-texture-pool.js";export * from "./model/model-store.js";diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.tsindex bc5aade3..e7d9b1eb 100644--- a/packages/vision/src/models.e2e.test.ts+++ b/packages/vision/src/models.e2e.test.ts@@ -11,9 +11,10 @@import { fileURLToPath } from "node:url";import { Canvas, loadImage } from "skia-canvas";-import { beforeAll, describe, expect, it } from "vitest";+import { afterAll, beforeAll, describe, expect, it } from "vitest";import { imageToTensor, preprocessSpec } from "./decode/preprocess.js";import { decodeRtmdetIns } from "./decode/rtmdet-ins.js";+import { FrameTensorPipeline } from "./gpu/frame-tensor.js";import { VisionModelStore } from "./model/model-store.js";import {COCO_CLASSES,@@ -22,6 +23,7 @@ import {type VisionModelKey,} from "./model/registry.js";import { VisionRunner } from "./runner/vision-runner.js";+import { NodeWebGPUProvider } from "./runtime/node-webgpu-provider.js";import { mergeSubjectMask } from "./segmentation/subject.js";import type { VisionImageInput } from "./types.js";@@ -71,6 +73,101 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {await session.release();}, 600_000);+ describe("GPU frames", () => {+ let device: GPUDevice;+ // Held for the test's lifetime: Dawn crashes if its instance is collected+ // while a device is alive.+ const keepAlive: unknown[] = [];+ const upload = (frame: VisionImageInput) => {+ const texture = device.createTexture({+ size: [frame.width, frame.height],+ format: "rgba8unorm",+ usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST,+ });+ device.queue.writeTexture(+ { texture },+ frame.data as Uint8Array<ArrayBuffer>,+ { bytesPerRow: frame.width * 4 },+ [frame.width, frame.height],+ );+ return { texture, device, width: frame.width, height: frame.height };+ };++ beforeAll(async () => {+ const { create, globals } = await import("webgpu");+ Object.assign(globalThis, globals);+ const instance = create([]);+ keepAlive.push(instance);+ const adapter = await instance.requestAdapter();+ if (!adapter) throw new Error("no WebGPU adapter");+ device = await adapter.requestDevice();+ });++ // Dawn can crash the worker at exit while a device is alive.+ afterAll(() => {+ device?.destroy();+ keepAlive.length = 0;+ });++ it("preprocesses on the GPU like imageToTensor", async () => {+ for (const family of ["rtmdet-ins", "rtmo", "selfie"] as const) {+ const spec = preprocessSpec(+ family,+ family === "selfie" ? [256, 144] : [640, 640],+ );+ const frame = frames.dancer;+ const expected = imageToTensor(frame, spec).tensor;+ const out = device.createBuffer({+ size: FrameTensorPipeline.tensorBytes(spec),+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,+ });+ const staging = device.createBuffer({+ size: out.size,+ usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,+ });+ FrameTensorPipeline.for(device).toTensor(upload(frame), spec, out);+ const encoder = device.createCommandEncoder();+ encoder.copyBufferToBuffer(out, 0, staging, 0, out.size);+ device.queue.submit([encoder.finish()]);+ await staging.mapAsync(GPUMapMode.READ);+ const actual = new Float32Array(staging.getMappedRange().slice(0));+ let worst = 0;+ for (let i = 0; i < expected.length; i++) {+ worst = Math.max(+ worst,+ Math.abs((actual[i] as number) - (expected[i] as number)),+ );+ }+ // A step of one 8-bit level is 1/std ≈ 0.017 (rtmdet), 1 (rtmo), 1/255 (selfie).+ expect(worst).toBeLessThan(family === "rtmo" ? 0.05 : 1e-3);+ }+ }, 120_000);++ it("gives the same detections, masks and poses as CPU frames", async () => {+ const gpu = VisionRunner.create({+ provider: new NodeWebGPUProvider({ device }),+ });+ const cpu = VisionRunner.create();+ for (const name of ["dancer", "crema"] as const) {+ const a = await gpu.segment(upload(frames[name]));+ const b = await cpu.segment(frames[name]);+ expect(a.detections.map((d) => d.category)).toEqual(+ b.detections.map((d) => d.category),+ );+ a.detections.forEach((d, i) => {+ expect(d.score).toBeCloseTo((b.detections[i] as typeof d).score, 2);+ });+ expect(a.masks.map((m) => Math.round(m.coverage * 1000))).toEqual(+ b.masks.map((m) => Math.round(m.coverage * 1000)),+ );+ }+ const pose = await gpu.pose(upload(frames.dancer));+ expect(pose.people).toHaveLength(1);+ gpu.close();+ cpu.close();+ }, 600_000);+ });+describe.each(VISION_VARIANTS)("variant %s", (variant) => {const runner = VisionRunner.create({ variant });diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.tsindex 23818f0d..8afe5300 100644--- a/packages/vision/src/runner/vision-runner.ts+++ b/packages/vision/src/runner/vision-runner.ts@@ -16,6 +16,7 @@ import {} from "../decode/rtmdet-ins-head.js";import { decodeRtmo } from "../decode/rtmo.js";import { decodeSelfie } from "../decode/selfie.js";+import { FrameTensorPipeline, readTexturePixels } from "../gpu/frame-tensor.js";import {type ModelDownloadStatus,VisionModelStore,@@ -36,11 +37,13 @@ import {type VisionTensor,type VisionTensorOutputs,} from "../runtime/session-provider.js";-import type {- PersonMatte,- PoseResult,- SegmentationResult,- VisionImageInput,+import {+ isGpuImage,+ type PersonMatte,+ type PoseResult,+ type SegmentationResult,+ type VisionFrame,+ type VisionImageInput,} from "../types.js";export interface VisionRunnerOptions {@@ -84,6 +87,11 @@ interface RunnerCore {readonly tensorScratch: Map<VisionModelKey, Float32Array>;/** Tail of each model's run queue: one inference per session at a time. */readonly runQueue: Map<VisionModelKey, Promise<unknown>>;+ /** Each model's input on the GPU (GPU frames); runs are queued, so one each. */+ readonly gpuInputs: Map<+ VisionModelKey,+ { readonly device: GPUDevice; readonly buffer: GPUBuffer }+ >;/*** Recent RTMDet-Ins passes, newest last, shared by detect() and segment()* and matched by content — layers showing the same pixels share one pass.@@ -135,6 +143,7 @@ export class VisionRunner {sessions: new Map(),tensorScratch: new Map(),runQueue: new Map(),+ gpuInputs: new Map(),instancePasses: [],};}@@ -211,9 +220,7 @@ export class VisionRunner {}/** COCO-80 boxes (sorted by score, source pixels). Masks are not decoded. */- public async detect(- image: VisionImageInput,- ): Promise<readonly DetectedObject[]> {+ public async detect(image: VisionFrame): Promise<readonly DetectedObject[]> {const pass = await this.instancePass(image);return decodeRtmdetIns({ ...pass.outputs, masks: undefined, maskPlane: undefined },@@ -222,7 +229,7 @@ export class VisionRunner {}/** COCO-80 boxes plus a frame-aligned soft mask per instance. */- public async segment(image: VisionImageInput): Promise<SegmentationResult> {+ public async segment(image: VisionFrame): Promise<SegmentationResult> {const pass = await this.instancePass(image);return decodeRtmdetIns(pass.outputs,@@ -231,7 +238,7 @@ export class VisionRunner {}/** COCO-17 keypoints per person (source pixels), highest score first. */- public async pose(image: VisionImageInput): Promise<PoseResult> {+ public async pose(image: VisionFrame): Promise<PoseResult> {const key = this.modelKeyFor("pose");const { outputs, transform } = await this.infer(key, image);const dets = output(outputs, "dets", key);@@ -253,7 +260,7 @@ export class VisionRunner {}/** Person-vs-background alpha for the whole frame (Selfie Segmenter). */- public async matte(image: VisionImageInput): Promise<PersonMatte> {+ public async matte(image: VisionFrame): Promise<PersonMatte> {const key = this.modelKeyFor("matte", image.width / image.height);const [w, h] = VISION_MODELS[key].input;const { outputs } = await this.infer(key, image);@@ -276,15 +283,21 @@ export class VisionRunner {}core.sessions.clear();core.runQueue.clear();+ for (const { buffer } of core.gpuInputs.values()) buffer.destroy();+ core.gpuInputs.clear();core.instancePasses.length = 0;core.store.clearMemoryCache();}// ── internals ──────────────────────────────────────────────────────────- private instancePass(image: VisionImageInput): Promise<InstancePass> {+ private async instancePass(image: VisionFrame): Promise<InstancePass> {const passes = this._core.instancePasses;- const content = contentKey(image);+ const content = isGpuImage(image)+ ? await FrameTensorPipeline.for(image.device).contentKey(image)+ : contentKey(image);+ // Checked after the await: a caller that hashed the same frame meanwhile+ // has registered its pass by now.const byContent = passes.find((pass) => pass.content === content);if (byContent) return byContent.result;const key = this.modelKeyFor("segment");@@ -305,10 +318,7 @@ export class VisionRunner {return result;}- private instanceDecodeOptions(- image: VisionImageInput,- transform: InputTransform,- ) {+ private instanceDecodeOptions(image: VisionFrame, transform: InputTransform) {return {confidence: this.confidence,classes: this.classes,@@ -329,7 +339,7 @@ export class VisionRunner {*/private infer(key: VisionModelKey,- image: VisionImageInput,+ image: VisionFrame,): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {const queue = this._core.runQueue;const previous = queue.get(key) ?? Promise.resolve();@@ -347,11 +357,34 @@ export class VisionRunner {private async inferNow(key: VisionModelKey,- image: VisionImageInput,+ frame: VisionFrame,): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {const session = await this.session(key);const desc = VISION_MODELS[key];const spec = preprocessSpec(desc.family, desc.input);+ const dims = [1, 3, spec.height, spec.width];+ let image: VisionImageInput;+ if (isGpuImage(frame)) {+ if (this._core.provider.device === frame.device) {+ // Same device as the sessions: preprocess on the GPU, no readback.+ const buffer = this.gpuInput(key, frame.device, spec);+ const transform = FrameTensorPipeline.for(frame.device).toTensor(+ frame,+ spec,+ buffer,+ );+ const outputs = await session.run({+ name: "input",+ data: EMPTY_TENSOR,+ gpuBuffer: buffer,+ dims,+ });+ return { outputs, transform };+ }+ image = await readTexturePixels(frame);+ } else {+ image = frame;+ }const size = 3 * spec.width * spec.height;const scratchByKey = this._core.tensorScratch;let scratch = scratchByKey.get(key);@@ -360,14 +393,31 @@ export class VisionRunner {scratchByKey.set(key, scratch);}const { tensor, transform } = imageToTensor(image, spec, scratch);- const outputs = await session.run({- name: "input",- data: tensor,- dims: [1, 3, spec.height, spec.width],- });+ const outputs = await session.run({ name: "input", data: tensor, dims });return { outputs, transform };}+ private gpuInput(+ key: VisionModelKey,+ device: GPUDevice,+ spec: ReturnType<typeof preprocessSpec>,+ ): GPUBuffer {+ const inputs = this._core.gpuInputs;+ const size = FrameTensorPipeline.tensorBytes(spec);+ const current = inputs.get(key);+ if (current?.device === device && current.buffer.size >= size) {+ return current.buffer;+ }+ current?.buffer.destroy();+ const buffer = device.createBuffer({+ label: `vision_input_${key}`,+ size,+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,+ });+ inputs.set(key, { device, buffer });+ return buffer;+ }+private session(key: VisionModelKey): Promise<VisionSession> {const { sessions, store, provider } = this._core;let promise = sessions.get(key);@@ -395,6 +445,8 @@ export class VisionRunner {}}+const EMPTY_TENSOR = new Float32Array(0);+interface InstancePass {readonly transform: InputTransform;readonly outputs: RtmdetInsOutputs;diff --git a/packages/vision/src/runtime/node-webgpu-provider.ts b/packages/vision/src/runtime/node-webgpu-provider.tsindex 2ebe9900..8778ddb9 100644--- a/packages/vision/src/runtime/node-webgpu-provider.ts+++ b/packages/vision/src/runtime/node-webgpu-provider.ts@@ -151,6 +151,7 @@ export type NodeWebGPUProviderOptions = WebGPUProviderOptions;export class NodeWebGPUProvider implements SessionProvider {public readonly kind = "webgpu" as const;+ public readonly device?: GPUDevice;private readonly _inner: WebGPUProvider;@@ -177,6 +178,7 @@ export class NodeWebGPUProvider implements SessionProvider {// No WASM fallback inside the WebGPU provider: when WebGPU isn't possible// the caller (AutoSessionProvider) falls back to the node CPU provider,// which is faster than wasm and already installed.+ this.device = options.device;this._inner = new WebGPUProvider({wasmFallback: false,...options,diff --git a/packages/vision/src/runtime/session-provider.ts b/packages/vision/src/runtime/session-provider.tsindex 1232bb42..84bde92a 100644--- a/packages/vision/src/runtime/session-provider.ts+++ b/packages/vision/src/runtime/session-provider.ts@@ -14,7 +14,10 @@ export interface VisionTensor {export interface VisionTensorInput {readonly name: string;+ /** Ignored (may be empty) when `gpuBuffer` is set. */readonly data: Float32Array;+ /** The input already on the GPU; only for providers with a `device` (it must be that device's). */+ readonly gpuBuffer?: GPUBuffer;readonly dims: readonly number[];}@@ -27,10 +30,23 @@ export interface VisionSession {export interface SessionProvider {readonly kind: "node" | "webgpu";+ /** The GPU device sessions run on, when they accept `gpuBuffer` inputs from it. */+ readonly device?: GPUDevice;createSession(modelBytes: Uint8Array): Promise<VisionSession>;}let defaultProvider: (() => SessionProvider) | undefined;+let visionDevice: GPUDevice | undefined;++/**+ * The GPU device the default provider runs WebGPU sessions on — the renderer's,+ * so frames can go from its textures into the models without a CPU round trip.+ * Applies to sessions created afterwards; without it, onnxruntime-web creates+ * its own device.+ */+export function setVisionDevice(device: GPUDevice | undefined): void {+ visionDevice = device;+}/*** Sets the provider runners use when none is passed, for hosts that create@@ -78,6 +94,10 @@ export class AutoSessionProvider implements SessionProvider {return this._resolved?.kind ?? "webgpu";}+ public get device(): GPUDevice | undefined {+ return this._resolved?.device;+ }+public async createSession(modelBytes: Uint8Array): Promise<VisionSession> {if (this._resolved) return this._resolved.createSession(modelBytes);if (!this._resolving) {@@ -106,7 +126,10 @@ export class AutoSessionProvider implements SessionProvider {}),};}- const webgpu = new NodeWebGPUProvider({ wasmFallback: false });+ const webgpu = new NodeWebGPUProvider({+ wasmFallback: false,+ device: visionDevice,+ });try {return {provider: webgpu,diff --git a/packages/vision/src/runtime/webgpu-provider.test.ts b/packages/vision/src/runtime/webgpu-provider.test.tsindex 7a4620a2..fd209a76 100644--- a/packages/vision/src/runtime/webgpu-provider.test.ts+++ b/packages/vision/src/runtime/webgpu-provider.test.ts@@ -17,12 +17,21 @@ class FakeOrtSession implements OrtWebSession {}function fakeModule(opts: { failWebGPU?: boolean } = {}) {- const calls: { executionProviders: string[] }[] = [];+ const calls: { executionProviders: string[]; devices: unknown[] }[] = [];+ const fromGpuBuffer: unknown[] = [];const module: OrtWebModule = {InferenceSession: {async create(_bytes, options) {- calls.push({ executionProviders: [...options.executionProviders] });- if (opts.failWebGPU && options.executionProviders.includes("webgpu")) {+ const names = options.executionProviders.map((p) =>+ typeof p === "string" ? p : p.name,+ );+ calls.push({+ executionProviders: names,+ devices: options.executionProviders.map((p) =>+ typeof p === "string" ? undefined : p.device,+ ),+ });+ if (opts.failWebGPU && names.includes("webgpu")) {throw new Error("webgpu unavailable");}return new FakeOrtSession();@@ -34,12 +43,33 @@ function fakeModule(opts: { failWebGPU?: boolean } = {}) {public data: Float32Array,public dims: readonly number[],) {}+ static fromGpuBuffer(buffer: unknown) {+ fromGpuBuffer.push(buffer);+ return { buffer };+ }} as unknown as OrtWebModule["Tensor"],};- return { module, calls };+ return { module, calls, fromGpuBuffer };}describe("WebGPUProvider", () => {+ it("runs sessions on a given device and feeds GPU-buffer inputs", async () => {+ const { module, calls, fromGpuBuffer } = fakeModule();+ const device = { label: "renderer" } as unknown as GPUDevice;+ const provider = new WebGPUProvider({ loader: async () => module, device });+ expect(provider.device).toBe(device);+ const session = await provider.createSession(new Uint8Array([1]));+ expect(calls[0]?.devices).toEqual([device, undefined]);+ const buffer = { label: "input" } as unknown as GPUBuffer;+ await session.run({+ name: "input",+ data: new Float32Array(0),+ gpuBuffer: buffer,+ dims: [1, 3, 8, 8],+ });+ expect(fromGpuBuffer).toEqual([buffer]);+ });+it("creates a session with the injected module and maps tensor outputs", async () => {const { module, calls } = fakeModule();let loads = 0;diff --git a/packages/vision/src/runtime/webgpu-provider.ts b/packages/vision/src/runtime/webgpu-provider.tsindex 62b6a2ad..29e66706 100644--- a/packages/vision/src/runtime/webgpu-provider.ts+++ b/packages/vision/src/runtime/webgpu-provider.ts@@ -34,19 +34,24 @@ export interface OrtWebModule {create(modelBytes: Uint8Array,options: {- executionProviders: string[];+ executionProviders: (string | { name: string; device?: GPUDevice })[];graphOptimizationLevel?: "disabled" | "basic" | "extended" | "all";/** 0 verbose … 4 fatal. */logSeverityLevel?: 0 | 1 | 2 | 3 | 4;},): Promise<OrtWebSession>;};- /** ORT tensor constructor (called with `new`). */- readonly Tensor: new (+ /** ORT tensor constructor (called with `new`), with `fromGpuBuffer` on WebGPU builds. */+ readonly Tensor: (new (type: "float32",data: Float32Array,dims: readonly number[],- ) => unknown;+ ) => unknown) & {+ fromGpuBuffer?(+ buffer: GPUBuffer,+ options: { dataType: "float32"; dims: readonly number[] },+ ): unknown;+ };}export interface WebGPUProviderOptions {@@ -56,6 +61,11 @@ export interface WebGPUProviderOptions {readonly executionProviders?: readonly string[];/** Fall back to WASM automatically if the WebGPU session fails to create. Default true. */readonly wasmFallback?: boolean;+ /**+ * Run WebGPU sessions on this device (e.g. the renderer's) instead of one+ * onnxruntime-web creates. Sessions then accept `gpuBuffer` inputs from it.+ */+ readonly device?: GPUDevice;}const ORT_WEB_SPECIFIER = "onnxruntime-web";@@ -72,11 +82,13 @@ export class WebGPUProvider implements SessionProvider {private readonly _providers: readonly string[];private readonly _wasmFallback: boolean;private _ortPromise?: Promise<OrtWebModule>;+ public readonly device?: GPUDevice;constructor(options: WebGPUProviderOptions = {}) {this._loader = options.loader ?? defaultLoader;this._providers = options.executionProviders ?? ["webgpu", "wasm"];this._wasmFallback = options.wasmFallback !== false;+ this.device = options.device;}private ort(): Promise<OrtWebModule> {@@ -97,8 +109,11 @@ export class WebGPUProvider implements SessionProvider {let session: OrtWebSession;try {+ const device = this.device;session = await ort.InferenceSession.create(modelBytes, {- executionProviders: [...this._providers],+ executionProviders: this._providers.map((name) =>+ name === "webgpu" && device ? { name, device } : name,+ ),graphOptimizationLevel: "all",// Errors only: node-placement notes print as console errors otherwise.logSeverityLevel: 3,@@ -118,9 +133,14 @@ export class WebGPUProvider implements SessionProvider {const inputName = session.inputNames.includes(input.name)? input.name: (session.inputNames[0] ?? input.name);- const results = await session.run({- [inputName]: new ort.Tensor("float32", input.data, [...input.dims]),- });+ const tensor =+ input.gpuBuffer && ort.Tensor.fromGpuBuffer+ ? ort.Tensor.fromGpuBuffer(input.gpuBuffer, {+ dataType: "float32",+ dims: [...input.dims],+ })+ : new ort.Tensor("float32", input.data, [...input.dims]);+ const results = await session.run({ [inputName]: tensor });const outputs: Record<string, VisionTensorOutputs[string]> = {};for (const [name, tensor] of Object.entries(results)) {outputs[name] = {diff --git a/packages/vision/src/types.ts b/packages/vision/src/types.tsindex 182bd1ad..f708d7c8 100644--- a/packages/vision/src/types.ts+++ b/packages/vision/src/types.ts@@ -13,6 +13,25 @@ export interface VisionImageInput {readonly height: number;}+/**+ * A frame that is already on the GPU: an 8-bit RGBA/BGRA texture of `device`.+ * When the runner's sessions run on the same device (see `setVisionDevice`),+ * preprocessing happens on the GPU and the frame never comes back to the CPU;+ * otherwise the runner reads it back.+ */+export interface VisionGpuImage {+ readonly texture: GPUTexture;+ readonly device: GPUDevice;+ readonly width: number;+ readonly height: number;+}++export type VisionFrame = VisionImageInput | VisionGpuImage;++export function isGpuImage(image: VisionFrame): image is VisionGpuImage {+ return (image as Partial<VisionGpuImage>).texture !== undefined;+}+/** Instance segmentation mask aligned to the ORIGINAL source frame. */export interface InstanceMask {readonly category: string;