Skip to content

Code-first video & image, rendered natively on WebGPU.

perf(vision): frames stay on the GPU

okanasl5c55a7b8348a
diff --git a/.changeset/vision-gpu-frames.md b/.changeset/vision-gpu-frames.md
new file mode 100644
index 00000000..5f8847f2
--- /dev/null
+++ b/.changeset/vision-gpu-frames.md
@@ -0,0 +1,9 @@
+---
+"framefields": patch
+"@framefields/vision": minor
+"@framefields/node-vision": patch
+---
+
+Vision layers no longer copy frames to the CPU. ONNX Runtime's WebGPU sessions now run on the renderer's own device (`setVisionDevice`), and a compute shader turns the layer's texture straight into each model's input tensor. It does the same resize, letterbox, channel order and normalisation as `imageToTensor`, within float32 rounding. A small GPU checksum replaces hashing the pixels on the CPU, so layers showing the same frame still share a pass. Frames come back to the CPU only for the opt-in background key (`keyBackground`).
+
+`VisionRunner` methods accept a `VisionGpuImage` (`{ texture, device, width, height }`) as well as RGBA pixels. When its sessions run on another device, or on the CPU, the runner reads the texture back itself. `WebGPUProvider` takes a `device` option and feeds `gpuBuffer` inputs with `Tensor.fromGpuBuffer`.
diff --git a/nodes/node-vision/src/renderers/webgpu-renderer.ts b/nodes/node-vision/src/renderers/webgpu-renderer.ts
index 1624d6c6..9173dff6 100644
--- a/nodes/node-vision/src/renderers/webgpu-renderer.ts
+++ b/nodes/node-vision/src/renderers/webgpu-renderer.ts
@@ -8,9 +8,12 @@ import {
type PersonMatte,
type PoseResult,
PoseSkeletonRenderer,
+ readTexturePixels,
+ setVisionDevice,
TemporalObjectTracker,
type TrackedObject,
type VisionBundle,
+ type VisionGpuImage,
VisionRunner,
} from "@framefields/vision";
import type { VisionOperation } from "../shared/config.js";
@@ -38,8 +41,9 @@ let sharedCompositorFormat: GPUTextureFormat | null = null;
const frameCache = new VisionFrameCache();
/** The frame each node's child texture holds (drawn on its last call). */
const heldFrameByNode = new Map<string, number>();
-/** Per node: a copy of the held frame, and the composited matte. */
+/** Per node: a copy of the held frame, the frame being analysed, and the composited matte. */
const heldTextures = new Map<string, GPUTexture>();
+const analysisTextures = new Map<string, GPUTexture>();
const matteTextures = new Map<string, GPUTexture>();
/** One tracker per vision node — track ids must not mix across nodes/sources. */
const objectTrackers = new Map<string, TemporalObjectTracker>();
@@ -106,6 +110,18 @@ const heldTexture = (device: GPUDevice, key: string, like: GPUTexture) =>
"vision_held_frame",
);
+const analysisTexture = (device: GPUDevice, key: string, like: GPUTexture) =>
+ nodeTexture(
+ analysisTextures,
+ device,
+ key,
+ like,
+ GPUTextureUsage.COPY_DST |
+ GPUTextureUsage.TEXTURE_BINDING |
+ GPUTextureUsage.COPY_SRC,
+ "vision_analysis_frame",
+ );
+
const matteTexture = (device: GPUDevice, key: string, like: GPUTexture) =>
nodeTexture(
matteTextures,
@@ -392,7 +408,7 @@ function growMaskIntoForeground(
mask: Uint8Array,
width: number,
height: number,
- pixels: Uint8ClampedArray,
+ pixels: Uint8ClampedArray | Uint8Array,
threshold: number,
featherRadius?: number,
): Uint8Array {
@@ -517,6 +533,9 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {
} = args;
const { virtualMedia } = props;
+ // Vision sessions run on the renderer's device, so frames reach the models
+ // without leaving the GPU.
+ setVisionDevice(ctx.device);
const rawOp = virtualMedia?.operation as
| VisionOperation
| Record<string, unknown>
@@ -634,48 +653,31 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {
]);
}
- // 3. Read back the held frame's pixels, only when it still needs inference.
- let framePixels: Uint8ClampedArray | null = null;
+ // 3. The held frame, when it still needs inference: copied now (submitted
+ // before this frame redraws the child) and analysed on the GPU. Pixels
+ // come back to the CPU only for the opt-in background key.
+ let frame: VisionGpuImage | undefined;
+ let framePixels: Uint8ClampedArray | Uint8Array | null = null;
if (hasPreviousFrame && !cached) {
- const bytesPerPixel = 4;
- const unalignedBytesPerRow = targetWidth * bytesPerPixel;
- const bytesPerRow = Math.ceil(unalignedBytesPerRow / 256) * 256;
- const bufferSize = bytesPerRow * targetHeight;
-
- const stagingBuffer = ctx.device.createBuffer({
- size: bufferSize,
- usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
- label: "vision_frame_staging",
- });
-
- const readbackEncoder = ctx.device.createCommandEncoder({
- label: "vision_readback_encoder",
+ const analysis = analysisTexture(ctx.device, nodeKeyStr, childTex);
+ const copy = ctx.device.createCommandEncoder({
+ label: "vision_analysis_copy",
});
- readbackEncoder.copyTextureToBuffer(
- { texture: childTex },
- { buffer: stagingBuffer, bytesPerRow, rowsPerImage: targetHeight },
- [targetWidth, targetHeight, 1],
- );
- ctx.device.queue.submit([readbackEncoder.finish()]);
-
- await stagingBuffer.mapAsync(GPUMapMode.READ);
- const mappedBytes = new Uint8Array(stagingBuffer.getMappedRange());
- framePixels = new Uint8ClampedArray(targetWidth * targetHeight * 4);
- if (bytesPerRow === unalignedBytesPerRow) {
- framePixels.set(mappedBytes.subarray(0, framePixels.length));
- } else {
- for (let y = 0; y < targetHeight; y++) {
- framePixels.set(
- mappedBytes.subarray(
- y * bytesPerRow,
- y * bytesPerRow + unalignedBytesPerRow,
- ),
- y * unalignedBytesPerRow,
- );
- }
+ copy.copyTextureToTexture({ texture: childTex }, { texture: analysis }, [
+ targetWidth,
+ targetHeight,
+ 1,
+ ]);
+ ctx.device.queue.submit([copy.finish()]);
+ frame = {
+ texture: analysis,
+ device: ctx.device,
+ width: targetWidth,
+ height: targetHeight,
+ };
+ if (isMatteMode && op.keyBackground === true) {
+ framePixels = (await readTexturePixels(frame)).data;
}
- stagingBuffer.unmap();
- stagingBuffer.destroy();
}
// 4. Draw THIS frame's child into the persistent texture (recorded into the
@@ -725,13 +727,9 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {
subject = fromCachedMask(cached.subject.mask);
lastSubjectByNode.set(nodeKeyStr, { mask: subject, frame: frameIdx });
}
- } else if (framePixels) {
+ } else if (frame) {
const runner = getSharedRunner(op);
- const image = {
- data: framePixels,
- width: targetWidth,
- height: targetHeight,
- };
+ const image = frame;
// detect + segment share one RTMDet-Ins pass per frame inside the runner.
// Tracked objects are kept locally (like masks below) so `boxes` /
@@ -791,7 +789,7 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {
// Opt-in: grow the subject into connected pixels that stand out from
// the backdrop (catches edges the model's soft mask leaves behind).
const threshold = keyThresholdAt(heldFrame ?? frameIdx);
- if (subject && threshold !== undefined) {
+ if (subject && threshold !== undefined && framePixels) {
subject = {
...subject,
mask: growMaskIntoForeground(
diff --git a/packages/vision/src/gpu/frame-tensor.ts b/packages/vision/src/gpu/frame-tensor.ts
new file mode 100644
index 00000000..b3e0b498
--- /dev/null
+++ b/packages/vision/src/gpu/frame-tensor.ts
@@ -0,0 +1,293 @@
+/**
+ * Frame preprocessing on the GPU: a texture becomes a model's NCHW float input
+ * in a storage buffer, so a frame drawn by the renderer reaches the model
+ * without a round trip through the CPU. The math is `imageToTensor`'s
+ * (pixel-centre bilinear sampling, letterbox padding, channel order,
+ * normalisation); see decode/preprocess.ts for the per-model contracts.
+ *
+ * Also: a content hash of a texture (two order-independent 32-bit sums, read
+ * back as 8 bytes), so layers showing the same pixels can share a pass, and a
+ * plain RGBA readback for paths that need pixels on the CPU.
+ */
+
+import {
+ computeInputTransform,
+ type InputTransform,
+ type PreprocessSpec,
+} from "../decode/preprocess.js";
+import type { VisionGpuImage, VisionImageInput } from "../types.js";
+
+const TENSOR_WGSL = /* wgsl */ `
+struct Params {
+ src: vec2u,
+ dst: vec2u,
+ scale: vec2f,
+ offset: vec2f,
+ lo: vec2i,
+ hi: vec2i,
+ mean: vec4f,
+ invStd: vec4f,
+ pad: vec4f,
+ order: vec4u,
+};
+
+@group(0) @binding(0) var<uniform> p: Params;
+@group(0) @binding(1) var src: texture_2d<f32>;
+@group(0) @binding(2) var<storage, read_write> dst: array<f32>;
+
+@compute @workgroup_size(16, 16)
+fn main(@builtin(global_invocation_id) id: vec3u) {
+ if (id.x >= p.dst.x || id.y >= p.dst.y) { return; }
+ let x = i32(id.x);
+ let y = i32(id.y);
+ let plane = p.dst.x * p.dst.y;
+ let o = id.y * p.dst.x + id.x;
+ var v = p.pad.xyz;
+ if (x >= p.lo.x && x <= p.hi.x && y >= p.lo.y && y <= p.hi.y) {
+ let maxX = i32(p.src.x) - 1;
+ let maxY = i32(p.src.y) - 1;
+ let sy = max(0.0, (f32(y) + 0.5 - p.offset.y) / p.scale.y - 0.5);
+ let iy0 = min(maxY, i32(floor(sy)));
+ let iy1 = min(maxY, iy0 + 1);
+ let fy = sy - f32(iy0);
+ let sx = max(0.0, (f32(x) + 0.5 - p.offset.x) / p.scale.x - 0.5);
+ let ix0 = min(maxX, i32(floor(sx)));
+ let ix1 = min(maxX, ix0 + 1);
+ let fx = sx - f32(ix0);
+ var rgba =
+ textureLoad(src, vec2i(ix0, iy0), 0) * ((1.0 - fx) * (1.0 - fy)) +
+ textureLoad(src, vec2i(ix1, iy0), 0) * (fx * (1.0 - fy)) +
+ textureLoad(src, vec2i(ix0, iy1), 0) * ((1.0 - fx) * fy) +
+ textureLoad(src, vec2i(ix1, iy1), 0) * (fx * fy);
+ rgba = rgba * 255.0;
+ let s = vec3f(rgba[p.order.x], rgba[p.order.y], rgba[p.order.z]);
+ v = (s - p.mean.xyz) * p.invStd.xyz;
+ }
+ dst[o] = v.x;
+ dst[plane + o] = v.y;
+ dst[2u * plane + o] = v.z;
+}
+`;
+
+const HASH_WGSL = /* wgsl */ `
+@group(0) @binding(0) var src: texture_2d<f32>;
+@group(0) @binding(1) var<storage, read_write> acc: array<atomic<u32>, 2>;
+
+var<workgroup> local: array<atomic<u32>, 2>;
+
+@compute @workgroup_size(16, 16)
+fn main(
+ @builtin(global_invocation_id) id: vec3u,
+ @builtin(local_invocation_index) li: u32,
+) {
+ let dim = textureDimensions(src);
+ if (id.x < dim.x && id.y < dim.y) {
+ let c = pack4x8unorm(textureLoad(src, vec2i(id.xy), 0));
+ let i = id.y * dim.x + id.x;
+ let a = (c ^ (i * 0x9e3779b9u)) * 0x85ebca6bu;
+ let b = (c + i * 0xc2b2ae35u) * 0x27d4eb2fu;
+ atomicAdd(&local[0], a ^ (a >> 15u));
+ atomicAdd(&local[1], b ^ (b >> 13u));
+ }
+ workgroupBarrier();
+ if (li == 0u) {
+ atomicAdd(&acc[0], atomicLoad(&local[0]));
+ atomicAdd(&acc[1], atomicLoad(&local[1]));
+ }
+}
+`;
+
+const PARAMS_BYTES = 112;
+
+const pipelines = new WeakMap<GPUDevice, FrameTensorPipeline>();
+
+export class FrameTensorPipeline {
+ private readonly tensorPipeline: GPUComputePipeline;
+ private readonly hashPipeline: GPUComputePipeline;
+ private readonly params: GPUBuffer;
+
+ /** One per device. */
+ public static for(device: GPUDevice): FrameTensorPipeline {
+ let pipeline = pipelines.get(device);
+ if (!pipeline) {
+ pipeline = new FrameTensorPipeline(device);
+ pipelines.set(device, pipeline);
+ }
+ return pipeline;
+ }
+
+ private constructor(private readonly device: GPUDevice) {
+ this.tensorPipeline = device.createComputePipeline({
+ label: "vision_frame_tensor",
+ layout: "auto",
+ compute: {
+ module: device.createShaderModule({ code: TENSOR_WGSL }),
+ entryPoint: "main",
+ },
+ });
+ this.hashPipeline = device.createComputePipeline({
+ label: "vision_frame_hash",
+ layout: "auto",
+ compute: {
+ module: device.createShaderModule({ code: HASH_WGSL }),
+ entryPoint: "main",
+ },
+ });
+ this.params = device.createBuffer({
+ label: "vision_frame_tensor_params",
+ size: PARAMS_BYTES,
+ usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+ });
+ }
+
+ /** Bytes of the NCHW float tensor for `spec`. */
+ public static tensorBytes(spec: PreprocessSpec): number {
+ return 3 * spec.width * spec.height * 4;
+ }
+
+ /**
+ * Writes `image` as the model input described by `spec` into `out` (at least
+ * `tensorBytes(spec)`, STORAGE usage). Submits its own command buffer, so it
+ * runs before anything submitted later (e.g. the inference reading `out`).
+ */
+ public toTensor(
+ image: VisionGpuImage,
+ spec: PreprocessSpec,
+ out: GPUBuffer,
+ ): InputTransform {
+ const transform = computeInputTransform(image.width, image.height, spec);
+ const { scaleX, scaleY, offsetX, offsetY } = transform;
+ const params = new ArrayBuffer(PARAMS_BYTES);
+ const u32 = new Uint32Array(params);
+ const f32 = new Float32Array(params);
+ const i32 = new Int32Array(params);
+ u32.set([image.width, image.height, spec.width, spec.height], 0);
+ f32.set([scaleX, scaleY, offsetX, offsetY], 4);
+ i32.set(
+ [
+ Math.ceil(offsetX),
+ Math.ceil(offsetY),
+ Math.min(spec.width, Math.floor(offsetX + image.width * scaleX)) - 1,
+ Math.min(spec.height, Math.floor(offsetY + image.height * scaleY)) - 1,
+ ],
+ 8,
+ );
+ const inv = spec.std.map((s) => 1 / s);
+ f32.set([...spec.mean, 0], 12);
+ f32.set([...inv, 0], 16);
+ f32.set(
+ [0, 1, 2].map(
+ (c) => (spec.padValue - (spec.mean[c] as number)) * (inv[c] as number),
+ ),
+ 20,
+ );
+ u32.set(spec.channels === "bgr" ? [2, 1, 0, 3] : [0, 1, 2, 3], 24);
+ this.device.queue.writeBuffer(this.params, 0, params);
+
+ const encoder = this.device.createCommandEncoder({
+ label: "vision_frame_tensor",
+ });
+ const pass = encoder.beginComputePass();
+ pass.setPipeline(this.tensorPipeline);
+ pass.setBindGroup(
+ 0,
+ this.device.createBindGroup({
+ layout: this.tensorPipeline.getBindGroupLayout(0),
+ entries: [
+ { binding: 0, resource: { buffer: this.params } },
+ { binding: 1, resource: image.texture.createView() },
+ { binding: 2, resource: { buffer: out } },
+ ],
+ }),
+ );
+ pass.dispatchWorkgroups(
+ Math.ceil(spec.width / 16),
+ Math.ceil(spec.height / 16),
+ );
+ pass.end();
+ this.device.queue.submit([encoder.finish()]);
+ return transform;
+ }
+
+ /** A key that is equal for textures with equal size and pixels. */
+ public async contentKey(image: VisionGpuImage): Promise<string> {
+ const acc = this.device.createBuffer({
+ size: 8,
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
+ });
+ const staging = this.device.createBuffer({
+ size: 8,
+ usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
+ });
+ const encoder = this.device.createCommandEncoder({
+ label: "vision_frame_hash",
+ });
+ const pass = encoder.beginComputePass();
+ pass.setPipeline(this.hashPipeline);
+ pass.setBindGroup(
+ 0,
+ this.device.createBindGroup({
+ layout: this.hashPipeline.getBindGroupLayout(0),
+ entries: [
+ { binding: 0, resource: image.texture.createView() },
+ { binding: 1, resource: { buffer: acc } },
+ ],
+ }),
+ );
+ pass.dispatchWorkgroups(
+ Math.ceil(image.width / 16),
+ Math.ceil(image.height / 16),
+ );
+ pass.end();
+ encoder.copyBufferToBuffer(acc, 0, staging, 0, 8);
+ this.device.queue.submit([encoder.finish()]);
+ await staging.mapAsync(GPUMapMode.READ);
+ const [a, b] = new Uint32Array(staging.getMappedRange().slice(0));
+ staging.unmap();
+ staging.destroy();
+ acc.destroy();
+ return `${image.width}x${image.height}:gpu:${(a as number).toString(36)}:${(b as number).toString(36)}`;
+ }
+}
+
+/** Reads an 8-bit RGBA/BGRA texture back as RGBA pixels. */
+export async function readTexturePixels(
+ image: VisionGpuImage,
+): Promise<VisionImageInput> {
+ const { device, texture, width, height } = image;
+ const unaligned = width * 4;
+ const bytesPerRow = Math.ceil(unaligned / 256) * 256;
+ const staging = device.createBuffer({
+ size: bytesPerRow * height,
+ usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
+ label: "vision_frame_readback",
+ });
+ const encoder = device.createCommandEncoder({
+ label: "vision_frame_readback",
+ });
+ encoder.copyTextureToBuffer(
+ { texture },
+ { buffer: staging, bytesPerRow, rowsPerImage: height },
+ [width, height, 1],
+ );
+ device.queue.submit([encoder.finish()]);
+ await staging.mapAsync(GPUMapMode.READ);
+ const mapped = new Uint8Array(staging.getMappedRange());
+ const data = new Uint8ClampedArray(width * height * 4);
+ for (let y = 0; y < height; y++) {
+ data.set(
+ mapped.subarray(y * bytesPerRow, y * bytesPerRow + unaligned),
+ y * unaligned,
+ );
+ }
+ staging.unmap();
+ staging.destroy();
+ if (texture.format.startsWith("bgra")) {
+ for (let i = 0; i < data.length; i += 4) {
+ const b = data[i] as number;
+ data[i] = data[i + 2] as number;
+ data[i + 2] = b;
+ }
+ }
+ return { data, width, height };
+}
diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.ts
index 873d78b6..dcd330d8 100644
--- a/packages/vision/src/index.ts
+++ b/packages/vision/src/index.ts
@@ -3,6 +3,7 @@ export * from "./decode/preprocess.js";
export * from "./decode/rtmdet-ins.js";
export * from "./decode/rtmo.js";
export * from "./decode/selfie.js";
+export * from "./gpu/frame-tensor.js";
export * from "./gpu/pose-skeleton-renderer.js";
export * from "./gpu/segmentation-texture-pool.js";
export * from "./model/model-store.js";
diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.ts
index bc5aade3..e7d9b1eb 100644
--- a/packages/vision/src/models.e2e.test.ts
+++ b/packages/vision/src/models.e2e.test.ts
@@ -11,9 +11,10 @@
import { fileURLToPath } from "node:url";
import { Canvas, loadImage } from "skia-canvas";
-import { beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { imageToTensor, preprocessSpec } from "./decode/preprocess.js";
import { decodeRtmdetIns } from "./decode/rtmdet-ins.js";
+import { FrameTensorPipeline } from "./gpu/frame-tensor.js";
import { VisionModelStore } from "./model/model-store.js";
import {
COCO_CLASSES,
@@ -22,6 +23,7 @@ import {
type VisionModelKey,
} from "./model/registry.js";
import { VisionRunner } from "./runner/vision-runner.js";
+import { NodeWebGPUProvider } from "./runtime/node-webgpu-provider.js";
import { mergeSubjectMask } from "./segmentation/subject.js";
import type { VisionImageInput } from "./types.js";
@@ -71,6 +73,101 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {
await session.release();
}, 600_000);
+ describe("GPU frames", () => {
+ let device: GPUDevice;
+ // Held for the test's lifetime: Dawn crashes if its instance is collected
+ // while a device is alive.
+ const keepAlive: unknown[] = [];
+ const upload = (frame: VisionImageInput) => {
+ const texture = device.createTexture({
+ size: [frame.width, frame.height],
+ format: "rgba8unorm",
+ usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST,
+ });
+ device.queue.writeTexture(
+ { texture },
+ frame.data as Uint8Array<ArrayBuffer>,
+ { bytesPerRow: frame.width * 4 },
+ [frame.width, frame.height],
+ );
+ return { texture, device, width: frame.width, height: frame.height };
+ };
+
+ beforeAll(async () => {
+ const { create, globals } = await import("webgpu");
+ Object.assign(globalThis, globals);
+ const instance = create([]);
+ keepAlive.push(instance);
+ const adapter = await instance.requestAdapter();
+ if (!adapter) throw new Error("no WebGPU adapter");
+ device = await adapter.requestDevice();
+ });
+
+ // Dawn can crash the worker at exit while a device is alive.
+ afterAll(() => {
+ device?.destroy();
+ keepAlive.length = 0;
+ });
+
+ it("preprocesses on the GPU like imageToTensor", async () => {
+ for (const family of ["rtmdet-ins", "rtmo", "selfie"] as const) {
+ const spec = preprocessSpec(
+ family,
+ family === "selfie" ? [256, 144] : [640, 640],
+ );
+ const frame = frames.dancer;
+ const expected = imageToTensor(frame, spec).tensor;
+ const out = device.createBuffer({
+ size: FrameTensorPipeline.tensorBytes(spec),
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
+ });
+ const staging = device.createBuffer({
+ size: out.size,
+ usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
+ });
+ FrameTensorPipeline.for(device).toTensor(upload(frame), spec, out);
+ const encoder = device.createCommandEncoder();
+ encoder.copyBufferToBuffer(out, 0, staging, 0, out.size);
+ device.queue.submit([encoder.finish()]);
+ await staging.mapAsync(GPUMapMode.READ);
+ const actual = new Float32Array(staging.getMappedRange().slice(0));
+ let worst = 0;
+ for (let i = 0; i < expected.length; i++) {
+ worst = Math.max(
+ worst,
+ Math.abs((actual[i] as number) - (expected[i] as number)),
+ );
+ }
+ // A step of one 8-bit level is 1/std ≈ 0.017 (rtmdet), 1 (rtmo), 1/255 (selfie).
+ expect(worst).toBeLessThan(family === "rtmo" ? 0.05 : 1e-3);
+ }
+ }, 120_000);
+
+ it("gives the same detections, masks and poses as CPU frames", async () => {
+ const gpu = VisionRunner.create({
+ provider: new NodeWebGPUProvider({ device }),
+ });
+ const cpu = VisionRunner.create();
+ for (const name of ["dancer", "crema"] as const) {
+ const a = await gpu.segment(upload(frames[name]));
+ const b = await cpu.segment(frames[name]);
+ expect(a.detections.map((d) => d.category)).toEqual(
+ b.detections.map((d) => d.category),
+ );
+ a.detections.forEach((d, i) => {
+ expect(d.score).toBeCloseTo((b.detections[i] as typeof d).score, 2);
+ });
+ expect(a.masks.map((m) => Math.round(m.coverage * 1000))).toEqual(
+ b.masks.map((m) => Math.round(m.coverage * 1000)),
+ );
+ }
+ const pose = await gpu.pose(upload(frames.dancer));
+ expect(pose.people).toHaveLength(1);
+ gpu.close();
+ cpu.close();
+ }, 600_000);
+ });
+
describe.each(VISION_VARIANTS)("variant %s", (variant) => {
const runner = VisionRunner.create({ variant });
diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.ts
index 23818f0d..8afe5300 100644
--- a/packages/vision/src/runner/vision-runner.ts
+++ b/packages/vision/src/runner/vision-runner.ts
@@ -16,6 +16,7 @@ import {
} from "../decode/rtmdet-ins-head.js";
import { decodeRtmo } from "../decode/rtmo.js";
import { decodeSelfie } from "../decode/selfie.js";
+import { FrameTensorPipeline, readTexturePixels } from "../gpu/frame-tensor.js";
import {
type ModelDownloadStatus,
VisionModelStore,
@@ -36,11 +37,13 @@ import {
type VisionTensor,
type VisionTensorOutputs,
} from "../runtime/session-provider.js";
-import type {
- PersonMatte,
- PoseResult,
- SegmentationResult,
- VisionImageInput,
+import {
+ isGpuImage,
+ type PersonMatte,
+ type PoseResult,
+ type SegmentationResult,
+ type VisionFrame,
+ type VisionImageInput,
} from "../types.js";
export interface VisionRunnerOptions {
@@ -84,6 +87,11 @@ interface RunnerCore {
readonly tensorScratch: Map<VisionModelKey, Float32Array>;
/** Tail of each model's run queue: one inference per session at a time. */
readonly runQueue: Map<VisionModelKey, Promise<unknown>>;
+ /** Each model's input on the GPU (GPU frames); runs are queued, so one each. */
+ readonly gpuInputs: Map<
+ VisionModelKey,
+ { readonly device: GPUDevice; readonly buffer: GPUBuffer }
+ >;
/**
* Recent RTMDet-Ins passes, newest last, shared by detect() and segment()
* and matched by content — layers showing the same pixels share one pass.
@@ -135,6 +143,7 @@ export class VisionRunner {
sessions: new Map(),
tensorScratch: new Map(),
runQueue: new Map(),
+ gpuInputs: new Map(),
instancePasses: [],
};
}
@@ -211,9 +220,7 @@ export class VisionRunner {
}
/** COCO-80 boxes (sorted by score, source pixels). Masks are not decoded. */
- public async detect(
- image: VisionImageInput,
- ): Promise<readonly DetectedObject[]> {
+ public async detect(image: VisionFrame): Promise<readonly DetectedObject[]> {
const pass = await this.instancePass(image);
return decodeRtmdetIns(
{ ...pass.outputs, masks: undefined, maskPlane: undefined },
@@ -222,7 +229,7 @@ export class VisionRunner {
}
/** COCO-80 boxes plus a frame-aligned soft mask per instance. */
- public async segment(image: VisionImageInput): Promise<SegmentationResult> {
+ public async segment(image: VisionFrame): Promise<SegmentationResult> {
const pass = await this.instancePass(image);
return decodeRtmdetIns(
pass.outputs,
@@ -231,7 +238,7 @@ export class VisionRunner {
}
/** COCO-17 keypoints per person (source pixels), highest score first. */
- public async pose(image: VisionImageInput): Promise<PoseResult> {
+ public async pose(image: VisionFrame): Promise<PoseResult> {
const key = this.modelKeyFor("pose");
const { outputs, transform } = await this.infer(key, image);
const dets = output(outputs, "dets", key);
@@ -253,7 +260,7 @@ export class VisionRunner {
}
/** Person-vs-background alpha for the whole frame (Selfie Segmenter). */
- public async matte(image: VisionImageInput): Promise<PersonMatte> {
+ public async matte(image: VisionFrame): Promise<PersonMatte> {
const key = this.modelKeyFor("matte", image.width / image.height);
const [w, h] = VISION_MODELS[key].input;
const { outputs } = await this.infer(key, image);
@@ -276,15 +283,21 @@ export class VisionRunner {
}
core.sessions.clear();
core.runQueue.clear();
+ for (const { buffer } of core.gpuInputs.values()) buffer.destroy();
+ core.gpuInputs.clear();
core.instancePasses.length = 0;
core.store.clearMemoryCache();
}
// ── internals ──────────────────────────────────────────────────────────
- private instancePass(image: VisionImageInput): Promise<InstancePass> {
+ private async instancePass(image: VisionFrame): Promise<InstancePass> {
const passes = this._core.instancePasses;
- const content = contentKey(image);
+ const content = isGpuImage(image)
+ ? await FrameTensorPipeline.for(image.device).contentKey(image)
+ : contentKey(image);
+ // Checked after the await: a caller that hashed the same frame meanwhile
+ // has registered its pass by now.
const byContent = passes.find((pass) => pass.content === content);
if (byContent) return byContent.result;
const key = this.modelKeyFor("segment");
@@ -305,10 +318,7 @@ export class VisionRunner {
return result;
}
- private instanceDecodeOptions(
- image: VisionImageInput,
- transform: InputTransform,
- ) {
+ private instanceDecodeOptions(image: VisionFrame, transform: InputTransform) {
return {
confidence: this.confidence,
classes: this.classes,
@@ -329,7 +339,7 @@ export class VisionRunner {
*/
private infer(
key: VisionModelKey,
- image: VisionImageInput,
+ image: VisionFrame,
): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {
const queue = this._core.runQueue;
const previous = queue.get(key) ?? Promise.resolve();
@@ -347,11 +357,34 @@ export class VisionRunner {
private async inferNow(
key: VisionModelKey,
- image: VisionImageInput,
+ frame: VisionFrame,
): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {
const session = await this.session(key);
const desc = VISION_MODELS[key];
const spec = preprocessSpec(desc.family, desc.input);
+ const dims = [1, 3, spec.height, spec.width];
+ let image: VisionImageInput;
+ if (isGpuImage(frame)) {
+ if (this._core.provider.device === frame.device) {
+ // Same device as the sessions: preprocess on the GPU, no readback.
+ const buffer = this.gpuInput(key, frame.device, spec);
+ const transform = FrameTensorPipeline.for(frame.device).toTensor(
+ frame,
+ spec,
+ buffer,
+ );
+ const outputs = await session.run({
+ name: "input",
+ data: EMPTY_TENSOR,
+ gpuBuffer: buffer,
+ dims,
+ });
+ return { outputs, transform };
+ }
+ image = await readTexturePixels(frame);
+ } else {
+ image = frame;
+ }
const size = 3 * spec.width * spec.height;
const scratchByKey = this._core.tensorScratch;
let scratch = scratchByKey.get(key);
@@ -360,14 +393,31 @@ export class VisionRunner {
scratchByKey.set(key, scratch);
}
const { tensor, transform } = imageToTensor(image, spec, scratch);
- const outputs = await session.run({
- name: "input",
- data: tensor,
- dims: [1, 3, spec.height, spec.width],
- });
+ const outputs = await session.run({ name: "input", data: tensor, dims });
return { outputs, transform };
}
+ private gpuInput(
+ key: VisionModelKey,
+ device: GPUDevice,
+ spec: ReturnType<typeof preprocessSpec>,
+ ): GPUBuffer {
+ const inputs = this._core.gpuInputs;
+ const size = FrameTensorPipeline.tensorBytes(spec);
+ const current = inputs.get(key);
+ if (current?.device === device && current.buffer.size >= size) {
+ return current.buffer;
+ }
+ current?.buffer.destroy();
+ const buffer = device.createBuffer({
+ label: `vision_input_${key}`,
+ size,
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
+ });
+ inputs.set(key, { device, buffer });
+ return buffer;
+ }
+
private session(key: VisionModelKey): Promise<VisionSession> {
const { sessions, store, provider } = this._core;
let promise = sessions.get(key);
@@ -395,6 +445,8 @@ export class VisionRunner {
}
}
+const EMPTY_TENSOR = new Float32Array(0);
+
interface InstancePass {
readonly transform: InputTransform;
readonly outputs: RtmdetInsOutputs;
diff --git a/packages/vision/src/runtime/node-webgpu-provider.ts b/packages/vision/src/runtime/node-webgpu-provider.ts
index 2ebe9900..8778ddb9 100644
--- a/packages/vision/src/runtime/node-webgpu-provider.ts
+++ b/packages/vision/src/runtime/node-webgpu-provider.ts
@@ -151,6 +151,7 @@ export type NodeWebGPUProviderOptions = WebGPUProviderOptions;
export class NodeWebGPUProvider implements SessionProvider {
public readonly kind = "webgpu" as const;
+ public readonly device?: GPUDevice;
private readonly _inner: WebGPUProvider;
@@ -177,6 +178,7 @@ export class NodeWebGPUProvider implements SessionProvider {
// No WASM fallback inside the WebGPU provider: when WebGPU isn't possible
// the caller (AutoSessionProvider) falls back to the node CPU provider,
// which is faster than wasm and already installed.
+ this.device = options.device;
this._inner = new WebGPUProvider({
wasmFallback: false,
...options,
diff --git a/packages/vision/src/runtime/session-provider.ts b/packages/vision/src/runtime/session-provider.ts
index 1232bb42..84bde92a 100644
--- a/packages/vision/src/runtime/session-provider.ts
+++ b/packages/vision/src/runtime/session-provider.ts
@@ -14,7 +14,10 @@ export interface VisionTensor {
export interface VisionTensorInput {
readonly name: string;
+ /** Ignored (may be empty) when `gpuBuffer` is set. */
readonly data: Float32Array;
+ /** The input already on the GPU; only for providers with a `device` (it must be that device's). */
+ readonly gpuBuffer?: GPUBuffer;
readonly dims: readonly number[];
}
@@ -27,10 +30,23 @@ export interface VisionSession {
export interface SessionProvider {
readonly kind: "node" | "webgpu";
+ /** The GPU device sessions run on, when they accept `gpuBuffer` inputs from it. */
+ readonly device?: GPUDevice;
createSession(modelBytes: Uint8Array): Promise<VisionSession>;
}
let defaultProvider: (() => SessionProvider) | undefined;
+let visionDevice: GPUDevice | undefined;
+
+/**
+ * The GPU device the default provider runs WebGPU sessions on — the renderer's,
+ * so frames can go from its textures into the models without a CPU round trip.
+ * Applies to sessions created afterwards; without it, onnxruntime-web creates
+ * its own device.
+ */
+export function setVisionDevice(device: GPUDevice | undefined): void {
+ visionDevice = device;
+}
/**
* Sets the provider runners use when none is passed, for hosts that create
@@ -78,6 +94,10 @@ export class AutoSessionProvider implements SessionProvider {
return this._resolved?.kind ?? "webgpu";
}
+ public get device(): GPUDevice | undefined {
+ return this._resolved?.device;
+ }
+
public async createSession(modelBytes: Uint8Array): Promise<VisionSession> {
if (this._resolved) return this._resolved.createSession(modelBytes);
if (!this._resolving) {
@@ -106,7 +126,10 @@ export class AutoSessionProvider implements SessionProvider {
}),
};
}
- const webgpu = new NodeWebGPUProvider({ wasmFallback: false });
+ const webgpu = new NodeWebGPUProvider({
+ wasmFallback: false,
+ device: visionDevice,
+ });
try {
return {
provider: webgpu,
diff --git a/packages/vision/src/runtime/webgpu-provider.test.ts b/packages/vision/src/runtime/webgpu-provider.test.ts
index 7a4620a2..fd209a76 100644
--- a/packages/vision/src/runtime/webgpu-provider.test.ts
+++ b/packages/vision/src/runtime/webgpu-provider.test.ts
@@ -17,12 +17,21 @@ class FakeOrtSession implements OrtWebSession {
}
function fakeModule(opts: { failWebGPU?: boolean } = {}) {
- const calls: { executionProviders: string[] }[] = [];
+ const calls: { executionProviders: string[]; devices: unknown[] }[] = [];
+ const fromGpuBuffer: unknown[] = [];
const module: OrtWebModule = {
InferenceSession: {
async create(_bytes, options) {
- calls.push({ executionProviders: [...options.executionProviders] });
- if (opts.failWebGPU && options.executionProviders.includes("webgpu")) {
+ const names = options.executionProviders.map((p) =>
+ typeof p === "string" ? p : p.name,
+ );
+ calls.push({
+ executionProviders: names,
+ devices: options.executionProviders.map((p) =>
+ typeof p === "string" ? undefined : p.device,
+ ),
+ });
+ if (opts.failWebGPU && names.includes("webgpu")) {
throw new Error("webgpu unavailable");
}
return new FakeOrtSession();
@@ -34,12 +43,33 @@ function fakeModule(opts: { failWebGPU?: boolean } = {}) {
public data: Float32Array,
public dims: readonly number[],
) {}
+ static fromGpuBuffer(buffer: unknown) {
+ fromGpuBuffer.push(buffer);
+ return { buffer };
+ }
} as unknown as OrtWebModule["Tensor"],
};
- return { module, calls };
+ return { module, calls, fromGpuBuffer };
}
describe("WebGPUProvider", () => {
+ it("runs sessions on a given device and feeds GPU-buffer inputs", async () => {
+ const { module, calls, fromGpuBuffer } = fakeModule();
+ const device = { label: "renderer" } as unknown as GPUDevice;
+ const provider = new WebGPUProvider({ loader: async () => module, device });
+ expect(provider.device).toBe(device);
+ const session = await provider.createSession(new Uint8Array([1]));
+ expect(calls[0]?.devices).toEqual([device, undefined]);
+ const buffer = { label: "input" } as unknown as GPUBuffer;
+ await session.run({
+ name: "input",
+ data: new Float32Array(0),
+ gpuBuffer: buffer,
+ dims: [1, 3, 8, 8],
+ });
+ expect(fromGpuBuffer).toEqual([buffer]);
+ });
+
it("creates a session with the injected module and maps tensor outputs", async () => {
const { module, calls } = fakeModule();
let loads = 0;
diff --git a/packages/vision/src/runtime/webgpu-provider.ts b/packages/vision/src/runtime/webgpu-provider.ts
index 62b6a2ad..29e66706 100644
--- a/packages/vision/src/runtime/webgpu-provider.ts
+++ b/packages/vision/src/runtime/webgpu-provider.ts
@@ -34,19 +34,24 @@ export interface OrtWebModule {
create(
modelBytes: Uint8Array,
options: {
- executionProviders: string[];
+ executionProviders: (string | { name: string; device?: GPUDevice })[];
graphOptimizationLevel?: "disabled" | "basic" | "extended" | "all";
/** 0 verbose … 4 fatal. */
logSeverityLevel?: 0 | 1 | 2 | 3 | 4;
},
): Promise<OrtWebSession>;
};
- /** ORT tensor constructor (called with `new`). */
- readonly Tensor: new (
+ /** ORT tensor constructor (called with `new`), with `fromGpuBuffer` on WebGPU builds. */
+ readonly Tensor: (new (
type: "float32",
data: Float32Array,
dims: readonly number[],
- ) => unknown;
+ ) => unknown) & {
+ fromGpuBuffer?(
+ buffer: GPUBuffer,
+ options: { dataType: "float32"; dims: readonly number[] },
+ ): unknown;
+ };
}
export interface WebGPUProviderOptions {
@@ -56,6 +61,11 @@ export interface WebGPUProviderOptions {
readonly executionProviders?: readonly string[];
/** Fall back to WASM automatically if the WebGPU session fails to create. Default true. */
readonly wasmFallback?: boolean;
+ /**
+ * Run WebGPU sessions on this device (e.g. the renderer's) instead of one
+ * onnxruntime-web creates. Sessions then accept `gpuBuffer` inputs from it.
+ */
+ readonly device?: GPUDevice;
}
const ORT_WEB_SPECIFIER = "onnxruntime-web";
@@ -72,11 +82,13 @@ export class WebGPUProvider implements SessionProvider {
private readonly _providers: readonly string[];
private readonly _wasmFallback: boolean;
private _ortPromise?: Promise<OrtWebModule>;
+ public readonly device?: GPUDevice;
constructor(options: WebGPUProviderOptions = {}) {
this._loader = options.loader ?? defaultLoader;
this._providers = options.executionProviders ?? ["webgpu", "wasm"];
this._wasmFallback = options.wasmFallback !== false;
+ this.device = options.device;
}
private ort(): Promise<OrtWebModule> {
@@ -97,8 +109,11 @@ export class WebGPUProvider implements SessionProvider {
let session: OrtWebSession;
try {
+ const device = this.device;
session = await ort.InferenceSession.create(modelBytes, {
- executionProviders: [...this._providers],
+ executionProviders: this._providers.map((name) =>
+ name === "webgpu" && device ? { name, device } : name,
+ ),
graphOptimizationLevel: "all",
// Errors only: node-placement notes print as console errors otherwise.
logSeverityLevel: 3,
@@ -118,9 +133,14 @@ export class WebGPUProvider implements SessionProvider {
const inputName = session.inputNames.includes(input.name)
? input.name
: (session.inputNames[0] ?? input.name);
- const results = await session.run({
- [inputName]: new ort.Tensor("float32", input.data, [...input.dims]),
- });
+ const tensor =
+ input.gpuBuffer && ort.Tensor.fromGpuBuffer
+ ? ort.Tensor.fromGpuBuffer(input.gpuBuffer, {
+ dataType: "float32",
+ dims: [...input.dims],
+ })
+ : new ort.Tensor("float32", input.data, [...input.dims]);
+ const results = await session.run({ [inputName]: tensor });
const outputs: Record<string, VisionTensorOutputs[string]> = {};
for (const [name, tensor] of Object.entries(results)) {
outputs[name] = {
diff --git a/packages/vision/src/types.ts b/packages/vision/src/types.ts
index 182bd1ad..f708d7c8 100644
--- a/packages/vision/src/types.ts
+++ b/packages/vision/src/types.ts
@@ -13,6 +13,25 @@ export interface VisionImageInput {
readonly height: number;
}
+/**
+ * A frame that is already on the GPU: an 8-bit RGBA/BGRA texture of `device`.
+ * When the runner's sessions run on the same device (see `setVisionDevice`),
+ * preprocessing happens on the GPU and the frame never comes back to the CPU;
+ * otherwise the runner reads it back.
+ */
+export interface VisionGpuImage {
+ readonly texture: GPUTexture;
+ readonly device: GPUDevice;
+ readonly width: number;
+ readonly height: number;
+}
+
+export type VisionFrame = VisionImageInput | VisionGpuImage;
+
+export function isGpuImage(image: VisionFrame): image is VisionGpuImage {
+ return (image as Partial<VisionGpuImage>).texture !== undefined;
+}
+
/** Instance segmentation mask aligned to the ORIGINAL source frame. */
export interface InstanceMask {
readonly category: string;