Skip to content

Code-first video & image, rendered natively on WebGPU.

perf(vision): RTMDet-Ins without in-graph NMS and full-size masks

okanasl931435fc6ec7
diff --git a/.changeset/vision-rtmdet-head.md b/.changeset/vision-rtmdet-head.md
new file mode 100644
index 00000000..2cd8e319
--- /dev/null
+++ b/.changeset/vision-rtmdet-head.md
@@ -0,0 +1,10 @@
+---
+"framefields": patch
+"@framefields/vision": patch
+---
+
+Instance segmentation does far less work per frame. The RTMDet-Ins export ran NMS on the CPU in the middle of its graph and returned a full 640×640 mask for every detection: about 40 per frame, 65 MB of floats, computed on the GPU and copied back. The runner now cuts the graph at load time before its first data-dependent step and does the rest itself: top-k, per-class NMS with the export's thresholds, and each mask's three dynamic 1×1 layers at 80×80, only for the detections a layer keeps. `detect()` computes no masks at all.
+
+Results match the end2end graph: the same detections, scores and boxes, and masks with IoU above 0.98 on the reference frames (edges differ by one interpolation step). The model file and its SHA-256 check are unchanged; a model without the expected head tensors still runs end to end.
+
+The smoke project's `vision` export on an Apple Silicon Mac: 22.3 s → 15 s.
diff --git a/packages/vision/src/decode/rtmdet-ins-head.test.ts b/packages/vision/src/decode/rtmdet-ins-head.test.ts
new file mode 100644
index 00000000..9e414c10
--- /dev/null
+++ b/packages/vision/src/decode/rtmdet-ins-head.test.ts
@@ -0,0 +1,118 @@
+import { describe, expect, it } from "vitest";
+import { logitToAlpha, probabilityToAlpha } from "./rtmdet-ins.js";
+import {
+ anchorPrior,
+ maskLogits,
+ type RtmdetInsHeadOutputs,
+ selectInstances,
+} from "./rtmdet-ins-head.js";
+
+const ANCHORS = 8400;
+const CLASSES = 3;
+
+function head(
+ entries: { anchor: number; box: number[]; scores: number[] }[],
+): RtmdetInsHeadOutputs {
+ const scores = new Float32Array(ANCHORS * CLASSES);
+ const boxes = new Float32Array(ANCHORS * 4);
+ for (const { anchor, box, scores: s } of entries) {
+ boxes.set(box, anchor * 4);
+ scores.set(s, anchor * CLASSES);
+ }
+ return {
+ scores,
+ boxes,
+ kernels: new Float32Array(ANCHORS * 169),
+ maskFeatures: new Float32Array(8 * 80 * 80),
+ anchors: ANCHORS,
+ classes: CLASSES,
+ maskWidth: 80,
+ maskHeight: 80,
+ inputWidth: 640,
+ inputHeight: 640,
+ };
+}
+
+describe("selectInstances", () => {
+ it("suppresses overlaps per class, keeps other classes, best first", () => {
+ const selected = selectInstances(
+ head([
+ { anchor: 10, box: [0, 0, 100, 100], scores: [0.9, 0, 0] },
+ // IoU 0.81 with anchor 10: suppressed for class 0…
+ { anchor: 11, box: [0, 0, 100, 90], scores: [0.8, 0, 0.7] },
+ // …IoU 0.5: kept.
+ { anchor: 12, box: [50, 0, 150, 100], scores: [0.6, 0, 0] },
+ // Below the score floor in every class: never selected.
+ { anchor: 13, box: [300, 300, 400, 400], scores: [0.05, 0.04, 0] },
+ ]),
+ );
+ expect(selected).toEqual([
+ { anchor: 10, classIndex: 0, score: expect.closeTo(0.9, 5) },
+ { anchor: 11, classIndex: 2, score: expect.closeTo(0.7, 5) },
+ { anchor: 12, classIndex: 0, score: expect.closeTo(0.6, 5) },
+ ]);
+ });
+
+ it("returns at most 100 detections", () => {
+ const entries = Array.from({ length: 150 }, (_, i) => ({
+ anchor: i,
+ box: [
+ (i % 15) * 40,
+ Math.floor(i / 15) * 40,
+ (i % 15) * 40 + 30,
+ Math.floor(i / 15) * 40 + 30,
+ ],
+ scores: [0.5 + i / 1000, 0, 0],
+ }));
+ const selected = selectInstances(head(entries));
+ expect(selected).toHaveLength(100);
+ expect(selected[0]?.anchor).toBe(149);
+ });
+});
+
+describe("anchorPrior", () => {
+ it("walks the stride-8, -16 and -32 grids row by row", () => {
+ expect(anchorPrior(0, 640, 640)).toEqual({ x: 0, y: 0, stride: 8 });
+ expect(anchorPrior(81, 640, 640)).toEqual({ x: 8, y: 8, stride: 8 });
+ expect(anchorPrior(6400, 640, 640)).toEqual({ x: 0, y: 0, stride: 16 });
+ expect(anchorPrior(8399, 640, 640)).toEqual({ x: 608, y: 608, stride: 32 });
+ expect(() => anchorPrior(8400, 640, 640)).toThrow(RangeError);
+ });
+});
+
+describe("maskLogits", () => {
+ it("runs the three dynamic layers over offsets and features", () => {
+ const h = head([]);
+ const anchor = 81; // prior (8, 8), stride 8
+ const k = h.kernels.subarray(anchor * 169, (anchor + 1) * 169);
+ // Layer 1: hidden0 = relu(-x offset), hidden1 = relu(feature 0).
+ k[0] = -1; // w1[0][relX]
+ k[12] = 1; // w1[1][feature 0]
+ // Layer 2: pass both through. Layer 3: out = hidden0 + 2·hidden1 + 0.5.
+ k[80] = 1;
+ k[80 + 8 + 1] = 1;
+ k[144] = 1;
+ k[145] = 2;
+ k[168] = 0.5;
+ h.maskFeatures[3 * 80 + 4] = 0.25; // feature 0 at (x 4, y 3)
+ const logits = maskLogits(h, anchor);
+ // relX = (8 - 8x) / 64; hidden0 = relu(-(relX)) = max(0, (8x - 8) / 64).
+ expect(logits[0]).toBeCloseTo(0.5, 6); // x 0: relX > 0
+ expect(logits[3 * 80 + 4]).toBeCloseTo(0.5 + 24 / 64 + 0.5, 6);
+ });
+});
+
+describe("logitToAlpha", () => {
+ it("matches probabilityToAlpha after a sigmoid", () => {
+ const fromLogit = logitToAlpha(0.5, 0.05);
+ const fromProbability = probabilityToAlpha(0.5, 0.05);
+ for (const l of [-8, -0.3, -0.1, 0, 0.05, 0.15, 0.3, 8]) {
+ expect(fromLogit(l)).toBeCloseTo(
+ fromProbability(1 / (1 + Math.exp(-l))),
+ 6,
+ );
+ }
+ const hard = logitToAlpha(0.5, 0);
+ expect([hard(-0.01), hard(0.01)]).toEqual([0, 255]);
+ });
+});
diff --git a/packages/vision/src/decode/rtmdet-ins-head.ts b/packages/vision/src/decode/rtmdet-ins-head.ts
new file mode 100644
index 00000000..9a6e6737
--- /dev/null
+++ b/packages/vision/src/decode/rtmdet-ins-head.ts
@@ -0,0 +1,232 @@
+/**
+ * RTMDet-Ins post-processing, done here instead of inside the ONNX graph.
+ *
+ * The mmdeploy end2end export runs top-k, per-class NMS and mask assembly in
+ * the graph and returns a full input-sized mask for each of up to 100
+ * detections (40 on a typical frame: ~65 MB of floats). The runner instead
+ * cuts the graph before its first data-dependent op (see `RTMDET_INS_HEAD`)
+ * and reproduces the tail exactly:
+ *
+ * 1. top 5000 anchors by their best class score;
+ * 2. ONNX NonMaxSuppression per class (IoU > 0.6 suppresses, scores > 0.05,
+ * at most 200 per class);
+ * 3. the 100 best (class, box) pairs;
+ * 4. masks on demand: a 3-layer dynamic 1×1 conv over the 8 mask features
+ * plus each pixel's offset from the anchor, at 1/8 of the input size.
+ * The logits are upsampled (bilinear) and squashed by the decoder.
+ *
+ * Constants and layer layout match the export's graph; `models.e2e.test.ts`
+ * checks the result against the end2end outputs.
+ */
+
+/** Head tensors of the mmdeploy RTMDet-Ins exports (all variants). */
+export const RTMDET_INS_HEAD = {
+ /** [1, A, 80] class probabilities (sigmoided). */
+ scores: "/Sigmoid_output_0",
+ /** [1, A, 4] x0, y0, x1, y1 in input pixels. */
+ boxes: "/Concat_7_output_0",
+ /** [1, A, 169] dynamic conv parameters per anchor. */
+ kernels: "/Concat_5_output_0",
+ /** [1, 8, H/8, W/8] mask features. */
+ maskFeatures: "/bbox_head/mask_head/projection/Conv_output_0",
+} as const;
+
+const PRE_NMS = 5000;
+// float32, as the graph compares them: a float32 score of 0.05 does not pass.
+const IOU_THRESHOLD = Math.fround(0.6);
+const SCORE_FLOOR = Math.fround(0.05);
+const MAX_PER_CLASS = 200;
+const MAX_DETECTIONS = 100;
+const STRIDES = [8, 16, 32] as const;
+/** Mask features are at stride 8; relative offsets are divided by stride × 8. */
+const MASK_STRIDE = 8;
+const MASK_CHANNELS = 8;
+const KERNEL_SIZE = 169;
+
+export interface RtmdetInsHeadOutputs {
+ readonly scores: Float32Array;
+ readonly boxes: Float32Array;
+ readonly kernels: Float32Array;
+ readonly maskFeatures: Float32Array;
+ readonly anchors: number;
+ readonly classes: number;
+ readonly maskWidth: number;
+ readonly maskHeight: number;
+ /** Model input size; anchor positions derive from it. */
+ readonly inputWidth: number;
+ readonly inputHeight: number;
+}
+
+export interface RtmdetInsInstance {
+ readonly anchor: number;
+ readonly classIndex: number;
+ readonly score: number;
+}
+
+/** Steps 1–3: the detections the end2end graph would return, best first. */
+export function selectInstances(
+ head: RtmdetInsHeadOutputs,
+): RtmdetInsInstance[] {
+ const { scores, boxes, anchors, classes } = head;
+
+ // 1. Best class score per anchor, then the top PRE_NMS anchors.
+ const best = new Float32Array(anchors);
+ for (let a = 0; a < anchors; a++) {
+ let max = 0;
+ const row = a * classes;
+ for (let c = 0; c < classes; c++) {
+ const s = scores[row + c] as number;
+ if (s > max) max = s;
+ }
+ best[a] = max;
+ }
+ // Anchors that can't pass the score floor in any class are never selected.
+ const order: number[] = [];
+ for (let a = 0; a < anchors; a++) {
+ if ((best[a] as number) > SCORE_FLOOR) order.push(a);
+ }
+ order.sort((x, y) => (best[y] as number) - (best[x] as number) || x - y);
+ if (order.length > PRE_NMS) order.length = PRE_NMS;
+
+ // 2. Per-class NMS over those anchors.
+ const picked: RtmdetInsInstance[] = [];
+ const candidates: number[] = [];
+ const kept: number[] = [];
+ for (let c = 0; c < classes; c++) {
+ candidates.length = 0;
+ for (const a of order) {
+ if ((scores[a * classes + c] as number) > SCORE_FLOOR) candidates.push(a);
+ }
+ if (candidates.length === 0) continue;
+ candidates.sort(
+ (x, y) =>
+ (scores[y * classes + c] as number) -
+ (scores[x * classes + c] as number) || x - y,
+ );
+ kept.length = 0;
+ for (const a of candidates) {
+ if (kept.length >= MAX_PER_CLASS) break;
+ let suppressed = false;
+ for (const k of kept) {
+ if (iou(boxes, a, k) > IOU_THRESHOLD) {
+ suppressed = true;
+ break;
+ }
+ }
+ if (!suppressed) {
+ kept.push(a);
+ picked.push({
+ anchor: a,
+ classIndex: c,
+ score: scores[a * classes + c] as number,
+ });
+ }
+ }
+ }
+
+ // 3. The best MAX_DETECTIONS pairs.
+ picked.sort((x, y) => y.score - x.score);
+ if (picked.length > MAX_DETECTIONS) picked.length = MAX_DETECTIONS;
+ return picked;
+}
+
+function iou(boxes: Float32Array, a: number, b: number): number {
+ const a0 = boxes[a * 4] as number;
+ const a1 = boxes[a * 4 + 1] as number;
+ const a2 = boxes[a * 4 + 2] as number;
+ const a3 = boxes[a * 4 + 3] as number;
+ const b0 = boxes[b * 4] as number;
+ const b1 = boxes[b * 4 + 1] as number;
+ const b2 = boxes[b * 4 + 2] as number;
+ const b3 = boxes[b * 4 + 3] as number;
+ const w = Math.min(a2, b2) - Math.max(a0, b0);
+ const h = Math.min(a3, b3) - Math.max(a1, b1);
+ if (w <= 0 || h <= 0) return 0;
+ const inter = w * h;
+ const union = (a2 - a0) * (a3 - a1) + (b2 - b0) * (b3 - b1) - inter;
+ return union > 0 ? inter / union : 0;
+}
+
+/** Anchor position and stride, in input pixels (mmdet point priors, offset 0). */
+export function anchorPrior(
+ anchor: number,
+ inputWidth: number,
+ inputHeight: number,
+): { x: number; y: number; stride: number } {
+ let base = 0;
+ for (const stride of STRIDES) {
+ const cols = Math.ceil(inputWidth / stride);
+ const count = cols * Math.ceil(inputHeight / stride);
+ if (anchor < base + count) {
+ const i = anchor - base;
+ return {
+ x: (i % cols) * stride,
+ y: Math.floor(i / cols) * stride,
+ stride,
+ };
+ }
+ base += count;
+ }
+ throw new RangeError(`anchor ${anchor} is out of range`);
+}
+
+/**
+ * Step 4: mask logits for one anchor, `maskHeight × maskWidth`, row-major.
+ * Writes into `out` when it is large enough.
+ */
+export function maskLogits(
+ head: RtmdetInsHeadOutputs,
+ anchor: number,
+ out?: Float32Array,
+): Float32Array {
+ const { kernels, maskFeatures, maskWidth: mw, maskHeight: mh } = head;
+ const plane = mw * mh;
+ const logits = out && out.length >= plane ? out : new Float32Array(plane);
+ const prior = anchorPrior(anchor, head.inputWidth, head.inputHeight);
+ const k = kernels.subarray(anchor * KERNEL_SIZE, (anchor + 1) * KERNEL_SIZE);
+ // Layout: w1 [8×10], w2 [8×8], w3 [1×8], b1 [8], b2 [8], b3 [1].
+ const w1 = 0;
+ const w2 = 80;
+ const w3 = 144;
+ const b1 = 152;
+ const b2 = 160;
+ const b3 = 168;
+ const scale = 1 / (prior.stride * MASK_STRIDE);
+ const input = new Float32Array(2 + MASK_CHANNELS);
+ const hidden1 = new Float32Array(8);
+ const hidden2 = new Float32Array(8);
+
+ for (let y = 0; y < mh; y++) {
+ const relY = (prior.y - y * MASK_STRIDE) * scale;
+ for (let x = 0; x < mw; x++) {
+ const p = y * mw + x;
+ input[0] = (prior.x - x * MASK_STRIDE) * scale;
+ input[1] = relY;
+ for (let c = 0; c < MASK_CHANNELS; c++) {
+ input[2 + c] = maskFeatures[c * plane + p] as number;
+ }
+ for (let o = 0; o < 8; o++) {
+ let sum = k[b1 + o] as number;
+ const row = w1 + o * 10;
+ for (let i = 0; i < 10; i++) {
+ sum += (k[row + i] as number) * (input[i] as number);
+ }
+ hidden1[o] = sum > 0 ? sum : 0;
+ }
+ for (let o = 0; o < 8; o++) {
+ let sum = k[b2 + o] as number;
+ const row = w2 + o * 8;
+ for (let i = 0; i < 8; i++) {
+ sum += (k[row + i] as number) * (hidden1[i] as number);
+ }
+ hidden2[o] = sum > 0 ? sum : 0;
+ }
+ let sum = k[b3] as number;
+ for (let i = 0; i < 8; i++) {
+ sum += (k[w3 + i] as number) * (hidden2[i] as number);
+ }
+ logits[p] = sum;
+ }
+ }
+ return logits;
+}
diff --git a/packages/vision/src/decode/rtmdet-ins.ts b/packages/vision/src/decode/rtmdet-ins.ts
index 9a1ffa50..4fd56256 100644
--- a/packages/vision/src/decode/rtmdet-ins.ts
+++ b/packages/vision/src/decode/rtmdet-ins.ts
@@ -13,8 +13,15 @@ import { type InputTransform, toSourcePoint } from "./preprocess.js";
export interface RtmdetInsOutputs {
readonly dets: ArrayLike<number>;
readonly labels: ArrayLike<number | bigint>;
- /** Omit to decode boxes only (detection without masks). */
+ /** Omit (and `maskPlane`) to decode boxes only (detection without masks). */
readonly masks?: ArrayLike<number>;
+ /**
+ * Alternative to `masks`: detection `i`'s plane, computed on demand, so only
+ * detections that pass the filters cost anything.
+ */
+ readonly maskPlane?: (index: number) => ArrayLike<number>;
+ /** The planes hold logits (sigmoid applied after upsampling), not probabilities. */
+ readonly maskLogits?: boolean;
readonly count: number;
readonly maskWidth: number;
readonly maskHeight: number;
@@ -88,12 +95,13 @@ export function decodeRtmdetIns(
const detections = kept.map((k) => k.detection);
const masks: InstanceMask[] = [];
- if (!out.masks) return { detections, masks };
+ if (!out.masks && !out.maskPlane) return { detections, masks };
- const alphaOf = probabilityToAlpha(
- opts.maskThreshold ?? 0.5,
- opts.featherRadius ?? 0.05,
- );
+ const threshold = opts.maskThreshold ?? 0.5;
+ const feather = opts.featherRadius ?? 0.05;
+ const alphaOf = out.maskLogits
+ ? logitToAlpha(threshold, feather)
+ : probabilityToAlpha(threshold, feather);
const { maskWidth: mw, maskHeight: mh } = out;
const plane = mw * mh;
// Mask grid px per input px (exports may emit masks below input resolution).
@@ -103,7 +111,10 @@ export function decodeRtmdetIns(
for (let d = 0; d < kept.length; d++) {
const { index, detection } = kept[d];
const box = detection.boundingBox;
- const base = index * plane;
+ const data = out.maskPlane
+ ? out.maskPlane(index)
+ : (out.masks as ArrayLike<number>);
+ const base = out.maskPlane ? 0 : index * plane;
const bx0 = Math.max(0, Math.floor(box.originX) - MASK_BOX_PAD_PX);
const by0 = Math.max(0, Math.floor(box.originY) - MASK_BOX_PAD_PX);
const bx1 = Math.min(
@@ -131,10 +142,10 @@ export function decodeRtmdetIns(
const mx1 = Math.min(mw - 1, mx0 + 1);
const fx = clamp(mx - mx0, 0, 1);
const p =
- out.masks[base + my0 * mw + mx0] * (1 - fx) * (1 - fy) +
- out.masks[base + my0 * mw + mx1] * fx * (1 - fy) +
- out.masks[base + my1 * mw + mx0] * (1 - fx) * fy +
- out.masks[base + my1 * mw + mx1] * fx * fy;
+ data[base + my0 * mw + mx0] * (1 - fx) * (1 - fy) +
+ data[base + my0 * mw + mx1] * fx * (1 - fy) +
+ data[base + my1 * mw + mx0] * (1 - fx) * fy +
+ data[base + my1 * mw + mx1] * fx * fy;
const alpha = alphaOf(p);
if (alpha > 0) {
mask[row + sx] = alpha;
@@ -175,6 +186,26 @@ export function probabilityToAlpha(
return (p) => (p <= lo ? 0 : p >= threshold + feather ? 255 : (p - lo) * inv);
}
+/**
+ * Logit → 0..255 alpha: `probabilityToAlpha` after a sigmoid, which is only
+ * evaluated inside the feather band.
+ */
+export function logitToAlpha(
+ threshold: number,
+ feather: number,
+): (logit: number) => number {
+ const toAlpha = probabilityToAlpha(threshold, feather);
+ const logit = (p: number) =>
+ p <= 0 ? -Infinity : p >= 1 ? Infinity : Math.log(p / (1 - p));
+ if (feather <= 0) {
+ const edge = logit(threshold);
+ return (l) => (l > edge ? 255 : 0);
+ }
+ const lo = logit(threshold - feather);
+ const hi = logit(threshold + feather);
+ return (l) => (l <= lo ? 0 : l >= hi ? 255 : toAlpha(1 / (1 + Math.exp(-l))));
+}
+
function clamp(v: number, lo: number, hi: number): number {
return v < lo ? lo : v > hi ? hi : v;
}
diff --git a/packages/vision/src/model/onnx-subgraph.test.ts b/packages/vision/src/model/onnx-subgraph.test.ts
new file mode 100644
index 00000000..303b7d67
--- /dev/null
+++ b/packages/vision/src/model/onnx-subgraph.test.ts
@@ -0,0 +1,125 @@
+import { describe, expect, it } from "vitest";
+import { extractSubgraph } from "./onnx-subgraph.js";
+
+// A hand-encoded ONNX graph: x → A → a → B → b → C → c, plus an initializer
+// `w` read by B and an unrelated node D → d.
+const enc = new TextEncoder();
+const dec = new TextDecoder();
+
+function varint(n: number): number[] {
+ const out: number[] = [];
+ while (n >= 0x80) {
+ out.push((n % 128) | 0x80);
+ n = Math.floor(n / 128);
+ }
+ out.push(n);
+ return out;
+}
+
+function field(number: number, payload: Uint8Array | string): number[] {
+ const bytes = typeof payload === "string" ? enc.encode(payload) : payload;
+ return [...varint(number * 8 + 2), ...varint(bytes.length), ...bytes];
+}
+
+const bytes = (parts: number[][]) => new Uint8Array(parts.flat());
+
+const node = (op: string, inputs: string[], outputs: string[]) =>
+ bytes([
+ ...inputs.map((i) => field(1, i)),
+ ...outputs.map((o) => field(2, o)),
+ field(4, op),
+ ]);
+
+const valueInfo = (name: string) => bytes([field(1, name)]);
+
+const graph = bytes([
+ field(1, node("A", ["x"], ["a"])),
+ field(1, node("B", ["a", "w"], ["b"])),
+ field(1, node("C", ["b"], ["c"])),
+ field(1, node("D", ["x"], ["d"])),
+ field(2, "g"),
+ field(5, bytes([field(8, "w")])),
+ field(5, bytes([field(8, "unused")])),
+ field(11, valueInfo("x")),
+ field(12, valueInfo("c")),
+ field(12, valueInfo("d")),
+ field(13, valueInfo("b")),
+]);
+// ir_version 8, graph, opset_import { version 17 }.
+const model = new Uint8Array([
+ 0x08,
+ 0x08,
+ ...field(7, graph),
+ ...field(8, new Uint8Array([0x10, 0x11])),
+]);
+
+/** Decodes the length-delimited fields we care about, by number. */
+function read(buf: Uint8Array): Map<number, Uint8Array[]> {
+ const out = new Map<number, Uint8Array[]>();
+ let pos = 0;
+ const v = () => {
+ let n = 0;
+ let s = 1;
+ for (;;) {
+ const b = buf[pos++] as number;
+ n += (b & 0x7f) * s;
+ if (b < 0x80) return n;
+ s *= 128;
+ }
+ };
+ while (pos < buf.length) {
+ const tag = v();
+ if ((tag & 7) === 0) {
+ v();
+ continue;
+ }
+ const len = v();
+ const list = out.get(tag >> 3) ?? [];
+ list.push(buf.subarray(pos, pos + len));
+ out.set(tag >> 3, list);
+ pos += len;
+ }
+ return out;
+}
+
+const names = (list: Uint8Array[] | undefined, nameField: number) =>
+ (list ?? []).map((m) => dec.decode(read(m).get(nameField)?.[0]));
+
+describe("extractSubgraph", () => {
+ it("keeps only what the requested tensors need, and makes them the outputs", () => {
+ const cut = read(extractSubgraph(model, ["b"]));
+ const g = read((cut.get(7) as Uint8Array[])[0] as Uint8Array);
+ expect(
+ (g.get(1) ?? []).map((n) => dec.decode(read(n).get(4)?.[0])),
+ ).toEqual(["A", "B"]);
+ expect(names(g.get(5), 8)).toEqual(["w"]);
+ expect(names(g.get(11), 1)).toEqual(["x"]);
+ expect(names(g.get(12), 1)).toEqual(["b"]);
+ // Other model fields (opset import) and the graph name are kept.
+ expect(cut.get(8)).toHaveLength(1);
+ expect(dec.decode(g.get(2)?.[0])).toBe("g");
+ });
+
+ it("declares outputs without value_info as float tensors", () => {
+ const g = read(
+ (
+ read(extractSubgraph(model, ["a", "c"])).get(7) as Uint8Array[]
+ )[0] as Uint8Array,
+ );
+ const outputs = g.get(12) ?? [];
+ expect(names(outputs, 1)).toEqual(["a", "c"]);
+ // type { tensor_type { elem_type: FLOAT } }
+ const type = read(outputs[0] as Uint8Array).get(2)?.[0] as Uint8Array;
+ expect([...(read(type).get(1)?.[0] ?? [])]).toEqual([0x08, 0x01]);
+ });
+
+ it("rejects tensors the graph doesn't produce", () => {
+ expect(() => extractSubgraph(model, ["nope"])).toThrow(/no tensor 'nope'/);
+ });
+
+ it("rejects bytes that aren't a model", () => {
+ expect(() =>
+ extractSubgraph(new Uint8Array([1, 2, 3, 4]), ["b"]),
+ ).toThrow();
+ });
+});
diff --git a/packages/vision/src/model/onnx-subgraph.ts b/packages/vision/src/model/onnx-subgraph.ts
new file mode 100644
index 00000000..c132d7dd
--- /dev/null
+++ b/packages/vision/src/model/onnx-subgraph.ts
@@ -0,0 +1,225 @@
+/**
+ * Cuts an ONNX model down to the nodes that compute a chosen set of tensors,
+ * which become the graph's outputs. Works on the protobuf wire format directly
+ * (only the fields it needs are decoded; everything else is copied byte for
+ * byte), so it needs no protobuf library and keeps the weights untouched.
+ *
+ * Used to run a detector's fixed-shape body on the GPU and do its
+ * data-dependent tail (top-k, NMS, mask assembly) ourselves.
+ */
+
+// ModelProto.graph, GraphProto.{node,initializer,input,output,value_info},
+// NodeProto.{input,output}, ValueInfoProto.name, TensorProto.name.
+const MODEL_GRAPH = 7;
+const GRAPH_NODE = 1;
+const GRAPH_INITIALIZER = 5;
+const GRAPH_INPUT = 11;
+const GRAPH_OUTPUT = 12;
+const GRAPH_VALUE_INFO = 13;
+const NODE_INPUT = 1;
+const NODE_OUTPUT = 2;
+const VALUE_INFO_NAME = 1;
+const TENSOR_NAME = 8;
+
+const WIRE_VARINT = 0;
+const WIRE_I64 = 1;
+const WIRE_LEN = 2;
+const WIRE_I32 = 5;
+
+interface Field {
+ readonly number: number;
+ readonly wire: number;
+ /** The whole field: tag, length prefix and payload. */
+ readonly raw: Uint8Array;
+ /** Payload of a length-delimited field. */
+ readonly body: Uint8Array;
+}
+
+function readVarint(buf: Uint8Array, pos: number): [number, number] {
+ let value = 0;
+ let scale = 1;
+ for (;;) {
+ if (pos >= buf.length) throw new Error("ONNX model: truncated varint");
+ const byte = buf[pos++] as number;
+ value += (byte & 0x7f) * scale;
+ if (byte < 0x80) return [value, pos];
+ scale *= 128;
+ }
+}
+
+function fields(buf: Uint8Array): Field[] {
+ const out: Field[] = [];
+ let pos = 0;
+ while (pos < buf.length) {
+ const start = pos;
+ const [tag, afterTag] = readVarint(buf, pos);
+ pos = afterTag;
+ const number = Math.floor(tag / 8);
+ const wire = tag & 7;
+ let body = buf.subarray(pos, pos);
+ if (wire === WIRE_VARINT) {
+ pos = readVarint(buf, pos)[1];
+ } else if (wire === WIRE_I64) {
+ pos += 8;
+ } else if (wire === WIRE_I32) {
+ pos += 4;
+ } else if (wire === WIRE_LEN) {
+ const [length, afterLength] = readVarint(buf, pos);
+ body = buf.subarray(afterLength, afterLength + length);
+ pos = afterLength + length;
+ } else {
+ throw new Error(`ONNX model: unsupported wire type ${wire}`);
+ }
+ if (pos > buf.length) throw new Error("ONNX model: truncated field");
+ out.push({ number, wire, raw: buf.subarray(start, pos), body });
+ }
+ return out;
+}
+
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+
+function strings(buf: Uint8Array, number: number): string[] {
+ return fields(buf)
+ .filter((f) => f.number === number && f.wire === WIRE_LEN)
+ .map((f) => decoder.decode(f.body));
+}
+
+function firstString(buf: Uint8Array, number: number): string | undefined {
+ return strings(buf, number)[0];
+}
+
+function varint(value: number): number[] {
+ const bytes: number[] = [];
+ while (value >= 0x80) {
+ bytes.push((value % 128) | 0x80);
+ value = Math.floor(value / 128);
+ }
+ bytes.push(value);
+ return bytes;
+}
+
+function lengthDelimited(number: number, payload: Uint8Array): Uint8Array {
+ const head = [...varint(number * 8 + WIRE_LEN), ...varint(payload.length)];
+ const out = new Uint8Array(head.length + payload.length);
+ out.set(head, 0);
+ out.set(payload, head.length);
+ return out;
+}
+
+function concat(parts: readonly Uint8Array[]): Uint8Array {
+ const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
+ let offset = 0;
+ for (const part of parts) {
+ out.set(part, offset);
+ offset += part.length;
+ }
+ return out;
+}
+
+/** ValueInfoProto for a float tensor of unknown shape. */
+function floatOutput(name: string): Uint8Array {
+ // TypeProto.tensor_type (1) { elem_type (1) = FLOAT (1) }
+ const tensorType = new Uint8Array([0x08, 0x01]);
+ const typeProto = lengthDelimited(1, tensorType);
+ return concat([
+ lengthDelimited(VALUE_INFO_NAME, encoder.encode(name)),
+ lengthDelimited(2, typeProto),
+ ]);
+}
+
+/**
+ * Returns a model whose outputs are `outputs` (tensors of the original graph),
+ * keeping only the nodes, initializers and inputs needed to compute them.
+ * Output types come from the graph's value_info when present; otherwise they
+ * are declared as float tensors.
+ */
+export function extractSubgraph(
+ model: Uint8Array,
+ outputs: readonly string[],
+): Uint8Array {
+ const modelFields = fields(model);
+ const graphField = modelFields.find(
+ (f) => f.number === MODEL_GRAPH && f.wire === WIRE_LEN,
+ );
+ if (!graphField) throw new Error("ONNX model has no graph");
+ const graphFields = fields(graphField.body);
+
+ const nodes = graphFields
+ .filter((f) => f.number === GRAPH_NODE)
+ .map((f) => ({
+ field: f,
+ inputs: strings(f.body, NODE_INPUT),
+ outputs: strings(f.body, NODE_OUTPUT),
+ }));
+ const producer = new Map<string, number>();
+ nodes.forEach((node, index) => {
+ for (const name of node.outputs) producer.set(name, index);
+ });
+
+ // Walk back from the requested tensors.
+ const keepNode = new Set<number>();
+ const needed = new Set<string>();
+ const stack = [...outputs];
+ for (const name of outputs) {
+ if (!producer.has(name)) {
+ throw new Error(`ONNX model has no tensor '${name}'`);
+ }
+ }
+ while (stack.length > 0) {
+ const name = stack.pop() as string;
+ if (needed.has(name)) continue;
+ needed.add(name);
+ const index = producer.get(name);
+ if (index === undefined || keepNode.has(index)) continue;
+ keepNode.add(index);
+ for (const input of (nodes[index] as { inputs: string[] }).inputs) {
+ if (input) stack.push(input);
+ }
+ }
+
+ const valueInfo = new Map<string, Uint8Array>();
+ for (const f of graphFields) {
+ if (f.number !== GRAPH_VALUE_INFO) continue;
+ const name = firstString(f.body, VALUE_INFO_NAME);
+ if (name !== undefined) valueInfo.set(name, f.body);
+ }
+
+ const parts: Uint8Array[] = [];
+ let nodeIndex = 0;
+ for (const f of graphFields) {
+ switch (f.number) {
+ case GRAPH_NODE:
+ if (keepNode.has(nodeIndex)) parts.push(f.raw);
+ nodeIndex++;
+ break;
+ case GRAPH_INITIALIZER: {
+ const name = firstString(f.body, TENSOR_NAME);
+ if (name !== undefined && needed.has(name)) parts.push(f.raw);
+ break;
+ }
+ case GRAPH_INPUT:
+ case GRAPH_VALUE_INFO: {
+ const name = firstString(f.body, VALUE_INFO_NAME);
+ if (name !== undefined && needed.has(name)) parts.push(f.raw);
+ break;
+ }
+ case GRAPH_OUTPUT:
+ break;
+ default:
+ parts.push(f.raw);
+ }
+ }
+ for (const name of outputs) {
+ parts.push(
+ lengthDelimited(GRAPH_OUTPUT, valueInfo.get(name) ?? floatOutput(name)),
+ );
+ }
+
+ const graph = concat(parts);
+ return concat(
+ modelFields.map((f) =>
+ f === graphField ? lengthDelimited(MODEL_GRAPH, graph) : f.raw,
+ ),
+ );
+}
diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.ts
index d1264659..bc5aade3 100644
--- a/packages/vision/src/models.e2e.test.ts
+++ b/packages/vision/src/models.e2e.test.ts
@@ -12,8 +12,11 @@
import { fileURLToPath } from "node:url";
import { Canvas, loadImage } from "skia-canvas";
import { beforeAll, describe, expect, it } from "vitest";
+import { imageToTensor, preprocessSpec } from "./decode/preprocess.js";
+import { decodeRtmdetIns } from "./decode/rtmdet-ins.js";
import { VisionModelStore } from "./model/model-store.js";
import {
+ COCO_CLASSES,
VISION_MODELS,
VISION_VARIANTS,
type VisionModelKey,
@@ -71,6 +74,84 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {
describe.each(VISION_VARIANTS)("variant %s", (variant) => {
const runner = VisionRunner.create({ variant });
+ it("matches the end2end graph (head cut + own NMS and masks)", async () => {
+ const key = runner.modelKeyFor("segment");
+ const ort = await import("onnxruntime-node");
+ const full = await ort.InferenceSession.create(
+ await new VisionModelStore().ensure(key),
+ );
+ const decodeOptions = {
+ confidence: 0.3,
+ classNames: COCO_CLASSES,
+ maskThreshold: 0.5,
+ };
+ for (const name of ["dancer", "crema", "tunnel", "spotlight"] as const) {
+ const frame = frames[name];
+ const spec = preprocessSpec("rtmdet-ins", VISION_MODELS[key].input);
+ const { tensor, transform } = imageToTensor(frame, spec);
+ const out = await full.run({
+ input: new ort.Tensor("float32", tensor, [
+ 1,
+ 3,
+ spec.height,
+ spec.width,
+ ]),
+ });
+ const dets = out.dets as unknown as {
+ data: Float32Array;
+ dims: readonly number[];
+ };
+ const masks = out.masks as unknown as {
+ data: Float32Array;
+ dims: readonly number[];
+ };
+ const expected = decodeRtmdetIns(
+ {
+ dets: dets.data,
+ labels: (out.labels as { data: BigInt64Array }).data,
+ masks: masks.data,
+ count: dets.dims[1] ?? 0,
+ maskHeight: masks.dims[2] ?? 0,
+ maskWidth: masks.dims[3] ?? 0,
+ },
+ {
+ ...decodeOptions,
+ transform,
+ sourceWidth: frame.width,
+ sourceHeight: frame.height,
+ },
+ );
+ const actual = await runner.segment(frame);
+
+ expect(actual.detections.map((d) => d.category)).toEqual(
+ expected.detections.map((d) => d.category),
+ );
+ actual.detections.forEach((d, i) => {
+ const e = expected.detections[i] as (typeof expected.detections)[0];
+ expect(d.score).toBeCloseTo(e.score, 4);
+ expect(d.boundingBox.originX).toBeCloseTo(e.boundingBox.originX, 2);
+ expect(d.boundingBox.width).toBeCloseTo(e.boundingBox.width, 2);
+ });
+ expect(actual.masks).toHaveLength(expected.masks.length);
+ actual.masks.forEach((m, i) => {
+ const e = expected.masks[i] as (typeof expected.masks)[0];
+ let inter = 0;
+ let union = 0;
+ for (let p = 0; p < m.mask.length; p++) {
+ const a = (m.mask[p] as number) >= 128;
+ const b = (e.mask[p] as number) >= 128;
+ if (a && b) inter++;
+ if (a || b) union++;
+ }
+ // Edges differ slightly: end2end upsamples twice (graph ×8, then to
+ // the frame) with the sigmoid in between; the head path upsamples the
+ // logits once, straight to the frame.
+ expect(union === 0 ? 1 : inter / union).toBeGreaterThan(0.98);
+ });
+ }
+ await full.release();
+ }, 600_000);
+
it("cuts out the whole dancer, dress included", async () => {
const seg = await runner.segment(frames.dancer);
const subject = mergeSubjectMask(seg.masks);
diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.ts
index 06167b15..23818f0d 100644
--- a/packages/vision/src/runner/vision-runner.ts
+++ b/packages/vision/src/runner/vision-runner.ts
@@ -4,13 +4,23 @@ import {
imageToTensor,
preprocessSpec,
} from "../decode/preprocess.js";
-import { decodeRtmdetIns } from "../decode/rtmdet-ins.js";
+import {
+ decodeRtmdetIns,
+ type RtmdetInsOutputs,
+} from "../decode/rtmdet-ins.js";
+import {
+ maskLogits,
+ RTMDET_INS_HEAD,
+ type RtmdetInsHeadOutputs,
+ selectInstances,
+} from "../decode/rtmdet-ins-head.js";
import { decodeRtmo } from "../decode/rtmo.js";
import { decodeSelfie } from "../decode/selfie.js";
import {
type ModelDownloadStatus,
VisionModelStore,
} from "../model/model-store.js";
+import { extractSubgraph } from "../model/onnx-subgraph.js";
import {
COCO_CLASSES,
modelKeyFor,
@@ -206,7 +216,7 @@ export class VisionRunner {
): Promise<readonly DetectedObject[]> {
const pass = await this.instancePass(image);
return decodeRtmdetIns(
- { ...pass.outputs, masks: undefined },
+ { ...pass.outputs, masks: undefined, maskPlane: undefined },
this.instanceDecodeOptions(image, pass.transform),
).detections;
}
@@ -278,21 +288,13 @@ export class VisionRunner {
const byContent = passes.find((pass) => pass.content === content);
if (byContent) return byContent.result;
const key = this.modelKeyFor("segment");
- const result = this.infer(key, image).then(({ outputs, transform }) => {
- const dets = output(outputs, "dets", key);
- const masks = output(outputs, "masks", key);
- return {
- transform,
- outputs: {
- dets: dets.data as Float32Array,
- labels: output(outputs, "labels", key).data,
- masks: masks.data as Float32Array,
- count: dets.dims[1] ?? 0,
- maskHeight: masks.dims[masks.dims.length - 2] ?? 0,
- maskWidth: masks.dims[masks.dims.length - 1] ?? 0,
- },
- };
- });
+ const result = this.infer(key, image).then(({ outputs, transform }) => ({
+ transform,
+ outputs:
+ RTMDET_INS_HEAD.scores in outputs
+ ? headPassOutputs(outputs, key, transform)
+ : end2endPassOutputs(outputs, key),
+ }));
const pass = { content, result };
passes.push(pass);
if (passes.length > MAX_INSTANCE_PASSES) passes.shift();
@@ -372,7 +374,13 @@ export class VisionRunner {
if (!promise) {
promise = store
.ensure(key)
- .then((bytes) => provider.createSession(bytes))
+ .then((bytes) =>
+ provider.createSession(
+ VISION_MODELS[key].family === "rtmdet-ins"
+ ? rtmdetInsBody(bytes)
+ : bytes,
+ ),
+ )
.catch((err: unknown) => {
// Drop both the session cache entry AND the cached file: a parse
// failure means the on-disk bytes are corrupt — the next call must
@@ -389,13 +397,84 @@ export class VisionRunner {
interface InstancePass {
readonly transform: InputTransform;
- readonly outputs: {
- readonly dets: Float32Array;
- readonly labels: VisionTensor["data"];
- readonly masks: Float32Array;
- readonly count: number;
- readonly maskWidth: number;
- readonly maskHeight: number;
+ readonly outputs: RtmdetInsOutputs;
+}
+
+/**
+ * The RTMDet-Ins graph up to its head (see `RTMDET_INS_HEAD`): no NMS or
+ * full-size masks in the graph, so the GPU part has fixed shapes and only
+ * ~9 MB comes back. Models without those tensors run end to end.
+ */
+function rtmdetInsBody(bytes: Uint8Array): Uint8Array {
+ try {
+ return extractSubgraph(bytes, Object.values(RTMDET_INS_HEAD));
+ } catch {
+ return bytes;
+ }
+}
+
+/** Outputs of the end2end graph: NMS done, one input-sized mask per detection. */
+function end2endPassOutputs(
+ outputs: VisionTensorOutputs,
+ key: VisionModelKey,
+): RtmdetInsOutputs {
+ const dets = output(outputs, "dets", key);
+ const masks = output(outputs, "masks", key);
+ return {
+ dets: dets.data as Float32Array,
+ labels: output(outputs, "labels", key).data,
+ masks: masks.data as Float32Array,
+ count: dets.dims[1] ?? 0,
+ maskHeight: masks.dims[masks.dims.length - 2] ?? 0,
+ maskWidth: masks.dims[masks.dims.length - 1] ?? 0,
+ };
+}
+
+/** Outputs of the head: selection here, mask logits only for detections asked for. */
+function headPassOutputs(
+ outputs: VisionTensorOutputs,
+ key: VisionModelKey,
+ transform: InputTransform,
+): RtmdetInsOutputs {
+ const scores = output(outputs, RTMDET_INS_HEAD.scores, key);
+ const features = output(outputs, RTMDET_INS_HEAD.maskFeatures, key);
+ const head: RtmdetInsHeadOutputs = {
+ scores: scores.data as Float32Array,
+ boxes: output(outputs, RTMDET_INS_HEAD.boxes, key).data as Float32Array,
+ kernels: output(outputs, RTMDET_INS_HEAD.kernels, key).data as Float32Array,
+ maskFeatures: features.data as Float32Array,
+ anchors: scores.dims[1] ?? 0,
+ classes: scores.dims[2] ?? 0,
+ maskHeight: features.dims[2] ?? 0,
+ maskWidth: features.dims[3] ?? 0,
+ inputWidth: transform.inputWidth,
+ inputHeight: transform.inputHeight,
+ };
+ const picked = selectInstances(head);
+ const dets = new Float32Array(picked.length * 5);
+ const labels = new Int32Array(picked.length);
+ picked.forEach(({ anchor, classIndex, score }, i) => {
+ dets.set(head.boxes.subarray(anchor * 4, anchor * 4 + 4), i * 5);
+ dets[i * 5 + 4] = score;
+ labels[i] = classIndex;
+ });
+ // Shared by every layer that reads this pass: each plane is computed once.
+ const planes = new Map<number, Float32Array>();
+ return {
+ dets,
+ labels,
+ count: picked.length,
+ maskWidth: head.maskWidth,
+ maskHeight: head.maskHeight,
+ maskLogits: true,
+ maskPlane: (index) => {
+ let plane = planes.get(index);
+ if (!plane) {
+ plane = maskLogits(head, (picked[index] as { anchor: number }).anchor);
+ planes.set(index, plane);
+ }
+ return plane;
+ },
};
}