Code-first video & image, rendered natively on WebGPU.
perf(vision): RTMDet-Ins without in-graph NMS and full-size masks
okanasl931435fc6ec7
diff --git a/.changeset/vision-rtmdet-head.md b/.changeset/vision-rtmdet-head.mdnew file mode 100644index 00000000..2cd8e319--- /dev/null+++ b/.changeset/vision-rtmdet-head.md@@ -0,0 +1,10 @@+---+"framefields": patch+"@framefields/vision": patch+---++Instance segmentation does far less work per frame. The RTMDet-Ins export ran NMS on the CPU in the middle of its graph and returned a full 640×640 mask for every detection: about 40 per frame, 65 MB of floats, computed on the GPU and copied back. The runner now cuts the graph at load time before its first data-dependent step and does the rest itself: top-k, per-class NMS with the export's thresholds, and each mask's three dynamic 1×1 layers at 80×80, only for the detections a layer keeps. `detect()` computes no masks at all.++Results match the end2end graph: the same detections, scores and boxes, and masks with IoU above 0.98 on the reference frames (edges differ by one interpolation step). The model file and its SHA-256 check are unchanged; a model without the expected head tensors still runs end to end.++The smoke project's `vision` export on an Apple Silicon Mac: 22.3 s → 15 s.diff --git a/packages/vision/src/decode/rtmdet-ins-head.test.ts b/packages/vision/src/decode/rtmdet-ins-head.test.tsnew file mode 100644index 00000000..9e414c10--- /dev/null+++ b/packages/vision/src/decode/rtmdet-ins-head.test.ts@@ -0,0 +1,118 @@+import { describe, expect, it } from "vitest";+import { logitToAlpha, probabilityToAlpha } from "./rtmdet-ins.js";+import {+ anchorPrior,+ maskLogits,+ type RtmdetInsHeadOutputs,+ selectInstances,+} from "./rtmdet-ins-head.js";++const ANCHORS = 8400;+const CLASSES = 3;++function head(+ entries: { anchor: number; box: number[]; scores: number[] }[],+): RtmdetInsHeadOutputs {+ const scores = new Float32Array(ANCHORS * CLASSES);+ const boxes = new Float32Array(ANCHORS * 4);+ for (const { anchor, box, scores: s } of entries) {+ boxes.set(box, anchor * 4);+ scores.set(s, anchor * CLASSES);+ }+ return {+ scores,+ boxes,+ kernels: new Float32Array(ANCHORS * 169),+ maskFeatures: new Float32Array(8 * 80 * 80),+ anchors: ANCHORS,+ classes: CLASSES,+ maskWidth: 80,+ maskHeight: 80,+ inputWidth: 640,+ inputHeight: 640,+ };+}++describe("selectInstances", () => {+ it("suppresses overlaps per class, keeps other classes, best first", () => {+ const selected = selectInstances(+ head([+ { anchor: 10, box: [0, 0, 100, 100], scores: [0.9, 0, 0] },+ // IoU 0.81 with anchor 10: suppressed for class 0…+ { anchor: 11, box: [0, 0, 100, 90], scores: [0.8, 0, 0.7] },+ // …IoU 0.5: kept.+ { anchor: 12, box: [50, 0, 150, 100], scores: [0.6, 0, 0] },+ // Below the score floor in every class: never selected.+ { anchor: 13, box: [300, 300, 400, 400], scores: [0.05, 0.04, 0] },+ ]),+ );+ expect(selected).toEqual([+ { anchor: 10, classIndex: 0, score: expect.closeTo(0.9, 5) },+ { anchor: 11, classIndex: 2, score: expect.closeTo(0.7, 5) },+ { anchor: 12, classIndex: 0, score: expect.closeTo(0.6, 5) },+ ]);+ });++ it("returns at most 100 detections", () => {+ const entries = Array.from({ length: 150 }, (_, i) => ({+ anchor: i,+ box: [+ (i % 15) * 40,+ Math.floor(i / 15) * 40,+ (i % 15) * 40 + 30,+ Math.floor(i / 15) * 40 + 30,+ ],+ scores: [0.5 + i / 1000, 0, 0],+ }));+ const selected = selectInstances(head(entries));+ expect(selected).toHaveLength(100);+ expect(selected[0]?.anchor).toBe(149);+ });+});++describe("anchorPrior", () => {+ it("walks the stride-8, -16 and -32 grids row by row", () => {+ expect(anchorPrior(0, 640, 640)).toEqual({ x: 0, y: 0, stride: 8 });+ expect(anchorPrior(81, 640, 640)).toEqual({ x: 8, y: 8, stride: 8 });+ expect(anchorPrior(6400, 640, 640)).toEqual({ x: 0, y: 0, stride: 16 });+ expect(anchorPrior(8399, 640, 640)).toEqual({ x: 608, y: 608, stride: 32 });+ expect(() => anchorPrior(8400, 640, 640)).toThrow(RangeError);+ });+});++describe("maskLogits", () => {+ it("runs the three dynamic layers over offsets and features", () => {+ const h = head([]);+ const anchor = 81; // prior (8, 8), stride 8+ const k = h.kernels.subarray(anchor * 169, (anchor + 1) * 169);+ // Layer 1: hidden0 = relu(-x offset), hidden1 = relu(feature 0).+ k[0] = -1; // w1[0][relX]+ k[12] = 1; // w1[1][feature 0]+ // Layer 2: pass both through. Layer 3: out = hidden0 + 2·hidden1 + 0.5.+ k[80] = 1;+ k[80 + 8 + 1] = 1;+ k[144] = 1;+ k[145] = 2;+ k[168] = 0.5;+ h.maskFeatures[3 * 80 + 4] = 0.25; // feature 0 at (x 4, y 3)+ const logits = maskLogits(h, anchor);+ // relX = (8 - 8x) / 64; hidden0 = relu(-(relX)) = max(0, (8x - 8) / 64).+ expect(logits[0]).toBeCloseTo(0.5, 6); // x 0: relX > 0+ expect(logits[3 * 80 + 4]).toBeCloseTo(0.5 + 24 / 64 + 0.5, 6);+ });+});++describe("logitToAlpha", () => {+ it("matches probabilityToAlpha after a sigmoid", () => {+ const fromLogit = logitToAlpha(0.5, 0.05);+ const fromProbability = probabilityToAlpha(0.5, 0.05);+ for (const l of [-8, -0.3, -0.1, 0, 0.05, 0.15, 0.3, 8]) {+ expect(fromLogit(l)).toBeCloseTo(+ fromProbability(1 / (1 + Math.exp(-l))),+ 6,+ );+ }+ const hard = logitToAlpha(0.5, 0);+ expect([hard(-0.01), hard(0.01)]).toEqual([0, 255]);+ });+});diff --git a/packages/vision/src/decode/rtmdet-ins-head.ts b/packages/vision/src/decode/rtmdet-ins-head.tsnew file mode 100644index 00000000..9a6e6737--- /dev/null+++ b/packages/vision/src/decode/rtmdet-ins-head.ts@@ -0,0 +1,232 @@+/**+ * RTMDet-Ins post-processing, done here instead of inside the ONNX graph.+ *+ * The mmdeploy end2end export runs top-k, per-class NMS and mask assembly in+ * the graph and returns a full input-sized mask for each of up to 100+ * detections (40 on a typical frame: ~65 MB of floats). The runner instead+ * cuts the graph before its first data-dependent op (see `RTMDET_INS_HEAD`)+ * and reproduces the tail exactly:+ *+ * 1. top 5000 anchors by their best class score;+ * 2. ONNX NonMaxSuppression per class (IoU > 0.6 suppresses, scores > 0.05,+ * at most 200 per class);+ * 3. the 100 best (class, box) pairs;+ * 4. masks on demand: a 3-layer dynamic 1×1 conv over the 8 mask features+ * plus each pixel's offset from the anchor, at 1/8 of the input size.+ * The logits are upsampled (bilinear) and squashed by the decoder.+ *+ * Constants and layer layout match the export's graph; `models.e2e.test.ts`+ * checks the result against the end2end outputs.+ */++/** Head tensors of the mmdeploy RTMDet-Ins exports (all variants). */+export const RTMDET_INS_HEAD = {+ /** [1, A, 80] class probabilities (sigmoided). */+ scores: "/Sigmoid_output_0",+ /** [1, A, 4] x0, y0, x1, y1 in input pixels. */+ boxes: "/Concat_7_output_0",+ /** [1, A, 169] dynamic conv parameters per anchor. */+ kernels: "/Concat_5_output_0",+ /** [1, 8, H/8, W/8] mask features. */+ maskFeatures: "/bbox_head/mask_head/projection/Conv_output_0",+} as const;++const PRE_NMS = 5000;+// float32, as the graph compares them: a float32 score of 0.05 does not pass.+const IOU_THRESHOLD = Math.fround(0.6);+const SCORE_FLOOR = Math.fround(0.05);+const MAX_PER_CLASS = 200;+const MAX_DETECTIONS = 100;+const STRIDES = [8, 16, 32] as const;+/** Mask features are at stride 8; relative offsets are divided by stride × 8. */+const MASK_STRIDE = 8;+const MASK_CHANNELS = 8;+const KERNEL_SIZE = 169;++export interface RtmdetInsHeadOutputs {+ readonly scores: Float32Array;+ readonly boxes: Float32Array;+ readonly kernels: Float32Array;+ readonly maskFeatures: Float32Array;+ readonly anchors: number;+ readonly classes: number;+ readonly maskWidth: number;+ readonly maskHeight: number;+ /** Model input size; anchor positions derive from it. */+ readonly inputWidth: number;+ readonly inputHeight: number;+}++export interface RtmdetInsInstance {+ readonly anchor: number;+ readonly classIndex: number;+ readonly score: number;+}++/** Steps 1–3: the detections the end2end graph would return, best first. */+export function selectInstances(+ head: RtmdetInsHeadOutputs,+): RtmdetInsInstance[] {+ const { scores, boxes, anchors, classes } = head;++ // 1. Best class score per anchor, then the top PRE_NMS anchors.+ const best = new Float32Array(anchors);+ for (let a = 0; a < anchors; a++) {+ let max = 0;+ const row = a * classes;+ for (let c = 0; c < classes; c++) {+ const s = scores[row + c] as number;+ if (s > max) max = s;+ }+ best[a] = max;+ }+ // Anchors that can't pass the score floor in any class are never selected.+ const order: number[] = [];+ for (let a = 0; a < anchors; a++) {+ if ((best[a] as number) > SCORE_FLOOR) order.push(a);+ }+ order.sort((x, y) => (best[y] as number) - (best[x] as number) || x - y);+ if (order.length > PRE_NMS) order.length = PRE_NMS;++ // 2. Per-class NMS over those anchors.+ const picked: RtmdetInsInstance[] = [];+ const candidates: number[] = [];+ const kept: number[] = [];+ for (let c = 0; c < classes; c++) {+ candidates.length = 0;+ for (const a of order) {+ if ((scores[a * classes + c] as number) > SCORE_FLOOR) candidates.push(a);+ }+ if (candidates.length === 0) continue;+ candidates.sort(+ (x, y) =>+ (scores[y * classes + c] as number) -+ (scores[x * classes + c] as number) || x - y,+ );+ kept.length = 0;+ for (const a of candidates) {+ if (kept.length >= MAX_PER_CLASS) break;+ let suppressed = false;+ for (const k of kept) {+ if (iou(boxes, a, k) > IOU_THRESHOLD) {+ suppressed = true;+ break;+ }+ }+ if (!suppressed) {+ kept.push(a);+ picked.push({+ anchor: a,+ classIndex: c,+ score: scores[a * classes + c] as number,+ });+ }+ }+ }++ // 3. The best MAX_DETECTIONS pairs.+ picked.sort((x, y) => y.score - x.score);+ if (picked.length > MAX_DETECTIONS) picked.length = MAX_DETECTIONS;+ return picked;+}++function iou(boxes: Float32Array, a: number, b: number): number {+ const a0 = boxes[a * 4] as number;+ const a1 = boxes[a * 4 + 1] as number;+ const a2 = boxes[a * 4 + 2] as number;+ const a3 = boxes[a * 4 + 3] as number;+ const b0 = boxes[b * 4] as number;+ const b1 = boxes[b * 4 + 1] as number;+ const b2 = boxes[b * 4 + 2] as number;+ const b3 = boxes[b * 4 + 3] as number;+ const w = Math.min(a2, b2) - Math.max(a0, b0);+ const h = Math.min(a3, b3) - Math.max(a1, b1);+ if (w <= 0 || h <= 0) return 0;+ const inter = w * h;+ const union = (a2 - a0) * (a3 - a1) + (b2 - b0) * (b3 - b1) - inter;+ return union > 0 ? inter / union : 0;+}++/** Anchor position and stride, in input pixels (mmdet point priors, offset 0). */+export function anchorPrior(+ anchor: number,+ inputWidth: number,+ inputHeight: number,+): { x: number; y: number; stride: number } {+ let base = 0;+ for (const stride of STRIDES) {+ const cols = Math.ceil(inputWidth / stride);+ const count = cols * Math.ceil(inputHeight / stride);+ if (anchor < base + count) {+ const i = anchor - base;+ return {+ x: (i % cols) * stride,+ y: Math.floor(i / cols) * stride,+ stride,+ };+ }+ base += count;+ }+ throw new RangeError(`anchor ${anchor} is out of range`);+}++/**+ * Step 4: mask logits for one anchor, `maskHeight × maskWidth`, row-major.+ * Writes into `out` when it is large enough.+ */+export function maskLogits(+ head: RtmdetInsHeadOutputs,+ anchor: number,+ out?: Float32Array,+): Float32Array {+ const { kernels, maskFeatures, maskWidth: mw, maskHeight: mh } = head;+ const plane = mw * mh;+ const logits = out && out.length >= plane ? out : new Float32Array(plane);+ const prior = anchorPrior(anchor, head.inputWidth, head.inputHeight);+ const k = kernels.subarray(anchor * KERNEL_SIZE, (anchor + 1) * KERNEL_SIZE);+ // Layout: w1 [8×10], w2 [8×8], w3 [1×8], b1 [8], b2 [8], b3 [1].+ const w1 = 0;+ const w2 = 80;+ const w3 = 144;+ const b1 = 152;+ const b2 = 160;+ const b3 = 168;+ const scale = 1 / (prior.stride * MASK_STRIDE);+ const input = new Float32Array(2 + MASK_CHANNELS);+ const hidden1 = new Float32Array(8);+ const hidden2 = new Float32Array(8);++ for (let y = 0; y < mh; y++) {+ const relY = (prior.y - y * MASK_STRIDE) * scale;+ for (let x = 0; x < mw; x++) {+ const p = y * mw + x;+ input[0] = (prior.x - x * MASK_STRIDE) * scale;+ input[1] = relY;+ for (let c = 0; c < MASK_CHANNELS; c++) {+ input[2 + c] = maskFeatures[c * plane + p] as number;+ }+ for (let o = 0; o < 8; o++) {+ let sum = k[b1 + o] as number;+ const row = w1 + o * 10;+ for (let i = 0; i < 10; i++) {+ sum += (k[row + i] as number) * (input[i] as number);+ }+ hidden1[o] = sum > 0 ? sum : 0;+ }+ for (let o = 0; o < 8; o++) {+ let sum = k[b2 + o] as number;+ const row = w2 + o * 8;+ for (let i = 0; i < 8; i++) {+ sum += (k[row + i] as number) * (hidden1[i] as number);+ }+ hidden2[o] = sum > 0 ? sum : 0;+ }+ let sum = k[b3] as number;+ for (let i = 0; i < 8; i++) {+ sum += (k[w3 + i] as number) * (hidden2[i] as number);+ }+ logits[p] = sum;+ }+ }+ return logits;+}diff --git a/packages/vision/src/decode/rtmdet-ins.ts b/packages/vision/src/decode/rtmdet-ins.tsindex 9a1ffa50..4fd56256 100644--- a/packages/vision/src/decode/rtmdet-ins.ts+++ b/packages/vision/src/decode/rtmdet-ins.ts@@ -13,8 +13,15 @@ import { type InputTransform, toSourcePoint } from "./preprocess.js";export interface RtmdetInsOutputs {readonly dets: ArrayLike<number>;readonly labels: ArrayLike<number | bigint>;- /** Omit to decode boxes only (detection without masks). */+ /** Omit (and `maskPlane`) to decode boxes only (detection without masks). */readonly masks?: ArrayLike<number>;+ /**+ * Alternative to `masks`: detection `i`'s plane, computed on demand, so only+ * detections that pass the filters cost anything.+ */+ readonly maskPlane?: (index: number) => ArrayLike<number>;+ /** The planes hold logits (sigmoid applied after upsampling), not probabilities. */+ readonly maskLogits?: boolean;readonly count: number;readonly maskWidth: number;readonly maskHeight: number;@@ -88,12 +95,13 @@ export function decodeRtmdetIns(const detections = kept.map((k) => k.detection);const masks: InstanceMask[] = [];- if (!out.masks) return { detections, masks };+ if (!out.masks && !out.maskPlane) return { detections, masks };- const alphaOf = probabilityToAlpha(- opts.maskThreshold ?? 0.5,- opts.featherRadius ?? 0.05,- );+ const threshold = opts.maskThreshold ?? 0.5;+ const feather = opts.featherRadius ?? 0.05;+ const alphaOf = out.maskLogits+ ? logitToAlpha(threshold, feather)+ : probabilityToAlpha(threshold, feather);const { maskWidth: mw, maskHeight: mh } = out;const plane = mw * mh;// Mask grid px per input px (exports may emit masks below input resolution).@@ -103,7 +111,10 @@ export function decodeRtmdetIns(for (let d = 0; d < kept.length; d++) {const { index, detection } = kept[d];const box = detection.boundingBox;- const base = index * plane;+ const data = out.maskPlane+ ? out.maskPlane(index)+ : (out.masks as ArrayLike<number>);+ const base = out.maskPlane ? 0 : index * plane;const bx0 = Math.max(0, Math.floor(box.originX) - MASK_BOX_PAD_PX);const by0 = Math.max(0, Math.floor(box.originY) - MASK_BOX_PAD_PX);const bx1 = Math.min(@@ -131,10 +142,10 @@ export function decodeRtmdetIns(const mx1 = Math.min(mw - 1, mx0 + 1);const fx = clamp(mx - mx0, 0, 1);const p =- out.masks[base + my0 * mw + mx0] * (1 - fx) * (1 - fy) +- out.masks[base + my0 * mw + mx1] * fx * (1 - fy) +- out.masks[base + my1 * mw + mx0] * (1 - fx) * fy +- out.masks[base + my1 * mw + mx1] * fx * fy;+ data[base + my0 * mw + mx0] * (1 - fx) * (1 - fy) ++ data[base + my0 * mw + mx1] * fx * (1 - fy) ++ data[base + my1 * mw + mx0] * (1 - fx) * fy ++ data[base + my1 * mw + mx1] * fx * fy;const alpha = alphaOf(p);if (alpha > 0) {mask[row + sx] = alpha;@@ -175,6 +186,26 @@ export function probabilityToAlpha(return (p) => (p <= lo ? 0 : p >= threshold + feather ? 255 : (p - lo) * inv);}+/**+ * Logit → 0..255 alpha: `probabilityToAlpha` after a sigmoid, which is only+ * evaluated inside the feather band.+ */+export function logitToAlpha(+ threshold: number,+ feather: number,+): (logit: number) => number {+ const toAlpha = probabilityToAlpha(threshold, feather);+ const logit = (p: number) =>+ p <= 0 ? -Infinity : p >= 1 ? Infinity : Math.log(p / (1 - p));+ if (feather <= 0) {+ const edge = logit(threshold);+ return (l) => (l > edge ? 255 : 0);+ }+ const lo = logit(threshold - feather);+ const hi = logit(threshold + feather);+ return (l) => (l <= lo ? 0 : l >= hi ? 255 : toAlpha(1 / (1 + Math.exp(-l))));+}+function clamp(v: number, lo: number, hi: number): number {return v < lo ? lo : v > hi ? hi : v;}diff --git a/packages/vision/src/model/onnx-subgraph.test.ts b/packages/vision/src/model/onnx-subgraph.test.tsnew file mode 100644index 00000000..303b7d67--- /dev/null+++ b/packages/vision/src/model/onnx-subgraph.test.ts@@ -0,0 +1,125 @@+import { describe, expect, it } from "vitest";+import { extractSubgraph } from "./onnx-subgraph.js";++// A hand-encoded ONNX graph: x → A → a → B → b → C → c, plus an initializer+// `w` read by B and an unrelated node D → d.+const enc = new TextEncoder();+const dec = new TextDecoder();++function varint(n: number): number[] {+ const out: number[] = [];+ while (n >= 0x80) {+ out.push((n % 128) | 0x80);+ n = Math.floor(n / 128);+ }+ out.push(n);+ return out;+}++function field(number: number, payload: Uint8Array | string): number[] {+ const bytes = typeof payload === "string" ? enc.encode(payload) : payload;+ return [...varint(number * 8 + 2), ...varint(bytes.length), ...bytes];+}++const bytes = (parts: number[][]) => new Uint8Array(parts.flat());++const node = (op: string, inputs: string[], outputs: string[]) =>+ bytes([+ ...inputs.map((i) => field(1, i)),+ ...outputs.map((o) => field(2, o)),+ field(4, op),+ ]);++const valueInfo = (name: string) => bytes([field(1, name)]);++const graph = bytes([+ field(1, node("A", ["x"], ["a"])),+ field(1, node("B", ["a", "w"], ["b"])),+ field(1, node("C", ["b"], ["c"])),+ field(1, node("D", ["x"], ["d"])),+ field(2, "g"),+ field(5, bytes([field(8, "w")])),+ field(5, bytes([field(8, "unused")])),+ field(11, valueInfo("x")),+ field(12, valueInfo("c")),+ field(12, valueInfo("d")),+ field(13, valueInfo("b")),+]);+// ir_version 8, graph, opset_import { version 17 }.+const model = new Uint8Array([+ 0x08,+ 0x08,+ ...field(7, graph),+ ...field(8, new Uint8Array([0x10, 0x11])),+]);++/** Decodes the length-delimited fields we care about, by number. */+function read(buf: Uint8Array): Map<number, Uint8Array[]> {+ const out = new Map<number, Uint8Array[]>();+ let pos = 0;+ const v = () => {+ let n = 0;+ let s = 1;+ for (;;) {+ const b = buf[pos++] as number;+ n += (b & 0x7f) * s;+ if (b < 0x80) return n;+ s *= 128;+ }+ };+ while (pos < buf.length) {+ const tag = v();+ if ((tag & 7) === 0) {+ v();+ continue;+ }+ const len = v();+ const list = out.get(tag >> 3) ?? [];+ list.push(buf.subarray(pos, pos + len));+ out.set(tag >> 3, list);+ pos += len;+ }+ return out;+}++const names = (list: Uint8Array[] | undefined, nameField: number) =>+ (list ?? []).map((m) => dec.decode(read(m).get(nameField)?.[0]));++describe("extractSubgraph", () => {+ it("keeps only what the requested tensors need, and makes them the outputs", () => {+ const cut = read(extractSubgraph(model, ["b"]));+ const g = read((cut.get(7) as Uint8Array[])[0] as Uint8Array);+ expect(+ (g.get(1) ?? []).map((n) => dec.decode(read(n).get(4)?.[0])),+ ).toEqual(["A", "B"]);+ expect(names(g.get(5), 8)).toEqual(["w"]);+ expect(names(g.get(11), 1)).toEqual(["x"]);+ expect(names(g.get(12), 1)).toEqual(["b"]);+ // Other model fields (opset import) and the graph name are kept.+ expect(cut.get(8)).toHaveLength(1);+ expect(dec.decode(g.get(2)?.[0])).toBe("g");+ });++ it("declares outputs without value_info as float tensors", () => {+ const g = read(+ (+ read(extractSubgraph(model, ["a", "c"])).get(7) as Uint8Array[]+ )[0] as Uint8Array,+ );+ const outputs = g.get(12) ?? [];+ expect(names(outputs, 1)).toEqual(["a", "c"]);+ // type { tensor_type { elem_type: FLOAT } }+ const type = read(outputs[0] as Uint8Array).get(2)?.[0] as Uint8Array;+ expect([...(read(type).get(1)?.[0] ?? [])]).toEqual([0x08, 0x01]);+ });++ it("rejects tensors the graph doesn't produce", () => {+ expect(() => extractSubgraph(model, ["nope"])).toThrow(/no tensor 'nope'/);+ });++ it("rejects bytes that aren't a model", () => {+ expect(() =>+ extractSubgraph(new Uint8Array([1, 2, 3, 4]), ["b"]),+ ).toThrow();+ });+});diff --git a/packages/vision/src/model/onnx-subgraph.ts b/packages/vision/src/model/onnx-subgraph.tsnew file mode 100644index 00000000..c132d7dd--- /dev/null+++ b/packages/vision/src/model/onnx-subgraph.ts@@ -0,0 +1,225 @@+/**+ * Cuts an ONNX model down to the nodes that compute a chosen set of tensors,+ * which become the graph's outputs. Works on the protobuf wire format directly+ * (only the fields it needs are decoded; everything else is copied byte for+ * byte), so it needs no protobuf library and keeps the weights untouched.+ *+ * Used to run a detector's fixed-shape body on the GPU and do its+ * data-dependent tail (top-k, NMS, mask assembly) ourselves.+ */++// ModelProto.graph, GraphProto.{node,initializer,input,output,value_info},+// NodeProto.{input,output}, ValueInfoProto.name, TensorProto.name.+const MODEL_GRAPH = 7;+const GRAPH_NODE = 1;+const GRAPH_INITIALIZER = 5;+const GRAPH_INPUT = 11;+const GRAPH_OUTPUT = 12;+const GRAPH_VALUE_INFO = 13;+const NODE_INPUT = 1;+const NODE_OUTPUT = 2;+const VALUE_INFO_NAME = 1;+const TENSOR_NAME = 8;++const WIRE_VARINT = 0;+const WIRE_I64 = 1;+const WIRE_LEN = 2;+const WIRE_I32 = 5;++interface Field {+ readonly number: number;+ readonly wire: number;+ /** The whole field: tag, length prefix and payload. */+ readonly raw: Uint8Array;+ /** Payload of a length-delimited field. */+ readonly body: Uint8Array;+}++function readVarint(buf: Uint8Array, pos: number): [number, number] {+ let value = 0;+ let scale = 1;+ for (;;) {+ if (pos >= buf.length) throw new Error("ONNX model: truncated varint");+ const byte = buf[pos++] as number;+ value += (byte & 0x7f) * scale;+ if (byte < 0x80) return [value, pos];+ scale *= 128;+ }+}++function fields(buf: Uint8Array): Field[] {+ const out: Field[] = [];+ let pos = 0;+ while (pos < buf.length) {+ const start = pos;+ const [tag, afterTag] = readVarint(buf, pos);+ pos = afterTag;+ const number = Math.floor(tag / 8);+ const wire = tag & 7;+ let body = buf.subarray(pos, pos);+ if (wire === WIRE_VARINT) {+ pos = readVarint(buf, pos)[1];+ } else if (wire === WIRE_I64) {+ pos += 8;+ } else if (wire === WIRE_I32) {+ pos += 4;+ } else if (wire === WIRE_LEN) {+ const [length, afterLength] = readVarint(buf, pos);+ body = buf.subarray(afterLength, afterLength + length);+ pos = afterLength + length;+ } else {+ throw new Error(`ONNX model: unsupported wire type ${wire}`);+ }+ if (pos > buf.length) throw new Error("ONNX model: truncated field");+ out.push({ number, wire, raw: buf.subarray(start, pos), body });+ }+ return out;+}++const decoder = new TextDecoder();+const encoder = new TextEncoder();++function strings(buf: Uint8Array, number: number): string[] {+ return fields(buf)+ .filter((f) => f.number === number && f.wire === WIRE_LEN)+ .map((f) => decoder.decode(f.body));+}++function firstString(buf: Uint8Array, number: number): string | undefined {+ return strings(buf, number)[0];+}++function varint(value: number): number[] {+ const bytes: number[] = [];+ while (value >= 0x80) {+ bytes.push((value % 128) | 0x80);+ value = Math.floor(value / 128);+ }+ bytes.push(value);+ return bytes;+}++function lengthDelimited(number: number, payload: Uint8Array): Uint8Array {+ const head = [...varint(number * 8 + WIRE_LEN), ...varint(payload.length)];+ const out = new Uint8Array(head.length + payload.length);+ out.set(head, 0);+ out.set(payload, head.length);+ return out;+}++function concat(parts: readonly Uint8Array[]): Uint8Array {+ const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));+ let offset = 0;+ for (const part of parts) {+ out.set(part, offset);+ offset += part.length;+ }+ return out;+}++/** ValueInfoProto for a float tensor of unknown shape. */+function floatOutput(name: string): Uint8Array {+ // TypeProto.tensor_type (1) { elem_type (1) = FLOAT (1) }+ const tensorType = new Uint8Array([0x08, 0x01]);+ const typeProto = lengthDelimited(1, tensorType);+ return concat([+ lengthDelimited(VALUE_INFO_NAME, encoder.encode(name)),+ lengthDelimited(2, typeProto),+ ]);+}++/**+ * Returns a model whose outputs are `outputs` (tensors of the original graph),+ * keeping only the nodes, initializers and inputs needed to compute them.+ * Output types come from the graph's value_info when present; otherwise they+ * are declared as float tensors.+ */+export function extractSubgraph(+ model: Uint8Array,+ outputs: readonly string[],+): Uint8Array {+ const modelFields = fields(model);+ const graphField = modelFields.find(+ (f) => f.number === MODEL_GRAPH && f.wire === WIRE_LEN,+ );+ if (!graphField) throw new Error("ONNX model has no graph");+ const graphFields = fields(graphField.body);++ const nodes = graphFields+ .filter((f) => f.number === GRAPH_NODE)+ .map((f) => ({+ field: f,+ inputs: strings(f.body, NODE_INPUT),+ outputs: strings(f.body, NODE_OUTPUT),+ }));+ const producer = new Map<string, number>();+ nodes.forEach((node, index) => {+ for (const name of node.outputs) producer.set(name, index);+ });++ // Walk back from the requested tensors.+ const keepNode = new Set<number>();+ const needed = new Set<string>();+ const stack = [...outputs];+ for (const name of outputs) {+ if (!producer.has(name)) {+ throw new Error(`ONNX model has no tensor '${name}'`);+ }+ }+ while (stack.length > 0) {+ const name = stack.pop() as string;+ if (needed.has(name)) continue;+ needed.add(name);+ const index = producer.get(name);+ if (index === undefined || keepNode.has(index)) continue;+ keepNode.add(index);+ for (const input of (nodes[index] as { inputs: string[] }).inputs) {+ if (input) stack.push(input);+ }+ }++ const valueInfo = new Map<string, Uint8Array>();+ for (const f of graphFields) {+ if (f.number !== GRAPH_VALUE_INFO) continue;+ const name = firstString(f.body, VALUE_INFO_NAME);+ if (name !== undefined) valueInfo.set(name, f.body);+ }++ const parts: Uint8Array[] = [];+ let nodeIndex = 0;+ for (const f of graphFields) {+ switch (f.number) {+ case GRAPH_NODE:+ if (keepNode.has(nodeIndex)) parts.push(f.raw);+ nodeIndex++;+ break;+ case GRAPH_INITIALIZER: {+ const name = firstString(f.body, TENSOR_NAME);+ if (name !== undefined && needed.has(name)) parts.push(f.raw);+ break;+ }+ case GRAPH_INPUT:+ case GRAPH_VALUE_INFO: {+ const name = firstString(f.body, VALUE_INFO_NAME);+ if (name !== undefined && needed.has(name)) parts.push(f.raw);+ break;+ }+ case GRAPH_OUTPUT:+ break;+ default:+ parts.push(f.raw);+ }+ }+ for (const name of outputs) {+ parts.push(+ lengthDelimited(GRAPH_OUTPUT, valueInfo.get(name) ?? floatOutput(name)),+ );+ }++ const graph = concat(parts);+ return concat(+ modelFields.map((f) =>+ f === graphField ? lengthDelimited(MODEL_GRAPH, graph) : f.raw,+ ),+ );+}diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.tsindex d1264659..bc5aade3 100644--- a/packages/vision/src/models.e2e.test.ts+++ b/packages/vision/src/models.e2e.test.ts@@ -12,8 +12,11 @@import { fileURLToPath } from "node:url";import { Canvas, loadImage } from "skia-canvas";import { beforeAll, describe, expect, it } from "vitest";+import { imageToTensor, preprocessSpec } from "./decode/preprocess.js";+import { decodeRtmdetIns } from "./decode/rtmdet-ins.js";import { VisionModelStore } from "./model/model-store.js";import {+ COCO_CLASSES,VISION_MODELS,VISION_VARIANTS,type VisionModelKey,@@ -71,6 +74,84 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {describe.each(VISION_VARIANTS)("variant %s", (variant) => {const runner = VisionRunner.create({ variant });+ it("matches the end2end graph (head cut + own NMS and masks)", async () => {+ const key = runner.modelKeyFor("segment");+ const ort = await import("onnxruntime-node");+ const full = await ort.InferenceSession.create(+ await new VisionModelStore().ensure(key),+ );+ const decodeOptions = {+ confidence: 0.3,+ classNames: COCO_CLASSES,+ maskThreshold: 0.5,+ };+ for (const name of ["dancer", "crema", "tunnel", "spotlight"] as const) {+ const frame = frames[name];+ const spec = preprocessSpec("rtmdet-ins", VISION_MODELS[key].input);+ const { tensor, transform } = imageToTensor(frame, spec);+ const out = await full.run({+ input: new ort.Tensor("float32", tensor, [+ 1,+ 3,+ spec.height,+ spec.width,+ ]),+ });+ const dets = out.dets as unknown as {+ data: Float32Array;+ dims: readonly number[];+ };+ const masks = out.masks as unknown as {+ data: Float32Array;+ dims: readonly number[];+ };+ const expected = decodeRtmdetIns(+ {+ dets: dets.data,+ labels: (out.labels as { data: BigInt64Array }).data,+ masks: masks.data,+ count: dets.dims[1] ?? 0,+ maskHeight: masks.dims[2] ?? 0,+ maskWidth: masks.dims[3] ?? 0,+ },+ {+ ...decodeOptions,+ transform,+ sourceWidth: frame.width,+ sourceHeight: frame.height,+ },+ );+ const actual = await runner.segment(frame);++ expect(actual.detections.map((d) => d.category)).toEqual(+ expected.detections.map((d) => d.category),+ );+ actual.detections.forEach((d, i) => {+ const e = expected.detections[i] as (typeof expected.detections)[0];+ expect(d.score).toBeCloseTo(e.score, 4);+ expect(d.boundingBox.originX).toBeCloseTo(e.boundingBox.originX, 2);+ expect(d.boundingBox.width).toBeCloseTo(e.boundingBox.width, 2);+ });+ expect(actual.masks).toHaveLength(expected.masks.length);+ actual.masks.forEach((m, i) => {+ const e = expected.masks[i] as (typeof expected.masks)[0];+ let inter = 0;+ let union = 0;+ for (let p = 0; p < m.mask.length; p++) {+ const a = (m.mask[p] as number) >= 128;+ const b = (e.mask[p] as number) >= 128;+ if (a && b) inter++;+ if (a || b) union++;+ }+ // Edges differ slightly: end2end upsamples twice (graph ×8, then to+ // the frame) with the sigmoid in between; the head path upsamples the+ // logits once, straight to the frame.+ expect(union === 0 ? 1 : inter / union).toBeGreaterThan(0.98);+ });+ }+ await full.release();+ }, 600_000);+it("cuts out the whole dancer, dress included", async () => {const seg = await runner.segment(frames.dancer);const subject = mergeSubjectMask(seg.masks);diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.tsindex 06167b15..23818f0d 100644--- a/packages/vision/src/runner/vision-runner.ts+++ b/packages/vision/src/runner/vision-runner.ts@@ -4,13 +4,23 @@ import {imageToTensor,preprocessSpec,} from "../decode/preprocess.js";-import { decodeRtmdetIns } from "../decode/rtmdet-ins.js";+import {+ decodeRtmdetIns,+ type RtmdetInsOutputs,+} from "../decode/rtmdet-ins.js";+import {+ maskLogits,+ RTMDET_INS_HEAD,+ type RtmdetInsHeadOutputs,+ selectInstances,+} from "../decode/rtmdet-ins-head.js";import { decodeRtmo } from "../decode/rtmo.js";import { decodeSelfie } from "../decode/selfie.js";import {type ModelDownloadStatus,VisionModelStore,} from "../model/model-store.js";+import { extractSubgraph } from "../model/onnx-subgraph.js";import {COCO_CLASSES,modelKeyFor,@@ -206,7 +216,7 @@ export class VisionRunner {): Promise<readonly DetectedObject[]> {const pass = await this.instancePass(image);return decodeRtmdetIns(- { ...pass.outputs, masks: undefined },+ { ...pass.outputs, masks: undefined, maskPlane: undefined },this.instanceDecodeOptions(image, pass.transform),).detections;}@@ -278,21 +288,13 @@ export class VisionRunner {const byContent = passes.find((pass) => pass.content === content);if (byContent) return byContent.result;const key = this.modelKeyFor("segment");- const result = this.infer(key, image).then(({ outputs, transform }) => {- const dets = output(outputs, "dets", key);- const masks = output(outputs, "masks", key);- return {- transform,- outputs: {- dets: dets.data as Float32Array,- labels: output(outputs, "labels", key).data,- masks: masks.data as Float32Array,- count: dets.dims[1] ?? 0,- maskHeight: masks.dims[masks.dims.length - 2] ?? 0,- maskWidth: masks.dims[masks.dims.length - 1] ?? 0,- },- };- });+ const result = this.infer(key, image).then(({ outputs, transform }) => ({+ transform,+ outputs:+ RTMDET_INS_HEAD.scores in outputs+ ? headPassOutputs(outputs, key, transform)+ : end2endPassOutputs(outputs, key),+ }));const pass = { content, result };passes.push(pass);if (passes.length > MAX_INSTANCE_PASSES) passes.shift();@@ -372,7 +374,13 @@ export class VisionRunner {if (!promise) {promise = store.ensure(key)- .then((bytes) => provider.createSession(bytes))+ .then((bytes) =>+ provider.createSession(+ VISION_MODELS[key].family === "rtmdet-ins"+ ? rtmdetInsBody(bytes)+ : bytes,+ ),+ ).catch((err: unknown) => {// Drop both the session cache entry AND the cached file: a parse// failure means the on-disk bytes are corrupt — the next call must@@ -389,13 +397,84 @@ export class VisionRunner {interface InstancePass {readonly transform: InputTransform;- readonly outputs: {- readonly dets: Float32Array;- readonly labels: VisionTensor["data"];- readonly masks: Float32Array;- readonly count: number;- readonly maskWidth: number;- readonly maskHeight: number;+ readonly outputs: RtmdetInsOutputs;+}++/**+ * The RTMDet-Ins graph up to its head (see `RTMDET_INS_HEAD`): no NMS or+ * full-size masks in the graph, so the GPU part has fixed shapes and only+ * ~9 MB comes back. Models without those tensors run end to end.+ */+function rtmdetInsBody(bytes: Uint8Array): Uint8Array {+ try {+ return extractSubgraph(bytes, Object.values(RTMDET_INS_HEAD));+ } catch {+ return bytes;+ }+}++/** Outputs of the end2end graph: NMS done, one input-sized mask per detection. */+function end2endPassOutputs(+ outputs: VisionTensorOutputs,+ key: VisionModelKey,+): RtmdetInsOutputs {+ const dets = output(outputs, "dets", key);+ const masks = output(outputs, "masks", key);+ return {+ dets: dets.data as Float32Array,+ labels: output(outputs, "labels", key).data,+ masks: masks.data as Float32Array,+ count: dets.dims[1] ?? 0,+ maskHeight: masks.dims[masks.dims.length - 2] ?? 0,+ maskWidth: masks.dims[masks.dims.length - 1] ?? 0,+ };+}++/** Outputs of the head: selection here, mask logits only for detections asked for. */+function headPassOutputs(+ outputs: VisionTensorOutputs,+ key: VisionModelKey,+ transform: InputTransform,+): RtmdetInsOutputs {+ const scores = output(outputs, RTMDET_INS_HEAD.scores, key);+ const features = output(outputs, RTMDET_INS_HEAD.maskFeatures, key);+ const head: RtmdetInsHeadOutputs = {+ scores: scores.data as Float32Array,+ boxes: output(outputs, RTMDET_INS_HEAD.boxes, key).data as Float32Array,+ kernels: output(outputs, RTMDET_INS_HEAD.kernels, key).data as Float32Array,+ maskFeatures: features.data as Float32Array,+ anchors: scores.dims[1] ?? 0,+ classes: scores.dims[2] ?? 0,+ maskHeight: features.dims[2] ?? 0,+ maskWidth: features.dims[3] ?? 0,+ inputWidth: transform.inputWidth,+ inputHeight: transform.inputHeight,+ };+ const picked = selectInstances(head);+ const dets = new Float32Array(picked.length * 5);+ const labels = new Int32Array(picked.length);+ picked.forEach(({ anchor, classIndex, score }, i) => {+ dets.set(head.boxes.subarray(anchor * 4, anchor * 4 + 4), i * 5);+ dets[i * 5 + 4] = score;+ labels[i] = classIndex;+ });+ // Shared by every layer that reads this pass: each plane is computed once.+ const planes = new Map<number, Float32Array>();+ return {+ dets,+ labels,+ count: picked.length,+ maskWidth: head.maskWidth,+ maskHeight: head.maskHeight,+ maskLogits: true,+ maskPlane: (index) => {+ let plane = planes.get(index);+ if (!plane) {+ plane = maskLogits(head, (picked[index] as { anchor: number }).anchor);+ planes.set(index, plane);+ }+ return plane;+ },};}