Code-first video & image, rendered natively on WebGPU.
feat(vision): soft alpha matting for clip subjects (ViTMatte Small)
okanasl52b7a616598c
diff --git a/.changeset/vision-alpha-matting.md b/.changeset/vision-alpha-matting.mdnew file mode 100644index 00000000..a3e155ce--- /dev/null+++ b/.changeset/vision-alpha-matting.md@@ -0,0 +1,5 @@+---+"framefields": minor+---++`comp.vision(clip, { matting: true })` resolves the subject's edge to soft alpha with ViTMatte Small, so cutouts keep hair, motion blur and see-through parts that a hard mask cuts off. It refines the automatic subject or the `sam2` / `sam3` tracks: one model pass per frame over an unknown band either side of the mask's edge (`band`, default about 1.2% of the short side), with each prompted track keeping its own alpha. Matte layers, `signals` masks and the clip's subject read the soft alpha. Mattes persist beside the masks, so toggling matting never re-runs SAM. The 104 MB model downloads on first use. `VisionRunner.refineMatte(image, mask)` exposes the same pass.diff --git a/packages/framefields/NOTICE.md b/packages/framefields/NOTICE.mdindex ee177004..920cffbc 100644--- a/packages/framefields/NOTICE.md+++ b/packages/framefields/NOTICE.md@@ -16,3 +16,7 @@ Before SAM 3 weights are downloaded or run, you must accept Meta's SAM License (## Depth`comp.vision(clip, { depth: true })` and `withVision({ mode: "depth" })` estimate depth with Depth Anything V2 **Small** (onnx-community export of `depth-anything/Depth-Anything-V2-Small`), whose weights are Apache-2.0. The larger Depth Anything V2 checkpoints are licensed CC-BY-NC-4.0 and are not used.++## Alpha matting++`comp.vision(clip, { matting: true })` and `VisionRunner.refineMatte` resolve mask edges with ViTMatte **Small** trained on Distinctions-646 (Xenova ONNX export of `hustvl/vitmatte-small-distinctions-646`). Its weights are Apache-2.0 and the ViTMatte code is MIT.diff --git a/packages/framefields/src/clip-vision-matte.test.ts b/packages/framefields/src/clip-vision-matte.test.tsindex 6551f150..7502ea82 100644--- a/packages/framefields/src/clip-vision-matte.test.ts+++ b/packages/framefields/src/clip-vision-matte.test.ts@@ -5,9 +5,14 @@import fs from "node:fs/promises";import os from "node:os";import path from "node:path";-import { CLIP_SUBJECT } from "@framefields/vision";+import {+ CLIP_SUBJECT,+ type InstanceMask,+ trimapFromMask,+ type VisionRunner,+} from "@framefields/vision";import { describe, expect, it } from "vitest";-import { seedClipVision } from "./clip-vision.js";+import { type ClipVision, seedClipVision } from "./clip-vision.js";import { Composition, HeadlessMediaRenderer, Layer } from "./index.js";import { decodePngPixels, getPixel } from "./test-utils/pixels.js";@@ -252,3 +257,128 @@ describe("matte reuses comp.vision", () => {expect(linked(other as never)).toBe(false);});});++/** Stands in for ViTMatte: every pixel of the trimap's unknown band gets alpha 128. */+function stubMatting(clip: ClipVision): { calls: number } {+ const count = { calls: 0 };+ const runner: Pick<VisionRunner, "refineMatte" | "close"> = {+ refineMatte: async (image, mask, options) => {+ count.calls++;+ const trimap = trimapFromMask(+ mask,+ image.width,+ image.height,+ options?.band ?? 4,+ );+ return {+ mask: trimap.data,+ width: image.width,+ height: image.height,+ nearest: trimap.nearest,+ };+ },+ close: () => undefined,+ };+ (clip as unknown as { runner: typeof runner }).runner = runner;+ return count;+}++const maskAt = (clip: ClipVision, frame: number, id?: number) =>+ (+ clip as unknown as {+ maskAt(frame: number, id?: number): InstanceMask | undefined;+ }+ ).maskAt(frame, id);++describe("matting", () => {+ /** Track 1: x < 50; track 2: x ≥ 150. */+ function twoTracks(): Uint8Array {+ const labels = new Uint8Array(SIZE * SIZE);+ for (let y = 0; y < SIZE; y++) {+ labels.fill(1, y * SIZE, y * SIZE + 50);+ labels.fill(2, y * SIZE + 150, y * SIZE + SIZE);+ }+ return labels;+ }++ const prompted = (cacheDir: string) => ({+ cacheDir,+ matting: { band: 4 },+ sam2: { prompts: [{ frame: 0, points: [{ x: 0.1, y: 0.5, label: 1 }] }] },+ });++ it("gives each track its own soft edge, and caches mattes apart from masks", async () => {+ const file = await greenClip();+ const cacheDir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-matting-"));+ const seed = (clip: ClipVision) =>+ seedClipVision(+ clip,+ { width: SIZE, height: SIZE },+ Array.from({ length: 10 }, () => ({+ kp: null,+ mask: null,+ tracks: twoTracks(),+ })),+ );+ const clip = seed(stage().vision(file, prompted(cacheDir)));+ const stub = stubMatting(clip);+ await clip.ready(0, 3);+ expect(stub.calls).toBe(4);+ expect(clip.isReady(3)).toBe(true);+ expect(clip.isReady(4)).toBe(false);++ const row = 100 * SIZE;+ const subject = clip.subject(0, FPS)?.mask;+ expect(subject?.[row + 10]).toBe(255);+ expect(subject?.[row + 52]).toBe(128);+ expect(subject?.[row + 100]).toBe(0);+ // The band past track 1's edge belongs to track 1 only, and likewise for track 2.+ expect(maskAt(clip, 0, 1)?.mask[row + 52]).toBe(128);+ expect(maskAt(clip, 0, 1)?.mask[row + 148]).toBe(0);+ expect(maskAt(clip, 0, 2)?.mask[row + 148]).toBe(128);+ expect(maskAt(clip, 0, 2)?.mask[row + 52]).toBe(0);++ // A second run reads the mattes from disk.+ const again = seed(stage().vision(file, prompted(cacheDir)));+ const rerun = stubMatting(again);+ await again.ready(0, 3);+ expect(rerun.calls).toBe(0);+ expect(again.subject(0, FPS)?.mask[row + 52]).toBe(128);++ // Without matting the same clip keeps its hard masks.+ const hard = seed(+ stage().vision(file, { ...prompted(cacheDir), matting: false }),+ );+ await hard.ready(0, 3);+ expect(hard.subject(0, FPS)?.mask[row + 52]).toBe(0);+ });++ it("cuts a matte layer out with the soft edge", async () => {+ const file = await greenClip();+ const c = stage("#0000ff");+ const layer = cutout(file);+ c.add(layer);+ const clip = c.vision(layer, {+ cacheDir: false,+ smoothing: 0,+ matting: { band: 4 },+ });+ seedClipVision(+ clip,+ { width: SIZE, height: SIZE },+ Array.from({ length: 40 }, () => ({+ kp: null,+ mask: null,+ subject: leftHalf(),+ })),+ );+ stubMatting(clip);+ const image = await decodePngPixels(await c.renderFrame({ frame: 0 }));+ expect(getPixel(image, 50, 100).g).toBeGreaterThan(240);+ const edge = getPixel(image, 101, 100);+ expect(edge.g).toBeGreaterThan(90);+ expect(edge.g).toBeLessThan(170);+ expect(edge.b).toBeGreaterThan(90);+ expect(getPixel(image, 150, 100).b).toBeGreaterThan(240);+ });+});diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex 852babb3..c0413eab 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -11,7 +11,9 @@ import {type ClipSubjectSource,COCO17_KEYPOINT_NAMES,type DepthMap,+ defaultMatteBand,type InstanceMask,+ type MatteRefineOptions,maskObject,mergeSubjectMask,VisionBundle,@@ -60,6 +62,14 @@ export interface ClipVisionOptions {* clip tracks no pose or silhouette unless `pose` / `mask` ask for them.*/depth?: boolean;+ /**+ * Resolve the subject's edge to soft alpha (ViTMatte Small): hair, motion+ * blur and see-through parts that a hard mask cuts off. `true`, or+ * `{ band, resolution }` (see `MatteRefineOptions`). Matte layers, `signals`+ * masks and `subject` read the soft alpha; each prompted track keeps its own.+ * Mattes persist apart from the masks, so turning this on reuses them.+ */+ matting?: boolean | MatteRefineOptions;/** Silhouette cell size in source pixels (default 8: a 1920×1080 clip gives a 240×135 grid). */maskCell?: number;/** Keypoints are averaged over ±`smoothing` frames, weighted by confidence (default 2; 0 = raw). */@@ -107,6 +117,26 @@ interface SubjectResult {scores?: number[];}+/** A frame's soft alpha, run-length encoded, as matting resolved it. */+interface MatteResult {+ alpha: Runs;+ /**+ * Prompted frames only: the track label map grown into the soft edge+ * (each edge pixel takes its nearest track's label), so every track keeps its own alpha.+ */+ labels?: Runs;+}++interface Runs {+ values: Uint8Array;+ lengths: Uint32Array;+}++interface StoredRuns {+ values: string;+ lengths: string;+}+/** A subject as written to the subjects cache file. */interface StoredSubject {category: string;@@ -128,6 +158,8 @@ const HOLD = 24;const CACHE_VERSION = 1;const DEPTH_CACHE_VERSION = 1;+const MATTE_CACHE_VERSION = 1;+const MATTE_MODEL = "vitmatte-s";const DEPTH_MODEL = "depth-anything-v2-s";/** Depth frames held in memory when they also live on disk (each is about 1 MB). */const DEPTH_MEMORY = 48;@@ -204,6 +236,10 @@ export class ClipVision implements ClipSubjectSource {private readonly subjects = new Map<number, SubjectResult | null>();private wantSubjects = false;private subjectsLoaded?: Promise<void>;+ /** Soft alpha per source frame (`matting`); null when the frame holds no subject. */+ private readonly mattes = new Map<number, MatteResult | null>();+ private mattesLoaded?: Promise<void>;+ private unsavedMattes = 0;/** Depth per source frame; null when the frame did not decode. Bounded when on disk. */private readonly depths = new Map<number, StoredDepth | null>();/** Source frames whose depth is on disk. */@@ -264,7 +300,8 @@ export class ClipVision implements ClipSubjectSource {Math.round(options.smoothing ?? (options.sam2 || options.sam3 ? 0 : 2)),);this.cell = Math.max(1, Math.round(options.maskCell ?? 8));- if (options.sam2 || options.sam3) this.wantSubjects = true;+ if (options.sam2 || options.sam3 || this.tracksMatting)+ this.wantSubjects = true;}/** @internal Explicit selections must never fall back to another detected subject. */@@ -374,9 +411,11 @@ export class ClipVision implements ClipSubjectSource {if (s === null || !this.sourceWidth) return null;const width = this.sourceWidth;const height = this.sourceHeight;+ const mask = this.alpha(sourceFrame, s);+ if (!mask) return undefined;return {category: s.category,- mask: subjectAlpha(s, width * height),+ mask,width,height,area: s.area,@@ -421,6 +460,43 @@ export class ClipVision implements ClipSubjectSource {return this.options.mask ?? !this.tracksDepth;}+ /** Whether subjects are matted to soft alpha. */+ get tracksMatting(): boolean {+ return this.options.matting !== undefined && this.options.matting !== false;+ }++ private get matteSettings(): Required<MatteRefineOptions> | undefined {+ const m = this.options.matting;+ if (!m) return undefined;+ const o = m === true ? {} : m;+ const band =+ o.band ??+ (this.sourceWidth+ ? defaultMatteBand(this.sourceWidth, this.sourceHeight)+ : 0);+ return { band, resolution: o.resolution ?? 1024 };+ }++ /**+ * A stored subject's alpha at a source frame: the matte when the clip mattes+ * (undefined until matted), else the mask. With `id`, that track only.+ */+ private alpha(+ c: number,+ stored: SubjectResult,+ id?: number,+ ): Uint8Array | undefined {+ const size = this.sourceWidth * this.sourceHeight;+ if (!this.tracksMatting) return subjectAlpha(stored, size, id);+ const matte = this.mattes.get(c);+ if (!matte) return undefined;+ const alpha = decodeRuns(matte.alpha, size);+ if (id === undefined || !matte.labels) return alpha;+ const labels = decodeRuns(matte.labels, size);+ for (let i = 0; i < size; i++) if (labels[i] !== id) alpha[i] = 0;+ return alpha;+ }+/** Whether frames carry relative depth. */get tracksDepth(): boolean {return this.options.depth === true;@@ -512,7 +588,11 @@ export class ClipVision implements ClipSubjectSource {}private analysed(c: number): boolean {- return this.analysedSubject(c) && (!this.tracksDepth || this.hasDepth(c));+ return (+ this.analysedSubject(c) &&+ (!this.tracksDepth || this.hasDepth(c)) &&+ (!this.tracksMatting || this.mattes.has(c))+ );}/** Everything but depth, which is a pass of its own. */@@ -539,6 +619,12 @@ export class ClipVision implements ClipSubjectSource {await this.subjectsLoaded;}if (this.tracksDepth) await this.computeDepth(from, to);+ await this.computeSubjects(from, to);+ if (this.tracksMatting) await this.computeMattes(from, to);+ }++ /** Pose, silhouette and subject of every frame in `from…to` not analysed yet. */+ private async computeSubjects(from: number, to: number): Promise<void> {let missing = false;for (let c = from; c <= to; c++)if (!this.analysedSubject(c)) missing = true;@@ -634,6 +720,60 @@ export class ClipVision implements ClipSubjectSource {return this.runner;}+ /**+ * The soft alpha of every subject in `from…to` not matted yet: one+ * ViTMatte pass per frame over every track's edge. Prompted label maps grow+ * into the soft edge, so each track keeps its own alpha.+ */+ private async computeMattes(from: number, to: number): Promise<void> {+ // The default band, and so the cache key, depends on the frame size.+ if (!this.sourceWidth) return;+ this.mattesLoaded ??= this.loadMattes();+ await this.mattesLoaded;+ let decoder: ReturnType<typeof mediaDecoderCache.getVideo> | undefined;+ for (let c = from; c <= to; c++) {+ if (this.mattes.has(c)) continue;+ const stored = this.subjects.get(c);+ if (stored === undefined) continue;+ if (stored === null) {+ this.mattes.set(c, null);+ continue;+ }+ decoder ??= this.decoder();+ const image = await decodedPixels(await decoder.getFrame(c / this.fps));+ if (!image) {+ this.mattes.set(c, null);+ continue;+ }+ const size = image.width * image.height;+ const values = decodeRuns(stored, size);+ const hard = stored.scores+ ? values.map((label) => (label ? 255 : 0))+ : values;+ const matte = await this.visionRunner().refineMatte(+ image,+ hard,+ this.matteSettings,+ );+ let labels: Uint8Array | undefined;+ if (stored.scores) {+ labels = new Uint8Array(size);+ for (let i = 0; i < size; i++) {+ const nearest = matte.nearest[i] as number;+ if (matte.mask[i] && nearest >= 0)+ labels[i] = values[nearest] as number;+ }+ }+ this.mattes.set(c, {+ alpha: encodeRuns(matte.mask),+ ...(labels && { labels: encodeRuns(labels) }),+ });+ this.signalMaskFrame = -1;+ if (++this.unsavedMattes >= 24) await this.saveMattes();+ }+ if (this.unsavedMattes > 0) await this.saveMattes();+ }+/** Depth of every source frame in `from…to` not analysed yet, saved one file per frame. */private async computeDepth(from: number, to: number): Promise<void> {const dir = await this.depthCacheDir();@@ -984,7 +1124,9 @@ export class ClipVision implements ClipSubjectSource {if (!this.results.has(c) || !this.sourceWidth) return undefined;const stored = this.subjects.get(c);const size = this.sourceWidth * this.sourceHeight;- const data = stored ? subjectAlpha(stored, size, id) : new Uint8Array(size);+ const alpha = stored ? this.alpha(c, stored, id) : new Uint8Array(size);+ if (!alpha) return undefined;+ const data = alpha;let area = stored?.area ?? 0;if (id !== undefined && stored?.scores) {area = 0;@@ -1298,6 +1440,87 @@ export class ClipVision implements ClipSubjectSource {}}+ /**+ * Mattes live beside the subjects they refine, keyed by the matting model+ * and settings, so toggling matting never recomputes the masks.+ */+ private async mattesPath(): Promise<string | null> {+ const file = await this.cachePath();+ const node = await nodeModules();+ const settings = this.matteSettings;+ if (!file || !node || !settings) return null;+ const hash = node.crypto+ .createHash("sha256")+ .update(+ JSON.stringify([+ MATTE_CACHE_VERSION,+ MATTE_MODEL,+ settings.band,+ settings.resolution,+ ]),+ )+ .digest("hex")+ .slice(0, 12);+ return file.replace(/\.json$/, `.matte-${hash}.json`);+ }++ private async loadMattes(): Promise<void> {+ const file = await this.mattesPath();+ if (!file) return;+ try {+ const fs = await import("node:fs");+ if (!fs.existsSync(file)) return;+ const json = JSON.parse(fs.readFileSync(file, "utf8")) as {+ width: number;+ height: number;+ frames: Record<+ string,+ { alpha: StoredRuns; labels?: StoredRuns } | null+ >;+ };+ if (json.width !== this.sourceWidth || json.height !== this.sourceHeight)+ return;+ for (const [c, m] of Object.entries(json.frames)) {+ this.mattes.set(+ Number(c),+ m && {+ alpha: fromStoredRuns(m.alpha),+ ...(m.labels && { labels: fromStoredRuns(m.labels) }),+ },+ );+ }+ } catch {+ // A damaged cache is recomputed.+ }+ }++ private async saveMattes(): Promise<void> {+ this.unsavedMattes = 0;+ const file = await this.mattesPath();+ if (!file || !this.sourceWidth) return;+ const frames: Record<+ string,+ { alpha: StoredRuns; labels?: StoredRuns } | null+ > = {};+ for (const [c, m] of this.mattes) {+ frames[c] = m && {+ alpha: toStoredRuns(m.alpha),+ ...(m.labels && { labels: toStoredRuns(m.labels) }),+ };+ }+ const fs = await import("node:fs");+ const tmp = `${file}.${process.pid}.tmp`;+ fs.writeFileSync(+ tmp,+ JSON.stringify({+ width: this.sourceWidth,+ height: this.sourceHeight,+ frames,+ }),+ );+ fs.renameSync(tmp, file);+ }+private async saveSubjects(): Promise<void> {this.unsavedSubjects = 0;const file = await this.subjectsPath();@@ -1620,6 +1843,30 @@ function downsampleMean(}}+function toStoredRuns(runs: Runs): StoredRuns {+ return {+ values: Buffer.from(runs.values).toString("base64"),+ lengths: Buffer.from(+ runs.lengths.buffer,+ runs.lengths.byteOffset,+ runs.lengths.byteLength,+ ).toString("base64"),+ };+}++function fromStoredRuns(stored: StoredRuns): Runs {+ const lengths = Buffer.from(stored.lengths, "base64");+ return {+ values: new Uint8Array(Buffer.from(stored.values, "base64")),+ lengths: new Uint32Array(+ lengths.buffer.slice(+ lengths.byteOffset,+ lengths.byteOffset + lengths.byteLength,+ ),+ ),+ };+}+/** A mask as (value, run length) pairs: long flat runs with soft edges. */function encodeRuns(mask: Uint8Array): {values: Uint8Array;@@ -1701,6 +1948,8 @@ type SeedFrame = {mask: Uint8Array | null;/** Source-sized subject mask for render-time mattes; null = no subject. */subject?: Uint8Array | null;+ /** Prompted frames: a source-sized track label map (track ID per pixel, 0 for none). */+ tracks?: Uint8Array;};/** @internal Test hook: a clip whose results are given instead of computed. */@@ -1735,6 +1984,21 @@ export function seedClipVision(kp: f.kp ? Float32Array.from(f.kp) : null,mask: f.mask,});+ if (f.tracks) {+ let area = 0;+ let count = 0;+ for (const id of f.tracks) {+ if (id) area++;+ if (id > count) count = id;+ }+ c.subjects.set(i, {+ category: "object",+ area,+ scores: Array.from({ length: count }, () => 1),+ ...encodeRuns(f.tracks),+ });+ return;+ }if (f.subject === undefined) return;let area = 0;for (const v of f.subject ?? []) area += v;diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.tsindex 62a670b1..c34cceae 100644--- a/packages/vision/src/index.ts+++ b/packages/vision/src/index.ts@@ -7,6 +7,8 @@ export * from "./decode/selfie.js";export * from "./gpu/frame-tensor.js";export * from "./gpu/pose-skeleton-renderer.js";export * from "./gpu/segmentation-texture-pool.js";+export * from "./matting/trimap.js";+export * from "./matting/vitmatte.js";export * from "./model/model-store.js";export * from "./model/onnx-proto.js";export * from "./model/registry.js";diff --git a/packages/vision/src/matting/matting.test.ts b/packages/vision/src/matting/matting.test.tsnew file mode 100644index 00000000..70941da9--- /dev/null+++ b/packages/vision/src/matting/matting.test.ts@@ -0,0 +1,187 @@+import { describe, expect, it } from "vitest";+import {+ defaultMatteBand,+ distanceTransform,+ TRIMAP_BACKGROUND,+ TRIMAP_FOREGROUND,+ TRIMAP_UNKNOWN,+ trimapFromMask,+} from "./trimap.js";+import { composeMatte, matteWindow, vitmatteInput } from "./vitmatte.js";++function disc(+ width: number,+ height: number,+ cx: number,+ cy: number,+ r: number,+) {+ const mask = new Uint8Array(width * height);+ for (let y = 0; y < height; y++)+ for (let x = 0; x < width; x++)+ if ((x - cx) ** 2 + (y - cy) ** 2 <= r * r) mask[y * width + x] = 255;+ return mask;+}++describe("distanceTransform", () => {+ it("matches brute force, with the nearest seed", () => {+ const width = 23;+ const height = 17;+ const seeds = [+ [3, 4],+ [18, 2],+ [11, 14],+ [20, 15],+ ] as const;+ const isSeed = (i: number) => seeds.some(([x, y]) => y * width + x === i);+ const { distance, nearest } = distanceTransform(width, height, isSeed);+ for (let y = 0; y < height; y++) {+ for (let x = 0; x < width; x++) {+ const best = Math.min(+ ...seeds.map(([sx, sy]) => (x - sx) ** 2 + (y - sy) ** 2),+ );+ const i = y * width + x;+ expect(distance[i]).toBe(best);+ const n = nearest[i] as number;+ const nx = n % width;+ const ny = Math.floor(n / width);+ expect(isSeed(n)).toBe(true);+ expect((x - nx) ** 2 + (y - ny) ** 2).toBe(best);+ }+ }+ });++ it("reports no seed as −1", () => {+ const { nearest } = distanceTransform(4, 3, () => false);+ expect([...nearest].every((n) => n === -1)).toBe(true);+ });+});++describe("trimapFromMask", () => {+ it("puts a band of `band` px either side of the edge", () => {+ const width = 64;+ const height = 48;+ const r = 14;+ const band = 3;+ const trimap = trimapFromMask(+ disc(width, height, 32, 24, r),+ width,+ height,+ band,+ );+ const at = (x: number, y: number) => trimap.data[y * width + x];+ expect(at(32, 24)).toBe(TRIMAP_FOREGROUND);+ expect(at(2, 2)).toBe(TRIMAP_BACKGROUND);+ // Along the horizontal through the centre: the edge sits at x = 32 ± 14.+ expect(at(32 + r, 24)).toBe(TRIMAP_UNKNOWN);+ expect(at(32 + r + band, 24)).toBe(TRIMAP_UNKNOWN);+ expect(at(32 + r + band + 1, 24)).toBe(TRIMAP_BACKGROUND);+ expect(at(32 + r - band + 1, 24)).toBe(TRIMAP_UNKNOWN);+ expect(at(32 + r - band - 1, 24)).toBe(TRIMAP_FOREGROUND);+ expect(trimap.unknown).toEqual([+ 32 - r - band,+ 24 - r - band,+ 32 + r + band + 1,+ 24 + r + band + 1,+ ]);+ });++ it("has no unknown band for an empty or a full mask", () => {+ expect(trimapFromMask(new Uint8Array(30), 6, 5, 2).unknown).toBeNull();+ const full = trimapFromMask(new Uint8Array(30).fill(255), 6, 5, 2);+ expect(full.unknown).toBeNull();+ expect([...full.data].every((v) => v === TRIMAP_FOREGROUND)).toBe(true);+ });++ it("points every pixel at the nearest mask pixel", () => {+ const width = 20;+ const height = 4;+ const mask = new Uint8Array(width * height);+ for (let y = 0; y < height; y++) {+ mask[y * width + 2] = 255;+ mask[y * width + 15] = 255;+ }+ const { nearest } = trimapFromMask(mask, width, height, 2);+ expect((nearest[1 * width + 6] as number) % width).toBe(2);+ expect((nearest[1 * width + 10] as number) % width).toBe(15);+ });++ it("scales the default band with the frame", () => {+ expect(defaultMatteBand(1920, 1080)).toBe(13);+ expect(defaultMatteBand(64, 64)).toBe(3);+ expect(defaultMatteBand(8000, 8000)).toBe(40);+ });+});++describe("ViTMatte input and output", () => {+ const width = 100;+ const height = 60;+ const trimap = trimapFromMask(+ disc(width, height, 50, 30, 16),+ width,+ height,+ 4,+ );++ it("grows the window by the context and pads to multiples of 32", () => {+ const window = matteWindow(trimap, 10, 1024);+ expect(window).toMatchObject({+ x0: 20,+ y0: 0,+ x1: 81,+ y1: 60,+ scale: 1,+ contentWidth: 61,+ contentHeight: 60,+ inputWidth: 64,+ inputHeight: 64,+ });+ });++ it("scales a window larger than the resolution", () => {+ const window = matteWindow(trimap, 100, 50);+ expect(window?.scale).toBeCloseTo(0.5);+ expect(window?.contentWidth).toBe(50);+ expect(window?.contentHeight).toBe(30);+ expect(window?.inputWidth).toBe(64);+ expect(window?.inputHeight).toBe(32);+ });++ it("normalizes RGB to −1…1 and the trimap to 0…1, zero padded", () => {+ const data = new Uint8Array(width * height * 4);+ for (let i = 0; i < width * height; i++) {+ data[i * 4] = 255;+ data[i * 4 + 1] = 0;+ data[i * 4 + 2] = 127.5;+ data[i * 4 + 3] = 255;+ }+ const window = matteWindow(trimap, 10, 1024);+ if (!window) throw new Error("no window");+ const tensor = vitmatteInput({ data, width, height }, trimap, window);+ const plane = window.inputWidth * window.inputHeight;+ expect(tensor).toHaveLength(4 * plane);+ expect(tensor[0]).toBeCloseTo(1);+ expect(tensor[plane]).toBeCloseTo(-1);+ expect(tensor[2 * plane]).toBeCloseTo(0, 1);+ // Window pixel (30, 30) is source (50, 30): the disc's centre.+ expect(tensor[3 * plane + 30 * window.inputWidth + 30]).toBe(1);+ expect(tensor[3 * plane]).toBe(0);+ // Padding column.+ expect(tensor[window.inputWidth - 1]).toBe(0);+ expect(tensor[3 * plane + window.inputWidth - 1]).toBe(0);+ });++ it("keeps the trimap's certain pixels and takes the model's alpha in the band", () => {+ const window = matteWindow(trimap, 10, 1024);+ if (!window) throw new Error("no window");+ const alphas = new Float32Array(+ window.inputWidth * window.inputHeight,+ ).fill(0.4);+ const alpha = composeMatte(trimap, window, alphas);+ for (let i = 0; i < alpha.length; i++) {+ const t = trimap.data[i];+ if (t === TRIMAP_UNKNOWN) expect(alpha[i]).toBe(102);+ else expect(alpha[i]).toBe(t);+ }+ });+});diff --git a/packages/vision/src/matting/trimap.ts b/packages/vision/src/matting/trimap.tsnew file mode 100644index 00000000..fb3f22cf--- /dev/null+++ b/packages/vision/src/matting/trimap.ts@@ -0,0 +1,174 @@+/**+ * Trimaps from hard masks, for alpha matting: pixels well inside the mask are+ * foreground, pixels well outside are background, and a band of `band` pixels+ * either side of the edge is unknown, for the matting model to resolve.+ */++/** Trimap values, as ViTMatte reads them (0, 0.5, 1 on its 0..1 scale). */+export const TRIMAP_BACKGROUND = 0;+export const TRIMAP_UNKNOWN = 128;+export const TRIMAP_FOREGROUND = 255;++const FAR = 1e20;++/**+ * Squared Euclidean distance from every pixel to the nearest pixel where+ * `inside(i)` holds, with that pixel's index (−1 when there is none).+ * Felzenszwalb and Huttenlocher's two-pass lower envelope, O(width × height).+ */+export function distanceTransform(+ width: number,+ height: number,+ inside: (index: number) => boolean,+): { distance: Float32Array; nearest: Int32Array } {+ const n = width * height;+ const columns = new Float32Array(n);+ const columnFrom = new Int32Array(n);+ const scratch = envelopeScratch(Math.max(width, height));+ for (let x = 0; x < width; x++) {+ for (let y = 0; y < height; y++)+ scratch.f[y] = inside(y * width + x) ? 0 : FAR;+ envelope(scratch, height);+ for (let y = 0; y < height; y++) {+ columns[y * width + x] = scratch.d[y] as number;+ columnFrom[y * width + x] = scratch.from[y] as number;+ }+ }+ const distance = new Float32Array(n);+ const nearest = new Int32Array(n);+ for (let y = 0; y < height; y++) {+ const row = y * width;+ for (let x = 0; x < width; x++) scratch.f[x] = columns[row + x] as number;+ envelope(scratch, width);+ for (let x = 0; x < width; x++) {+ const d = scratch.d[x] as number;+ const fromX = scratch.from[x] as number;+ distance[row + x] = d;+ nearest[row + x] =+ d >= FAR ? -1 : (columnFrom[row + fromX] as number) * width + fromX;+ }+ }+ return { distance, nearest };+}++interface EnvelopeScratch {+ /** Input samples. */+ readonly f: Float64Array;+ /** Output: squared distance, and which sample it came from. */+ readonly d: Float64Array;+ readonly from: Int32Array;+ readonly v: Int32Array;+ readonly z: Float64Array;+}++function envelopeScratch(length: number): EnvelopeScratch {+ return {+ f: new Float64Array(length),+ d: new Float64Array(length),+ from: new Int32Array(length),+ v: new Int32Array(length),+ z: new Float64Array(length + 1),+ };+}++/** One dimension: d[q] = min over p of (q − p)² + f[p]. */+function envelope(s: EnvelopeScratch, length: number): void {+ const { f, d, from, v, z } = s;+ let k = -1;+ for (let q = 0; q < length; q++) {+ const fq = f[q] as number;+ if (fq >= FAR) continue;+ let cut = Number.NEGATIVE_INFINITY;+ while (k >= 0) {+ const p = v[k] as number;+ cut = (fq + q * q - ((f[p] as number) + p * p)) / (2 * (q - p));+ if (cut > (z[k] as number)) break;+ k--;+ }+ k++;+ v[k] = q;+ z[k] = k === 0 ? Number.NEGATIVE_INFINITY : cut;+ z[k + 1] = Number.POSITIVE_INFINITY;+ }+ if (k < 0) {+ d.fill(FAR, 0, length);+ from.fill(0, 0, length);+ return;+ }+ let j = 0;+ for (let q = 0; q < length; q++) {+ while ((z[j + 1] as number) < q) j++;+ const p = v[j] as number;+ d[q] = (q - p) * (q - p) + (f[p] as number);+ from[q] = p;+ }+}++export interface Trimap {+ /** Per pixel: {@link TRIMAP_BACKGROUND}, {@link TRIMAP_UNKNOWN} or {@link TRIMAP_FOREGROUND}. */+ readonly data: Uint8Array;+ readonly width: number;+ readonly height: number;+ /** Bounds of the unknown band, `[x0, y0, x1, y1)`; null when the band is empty. */+ readonly unknown: readonly [number, number, number, number] | null;+ /** Per pixel, the index of the nearest mask pixel (itself inside the mask; −1 when the mask is empty). */+ readonly nearest: Int32Array;+}++/**+ * The trimap of a mask (`mask[i] >= 128` is inside): an unknown band of+ * `band` px either side of the edge, foreground inside it, background outside.+ */+export function trimapFromMask(+ mask: ArrayLike<number>,+ width: number,+ height: number,+ band: number,+): Trimap {+ const outside = distanceTransform(+ width,+ height,+ (i) => (mask[i] as number) >= 128,+ );+ const inside = distanceTransform(+ width,+ height,+ (i) => (mask[i] as number) < 128,+ );+ const r2 = band * band;+ const data = new Uint8Array(width * height);+ let x0 = width;+ let y0 = height;+ let x1 = 0;+ let y1 = 0;+ for (let y = 0; y < height; y++) {+ for (let x = 0; x < width; x++) {+ const i = y * width + x;+ const isIn = (mask[i] as number) >= 128;+ const d = isIn+ ? (inside.distance[i] as number)+ : (outside.distance[i] as number);+ if (d > r2) {+ data[i] = isIn ? TRIMAP_FOREGROUND : TRIMAP_BACKGROUND;+ continue;+ }+ data[i] = TRIMAP_UNKNOWN;+ if (x < x0) x0 = x;+ if (y < y0) y0 = y;+ if (x >= x1) x1 = x + 1;+ if (y >= y1) y1 = y + 1;+ }+ }+ return {+ data,+ width,+ height,+ unknown: x1 > x0 ? [x0, y0, x1, y1] : null,+ nearest: outside.nearest,+ };+}++/** Default band for a frame: about 1.2% of its short side, 3…40 px (13 px at 1080p). */+export function defaultMatteBand(width: number, height: number): number {+ return Math.min(40, Math.max(3, Math.round(Math.min(width, height) * 0.012)));+}diff --git a/packages/vision/src/matting/vitmatte.ts b/packages/vision/src/matting/vitmatte.tsnew file mode 100644index 00000000..47c4fe80--- /dev/null+++ b/packages/vision/src/matting/vitmatte.ts@@ -0,0 +1,156 @@+import type { RgbaImage } from "../decode/preprocess.js";+import { TRIMAP_UNKNOWN, type Trimap } from "./trimap.js";++/** ViTMatte's patch grid: both input sides pad to a multiple of 32 (as its image processor does). */+const DIVISOR = 32;++/** Where a matting pass looks: a source rectangle, scaled into a padded model input. */+export interface MatteWindow {+ /** Source rectangle `[x0, y0, x1, y1)`. */+ readonly x0: number;+ readonly y0: number;+ readonly x1: number;+ readonly y1: number;+ /** Source px → model px. */+ readonly scale: number;+ /** Scaled content size, before padding. */+ readonly contentWidth: number;+ readonly contentHeight: number;+ /** Model input size, padded to a multiple of 32. */+ readonly inputWidth: number;+ readonly inputHeight: number;+}++/**+ * The window around a trimap's unknown band, grown by `context` px so the+ * model sees what is on either side, and scaled down when its long side+ * exceeds `resolution`. Null when there is nothing to resolve.+ */+export function matteWindow(+ trimap: Trimap,+ context: number,+ resolution: number,+): MatteWindow | null {+ if (!trimap.unknown) return null;+ const [ux0, uy0, ux1, uy1] = trimap.unknown;+ const x0 = Math.max(0, ux0 - context);+ const y0 = Math.max(0, uy0 - context);+ const x1 = Math.min(trimap.width, ux1 + context);+ const y1 = Math.min(trimap.height, uy1 + context);+ const scale = Math.min(1, resolution / Math.max(x1 - x0, y1 - y0));+ const contentWidth = Math.max(1, Math.round((x1 - x0) * scale));+ const contentHeight = Math.max(1, Math.round((y1 - y0) * scale));+ return {+ x0,+ y0,+ x1,+ y1,+ scale,+ contentWidth,+ contentHeight,+ inputWidth: Math.ceil(contentWidth / DIVISOR) * DIVISOR,+ inputHeight: Math.ceil(contentHeight / DIVISOR) * DIVISOR,+ };+}++/**+ * The model input `[1, 4, H, W]`: RGB as (v / 255 − 0.5) / 0.5, then the+ * trimap on 0…1. The image is sampled bilinearly, the trimap nearest so its+ * three levels stay exact; padding is zero in every channel.+ */+export function vitmatteInput(+ image: RgbaImage,+ trimap: Trimap,+ window: MatteWindow,+ out?: Float32Array,+): Float32Array {+ const { inputWidth: w, inputHeight: h } = window;+ const plane = w * h;+ const tensor =+ out && out.length >= 4 * plane+ ? out.subarray(0, 4 * plane)+ : new Float32Array(4 * plane);+ tensor.fill(0);+ const { data, width: sw, height: sh } = image;+ const inv = 1 / window.scale;+ for (let y = 0; y < window.contentHeight; y++) {+ const sy = Math.min(sh - 1, Math.max(0, window.y0 + (y + 0.5) * inv - 0.5));+ const y0 = Math.floor(sy);+ const y1 = Math.min(sh - 1, y0 + 1);+ const fy = sy - y0;+ const ty = Math.min(window.y1 - 1, Math.floor(window.y0 + (y + 0.5) * inv));+ for (let x = 0; x < window.contentWidth; x++) {+ const sx = Math.min(+ sw - 1,+ Math.max(0, window.x0 + (x + 0.5) * inv - 0.5),+ );+ const x0 = Math.floor(sx);+ const x1 = Math.min(sw - 1, x0 + 1);+ const fx = sx - x0;+ const o = y * w + x;+ for (let c = 0; c < 3; c++) {+ const a = data[(y0 * sw + x0) * 4 + c] as number;+ const b = data[(y0 * sw + x1) * 4 + c] as number;+ const d = data[(y1 * sw + x0) * 4 + c] as number;+ const e = data[(y1 * sw + x1) * 4 + c] as number;+ const top = a + (b - a) * fx;+ const v = top + (d + (e - d) * fx - top) * fy;+ tensor[c * plane + o] = v / 127.5 - 1;+ }+ const tx = Math.min(+ window.x1 - 1,+ Math.floor(window.x0 + (x + 0.5) * inv),+ );+ tensor[3 * plane + o] =+ (trimap.data[ty * trimap.width + tx] as number) / 255;+ }+ }+ return tensor;+}++/**+ * The frame's alpha: the trimap's foreground and background as they are,+ * and the model's alpha (sampled back to source px) across the unknown band.+ */+export function composeMatte(+ trimap: Trimap,+ window: MatteWindow | null,+ alphas: ArrayLike<number>,+): Uint8Array {+ const { width } = trimap;+ const out = Uint8Array.from(trimap.data, (v) =>+ v === TRIMAP_UNKNOWN ? 0 : v,+ );+ if (!window) return out;+ const w = window.inputWidth;+ const maxX = window.contentWidth - 1;+ const maxY = window.contentHeight - 1;+ for (let y = window.y0; y < window.y1; y++) {+ const my = Math.min(+ maxY,+ Math.max(0, (y - window.y0 + 0.5) * window.scale - 0.5),+ );+ const my0 = Math.floor(my);+ const my1 = Math.min(maxY, my0 + 1);+ const fy = my - my0;+ for (let x = window.x0; x < window.x1; x++) {+ const i = y * width + x;+ if (trimap.data[i] !== TRIMAP_UNKNOWN) continue;+ const mx = Math.min(+ maxX,+ Math.max(0, (x - window.x0 + 0.5) * window.scale - 0.5),+ );+ const mx0 = Math.floor(mx);+ const mx1 = Math.min(maxX, mx0 + 1);+ const fx = mx - mx0;+ const a = alphas[my0 * w + mx0] as number;+ const b = alphas[my0 * w + mx1] as number;+ const c = alphas[my1 * w + mx0] as number;+ const d = alphas[my1 * w + mx1] as number;+ const top = a + (b - a) * fx;+ const v = top + (c + (d - c) * fx - top) * fy;+ out[i] = Math.round(Math.min(1, Math.max(0, v)) * 255);+ }+ }+ return out;+}diff --git a/packages/vision/src/model/registry.ts b/packages/vision/src/model/registry.tsindex 6cb083ad..d7fe030b 100644--- a/packages/vision/src/model/registry.ts+++ b/packages/vision/src/model/registry.ts@@ -34,6 +34,7 @@ export type VisionModelKey =| "selfie-square"| "selfie-landscape"| "depth-anything-v2-s"+ | "vitmatte-s"| (string & {});/** Which decoder a model's outputs feed (see runner/vision-runner.ts). */@@ -42,6 +43,7 @@ export type VisionModelFamily =| "rtmo"| "selfie"| "depth-anything"+ | "vitmatte"| (string & {});export interface VisionModelDescriptor {@@ -75,6 +77,8 @@ const SELFIE_SOURCE ="MediaPipe Selfie Segmenter (Google, Apache-2.0), ONNX export";const DEPTH_SOURCE ="Depth Anything V2 Small (TikTok / HKU, Apache-2.0), ONNX export";+const VITMATTE_SOURCE =+ "ViTMatte Small, Distinctions-646 (HUST Vision Lab, code MIT, weights Apache-2.0), ONNX export";const models: Record<VisionModelKey, VisionModelDescriptor> = {// ── Detection + instance segmentation (COCO 80) ─────────────────────@@ -209,6 +213,23 @@ const models: Record<VisionModelKey, VisionModelDescriptor> = {input: [518, 518],source: DEPTH_SOURCE,},+ // ── Alpha matting (a trimap from a mask → soft alpha) ───────────────+ // The graph takes any size in multiples of 32; `input` is the longest+ // side a matting window is scaled to.+ "vitmatte-s": {+ key: "vitmatte-s",+ family: "vitmatte",+ filename: "vitmatte-s.onnx",+ url: hf(+ "Xenova/vitmatte-small-distinctions-646",+ "358d428c452e5e0cd52955011a8b51944731d28e",+ "onnx/model.onnx",+ ),+ bytes: 103_885_865,+ sha256: "a1cf48234c369faa3ea1711981d961fe1ec71f51e593f9d6553aa5a0e7d557e3",+ input: [1024, 1024],+ source: VITMATTE_SOURCE,+ },};export const VISION_MODELS: Readonly<diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.tsindex 111a23d7..93f2973e 100644--- a/packages/vision/src/models.e2e.test.ts+++ b/packages/vision/src/models.e2e.test.ts@@ -70,7 +70,11 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {const meta = session.inputMetadata[0];const shape = meta?.isTensor ? meta.shape : [];const [w, h] = VISION_MODELS[key].input;- expect(shape.slice(2)).toEqual([h, w]);+ const spatial = shape.slice(2);+ // Graphs with named spatial dims take any size; `input` is then the runner's choice.+ if (spatial.every((d) => typeof d === "string"))+ expect(spatial).toHaveLength(2);+ else expect(spatial).toEqual([h, w]);await session.release();}, 600_000);@@ -322,4 +326,65 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {expect(matte.coverage).toBeLessThan(0.6);}, 120_000);});++ describe("alpha matting (ViTMatte Small)", () => {+ const runner = VisionRunner.create();+ afterAll(() => runner.close());++ it("recovers a soft edge a hard mask cuts off", async () => {+ // A textured disc whose alpha falls off over 12 px, over a textured background.+ const width = 320;+ const height = 240;+ const truth = new Float32Array(width * height);+ const data = new Uint8Array(width * height * 4);+ for (let y = 0; y < height; y++) {+ for (let x = 0; x < width; x++) {+ const i = y * width + x;+ const r = Math.hypot(x - 160, y - 120);+ const a = Math.min(1, Math.max(0, (80 - r) / 12));+ truth[i] = a;+ const fg = [220, 170 + 40 * Math.sin(x / 3), 60];+ const bg = [40 + 30 * Math.sin(y / 5), 90, 200];+ for (let c = 0; c < 3; c++)+ data[i * 4 + c] =+ (fg[c] as number) * a + (bg[c] as number) * (1 - a);+ data[i * 4 + 3] = 255;+ }+ }+ const hard = Uint8Array.from(truth, (a) => (a >= 0.5 ? 255 : 0));+ const matte = await runner.refineMatte({ data, width, height }, hard, {+ band: 12,+ });+ let hardError = 0;+ let softError = 0;+ for (let i = 0; i < truth.length; i++) {+ hardError += Math.abs((hard[i] as number) / 255 - (truth[i] as number));+ softError += Math.abs(+ (matte.mask[i] as number) / 255 - (truth[i] as number),+ );+ }+ expect(softError).toBeLessThan(hardError * 0.5);+ }, 600_000);++ it("softens a portrait's segmentation edge and keeps its core", async () => {+ const frame = frames.portrait;+ const subject = mergeSubjectMask((await runner.segment(frame)).masks);+ if (!subject) throw new Error("no subject in the portrait");+ const hard = Uint8Array.from(subject.mask, (v) => (v >= 128 ? 255 : 0));+ const matte = await runner.refineMatte(frame, hard);+ let soft = 0;+ let inside = 0;+ let kept = 0;+ for (let i = 0; i < hard.length; i++) {+ const a = matte.mask[i] as number;+ if (a > 10 && a < 245) soft++;+ if (hard[i]) {+ inside++;+ if (a >= 128) kept++;+ }+ }+ expect(soft).toBeGreaterThan(frame.width);+ expect(kept / inside).toBeGreaterThan(0.95);+ }, 600_000);+ });});diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.tsindex 078d41e2..fed44026 100644--- a/packages/vision/src/runner/vision-runner.ts+++ b/packages/vision/src/runner/vision-runner.ts@@ -19,6 +19,12 @@ import {import { decodeRtmo } from "../decode/rtmo.js";import { decodeSelfie } from "../decode/selfie.js";import { FrameTensorPipeline, readTexturePixels } from "../gpu/frame-tensor.js";+import { defaultMatteBand, trimapFromMask } from "../matting/trimap.js";+import {+ composeMatte,+ matteWindow,+ vitmatteInput,+} from "../matting/vitmatte.js";import {type ModelDownloadStatus,VisionModelStore,@@ -42,8 +48,10 @@ import {import {type DepthMap,isGpuImage,+ type MatteRefineOptions,type PersonMatte,type PoseResult,+ type RefinedMatte,type SegmentationResult,type VisionFrame,type VisionImageInput,@@ -296,6 +304,58 @@ export class VisionRunner {);}+ /**+ * Resolves a hard mask's edge to soft alpha (ViTMatte Small): hair, motion+ * blur and see-through parts. `mask` is frame-sized, inside where ≥ 128.+ * Only a band around the edge changes; one model pass covers every part of+ * the mask, cropped to the band and scaled to `resolution`.+ */+ public async refineMatte(+ image: VisionFrame,+ mask: ArrayLike<number>,+ options: MatteRefineOptions = {},+ ): Promise<RefinedMatte> {+ const { width, height } = image;+ if (mask.length !== width * height)+ throw new Error(+ `refineMatte: the mask has ${mask.length} pixels; the frame has ${width}×${height}.`,+ );+ const band = Math.max(+ 1,+ Math.round(options.band ?? defaultMatteBand(width, height)),+ );+ const key: VisionModelKey = "vitmatte-s";+ const trimap = trimapFromMask(mask, width, height, band);+ const window = matteWindow(+ trimap,+ // Enough of either side for the model to tell subject from background.+ Math.max(32, band * 2),+ options.resolution ?? VISION_MODELS[key].input[0],+ );+ let alphas: ArrayLike<number> = [];+ if (window) {+ const pixels = isGpuImage(image) ? await readTexturePixels(image) : image;+ const outputs = await this.queued(key, async () => {+ const session = await this.session(key);+ const scratch = this._core.tensorScratch;+ const data = vitmatteInput(pixels, trimap, window, scratch.get(key));+ if (data.buffer !== scratch.get(key)?.buffer) scratch.set(key, data);+ return session.run({+ name: "pixel_values",+ data,+ dims: [1, 4, window.inputHeight, window.inputWidth],+ });+ });+ alphas = output(outputs, "alphas", key).data as Float32Array;+ }+ return {+ mask: composeMatte(trimap, window, alphas),+ width,+ height,+ nearest: trimap.nearest,+ };+ }+public close(): void {if (!this._ownsCore) return;const core = this._core;@@ -364,12 +424,14 @@ export class VisionRunner {/** Input size for graphs that take any (default: the registry's). */input?: readonly [number, number],): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {+ return this.queued(key, () => this.inferNow(key, image, input));+ }++ /** Runs `fn` after the model's earlier runs: one inference per session at a time. */+ private queued<T>(key: VisionModelKey, fn: () => Promise<T>): Promise<T> {const queue = this._core.runQueue;const previous = queue.get(key) ?? Promise.resolve();- const run = previous.then(- () => this.inferNow(key, image, input),- () => this.inferNow(key, image, input),- );+ const run = previous.then(fn, fn);const tail = run.catch(() => undefined);queue.set(key, tail);void tail.then(() => {diff --git a/packages/vision/src/types.ts b/packages/vision/src/types.tsindex 6afa7b13..80a54737 100644--- a/packages/vision/src/types.ts+++ b/packages/vision/src/types.ts@@ -61,6 +61,31 @@ export interface DepthMap {readonly height: number;}+/** How {@link VisionRunner.refineMatte} resolves a mask's edge. */+export interface MatteRefineOptions {+ /**+ * Pixels either side of the mask's edge the model may change (default about+ * 1.2% of the frame's short side: 13 px at 1080p). Wider reaches loose hair+ * the mask missed; narrower keeps more of the mask's own edge.+ */+ readonly band?: number;+ /** Longest side, in model px, the region around the edge is scaled to (default 1024). */+ readonly resolution?: number;+}++/** A mask with its edge resolved to soft alpha, aligned to the source frame. */+export interface RefinedMatte {+ /** Frame-sized alpha, 0..255. */+ readonly mask: Uint8Array;+ readonly width: number;+ readonly height: number;+ /**+ * Per pixel, the index of the nearest pixel inside the input mask (−1 when+ * the mask is empty), so alpha past the edge can follow labelled objects.+ */+ readonly nearest: Int32Array;+}+/** Person-vs-background alpha (Selfie Segmenter), aligned to the ORIGINAL source frame. */export interface PersonMatte {readonly mask: Uint8Array; // frame-sized, continuous alpha 0..255diff --git a/plugins/framefields/skills/framefields-vision/SKILL.md b/plugins/framefields/skills/framefields-vision/SKILL.mdindex e265b940..97aaf3e4 100644--- a/plugins/framefields/skills/framefields-vision/SKILL.md+++ b/plugins/framefields/skills/framefields-vision/SKILL.md@@ -1,6 +1,6 @@---name: framefields-vision-description: On-device computer vision in framefields — object detection and tracking, instance segmentation, pose, person mattes and cutouts (RTMDet-Ins, RTMO, Selfie Segmenter), and prompted video tracking of a specific object or several people with SAM 2.1 Tiny or Small (point/box prompts) and SAM 3 (text prompts such as "man in white shirt", multiple objects at once). Covers which API to use (withVision, comp.vision, analyzeVisionSequence), every pinned ONNX model with sizes and licenses, GPU/CUDA/CPU backends, model downloads and caching, performance, and troubleshooting. Use when a framefields composition needs to know where people or objects are, cut a subject out, follow someone through a clip, or select one person in a crowd.+description: On-device computer vision in framefields — object detection and tracking, instance segmentation, pose, person mattes and cutouts (RTMDet-Ins, RTMO, Selfie Segmenter), soft alpha matting of hair and motion blur (ViTMatte), and prompted video tracking of a specific object or several people with SAM 2.1 Tiny or Small (point/box prompts) and SAM 3 (text prompts such as "man in white shirt", multiple objects at once). Covers which API to use (withVision, comp.vision, analyzeVisionSequence), every pinned ONNX model with sizes and licenses, GPU/CUDA/CPU backends, model downloads and caching, performance, and troubleshooting. Use when a framefields composition needs to know where people or objects are, cut a subject out, follow someone through a clip, or select one person in a crowd.---# framefields-vision@@ -27,8 +27,9 @@ Vision runs ONNX models locally from [`framefields`](https://www.npmjs.com/packa| Pose, 17 COCO keypoints | RTMO | 27 MB (416²) | 40 MB | 89 MB | 640² || Person matte | MediaPipe Selfie Segmenter | square 0.5 MB (256²), landscape 0.5 MB (256×144) for wide frames | | | || Relative depth (`depth: true`) | Depth Anything V2 Small | 99 MB, one size | | | short side 518, frame aspect |+| Alpha matting (`matting: true`) | ViTMatte Small (Distinctions-646) | 104 MB, one size | | | the subject's edge, long side ≤ 1024 |-About 480 MB for all of them. `variant: "t" | "s" | "m"` picks the size.+About 585 MB for all of them. `variant: "t" | "s" | "m"` picks the size.### SAM 2.1 Tiny and Small (`sam2`, Apache-2.0)Meta SAM 2.1 Hiera checkpoints (Diffusion Studio fp16 video exports; memory of the prompted frame, the 6 latest frames, 16 object pointers). Point and box prompts only. `sam2: { model }`:@@ -126,6 +127,7 @@ await dancer.ready(0); // analyse a frame (or ready(0, 120)) to read- Each frame is analysed just before it draws, in every render path and the preview; nothing runs up front. Results persist, keyed by file and settings (`cacheDir: false` opts out), so later renders are instant.- Options: `startFrame` / `trimStartSec` (where the clip plays; pass `trimStartSec` yourself when you give a path instead of the layer), `fps` (analysis rate), `pose`, `mask`, `maskCell` (silhouette grid, default 8 px), `smoothing` (frames, default 2), `variant`. A lost person's last pose is held for up to a second.- `depth: true` adds each frame's relative depth (Depth Anything V2 Small; 0 far … 1 near, per-frame range, not metric) at about 924×518 for 16:9. A depth clip tracks no pose or silhouette unless `pose` / `mask` ask. Read it with `clip.depthMap(frame)` (`{ data, width, height }`), or draw it with `withVision({ mode: "depth" })` and lift a 3D layer with it via `withRelief` (framefields-compose). Depth frames persist one file each, keyed by the file only, so other options never invalidate them. About 0.4 s per frame on an M1 Pro GPU.+- `matting: true` resolves the subject's edge to soft alpha with ViTMatte Small: hair, motion blur and see-through parts that a hard mask cuts off. It refines whatever the clip selects (the automatic subject, or the `sam2` / `sam3` tracks), so it never picks a subject itself. Each frame's mask becomes a trimap: certain inside and outside, plus an unknown band of `band` px either side of the edge (default about 1.2% of the short side, 13 px at 1080p). One model pass per frame covers the band of every track. The window around the band is scaled to `resolution` (default 1024) on its long side, and each prompted track keeps its own alpha. Matte layers, `signals` masks and the clip's subject then read the soft alpha. Mattes persist beside the masks, keyed by `band` and `resolution`, so turning matting on or off never re-runs SAM. Widen `band` when hair reaches well past the mask. Only the band changes: matting does not fix a wrong selection, and an edge more than `band` px off stays off.- A `withVision({ mode: "matte" })` cutout (or `mask` / `crop`) of the same clip reuses the clip's subject for the frame being drawn: segmented once, aligned with the plate. Layers with `classes`, `keyBackground`, `matteSource: "selfie"` or different model settings keep their own pass. With several `comp.vision` clips of one source, the matte links to the **first one created**.## Prompted tracking with SAM@@ -158,6 +160,9 @@ const plate = Layer.video("assets/source.mp4", { width: 1920, height: 1080, fit:.withVision({ mode: "matte" });comp.add(plate);+// SAM 3 + matting: hair and motion blur keep their soft edge in the cutout+comp.vision(layer, { sam3: { text: "woman with long hair" }, acceptSamLicense: true, matting: true });+// SAM 3: one person by descriptionconst hero = comp.vision(plate, {acceptSamLicense: true,