Code-first video & image, rendered natively on WebGPU.
fix(vision): classes pick the cutout subject, not which parts merge
okanasl25edbbab885f
diff --git a/.changeset/vision-matte-classes.md b/.changeset/vision-matte-classes.mdnew file mode 100644index 00000000..266ab7b9--- /dev/null+++ b/.changeset/vision-matte-classes.md@@ -0,0 +1,8 @@+---+"framefields": patch+"@framefields/vision": patch+---++A `mask` / `matte` / `crop` cutout with `classes` keeps the whole subject again. The class filter ran before the subject was assembled, so the parts COCO labels as other classes were thrown away before they could merge: a flowing dress comes back as a "kite", "umbrella" or "horse", and a `classes: ["person"]` cutout of a dancer kept only head, arms and torso (about 3% of the frame instead of 12%). The text behind the dancer then showed through the dress on most frames, on every platform. `classes` now only decides which instance is the subject; overlapping parts of any class merge into it, and the masks a vision bundle receives are still filtered by `classes`.++`mergeSubjectMask(masks, options)` takes `{ classes, overlap, maxGrowth }` instead of positional `overlap` and `maxGrowth`, and `selectSubjectMask(masks, classes)` takes an optional class list.diff --git a/nodes/node-vision/src/renderers/webgpu-renderer.ts b/nodes/node-vision/src/renderers/webgpu-renderer.tsindex 9173dff6..9fedcb3d 100644--- a/nodes/node-vision/src/renderers/webgpu-renderer.ts+++ b/nodes/node-vision/src/renderers/webgpu-renderer.ts@@ -274,6 +274,16 @@ function normalizePoseKeypoints(};}+/** Case-insensitive class filter, as the runner's decode applies it. */+function matchesClasses(+ category: string,+ classes: readonly string[] | undefined,+): boolean {+ if (!classes) return true;+ const c = category.toLowerCase();+ return classes.some((k) => k.toLowerCase() === c);+}+/** Assigns each decoded instance mask to the temporal track whose box center is nearest the mask's centroid. */function assignMasksToTracks(masks: readonly InstanceMask[],@@ -750,9 +760,24 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {// Decoded masks are kept locally so mask-family modes render even without// an attached visionBundle (the round-trip through the bundle is for signal DX).+ // The subject is built from every class: `classes` picks the subject, but+ // parts COCO mislabels (a dress as a "kite") must still merge into it.+ let subjectMasks: readonly InstanceMask[] = [];if (wantsSegmentation) {- const segRes = await runner.segment(image);- frameMasks = assignMasksToTracks(segRes.masks, frameObjects);+ const subjectFromAllClasses =+ isMatteMode && op.matteSource !== "selfie" && op.classes !== undefined;+ // Same models and per-frame pass as `runner`, decoded without the filter.+ const segRunner = subjectFromAllClasses+ ? getSharedRunner({ ...op, classes: undefined })+ : runner;+ const segRes = await segRunner.segment(image);+ frameMasks = assignMasksToTracks(+ subjectFromAllClasses+ ? segRes.masks.filter((m) => matchesClasses(m.category, op.classes))+ : segRes.masks,+ frameObjects,+ );+ subjectMasks = subjectFromAllClasses ? segRes.masks : frameMasks;visionBundle?.setMaskResult(frameIdx, frameMasks);}@@ -779,9 +804,10 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {op.matteSource === "selfie"? personMatteAsMask(personMatte): mergeSubjectMask(- frameMasks.length > 0- ? frameMasks+ subjectMasks.length > 0+ ? subjectMasks: (visionBundle?.getMaskResult(frameIdx) ?? []),+ { classes: op.classes },),nodeKeyStr,frameIdx,diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.tsindex 03eb50b6..111a23d7 100644--- a/packages/vision/src/models.e2e.test.ts+++ b/packages/vision/src/models.e2e.test.ts@@ -266,6 +266,16 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {expect(subject?.coverage).toBeLessThan(0.2);}, 120_000);+ it("cuts out the whole dancer when the subject is limited to people", async () => {+ // The dress comes back as other classes ("kite", "umbrella"); a person-only+ // filter before the merge leaves just the bare person (~3%).+ const seg = await runner.segment(frames.dancer);+ const subject = mergeSubjectMask(seg.masks, { classes: ["person"] });+ expect(subject?.category).toBe("person");+ expect(subject?.coverage).toBeGreaterThan(0.08);+ expect(subject?.coverage).toBeLessThan(0.2);+ }, 120_000);+it("finds one dancer with a full skeleton", async () => {const { people } = await runner.pose(frames.dancer);expect(people).toHaveLength(1);diff --git a/packages/vision/src/segmentation/subject.test.ts b/packages/vision/src/segmentation/subject.test.tsindex b815942a..87da684e 100644--- a/packages/vision/src/segmentation/subject.test.ts+++ b/packages/vision/src/segmentation/subject.test.ts@@ -64,6 +64,23 @@ describe("subject selection", () => {expect(merged?.area).toBe(body.area + dress.area - 5 * 11);});+ it("limits the subject to `classes` but merges parts of any class", () => {+ // The smoke project's cutout: classes ["person"], a dress detected as a "kite".+ const body = rect("person", 8, 2, 12, 17, 1);+ const dress = rect("kite", 4, 6, 16, 16, 0);+ const merged = mergeSubjectMask([dress, body], { classes: ["Person"] });+ expect(merged?.category).toBe("person");+ expect(merged?.mask[10 * W + 5]).toBe(255);+ expect(merged?.area).toBe(body.area + dress.area - 5 * 11);+ });++ it("finds no subject when no instance is of `classes`", () => {+ const car = rect("car", 5, 5, 15, 15);+ expect(selectSubjectMask([car], ["person"])).toBeUndefined();+ expect(mergeSubjectMask([car], { classes: ["person"] })).toBeUndefined();+ expect(selectSubjectMask([car], ["car"])).toBe(car);+ });+it("maskBounds returns tight bounds or null", () => {expect(maskBounds(rect("x", 3, 4, 7, 9))).toEqual({x0: 3,diff --git a/packages/vision/src/segmentation/subject.ts b/packages/vision/src/segmentation/subject.tsindex da936207..9d3b5f39 100644--- a/packages/vision/src/segmentation/subject.ts+++ b/packages/vision/src/segmentation/subject.ts@@ -4,12 +4,16 @@ import type { InstanceMask } from "../types.js";* The frame's primary instance: the largest person when present, else the most confident* instance (lowest `detectionIndex` — detections are score-sorted). Size alone is a poor* signal without a person: the largest instance is usually a backdrop ("dining table").+ * `classes` (case-insensitive) limits which instances can be the subject.*/export function selectSubjectMask(masks: readonly InstanceMask[],+ classes?: readonly string[],): InstanceMask | undefined {+ const allowed = classes && new Set(classes.map((c) => c.toLowerCase()));let best: InstanceMask | undefined;for (const m of masks) {+ if (allowed && !allowed.has(m.category.toLowerCase())) continue;if (!best) {best = m;continue;@@ -58,22 +62,34 @@ export function maskBounds(mask: InstanceMask): MaskBounds | null {return x1 < 0 ? null : { x0, y0, x1, y1 };}+export interface MergeSubjectOptions {+ /**+ * Classes the subject may be (case-insensitive; any when omitted). Parts merged into it+ * may be of any class — pass every instance, not only these classes.+ */+ readonly classes?: readonly string[];+ /** Minimum intersection / smaller box area for a part. Default 0.5. */+ readonly overlap?: number;+ /** Largest part box area relative to the subject's. Default 2. */+ readonly maxGrowth?: number;+}+/*** Builds the full subject silhouette: the primary instance plus every comparably sized- * instance whose bounds sit mostly inside or across it (`overlap` = intersection / smaller- * box area; `maxGrowth` caps a part's box area relative to the subject's).+ * instance whose bounds sit mostly inside or across it.** COCO has no "clothing" class, so a flowing dress, a held guitar or a ridden bike comes back* as its own instance (often mislabeled) — merging them keeps the whole figure in the cutout.- * The size cap keeps containers out: a small figure inside a tunnel or window detected as a- * huge "clock" must not drag the whole frame into the subject.+ * That is why `classes` only picks the subject: a dress detected as a "kite" still merges+ * into a "person" subject. The size cap keeps containers out: a small figure inside a tunnel+ * or window detected as a huge "clock" must not drag the whole frame into the subject.*/export function mergeSubjectMask(masks: readonly InstanceMask[],- overlap = 0.5,- maxGrowth = 2,+ options: MergeSubjectOptions = {},): InstanceMask | undefined {- const subject = selectSubjectMask(masks);+ const { classes, overlap = 0.5, maxGrowth = 2 } = options;+ const subject = selectSubjectMask(masks, classes);if (!subject) return undefined;const sb = maskBounds(subject);if (!sb) return subject;diff --git a/plugins/framefields/skills/framefields-effects/SKILL.md b/plugins/framefields/skills/framefields-effects/SKILL.mdindex 500a85a7..a139af5b 100644--- a/plugins/framefields/skills/framefields-effects/SKILL.md+++ b/plugins/framefields/skills/framefields-effects/SKILL.md@@ -173,7 +173,7 @@ Inference runs on the GPU through `onnxruntime-web`'s WebGPU backend (an optiona- **One-frame delay.** Vision reads the layer's *previous* rendered frame, so the very first frame has no results (a cutout renders transparent) and masks trail the plate by one frame. This is invisible at normal playback speed.- **Render vision frames in order.** `renderVideo` does this for you. For stills, render at least two consecutive frames with the same renderer and keep the last one. Jumping straight to a later frame (or using a frame grid) cuts out pixels from whatever frame was rendered before it, which shows up as a ghosted double of the subject.-- **Subject choice.** `mask` / `matte` / `crop` use the largest person; with no person, the most confident instance. Instances overlapping the subject and no more than twice its size are merged in (a dress, a held instrument) — large containers around it (a tunnel, a window frame) are not.+- **Subject choice.** `mask` / `matte` / `crop` use the largest person; with no person, the most confident instance. Instances overlapping the subject and no more than twice its size are merged in (a dress, a held instrument) — large containers around it (a tunnel, a window frame) are not. `classes` limits what the subject can be, not what merges into it: COCO has no clothing class, so a flowing dress often comes back as a "kite" or "umbrella" and still joins a `classes: ["person"]` cutout.- Each vision layer tracks objects independently; track ids from two layers are unrelated.### Troubleshooting