Code-first video & image, rendered natively on WebGPU.
feat(vision-sam): select every SAM 3 text match as one object
okanaslaec869b1767f
diff --git a/.changeset/sam3-merge.md b/.changeset/sam3-merge.mdnew file mode 100644index 00000000..895312c6--- /dev/null+++ b/.changeset/sam3-merge.md@@ -0,0 +1,6 @@+---+"@framefields/vision-sam": minor+"framefields": minor+---++`sam3: { text, merge: true }` selects every instance the text matches as one object: one mask and one memory, so a whole crowd costs the same as a single track. `maxObjects` caps the instances merged (default: all).diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex e0f49c26..edcbaced 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -602,6 +602,7 @@ export class ClipVision implements ClipSubjectSource {frameCount,textThreshold: settings.textThreshold,maxObjects: settings.maxObjects,+ merge: settings.merge,precision: settings.precision,acceptLicense:this.options.acceptSamLicense ?? this.options.sam3?.acceptLicense,diff --git a/packages/vision-sam/src/sam3/tracker.ts b/packages/vision-sam/src/sam3/tracker.tsindex de5454f6..7ffe9461 100644--- a/packages/vision-sam/src/sam3/tracker.ts+++ b/packages/vision-sam/src/sam3/tracker.ts@@ -54,8 +54,10 @@ export interface Sam3TrackerOptions extends VisionModelStoreOptions {readonly store?: VisionModelStore;readonly frameCount?: number;readonly textThreshold?: number;- /** Most instances a text prompt selects. Default 1. */+ /** Most instances a text prompt selects. Default 1, or every match with `merge`. */readonly maxObjects?: number;+ /** Track every instance a text prompt selects as one object (one mask, one memory). */+ readonly merge?: boolean;/** Image-encoder weights; defaults to the fastest for the provider's backend (`SAM3_ENCODER`). */readonly precision?: Sam3Precision;/** Explicitly agree to Meta's SAM License terms to download or run SAM models. */@@ -260,12 +262,13 @@ export class Sam3Tracker {this.provider,);if (!this.options.store) this.store.clearMemoryCache();- const masks = await this.selector.select(+ const selected = await this.selector.select(pixels,prompt.text,this.options.textThreshold ?? 0.5,- this.options.maxObjects ?? 1,+ this.options.maxObjects ?? (this.options.merge ? Infinity : 1),);+ const masks = this.options.merge ? [unionLogits(selected)] : selected;this.banks.length = 0;for (const [index, logits] of masks.entries()) {this.banks.push(this.newBank());@@ -463,3 +466,12 @@ function seedLogits(logits: Float32Array): Float32Array {}return out;}++/** Instances as one mask: each pixel keeps its most confident instance's logit. */+function unionLogits(masks: readonly Float32Array[]): Float32Array {+ const union = new Float32Array(masks[0]?.length ?? 0).fill(-Infinity);+ for (const mask of masks)+ for (let i = 0; i < union.length; i++)+ union[i] = Math.max(union[i] ?? -Infinity, mask[i] ?? -Infinity);+ return union;+}diff --git a/packages/vision-sam/src/sam3/types.ts b/packages/vision-sam/src/sam3/types.tsindex e24d5098..b65fe40f 100644--- a/packages/vision-sam/src/sam3/types.ts+++ b/packages/vision-sam/src/sam3/types.ts@@ -34,8 +34,13 @@ export interface Sam3Options extends SamLicenseOptions {readonly category?: string;/** Minimum sigmoid score for a text-selected instance. Default 0.5. */readonly textThreshold?: number;- /** Most instances a text prompt selects, as track IDs 1…N by score. Default 1. */+ /** Most instances a text prompt selects, as track IDs 1…N by score. Default 1, or every match with `merge`. */readonly maxObjects?: number;+ /**+ * Select every instance matching the text as **one** object: one mask and one memory, so+ * tracking costs the same as a single object however many match (e.g. a whole crowd).+ */+ readonly merge?: boolean;/** Image-encoder weights; defaults to the fastest for the backend in use (`SAM3_ENCODER`). */readonly precision?: Sam3Precision;}diff --git a/plugins/framefields/skills/framefields-vision/SKILL.md b/plugins/framefields/skills/framefields-vision/SKILL.mdindex bd012bd0..5aa8602b 100644--- a/plugins/framefields/skills/framefields-vision/SKILL.md+++ b/plugins/framefields/skills/framefields-vision/SKILL.md@@ -179,6 +179,15 @@ const crowd = comp.vision("assets/source.mp4", {sam3: { category: "person", text: "student in blazer", maxObjects: 5 },});+// SAM 3: every match as ONE object — one mask, one memory, the cost of a single track+const students = Layer.video("assets/source.mp4", { width: 1920, height: 1080, fit: "contain" })+ .withVision({ mode: "matte" }); // cuts out all of them as one entity+comp.add(students);+comp.vision(students, {+ acceptSamLicense: true,+ sam3: { category: "person", text: "student in black jacket", merge: true },+});+// SAM 2.1 Small: a small subject, a box plus a point, and a correction laterconst ball = comp.vision(plate, {acceptSamLicense: true,@@ -214,6 +223,7 @@ Layer.box({- **Prompt frames** are increasing integers on the source analysis clock, before trimming. SAM propagates forward from the first prompt; earlier frames are empty and are not analysed. To track only a stretch of a long clip, trim the layer and prompt on the stretch's first source frame.- **Text prompts are concepts.** Name what sets the target apart in that shot. In a crowd where everyone shows a white collar, "man in white shirt" picks a student. In the same film's wide shot, where only he has no blazer, it picks him. Check the first frame of every prompt.- **Several objects** (`maxObjects`): one encoder pass per frame serves every track, and each track keeps its own memory. Where tracks overlap, the more confident one keeps the pixel. `signals.objects.get(k)` and `signals.masks` carry tracks `1…N`; the clip's matte and subject are all tracks merged. A later text prompt re-selects every track (IDs renumber); a points prompt with `object: k` seeds or corrects track `k` only (the next unused ID adds one).+- **One entity for a whole group** (`merge: true`): every instance the text matches (up to `maxObjects`, default all) becomes one seed mask, tracked as one object with one memory. Use it when you need the group's mask or matte (cut out a crowd, grade everyone but the hero), not each member's box; it costs the same as one track however many match (4.1 s per frame for a whole crowd of students on an M1 Pro). Use `maxObjects` without `merge` when each member needs its own track, at about 0.7 s per member per frame.- Two text-prompted clips of one shot can claim the same person (a "student" track on the hero). Prefer prompts that exclude each other.- **Corrections and cuts.** A prompt on a later frame resets that object's memory. Re-prompt after hard cuts or long occlusions; appearance tracking cannot guarantee re-identification.- An explicit selection never falls back to another detected person. If its mask cannot align with a matte layer (a different aspect ratio), the render fails. Matte at the source aspect ratio and crop afterwards.