Skip to content

Code-first video & image, rendered natively on WebGPU.

test(vision-sam): cover merged SAM 3 selections; refresh README SAM section

okanaslc583f001c3b0
diff --git a/README.md b/README.md
index e16b93d6..75e9339e 100644
--- a/README.md
+++ b/README.md
@@ -194,7 +194,7 @@ Installed components import only `framefields` and the shared toolkit in `framef
## Select and track a subject
-Use SAM 2.1 Tiny to select a specific object with a point or box and track it through video. Its masks, bounds, motion and optional matched pose feed the existing vision signals; the same clip masks drive cutouts. SAM ships separately under Meta's SAM License, so install it next to framefields:
+Select a specific object with a point or box (SAM 2.1 Tiny or Small), or by description, several at once, or a whole group as one mask (SAM 3), and track it through video. Masks, bounds, motion and optional matched pose feed the existing vision signals; the same clip masks drive cutouts. SAM ships separately under Meta's SAM License, so install it next to framefields:
```bash
pnpm add @framefields/vision-sam
@@ -206,12 +206,19 @@ const subject = Layer.video("assets/source.mp4", {
}).withVision({ mode: "matte" });
comp.add(subject);
const tracked = comp.vision(subject, {
+ acceptSamLicense: true,
sam2: { prompts: [{ frame: 0, points: [{ x: 0.5, y: 0.5, label: 1 }] }] },
});
const target = tracked.signals.objects.get(1);
+
+// SAM 3: every student in the shot as one mask, from one prompt
+comp.vision(subject, {
+ acceptSamLicense: true,
+ sam3: { text: "students in black jackets", merge: true },
+});
```
-Prompts use normalized source coordinates. Model files download lazily, inference runs locally, and cached source-frame results support seeking and time remapping. See the [vision guide](https://framefields.dev/docs/guides/vision/#select-a-specific-object-with-sam-21-tiny) for corrections, signal tracks, model/runtime requirements and mask alignment limits.
+Prompts use normalized source coordinates. Model files download lazily, inference runs locally, and cached source-frame results support seeking and time remapping. See the [vision guide](https://framefields.dev/docs/guides/vision/#select-specific-objects-with-sam) for choosing a model, corrections, several objects, backends, costs and mask alignment limits.
---
@@ -232,7 +239,7 @@ Everything else—including full API references, interactive tutorials, and guid
- **[WebGPU 2D VFX](https://framefields.dev/docs)** — 50+ shaders: 3D LUTs, Curves, Film Grain, Glassmorphism, and Blur
- **[Unified 3D Scenes](https://framefields.dev/docs)** — LookAt cameras, PBR materials, studio lighting, shadows, and glTF/GLB models
- **[Audio & Procedural SFX](https://framefields.dev/docs)** — BeatGrids, procedural whooshes/impacts, and audio-reactive signals
-- **[On-Device Vision](https://framefields.dev/docs/guides/vision/)** — Apache-2.0 ONNX models for detection, pose, cutouts, and prompted SAM 2.1 video tracking
+- **[On-Device Vision](https://framefields.dev/docs/guides/vision/)** — ONNX models for detection, pose and cutouts (Apache-2.0), and prompted SAM 2.1 / SAM 3 video tracking (optional, Meta SAM License)
- **[Live Browser Preview](https://framefields.dev/docs)** — Localhost WebGPU player with scrubbing, waveform timeline, and frame-stepping
---
diff --git a/packages/vision-sam/src/sam3/selection.test.ts b/packages/vision-sam/src/sam3/selection.test.ts
new file mode 100644
index 00000000..fc9a7a6d
--- /dev/null
+++ b/packages/vision-sam/src/sam3/selection.test.ts
@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import { describe, expect, it } from "vitest";
+import { mergeSelections } from "./selection.js";
+
+describe("mergeSelections", () => {
+ it("keeps every instance in one mask, at its most confident logit", () => {
+ const a = Float32Array.of(3, -2, -5, -1);
+ const b = Float32Array.of(-4, 1, -6, -3);
+ expect(Array.from(mergeSelections([a, b]))).toEqual([3, 1, -5, -1]);
+ // Positive (selected) wherever any instance is.
+ expect(Array.from(mergeSelections([a, b]), (v) => v > 0)).toEqual([
+ true,
+ true,
+ false,
+ false,
+ ]);
+ });
+
+ it("returns a single instance unchanged", () => {
+ const a = Float32Array.of(2, -2);
+ expect(Array.from(mergeSelections([a]))).toEqual([2, -2]);
+ });
+});
diff --git a/packages/vision-sam/src/sam3/selection.ts b/packages/vision-sam/src/sam3/selection.ts
new file mode 100644
index 00000000..d740ec46
--- /dev/null
+++ b/packages/vision-sam/src/sam3/selection.ts
@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * Text-selected instances as one mask (`sam3.merge`): each pixel keeps its most confident
+ * instance's logit, so the union is positive wherever any instance is.
+ */
+export function mergeSelections(masks: readonly Float32Array[]): Float32Array {
+ const merged = new Float32Array(masks[0]?.length ?? 0).fill(-Infinity);
+ for (const mask of masks)
+ for (let i = 0; i < merged.length; i++)
+ merged[i] = Math.max(merged[i] ?? -Infinity, mask[i] ?? -Infinity);
+ return merged;
+}
diff --git a/packages/vision-sam/src/sam3/tracker.ts b/packages/vision-sam/src/sam3/tracker.ts
index 7ffe9461..2075e579 100644
--- a/packages/vision-sam/src/sam3/tracker.ts
+++ b/packages/vision-sam/src/sam3/tracker.ts
@@ -28,6 +28,7 @@ import type { Sam2Point } from "../sam2/types.js";
import { holdSam3Encoder, type Sam3EncoderHandle } from "./encoder.js";
import { sam3DenseDecoder, sam3SupportGraphs } from "./graphs.js";
import { Sam3MemoryBank, sam3Position } from "./memory.js";
+import { mergeSelections } from "./selection.js";
import { Sam3TextSelector, sam3Floats, sam3Run } from "./text-selector.js";
import {
type Sam3FrameResult,
@@ -268,7 +269,7 @@ export class Sam3Tracker {
this.options.textThreshold ?? 0.5,
this.options.maxObjects ?? (this.options.merge ? Infinity : 1),
);
- const masks = this.options.merge ? [unionLogits(selected)] : selected;
+ const masks = this.options.merge ? [mergeSelections(selected)] : selected;
this.banks.length = 0;
for (const [index, logits] of masks.entries()) {
this.banks.push(this.newBank());
@@ -466,12 +467,3 @@ function seedLogits(logits: Float32Array): Float32Array {
}
return out;
}
-
-/** Instances as one mask: each pixel keeps its most confident instance's logit. */
-function unionLogits(masks: readonly Float32Array[]): Float32Array {
- const union = new Float32Array(masks[0]?.length ?? 0).fill(-Infinity);
- for (const mask of masks)
- for (let i = 0; i < union.length; i++)
- union[i] = Math.max(union[i] ?? -Infinity, mask[i] ?? -Infinity);
- return union;
-}