Code-first video & image, rendered natively on WebGPU.
test(vision-sam): cover merged SAM 3 selections; refresh README SAM section
okanaslc583f001c3b0
diff --git a/README.md b/README.mdindex e16b93d6..75e9339e 100644--- a/README.md+++ b/README.md@@ -194,7 +194,7 @@ Installed components import only `framefields` and the shared toolkit in `framef## Select and track a subject-Use SAM 2.1 Tiny to select a specific object with a point or box and track it through video. Its masks, bounds, motion and optional matched pose feed the existing vision signals; the same clip masks drive cutouts. SAM ships separately under Meta's SAM License, so install it next to framefields:+Select a specific object with a point or box (SAM 2.1 Tiny or Small), or by description, several at once, or a whole group as one mask (SAM 3), and track it through video. Masks, bounds, motion and optional matched pose feed the existing vision signals; the same clip masks drive cutouts. SAM ships separately under Meta's SAM License, so install it next to framefields:```bashpnpm add @framefields/vision-sam@@ -206,12 +206,19 @@ const subject = Layer.video("assets/source.mp4", {}).withVision({ mode: "matte" });comp.add(subject);const tracked = comp.vision(subject, {+ acceptSamLicense: true,sam2: { prompts: [{ frame: 0, points: [{ x: 0.5, y: 0.5, label: 1 }] }] },});const target = tracked.signals.objects.get(1);++// SAM 3: every student in the shot as one mask, from one prompt+comp.vision(subject, {+ acceptSamLicense: true,+ sam3: { text: "students in black jackets", merge: true },+});```-Prompts use normalized source coordinates. Model files download lazily, inference runs locally, and cached source-frame results support seeking and time remapping. See the [vision guide](https://framefields.dev/docs/guides/vision/#select-a-specific-object-with-sam-21-tiny) for corrections, signal tracks, model/runtime requirements and mask alignment limits.+Prompts use normalized source coordinates. Model files download lazily, inference runs locally, and cached source-frame results support seeking and time remapping. See the [vision guide](https://framefields.dev/docs/guides/vision/#select-specific-objects-with-sam) for choosing a model, corrections, several objects, backends, costs and mask alignment limits.---@@ -232,7 +239,7 @@ Everything else—including full API references, interactive tutorials, and guid- **[WebGPU 2D VFX](https://framefields.dev/docs)** — 50+ shaders: 3D LUTs, Curves, Film Grain, Glassmorphism, and Blur- **[Unified 3D Scenes](https://framefields.dev/docs)** — LookAt cameras, PBR materials, studio lighting, shadows, and glTF/GLB models- **[Audio & Procedural SFX](https://framefields.dev/docs)** — BeatGrids, procedural whooshes/impacts, and audio-reactive signals-- **[On-Device Vision](https://framefields.dev/docs/guides/vision/)** — Apache-2.0 ONNX models for detection, pose, cutouts, and prompted SAM 2.1 video tracking+- **[On-Device Vision](https://framefields.dev/docs/guides/vision/)** — ONNX models for detection, pose and cutouts (Apache-2.0), and prompted SAM 2.1 / SAM 3 video tracking (optional, Meta SAM License)- **[Live Browser Preview](https://framefields.dev/docs)** — Localhost WebGPU player with scrubbing, waveform timeline, and frame-stepping---diff --git a/packages/vision-sam/src/sam3/selection.test.ts b/packages/vision-sam/src/sam3/selection.test.tsnew file mode 100644index 00000000..fc9a7a6d--- /dev/null+++ b/packages/vision-sam/src/sam3/selection.test.ts@@ -0,0 +1,30 @@+/**+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ * All rights reserved.+ *+ * This source code is licensed under the SAM License found in the+ * LICENSE file in the root directory of this source tree.+ */++import { describe, expect, it } from "vitest";+import { mergeSelections } from "./selection.js";++describe("mergeSelections", () => {+ it("keeps every instance in one mask, at its most confident logit", () => {+ const a = Float32Array.of(3, -2, -5, -1);+ const b = Float32Array.of(-4, 1, -6, -3);+ expect(Array.from(mergeSelections([a, b]))).toEqual([3, 1, -5, -1]);+ // Positive (selected) wherever any instance is.+ expect(Array.from(mergeSelections([a, b]), (v) => v > 0)).toEqual([+ true,+ true,+ false,+ false,+ ]);+ });++ it("returns a single instance unchanged", () => {+ const a = Float32Array.of(2, -2);+ expect(Array.from(mergeSelections([a]))).toEqual([2, -2]);+ });+});diff --git a/packages/vision-sam/src/sam3/selection.ts b/packages/vision-sam/src/sam3/selection.tsnew file mode 100644index 00000000..d740ec46--- /dev/null+++ b/packages/vision-sam/src/sam3/selection.ts@@ -0,0 +1,19 @@+/**+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ * All rights reserved.+ *+ * This source code is licensed under the SAM License found in the+ * LICENSE file in the root directory of this source tree.+ */++/**+ * Text-selected instances as one mask (`sam3.merge`): each pixel keeps its most confident+ * instance's logit, so the union is positive wherever any instance is.+ */+export function mergeSelections(masks: readonly Float32Array[]): Float32Array {+ const merged = new Float32Array(masks[0]?.length ?? 0).fill(-Infinity);+ for (const mask of masks)+ for (let i = 0; i < merged.length; i++)+ merged[i] = Math.max(merged[i] ?? -Infinity, mask[i] ?? -Infinity);+ return merged;+}diff --git a/packages/vision-sam/src/sam3/tracker.ts b/packages/vision-sam/src/sam3/tracker.tsindex 7ffe9461..2075e579 100644--- a/packages/vision-sam/src/sam3/tracker.ts+++ b/packages/vision-sam/src/sam3/tracker.ts@@ -28,6 +28,7 @@ import type { Sam2Point } from "../sam2/types.js";import { holdSam3Encoder, type Sam3EncoderHandle } from "./encoder.js";import { sam3DenseDecoder, sam3SupportGraphs } from "./graphs.js";import { Sam3MemoryBank, sam3Position } from "./memory.js";+import { mergeSelections } from "./selection.js";import { Sam3TextSelector, sam3Floats, sam3Run } from "./text-selector.js";import {type Sam3FrameResult,@@ -268,7 +269,7 @@ export class Sam3Tracker {this.options.textThreshold ?? 0.5,this.options.maxObjects ?? (this.options.merge ? Infinity : 1),);- const masks = this.options.merge ? [unionLogits(selected)] : selected;+ const masks = this.options.merge ? [mergeSelections(selected)] : selected;this.banks.length = 0;for (const [index, logits] of masks.entries()) {this.banks.push(this.newBank());@@ -466,12 +467,3 @@ function seedLogits(logits: Float32Array): Float32Array {}return out;}--/** Instances as one mask: each pixel keeps its most confident instance's logit. */-function unionLogits(masks: readonly Float32Array[]): Float32Array {- const union = new Float32Array(masks[0]?.length ?? 0).fill(-Infinity);- for (const mask of masks)- for (let i = 0; i < union.length; i++)- union[i] = Math.max(union[i] ?? -Infinity, mask[i] ?? -Infinity);- return union;-}