Code-first video & image, rendered natively on WebGPU.
feat(vision-sam): add SAM 2.1 Small, chosen per shot
okanasl9a82bdbb6277
diff --git a/.changeset/sam2-small.md b/.changeset/sam2-small.mdnew file mode 100644index 00000000..aad1fb86--- /dev/null+++ b/.changeset/sam2-small.md@@ -0,0 +1,6 @@+---+"@framefields/vision-sam": minor+"framefields": minor+---++SAM 2.1 Small: `sam2: { model: "small" }` tracks at SAM 2's native 1024² from Diffusion Studio's pinned fp16 export (~112 MB, ~1.1 s/frame on an M1 Pro against Tiny's ~0.27 s), for small, thin or detailed subjects and identity after cuts. `Sam2Tracker` derives every size from the model's constants; `sam2Revision(model)` replaces `SAM2_TINY_REVISION`. The framefields-vision skill says when to choose Tiny, Small or SAM 3.diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex 84d2d548..e0f49c26 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -546,6 +546,7 @@ export class ClipVision implements ClipSubjectSource {modelsDir: this.options.modelsDir,baseUrl: this.options.baseUrl,frameCount,+ model: settings.model,acceptLicense:this.options.acceptSamLicense ?? this.options.sam2?.acceptLicense,});@@ -936,7 +937,7 @@ export class ClipVision implements ClipSubjectSource {o.variant ?? "s",o.confidence ?? null,// Prompted subjects are track-ID label maps (format 2).- ...(o.sam2 ? [[sam?.SAM2_TINY_REVISION, o.sam2, 2]] : []),+ ...(o.sam2 ? [[sam?.sam2Revision(o.sam2.model), o.sam2, 2]] : []),...(o.sam3 && sam? [[diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.tsindex 4e78df54..d0224bee 100644--- a/packages/framefields/src/index.ts+++ b/packages/framefields/src/index.ts@@ -139,6 +139,7 @@ export {// Types only: the trackers ship in the optional @framefields/vision-sam peer (Meta SAM License).export type {Sam2FrameResult,+ Sam2Model,Sam2Options,Sam2Point,Sam2Prompt,diff --git a/packages/vision-sam/src/models/registry.ts b/packages/vision-sam/src/models/registry.tsindex bfd42cb4..bcfd89a5 100644--- a/packages/vision-sam/src/models/registry.ts+++ b/packages/vision-sam/src/models/registry.ts@@ -20,13 +20,43 @@ import {* registry on first tracker creation so `@framefields/vision` never lists SAM weights.*/-export type Sam2ModelKey =- | "sam2.1-tiny-constants"- | "sam2.1-tiny-vision_encoder"- | "sam2.1-tiny-mask_decoder"- | "sam2.1-tiny-memory_encoder"- | "sam2.1-tiny-memory_attention"- | "sam2.1-tiny-pointer_tpos";+/** SAM 2.1 checkpoints with a pinned video export. */+export type Sam2Model = "tiny" | "small";++export type Sam2Graph =+ | "vision_encoder"+ | "mask_decoder"+ | "memory_encoder"+ | "memory_attention"+ | "pointer_tpos";++export type Sam2ModelKey = `sam2.1-${Sam2Model}-${Sam2Graph | "constants"}`;++/**+ * Diffusion Studio's fp16 video exports of Meta SAM 2.1 (Apache-2.0). Tiny is built for+ * speed at 512² (32×32 features); Small runs SAM 2's native 1024² (64×64), resolving small+ * objects, thin structures and identity after cuts better at roughly 4× the encoder and+ * 16× the memory-attention work.+ */+export const SAM2_EXPORTS: Readonly<+ Record<+ Sam2Model,+ { repo: string; revision: string; size: number; name: string }+ >+> = {+ tiny: {+ repo: "diffusionstudio/sam2.1-tiny-video-onnx-fp16",+ revision: "66673b5db39371b7dd7847f4d3bc0d4f4179b79e",+ size: 512,+ name: "Hiera Tiny",+ },+ small: {+ repo: "diffusionstudio/sam2.1-small-video-onnx-fp16",+ revision: "927ecec6e6ae2727d0af7720eefdb07f2efe4689",+ size: 1024,+ name: "Hiera Small",+ },+};/** Weight formats published for the SAM 3 tracker's image encoder. */export type Sam3Precision = "fp16" | "q4f16" | "q4";@@ -68,24 +98,25 @@ const hf = (repo: string, revision: string, path: string): string =>type Pin = readonly [path: string, bytes: number, sha256: string];-const SAM2_REPO = "diffusionstudio/sam2.1-tiny-video-onnx-fp16";-const SAM2_REV = "66673b5db39371b7dd7847f4d3bc0d4f4179b79e";-const sam2 = (- key: Sam2ModelKey,+ model: Sam2Model,+ graph: Sam2Graph | "constants",[path, bytes, sha256]: Pin,-): VisionModelDescriptor => ({- key,- family: "sam2",- filename: `${key}${path.slice(path.lastIndexOf("."))}`,- url: hf(SAM2_REPO, SAM2_REV, path),- bytes,- sha256,- input: [512, 512],- source:- "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",- license: "Apache-2.0",-});+): VisionModelDescriptor => {+ const { repo, revision, size, name } = SAM2_EXPORTS[model];+ const key: Sam2ModelKey = `sam2.1-${model}-${graph}`;+ return {+ key,+ family: "sam2",+ filename: `${key}${path.slice(path.lastIndexOf("."))}`,+ url: hf(repo, revision, path),+ bytes,+ sha256,+ input: [size, size],+ source: `Meta SAM 2.1 ${name} (Apache-2.0), Diffusion Studio ${size}px fp16 video export`,+ license: "Apache-2.0",+ };+};const sam3 =(repo: string, revision: string) =>@@ -155,36 +186,66 @@ const SUPPORT_GRAPH ="bundles/sam3-base-video-tracking-ortcuda-v2/graphs/base_tracker_preview_single1.onnx";export const SAM_MODELS: readonly VisionModelDescriptor[] = [- sam2("sam2.1-tiny-constants", [+ sam2("tiny", "constants", ["constants.json",9781,"67583810f4b80b762285377a5cc1066bbe19cdf8ef7fc87ff6aa78fc79830ef2",]),- sam2("sam2.1-tiny-vision_encoder", [+ sam2("tiny", "vision_encoder", ["onnx/vision_encoder.onnx",58385353,"24e9ccf5994be5c8dd93ebe771a4cc4f8bb122d0524555ac3ba4e7141710b133",]),- sam2("sam2.1-tiny-mask_decoder", [+ sam2("tiny", "mask_decoder", ["onnx/mask_decoder.onnx",8898805,"6bc158b0d028777ca2e25203ece0f27429c436de39a5c7fe301b3858d6059789",]),- sam2("sam2.1-tiny-memory_encoder", [+ sam2("tiny", "memory_encoder", ["onnx/memory_encoder.onnx",2807753,"8b2bb6810444cee684ef938b7464c5645653f236014f5497c3db3654818cc7ce",]),- sam2("sam2.1-tiny-memory_attention", [+ sam2("tiny", "memory_attention", ["onnx/memory_attention.onnx",13029470,"67c4e4da01bc7f501b88294bb19a8200362915ece04e9d7e367a437658f342c9",]),- sam2("sam2.1-tiny-pointer_tpos", [+ sam2("tiny", "pointer_tpos", ["onnx/pointer_tpos.onnx",34228,"3ab5bfce40ed01208301b2d8f596b9c9f7f493dcd7e09ef01ec22e5be3694bd7",]),+ sam2("small", "constants", [+ "constants.json",+ 9820,+ "09397f7251c159b64a2d581cc3ea0dbb3cf720c2e60777cb14e244405ed1c305",+ ]),+ sam2("small", "vision_encoder", [+ "onnx/vision_encoder.onnx",+ 83679474,+ "fb570036f5afac59ef20848958b075c84da746e333efd863104d94604652efc4",+ ]),+ sam2("small", "mask_decoder", [+ "onnx/mask_decoder.onnx",+ 8910345,+ "999b3b47bf2e924d522ed6cc011fa2c9b651754cc07895eb0da16a28a5ce7c4d",+ ]),+ sam2("small", "memory_encoder", [+ "onnx/memory_encoder.onnx",+ 2807884,+ "f36324e383dbf21fcd549169aedf698db00c307fd9a49538c0e5f0aa168e4f99",+ ]),+ sam2("small", "memory_attention", [+ "onnx/memory_attention.onnx",+ 16175204,+ "6666be8ca502b5c16e9a113d3d3934996d32f27e93bee5e66a8317fcceec9985",+ ]),+ sam2("small", "pointer_tpos", [+ "onnx/pointer_tpos.onnx",+ 34228,+ "f9c1b36e0aa9dc6b7c05b02a8fdaa17cc671d967ffcacc14fcd3177e65ec25e9",+ ]),...encoder("fp16",[diff --git a/packages/vision-sam/src/sam2/sam2.e2e.test.ts b/packages/vision-sam/src/sam2/sam2.e2e.test.tsindex 16ffe036..217481c0 100644--- a/packages/vision-sam/src/sam2/sam2.e2e.test.ts+++ b/packages/vision-sam/src/sam2/sam2.e2e.test.ts@@ -34,11 +34,15 @@ function picture(frame: number) {return { width, height, data };}-describe.skipIf(!enabled)("SAM 2.1 Tiny on native Dawn", () => {- it("runs every graph and propagates a selected object without selecting a distractor", async () => {+describe.skipIf(!enabled)("SAM 2.1 on native Dawn", () => {+ it.each([+ "tiny",+ "small",+ ] as const)("%s runs every graph and propagates a selected object without selecting a distractor", async (model) => {const tracker = await Sam2Tracker.create({provider: new NodeWebGPUProvider(),frameCount: 8,+ model,acceptLicense: true,});try {@@ -55,5 +59,5 @@ describe.skipIf(!enabled)("SAM 2.1 Tiny on native Dawn", () => {} finally {tracker.release();}- }, 120_000);+ }, 300_000);});diff --git a/packages/vision-sam/src/sam2/sam2.test.ts b/packages/vision-sam/src/sam2/sam2.test.tsindex e59d6a5d..3352bbbc 100644--- a/packages/vision-sam/src/sam2/sam2.test.ts+++ b/packages/vision-sam/src/sam2/sam2.test.ts@@ -8,6 +8,7 @@import { VisionModelStore } from "@framefields/vision";import { describe, expect, it } from "vitest";+import { SAM_MODELS, SAM2_EXPORTS } from "../models/registry.js";import { ScriptedProvider } from "../test-support.js";import { Sam2MemoryBank, sam2Tokens } from "./memory.js";import { decodeSam2Mask, Sam2Tracker } from "./tracker.js";@@ -141,6 +142,43 @@ describe("SAM 2 video pipeline", () => {).toThrow("corners");});+ it("pins every graph of every SAM 2.1 model, sized to its export", () => {+ for (const [model, { size }] of Object.entries(SAM2_EXPORTS))+ for (const graph of [+ "constants",+ "vision_encoder",+ "mask_decoder",+ "memory_encoder",+ "memory_attention",+ "pointer_tpos",+ ])+ expect(+ SAM_MODELS.find((m) => m.key === `sam2.1-${model}-${graph}`)?.input,+ ).toEqual([size, size]);+ });++ it("rejects an unknown model before downloading anything", async () => {+ await expect(+ Sam2Tracker.create({+ acceptLicense: true,+ // @ts-expect-error not a SAM 2.1 model+ model: "huge",+ }),+ ).rejects.toThrow('Unknown SAM 2.1 model "huge"');+ });++ it("decodes logits of any square model size to a source-sized mask", () => {+ const logits = new Float32Array(1024 * 1024).fill(-1);+ for (let y = 0; y < 1024; y++)+ for (let x = 0; x < 512; x++) logits[y * 1024 + x] = 2;+ expect(Array.from(decodeSam2Mask(logits, 4, 1, 1, 0.9).mask)).toEqual([+ 255, 255, 0, 0,+ ]);+ expect(() => decodeSam2Mask(new Float32Array(500), 4, 1, 1, 0.9)).toThrow(+ "Invalid SAM 2 mask shape.",+ );+ });+it("decodes logits to a source-sized binary mask", () => {const logits = new Float32Array(512 * 512).fill(-1);for (let y = 0; y < 512; y++)diff --git a/packages/vision-sam/src/sam2/tracker.ts b/packages/vision-sam/src/sam2/tracker.tsindex 2ad52e68..b821c637 100644--- a/packages/vision-sam/src/sam2/tracker.ts+++ b/packages/vision-sam/src/sam2/tracker.ts@@ -20,7 +20,12 @@ import {} from "@framefields/vision";import { z } from "zod";import { assertSamLicenseAccepted } from "../license.js";-import { ensureSamModelsRegistered } from "../models/registry.js";+import {+ ensureSamModelsRegistered,+ SAM2_EXPORTS,+ type Sam2Graph,+ type Sam2Model,+} from "../models/registry.js";import { Sam2MemoryBank, sam2Tokens } from "./memory.js";import {type Sam2FrameResult,@@ -29,7 +34,7 @@ import {} from "./types.js";const constantsSchema = z.object({- image_size: z.literal(512),+ image_size: z.number().int().positive().multipleOf(16),memory_frames: z.literal(7),image_mean: z.tuple([z.number(), z.number(), z.number()]),image_std: z.tuple([@@ -42,12 +47,7 @@ const constantsSchema = z.object({.length(7),});type Constants = z.infer<typeof constantsSchema>;-type Graph =- | "vision_encoder"- | "mask_decoder"- | "memory_encoder"- | "memory_attention"- | "pointer_tpos";+type Graph = Sam2Graph;const GRAPHS: readonly Graph[] = ["vision_encoder","mask_decoder",@@ -55,13 +55,18 @@ const GRAPHS: readonly Graph[] = ["memory_attention","pointer_tpos",];-export const SAM2_TINY_REVISION = "66673b5db39371b7dd7847f4d3bc0d4f4179b79e";+/** The pinned export revision of `model`: part of ClipVision's persistent cache identity. */+export function sam2Revision(model: Sam2Model = "tiny"): string {+ return `sam2.1-${model}-${SAM2_EXPORTS[model].revision}`;+}export interface Sam2TrackerOptions extends VisionModelStoreOptions {readonly provider?: SessionProvider;readonly store?: VisionModelStore;/** Source sequence length; used to normalize object-pointer time positions for short videos. */readonly frameCount?: number;+ /** Checkpoint: `"tiny"` (default, 512², fast) or `"small"` (1024², finer detail). */+ readonly model?: Sam2Model;/** Explicitly agree to Meta's SAM License terms to download or run SAM models. */readonly acceptLicense?: boolean;}@@ -85,10 +90,13 @@ function tensor(return { name, data, dims };}-/** Complete pinned 512px SAM 2.1 Tiny video graph pipeline, one explicitly selected object. */+/** Complete pinned SAM 2.1 video graph pipeline (Tiny or Small), one explicitly selected object. */export class Sam2Tracker {private readonly sessions = new Map<Graph, VisionSession>();private readonly bank: Sam2MemoryBank;+ /** Model input side; image features are `size / 16` square, one memory token per cell. */+ private readonly size: number;+ private readonly tokens: number;private lastFrame = -1;private released = false;private constructor(@@ -98,6 +106,8 @@ export class Sam2Tracker {this.bank = new Sam2MemoryBank(constants.memory_temporal_positional_encoding,);+ this.size = constants.image_size;+ this.tokens = (this.size / 16) ** 2;}static async create(options: Sam2TrackerOptions = {}): Promise<Sam2Tracker> {@@ -109,7 +119,12 @@ export class Sam2Tracker {...options,acceptLicense: options.acceptLicense ?? true,});- const bytes = await store.ensure("sam2.1-tiny-constants");+ const model = options.model ?? "tiny";+ if (!(model in SAM2_EXPORTS))+ throw new Error(+ `Unknown SAM 2.1 model "${model}"; use "tiny" or "small".`,+ );+ const bytes = await store.ensure(`sam2.1-${model}-constants`);const constants = constantsSchema.parse(JSON.parse(new TextDecoder().decode(bytes)),);@@ -123,7 +138,7 @@ export class Sam2Tracker {try {for (const graph of GRAPHS) {const session = await provider.createSession(- await store.ensure(`sam2.1-tiny-${graph}`),+ await store.ensure(`sam2.1-${model}-${graph}`),);tracker.sessions.set(graph, session);}@@ -161,8 +176,8 @@ export class Sam2Tracker {if (this.lastFrame < 0 && !points)throw new Error("SAM 2 needs a prompt on its first frame.");const { tensor: pixels } = imageToTensor(image, {- width: 512,- height: 512,+ width: this.size,+ height: this.size,fit: "stretch",channels: "rgb",padValue: 0,@@ -178,7 +193,7 @@ export class Sam2Tracker {],});const vision = await this.run("vision_encoder", [- tensor("pixel_values", pixels, [1, 3, 512, 512]),+ tensor("pixel_values", pixels, [1, 3, this.size, this.size]),]);const conditioned = points? vision.feats2_no_mem@@ -192,7 +207,9 @@ export class Sam2Tracker {input("feats2_cond", conditioned),tensor("input_points",- Float32Array.from(prompt.flatMap((p) => [p.x * 512, p.y * 512])),+ Float32Array.from(+ prompt.flatMap((p) => [p.x * this.size, p.y * this.size]),+ ),[1, 1, prompt.length, 2],),tensor("input_labels", Int32Array.from(prompt.map((p) => p.label)), [@@ -225,21 +242,22 @@ export class Sam2Tracker {for (let token = 0; token < 4; token++)bank.positions.set(positions.subarray(i * 64, (i + 1) * 64),- 7 * 1024 * 64 + (i * 4 + token) * 64,+ 7 * this.tokens * 64 + (i * 4 + token) * 64,);+ const memoryLength = 7 * this.tokens + 16 * 4;const result = await this.run("memory_attention", [- tensor(- "current_vision_features",- sam2Tokens(floats(vision, "feats2")),- [1024, 1, 256],- ),+ tensor("current_vision_features", sam2Tokens(floats(vision, "feats2")), [+ this.tokens,+ 1,+ 256,+ ]),tensor("current_vision_position_embeddings",sam2Tokens(floats(vision, "vision_pos_embed")),- [1024, 1, 256],+ [this.tokens, 1, 256],),- tensor("memory", bank.memory, [7232, 1, 64]),- tensor("memory_pos", bank.positions, [7232, 1, 64]),+ tensor("memory", bank.memory, [memoryLength, 1, 64]),+ tensor("memory_pos", bank.positions, [memoryLength, 1, 64]),]);const conditioned = result.conditioned_feats;if (!conditioned) throw new Error("Missing SAM 2 conditioned features.");@@ -282,7 +300,7 @@ export class Sam2Tracker {}}-/** Bilinear resize of logits before thresholding, matching SAM's source-image mask contract. */+/** Bilinear resize of square logits before thresholding, matching SAM's source-image mask contract. */export function decodeSam2Mask(logits: Float32Array,width: number,@@ -290,25 +308,31 @@ export function decodeSam2Mask(score: number,iou: number,): Sam2FrameResult {- if (logits.length !== 512 * 512) throw new Error("Invalid SAM 2 mask shape.");+ const side = Math.round(Math.sqrt(logits.length));+ if (side * side !== logits.length || side % 16 !== 0)+ throw new Error("Invalid SAM 2 mask shape.");+ const last = side - 1;const mask = new Uint8Array(width * height);if (score > 0) {for (let y = 0; y < height; y++) {- const sy = Math.max(0, Math.min(511, ((y + 0.5) * 512) / height - 0.5));+ const sy = Math.max(0, Math.min(last, ((y + 0.5) * side) / height - 0.5));const y0 = Math.floor(sy),- y1 = Math.min(511, y0 + 1),+ y1 = Math.min(last, y0 + 1),fy = sy - y0;for (let x = 0; x < width; x++) {- const sx = Math.max(0, Math.min(511, ((x + 0.5) * 512) / width - 0.5));+ const sx = Math.max(+ 0,+ Math.min(last, ((x + 0.5) * side) / width - 0.5),+ );const x0 = Math.floor(sx),- x1 = Math.min(511, x0 + 1),+ x1 = Math.min(last, x0 + 1),fx = sx - x0;const a =- (logits[y0 * 512 + x0] ?? 0) * (1 - fx) +- (logits[y0 * 512 + x1] ?? 0) * fx;+ (logits[y0 * side + x0] ?? 0) * (1 - fx) ++ (logits[y0 * side + x1] ?? 0) * fx;const b =- (logits[y1 * 512 + x0] ?? 0) * (1 - fx) +- (logits[y1 * 512 + x1] ?? 0) * fx;+ (logits[y1 * side + x0] ?? 0) * (1 - fx) ++ (logits[y1 * side + x1] ?? 0) * fx;mask[y * width + x] = a * (1 - fy) + b * fy > 0 ? 255 : 0;}}diff --git a/packages/vision-sam/src/sam2/types.ts b/packages/vision-sam/src/sam2/types.tsindex c9bf8747..7ff2539e 100644--- a/packages/vision-sam/src/sam2/types.ts+++ b/packages/vision-sam/src/sam2/types.ts@@ -8,6 +8,7 @@import type { VisionImageInput } from "@framefields/vision";import type { SamLicenseOptions } from "../license.js";+import type { Sam2Model } from "../models/registry.js";/** Normalized source-image coordinates. Labels 2/3 form a box's opposite corners. */export interface Sam2Point {@@ -26,6 +27,12 @@ export interface Sam2Options extends SamLicenseOptions {readonly prompts: readonly Sam2Prompt[];/** Metadata for object and class signals; SAM does not classify objects. */readonly category?: string;+ /**+ * Checkpoint: `"tiny"` (default; 512², ~0.3 s/frame on an M1 Pro) or `"small"` (SAM 2's+ * native 1024²; finer edges, small objects and thin structures, steadier identity after+ * cuts, at several times the cost).+ */+ readonly model?: Sam2Model;}export interface Sam2FrameResult {diff --git a/plugins/framefields/skills/framefields-vision/SKILL.md b/plugins/framefields/skills/framefields-vision/SKILL.mdindex 30703446..bd012bd0 100644--- a/plugins/framefields/skills/framefields-vision/SKILL.md+++ b/plugins/framefields/skills/framefields-vision/SKILL.md@@ -1,6 +1,6 @@---name: framefields-vision-description: On-device computer vision in framefields — object detection and tracking, instance segmentation, pose, person mattes and cutouts (RTMDet-Ins, RTMO, Selfie Segmenter), and prompted video tracking of a specific object or several people with SAM 2.1 Tiny (point/box prompts) and SAM 3 (text prompts such as "man in white shirt", multiple objects at once). Covers which API to use (withVision, comp.vision, analyzeVisionSequence), every pinned ONNX model with sizes and licenses, GPU/CUDA/CPU backends, model downloads and caching, performance, and troubleshooting. Use when a framefields composition needs to know where people or objects are, cut a subject out, follow someone through a clip, or select one person in a crowd.+description: On-device computer vision in framefields — object detection and tracking, instance segmentation, pose, person mattes and cutouts (RTMDet-Ins, RTMO, Selfie Segmenter), and prompted video tracking of a specific object or several people with SAM 2.1 Tiny or Small (point/box prompts) and SAM 3 (text prompts such as "man in white shirt", multiple objects at once). Covers which API to use (withVision, comp.vision, analyzeVisionSequence), every pinned ONNX model with sizes and licenses, GPU/CUDA/CPU backends, model downloads and caching, performance, and troubleshooting. Use when a framefields composition needs to know where people or objects are, cut a subject out, follow someone through a clip, or select one person in a crowd.---# framefields-vision@@ -32,8 +32,13 @@ Vision runs ONNX models locally. Nothing is sent anywhere. Models download on fiAbout 380 MB for all of them. `variant: "t" | "s" | "m"` picks the size.-### SAM 2.1 Tiny (`sam2`, Apache-2.0)-Meta SAM 2.1 Hiera Tiny, Diffusion Studio's 512 px fp16 video export: vision encoder 58 MB, mask decoder 9 MB, memory attention 13 MB, memory encoder 3 MB, pointer encoding and constants (under 0.1 MB). About 83 MB in total. Point and box prompts only.+### SAM 2.1 Tiny and Small (`sam2`, Apache-2.0)+Meta SAM 2.1 Hiera checkpoints in Diffusion Studio's fp16 video exports, with the same five graphs and memory bank (the prompted frame, the 6 latest frames, 16 object pointers). Point and box prompts only. Pick one with `sam2: { model }`:++| `model` | Input | Image encoder | Memory attention | Total | Per frame, M1 Pro WebGPU |+| --- | --- | --- | --- | --- | --- |+| `"tiny"` (default) | 512² (32×32 features) | 58 MB | 13 MB | ~83 MB | ~0.27 s |+| `"small"` | 1024², SAM 2's native size (64×64) | 84 MB | 16 MB | ~112 MB | ~1.1 s |### SAM 3 (`sam3`, Meta SAM License)Video tracking at 1008², plus text ("concept") selection. Only the image encoder the backend runs fastest is downloaded:@@ -131,14 +136,20 @@ await dancer.ready(0); // analyse a frame (or ready(0, 120)) to readDetection picks "the largest person". Use SAM when the subject is **a specific** person or object, especially in a crowd, or when you can only describe it.-| | SAM 2.1 Tiny (`sam2`) | SAM 3 (`sam3`) |-| --- | --- | --- |-| Select by | points and boxes | points, boxes, or **text** |-| Objects per clip | 1 | 1 to `maxObjects` |-| Resolution | 512² | 1008² |-| Cost per frame, M1 Pro | ~0.3 s | ~3.6 s image encoder (shared by every SAM 3 clip of the source) + ~0.7 s per tracked object |-| Download | ~83 MB | ~1.75 GB (GPU) / ~1.2 GB (CPU) |-| License | Apache-2.0 | Meta SAM License |+| | SAM 2.1 Tiny (`sam2`) | SAM 2.1 Small (`sam2: { model: "small" }`) | SAM 3 (`sam3`) |+| --- | --- | --- | --- |+| Select by | points and boxes | points and boxes | points, boxes, or **text** |+| Objects per clip | 1 | 1 | 1 to `maxObjects` |+| Resolution | 512² | 1024² | 1008² |+| Cost per frame, M1 Pro | ~0.27 s | ~1.1 s | ~3.6 s image encoder (shared by every SAM 3 clip of the source) + ~0.7 s per tracked object |+| Download | ~83 MB | ~112 MB | ~1.75 GB (GPU) / ~1.2 GB (CPU) |+| Weights license | Apache-2.0 | Apache-2.0 | Meta SAM License |++**Choosing.** Decide from the shot and the request, and say which model you chose and why:+- **SAM 2.1 Tiny:** one subject you can point at, which is large in frame (a close-up, a dancer, a speaker). It is the fastest by far; on such subjects its masks match Small's closely (IoU 0.98 on a close-up).+- **SAM 2.1 Small:** still one subject from points or a box, but small in frame, thin or detailed (a ball, a hand, a bike, hair, a distant runner), or a shot with cuts or look-alikes where Tiny latches onto a similar object. About 4× Tiny's cost.+- **SAM 3:** the subject has to be found from a description, or several people or objects must be tracked at once. It is the most expensive, so trim to the stretch you need.+- Unsure between Tiny and Small: prototype with Tiny on a short trimmed stretch, inspect the masks, and switch to Small if edges, small parts or identity fail.### Install and license@@ -168,10 +179,11 @@ const crowd = comp.vision("assets/source.mp4", {sam3: { category: "person", text: "student in blazer", maxObjects: 5 },});-// SAM 2.1: a box plus a point, and a correction later+// SAM 2.1 Small: a small subject, a box plus a point, and a correction laterconst ball = comp.vision(plate, {acceptSamLicense: true,sam2: {+ model: "small",category: "ball",prompts: [{ frame: 0, points: [@@ -279,4 +291,5 @@ report.tracks; // [{ trackId, category, frames: [start, end], centerPath, … }]| Log says `Inference on the CPU` on a GPU machine | The log's `reason` says why; on Linux NVIDIA check the CUDA 12 / cuDNN 9 libraries, or force `FRAMEFIELDS_VISION_PROVIDER` to see the error || A preview of a clip outside the project shows nothing | Add its folder to `startPreview(…, { root: [process.cwd(), dir] })` || Renders offline or in CI | Copy the model cache, or set `FRAMEFIELDS_MODELS_BASE_URL` |-| Check the models themselves | `await runner.preload([...])` on a fresh `FRAMEFIELDS_MODELS_DIR`; SAM: `pnpm --filter @framefields/vision-sam test:sam2` / `test:sam3` on a GPU machine |+| SAM 2.1 Tiny mask loses a small object, its edges, or the subject after a cut | `sam2: { model: "small" }` |+| Check the models themselves | `await runner.preload([...])` on a fresh `FRAMEFIELDS_MODELS_DIR`; SAM: `pnpm --filter @framefields/vision-sam test:sam2` (Tiny and Small) / `test:sam3` on a GPU machine |