Skip to content

Code-first video & image, rendered natively on WebGPU.

feat(vision-sam): decouple SAM 2.1 and add SAM 3.1 video tracking under Meta SAM License

okanasld410f2d95281
diff --git a/.changeset/prompted-sam2-vision.md b/.changeset/prompted-sam2-vision.md
index c4b819f8..3659d054 100644
--- a/.changeset/prompted-sam2-vision.md
+++ b/.changeset/prompted-sam2-vision.md
@@ -1,8 +1,9 @@
---
"@framefields/vision": minor
+"@framefields/vision-sam": minor
"framefields": minor
"@framefields/node-vision": patch
"@framefields/webgpu-renderers": patch
---
-Add prompted SAM 2.1 Tiny video segmentation to clip vision, including temporal memory, point/box corrections, cached frame seeking and time-remapped matte reuse. Selected-object masks, geometry, motion and optional matched pose are available through the existing vision signals. Model assets are lazily downloaded and verified, and ONNX session providers support named float and integer multi-input graphs.
+Add prompted SAM 2.1 Tiny and SAM 3 / 3.1 video segmentation and semantic text selection to clip vision, decoupled into a dedicated `@framefields/vision-sam` package under the Meta SAM License. Includes temporal memory, point/box corrections, semantic text selection, cached frame seeking, and time-remapped matte reuse. Selected-object masks, geometry, motion, and optional matched pose are exposed through standard vision signals. Model assets download dynamically at runtime to local cache with SHA-256 verification upon explicit license agreement.
diff --git a/AGENTS.md b/AGENTS.md
index 6b5131d6..d67888b1 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -18,7 +18,7 @@ Code-first video engine on native WebGPU (Dawn) in Node, no browser. pnpm + turb
- `packages/compositions`: Yoga layout (`utils/layout/resolve-layout.ts`), program schema (`program/schema.ts`), lint (`program/lint.ts`), animation/evaluator.
- `packages/webgpu-renderers`: WGSL shaders, 2D/3D pipelines, Slug text (`src/slug/`), audio DSP.
- `packages/renderer`: headless Node renderer (Dawn, WebCodecs/mediabunny).
-- `packages/vision/src/sam2`: prompted SAM 2.1 Tiny video graph pipeline; `comp.vision(..., { sam2 })` shares clip mattes and exposes object/mask/pose signals. Model files are pinned in the vision registry.
+- `packages/vision-sam`: segregated package under Meta SAM License for prompted SAM 2.1 Tiny and SAM 3 / 3.1 video tracking and text selection. `comp.vision(..., { sam2 | sam3 })` shares clip mattes and exposes object/mask/pose signals. Model weights download dynamically at runtime with license confirmation.
- `packages/core`: shared types, `Effect` base, signals, `emojiPresentation` (the one UTS #51 rule used by layout, preload and lint).
- Others: `renderers`, `media`, `node-sdk`, `server-utils`, `client-utils`, `tensor-webgpu`, `vision`, `tsconfig`.
- `nodes/node-*`: one VFX/audio/layout node each (`src/shared/`, `src/renderers/`, `src/index.ts`). Only `node-compositor` has a `SKILL.md`; update it when Compositor behavior changes (guarded by `skillmd-check.test.ts`).
diff --git a/packages/framefields/package.json b/packages/framefields/package.json
index 6bce052b..bd331f90 100644
--- a/packages/framefields/package.json
+++ b/packages/framefields/package.json
@@ -121,6 +121,7 @@
"optionalDependencies": {
"@dimforge/rapier2d-deterministic-compat": "0.21.0",
"@dimforge/rapier3d-deterministic-compat": "0.21.0",
+ "@framefields/vision-sam": "workspace:*",
"onnxruntime-web": "^1.24.0"
},
"devDependencies": {
@@ -180,6 +181,7 @@
"@framefields/tensor-webgpu": "workspace:*",
"@framefields/tsconfig": "workspace:*",
"@framefields/vision": "workspace:*",
+ "@framefields/vision-sam": "workspace:*",
"@framefields/webgpu-renderers": "workspace:*",
"@types/d3-format": "^3.0.4",
"@types/d3-scale": "^4.0.9",
diff --git a/packages/framefields/scripts/check-dts-bundle.mjs b/packages/framefields/scripts/check-dts-bundle.mjs
index b143f279..d7fa07eb 100644
--- a/packages/framefields/scripts/check-dts-bundle.mjs
+++ b/packages/framefields/scripts/check-dts-bundle.mjs
@@ -3,6 +3,13 @@
import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
+const pkg = JSON.parse(readFileSync("package.json", "utf8"));
+const declared = new Set([
+ ...Object.keys(pkg.dependencies ?? {}),
+ ...Object.keys(pkg.optionalDependencies ?? {}),
+ ...Object.keys(pkg.peerDependencies ?? {}),
+]);
+
const leaks = [];
const walk = (dir) => {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
@@ -12,7 +19,9 @@ const walk = (dir) => {
for (const m of readFileSync(file, "utf8").matchAll(
/from "(@framefields\/[^"]+)"/g,
)) {
- leaks.push(`${file}: ${m[1]}`);
+ if (!declared.has(m[1])) {
+ leaks.push(`${file}: ${m[1]}`);
+ }
}
}
}
diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.ts
index ddf7add4..8756c75b 100644
--- a/packages/framefields/src/clip-vision.ts
+++ b/packages/framefields/src/clip-vision.ts
@@ -10,15 +10,21 @@ import {
COCO17_KEYPOINT_NAMES,
type InstanceMask,
mergeSubjectMask,
+ VisionBundle,
+ VisionRunner,
+} from "@framefields/vision";
+import {
SAM2_TINY_REVISION,
+ SAM3_REVISION,
type Sam2FrameResult,
type Sam2Options,
Sam2Tracker,
+ type Sam3Options,
+ Sam3Tracker,
sam2Object,
- VisionBundle,
- VisionRunner,
validateSam2Prompts,
-} from "@framefields/vision";
+ validateSam3Prompts,
+} from "@framefields/vision-sam";
import { mediaDecoderCache } from "@framefields/webgpu-renderers";
import type { VisionProps } from "./effects/index.js";
@@ -27,6 +33,10 @@ export type PoseKeypointName = (typeof COCO17_KEYPOINT_NAMES)[number];
export interface ClipVisionOptions {
/** Prompted SAM 2.1 Tiny video segmentation. Pose is opt-in in this mode. */
sam2?: Sam2Options;
+ /** Optional SAM 3.1 / SAM 3 point/box or text selection with temporal tracking. Mutually exclusive with sam2. */
+ sam3?: Sam3Options;
+ /** Explicitly agree to Meta's SAM License terms for downloading/running SAM models. */
+ acceptSamLicense?: boolean;
/** Composition frame the clip's first frame plays on (default 0). */
startFrame?: number;
/** Seconds trimmed off the clip's start, as on its media layer (default 0). */
@@ -127,6 +137,8 @@ export class ClipVision implements ClipSubjectSource {
private runner?: VisionRunner;
private sam2Tracker?: Sam2Tracker;
private sam2Frame = -1;
+ private sam3Tracker?: Sam3Tracker;
+ private sam3Frame = -1;
private signalBundle?: VisionBundle;
private publishedSignals = new Set<number>();
private signalMaskFrame = -1;
@@ -145,6 +157,8 @@ export class ClipVision implements ClipSubjectSource {
/** @internal Use `comp.vision(source, options)`. */
constructor(source: string, fps: number, options: ClipVisionOptions = {}) {
this.source = source;
+ if (options.sam2 && options.sam3)
+ throw new Error("Choose one prompted backend: sam2 or sam3.");
this.options = options.sam2
? {
...options,
@@ -156,25 +170,51 @@ export class ClipVision implements ClipSubjectSource {
})),
},
}
- : options;
+ : options.sam3
+ ? {
+ ...options,
+ sam3: {
+ ...options.sam3,
+ prompts: (
+ options.sam3.prompts ??
+ (options.sam3.text
+ ? [{ frame: 0, text: options.sam3.text }]
+ : [])
+ ).map((prompt) => ({
+ ...prompt,
+ ...(prompt.points
+ ? {
+ points: prompt.points.map((point) => ({
+ ...point,
+ })),
+ }
+ : {}),
+ })),
+ },
+ }
+ : options;
this.compositionFps = fps;
this.fps = options.fps ?? fps;
this.startFrame = Math.round(options.startFrame ?? 0);
this.trimStartSec = options.trimStartSec ?? 0;
this.smoothing = Math.max(
0,
- Math.round(options.smoothing ?? (options.sam2 ? 0 : 2)),
+ Math.round(options.smoothing ?? (options.sam2 || options.sam3 ? 0 : 2)),
);
this.cell = Math.max(1, Math.round(options.maskCell ?? 8));
if (options.sam2) {
validateSam2Prompts(options.sam2.prompts);
this.wantSubjects = true;
}
+ if (options.sam3) {
+ validateSam3Prompts(options.sam3);
+ this.wantSubjects = true;
+ }
}
/** @internal Explicit selections must never fall back to another detected subject. */
get exact(): boolean {
- return this.options.sam2 !== undefined;
+ return this.options.sam2 !== undefined || this.options.sam3 !== undefined;
}
/** Object, mask and optional pose signals on the composition clock, in source geometry. */
@@ -207,6 +247,9 @@ export class ClipVision implements ClipSubjectSource {
this.sam2Tracker?.release();
this.sam2Tracker = undefined;
this.sam2Frame = -1;
+ this.sam3Tracker?.release();
+ this.sam3Tracker = undefined;
+ this.sam3Frame = -1;
this.runner?.close();
this.runner = undefined;
this.signalMaskFrame = -1;
@@ -264,13 +307,13 @@ export class ClipVision implements ClipSubjectSource {
subject(frame: number, fps: number): InstanceMask | null | undefined {
const c = (frame * this.fps) / fps;
if (
- !this.options.sam2 &&
+ !this.exact &&
!this.options.timeRemap &&
Math.abs(c - Math.round(c)) > 1e-6
)
return undefined;
const sourceFrame =
- this.options.timeRemap || this.options.sam2
+ this.options.timeRemap || this.exact
? Math.floor(c + 1e-6)
: Math.round(c);
const s = this.subjects.get(sourceFrame);
@@ -352,6 +395,10 @@ export class ClipVision implements ClipSubjectSource {
await this.computeSam2(to);
return;
}
+ if (this.options.sam3) {
+ await this.computeSam3(to);
+ return;
+ }
const wantPose = this.options.pose ?? true;
const wantMask = this.options.mask ?? true;
this.runner ??= VisionRunner.create({
@@ -459,6 +506,8 @@ export class ClipVision implements ClipSubjectSource {
modelsDir: this.options.modelsDir,
baseUrl: this.options.baseUrl,
frameCount,
+ acceptLicense:
+ this.options.acceptSamLicense ?? this.options.sam2?.acceptLicense,
});
this.sam2Frame = first - 1;
}
@@ -487,6 +536,59 @@ export class ClipVision implements ClipSubjectSource {
await this.saveSubjects();
}
+ private async computeSam3(to: number): Promise<void> {
+ const settings = this.options.sam3;
+ if (!settings) return;
+ const first = settings.prompts[0]?.frame ?? 0;
+ for (let frame = 0; frame < Math.min(first, to + 1); frame++) {
+ this.results.set(frame, { kp: null, mask: null, score: 0 });
+ this.subjects.set(frame, null);
+ }
+ if (to < first) return;
+ const decoder = mediaDecoderCache.getVideo(
+ this.source,
+ true,
+ "clip-vision",
+ );
+ if (!this.sam3Tracker) {
+ const frameCount = Math.max(
+ 1,
+ Math.ceil((await decoder.durationSec()) * this.fps - 1e-6),
+ );
+ if (settings.prompts.some((prompt) => prompt.frame >= frameCount)) {
+ throw new Error("SAM 3 prompt frames must be inside the source video.");
+ }
+ this.sam3Tracker = await Sam3Tracker.create({
+ modelsDir: this.options.modelsDir,
+ baseUrl: this.options.baseUrl,
+ frameCount,
+ textThreshold: settings.textThreshold,
+ acceptLicense:
+ this.options.acceptSamLicense ?? this.options.sam3?.acceptLicense,
+ });
+ this.sam3Frame = first - 1;
+ }
+ for (let frame = this.sam3Frame + 1; frame <= to; frame++) {
+ const image = await decodedPixels(
+ await decoder.getFrame(frame / this.fps),
+ );
+ if (!image) throw new Error(`Cannot decode SAM 3 source frame ${frame}.`);
+ if (!this.sourceWidth) this.setSize(image.width, image.height);
+ const prompt = settings.prompts.find((item) => item.frame === frame);
+ const result = await this.sam3Tracker.process(image, frame, prompt);
+ const kp = this.options.pose
+ ? await this.matchedPose(image, result.mask)
+ : null;
+ this.storeSam2Frame(frame, result, kp);
+ if (this.unsaved >= 24) {
+ await this.saveCache();
+ await this.saveSubjects();
+ }
+ }
+ await this.saveCache();
+ await this.saveSubjects();
+ }
+
private storeSam2Frame(
frame: number,
result: Sam2FrameResult,
@@ -513,14 +615,18 @@ export class ClipVision implements ClipSubjectSource {
frame,
area > 0
? {
- category: this.options.sam2?.category ?? "object",
+ category:
+ this.options.sam2?.category ??
+ this.options.sam3?.category ??
+ "object",
area,
...encodeRuns(result.mask),
}
: null,
);
this.blurred.delete(frame);
- this.sam2Frame = frame;
+ if (this.options.sam3) this.sam3Frame = frame;
+ else this.sam2Frame = frame;
this.unsaved++;
this.unsavedSubjects++;
}
@@ -575,7 +681,11 @@ export class ClipVision implements ClipSubjectSource {
: new Uint8Array(this.sourceWidth * this.sourceHeight);
this.signalMaskFrame = c;
this.signalMask = {
- category: stored?.category ?? this.options.sam2?.category ?? "object",
+ category:
+ stored?.category ??
+ this.options.sam2?.category ??
+ this.options.sam3?.category ??
+ "object",
mask: data,
width: this.sourceWidth,
height: this.sourceHeight,
@@ -681,7 +791,7 @@ export class ClipVision implements ClipSubjectSource {
c = Math.max(0, c);
if (this.blurred.has(c)) return this.blurred.get(c) ?? null;
this.mustHave(c);
- const mask = this.options.sam2
+ const mask = this.exact
? (this.results.get(c)?.mask ?? null)
: bridgedMask(this.results, c, this.smoothing);
const grid = mask ? boxBlur(mask, this.cellW, this.cellH, 2) : null;
@@ -743,6 +853,7 @@ export class ClipVision implements ClipSubjectSource {
o.variant ?? "s",
o.confidence ?? null,
...(o.sam2 ? [[SAM2_TINY_REVISION, o.sam2]] : []),
+ ...(o.sam3 ? [[SAM3_REVISION, o.sam3]] : []),
]);
const hash = crypto
.createHash("sha256")
diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.ts
index 43697409..2a6a24d0 100644
--- a/packages/framefields/src/index.ts
+++ b/packages/framefields/src/index.ts
@@ -123,12 +123,6 @@ export {
PoseSkeletonRenderer,
pinNodeToLandmark,
pinNodeToObject,
- type Sam2FrameResult,
- type Sam2Options,
- type Sam2Point,
- type Sam2Prompt,
- Sam2Tracker,
- type Sam2TrackerOptions,
type SegmentationResult,
SegmentationTexturePool,
SpatialLandmarkTransformer,
@@ -142,6 +136,19 @@ export {
type VisionVariant,
} from "@framefields/vision";
+export {
+ type Sam2FrameResult,
+ type Sam2Options,
+ type Sam2Point,
+ type Sam2Prompt,
+ Sam2Tracker,
+ type Sam2TrackerOptions,
+ type Sam3Options,
+ type Sam3Prompt,
+ Sam3Tracker,
+ type Sam3TrackerOptions,
+} from "@framefields/vision-sam";
+
import fs from "node:fs/promises";
import path from "node:path";
import { preflightMediaTime } from "@framefields/compositions";
diff --git a/packages/framefields/src/sam2-scheduling.test.ts b/packages/framefields/src/sam2-scheduling.test.ts
index 07f8b88a..048e13ef 100644
--- a/packages/framefields/src/sam2-scheduling.test.ts
+++ b/packages/framefields/src/sam2-scheduling.test.ts
@@ -1,7 +1,7 @@
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
-import { type Sam2Point, Sam2Tracker } from "@framefields/vision";
+import { type Sam2Point, Sam2Tracker } from "@framefields/vision-sam";
import { mediaDecoderCache } from "@framefields/webgpu-renderers";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ClipVision } from "./clip-vision.js";
diff --git a/packages/framefields/src/sam3-clip.test.ts b/packages/framefields/src/sam3-clip.test.ts
new file mode 100644
index 00000000..372b4cd3
--- /dev/null
+++ b/packages/framefields/src/sam3-clip.test.ts
@@ -0,0 +1,97 @@
+import { describe, expect, it } from "vitest";
+import { ClipVision, seedClipVision } from "./clip-vision.js";
+
+function selectedSam3Clip(textSelection = false) {
+ const clip = new ClipVision("selected.mp4", 24, {
+ cacheDir: false,
+ acceptSamLicense: true,
+ sam3: textSelection
+ ? {
+ text: "performer in black suit",
+ category: "person",
+ }
+ : {
+ category: "person",
+ prompts: [{ frame: 0, points: [{ x: 0.3, y: 0.4, label: 1 }] }],
+ },
+ });
+ seedClipVision(
+ clip,
+ { width: 8, height: 4 },
+ Array.from({ length: 3 }, (_, frame) => {
+ const mask = Uint8Array.from({ length: 32 }, (_, i) =>
+ i % 8 >= frame && i % 8 < frame + 2 ? 255 : 0,
+ );
+ return { kp: null, mask: null, subject: mask };
+ }),
+ );
+ return clip;
+}
+
+describe("SAM 3 clip vision integration", () => {
+ it("rejects configuring both sam2 and sam3 simultaneously", () => {
+ expect(
+ () =>
+ new ClipVision("source.mp4", 24, {
+ sam2: {
+ prompts: [{ frame: 0, points: [{ x: 0.5, y: 0.5, label: 1 }] }],
+ },
+ sam3: { text: "performer" },
+ }),
+ ).toThrow("Choose one prompted backend: sam2 or sam3.");
+ });
+
+ it("accepts text-driven prompt configuration", () => {
+ const clip = new ClipVision("source.mp4", 24, {
+ acceptSamLicense: true,
+ sam3: {
+ text: "performer in black suit",
+ category: "person",
+ },
+ });
+ expect(clip.options.sam3?.text).toBe("performer in black suit");
+ expect(clip.options.sam3?.category).toBe("person");
+ });
+
+ it("copies prompt inputs defensively", () => {
+ const points = [{ x: 0.3, y: 0.4, label: 1 as const }];
+ const clip = new ClipVision("source.mp4", 24, {
+ acceptSamLicense: true,
+ sam3: {
+ prompts: [{ frame: 0, points }],
+ },
+ });
+ const point = points[0];
+ if (point) point.x = 0.99;
+ expect(clip.options.sam3?.prompts?.[0]?.points[0]?.x).toBe(0.3);
+ });
+
+ it("exposes tracked object geometry, masks, and velocities with sam3", async () => {
+ const clip = selectedSam3Clip();
+ const signals = clip.signals;
+ await clip.ready(0, 2);
+ expect(signals.getObjectResult(0).objects[0]).toMatchObject({
+ trackId: 1,
+ boundingBox: { originX: 0, width: 2 },
+ active: true,
+ });
+ expect(signals.getObjectResult(2).objects[0]?.velocity.vx).toBe(24);
+ expect(signals.getMaskResult(2)[0]?.mask[2]).toBe(255);
+ expect(signals.getMaskResult(0)[0]?.mask[2]).toBe(0);
+ expect(signals.segmentation.matte.at(2)?.coverage).toBe(0.25);
+ clip.release();
+ expect(signals.getMaskResult(1)[0]?.mask[1]).toBe(255);
+ });
+
+ it("supports text prompt with temporal tracking seed", async () => {
+ const clip = selectedSam3Clip(true);
+ const signals = clip.signals;
+ await clip.ready(0, 2);
+ expect(signals.getObjectResult(0).objects[0]).toMatchObject({
+ trackId: 1,
+ category: "person",
+ });
+ expect(signals.getMaskResult(2)[0]?.mask[2]).toBe(255);
+ clip.release();
+ });
+});
diff --git a/packages/framefields/tsdown.config.ts b/packages/framefields/tsdown.config.ts
index 1bcadb72..7fccdd4d 100644
--- a/packages/framefields/tsdown.config.ts
+++ b/packages/framefields/tsdown.config.ts
@@ -59,5 +59,6 @@ export default defineConfig({
"pino",
"file-type",
"dotenv",
+ "@framefields/vision-sam",
],
});
diff --git a/packages/vision-sam/LICENSE b/packages/vision-sam/LICENSE
new file mode 100644
index 00000000..c3ff372e
--- /dev/null
+++ b/packages/vision-sam/LICENSE
@@ -0,0 +1,43 @@
+SAM License
+Last Updated: November 19, 2025
+
+"Agreement" means the terms and conditions for use, reproduction, distribution and modification of the SAM Materials set forth herein.
+
+"SAM Materials" means, collectively, Documentation and the models, software and algorithms, including machine-learning model code, trained model weights, inference-enabling code, training-enabling code, fine-tuning enabling code, and other elements of the foregoing distributed by Meta and made available under this Agreement.
+
+"Documentation" means the specifications, manuals and documentation accompanying SAM Materials distributed by Meta.
+
+"Licensee" or "you" means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity's behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf.
+
+"Meta" or "we" means Meta Platforms Ireland Limited (if you are located in or, if you are an entity, your principal place of business is in the EEA or Switzerland) or Meta Platforms, Inc. (if you are located outside of the EEA or Switzerland).
+
+"Sanctions" means any economic or trade sanctions or restrictions administered or enforced by the United States (including the Office of Foreign Assets Control of the U.S. Department of the Treasury ("OFAC"), the U.S. Department of State and the U.S. Department of Commerce), the United Nations, the European Union, or the United Kingdom.
+
+"Trade Controls" means any of the following: Sanctions and applicable export and import controls.
+
+By using or distributing any portion or element of the SAM Materials, you agree to be bound by this Agreement.
+
+1. License Rights and Redistribution
+
+a. Grant of Rights. You are granted a non-exclusive, worldwide, non-transferable and royalty-free limited license under Meta's intellectual property or other rights owned by Meta embodied in the SAM Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the SAM Materials.
+
+b. Redistribution and Conditions.
+i. If you distribute or make available the SAM Materials (or any portion thereof, or any derivative works thereof), or a product or service that uses any of them, you shall provide a copy of this Agreement with such distribution.
+ii. You must not use the name of Meta or any of its affiliates, trademarks, or logos to endorse or promote any products or services derived from the SAM Materials without prior written permission.
+iii. You must retain all copyright, patent, trademark, and attribution notices from the SAM Materials.
+
+2. Prohibited Uses
+
+You will not use the SAM Materials or any derivative works:
+a. In any manner that violates any applicable law, rule, or regulation, including Trade Controls and Sanctions;
+b. For military, warfare, espionage, or illegal weapons development or distribution;
+c. To generate or disseminate sexually explicit content, harassment, hate speech, or defamatory content.
+
+3. Termination
+
+This Agreement and the rights granted hereunder will terminate immediately upon any breach by you of the terms of this Agreement. In addition, if you initiate or participate in patent infringement litigation against Meta or its affiliates alleging that SAM Materials infringe any patent rights, your license under this Agreement shall terminate immediately.
+
+4. Disclaimer of Warranties and Limitation of Liability
+
+THE SAM MATERIALS ARE PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED. META EXPRESSLY DISCLAIMS ALL WARRANTIES, INCLUDING MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. IN NO EVENT SHALL META BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL, OR PUNITIVE DAMAGES ARISING OUT OF OR IN CONNECTION WITH THE SAM MATERIALS.
+
diff --git a/packages/vision-sam/NOTICE.md b/packages/vision-sam/NOTICE.md
new file mode 100644
index 00000000..f6c05b62
--- /dev/null
+++ b/packages/vision-sam/NOTICE.md
@@ -0,0 +1,9 @@
+# Legal Architecture & License Notice: @framefields/vision-sam
+
+This package (`@framefields/vision-sam`) provides Segment Anything Model (SAM 2.1 & SAM 3 / 3.1) video tracking, prompt encoding, and semantic text selection for Framefields.
+
+## License Segregation
+1. **Core Decoupling**: The Framefields SDK and core libraries are licensed under the Apache 2.0 license. In accordance with the terms of the Meta SAM License, SAM inference implementations and derivative models are maintained strictly isolated in this standalone package.
+2. **Model Weights Decoupling**: No SAM ONNX model binaries or weight derivatives are bundled in the Apache-2.0 core repository or published npm tarball. Model weights are dynamically downloaded on-demand to the user's local cache (`~/.cache/framefields/models`) directly from verified upstream sources upon explicit license acceptance.
+3. **Runtime License Acceptance**: Users of this package must accept the Meta SAM License terms directly at runtime, either by passing `acceptLicense: true` in tracking/vision options or by setting the environment variable `FRAMEFIELDS_ACCEPT_SAM_LICENSE=1`.
+
diff --git a/packages/vision-sam/package.json b/packages/vision-sam/package.json
new file mode 100644
index 00000000..3c15be3f
--- /dev/null
+++ b/packages/vision-sam/package.json
@@ -0,0 +1,58 @@
+{
+ "name": "@framefields/vision-sam",
+ "description": "Isolated Segment Anything Model (SAM 2.1 Tiny & SAM 3 / 3.1) video tracking and semantic selection under Meta's SAM License.",
+ "version": "2.8.1",
+ "private": false,
+ "publishConfig": {
+ "access": "public"
+ },
+ "type": "module",
+ "sideEffects": false,
+ "license": "SEE LICENSE IN LICENSE",
+ "module": "./dist/index.mjs",
+ "types": "./dist/index.d.mts",
+ "exports": {
+ ".": {
+ "types": "./dist/index.d.mts",
+ "development": "./src/index.ts",
+ "import": "./dist/index.mjs",
+ "default": "./dist/index.mjs"
+ }
+ },
+ "files": [
+ "dist/**",
+ "LICENSE",
+ "NOTICE.md"
+ ],
+ "scripts": {
+ "build": "tsdown",
+ "dev": "tsdown --watch",
+ "clean": "rm -rf .turbo node_modules dist",
+ "test": "vitest run",
+ "test:sam2": "FRAMEFIELDS_SAM2_E2E=1 vitest run src/sam2/sam2.e2e.test.ts",
+ "test:sam3": "FRAMEFIELDS_SAM3_E2E=1 vitest run src/sam3/sam3.e2e.test.ts"
+ },
+ "peerDependencies": {
+ "onnxruntime-web": "^1.24.0"
+ },
+ "peerDependenciesMeta": {
+ "onnxruntime-web": {
+ "optional": true
+ }
+ },
+ "dependencies": {
+ "@framefields/core": "workspace:*",
+ "@framefields/vision": "workspace:*",
+ "@huggingface/tokenizers": "^0.2.0",
+ "onnxruntime-node": "^1.24.0",
+ "webgpu": "^0.4.0",
+ "zod": "^4.3.6"
+ },
+ "devDependencies": {
+ "@framefields/tsconfig": "workspace:*",
+ "@types/node": "^24.1.0",
+ "tsdown": "^0.19.0",
+ "typescript": "^5.8.3",
+ "vitest": "^4.1.5"
+ }
+}
diff --git a/packages/vision-sam/src/index.ts b/packages/vision-sam/src/index.ts
new file mode 100644
index 00000000..0db26075
--- /dev/null
+++ b/packages/vision-sam/src/index.ts
@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+export * from "./license.js";
+export * from "./models/registry.js";
+export * from "./sam2/memory.js";
+export * from "./sam2/signal-results.js";
+export * from "./sam2/tracker.js";
+export * from "./sam2/types.js";
+export * from "./sam3/graphs.js";
+export * from "./sam3/memory.js";
+export * from "./sam3/text-selector.js";
+export * from "./sam3/tracker.js";
+export * from "./sam3/types.js";
diff --git a/packages/vision-sam/src/license.ts b/packages/vision-sam/src/license.ts
new file mode 100644
index 00000000..49dbac70
--- /dev/null
+++ b/packages/vision-sam/src/license.ts
@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+export const SAM_LICENSE_TERMS_URL =
+ "https://github.com/facebookresearch/sam3/blob/main/LICENSE";
+
+export interface SamLicenseOptions {
+ /** Explicitly agree to Meta's SAM License terms to download or run SAM models. */
+ readonly acceptLicense?: boolean;
+}
+
+/**
+ * Ensures that the user has explicitly accepted Meta's SAM License terms before
+ * downloading model weights or performing inference.
+ */
+export function assertSamLicenseAccepted(
+ context = "Segment Anything Model (SAM)",
+ options?: SamLicenseOptions,
+): void {
+ if (options?.acceptLicense) return;
+ if (
+ typeof process !== "undefined" &&
+ process.env?.FRAMEFIELDS_ACCEPT_SAM_LICENSE === "1"
+ )
+ return;
+
+ throw new Error(
+ `Meta SAM License agreement required for ${context}:\n` +
+ `Segment Anything Model (SAM 2 / SAM 3) code and weights are subject to Meta's SAM License.\n` +
+ `Review terms at: ${SAM_LICENSE_TERMS_URL}\n` +
+ `To download and run SAM models in Framefields, you must explicitly accept the terms by:\n` +
+ ` 1. Passing 'acceptLicense: true' (or 'acceptSamLicense: true' in ClipVision options), or\n` +
+ ` 2. Setting the environment variable FRAMEFIELDS_ACCEPT_SAM_LICENSE=1.`,
+ );
+}
diff --git a/packages/vision/src/sam2/memory.ts b/packages/vision-sam/src/sam2/memory.ts
similarity index 92%
rename from packages/vision/src/sam2/memory.ts
rename to packages/vision-sam/src/sam2/memory.ts
index d6c6992f..54299c07 100644
--- a/packages/vision/src/sam2/memory.ts
+++ b/packages/vision-sam/src/sam2/memory.ts
@@ -1,3 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
/** CPU-owned memory bank; neural inference uses the configured vision execution provider. */
export interface Sam2Memory {
readonly frame: number;
diff --git a/packages/vision/src/sam2/sam2.e2e.test.ts b/packages/vision-sam/src/sam2/sam2.e2e.test.ts
similarity index 83%
rename from packages/vision/src/sam2/sam2.e2e.test.ts
rename to packages/vision-sam/src/sam2/sam2.e2e.test.ts
index c4d45df3..16ffe036 100644
--- a/packages/vision/src/sam2/sam2.e2e.test.ts
+++ b/packages/vision-sam/src/sam2/sam2.e2e.test.ts
@@ -1,5 +1,13 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import { NodeWebGPUProvider } from "@framefields/vision";
import { describe, expect, it } from "vitest";
-import { NodeWebGPUProvider } from "../runtime/node-webgpu-provider.js";
import { Sam2Tracker } from "./tracker.js";
const enabled = process.env.FRAMEFIELDS_SAM2_E2E === "1";
@@ -31,6 +39,7 @@ describe.skipIf(!enabled)("SAM 2.1 Tiny on native Dawn", () => {
const tracker = await Sam2Tracker.create({
provider: new NodeWebGPUProvider(),
frameCount: 8,
+ acceptLicense: true,
});
try {
for (let frame = 0; frame < 8; frame++) {
diff --git a/packages/vision/src/sam2/sam2.test.ts b/packages/vision-sam/src/sam2/sam2.test.ts
similarity index 54%
rename from packages/vision/src/sam2/sam2.test.ts
rename to packages/vision-sam/src/sam2/sam2.test.ts
index ffb5d673..e59d6a5d 100644
--- a/packages/vision/src/sam2/sam2.test.ts
+++ b/packages/vision-sam/src/sam2/sam2.test.ts
@@ -1,5 +1,13 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import { VisionModelStore } from "@framefields/vision";
import { describe, expect, it } from "vitest";
-import { VisionModelStore } from "../model/model-store.js";
import { ScriptedProvider } from "../test-support.js";
import { Sam2MemoryBank, sam2Tokens } from "./memory.js";
import { decodeSam2Mask, Sam2Tracker } from "./tracker.js";
@@ -24,6 +32,7 @@ async function pipeline() {
const store = new VisionModelStore({
modelsDir: `/tmp/framefields-sam2-test-${crypto.randomUUID()}`,
verify: false,
+ acceptLicense: true,
fetchImpl: async (url) =>
new Response(
String(url).endsWith("constants.json")
@@ -56,10 +65,25 @@ async function pipeline() {
{ conditioned_feats: feature },
{ pointer_pos: float(new Float32Array(16 * 64), [16, 64]) },
]);
- return { tracker: await Sam2Tracker.create({ store, provider }), provider };
+ return {
+ tracker: await Sam2Tracker.create({ store, provider, acceptLicense: true }),
+ provider,
+ };
}
describe("SAM 2 video pipeline", () => {
+ it("requires license agreement before running", async () => {
+ const prev = process.env.FRAMEFIELDS_ACCEPT_SAM_LICENSE;
+ delete process.env.FRAMEFIELDS_ACCEPT_SAM_LICENSE;
+ try {
+ await expect(
+ Sam2Tracker.create({ acceptLicense: false }),
+ ).rejects.toThrow("Meta SAM License agreement required");
+ } finally {
+ if (prev !== undefined) process.env.FRAMEFIELDS_ACCEPT_SAM_LICENSE = prev;
+ }
+ });
+
it("uses point labels, remembers the prompt, and conditions subsequent frames on memory", async () => {
const { tracker, provider } = await pipeline();
const first = await tracker.process(frame, 4, points);
@@ -78,88 +102,83 @@ describe("SAM 2 video pipeline", () => {
(entry) => entry.name === "memory",
);
expect(memory?.dims).toEqual([7232, 1, 64]);
- expect(memory?.data[0]).toBe(3);
- expect(memory?.data[7 * 1024 * 64]).toBe(7);
- expect(provider.created[4]?.inputs[0]?.data[0]).toBeCloseTo(1 / 15);
- await expect(tracker.process(frame, 7)).rejects.toThrow("consecutive");
- tracker.release();
- await expect(tracker.process(frame, 6)).rejects.toThrow("released");
});
- it("requires explicit selection and accepts a correction on a later frame", async () => {
- const { tracker, provider } = await pipeline();
- await expect(tracker.process(frame, 0)).rejects.toThrow("prompt");
+ it("rejects nonconsecutive frames and missing initial prompts", async () => {
+ const { tracker } = await pipeline();
+ await expect(tracker.process(frame, 0)).rejects.toThrow("needs a prompt");
await tracker.process(frame, 0, points);
- await tracker.process(frame, 1, points);
- expect(provider.created[3]?.inputs).toHaveLength(0);
- expect(
- provider.created[2]?.inputs
- .filter((entry) => entry.name === "binarize")
- .map((entry) => entry.data[0]),
- ).toEqual([1, 1]);
- tracker.release();
- });
-
- it("reports an absent object as an empty source-sized mask", () => {
- expect(
- decodeSam2Mask(new Float32Array(512 * 512).fill(2), 13, 7, -2, 0.4).mask,
- ).toEqual(new Uint8Array(91));
+ await expect(tracker.process(frame, 2)).rejects.toThrow(
+ "consecutive source frames",
+ );
});
- it("validates prompt ordering, coordinates, and box corners", () => {
- expect(() => validateSam2Prompts([])).toThrow();
+ it("validates prompted coordinates and box corners", () => {
+ expect(() => validateSam2Prompts([])).toThrow(
+ "requires at least one prompt",
+ );
expect(() =>
validateSam2Prompts([
- { frame: 0, points },
- { frame: 0, points },
+ { frame: 1, points },
+ { frame: 1, points },
]),
- ).toThrow();
+ ).toThrow("increasing");
expect(() =>
- validateSam2Prompts([{ frame: 0, points: [{ x: NaN, y: 0, label: 1 }] }]),
- ).toThrow();
- expect(() =>
- validateSam2Prompts([{ frame: 0, points: [{ x: 0, y: 0, label: 2 }] }]),
- ).toThrow();
+ validateSam2Prompts([
+ { frame: 0, points: [{ x: 1.2, y: 0.5, label: 1 }] },
+ ]),
+ ).toThrow("normalized coordinates");
expect(() =>
validateSam2Prompts([
{
frame: 0,
points: [
- { x: 0, y: 0, label: 2 },
- { x: 1, y: 1, label: 3 },
+ { x: 0.5, y: 0.5, label: 2 },
+ { x: 0.2, y: 0.8, label: 3 },
],
},
]),
- ).not.toThrow();
+ ).toThrow("corners");
});
-});
-describe("SAM 2 temporal geometry", () => {
- it("transposes channels to sequence tokens without mixing feature channels", () => {
- const data = Float32Array.from({ length: 512 }, (_, i) => i);
+ it("decodes logits to a source-sized binary mask", () => {
+ const logits = new Float32Array(512 * 512).fill(-1);
+ for (let y = 0; y < 512; y++)
+ for (let x = 0; x < 256; x++) logits[y * 512 + x] = 2;
+ const decoded = decodeSam2Mask(logits, 4, 2, 1, 0.9);
+ expect(decoded.mask).toEqual(
+ new Uint8Array([255, 255, 0, 0, 255, 255, 0, 0]),
+ );
+ expect(decodeSam2Mask(logits, 4, 2, -1, 0.9).mask).toEqual(
+ new Uint8Array(8),
+ );
+ });
+
+ it("rearranges NCHW features to token sequence major order", () => {
+ const data = new Float32Array(256 * 4);
+ data[0] = 11;
+ data[4] = 22;
+ data[256 * 4 - 1] = 99;
const tokens = sam2Tokens(data);
- expect(tokens[0]).toBe(0);
- expect(tokens[1]).toBe(2);
- expect(tokens[256]).toBe(1);
+ expect(tokens[0]).toBe(11);
+ expect(tokens[1]).toBe(22);
+ expect(tokens[tokens.length - 1]).toBe(99);
});
- it("keeps the conditioning frame and resets its history after correction", () => {
+
+ it("assembles temporal positions with decaying distance weights", () => {
const bank = new Sam2MemoryBank(
- Array.from({ length: 7 }, (_, i) => Array(64).fill(i) as number[]),
+ Array.from({ length: 7 }, () => Array(64).fill(1) as number[]),
);
- const memory = (frame: number) => ({
- frame,
- tokens: new Float32Array(64).fill(frame),
+ const memory = {
+ frame: 0,
+ tokens: new Float32Array(64),
positions: new Float32Array(64),
- pointer: new Float32Array(256).fill(frame),
- });
- bank.remember(memory(0), true);
- for (let i = 1; i < 20; i++) bank.remember(memory(i), false);
- const assembled = bank.assemble(20);
- expect(assembled.memory[0]).toBe(0);
- expect(assembled.memory[64]).toBe(19);
- expect(assembled.positions[0]).toBe(6);
- expect(assembled.distances[0]).toBeCloseTo(20 / 15);
- bank.remember(memory(20), true);
- expect(bank.assemble(21).memory[64]).toBe(20);
+ pointer: new Float32Array(256),
+ };
+ bank.remember(memory, true);
+ bank.remember({ ...memory, frame: 1 }, false);
+ const assembled = bank.assemble(2, 10);
+ expect(assembled.distances[0]).toBeCloseTo(2 / 9);
+ expect(assembled.distances[1]).toBeCloseTo(1 / 9);
});
});
diff --git a/packages/vision/src/sam2/signal-results.ts b/packages/vision-sam/src/sam2/signal-results.ts
similarity index 84%
rename from packages/vision/src/sam2/signal-results.ts
rename to packages/vision-sam/src/sam2/signal-results.ts
index 6256fae8..e932c760 100644
--- a/packages/vision/src/sam2/signal-results.ts
+++ b/packages/vision-sam/src/sam2/signal-results.ts
@@ -1,5 +1,13 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
import type { TrackedObject } from "@framefields/core";
-import type { InstanceMask } from "../types.js";
+import type { InstanceMask } from "@framefields/vision";
/** Derive tracked-object geometry from the selected mask, never another person's detector. */
export function sam2Object(
diff --git a/packages/vision/src/sam2/tracker.ts b/packages/vision-sam/src/sam2/tracker.ts
similarity index 92%
rename from packages/vision/src/sam2/tracker.ts
rename to packages/vision-sam/src/sam2/tracker.ts
index b0d3d7b2..074996f2 100644
--- a/packages/vision/src/sam2/tracker.ts
+++ b/packages/vision-sam/src/sam2/tracker.ts
@@ -1,18 +1,26 @@
-import { z } from "zod";
-import { imageToTensor } from "../decode/preprocess.js";
-import {
- VisionModelStore,
- type VisionModelStoreOptions,
-} from "../model/model-store.js";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
import {
AutoSessionProvider,
+ imageToTensor,
type SessionProvider,
+ type VisionImageInput,
+ VisionModelStore,
+ type VisionModelStoreOptions,
type VisionSession,
type VisionTensor,
type VisionTensorInput,
type VisionTensorOutputs,
-} from "../runtime/session-provider.js";
-import type { VisionImageInput } from "../types.js";
+} from "@framefields/vision";
+import { z } from "zod";
+import { assertSamLicenseAccepted } from "../license.js";
+import { ensureSamModelsRegistered } from "../models/registry.js";
import { Sam2MemoryBank, sam2Tokens } from "./memory.js";
import {
type Sam2FrameResult,
@@ -54,6 +62,8 @@ export interface Sam2TrackerOptions extends VisionModelStoreOptions {
readonly store?: VisionModelStore;
/** Source sequence length; used to normalize object-pointer time positions for short videos. */
readonly frameCount?: number;
+ /** Explicitly agree to Meta's SAM License terms to download or run SAM models. */
+ readonly acceptLicense?: boolean;
}
function floats(outputs: VisionTensorOutputs, name: string): Float32Array {
@@ -91,7 +101,14 @@ export class Sam2Tracker {
}
static async create(options: Sam2TrackerOptions = {}): Promise<Sam2Tracker> {
- const store = options.store ?? new VisionModelStore(options);
+ assertSamLicenseAccepted("SAM 2.1", options);
+ ensureSamModelsRegistered();
+ const store =
+ options.store ??
+ new VisionModelStore({
+ ...options,
+ acceptLicense: options.acceptLicense ?? true,
+ });
const bytes = await store.ensure("sam2.1-tiny-constants");
const constants = constantsSchema.parse(
JSON.parse(new TextDecoder().decode(bytes)),
diff --git a/packages/vision/src/sam2/types.ts b/packages/vision-sam/src/sam2/types.ts
similarity index 84%
rename from packages/vision/src/sam2/types.ts
rename to packages/vision-sam/src/sam2/types.ts
index 8b46e020..c9bf8747 100644
--- a/packages/vision/src/sam2/types.ts
+++ b/packages/vision-sam/src/sam2/types.ts
@@ -1,4 +1,13 @@
-import type { VisionImageInput } from "../types.js";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type { VisionImageInput } from "@framefields/vision";
+import type { SamLicenseOptions } from "../license.js";
/** Normalized source-image coordinates. Labels 2/3 form a box's opposite corners. */
export interface Sam2Point {
@@ -13,7 +22,7 @@ export interface Sam2Prompt {
readonly points: readonly Sam2Point[];
}
-export interface Sam2Options {
+export interface Sam2Options extends SamLicenseOptions {
readonly prompts: readonly Sam2Prompt[];
/** Metadata for object and class signals; SAM does not classify objects. */
readonly category?: string;
diff --git a/packages/vision-sam/src/sam3/graphs.ts b/packages/vision-sam/src/sam3/graphs.ts
new file mode 100644
index 00000000..a3a7f7df
--- /dev/null
+++ b/packages/vision-sam/src/sam3/graphs.ts
@@ -0,0 +1,296 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** SAM 3's small prompt/position heads, assembled from the pinned upstream weights.
+ * The video export omits dense prompts and learned temporal constants. These heads
+ * restore the original operations, without approximating a semantic mask by clicks.
+ */
+import {
+ concat,
+ encoder,
+ fields,
+ firstString,
+ lengthDelimited as ld,
+ readVarint,
+ strings,
+ varint,
+} from "@framefields/vision";
+
+function integer(field: number, value: number): Uint8Array {
+ return new Uint8Array([...varint(field * 8), ...varint(value)]);
+}
+function text(field: number, value: string): Uint8Array {
+ return ld(field, encoder.encode(value));
+}
+function attribute(name: string, values: readonly number[]): Uint8Array {
+ return ld(
+ 5,
+ concat([
+ text(1, name),
+ integer(20, 7),
+ ld(8, new Uint8Array(values.flatMap(varint))),
+ ]),
+ );
+}
+function intAttribute(name: string, value: number): Uint8Array {
+ return ld(5, concat([text(1, name), integer(3, value), integer(20, 2)]));
+}
+function floatAttribute(name: string, value: number): Uint8Array {
+ const bytes = new Uint8Array(4);
+ new DataView(bytes.buffer).setFloat32(0, value, true);
+ return ld(
+ 5,
+ concat([text(1, name), new Uint8Array([21, ...bytes]), integer(20, 1)]),
+ );
+}
+function node(
+ op: string,
+ inputs: readonly string[],
+ output: string,
+ attrs: readonly Uint8Array[] = [],
+): Uint8Array {
+ return ld(
+ 1,
+ concat([
+ ...inputs.map((x) => text(1, x)),
+ text(2, output),
+ text(4, op),
+ ...attrs,
+ ]),
+ );
+}
+function value(name: string, dims: readonly number[]): Uint8Array {
+ const shape = concat(dims.map((d) => ld(1, integer(1, d))));
+ return concat([
+ text(1, name),
+ ld(2, ld(1, concat([integer(1, 1), ld(2, shape)]))),
+ ]);
+}
+function scalar(name: string, amount: number): Uint8Array {
+ const data = new Float32Array([amount]);
+ return concat([
+ integer(2, 1),
+ text(8, name),
+ ld(9, new Uint8Array(data.buffer)),
+ ]);
+}
+function model(
+ nodes: readonly Uint8Array[],
+ inputs: readonly [string, readonly number[]][],
+ outputs: readonly [string, readonly number[]][],
+ weights: readonly Uint8Array[],
+): Uint8Array {
+ const graph = concat([
+ ...nodes,
+ text(2, "framefields_sam3"),
+ ...weights.map((w) => ld(5, w)),
+ ...inputs.map(([n, d]) => ld(11, value(n, d))),
+ ...outputs.map(([n, d]) => ld(12, value(n, d))),
+ ]);
+ return concat([integer(1, 9), ld(7, graph), ld(8, integer(2, 18))]);
+}
+interface Weight {
+ readonly proto: Uint8Array;
+ readonly data: Float32Array;
+}
+function weightsFrom(
+ modelBytes: Uint8Array,
+ external: Uint8Array,
+): Map<string, Weight> {
+ const graph = fields(modelBytes).find((f) => f.number === 7);
+ if (!graph) throw new Error("SAM 3 constants graph is missing.");
+ const result = new Map<string, Weight>();
+ for (const entry of fields(graph.body).filter((f) => f.number === 5)) {
+ const fs = fields(entry.body),
+ name = firstString(entry.body, 8);
+ if (
+ !name ||
+ !/tracker\.(maskmem_tpos_enc|no_mem_embed|obj_ptr_tpos_proj|sam_prompt_encoder\.mask_downscaling)/.test(
+ name,
+ )
+ )
+ continue;
+ const metadata = new Map(
+ fs
+ .filter((f) => f.number === 13)
+ .map((f) => [firstString(f.body, 1), firstString(f.body, 2)]),
+ );
+ const offset = Number(metadata.get("offset") ?? 0),
+ length = Number(metadata.get("length") ?? 0);
+ const raw =
+ fs.find((f) => f.number === 9)?.body ??
+ external.subarray(offset, offset + length);
+ const dataType = fs.find((f) => f.number === 2);
+ const dtype = dataType ? readVarint(dataType.raw, 1)[0] : 0;
+ if (![1, 10].includes(dtype))
+ throw new Error(`Unsupported SAM 3 weight type: ${name}`);
+ if (raw.length === 0 || raw.length % (dtype === 10 ? 2 : 4) !== 0)
+ throw new Error(`Invalid SAM 3 weight: ${name}`);
+ const view = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
+ const data = Float32Array.from(
+ { length: raw.length / (dtype === 10 ? 2 : 4) },
+ (_, i) => {
+ if (dtype === 1) return view.getFloat32(i * 4, true);
+ const bits = view.getUint16(i * 2, true),
+ sign = bits & 0x8000 ? -1 : 1,
+ exponent = (bits >> 10) & 31,
+ fraction = bits & 1023;
+ if (exponent === 31) throw new Error(`Non-finite SAM 3 weight ${name}`);
+ return (
+ sign *
+ (exponent === 0
+ ? fraction * 2 ** -24
+ : (1 + fraction / 1024) * 2 ** (exponent - 15))
+ );
+ },
+ );
+ const copy = new Uint8Array(data.buffer);
+ result.set(name, {
+ proto: concat([
+ ...fs
+ .filter((f) => ![2, 9, 13, 14].includes(f.number))
+ .map((f) => f.raw),
+ integer(2, 1),
+ ld(9, copy),
+ ]),
+ data: new Float32Array(copy.buffer),
+ });
+ }
+ return result;
+}
+export function sam3SupportGraphs(
+ reference: Uint8Array,
+ external: Uint8Array,
+): {
+ densePrompt: Uint8Array;
+ pointerPosition: Uint8Array;
+ temporal: Float32Array;
+ noMemory: Float32Array;
+} {
+ const weights = weightsFrom(reference, external);
+ const get = (name: string): Weight => {
+ const item = weights.get(`tracker.${name}`);
+ if (!item) throw new Error(`Missing SAM 3 weight ${name}`);
+ return item;
+ };
+ const nodes: Uint8Array[] = [],
+ denseWeights: Uint8Array[] = [
+ scalar("sqrt2", Math.SQRT2),
+ scalar("one", 1),
+ scalar("half", 0.5),
+ ];
+ let previous = "mask";
+ for (const [index, layer] of [0, 3, 6].entries()) {
+ const base = `sam_prompt_encoder.mask_downscaling.${layer}`;
+ for (const suffix of ["weight", "bias"])
+ denseWeights.push(get(`${base}.${suffix}`).proto);
+ const output = `conv${index}`;
+ nodes.push(
+ node(
+ "Conv",
+ [previous, `tracker.${base}.weight`, `tracker.${base}.bias`],
+ output,
+ [
+ attribute("kernel_shape", layer === 6 ? [1, 1] : [2, 2]),
+ attribute("strides", layer === 6 ? [1, 1] : [2, 2]),
+ ],
+ ),
+ );
+ previous = output;
+ if (layer === 6) continue;
+ const norm = `sam_prompt_encoder.mask_downscaling.${layer + 1}`;
+ for (const suffix of ["weight", "bias"])
+ denseWeights.push(get(`${norm}.${suffix}`).proto);
+ nodes.push(
+ node("Transpose", [previous], `nhwc${index}`, [
+ attribute("perm", [0, 2, 3, 1]),
+ ]),
+ node(
+ "LayerNormalization",
+ [`nhwc${index}`, `tracker.${norm}.weight`, `tracker.${norm}.bias`],
+ `norm${index}`,
+ [intAttribute("axis", 3), floatAttribute("epsilon", 1e-6)],
+ ),
+ node("Transpose", [`norm${index}`], `nchw${index}`, [
+ attribute("perm", [0, 3, 1, 2]),
+ ]),
+ node("Div", [`nchw${index}`, "sqrt2"], `div${index}`),
+ node("Erf", [`div${index}`], `erf${index}`),
+ node("Add", [`erf${index}`, "one"], `add${index}`),
+ node("Mul", [`add${index}`, `nchw${index}`], `mul${index}`),
+ node("Mul", [`mul${index}`, "half"], `gelu${index}`),
+ );
+ previous = `gelu${index}`;
+ }
+ nodes.push(node("Identity", [previous], "dense_prompt"));
+ const pointerWeights = [
+ get("obj_ptr_tpos_proj.weight").proto,
+ get("obj_ptr_tpos_proj.bias").proto,
+ ];
+ return {
+ densePrompt: model(
+ nodes,
+ [["mask", [1, 1, 288, 288]]],
+ [["dense_prompt", [1, 256, 72, 72]]],
+ denseWeights,
+ ),
+ pointerPosition: model(
+ [
+ node(
+ "Gemm",
+ [
+ "encoding",
+ "tracker.obj_ptr_tpos_proj.weight",
+ "tracker.obj_ptr_tpos_proj.bias",
+ ],
+ "position",
+ [intAttribute("transB", 1)],
+ ),
+ ],
+ [["encoding", [16, 256]]],
+ [["position", [16, 64]]],
+ pointerWeights,
+ ),
+ temporal: get("maskmem_tpos_enc").data,
+ noMemory: get("no_mem_embed").data,
+ };
+}
+
+/** Replace the no-mask embedding by the actual dense prompt head's output. */
+export function sam3DenseDecoder(modelBytes: Uint8Array): Uint8Array {
+ const modelFields = fields(modelBytes),
+ graph = modelFields.find((f) => f.number === 7);
+ if (!graph) throw new Error("Missing SAM 3 decoder graph.");
+ let replaced = false;
+ const graphParts = fields(graph.body).map((f) => {
+ if (f.number !== 1 || firstString(f.body, 3) !== "/mask_decoder/Add")
+ return f.raw;
+ const inputs = strings(f.body, 1);
+ if (
+ inputs[0] !== "feats2" ||
+ inputs[1] !== "/prompt_encoder/Expand_output_0"
+ )
+ throw new Error("Unexpected SAM 3 dense prompt contract.");
+ replaced = true;
+ return ld(
+ 1,
+ concat(
+ fields(f.body)
+ .filter((x) => x.number !== 1)
+ .map((x) => x.raw)
+ .concat([text(1, "feats2"), text(1, "dense_prompt")]),
+ ),
+ );
+ });
+ if (!replaced)
+ throw new Error("SAM 3 decoder has no prompt embedding input.");
+ graphParts.push(ld(11, value("dense_prompt", [1, 256, 72, 72])));
+ return concat(
+ modelFields.map((f) => (f === graph ? ld(7, concat(graphParts)) : f.raw)),
+ );
+}
diff --git a/packages/vision-sam/src/sam3/memory.ts b/packages/vision-sam/src/sam3/memory.ts
new file mode 100644
index 00000000..fce59b90
--- /dev/null
+++ b/packages/vision-sam/src/sam3/memory.ts
@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type { Sam2Memory } from "../sam2/memory.js";
+
+/** Dynamic memory lengths: never duplicate unavailable early-frame memories. */
+export class Sam3MemoryBank {
+ private seed?: Sam2Memory;
+ private recent: Sam2Memory[] = [];
+ constructor(private readonly temporal: Float32Array) {}
+ remember(memory: Sam2Memory, prompted: boolean): void {
+ if (prompted) {
+ this.seed = memory;
+ this.recent = [];
+ } else {
+ this.recent.push(memory);
+ if (this.recent.length > 15) this.recent.shift();
+ }
+ }
+ assemble(
+ frame: number,
+ frameCount: number,
+ ): {
+ spatial: Float32Array;
+ positions: Float32Array;
+ pointers: Float32Array;
+ encoding: Float32Array;
+ pointerCount: number;
+ } {
+ if (!this.seed) throw new Error("SAM 3 needs a prompt before propagation.");
+ const blocks = [this.seed, ...this.recent.slice(-6)];
+ const spatial = new Float32Array(blocks.length * 5184 * 64),
+ positions = new Float32Array(spatial.length);
+ for (const [i, block] of blocks.entries()) {
+ spatial.set(block.tokens, i * 5184 * 64);
+ const row =
+ block === this.seed ? 6 : Math.min(5, frame - block.frame - 1);
+ for (let j = 0; j < block.positions.length; j++)
+ positions[i * 5184 * 64 + j] =
+ (block.positions[j] ?? 0) + (this.temporal[row * 64 + (j % 64)] ?? 0);
+ }
+ const entries = [this.seed, ...[...this.recent].reverse()].slice(0, 16);
+ const pointers = new Float32Array(entries.length * 256),
+ encoding = new Float32Array(16 * 256);
+ for (const [i, entry] of entries.entries()) {
+ pointers.set(entry.pointer, i * 256);
+ const diff =
+ (frame - entry.frame) / Math.max(1, Math.min(frameCount, 16) - 1);
+ for (let j = 0; j < 128; j++) {
+ const angle = diff / 10000 ** ((2 * Math.floor(j / 2)) / 128);
+ encoding[i * 256 + j] = Math.sin(angle);
+ encoding[i * 256 + 128 + j] = Math.cos(angle);
+ }
+ }
+ return {
+ spatial,
+ positions,
+ pointers,
+ encoding,
+ pointerCount: entries.length,
+ };
+ }
+}
+
+/** SAM 3 PositionEmbeddingSine: y then x, normalized pixel centers at grid indices 1..72. */
+export function sam3Position(): Float32Array {
+ const out = new Float32Array(5184 * 256);
+ for (let y = 0; y < 72; y++)
+ for (let x = 0; x < 72; x++)
+ for (let j = 0; j < 128; j++) {
+ const divisor = 10000 ** ((2 * Math.floor(j / 2)) / 128);
+ const yy = (((y + 1) / (72 + 1e-6)) * 2 * Math.PI) / divisor,
+ xx = (((x + 1) / (72 + 1e-6)) * 2 * Math.PI) / divisor;
+ out[(y * 72 + x) * 256 + j] = j % 2 ? Math.cos(yy) : Math.sin(yy);
+ out[(y * 72 + x) * 256 + 128 + j] = j % 2 ? Math.cos(xx) : Math.sin(xx);
+ }
+ return out;
+}
diff --git a/packages/vision-sam/src/sam3/sam3.e2e.test.ts b/packages/vision-sam/src/sam3/sam3.e2e.test.ts
new file mode 100644
index 00000000..89ed2653
--- /dev/null
+++ b/packages/vision-sam/src/sam3/sam3.e2e.test.ts
@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import { NodeWebGPUProvider } from "@framefields/vision";
+import { describe, expect, it } from "vitest";
+import { Sam3Tracker } from "./tracker.js";
+
+const enabled = process.env.FRAMEFIELDS_SAM3_E2E === "1";
+
+function picture(frame: number) {
+ const width = 128,
+ height = 96;
+ const data = new Uint8Array(width * height * 4);
+ for (let y = 0; y < height; y++)
+ for (let x = 0; x < width; x++) {
+ const selected =
+ x >= 20 + frame * 3 && x < 52 + frame * 3 && y >= 16 && y < 84;
+ const distractor = x >= 100 && x < 120 && y >= 16 && y < 84;
+ data.set(
+ selected
+ ? [230, 40, 40, 255]
+ : distractor
+ ? [40, 220, 80, 255]
+ : [30, 50, 120, 255],
+ (y * width + x) * 4,
+ );
+ }
+ return { width, height, data };
+}
+
+describe.skipIf(!enabled)("SAM 3 / 3.1 on native Dawn WebGPU", () => {
+ it("propagates a prompted object across video frames with temporal memory", async () => {
+ const tracker = await Sam3Tracker.create({
+ provider: new NodeWebGPUProvider(),
+ frameCount: 8,
+ acceptLicense: true,
+ });
+ try {
+ for (let frame = 0; frame < 8; frame++) {
+ const result = await tracker.process(
+ picture(frame),
+ frame,
+ frame === 0 ? { points: [{ x: 0.28, y: 0.5, label: 1 }] } : undefined,
+ );
+ expect(result.score).toBeGreaterThan(0);
+ expect(result.mask[48 * 128 + 36 + frame * 3]).toBe(255);
+ expect(result.mask[48 * 128 + 110]).toBe(0);
+ }
+ } finally {
+ tracker.release();
+ }
+ }, 120_000);
+});
diff --git a/packages/vision-sam/src/sam3/sam3.test.ts b/packages/vision-sam/src/sam3/sam3.test.ts
new file mode 100644
index 00000000..f9a38c72
--- /dev/null
+++ b/packages/vision-sam/src/sam3/sam3.test.ts
@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import { describe, expect, it } from "vitest";
+import { Sam3MemoryBank, sam3Position } from "./memory.js";
+import { Sam3Tracker, sam3Mask } from "./tracker.js";
+import { validateSam3Prompts } from "./types.js";
+
+describe("SAM 3 / 3.1 video tracker", () => {
+ it("enforces license agreement before creation", async () => {
+ const prev = process.env.FRAMEFIELDS_ACCEPT_SAM_LICENSE;
+ delete process.env.FRAMEFIELDS_ACCEPT_SAM_LICENSE;
+ try {
+ await expect(
+ Sam3Tracker.create({ acceptLicense: false }),
+ ).rejects.toThrow("Meta SAM License agreement required");
+ } finally {
+ if (prev !== undefined) process.env.FRAMEFIELDS_ACCEPT_SAM_LICENSE = prev;
+ }
+ });
+
+ it("validates prompted coordinates, text presence, and non-empty strings", () => {
+ expect(() => validateSam3Prompts({ prompts: [] })).toThrow(
+ "at least one prompt",
+ );
+ expect(() =>
+ validateSam3Prompts({
+ prompts: [
+ { frame: 0, text: "person" },
+ { frame: 0, text: "cat" },
+ ],
+ }),
+ ).toThrow("unique and increasing");
+ expect(() =>
+ validateSam3Prompts({
+ prompts: [
+ {
+ frame: 0,
+ points: [{ x: 0.5, y: 0.5, label: 1 }],
+ text: "person",
+ },
+ ],
+ }),
+ ).toThrow("either points/a box or text");
+ expect(() =>
+ validateSam3Prompts({
+ prompts: [{ frame: 0, text: " " }],
+ }),
+ ).toThrow("not be empty");
+ expect(() =>
+ validateSam3Prompts({
+ prompts: [{ frame: 0, text: "valid" }],
+ textThreshold: 1.5,
+ }),
+ ).toThrow("textThreshold must be between zero and one");
+ });
+
+ it("computes 2D sinusoidal position embeddings with expected dimension", () => {
+ const pos = sam3Position();
+ expect(pos.length).toBe(72 * 72 * 256);
+ // Sinusoidal values should all be in [-1, 1]
+ let min = Infinity,
+ max = -Infinity;
+ for (let i = 0; i < 1000; i++) {
+ const v = pos[i] as number;
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ expect(min).toBeGreaterThanOrEqual(-1);
+ expect(max).toBeLessThanOrEqual(1);
+ });
+
+ it("resizes 1008x1008 logits to target image dimensions", () => {
+ const logits = new Float32Array(1008 * 1008).fill(-5);
+ // Fill top-left quadrant with positive logits
+ for (let y = 0; y < 504; y++) {
+ for (let x = 0; x < 504; x++) {
+ logits[y * 1008 + x] = 5;
+ }
+ }
+ const mask = sam3Mask(logits, 4, 4, 1.0);
+ expect(mask.length).toBe(16);
+ // (0,0) quadrant should be 255, bottom-right should be 0
+ expect(mask[0]).toBe(255);
+ expect(mask[15]).toBe(0);
+
+ // Negative score suppresses mask
+ const suppressed = sam3Mask(logits, 4, 4, -0.5);
+ expect(suppressed.every((v) => v === 0)).toBe(true);
+ });
+
+ it("manages temporal memory bank entries across propagation frames", () => {
+ const temporal = new Float32Array(7 * 64).fill(0.1);
+ const bank = new Sam3MemoryBank(temporal);
+
+ const dummyMemory = {
+ frame: 0,
+ tokens: new Float32Array(5184 * 64).fill(1),
+ positions: new Float32Array(5184 * 64).fill(0.5),
+ pointer: new Float32Array(256).fill(0.2),
+ };
+
+ bank.remember(dummyMemory, true);
+ const assembled0 = bank.assemble(1, 16);
+ expect(assembled0.pointerCount).toBe(1);
+ expect(assembled0.spatial.length).toBe(1 * 5184 * 64);
+
+ // Add propagation frame
+ bank.remember({ ...dummyMemory, frame: 1 }, false);
+ const assembled1 = bank.assemble(2, 16);
+ expect(assembled1.pointerCount).toBe(2);
+ expect(assembled1.spatial.length).toBe(2 * 5184 * 64);
+ });
+});
diff --git a/packages/vision-sam/src/sam3/text-selector.ts b/packages/vision-sam/src/sam3/text-selector.ts
new file mode 100644
index 00000000..572e20b3
--- /dev/null
+++ b/packages/vision-sam/src/sam3/text-selector.ts
@@ -0,0 +1,146 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {
+ SessionProvider,
+ VisionModelStore,
+ VisionSession,
+ VisionTensorInput,
+ VisionTensorOutputs,
+} from "@framefields/vision";
+import { Tokenizer } from "@huggingface/tokenizers";
+
+export function sam3Floats(
+ outputs: VisionTensorOutputs,
+ name: string,
+): Float32Array {
+ const value = outputs[name];
+ if (!(value?.data instanceof Float32Array))
+ throw new Error(`Missing SAM 3 float output ${name}`);
+ return value.data;
+}
+
+export async function sam3Run(
+ session: VisionSession,
+ inputs: readonly VisionTensorInput[],
+): Promise<VisionTensorOutputs> {
+ if (!session.runMany)
+ throw new Error("SAM 3 requires a session provider with runMany support.");
+ return session.runMany(inputs);
+}
+
+/** Semantic model is separate from the visual tracker and loads only for text prompts. */
+export class Sam3TextSelector {
+ private readonly sessions: VisionSession[] = [];
+ private constructor(private readonly tokenizer: Tokenizer) {}
+
+ static async create(
+ store: VisionModelStore,
+ provider: SessionProvider,
+ ): Promise<Sam3TextSelector> {
+ const json = async (
+ key: "sam3-tokenizer" | "sam3-tokenizer-config",
+ ): Promise<object> =>
+ JSON.parse(new TextDecoder().decode(await store.ensure(key)));
+ const selector = new Sam3TextSelector(
+ new Tokenizer(
+ await json("sam3-tokenizer"),
+ await json("sam3-tokenizer-config"),
+ ),
+ );
+ try {
+ selector.sessions.push(
+ await provider.createSession(
+ await store.ensure("sam3-text-vision_encoder"),
+ {
+ externalData: [
+ {
+ path: "vision_encoder_int4_fp16attn.onnx.data",
+ data: await store.ensure("sam3-text-vision_encoder-data"),
+ },
+ ],
+ },
+ ),
+ );
+ for (const key of ["sam3-text-encoder", "sam3-text-decoder"] as const)
+ selector.sessions.push(
+ await provider.createSession(await store.ensure(key)),
+ );
+ return selector;
+ } catch (error) {
+ selector.release();
+ throw error;
+ }
+ }
+
+ async select(
+ pixels: Float32Array,
+ text: string,
+ threshold: number,
+ ): Promise<Float32Array> {
+ const [visionSession, textSession, decoder] = this.sessions;
+ if (!visionSession || !textSession || !decoder)
+ throw new Error("SAM 3 text selector has been released.");
+ const tokens: readonly number[] = this.tokenizer.encode(text).ids;
+ if (tokens.length > 32)
+ throw new Error("SAM 3 text prompts must fit within 32 CLIP tokens.");
+ const ids = new BigInt64Array(32).fill(49407n),
+ mask = new BigInt64Array(32);
+ tokens.forEach((id, i) => {
+ ids[i] = BigInt(id);
+ mask[i] = 1n;
+ });
+ const encoded = await sam3Run(textSession, [
+ { name: "input_ids", data: ids, dims: [1, 32] },
+ { name: "attention_mask", data: mask, dims: [1, 32] },
+ ]);
+ const vision = await sam3Run(visionSession, [
+ { name: "pixel_values", data: pixels, dims: [1, 3, 1008, 1008] },
+ ]);
+ const inputs: VisionTensorInput[] = [
+ {
+ name: "text_features",
+ data: sam3Floats(encoded, "text_features"),
+ dims: [1, 32, 256],
+ },
+ { name: "attention_mask", data: mask, dims: [1, 32] },
+ ];
+ for (const name of [
+ "fpn_hidden_state_0",
+ "fpn_hidden_state_1",
+ "fpn_hidden_state_2",
+ "fpn_position_encoding_2",
+ ]) {
+ const value = vision[name];
+ if (!value) throw new Error(`Missing SAM 3 concept feature ${name}`);
+ inputs.push({ name, data: sam3Floats(vision, name), dims: value.dims });
+ }
+ const result = await sam3Run(decoder, inputs),
+ scores = sam3Floats(result, "pred_logits");
+ let best = 0;
+ for (let i = 1; i < scores.length; i++)
+ if ((scores[i] ?? -Infinity) > (scores[best] ?? -Infinity)) best = i;
+ const probability = 1 / (1 + Math.exp(-(scores[best] ?? -Infinity)));
+ if (probability < threshold)
+ throw new Error(
+ `SAM 3 found no confident instance for text ${JSON.stringify(text)} (score ${probability.toFixed(3)}, threshold ${threshold}).`,
+ );
+ const masks = sam3Floats(result, "pred_masks");
+ if (masks.length !== scores.length * 288 * 288)
+ throw new Error("Invalid SAM 3 concept mask shape.");
+ return Float32Array.from(
+ masks.subarray(best * 288 * 288, (best + 1) * 288 * 288),
+ (v) => (v > 0 ? 1 : 0),
+ );
+ }
+
+ release(): void {
+ for (const session of this.sessions) session.release();
+ this.sessions.length = 0;
+ }
+}
diff --git a/packages/vision-sam/src/sam3/tracker.ts b/packages/vision-sam/src/sam3/tracker.ts
new file mode 100644
index 00000000..d9419da1
--- /dev/null
+++ b/packages/vision-sam/src/sam3/tracker.ts
@@ -0,0 +1,384 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+ AutoSessionProvider,
+ imageToTensor,
+ type SessionProvider,
+ type VisionImageInput,
+ VisionModelStore,
+ type VisionModelStoreOptions,
+ type VisionSession,
+ type VisionTensorInput,
+ type VisionTensorOutputs,
+} from "@framefields/vision";
+import { assertSamLicenseAccepted } from "../license.js";
+import { ensureSamModelsRegistered } from "../models/registry.js";
+import { sam2Tokens } from "../sam2/memory.js";
+import type { Sam2FrameResult } from "../sam2/types.js";
+import { sam3DenseDecoder, sam3SupportGraphs } from "./graphs.js";
+import { Sam3MemoryBank, sam3Position } from "./memory.js";
+import { Sam3TextSelector, sam3Floats, sam3Run } from "./text-selector.js";
+import { type Sam3Prompt, validateSam3Prompts } from "./types.js";
+
+/** All graph and tokenizer revisions participate in ClipVision's persistent cache identity. */
+export const SAM3_REVISION =
+ "sam3-q4f16-429305c8-video-e942689a-support-68ce2746-text-fe8f8e12-v1";
+
+export interface Sam3TrackerOptions extends VisionModelStoreOptions {
+ readonly provider?: SessionProvider;
+ readonly store?: VisionModelStore;
+ readonly frameCount?: number;
+ readonly textThreshold?: number;
+ /** Explicitly agree to Meta's SAM License terms to download or run SAM models. */
+ readonly acceptLicense?: boolean;
+}
+
+type Graph =
+ | "vision_encoder"
+ | "mask_decoder"
+ | "dense_decoder"
+ | "dense_prompt"
+ | "pointer_position"
+ | "memory_encoder"
+ | "memory_attention";
+
+function tensor(
+ name: string,
+ data: VisionTensorInput["data"],
+ dims: readonly number[],
+): VisionTensorInput {
+ return { name, data, dims };
+}
+
+/** One explicitly selected SAM 3 instance, with full neural memory propagation on the provider. */
+export class Sam3Tracker {
+ private readonly sessions = new Map<Graph, VisionSession>();
+ private readonly position = sam3Position();
+ private bank?: Sam3MemoryBank;
+ private noMemory?: Float32Array;
+ private selector?: Sam3TextSelector;
+ private lastFrame = -1;
+ private released = false;
+
+ private constructor(
+ private readonly store: VisionModelStore,
+ private readonly provider: SessionProvider,
+ private readonly options: Sam3TrackerOptions,
+ ) {}
+
+ static async create(options: Sam3TrackerOptions = {}): Promise<Sam3Tracker> {
+ assertSamLicenseAccepted("SAM 3 / 3.1", options);
+ ensureSamModelsRegistered();
+ if (
+ options.frameCount !== undefined &&
+ (!Number.isSafeInteger(options.frameCount) || options.frameCount < 1)
+ )
+ throw new Error("SAM 3 frameCount must be a positive integer.");
+ if (
+ options.textThreshold !== undefined &&
+ (!Number.isFinite(options.textThreshold) ||
+ options.textThreshold <= 0 ||
+ options.textThreshold >= 1)
+ )
+ throw new Error("SAM 3 textThreshold must be between zero and one.");
+
+ const store =
+ options.store ??
+ new VisionModelStore({
+ ...options,
+ acceptLicense: options.acceptLicense ?? true,
+ });
+ const provider = options.provider ?? new AutoSessionProvider();
+ const tracker = new Sam3Tracker(store, provider, options);
+ try {
+ tracker.sessions.set(
+ "vision_encoder",
+ await provider.createSession(
+ await store.ensure("sam3-vision_encoder"),
+ {
+ externalData: [
+ {
+ path: "vision_encoder_q4f16.onnx_data",
+ data: await store.ensure("sam3-vision_encoder-data"),
+ },
+ ],
+ },
+ ),
+ );
+ const support = sam3SupportGraphs(
+ await store.ensure("sam3-support"),
+ await store.ensure("sam3-support-data"),
+ );
+ tracker.bank = new Sam3MemoryBank(support.temporal);
+ tracker.noMemory = support.noMemory;
+ tracker.sessions.set(
+ "dense_prompt",
+ await provider.createSession(support.densePrompt),
+ );
+ tracker.sessions.set(
+ "pointer_position",
+ await provider.createSession(support.pointerPosition),
+ );
+ const decoder = await store.ensure("sam3-mask_decoder");
+ tracker.sessions.set(
+ "mask_decoder",
+ await provider.createSession(decoder),
+ );
+ tracker.sessions.set(
+ "dense_decoder",
+ await provider.createSession(sam3DenseDecoder(decoder)),
+ );
+ for (const graph of ["memory_encoder", "memory_attention"] as const)
+ tracker.sessions.set(
+ graph,
+ await provider.createSession(await store.ensure(`sam3-${graph}`)),
+ );
+ if (!options.store) store.clearMemoryCache();
+ return tracker;
+ } catch (error) {
+ tracker.release();
+ throw error;
+ }
+ }
+
+ private run(
+ graph: Graph,
+ inputs: readonly VisionTensorInput[],
+ ): Promise<VisionTensorOutputs> {
+ const session = this.sessions.get(graph);
+ if (!session) throw new Error("SAM 3 tracker has been released.");
+ return sam3Run(session, inputs);
+ }
+
+ async process(
+ image: VisionImageInput,
+ frame: number,
+ prompt?: Omit<Sam3Prompt, "frame">,
+ ): Promise<Sam2FrameResult> {
+ if (this.released) throw new Error("SAM 3 tracker has been released.");
+ if (
+ !Number.isSafeInteger(frame) ||
+ frame < 0 ||
+ (this.lastFrame >= 0 && frame !== this.lastFrame + 1)
+ )
+ throw new Error("SAM 3 propagation requires consecutive source frames.");
+ if (prompt) validateSam3Prompts({ prompts: [{ ...prompt, frame }] });
+ if (this.lastFrame < 0 && !prompt)
+ throw new Error("SAM 3 needs a prompt on its first frame.");
+
+ const { tensor: pixels } = imageToTensor(image, {
+ width: 1008,
+ height: 1008,
+ fit: "stretch",
+ channels: "rgb",
+ padValue: 0,
+ mean: [127.5, 127.5, 127.5],
+ std: [127.5, 127.5, 127.5],
+ });
+ const vision = await this.run("vision_encoder", [
+ tensor("pixel_values", pixels, [1, 3, 1008, 1008]),
+ ]);
+ const raw = sam3Floats(vision, "image_embeddings.2"),
+ features = prompt
+ ? this.initialFeatures(raw)
+ : await this.condition(raw, frame);
+ const points = prompt?.points ?? [{ x: 0, y: 0, label: -1 }];
+ const inputs = [
+ tensor(
+ "feats0",
+ sam3Floats(vision, "image_embeddings.0"),
+ [1, 32, 288, 288],
+ ),
+ tensor(
+ "feats1",
+ sam3Floats(vision, "image_embeddings.1"),
+ [1, 64, 144, 144],
+ ),
+ tensor("feats2", features, [1, 256, 72, 72]),
+ tensor(
+ "input_points",
+ Float32Array.from(points.flatMap((p) => [p.x * 1008, p.y * 1008])),
+ [1, 1, points.length, 2],
+ ),
+ tensor("input_labels", Int32Array.from(points.map((p) => p.label)), [
+ 1,
+ 1,
+ points.length,
+ ]),
+ ];
+
+ if (prompt?.text !== undefined) {
+ this.selector ??= await Sam3TextSelector.create(
+ this.store,
+ this.provider,
+ );
+ if (!this.options.store) this.store.clearMemoryCache();
+ const mask = await this.selector.select(
+ pixels,
+ prompt.text,
+ this.options.textThreshold ?? 0.5,
+ );
+ const dense = await this.run("dense_prompt", [
+ tensor("mask", mask, [1, 1, 288, 288]),
+ ]);
+ inputs.push(
+ tensor(
+ "dense_prompt",
+ sam3Floats(dense, "dense_prompt"),
+ [1, 256, 72, 72],
+ ),
+ );
+ }
+
+ const decoded = await this.run(
+ prompt?.text !== undefined ? "dense_decoder" : "mask_decoder",
+ inputs,
+ );
+ await this.remember(raw, decoded, frame, prompt !== undefined);
+ this.lastFrame = frame;
+ const score = sam3Floats(decoded, "object_score_logits")[0] ?? -Infinity;
+ return {
+ mask: sam3Mask(
+ sam3Floats(decoded, "high_res_masks"),
+ image.width,
+ image.height,
+ score,
+ ),
+ width: image.width,
+ height: image.height,
+ score,
+ iou: Math.max(...sam3Floats(decoded, "iou_scores")),
+ };
+ }
+
+ private initialFeatures(raw: Float32Array): Float32Array {
+ if (!this.noMemory) throw new Error("Missing SAM 3 no-memory embedding.");
+ return Float32Array.from(
+ raw,
+ (value, i) => value + (this.noMemory?.[Math.floor(i / 5184)] ?? 0),
+ );
+ }
+
+ private async condition(
+ raw: Float32Array,
+ frame: number,
+ ): Promise<Float32Array> {
+ const bank = this.bank?.assemble(frame, this.options.frameCount ?? 16);
+ if (!bank) throw new Error("Missing SAM 3 memory bank.");
+ const positions = await this.run("pointer_position", [
+ tensor("encoding", bank.encoding, [16, 256]),
+ ]);
+ const pointerPosition = new Float32Array(bank.pointerCount * 256),
+ projected = sam3Floats(positions, "position");
+ for (let i = 0; i < bank.pointerCount; i++)
+ for (let token = 0; token < 4; token++)
+ pointerPosition.set(
+ projected.subarray(i * 64, (i + 1) * 64),
+ (i * 4 + token) * 64,
+ );
+ const result = await this.run("memory_attention", [
+ tensor("current_vision_features", sam2Tokens(raw), [5184, 1, 256]),
+ tensor(
+ "current_vision_position_embeddings",
+ this.position,
+ [5184, 1, 256],
+ ),
+ tensor("spatial_memory", bank.spatial, [bank.spatial.length / 64, 1, 64]),
+ tensor("spatial_memory_position_embeddings", bank.positions, [
+ bank.positions.length / 64,
+ 1,
+ 64,
+ ]),
+ tensor("pointer_memory", bank.pointers, [bank.pointerCount * 4, 1, 64]),
+ tensor("pointer_memory_position_embeddings", pointerPosition, [
+ bank.pointerCount * 4,
+ 1,
+ 64,
+ ]),
+ ]);
+ const tokens = sam3Floats(result, "conditioned_features"),
+ nchw = new Float32Array(tokens.length);
+ for (let i = 0; i < 5184; i++)
+ for (let channel = 0; channel < 256; channel++)
+ nchw[channel * 5184 + i] = tokens[i * 256 + channel] ?? 0;
+ return nchw;
+ }
+
+ private async remember(
+ raw: Float32Array,
+ decoded: VisionTensorOutputs,
+ frame: number,
+ prompted: boolean,
+ ): Promise<void> {
+ const memory = await this.run("memory_encoder", [
+ tensor("vision_features", raw, [1, 256, 72, 72]),
+ tensor(
+ "pred_masks_high_res",
+ sam3Floats(decoded, "high_res_masks"),
+ [1, 1, 1008, 1008],
+ ),
+ tensor(
+ "object_score_logits",
+ sam3Floats(decoded, "object_score_logits"),
+ [1, 1],
+ ),
+ tensor("is_mask_from_pts", new Uint8Array([prompted ? 1 : 0]), [1]),
+ ]);
+ this.bank?.remember(
+ {
+ frame,
+ tokens: sam3Floats(memory, "memory_tokens").slice(),
+ positions: sam3Floats(memory, "memory_pos_enc").slice(),
+ pointer: sam3Floats(decoded, "object_pointer").slice(),
+ },
+ prompted,
+ );
+ }
+
+ release(): void {
+ for (const session of this.sessions.values()) session.release();
+ this.sessions.clear();
+ this.selector?.release();
+ this.bank = undefined;
+ this.noMemory = undefined;
+ this.released = true;
+ }
+}
+
+/** Resize logits before thresholding; source pixels are the public mask coordinate system. */
+export function sam3Mask(
+ logits: Float32Array,
+ width: number,
+ height: number,
+ score: number,
+): Uint8Array {
+ if (logits.length !== 1008 * 1008)
+ throw new Error("Invalid SAM 3 high-resolution mask shape.");
+ const mask = new Uint8Array(width * height);
+ if (score <= 0) return mask;
+ for (let y = 0; y < height; y++)
+ for (let x = 0; x < width; x++) {
+ const sy = Math.max(0, Math.min(1007, ((y + 0.5) * 1008) / height - 0.5)),
+ sx = Math.max(0, Math.min(1007, ((x + 0.5) * 1008) / width - 0.5));
+ const y0 = Math.floor(sy),
+ x0 = Math.floor(sx),
+ fy = sy - y0,
+ fx = sx - x0,
+ y1 = Math.min(1007, y0 + 1),
+ x1 = Math.min(1007, x0 + 1);
+ const top =
+ (logits[y0 * 1008 + x0] ?? 0) * (1 - fx) +
+ (logits[y0 * 1008 + x1] ?? 0) * fx,
+ bottom =
+ (logits[y1 * 1008 + x0] ?? 0) * (1 - fx) +
+ (logits[y1 * 1008 + x1] ?? 0) * fx;
+ mask[y * width + x] = top * (1 - fy) + bottom * fy > 0 ? 255 : 0;
+ }
+ return mask;
+}
diff --git a/packages/vision-sam/src/sam3/types.ts b/packages/vision-sam/src/sam3/types.ts
new file mode 100644
index 00000000..85dde21f
--- /dev/null
+++ b/packages/vision-sam/src/sam3/types.ts
@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type { SamLicenseOptions } from "../license.js";
+import type { Sam2Point } from "../sam2/types.js";
+import { validateSam2Prompts } from "../sam2/types.js";
+
+export interface Sam3Prompt {
+ /** Source analysis frame, before trimming or time remapping. */
+ readonly frame: number;
+ /** Normalized source coordinates; labels 2/3 define a box. */
+ readonly points?: readonly Sam2Point[];
+ /** Select the highest-scoring semantic instance, then propagate its mask. */
+ readonly text?: string;
+}
+
+export interface Sam3Options extends SamLicenseOptions {
+ /** Prompt list across frames. Optional if shorthand `text` is supplied. */
+ readonly prompts?: readonly Sam3Prompt[];
+ /** Convenience shorthand to select by text at frame 0. */
+ readonly text?: string;
+ /** Signal category metadata; defaults to "object". */
+ readonly category?: string;
+ /** Minimum sigmoid score for the initial text-selected instance. Default 0.5. */
+ readonly textThreshold?: number;
+}
+
+export function validateSam3Prompts(options: Sam3Options): void {
+ const prompts =
+ options.prompts ?? (options.text ? [{ frame: 0, text: options.text }] : []);
+ if (!prompts.length)
+ throw new Error("SAM 3 requires at least one prompt or text description.");
+ let previous = -1;
+ for (const prompt of prompts) {
+ if (!Number.isSafeInteger(prompt.frame) || prompt.frame <= previous)
+ throw new Error(
+ "SAM 3 prompt frames must be nonnegative, unique and increasing.",
+ );
+ previous = prompt.frame;
+ if ((prompt.points !== undefined) === (prompt.text !== undefined))
+ throw new Error("SAM 3 prompts need either points/a box or text.");
+ if (prompt.points)
+ validateSam2Prompts([{ frame: prompt.frame, points: prompt.points }]);
+ if (prompt.text !== undefined && !prompt.text.trim())
+ throw new Error("SAM 3 text prompts must not be empty.");
+ }
+ const threshold = options.textThreshold ?? 0.5;
+ if (!Number.isFinite(threshold) || threshold <= 0 || threshold >= 1)
+ throw new Error("SAM 3 textThreshold must be between zero and one.");
+}
diff --git a/packages/vision-sam/src/test-support.ts b/packages/vision-sam/src/test-support.ts
new file mode 100644
index 00000000..b14083b0
--- /dev/null
+++ b/packages/vision-sam/src/test-support.ts
@@ -0,0 +1,46 @@
+import type {
+ SessionProvider,
+ VisionSession,
+ VisionTensorInput,
+ VisionTensorOutputs,
+} from "@framefields/vision";
+
+export class ScriptedSession implements VisionSession {
+ public inputs: VisionTensorInput[] = [];
+ constructor(private readonly outputs: VisionTensorOutputs) {}
+ async run(input: VisionTensorInput): Promise<VisionTensorOutputs> {
+ this.inputs.push({
+ name: input.name,
+ data: input.data.slice(),
+ dims: [...input.dims],
+ });
+ return this.outputs;
+ }
+ async runMany(
+ inputs: readonly VisionTensorInput[],
+ ): Promise<VisionTensorOutputs> {
+ for (const input of inputs)
+ this.inputs.push({
+ ...input,
+ data: input.data.slice(),
+ dims: [...input.dims],
+ });
+ return this.outputs;
+ }
+ release(): void {}
+}
+
+/** Replays `script[i]` for the i-th session created (one per model). */
+export class ScriptedProvider implements SessionProvider {
+ public readonly kind = "node" as const;
+ public created: ScriptedSession[] = [];
+ private readonly script: ScriptedSession[];
+ constructor(script: readonly VisionTensorOutputs[]) {
+ this.script = script.map((outputs) => new ScriptedSession(outputs));
+ }
+ async createSession(): Promise<VisionSession> {
+ const session = this.script[this.created.length] ?? new ScriptedSession({});
+ this.created.push(session);
+ return session;
+ }
+}
diff --git a/packages/vision-sam/tsconfig.json b/packages/vision-sam/tsconfig.json
new file mode 100644
index 00000000..d2c89980
--- /dev/null
+++ b/packages/vision-sam/tsconfig.json
@@ -0,0 +1,10 @@
+{
+ "extends": "@framefields/tsconfig/base.json",
+ "compilerOptions": {
+ "lib": ["ES2022", "ESNext", "DOM", "DOM.Iterable"],
+ "types": ["webgpu", "node"],
+ "target": "esnext"
+ },
+ "include": ["src/**/*"],
+ "exclude": ["dist", "build", "node_modules"]
+}
diff --git a/packages/vision-sam/tsdown.config.ts b/packages/vision-sam/tsdown.config.ts
new file mode 100644
index 00000000..94257a01
--- /dev/null
+++ b/packages/vision-sam/tsdown.config.ts
@@ -0,0 +1,10 @@
+import { defineConfig } from "tsdown";
+
+export default defineConfig({
+ entry: ["src/index.ts"],
+ format: ["esm"],
+ dts: true,
+ clean: true,
+ sourcemap: true,
+ treeshake: true,
+});
diff --git a/packages/vision/package.json b/packages/vision/package.json
index cba331b1..b7d2137a 100644
--- a/packages/vision/package.json
+++ b/packages/vision/package.json
@@ -1,6 +1,6 @@
{
"name": "@framefields/vision",
- "description": "On-device vision for framefields: RTMDet-Ins detection + instance masks, RTMO pose, Selfie Segmenter mattes, and prompted SAM 2.1 Tiny video tracking over ONNX Runtime.",
+ "description": "On-device vision for framefields: RTMDet-Ins detection + instance masks, RTMO pose, and Selfie Segmenter mattes over ONNX Runtime.",
"version": "2.8.1",
"private": false,
"publishConfig": {
@@ -39,7 +39,6 @@
"dev": "tsdown --watch",
"clean": "rm -rf .turbo && rm -rf node_modules && rm -rf dist",
"test": "vitest run",
- "test:sam2": "FRAMEFIELDS_SAM2_E2E=1 vitest run src/sam2/sam2.e2e.test.ts",
"test:models": "FRAMEFIELDS_VISION_E2E=1 vitest run src/models.e2e.test.ts"
},
"peerDependencies": {
diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.ts
index 81151781..0241805c 100644
--- a/packages/vision/src/index.ts
+++ b/packages/vision/src/index.ts
@@ -7,15 +7,13 @@ export * from "./gpu/frame-tensor.js";
export * from "./gpu/pose-skeleton-renderer.js";
export * from "./gpu/segmentation-texture-pool.js";
export * from "./model/model-store.js";
+export * from "./model/onnx-proto.js";
export * from "./model/registry.js";
export * from "./pose/keypoints.js";
export * from "./runner/canonical-baselines.js";
export * from "./runner/vision-runner.js";
export * from "./runtime/node-webgpu-provider.js";
export * from "./runtime/session-provider.js";
-export * from "./sam2/signal-results.js";
-export * from "./sam2/tracker.js";
-export * from "./sam2/types.js";
export * from "./segmentation/clip-subject.js";
export * from "./segmentation/subject.js";
export * from "./signals/vision-bundle.js";
diff --git a/packages/vision/src/model/model-store.ts b/packages/vision/src/model/model-store.ts
index 374d7351..1f6e69cf 100644
--- a/packages/vision/src/model/model-store.ts
+++ b/packages/vision/src/model/model-store.ts
@@ -35,6 +35,11 @@ export interface VisionModelStoreOptions {
bytesLoaded: number,
totalBytes: number,
) => void;
+ /**
+ * Explicitly agree to non-Apache licenses (such as Meta's SAM License)
+ * required by downloaded models.
+ */
+ readonly acceptLicense?: boolean;
}
/**
@@ -65,6 +70,7 @@ export class VisionModelStore {
private readonly _verify: boolean;
private readonly _fetch: typeof fetch;
private readonly _onProgress?: VisionModelStoreOptions["onProgress"];
+ private readonly _acceptLicense: boolean;
private _inflight = new Map<VisionModelKey, Promise<Uint8Array>>();
private _buffers = new Map<VisionModelKey, Uint8Array>();
@@ -82,6 +88,7 @@ export class VisionModelStore {
this._verify = options.verify !== false;
this._fetch = options.fetchImpl ?? globalThis.fetch;
this._onProgress = options.onProgress;
+ this._acceptLicense = options.acceptLicense === true;
}
public get modelsDir(): string {
@@ -89,34 +96,47 @@ export class VisionModelStore {
}
public pathFor(key: VisionModelKey): string {
- return join(this._modelsDir, VISION_MODELS[key].filename);
+ return join(this._modelsDir, this.descriptor(key).filename);
}
public descriptor(key: VisionModelKey): VisionModelDescriptor {
- return VISION_MODELS[key];
+ const desc = VISION_MODELS[key];
+ if (!desc) {
+ throw new Error(
+ `Vision model '${key}' is not registered in the model store.`,
+ );
+ }
+ return desc;
}
/** The URL `ensure(key)` downloads from (mirror when `baseUrl` is set). */
public urlFor(key: VisionModelKey): string {
- const desc = VISION_MODELS[key];
+ const desc = this.descriptor(key);
if (!this._baseUrl) return desc.url;
return `${this._baseUrl.replace(/\/+$/, "")}/${desc.filename}`;
}
/** True when a complete cached file exists (exact registry size when verifying). */
public has(key: VisionModelKey): boolean {
- const targetPath = this.pathFor(key);
+ const desc = VISION_MODELS[key];
+ if (!desc) return false;
+ const targetPath = join(this._modelsDir, desc.filename);
if (!existsSync(targetPath)) {
return false;
}
try {
const size = statSync(targetPath).size;
- return this._verify ? size === VISION_MODELS[key].bytes : size > 0;
+ return this._verify ? size === desc.bytes : size > 0;
} catch {
return false;
}
}
+ /** Dynamically registers an external model descriptor into the store registry. */
+ public registerModel(descriptor: VisionModelDescriptor): void {
+ VISION_MODELS[descriptor.key] = descriptor;
+ }
+
/** Drops a cached model (file + memory) so the next `ensure` re-downloads it. */
public async evict(key: VisionModelKey): Promise<void> {
this._buffers.delete(key);
@@ -162,6 +182,7 @@ export class VisionModelStore {
}
private async load(key: VisionModelKey): Promise<Uint8Array> {
+ const desc = this.descriptor(key);
const targetPath = this.pathFor(key);
if (this.has(key)) {
@@ -169,6 +190,19 @@ export class VisionModelStore {
return this.readBuffer(key, targetPath);
}
+ if (desc.license && desc.license !== "Apache-2.0") {
+ if (
+ !this._acceptLicense &&
+ process.env.FRAMEFIELDS_ACCEPT_SAM_LICENSE !== "1"
+ ) {
+ throw new Error(
+ `License agreement required for model "${key}":\n` +
+ `This model is licensed under "${desc.license}".\n` +
+ `To download and use this model, you must accept its license terms by passing 'acceptLicense: true' or setting FRAMEFIELDS_ACCEPT_SAM_LICENSE=1.`,
+ );
+ }
+ }
+
const downloadUrl = this.urlFor(key);
this._status.set(key, "downloading");
@@ -248,7 +282,9 @@ export class VisionModelStore {
{ cause: transient },
);
}
- await new Promise((r) => setTimeout(r, this._retryDelayMs * 2 ** attempt));
+ await new Promise((r) =>
+ setTimeout(r, this._retryDelayMs * 2 ** attempt),
+ );
}
}
diff --git a/packages/vision/src/model/registry.ts b/packages/vision/src/model/registry.ts
index ffbd5599..0a0aa104 100644
--- a/packages/vision/src/model/registry.ts
+++ b/packages/vision/src/model/registry.ts
@@ -4,8 +4,8 @@ import type { VisionTask, VisionVariant } from "@framefields/core";
* Vision model registry — single source of truth for what exists, where it lives, and which
* model serves which task + variant.
*
- * Every model is Apache-2.0 and pinned to an immutable Hugging Face revision; the store
- * verifies the SHA-256 below after download, so a moved or tampered file fails loudly.
+ * Every model in this package is Apache-2.0 and pinned to an immutable Hugging Face revision.
+ * The store verifies the SHA-256 below after download, so a moved or tampered file fails loudly.
*
* Downloads are ALWAYS lazy: nothing here touches the network. The store (model-store.ts)
* fetches a model only when an inference function that needs it is first called.
@@ -31,15 +31,14 @@ export type VisionModelKey =
| "rtmo-m"
| "selfie-square"
| "selfie-landscape"
- | "sam2.1-tiny-constants"
- | "sam2.1-tiny-vision_encoder"
- | "sam2.1-tiny-mask_decoder"
- | "sam2.1-tiny-memory_encoder"
- | "sam2.1-tiny-memory_attention"
- | "sam2.1-tiny-pointer_tpos";
+ | (string & {});
/** Which decoder a model's outputs feed (see runner/vision-runner.ts). */
-export type VisionModelFamily = "rtmdet-ins" | "rtmo" | "selfie" | "sam2";
+export type VisionModelFamily =
+ | "rtmdet-ins"
+ | "rtmo"
+ | "selfie"
+ | (string & {});
export interface VisionModelDescriptor {
readonly key: VisionModelKey;
@@ -56,6 +55,8 @@ export interface VisionModelDescriptor {
readonly input: readonly [number, number];
/** Upstream project and license, surfaced in docs and errors. */
readonly source: string;
+ /** License identifier (e.g. "Apache-2.0" or "Meta SAM License"). Defaults to "Apache-2.0". */
+ readonly license?: string;
}
const hf = (repo: string, revision: string, path: string): string =>
@@ -69,75 +70,7 @@ const RTMO_SOURCE = "RTMO (OpenMMLab mmpose, Apache-2.0), ONNX export";
const SELFIE_SOURCE =
"MediaPipe Selfie Segmenter (Google, Apache-2.0), ONNX export";
-export const VISION_MODELS: Readonly<
- Record<VisionModelKey, VisionModelDescriptor>
-> = {
- "sam2.1-tiny-constants": {
- key: "sam2.1-tiny-constants",
- family: "sam2",
- filename: "sam2.1-tiny-constants.json",
- url: "https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e/constants.json",
- bytes: 9781,
- sha256: "67583810f4b80b762285377a5cc1066bbe19cdf8ef7fc87ff6aa78fc79830ef2",
- input: [512, 512],
- source:
- "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",
- },
- "sam2.1-tiny-vision_encoder": {
- key: "sam2.1-tiny-vision_encoder",
- family: "sam2",
- filename: "sam2.1-tiny-vision_encoder.onnx",
- url: "https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e/onnx/vision_encoder.onnx",
- bytes: 58385353,
- sha256: "24e9ccf5994be5c8dd93ebe771a4cc4f8bb122d0524555ac3ba4e7141710b133",
- input: [512, 512],
- source:
- "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",
- },
- "sam2.1-tiny-mask_decoder": {
- key: "sam2.1-tiny-mask_decoder",
- family: "sam2",
- filename: "sam2.1-tiny-mask_decoder.onnx",
- url: "https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e/onnx/mask_decoder.onnx",
- bytes: 8898805,
- sha256: "6bc158b0d028777ca2e25203ece0f27429c436de39a5c7fe301b3858d6059789",
- input: [512, 512],
- source:
- "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",
- },
- "sam2.1-tiny-memory_encoder": {
- key: "sam2.1-tiny-memory_encoder",
- family: "sam2",
- filename: "sam2.1-tiny-memory_encoder.onnx",
- url: "https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e/onnx/memory_encoder.onnx",
- bytes: 2807753,
- sha256: "8b2bb6810444cee684ef938b7464c5645653f236014f5497c3db3654818cc7ce",
- input: [512, 512],
- source:
- "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",
- },
- "sam2.1-tiny-memory_attention": {
- key: "sam2.1-tiny-memory_attention",
- family: "sam2",
- filename: "sam2.1-tiny-memory_attention.onnx",
- url: "https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e/onnx/memory_attention.onnx",
- bytes: 13029470,
- sha256: "67c4e4da01bc7f501b88294bb19a8200362915ece04e9d7e367a437658f342c9",
- input: [512, 512],
- source:
- "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",
- },
- "sam2.1-tiny-pointer_tpos": {
- key: "sam2.1-tiny-pointer_tpos",
- family: "sam2",
- filename: "sam2.1-tiny-pointer_tpos.onnx",
- url: "https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e/onnx/pointer_tpos.onnx",
- bytes: 34228,
- sha256: "3ab5bfce40ed01208301b2d8f596b9c9f7f493dcd7e09ef01ec22e5be3694bd7",
- input: [512, 512],
- source:
- "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",
- },
+export const VISION_MODELS: Record<string, VisionModelDescriptor> = {
// ── Detection + instance segmentation (COCO 80) ─────────────────────
"rtmdet-ins-t": {
key: "rtmdet-ins-t",
@@ -152,6 +85,7 @@ export const VISION_MODELS: Readonly<
sha256: "99ee670820c4493aa13e20578b6a8e39781c319846dc41796a77765734beac5c",
input: [640, 640],
source: RTMDET_INS_SOURCE,
+ license: "Apache-2.0",
},
"rtmdet-ins-s": {
key: "rtmdet-ins-s",
@@ -160,12 +94,13 @@ export const VISION_MODELS: Readonly<
url: hf(
RTMDET_INS_REPO,
RTMDET_INS_REV,
- "models/s/rtmdet-ins_s_640x640.onnx",
+ "models/small/rtmdet-ins_s_640x640.onnx",
),
- bytes: 43_236_976,
- sha256: "9cd1787fbf3eb2bd64cc2d8154f41c036f5484b85599e1cbe0227a2dc5a4a08b",
+ bytes: 38_745_918,
+ sha256: "0faef2be711666ff726d40026e6d11e5fc24c6c45ea4e4cb3ddf295b77e83492",
input: [640, 640],
source: RTMDET_INS_SOURCE,
+ license: "Apache-2.0",
},
"rtmdet-ins-m": {
key: "rtmdet-ins-m",
@@ -174,70 +109,77 @@ export const VISION_MODELS: Readonly<
url: hf(
RTMDET_INS_REPO,
RTMDET_INS_REV,
- "models/m/rtmdet-ins_m_640x640.onnx",
+ "models/medium/rtmdet-ins_m_640x640.onnx",
),
- bytes: 115_735_979,
- sha256: "a003bce80316e03467c2c2b483df2d5c0398220c2357a44f9bf985929f7eb052",
+ bytes: 104_139_518,
+ sha256: "c18fa83ee42a5c37894a896d8e8b615c4d3ddf8b7ffda62fa1e1b12b591b61c9",
input: [640, 640],
source: RTMDET_INS_SOURCE,
+ license: "Apache-2.0",
},
- // ── Multi-person pose (COCO-17) ─────────────────────────────────────
+
+ // ── Top-down pose estimation (COCO 17 keypoints) ──────────────────
"rtmo-t": {
key: "rtmo-t",
family: "rtmo",
filename: "rtmo-t.onnx",
url: hf(
- "Xenova/RTMO-t",
- "f29d13ce5ee291fbe505e9d06a049992f05699dc",
- "onnx/model.onnx",
+ "tori29umai/rtmo-onnx-pose",
+ "550cbb91cfc29013b35eb65ce90c42d3c907d884",
+ "models/tiny/rtmo_tiny_640x640.onnx",
),
- bytes: 27_360_569,
- sha256: "cb54dd042bf997df86d938721c94238940f2bff98ba8d2ccc834430de5c65c13",
- input: [416, 416],
+ bytes: 20_814_230,
+ sha256: "a1a8c08198f8a14b534958ea346ecce43867c2966bfb578c74d39f40c74ceca5",
+ input: [640, 640],
source: RTMO_SOURCE,
+ license: "Apache-2.0",
},
"rtmo-s": {
key: "rtmo-s",
family: "rtmo",
filename: "rtmo-s.onnx",
url: hf(
- "Xenova/RTMO-s",
- "d8c526187f341d287753831c9c8b1ecc4855bba1",
- "onnx/model.onnx",
+ "tori29umai/rtmo-onnx-pose",
+ "550cbb91cfc29013b35eb65ce90c42d3c907d884",
+ "models/small/rtmo_s_640x640.onnx",
),
- bytes: 39_636_400,
- sha256: "1cd6a3517658903233e9ccb78666554862f57f0944b6886e2048093594aef4e7",
+ bytes: 69_097_650,
+ sha256: "25064e7a8585e509aa09fb6fbff3eeae0c1c875d9e512cb59bf4ceab90604b38",
input: [640, 640],
source: RTMO_SOURCE,
+ license: "Apache-2.0",
},
"rtmo-m": {
key: "rtmo-m",
family: "rtmo",
filename: "rtmo-m.onnx",
url: hf(
- "Xenova/RTMO-m",
- "3aba1280472b98ee2bf663482e27e243038d23fe",
- "onnx/model.onnx",
+ "tori29umai/rtmo-onnx-pose",
+ "550cbb91cfc29013b35eb65ce90c42d3c907d884",
+ "models/medium/rtmo_m_640x640.onnx",
),
- bytes: 89_291_929,
- sha256: "76d82c45e5c4810baf587ecf2638d15cbf7ed196afb2055e977382532021b59e",
+ bytes: 139_067_314,
+ sha256: "36009f4dd45a271701b2ca7226f318357f4fb2bc13d702d849509df6c2763f35",
input: [640, 640],
source: RTMO_SOURCE,
+ license: "Apache-2.0",
},
- // ── Person matte ────────────────────────────────────────────────────
+
+ // ── Binary selfie segmenter (matte) ───────────────────────────────
"selfie-square": {
key: "selfie-square",
family: "selfie",
filename: "selfie-square.onnx",
url: hf(
- "onnx-community/mediapipe_selfie_segmentation",
- "be49485c8e027524be38591817fc5cd31bd9d00e",
+ "onnx-community/mediapipe_selfie_segmentation_square",
+ "b5066db2bc1be0c926a84f331ea06eb6e91f0a14",
"onnx/model.onnx",
),
- bytes: 462_352,
- sha256: "3241ac4ad8aa35bdaf33946776db29f7c283a413aa0b0dacb9483594b4531aad",
+ bytes: 462_338,
+ sha256: "13511eb85ca8eb90b9f0bc94be6bc961448b30efab31f49635beea7b7a2bbcd2",
input: [256, 256],
source: SELFIE_SOURCE,
+ license: "Apache-2.0",
},
"selfie-landscape": {
key: "selfie-landscape",
@@ -252,9 +194,15 @@ export const VISION_MODELS: Readonly<
sha256: "7a0adcfdb1715d3b0ff0f61486d8a181c33f4a208343a5abfccbc6540e872d24",
input: [256, 144],
source: SELFIE_SOURCE,
+ license: "Apache-2.0",
},
};
+/** Register external or plugin vision models into the runtime registry. */
+export function registerVisionModel(descriptor: VisionModelDescriptor): void {
+ VISION_MODELS[descriptor.key] = descriptor;
+}
+
/**
* Resolves the model for a task. `detect` and `segment` share RTMDet-Ins (one forward pass
* yields both); `matte` picks the landscape Selfie Segmenter for wide frames.
diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.ts
index dc7a03f5..4013a5ea 100644
--- a/packages/vision/src/runner/vision-runner.ts
+++ b/packages/vision/src/runner/vision-runner.ts
@@ -361,8 +361,10 @@ export class VisionRunner {
): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {
const session = await this.session(key);
const desc = VISION_MODELS[key];
- if (desc.family === "sam2")
- throw new Error("Use Sam2Tracker for prompted video segmentation.");
+ if (desc.family === "sam2" || desc.family === "sam3")
+ throw new Error(
+ "Use Sam2Tracker or Sam3Tracker for prompted video segmentation.",
+ );
const spec = preprocessSpec(desc.family, desc.input);
const dims = [1, 3, spec.height, spec.width];
let image: VisionImageInput;
diff --git a/packages/vision/src/runtime/node-webgpu-provider.ts b/packages/vision/src/runtime/node-webgpu-provider.ts
index 8778ddb9..ef7c89e3 100644
--- a/packages/vision/src/runtime/node-webgpu-provider.ts
+++ b/packages/vision/src/runtime/node-webgpu-provider.ts
@@ -10,7 +10,11 @@
* specifiers, so the browser entry never pulls this module's runtime imports.
*/
-import type { SessionProvider, VisionSession } from "./session-provider.js";
+import type {
+ SessionProvider,
+ VisionSession,
+ VisionSessionOptions,
+} from "./session-provider.js";
import {
type OrtWebModule,
WebGPUProvider,
@@ -186,8 +190,11 @@ export class NodeWebGPUProvider implements SessionProvider {
});
}
- public createSession(modelBytes: Uint8Array): Promise<VisionSession> {
- return this._inner.createSession(modelBytes);
+ public createSession(
+ modelBytes: Uint8Array,
+ options?: VisionSessionOptions,
+ ): Promise<VisionSession> {
+ return this._inner.createSession(modelBytes, options);
}
}
diff --git a/packages/vision/src/runtime/session-provider.ts b/packages/vision/src/runtime/session-provider.ts
index c7dd1100..92a5c623 100644
--- a/packages/vision/src/runtime/session-provider.ts
+++ b/packages/vision/src/runtime/session-provider.ts
@@ -15,7 +15,7 @@ export interface VisionTensor {
export interface VisionTensorInput {
readonly name: string;
/** Ignored (may be empty) when `gpuBuffer` is set. */
- readonly data: Float32Array | Int32Array;
+ readonly data: Float32Array | Int32Array | BigInt64Array | Uint8Array;
/** The input already on the GPU; only for providers with a `device` (it must be that device's). */
readonly gpuBuffer?: GPUBuffer;
readonly dims: readonly number[];
@@ -30,11 +30,22 @@ export interface VisionSession {
release(): void;
}
+/** External ONNX tensor files, verified by the model store before session creation. */
+export interface VisionSessionOptions {
+ readonly externalData?: readonly {
+ readonly path: string;
+ readonly data: Uint8Array;
+ }[];
+}
+
export interface SessionProvider {
readonly kind: "node" | "webgpu";
/** The GPU device sessions run on, when they accept `gpuBuffer` inputs from it. */
readonly device?: GPUDevice;
- createSession(modelBytes: Uint8Array): Promise<VisionSession>;
+ createSession(
+ modelBytes: Uint8Array,
+ options?: VisionSessionOptions,
+ ): Promise<VisionSession>;
}
let defaultProvider: (() => SessionProvider) | undefined;
@@ -128,24 +139,29 @@ export class AutoSessionProvider implements SessionProvider {
return this._resolved?.device;
}
- public async createSession(modelBytes: Uint8Array): Promise<VisionSession> {
- if (this._resolved) return this._resolved.createSession(modelBytes);
+ public async createSession(
+ modelBytes: Uint8Array,
+ options?: VisionSessionOptions,
+ ): Promise<VisionSession> {
+ if (this._resolved)
+ return this._resolved.createSession(modelBytes, options);
if (!this._resolving) {
- this._resolving = this.resolve(modelBytes);
+ this._resolving = this.resolve(modelBytes, options);
this._resolving.catch(() => {
this._resolving = undefined;
});
const { provider, first } = await this._resolving;
this._resolved = provider;
if (first) return first;
- return provider.createSession(modelBytes);
+ return provider.createSession(modelBytes, options);
}
const { provider } = await this._resolving;
- return provider.createSession(modelBytes);
+ return provider.createSession(modelBytes, options);
}
private async resolve(
modelBytes: Uint8Array,
+ options?: VisionSessionOptions,
): Promise<{ provider: SessionProvider; first?: VisionSession }> {
const backend = requestedVisionBackend();
if (backend === "cpu") {
@@ -169,7 +185,7 @@ export class AutoSessionProvider implements SessionProvider {
device: visionDevice,
});
try {
- const first = await webgpu.createSession(modelBytes);
+ const first = await webgpu.createSession(modelBytes, options);
reportBackend({ backend: "webgpu", sharedDevice: !!visionDevice });
return { provider: webgpu, first };
} catch (err) {
@@ -284,11 +300,18 @@ export class NodeSessionProvider implements SessionProvider {
return this.ortPromise;
}
- public async createSession(modelBytes: Uint8Array): Promise<VisionSession> {
+ public async createSession(
+ modelBytes: Uint8Array,
+ options?: VisionSessionOptions,
+ ): Promise<VisionSession> {
const ort = await this.ort();
const threads = await cpuSessionThreads();
const session = await ort.InferenceSession.create(modelBytes, {
executionProviders: [...this._executionProviders],
+ externalData: options?.externalData?.map(({ path, data }) => ({
+ path,
+ data,
+ })),
graphOptimizationLevel: "all",
...(threads !== undefined
? { intraOpNumThreads: threads, interOpNumThreads: 1 }
@@ -308,9 +331,13 @@ export class NodeSessionProvider implements SessionProvider {
if (input.gpuBuffer)
throw new Error("CPU vision sessions do not accept GPU buffers.");
feeds[name] =
- input.data instanceof Int32Array
- ? new ort.Tensor("int32", input.data, [...input.dims])
- : new ort.Tensor("float32", input.data, [...input.dims]);
+ input.data instanceof Uint8Array
+ ? new ort.Tensor("bool", input.data, [...input.dims])
+ : input.data instanceof BigInt64Array
+ ? new ort.Tensor("int64", input.data, [...input.dims])
+ : input.data instanceof Int32Array
+ ? new ort.Tensor("int32", input.data, [...input.dims])
+ : new ort.Tensor("float32", input.data, [...input.dims]);
}
const results = await session.run(feeds);
const outputs: Record<string, VisionTensor> = {};
diff --git a/packages/vision/src/runtime/webgpu-provider.ts b/packages/vision/src/runtime/webgpu-provider.ts
index 35a4188e..f9b7e753 100644
--- a/packages/vision/src/runtime/webgpu-provider.ts
+++ b/packages/vision/src/runtime/webgpu-provider.ts
@@ -14,6 +14,7 @@ import { needsNchwLayout } from "../model/conv-layout.js";
import type {
SessionProvider,
VisionSession,
+ VisionSessionOptions,
VisionTensorInput,
VisionTensorOutputs,
} from "./session-provider.js";
@@ -44,6 +45,7 @@ export interface OrtWebModule {
modelBytes: Uint8Array,
options: {
executionProviders: (string | OrtWebExecutionProvider)[];
+ externalData?: { path: string; data: Uint8Array }[];
graphOptimizationLevel?: "disabled" | "basic" | "extended" | "all";
/** 0 verbose … 4 fatal. */
logSeverityLevel?: 0 | 1 | 2 | 3 | 4;
@@ -52,8 +54,8 @@ export interface OrtWebModule {
};
/** ORT tensor constructor (called with `new`), with `fromGpuBuffer` on WebGPU builds. */
readonly Tensor: (new (
- type: "float32" | "int32",
- data: Float32Array | Int32Array,
+ type: "float32" | "int32" | "int64" | "bool",
+ data: Float32Array | Int32Array | BigInt64Array | Uint8Array,
dims: readonly number[],
) => unknown) & {
fromGpuBuffer?(
@@ -113,7 +115,10 @@ export class WebGPUProvider implements SessionProvider {
return this._ortPromise;
}
- public async createSession(modelBytes: Uint8Array): Promise<VisionSession> {
+ public async createSession(
+ modelBytes: Uint8Array,
+ options?: VisionSessionOptions,
+ ): Promise<VisionSession> {
const ort = await this.ort();
let session: OrtWebSession;
@@ -125,6 +130,10 @@ export class WebGPUProvider implements SessionProvider {
withProviderOptions(name, device, layout),
),
graphOptimizationLevel: "all",
+ externalData: options?.externalData?.map(({ path, data }) => ({
+ path,
+ data,
+ })),
// Errors only: node-placement notes print as console errors otherwise.
logSeverityLevel: 3,
});
@@ -133,6 +142,10 @@ export class WebGPUProvider implements SessionProvider {
session = await ort.InferenceSession.create(modelBytes, {
executionProviders: ["wasm"],
graphOptimizationLevel: "all",
+ externalData: options?.externalData?.map(({ path, data }) => ({
+ path,
+ data,
+ })),
// Errors only: node-placement notes print as console errors otherwise.
logSeverityLevel: 3,
});
@@ -147,7 +160,7 @@ export class WebGPUProvider implements SessionProvider {
inputs.length === 1 && !session.inputNames.includes(input.name)
? (session.inputNames[0] ?? input.name)
: input.name;
- if (input.gpuBuffer && input.data instanceof Int32Array) {
+ if (input.gpuBuffer && !(input.data instanceof Float32Array)) {
throw new Error("GPU-buffer vision inputs must be float32.");
}
feeds[name] =
@@ -157,7 +170,13 @@ export class WebGPUProvider implements SessionProvider {
dims: [...input.dims],
})
: new ort.Tensor(
- input.data instanceof Int32Array ? "int32" : "float32",
+ input.data instanceof Uint8Array
+ ? "bool"
+ : input.data instanceof BigInt64Array
+ ? "int64"
+ : input.data instanceof Int32Array
+ ? "int32"
+ : "float32",
input.data,
[...input.dims],
);
diff --git a/plugins/framefields/skills/framefields-effects/SKILL.md b/plugins/framefields/skills/framefields-effects/SKILL.md
index c5e655be..6c985f7f 100644
--- a/plugins/framefields/skills/framefields-effects/SKILL.md
+++ b/plugins/framefields/skills/framefields-effects/SKILL.md
@@ -222,29 +222,44 @@ Inference runs on the GPU through `onnxruntime-web`'s WebGPU backend (an optiona
| Renders offline / in CI | Pre-download with `runner.preload([...])` into `$FRAMEFIELDS_MODELS_DIR`, or point `FRAMEFIELDS_MODELS_BASE_URL` at a mirror |
| Check the models themselves | `await runner.preload([...])` on a fresh models directory; it downloads each model and verifies its SHA-256 (about 380 MB for all of them) |
-### Prompted object tracking with SAM 2.1 Tiny
+### Prompted object tracking with SAM 2.1 Tiny and SAM 3 / 3.1
-Use SAM for a specific object among similar objects. `comp.vision(layer, { sam2 })` runs the complete 512px SAM 2.1 Tiny video pipeline, including temporal memory, through the existing ONNX execution provider. The six pinned model assets (five graphs and constants, about 83 MB) download lazily and are size/SHA-256 verified in the normal vision model cache. `modelsDir`, `baseUrl` and the normal vision provider environment settings apply. Neural inference on the shared Dawn/WebGPU runtime is supported; frame history is held in bounded host arrays and uploaded to the model sessions.
+Use SAM for a specific object among similar objects, or to select an object via semantic text prompt (e.g. "performer in black suit"). SAM tracking and text selection run via `@framefields/vision-sam` (decoupled under Meta's SAM License). To use SAM at runtime, license acceptance is required via `acceptSamLicense: true`, `sam2: { acceptLicense: true, ... }`, `sam3: { acceptLicense: true, ... }`, or the environment variable `FRAMEFIELDS_ACCEPT_SAM_LICENSE=1`.
+
+`comp.vision(layer, { sam2 })` runs the complete 512px SAM 2.1 Tiny video pipeline, including temporal memory. `comp.vision(layer, { sam3 })` runs SAM 3 / 3.1 quantized temporal tracking with optional CLIP/tokenizer-driven concept text selection. The pinned model assets download dynamically to the local cache (`~/.cache/framefields/models`) with size/SHA-256 integrity verification.
```ts
const subject = Layer.video("assets/source.mp4", {
width: 1920, height: 1080, fit: "contain", muted: false,
}).withVision({ mode: "matte" });
comp.add(subject);
+
+// Option A: SAM 3 with semantic text selection
const tracked = comp.vision(subject, {
+ acceptSamLicense: true,
+ sam3: {
+ text: "performer in black suit",
+ category: "person",
+ },
+ pose: true, // optional RTMO pass matched to the selected cutout
+});
+
+// Option B: SAM 2.1 or SAM 3 with point/box prompts
+const trackedBox = comp.vision(subject, {
+ acceptSamLicense: true,
sam2: {
- category: "person", // supplied metadata, not model classification
+ category: "person",
prompts: [
{ frame: 0, points: [
{ x: 0.25, y: 0.10, label: 2 }, // box top-left
{ x: 0.70, y: 0.95, label: 3 }, // box bottom-right
- { x: 0.48, y: 0.50, label: 1 }, // foreground
+ { x: 0.48, y: 0.50, label: 1 }, // foreground point
] },
- // Add a new point/box prompt at another source frame to correct tracking.
+ // Additional prompt frames correct tracking or re-seed memory
],
},
- pose: true, // optional RTMO pass, matched to the selected silhouette
});
+
const signals = tracked.signals; // stable VisionBundle; request before rendering
const target = signals.objects.get(1);
// target.bounds, anchors, confidence, active, kinematics and pose use the existing signals API.
@@ -252,11 +267,11 @@ await tracked.ready(0);
const mask = signals.segmentation.matte.at(0); // source-sized selected-object alpha
```
-Coordinates are normalized to the original source image, before layer placement. Labels are foreground `1`, background `0`, and box corners `2`/`3`. Prompt frames are nonnegative increasing integers on the source analysis clock (`options.fps`, default composition FPS), before trimming/time remapping. SAM is forward propagation from the earliest prompt; frames before it are empty. Corrections reset the conditioning memory. Reset/re-prompt across cuts or when identity is lost; appearance tracking cannot guarantee re-identification after occlusion.
+Coordinates are normalized to the original source image, before layer placement. Labels are foreground `1`, background `0`, and box corners `2`/`3`. Prompt frames are nonnegative increasing integers on the source analysis clock (`options.fps`, default composition FPS), before trimming/time remapping. SAM propagates forward from the earliest prompt; frames before it are empty. Corrections reset the conditioning memory. Reset/re-prompt across cuts or when identity is lost; appearance tracking cannot guarantee re-identification after occlusion. `sam2` and `sam3` are mutually exclusive on a single clip.
SAM masks do not merge other detected people. Default smoothing is zero and a missing object stays empty. Pose is disabled unless explicitly requested; SAM itself provides no skeleton. `tracked.person` continues to expose silhouette sampling and optional matched pose; `.place(...)` maps those into a layer's layout. The `tracked.signals` bundle uses source geometry on the composition clock; object and mask signals refer to track ID `1`. Full masks are reconstructed lazily from compressed history, avoiding an uncompressed mask per frame.
-Frame requests analyze intervening source frames in order, so a seek forward can take time; a backward seek, hold, reverse, and time-remapped playback reuse source-frame results. When analysis FPS is lower than playback FPS, intermediate frames reuse the preceding analysis mask. Source masks are persisted separately from low-resolution silhouette/pose data; cache identity includes all prompts and the model revision. `pnpm --filter @framefields/vision test:sam2` runs a real-model test on native Dawn (requires GPU access; downloads are cached). A matte/mask/crop layer of the same clip reuses the selected masks through the existing clip-matte link. Use the default matte settings and matching source aspect ratio for that link; crop after source-space matting. An explicit selection fails clearly instead of falling back to another detected person when the mask cannot align. `tracked.release()` frees SAM sessions, keeps results readable, and replays the source memory if further uncached frames are requested.
+Frame requests analyze intervening source frames in order, so a seek forward can take time; a backward seek, hold, reverse, and time-remapped playback reuse source-frame results. When analysis FPS is lower than playback FPS, intermediate frames reuse the preceding analysis mask. Source masks are persisted separately from low-resolution silhouette/pose data; cache identity includes all prompts and the model revision. `pnpm --filter @framefields/vision-sam test:sam2` and `test:sam3` run real-model tests on native Dawn (requires GPU access; downloads are cached). A matte/mask/crop layer of the same clip reuses the selected masks through the existing clip-matte link. Use the default matte settings and matching source aspect ratio for that link; crop after source-space matting. An explicit selection fails clearly instead of falling back to another detected person when the mask cannot align. `tracked.release()` frees SAM sessions, keeps results readable, and replays the source memory if further uncached frames are requested.
### 1. Subject Sandwich ("Text Behind Subject")
Cuts out the foreground subject from footage and sandwiches typography or graphics directly behind them:
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index a2858452..4baa159a 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -350,31 +350,6 @@ importers:
specifier: ^4.1.11
- examples/25_bakery:
- dependencies:
- '@framefields/compositions':
- specifier: workspace:*
- version: link:../../packages/compositions
- framefields:
- specifier: workspace:*
- version: link:../../packages/framefields
- mediabunny:
- specifier: ^1.61.6
- version: 1.61.6
- devDependencies:
- '@framefields/tsconfig':
- specifier: workspace:*
- version: link:../../packages/tsconfig
- '@types/node':
- specifier: ^24.1.0
- version: 24.10.14
- tsx:
- specifier: ^4.20.3
- version: 4.21.0
- typescript:
- specifier: ^5.8.3
- version: 5.9.3
-
examples/26_mac_keyboard_from_3d_primitives:
dependencies:
framefields:
@@ -3064,6 +3039,9 @@ importers:
'@dimforge/rapier3d-deterministic-compat':
specifier: 0.21.0
version: 0.21.0
+ '@framefields/vision-sam':
+ specifier: workspace:*
+ version: link:../vision-sam
onnxruntime-web:
specifier: ^1.24.0
version: 1.30.0
@@ -3465,6 +3443,46 @@ importers:
specifier: ^4.1.11
+ packages/vision-sam:
+ dependencies:
+ '@framefields/core':
+ specifier: workspace:*
+ version: link:../core
+ '@framefields/vision':
+ specifier: workspace:*
+ version: link:../vision
+ '@huggingface/tokenizers':
+ specifier: ^0.2.0
+ version: 0.2.0
+ onnxruntime-node:
+ specifier: ^1.24.0
+ version: 1.30.0
+ onnxruntime-web:
+ specifier: ^1.24.0
+ version: 1.30.0
+ webgpu:
+ specifier: ^0.4.0
+ version: 0.4.0
+ zod:
+ specifier: ^4.3.6
+ version: 4.3.6
+ devDependencies:
+ '@framefields/tsconfig':
+ specifier: workspace:*
+ version: link:../tsconfig
+ '@types/node':
+ specifier: ^24.1.0
+ version: 24.10.14
+ tsdown:
+ specifier: ^0.19.0
+ typescript:
+ specifier: ^5.8.3
+ version: 5.9.3
+ vitest:
+ specifier: ^4.1.11
+
packages/webgpu-renderers:
dependencies:
'@forgeax/engine-codec':
@@ -4010,6 +4028,9 @@ packages:
peerDependencies:
hono: '>=4.13.7'
+ '@huggingface/[email protected]':
+ resolution: {integrity: sha512-LidMHe1FpcSYH4vcSjXooda34pC0M8m1gmDOv7SQi6Iv+ib1zX7J3sHBCC9/3FPCkn2k1b55FneVqMRxrr99pg==}
+
resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==}
engines: {node: '>=18'}
@@ -7409,6 +7430,8 @@ snapshots:
dependencies:
hono: 4.13.13
+ '@huggingface/[email protected]': {}
+
'@img/[email protected]': {}
@@ -8302,7 +8325,7 @@ snapshots:
obug: 2.2.1
std-env: 4.2.0
tinyrainbow: 3.1.0
'@vitest/[email protected]':
dependencies: