Code-first video & image, rendered natively on WebGPU.
import {
createDefaultSessionProvider,
type SessionProvider,
sessionCompute,
type VisionImageInput,
VisionModelStore,
type VisionModelStoreOptions,
type VisionSession,
type VisionTensorInput,
type VisionTensorOutputs,
} from "@framefields/vision";
import { assertSamLicenseAccepted } from "../license.js";
import {
ensureSamModelsRegistered,
SAM3_ENCODER,
type Sam3Precision,
} from "../models/registry.js";
import { sam2Tokens } from "../sam2/memory.js";
import type { Sam2Point } from "../sam2/types.js";
import { holdSam3Encoder, type Sam3EncoderHandle } from "./encoder.js";
import { sam3DenseDecoder, sam3SupportGraphs } from "./graphs.js";
import { Sam3MemoryBank, sam3Position } from "./memory.js";
import {
coveredCells,
keepTracked,
MASK_LOGIT,
mergeSelections,
uncovered,
} from "./selection.js";
import { Sam3TextSelector, sam3Floats, sam3Run } from "./text-selector.js";
import {
type Sam3FrameResult,
type Sam3ObjectResult,
type Sam3Prompt,
validateSam3Prompts,
} from "./types.js";
/** Every graph and tokenizer revision, and the encoder precision: ClipVision's cache identity. */
export function sam3Revision(precision: Sam3Precision): string {
return `sam3-${precision}-429305c8-video-e942689a-support-68ce2746-text-fe8f8e12-v2`;
}
/** The encoder precision a tracker on `provider` uses: the override, else the backend's best. */
export async function sam3Precision(
provider: SessionProvider = createDefaultSessionProvider(),
override?: Sam3Precision,
): Promise<Sam3Precision> {
return override ?? SAM3_ENCODER[await sessionCompute(provider)];
}
export interface Sam3TrackerOptions extends VisionModelStoreOptions {
readonly provider?: SessionProvider;
readonly store?: VisionModelStore;
readonly frameCount?: number;
readonly textThreshold?: number;
/** Most instances a text prompt selects. Default 1, or every match with `merge`. */
readonly maxObjects?: number;
/** Track every instance a text prompt selects as one object (one mask, one memory). */
readonly merge?: boolean;
/** Re-run the last text prompt every this many frames and add what no object covers (see `Sam3Options`). */
readonly redetect?: number;
/** Image-encoder weights; defaults to the fastest for the provider's backend (`SAM3_ENCODER`). */
readonly precision?: Sam3Precision;
/** Explicitly agree to Meta's SAM License terms to download or run SAM 3's weights. */
readonly acceptLicense?: boolean;
}
type Graph =
| "mask_decoder"
| "dense_decoder"
| "dense_prompt"
| "pointer_position"
| "memory_encoder"
| "memory_attention";
/** How an object is (re)seeded on a prompted frame. */
interface Seed {
readonly points?: readonly Sam2Point[];
readonly dense?: Float32Array;
/**
* A mask seed's own 1008² logits (±10). Like SAM's `_use_mask_as_output`, the seed
* frame outputs and remembers the given mask; the decoder only supplies its pointer.
*/
readonly mask?: Float32Array;
}
/** One object's decoded frame, before it is remembered. */
interface Tracked {
readonly seed?: Seed;
readonly decoded: VisionTensorOutputs;
readonly logits: Float32Array;
readonly score: number;
}
function tensor(
name: string,
data: VisionTensorInput["data"],
dims: readonly number[],
): VisionTensorInput {
return { name, data, dims };
}
/**
* Explicitly selected SAM 3 instances with full neural memory propagation on the provider.
* The image encoder runs once per frame; each object keeps its own memory bank.
*/
export class Sam3Tracker {
private readonly sessions = new Map<Graph, VisionSession>();
private encoder?: Sam3EncoderHandle;
private readonly position = sam3Position();
private readonly banks: Sam3MemoryBank[] = [];
private temporal?: Float32Array;
private noMemory?: Float32Array;
private selector?: Sam3TextSelector;
/** The last text prompt and its frame, which re-detection repeats. */
private text?: { value: string; frame: number };
private lastFrame = -1;
private released = false;
private constructor(
private readonly store: VisionModelStore,
private readonly provider: SessionProvider,
private readonly options: Sam3TrackerOptions,
) {}
static async create(options: Sam3TrackerOptions = {}): Promise<Sam3Tracker> {
assertSamLicenseAccepted("SAM 3", options);
ensureSamModelsRegistered();
if (
options.frameCount !== undefined &&
(!Number.isSafeInteger(options.frameCount) || options.frameCount < 1)
)
throw new Error("SAM 3 frameCount must be a positive integer.");
if (
options.textThreshold !== undefined &&
(!Number.isFinite(options.textThreshold) ||
options.textThreshold <= 0 ||
options.textThreshold >= 1)
)
throw new Error("SAM 3 textThreshold must be between zero and one.");
if (
options.maxObjects !== undefined &&
(!Number.isSafeInteger(options.maxObjects) || options.maxObjects < 1)
)
throw new Error("SAM 3 maxObjects must be a positive integer.");
if (
options.redetect !== undefined &&
(!Number.isSafeInteger(options.redetect) || options.redetect < 0)
)
throw new Error(
"SAM 3 redetect must be a nonnegative integer frame interval.",
);
const store =
options.store ??
new VisionModelStore({
...options,
acceptLicense: options.acceptLicense ?? true,
});
const provider = options.provider ?? createDefaultSessionProvider();
const tracker = new Sam3Tracker(store, provider, options);
try {
tracker.encoder = await holdSam3Encoder(
store,
provider,
await sam3Precision(provider, options.precision),
);
const support = sam3SupportGraphs(
await store.ensure("sam3-support"),
await store.ensure("sam3-support-data"),
);
tracker.temporal = support.temporal;
tracker.noMemory = support.noMemory;
tracker.sessions.set(
"dense_prompt",
await provider.createSession(support.densePrompt),
);
tracker.sessions.set(
"pointer_position",
await provider.createSession(support.pointerPosition),
);
const decoder = await store.ensure("sam3-mask_decoder");
tracker.sessions.set(
"mask_decoder",
await provider.createSession(decoder),
);
tracker.sessions.set(
"dense_decoder",
await provider.createSession(sam3DenseDecoder(decoder)),
);
for (const graph of ["memory_encoder", "memory_attention"] as const)
tracker.sessions.set(
graph,
await provider.createSession(await store.ensure(`sam3-${graph}`)),
);
if (!options.store) store.clearMemoryCache();
return tracker;
} catch (error) {
tracker.release();
throw error;
}
}
private run(
graph: Graph,
inputs: readonly VisionTensorInput[],
): Promise<VisionTensorOutputs> {
const session = this.sessions.get(graph);
if (!session) throw new Error("SAM 3 tracker has been released.");
return sam3Run(session, inputs);
}
async process(
image: VisionImageInput,
frame: number,
prompt?: Omit<Sam3Prompt, "frame">,
): Promise<Sam3FrameResult> {
if (this.released) throw new Error("SAM 3 tracker has been released.");
if (
!Number.isSafeInteger(frame) ||
frame < 0 ||
(this.lastFrame >= 0 && frame !== this.lastFrame + 1)
)
throw new Error("SAM 3 propagation requires consecutive source frames.");
if (prompt) validateSam3Prompts({ prompts: [{ ...prompt, frame }] });
if (this.lastFrame < 0 && !prompt)
throw new Error("SAM 3 needs a prompt on its first frame.");
if (!this.encoder) throw new Error("SAM 3 tracker has been released.");
const { pixels, feats0, feats1, raw } = await this.encoder.encode(image);
const shared = [
tensor("feats0", feats0, [1, 32, 288, 288]),
tensor("feats1", feats1, [1, 64, 144, 144]),
];
const seeds = await this.seeds(pixels, frame, prompt);
const tracked: Tracked[] = [];
for (const [index, bank] of this.banks.entries()) {
const seed = seeds.get(index);
const features = seed
? this.initialFeatures(raw)
: await this.condition(raw, frame, bank);
tracked.push(this.track(seed, await this.decode(shared, features, seed)));
}
if (!prompt) await this.redetect(pixels, frame, tracked, shared, raw);
const objects: Sam3ObjectResult[] = [];
for (const [index, bank] of this.banks.entries()) {
const item = tracked[index];
if (!item) continue;
const { seed, decoded, logits, score } = item;
await this.remember(bank, raw, decoded, frame, seed, logits, score);
objects.push({
id: index + 1,
mask: sam3Mask(logits, image.width, image.height, score),
score,
iou: seed?.mask ? 1 : Math.max(...sam3Floats(decoded, "iou_scores")),
});
}
this.lastFrame = frame;
return { width: image.width, height: image.height, objects };
}
private track(seed: Seed | undefined, decoded: VisionTensorOutputs): Tracked {
return {
seed,
decoded,
logits: seed?.mask ?? sam3Floats(decoded, "high_res_masks"),
score: seed?.mask
? MASK_LOGIT
: (sam3Floats(decoded, "object_score_logits")[0] ?? -Infinity),
};
}
/**
* On a `redetect` frame, runs the last text prompt again and seeds what no tracked object
* covers: merged into the one object (keeping what it tracks), or as new objects.
*/
private async redetect(
pixels: Float32Array,
frame: number,
tracked: Tracked[],
shared: readonly VisionTensorInput[],
raw: Float32Array,
): Promise<void> {
const every = this.options.redetect ?? 0;
if (!this.text || every <= 0 || (frame - this.text.frame) % every !== 0)
return;
const room = this.options.merge
? Infinity
: (this.options.maxObjects ?? 1) - this.banks.length;
if (room <= 0) return;
const selector = await this.textSelector();
const found = await selector.select(
pixels,
this.text.value,
this.options.textThreshold ?? 0.5,
Infinity,
false,
);
const covered = coveredCells(tracked);
const fresh = uncovered(found, covered).slice(0, room);
if (!fresh.length) return;
const seeded = async (seed: Seed): Promise<Tracked> =>
this.track(
seed,
await this.decode(shared, this.initialFeatures(raw), seed),
);
if (this.options.merge) {
// One object: what it tracks (at full resolution) plus every new instance.
const added = mergeSelections(fresh);
const kept = tracked[0];
if (!kept) this.banks.push(this.newBank());
tracked[0] = await seeded({
dense: await this.densePrompt(mergeSelections([added, covered])),
mask: keepTracked(kept, seedLogits(added)),
});
return;
}
for (const logits of fresh) {
this.banks.push(this.newBank());
tracked.push(await seeded(await this.textSeed(logits)));
}
}
private async textSelector(): Promise<Sam3TextSelector> {
this.selector ??= await Sam3TextSelector.create(this.store, this.provider);
if (!this.options.store) this.store.clearMemoryCache();
return this.selector;
}
/** A 288² concept mask as a seed that outputs (and remembers) that mask. */
private async textSeed(logits: Float32Array): Promise<Seed> {
return { dense: await this.densePrompt(logits), mask: seedLogits(logits) };
}
private async densePrompt(logits: Float32Array): Promise<Float32Array> {
const binary = Float32Array.from(logits, (v) => (v > 0 ? 1 : 0));
const dense = await this.run("dense_prompt", [
tensor("mask", binary, [1, 1, 288, 288]),
]);
return sam3Floats(dense, "dense_prompt");
}
/** Objects a prompt (re)seeds, by index. Text re-selects every object; points address one. */
private async seeds(
pixels: Float32Array,
frame: number,
prompt?: Omit<Sam3Prompt, "frame">,
): Promise<Map<number, Seed>> {
const seeds = new Map<number, Seed>();
if (prompt?.points) {
const index = (prompt.object ?? 1) - 1;
if (index > this.banks.length)
throw new Error(
`SAM 3 prompt object ${index + 1} skips track IDs; the next new ID is ${this.banks.length + 1}.`,
);
if (index === this.banks.length) this.banks.push(this.newBank());
seeds.set(index, { points: prompt.points });
}
if (prompt?.text !== undefined) {
const selector = await this.textSelector();
const selected = await selector.select(
pixels,
prompt.text,
this.options.textThreshold ?? 0.5,
this.options.maxObjects ?? (this.options.merge ? Infinity : 1),
);
const masks = this.options.merge ? [mergeSelections(selected)] : selected;
this.text = { value: prompt.text, frame };
this.banks.length = 0;
for (const [index, logits] of masks.entries()) {
this.banks.push(this.newBank());
seeds.set(index, await this.textSeed(logits));
}
}
return seeds;
}
private newBank(): Sam3MemoryBank {
if (!this.temporal) throw new Error("SAM 3 tracker has been released.");
return new Sam3MemoryBank(this.temporal);
}
private decode(
shared: readonly VisionTensorInput[],
features: Float32Array,
seed?: Seed,
): Promise<VisionTensorOutputs> {
const points = seed?.points ?? [{ x: 0, y: 0, label: -1 }];
const inputs = [
...shared,
tensor("feats2", features, [1, 256, 72, 72]),
tensor(
"input_points",
Float32Array.from(points.flatMap((p) => [p.x * 1008, p.y * 1008])),
[1, 1, points.length, 2],
),
tensor("input_labels", Int32Array.from(points.map((p) => p.label)), [
1,
1,
points.length,
]),
];
if (!seed?.dense) return this.run("mask_decoder", inputs);
return this.run("dense_decoder", [
...inputs,
tensor("dense_prompt", seed.dense, [1, 256, 72, 72]),
]);
}
private initialFeatures(raw: Float32Array): Float32Array {
if (!this.noMemory) throw new Error("Missing SAM 3 no-memory embedding.");
return Float32Array.from(
raw,
(value, i) => value + (this.noMemory?.[Math.floor(i / 5184)] ?? 0),
);
}
private async condition(
raw: Float32Array,
frame: number,
memory: Sam3MemoryBank,
): Promise<Float32Array> {
const bank = memory.assemble(frame, this.options.frameCount ?? 16);
const positions = await this.run("pointer_position", [
tensor("encoding", bank.encoding, [16, 256]),
]);
const pointerPosition = new Float32Array(bank.pointerCount * 256),
projected = sam3Floats(positions, "position");
for (let i = 0; i < bank.pointerCount; i++)
for (let token = 0; token < 4; token++)
pointerPosition.set(
projected.subarray(i * 64, (i + 1) * 64),
(i * 4 + token) * 64,
);
const result = await this.run("memory_attention", [
tensor("current_vision_features", sam2Tokens(raw), [5184, 1, 256]),
tensor(
"current_vision_position_embeddings",
this.position,
[5184, 1, 256],
),
tensor("spatial_memory", bank.spatial, [bank.spatial.length / 64, 1, 64]),
tensor("spatial_memory_position_embeddings", bank.positions, [
bank.positions.length / 64,
1,
64,
]),
tensor("pointer_memory", bank.pointers, [bank.pointerCount * 4, 1, 64]),
tensor("pointer_memory_position_embeddings", pointerPosition, [
bank.pointerCount * 4,
1,
64,
]),
]);
const tokens = sam3Floats(result, "conditioned_features"),
nchw = new Float32Array(tokens.length);
for (let i = 0; i < 5184; i++)
for (let channel = 0; channel < 256; channel++)
nchw[channel * 5184 + i] = tokens[i * 256 + channel] ?? 0;
return nchw;
}
private async remember(
bank: Sam3MemoryBank,
raw: Float32Array,
decoded: VisionTensorOutputs,
frame: number,
seed: Seed | undefined,
logits: Float32Array,
score: number,
): Promise<void> {
const memory = await this.run("memory_encoder", [
tensor("vision_features", raw, [1, 256, 72, 72]),
tensor("pred_masks_high_res", logits, [1, 1, 1008, 1008]),
tensor("object_score_logits", Float32Array.of(score), [1, 1]),
tensor("is_mask_from_pts", new Uint8Array([seed ? 1 : 0]), [1]),
]);
bank.remember(
{
frame,
tokens: sam3Floats(memory, "memory_tokens").slice(),
positions: sam3Floats(memory, "memory_pos_enc").slice(),
pointer: sam3Floats(decoded, "object_pointer").slice(),
},
seed !== undefined,
);
}
release(): void {
for (const session of this.sessions.values()) session.release();
this.sessions.clear();
this.encoder?.release();
this.encoder = undefined;
this.selector?.release();
this.banks.length = 0;
this.temporal = undefined;
this.noMemory = undefined;
this.released = true;
}
}
/** Resize logits before thresholding; source pixels are the public mask coordinate system. */
export function sam3Mask(
logits: Float32Array,
width: number,
height: number,
score: number,
): Uint8Array {
if (logits.length !== 1008 * 1008)
throw new Error("Invalid SAM 3 high-resolution mask shape.");
const mask = new Uint8Array(width * height);
if (score <= 0) return mask;
for (let y = 0; y < height; y++)
for (let x = 0; x < width; x++) {
const sy = Math.max(0, Math.min(1007, ((y + 0.5) * 1008) / height - 0.5)),
sx = Math.max(0, Math.min(1007, ((x + 0.5) * 1008) / width - 0.5));
const y0 = Math.floor(sy),
x0 = Math.floor(sx),
fy = sy - y0,
fx = sx - x0,
y1 = Math.min(1007, y0 + 1),
x1 = Math.min(1007, x0 + 1);
const top =
(logits[y0 * 1008 + x0] ?? 0) * (1 - fx) +
(logits[y0 * 1008 + x1] ?? 0) * fx,
bottom =
(logits[y1 * 1008 + x0] ?? 0) * (1 - fx) +
(logits[y1 * 1008 + x1] ?? 0) * fx;
mask[y * width + x] = top * (1 - fy) + bottom * fy > 0 ? 255 : 0;
}
return mask;
}
/** A 288² concept mask's logits as SAM's 1008² seed logits: bilinear, then ±10. */
function seedLogits(logits: Float32Array): Float32Array {
const out = new Float32Array(1008 * 1008);
for (let y = 0; y < 1008; y++) {
const sy = Math.max(0, Math.min(287, ((y + 0.5) * 288) / 1008 - 0.5)),
y0 = Math.floor(sy),
y1 = Math.min(287, y0 + 1),
fy = sy - y0;
for (let x = 0; x < 1008; x++) {
const sx = Math.max(0, Math.min(287, ((x + 0.5) * 288) / 1008 - 0.5)),
x0 = Math.floor(sx),
x1 = Math.min(287, x0 + 1),
fx = sx - x0;
const top =
(logits[y0 * 288 + x0] ?? 0) * (1 - fx) +
(logits[y0 * 288 + x1] ?? 0) * fx,
bottom =
(logits[y1 * 288 + x0] ?? 0) * (1 - fx) +
(logits[y1 * 288 + x1] ?? 0) * fx;
out[y * 1008 + x] =
top * (1 - fy) + bottom * fy > 0 ? MASK_LOGIT : -MASK_LOGIT;
}
}
return out;
}