Skip to content

Code-first video & image, rendered natively on WebGPU.

9.8 KB
Raw Blame
import {
	createDefaultSessionProvider,
	imageToTensor,
	type SessionProvider,
	type VisionImageInput,
	VisionModelStore,
	type VisionModelStoreOptions,
	type VisionSession,
	type VisionTensor,
	type VisionTensorInput,
	type VisionTensorOutputs,
} from "@framefields/vision";
import { z } from "zod";
import {
	ensureSamModelsRegistered,
	SAM2_EXPORTS,
	type Sam2Graph,
	type Sam2Model,
} from "../models/registry.js";
import { Sam2MemoryBank, sam2Tokens } from "./memory.js";
import {
	type Sam2FrameResult,
	type Sam2Point,
	validateSam2Prompts,
} from "./types.js";

const constantsSchema = z.object({
	image_size: z.number().int().positive().multipleOf(16),
	memory_frames: z.literal(7),
	image_mean: z.tuple([z.number(), z.number(), z.number()]),
	image_std: z.tuple([
		z.number().positive(),
		z.number().positive(),
		z.number().positive(),
	]),
	memory_temporal_positional_encoding: z
		.array(z.array(z.number()).length(64))
		.length(7),
});
type Constants = z.infer<typeof constantsSchema>;
type Graph = Sam2Graph;
const GRAPHS: readonly Graph[] = [
	"vision_encoder",
	"mask_decoder",
	"memory_encoder",
	"memory_attention",
	"pointer_tpos",
];
/** The pinned export revision of `model`: part of ClipVision's persistent cache identity. */
export function sam2Revision(model: Sam2Model = "tiny"): string {
	return `sam2.1-${model}-${SAM2_EXPORTS[model].revision}`;
}

export interface Sam2TrackerOptions extends VisionModelStoreOptions {
	readonly provider?: SessionProvider;
	readonly store?: VisionModelStore;
	/** Source sequence length; used to normalize object-pointer time positions for short videos. */
	readonly frameCount?: number;
	/** Checkpoint: `"tiny"` (default, 512², fast) or `"small"` (1024², finer detail). */
	readonly model?: Sam2Model;
}

function floats(outputs: VisionTensorOutputs, name: string): Float32Array {
	const tensor = outputs[name];
	if (!tensor || !(tensor.data instanceof Float32Array))
		throw new Error(`Missing SAM 2 float output: ${name}`);
	return tensor.data;
}
function input(name: string, tensor: VisionTensor): VisionTensorInput {
	if (tensor.data instanceof BigInt64Array)
		throw new Error(`Unsupported SAM 2 input type: ${name}`);
	return { name, data: tensor.data, dims: tensor.dims };
}
function tensor(
	name: string,
	data: Float32Array | Int32Array,
	dims: readonly number[],
): VisionTensorInput {
	return { name, data, dims };
}

/** Complete pinned SAM 2.1 video graph pipeline (Tiny or Small), one explicitly selected object. */
export class Sam2Tracker {
	private readonly sessions = new Map<Graph, VisionSession>();
	private readonly bank: Sam2MemoryBank;
	/** Model input side; image features are `size / 16` square, one memory token per cell. */
	private readonly size: number;
	private readonly tokens: number;
	private lastFrame = -1;
	private released = false;
	private constructor(
		private readonly constants: Constants,
		private readonly frameCount = 16,
	) {
		this.bank = new Sam2MemoryBank(
			constants.memory_temporal_positional_encoding,
		);
		this.size = constants.image_size;
		this.tokens = (this.size / 16) ** 2;
	}

	static async create(options: Sam2TrackerOptions = {}): Promise<Sam2Tracker> {
		ensureSamModelsRegistered();
		// SAM 2.1's weights are Apache-2.0: no SAM License acceptance.
		const store = options.store ?? new VisionModelStore(options);
		const model = options.model ?? "tiny";
		if (!(model in SAM2_EXPORTS))
			throw new Error(
				`Unknown SAM 2.1 model "${model}"; use "tiny" or "small".`,
			);
		const bytes = await store.ensure(`sam2.1-${model}-constants`);
		const constants = constantsSchema.parse(
			JSON.parse(new TextDecoder().decode(bytes)),
		);
		if (
			options.frameCount !== undefined &&
			(!Number.isSafeInteger(options.frameCount) || options.frameCount < 1)
		)
			throw new Error("SAM 2 frameCount must be a positive integer.");
		const tracker = new Sam2Tracker(constants, options.frameCount);
		const provider = options.provider ?? createDefaultSessionProvider();
		try {
			for (const graph of GRAPHS) {
				const session = await provider.createSession(
					await store.ensure(`sam2.1-${model}-${graph}`),
				);
				tracker.sessions.set(graph, session);
			}
			return tracker;
		} catch (error) {
			tracker.release();
			throw error;
		}
	}

	private run(
		graph: Graph,
		inputs: readonly VisionTensorInput[],
	): Promise<VisionTensorOutputs> {
		const session = this.sessions.get(graph);
		if (!session) throw new Error("SAM 2 tracker has been released.");
		if (session.runMany) return session.runMany(inputs);
		if (inputs.length === 1 && inputs[0]) return session.run(inputs[0]);
		throw new Error("SAM 2 requires a session provider with runMany support.");
	}

	async process(
		image: VisionImageInput,
		frame: number,
		points?: readonly Sam2Point[],
	): Promise<Sam2FrameResult> {
		if (this.released) throw new Error("SAM 2 tracker has been released.");
		if (
			!Number.isSafeInteger(frame) ||
			frame < 0 ||
			(this.lastFrame >= 0 && frame !== this.lastFrame + 1)
		)
			throw new Error("SAM 2 propagation requires consecutive source frames.");
		if (points) validateSam2Prompts([{ frame, points }]);
		if (this.lastFrame < 0 && !points)
			throw new Error("SAM 2 needs a prompt on its first frame.");
		const { tensor: pixels } = imageToTensor(image, {
			width: this.size,
			height: this.size,
			fit: "stretch",
			channels: "rgb",
			padValue: 0,
			mean: this.constants.image_mean.map((v) => v * 255) as [
				number,
				number,
				number,
			],
			std: this.constants.image_std.map((v) => v * 255) as [
				number,
				number,
				number,
			],
		});
		const vision = await this.run("vision_encoder", [
			tensor("pixel_values", pixels, [1, 3, this.size, this.size]),
		]);
		const conditioned = points
			? vision.feats2_no_mem
			: await this.condition(vision, frame);
		if (!conditioned || !vision.feats0 || !vision.feats1)
			throw new Error("Invalid SAM 2 vision outputs.");
		const prompt = points ?? [{ x: 0, y: 0, label: -1 }];
		const decoded = await this.run("mask_decoder", [
			input("feats0", vision.feats0),
			input("feats1", vision.feats1),
			input("feats2_cond", conditioned),
			tensor(
				"input_points",
				Float32Array.from(
					prompt.flatMap((p) => [p.x * this.size, p.y * this.size]),
				),
				[1, 1, prompt.length, 2],
			),
			tensor("input_labels", Int32Array.from(prompt.map((p) => p.label)), [
				1,
				1,
				prompt.length,
			]),
		]);
		await this.remember(vision, decoded, frame, points !== undefined);
		this.lastFrame = frame;
		return decodeSam2Mask(
			floats(decoded, "high_res_mask"),
			image.width,
			image.height,
			floats(decoded, "object_score_logits")[0] ?? -Infinity,
			floats(decoded, "iou")[0] ?? 0,
		);
	}

	private async condition(
		vision: VisionTensorOutputs,
		frame: number,
	): Promise<VisionTensor> {
		const bank = this.bank.assemble(frame, this.frameCount);
		const pointer = await this.run("pointer_tpos", [
			tensor("normalized_diffs", bank.distances, [16]),
		]);
		const positions = floats(pointer, "pointer_pos");
		for (let i = 0; i < 16; i++)
			for (let token = 0; token < 4; token++)
				bank.positions.set(
					positions.subarray(i * 64, (i + 1) * 64),
					7 * this.tokens * 64 + (i * 4 + token) * 64,
				);
		const memoryLength = 7 * this.tokens + 16 * 4;
		const result = await this.run("memory_attention", [
			tensor("current_vision_features", sam2Tokens(floats(vision, "feats2")), [
				this.tokens,
				1,
				256,
			]),
			tensor(
				"current_vision_position_embeddings",
				sam2Tokens(floats(vision, "vision_pos_embed")),
				[this.tokens, 1, 256],
			),
			tensor("memory", bank.memory, [memoryLength, 1, 64]),
			tensor("memory_pos", bank.positions, [memoryLength, 1, 64]),
		]);
		const conditioned = result.conditioned_feats;
		if (!conditioned) throw new Error("Missing SAM 2 conditioned features.");
		return conditioned;
	}

	private async remember(
		vision: VisionTensorOutputs,
		decoded: VisionTensorOutputs,
		frame: number,
		prompted: boolean,
	): Promise<void> {
		if (!vision.feats2 || !decoded.high_res_mask)
			throw new Error("Invalid SAM 2 memory inputs.");
		const memory = await this.run("memory_encoder", [
			input("feats2", vision.feats2),
			input("high_res_mask", decoded.high_res_mask),
			tensor(
				"object_score_logits",
				floats(decoded, "object_score_logits"),
				[1, 1],
			),
			tensor("binarize", new Float32Array([prompted ? 1 : 0]), []),
		]);
		this.bank.remember(
			{
				frame,
				tokens: floats(memory, "memory_tokens").slice(),
				positions: floats(memory, "memory_pos").slice(),
				pointer: floats(decoded, "object_pointer").slice(),
			},
			prompted,
		);
	}

	release(): void {
		for (const session of this.sessions.values()) session.release();
		this.sessions.clear();
		this.released = true;
	}
}

/** Bilinear resize of square logits before thresholding, matching SAM's source-image mask contract. */
export function decodeSam2Mask(
	logits: Float32Array,
	width: number,
	height: number,
	score: number,
	iou: number,
): Sam2FrameResult {
	const side = Math.round(Math.sqrt(logits.length));
	if (side * side !== logits.length || side % 16 !== 0)
		throw new Error("Invalid SAM 2 mask shape.");
	const last = side - 1;
	const mask = new Uint8Array(width * height);
	if (score > 0) {
		for (let y = 0; y < height; y++) {
			const sy = Math.max(0, Math.min(last, ((y + 0.5) * side) / height - 0.5));
			const y0 = Math.floor(sy),
				y1 = Math.min(last, y0 + 1),
				fy = sy - y0;
			for (let x = 0; x < width; x++) {
				const sx = Math.max(
					0,
					Math.min(last, ((x + 0.5) * side) / width - 0.5),
				);
				const x0 = Math.floor(sx),
					x1 = Math.min(last, x0 + 1),
					fx = sx - x0;
				const a =
					(logits[y0 * side + x0] ?? 0) * (1 - fx) +
					(logits[y0 * side + x1] ?? 0) * fx;
				const b =
					(logits[y1 * side + x0] ?? 0) * (1 - fx) +
					(logits[y1 * side + x1] ?? 0) * fx;
				mask[y * width + x] = a * (1 - fy) + b * fy > 0 ? 255 : 0;
			}
		}
	}
	return { mask, width, height, score, iou };
}