Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add SAM 2.1 video object segmentation and tracking package

konstantin-paulus6c3901e5430c
diff --git a/packages/sam2/package.json b/packages/sam2/package.json
new file mode 100644
index 00000000..09e8f9e8
--- /dev/null
+++ b/packages/sam2/package.json
@@ -0,0 +1,30 @@
+{
+ "name": "@diffusionstudio/sam2",
+ "version": "0.1.0",
+ "description": "SAM 2.1 video object segmentation and tracking in the browser: ONNX Runtime on WebGPU, frames decoded with mediabunny. Weights are fetched from Hugging Face on first use and cached locally.",
+ "license": "MPL-2.0",
+ "type": "module",
+ "sideEffects": false,
+ "exports": {
+ ".": "./src/index.ts",
+ "./mask": "./src/mask.ts"
+ },
+ "scripts": {
+ "check": "tsc --noEmit"
+ },
+ "keywords": [
+ "sam2",
+ "segmentation",
+ "tracking",
+ "webgpu",
+ "onnx"
+ ],
+ "dependencies": {
+ "mediabunny": "^1.32.2",
+ "onnxruntime-web": "^1.30.0"
+ },
+ "devDependencies": {
+ "@webgpu/types": "^0.1.71",
+ "typescript": "~5.9.3"
+ }
+}
diff --git a/packages/sam2/src/constants.ts b/packages/sam2/src/constants.ts
new file mode 100644
index 00000000..36550a45
--- /dev/null
+++ b/packages/sam2/src/constants.ts
@@ -0,0 +1,37 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export const MODEL_REPO = 'https://huggingface.co/square-zero-labs/sam2.1-tiny-video-onnx/resolve/main';
+
+export const MODEL_FILES = {
+ constants: { path: 'constants.json', size: 9_922 },
+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 134_335_567 },
+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 17_794_355 },
+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 5_616_496 },
+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 32_259_165 },
+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 67_289 },
+} as const;
+
+export const IMAGE_SIZE = 1024;
+export const MASK_SIZE = 256;
+export const FEAT_SIZE = 64;
+export const FEAT_TOKENS = FEAT_SIZE * FEAT_SIZE;
+export const HIDDEN_DIM = 256;
+export const MEM_DIM = 64;
+export const NUM_MASKMEM = 7;
+export const MAX_POINTERS = 16;
+export const POINTER_TOKENS = HIDDEN_DIM / MEM_DIM;
+export const MEMORY_ROWS = NUM_MASKMEM * FEAT_TOKENS + MAX_POINTERS * POINTER_TOKENS;
+
+export const BYTES_PER_FLOAT = 4;
+export const MEMORY_ROW_BYTES = MEM_DIM * BYTES_PER_FLOAT;
+export const MEMORY_BLOCK_BYTES = FEAT_TOKENS * MEMORY_ROW_BYTES;
+export const POINTER_BYTES = HIDDEN_DIM * BYTES_PER_FLOAT;
+
+/** The values `constants.json` in the model repo carries. */
+export type ModelConstants = {
+ image_mean: [number, number, number];
+ image_std: [number, number, number];
+ memory_temporal_positional_encoding: number[][];
+};
diff --git a/packages/sam2/src/env.d.ts b/packages/sam2/src/env.d.ts
new file mode 100644
index 00000000..f72e628d
--- /dev/null
+++ b/packages/sam2/src/env.d.ts
@@ -0,0 +1,4 @@
+declare module '*.wasm?url' {
+ const url: string;
+ export default url;
+}
diff --git a/packages/sam2/src/gpu.ts b/packages/sam2/src/gpu.ts
new file mode 100644
index 00000000..3785120a
--- /dev/null
+++ b/packages/sam2/src/gpu.ts
@@ -0,0 +1,131 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { BYTES_PER_FLOAT, FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE } from './constants';
+import { PREPROCESS_SHADER, TRANSPOSE_SHADER, WORKGROUP_SIZE } from './shaders';
+
+/** Degrees clockwise a decoded frame is turned for display. */
+export type Rotation = 0 | 90 | 180 | 270;
+
+export function createStorageBuffer(device: GPUDevice, size: number, label: string): GPUBuffer {
+ return device.createBuffer({
+ label,
+ size,
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST,
+ });
+}
+
+function createPipeline(device: GPUDevice, code: string, label: string): GPUComputePipeline {
+ return device.createComputePipeline({
+ label,
+ layout: 'auto',
+ compute: { module: device.createShaderModule({ label, code }), entryPoint: 'main' },
+ });
+}
+
+function dispatch(device: GPUDevice, pipeline: GPUComputePipeline, bindGroup: GPUBindGroup, x: number, y: number): void {
+ const encoder = device.createCommandEncoder();
+ const pass = encoder.beginComputePass();
+ pass.setPipeline(pipeline);
+ pass.setBindGroup(0, bindGroup);
+ pass.dispatchWorkgroups(x, y);
+ pass.end();
+ device.queue.submit([encoder.finish()]);
+}
+
+const PARAMS_BYTES = 48;
+const MAX_TAPS = 4;
+
+/** Uploads a frame and turns it into the vision encoder's `pixel_values`. */
+export class FramePreprocessor {
+ readonly pixels: GPUBuffer;
+ private readonly pipeline: GPUComputePipeline;
+ private readonly sampler: GPUSampler;
+ private readonly params: GPUBuffer;
+ private texture: GPUTexture | null = null;
+
+ constructor(
+ private readonly device: GPUDevice,
+ private readonly mean: readonly number[],
+ private readonly std: readonly number[],
+ ) {
+ this.pixels = createStorageBuffer(device, 3 * IMAGE_SIZE * IMAGE_SIZE * BYTES_PER_FLOAT, 'sam2 pixel_values');
+ this.pipeline = createPipeline(device, PREPROCESS_SHADER, 'sam2 preprocess');
+ this.sampler = device.createSampler({ magFilter: 'linear', minFilter: 'linear' });
+ this.params = device.createBuffer({
+ label: 'sam2 preprocess params',
+ size: PARAMS_BYTES,
+ usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+ });
+ }
+
+ run(frame: VideoFrame, rotation: Rotation): void {
+ const width = frame.displayWidth;
+ const height = frame.displayHeight;
+ const texture = this.textureFor(width, height);
+ this.device.queue.copyExternalImageToTexture({ source: frame }, { texture }, [width, height]);
+
+ const params = new ArrayBuffer(PARAMS_BYTES);
+ const floats = new Float32Array(params);
+ const uints = new Uint32Array(params);
+ floats[0] = width;
+ floats[1] = height;
+ uints[2] = rotation / 90;
+ uints[3] = Math.min(MAX_TAPS, Math.max(1, Math.ceil(Math.max(width, height) / IMAGE_SIZE)));
+ floats.set(this.mean, 4);
+ floats.set(this.std, 8);
+ this.device.queue.writeBuffer(this.params, 0, params);
+
+ const bindGroup = this.device.createBindGroup({
+ layout: this.pipeline.getBindGroupLayout(0),
+ entries: [
+ { binding: 0, resource: { buffer: this.params } },
+ { binding: 1, resource: texture.createView() },
+ { binding: 2, resource: this.sampler },
+ { binding: 3, resource: { buffer: this.pixels } },
+ ],
+ });
+ dispatch(this.device, this.pipeline, bindGroup, IMAGE_SIZE / WORKGROUP_SIZE, IMAGE_SIZE / WORKGROUP_SIZE);
+ }
+
+ private textureFor(width: number, height: number): GPUTexture {
+ if (this.texture?.width === width && this.texture.height === height) return this.texture;
+
+ this.texture?.destroy();
+ this.texture = this.device.createTexture({
+ label: 'sam2 frame',
+ size: [width, height],
+ format: 'rgba8unorm',
+ usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.RENDER_ATTACHMENT,
+ });
+ return this.texture;
+ }
+
+ dispose(): void {
+ this.texture?.destroy();
+ this.pixels.destroy();
+ this.params.destroy();
+ }
+}
+
+/** Rewrites a [1, C, H, W] feature map as the [H*W, 1, C] tokens memory attention reads. */
+export class TokenTransposer {
+ private readonly pipeline: GPUComputePipeline;
+
+ constructor(private readonly device: GPUDevice) {
+ this.pipeline = createPipeline(device, TRANSPOSE_SHADER, 'sam2 transpose');
+ }
+
+ run(channels: GPUBuffer, tokens: GPUBuffer): void {
+ const size = FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT;
+ const bindGroup = this.device.createBindGroup({
+ layout: this.pipeline.getBindGroupLayout(0),
+ entries: [
+ { binding: 0, resource: { buffer: channels, size } },
+ { binding: 1, resource: { buffer: tokens, size } },
+ ],
+ });
+ dispatch(this.device, this.pipeline, bindGroup, FEAT_TOKENS / WORKGROUP_SIZE, HIDDEN_DIM / WORKGROUP_SIZE);
+ }
+}
diff --git a/packages/sam2/src/index.ts b/packages/sam2/src/index.ts
new file mode 100644
index 00000000..88aff151
--- /dev/null
+++ b/packages/sam2/src/index.ts
@@ -0,0 +1,13 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export { loadSam2 } from './load';
+export { Sam2Video } from './tracker';
+export { trackObject } from './video';
+export { MASK_SIZE, maskHas, maskIsEmpty, renderMask } from './mask';
+
+export type { Sam2LoadOptions, Sam2Progress } from './load';
+export type { Sam2Point } from './tracker';
+export type { TrackRequest } from './video';
+export type { Sam2Mask } from './mask';
diff --git a/packages/sam2/src/load.ts b/packages/sam2/src/load.ts
new file mode 100644
index 00000000..e8a9e09d
--- /dev/null
+++ b/packages/sam2/src/load.ts
@@ -0,0 +1,48 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { configureRuntime, createSession, runtimeDevice } from './sessions';
+import { Sam2Video } from './tracker';
+import { fetchModelFiles } from './weights';
+
+import type { ModelConstants } from './constants';
+
+export type Sam2Progress =
+ | { phase: 'download'; loaded: number; total: number }
+ | { phase: 'compile' };
+
+export type Sam2LoadOptions = {
+ onProgress?: (progress: Sam2Progress) => void;
+ signal?: AbortSignal;
+};
+
+let loading: Promise<Sam2Video> | null = null;
+
+/** The model, loaded once per page: the first call fetches and compiles it, later calls share the result. */
+export function loadSam2(options: Sam2LoadOptions = {}): Promise<Sam2Video> {
+ loading ??= load(options).catch((error: unknown) => {
+ loading = null;
+ throw error;
+ });
+ return loading;
+}
+
+async function load({ onProgress, signal }: Sam2LoadOptions): Promise<Sam2Video> {
+ if (!navigator.gpu) throw new Error('WebGPU is not available in this browser');
+ configureRuntime();
+
+ const files = await fetchModelFiles((progress) => onProgress?.({ phase: 'download', ...progress }), signal);
+ const constants = JSON.parse(new TextDecoder().decode(files.constants)) as ModelConstants;
+
+ onProgress?.({ phase: 'compile' });
+ // The runtime builds one WebGPU session at a time.
+ const visionEncoder = await createSession(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed']);
+ const maskDecoder = await createSession(files.maskDecoder, ['high_res_mask', 'object_pointer']);
+ const memoryEncoder = await createSession(files.memoryEncoder, ['memory_tokens', 'memory_pos']);
+ const memoryAttention = await createSession(files.memoryAttention, ['conditioned_feats']);
+ const pointerTpos = await createSession(files.pointerTpos, ['pointer_pos']);
+ const device = await runtimeDevice();
+
+ return new Sam2Video(device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants);
+}
diff --git a/packages/sam2/src/mask.ts b/packages/sam2/src/mask.ts
new file mode 100644
index 00000000..b5083a24
--- /dev/null
+++ b/packages/sam2/src/mask.ts
@@ -0,0 +1,55 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { MASK_SIZE } from './constants';
+
+export { MASK_SIZE };
+
+/** One frame's segmentation, at the model's mask resolution. */
+export type Sam2Mask = {
+ /** Row-major, one bit per pixel, most significant bit first. */
+ bits: Uint8Array;
+ /** Whether the object is in frame: at or below zero the model saw it occluded. */
+ score: number;
+ /** The model's own estimate of the mask's quality, 0 to 1. */
+ iou: number;
+};
+
+const MASK_BYTES = (MASK_SIZE * MASK_SIZE) / 8;
+
+/** Packs mask logits into bits; positive logits are foreground. */
+export function packMask(logits: Float32Array, score: number, iou: number): Sam2Mask {
+ const bits = new Uint8Array(MASK_BYTES);
+ for (let i = 0; i < logits.length; i++) {
+ if (logits[i]! > 0) bits[i >> 3]! |= 0x80 >> (i & 7);
+ }
+ return { bits, score, iou };
+}
+
+export function maskHas(mask: Sam2Mask, x: number, y: number): boolean {
+ const i = y * MASK_SIZE + x;
+ return (mask.bits[i >> 3]! & (0x80 >> (i & 7))) !== 0;
+}
+
+/** Whether any pixel is foreground. */
+export function maskIsEmpty(mask: Sam2Mask): boolean {
+ return mask.bits.every((byte) => byte === 0);
+}
+
+/** The mask as RGBA pixels, `color` where it is foreground and transparent elsewhere. */
+export function renderMask(mask: Sam2Mask, color: [r: number, g: number, b: number, a: number]): ImageData {
+ const image = new ImageData(MASK_SIZE, MASK_SIZE);
+ const pixels = new Uint32Array(image.data.buffer);
+ const packed = (color[3] << 24 | color[2] << 16 | color[1] << 8 | color[0]) >>> 0;
+
+ for (let byte = 0; byte < mask.bits.length; byte++) {
+ const value = mask.bits[byte]!;
+ if (value === 0) continue;
+ for (let bit = 0; bit < 8; bit++) {
+ if (value & (0x80 >> bit)) pixels[byte * 8 + bit] = packed;
+ }
+ }
+
+ return image;
+}
diff --git a/packages/sam2/src/memory.ts b/packages/sam2/src/memory.ts
new file mode 100644
index 00000000..0dc89a29
--- /dev/null
+++ b/packages/sam2/src/memory.ts
@@ -0,0 +1,156 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import {
+ MAX_POINTERS, MEM_DIM, MEMORY_BLOCK_BYTES, MEMORY_ROW_BYTES, MEMORY_ROWS,
+ NUM_MASKMEM, POINTER_BYTES, POINTER_TOKENS,
+} from './constants';
+import { createStorageBuffer } from './gpu';
+
+type FrameMemory = {
+ index: number;
+ tokens: GPUBuffer;
+ pointer: GPUBuffer;
+};
+
+/**
+ * What memory attention sees of the frames already tracked: the memory of the
+ * prompted frame, the most recent frames' memories, and the object pointers of
+ * the last sixteen. Assembled on the GPU from copies of the encoder's outputs,
+ * so nothing crosses to the CPU between frames.
+ */
+export class MemoryBank {
+ readonly memory: GPUBuffer;
+ readonly memoryPos: GPUBuffer;
+ private positions: GPUBuffer[] | null = null;
+ private conditioning: FrameMemory | null = null;
+ private recent: FrameMemory[] = [];
+ private readonly pool: FrameMemory[] = [];
+
+ constructor(
+ private readonly device: GPUDevice,
+ private readonly temporalEncoding: readonly (readonly number[])[],
+ ) {
+ this.memory = createStorageBuffer(device, MEMORY_ROWS * MEMORY_ROW_BYTES, 'sam2 memory');
+ this.memoryPos = createStorageBuffer(device, MEMORY_ROWS * MEMORY_ROW_BYTES, 'sam2 memory_pos');
+ }
+
+ get hasPositions(): boolean {
+ return this.positions !== null;
+ }
+
+ /**
+ * The memory encoder's `memory_pos` is the same for every frame, so each
+ * temporal slot's positions are laid down once: the spatial encoding plus
+ * the slot's row of the temporal table.
+ */
+ setPositions(spatial: Float32Array): void {
+ this.positions = this.temporalEncoding.map((row, slot) => {
+ const data = new Float32Array(spatial.length);
+ for (let i = 0; i < data.length; i++) data[i] = spatial[i]! + row[i % MEM_DIM]!;
+
+ const buffer = createStorageBuffer(this.device, data.byteLength, `sam2 memory_pos slot ${slot}`);
+ this.device.queue.writeBuffer(buffer, 0, data);
+ return buffer;
+ });
+ }
+
+ condition(index: number, tokens: GPUBuffer, pointer: GPUBuffer): void {
+ this.reset();
+ this.conditioning = this.retain(index, tokens, pointer);
+ }
+
+ push(index: number, tokens: GPUBuffer, pointer: GPUBuffer): void {
+ this.recent.push(this.retain(index, tokens, pointer));
+ while (this.recent.length > MAX_POINTERS - 1) this.pool.push(this.recent.shift()!);
+ }
+
+ /** Back to the prompted frame alone, for a pass in the other direction. */
+ rewind(): void {
+ this.pool.push(...this.recent);
+ this.recent = [];
+ }
+
+ reset(): void {
+ this.rewind();
+ if (this.conditioning) this.pool.push(this.conditioning);
+ this.conditioning = null;
+ }
+
+ /**
+ * Lays the bank out for frame `index` and returns each pointer's distance
+ * to it, normalized the way `pointer_tpos` expects. Slot 0 is the prompted
+ * frame, slots 1 to 6 the newest frames first; empty slots and pointers
+ * repeat the newest entry, which the graph's fixed shapes require.
+ */
+ assemble(index: number, totalFrames: number): number[] {
+ const conditioning = this.conditioning;
+ const positions = this.positions;
+ if (!conditioning || !positions) throw new Error('The memory bank has no prompted frame');
+
+ const newestFirst = [...this.recent].reverse();
+ const blocks = [{ tokens: conditioning.tokens, pos: positions[NUM_MASKMEM - 1]! }];
+ for (let slot = 1; slot < NUM_MASKMEM; slot++) {
+ const entry = newestFirst[slot - 1];
+ blocks.push(entry ? { tokens: entry.tokens, pos: positions[slot - 1]! } : blocks[1] ?? blocks[0]!);
+ }
+
+ const pointers = [conditioning, ...newestFirst].slice(0, MAX_POINTERS);
+ while (pointers.length < MAX_POINTERS) pointers.push(pointers[pointers.length - 1]!);
+
+ const encoder = this.device.createCommandEncoder();
+ blocks.forEach((block, slot) => {
+ encoder.copyBufferToBuffer(block.tokens, 0, this.memory, slot * MEMORY_BLOCK_BYTES, MEMORY_BLOCK_BYTES);
+ encoder.copyBufferToBuffer(block.pos, 0, this.memoryPos, slot * MEMORY_BLOCK_BYTES, MEMORY_BLOCK_BYTES);
+ });
+ pointers.forEach((entry, i) => {
+ encoder.copyBufferToBuffer(entry.pointer, 0, this.memory, POINTER_OFFSET + i * POINTER_BYTES, POINTER_BYTES);
+ });
+ this.device.queue.submit([encoder.finish()]);
+
+ const span = Math.max(1, Math.min(totalFrames, MAX_POINTERS) - 1);
+ return pointers.map((entry) => Math.abs(index - entry.index) / span);
+ }
+
+ /** Writes the pointers' temporal positions, one [MEM_DIM] row per pointer repeated over its tokens. */
+ setPointerPositions(positions: GPUBuffer): void {
+ const encoder = this.device.createCommandEncoder();
+ for (let i = 0; i < MAX_POINTERS; i++) {
+ for (let token = 0; token < POINTER_TOKENS; token++) {
+ const row = i * POINTER_TOKENS + token;
+ encoder.copyBufferToBuffer(positions, i * MEMORY_ROW_BYTES, this.memoryPos, POINTER_OFFSET + row * MEMORY_ROW_BYTES, MEMORY_ROW_BYTES);
+ }
+ }
+ this.device.queue.submit([encoder.finish()]);
+ }
+
+ dispose(): void {
+ this.reset();
+ for (const entry of this.pool) {
+ entry.tokens.destroy();
+ entry.pointer.destroy();
+ }
+ this.pool.length = 0;
+ this.positions?.forEach((buffer) => buffer.destroy());
+ this.memory.destroy();
+ this.memoryPos.destroy();
+ }
+
+ private retain(index: number, tokens: GPUBuffer, pointer: GPUBuffer): FrameMemory {
+ const entry = this.pool.pop() ?? {
+ index,
+ tokens: createStorageBuffer(this.device, MEMORY_BLOCK_BYTES, 'sam2 frame memory'),
+ pointer: createStorageBuffer(this.device, POINTER_BYTES, 'sam2 frame pointer'),
+ };
+ entry.index = index;
+
+ const encoder = this.device.createCommandEncoder();
+ encoder.copyBufferToBuffer(tokens, 0, entry.tokens, 0, MEMORY_BLOCK_BYTES);
+ encoder.copyBufferToBuffer(pointer, 0, entry.pointer, 0, POINTER_BYTES);
+ this.device.queue.submit([encoder.finish()]);
+ return entry;
+ }
+}
+
+const POINTER_OFFSET = NUM_MASKMEM * MEMORY_BLOCK_BYTES;
diff --git a/packages/sam2/src/sessions.ts b/packages/sam2/src/sessions.ts
new file mode 100644
index 00000000..29547cd1
--- /dev/null
+++ b/packages/sam2/src/sessions.ts
@@ -0,0 +1,57 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import * as ort from 'onnxruntime-web/webgpu';
+import wasmUrl from 'onnxruntime-web/ort-wasm-simd-threaded.asyncify.wasm?url';
+
+export type Session = ort.InferenceSession;
+export type Tensor = ort.Tensor;
+export type Outputs = Awaited<ReturnType<Session['run']>>;
+
+/** Points the runtime at its bundled wasm; it creates the GPU device itself when the first session is built. */
+export function configureRuntime(): void {
+ ort.env.wasm.wasmPaths = { wasm: wasmUrl };
+ ort.env.webgpu.powerPreference = 'high-performance';
+}
+
+/** The device the runtime built its sessions on, which the pipeline's own passes share so buffers need no copies. */
+export function runtimeDevice(): Promise<GPUDevice> {
+ return ort.env.webgpu.device;
+}
+
+/** A WebGPU session whose listed outputs stay on the GPU; the rest are downloaded after each run. */
+export function createSession(model: ArrayBuffer, gpuOutputs: readonly string[]): Promise<Session> {
+ return ort.InferenceSession.create(model, {
+ executionProviders: ['webgpu'],
+ graphOptimizationLevel: 'all',
+ preferredOutputLocation: Object.fromEntries(gpuOutputs.map((name) => [name, 'gpu-buffer' as const])),
+ });
+}
+
+export function gpuTensor(buffer: GPUBuffer, dims: number[]): Tensor {
+ return ort.Tensor.fromGpuBuffer(buffer, { dataType: 'float32', dims });
+}
+
+export function floatTensor(data: ArrayLike<number>, dims: number[]): Tensor {
+ return new ort.Tensor('float32', Float32Array.from(data), dims);
+}
+
+export function intTensor(data: ArrayLike<number>, dims: number[]): Tensor {
+ return new ort.Tensor('int32', Int32Array.from(data), dims);
+}
+
+export function floats(outputs: Outputs, name: string): Float32Array {
+ return outputs[name]!.data as Float32Array;
+}
+
+export function gpuBuffer(outputs: Outputs, name: string): GPUBuffer {
+ return outputs[name]!.gpuBuffer;
+}
+
+/** Hands GPU-resident outputs back to the runtime's pool. */
+export function disposeOutputs(outputs: Outputs): void {
+ for (const tensor of Object.values(outputs)) {
+ if (tensor.location === 'gpu-buffer') tensor.dispose();
+ }
+}
diff --git a/packages/sam2/src/shaders.ts b/packages/sam2/src/shaders.ts
new file mode 100644
index 00000000..3d30f28c
--- /dev/null
+++ b/packages/sam2/src/shaders.ts
@@ -0,0 +1,80 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE } from './constants';
+
+export const WORKGROUP_SIZE = 16;
+
+/**
+ * Resizes a video frame to the model's square input, applies the track's
+ * display rotation, and normalizes into planar RGB floats. Frames larger than
+ * the input are box-filtered with a few taps per output pixel so downscaling
+ * does not alias.
+ */
+export const PREPROCESS_SHADER = /* wgsl */ `
+struct Params {
+ size: vec2f,
+ rotation: u32,
+ taps: u32,
+ mean: vec3f,
+ _pad0: f32,
+ deviation: vec3f,
+ _pad1: f32,
+};
+
+@group(0) @binding(0) var<uniform> params: Params;
+@group(0) @binding(1) var frame: texture_2d<f32>;
+@group(0) @binding(2) var frameSampler: sampler;
+@group(0) @binding(3) var<storage, read_write> pixels: array<f32>;
+
+const SIZE: u32 = ${IMAGE_SIZE}u;
+const PLANE: u32 = SIZE * SIZE;
+
+fn toSource(uv: vec2f) -> vec2f {
+ switch params.rotation {
+ case 1u: { return vec2f(uv.y, 1.0 - uv.x); }
+ case 2u: { return vec2f(1.0 - uv.x, 1.0 - uv.y); }
+ case 3u: { return vec2f(1.0 - uv.y, uv.x); }
+ default: { return uv; }
+ }
+}
+
+@compute @workgroup_size(${WORKGROUP_SIZE}, ${WORKGROUP_SIZE})
+fn main(@builtin(global_invocation_id) id: vec3u) {
+ if (id.x >= SIZE || id.y >= SIZE) { return; }
+
+ let taps = max(params.taps, 1u);
+ var rgb = vec3f(0.0);
+ for (var y = 0u; y < taps; y++) {
+ for (var x = 0u; x < taps; x++) {
+ let offset = (vec2f(f32(x), f32(y)) + 0.5) / f32(taps);
+ let uv = (vec2f(id.xy) + offset) / f32(SIZE);
+ rgb += textureSampleLevel(frame, frameSampler, toSource(uv), 0.0).rgb;
+ }
+ }
+ rgb = (rgb / f32(taps * taps) - params.mean) / params.deviation;
+
+ let index = id.y * SIZE + id.x;
+ pixels[index] = rgb.r;
+ pixels[PLANE + index] = rgb.g;
+ pixels[2u * PLANE + index] = rgb.b;
+}
+`;
+
+/** Turns a [C, H, W] feature map into the [H*W, C] token layout memory attention takes. */
+export const TRANSPOSE_SHADER = /* wgsl */ `
+@group(0) @binding(0) var<storage, read> channels: array<f32>;
+@group(0) @binding(1) var<storage, read_write> tokens: array<f32>;
+
+const CHANNELS: u32 = ${HIDDEN_DIM}u;
+const TOKENS: u32 = ${FEAT_TOKENS}u;
+
+@compute @workgroup_size(${WORKGROUP_SIZE}, ${WORKGROUP_SIZE})
+fn main(@builtin(global_invocation_id) id: vec3u) {
+ let token = id.x;
+ let channel = id.y;
+ if (token >= TOKENS || channel >= CHANNELS) { return; }
+ tokens[token * CHANNELS + channel] = channels[channel * TOKENS + token];
+}
+`;
diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.ts
new file mode 100644
index 00000000..ac858eab
--- /dev/null
+++ b/packages/sam2/src/tracker.ts
@@ -0,0 +1,165 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import {
+ BYTES_PER_FLOAT, FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE, MAX_POINTERS, MEM_DIM, MEMORY_ROWS,
+} from './constants';
+import { FramePreprocessor, TokenTransposer, createStorageBuffer } from './gpu';
+import { MemoryBank } from './memory';
+import { packMask } from './mask';
+import { disposeOutputs, floatTensor, floats, gpuBuffer, gpuTensor, intTensor } from './sessions';
+
+import type { ModelConstants } from './constants';
+import type { Rotation } from './gpu';
+import type { Sam2Mask } from './mask';
+import type { Outputs, Session, Tensor } from './sessions';
+
+/** A prompt on the displayed frame, in 0..1 of its width and height; label 1 is the object, 0 is background. */
+export type Sam2Point = { x: number; y: number; label: 0 | 1 };
+
+export type Sam2Sessions = {
+ visionEncoder: Session;
+ maskDecoder: Session;
+ memoryEncoder: Session;
+ memoryAttention: Session;
+ pointerTpos: Session;
+};
+
+type Prompt = { points: Tensor; labels: Tensor };
+
+/**
+ * SAM 2.1 video tracking over one object. `seed` segments it on the prompted
+ * frame; `track` follows it through the frames that come after, each one
+ * conditioned on the memory of those before. Frames are numbered by the
+ * caller, and their distances are what the model's temporal encodings read.
+ */
+export class Sam2Video {
+ private readonly preprocessor: FramePreprocessor;
+ private readonly transposer: TokenTransposer;
+ private readonly bank: MemoryBank;
+ private readonly featureTokens: GPUBuffer;
+ private readonly positionTokens: GPUBuffer;
+
+ constructor(
+ device: GPUDevice,
+ private readonly sessions: Sam2Sessions,
+ constants: ModelConstants,
+ ) {
+ this.preprocessor = new FramePreprocessor(device, constants.image_mean, constants.image_std);
+ this.transposer = new TokenTransposer(device);
+ this.bank = new MemoryBank(device, constants.memory_temporal_positional_encoding);
+ this.featureTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 vision tokens');
+ this.positionTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 position tokens');
+ }
+
+ /** Segments the object at `points` and makes this the frame every later one is tracked from. */
+ async seed(frame: VideoFrame, rotation: Rotation, points: Sam2Point[], index: number): Promise<Sam2Mask> {
+ if (points.length === 0) throw new Error('A prompt needs at least one point');
+
+ const prompt: Prompt = {
+ points: floatTensor(points.flatMap((point) => [point.x * IMAGE_SIZE, point.y * IMAGE_SIZE]), [1, 1, points.length, 2]),
+ labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),
+ };
+
+ const vision = await this.encode(frame, rotation);
+ try {
+ return await this.segment(vision, vision.feats2_no_mem!, prompt, { index, prompted: true });
+ } finally {
+ disposeOutputs(vision);
+ }
+ }
+
+ /** Finds the seeded object in a frame `index` steps along from the last one. */
+ async track(frame: VideoFrame, rotation: Rotation, index: number, totalFrames: number): Promise<Sam2Mask> {
+ const vision = await this.encode(frame, rotation);
+ try {
+ this.transposer.run(gpuBuffer(vision, 'feats2'), this.featureTokens);
+ this.transposer.run(gpuBuffer(vision, 'vision_pos_embed'), this.positionTokens);
+
+ const distances = this.bank.assemble(index, totalFrames);
+ const temporal = await this.sessions.pointerTpos.run({ normalized_diffs: floatTensor(distances, [MAX_POINTERS]) });
+ this.bank.setPointerPositions(gpuBuffer(temporal, 'pointer_pos'));
+ disposeOutputs(temporal);
+
+ const attended = await this.sessions.memoryAttention.run({
+ current_vision_features: gpuTensor(this.featureTokens, [FEAT_TOKENS, 1, HIDDEN_DIM]),
+ current_vision_position_embeddings: gpuTensor(this.positionTokens, [FEAT_TOKENS, 1, HIDDEN_DIM]),
+ memory: gpuTensor(this.bank.memory, [MEMORY_ROWS, 1, MEM_DIM]),
+ memory_pos: gpuTensor(this.bank.memoryPos, [MEMORY_ROWS, 1, MEM_DIM]),
+ });
+ try {
+ return await this.segment(vision, attended.conditioned_feats!, paddingPrompt(), { index, prompted: false });
+ } finally {
+ disposeOutputs(attended);
+ }
+ } finally {
+ disposeOutputs(vision);
+ }
+ }
+
+ /** Forgets the tracked frames but not the seed, to track away from it in the other direction. */
+ rewind(): void {
+ this.bank.rewind();
+ }
+
+ reset(): void {
+ this.bank.reset();
+ }
+
+ dispose(): void {
+ this.bank.dispose();
+ this.preprocessor.dispose();
+ this.featureTokens.destroy();
+ this.positionTokens.destroy();
+ for (const session of Object.values(this.sessions)) void session.release();
+ }
+
+ private async encode(frame: VideoFrame, rotation: Rotation): Promise<Outputs> {
+ this.preprocessor.run(frame, rotation);
+ return this.sessions.visionEncoder.run({
+ pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, IMAGE_SIZE, IMAGE_SIZE]),
+ });
+ }
+
+ /** Decodes the mask, then encodes the frame into memory as the prompted frame or the next tracked one. */
+ private async segment(vision: Outputs, conditioned: Tensor, prompt: Prompt, frame: { index: number; prompted: boolean }): Promise<Sam2Mask> {
+ const decoded = await this.sessions.maskDecoder.run({
+ feats0: vision.feats0!,
+ feats1: vision.feats1!,
+ feats2_cond: conditioned,
+ input_points: prompt.points,
+ input_labels: prompt.labels,
+ });
+ try {
+ const score = floats(decoded, 'object_score_logits');
+ const memory = await this.sessions.memoryEncoder.run({
+ feats2: vision.feats2!,
+ high_res_mask: decoded.high_res_mask!,
+ object_score_logits: floatTensor(score, [1, 1]),
+ binarize: floatTensor([frame.prompted ? 1 : 0], []),
+ });
+ try {
+ if (!this.bank.hasPositions) {
+ this.bank.setPositions((await memory.memory_pos!.getData()) as Float32Array);
+ }
+
+ const tokens = gpuBuffer(memory, 'memory_tokens');
+ const pointer = gpuBuffer(decoded, 'object_pointer');
+ if (frame.prompted) this.bank.condition(frame.index, tokens, pointer);
+ else this.bank.push(frame.index, tokens, pointer);
+ } finally {
+ disposeOutputs(memory);
+ }
+
+ return packMask(floats(decoded, 'low_res_mask'), score[0]!, floats(decoded, 'iou')[0]!);
+ } finally {
+ disposeOutputs(decoded);
+ }
+ }
+}
+
+/** What the decoder is prompted with on frames that have no points of their own. */
+function paddingPrompt(): Prompt {
+ return { points: floatTensor([0, 0], [1, 1, 1, 2]), labels: intTensor([-1], [1, 1, 1]) };
+}
diff --git a/packages/sam2/src/video.ts b/packages/sam2/src/video.ts
new file mode 100644
index 00000000..71049fbb
--- /dev/null
+++ b/packages/sam2/src/video.ts
@@ -0,0 +1,114 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { VideoSampleSink } from 'mediabunny';
+
+import type { InputVideoTrack, VideoSample } from 'mediabunny';
+import type { Sam2Mask } from './mask';
+import type { Sam2Point, Sam2Video } from './tracker';
+
+export type TrackRequest = {
+ track: InputVideoTrack;
+ /** Where each frame to segment is in the file, in seconds, ascending. */
+ timestamps: number[];
+ /** Which of `timestamps` the points were placed on. */
+ seedIndex: number;
+ points: Sam2Point[];
+ signal?: AbortSignal;
+ /** Called as each frame's mask is ready, in tracking order: the seed, then forward, then backward. */
+ onMask: (index: number, mask: Sam2Mask) => void;
+};
+
+/**
+ * Frames decoded in reverse are read a batch at a time, in file order, and
+ * walked backward: seeking to every frame on its own would decode from its
+ * keyframe each time.
+ */
+const REVERSE_BATCH = 12;
+
+/**
+ * Segments the object at the seed and follows it through every frame of the
+ * request, away from the seed in both directions. Frames that decode to the
+ * same picture as the one before (a source slower than the timestamps) share
+ * its mask instead of being run again.
+ */
+export async function trackObject(model: Sam2Video, request: TrackRequest): Promise<void> {
+ const { track, timestamps, seedIndex, points, signal, onMask } = request;
+ const sink = new VideoSampleSink(track);
+ const rotation = track.rotation;
+ const total = timestamps.length;
+
+ const seed = await sink.getSample(timestamps[seedIndex]!);
+ if (!seed) throw new Error('The prompted frame could not be decoded');
+
+ const seedTimestamp = seed.timestamp;
+ const seedMask = await withVideoFrame(seed, (frame) => model.seed(frame, rotation, points, seedIndex));
+ onMask(seedIndex, seedMask);
+
+ let previous = { timestamp: seedTimestamp, mask: seedMask };
+
+ const process = async (sample: VideoSample, index: number) => {
+ if (sample.timestamp === previous.timestamp) {
+ sample.close();
+ onMask(index, previous.mask);
+ return;
+ }
+ const timestamp = sample.timestamp;
+ const mask = await withVideoFrame(sample, (frame) => model.track(frame, rotation, index, total));
+ previous = { timestamp, mask };
+ onMask(index, mask);
+ };
+
+ const forward = sink.samplesAtTimestamps(timestamps.slice(seedIndex + 1));
+ let index = seedIndex + 1;
+ // The next decode is requested before the current frame is processed, so
+ // decoding and inference overlap.
+ let pending = forward.next();
+ try {
+ for (;;) {
+ const { done, value: sample } = await pending;
+ if (done) break;
+ pending = forward.next();
+
+ if (signal?.aborted) {
+ sample?.close();
+ return;
+ }
+ if (sample) await process(sample, index);
+ index++;
+ }
+ } finally {
+ await forward.return();
+ }
+
+ model.rewind();
+ previous = { timestamp: seedTimestamp, mask: seedMask };
+
+ for (let end = seedIndex; end > 0; end -= REVERSE_BATCH) {
+ const start = Math.max(0, end - REVERSE_BATCH);
+ const samples: (VideoSample | null)[] = [];
+ for await (const sample of sink.samplesAtTimestamps(timestamps.slice(start, end))) samples.push(sample);
+
+ for (let i = samples.length - 1; i >= 0; i--) {
+ const sample = samples[i];
+ if (!sample) continue;
+ if (signal?.aborted) {
+ sample.close();
+ continue;
+ }
+ await process(sample, start + i);
+ }
+ if (signal?.aborted) return;
+ }
+}
+
+async function withVideoFrame<T>(sample: VideoSample, run: (frame: VideoFrame) => Promise<T>): Promise<T> {
+ const frame = sample.toVideoFrame();
+ try {
+ return await run(frame);
+ } finally {
+ frame.close();
+ sample.close();
+ }
+}
diff --git a/packages/sam2/src/weights.ts b/packages/sam2/src/weights.ts
new file mode 100644
index 00000000..c3d9c9d7
--- /dev/null
+++ b/packages/sam2/src/weights.ts
@@ -0,0 +1,104 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { MODEL_FILES, MODEL_REPO } from './constants';
+
+export type ModelFileKey = keyof typeof MODEL_FILES;
+export type ModelFiles = Record<ModelFileKey, ArrayBuffer>;
+export type DownloadProgress = { loaded: number; total: number };
+
+const DB_NAME = 'diffusion-sam2';
+const STORE_NAME = 'files';
+
+const TOTAL_BYTES = Object.values(MODEL_FILES).reduce((sum, file) => sum + file.size, 0);
+
+/**
+ * The model files, from the local cache when they have been fetched before
+ * and from the Hugging Face repo otherwise. Files are keyed by URL, so a
+ * change of repo or revision invalidates them.
+ */
+export async function fetchModelFiles(
+ onProgress?: (progress: DownloadProgress) => void,
+ signal?: AbortSignal,
+): Promise<ModelFiles> {
+ const store = await openFileStore();
+ const loaded = new Map<ModelFileKey, number>();
+ const report = () => onProgress?.({ loaded: [...loaded.values()].reduce((a, b) => a + b, 0), total: TOTAL_BYTES });
+
+ const entries = await Promise.all(
+ (Object.keys(MODEL_FILES) as ModelFileKey[]).map(async (key) => {
+ const file = MODEL_FILES[key];
+ const url = `${MODEL_REPO}/${file.path}`;
+
+ let blob = await store?.read(url);
+ if (!blob) {
+ blob = await download(url, signal, (bytes) => {
+ loaded.set(key, bytes);
+ report();
+ });
+ await store?.write(url, blob);
+ }
+
+ loaded.set(key, file.size);
+ report();
+ return [key, await blob.arrayBuffer()] as const;
+ }),
+ );
+
+ return Object.fromEntries(entries) as ModelFiles;
+}
+
+async function download(url: string, signal: AbortSignal | undefined, onBytes: (bytes: number) => void): Promise<Blob> {
+ const response = await fetch(url, { signal });
+ if (!response.ok || !response.body) {
+ throw new Error(`Could not fetch ${url} (${response.status})`);
+ }
+
+ const reader = response.body.getReader();
+ const chunks: BlobPart[] = [];
+ let received = 0;
+
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ chunks.push(value);
+ received += value.byteLength;
+ onBytes(received);
+ }
+
+ return new Blob(chunks);
+}
+
+type FileStore = {
+ read(key: string): Promise<Blob | undefined>;
+ write(key: string, blob: Blob): Promise<void>;
+};
+
+async function openFileStore(): Promise<FileStore | null> {
+ if (typeof indexedDB === 'undefined') return null;
+
+ try {
+ const db = await new Promise<IDBDatabase>((resolve, reject) => {
+ const request = indexedDB.open(DB_NAME, 1);
+ request.onupgradeneeded = () => request.result.createObjectStore(STORE_NAME);
+ request.onsuccess = () => resolve(request.result);
+ request.onerror = () => reject(request.error);
+ });
+
+ const run = <T>(mode: IDBTransactionMode, operation: (store: IDBObjectStore) => IDBRequest<T>) =>
+ new Promise<T>((resolve, reject) => {
+ const request = operation(db.transaction(STORE_NAME, mode).objectStore(STORE_NAME));
+ request.onsuccess = () => resolve(request.result);
+ request.onerror = () => reject(request.error);
+ });
+
+ return {
+ read: (key) => run<Blob | undefined>('readonly', (store) => store.get(key)),
+ // A cache that cannot be written (quota, private mode) is not a reason to fail the load.
+ write: (key, blob) => run('readwrite', (store) => store.put(blob, key)).then(() => undefined, () => undefined),
+ };
+ } catch {
+ return null;
+ }
+}
diff --git a/packages/sam2/tsconfig.json b/packages/sam2/tsconfig.json
new file mode 100644
index 00000000..c3c72503
--- /dev/null
+++ b/packages/sam2/tsconfig.json
@@ -0,0 +1,18 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo",
+ "erasableSyntaxOnly": false,
+ "lib": [
+ "es2024",
+ "DOM",
+ "DOM.Iterable"
+ ],
+ "types": [
+ "@webgpu/types"
+ ]
+ },
+ "include": [
+ "src"
+ ]
+}