The VS Code of video editing, built for agents.
TypeScript
Add SAM 2.1 video object segmentation and tracking package
konstantin-paulus6c3901e5430c
diff --git a/packages/sam2/package.json b/packages/sam2/package.jsonnew file mode 100644index 00000000..09e8f9e8--- /dev/null+++ b/packages/sam2/package.json@@ -0,0 +1,30 @@+{+ "name": "@diffusionstudio/sam2",+ "version": "0.1.0",+ "description": "SAM 2.1 video object segmentation and tracking in the browser: ONNX Runtime on WebGPU, frames decoded with mediabunny. Weights are fetched from Hugging Face on first use and cached locally.",+ "license": "MPL-2.0",+ "type": "module",+ "sideEffects": false,+ "exports": {+ ".": "./src/index.ts",+ "./mask": "./src/mask.ts"+ },+ "scripts": {+ "check": "tsc --noEmit"+ },+ "keywords": [+ "sam2",+ "segmentation",+ "tracking",+ "webgpu",+ "onnx"+ ],+ "dependencies": {+ "mediabunny": "^1.32.2",+ "onnxruntime-web": "^1.30.0"+ },+ "devDependencies": {+ "@webgpu/types": "^0.1.71",+ "typescript": "~5.9.3"+ }+}diff --git a/packages/sam2/src/constants.ts b/packages/sam2/src/constants.tsnew file mode 100644index 00000000..36550a45--- /dev/null+++ b/packages/sam2/src/constants.ts@@ -0,0 +1,37 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++export const MODEL_REPO = 'https://huggingface.co/square-zero-labs/sam2.1-tiny-video-onnx/resolve/main';++export const MODEL_FILES = {+ constants: { path: 'constants.json', size: 9_922 },+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 134_335_567 },+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 17_794_355 },+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 5_616_496 },+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 32_259_165 },+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 67_289 },+} as const;++export const IMAGE_SIZE = 1024;+export const MASK_SIZE = 256;+export const FEAT_SIZE = 64;+export const FEAT_TOKENS = FEAT_SIZE * FEAT_SIZE;+export const HIDDEN_DIM = 256;+export const MEM_DIM = 64;+export const NUM_MASKMEM = 7;+export const MAX_POINTERS = 16;+export const POINTER_TOKENS = HIDDEN_DIM / MEM_DIM;+export const MEMORY_ROWS = NUM_MASKMEM * FEAT_TOKENS + MAX_POINTERS * POINTER_TOKENS;++export const BYTES_PER_FLOAT = 4;+export const MEMORY_ROW_BYTES = MEM_DIM * BYTES_PER_FLOAT;+export const MEMORY_BLOCK_BYTES = FEAT_TOKENS * MEMORY_ROW_BYTES;+export const POINTER_BYTES = HIDDEN_DIM * BYTES_PER_FLOAT;++/** The values `constants.json` in the model repo carries. */+export type ModelConstants = {+ image_mean: [number, number, number];+ image_std: [number, number, number];+ memory_temporal_positional_encoding: number[][];+};diff --git a/packages/sam2/src/env.d.ts b/packages/sam2/src/env.d.tsnew file mode 100644index 00000000..f72e628d--- /dev/null+++ b/packages/sam2/src/env.d.ts@@ -0,0 +1,4 @@+declare module '*.wasm?url' {+ const url: string;+ export default url;+}diff --git a/packages/sam2/src/gpu.ts b/packages/sam2/src/gpu.tsnew file mode 100644index 00000000..3785120a--- /dev/null+++ b/packages/sam2/src/gpu.ts@@ -0,0 +1,131 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { BYTES_PER_FLOAT, FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE } from './constants';+import { PREPROCESS_SHADER, TRANSPOSE_SHADER, WORKGROUP_SIZE } from './shaders';++/** Degrees clockwise a decoded frame is turned for display. */+export type Rotation = 0 | 90 | 180 | 270;++export function createStorageBuffer(device: GPUDevice, size: number, label: string): GPUBuffer {+ return device.createBuffer({+ label,+ size,+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST,+ });+}++function createPipeline(device: GPUDevice, code: string, label: string): GPUComputePipeline {+ return device.createComputePipeline({+ label,+ layout: 'auto',+ compute: { module: device.createShaderModule({ label, code }), entryPoint: 'main' },+ });+}++function dispatch(device: GPUDevice, pipeline: GPUComputePipeline, bindGroup: GPUBindGroup, x: number, y: number): void {+ const encoder = device.createCommandEncoder();+ const pass = encoder.beginComputePass();+ pass.setPipeline(pipeline);+ pass.setBindGroup(0, bindGroup);+ pass.dispatchWorkgroups(x, y);+ pass.end();+ device.queue.submit([encoder.finish()]);+}++const PARAMS_BYTES = 48;+const MAX_TAPS = 4;++/** Uploads a frame and turns it into the vision encoder's `pixel_values`. */+export class FramePreprocessor {+ readonly pixels: GPUBuffer;+ private readonly pipeline: GPUComputePipeline;+ private readonly sampler: GPUSampler;+ private readonly params: GPUBuffer;+ private texture: GPUTexture | null = null;++ constructor(+ private readonly device: GPUDevice,+ private readonly mean: readonly number[],+ private readonly std: readonly number[],+ ) {+ this.pixels = createStorageBuffer(device, 3 * IMAGE_SIZE * IMAGE_SIZE * BYTES_PER_FLOAT, 'sam2 pixel_values');+ this.pipeline = createPipeline(device, PREPROCESS_SHADER, 'sam2 preprocess');+ this.sampler = device.createSampler({ magFilter: 'linear', minFilter: 'linear' });+ this.params = device.createBuffer({+ label: 'sam2 preprocess params',+ size: PARAMS_BYTES,+ usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,+ });+ }++ run(frame: VideoFrame, rotation: Rotation): void {+ const width = frame.displayWidth;+ const height = frame.displayHeight;+ const texture = this.textureFor(width, height);+ this.device.queue.copyExternalImageToTexture({ source: frame }, { texture }, [width, height]);++ const params = new ArrayBuffer(PARAMS_BYTES);+ const floats = new Float32Array(params);+ const uints = new Uint32Array(params);+ floats[0] = width;+ floats[1] = height;+ uints[2] = rotation / 90;+ uints[3] = Math.min(MAX_TAPS, Math.max(1, Math.ceil(Math.max(width, height) / IMAGE_SIZE)));+ floats.set(this.mean, 4);+ floats.set(this.std, 8);+ this.device.queue.writeBuffer(this.params, 0, params);++ const bindGroup = this.device.createBindGroup({+ layout: this.pipeline.getBindGroupLayout(0),+ entries: [+ { binding: 0, resource: { buffer: this.params } },+ { binding: 1, resource: texture.createView() },+ { binding: 2, resource: this.sampler },+ { binding: 3, resource: { buffer: this.pixels } },+ ],+ });+ dispatch(this.device, this.pipeline, bindGroup, IMAGE_SIZE / WORKGROUP_SIZE, IMAGE_SIZE / WORKGROUP_SIZE);+ }++ private textureFor(width: number, height: number): GPUTexture {+ if (this.texture?.width === width && this.texture.height === height) return this.texture;++ this.texture?.destroy();+ this.texture = this.device.createTexture({+ label: 'sam2 frame',+ size: [width, height],+ format: 'rgba8unorm',+ usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.RENDER_ATTACHMENT,+ });+ return this.texture;+ }++ dispose(): void {+ this.texture?.destroy();+ this.pixels.destroy();+ this.params.destroy();+ }+}++/** Rewrites a [1, C, H, W] feature map as the [H*W, 1, C] tokens memory attention reads. */+export class TokenTransposer {+ private readonly pipeline: GPUComputePipeline;++ constructor(private readonly device: GPUDevice) {+ this.pipeline = createPipeline(device, TRANSPOSE_SHADER, 'sam2 transpose');+ }++ run(channels: GPUBuffer, tokens: GPUBuffer): void {+ const size = FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT;+ const bindGroup = this.device.createBindGroup({+ layout: this.pipeline.getBindGroupLayout(0),+ entries: [+ { binding: 0, resource: { buffer: channels, size } },+ { binding: 1, resource: { buffer: tokens, size } },+ ],+ });+ dispatch(this.device, this.pipeline, bindGroup, FEAT_TOKENS / WORKGROUP_SIZE, HIDDEN_DIM / WORKGROUP_SIZE);+ }+}diff --git a/packages/sam2/src/index.ts b/packages/sam2/src/index.tsnew file mode 100644index 00000000..88aff151--- /dev/null+++ b/packages/sam2/src/index.ts@@ -0,0 +1,13 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++export { loadSam2 } from './load';+export { Sam2Video } from './tracker';+export { trackObject } from './video';+export { MASK_SIZE, maskHas, maskIsEmpty, renderMask } from './mask';++export type { Sam2LoadOptions, Sam2Progress } from './load';+export type { Sam2Point } from './tracker';+export type { TrackRequest } from './video';+export type { Sam2Mask } from './mask';diff --git a/packages/sam2/src/load.ts b/packages/sam2/src/load.tsnew file mode 100644index 00000000..e8a9e09d--- /dev/null+++ b/packages/sam2/src/load.ts@@ -0,0 +1,48 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { configureRuntime, createSession, runtimeDevice } from './sessions';+import { Sam2Video } from './tracker';+import { fetchModelFiles } from './weights';++import type { ModelConstants } from './constants';++export type Sam2Progress =+ | { phase: 'download'; loaded: number; total: number }+ | { phase: 'compile' };++export type Sam2LoadOptions = {+ onProgress?: (progress: Sam2Progress) => void;+ signal?: AbortSignal;+};++let loading: Promise<Sam2Video> | null = null;++/** The model, loaded once per page: the first call fetches and compiles it, later calls share the result. */+export function loadSam2(options: Sam2LoadOptions = {}): Promise<Sam2Video> {+ loading ??= load(options).catch((error: unknown) => {+ loading = null;+ throw error;+ });+ return loading;+}++async function load({ onProgress, signal }: Sam2LoadOptions): Promise<Sam2Video> {+ if (!navigator.gpu) throw new Error('WebGPU is not available in this browser');+ configureRuntime();++ const files = await fetchModelFiles((progress) => onProgress?.({ phase: 'download', ...progress }), signal);+ const constants = JSON.parse(new TextDecoder().decode(files.constants)) as ModelConstants;++ onProgress?.({ phase: 'compile' });+ // The runtime builds one WebGPU session at a time.+ const visionEncoder = await createSession(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed']);+ const maskDecoder = await createSession(files.maskDecoder, ['high_res_mask', 'object_pointer']);+ const memoryEncoder = await createSession(files.memoryEncoder, ['memory_tokens', 'memory_pos']);+ const memoryAttention = await createSession(files.memoryAttention, ['conditioned_feats']);+ const pointerTpos = await createSession(files.pointerTpos, ['pointer_pos']);+ const device = await runtimeDevice();++ return new Sam2Video(device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants);+}diff --git a/packages/sam2/src/mask.ts b/packages/sam2/src/mask.tsnew file mode 100644index 00000000..b5083a24--- /dev/null+++ b/packages/sam2/src/mask.ts@@ -0,0 +1,55 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { MASK_SIZE } from './constants';++export { MASK_SIZE };++/** One frame's segmentation, at the model's mask resolution. */+export type Sam2Mask = {+ /** Row-major, one bit per pixel, most significant bit first. */+ bits: Uint8Array;+ /** Whether the object is in frame: at or below zero the model saw it occluded. */+ score: number;+ /** The model's own estimate of the mask's quality, 0 to 1. */+ iou: number;+};++const MASK_BYTES = (MASK_SIZE * MASK_SIZE) / 8;++/** Packs mask logits into bits; positive logits are foreground. */+export function packMask(logits: Float32Array, score: number, iou: number): Sam2Mask {+ const bits = new Uint8Array(MASK_BYTES);+ for (let i = 0; i < logits.length; i++) {+ if (logits[i]! > 0) bits[i >> 3]! |= 0x80 >> (i & 7);+ }+ return { bits, score, iou };+}++export function maskHas(mask: Sam2Mask, x: number, y: number): boolean {+ const i = y * MASK_SIZE + x;+ return (mask.bits[i >> 3]! & (0x80 >> (i & 7))) !== 0;+}++/** Whether any pixel is foreground. */+export function maskIsEmpty(mask: Sam2Mask): boolean {+ return mask.bits.every((byte) => byte === 0);+}++/** The mask as RGBA pixels, `color` where it is foreground and transparent elsewhere. */+export function renderMask(mask: Sam2Mask, color: [r: number, g: number, b: number, a: number]): ImageData {+ const image = new ImageData(MASK_SIZE, MASK_SIZE);+ const pixels = new Uint32Array(image.data.buffer);+ const packed = (color[3] << 24 | color[2] << 16 | color[1] << 8 | color[0]) >>> 0;++ for (let byte = 0; byte < mask.bits.length; byte++) {+ const value = mask.bits[byte]!;+ if (value === 0) continue;+ for (let bit = 0; bit < 8; bit++) {+ if (value & (0x80 >> bit)) pixels[byte * 8 + bit] = packed;+ }+ }++ return image;+}diff --git a/packages/sam2/src/memory.ts b/packages/sam2/src/memory.tsnew file mode 100644index 00000000..0dc89a29--- /dev/null+++ b/packages/sam2/src/memory.ts@@ -0,0 +1,156 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import {+ MAX_POINTERS, MEM_DIM, MEMORY_BLOCK_BYTES, MEMORY_ROW_BYTES, MEMORY_ROWS,+ NUM_MASKMEM, POINTER_BYTES, POINTER_TOKENS,+} from './constants';+import { createStorageBuffer } from './gpu';++type FrameMemory = {+ index: number;+ tokens: GPUBuffer;+ pointer: GPUBuffer;+};++/**+ * What memory attention sees of the frames already tracked: the memory of the+ * prompted frame, the most recent frames' memories, and the object pointers of+ * the last sixteen. Assembled on the GPU from copies of the encoder's outputs,+ * so nothing crosses to the CPU between frames.+ */+export class MemoryBank {+ readonly memory: GPUBuffer;+ readonly memoryPos: GPUBuffer;+ private positions: GPUBuffer[] | null = null;+ private conditioning: FrameMemory | null = null;+ private recent: FrameMemory[] = [];+ private readonly pool: FrameMemory[] = [];++ constructor(+ private readonly device: GPUDevice,+ private readonly temporalEncoding: readonly (readonly number[])[],+ ) {+ this.memory = createStorageBuffer(device, MEMORY_ROWS * MEMORY_ROW_BYTES, 'sam2 memory');+ this.memoryPos = createStorageBuffer(device, MEMORY_ROWS * MEMORY_ROW_BYTES, 'sam2 memory_pos');+ }++ get hasPositions(): boolean {+ return this.positions !== null;+ }++ /**+ * The memory encoder's `memory_pos` is the same for every frame, so each+ * temporal slot's positions are laid down once: the spatial encoding plus+ * the slot's row of the temporal table.+ */+ setPositions(spatial: Float32Array): void {+ this.positions = this.temporalEncoding.map((row, slot) => {+ const data = new Float32Array(spatial.length);+ for (let i = 0; i < data.length; i++) data[i] = spatial[i]! + row[i % MEM_DIM]!;++ const buffer = createStorageBuffer(this.device, data.byteLength, `sam2 memory_pos slot ${slot}`);+ this.device.queue.writeBuffer(buffer, 0, data);+ return buffer;+ });+ }++ condition(index: number, tokens: GPUBuffer, pointer: GPUBuffer): void {+ this.reset();+ this.conditioning = this.retain(index, tokens, pointer);+ }++ push(index: number, tokens: GPUBuffer, pointer: GPUBuffer): void {+ this.recent.push(this.retain(index, tokens, pointer));+ while (this.recent.length > MAX_POINTERS - 1) this.pool.push(this.recent.shift()!);+ }++ /** Back to the prompted frame alone, for a pass in the other direction. */+ rewind(): void {+ this.pool.push(...this.recent);+ this.recent = [];+ }++ reset(): void {+ this.rewind();+ if (this.conditioning) this.pool.push(this.conditioning);+ this.conditioning = null;+ }++ /**+ * Lays the bank out for frame `index` and returns each pointer's distance+ * to it, normalized the way `pointer_tpos` expects. Slot 0 is the prompted+ * frame, slots 1 to 6 the newest frames first; empty slots and pointers+ * repeat the newest entry, which the graph's fixed shapes require.+ */+ assemble(index: number, totalFrames: number): number[] {+ const conditioning = this.conditioning;+ const positions = this.positions;+ if (!conditioning || !positions) throw new Error('The memory bank has no prompted frame');++ const newestFirst = [...this.recent].reverse();+ const blocks = [{ tokens: conditioning.tokens, pos: positions[NUM_MASKMEM - 1]! }];+ for (let slot = 1; slot < NUM_MASKMEM; slot++) {+ const entry = newestFirst[slot - 1];+ blocks.push(entry ? { tokens: entry.tokens, pos: positions[slot - 1]! } : blocks[1] ?? blocks[0]!);+ }++ const pointers = [conditioning, ...newestFirst].slice(0, MAX_POINTERS);+ while (pointers.length < MAX_POINTERS) pointers.push(pointers[pointers.length - 1]!);++ const encoder = this.device.createCommandEncoder();+ blocks.forEach((block, slot) => {+ encoder.copyBufferToBuffer(block.tokens, 0, this.memory, slot * MEMORY_BLOCK_BYTES, MEMORY_BLOCK_BYTES);+ encoder.copyBufferToBuffer(block.pos, 0, this.memoryPos, slot * MEMORY_BLOCK_BYTES, MEMORY_BLOCK_BYTES);+ });+ pointers.forEach((entry, i) => {+ encoder.copyBufferToBuffer(entry.pointer, 0, this.memory, POINTER_OFFSET + i * POINTER_BYTES, POINTER_BYTES);+ });+ this.device.queue.submit([encoder.finish()]);++ const span = Math.max(1, Math.min(totalFrames, MAX_POINTERS) - 1);+ return pointers.map((entry) => Math.abs(index - entry.index) / span);+ }++ /** Writes the pointers' temporal positions, one [MEM_DIM] row per pointer repeated over its tokens. */+ setPointerPositions(positions: GPUBuffer): void {+ const encoder = this.device.createCommandEncoder();+ for (let i = 0; i < MAX_POINTERS; i++) {+ for (let token = 0; token < POINTER_TOKENS; token++) {+ const row = i * POINTER_TOKENS + token;+ encoder.copyBufferToBuffer(positions, i * MEMORY_ROW_BYTES, this.memoryPos, POINTER_OFFSET + row * MEMORY_ROW_BYTES, MEMORY_ROW_BYTES);+ }+ }+ this.device.queue.submit([encoder.finish()]);+ }++ dispose(): void {+ this.reset();+ for (const entry of this.pool) {+ entry.tokens.destroy();+ entry.pointer.destroy();+ }+ this.pool.length = 0;+ this.positions?.forEach((buffer) => buffer.destroy());+ this.memory.destroy();+ this.memoryPos.destroy();+ }++ private retain(index: number, tokens: GPUBuffer, pointer: GPUBuffer): FrameMemory {+ const entry = this.pool.pop() ?? {+ index,+ tokens: createStorageBuffer(this.device, MEMORY_BLOCK_BYTES, 'sam2 frame memory'),+ pointer: createStorageBuffer(this.device, POINTER_BYTES, 'sam2 frame pointer'),+ };+ entry.index = index;++ const encoder = this.device.createCommandEncoder();+ encoder.copyBufferToBuffer(tokens, 0, entry.tokens, 0, MEMORY_BLOCK_BYTES);+ encoder.copyBufferToBuffer(pointer, 0, entry.pointer, 0, POINTER_BYTES);+ this.device.queue.submit([encoder.finish()]);+ return entry;+ }+}++const POINTER_OFFSET = NUM_MASKMEM * MEMORY_BLOCK_BYTES;diff --git a/packages/sam2/src/sessions.ts b/packages/sam2/src/sessions.tsnew file mode 100644index 00000000..29547cd1--- /dev/null+++ b/packages/sam2/src/sessions.ts@@ -0,0 +1,57 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import * as ort from 'onnxruntime-web/webgpu';+import wasmUrl from 'onnxruntime-web/ort-wasm-simd-threaded.asyncify.wasm?url';++export type Session = ort.InferenceSession;+export type Tensor = ort.Tensor;+export type Outputs = Awaited<ReturnType<Session['run']>>;++/** Points the runtime at its bundled wasm; it creates the GPU device itself when the first session is built. */+export function configureRuntime(): void {+ ort.env.wasm.wasmPaths = { wasm: wasmUrl };+ ort.env.webgpu.powerPreference = 'high-performance';+}++/** The device the runtime built its sessions on, which the pipeline's own passes share so buffers need no copies. */+export function runtimeDevice(): Promise<GPUDevice> {+ return ort.env.webgpu.device;+}++/** A WebGPU session whose listed outputs stay on the GPU; the rest are downloaded after each run. */+export function createSession(model: ArrayBuffer, gpuOutputs: readonly string[]): Promise<Session> {+ return ort.InferenceSession.create(model, {+ executionProviders: ['webgpu'],+ graphOptimizationLevel: 'all',+ preferredOutputLocation: Object.fromEntries(gpuOutputs.map((name) => [name, 'gpu-buffer' as const])),+ });+}++export function gpuTensor(buffer: GPUBuffer, dims: number[]): Tensor {+ return ort.Tensor.fromGpuBuffer(buffer, { dataType: 'float32', dims });+}++export function floatTensor(data: ArrayLike<number>, dims: number[]): Tensor {+ return new ort.Tensor('float32', Float32Array.from(data), dims);+}++export function intTensor(data: ArrayLike<number>, dims: number[]): Tensor {+ return new ort.Tensor('int32', Int32Array.from(data), dims);+}++export function floats(outputs: Outputs, name: string): Float32Array {+ return outputs[name]!.data as Float32Array;+}++export function gpuBuffer(outputs: Outputs, name: string): GPUBuffer {+ return outputs[name]!.gpuBuffer;+}++/** Hands GPU-resident outputs back to the runtime's pool. */+export function disposeOutputs(outputs: Outputs): void {+ for (const tensor of Object.values(outputs)) {+ if (tensor.location === 'gpu-buffer') tensor.dispose();+ }+}diff --git a/packages/sam2/src/shaders.ts b/packages/sam2/src/shaders.tsnew file mode 100644index 00000000..3d30f28c--- /dev/null+++ b/packages/sam2/src/shaders.ts@@ -0,0 +1,80 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE } from './constants';++export const WORKGROUP_SIZE = 16;++/**+ * Resizes a video frame to the model's square input, applies the track's+ * display rotation, and normalizes into planar RGB floats. Frames larger than+ * the input are box-filtered with a few taps per output pixel so downscaling+ * does not alias.+ */+export const PREPROCESS_SHADER = /* wgsl */ `+struct Params {+ size: vec2f,+ rotation: u32,+ taps: u32,+ mean: vec3f,+ _pad0: f32,+ deviation: vec3f,+ _pad1: f32,+};++@group(0) @binding(0) var<uniform> params: Params;+@group(0) @binding(1) var frame: texture_2d<f32>;+@group(0) @binding(2) var frameSampler: sampler;+@group(0) @binding(3) var<storage, read_write> pixels: array<f32>;++const SIZE: u32 = ${IMAGE_SIZE}u;+const PLANE: u32 = SIZE * SIZE;++fn toSource(uv: vec2f) -> vec2f {+ switch params.rotation {+ case 1u: { return vec2f(uv.y, 1.0 - uv.x); }+ case 2u: { return vec2f(1.0 - uv.x, 1.0 - uv.y); }+ case 3u: { return vec2f(1.0 - uv.y, uv.x); }+ default: { return uv; }+ }+}++@compute @workgroup_size(${WORKGROUP_SIZE}, ${WORKGROUP_SIZE})+fn main(@builtin(global_invocation_id) id: vec3u) {+ if (id.x >= SIZE || id.y >= SIZE) { return; }++ let taps = max(params.taps, 1u);+ var rgb = vec3f(0.0);+ for (var y = 0u; y < taps; y++) {+ for (var x = 0u; x < taps; x++) {+ let offset = (vec2f(f32(x), f32(y)) + 0.5) / f32(taps);+ let uv = (vec2f(id.xy) + offset) / f32(SIZE);+ rgb += textureSampleLevel(frame, frameSampler, toSource(uv), 0.0).rgb;+ }+ }+ rgb = (rgb / f32(taps * taps) - params.mean) / params.deviation;++ let index = id.y * SIZE + id.x;+ pixels[index] = rgb.r;+ pixels[PLANE + index] = rgb.g;+ pixels[2u * PLANE + index] = rgb.b;+}+`;++/** Turns a [C, H, W] feature map into the [H*W, C] token layout memory attention takes. */+export const TRANSPOSE_SHADER = /* wgsl */ `+@group(0) @binding(0) var<storage, read> channels: array<f32>;+@group(0) @binding(1) var<storage, read_write> tokens: array<f32>;++const CHANNELS: u32 = ${HIDDEN_DIM}u;+const TOKENS: u32 = ${FEAT_TOKENS}u;++@compute @workgroup_size(${WORKGROUP_SIZE}, ${WORKGROUP_SIZE})+fn main(@builtin(global_invocation_id) id: vec3u) {+ let token = id.x;+ let channel = id.y;+ if (token >= TOKENS || channel >= CHANNELS) { return; }+ tokens[token * CHANNELS + channel] = channels[channel * TOKENS + token];+}+`;diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.tsnew file mode 100644index 00000000..ac858eab--- /dev/null+++ b/packages/sam2/src/tracker.ts@@ -0,0 +1,165 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import {+ BYTES_PER_FLOAT, FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE, MAX_POINTERS, MEM_DIM, MEMORY_ROWS,+} from './constants';+import { FramePreprocessor, TokenTransposer, createStorageBuffer } from './gpu';+import { MemoryBank } from './memory';+import { packMask } from './mask';+import { disposeOutputs, floatTensor, floats, gpuBuffer, gpuTensor, intTensor } from './sessions';++import type { ModelConstants } from './constants';+import type { Rotation } from './gpu';+import type { Sam2Mask } from './mask';+import type { Outputs, Session, Tensor } from './sessions';++/** A prompt on the displayed frame, in 0..1 of its width and height; label 1 is the object, 0 is background. */+export type Sam2Point = { x: number; y: number; label: 0 | 1 };++export type Sam2Sessions = {+ visionEncoder: Session;+ maskDecoder: Session;+ memoryEncoder: Session;+ memoryAttention: Session;+ pointerTpos: Session;+};++type Prompt = { points: Tensor; labels: Tensor };++/**+ * SAM 2.1 video tracking over one object. `seed` segments it on the prompted+ * frame; `track` follows it through the frames that come after, each one+ * conditioned on the memory of those before. Frames are numbered by the+ * caller, and their distances are what the model's temporal encodings read.+ */+export class Sam2Video {+ private readonly preprocessor: FramePreprocessor;+ private readonly transposer: TokenTransposer;+ private readonly bank: MemoryBank;+ private readonly featureTokens: GPUBuffer;+ private readonly positionTokens: GPUBuffer;++ constructor(+ device: GPUDevice,+ private readonly sessions: Sam2Sessions,+ constants: ModelConstants,+ ) {+ this.preprocessor = new FramePreprocessor(device, constants.image_mean, constants.image_std);+ this.transposer = new TokenTransposer(device);+ this.bank = new MemoryBank(device, constants.memory_temporal_positional_encoding);+ this.featureTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 vision tokens');+ this.positionTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 position tokens');+ }++ /** Segments the object at `points` and makes this the frame every later one is tracked from. */+ async seed(frame: VideoFrame, rotation: Rotation, points: Sam2Point[], index: number): Promise<Sam2Mask> {+ if (points.length === 0) throw new Error('A prompt needs at least one point');++ const prompt: Prompt = {+ points: floatTensor(points.flatMap((point) => [point.x * IMAGE_SIZE, point.y * IMAGE_SIZE]), [1, 1, points.length, 2]),+ labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),+ };++ const vision = await this.encode(frame, rotation);+ try {+ return await this.segment(vision, vision.feats2_no_mem!, prompt, { index, prompted: true });+ } finally {+ disposeOutputs(vision);+ }+ }++ /** Finds the seeded object in a frame `index` steps along from the last one. */+ async track(frame: VideoFrame, rotation: Rotation, index: number, totalFrames: number): Promise<Sam2Mask> {+ const vision = await this.encode(frame, rotation);+ try {+ this.transposer.run(gpuBuffer(vision, 'feats2'), this.featureTokens);+ this.transposer.run(gpuBuffer(vision, 'vision_pos_embed'), this.positionTokens);++ const distances = this.bank.assemble(index, totalFrames);+ const temporal = await this.sessions.pointerTpos.run({ normalized_diffs: floatTensor(distances, [MAX_POINTERS]) });+ this.bank.setPointerPositions(gpuBuffer(temporal, 'pointer_pos'));+ disposeOutputs(temporal);++ const attended = await this.sessions.memoryAttention.run({+ current_vision_features: gpuTensor(this.featureTokens, [FEAT_TOKENS, 1, HIDDEN_DIM]),+ current_vision_position_embeddings: gpuTensor(this.positionTokens, [FEAT_TOKENS, 1, HIDDEN_DIM]),+ memory: gpuTensor(this.bank.memory, [MEMORY_ROWS, 1, MEM_DIM]),+ memory_pos: gpuTensor(this.bank.memoryPos, [MEMORY_ROWS, 1, MEM_DIM]),+ });+ try {+ return await this.segment(vision, attended.conditioned_feats!, paddingPrompt(), { index, prompted: false });+ } finally {+ disposeOutputs(attended);+ }+ } finally {+ disposeOutputs(vision);+ }+ }++ /** Forgets the tracked frames but not the seed, to track away from it in the other direction. */+ rewind(): void {+ this.bank.rewind();+ }++ reset(): void {+ this.bank.reset();+ }++ dispose(): void {+ this.bank.dispose();+ this.preprocessor.dispose();+ this.featureTokens.destroy();+ this.positionTokens.destroy();+ for (const session of Object.values(this.sessions)) void session.release();+ }++ private async encode(frame: VideoFrame, rotation: Rotation): Promise<Outputs> {+ this.preprocessor.run(frame, rotation);+ return this.sessions.visionEncoder.run({+ pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, IMAGE_SIZE, IMAGE_SIZE]),+ });+ }++ /** Decodes the mask, then encodes the frame into memory as the prompted frame or the next tracked one. */+ private async segment(vision: Outputs, conditioned: Tensor, prompt: Prompt, frame: { index: number; prompted: boolean }): Promise<Sam2Mask> {+ const decoded = await this.sessions.maskDecoder.run({+ feats0: vision.feats0!,+ feats1: vision.feats1!,+ feats2_cond: conditioned,+ input_points: prompt.points,+ input_labels: prompt.labels,+ });+ try {+ const score = floats(decoded, 'object_score_logits');+ const memory = await this.sessions.memoryEncoder.run({+ feats2: vision.feats2!,+ high_res_mask: decoded.high_res_mask!,+ object_score_logits: floatTensor(score, [1, 1]),+ binarize: floatTensor([frame.prompted ? 1 : 0], []),+ });+ try {+ if (!this.bank.hasPositions) {+ this.bank.setPositions((await memory.memory_pos!.getData()) as Float32Array);+ }++ const tokens = gpuBuffer(memory, 'memory_tokens');+ const pointer = gpuBuffer(decoded, 'object_pointer');+ if (frame.prompted) this.bank.condition(frame.index, tokens, pointer);+ else this.bank.push(frame.index, tokens, pointer);+ } finally {+ disposeOutputs(memory);+ }++ return packMask(floats(decoded, 'low_res_mask'), score[0]!, floats(decoded, 'iou')[0]!);+ } finally {+ disposeOutputs(decoded);+ }+ }+}++/** What the decoder is prompted with on frames that have no points of their own. */+function paddingPrompt(): Prompt {+ return { points: floatTensor([0, 0], [1, 1, 1, 2]), labels: intTensor([-1], [1, 1, 1]) };+}diff --git a/packages/sam2/src/video.ts b/packages/sam2/src/video.tsnew file mode 100644index 00000000..71049fbb--- /dev/null+++ b/packages/sam2/src/video.ts@@ -0,0 +1,114 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { VideoSampleSink } from 'mediabunny';++import type { InputVideoTrack, VideoSample } from 'mediabunny';+import type { Sam2Mask } from './mask';+import type { Sam2Point, Sam2Video } from './tracker';++export type TrackRequest = {+ track: InputVideoTrack;+ /** Where each frame to segment is in the file, in seconds, ascending. */+ timestamps: number[];+ /** Which of `timestamps` the points were placed on. */+ seedIndex: number;+ points: Sam2Point[];+ signal?: AbortSignal;+ /** Called as each frame's mask is ready, in tracking order: the seed, then forward, then backward. */+ onMask: (index: number, mask: Sam2Mask) => void;+};++/**+ * Frames decoded in reverse are read a batch at a time, in file order, and+ * walked backward: seeking to every frame on its own would decode from its+ * keyframe each time.+ */+const REVERSE_BATCH = 12;++/**+ * Segments the object at the seed and follows it through every frame of the+ * request, away from the seed in both directions. Frames that decode to the+ * same picture as the one before (a source slower than the timestamps) share+ * its mask instead of being run again.+ */+export async function trackObject(model: Sam2Video, request: TrackRequest): Promise<void> {+ const { track, timestamps, seedIndex, points, signal, onMask } = request;+ const sink = new VideoSampleSink(track);+ const rotation = track.rotation;+ const total = timestamps.length;++ const seed = await sink.getSample(timestamps[seedIndex]!);+ if (!seed) throw new Error('The prompted frame could not be decoded');++ const seedTimestamp = seed.timestamp;+ const seedMask = await withVideoFrame(seed, (frame) => model.seed(frame, rotation, points, seedIndex));+ onMask(seedIndex, seedMask);++ let previous = { timestamp: seedTimestamp, mask: seedMask };++ const process = async (sample: VideoSample, index: number) => {+ if (sample.timestamp === previous.timestamp) {+ sample.close();+ onMask(index, previous.mask);+ return;+ }+ const timestamp = sample.timestamp;+ const mask = await withVideoFrame(sample, (frame) => model.track(frame, rotation, index, total));+ previous = { timestamp, mask };+ onMask(index, mask);+ };++ const forward = sink.samplesAtTimestamps(timestamps.slice(seedIndex + 1));+ let index = seedIndex + 1;+ // The next decode is requested before the current frame is processed, so+ // decoding and inference overlap.+ let pending = forward.next();+ try {+ for (;;) {+ const { done, value: sample } = await pending;+ if (done) break;+ pending = forward.next();++ if (signal?.aborted) {+ sample?.close();+ return;+ }+ if (sample) await process(sample, index);+ index++;+ }+ } finally {+ await forward.return();+ }++ model.rewind();+ previous = { timestamp: seedTimestamp, mask: seedMask };++ for (let end = seedIndex; end > 0; end -= REVERSE_BATCH) {+ const start = Math.max(0, end - REVERSE_BATCH);+ const samples: (VideoSample | null)[] = [];+ for await (const sample of sink.samplesAtTimestamps(timestamps.slice(start, end))) samples.push(sample);++ for (let i = samples.length - 1; i >= 0; i--) {+ const sample = samples[i];+ if (!sample) continue;+ if (signal?.aborted) {+ sample.close();+ continue;+ }+ await process(sample, start + i);+ }+ if (signal?.aborted) return;+ }+}++async function withVideoFrame<T>(sample: VideoSample, run: (frame: VideoFrame) => Promise<T>): Promise<T> {+ const frame = sample.toVideoFrame();+ try {+ return await run(frame);+ } finally {+ frame.close();+ sample.close();+ }+}diff --git a/packages/sam2/src/weights.ts b/packages/sam2/src/weights.tsnew file mode 100644index 00000000..c3d9c9d7--- /dev/null+++ b/packages/sam2/src/weights.ts@@ -0,0 +1,104 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { MODEL_FILES, MODEL_REPO } from './constants';++export type ModelFileKey = keyof typeof MODEL_FILES;+export type ModelFiles = Record<ModelFileKey, ArrayBuffer>;+export type DownloadProgress = { loaded: number; total: number };++const DB_NAME = 'diffusion-sam2';+const STORE_NAME = 'files';++const TOTAL_BYTES = Object.values(MODEL_FILES).reduce((sum, file) => sum + file.size, 0);++/**+ * The model files, from the local cache when they have been fetched before+ * and from the Hugging Face repo otherwise. Files are keyed by URL, so a+ * change of repo or revision invalidates them.+ */+export async function fetchModelFiles(+ onProgress?: (progress: DownloadProgress) => void,+ signal?: AbortSignal,+): Promise<ModelFiles> {+ const store = await openFileStore();+ const loaded = new Map<ModelFileKey, number>();+ const report = () => onProgress?.({ loaded: [...loaded.values()].reduce((a, b) => a + b, 0), total: TOTAL_BYTES });++ const entries = await Promise.all(+ (Object.keys(MODEL_FILES) as ModelFileKey[]).map(async (key) => {+ const file = MODEL_FILES[key];+ const url = `${MODEL_REPO}/${file.path}`;++ let blob = await store?.read(url);+ if (!blob) {+ blob = await download(url, signal, (bytes) => {+ loaded.set(key, bytes);+ report();+ });+ await store?.write(url, blob);+ }++ loaded.set(key, file.size);+ report();+ return [key, await blob.arrayBuffer()] as const;+ }),+ );++ return Object.fromEntries(entries) as ModelFiles;+}++async function download(url: string, signal: AbortSignal | undefined, onBytes: (bytes: number) => void): Promise<Blob> {+ const response = await fetch(url, { signal });+ if (!response.ok || !response.body) {+ throw new Error(`Could not fetch ${url} (${response.status})`);+ }++ const reader = response.body.getReader();+ const chunks: BlobPart[] = [];+ let received = 0;++ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ chunks.push(value);+ received += value.byteLength;+ onBytes(received);+ }++ return new Blob(chunks);+}++type FileStore = {+ read(key: string): Promise<Blob | undefined>;+ write(key: string, blob: Blob): Promise<void>;+};++async function openFileStore(): Promise<FileStore | null> {+ if (typeof indexedDB === 'undefined') return null;++ try {+ const db = await new Promise<IDBDatabase>((resolve, reject) => {+ const request = indexedDB.open(DB_NAME, 1);+ request.onupgradeneeded = () => request.result.createObjectStore(STORE_NAME);+ request.onsuccess = () => resolve(request.result);+ request.onerror = () => reject(request.error);+ });++ const run = <T>(mode: IDBTransactionMode, operation: (store: IDBObjectStore) => IDBRequest<T>) =>+ new Promise<T>((resolve, reject) => {+ const request = operation(db.transaction(STORE_NAME, mode).objectStore(STORE_NAME));+ request.onsuccess = () => resolve(request.result);+ request.onerror = () => reject(request.error);+ });++ return {+ read: (key) => run<Blob | undefined>('readonly', (store) => store.get(key)),+ // A cache that cannot be written (quota, private mode) is not a reason to fail the load.+ write: (key, blob) => run('readwrite', (store) => store.put(blob, key)).then(() => undefined, () => undefined),+ };+ } catch {+ return null;+ }+}diff --git a/packages/sam2/tsconfig.json b/packages/sam2/tsconfig.jsonnew file mode 100644index 00000000..c3c72503--- /dev/null+++ b/packages/sam2/tsconfig.json@@ -0,0 +1,18 @@+{+ "extends": "../../tsconfig.base.json",+ "compilerOptions": {+ "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo",+ "erasableSyntaxOnly": false,+ "lib": [+ "es2024",+ "DOM",+ "DOM.Iterable"+ ],+ "types": [+ "@webgpu/types"+ ]+ },+ "include": [+ "src"+ ]+}