The VS Code of video editing, built for agents.
TypeScript
Enhance SAM 2.1 model structure and GPU processing
konstantin-paulus88f74f8bfe5d
diff --git a/packages/sam2/package.json b/packages/sam2/package.jsonindex 09e8f9e8..69e0b303 100644--- a/packages/sam2/package.json+++ b/packages/sam2/package.json@@ -7,7 +7,8 @@"sideEffects": false,"exports": {".": "./src/index.ts",- "./mask": "./src/mask.ts"+ "./mask": "./src/mask.ts",+ "./models": "./src/constants.ts"},"scripts": {"check": "tsc --noEmit"diff --git a/packages/sam2/src/constants.ts b/packages/sam2/src/constants.tsindex 7794c085..9de98894 100644--- a/packages/sam2/src/constants.ts+++ b/packages/sam2/src/constants.ts@@ -2,40 +2,140 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */+export type Sam2ModelId = 'tiny' | 'small' | 'base-plus' | 'large';++export type ModelFileKey = 'constants' | 'visionEncoder' | 'maskDecoder' | 'memoryEncoder' | 'memoryAttention' | 'pointerTpos';+/**- * Made by `scripts/export.py 512 7`: fixed shapes for `IMAGE_SIZE` and- * `MEMORY_FRAMES` below, fp16 compute behind fp32 inputs and outputs.- * Pinned to a commit, which is part of the cache key.+ * A SAM 2.1 export, made by `scripts/export.py <size> <imageSize> 7`: fixed+ * shapes for `imageSize` and `MEMORY_FRAMES`, fp16 compute behind fp32 inputs+ * and outputs. The repo is pinned to a commit, which is part of the cache key+ * and what a mask's recipe names.*/-export const MODEL_REPO = 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e';--export const MODEL_FILES = {- constants: { path: 'constants.json', size: 9_781 },- visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 58_385_353 },- maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_898_805 },- memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_753 },- memoryAttention: { path: 'onnx/memory_attention.onnx', size: 13_029_470 },- pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },-} as const;--/** The model's input, half the resolution SAM 2 was trained at: a quarter of the encoder's work. */-export const IMAGE_SIZE = 512;-/** The decoder's masks, upsampled in-graph from its own `IMAGE_SIZE / 4`. */-export const MASK_SIZE = 256;-export const FEAT_SIZE = IMAGE_SIZE / 16;-export const FEAT_TOKENS = FEAT_SIZE * FEAT_SIZE;+export type Sam2Model = {+ id: Sam2ModelId;+ label: string;+ repo: string;+ imageSize: number;+ files: Record<ModelFileKey, { path: string; size: number }>;+};++/**+ * The models on offer, fastest first. Tiny runs at half the resolution+ * SAM 2 was trained at, a quarter of the encoder's work; the larger ones at+ * the full 1024, for the detail and small objects they are chosen for.+ */+export const SAM2_MODELS: readonly Sam2Model[] = [+ {+ id: 'tiny',+ label: 'Tiny',+ repo: 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e',+ imageSize: 512,+ files: {+ constants: { path: 'constants.json', size: 9_781 },+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 58_385_353 },+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_898_805 },+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_753 },+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 13_029_470 },+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },+ },+ },+ {+ id: 'small',+ label: 'Small',+ repo: 'https://huggingface.co/diffusionstudio/sam2.1-small-video-onnx-fp16/resolve/927ecec6e6ae2727d0af7720eefdb07f2efe4689',+ imageSize: 1024,+ files: {+ constants: { path: 'constants.json', size: 9_820 },+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 83_679_474 },+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_910_345 },+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_884 },+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_175_204 },+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },+ },+ },+ {+ id: 'base-plus',+ label: 'Base+',+ repo: 'https://huggingface.co/diffusionstudio/sam2.1-base-plus-video-onnx-fp16/resolve/386b83e09a774fe2782d86fd1b8cb5aaf718aa9d',+ imageSize: 1024,+ files: {+ constants: { path: 'constants.json', size: 9_857 },+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 155_490_079 },+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_910_345 },+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_648 },+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_169_156 },+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },+ },+ },+ {+ id: 'large',+ label: 'Large',+ repo: 'https://huggingface.co/diffusionstudio/sam2.1-large-video-onnx-fp16/resolve/98dbd222d9b3f75d15fe63a5c2a22b0b0029eea1',+ imageSize: 1024,+ files: {+ constants: { path: 'constants.json', size: 9_752 },+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 447_430_045 },+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_910_364 },+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_886 },+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_175_204 },+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },+ },+ },+];++export const DEFAULT_SAM2_MODEL: Sam2ModelId = 'tiny';++export function sam2Model(id: Sam2ModelId): Sam2Model {+ const model = SAM2_MODELS.find((entry) => entry.id === id);+ if (!model) throw new Error(`There is no SAM 2.1 model ${id}`);+ return model;+}++/** The model a mask's recipe names by its pinned repo; undefined for one no longer offered. */+export function sam2ModelOfRepo(repo: string): Sam2Model | undefined {+ return SAM2_MODELS.find((entry) => entry.repo === repo);+}++/** Bytes to fetch for `model` when none of it is cached. */+export function downloadSize(model: Sam2Model): number {+ return Object.values(model.files).reduce((sum, file) => sum + file.size, 0);+}+export const HIDDEN_DIM = 256;export const MEM_DIM = 64;export const MEMORY_FRAMES = 7;export const MAX_POINTERS = 16;export const POINTER_TOKENS = HIDDEN_DIM / MEM_DIM;-export const MEMORY_ROWS = MEMORY_FRAMES * FEAT_TOKENS + MAX_POINTERS * POINTER_TOKENS;+export const DECODED_MASK_SIZE = 256;export const BYTES_PER_FLOAT = 4;export const MEMORY_ROW_BYTES = MEM_DIM * BYTES_PER_FLOAT;-export const MEMORY_BLOCK_BYTES = FEAT_TOKENS * MEMORY_ROW_BYTES;export const POINTER_BYTES = HIDDEN_DIM * BYTES_PER_FLOAT;+/** The sizes a model's input resolution sets: its grids, its buffers and its shaders. */+export type Sam2Geometry = {+ imageSize: number;+ maskSize: number;+ featSize: number;+ featTokens: number;+ memoryRows: number;+ memoryBlockBytes: number;+};++export function geometryOf(imageSize: number): Sam2Geometry {+ const featSize = imageSize / 16;+ const featTokens = featSize * featSize;+ return {+ imageSize,+ maskSize: imageSize / 4,+ featSize,+ featTokens,+ memoryRows: MEMORY_FRAMES * featTokens + MAX_POINTERS * POINTER_TOKENS,+ memoryBlockBytes: featTokens * MEMORY_ROW_BYTES,+ };+}+/** The values `constants.json` in the model repo carries. */export type ModelConstants = {image_size: number;diff --git a/packages/sam2/src/gpu.ts b/packages/sam2/src/gpu.tsindex 2b4f451a..9b8c57ab 100644--- a/packages/sam2/src/gpu.ts+++ b/packages/sam2/src/gpu.ts@@ -2,8 +2,10 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { BYTES_PER_FLOAT, FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE } from './constants';-import { PREPROCESS_SHADER, TRANSPOSE_SHADER, WORKGROUP_SIZE } from './shaders';+import { BYTES_PER_FLOAT, HIDDEN_DIM } from './constants';+import { WORKGROUP_SIZE, preprocessShader, transposeShader } from './shaders';++import type { Sam2Geometry } from './constants';/** Degrees clockwise a decoded frame is turned for display. */export type Rotation = 0 | 90 | 180 | 270;@@ -47,11 +49,12 @@ export class FramePreprocessor {constructor(private readonly device: GPUDevice,+ private readonly imageSize: number,private readonly mean: readonly number[],private readonly std: readonly number[],) {- this.pixels = createStorageBuffer(device, 3 * IMAGE_SIZE * IMAGE_SIZE * BYTES_PER_FLOAT, 'sam2 pixel_values');- this.pipeline = createPipeline(device, PREPROCESS_SHADER, 'sam2 preprocess');+ this.pixels = createStorageBuffer(device, 3 * imageSize * imageSize * BYTES_PER_FLOAT, 'sam2 pixel_values');+ this.pipeline = createPipeline(device, preprocessShader(imageSize), 'sam2 preprocess');this.sampler = device.createSampler({ magFilter: 'linear', minFilter: 'linear' });this.params = device.createBuffer({label: 'sam2 preprocess params',@@ -72,7 +75,7 @@ export class FramePreprocessor {floats[0] = width;floats[1] = height;uints[2] = rotation / 90;- uints[3] = Math.min(MAX_TAPS, Math.max(1, Math.ceil(Math.max(width, height) / IMAGE_SIZE)));+ uints[3] = Math.min(MAX_TAPS, Math.max(1, Math.ceil(Math.max(width, height) / this.imageSize)));floats.set(this.mean, 4);floats.set(this.std, 8);this.device.queue.writeBuffer(this.params, 0, params);@@ -86,7 +89,7 @@ export class FramePreprocessor {{ binding: 3, resource: { buffer: this.pixels } },],});- dispatch(this.device, this.pipeline, bindGroup, IMAGE_SIZE / WORKGROUP_SIZE, IMAGE_SIZE / WORKGROUP_SIZE);+ dispatch(this.device, this.pipeline, bindGroup, this.imageSize / WORKGROUP_SIZE, this.imageSize / WORKGROUP_SIZE);}private textureFor(width: number, height: number): GPUTexture {@@ -113,12 +116,15 @@ export class FramePreprocessor {export class TokenTransposer {private readonly pipeline: GPUComputePipeline;- constructor(private readonly device: GPUDevice) {- this.pipeline = createPipeline(device, TRANSPOSE_SHADER, 'sam2 transpose');+ constructor(+ private readonly device: GPUDevice,+ private readonly geometry: Sam2Geometry,+ ) {+ this.pipeline = createPipeline(device, transposeShader(geometry.featTokens), 'sam2 transpose');}run(channels: GPUBuffer, tokens: GPUBuffer): void {- const size = FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT;+ const size = this.geometry.featTokens * HIDDEN_DIM * BYTES_PER_FLOAT;const bindGroup = this.device.createBindGroup({layout: this.pipeline.getBindGroupLayout(0),entries: [@@ -126,6 +132,6 @@ export class TokenTransposer {{ binding: 1, resource: { buffer: tokens, size } },],});- dispatch(this.device, this.pipeline, bindGroup, FEAT_TOKENS / WORKGROUP_SIZE, HIDDEN_DIM / WORKGROUP_SIZE);+ dispatch(this.device, this.pipeline, bindGroup, this.geometry.featTokens / WORKGROUP_SIZE, HIDDEN_DIM / WORKGROUP_SIZE);}}diff --git a/packages/sam2/src/index.ts b/packages/sam2/src/index.tsindex 337d766f..bc25c1c7 100644--- a/packages/sam2/src/index.ts+++ b/packages/sam2/src/index.ts@@ -5,9 +5,13 @@export { loadSam2 } from './load';export { Sam2Video } from './tracker';export { holdFrame, trackObject } from './video';-export { MASK_SIZE, maskHas, maskIsEmpty, renderMask } from './mask';+export { cachedSam2Models } from './weights';+export { DEFAULT_SAM2_MODEL, SAM2_MODELS, downloadSize, sam2Model, sam2ModelOfRepo } from './constants';+export { maskIsEmpty, paintMask, paintStroke } from './mask';++export type { Sam2Model, Sam2ModelId } from './constants';export type { Sam2LoadOptions, Sam2Progress } from './load';-export type { Sam2Point } from './tracker';+export type { Sam2Correction, Sam2Point } from './tracker';export type { FrameRequest, TrackRequest } from './video';-export type { Sam2Mask } from './mask';+export type { Sam2Mask, Sam2Stroke } from './mask';diff --git a/packages/sam2/src/load.ts b/packages/sam2/src/load.tsindex 42a7f71a..caee9344 100644--- a/packages/sam2/src/load.ts+++ b/packages/sam2/src/load.ts@@ -2,13 +2,14 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { IMAGE_SIZE, MEMORY_FRAMES } from './constants';+import { DEFAULT_SAM2_MODEL, MEMORY_FRAMES, sam2Model } from './constants';import { readFloat16Initializer } from './onnx';import { configureRuntime, createSession, runtimeDevice } from './sessions';import { Sam2Video } from './tracker';import { fetchModelFiles } from './weights';-import type { ModelConstants } from './constants';+import type { ModelConstants, Sam2Model, Sam2ModelId } from './constants';+import type { Session } from './sessions';export type Sam2Progress =| { phase: 'download'; loaded: number; total: number }@@ -16,40 +17,105 @@ export type Sam2Progress =export type Sam2LoadOptions = {onProgress?: (progress: Sam2Progress) => void;- signal?: AbortSignal;};-let loading: Promise<Sam2Video> | null = null;+type Loading = {+ id: Sam2ModelId;+ model: Promise<Sam2Video>;+ listeners: Set<(progress: Sam2Progress) => void>;+ last: Sam2Progress | null;+ controller: AbortController;+ settled: boolean;+};-/** The model, loaded once per page: the first call fetches and compiles it, later calls share the result. */-export function loadSam2(options: Sam2LoadOptions = {}): Promise<Sam2Video> {- loading ??= load(options).catch((error: unknown) => {- loading = null;- throw error;- });- return loading;+let loaded: Loading | null = null;++/**+ * A model, loaded once per page: the first call fetches and compiles it,+ * later calls for it share the result, and hear its progress from where it+ * stands. One model is loaded at a time, the large one being most of a+ * laptop GPU's memory: asking for another disposes the one before, so a+ * caller must be done with it first, and cancels it while it is still on+ * its way — the files it fetched whole stay cached, the rest are left.+ * A cancelled load rejects with an `AbortError`.+ */+export function loadSam2(id: Sam2ModelId = DEFAULT_SAM2_MODEL, { onProgress }: Sam2LoadOptions = {}): Promise<Sam2Video> {+ if (loaded?.id !== id) loaded = startLoad(id);+ if (onProgress) {+ loaded.listeners.add(onProgress);+ if (loaded.last) onProgress(loaded.last);+ }+ return loaded.model;}-async function load({ onProgress, signal }: Sam2LoadOptions): Promise<Sam2Video> {+function startLoad(id: Sam2ModelId): Loading {+ const previous = loaded;+ if (previous && !previous.settled) previous.controller.abort(new DOMException(`${sam2Model(id).label} was asked for instead`, 'AbortError'));++ const controller = new AbortController();+ const report = (progress: Sam2Progress) => {+ entry.last = progress;+ for (const listener of entry.listeners) listener(progress);+ };+ const model = (async () => {+ await previous?.model.then((model) => model.dispose(), () => undefined);+ return load(sam2Model(id), report, controller.signal);+ })();++ const entry: Loading = {+ id,+ model: model+ .catch((error: unknown) => {+ if (loaded === entry) loaded = null;+ throw error;+ })+ .finally(() => {+ entry.settled = true;+ entry.listeners.clear();+ }),+ listeners: new Set(),+ last: null,+ controller,+ settled: false,+ };+ return entry;+}++async function load(model: Sam2Model, onProgress: (progress: Sam2Progress) => void, signal: AbortSignal): Promise<Sam2Video> {+ signal.throwIfAborted();if (!navigator.gpu) throw new Error('WebGPU is not available in this browser');configureRuntime();- const files = await fetchModelFiles((progress) => onProgress?.({ phase: 'download', ...progress }), signal);+ const files = await fetchModelFiles(model, (progress) => onProgress({ phase: 'download', ...progress }), signal);const constants = JSON.parse(new TextDecoder().decode(files.constants)) as ModelConstants;- if (constants.image_size !== IMAGE_SIZE || constants.memory_frames !== MEMORY_FRAMES) {- throw new Error(`The model is for ${constants.image_size}px and ${constants.memory_frames} memories, the pipeline for ${IMAGE_SIZE}px and ${MEMORY_FRAMES}`);+ if (constants.image_size !== model.imageSize || constants.memory_frames !== MEMORY_FRAMES) {+ throw new Error(`The model is for ${constants.image_size}px and ${constants.memory_frames} memories, the pipeline for ${model.imageSize}px and ${MEMORY_FRAMES}`);}- onProgress?.({ phase: 'compile' });- // The runtime builds one WebGPU session at a time.- const visionEncoder = await createSession(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed'], true);- const maskDecoder = await createSession(files.maskDecoder, ['high_res_mask', 'object_pointer']);- const memoryEncoder = await createSession(files.memoryEncoder, ['memory_tokens', 'memory_pos']);- const memoryAttention = await createSession(files.memoryAttention, ['conditioned_feats'], { enableGraphCapture: true });- const pointerTpos = await createSession(files.pointerTpos, ['pointer_pos']);- const device = await runtimeDevice();-- const positions = readFloat16Initializer(files.visionEncoder, 'pos_embed');+ onProgress({ phase: 'compile' });+ // Compiling takes seconds and a share of the GPU: a model no longer+ // wanted stops at the next graph, and what it built is let go. The runtime+ // builds one WebGPU session at a time.+ const built: Session[] = [];+ const build = async (...args: Parameters<typeof createSession>) => {+ signal.throwIfAborted();+ const session = await createSession(...args);+ built.push(session);+ return session;+ };+ try {+ const visionEncoder = await build(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed'], { enableGraphCapture: true });+ const maskDecoder = await build(files.maskDecoder, ['high_res_mask', 'object_pointer']);+ const memoryEncoder = await build(files.memoryEncoder, ['memory_tokens', 'memory_pos']);+ const memoryAttention = await build(files.memoryAttention, ['conditioned_feats'], { enableGraphCapture: true });+ const pointerTpos = await build(files.pointerTpos, ['pointer_pos']);+ signal.throwIfAborted();+ const device = await runtimeDevice();- return new Sam2Video(device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants, positions);+ const positions = readFloat16Initializer(files.visionEncoder, 'pos_embed');+ return new Sam2Video(model, device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants, positions);+ } catch (error) {+ for (const session of built) void session.release();+ throw error;+ }}diff --git a/packages/sam2/src/mask.ts b/packages/sam2/src/mask.tsindex b5083a24..14d1b7ae 100644--- a/packages/sam2/src/mask.ts+++ b/packages/sam2/src/mask.ts@@ -2,54 +2,195 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { MASK_SIZE } from './constants';+import { DECODED_MASK_SIZE } from './constants';-export { MASK_SIZE };--/** One frame's segmentation, at the model's mask resolution. */+/** One frame's segmentation, as the decoder sees it. */export type Sam2Mask = {- /** Row-major, one bit per pixel, most significant bit first. */- bits: Uint8Array;+ /**+ * The decoder's logits on its own grid, `size` square, row-major: the+ * object is where they are above zero.+ */+ logits: Float32Array;+ /** The grid's side, a quarter of the model's input: set by the model the mask came from. */+ size: number;/** Whether the object is in frame: at or below zero the model saw it occluded. */score: number;/** The model's own estimate of the mask's quality, 0 to 1. */iou: number;};-const MASK_BYTES = (MASK_SIZE * MASK_SIZE) / 8;+/**+ * How sure a mask that is given rather than decoded is, as a logit: of its+ * cells, and of the object being in frame. SAM 2 scales a mask prompt so.+ */+export const GIVEN_LOGIT = 10;-/** Packs mask logits into bits; positive logits are foreground. */-export function packMask(logits: Float32Array, score: number, iou: number): Sam2Mask {- const bits = new Uint8Array(MASK_BYTES);- for (let i = 0; i < logits.length; i++) {- if (logits[i]! > 0) bits[i >> 3]! |= 0x80 >> (i & 7);- }- return { bits, score, iou };+/**+ * The decoder's mask as a `Sam2Mask` on its `size` grid: its logits, which a+ * decoder with a smaller grid than `DECODED_MASK_SIZE` upsampled twice before+ * handing them over, brought back to its own grid — exactly, since the+ * upsampling is linear and invertible.+ */+export function packMask(decoded: Float32Array, size: number, score: number, iou: number): Sam2Mask {+ if (size === DECODED_MASK_SIZE) return { logits: decoded.slice(), size, score, iou };+ if (size * 2 !== DECODED_MASK_SIZE) throw new Error(`A ${size} grid is not upsampled to ${DECODED_MASK_SIZE} twice`);+ return { logits: downsample(decoded, size), size, score, iou };}-export function maskHas(mask: Sam2Mask, x: number, y: number): boolean {- const i = y * MASK_SIZE + x;- return (mask.bits[i >> 3]! & (0x80 >> (i & 7))) !== 0;+/** `mask` with nothing in it. */+export function clearMask(mask: Sam2Mask): Sam2Mask {+ return { ...mask, logits: new Float32Array(mask.logits.length).fill(-GIVEN_LOGIT) };}-/** Whether any pixel is foreground. */+/** Whether no cell is in. */export function maskIsEmpty(mask: Sam2Mask): boolean {- return mask.bits.every((byte) => byte === 0);+ return mask.logits.every((value) => value <= 0);}-/** The mask as RGBA pixels, `color` where it is foreground and transparent elsewhere. */-export function renderMask(mask: Sam2Mask, color: [r: number, g: number, b: number, a: number]): ImageData {- const image = new ImageData(MASK_SIZE, MASK_SIZE);- const pixels = new Uint32Array(image.data.buffer);- const packed = (color[3] << 24 | color[2] << 16 | color[1] << 8 | color[0]) >>> 0;+// ── The decoder's upsampling, undone ─────────────────────────- for (let byte = 0; byte < mask.bits.length; byte++) {- const value = mask.bits[byte]!;- if (value === 0) continue;- for (let bit = 0; bit < 8; bit++) {- if (value & (0x80 >> bit)) pixels[byte * 8 + bit] = packed;- }+// The decoder upsamples its grid twice, bilinear with cell centers aligned+// and the border held: of each pair of new cells, one is three quarters the+// old cell and a quarter the one before it, the other three quarters the old+// cell and a quarter the one after. So a pair sums to a quarter of each+// neighbor and one and a half of the cell — one and three quarters at the+// ends — a tridiagonal system, solved along each row and then each column.++const NEIGHBOR = 0.25;++type Thomas = { upper: Float64Array; pivot: Float64Array };+const thomasBySize = new Map<number, Thomas>();++/** The Thomas algorithm's forward coefficients for a line of `n` cells, the same for every line. */+function thomas(n: number): Thomas {+ let coefficients = thomasBySize.get(n);+ if (coefficients) return coefficients;++ const upper = new Float64Array(n);+ const pivot = new Float64Array(n);+ for (let i = 0; i < n; i++) {+ const diagonal = i === 0 || i === n - 1 ? 1.75 : 1.5;+ pivot[i] = diagonal - (i > 0 ? NEIGHBOR * upper[i - 1]! : 0);+ upper[i] = NEIGHBOR / pivot[i]!;+ }+ coefficients = { upper, pivot };+ thomasBySize.set(n, coefficients);+ return coefficients;+}++/** Solves one line in place: `line` holds the pair sums, and then the cells. */+function solveLine(line: Float64Array): void {+ const n = line.length;+ const { upper, pivot } = thomas(n);+ for (let i = 0; i < n; i++) line[i] = (line[i]! - (i > 0 ? NEIGHBOR * line[i - 1]! : 0)) / pivot[i]!;+ for (let i = n - 2; i >= 0; i--) line[i] = line[i]! - upper[i]! * line[i + 1]!;+}++/** The decoder's `DECODED_MASK_SIZE` logits back on its own grid, `n` square. */+function downsample(decoded: Float32Array, n: number): Float32Array {+ const size = DECODED_MASK_SIZE;+ const line = new Float64Array(n);++ const rows = new Float64Array(size * n);+ for (let y = 0; y < size; y++) {+ for (let i = 0; i < n; i++) line[i] = decoded[y * size + 2 * i]! + decoded[y * size + 2 * i + 1]!;+ solveLine(line);+ rows.set(line, y * n);+ }++ const out = new Float32Array(n * n);+ for (let x = 0; x < n; x++) {+ for (let i = 0; i < n; i++) line[i] = rows[2 * i * n + x]! + rows[(2 * i + 1) * n + x]!;+ solveLine(line);+ for (let i = 0; i < n; i++) out[i * n + x] = line[i]!;}+ return out;+}++/**+ * A brush stroke over a mask: its points in 0..1 of the frame's width and+ * height, its radius in 0..1 of the frame's height, so the brush is round on+ * the footage however the square grid is stretched over it. Label 1 adds to+ * the mask, 0 erases.+ */+export type Sam2Stroke = {+ label: 0 | 1;+ radius: number;+ points: { x: number; y: number }[];+};++/**+ * How steeply a stroke's logits rise across its edge, per cell: about as+ * steep as the model's own edges, so a stroke's edge falls between cells+ * where the brush's does. They level off at `GIVEN_LOGIT`.+ */+const BRUSH_SLOPE = 4;- return image;+/** How far past its radius a stroke reaches, in cells: to where its logits have levelled off. */+const BRUSH_REACH = GIVEN_LOGIT / BRUSH_SLOPE;++/** `mask` with `strokes` painted over it in order; `aspect` is the frame's width over its height. */+export function paintMask(mask: Sam2Mask, strokes: readonly Sam2Stroke[], aspect: number): Sam2Mask {+ if (strokes.length === 0) return mask;+ const painted = { ...mask, logits: mask.logits.slice() };+ for (const stroke of strokes) paintStroke(painted, stroke, aspect);+ return painted;+}++/**+ * Paints `stroke` into `mask`'s logits, in place, from its point `from` on: a+ * disc at each point and a capsule along each segment, so a fast stroke+ * leaves no gaps. Painting from the last point painted is how a stroke is+ * drawn as it grows.+ */+export function paintStroke(mask: Sam2Mask, stroke: Sam2Stroke, aspect: number, from = 0): void {+ const { points, radius, label } = stroke;+ for (let i = Math.max(0, from); i < points.length; i++) {+ paintSegment(mask.logits, mask.size, points[Math.max(0, i - 1)]!, points[i]!, radius, aspect, label);+ }+}++/**+ * Raises the logits under a segment's capsule, or lowers them to erase: each+ * cell to its distance inside the capsule's edge, measured on the frame, as a+ * logit, where that is further in than the cell already is. The edge is then+ * where the capsule's is, to a fraction of a cell, and the model's own edge+ * stands where the stroke does not reach it.+ */+function paintSegment(+ logits: Float32Array,+ size: number,+ a: { x: number; y: number },+ b: { x: number; y: number },+ radius: number,+ aspect: number,+ label: 0 | 1,+): void {+ // Distances are taken in frame heights: x is scaled by the aspect so the brush is round.+ const au = a.x * aspect;+ const bu = b.x * aspect;+ const du = bu - au;+ const dv = b.y - a.y;+ const length2 = du * du + dv * dv;+ const reach = radius + BRUSH_REACH / size;++ const cell = (value: number) => Math.floor(value * size);+ const x0 = Math.max(0, cell(Math.min(a.x, b.x) - reach / aspect));+ const x1 = Math.min(size - 1, cell(Math.max(a.x, b.x) + reach / aspect));+ const y0 = Math.max(0, cell(Math.min(a.y, b.y) - reach));+ const y1 = Math.min(size - 1, cell(Math.max(a.y, b.y) + reach));++ for (let y = y0; y <= y1; y++) {+ const v = (y + 0.5) / size;+ for (let x = x0; x <= x1; x++) {+ const u = ((x + 0.5) / size) * aspect;+ const t = length2 === 0 ? 0 : Math.min(1, Math.max(0, ((u - au) * du + (v - a.y) * dv) / length2));+ const inside = radius - Math.hypot(u - (au + t * du), v - (a.y + t * dv));+ if (inside < radius - reach) continue;++ const logit = Math.min(GIVEN_LOGIT, Math.max(-GIVEN_LOGIT, inside * size * BRUSH_SLOPE));+ const i = y * size + x;+ logits[i] = label === 1 ? Math.max(logits[i]!, logit) : Math.min(logits[i]!, -logit);+ }+ }}diff --git a/packages/sam2/src/memory.ts b/packages/sam2/src/memory.tsindex 1ad3d757..af62adbb 100644--- a/packages/sam2/src/memory.ts+++ b/packages/sam2/src/memory.ts@@ -2,12 +2,11 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import {- MAX_POINTERS, MEM_DIM, MEMORY_BLOCK_BYTES, MEMORY_FRAMES, MEMORY_ROW_BYTES, MEMORY_ROWS,- POINTER_BYTES, POINTER_TOKENS,-} from './constants';+import { MAX_POINTERS, MEM_DIM, MEMORY_FRAMES, MEMORY_ROW_BYTES, POINTER_BYTES, POINTER_TOKENS } from './constants';import { createStorageBuffer } from './gpu';+import type { Sam2Geometry } from './constants';+type FrameMemory = {index: number;tokens: GPUBuffer;@@ -27,13 +26,19 @@ export class MemoryBank {private conditioning: FrameMemory | null = null;private recent: FrameMemory[] = [];private readonly pool: FrameMemory[] = [];+ /** A frame memory's bytes; the object pointers follow the frames'. */+ private readonly blockBytes: number;+ private readonly pointerOffset: number;constructor(private readonly device: GPUDevice,+ geometry: Sam2Geometry,private readonly temporalEncoding: readonly (readonly number[])[],) {- this.memory = createStorageBuffer(device, MEMORY_ROWS * MEMORY_ROW_BYTES, 'sam2 memory');- this.memoryPos = createStorageBuffer(device, MEMORY_ROWS * MEMORY_ROW_BYTES, 'sam2 memory_pos');+ this.blockBytes = geometry.memoryBlockBytes;+ this.pointerOffset = MEMORY_FRAMES * geometry.memoryBlockBytes;+ this.memory = createStorageBuffer(device, geometry.memoryRows * MEMORY_ROW_BYTES, 'sam2 memory');+ this.memoryPos = createStorageBuffer(device, geometry.memoryRows * MEMORY_ROW_BYTES, 'sam2 memory_pos');}get hasPositions(): boolean {@@ -102,11 +107,11 @@ export class MemoryBank {const encoder = this.device.createCommandEncoder();blocks.forEach((block, slot) => {- encoder.copyBufferToBuffer(block.tokens, 0, this.memory, slot * MEMORY_BLOCK_BYTES, MEMORY_BLOCK_BYTES);- encoder.copyBufferToBuffer(block.pos, 0, this.memoryPos, slot * MEMORY_BLOCK_BYTES, MEMORY_BLOCK_BYTES);+ encoder.copyBufferToBuffer(block.tokens, 0, this.memory, slot * this.blockBytes, this.blockBytes);+ encoder.copyBufferToBuffer(block.pos, 0, this.memoryPos, slot * this.blockBytes, this.blockBytes);});pointers.forEach((entry, i) => {- encoder.copyBufferToBuffer(entry.pointer, 0, this.memory, POINTER_OFFSET + i * POINTER_BYTES, POINTER_BYTES);+ encoder.copyBufferToBuffer(entry.pointer, 0, this.memory, this.pointerOffset + i * POINTER_BYTES, POINTER_BYTES);});this.device.queue.submit([encoder.finish()]);@@ -120,7 +125,7 @@ export class MemoryBank {for (let i = 0; i < MAX_POINTERS; i++) {for (let token = 0; token < POINTER_TOKENS; token++) {const row = i * POINTER_TOKENS + token;- encoder.copyBufferToBuffer(positions, i * MEMORY_ROW_BYTES, this.memoryPos, POINTER_OFFSET + row * MEMORY_ROW_BYTES, MEMORY_ROW_BYTES);+ encoder.copyBufferToBuffer(positions, i * MEMORY_ROW_BYTES, this.memoryPos, this.pointerOffset + row * MEMORY_ROW_BYTES, MEMORY_ROW_BYTES);}}this.device.queue.submit([encoder.finish()]);@@ -141,17 +146,15 @@ export class MemoryBank {private retain(index: number, tokens: GPUBuffer, pointer: GPUBuffer): FrameMemory {const entry = this.pool.pop() ?? {index,- tokens: createStorageBuffer(this.device, MEMORY_BLOCK_BYTES, 'sam2 frame memory'),+ tokens: createStorageBuffer(this.device, this.blockBytes, 'sam2 frame memory'),pointer: createStorageBuffer(this.device, POINTER_BYTES, 'sam2 frame pointer'),};entry.index = index;const encoder = this.device.createCommandEncoder();- encoder.copyBufferToBuffer(tokens, 0, entry.tokens, 0, MEMORY_BLOCK_BYTES);+ encoder.copyBufferToBuffer(tokens, 0, entry.tokens, 0, this.blockBytes);encoder.copyBufferToBuffer(pointer, 0, entry.pointer, 0, POINTER_BYTES);this.device.queue.submit([encoder.finish()]);return entry;}}--const POINTER_OFFSET = MEMORY_FRAMES * MEMORY_BLOCK_BYTES;diff --git a/packages/sam2/src/shaders.ts b/packages/sam2/src/shaders.tsindex 3d30f28c..d1dc4769 100644--- a/packages/sam2/src/shaders.ts+++ b/packages/sam2/src/shaders.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE } from './constants';+import { HIDDEN_DIM } from './constants';export const WORKGROUP_SIZE = 16;@@ -12,7 +12,7 @@ export const WORKGROUP_SIZE = 16;* the input are box-filtered with a few taps per output pixel so downscaling* does not alias.*/-export const PREPROCESS_SHADER = /* wgsl */ `+export const preprocessShader = (imageSize: number) => /* wgsl */ `struct Params {size: vec2f,rotation: u32,@@ -28,7 +28,7 @@ struct Params {@group(0) @binding(2) var frameSampler: sampler;@group(0) @binding(3) var<storage, read_write> pixels: array<f32>;-const SIZE: u32 = ${IMAGE_SIZE}u;+const SIZE: u32 = ${imageSize}u;const PLANE: u32 = SIZE * SIZE;fn toSource(uv: vec2f) -> vec2f {@@ -63,12 +63,12 @@ fn main(@builtin(global_invocation_id) id: vec3u) {`;/** Turns a [C, H, W] feature map into the [H*W, C] token layout memory attention takes. */-export const TRANSPOSE_SHADER = /* wgsl */ `+export const transposeShader = (featTokens: number) => /* wgsl */ `@group(0) @binding(0) var<storage, read> channels: array<f32>;@group(0) @binding(1) var<storage, read_write> tokens: array<f32>;const CHANNELS: u32 = ${HIDDEN_DIM}u;-const TOKENS: u32 = ${FEAT_TOKENS}u;+const TOKENS: u32 = ${featTokens}u;@compute @workgroup_size(${WORKGROUP_SIZE}, ${WORKGROUP_SIZE})fn main(@builtin(global_invocation_id) id: vec3u) {diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.tsindex 49efae6a..7c604d3f 100644--- a/packages/sam2/src/tracker.ts+++ b/packages/sam2/src/tracker.ts@@ -2,15 +2,13 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import {- BYTES_PER_FLOAT, FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE, MAX_POINTERS, MEM_DIM, MEMORY_ROWS,-} from './constants';+import { BYTES_PER_FLOAT, HIDDEN_DIM, MAX_POINTERS, MEM_DIM, geometryOf } from './constants';import { FramePreprocessor, TokenTransposer, createStorageBuffer } from './gpu';import { MemoryBank } from './memory';-import { packMask } from './mask';+import { GIVEN_LOGIT, clearMask, maskIsEmpty, packMask } from './mask';import { disposeOutputs, floatTensor, floats, gpuBuffer, gpuTensor, intTensor } from './sessions';-import type { ModelConstants } from './constants';+import type { ModelConstants, Sam2Geometry, Sam2Model } from './constants';import type { Rotation } from './gpu';import type { Sam2Mask } from './mask';import type { Outputs, Session, Tensor } from './sessions';@@ -18,6 +16,13 @@ import type { Outputs, Session, Tensor } from './sessions';/** A prompt on the displayed frame, in 0..1 of its width and height; label 1 is the object, 0 is background. */export type Sam2Point = { x: number; y: number; label: 0 | 1 };+/**+ * The prompted frame's mask as the user corrected it, from the model's own:+ * what the frame is remembered as, and so what every tracked frame follows.+ * Returning the mask it was given leaves the model's.+ */+export type Sam2Correction = (mask: Sam2Mask) => Sam2Mask;+export type Sam2Sessions = {visionEncoder: Session;maskDecoder: Session;@@ -28,6 +33,8 @@ export type Sam2Sessions = {type Prompt = { points: Tensor; labels: Tensor };+type SegmentedFrame = { index: number; prompted: boolean; correct?: Sam2Correction };+/*** SAM 2.1 video tracking over one object. `seed` segments it on the prompted* frame; `track` follows it through the frames that come after, each one@@ -35,6 +42,7 @@ type Prompt = { points: Tensor; labels: Tensor };* caller, and their distances are what the model's temporal encodings read.*/export class Sam2Video {+ readonly geometry: Sam2Geometry;private readonly preprocessor: FramePreprocessor;private readonly transposer: TokenTransposer;private readonly bank: MemoryBank;@@ -43,26 +51,36 @@ export class Sam2Video {/** The frame kept encoded for `preview` and `seedHeld`, so a prompt costs the decoder alone. */private held: Outputs | null = null;- /** `positions` is the vision encoder's `pos_embed`, [HIDDEN_DIM, FEAT_TOKENS]. */+ /** `positions` is the vision encoder's `pos_embed`, [HIDDEN_DIM, featTokens]. */constructor(+ readonly model: Sam2Model,device: GPUDevice,private readonly sessions: Sam2Sessions,constants: ModelConstants,positions: Float32Array,) {- this.preprocessor = new FramePreprocessor(device, constants.image_mean, constants.image_std);- this.transposer = new TokenTransposer(device);- this.bank = new MemoryBank(device, constants.memory_temporal_positional_encoding);- this.featureTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 vision tokens');- this.positionTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 position tokens');- device.queue.writeBuffer(this.positionTokens, 0, toTokens(positions));+ const geometry = (this.geometry = geometryOf(model.imageSize));+ this.preprocessor = new FramePreprocessor(device, geometry.imageSize, constants.image_mean, constants.image_std);+ this.transposer = new TokenTransposer(device, geometry);+ this.bank = new MemoryBank(device, geometry, constants.memory_temporal_positional_encoding);+ this.featureTokens = createStorageBuffer(device, geometry.featTokens * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 vision tokens');+ this.positionTokens = createStorageBuffer(device, geometry.featTokens * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 position tokens');+ device.queue.writeBuffer(this.positionTokens, 0, toTokens(positions, geometry.featTokens));+ }++ /** The side of its masks' grid. */+ get maskSize(): number {+ return this.geometry.maskSize;}- /** Segments the object at `points` and makes this the frame every later one is tracked from. */- async seed(frame: VideoFrame, rotation: Rotation, points: Sam2Point[], index: number): Promise<Sam2Mask> {+ /**+ * Segments the object at `points` and makes this the frame every later one+ * is tracked from, as `correct` has it when given.+ */+ async seed(frame: VideoFrame, rotation: Rotation, points: Sam2Point[], index: number, correct?: Sam2Correction): Promise<Sam2Mask> {const vision = await this.encode(frame, rotation);try {- return await this.segment(vision, vision.feats2_no_mem!, pointPrompt(points), { index, prompted: true });+ return await this.segment(vision, vision.feats2_no_mem!, this.pointPrompt(points), { index, prompted: true, correct });} finally {disposeOutputs(vision);}@@ -90,7 +108,7 @@ export class Sam2Video {/** Segments the held frame at `points` without remembering it: what a click there would pick. */async preview(points: Sam2Point[]): Promise<Sam2Mask> {const vision = this.heldFrame();- const prompt = pointPrompt(points);+ const prompt = this.pointPrompt(points);const decoded = await this.sessions.maskDecoder.run({feats0: vision.feats0!,feats1: vision.feats1!,@@ -99,16 +117,16 @@ export class Sam2Video {input_labels: prompt.labels,});try {- return packMask(floats(decoded, 'low_res_mask'), floats(decoded, 'object_score_logits')[0]!, floats(decoded, 'iou')[0]!);+ return this.unpack(decoded);} finally {disposeOutputs(decoded);}}/** `seed` on the held frame. */- seedHeld(points: Sam2Point[], index: number): Promise<Sam2Mask> {+ seedHeld(points: Sam2Point[], index: number, correct?: Sam2Correction): Promise<Sam2Mask> {const vision = this.heldFrame();- return this.segment(vision, vision.feats2_no_mem!, pointPrompt(points), { index, prompted: true });+ return this.segment(vision, vision.feats2_no_mem!, this.pointPrompt(points), { index, prompted: true, correct });}private heldFrame(): Outputs {@@ -127,11 +145,12 @@ export class Sam2Video {this.bank.setPointerPositions(gpuBuffer(temporal, 'pointer_pos'));disposeOutputs(temporal);+ const { featTokens, memoryRows } = this.geometry;const attended = await this.sessions.memoryAttention.run({- current_vision_features: gpuTensor(this.featureTokens, [FEAT_TOKENS, 1, HIDDEN_DIM]),- current_vision_position_embeddings: gpuTensor(this.positionTokens, [FEAT_TOKENS, 1, HIDDEN_DIM]),- memory: gpuTensor(this.bank.memory, [MEMORY_ROWS, 1, MEM_DIM]),- memory_pos: gpuTensor(this.bank.memoryPos, [MEMORY_ROWS, 1, MEM_DIM]),+ current_vision_features: gpuTensor(this.featureTokens, [featTokens, 1, HIDDEN_DIM]),+ current_vision_position_embeddings: gpuTensor(this.positionTokens, [featTokens, 1, HIDDEN_DIM]),+ memory: gpuTensor(this.bank.memory, [memoryRows, 1, MEM_DIM]),+ memory_pos: gpuTensor(this.bank.memoryPos, [memoryRows, 1, MEM_DIM]),});try {return await this.segment(vision, attended.conditioned_feats!, paddingPrompt(), { index, prompted: false });@@ -165,8 +184,9 @@ export class Sam2Video {private async encode(frame: VideoFrame, rotation: Rotation): Promise<Outputs> {this.release();this.preprocessor.run(frame, rotation);+ const size = this.geometry.imageSize;return this.sessions.visionEncoder.run(- { pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, IMAGE_SIZE, IMAGE_SIZE]) },+ { pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, size, size]) },VISION_OUTPUTS,);}@@ -177,7 +197,7 @@ export class Sam2Video {* not remembered: a wrong guess kept in memory is what the next frames* would match, so it would be reinforced rather than corrected.*/- private async segment(vision: Outputs, conditioned: Tensor, prompt: Prompt, frame: { index: number; prompted: boolean }): Promise<Sam2Mask> {+ private async segment(vision: Outputs, conditioned: Tensor, prompt: Prompt, frame: SegmentedFrame): Promise<Sam2Mask> {const decoded = await this.sessions.maskDecoder.run({feats0: vision.feats0!,feats1: vision.feats1!,@@ -186,26 +206,37 @@ export class Sam2Video {input_labels: prompt.labels,});try {- const score = floats(decoded, 'object_score_logits');- const mask = packMask(floats(decoded, 'low_res_mask'), score[0]!, floats(decoded, 'iou')[0]!);+ const mask = this.unpack(decoded);if (frame.prompted) {- await this.remember(vision, decoded, frame.index, true);- return mask;+ const corrected = frame.correct?.(mask) ?? mask;+ if (corrected === mask) {+ await this.remember(vision, decoded, frame.index, true);+ return mask;+ }+ // As SAM 2 takes a mask prompt: the object is there exactly where the mask says.+ const given = { ...corrected, score: maskIsEmpty(corrected) ? -GIVEN_LOGIT : GIVEN_LOGIT };+ await this.remember(vision, decoded, frame.index, true, given);+ return given;}if (isReliable(mask)) await this.remember(vision, decoded, frame.index, false);- return isPlausible(mask) ? mask : { ...mask, bits: new Uint8Array(mask.bits.length) };+ return isPlausible(mask) ? mask : clearMask(mask);} finally {disposeOutputs(decoded);}}- /** Encodes the frame's mask into memory, as the prompted frame or the next tracked one. */- private async remember(vision: Outputs, decoded: Outputs, index: number, prompted: boolean): Promise<void> {+ /**+ * Encodes the frame's mask into memory, as the prompted frame or the next+ * tracked one: the decoder's, or `given` in its place. A given mask keeps+ * the decoder's object pointer, which the points it was corrected from made.+ */+ private async remember(vision: Outputs, decoded: Outputs, index: number, prompted: boolean, given?: Sam2Mask): Promise<void> {+ const size = this.geometry.imageSize;const memory = await this.sessions.memoryEncoder.run({feats2: vision.feats2!,- high_res_mask: decoded.high_res_mask!,- object_score_logits: floatTensor(floats(decoded, 'object_score_logits'), [1, 1]),+ high_res_mask: given ? floatTensor(maskLogits(given, size), [1, 1, size, size]) : decoded.high_res_mask!,+ object_score_logits: floatTensor(given ? [given.score] : floats(decoded, 'object_score_logits'), [1, 1]),binarize: floatTensor([prompted ? 1 : 0], []),});try {@@ -221,6 +252,19 @@ export class Sam2Video {disposeOutputs(memory);}}++ private unpack(decoded: Outputs): Sam2Mask {+ return packMask(floats(decoded, 'low_res_mask'), this.geometry.maskSize, floats(decoded, 'object_score_logits')[0]!, floats(decoded, 'iou')[0]!);+ }++ private pointPrompt(points: Sam2Point[]): Prompt {+ if (points.length === 0) throw new Error('A prompt needs at least one point');+ const size = this.geometry.imageSize;+ return {+ points: floatTensor(points.flatMap((point) => [point.x * size, point.y * size]), [1, 1, points.length, 2]),+ labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),+ };+ }}/**@@ -231,10 +275,10 @@ export class Sam2Video {const VISION_OUTPUTS = ['feats0', 'feats1', 'feats2', 'feats2_no_mem'];/** Rewrites [C, N] channels as the [N, 1, C] tokens memory attention reads. */-function toTokens(channels: Float32Array): Float32Array<ArrayBuffer> {+function toTokens(channels: Float32Array, count: number): Float32Array<ArrayBuffer> {const tokens = new Float32Array(channels.length);for (let c = 0; c < HIDDEN_DIM; c++) {- for (let n = 0; n < FEAT_TOKENS; n++) tokens[n * HIDDEN_DIM + c] = channels[c * FEAT_TOKENS + n]!;+ for (let n = 0; n < count; n++) tokens[n * HIDDEN_DIM + c] = channels[c * count + n]!;}return tokens;}@@ -255,6 +299,42 @@ const RELIABLE_IOU = 0.25;*/const PLAUSIBLE_IOU = 0.15;+/**+ * A mask's logits as the decoder's `high_res_mask`: upsampled to the+ * image as the decoder upsamples its own, bilinear with cell centers aligned.+ * A prompted frame's memory is made from where they are above zero.+ */+function maskLogits(mask: Sam2Mask, imageSize: number): Float32Array {+ // Each image row or column reads two cells, at a weight for the second.+ const scale = mask.size / imageSize;+ const lower = new Int32Array(imageSize);+ const upper = new Int32Array(imageSize);+ const weight = new Float32Array(imageSize);+ for (let o = 0; o < imageSize; o++) {+ const source = Math.max(0, (o + 0.5) * scale - 0.5);+ lower[o] = Math.floor(source);+ upper[o] = Math.min(lower[o]! + 1, mask.size - 1);+ weight[o] = source - lower[o]!;+ }++ const at = (x: number, y: number) => mask.logits[y * mask.size + x]!;+ const logits = new Float32Array(imageSize * imageSize);+ for (let y = 0; y < imageSize; y++) {+ const y0 = lower[y]!;+ const y1 = upper[y]!;+ const wy = weight[y]!;+ for (let x = 0; x < imageSize; x++) {+ const x0 = lower[x]!;+ const x1 = upper[x]!;+ const wx = weight[x]!;+ const top = at(x0, y0) * (1 - wx) + at(x1, y0) * wx;+ const bottom = at(x0, y1) * (1 - wx) + at(x1, y1) * wx;+ logits[y * imageSize + x] = top * (1 - wy) + bottom * wy;+ }+ }+ return logits;+}+function isReliable(mask: Sam2Mask): boolean {return mask.score > 0 && mask.iou >= RELIABLE_IOU;}@@ -263,14 +343,6 @@ function isPlausible(mask: Sam2Mask): boolean {return mask.iou >= PLAUSIBLE_IOU;}-function pointPrompt(points: Sam2Point[]): Prompt {- if (points.length === 0) throw new Error('A prompt needs at least one point');- return {- points: floatTensor(points.flatMap((point) => [point.x * IMAGE_SIZE, point.y * IMAGE_SIZE]), [1, 1, points.length, 2]),- labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),- };-}-/** What the decoder is prompted with on frames that have no points of their own. */function paddingPrompt(): Prompt {return { points: floatTensor([0, 0], [1, 1, 1, 2]), labels: intTensor([-1], [1, 1, 1]) };diff --git a/packages/sam2/src/weights.ts b/packages/sam2/src/weights.tsindex 56092ed8..60baa57c 100644--- a/packages/sam2/src/weights.ts+++ b/packages/sam2/src/weights.ts@@ -2,111 +2,220 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { MODEL_FILES, MODEL_REPO } from './constants';+import { SAM2_MODELS, downloadSize } from './constants';++import type { ModelFileKey, Sam2Model, Sam2ModelId } from './constants';-export type ModelFileKey = keyof typeof MODEL_FILES;export type ModelFiles = Record<ModelFileKey, ArrayBuffer>;export type DownloadProgress = { loaded: number; total: number };-const DB_NAME = 'diffusion-sam2';-const STORE_NAME = 'files';--const TOTAL_BYTES = Object.values(MODEL_FILES).reduce((sum, file) => sum + file.size, 0);+/**+ * The folder in the origin private file system models are kept in, shared+ * with whatever else downloads models: a folder a repo revision, named by+ * `folderName`, so each keeps to its own repos.+ */+const ROOT = 'models';/**- * The model files, from the local cache when they have been fetched before- * and from the Hugging Face repo otherwise. Files are keyed by URL, so a- * change of repo or revision invalidates them.+ * A model's files, from the local cache when they have been fetched before+ * and from the Hugging Face repo otherwise. Downloads stream to disk as they+ * arrive, so a file is in memory once, when it is read back; one cut short,+ * by `signal` or the network, keeps what arrived and resumes from there. A+ * file is whole when it has the size the catalogue gives it. Without a+ * private file system, files are fetched into memory and not kept.*/export async function fetchModelFiles(+ model: Sam2Model,onProgress?: (progress: DownloadProgress) => void,signal?: AbortSignal,): Promise<ModelFiles> {- const store = await openFileStore();- await store?.retain(Object.values(MODEL_FILES).map((file) => `${MODEL_REPO}/${file.path}`));+ const root = await openRoot();+ if (root) await retain(root);+ const folder = root && (await root.getDirectoryHandle(folderName(model), { create: true }).catch(() => null));++ const total = downloadSize(model);const loaded = new Map<ModelFileKey, number>();- const report = () => onProgress?.({ loaded: [...loaded.values()].reduce((a, b) => a + b, 0), total: TOTAL_BYTES });+ const report = () => onProgress?.({ loaded: [...loaded.values()].reduce((a, b) => a + b, 0), total });const entries = await Promise.all(- (Object.keys(MODEL_FILES) as ModelFileKey[]).map(async (key) => {- const file = MODEL_FILES[key];- const url = `${MODEL_REPO}/${file.path}`;-- let blob = await store?.read(url);- if (!blob) {- blob = await download(url, signal, (bytes) => {- loaded.set(key, bytes);- report();- });- await store?.write(url, blob);+ (Object.keys(model.files) as ModelFileKey[]).map(async (key) => {+ const { path, size } = model.files[key];+ const url = `${model.repo}/${path}`;+ const onBytes = (bytes: number) => {+ loaded.set(key, bytes);+ report();+ };++ let bytes: ArrayBuffer | null = null;+ if (folder) {+ try {+ bytes = await fetchStored(folder, fileName(path), url, size, onBytes, signal);+ } catch (error) {+ // A cache that cannot be written (quota, a browser without writable files) is not a reason to fail the load.+ if (!isStorageError(error)) throw error;+ }}-- loaded.set(key, file.size);- report();- return [key, await blob.arrayBuffer()] as const;+ bytes ??= await fetchInMemory(url, size, onBytes, signal);+ onBytes(size);+ return [key, bytes] as const;}),);return Object.fromEntries(entries) as ModelFiles;}-async function download(url: string, signal: AbortSignal | undefined, onBytes: (bytes: number) => void): Promise<Blob> {+/** The models whose every file is in the local cache, whole, so loading them costs no download. */+export async function cachedSam2Models(): Promise<Set<Sam2ModelId>> {+ const root = await openRoot();+ if (!root) return new Set();++ const cached = await Promise.all(+ SAM2_MODELS.map(async (model) => {+ try {+ const folder = await root.getDirectoryHandle(folderName(model));+ const sizes = await Promise.all(+ Object.values(model.files).map(async (file) => (await (await folder.getFileHandle(fileName(file.path))).getFile()).size === file.size),+ );+ return sizes.every(Boolean) ? model.id : null;+ } catch {+ return null;+ }+ }),+ );+ return new Set(cached.filter((id) => id !== null));+}++/** The file from `folder`, fetched first as far as it is missing. */+async function fetchStored(+ folder: FileSystemDirectoryHandle,+ name: string,+ url: string,+ size: number,+ onBytes: (bytes: number) => void,+ signal: AbortSignal | undefined,+): Promise<ArrayBuffer> {+ const handle = await folder.getFileHandle(name, { create: true });+ const stored = (await handle.getFile()).size;+ // Longer than the file should be, it is not this file: it is fetched again.+ if (stored !== size) {+ await download(handle, url, stored < size ? stored : 0, onBytes, signal);+ }++ const file = await handle.getFile();+ if (file.size !== size) {+ throw new Error(`${url} came to ${file.size} bytes, not ${size}`);+ }+ return file.arrayBuffer();+}++/**+ * Streams `url` into the file from byte `offset` on, asking the server for+ * the rest only. Closing commits what arrived, whole or not.+ */+async function download(+ handle: FileSystemFileHandle,+ url: string,+ offset: number,+ onBytes: (bytes: number) => void,+ signal: AbortSignal | undefined,+): Promise<void> {+ onBytes(offset);+ const response = await fetch(url, { signal, headers: offset > 0 ? { Range: `bytes=${offset}-` } : undefined });+ if (!response.ok || !response.body) {+ throw new Error(`Could not fetch ${url} (${response.status})`);+ }++ // A server that ignores the range sends the whole file, which replaces the part kept.+ const start = response.status === 206 ? offset : 0;+ const writable = await handle.createWritable({ keepExistingData: start > 0 });+ let received = start;+ const counter = new TransformStream<Uint8Array, Uint8Array>({+ transform(chunk, controller) {+ received += chunk.byteLength;+ onBytes(received);+ controller.enqueue(chunk);+ },+ });++ try {+ if (start > 0) await writable.seek(start);+ await response.body.pipeThrough(counter).pipeTo(writable, { preventClose: true, preventAbort: true, signal });+ } finally {+ await writable.close().catch(() => undefined);+ }+}++/** `url`, whose size is known, read into one buffer as it arrives. */+async function fetchInMemory(url: string, size: number, onBytes: (bytes: number) => void, signal: AbortSignal | undefined): Promise<ArrayBuffer> {const response = await fetch(url, { signal });if (!response.ok || !response.body) {throw new Error(`Could not fetch ${url} (${response.status})`);}+ const bytes = new Uint8Array(size);const reader = response.body.getReader();- const chunks: BlobPart[] = [];let received = 0;-- for (;;) {+ while (true) {const { done, value } = await reader.read();if (done) break;- chunks.push(value);+ if (received + value.byteLength > size) {+ throw new Error(`${url} is longer than ${size} bytes`);+ }+ bytes.set(value, received);received += value.byteLength;onBytes(received);}-- return new Blob(chunks);+ if (received !== size) {+ throw new Error(`${url} came to ${received} bytes, not ${size}`);+ }+ return bytes.buffer;}-type FileStore = {- read(key: string): Promise<Blob | undefined>;- write(key: string, blob: Blob): Promise<void>;- /** Deletes every file but these, such as those of a model revision no longer loaded. */- retain(keys: string[]): Promise<void>;-};--async function openFileStore(): Promise<FileStore | null> {- if (typeof indexedDB === 'undefined') return null;-+async function openRoot(): Promise<FileSystemDirectoryHandle | null> {try {- const db = await new Promise<IDBDatabase>((resolve, reject) => {- const request = indexedDB.open(DB_NAME, 1);- request.onupgradeneeded = () => request.result.createObjectStore(STORE_NAME);- request.onsuccess = () => resolve(request.result);- request.onerror = () => reject(request.error);- });-- const run = <T>(mode: IDBTransactionMode, operation: (store: IDBObjectStore) => IDBRequest<T>) =>- new Promise<T>((resolve, reject) => {- const request = operation(db.transaction(STORE_NAME, mode).objectStore(STORE_NAME));- request.onsuccess = () => resolve(request.result);- request.onerror = () => reject(request.error);- });-- return {- read: (key) => run<Blob | undefined>('readonly', (store) => store.get(key)),- // A cache that cannot be written (quota, private mode) is not a reason to fail the load.- write: (key, blob) => run('readwrite', (store) => store.put(blob, key)).then(() => undefined, () => undefined),- // Like a failed write, a failed cleanup only costs disk space.- retain: (keys) =>- run('readonly', (store) => store.getAllKeys())- .then((all) => Promise.all(all.filter((key) => !keys.includes(String(key))).map((key) => run('readwrite', (store) => store.delete(key)))))- .then(() => undefined, () => undefined),- };+ return await (await navigator.storage.getDirectory()).getDirectoryHandle(ROOT, { create: true });} catch {return null;}}++/**+ * Deletes the revisions of these models' repos that are no longer offered;+ * the revisions on offer stay, fetched or not, and other repos' folders are+ * not these models' to touch.+ */+async function retain(root: FileSystemDirectoryHandle): Promise<void> {+ const keep = new Set(SAM2_MODELS.map(folderName));+ const repos = new Set(SAM2_MODELS.map((model) => repoOf(folderName(model))));+ try {+ // The DOM library types the folder's listing only with `DOM.AsyncIterable`.+ const names: string[] = [];+ for await (const name of (root as unknown as { keys(): AsyncIterable<string> }).keys()) {+ names.push(name);+ }+ const stale = names.filter((name) => repos.has(repoOf(name)) && !keep.has(name));+ await Promise.all(stale.map((name) => root.removeEntry(name, { recursive: true })));+ } catch { /* noop */ }+}++/** A model revision's folder: `https://huggingface.co/<org>/<repo>/resolve/<commit>` as `<org>--<repo>@<commit>`. */+function folderName(model: Sam2Model): string {+ return new URL(model.repo).pathname.slice(1).replace('/resolve/', '@').replaceAll('/', '--');+}++/** The repo part of a folder's name, without its revision. */+function repoOf(folder: string): string {+ return folder.split('@')[0]!;+}++function fileName(path: string): string {+ return path.replaceAll('/', '--');+}++/** Whether `error` is the file system's, rather than the download's. */+function isStorageError(error: unknown): boolean {+ return (+ error instanceof DOMException &&+ ['QuotaExceededError', 'NoModificationAllowedError', 'NotAllowedError', 'InvalidModificationError', 'InvalidStateError', 'NotFoundError', 'SecurityError'].includes(error.name)+ );+}