Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance SAM 2.1 model structure and GPU processing

konstantin-paulus88f74f8bfe5d
diff --git a/packages/sam2/package.json b/packages/sam2/package.json
index 09e8f9e8..69e0b303 100644
--- a/packages/sam2/package.json
+++ b/packages/sam2/package.json
@@ -7,7 +7,8 @@
"sideEffects": false,
"exports": {
".": "./src/index.ts",
- "./mask": "./src/mask.ts"
+ "./mask": "./src/mask.ts",
+ "./models": "./src/constants.ts"
},
"scripts": {
"check": "tsc --noEmit"
diff --git a/packages/sam2/src/constants.ts b/packages/sam2/src/constants.ts
index 7794c085..9de98894 100644
--- a/packages/sam2/src/constants.ts
+++ b/packages/sam2/src/constants.ts
@@ -2,40 +2,140 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+export type Sam2ModelId = 'tiny' | 'small' | 'base-plus' | 'large';
+
+export type ModelFileKey = 'constants' | 'visionEncoder' | 'maskDecoder' | 'memoryEncoder' | 'memoryAttention' | 'pointerTpos';
+
/**
- * Made by `scripts/export.py 512 7`: fixed shapes for `IMAGE_SIZE` and
- * `MEMORY_FRAMES` below, fp16 compute behind fp32 inputs and outputs.
- * Pinned to a commit, which is part of the cache key.
+ * A SAM 2.1 export, made by `scripts/export.py <size> <imageSize> 7`: fixed
+ * shapes for `imageSize` and `MEMORY_FRAMES`, fp16 compute behind fp32 inputs
+ * and outputs. The repo is pinned to a commit, which is part of the cache key
+ * and what a mask's recipe names.
*/
-export const MODEL_REPO = 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e';
-
-export const MODEL_FILES = {
- constants: { path: 'constants.json', size: 9_781 },
- visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 58_385_353 },
- maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_898_805 },
- memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_753 },
- memoryAttention: { path: 'onnx/memory_attention.onnx', size: 13_029_470 },
- pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },
-} as const;
-
-/** The model's input, half the resolution SAM 2 was trained at: a quarter of the encoder's work. */
-export const IMAGE_SIZE = 512;
-/** The decoder's masks, upsampled in-graph from its own `IMAGE_SIZE / 4`. */
-export const MASK_SIZE = 256;
-export const FEAT_SIZE = IMAGE_SIZE / 16;
-export const FEAT_TOKENS = FEAT_SIZE * FEAT_SIZE;
+export type Sam2Model = {
+ id: Sam2ModelId;
+ label: string;
+ repo: string;
+ imageSize: number;
+ files: Record<ModelFileKey, { path: string; size: number }>;
+};
+
+/**
+ * The models on offer, fastest first. Tiny runs at half the resolution
+ * SAM 2 was trained at, a quarter of the encoder's work; the larger ones at
+ * the full 1024, for the detail and small objects they are chosen for.
+ */
+export const SAM2_MODELS: readonly Sam2Model[] = [
+ {
+ id: 'tiny',
+ label: 'Tiny',
+ repo: 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e',
+ imageSize: 512,
+ files: {
+ constants: { path: 'constants.json', size: 9_781 },
+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 58_385_353 },
+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_898_805 },
+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_753 },
+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 13_029_470 },
+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },
+ },
+ },
+ {
+ id: 'small',
+ label: 'Small',
+ repo: 'https://huggingface.co/diffusionstudio/sam2.1-small-video-onnx-fp16/resolve/927ecec6e6ae2727d0af7720eefdb07f2efe4689',
+ imageSize: 1024,
+ files: {
+ constants: { path: 'constants.json', size: 9_820 },
+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 83_679_474 },
+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_910_345 },
+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_884 },
+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_175_204 },
+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },
+ },
+ },
+ {
+ id: 'base-plus',
+ label: 'Base+',
+ repo: 'https://huggingface.co/diffusionstudio/sam2.1-base-plus-video-onnx-fp16/resolve/386b83e09a774fe2782d86fd1b8cb5aaf718aa9d',
+ imageSize: 1024,
+ files: {
+ constants: { path: 'constants.json', size: 9_857 },
+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 155_490_079 },
+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_910_345 },
+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_648 },
+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_169_156 },
+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },
+ },
+ },
+ {
+ id: 'large',
+ label: 'Large',
+ repo: 'https://huggingface.co/diffusionstudio/sam2.1-large-video-onnx-fp16/resolve/98dbd222d9b3f75d15fe63a5c2a22b0b0029eea1',
+ imageSize: 1024,
+ files: {
+ constants: { path: 'constants.json', size: 9_752 },
+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 447_430_045 },
+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_910_364 },
+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_886 },
+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_175_204 },
+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },
+ },
+ },
+];
+
+export const DEFAULT_SAM2_MODEL: Sam2ModelId = 'tiny';
+
+export function sam2Model(id: Sam2ModelId): Sam2Model {
+ const model = SAM2_MODELS.find((entry) => entry.id === id);
+ if (!model) throw new Error(`There is no SAM 2.1 model ${id}`);
+ return model;
+}
+
+/** The model a mask's recipe names by its pinned repo; undefined for one no longer offered. */
+export function sam2ModelOfRepo(repo: string): Sam2Model | undefined {
+ return SAM2_MODELS.find((entry) => entry.repo === repo);
+}
+
+/** Bytes to fetch for `model` when none of it is cached. */
+export function downloadSize(model: Sam2Model): number {
+ return Object.values(model.files).reduce((sum, file) => sum + file.size, 0);
+}
+
export const HIDDEN_DIM = 256;
export const MEM_DIM = 64;
export const MEMORY_FRAMES = 7;
export const MAX_POINTERS = 16;
export const POINTER_TOKENS = HIDDEN_DIM / MEM_DIM;
-export const MEMORY_ROWS = MEMORY_FRAMES * FEAT_TOKENS + MAX_POINTERS * POINTER_TOKENS;
+export const DECODED_MASK_SIZE = 256;
export const BYTES_PER_FLOAT = 4;
export const MEMORY_ROW_BYTES = MEM_DIM * BYTES_PER_FLOAT;
-export const MEMORY_BLOCK_BYTES = FEAT_TOKENS * MEMORY_ROW_BYTES;
export const POINTER_BYTES = HIDDEN_DIM * BYTES_PER_FLOAT;
+/** The sizes a model's input resolution sets: its grids, its buffers and its shaders. */
+export type Sam2Geometry = {
+ imageSize: number;
+ maskSize: number;
+ featSize: number;
+ featTokens: number;
+ memoryRows: number;
+ memoryBlockBytes: number;
+};
+
+export function geometryOf(imageSize: number): Sam2Geometry {
+ const featSize = imageSize / 16;
+ const featTokens = featSize * featSize;
+ return {
+ imageSize,
+ maskSize: imageSize / 4,
+ featSize,
+ featTokens,
+ memoryRows: MEMORY_FRAMES * featTokens + MAX_POINTERS * POINTER_TOKENS,
+ memoryBlockBytes: featTokens * MEMORY_ROW_BYTES,
+ };
+}
+
/** The values `constants.json` in the model repo carries. */
export type ModelConstants = {
image_size: number;
diff --git a/packages/sam2/src/gpu.ts b/packages/sam2/src/gpu.ts
index 2b4f451a..9b8c57ab 100644
--- a/packages/sam2/src/gpu.ts
+++ b/packages/sam2/src/gpu.ts
@@ -2,8 +2,10 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { BYTES_PER_FLOAT, FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE } from './constants';
-import { PREPROCESS_SHADER, TRANSPOSE_SHADER, WORKGROUP_SIZE } from './shaders';
+import { BYTES_PER_FLOAT, HIDDEN_DIM } from './constants';
+import { WORKGROUP_SIZE, preprocessShader, transposeShader } from './shaders';
+
+import type { Sam2Geometry } from './constants';
/** Degrees clockwise a decoded frame is turned for display. */
export type Rotation = 0 | 90 | 180 | 270;
@@ -47,11 +49,12 @@ export class FramePreprocessor {
constructor(
private readonly device: GPUDevice,
+ private readonly imageSize: number,
private readonly mean: readonly number[],
private readonly std: readonly number[],
) {
- this.pixels = createStorageBuffer(device, 3 * IMAGE_SIZE * IMAGE_SIZE * BYTES_PER_FLOAT, 'sam2 pixel_values');
- this.pipeline = createPipeline(device, PREPROCESS_SHADER, 'sam2 preprocess');
+ this.pixels = createStorageBuffer(device, 3 * imageSize * imageSize * BYTES_PER_FLOAT, 'sam2 pixel_values');
+ this.pipeline = createPipeline(device, preprocessShader(imageSize), 'sam2 preprocess');
this.sampler = device.createSampler({ magFilter: 'linear', minFilter: 'linear' });
this.params = device.createBuffer({
label: 'sam2 preprocess params',
@@ -72,7 +75,7 @@ export class FramePreprocessor {
floats[0] = width;
floats[1] = height;
uints[2] = rotation / 90;
- uints[3] = Math.min(MAX_TAPS, Math.max(1, Math.ceil(Math.max(width, height) / IMAGE_SIZE)));
+ uints[3] = Math.min(MAX_TAPS, Math.max(1, Math.ceil(Math.max(width, height) / this.imageSize)));
floats.set(this.mean, 4);
floats.set(this.std, 8);
this.device.queue.writeBuffer(this.params, 0, params);
@@ -86,7 +89,7 @@ export class FramePreprocessor {
{ binding: 3, resource: { buffer: this.pixels } },
],
});
- dispatch(this.device, this.pipeline, bindGroup, IMAGE_SIZE / WORKGROUP_SIZE, IMAGE_SIZE / WORKGROUP_SIZE);
+ dispatch(this.device, this.pipeline, bindGroup, this.imageSize / WORKGROUP_SIZE, this.imageSize / WORKGROUP_SIZE);
}
private textureFor(width: number, height: number): GPUTexture {
@@ -113,12 +116,15 @@ export class FramePreprocessor {
export class TokenTransposer {
private readonly pipeline: GPUComputePipeline;
- constructor(private readonly device: GPUDevice) {
- this.pipeline = createPipeline(device, TRANSPOSE_SHADER, 'sam2 transpose');
+ constructor(
+ private readonly device: GPUDevice,
+ private readonly geometry: Sam2Geometry,
+ ) {
+ this.pipeline = createPipeline(device, transposeShader(geometry.featTokens), 'sam2 transpose');
}
run(channels: GPUBuffer, tokens: GPUBuffer): void {
- const size = FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT;
+ const size = this.geometry.featTokens * HIDDEN_DIM * BYTES_PER_FLOAT;
const bindGroup = this.device.createBindGroup({
layout: this.pipeline.getBindGroupLayout(0),
entries: [
@@ -126,6 +132,6 @@ export class TokenTransposer {
{ binding: 1, resource: { buffer: tokens, size } },
],
});
- dispatch(this.device, this.pipeline, bindGroup, FEAT_TOKENS / WORKGROUP_SIZE, HIDDEN_DIM / WORKGROUP_SIZE);
+ dispatch(this.device, this.pipeline, bindGroup, this.geometry.featTokens / WORKGROUP_SIZE, HIDDEN_DIM / WORKGROUP_SIZE);
}
}
diff --git a/packages/sam2/src/index.ts b/packages/sam2/src/index.ts
index 337d766f..bc25c1c7 100644
--- a/packages/sam2/src/index.ts
+++ b/packages/sam2/src/index.ts
@@ -5,9 +5,13 @@
export { loadSam2 } from './load';
export { Sam2Video } from './tracker';
export { holdFrame, trackObject } from './video';
-export { MASK_SIZE, maskHas, maskIsEmpty, renderMask } from './mask';
+export { cachedSam2Models } from './weights';
+export { DEFAULT_SAM2_MODEL, SAM2_MODELS, downloadSize, sam2Model, sam2ModelOfRepo } from './constants';
+export { maskIsEmpty, paintMask, paintStroke } from './mask';
+
+export type { Sam2Model, Sam2ModelId } from './constants';
export type { Sam2LoadOptions, Sam2Progress } from './load';
-export type { Sam2Point } from './tracker';
+export type { Sam2Correction, Sam2Point } from './tracker';
export type { FrameRequest, TrackRequest } from './video';
-export type { Sam2Mask } from './mask';
+export type { Sam2Mask, Sam2Stroke } from './mask';
diff --git a/packages/sam2/src/load.ts b/packages/sam2/src/load.ts
index 42a7f71a..caee9344 100644
--- a/packages/sam2/src/load.ts
+++ b/packages/sam2/src/load.ts
@@ -2,13 +2,14 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { IMAGE_SIZE, MEMORY_FRAMES } from './constants';
+import { DEFAULT_SAM2_MODEL, MEMORY_FRAMES, sam2Model } from './constants';
import { readFloat16Initializer } from './onnx';
import { configureRuntime, createSession, runtimeDevice } from './sessions';
import { Sam2Video } from './tracker';
import { fetchModelFiles } from './weights';
-import type { ModelConstants } from './constants';
+import type { ModelConstants, Sam2Model, Sam2ModelId } from './constants';
+import type { Session } from './sessions';
export type Sam2Progress =
| { phase: 'download'; loaded: number; total: number }
@@ -16,40 +17,105 @@ export type Sam2Progress =
export type Sam2LoadOptions = {
onProgress?: (progress: Sam2Progress) => void;
- signal?: AbortSignal;
};
-let loading: Promise<Sam2Video> | null = null;
+type Loading = {
+ id: Sam2ModelId;
+ model: Promise<Sam2Video>;
+ listeners: Set<(progress: Sam2Progress) => void>;
+ last: Sam2Progress | null;
+ controller: AbortController;
+ settled: boolean;
+};
-/** The model, loaded once per page: the first call fetches and compiles it, later calls share the result. */
-export function loadSam2(options: Sam2LoadOptions = {}): Promise<Sam2Video> {
- loading ??= load(options).catch((error: unknown) => {
- loading = null;
- throw error;
- });
- return loading;
+let loaded: Loading | null = null;
+
+/**
+ * A model, loaded once per page: the first call fetches and compiles it,
+ * later calls for it share the result, and hear its progress from where it
+ * stands. One model is loaded at a time, the large one being most of a
+ * laptop GPU's memory: asking for another disposes the one before, so a
+ * caller must be done with it first, and cancels it while it is still on
+ * its way — the files it fetched whole stay cached, the rest are left.
+ * A cancelled load rejects with an `AbortError`.
+ */
+export function loadSam2(id: Sam2ModelId = DEFAULT_SAM2_MODEL, { onProgress }: Sam2LoadOptions = {}): Promise<Sam2Video> {
+ if (loaded?.id !== id) loaded = startLoad(id);
+ if (onProgress) {
+ loaded.listeners.add(onProgress);
+ if (loaded.last) onProgress(loaded.last);
+ }
+ return loaded.model;
}
-async function load({ onProgress, signal }: Sam2LoadOptions): Promise<Sam2Video> {
+function startLoad(id: Sam2ModelId): Loading {
+ const previous = loaded;
+ if (previous && !previous.settled) previous.controller.abort(new DOMException(`${sam2Model(id).label} was asked for instead`, 'AbortError'));
+
+ const controller = new AbortController();
+ const report = (progress: Sam2Progress) => {
+ entry.last = progress;
+ for (const listener of entry.listeners) listener(progress);
+ };
+ const model = (async () => {
+ await previous?.model.then((model) => model.dispose(), () => undefined);
+ return load(sam2Model(id), report, controller.signal);
+ })();
+
+ const entry: Loading = {
+ id,
+ model: model
+ .catch((error: unknown) => {
+ if (loaded === entry) loaded = null;
+ throw error;
+ })
+ .finally(() => {
+ entry.settled = true;
+ entry.listeners.clear();
+ }),
+ listeners: new Set(),
+ last: null,
+ controller,
+ settled: false,
+ };
+ return entry;
+}
+
+async function load(model: Sam2Model, onProgress: (progress: Sam2Progress) => void, signal: AbortSignal): Promise<Sam2Video> {
+ signal.throwIfAborted();
if (!navigator.gpu) throw new Error('WebGPU is not available in this browser');
configureRuntime();
- const files = await fetchModelFiles((progress) => onProgress?.({ phase: 'download', ...progress }), signal);
+ const files = await fetchModelFiles(model, (progress) => onProgress({ phase: 'download', ...progress }), signal);
const constants = JSON.parse(new TextDecoder().decode(files.constants)) as ModelConstants;
- if (constants.image_size !== IMAGE_SIZE || constants.memory_frames !== MEMORY_FRAMES) {
- throw new Error(`The model is for ${constants.image_size}px and ${constants.memory_frames} memories, the pipeline for ${IMAGE_SIZE}px and ${MEMORY_FRAMES}`);
+ if (constants.image_size !== model.imageSize || constants.memory_frames !== MEMORY_FRAMES) {
+ throw new Error(`The model is for ${constants.image_size}px and ${constants.memory_frames} memories, the pipeline for ${model.imageSize}px and ${MEMORY_FRAMES}`);
}
- onProgress?.({ phase: 'compile' });
- // The runtime builds one WebGPU session at a time.
- const visionEncoder = await createSession(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed'], true);
- const maskDecoder = await createSession(files.maskDecoder, ['high_res_mask', 'object_pointer']);
- const memoryEncoder = await createSession(files.memoryEncoder, ['memory_tokens', 'memory_pos']);
- const memoryAttention = await createSession(files.memoryAttention, ['conditioned_feats'], { enableGraphCapture: true });
- const pointerTpos = await createSession(files.pointerTpos, ['pointer_pos']);
- const device = await runtimeDevice();
-
- const positions = readFloat16Initializer(files.visionEncoder, 'pos_embed');
+ onProgress({ phase: 'compile' });
+ // Compiling takes seconds and a share of the GPU: a model no longer
+ // wanted stops at the next graph, and what it built is let go. The runtime
+ // builds one WebGPU session at a time.
+ const built: Session[] = [];
+ const build = async (...args: Parameters<typeof createSession>) => {
+ signal.throwIfAborted();
+ const session = await createSession(...args);
+ built.push(session);
+ return session;
+ };
+ try {
+ const visionEncoder = await build(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed'], { enableGraphCapture: true });
+ const maskDecoder = await build(files.maskDecoder, ['high_res_mask', 'object_pointer']);
+ const memoryEncoder = await build(files.memoryEncoder, ['memory_tokens', 'memory_pos']);
+ const memoryAttention = await build(files.memoryAttention, ['conditioned_feats'], { enableGraphCapture: true });
+ const pointerTpos = await build(files.pointerTpos, ['pointer_pos']);
+ signal.throwIfAborted();
+ const device = await runtimeDevice();
- return new Sam2Video(device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants, positions);
+ const positions = readFloat16Initializer(files.visionEncoder, 'pos_embed');
+ return new Sam2Video(model, device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants, positions);
+ } catch (error) {
+ for (const session of built) void session.release();
+ throw error;
+ }
}
diff --git a/packages/sam2/src/mask.ts b/packages/sam2/src/mask.ts
index b5083a24..14d1b7ae 100644
--- a/packages/sam2/src/mask.ts
+++ b/packages/sam2/src/mask.ts
@@ -2,54 +2,195 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { MASK_SIZE } from './constants';
+import { DECODED_MASK_SIZE } from './constants';
-export { MASK_SIZE };
-
-/** One frame's segmentation, at the model's mask resolution. */
+/** One frame's segmentation, as the decoder sees it. */
export type Sam2Mask = {
- /** Row-major, one bit per pixel, most significant bit first. */
- bits: Uint8Array;
+ /**
+ * The decoder's logits on its own grid, `size` square, row-major: the
+ * object is where they are above zero.
+ */
+ logits: Float32Array;
+ /** The grid's side, a quarter of the model's input: set by the model the mask came from. */
+ size: number;
/** Whether the object is in frame: at or below zero the model saw it occluded. */
score: number;
/** The model's own estimate of the mask's quality, 0 to 1. */
iou: number;
};
-const MASK_BYTES = (MASK_SIZE * MASK_SIZE) / 8;
+/**
+ * How sure a mask that is given rather than decoded is, as a logit: of its
+ * cells, and of the object being in frame. SAM 2 scales a mask prompt so.
+ */
+export const GIVEN_LOGIT = 10;
-/** Packs mask logits into bits; positive logits are foreground. */
-export function packMask(logits: Float32Array, score: number, iou: number): Sam2Mask {
- const bits = new Uint8Array(MASK_BYTES);
- for (let i = 0; i < logits.length; i++) {
- if (logits[i]! > 0) bits[i >> 3]! |= 0x80 >> (i & 7);
- }
- return { bits, score, iou };
+/**
+ * The decoder's mask as a `Sam2Mask` on its `size` grid: its logits, which a
+ * decoder with a smaller grid than `DECODED_MASK_SIZE` upsampled twice before
+ * handing them over, brought back to its own grid — exactly, since the
+ * upsampling is linear and invertible.
+ */
+export function packMask(decoded: Float32Array, size: number, score: number, iou: number): Sam2Mask {
+ if (size === DECODED_MASK_SIZE) return { logits: decoded.slice(), size, score, iou };
+ if (size * 2 !== DECODED_MASK_SIZE) throw new Error(`A ${size} grid is not upsampled to ${DECODED_MASK_SIZE} twice`);
+ return { logits: downsample(decoded, size), size, score, iou };
}
-export function maskHas(mask: Sam2Mask, x: number, y: number): boolean {
- const i = y * MASK_SIZE + x;
- return (mask.bits[i >> 3]! & (0x80 >> (i & 7))) !== 0;
+/** `mask` with nothing in it. */
+export function clearMask(mask: Sam2Mask): Sam2Mask {
+ return { ...mask, logits: new Float32Array(mask.logits.length).fill(-GIVEN_LOGIT) };
}
-/** Whether any pixel is foreground. */
+/** Whether no cell is in. */
export function maskIsEmpty(mask: Sam2Mask): boolean {
- return mask.bits.every((byte) => byte === 0);
+ return mask.logits.every((value) => value <= 0);
}
-/** The mask as RGBA pixels, `color` where it is foreground and transparent elsewhere. */
-export function renderMask(mask: Sam2Mask, color: [r: number, g: number, b: number, a: number]): ImageData {
- const image = new ImageData(MASK_SIZE, MASK_SIZE);
- const pixels = new Uint32Array(image.data.buffer);
- const packed = (color[3] << 24 | color[2] << 16 | color[1] << 8 | color[0]) >>> 0;
+// ── The decoder's upsampling, undone ─────────────────────────
- for (let byte = 0; byte < mask.bits.length; byte++) {
- const value = mask.bits[byte]!;
- if (value === 0) continue;
- for (let bit = 0; bit < 8; bit++) {
- if (value & (0x80 >> bit)) pixels[byte * 8 + bit] = packed;
- }
+// The decoder upsamples its grid twice, bilinear with cell centers aligned
+// and the border held: of each pair of new cells, one is three quarters the
+// old cell and a quarter the one before it, the other three quarters the old
+// cell and a quarter the one after. So a pair sums to a quarter of each
+// neighbor and one and a half of the cell — one and three quarters at the
+// ends — a tridiagonal system, solved along each row and then each column.
+
+const NEIGHBOR = 0.25;
+
+type Thomas = { upper: Float64Array; pivot: Float64Array };
+const thomasBySize = new Map<number, Thomas>();
+
+/** The Thomas algorithm's forward coefficients for a line of `n` cells, the same for every line. */
+function thomas(n: number): Thomas {
+ let coefficients = thomasBySize.get(n);
+ if (coefficients) return coefficients;
+
+ const upper = new Float64Array(n);
+ const pivot = new Float64Array(n);
+ for (let i = 0; i < n; i++) {
+ const diagonal = i === 0 || i === n - 1 ? 1.75 : 1.5;
+ pivot[i] = diagonal - (i > 0 ? NEIGHBOR * upper[i - 1]! : 0);
+ upper[i] = NEIGHBOR / pivot[i]!;
+ }
+ coefficients = { upper, pivot };
+ thomasBySize.set(n, coefficients);
+ return coefficients;
+}
+
+/** Solves one line in place: `line` holds the pair sums, and then the cells. */
+function solveLine(line: Float64Array): void {
+ const n = line.length;
+ const { upper, pivot } = thomas(n);
+ for (let i = 0; i < n; i++) line[i] = (line[i]! - (i > 0 ? NEIGHBOR * line[i - 1]! : 0)) / pivot[i]!;
+ for (let i = n - 2; i >= 0; i--) line[i] = line[i]! - upper[i]! * line[i + 1]!;
+}
+
+/** The decoder's `DECODED_MASK_SIZE` logits back on its own grid, `n` square. */
+function downsample(decoded: Float32Array, n: number): Float32Array {
+ const size = DECODED_MASK_SIZE;
+ const line = new Float64Array(n);
+
+ const rows = new Float64Array(size * n);
+ for (let y = 0; y < size; y++) {
+ for (let i = 0; i < n; i++) line[i] = decoded[y * size + 2 * i]! + decoded[y * size + 2 * i + 1]!;
+ solveLine(line);
+ rows.set(line, y * n);
+ }
+
+ const out = new Float32Array(n * n);
+ for (let x = 0; x < n; x++) {
+ for (let i = 0; i < n; i++) line[i] = rows[2 * i * n + x]! + rows[(2 * i + 1) * n + x]!;
+ solveLine(line);
+ for (let i = 0; i < n; i++) out[i * n + x] = line[i]!;
}
+ return out;
+}
+
+/**
+ * A brush stroke over a mask: its points in 0..1 of the frame's width and
+ * height, its radius in 0..1 of the frame's height, so the brush is round on
+ * the footage however the square grid is stretched over it. Label 1 adds to
+ * the mask, 0 erases.
+ */
+export type Sam2Stroke = {
+ label: 0 | 1;
+ radius: number;
+ points: { x: number; y: number }[];
+};
+
+/**
+ * How steeply a stroke's logits rise across its edge, per cell: about as
+ * steep as the model's own edges, so a stroke's edge falls between cells
+ * where the brush's does. They level off at `GIVEN_LOGIT`.
+ */
+const BRUSH_SLOPE = 4;
- return image;
+/** How far past its radius a stroke reaches, in cells: to where its logits have levelled off. */
+const BRUSH_REACH = GIVEN_LOGIT / BRUSH_SLOPE;
+
+/** `mask` with `strokes` painted over it in order; `aspect` is the frame's width over its height. */
+export function paintMask(mask: Sam2Mask, strokes: readonly Sam2Stroke[], aspect: number): Sam2Mask {
+ if (strokes.length === 0) return mask;
+ const painted = { ...mask, logits: mask.logits.slice() };
+ for (const stroke of strokes) paintStroke(painted, stroke, aspect);
+ return painted;
+}
+
+/**
+ * Paints `stroke` into `mask`'s logits, in place, from its point `from` on: a
+ * disc at each point and a capsule along each segment, so a fast stroke
+ * leaves no gaps. Painting from the last point painted is how a stroke is
+ * drawn as it grows.
+ */
+export function paintStroke(mask: Sam2Mask, stroke: Sam2Stroke, aspect: number, from = 0): void {
+ const { points, radius, label } = stroke;
+ for (let i = Math.max(0, from); i < points.length; i++) {
+ paintSegment(mask.logits, mask.size, points[Math.max(0, i - 1)]!, points[i]!, radius, aspect, label);
+ }
+}
+
+/**
+ * Raises the logits under a segment's capsule, or lowers them to erase: each
+ * cell to its distance inside the capsule's edge, measured on the frame, as a
+ * logit, where that is further in than the cell already is. The edge is then
+ * where the capsule's is, to a fraction of a cell, and the model's own edge
+ * stands where the stroke does not reach it.
+ */
+function paintSegment(
+ logits: Float32Array,
+ size: number,
+ a: { x: number; y: number },
+ b: { x: number; y: number },
+ radius: number,
+ aspect: number,
+ label: 0 | 1,
+): void {
+ // Distances are taken in frame heights: x is scaled by the aspect so the brush is round.
+ const au = a.x * aspect;
+ const bu = b.x * aspect;
+ const du = bu - au;
+ const dv = b.y - a.y;
+ const length2 = du * du + dv * dv;
+ const reach = radius + BRUSH_REACH / size;
+
+ const cell = (value: number) => Math.floor(value * size);
+ const x0 = Math.max(0, cell(Math.min(a.x, b.x) - reach / aspect));
+ const x1 = Math.min(size - 1, cell(Math.max(a.x, b.x) + reach / aspect));
+ const y0 = Math.max(0, cell(Math.min(a.y, b.y) - reach));
+ const y1 = Math.min(size - 1, cell(Math.max(a.y, b.y) + reach));
+
+ for (let y = y0; y <= y1; y++) {
+ const v = (y + 0.5) / size;
+ for (let x = x0; x <= x1; x++) {
+ const u = ((x + 0.5) / size) * aspect;
+ const t = length2 === 0 ? 0 : Math.min(1, Math.max(0, ((u - au) * du + (v - a.y) * dv) / length2));
+ const inside = radius - Math.hypot(u - (au + t * du), v - (a.y + t * dv));
+ if (inside < radius - reach) continue;
+
+ const logit = Math.min(GIVEN_LOGIT, Math.max(-GIVEN_LOGIT, inside * size * BRUSH_SLOPE));
+ const i = y * size + x;
+ logits[i] = label === 1 ? Math.max(logits[i]!, logit) : Math.min(logits[i]!, -logit);
+ }
+ }
}
diff --git a/packages/sam2/src/memory.ts b/packages/sam2/src/memory.ts
index 1ad3d757..af62adbb 100644
--- a/packages/sam2/src/memory.ts
+++ b/packages/sam2/src/memory.ts
@@ -2,12 +2,11 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import {
- MAX_POINTERS, MEM_DIM, MEMORY_BLOCK_BYTES, MEMORY_FRAMES, MEMORY_ROW_BYTES, MEMORY_ROWS,
- POINTER_BYTES, POINTER_TOKENS,
-} from './constants';
+import { MAX_POINTERS, MEM_DIM, MEMORY_FRAMES, MEMORY_ROW_BYTES, POINTER_BYTES, POINTER_TOKENS } from './constants';
import { createStorageBuffer } from './gpu';
+import type { Sam2Geometry } from './constants';
+
type FrameMemory = {
index: number;
tokens: GPUBuffer;
@@ -27,13 +26,19 @@ export class MemoryBank {
private conditioning: FrameMemory | null = null;
private recent: FrameMemory[] = [];
private readonly pool: FrameMemory[] = [];
+ /** A frame memory's bytes; the object pointers follow the frames'. */
+ private readonly blockBytes: number;
+ private readonly pointerOffset: number;
constructor(
private readonly device: GPUDevice,
+ geometry: Sam2Geometry,
private readonly temporalEncoding: readonly (readonly number[])[],
) {
- this.memory = createStorageBuffer(device, MEMORY_ROWS * MEMORY_ROW_BYTES, 'sam2 memory');
- this.memoryPos = createStorageBuffer(device, MEMORY_ROWS * MEMORY_ROW_BYTES, 'sam2 memory_pos');
+ this.blockBytes = geometry.memoryBlockBytes;
+ this.pointerOffset = MEMORY_FRAMES * geometry.memoryBlockBytes;
+ this.memory = createStorageBuffer(device, geometry.memoryRows * MEMORY_ROW_BYTES, 'sam2 memory');
+ this.memoryPos = createStorageBuffer(device, geometry.memoryRows * MEMORY_ROW_BYTES, 'sam2 memory_pos');
}
get hasPositions(): boolean {
@@ -102,11 +107,11 @@ export class MemoryBank {
const encoder = this.device.createCommandEncoder();
blocks.forEach((block, slot) => {
- encoder.copyBufferToBuffer(block.tokens, 0, this.memory, slot * MEMORY_BLOCK_BYTES, MEMORY_BLOCK_BYTES);
- encoder.copyBufferToBuffer(block.pos, 0, this.memoryPos, slot * MEMORY_BLOCK_BYTES, MEMORY_BLOCK_BYTES);
+ encoder.copyBufferToBuffer(block.tokens, 0, this.memory, slot * this.blockBytes, this.blockBytes);
+ encoder.copyBufferToBuffer(block.pos, 0, this.memoryPos, slot * this.blockBytes, this.blockBytes);
});
pointers.forEach((entry, i) => {
- encoder.copyBufferToBuffer(entry.pointer, 0, this.memory, POINTER_OFFSET + i * POINTER_BYTES, POINTER_BYTES);
+ encoder.copyBufferToBuffer(entry.pointer, 0, this.memory, this.pointerOffset + i * POINTER_BYTES, POINTER_BYTES);
});
this.device.queue.submit([encoder.finish()]);
@@ -120,7 +125,7 @@ export class MemoryBank {
for (let i = 0; i < MAX_POINTERS; i++) {
for (let token = 0; token < POINTER_TOKENS; token++) {
const row = i * POINTER_TOKENS + token;
- encoder.copyBufferToBuffer(positions, i * MEMORY_ROW_BYTES, this.memoryPos, POINTER_OFFSET + row * MEMORY_ROW_BYTES, MEMORY_ROW_BYTES);
+ encoder.copyBufferToBuffer(positions, i * MEMORY_ROW_BYTES, this.memoryPos, this.pointerOffset + row * MEMORY_ROW_BYTES, MEMORY_ROW_BYTES);
}
}
this.device.queue.submit([encoder.finish()]);
@@ -141,17 +146,15 @@ export class MemoryBank {
private retain(index: number, tokens: GPUBuffer, pointer: GPUBuffer): FrameMemory {
const entry = this.pool.pop() ?? {
index,
- tokens: createStorageBuffer(this.device, MEMORY_BLOCK_BYTES, 'sam2 frame memory'),
+ tokens: createStorageBuffer(this.device, this.blockBytes, 'sam2 frame memory'),
pointer: createStorageBuffer(this.device, POINTER_BYTES, 'sam2 frame pointer'),
};
entry.index = index;
const encoder = this.device.createCommandEncoder();
- encoder.copyBufferToBuffer(tokens, 0, entry.tokens, 0, MEMORY_BLOCK_BYTES);
+ encoder.copyBufferToBuffer(tokens, 0, entry.tokens, 0, this.blockBytes);
encoder.copyBufferToBuffer(pointer, 0, entry.pointer, 0, POINTER_BYTES);
this.device.queue.submit([encoder.finish()]);
return entry;
}
}
-
-const POINTER_OFFSET = MEMORY_FRAMES * MEMORY_BLOCK_BYTES;
diff --git a/packages/sam2/src/shaders.ts b/packages/sam2/src/shaders.ts
index 3d30f28c..d1dc4769 100644
--- a/packages/sam2/src/shaders.ts
+++ b/packages/sam2/src/shaders.ts
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE } from './constants';
+import { HIDDEN_DIM } from './constants';
export const WORKGROUP_SIZE = 16;
@@ -12,7 +12,7 @@ export const WORKGROUP_SIZE = 16;
* the input are box-filtered with a few taps per output pixel so downscaling
* does not alias.
*/
-export const PREPROCESS_SHADER = /* wgsl */ `
+export const preprocessShader = (imageSize: number) => /* wgsl */ `
struct Params {
size: vec2f,
rotation: u32,
@@ -28,7 +28,7 @@ struct Params {
@group(0) @binding(2) var frameSampler: sampler;
@group(0) @binding(3) var<storage, read_write> pixels: array<f32>;
-const SIZE: u32 = ${IMAGE_SIZE}u;
+const SIZE: u32 = ${imageSize}u;
const PLANE: u32 = SIZE * SIZE;
fn toSource(uv: vec2f) -> vec2f {
@@ -63,12 +63,12 @@ fn main(@builtin(global_invocation_id) id: vec3u) {
`;
/** Turns a [C, H, W] feature map into the [H*W, C] token layout memory attention takes. */
-export const TRANSPOSE_SHADER = /* wgsl */ `
+export const transposeShader = (featTokens: number) => /* wgsl */ `
@group(0) @binding(0) var<storage, read> channels: array<f32>;
@group(0) @binding(1) var<storage, read_write> tokens: array<f32>;
const CHANNELS: u32 = ${HIDDEN_DIM}u;
-const TOKENS: u32 = ${FEAT_TOKENS}u;
+const TOKENS: u32 = ${featTokens}u;
@compute @workgroup_size(${WORKGROUP_SIZE}, ${WORKGROUP_SIZE})
fn main(@builtin(global_invocation_id) id: vec3u) {
diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.ts
index 49efae6a..7c604d3f 100644
--- a/packages/sam2/src/tracker.ts
+++ b/packages/sam2/src/tracker.ts
@@ -2,15 +2,13 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import {
- BYTES_PER_FLOAT, FEAT_TOKENS, HIDDEN_DIM, IMAGE_SIZE, MAX_POINTERS, MEM_DIM, MEMORY_ROWS,
-} from './constants';
+import { BYTES_PER_FLOAT, HIDDEN_DIM, MAX_POINTERS, MEM_DIM, geometryOf } from './constants';
import { FramePreprocessor, TokenTransposer, createStorageBuffer } from './gpu';
import { MemoryBank } from './memory';
-import { packMask } from './mask';
+import { GIVEN_LOGIT, clearMask, maskIsEmpty, packMask } from './mask';
import { disposeOutputs, floatTensor, floats, gpuBuffer, gpuTensor, intTensor } from './sessions';
-import type { ModelConstants } from './constants';
+import type { ModelConstants, Sam2Geometry, Sam2Model } from './constants';
import type { Rotation } from './gpu';
import type { Sam2Mask } from './mask';
import type { Outputs, Session, Tensor } from './sessions';
@@ -18,6 +16,13 @@ import type { Outputs, Session, Tensor } from './sessions';
/** A prompt on the displayed frame, in 0..1 of its width and height; label 1 is the object, 0 is background. */
export type Sam2Point = { x: number; y: number; label: 0 | 1 };
+/**
+ * The prompted frame's mask as the user corrected it, from the model's own:
+ * what the frame is remembered as, and so what every tracked frame follows.
+ * Returning the mask it was given leaves the model's.
+ */
+export type Sam2Correction = (mask: Sam2Mask) => Sam2Mask;
+
export type Sam2Sessions = {
visionEncoder: Session;
maskDecoder: Session;
@@ -28,6 +33,8 @@ export type Sam2Sessions = {
type Prompt = { points: Tensor; labels: Tensor };
+type SegmentedFrame = { index: number; prompted: boolean; correct?: Sam2Correction };
+
/**
* SAM 2.1 video tracking over one object. `seed` segments it on the prompted
* frame; `track` follows it through the frames that come after, each one
@@ -35,6 +42,7 @@ type Prompt = { points: Tensor; labels: Tensor };
* caller, and their distances are what the model's temporal encodings read.
*/
export class Sam2Video {
+ readonly geometry: Sam2Geometry;
private readonly preprocessor: FramePreprocessor;
private readonly transposer: TokenTransposer;
private readonly bank: MemoryBank;
@@ -43,26 +51,36 @@ export class Sam2Video {
/** The frame kept encoded for `preview` and `seedHeld`, so a prompt costs the decoder alone. */
private held: Outputs | null = null;
- /** `positions` is the vision encoder's `pos_embed`, [HIDDEN_DIM, FEAT_TOKENS]. */
+ /** `positions` is the vision encoder's `pos_embed`, [HIDDEN_DIM, featTokens]. */
constructor(
+ readonly model: Sam2Model,
device: GPUDevice,
private readonly sessions: Sam2Sessions,
constants: ModelConstants,
positions: Float32Array,
) {
- this.preprocessor = new FramePreprocessor(device, constants.image_mean, constants.image_std);
- this.transposer = new TokenTransposer(device);
- this.bank = new MemoryBank(device, constants.memory_temporal_positional_encoding);
- this.featureTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 vision tokens');
- this.positionTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 position tokens');
- device.queue.writeBuffer(this.positionTokens, 0, toTokens(positions));
+ const geometry = (this.geometry = geometryOf(model.imageSize));
+ this.preprocessor = new FramePreprocessor(device, geometry.imageSize, constants.image_mean, constants.image_std);
+ this.transposer = new TokenTransposer(device, geometry);
+ this.bank = new MemoryBank(device, geometry, constants.memory_temporal_positional_encoding);
+ this.featureTokens = createStorageBuffer(device, geometry.featTokens * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 vision tokens');
+ this.positionTokens = createStorageBuffer(device, geometry.featTokens * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 position tokens');
+ device.queue.writeBuffer(this.positionTokens, 0, toTokens(positions, geometry.featTokens));
+ }
+
+ /** The side of its masks' grid. */
+ get maskSize(): number {
+ return this.geometry.maskSize;
}
- /** Segments the object at `points` and makes this the frame every later one is tracked from. */
- async seed(frame: VideoFrame, rotation: Rotation, points: Sam2Point[], index: number): Promise<Sam2Mask> {
+ /**
+ * Segments the object at `points` and makes this the frame every later one
+ * is tracked from, as `correct` has it when given.
+ */
+ async seed(frame: VideoFrame, rotation: Rotation, points: Sam2Point[], index: number, correct?: Sam2Correction): Promise<Sam2Mask> {
const vision = await this.encode(frame, rotation);
try {
- return await this.segment(vision, vision.feats2_no_mem!, pointPrompt(points), { index, prompted: true });
+ return await this.segment(vision, vision.feats2_no_mem!, this.pointPrompt(points), { index, prompted: true, correct });
} finally {
disposeOutputs(vision);
}
@@ -90,7 +108,7 @@ export class Sam2Video {
/** Segments the held frame at `points` without remembering it: what a click there would pick. */
async preview(points: Sam2Point[]): Promise<Sam2Mask> {
const vision = this.heldFrame();
- const prompt = pointPrompt(points);
+ const prompt = this.pointPrompt(points);
const decoded = await this.sessions.maskDecoder.run({
feats0: vision.feats0!,
feats1: vision.feats1!,
@@ -99,16 +117,16 @@ export class Sam2Video {
input_labels: prompt.labels,
});
try {
- return packMask(floats(decoded, 'low_res_mask'), floats(decoded, 'object_score_logits')[0]!, floats(decoded, 'iou')[0]!);
+ return this.unpack(decoded);
} finally {
disposeOutputs(decoded);
}
}
/** `seed` on the held frame. */
- seedHeld(points: Sam2Point[], index: number): Promise<Sam2Mask> {
+ seedHeld(points: Sam2Point[], index: number, correct?: Sam2Correction): Promise<Sam2Mask> {
const vision = this.heldFrame();
- return this.segment(vision, vision.feats2_no_mem!, pointPrompt(points), { index, prompted: true });
+ return this.segment(vision, vision.feats2_no_mem!, this.pointPrompt(points), { index, prompted: true, correct });
}
private heldFrame(): Outputs {
@@ -127,11 +145,12 @@ export class Sam2Video {
this.bank.setPointerPositions(gpuBuffer(temporal, 'pointer_pos'));
disposeOutputs(temporal);
+ const { featTokens, memoryRows } = this.geometry;
const attended = await this.sessions.memoryAttention.run({
- current_vision_features: gpuTensor(this.featureTokens, [FEAT_TOKENS, 1, HIDDEN_DIM]),
- current_vision_position_embeddings: gpuTensor(this.positionTokens, [FEAT_TOKENS, 1, HIDDEN_DIM]),
- memory: gpuTensor(this.bank.memory, [MEMORY_ROWS, 1, MEM_DIM]),
- memory_pos: gpuTensor(this.bank.memoryPos, [MEMORY_ROWS, 1, MEM_DIM]),
+ current_vision_features: gpuTensor(this.featureTokens, [featTokens, 1, HIDDEN_DIM]),
+ current_vision_position_embeddings: gpuTensor(this.positionTokens, [featTokens, 1, HIDDEN_DIM]),
+ memory: gpuTensor(this.bank.memory, [memoryRows, 1, MEM_DIM]),
+ memory_pos: gpuTensor(this.bank.memoryPos, [memoryRows, 1, MEM_DIM]),
});
try {
return await this.segment(vision, attended.conditioned_feats!, paddingPrompt(), { index, prompted: false });
@@ -165,8 +184,9 @@ export class Sam2Video {
private async encode(frame: VideoFrame, rotation: Rotation): Promise<Outputs> {
this.release();
this.preprocessor.run(frame, rotation);
+ const size = this.geometry.imageSize;
return this.sessions.visionEncoder.run(
- { pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, IMAGE_SIZE, IMAGE_SIZE]) },
+ { pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, size, size]) },
VISION_OUTPUTS,
);
}
@@ -177,7 +197,7 @@ export class Sam2Video {
* not remembered: a wrong guess kept in memory is what the next frames
* would match, so it would be reinforced rather than corrected.
*/
- private async segment(vision: Outputs, conditioned: Tensor, prompt: Prompt, frame: { index: number; prompted: boolean }): Promise<Sam2Mask> {
+ private async segment(vision: Outputs, conditioned: Tensor, prompt: Prompt, frame: SegmentedFrame): Promise<Sam2Mask> {
const decoded = await this.sessions.maskDecoder.run({
feats0: vision.feats0!,
feats1: vision.feats1!,
@@ -186,26 +206,37 @@ export class Sam2Video {
input_labels: prompt.labels,
});
try {
- const score = floats(decoded, 'object_score_logits');
- const mask = packMask(floats(decoded, 'low_res_mask'), score[0]!, floats(decoded, 'iou')[0]!);
+ const mask = this.unpack(decoded);
if (frame.prompted) {
- await this.remember(vision, decoded, frame.index, true);
- return mask;
+ const corrected = frame.correct?.(mask) ?? mask;
+ if (corrected === mask) {
+ await this.remember(vision, decoded, frame.index, true);
+ return mask;
+ }
+ // As SAM 2 takes a mask prompt: the object is there exactly where the mask says.
+ const given = { ...corrected, score: maskIsEmpty(corrected) ? -GIVEN_LOGIT : GIVEN_LOGIT };
+ await this.remember(vision, decoded, frame.index, true, given);
+ return given;
}
if (isReliable(mask)) await this.remember(vision, decoded, frame.index, false);
- return isPlausible(mask) ? mask : { ...mask, bits: new Uint8Array(mask.bits.length) };
+ return isPlausible(mask) ? mask : clearMask(mask);
} finally {
disposeOutputs(decoded);
}
}
- /** Encodes the frame's mask into memory, as the prompted frame or the next tracked one. */
- private async remember(vision: Outputs, decoded: Outputs, index: number, prompted: boolean): Promise<void> {
+ /**
+ * Encodes the frame's mask into memory, as the prompted frame or the next
+ * tracked one: the decoder's, or `given` in its place. A given mask keeps
+ * the decoder's object pointer, which the points it was corrected from made.
+ */
+ private async remember(vision: Outputs, decoded: Outputs, index: number, prompted: boolean, given?: Sam2Mask): Promise<void> {
+ const size = this.geometry.imageSize;
const memory = await this.sessions.memoryEncoder.run({
feats2: vision.feats2!,
- high_res_mask: decoded.high_res_mask!,
- object_score_logits: floatTensor(floats(decoded, 'object_score_logits'), [1, 1]),
+ high_res_mask: given ? floatTensor(maskLogits(given, size), [1, 1, size, size]) : decoded.high_res_mask!,
+ object_score_logits: floatTensor(given ? [given.score] : floats(decoded, 'object_score_logits'), [1, 1]),
binarize: floatTensor([prompted ? 1 : 0], []),
});
try {
@@ -221,6 +252,19 @@ export class Sam2Video {
disposeOutputs(memory);
}
}
+
+ private unpack(decoded: Outputs): Sam2Mask {
+ return packMask(floats(decoded, 'low_res_mask'), this.geometry.maskSize, floats(decoded, 'object_score_logits')[0]!, floats(decoded, 'iou')[0]!);
+ }
+
+ private pointPrompt(points: Sam2Point[]): Prompt {
+ if (points.length === 0) throw new Error('A prompt needs at least one point');
+ const size = this.geometry.imageSize;
+ return {
+ points: floatTensor(points.flatMap((point) => [point.x * size, point.y * size]), [1, 1, points.length, 2]),
+ labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),
+ };
+ }
}
/**
@@ -231,10 +275,10 @@ export class Sam2Video {
const VISION_OUTPUTS = ['feats0', 'feats1', 'feats2', 'feats2_no_mem'];
/** Rewrites [C, N] channels as the [N, 1, C] tokens memory attention reads. */
-function toTokens(channels: Float32Array): Float32Array<ArrayBuffer> {
+function toTokens(channels: Float32Array, count: number): Float32Array<ArrayBuffer> {
const tokens = new Float32Array(channels.length);
for (let c = 0; c < HIDDEN_DIM; c++) {
- for (let n = 0; n < FEAT_TOKENS; n++) tokens[n * HIDDEN_DIM + c] = channels[c * FEAT_TOKENS + n]!;
+ for (let n = 0; n < count; n++) tokens[n * HIDDEN_DIM + c] = channels[c * count + n]!;
}
return tokens;
}
@@ -255,6 +299,42 @@ const RELIABLE_IOU = 0.25;
*/
const PLAUSIBLE_IOU = 0.15;
+/**
+ * A mask's logits as the decoder's `high_res_mask`: upsampled to the
+ * image as the decoder upsamples its own, bilinear with cell centers aligned.
+ * A prompted frame's memory is made from where they are above zero.
+ */
+function maskLogits(mask: Sam2Mask, imageSize: number): Float32Array {
+ // Each image row or column reads two cells, at a weight for the second.
+ const scale = mask.size / imageSize;
+ const lower = new Int32Array(imageSize);
+ const upper = new Int32Array(imageSize);
+ const weight = new Float32Array(imageSize);
+ for (let o = 0; o < imageSize; o++) {
+ const source = Math.max(0, (o + 0.5) * scale - 0.5);
+ lower[o] = Math.floor(source);
+ upper[o] = Math.min(lower[o]! + 1, mask.size - 1);
+ weight[o] = source - lower[o]!;
+ }
+
+ const at = (x: number, y: number) => mask.logits[y * mask.size + x]!;
+ const logits = new Float32Array(imageSize * imageSize);
+ for (let y = 0; y < imageSize; y++) {
+ const y0 = lower[y]!;
+ const y1 = upper[y]!;
+ const wy = weight[y]!;
+ for (let x = 0; x < imageSize; x++) {
+ const x0 = lower[x]!;
+ const x1 = upper[x]!;
+ const wx = weight[x]!;
+ const top = at(x0, y0) * (1 - wx) + at(x1, y0) * wx;
+ const bottom = at(x0, y1) * (1 - wx) + at(x1, y1) * wx;
+ logits[y * imageSize + x] = top * (1 - wy) + bottom * wy;
+ }
+ }
+ return logits;
+}
+
function isReliable(mask: Sam2Mask): boolean {
return mask.score > 0 && mask.iou >= RELIABLE_IOU;
}
@@ -263,14 +343,6 @@ function isPlausible(mask: Sam2Mask): boolean {
return mask.iou >= PLAUSIBLE_IOU;
}
-function pointPrompt(points: Sam2Point[]): Prompt {
- if (points.length === 0) throw new Error('A prompt needs at least one point');
- return {
- points: floatTensor(points.flatMap((point) => [point.x * IMAGE_SIZE, point.y * IMAGE_SIZE]), [1, 1, points.length, 2]),
- labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),
- };
-}
-
/** What the decoder is prompted with on frames that have no points of their own. */
function paddingPrompt(): Prompt {
return { points: floatTensor([0, 0], [1, 1, 1, 2]), labels: intTensor([-1], [1, 1, 1]) };
diff --git a/packages/sam2/src/weights.ts b/packages/sam2/src/weights.ts
index 56092ed8..60baa57c 100644
--- a/packages/sam2/src/weights.ts
+++ b/packages/sam2/src/weights.ts
@@ -2,111 +2,220 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { MODEL_FILES, MODEL_REPO } from './constants';
+import { SAM2_MODELS, downloadSize } from './constants';
+
+import type { ModelFileKey, Sam2Model, Sam2ModelId } from './constants';
-export type ModelFileKey = keyof typeof MODEL_FILES;
export type ModelFiles = Record<ModelFileKey, ArrayBuffer>;
export type DownloadProgress = { loaded: number; total: number };
-const DB_NAME = 'diffusion-sam2';
-const STORE_NAME = 'files';
-
-const TOTAL_BYTES = Object.values(MODEL_FILES).reduce((sum, file) => sum + file.size, 0);
+/**
+ * The folder in the origin private file system models are kept in, shared
+ * with whatever else downloads models: a folder a repo revision, named by
+ * `folderName`, so each keeps to its own repos.
+ */
+const ROOT = 'models';
/**
- * The model files, from the local cache when they have been fetched before
- * and from the Hugging Face repo otherwise. Files are keyed by URL, so a
- * change of repo or revision invalidates them.
+ * A model's files, from the local cache when they have been fetched before
+ * and from the Hugging Face repo otherwise. Downloads stream to disk as they
+ * arrive, so a file is in memory once, when it is read back; one cut short,
+ * by `signal` or the network, keeps what arrived and resumes from there. A
+ * file is whole when it has the size the catalogue gives it. Without a
+ * private file system, files are fetched into memory and not kept.
*/
export async function fetchModelFiles(
+ model: Sam2Model,
onProgress?: (progress: DownloadProgress) => void,
signal?: AbortSignal,
): Promise<ModelFiles> {
- const store = await openFileStore();
- await store?.retain(Object.values(MODEL_FILES).map((file) => `${MODEL_REPO}/${file.path}`));
+ const root = await openRoot();
+ if (root) await retain(root);
+ const folder = root && (await root.getDirectoryHandle(folderName(model), { create: true }).catch(() => null));
+
+ const total = downloadSize(model);
const loaded = new Map<ModelFileKey, number>();
- const report = () => onProgress?.({ loaded: [...loaded.values()].reduce((a, b) => a + b, 0), total: TOTAL_BYTES });
+ const report = () => onProgress?.({ loaded: [...loaded.values()].reduce((a, b) => a + b, 0), total });
const entries = await Promise.all(
- (Object.keys(MODEL_FILES) as ModelFileKey[]).map(async (key) => {
- const file = MODEL_FILES[key];
- const url = `${MODEL_REPO}/${file.path}`;
-
- let blob = await store?.read(url);
- if (!blob) {
- blob = await download(url, signal, (bytes) => {
- loaded.set(key, bytes);
- report();
- });
- await store?.write(url, blob);
+ (Object.keys(model.files) as ModelFileKey[]).map(async (key) => {
+ const { path, size } = model.files[key];
+ const url = `${model.repo}/${path}`;
+ const onBytes = (bytes: number) => {
+ loaded.set(key, bytes);
+ report();
+ };
+
+ let bytes: ArrayBuffer | null = null;
+ if (folder) {
+ try {
+ bytes = await fetchStored(folder, fileName(path), url, size, onBytes, signal);
+ } catch (error) {
+ // A cache that cannot be written (quota, a browser without writable files) is not a reason to fail the load.
+ if (!isStorageError(error)) throw error;
+ }
}
-
- loaded.set(key, file.size);
- report();
- return [key, await blob.arrayBuffer()] as const;
+ bytes ??= await fetchInMemory(url, size, onBytes, signal);
+ onBytes(size);
+ return [key, bytes] as const;
}),
);
return Object.fromEntries(entries) as ModelFiles;
}
-async function download(url: string, signal: AbortSignal | undefined, onBytes: (bytes: number) => void): Promise<Blob> {
+/** The models whose every file is in the local cache, whole, so loading them costs no download. */
+export async function cachedSam2Models(): Promise<Set<Sam2ModelId>> {
+ const root = await openRoot();
+ if (!root) return new Set();
+
+ const cached = await Promise.all(
+ SAM2_MODELS.map(async (model) => {
+ try {
+ const folder = await root.getDirectoryHandle(folderName(model));
+ const sizes = await Promise.all(
+ Object.values(model.files).map(async (file) => (await (await folder.getFileHandle(fileName(file.path))).getFile()).size === file.size),
+ );
+ return sizes.every(Boolean) ? model.id : null;
+ } catch {
+ return null;
+ }
+ }),
+ );
+ return new Set(cached.filter((id) => id !== null));
+}
+
+/** The file from `folder`, fetched first as far as it is missing. */
+async function fetchStored(
+ folder: FileSystemDirectoryHandle,
+ name: string,
+ url: string,
+ size: number,
+ onBytes: (bytes: number) => void,
+ signal: AbortSignal | undefined,
+): Promise<ArrayBuffer> {
+ const handle = await folder.getFileHandle(name, { create: true });
+ const stored = (await handle.getFile()).size;
+ // Longer than the file should be, it is not this file: it is fetched again.
+ if (stored !== size) {
+ await download(handle, url, stored < size ? stored : 0, onBytes, signal);
+ }
+
+ const file = await handle.getFile();
+ if (file.size !== size) {
+ throw new Error(`${url} came to ${file.size} bytes, not ${size}`);
+ }
+ return file.arrayBuffer();
+}
+
+/**
+ * Streams `url` into the file from byte `offset` on, asking the server for
+ * the rest only. Closing commits what arrived, whole or not.
+ */
+async function download(
+ handle: FileSystemFileHandle,
+ url: string,
+ offset: number,
+ onBytes: (bytes: number) => void,
+ signal: AbortSignal | undefined,
+): Promise<void> {
+ onBytes(offset);
+ const response = await fetch(url, { signal, headers: offset > 0 ? { Range: `bytes=${offset}-` } : undefined });
+ if (!response.ok || !response.body) {
+ throw new Error(`Could not fetch ${url} (${response.status})`);
+ }
+
+ // A server that ignores the range sends the whole file, which replaces the part kept.
+ const start = response.status === 206 ? offset : 0;
+ const writable = await handle.createWritable({ keepExistingData: start > 0 });
+ let received = start;
+ const counter = new TransformStream<Uint8Array, Uint8Array>({
+ transform(chunk, controller) {
+ received += chunk.byteLength;
+ onBytes(received);
+ controller.enqueue(chunk);
+ },
+ });
+
+ try {
+ if (start > 0) await writable.seek(start);
+ await response.body.pipeThrough(counter).pipeTo(writable, { preventClose: true, preventAbort: true, signal });
+ } finally {
+ await writable.close().catch(() => undefined);
+ }
+}
+
+/** `url`, whose size is known, read into one buffer as it arrives. */
+async function fetchInMemory(url: string, size: number, onBytes: (bytes: number) => void, signal: AbortSignal | undefined): Promise<ArrayBuffer> {
const response = await fetch(url, { signal });
if (!response.ok || !response.body) {
throw new Error(`Could not fetch ${url} (${response.status})`);
}
+ const bytes = new Uint8Array(size);
const reader = response.body.getReader();
- const chunks: BlobPart[] = [];
let received = 0;
-
- for (;;) {
+ while (true) {
const { done, value } = await reader.read();
if (done) break;
- chunks.push(value);
+ if (received + value.byteLength > size) {
+ throw new Error(`${url} is longer than ${size} bytes`);
+ }
+ bytes.set(value, received);
received += value.byteLength;
onBytes(received);
}
-
- return new Blob(chunks);
+ if (received !== size) {
+ throw new Error(`${url} came to ${received} bytes, not ${size}`);
+ }
+ return bytes.buffer;
}
-type FileStore = {
- read(key: string): Promise<Blob | undefined>;
- write(key: string, blob: Blob): Promise<void>;
- /** Deletes every file but these, such as those of a model revision no longer loaded. */
- retain(keys: string[]): Promise<void>;
-};
-
-async function openFileStore(): Promise<FileStore | null> {
- if (typeof indexedDB === 'undefined') return null;
-
+async function openRoot(): Promise<FileSystemDirectoryHandle | null> {
try {
- const db = await new Promise<IDBDatabase>((resolve, reject) => {
- const request = indexedDB.open(DB_NAME, 1);
- request.onupgradeneeded = () => request.result.createObjectStore(STORE_NAME);
- request.onsuccess = () => resolve(request.result);
- request.onerror = () => reject(request.error);
- });
-
- const run = <T>(mode: IDBTransactionMode, operation: (store: IDBObjectStore) => IDBRequest<T>) =>
- new Promise<T>((resolve, reject) => {
- const request = operation(db.transaction(STORE_NAME, mode).objectStore(STORE_NAME));
- request.onsuccess = () => resolve(request.result);
- request.onerror = () => reject(request.error);
- });
-
- return {
- read: (key) => run<Blob | undefined>('readonly', (store) => store.get(key)),
- // A cache that cannot be written (quota, private mode) is not a reason to fail the load.
- write: (key, blob) => run('readwrite', (store) => store.put(blob, key)).then(() => undefined, () => undefined),
- // Like a failed write, a failed cleanup only costs disk space.
- retain: (keys) =>
- run('readonly', (store) => store.getAllKeys())
- .then((all) => Promise.all(all.filter((key) => !keys.includes(String(key))).map((key) => run('readwrite', (store) => store.delete(key)))))
- .then(() => undefined, () => undefined),
- };
+ return await (await navigator.storage.getDirectory()).getDirectoryHandle(ROOT, { create: true });
} catch {
return null;
}
}
+
+/**
+ * Deletes the revisions of these models' repos that are no longer offered;
+ * the revisions on offer stay, fetched or not, and other repos' folders are
+ * not these models' to touch.
+ */
+async function retain(root: FileSystemDirectoryHandle): Promise<void> {
+ const keep = new Set(SAM2_MODELS.map(folderName));
+ const repos = new Set(SAM2_MODELS.map((model) => repoOf(folderName(model))));
+ try {
+ // The DOM library types the folder's listing only with `DOM.AsyncIterable`.
+ const names: string[] = [];
+ for await (const name of (root as unknown as { keys(): AsyncIterable<string> }).keys()) {
+ names.push(name);
+ }
+ const stale = names.filter((name) => repos.has(repoOf(name)) && !keep.has(name));
+ await Promise.all(stale.map((name) => root.removeEntry(name, { recursive: true })));
+ } catch { /* noop */ }
+}
+
+/** A model revision's folder: `https://huggingface.co/<org>/<repo>/resolve/<commit>` as `<org>--<repo>@<commit>`. */
+function folderName(model: Sam2Model): string {
+ return new URL(model.repo).pathname.slice(1).replace('/resolve/', '@').replaceAll('/', '--');
+}
+
+/** The repo part of a folder's name, without its revision. */
+function repoOf(folder: string): string {
+ return folder.split('@')[0]!;
+}
+
+function fileName(path: string): string {
+ return path.replaceAll('/', '--');
+}
+
+/** Whether `error` is the file system's, rather than the download's. */
+function isStorageError(error: unknown): boolean {
+ return (
+ error instanceof DOMException &&
+ ['QuotaExceededError', 'NoModificationAllowedError', 'NotAllowedError', 'InvalidModificationError', 'InvalidStateError', 'NotFoundError', 'SecurityError'].includes(error.name)
+ );
+}