Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance SAM 2.1 video tracking functionality and update export parameters

konstantin-paulus1882b539c9fd
diff --git a/packages/sam2/scripts/export.py b/packages/sam2/scripts/export.py
index 4ec0a9c8..c6143482 100644
--- a/packages/sam2/scripts/export.py
+++ b/packages/sam2/scripts/export.py
@@ -3,16 +3,17 @@
# dependencies = ["numpy", "onnx==1.23.0", "onnxslim==0.1.96", "torch==2.14.0", "transformers==5.17.0"]
# ///
"""
-Exports the graphs the package loads: SAM 2.1 Hiera-Tiny video tracking as
-five fixed-shape ONNX graphs, from the `transformers` port of the checkpoint.
+Exports the graphs the package loads: SAM 2.1 video tracking as five
+fixed-shape ONNX graphs, from the `transformers` port of a checkpoint.
-The input resolution and the number of frame memories attended to are
-parameters; the package's constants must match the ones exported. The
+The model size (tiny, small, base-plus or large), the input resolution and
+the number of frame memories attended to are parameters; the package's
+constants must match the ones exported. The
graphs compute in fp16 but take and return fp32, with a cast at each boundary.
Constants are folded in fp32 first, so positional encodings are computed at
full precision before they are stored as halves.
- uv run packages/sam2/scripts/export.py <image-size> <memory-frames> <out-dir>
+ uv run packages/sam2/scripts/export.py <model-size> <image-size> <memory-frames> <out-dir>
"""
import hashlib
@@ -31,7 +32,6 @@ from torch import nn
from transformers import Sam2VideoConfig, Sam2VideoModel
from transformers.models.sam2_video.modeling_sam2_video import NO_OBJ_SCORE, get_1d_sine_pe
-CHECKPOINT = 'facebook/sam2.1-hiera-tiny'
# Masks come out at this size whatever the input resolution, upsampled from the decoder's.
MASK_SIZE = 256
MAX_POINTERS = 16
@@ -40,8 +40,8 @@ MAX_POINTERS = 16
FLOAT32_INPUTS = {('Resize', 1), ('Resize', 2)}
-def load(image_size: int) -> Sam2VideoModel:
- config = Sam2VideoConfig.from_pretrained(CHECKPOINT)
+def load(checkpoint: str, image_size: int) -> Sam2VideoModel:
+ config = Sam2VideoConfig.from_pretrained(checkpoint)
feat = image_size // 16
config.image_size = image_size
config.vision_config.backbone_config.image_size = [image_size, image_size]
@@ -49,7 +49,7 @@ def load(image_size: int) -> Sam2VideoModel:
config.prompt_encoder_config.image_size = image_size
config.mask_decoder_config.image_size = image_size
config.memory_attention_rope_feat_sizes = [feat, feat]
- model = Sam2VideoModel.from_pretrained(CHECKPOINT, config=config, attn_implementation='eager').eval()
+ model = Sam2VideoModel.from_pretrained(checkpoint, config=config, attn_implementation='eager').eval()
# The backbone's position embedding depends only on the input size; exported as the
# computation it is, it brings `If` nodes that the runtime evaluates on the CPU.
@@ -301,8 +301,8 @@ def dedupe_constants(model: onnx.ModelProto) -> onnx.ModelProto:
return model
-def main(image_size: int, memory_frames: int, out: Path) -> None:
- model = load(image_size)
+def main(model_size: str, image_size: int, memory_frames: int, out: Path) -> None:
+ model = load(f'facebook/sam2.1-hiera-{model_size}', image_size)
(out / 'onnx').mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory() as fp32:
export_fp32(model, memory_frames, Path(fp32))
@@ -313,6 +313,7 @@ def main(image_size: int, memory_frames: int, out: Path) -> None:
print(f'{path.stem}: {(out / "onnx" / path.name).stat().st_size:,} bytes')
constants = {
+ 'model': f'sam2.1-hiera-{model_size}',
'image_size': image_size,
'memory_frames': memory_frames,
'image_mean': [0.485, 0.456, 0.406],
@@ -323,4 +324,4 @@ def main(image_size: int, memory_frames: int, out: Path) -> None:
if __name__ == '__main__':
- main(int(sys.argv[1]), int(sys.argv[2]), Path(sys.argv[3]))
+ main(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), Path(sys.argv[4]))
diff --git a/packages/sam2/src/index.ts b/packages/sam2/src/index.ts
index 88aff151..337d766f 100644
--- a/packages/sam2/src/index.ts
+++ b/packages/sam2/src/index.ts
@@ -4,10 +4,10 @@
export { loadSam2 } from './load';
export { Sam2Video } from './tracker';
-export { trackObject } from './video';
+export { holdFrame, trackObject } from './video';
export { MASK_SIZE, maskHas, maskIsEmpty, renderMask } from './mask';
export type { Sam2LoadOptions, Sam2Progress } from './load';
export type { Sam2Point } from './tracker';
-export type { TrackRequest } from './video';
+export type { FrameRequest, TrackRequest } from './video';
export type { Sam2Mask } from './mask';
diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.ts
index ac858eab..fa1558b7 100644
--- a/packages/sam2/src/tracker.ts
+++ b/packages/sam2/src/tracker.ts
@@ -40,6 +40,8 @@ export class Sam2Video {
private readonly bank: MemoryBank;
private readonly featureTokens: GPUBuffer;
private readonly positionTokens: GPUBuffer;
+ /** The frame kept encoded for `preview` and `seedHeld`, so a prompt costs the decoder alone. */
+ private held: Outputs | null = null;
constructor(
device: GPUDevice,
@@ -55,21 +57,64 @@ export class Sam2Video {
/** Segments the object at `points` and makes this the frame every later one is tracked from. */
async seed(frame: VideoFrame, rotation: Rotation, points: Sam2Point[], index: number): Promise<Sam2Mask> {
- if (points.length === 0) throw new Error('A prompt needs at least one point');
-
- const prompt: Prompt = {
- points: floatTensor(points.flatMap((point) => [point.x * IMAGE_SIZE, point.y * IMAGE_SIZE]), [1, 1, points.length, 2]),
- labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),
- };
-
const vision = await this.encode(frame, rotation);
try {
- return await this.segment(vision, vision.feats2_no_mem!, prompt, { index, prompted: true });
+ return await this.segment(vision, vision.feats2_no_mem!, pointPrompt(points), { index, prompted: true });
} finally {
disposeOutputs(vision);
}
}
+ /**
+ * Encodes a frame and keeps it hot: `preview` and `seedHeld` then run
+ * the mask decoder alone, which is what makes prompting feel immediate.
+ * The frame held before is dropped.
+ */
+ async hold(frame: VideoFrame, rotation: Rotation): Promise<void> {
+ const vision = await this.encode(frame, rotation);
+ this.release();
+ this.held = vision;
+ }
+
+ get holding(): boolean {
+ return this.held !== null;
+ }
+
+ /** Drops the held frame. */
+ release(): void {
+ if (this.held) disposeOutputs(this.held);
+ this.held = null;
+ }
+
+ /** Segments the held frame at `points` without remembering it: what a click there would pick. */
+ async preview(points: Sam2Point[]): Promise<Sam2Mask> {
+ const vision = this.heldFrame();
+ const prompt = pointPrompt(points);
+ const decoded = await this.sessions.maskDecoder.run({
+ feats0: vision.feats0!,
+ feats1: vision.feats1!,
+ feats2_cond: vision.feats2_no_mem!,
+ input_points: prompt.points,
+ input_labels: prompt.labels,
+ });
+ try {
+ return packMask(floats(decoded, 'low_res_mask'), floats(decoded, 'object_score_logits')[0]!, floats(decoded, 'iou')[0]!);
+ } finally {
+ disposeOutputs(decoded);
+ }
+ }
+
+ /** `seed` on the held frame. */
+ seedHeld(points: Sam2Point[], index: number): Promise<Sam2Mask> {
+ const vision = this.heldFrame();
+ return this.segment(vision, vision.feats2_no_mem!, pointPrompt(points), { index, prompted: true });
+ }
+
+ private heldFrame(): Outputs {
+ if (!this.held) throw new Error('No frame is held');
+ return this.held;
+ }
+
/** Finds the seeded object in a frame `index` steps along from the last one. */
async track(frame: VideoFrame, rotation: Rotation, index: number, totalFrames: number): Promise<Sam2Mask> {
const vision = await this.encode(frame, rotation);
@@ -108,6 +153,7 @@ export class Sam2Video {
}
dispose(): void {
+ this.release();
this.bank.dispose();
this.preprocessor.dispose();
this.featureTokens.destroy();
@@ -122,7 +168,12 @@ export class Sam2Video {
});
}
- /** Decodes the mask, then encodes the frame into memory as the prompted frame or the next tracked one. */
+ /**
+ * Decodes the mask, then encodes the frame into memory as the prompted
+ * frame or the next tracked one. A tracked frame the model is unsure of is
+ * not remembered: a wrong guess kept in memory is what the next frames
+ * would match, so it would be reinforced rather than corrected.
+ */
private async segment(vision: Outputs, conditioned: Tensor, prompt: Prompt, frame: { index: number; prompted: boolean }): Promise<Sam2Mask> {
const decoded = await this.sessions.maskDecoder.run({
feats0: vision.feats0!,
@@ -133,30 +184,72 @@ export class Sam2Video {
});
try {
const score = floats(decoded, 'object_score_logits');
- const memory = await this.sessions.memoryEncoder.run({
- feats2: vision.feats2!,
- high_res_mask: decoded.high_res_mask!,
- object_score_logits: floatTensor(score, [1, 1]),
- binarize: floatTensor([frame.prompted ? 1 : 0], []),
- });
- try {
- if (!this.bank.hasPositions) {
- this.bank.setPositions((await memory.memory_pos!.getData()) as Float32Array);
- }
-
- const tokens = gpuBuffer(memory, 'memory_tokens');
- const pointer = gpuBuffer(decoded, 'object_pointer');
- if (frame.prompted) this.bank.condition(frame.index, tokens, pointer);
- else this.bank.push(frame.index, tokens, pointer);
- } finally {
- disposeOutputs(memory);
+ const mask = packMask(floats(decoded, 'low_res_mask'), score[0]!, floats(decoded, 'iou')[0]!);
+ if (frame.prompted) {
+ await this.remember(vision, decoded, frame.index, true);
+ return mask;
}
- return packMask(floats(decoded, 'low_res_mask'), score[0]!, floats(decoded, 'iou')[0]!);
+ if (isReliable(mask)) await this.remember(vision, decoded, frame.index, false);
+ return isPlausible(mask) ? mask : { ...mask, bits: new Uint8Array(mask.bits.length) };
} finally {
disposeOutputs(decoded);
}
}
+
+ /** Encodes the frame's mask into memory, as the prompted frame or the next tracked one. */
+ private async remember(vision: Outputs, decoded: Outputs, index: number, prompted: boolean): Promise<void> {
+ const memory = await this.sessions.memoryEncoder.run({
+ feats2: vision.feats2!,
+ high_res_mask: decoded.high_res_mask!,
+ object_score_logits: floatTensor(floats(decoded, 'object_score_logits'), [1, 1]),
+ binarize: floatTensor([prompted ? 1 : 0], []),
+ });
+ try {
+ if (!this.bank.hasPositions) {
+ this.bank.setPositions((await memory.memory_pos!.getData()) as Float32Array);
+ }
+
+ const tokens = gpuBuffer(memory, 'memory_tokens');
+ const pointer = gpuBuffer(decoded, 'object_pointer');
+ if (prompted) this.bank.condition(index, tokens, pointer);
+ else this.bank.push(index, tokens, pointer);
+ } finally {
+ disposeOutputs(memory);
+ }
+ }
+}
+
+/**
+ * Tracked frames enter memory only when the model sees the object and trusts
+ * its mask, as SAMURAI selects them. Around a cut or a lookalike the
+ * predicted IoU collapses while the object score can stay just above zero;
+ * when the object turns or changes shape it only sags, and those frames must
+ * still be remembered for the tracker to follow the new appearance.
+ */
+const RELIABLE_IOU = 0.25;
+
+/**
+ * Below this predicted IoU a tracked mask is reported as empty whatever the
+ * object score: on the test footage only masks of the wrong object, just after
+ * a cut, came out this low.
+ */
+const PLAUSIBLE_IOU = 0.15;
+
+function isReliable(mask: Sam2Mask): boolean {
+ return mask.score > 0 && mask.iou >= RELIABLE_IOU;
+}
+
+function isPlausible(mask: Sam2Mask): boolean {
+ return mask.iou >= PLAUSIBLE_IOU;
+}
+
+function pointPrompt(points: Sam2Point[]): Prompt {
+ if (points.length === 0) throw new Error('A prompt needs at least one point');
+ return {
+ points: floatTensor(points.flatMap((point) => [point.x * IMAGE_SIZE, point.y * IMAGE_SIZE]), [1, 1, points.length, 2]),
+ labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),
+ };
}
/** What the decoder is prompted with on frames that have no points of their own. */
diff --git a/packages/sam2/src/video.ts b/packages/sam2/src/video.ts
index 71049fbb..fa358379 100644
--- a/packages/sam2/src/video.ts
+++ b/packages/sam2/src/video.ts
@@ -8,6 +8,11 @@ import type { InputVideoTrack, VideoSample } from 'mediabunny';
import type { Sam2Mask } from './mask';
import type { Sam2Point, Sam2Video } from './tracker';
+export type FrameRequest = {
+ track: InputVideoTrack;
+ timestamp: number;
+};
+
export type TrackRequest = {
track: InputVideoTrack;
/** Where each frame to segment is in the file, in seconds, ascending. */
@@ -27,6 +32,17 @@ export type TrackRequest = {
*/
const REVERSE_BATCH = 12;
+/**
+ * Decodes one frame and holds it in the model, encoded, so prompts on it
+ * (`model.preview`, `model.seedHeld`) cost the mask decoder alone.
+ */
+export async function holdFrame(model: Sam2Video, request: FrameRequest): Promise<void> {
+ const { track, timestamp } = request;
+ const sample = await new VideoSampleSink(track).getSample(timestamp);
+ if (!sample) throw new Error('The frame could not be decoded');
+ await withVideoFrame(sample, (frame) => model.hold(frame, track.rotation));
+}
+
/**
* Segments the object at the seed and follows it through every frame of the
* request, away from the seed in both directions. Frames that decode to the