The VS Code of video editing, built for agents.
TypeScript
Enhance SAM 2.1 video tracking functionality and update export parameters
konstantin-paulus1882b539c9fd
diff --git a/packages/sam2/scripts/export.py b/packages/sam2/scripts/export.pyindex 4ec0a9c8..c6143482 100644--- a/packages/sam2/scripts/export.py+++ b/packages/sam2/scripts/export.py@@ -3,16 +3,17 @@# dependencies = ["numpy", "onnx==1.23.0", "onnxslim==0.1.96", "torch==2.14.0", "transformers==5.17.0"]# ///"""-Exports the graphs the package loads: SAM 2.1 Hiera-Tiny video tracking as-five fixed-shape ONNX graphs, from the `transformers` port of the checkpoint.+Exports the graphs the package loads: SAM 2.1 video tracking as five+fixed-shape ONNX graphs, from the `transformers` port of a checkpoint.-The input resolution and the number of frame memories attended to are-parameters; the package's constants must match the ones exported. The+The model size (tiny, small, base-plus or large), the input resolution and+the number of frame memories attended to are parameters; the package's+constants must match the ones exported. Thegraphs compute in fp16 but take and return fp32, with a cast at each boundary.Constants are folded in fp32 first, so positional encodings are computed atfull precision before they are stored as halves.- uv run packages/sam2/scripts/export.py <image-size> <memory-frames> <out-dir>+ uv run packages/sam2/scripts/export.py <model-size> <image-size> <memory-frames> <out-dir>"""import hashlib@@ -31,7 +32,6 @@ from torch import nnfrom transformers import Sam2VideoConfig, Sam2VideoModelfrom transformers.models.sam2_video.modeling_sam2_video import NO_OBJ_SCORE, get_1d_sine_pe-CHECKPOINT = 'facebook/sam2.1-hiera-tiny'# Masks come out at this size whatever the input resolution, upsampled from the decoder's.MASK_SIZE = 256MAX_POINTERS = 16@@ -40,8 +40,8 @@ MAX_POINTERS = 16FLOAT32_INPUTS = {('Resize', 1), ('Resize', 2)}-def load(image_size: int) -> Sam2VideoModel:- config = Sam2VideoConfig.from_pretrained(CHECKPOINT)+def load(checkpoint: str, image_size: int) -> Sam2VideoModel:+ config = Sam2VideoConfig.from_pretrained(checkpoint)feat = image_size // 16config.image_size = image_sizeconfig.vision_config.backbone_config.image_size = [image_size, image_size]@@ -49,7 +49,7 @@ def load(image_size: int) -> Sam2VideoModel:config.prompt_encoder_config.image_size = image_sizeconfig.mask_decoder_config.image_size = image_sizeconfig.memory_attention_rope_feat_sizes = [feat, feat]- model = Sam2VideoModel.from_pretrained(CHECKPOINT, config=config, attn_implementation='eager').eval()+ model = Sam2VideoModel.from_pretrained(checkpoint, config=config, attn_implementation='eager').eval()# The backbone's position embedding depends only on the input size; exported as the# computation it is, it brings `If` nodes that the runtime evaluates on the CPU.@@ -301,8 +301,8 @@ def dedupe_constants(model: onnx.ModelProto) -> onnx.ModelProto:return model-def main(image_size: int, memory_frames: int, out: Path) -> None:- model = load(image_size)+def main(model_size: str, image_size: int, memory_frames: int, out: Path) -> None:+ model = load(f'facebook/sam2.1-hiera-{model_size}', image_size)(out / 'onnx').mkdir(parents=True, exist_ok=True)with tempfile.TemporaryDirectory() as fp32:export_fp32(model, memory_frames, Path(fp32))@@ -313,6 +313,7 @@ def main(image_size: int, memory_frames: int, out: Path) -> None:print(f'{path.stem}: {(out / "onnx" / path.name).stat().st_size:,} bytes')constants = {+ 'model': f'sam2.1-hiera-{model_size}','image_size': image_size,'memory_frames': memory_frames,'image_mean': [0.485, 0.456, 0.406],@@ -323,4 +324,4 @@ def main(image_size: int, memory_frames: int, out: Path) -> None:if __name__ == '__main__':- main(int(sys.argv[1]), int(sys.argv[2]), Path(sys.argv[3]))+ main(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), Path(sys.argv[4]))diff --git a/packages/sam2/src/index.ts b/packages/sam2/src/index.tsindex 88aff151..337d766f 100644--- a/packages/sam2/src/index.ts+++ b/packages/sam2/src/index.ts@@ -4,10 +4,10 @@export { loadSam2 } from './load';export { Sam2Video } from './tracker';-export { trackObject } from './video';+export { holdFrame, trackObject } from './video';export { MASK_SIZE, maskHas, maskIsEmpty, renderMask } from './mask';export type { Sam2LoadOptions, Sam2Progress } from './load';export type { Sam2Point } from './tracker';-export type { TrackRequest } from './video';+export type { FrameRequest, TrackRequest } from './video';export type { Sam2Mask } from './mask';diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.tsindex ac858eab..fa1558b7 100644--- a/packages/sam2/src/tracker.ts+++ b/packages/sam2/src/tracker.ts@@ -40,6 +40,8 @@ export class Sam2Video {private readonly bank: MemoryBank;private readonly featureTokens: GPUBuffer;private readonly positionTokens: GPUBuffer;+ /** The frame kept encoded for `preview` and `seedHeld`, so a prompt costs the decoder alone. */+ private held: Outputs | null = null;constructor(device: GPUDevice,@@ -55,21 +57,64 @@ export class Sam2Video {/** Segments the object at `points` and makes this the frame every later one is tracked from. */async seed(frame: VideoFrame, rotation: Rotation, points: Sam2Point[], index: number): Promise<Sam2Mask> {- if (points.length === 0) throw new Error('A prompt needs at least one point');-- const prompt: Prompt = {- points: floatTensor(points.flatMap((point) => [point.x * IMAGE_SIZE, point.y * IMAGE_SIZE]), [1, 1, points.length, 2]),- labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),- };-const vision = await this.encode(frame, rotation);try {- return await this.segment(vision, vision.feats2_no_mem!, prompt, { index, prompted: true });+ return await this.segment(vision, vision.feats2_no_mem!, pointPrompt(points), { index, prompted: true });} finally {disposeOutputs(vision);}}+ /**+ * Encodes a frame and keeps it hot: `preview` and `seedHeld` then run+ * the mask decoder alone, which is what makes prompting feel immediate.+ * The frame held before is dropped.+ */+ async hold(frame: VideoFrame, rotation: Rotation): Promise<void> {+ const vision = await this.encode(frame, rotation);+ this.release();+ this.held = vision;+ }++ get holding(): boolean {+ return this.held !== null;+ }++ /** Drops the held frame. */+ release(): void {+ if (this.held) disposeOutputs(this.held);+ this.held = null;+ }++ /** Segments the held frame at `points` without remembering it: what a click there would pick. */+ async preview(points: Sam2Point[]): Promise<Sam2Mask> {+ const vision = this.heldFrame();+ const prompt = pointPrompt(points);+ const decoded = await this.sessions.maskDecoder.run({+ feats0: vision.feats0!,+ feats1: vision.feats1!,+ feats2_cond: vision.feats2_no_mem!,+ input_points: prompt.points,+ input_labels: prompt.labels,+ });+ try {+ return packMask(floats(decoded, 'low_res_mask'), floats(decoded, 'object_score_logits')[0]!, floats(decoded, 'iou')[0]!);+ } finally {+ disposeOutputs(decoded);+ }+ }++ /** `seed` on the held frame. */+ seedHeld(points: Sam2Point[], index: number): Promise<Sam2Mask> {+ const vision = this.heldFrame();+ return this.segment(vision, vision.feats2_no_mem!, pointPrompt(points), { index, prompted: true });+ }++ private heldFrame(): Outputs {+ if (!this.held) throw new Error('No frame is held');+ return this.held;+ }+/** Finds the seeded object in a frame `index` steps along from the last one. */async track(frame: VideoFrame, rotation: Rotation, index: number, totalFrames: number): Promise<Sam2Mask> {const vision = await this.encode(frame, rotation);@@ -108,6 +153,7 @@ export class Sam2Video {}dispose(): void {+ this.release();this.bank.dispose();this.preprocessor.dispose();this.featureTokens.destroy();@@ -122,7 +168,12 @@ export class Sam2Video {});}- /** Decodes the mask, then encodes the frame into memory as the prompted frame or the next tracked one. */+ /**+ * Decodes the mask, then encodes the frame into memory as the prompted+ * frame or the next tracked one. A tracked frame the model is unsure of is+ * not remembered: a wrong guess kept in memory is what the next frames+ * would match, so it would be reinforced rather than corrected.+ */private async segment(vision: Outputs, conditioned: Tensor, prompt: Prompt, frame: { index: number; prompted: boolean }): Promise<Sam2Mask> {const decoded = await this.sessions.maskDecoder.run({feats0: vision.feats0!,@@ -133,30 +184,72 @@ export class Sam2Video {});try {const score = floats(decoded, 'object_score_logits');- const memory = await this.sessions.memoryEncoder.run({- feats2: vision.feats2!,- high_res_mask: decoded.high_res_mask!,- object_score_logits: floatTensor(score, [1, 1]),- binarize: floatTensor([frame.prompted ? 1 : 0], []),- });- try {- if (!this.bank.hasPositions) {- this.bank.setPositions((await memory.memory_pos!.getData()) as Float32Array);- }-- const tokens = gpuBuffer(memory, 'memory_tokens');- const pointer = gpuBuffer(decoded, 'object_pointer');- if (frame.prompted) this.bank.condition(frame.index, tokens, pointer);- else this.bank.push(frame.index, tokens, pointer);- } finally {- disposeOutputs(memory);+ const mask = packMask(floats(decoded, 'low_res_mask'), score[0]!, floats(decoded, 'iou')[0]!);+ if (frame.prompted) {+ await this.remember(vision, decoded, frame.index, true);+ return mask;}- return packMask(floats(decoded, 'low_res_mask'), score[0]!, floats(decoded, 'iou')[0]!);+ if (isReliable(mask)) await this.remember(vision, decoded, frame.index, false);+ return isPlausible(mask) ? mask : { ...mask, bits: new Uint8Array(mask.bits.length) };} finally {disposeOutputs(decoded);}}++ /** Encodes the frame's mask into memory, as the prompted frame or the next tracked one. */+ private async remember(vision: Outputs, decoded: Outputs, index: number, prompted: boolean): Promise<void> {+ const memory = await this.sessions.memoryEncoder.run({+ feats2: vision.feats2!,+ high_res_mask: decoded.high_res_mask!,+ object_score_logits: floatTensor(floats(decoded, 'object_score_logits'), [1, 1]),+ binarize: floatTensor([prompted ? 1 : 0], []),+ });+ try {+ if (!this.bank.hasPositions) {+ this.bank.setPositions((await memory.memory_pos!.getData()) as Float32Array);+ }++ const tokens = gpuBuffer(memory, 'memory_tokens');+ const pointer = gpuBuffer(decoded, 'object_pointer');+ if (prompted) this.bank.condition(index, tokens, pointer);+ else this.bank.push(index, tokens, pointer);+ } finally {+ disposeOutputs(memory);+ }+ }+}++/**+ * Tracked frames enter memory only when the model sees the object and trusts+ * its mask, as SAMURAI selects them. Around a cut or a lookalike the+ * predicted IoU collapses while the object score can stay just above zero;+ * when the object turns or changes shape it only sags, and those frames must+ * still be remembered for the tracker to follow the new appearance.+ */+const RELIABLE_IOU = 0.25;++/**+ * Below this predicted IoU a tracked mask is reported as empty whatever the+ * object score: on the test footage only masks of the wrong object, just after+ * a cut, came out this low.+ */+const PLAUSIBLE_IOU = 0.15;++function isReliable(mask: Sam2Mask): boolean {+ return mask.score > 0 && mask.iou >= RELIABLE_IOU;+}++function isPlausible(mask: Sam2Mask): boolean {+ return mask.iou >= PLAUSIBLE_IOU;+}++function pointPrompt(points: Sam2Point[]): Prompt {+ if (points.length === 0) throw new Error('A prompt needs at least one point');+ return {+ points: floatTensor(points.flatMap((point) => [point.x * IMAGE_SIZE, point.y * IMAGE_SIZE]), [1, 1, points.length, 2]),+ labels: intTensor(points.map((point) => point.label), [1, 1, points.length]),+ };}/** What the decoder is prompted with on frames that have no points of their own. */diff --git a/packages/sam2/src/video.ts b/packages/sam2/src/video.tsindex 71049fbb..fa358379 100644--- a/packages/sam2/src/video.ts+++ b/packages/sam2/src/video.ts@@ -8,6 +8,11 @@ import type { InputVideoTrack, VideoSample } from 'mediabunny';import type { Sam2Mask } from './mask';import type { Sam2Point, Sam2Video } from './tracker';+export type FrameRequest = {+ track: InputVideoTrack;+ timestamp: number;+};+export type TrackRequest = {track: InputVideoTrack;/** Where each frame to segment is in the file, in seconds, ascending. */@@ -27,6 +32,17 @@ export type TrackRequest = {*/const REVERSE_BATCH = 12;+/**+ * Decodes one frame and holds it in the model, encoded, so prompts on it+ * (`model.preview`, `model.seedHeld`) cost the mask decoder alone.+ */+export async function holdFrame(model: Sam2Video, request: FrameRequest): Promise<void> {+ const { track, timestamp } = request;+ const sample = await new VideoSampleSink(track).getSample(timestamp);+ if (!sample) throw new Error('The frame could not be decoded');+ await withVideoFrame(sample, (frame) => model.hold(frame, track.rotation));+}+/*** Segments the object at the seed and follows it through every frame of the* request, away from the seed in both directions. Frames that decode to the