Skip to content

The VS Code of video editing, built for agents.

TypeScript

Remove `convert_fp16.py` script and introduce `export.py` for ONNX model export

konstantin-paulus70b2fa3ddfae
diff --git a/packages/sam2/scripts/convert_fp16.py b/packages/sam2/scripts/convert_fp16.py
deleted file mode 100644
index a665cc9e..00000000
--- a/packages/sam2/scripts/convert_fp16.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = ["huggingface_hub", "numpy", "onnx==1.23.0", "onnxslim==0.1.96"]
-# ///
-"""
-Builds the fp16 graphs the package loads from the fp32 SAM 2.1 video export.
-
-Constants are folded in fp32 first, so positional encodings are computed at
-full precision before they are stored as halves. The graphs then compute in
-fp16 but keep their fp32 inputs and outputs, with a cast at each boundary, so
-the pipeline feeding them is the same for either precision.
-
- uv run packages/sam2/scripts/convert_fp16.py <out-dir>
-"""
-
-import shutil
-import sys
-from pathlib import Path
-
-import numpy as np
-import onnx
-import onnxslim
-from huggingface_hub import hf_hub_download
-from onnx import TensorProto, helper, numpy_helper
-
-SOURCE_REPO = 'square-zero-labs/sam2.1-tiny-video-onnx'
-SOURCE_REVISION = '3b2984dd865f6e9d2cc6aed0be6a5a5c2eb352ce'
-GRAPHS = ['vision_encoder', 'mask_decoder', 'memory_encoder', 'memory_attention', 'pointer_tpos']
-
-# Folding memory attention's RoPE would store the same table once per layer, doubling the download.
-UNFOLDED = {'memory_attention'}
-FOLD_LIMIT_BYTES = 8 << 20
-
-# Inputs the ONNX spec types as float32 whatever the op computes in, by (op, input index).
-FLOAT32_INPUTS = {('Resize', 1), ('Resize', 2)}
-
-
-def to_fp16(model: onnx.ModelProto) -> onnx.ModelProto:
- graph = model.graph
- keep_float = {
- node.input[i]
- for node in graph.node
- for op, i in FLOAT32_INPUTS
- if node.op_type == op and len(node.input) > i
- }
-
- for init in graph.initializer:
- if init.data_type == TensorProto.FLOAT and init.name not in keep_float:
- init.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(init)), init.name))
-
- for node in graph.node:
- for attr in node.attribute:
- if node.op_type == 'Cast' and attr.name == 'to' and attr.i == TensorProto.FLOAT:
- attr.i = TensorProto.FLOAT16
- elif (
- attr.type == onnx.AttributeProto.TENSOR
- and attr.t.data_type == TensorProto.FLOAT
- and node.op_type in ('Constant', 'ConstantOfShape')
- and node.output[0] not in keep_float
- ):
- attr.t.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(attr.t)), attr.t.name))
-
- # Float inputs are cast down on entry and float outputs cast back up on exit.
- entry, exit = [], []
- for vi in graph.input:
- if vi.type.tensor_type.elem_type == TensorProto.FLOAT:
- half = f'{vi.name}__fp16'
- for node in graph.node:
- node.input[:] = [half if name == vi.name else name for name in node.input]
- entry.append(helper.make_node('Cast', [vi.name], [half], name=f'{vi.name}__to_fp16', to=TensorProto.FLOAT16))
- for vi in graph.output:
- if vi.type.tensor_type.elem_type == TensorProto.FLOAT:
- half = f'{vi.name}__fp16'
- for node in graph.node:
- node.output[:] = [half if name == vi.name else name for name in node.output]
- node.input[:] = [half if name == vi.name else name for name in node.input]
- exit.append(helper.make_node('Cast', [half], [vi.name], name=f'{vi.name}__to_fp32', to=TensorProto.FLOAT))
-
- nodes = entry + list(graph.node) + exit
- del graph.node[:]
- graph.node.extend(nodes)
- del graph.value_info[:]
- onnx.checker.check_model(model)
- return model
-
-
-def to_half(array: np.ndarray) -> np.ndarray:
- return np.clip(array, -65504, 65504).astype(np.float16)
-
-
-def main(out: Path) -> None:
- (out / 'onnx').mkdir(parents=True, exist_ok=True)
- fetch = lambda path: hf_hub_download(SOURCE_REPO, path, revision=SOURCE_REVISION)
-
- shutil.copy(fetch('constants.json'), out / 'constants.json')
- for name in GRAPHS:
- model = onnx.load(fetch(f'onnx/{name}.onnx'))
- if name not in UNFOLDED:
- model = onnxslim.slim(model, size_threshold=FOLD_LIMIT_BYTES)
- onnx.save(to_fp16(model), out / 'onnx' / f'{name}.onnx')
- print(f'{name}: {(out / "onnx" / f"{name}.onnx").stat().st_size:,} bytes')
-
-
-if __name__ == '__main__':
- main(Path(sys.argv[1]))
diff --git a/packages/sam2/scripts/export.py b/packages/sam2/scripts/export.py
new file mode 100644
index 00000000..4ec0a9c8
--- /dev/null
+++ b/packages/sam2/scripts/export.py
@@ -0,0 +1,326 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["numpy", "onnx==1.23.0", "onnxslim==0.1.96", "torch==2.14.0", "transformers==5.17.0"]
+# ///
+"""
+Exports the graphs the package loads: SAM 2.1 Hiera-Tiny video tracking as
+five fixed-shape ONNX graphs, from the `transformers` port of the checkpoint.
+
+The input resolution and the number of frame memories attended to are
+parameters; the package's constants must match the ones exported. The
+graphs compute in fp16 but take and return fp32, with a cast at each boundary.
+Constants are folded in fp32 first, so positional encodings are computed at
+full precision before they are stored as halves.
+
+ uv run packages/sam2/scripts/export.py <image-size> <memory-frames> <out-dir>
+"""
+
+import hashlib
+import json
+import sys
+import tempfile
+from pathlib import Path
+
+import numpy as np
+import onnx
+import onnxslim
+import torch
+import torch.nn.functional as F
+from onnx import TensorProto, helper, numpy_helper
+from torch import nn
+from transformers import Sam2VideoConfig, Sam2VideoModel
+from transformers.models.sam2_video.modeling_sam2_video import NO_OBJ_SCORE, get_1d_sine_pe
+
+CHECKPOINT = 'facebook/sam2.1-hiera-tiny'
+# Masks come out at this size whatever the input resolution, upsampled from the decoder's.
+MASK_SIZE = 256
+MAX_POINTERS = 16
+
+# Inputs the ONNX spec types as float32 whatever the op computes in, by (op, input index).
+FLOAT32_INPUTS = {('Resize', 1), ('Resize', 2)}
+
+
+def load(image_size: int) -> Sam2VideoModel:
+ config = Sam2VideoConfig.from_pretrained(CHECKPOINT)
+ feat = image_size // 16
+ config.image_size = image_size
+ config.vision_config.backbone_config.image_size = [image_size, image_size]
+ config.vision_config.backbone_feature_sizes = [[feat * 4, feat * 4], [feat * 2, feat * 2], [feat, feat]]
+ config.prompt_encoder_config.image_size = image_size
+ config.mask_decoder_config.image_size = image_size
+ config.memory_attention_rope_feat_sizes = [feat, feat]
+ model = Sam2VideoModel.from_pretrained(CHECKPOINT, config=config, attn_implementation='eager').eval()
+
+ # The backbone's position embedding depends only on the input size; exported as the
+ # computation it is, it brings `If` nodes that the runtime evaluates on the CPU.
+ backbone = model.vision_encoder.backbone
+ with torch.no_grad():
+ pos_embed = backbone._get_pos_embed((image_size // 4, image_size // 4))
+ backbone._get_pos_embed = lambda hw: pos_embed
+ return model
+
+
+class VisionEncoder(nn.Module):
+ """A frame's features at three scales, the smallest also with the prompted frame's no-memory embedding."""
+
+ def __init__(self, model: Sam2VideoModel):
+ super().__init__()
+ self.model = model
+ # Depends only on the input size, so it is a constant rather than a computation in fp16.
+ size = model.image_size
+ with torch.no_grad():
+ self.register_buffer('pos_embed', model.vision_encoder(torch.zeros(1, 3, size, size)).fpn_position_encoding[2])
+
+ def forward(self, pixel_values):
+ m = self.model
+ feats = m.vision_encoder(pixel_values, return_dict=True).fpn_hidden_states
+ feats0 = m.mask_decoder.conv_s0(feats[0])
+ feats1 = m.mask_decoder.conv_s1(feats[1])
+ feats2_no_mem = feats[2] + m.no_memory_embedding.view(1, -1, 1, 1)
+ return feats0, feats1, feats[2], feats2_no_mem, self.pos_embed
+
+
+class MaskDecoder(nn.Module):
+ """Prompt encoder and mask decoder, picking the mask and object pointer as the video predictor does."""
+
+ def __init__(self, model: Sam2VideoModel):
+ super().__init__()
+ self.model = model
+
+ def forward(self, feats0, feats1, feats2_cond, input_points, input_labels):
+ m = self.model
+ d = m.mask_decoder
+ sparse, dense = m.prompt_encoder(input_points=input_points, input_labels=input_labels, input_boxes=None, input_masks=None)
+
+ _, channels, height, width = feats2_cond.shape
+ tokens = torch.cat([d.obj_score_token.weight, d.iou_token.weight, d.mask_tokens.weight], dim=0)
+ tokens = torch.cat([tokens[None, None], sparse], dim=2)
+ point_embeddings, image_embeddings = d.transformer(
+ point_embeddings=tokens,
+ image_embeddings=feats2_cond + dense,
+ image_positional_embeddings=m.get_image_wide_positional_embeddings(),
+ attention_similarity=None,
+ target_embedding=None,
+ )
+ iou_token_out = point_embeddings[:, :, 1, :]
+ mask_tokens_out = point_embeddings[:, :, 2 : 2 + d.num_mask_tokens, :]
+
+ image_embeddings = image_embeddings.transpose(2, 3).reshape(1, channels, height, width)
+ upscaled = d.activation(d.upscale_layer_norm(d.upscale_conv1(image_embeddings) + feats1))
+ upscaled = d.activation(d.upscale_conv2(upscaled) + feats0)
+ hyper_in = torch.stack([mlp(mask_tokens_out[:, :, i, :]) for i, mlp in enumerate(d.output_hypernetworks_mlps)], dim=2)
+ _, c, h, w = upscaled.shape
+ masks = (hyper_in @ upscaled.view(1, 1, c, h * w)).view(1, 1, -1, h, w)
+ iou_pred = d.iou_prediction_head(iou_token_out)
+ object_score_logits = d.pred_obj_score_head(point_embeddings[:, :, 0, :])
+
+ # Several candidate masks when there is at most one real point: every tracked frame, and a single click.
+ multimask = (input_labels != -1).sum() <= m.config.multimask_max_pt_num
+ best = torch.argmax(iou_pred[0, 0, 1:]) + 1
+ multi_mask, multi_iou, multi_token = masks[0, 0, best], iou_pred[0, 0, best], mask_tokens_out[0, 0, best]
+
+ # One mask: token 0's, unless it is unstable and the best candidate's is taken instead.
+ single = masks[0, 0, 0]
+ delta = d.dynamic_multimask_stability_delta
+ area_i = (single > delta).sum().float()
+ area_u = (single > -delta).sum().float()
+ stability = torch.where(area_u > 0, area_i / area_u, torch.ones_like(area_u))
+ stable = stability >= d.dynamic_multimask_stability_thresh
+ single_mask = torch.where(stable, single, multi_mask)
+ single_iou = torch.where(stable, iou_pred[0, 0, 0], multi_iou)
+
+ low_res = torch.where(multimask, multi_mask, single_mask)
+ iou = torch.where(multimask, multi_iou, single_iou)
+ token = torch.where(multimask, multi_token, mask_tokens_out[0, 0, 0])
+
+ appearing = object_score_logits[0, 0, 0] > 0
+ low_res = torch.where(appearing, low_res, torch.full_like(low_res, NO_OBJ_SCORE))[None, None]
+ high_res = F.interpolate(low_res, size=(m.image_size, m.image_size), mode='bilinear', align_corners=False)
+ mask = low_res if w == MASK_SIZE else F.interpolate(low_res, size=(MASK_SIZE, MASK_SIZE), mode='bilinear', align_corners=False)
+
+ weight = appearing.float()
+ pointer = weight * m.object_pointer_proj(token[None, None]) + (1 - weight) * m.no_object_pointer
+ return mask, high_res, iou.view(1, 1), object_score_logits, pointer.view(1, 1, -1)
+
+
+class MemoryEncoder(nn.Module):
+ """A frame's memory from its features and mask; `binarize` is 1 for a prompted frame."""
+
+ def __init__(self, model: Sam2VideoModel):
+ super().__init__()
+ self.model = model
+
+ def forward(self, feats2, high_res_mask, object_score_logits, binarize):
+ m = self.model
+ mask = torch.where(binarize > 0.5, (high_res_mask > 0).float(), torch.sigmoid(high_res_mask))
+ mask = mask * m.config.sigmoid_scale_for_mem_enc + m.config.sigmoid_bias_for_mem_enc
+ features, pos = m.memory_encoder(feats2, mask)
+ occluded = 1 - (object_score_logits > 0).float()
+ features = features + occluded.view(1, 1, 1, 1) * m.occlusion_spatial_embedding_parameter.view(1, -1, 1, 1)
+ return features.flatten(2).permute(2, 0, 1), pos.flatten(2).permute(2, 0, 1)
+
+
+class MemoryAttention(nn.Module):
+ """The frame's features conditioned on the memory bank: frame memories, then object pointer tokens."""
+
+ def __init__(self, model: Sam2VideoModel):
+ super().__init__()
+ self.model = model
+
+ def forward(self, current_vision_features, current_vision_position_embeddings, memory, memory_pos):
+ m = self.model
+ feat = m.backbone_feature_sizes[-1][0]
+ out = m.memory_attention(
+ current_vision_features=current_vision_features,
+ current_vision_position_embeddings=current_vision_position_embeddings,
+ memory=memory,
+ memory_posision_embeddings=memory_pos,
+ num_object_pointer_tokens=MAX_POINTERS * (m.hidden_dim // m.mem_dim),
+ )
+ return out.squeeze(1).transpose(1, 2).reshape(1, m.hidden_dim, feat, feat)
+
+
+class PointerTpos(nn.Module):
+ """Temporal positions of the object pointers, from their normalized frame distances."""
+
+ def __init__(self, model: Sam2VideoModel):
+ super().__init__()
+ self.model = model
+
+ def forward(self, normalized_diffs):
+ m = self.model
+ return m.temporal_positional_encoding_projection_layer(get_1d_sine_pe(normalized_diffs, dim=m.hidden_dim))
+
+
+def export_fp32(model: Sam2VideoModel, memory_frames: int, out: Path) -> None:
+ s = model.image_size
+ f = s // 16
+ rows = memory_frames * f * f + MAX_POINTERS * (model.hidden_dim // model.mem_dim)
+
+ def save(module, name, args, inputs, outputs, dynamic_axes=None):
+ with torch.no_grad():
+ torch.onnx.export(
+ module, args, str(out / f'{name}.onnx'), input_names=inputs, output_names=outputs,
+ opset_version=17, dynamo=False, dynamic_axes=dynamic_axes,
+ )
+
+ save(VisionEncoder(model), 'vision_encoder', (torch.randn(1, 3, s, s),),
+ ['pixel_values'], ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed'])
+ save(MaskDecoder(model), 'mask_decoder',
+ (torch.randn(1, 32, 4 * f, 4 * f), torch.randn(1, 64, 2 * f, 2 * f), torch.randn(1, 256, f, f),
+ torch.rand(1, 1, 2, 2) * s, torch.tensor([[[1, 0]]], dtype=torch.int32)),
+ ['feats0', 'feats1', 'feats2_cond', 'input_points', 'input_labels'],
+ ['low_res_mask', 'high_res_mask', 'iou', 'object_score_logits', 'object_pointer'],
+ {'input_points': {2: 'num_points'}, 'input_labels': {2: 'num_points'}})
+ save(MemoryEncoder(model), 'memory_encoder',
+ (torch.randn(1, 256, f, f), torch.randn(1, 1, s, s), torch.randn(1, 1), torch.tensor(1.0)),
+ ['feats2', 'high_res_mask', 'object_score_logits', 'binarize'], ['memory_tokens', 'memory_pos'])
+ save(MemoryAttention(model), 'memory_attention',
+ (torch.randn(f * f, 1, 256), torch.randn(f * f, 1, 256), torch.randn(rows, 1, 64), torch.randn(rows, 1, 64)),
+ ['current_vision_features', 'current_vision_position_embeddings', 'memory', 'memory_pos'], ['conditioned_feats'])
+ save(PointerTpos(model), 'pointer_tpos', (torch.rand(MAX_POINTERS),), ['normalized_diffs'], ['pointer_pos'])
+
+
+def to_fp16(model: onnx.ModelProto) -> onnx.ModelProto:
+ graph = model.graph
+ keep_float = {
+ node.input[i]
+ for node in graph.node
+ for op, i in FLOAT32_INPUTS
+ if node.op_type == op and len(node.input) > i
+ }
+
+ for init in graph.initializer:
+ if init.data_type == TensorProto.FLOAT and init.name not in keep_float:
+ init.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(init)), init.name))
+
+ for node in graph.node:
+ for attr in node.attribute:
+ if node.op_type == 'Cast' and attr.name == 'to' and attr.i == TensorProto.FLOAT:
+ attr.i = TensorProto.FLOAT16
+ elif (
+ attr.type == onnx.AttributeProto.TENSOR
+ and attr.t.data_type == TensorProto.FLOAT
+ and node.op_type in ('Constant', 'ConstantOfShape')
+ and node.output[0] not in keep_float
+ ):
+ attr.t.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(attr.t)), attr.t.name))
+
+ # Float inputs are cast down on entry and float outputs cast back up on exit.
+ entry, exit = [], []
+ for vi in graph.input:
+ if vi.type.tensor_type.elem_type == TensorProto.FLOAT:
+ half = f'{vi.name}__fp16'
+ for node in graph.node:
+ node.input[:] = [half if name == vi.name else name for name in node.input]
+ entry.append(helper.make_node('Cast', [vi.name], [half], name=f'{vi.name}__to_fp16', to=TensorProto.FLOAT16))
+ for vi in graph.output:
+ if vi.type.tensor_type.elem_type == TensorProto.FLOAT:
+ half = f'{vi.name}__fp16'
+ for node in graph.node:
+ node.output[:] = [half if name == vi.name else name for name in node.output]
+ node.input[:] = [half if name == vi.name else name for name in node.input]
+ exit.append(helper.make_node('Cast', [half], [vi.name], name=f'{vi.name}__to_fp32', to=TensorProto.FLOAT))
+
+ nodes = entry + list(graph.node) + exit
+ del graph.node[:]
+ graph.node.extend(nodes)
+ del graph.value_info[:]
+ return model
+
+
+def to_half(array: np.ndarray) -> np.ndarray:
+ return np.clip(array, -65504, 65504).astype(np.float16)
+
+
+def dedupe_constants(model: onnx.ModelProto) -> onnx.ModelProto:
+ """Stores each distinct constant once: folding leaves a copy of the RoPE tables in every attention block."""
+ graph = model.graph
+ for node in [n for n in graph.node if n.op_type == 'Constant' and n.attribute[0].name == 'value']:
+ tensor = onnx.TensorProto()
+ tensor.CopyFrom(node.attribute[0].t)
+ tensor.name = node.output[0]
+ graph.initializer.append(tensor)
+ graph.node.remove(node)
+
+ first: dict[str, str] = {}
+ rename: dict[str, str] = {}
+ kept = []
+ for init in graph.initializer:
+ array = numpy_helper.to_array(init)
+ key = hashlib.sha1(f'{array.dtype}{array.shape}'.encode() + array.tobytes()).hexdigest()
+ if key in first:
+ rename[init.name] = first[key]
+ else:
+ first[key] = init.name
+ kept.append(init)
+ del graph.initializer[:]
+ graph.initializer.extend(kept)
+ for node in graph.node:
+ node.input[:] = [rename.get(name, name) for name in node.input]
+ return model
+
+
+def main(image_size: int, memory_frames: int, out: Path) -> None:
+ model = load(image_size)
+ (out / 'onnx').mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory() as fp32:
+ export_fp32(model, memory_frames, Path(fp32))
+ for path in sorted(Path(fp32).glob('*.onnx')):
+ graph = dedupe_constants(to_fp16(onnxslim.slim(onnx.load(path))))
+ onnx.checker.check_model(graph)
+ onnx.save(graph, out / 'onnx' / path.name)
+ print(f'{path.stem}: {(out / "onnx" / path.name).stat().st_size:,} bytes')
+
+ constants = {
+ 'image_size': image_size,
+ 'memory_frames': memory_frames,
+ 'image_mean': [0.485, 0.456, 0.406],
+ 'image_std': [0.229, 0.224, 0.225],
+ 'memory_temporal_positional_encoding': model.memory_temporal_positional_encoding.detach().reshape(model.num_maskmem, -1).tolist(),
+ }
+ (out / 'constants.json').write_text(json.dumps(constants))
+
+
+if __name__ == '__main__':
+ main(int(sys.argv[1]), int(sys.argv[2]), Path(sys.argv[3]))
diff --git a/packages/sam2/src/constants.ts b/packages/sam2/src/constants.ts
index a2ee6eff..7794c085 100644
--- a/packages/sam2/src/constants.ts
+++ b/packages/sam2/src/constants.ts
@@ -3,31 +3,33 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
- * The graphs compute in fp16 but take and return fp32, so the pipeline is the
- * same as for the fp32 export they are built from; `scripts/convert_fp16.py`
- * makes them. Pinned to a commit, which is part of the cache key.
+ * Made by `scripts/export.py 512 7`: fixed shapes for `IMAGE_SIZE` and
+ * `MEMORY_FRAMES` below, fp16 compute behind fp32 inputs and outputs.
+ * Pinned to a commit, which is part of the cache key.
*/
-export const MODEL_REPO = 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/9a1ecaa5b194cc9c5bd7840036301037fe3f6b13';
+export const MODEL_REPO = 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e';
export const MODEL_FILES = {
- constants: { path: 'constants.json', size: 9_922 },
- visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 67_221_881 },
- maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_928_987 },
- memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_822_478 },
- memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_220_704 },
- pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_236 },
+ constants: { path: 'constants.json', size: 9_781 },
+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 58_385_353 },
+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_898_805 },
+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_807_753 },
+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 13_029_470 },
+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_228 },
} as const;
-export const IMAGE_SIZE = 1024;
+/** The model's input, half the resolution SAM 2 was trained at: a quarter of the encoder's work. */
+export const IMAGE_SIZE = 512;
+/** The decoder's masks, upsampled in-graph from its own `IMAGE_SIZE / 4`. */
export const MASK_SIZE = 256;
-export const FEAT_SIZE = 64;
+export const FEAT_SIZE = IMAGE_SIZE / 16;
export const FEAT_TOKENS = FEAT_SIZE * FEAT_SIZE;
export const HIDDEN_DIM = 256;
export const MEM_DIM = 64;
-export const NUM_MASKMEM = 7;
+export const MEMORY_FRAMES = 7;
export const MAX_POINTERS = 16;
export const POINTER_TOKENS = HIDDEN_DIM / MEM_DIM;
-export const MEMORY_ROWS = NUM_MASKMEM * FEAT_TOKENS + MAX_POINTERS * POINTER_TOKENS;
+export const MEMORY_ROWS = MEMORY_FRAMES * FEAT_TOKENS + MAX_POINTERS * POINTER_TOKENS;
export const BYTES_PER_FLOAT = 4;
export const MEMORY_ROW_BYTES = MEM_DIM * BYTES_PER_FLOAT;
@@ -36,6 +38,8 @@ export const POINTER_BYTES = HIDDEN_DIM * BYTES_PER_FLOAT;
/** The values `constants.json` in the model repo carries. */
export type ModelConstants = {
+ image_size: number;
+ memory_frames: number;
image_mean: [number, number, number];
image_std: [number, number, number];
memory_temporal_positional_encoding: number[][];
diff --git a/packages/sam2/src/gpu.ts b/packages/sam2/src/gpu.ts
index 3785120a..2b4f451a 100644
--- a/packages/sam2/src/gpu.ts
+++ b/packages/sam2/src/gpu.ts
@@ -35,7 +35,7 @@ function dispatch(device: GPUDevice, pipeline: GPUComputePipeline, bindGroup: GP
}
const PARAMS_BYTES = 48;
-const MAX_TAPS = 4;
+const MAX_TAPS = 8;
/** Uploads a frame and turns it into the vision encoder's `pixel_values`. */
export class FramePreprocessor {
diff --git a/packages/sam2/src/load.ts b/packages/sam2/src/load.ts
index e8a9e09d..a6b90db1 100644
--- a/packages/sam2/src/load.ts
+++ b/packages/sam2/src/load.ts
@@ -2,6 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+import { IMAGE_SIZE, MEMORY_FRAMES } from './constants';
import { configureRuntime, createSession, runtimeDevice } from './sessions';
import { Sam2Video } from './tracker';
import { fetchModelFiles } from './weights';
@@ -34,6 +35,9 @@ async function load({ onProgress, signal }: Sam2LoadOptions): Promise<Sam2Video>
const files = await fetchModelFiles((progress) => onProgress?.({ phase: 'download', ...progress }), signal);
const constants = JSON.parse(new TextDecoder().decode(files.constants)) as ModelConstants;
+ if (constants.image_size !== IMAGE_SIZE || constants.memory_frames !== MEMORY_FRAMES) {
+ throw new Error(`The model is for ${constants.image_size}px and ${constants.memory_frames} memories, the pipeline for ${IMAGE_SIZE}px and ${MEMORY_FRAMES}`);
+ }
onProgress?.({ phase: 'compile' });
// The runtime builds one WebGPU session at a time.
diff --git a/packages/sam2/src/memory.ts b/packages/sam2/src/memory.ts
index 0dc89a29..1ad3d757 100644
--- a/packages/sam2/src/memory.ts
+++ b/packages/sam2/src/memory.ts
@@ -3,8 +3,8 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import {
- MAX_POINTERS, MEM_DIM, MEMORY_BLOCK_BYTES, MEMORY_ROW_BYTES, MEMORY_ROWS,
- NUM_MASKMEM, POINTER_BYTES, POINTER_TOKENS,
+ MAX_POINTERS, MEM_DIM, MEMORY_BLOCK_BYTES, MEMORY_FRAMES, MEMORY_ROW_BYTES, MEMORY_ROWS,
+ POINTER_BYTES, POINTER_TOKENS,
} from './constants';
import { createStorageBuffer } from './gpu';
@@ -81,7 +81,8 @@ export class MemoryBank {
/**
* Lays the bank out for frame `index` and returns each pointer's distance
* to it, normalized the way `pointer_tpos` expects. Slot 0 is the prompted
- * frame, slots 1 to 6 the newest frames first; empty slots and pointers
+ * frame, with the temporal table's last row; the other slots are the newest
+ * frames first, a frame `k` back with row `k - 1`. Empty slots and pointers
* repeat the newest entry, which the graph's fixed shapes require.
*/
assemble(index: number, totalFrames: number): number[] {
@@ -90,8 +91,8 @@ export class MemoryBank {
if (!conditioning || !positions) throw new Error('The memory bank has no prompted frame');
const newestFirst = [...this.recent].reverse();
- const blocks = [{ tokens: conditioning.tokens, pos: positions[NUM_MASKMEM - 1]! }];
- for (let slot = 1; slot < NUM_MASKMEM; slot++) {
+ const blocks = [{ tokens: conditioning.tokens, pos: positions[positions.length - 1]! }];
+ for (let slot = 1; slot < MEMORY_FRAMES; slot++) {
const entry = newestFirst[slot - 1];
blocks.push(entry ? { tokens: entry.tokens, pos: positions[slot - 1]! } : blocks[1] ?? blocks[0]!);
}
@@ -153,4 +154,4 @@ export class MemoryBank {
}
}
-const POINTER_OFFSET = NUM_MASKMEM * MEMORY_BLOCK_BYTES;
+const POINTER_OFFSET = MEMORY_FRAMES * MEMORY_BLOCK_BYTES;