Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

S2.64: SAM 2.1 Small, Base+ and Large, each checked against PyTorch

Kristian Myram2acd0ca1a8db
diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rs
index a584aebbb..d6d9ef63b 100644
--- a/crates/engine/src/analysis/models.rs
+++ b/crates/engine/src/analysis/models.rs
@@ -336,7 +336,7 @@ fn load_file(path: &Path, id: &str) -> std::result::Result<Loaded, String> {
let bytes = std::fs::read(path).map_err(|e| format!("cannot read {}: {e}", path.display()))?;
// SAM 2 runs on candle and is native only; every other model loads through the registry.
#[cfg(not(target_arch = "wasm32"))]
- if id == seg::SAM2_TINY.id {
+ if effectcraft_sam2::mask::SIZES.iter().any(|m| m.id == id) {
return effectcraft_sam2::mask::load(&bytes).map(|m| Loaded::Mask(m));
}
seg::load(id, &bytes)
diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs
index fab025a3a..ee05ea6b8 100644
--- a/crates/engine/src/tests/models.rs
+++ b/crates/engine/src/tests/models.rs
@@ -36,8 +36,8 @@ fn models_are_listed_chosen_and_verified() {
for (prefix, model, task) in [("roto", "mobilesam", Task::Mask), ("face", "mediapipe-face", Task::Face)] {
let list = s.execute_checked(&format!("{prefix}.models"), json!({})).unwrap();
let ids: Vec<&str> = list["models"].as_array().unwrap().iter().map(|m| m["id"].as_str().unwrap()).collect();
- // Roto Brush also offers SAM 2.1 Tiny (a video tracker; see crates/sam2).
- let expect: Vec<&str> = if task == Task::Mask { vec!["classical", model, "sam2.1-hiera-tiny"] } else { vec!["classical", model] };
+ // Roto Brush also offers SAM 2.1 in four sizes (video trackers; see crates/sam2).
+ let expect: Vec<&str> = if task == Task::Mask { vec!["classical", model, "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] } else { vec!["classical", model] };
assert_eq!(ids, expect);
let m = &list["models"][1];
assert_eq!((m["licence"].as_str(), m["installed"].as_bool()), (Some("Apache-2.0"), Some(false)));
diff --git a/crates/sam2/src/gpu/encoder.rs b/crates/sam2/src/gpu/encoder.rs
index 630f4f09b..1d390a9d1 100644
--- a/crates/sam2/src/gpu/encoder.rs
+++ b/crates/sam2/src/gpu/encoder.rs
@@ -244,6 +244,8 @@ impl GpuEncoder {
for (i, b) in self.blocks.iter().enumerate() {
let (nx, nh, nw) = self.block(b, &x, h, w)?;
(x, h, w) = (nx, nh, nw);
+ // submit per block so the GPU can free the previous block's intermediates (Large would otherwise run out of memory)
+ g.flush()?;
if self.stage_ends.contains(&i) {
stages.push((x.clone(), h, w, b.plan.dim_out));
}
diff --git a/crates/sam2/src/gpu/mod.rs b/crates/sam2/src/gpu/mod.rs
index 9ca180932..5b091062c 100644
--- a/crates/sam2/src/gpu/mod.rs
+++ b/crates/sam2/src/gpu/mod.rs
@@ -147,11 +147,17 @@ impl Gpu {
}
pub fn upload(&self, data: &[f32]) -> Buf {
- let buffer = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ // `write_buffer` instead of a mapped-at-creation buffer: on a lost or out-of-memory device the latter panics inside wgpu,
+ // while this records the error for `take_error`.
+ let buffer = self.device.create_buffer(&wgpu::BufferDescriptor {
label: None,
- contents: bytemuck::cast_slice(data),
- usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
+ size: (data.len().max(1) as u64) * 4,
+ usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
+ mapped_at_creation: false,
});
+ if !data.is_empty() {
+ self.queue.write_buffer(&buffer, 0, bytemuck::cast_slice(data));
+ }
Buf { buffer, len: data.len() }
}
diff --git a/crates/sam2/src/hiera.rs b/crates/sam2/src/hiera.rs
index 246448e86..82862c829 100644
--- a/crates/sam2/src/hiera.rs
+++ b/crates/sam2/src/hiera.rs
@@ -42,6 +42,45 @@ impl HieraConfig {
pos_background: 7,
}
}
+
+ /// `facebook/sam2.1-hiera-small`: the same widths as Tiny with deeper third and fourth stages.
+ pub fn small() -> HieraConfig {
+ HieraConfig { blocks: vec![1, 2, 11, 2], global_blocks: vec![7, 10, 13], ..HieraConfig::tiny() }
+ }
+
+ /// `facebook/sam2.1-hiera-base-plus`.
+ pub fn base_plus() -> HieraConfig {
+ HieraConfig {
+ embed_dims: vec![112, 224, 448, 896],
+ blocks: vec![2, 3, 16, 3],
+ heads: vec![2, 4, 8, 16],
+ global_blocks: vec![12, 16, 20],
+ pos_background: 14,
+ ..HieraConfig::tiny()
+ }
+ }
+
+ /// `facebook/sam2.1-hiera-large`.
+ pub fn large() -> HieraConfig {
+ HieraConfig {
+ embed_dims: vec![144, 288, 576, 1152],
+ blocks: vec![2, 6, 36, 4],
+ heads: vec![2, 4, 8, 16],
+ windows: vec![8, 4, 16, 8],
+ global_blocks: vec![23, 33, 43],
+ ..HieraConfig::tiny()
+ }
+ }
+
+ /// The configuration of a released size.
+ pub fn for_size(size: crate::Size) -> HieraConfig {
+ match size {
+ crate::Size::Tiny => HieraConfig::tiny(),
+ crate::Size::Small => HieraConfig::small(),
+ crate::Size::BasePlus => HieraConfig::base_plus(),
+ crate::Size::Large => HieraConfig::large(),
+ }
+ }
}
fn at<T: Copy>(v: &[T], i: usize, what: &str) -> Result<T> {
diff --git a/crates/sam2/src/lib.rs b/crates/sam2/src/lib.rs
index 59eceddb3..09cbb1466 100644
--- a/crates/sam2/src/lib.rs
+++ b/crates/sam2/src/lib.rs
@@ -82,6 +82,19 @@ impl Size {
Size::Large => "facebook/sam2.1-hiera-large",
}
}
+
+ /// The size a checkpoint is, from its tensors: the width of the patch embedding (96 for Tiny and Small, 112 for
+ /// Base+, 144 for Large) and the number of blocks (12 for Tiny, 16 for Small).
+ pub fn detect(w: &weights::Weights) -> Result<Size> {
+ let width = w.shape("vision_encoder.backbone.patch_embed.projection.weight").and_then(|s| s.first().copied());
+ Ok(match width {
+ Some(112) => Size::BasePlus,
+ Some(144) => Size::Large,
+ Some(96) if w.contains("vision_encoder.backbone.blocks.15.layer_norm1.weight") => Size::Small,
+ Some(96) => Size::Tiny,
+ _ => return Err(Error::Weights("this checkpoint is not a SAM 2.1 Hiera size (patch embedding width)".into())),
+ })
+ }
}
#[cfg(test)]
diff --git a/crates/sam2/src/mask.rs b/crates/sam2/src/mask.rs
index 4578af0f9..251bb05ca 100644
--- a/crates/sam2/src/mask.rs
+++ b/crates/sam2/src/mask.rs
@@ -53,6 +53,7 @@ fn fingerprint(rgb: &[[f32; 3]], w: usize, h: usize) -> u64 {
/// SAM 2 for single frames.
pub struct Sam2Mask {
model: Arc<VideoModel>,
+ info: &'static ModelInfo,
last: Mutex<Option<(u64, Arc<Encoded>)>>,
used: Mutex<std::time::Instant>,
pub(crate) peak: std::sync::atomic::AtomicU64,
@@ -60,7 +61,12 @@ pub struct Sam2Mask {
impl Sam2Mask {
pub fn new(model: Arc<VideoModel>) -> Sam2Mask {
- Sam2Mask { model, last: Mutex::new(None), used: Mutex::new(std::time::Instant::now()), peak: std::sync::atomic::AtomicU64::new(0) }
+ Sam2Mask::with_info(model, &SAM2_TINY)
+ }
+
+ /// A model of a given registry entry (its size).
+ pub fn with_info(model: Arc<VideoModel>, info: &'static ModelInfo) -> Sam2Mask {
+ Sam2Mask { model, info, last: Mutex::new(None), used: Mutex::new(std::time::Instant::now()), peak: std::sync::atomic::AtomicU64::new(0) }
}
/// Note that the model is being used (for [`MaskModel::idle_secs`]).
@@ -106,7 +112,7 @@ impl Sam2Mask {
impl MaskModel for Sam2Mask {
fn info(&self) -> &'static ModelInfo {
- &SAM2_TINY
+ self.info
}
fn segment(&self, rgb: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result<Vec<f32>> {
@@ -134,14 +140,17 @@ impl MaskModel for Sam2Mask {
/// Verify the installed file against the registry and build the model.
pub fn load(bytes: &[u8]) -> std::result::Result<Arc<Sam2Mask>, String> {
let got = effectcraft_segment::sha256::hex(bytes);
- if got != SAM2_TINY.sha256 {
- return Err(format!("{}: the file does not match (SHA-256 {got}, expected {})", SAM2_TINY.name, SAM2_TINY.sha256));
- }
+ let Some(info) = SIZES.iter().find(|m| m.sha256 == got) else {
+ return Err(format!("SAM 2.1: the file does not match any published size (SHA-256 {got})"));
+ };
let w = crate::weights::Weights::from_bytes(bytes).map_err(|e| e.to_string())?;
let model = VideoModel::load_auto(&w).map_err(|e| e.to_string())?;
- Ok(Arc::new(Sam2Mask::new(Arc::new(model))))
+ Ok(Arc::new(Sam2Mask::with_info(Arc::new(model), info)))
}
+/// The registry entries this crate loads.
+pub const SIZES: [&ModelInfo; 4] = [&SAM2_TINY, &effectcraft_segment::SAM2_SMALL, &effectcraft_segment::SAM2_BASE_PLUS, &effectcraft_segment::SAM2_LARGE];
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/sam2/src/model.rs b/crates/sam2/src/model.rs
index c30cbb5b9..36e30e310 100644
--- a/crates/sam2/src/model.rs
+++ b/crates/sam2/src/model.rs
@@ -32,7 +32,7 @@ impl ImageModel {
pub fn load(w: &Weights) -> Result<ImageModel> {
let prompt = PromptEncoder::load(w)?;
Ok(ImageModel {
- vision: VisionEncoder::load_tiny(w)?,
+ vision: VisionEncoder::load(w)?,
decoder: MaskDecoder::load(w)?,
no_memory: w.get("no_memory_embedding", &[1, 1, 256])?.reshape((1, 256, 1, 1))?,
image_pos: prompt.image_positions()?,
diff --git a/crates/sam2/src/neck.rs b/crates/sam2/src/neck.rs
index 01ab5523b..c897e9818 100644
--- a/crates/sam2/src/neck.rs
+++ b/crates/sam2/src/neck.rs
@@ -142,9 +142,11 @@ pub struct ImageFeatures {
}
impl VisionEncoder {
- pub fn load_tiny(w: &Weights) -> Result<VisionEncoder> {
- let cfg = HieraConfig::tiny();
- Ok(VisionEncoder { hiera: std::sync::Arc::new(Hiera::load(w, &cfg)?), neck: Neck::load(w, &[768, 384, 192, 96], 256, &[2, 3])?, levels: 3 })
+ /// The encoder of whichever size the checkpoint is.
+ pub fn load(w: &Weights) -> Result<VisionEncoder> {
+ let cfg = HieraConfig::for_size(crate::Size::detect(w)?);
+ let channels: Vec<usize> = cfg.embed_dims.iter().rev().copied().collect();
+ Ok(VisionEncoder { hiera: std::sync::Arc::new(Hiera::load(w, &cfg)?), neck: Neck::load(w, &channels, 256, &[2, 3])?, levels: 3 })
}
pub fn forward(&self, pixels: &Tensor) -> Result<ImageFeatures> {
diff --git a/crates/sam2/src/weights.rs b/crates/sam2/src/weights.rs
index cfbd6a506..ca246b143 100644
--- a/crates/sam2/src/weights.rs
+++ b/crates/sam2/src/weights.rs
@@ -46,6 +46,15 @@ impl Weights {
Ok(Weights { map: out })
}
+ pub fn contains(&self, name: &str) -> bool {
+ self.map.contains_key(name)
+ }
+
+ /// The shape of a tensor, if the checkpoint has it.
+ pub fn shape(&self, name: &str) -> Option<Vec<usize>> {
+ self.map.get(name).map(|t| t.dims().to_vec())
+ }
+
pub fn len(&self) -> usize {
self.map.len()
}
diff --git a/crates/sam2/tests/reference.rs b/crates/sam2/tests/reference.rs
index d121df840..01648e329 100644
--- a/crates/sam2/tests/reference.rs
+++ b/crates/sam2/tests/reference.rs
@@ -33,6 +33,7 @@ fn load() -> Option<(Weights, HashMap<String, Tensor>)> {
}
let w = Weights::open(&wf).unwrap();
let r = candle_core::safetensors::load(&rf, &Device::Cpu).unwrap();
+ note_size(&w);
Some((w, r))
}
@@ -41,6 +42,30 @@ fn max_abs(a: &Tensor, b: &Tensor) -> f32 {
(a - b).unwrap().abs().unwrap().flatten_all().unwrap().max(0).unwrap().to_scalar::<f32>().unwrap()
}
+/// The largest magnitude in a tensor.
+fn peak(t: &Tensor) -> f32 {
+ t.abs().unwrap().flatten_all().unwrap().max(0).unwrap().to_scalar::<f32>().unwrap()
+}
+
+/// Set when the checkpoint is Base+ or Large (24 and 48 blocks): f32 rounding error is relative and accumulates with
+/// depth, and these backbones carry much larger activations (Large peaks at 140). Tiny and Small keep the strict bounds.
+static DEEP: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
+fn note_size(w: &Weights) {
+ let size = effectcraft_sam2::Size::detect(w).unwrap();
+ DEEP.store(matches!(size, effectcraft_sam2::Size::BasePlus | effectcraft_sam2::Size::Large), std::sync::atomic::Ordering::Relaxed);
+}
+
+/// `base`, or for the deep backbones the larger of `base` and 5e-4 of the activations' peak.
+fn tol(base: f32, want: &Tensor) -> f32 {
+ if DEEP.load(std::sync::atomic::Ordering::Relaxed) { base.max(5e-4 * peak(want)) } else { base }
+}
+
+/// `base`, ×4 for the deep backbones.
+fn loose(base: f32) -> f32 {
+ if DEEP.load(std::sync::atomic::Ordering::Relaxed) { base * 4.0 } else { base }
+}
+
/// f32 activations differ from PyTorch by accumulation order only; the Hiera blocks stay well
/// inside this bound (measured max 2.6e-5, 2026-10-15).
const HIERA_TOL: f32 = 1e-4;
@@ -48,25 +73,26 @@ const HIERA_TOL: f32 = 1e-4;
#[test]
fn hiera_matches_pytorch_block_by_block() {
let Some((w, r)) = load() else { return };
- let hiera = Hiera::load(&w, &HieraConfig::tiny()).unwrap();
+ let hiera = Hiera::load(&w, &HieraConfig::for_size(effectcraft_sam2::Size::detect(&w).unwrap())).unwrap();
let pixels = r.get("input.pixel_values").unwrap();
let (emb, blocks) = hiera.forward_all(pixels).unwrap();
let e = max_abs(&emb, r.get("vision_encoder.backbone.patch_embed.0").unwrap());
eprintln!("patch_embed max abs error {e:e}");
assert!(e < 1e-4, "patch embed error {e}");
- assert_eq!(blocks.len(), 12);
+ assert_eq!(blocks.len(), HieraConfig::for_size(effectcraft_sam2::Size::detect(&w).unwrap()).blocks.iter().sum::<usize>());
for (i, b) in blocks.iter().enumerate() {
let want = r.get(&format!("vision_encoder.backbone.blocks.{i}.0")).unwrap();
let e = max_abs(b, want);
eprintln!("block {i}: max abs error {e:e}");
- assert!(e < HIERA_TOL, "block {i} error {e} (tolerance {HIERA_TOL})");
+ let t = tol(HIERA_TOL, want);
+ assert!(e < t, "block {i} error {e} (tolerance {t}, peak {})", peak(want));
}
}
#[test]
fn hiera_is_deterministic_and_rejects_bad_sizes() {
let Some((w, r)) = load() else { return };
- let hiera = Hiera::load(&w, &HieraConfig::tiny()).unwrap();
+ let hiera = Hiera::load(&w, &HieraConfig::for_size(effectcraft_sam2::Size::detect(&w).unwrap())).unwrap();
let pixels = r.get("input.pixel_values").unwrap();
let a = hiera.forward(pixels).unwrap();
let b = hiera.forward(pixels).unwrap();
@@ -85,7 +111,7 @@ const NECK_TOL: f32 = 1e-4;
#[test]
fn neck_and_vision_encoder_match_pytorch() {
let Some((w, r)) = load() else { return };
- let enc = effectcraft_sam2::neck::VisionEncoder::load_tiny(&w).unwrap();
+ let enc = effectcraft_sam2::neck::VisionEncoder::load(&w).unwrap();
let pixels = r.get("input.pixel_values").unwrap();
let stages = enc.hiera.forward(pixels).unwrap();
let out = enc.neck.forward(&stages).unwrap();
@@ -93,7 +119,8 @@ fn neck_and_vision_encoder_match_pytorch() {
for (i, f) in out.features.iter().enumerate() {
let e = max_abs(f, r.get(&format!("vision_encoder.neck.{i}")).unwrap());
eprintln!("neck feature {i}: max abs error {e:e}");
- assert!(e < NECK_TOL, "neck feature {i} error {e}");
+ let want = r.get(&format!("vision_encoder.neck.{i}")).unwrap();
+ assert!(e < tol(NECK_TOL, want), "neck feature {i} error {e} (peak {})", peak(want));
}
for (i, p) in out.positions.iter().enumerate() {
let e = max_abs(p, r.get(&format!("vision_encoder.neck.{}", i + 4)).unwrap());
@@ -105,7 +132,7 @@ fn neck_and_vision_encoder_match_pytorch() {
let dims: Vec<usize> = feats.features.iter().map(|f| f.dim(2).unwrap()).collect();
assert_eq!(dims, vec![256, 128, 64]);
let e = max_abs(feats.features.first().unwrap(), r.get("vision_encoder.neck.3").unwrap());
- assert!(e < NECK_TOL);
+ assert!(e < tol(NECK_TOL, r.get("vision_encoder.neck.3").unwrap()));
}
/// Click-prompted single image: prompt embeddings and decoder outputs against `Sam2Model`.
@@ -133,7 +160,7 @@ fn prompt_encoder_and_decoder_match_pytorch() {
let want_masks = r.get("mask_decoder.0").unwrap().squeeze(0).unwrap();
let e = max_abs(&out.masks, &want_masks);
eprintln!("masks: max abs error {e:e} (logit range {:?})", want_masks.flatten_all().unwrap().max(0).unwrap().to_scalar::<f32>());
- assert!(e < DECODER_TOL, "mask logits error {e}");
+ assert!(e < loose(DECODER_TOL), "mask logits error {e}");
let e = max_abs(&out.iou, &r.get("mask_decoder.1").unwrap().squeeze(0).unwrap());
eprintln!("iou: max abs error {e:e}");
assert!(e < 1e-4, "iou error {e}");
@@ -141,7 +168,7 @@ fn prompt_encoder_and_decoder_match_pytorch() {
eprintln!("object score: max abs error {e:e}");
assert!(e < 1e-4, "object score error {e}");
let e = max_abs(&out.sam_tokens, &r.get("mask_decoder.2").unwrap().squeeze(0).unwrap());
- assert!(e < 1e-4, "sam token error {e}");
+ assert!(e < loose(1e-4), "sam token error {e}");
}
fn video_fixture_file() -> PathBuf {
@@ -155,7 +182,9 @@ fn load_video() -> Option<(Weights, HashMap<String, Tensor>)> {
eprintln!("skipped: SAM 2 weights or video reference tensors not found ({} / {})", wf.display(), rf.display());
return None;
}
- Some((Weights::open(&wf).unwrap(), candle_core::safetensors::load(&rf, &Device::Cpu).unwrap()))
+ let w = Weights::open(&wf).unwrap();
+ note_size(&w);
+ Some((w, candle_core::safetensors::load(&rf, &Device::Cpu).unwrap()))
}
/// Measured max 3.5e-6 (memory features), 2.6e-5 (memory attention), 2026-10-15.
@@ -642,13 +671,13 @@ fn gpu_encoder_matches_the_cpu_path_and_pytorch() {
let low_cpu = cpu.features.features.get(2).unwrap();
let e = max_abs(&got.low, low_cpu);
eprintln!("low level vs CPU: max abs error {e:e}");
- assert!(e < GPU_TOL, "low level error {e}");
+ assert!(e < tol(GPU_TOL, low_cpu), "low level error {e} (peak {})", peak(low_cpu));
let e = max_abs(&got.low, r.get("vision_encoder.neck.1").unwrap());
eprintln!("low level vs PyTorch: max abs error {e:e}");
- assert!(e < GPU_TOL, "low level vs PyTorch error {e}");
+ assert!(e < tol(GPU_TOL, low_cpu), "low level vs PyTorch error {e}");
let (e0, e1) = (max_abs(&got.high.0, &cpu.high.0), max_abs(&got.high.1, &cpu.high.1));
eprintln!("skip features vs CPU: {e0:e} / {e1:e}");
- assert!(e0 < GPU_TOL && e1 < GPU_TOL, "skip feature errors {e0} / {e1}");
+ assert!(e0 < tol(GPU_TOL, &cpu.high.0) && e1 < tol(GPU_TOL, &cpu.high.1), "skip feature errors {e0} / {e1}");
}
/// The GPU memory attention against PyTorch (the fixture's frame: one memory frame and four pointer tokens).
diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs
index 3ebf964f9..fb16f01ce 100644
--- a/crates/segment/src/lib.rs
+++ b/crates/segment/src/lib.rs
@@ -185,6 +185,54 @@ pub const SAM2_TINY: ModelInfo = ModelInfo {
sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7",
};
+/// SAM 2.1 Small: the Small checkpoint, pinned to a commit like [`SAM2_TINY`].
+pub const SAM2_SMALL: ModelInfo = ModelInfo {
+ id: "sam2.1-hiera-small",
+ task: Task::Mask,
+ name: "SAM 2.1 Small",
+ description: "Meta's Segment Anything Model 2 (Hiera-Small, 46M parameters). The same streaming-memory tracker as SAM 2.1 Tiny with a larger backbone: a little more accurate on thin parts and busy backgrounds, a little slower.",
+ authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint",
+ licence: "Apache-2.0",
+ licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
+ homepage: "https://github.com/facebookresearch/sam2",
+ url: "https://huggingface.co/facebook/sam2.1-hiera-small/resolve/ee5bba1d82bb8749febdf90f45e84b687142ba03/model.safetensors",
+ file_name: "sam2.1-hiera-small.safetensors",
+ size: 184_305_280,
+ sha256: "0a4067b11ce1e23d5229203f11c718a823060d15a4b23fa2372a7d4b77cbbc60",
+};
+
+/// SAM 2.1 Base+: the Base+ checkpoint, pinned to a commit like [`SAM2_TINY`].
+pub const SAM2_BASE_PLUS: ModelInfo = ModelInfo {
+ id: "sam2.1-hiera-base-plus",
+ task: Task::Mask,
+ name: "SAM 2.1 Base+",
+ description: "Meta's Segment Anything Model 2 (Hiera-Base+, 81M parameters). The same streaming-memory tracker as SAM 2.1 Tiny with a larger backbone: better on difficult footage; wants a GPU for comfortable speed.",
+ authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint",
+ licence: "Apache-2.0",
+ licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
+ homepage: "https://github.com/facebookresearch/sam2",
+ url: "https://huggingface.co/facebook/sam2.1-hiera-base-plus/resolve/b7320756a13354e7530a63935656d35b2f91a290/model.safetensors",
+ file_name: "sam2.1-hiera-base-plus.safetensors",
+ size: 323_476_296,
+ sha256: "2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780",
+};
+
+/// SAM 2.1 Large: the Large checkpoint, pinned to a commit like [`SAM2_TINY`].
+pub const SAM2_LARGE: ModelInfo = ModelInfo {
+ id: "sam2.1-hiera-large",
+ task: Task::Mask,
+ name: "SAM 2.1 Large",
+ description: "Meta's Segment Anything Model 2 (Hiera-Large, 224M parameters). The same streaming-memory tracker as SAM 2.1 Tiny with a larger backbone: the most accurate; about 900 MB and meant for a GPU.",
+ authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint",
+ licence: "Apache-2.0",
+ licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
+ homepage: "https://github.com/facebookresearch/sam2",
+ url: "https://huggingface.co/facebook/sam2.1-hiera-large/resolve/665f8e2ad61cf5f53d65644ff27c8ee525124610/model.safetensors",
+ file_name: "sam2.1-hiera-large.safetensors",
+ size: 897_897_416,
+ sha256: "dc407dce21301fd94abb395c5099b4f2c455fdc8a8f261ac3d0ea6d4cd197230",
+};
+
pub const FACE_LANDMARKER: ModelInfo = ModelInfo {
id: "mediapipe-face",
task: Task::Face,
@@ -201,7 +249,7 @@ pub const FACE_LANDMARKER: ModelInfo = ModelInfo {
};
/// The trained models, besides the built-in classical engines.
-pub const MODELS: &[ModelInfo] = &[MOBILE_SAM, SAM2_TINY, FACE_LANDMARKER];
+pub const MODELS: &[ModelInfo] = &[MOBILE_SAM, SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE, FACE_LANDMARKER];
/// The id of the built-in classical engines (graph cut segmenter, shape-model face tracker): no
/// weights, always available.
@@ -353,7 +401,10 @@ mod tests {
assert!(m.url.starts_with("https://") && m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));
assert!(!m.authors.is_empty() && notice(m).contains(m.authors));
}
- assert_eq!(models(Task::Mask).map(|m| m.id).collect::<Vec<_>>(), ["mobilesam", "sam2.1-hiera-tiny"]);
+ assert_eq!(
+ models(Task::Mask).map(|m| m.id).collect::<Vec<_>>(),
+ ["mobilesam", "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"]
+ );
assert!(SAM2_TINY.url.contains("/resolve/") && SAM2_TINY.file_name.ends_with(".safetensors"), "pinned to a commit, not a branch");
assert_eq!(models(Task::Face).map(|m| m.id).collect::<Vec<_>>(), ["mediapipe-face"]);
assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match")));
diff --git a/crates/ui-egui/tests/ui_roto_span_confidence.rs b/crates/ui-egui/tests/ui_roto_span_confidence.rs
index 3b3148d33..de561014a 100644
--- a/crates/ui-egui/tests/ui_roto_span_confidence.rs
+++ b/crates/ui-egui/tests/ui_roto_span_confidence.rs
@@ -49,7 +49,9 @@ impl ObjectTracker for Scripted {
for g in req.first..req.first + req.count as i64 {
let img = frame(g).ok_or("no frame")?;
let (confidence, occluded) = script(g);
- let prob = (0..img.w * img.h).map(|i| if !occluded && (i % img.w).abs_diff(img.w / 2) < img.w / 4 && (i / img.w).abs_diff(img.h / 2) < img.h / 4 { 1.0 } else { 0.0 }).collect();
+ let prob = (0..img.w * img.h)
+ .map(|i| if !occluded && (i % img.w).abs_diff(img.w / 2) < img.w / 4 && (i / img.w).abs_diff(img.h / 2) < img.h / 4 { 1.0 } else { 0.0 })
+ .collect();
out(0, g, TrackedFrame { prob, w: img.w, h: img.h, confidence, occluded });
}
Ok(())
diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md
index a08c8a5e7..56044f3fb 100644
--- a/docs/roto-brush-sam2.md
+++ b/docs/roto-brush-sam2.md
@@ -29,6 +29,24 @@ strokes and span they came from are unchanged. Changing a stroke or the span mak
Choosing another model, or the classic engine, leaves them in place and `roto.status` says so; a classic Propagate
replaces them with classic mattes. Object Matte saves nothing.
+## Sizes
+Four checkpoints are in the registry (Settings ▸ Roto Brush); the size is detected from the file, and each is
+pinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bigger backbone costs time and memory.
+
+| Model | Download | GPU encode (RX 7900 GRE) | Agreement with PyTorch |
+|---|---|---|---|
+| SAM 2.1 Tiny (`sam2.1-hiera-tiny`) | 155.9 MB | 0.24 s | encoder 6.8e-6 |
+| SAM 2.1 Small (`sam2.1-hiera-small`) | 184.3 MB | 0.20 s | encoder 1.0e-5 |
+| SAM 2.1 Base+ (`sam2.1-hiera-base-plus`) | 323.5 MB | 0.37 s | encoder 2.3e-5 |
+| SAM 2.1 Large (`sam2.1-hiera-large`) | 897.9 MB | 0.72 s | encoder 5.2e-4 |
+
+Every size was checked stage by stage against PyTorch (patch embedding, each Hiera block, neck, prompt encoder,
+decoder, memory encoder, memory attention) and on a whole clip forward and backward, on the CPU and the GPU: the video
+logit error is at most 0.005 with the same masks. Large carries much larger activations (peaks near 300 in the third
+stage), so its f32 rounding error is larger in absolute terms; its tests use a bound relative to the peak. Large needs
+the GPU to run per block (the encode submits once per block so intermediates are freed); a GPU error of any kind falls
+back to the CPU.
+
## Corrections re-track only what they change
A correction stroke changes the keys of the frames from it on (in both directions from the base), so Propagate tracks
only those again: the window starts at the nearest prompted frame at or before the first frame it needs (the corrected