Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

S2.65: help and bench follow all four SAM 2 sizes; tests for the size configs and detection

Kristian Myramac8765bf67b4
diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs
index e6a3352dc..1c528c8fb 100644
--- a/apps/effectcraft-cli/src/main.rs
+++ b/apps/effectcraft-cli/src/main.rs
@@ -28,7 +28,7 @@
//! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings
//! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)
//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--fixes] [--clip FRAMES --truth MASKS] [--json] Roto Brush trackers on synthetic clips with known mattes
-//! (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame)
+//! (classical engine, MobileSAM and SAM 2.1 (any installed size) when installed; IoU, flicker, re-appearance, seconds per frame)
//! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script
//! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio
//!
@@ -1030,11 +1030,11 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
note!("bench --roto: MobileSAM is not installed; comparing the classical engine only");
}
// `--sam2 FILE` uses the weights at FILE; otherwise the installed ones.
- let sam2_file = args.opt("--sam2").map(std::path::PathBuf::from).or_else(|| session.model_path(&effectcraft_segment::SAM2_TINY));
+ let sam2_file = args.opt("--sam2").map(std::path::PathBuf::from).or_else(|| effectcraft_sam2::mask::SIZES.iter().find_map(|m| session.model_path(m)));
let sam2: Option<std::sync::Arc<dyn MaskModel>> =
sam2_file.and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_sam2::mask::load(&bytes).ok()).map(|m| m as std::sync::Arc<dyn MaskModel>);
if sam2.is_none() {
- note!("bench --roto: SAM 2.1 Tiny is not installed; no SAM 2 column");
+ note!("bench --roto: no SAM 2.1 size is installed; no SAM 2 column");
}
// `--fixes`: also count the correction strokes each needs before every frame reaches IoU 0.9 (up to 6).
let fixes = args.flag("--fixes");
@@ -1053,7 +1053,8 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "mobilesam", Some(m.as_ref())))));
}
if let Some(m) = &sam2 {
- runs.extend(measure(&sc, &|s| bench::run_tracker(s, "sam2-tiny", m.as_ref())));
+ let label = m.info().id.replacen("sam2.1-hiera-", "sam2-", 1);
+ runs.extend(measure(&sc, &|s| bench::run_tracker(s, &label, m.as_ref())));
}
}
if args.flag("--json") {
diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rs
index d6d9ef63b..a00dd660e 100644
--- a/crates/engine/src/analysis/models.rs
+++ b/crates/engine/src/analysis/models.rs
@@ -544,7 +544,7 @@ pub fn specs() -> Vec<CommandSpec> {
"Use Roto Brush Model",
[],
None,
- "{id: classical|mobilesam|sam2.1-hiera-tiny, wait?} (Roto Brush 2.0 / 3.0 use it; wait: loaded before returning)",
+ "{id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large, wait?} (Roto Brush 2.0 / 3.0 use it; wait: loaded before returning)",
always,
|s, p| select(s, p, Task::Mask)
),
diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs
index ee05ea6b8..3dee87e74 100644
--- a/crates/engine/src/tests/models.rs
+++ b/crates/engine/src/tests/models.rs
@@ -37,7 +37,11 @@ fn models_are_listed_chosen_and_verified() {
let list = s.execute_checked(&format!("{prefix}.models"), json!({})).unwrap();
let ids: Vec<&str> = list["models"].as_array().unwrap().iter().map(|m| m["id"].as_str().unwrap()).collect();
// Roto Brush also offers SAM 2.1 in four sizes (video trackers; see crates/sam2).
- let expect: Vec<&str> = if task == Task::Mask { vec!["classical", model, "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] } else { vec!["classical", model] };
+ let expect: Vec<&str> = if task == Task::Mask {
+ vec!["classical", model, "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"]
+ } else {
+ vec!["classical", model]
+ };
assert_eq!(ids, expect);
let m = &list["models"][1];
assert_eq!((m["licence"].as_str(), m["installed"].as_bool()), (Some("Apache-2.0"), Some(false)));
diff --git a/crates/sam2/src/lib.rs b/crates/sam2/src/lib.rs
index 09cbb1466..f2aba0390 100644
--- a/crates/sam2/src/lib.rs
+++ b/crates/sam2/src/lib.rs
@@ -107,6 +107,36 @@ mod tests {
assert!(Size::Large.repo().ends_with("large"));
}
+ #[test]
+ fn configs_match_the_released_sizes() {
+ let blocks = |s| hiera::HieraConfig::for_size(s).blocks.iter().sum::<usize>();
+ assert_eq!([Size::Tiny, Size::Small, Size::BasePlus, Size::Large].map(blocks), [12, 16, 24, 48]);
+ for s in [Size::Tiny, Size::Small, Size::BasePlus, Size::Large] {
+ let c = hiera::HieraConfig::for_size(s);
+ assert_eq!((c.embed_dims.len(), c.heads.len(), c.windows.len()), (4, 4, 4));
+ assert!(c.global_blocks.iter().all(|g| *g < blocks(s)));
+ }
+ }
+
+ #[test]
+ fn size_comes_from_the_patch_embedding_and_depth() {
+ use candle_core::{DType, Device, Tensor};
+ let make = |width: usize, deep: bool| {
+ let mut m = std::collections::HashMap::new();
+ m.insert("vision_encoder.backbone.patch_embed.projection.weight".to_string(), Tensor::zeros((width, 3, 7, 7), DType::F32, &Device::Cpu).unwrap());
+ if deep {
+ m.insert("vision_encoder.backbone.blocks.15.layer_norm1.weight".to_string(), Tensor::zeros(4, DType::F32, &Device::Cpu).unwrap());
+ }
+ weights::Weights::from_map(m).unwrap()
+ };
+ assert_eq!(Size::detect(&make(96, false)).unwrap(), Size::Tiny);
+ assert_eq!(Size::detect(&make(96, true)).unwrap(), Size::Small);
+ assert_eq!(Size::detect(&make(112, false)).unwrap(), Size::BasePlus);
+ assert_eq!(Size::detect(&make(144, false)).unwrap(), Size::Large);
+ assert!(Size::detect(&make(64, false)).is_err());
+ assert!(Size::detect(&weights::Weights::from_map(Default::default()).unwrap()).is_err());
+ }
+
#[test]
fn errors_say_what_failed() {
assert!(Error::Input("empty frame".into()).to_string().contains("empty frame"));
diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md
index 56044f3fb..b25506b2b 100644
--- a/docs/roto-brush-sam2.md
+++ b/docs/roto-brush-sam2.md
@@ -106,7 +106,7 @@ encoder's 64 × 64 level matches PyTorch to 7e-6, the memory attention to 1.8e-5
times the pieces on both devices (`EFFECTCRAFT_SAM2_PROFILE=1` prints a frame's breakdown).
## Benchmark
-`effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt]` runs synthetic clips with
+`effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors (any size; default: the first installed)] [--mobilesam mobile_sam.pt]` runs synthetic clips with
known mattes through all three. 24 frames, debug build, SAM 2 on the GPU; mean IoU (mean IoU after the object returns), seconds
per frame: