EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
S2.65: help and bench follow all four SAM 2 sizes; tests for the size configs and detection
Kristian Myramac8765bf67b4
diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rsindex e6a3352dc..1c528c8fb 100644--- a/apps/effectcraft-cli/src/main.rs+++ b/apps/effectcraft-cli/src/main.rs@@ -28,7 +28,7 @@//! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings//! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--fixes] [--clip FRAMES --truth MASKS] [--json] Roto Brush trackers on synthetic clips with known mattes-//! (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame)+//! (classical engine, MobileSAM and SAM 2.1 (any installed size) when installed; IoU, flicker, re-appearance, seconds per frame)//! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script//! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio//!@@ -1030,11 +1030,11 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {note!("bench --roto: MobileSAM is not installed; comparing the classical engine only");}// `--sam2 FILE` uses the weights at FILE; otherwise the installed ones.- let sam2_file = args.opt("--sam2").map(std::path::PathBuf::from).or_else(|| session.model_path(&effectcraft_segment::SAM2_TINY));+ let sam2_file = args.opt("--sam2").map(std::path::PathBuf::from).or_else(|| effectcraft_sam2::mask::SIZES.iter().find_map(|m| session.model_path(m)));let sam2: Option<std::sync::Arc<dyn MaskModel>> =sam2_file.and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_sam2::mask::load(&bytes).ok()).map(|m| m as std::sync::Arc<dyn MaskModel>);if sam2.is_none() {- note!("bench --roto: SAM 2.1 Tiny is not installed; no SAM 2 column");+ note!("bench --roto: no SAM 2.1 size is installed; no SAM 2 column");}// `--fixes`: also count the correction strokes each needs before every frame reaches IoU 0.9 (up to 6).let fixes = args.flag("--fixes");@@ -1053,7 +1053,8 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "mobilesam", Some(m.as_ref())))));}if let Some(m) = &sam2 {- runs.extend(measure(&sc, &|s| bench::run_tracker(s, "sam2-tiny", m.as_ref())));+ let label = m.info().id.replacen("sam2.1-hiera-", "sam2-", 1);+ runs.extend(measure(&sc, &|s| bench::run_tracker(s, &label, m.as_ref())));}}if args.flag("--json") {diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rsindex d6d9ef63b..a00dd660e 100644--- a/crates/engine/src/analysis/models.rs+++ b/crates/engine/src/analysis/models.rs@@ -544,7 +544,7 @@ pub fn specs() -> Vec<CommandSpec> {"Use Roto Brush Model",[],None,- "{id: classical|mobilesam|sam2.1-hiera-tiny, wait?} (Roto Brush 2.0 / 3.0 use it; wait: loaded before returning)",+ "{id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large, wait?} (Roto Brush 2.0 / 3.0 use it; wait: loaded before returning)",always,|s, p| select(s, p, Task::Mask)),diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rsindex ee05ea6b8..3dee87e74 100644--- a/crates/engine/src/tests/models.rs+++ b/crates/engine/src/tests/models.rs@@ -37,7 +37,11 @@ fn models_are_listed_chosen_and_verified() {let list = s.execute_checked(&format!("{prefix}.models"), json!({})).unwrap();let ids: Vec<&str> = list["models"].as_array().unwrap().iter().map(|m| m["id"].as_str().unwrap()).collect();// Roto Brush also offers SAM 2.1 in four sizes (video trackers; see crates/sam2).- let expect: Vec<&str> = if task == Task::Mask { vec!["classical", model, "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] } else { vec!["classical", model] };+ let expect: Vec<&str> = if task == Task::Mask {+ vec!["classical", model, "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"]+ } else {+ vec!["classical", model]+ };assert_eq!(ids, expect);let m = &list["models"][1];assert_eq!((m["licence"].as_str(), m["installed"].as_bool()), (Some("Apache-2.0"), Some(false)));diff --git a/crates/sam2/src/lib.rs b/crates/sam2/src/lib.rsindex 09cbb1466..f2aba0390 100644--- a/crates/sam2/src/lib.rs+++ b/crates/sam2/src/lib.rs@@ -107,6 +107,36 @@ mod tests {assert!(Size::Large.repo().ends_with("large"));}+ #[test]+ fn configs_match_the_released_sizes() {+ let blocks = |s| hiera::HieraConfig::for_size(s).blocks.iter().sum::<usize>();+ assert_eq!([Size::Tiny, Size::Small, Size::BasePlus, Size::Large].map(blocks), [12, 16, 24, 48]);+ for s in [Size::Tiny, Size::Small, Size::BasePlus, Size::Large] {+ let c = hiera::HieraConfig::for_size(s);+ assert_eq!((c.embed_dims.len(), c.heads.len(), c.windows.len()), (4, 4, 4));+ assert!(c.global_blocks.iter().all(|g| *g < blocks(s)));+ }+ }++ #[test]+ fn size_comes_from_the_patch_embedding_and_depth() {+ use candle_core::{DType, Device, Tensor};+ let make = |width: usize, deep: bool| {+ let mut m = std::collections::HashMap::new();+ m.insert("vision_encoder.backbone.patch_embed.projection.weight".to_string(), Tensor::zeros((width, 3, 7, 7), DType::F32, &Device::Cpu).unwrap());+ if deep {+ m.insert("vision_encoder.backbone.blocks.15.layer_norm1.weight".to_string(), Tensor::zeros(4, DType::F32, &Device::Cpu).unwrap());+ }+ weights::Weights::from_map(m).unwrap()+ };+ assert_eq!(Size::detect(&make(96, false)).unwrap(), Size::Tiny);+ assert_eq!(Size::detect(&make(96, true)).unwrap(), Size::Small);+ assert_eq!(Size::detect(&make(112, false)).unwrap(), Size::BasePlus);+ assert_eq!(Size::detect(&make(144, false)).unwrap(), Size::Large);+ assert!(Size::detect(&make(64, false)).is_err());+ assert!(Size::detect(&weights::Weights::from_map(Default::default()).unwrap()).is_err());+ }+#[test]fn errors_say_what_failed() {assert!(Error::Input("empty frame".into()).to_string().contains("empty frame"));diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.mdindex 56044f3fb..b25506b2b 100644--- a/docs/roto-brush-sam2.md+++ b/docs/roto-brush-sam2.md@@ -106,7 +106,7 @@ encoder's 64 × 64 level matches PyTorch to 7e-6, the memory attention to 1.8e-5times the pieces on both devices (`EFFECTCRAFT_SAM2_PROFILE=1` prints a frame's breakdown).## Benchmark-`effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt]` runs synthetic clips with+`effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors (any size; default: the first installed)] [--mobilesam mobile_sam.pt]` runs synthetic clips withknown mattes through all three. 24 frames, debug build, SAM 2 on the GPU; mean IoU (mean IoU after the object returns), secondsper frame: