EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
S2.50: bench --roto adds the SAM 2 tracker column
Kristian Myram4655425d0113
diff --git a/Cargo.lock b/Cargo.lockindex 7d951deb7..423abb85a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1455,6 +1455,7 @@ dependencies = ["effectcraft-gpu","effectcraft-host","effectcraft-render",+ "effectcraft-sam2","effectcraft-segment","effectcraft-time","effectcraft-track",diff --git a/apps/effectcraft-cli/Cargo.toml b/apps/effectcraft-cli/Cargo.tomlindex 7d8fe96ff..abfc014eb 100644--- a/apps/effectcraft-cli/Cargo.toml+++ b/apps/effectcraft-cli/Cargo.toml@@ -22,6 +22,7 @@ effectcraft-gpu = { workspace = true }effectcraft-time = { workspace = true }effectcraft-track = { workspace = true }effectcraft-segment = { workspace = true }+effectcraft-sam2 = { workspace = true }serde_json = { workspace = true }image = { workspace = true }cpu-time = "1"diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rsindex 3de4449b6..b60918304 100644--- a/apps/effectcraft-cli/src/main.rs+++ b/apps/effectcraft-cli/src/main.rs@@ -27,8 +27,8 @@//! (--gpu: CPU vs GPU ms/frame for every comp at Full and Half)//! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings//! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)-//! effectcraft-cli bench --roto [--frames N] [--json] Roto Brush trackers on synthetic clips with known mattes-//! (classical engine, MobileSAM when installed; IoU, flicker, re-appearance, seconds per frame)+//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--json] Roto Brush trackers on synthetic clips with known mattes+//! (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame)//! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script//! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio//!@@ -126,7 +126,8 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|sebench --ops [--small] [--layers N] [--comps N] [--footage N]everyday operations on a large generated project: startup, open,save, auto-save, edits + undo/redo, timeline, Project panel- bench --roto [--frames N] [--json] Roto Brush trackers on synthetic clips with known mattes: movement,+ bench --roto [--frames N] [--sam2 FILE] [--json]+ Roto Brush trackers on synthetic clips with known mattes: movement,lighting change, occlusion, leaving and re-entering the framescript FILE.jsx [F.ecproj] | --eval CODE run JavaScript with the After Effects-style object model(app.project, comps, layers, properties…); prints writeLn@@ -142,6 +143,7 @@ options: --project F.ecproj | --demo | --empty --save | --save-as F --bridgeconst VALUED: &[&str] = &["--n","--frames",+ "--sam2","--play","--layers","--comps",@@ -1023,12 +1025,22 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {if mobilesam.is_none() {note!("bench --roto: MobileSAM is not installed; comparing the classical engine only");}+ // `--sam2 FILE` uses the weights at FILE; otherwise the installed ones.+ let sam2_file = args.opt("--sam2").map(std::path::PathBuf::from).or_else(|| session.model_path(&effectcraft_segment::SAM2_TINY));+ let sam2: Option<std::sync::Arc<dyn MaskModel>> =+ sam2_file.and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_sam2::mask::load(&bytes).ok()).map(|m| m as std::sync::Arc<dyn MaskModel>);+ if sam2.is_none() {+ note!("bench --roto: SAM 2.1 Tiny is not installed; no SAM 2 column");+ }let mut runs = vec![];for sc in bench::scenarios(frames) {runs.push(bench::run_flow(&sc, "classical", None));if let Some(m) = &mobilesam {runs.push(bench::run_flow(&sc, "mobilesam", Some(m.as_ref())));}+ if let Some(run) = sam2.as_ref().and_then(|m| bench::run_tracker(&sc, "sam2-tiny", m.as_ref())) {+ runs.push(run);+ }}if args.flag("--json") {say!("{}", serde_json::to_string_pretty(&runs).map_err(|e| Failure::Error(e.to_string()))?);diff --git a/crates/track/src/roto/bench.rs b/crates/track/src/roto/bench.rsindex d52836979..810bf34e5 100644--- a/crates/track/src/roto/bench.rs+++ b/crates/track/src/roto/bench.rs@@ -5,11 +5,13 @@//! later SAM 2) is added when the caller has one. `effectcraft-cli bench --roto` prints the table.use effectcraft_raster::Image;-use effectcraft_segment::MaskModel;+use std::sync::Arc;++use effectcraft_segment::{ClipFrame, MaskModel, TrackRequest};use serde::Serialize;use web_time::Instant;-use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, segment_with};+use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, rgb_of, segment_tracked, segment_with, tracker_prompt};pub const W: u32 = 200;pub const H: u32 = 150;@@ -210,6 +212,30 @@ pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) ->metrics(sc, tracker, &mattes, secs)}+/// Follow the object through the whole clip with the model's video tracker, as the Roto Brush's+/// tracking pass does (prompt from the strokes, probability maps refined by the same graph cut).+/// `None` when the model has no tracker or the tracker fails.+pub fn run_tracker(sc: &Scenario, tracker: &str, model: &dyn MaskModel) -> Option<Run> {+ let t = model.tracker()?;+ let opts = SegOpts::default();+ let refs: Vec<&Stroke> = sc.strokes.iter().collect();+ let clip: Vec<Arc<ClipFrame>> = sc.frames.iter().map(|i| Arc::new(ClipFrame { rgb: rgb_of(i), w: i.width as usize, h: i.height as usize })).collect();+ let req = TrackRequest { first: 0, count: sc.frames.len(), prompts: vec![(sc.base as i64, tracker_prompt(&refs, 1.0))] };+ let start = Instant::now();+ let mut mattes: Vec<Vec<u8>> = vec![vec![0u8; (W * H) as usize]; sc.frames.len()];+ let mut get = |f: i64| usize::try_from(f).ok().and_then(|i| clip.get(i).cloned());+ let r = t.track(&req, &mut get, &|| false, &mut |f, tf| {+ let (Ok(i), true) = (usize::try_from(f), tf.prob.len() == (W * H) as usize) else { return };+ let Some(img) = sc.frames.get(i) else { return };+ let strokes: Vec<&Stroke> = if i == sc.base { refs.clone() } else { vec![] };+ if let Some(slot) = mattes.get_mut(i) {+ *slot = segment_tracked(img, &strokes, tf.prob, tf.occluded, &opts, 0.0).matte;+ }+ });+ r.ok()?;+ Some(metrics(sc, tracker, &mattes, start.elapsed().as_secs_f64()))+}+/// A fixed-width table of the runs, grouped by scenario.pub fn table(runs: &[Run]) -> String {let mut out = format!("{:<10} {:<12} {:>8} {:>8} {:>8} {:>10} {:>9}\n", "clip", "tracker", "meanIoU", "minIoU", "flicker", "after ret.", "s/frame");