Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

S2.50: bench --roto adds the SAM 2 tracker column

Kristian Myram4655425d0113
diff --git a/Cargo.lock b/Cargo.lock
index 7d951deb7..423abb85a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1455,6 +1455,7 @@ dependencies = [
"effectcraft-gpu",
"effectcraft-host",
"effectcraft-render",
+ "effectcraft-sam2",
"effectcraft-segment",
"effectcraft-time",
"effectcraft-track",
diff --git a/apps/effectcraft-cli/Cargo.toml b/apps/effectcraft-cli/Cargo.toml
index 7d8fe96ff..abfc014eb 100644
--- a/apps/effectcraft-cli/Cargo.toml
+++ b/apps/effectcraft-cli/Cargo.toml
@@ -22,6 +22,7 @@ effectcraft-gpu = { workspace = true }
effectcraft-time = { workspace = true }
effectcraft-track = { workspace = true }
effectcraft-segment = { workspace = true }
+effectcraft-sam2 = { workspace = true }
serde_json = { workspace = true }
image = { workspace = true }
cpu-time = "1"
diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs
index 3de4449b6..b60918304 100644
--- a/apps/effectcraft-cli/src/main.rs
+++ b/apps/effectcraft-cli/src/main.rs
@@ -27,8 +27,8 @@
//! (--gpu: CPU vs GPU ms/frame for every comp at Full and Half)
//! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings
//! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)
-//! effectcraft-cli bench --roto [--frames N] [--json] Roto Brush trackers on synthetic clips with known mattes
-//! (classical engine, MobileSAM when installed; IoU, flicker, re-appearance, seconds per frame)
+//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--json] Roto Brush trackers on synthetic clips with known mattes
+//! (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame)
//! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script
//! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio
//!
@@ -126,7 +126,8 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se
bench --ops [--small] [--layers N] [--comps N] [--footage N]
everyday operations on a large generated project: startup, open,
save, auto-save, edits + undo/redo, timeline, Project panel
- bench --roto [--frames N] [--json] Roto Brush trackers on synthetic clips with known mattes: movement,
+ bench --roto [--frames N] [--sam2 FILE] [--json]
+ Roto Brush trackers on synthetic clips with known mattes: movement,
lighting change, occlusion, leaving and re-entering the frame
script FILE.jsx [F.ecproj] | --eval CODE run JavaScript with the After Effects-style object model
(app.project, comps, layers, properties…); prints writeLn
@@ -142,6 +143,7 @@ options: --project F.ecproj | --demo | --empty --save | --save-as F --bridge
const VALUED: &[&str] = &[
"--n",
"--frames",
+ "--sam2",
"--play",
"--layers",
"--comps",
@@ -1023,12 +1025,22 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
if mobilesam.is_none() {
note!("bench --roto: MobileSAM is not installed; comparing the classical engine only");
}
+ // `--sam2 FILE` uses the weights at FILE; otherwise the installed ones.
+ let sam2_file = args.opt("--sam2").map(std::path::PathBuf::from).or_else(|| session.model_path(&effectcraft_segment::SAM2_TINY));
+ let sam2: Option<std::sync::Arc<dyn MaskModel>> =
+ sam2_file.and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_sam2::mask::load(&bytes).ok()).map(|m| m as std::sync::Arc<dyn MaskModel>);
+ if sam2.is_none() {
+ note!("bench --roto: SAM 2.1 Tiny is not installed; no SAM 2 column");
+ }
let mut runs = vec![];
for sc in bench::scenarios(frames) {
runs.push(bench::run_flow(&sc, "classical", None));
if let Some(m) = &mobilesam {
runs.push(bench::run_flow(&sc, "mobilesam", Some(m.as_ref())));
}
+ if let Some(run) = sam2.as_ref().and_then(|m| bench::run_tracker(&sc, "sam2-tiny", m.as_ref())) {
+ runs.push(run);
+ }
}
if args.flag("--json") {
say!("{}", serde_json::to_string_pretty(&runs).map_err(|e| Failure::Error(e.to_string()))?);
diff --git a/crates/track/src/roto/bench.rs b/crates/track/src/roto/bench.rs
index d52836979..810bf34e5 100644
--- a/crates/track/src/roto/bench.rs
+++ b/crates/track/src/roto/bench.rs
@@ -5,11 +5,13 @@
//! later SAM 2) is added when the caller has one. `effectcraft-cli bench --roto` prints the table.
use effectcraft_raster::Image;
-use effectcraft_segment::MaskModel;
+use std::sync::Arc;
+
+use effectcraft_segment::{ClipFrame, MaskModel, TrackRequest};
use serde::Serialize;
use web_time::Instant;
-use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, segment_with};
+use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, rgb_of, segment_tracked, segment_with, tracker_prompt};
pub const W: u32 = 200;
pub const H: u32 = 150;
@@ -210,6 +212,30 @@ pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) ->
metrics(sc, tracker, &mattes, secs)
}
+/// Follow the object through the whole clip with the model's video tracker, as the Roto Brush's
+/// tracking pass does (prompt from the strokes, probability maps refined by the same graph cut).
+/// `None` when the model has no tracker or the tracker fails.
+pub fn run_tracker(sc: &Scenario, tracker: &str, model: &dyn MaskModel) -> Option<Run> {
+ let t = model.tracker()?;
+ let opts = SegOpts::default();
+ let refs: Vec<&Stroke> = sc.strokes.iter().collect();
+ let clip: Vec<Arc<ClipFrame>> = sc.frames.iter().map(|i| Arc::new(ClipFrame { rgb: rgb_of(i), w: i.width as usize, h: i.height as usize })).collect();
+ let req = TrackRequest { first: 0, count: sc.frames.len(), prompts: vec![(sc.base as i64, tracker_prompt(&refs, 1.0))] };
+ let start = Instant::now();
+ let mut mattes: Vec<Vec<u8>> = vec![vec![0u8; (W * H) as usize]; sc.frames.len()];
+ let mut get = |f: i64| usize::try_from(f).ok().and_then(|i| clip.get(i).cloned());
+ let r = t.track(&req, &mut get, &|| false, &mut |f, tf| {
+ let (Ok(i), true) = (usize::try_from(f), tf.prob.len() == (W * H) as usize) else { return };
+ let Some(img) = sc.frames.get(i) else { return };
+ let strokes: Vec<&Stroke> = if i == sc.base { refs.clone() } else { vec![] };
+ if let Some(slot) = mattes.get_mut(i) {
+ *slot = segment_tracked(img, &strokes, tf.prob, tf.occluded, &opts, 0.0).matte;
+ }
+ });
+ r.ok()?;
+ Some(metrics(sc, tracker, &mattes, start.elapsed().as_secs_f64()))
+}
+
/// A fixed-width table of the runs, grouped by scenario.
pub fn table(runs: &[Run]) -> String {
let mut out = format!("{:<10} {:<12} {:>8} {:>8} {:>8} {:>10} {:>9}\n", "clip", "tracker", "meanIoU", "minIoU", "flicker", "after ret.", "s/frame");