Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

S2.50: effectcraft-cli bench --roto on synthetic clips

Kristian Myram71e44deb8424
diff --git a/Cargo.lock b/Cargo.lock
index 07564b7df..6a694299d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1394,7 +1394,9 @@ dependencies = [
"effectcraft-gpu",
"effectcraft-host",
"effectcraft-render",
+ "effectcraft-segment",
"effectcraft-time",
+ "effectcraft-track",
"effectcraft-ui-egui",
"image",
"serde_json",
diff --git a/apps/effectcraft-cli/Cargo.toml b/apps/effectcraft-cli/Cargo.toml
index b5784f4eb..7d8fe96ff 100644
--- a/apps/effectcraft-cli/Cargo.toml
+++ b/apps/effectcraft-cli/Cargo.toml
@@ -20,6 +20,8 @@ effectcraft-ui-egui = { workspace = true }
effectcraft-render = { workspace = true }
effectcraft-gpu = { workspace = true }
effectcraft-time = { workspace = true }
+effectcraft-track = { workspace = true }
+effectcraft-segment = { workspace = true }
serde_json = { workspace = true }
image = { workspace = true }
cpu-time = "1"
diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs
index fd77162ed..3de4449b6 100644
--- a/apps/effectcraft-cli/src/main.rs
+++ b/apps/effectcraft-cli/src/main.rs
@@ -27,6 +27,8 @@
//! (--gpu: CPU vs GPU ms/frame for every comp at Full and Half)
//! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings
//! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)
+//! effectcraft-cli bench --roto [--frames N] [--json] Roto Brush trackers on synthetic clips with known mattes
+//! (classical engine, MobileSAM when installed; IoU, flicker, re-appearance, seconds per frame)
//! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script
//! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio
//!
@@ -124,6 +126,8 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se
bench --ops [--small] [--layers N] [--comps N] [--footage N]
everyday operations on a large generated project: startup, open,
save, auto-save, edits + undo/redo, timeline, Project panel
+ bench --roto [--frames N] [--json] Roto Brush trackers on synthetic clips with known mattes: movement,
+ lighting change, occlusion, leaving and re-entering the frame
script FILE.jsx [F.ecproj] | --eval CODE run JavaScript with the After Effects-style object model
(app.project, comps, layers, properties…); prints writeLn
output and the result; errors exit 1 with file:line:col
@@ -137,6 +141,7 @@ options: --project F.ecproj | --demo | --empty --save | --save-as F --bridge
/// Options that take a value.
const VALUED: &[&str] = &[
"--n",
+ "--frames",
"--play",
"--layers",
"--comps",
@@ -199,6 +204,7 @@ const FLAGS: &[&str] = &[
"--work-area",
"--queue",
"--ops",
+ "--roto",
"--small",
"--adv3d",
"--autosave",
@@ -976,6 +982,9 @@ fn bench_cmd(args: &Args) -> Result<(), Failure> {
if args.flag("--ops") {
return bench_ops(args);
}
+ if args.flag("--roto") {
+ return bench_roto(args);
+ }
let mut s = effectcraft_host::session();
match &args.project {
Some(p) => s.execute("file.open", json!({"path": p})),
@@ -998,6 +1007,37 @@ fn bench_cmd(args: &Args) -> Result<(), Failure> {
Ok(())
}
+/// `bench --roto`: the Roto Brush's trackers on synthetic clips whose true mattes are known.
+fn bench_roto(args: &Args) -> Result<(), Failure> {
+ use effectcraft_segment::{Loaded, MOBILE_SAM, MaskModel};
+ use effectcraft_track::roto::bench;
+ let frames = args.num("--frames")?.unwrap_or(24.0).clamp(12.0, 240.0) as usize;
+ let session = effectcraft_host::session();
+ let mobilesam: Option<std::sync::Arc<dyn MaskModel>> =
+ session.model_path(&MOBILE_SAM).and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_segment::load(MOBILE_SAM.id, &bytes).ok()).and_then(
+ |m| match m {
+ Loaded::Mask(m) => Some(m),
+ _ => None,
+ },
+ );
+ if mobilesam.is_none() {
+ note!("bench --roto: MobileSAM is not installed; comparing the classical engine only");
+ }
+ let mut runs = vec![];
+ for sc in bench::scenarios(frames) {
+ runs.push(bench::run_flow(&sc, "classical", None));
+ if let Some(m) = &mobilesam {
+ runs.push(bench::run_flow(&sc, "mobilesam", Some(m.as_ref())));
+ }
+ }
+ if args.flag("--json") {
+ say!("{}", serde_json::to_string_pretty(&runs).map_err(|e| Failure::Error(e.to_string()))?);
+ } else {
+ say!("{}", bench::table(&runs));
+ }
+ Ok(())
+}
+
/// `bench --ops`: everyday operations on a large generated project (200 comps, 5,000 layers in
/// the main comp, 300 footage items, 20-deep precomp nesting, expressions): startup to first
/// frame, open, save, auto-save, small and large edits with undo/redo, first frame, timeline
diff --git a/crates/track/src/roto.rs b/crates/track/src/roto.rs
index 103d695dd..abcf28aa0 100644
--- a/crates/track/src/roto.rs
+++ b/crates/track/src/roto.rs
@@ -31,6 +31,7 @@
//! deterministic function of its *chain*: the base frame's strokes, then every frame's strokes
//! on the way out to it ([`RotoData::chain_keys`]).
+pub mod bench;
pub mod gmm;
pub mod matting;
pub mod maxflow;
diff --git a/crates/track/src/roto/bench.rs b/crates/track/src/roto/bench.rs
new file mode 100644
index 000000000..d52836979
--- /dev/null
+++ b/crates/track/src/roto/bench.rs
@@ -0,0 +1,264 @@
+//! Roto Brush benchmark: synthetic clips with known mattes (movement, colour change, occlusion,
+//! leaving and re-entering the frame) and the metrics that compare trackers on them.
+//!
+//! Needs no weights: the classical engine always runs, and a trained [`MaskModel`] (MobileSAM,
+//! later SAM 2) is added when the caller has one. `effectcraft-cli bench --roto` prints the table.
+
+use effectcraft_raster::Image;
+use effectcraft_segment::MaskModel;
+use serde::Serialize;
+use web_time::Instant;
+
+use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, segment_with};
+
+pub const W: u32 = 200;
+pub const H: u32 = 150;
+const R: f64 = 32.0;
+
+/// A clip with the true matte of the visible object on every frame.
+pub struct Scenario {
+ pub name: &'static str,
+ pub what: &'static str,
+ pub frames: Vec<Image>,
+ /// Visible-object matte per frame (`W×H`, 0/1); empty when the object is out of view.
+ pub truth: Vec<Vec<u8>>,
+ /// Frame the strokes are painted on.
+ pub base: usize,
+ pub strokes: Vec<Stroke>,
+}
+
+/// One tracker's result on one scenario.
+#[derive(Clone, Debug, Serialize)]
+pub struct Run {
+ pub scenario: &'static str,
+ pub tracker: String,
+ pub mean_iou: f64,
+ pub min_iou: f64,
+ /// Mean IoU between consecutive mattes minus the same for the truth, as a loss (0 is steady).
+ pub flicker: f64,
+ pub secs_per_frame: f64,
+ /// Mean IoU over the frames after the object's last absence (re-appearance), if it had one.
+ pub after_return: Option<f64>,
+ pub per_frame: Vec<f64>,
+}
+
+fn noise(x: f64, y: f64, seed: u32) -> f64 {
+ fn h(i: i64, j: i64, s: u32) -> f64 {
+ let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ i64::from(s).wrapping_mul(83492791)) as u64;
+ v ^= v >> 13;
+ v = v.wrapping_mul(0x5bd1e995);
+ v ^= v >> 15;
+ (v % 10_000) as f64 / 10_000.0
+ }
+ let mut out = 0.0;
+ for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] {
+ let (fx, fy) = (x / cell, y / cell);
+ let (i, j) = (fx.floor() as i64, fy.floor() as i64);
+ let (tx, ty) = (fx - i as f64, fy - j as f64);
+ let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty));
+ let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx;
+ let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx;
+ out += amp * (a + (b - a) * sy);
+ }
+ out
+}
+
+/// What a frame shows: the disk centre, an optional occluding bar `[x0, y0, x1, y1]` and a
+/// brightness gain applied to the whole frame.
+#[derive(Clone, Copy)]
+struct Look {
+ c: [f64; 2],
+ bar: Option<[f64; 4]>,
+ gain: f32,
+}
+
+fn render(l: Look) -> (Image, Vec<u8>) {
+ let mut img = Image::new(W, H);
+ let mut gt = vec![0u8; (W * H) as usize];
+ for y in 0..H {
+ for x in 0..W {
+ let p = [f64::from(x) + 0.5, f64::from(y) + 0.5];
+ let d = (p[0] - l.c[0]).hypot(p[1] - l.c[1]);
+ let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32;
+ let n = noise(p[0] - l.c[0], p[1] - l.c[1], 11);
+ let f = [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32];
+ let (bn, bm) = (noise(p[0], p[1], 5), noise(p[0] * 1.7, p[1] * 1.7, 9));
+ let b = [(0.05 + 0.25 * bm) as f32, (0.3 + 0.4 * bn) as f32, (0.45 + 0.4 * bm) as f32];
+ let mut px = [0.0f32; 3];
+ for k in 0..3 {
+ px[k] = f[k] * a + b[k] * (1.0 - a);
+ }
+ let under_bar = l.bar.is_some_and(|r| p[0] >= r[0] && p[0] < r[2] && p[1] >= r[1] && p[1] < r[3]);
+ if under_bar {
+ let t = noise(p[0] * 0.8, p[1] * 0.8, 23) as f32;
+ px = [0.15 + 0.1 * t, 0.15 + 0.1 * t, 0.2 + 0.1 * t];
+ }
+ let g = l.gain;
+ img.set(x, y, [(px[0] * g).min(1.0), (px[1] * g).min(1.0), (px[2] * g).min(1.0), 1.0]);
+ if let Some(v) = gt.get_mut((y * W + x) as usize) {
+ *v = (d <= R && !under_bar) as u8;
+ }
+ }
+ }
+ (img, gt)
+}
+
+fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) -> Stroke {
+ let pt = |k: u32| {
+ let t = f64::from(k) / 10.0;
+ [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]
+ };
+ Stroke { kind, frame, radius, points: (0..=10).map(pt).collect() }
+}
+
+/// The strokes an artist would paint on the base frame: one through the disk, two around it.
+fn strokes_for(c: [f64; 2]) -> Vec<Stroke> {
+ vec![
+ line(StrokeKind::Fg, 0, [c[0] - 18.0, c[1] - 6.0], [c[0] + 18.0, c[1] + 6.0], 5.0),
+ line(StrokeKind::Bg, 0, [c[0] - 50.0, c[1] - 45.0], [c[0] + 50.0, c[1] - 45.0], 5.0),
+ line(StrokeKind::Bg, 0, [c[0] + 50.0, c[1] + 45.0], [c[0] - 45.0, c[1] + 45.0], 5.0),
+ ]
+}
+
+fn scenario(name: &'static str, what: &'static str, n: usize, look: impl Fn(usize) -> Look) -> Scenario {
+ let (mut frames, mut truth) = (Vec::with_capacity(n), Vec::with_capacity(n));
+ for f in 0..n {
+ let (img, gt) = render(look(f));
+ frames.push(img);
+ truth.push(gt);
+ }
+ Scenario { name, what, frames, truth, base: 0, strokes: strokes_for(look(0).c) }
+}
+
+/// The benchmark clips, `n` frames each (at least 12, so every event happens).
+pub fn scenarios(n: usize) -> Vec<Scenario> {
+ let n = n.clamp(12, 240);
+ let k = n as f64 / 24.0;
+ let drift = move |f: usize| {
+ let t = f as f64 / k;
+ [60.0 + 3.1 * t, 70.0 + 1.3 * t + 4.0 * (t * 0.3).sin()]
+ };
+ vec![
+ scenario("move", "disk drifts across a textured background", n, move |f| Look { c: drift(f), bar: None, gain: 1.0 }),
+ scenario("light", "same motion while the whole frame brightens by 40 %", n, move |f| Look {
+ c: drift(f),
+ bar: None,
+ gain: 1.0 + 0.4 * f as f32 / n as f32,
+ }),
+ scenario("occlusion", "a bar crosses and hides most of the disk, then clears it", n, move |f| {
+ let t = f as f64 / n as f64;
+ let c = drift(f);
+ let x = -40.0 + 320.0 * ((t - 0.3) / 0.4).clamp(0.0, 1.0);
+ Look { c, bar: (0.3..0.7).contains(&t).then_some([x - 40.0, 0.0, x + 40.0, f64::from(H)]), gain: 1.0 }
+ }),
+ scenario("reenter", "disk leaves through the right edge and comes back", n, move |f| {
+ let t = f as f64 / n as f64;
+ let x = 60.0 + 190.0 * (1.0 - (2.0 * t - 1.0).powi(2));
+ Look { c: [x, 70.0 + 4.0 * (t * 6.0).sin()], bar: None, gain: 1.0 }
+ }),
+ ]
+}
+
+fn metrics(sc: &Scenario, tracker: &str, mattes: &[Vec<u8>], secs: f64) -> Run {
+ let per_frame: Vec<f64> = mattes.iter().zip(&sc.truth).map(|(m, t)| iou(m, t)).collect();
+ let n = per_frame.len().max(1) as f64;
+ let steady = |v: &[Vec<u8>]| {
+ let pairs: Vec<f64> = v.windows(2).filter_map(|w| Some(iou(w.first()?, w.get(1)?))).collect();
+ pairs.iter().sum::<f64>() / pairs.len().max(1) as f64
+ };
+ let last_gap = sc.truth.iter().rposition(|t| t.iter().all(|v| *v == 0));
+ let after_return = last_gap.and_then(|g| {
+ let tail = per_frame.get(g + 1..)?;
+ (!tail.is_empty()).then(|| tail.iter().sum::<f64>() / tail.len() as f64)
+ });
+ Run {
+ scenario: sc.name,
+ tracker: tracker.to_string(),
+ mean_iou: per_frame.iter().sum::<f64>() / n,
+ min_iou: per_frame.iter().copied().fold(1.0, f64::min),
+ flicker: (steady(&sc.truth) - steady(mattes)).max(0.0),
+ secs_per_frame: secs / n,
+ after_return,
+ per_frame,
+ }
+}
+
+/// Segment the base frame from the strokes and propagate to the end of the clip, as the Roto
+/// Brush does, with `model` (or none: the classical engine).
+pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) -> Run {
+ let opts = SegOpts::default();
+ let start = Instant::now();
+ let refs: Vec<&Stroke> = sc.strokes.iter().collect();
+ let mut segs: Vec<Option<FrameSeg>> = vec![None; sc.frames.len()];
+ if let Some(first) = sc.frames.get(sc.base) {
+ let base = segment_with(first, &refs, None, &opts, 0.0, model);
+ if let Some(slot) = segs.get_mut(sc.base) {
+ *slot = Some(base);
+ }
+ }
+ for f in sc.base + 1..sc.frames.len() {
+ let (Some(prev_img), Some(img)) = (sc.frames.get(f - 1), sc.frames.get(f)) else { break };
+ let Some(Some(prev)) = segs.get(f - 1) else { break };
+ let next = propagate_with(prev_img, prev, img, &[], &opts, 0.0, model);
+ if let Some(slot) = segs.get_mut(f) {
+ *slot = Some(next);
+ }
+ }
+ let secs = start.elapsed().as_secs_f64();
+ let empty = vec![0u8; (W * H) as usize];
+ let mattes: Vec<Vec<u8>> = segs.into_iter().map(|s| s.map_or_else(|| empty.clone(), |s| s.matte)).collect();
+ metrics(sc, tracker, &mattes, secs)
+}
+
+/// A fixed-width table of the runs, grouped by scenario.
+pub fn table(runs: &[Run]) -> String {
+ let mut out = format!("{:<10} {:<12} {:>8} {:>8} {:>8} {:>10} {:>9}\n", "clip", "tracker", "meanIoU", "minIoU", "flicker", "after ret.", "s/frame");
+ for r in runs {
+ let back = r.after_return.map_or_else(|| "-".to_string(), |v| format!("{v:.3}"));
+ out.push_str(&format!(
+ "{:<10} {:<12} {:>8.3} {:>8.3} {:>8.3} {:>10} {:>9.3}\n",
+ r.scenario, r.tracker, r.mean_iou, r.min_iou, r.flicker, back, r.secs_per_frame
+ ));
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn area(v: &[u8]) -> usize {
+ v.iter().filter(|x| **x != 0).count()
+ }
+
+ #[test]
+ fn scenarios_have_the_events_they_claim() {
+ let all = scenarios(24);
+ assert_eq!(all.len(), 4);
+ for s in &all {
+ assert_eq!((s.frames.len(), s.truth.len()), (24, 24), "{}", s.name);
+ }
+ let full = all.first().map(|s| area(&s.truth[0])).unwrap_or(0);
+ assert!(full > 3000, "{full}");
+ let occ = all.iter().find(|s| s.name == "occlusion").map(|s| s.truth.iter().map(|t| area(t)).min().unwrap_or(0)).unwrap_or(full);
+ assert!(occ < full / 2, "occluded minimum {occ} of {full}");
+ let gone = all.iter().find(|s| s.name == "reenter").map(|s| s.truth.iter().filter(|t| area(t) == 0).count()).unwrap_or(0);
+ assert!(gone > 0, "the disk never leaves");
+ assert!(all.iter().find(|s| s.name == "reenter").is_some_and(|s| area(s.truth.last().map_or(&[][..], |v| v)) > 0), "it must come back");
+ }
+
+ #[test]
+ fn classical_tracks_plain_motion_and_the_table_lists_it() {
+ let all = scenarios(24);
+ let Some(sc) = all.first() else { return };
+ let run = run_flow(sc, "classical", None);
+ assert!(run.mean_iou > 0.9, "mean IoU {}", run.mean_iou);
+ assert_eq!(run.per_frame.len(), 24);
+ assert!(table(&[run]).contains("classical"));
+ }
+
+ #[test]
+ fn tiny_clip_requests_are_clamped() {
+ assert!(scenarios(0).iter().all(|s| s.frames.len() == 12));
+ }
+}