Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

#1539: validate benchmark inputs and preserve shared refinement strokes

Kristian Myrama4f5c516ed8c
diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs
index 69a56c696..b1b53b488 100644
--- a/apps/effectcraft-cli/src/main.rs
+++ b/apps/effectcraft-cli/src/main.rs
@@ -1035,7 +1035,7 @@ fn bench_weights<T>(
fn bench_roto(args: &Args) -> Result<(), Failure> {
use effectcraft_segment::{Loaded, MOBILE_SAM, MaskModel};
use effectcraft_track::roto::bench;
- let frames = args.num("--frames")?.unwrap_or(24.0).clamp(12.0, 240.0) as usize;
+ let frames = args.num("--frames")?.unwrap_or(24.0).clamp(1.0, 240.0) as usize;
let session = effectcraft_host::session();
// `--mobilesam FILE` uses the weights at FILE (a file that cannot be used is an error);
// otherwise the installed ones.
@@ -1060,13 +1060,14 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
let fixes = args.flag("--fixes");
let measure = |sc: &bench::Scenario, run: &dyn Fn(&bench::Scenario) -> Option<bench::Run>| if fixes { bench::with_fixes(sc, 0.9, 6, run) } else { run(sc) };
let mut runs = vec![];
- let mut scenarios = bench::scenarios(frames);
- // `--clip FRAMES_DIR --truth MASKS_DIR`: the user's own clip (image sequences, same file order) with its true mattes.
- if let Some(dir) = args.opt("--clip") {
+ // User clips pair unique filename stems, allowing JPEG frames with PNG truth masks.
+ let scenarios = if let Some(dir) = args.opt("--clip") {
let truth = args.opt("--truth").ok_or_else(|| Failure::Error("--clip needs --truth MASKS_DIR".into()))?;
let name = std::path::Path::new(dir).file_name().map_or_else(|| "clip".to_string(), |n| n.to_string_lossy().chars().take(10).collect());
- scenarios = vec![load_bench_clip(&name, dir, truth, frames).map_err(Failure::Error)?];
- }
+ vec![load_bench_clip(&name, dir, truth, frames).map_err(Failure::Error)?]
+ } else {
+ bench::scenarios(frames)
+ };
for sc in scenarios {
runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "classical", None))));
if let Some(m) = &mobilesam {
@@ -1085,14 +1086,14 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
Ok(())
}
-/// The image files of `dir` (png, jpg, jpeg), in file-name order.
+/// The image files of `dir` (png, jpg, jpeg), ordered by stem then full file name.
fn image_files(dir: &str) -> Result<Vec<std::path::PathBuf>, String> {
let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(dir)
.map_err(|e| format!("cannot read {dir}: {e}"))?
.filter_map(|e| e.ok().map(|e| e.path()))
.filter(|p| p.extension().and_then(|e| e.to_str()).is_some_and(|e| ["png", "jpg", "jpeg"].contains(&e.to_ascii_lowercase().as_str())))
.collect();
- files.sort();
+ files.sort_by(|a, b| a.file_stem().cmp(&b.file_stem()).then_with(|| a.file_name().cmp(&b.file_name())));
Ok(files)
}
@@ -1103,6 +1104,11 @@ fn load_bench_clip(name: &str, frames_dir: &str, truth_dir: &str, limit: usize)
if frames.is_empty() || frames.len() != masks.len() {
return Err(format!("{} frames in {frames_dir} but {} masks in {truth_dir}: they must match one to one", frames.len(), masks.len()));
}
+ if frames.iter().map(|p| p.file_stem()).ne(masks.iter().map(|p| p.file_stem()))
+ || frames.windows(2).chain(masks.windows(2)).any(|pair| pair.first().and_then(|p| p.file_stem()) == pair.get(1).and_then(|p| p.file_stem()))
+ {
+ return Err("frame and truth file names must match one to one, with unique stems (extensions may differ)".into());
+ }
let n = frames.len().min(limit.clamp(1, 240));
let (mut imgs, mut truth) = (vec![], vec![]);
for (f, m) in frames.iter().zip(&masks).take(n) {
diff --git a/apps/effectcraft-cli/tests/it/cli.rs b/apps/effectcraft-cli/tests/it/cli.rs
index 93c899839..b3279260a 100644
--- a/apps/effectcraft-cli/tests/it/cli.rs
+++ b/apps/effectcraft-cli/tests/it/cli.rs
@@ -689,7 +689,7 @@ fn bench_roto_runs_on_a_users_clip() {
std::fs::create_dir_all(d).unwrap();
}
let (w, h) = (96u32, 72u32);
- for f in 0..6u32 {
+ for f in 0..14u32 {
let (cx, cy) = (30.0 + 4.0 * f as f32, 36.0f32);
let (mut img, mut mask) = (image::RgbImage::new(w, h), image::GrayImage::new(w, h));
for y in 0..h {
@@ -700,15 +700,23 @@ fn bench_roto_runs_on_a_users_clip() {
mask.put_pixel(x, y, image::Luma([if inside { 255 } else { 0 }]));
}
}
- img.save(frames.join(format!("f{f:03}.png"))).unwrap();
- mask.save(masks.join(format!("f{f:03}.png"))).unwrap();
+ let extension = if f == 0 { "jpg" } else { "png" };
+ let stem = match f {
+ 0 => "a".into(),
+ 1 => "a.jpg".into(),
+ _ => format!("f{f:03}"),
+ };
+ img.save(frames.join(format!("{stem}.{extension}"))).unwrap();
+ mask.save(masks.join(format!("{stem}.png"))).unwrap();
}
let (f, m) = (frames.to_str().unwrap(), masks.to_str().unwrap());
- let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6"]).output().unwrap();
+ let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6", "--json"]).output().unwrap();
let text = String::from_utf8_lossy(&out.stdout).to_string();
assert!(out.status.success(), "{text}\n{}", String::from_utf8_lossy(&out.stderr));
- let row = text.lines().find(|l| l.contains("classical")).unwrap_or_else(|| panic!("no classical row: {text}"));
- let mean: f64 = row.split_whitespace().nth(2).unwrap().parse().unwrap();
+ let rows: Value = serde_json::from_str(&text).unwrap();
+ let row = rows.as_array().unwrap().iter().find(|row| row["tracker"] == "classical").unwrap();
+ assert_eq!(row["per_frame"].as_array().unwrap().len(), 6, "a user clip respects the requested frame limit");
+ let mean = row["mean_iou"].as_f64().unwrap();
assert!(mean > 0.8, "classical mean IoU on the user's clip: {mean}\n{text}");
// one mask short: refused
std::fs::remove_file(masks.join("f005.png")).unwrap();
@@ -720,6 +728,40 @@ fn bench_roto_runs_on_a_users_clip() {
assert!(!out.status.success());
}
+#[test]
+fn bench_roto_rejects_mismatched_names_before_loading_images() {
+ let root = tmp("bench-clip-names");
+ let (frames, masks) = (root.join("frames"), root.join("masks"));
+ let _ = std::fs::remove_dir_all(&root);
+ for dir in [&frames, &masks] {
+ std::fs::create_dir_all(dir).unwrap();
+ std::fs::write(dir.join("f000.png"), []).unwrap();
+ }
+ std::fs::write(frames.join("f001.png"), []).unwrap();
+ std::fs::write(masks.join("f002.png"), []).unwrap();
+ let out = bin()
+ .env("EFFECTCRAFT_CONFIG_DIR", root.join("config"))
+ .args(["bench", "--roto", "--frames", "1", "--clip"])
+ .arg(&frames)
+ .arg("--truth")
+ .arg(&masks)
+ .output()
+ .unwrap();
+ let error = String::from_utf8_lossy(&out.stderr);
+ assert!(!out.status.success());
+ assert!(error.contains("names must match"), "validate the entire name set before decoding: {error}");
+ std::fs::remove_file(frames.join("f001.png")).unwrap();
+ std::fs::remove_file(masks.join("f002.png")).unwrap();
+ for dir in [&frames, &masks] {
+ std::fs::write(dir.join("f000.jpg"), []).unwrap();
+ }
+ let out =
+ bin().env("EFFECTCRAFT_CONFIG_DIR", root.join("config")).args(["bench", "--roto", "--clip"]).arg(&frames).arg("--truth").arg(&masks).output().unwrap();
+ assert!(!out.status.success());
+ assert!(String::from_utf8_lossy(&out.stderr).contains("unique stems"));
+ std::fs::remove_dir_all(root).unwrap();
+}
+
/// `bench --roto --sam2 FILE` / `--mobilesam FILE` say why a file named on the command line cannot be
/// used (a missing file, one that is not the published weights), instead of "not installed".
#[test]
diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs
index 1a1ce105f..379c772c5 100644
--- a/crates/effects/src/roto.rs
+++ b/crates/effects/src/roto.rs
@@ -694,6 +694,9 @@ impl Chain {
let mut prompts = vec![];
for f in self.data.stroke_frames().into_iter().filter(|f| self.data.in_span(*f)) {
let strokes = self.data.strokes_of(f, obj);
+ if !strokes.iter().any(|s| s.kind == StrokeKind::Fg) {
+ continue;
+ }
let p = match frame(f) {
Some(img) => rb::tracker_prompt_boxed(&strokes, self.opts.scale, &rb::segment(&img, &strokes, None, &self.opts, self.refine_radius)),
None => return Err(format!("cannot read Roto Brush prompt frame {f}")),
@@ -1429,6 +1432,30 @@ mod tests {
}
}
+ #[test]
+ fn prompt_segmentation_reads_only_frames_with_that_objects_foreground() {
+ let mut data = RotoData::default();
+ for (kind, frame, object) in [(StrokeKind::Fg, 0, 0), (StrokeKind::Refine, 1, 0), (StrokeKind::Fg, 2, 1)] {
+ data.add(Stroke { kind, frame, radius: 3.0, points: vec![[35.0, 40.0]], object }, 2, [0, 2]);
+ }
+ data.span = [0, 2];
+ let mut params = params_with(&data);
+ params.values.insert(INPUT_KEY.into(), Value::Str("foreground-only-prompts".into()));
+ let mut chain = Chain::new(&params, [100.0, 80.0], 25.0, 1.0);
+ let fake = Arc::new(TwoObjects(Default::default()));
+ chain.model = Some(fake.clone());
+ let mut prompt_reads = vec![];
+ let mut get = |f: i64| {
+ if fake.0.lock().unwrap().is_empty() {
+ prompt_reads.push(f);
+ }
+ Some(Arc::new(frame(f as f64 / 25.0)))
+ };
+ assert_eq!(chain.track(&mut get, &|| false, &mut |_| {}).unwrap().unwrap().frames, 3);
+ assert_eq!(prompt_reads, [0, 2], "shared Refine Edge and other objects' frames need no prompt graph cut");
+ assert_eq!(fake.0.lock().unwrap()[0], [1, 1]);
+ }
+
#[test]
fn separate_object_passes_cancel_at_the_pending_mask_budget() {
use std::sync::atomic::Ordering::Relaxed;
diff --git a/crates/engine/src/cmd/roto_cmds.rs b/crates/engine/src/cmd/roto_cmds.rs
index 5124d413d..792c60019 100644
--- a/crates/engine/src/cmd/roto_cmds.rs
+++ b/crates/engine/src/cmd/roto_cmds.rs
@@ -378,9 +378,20 @@ fn clear_strokes(s: &mut Session, p: &Value) -> Result<Value> {
Some(k) => Some(StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}`")))?),
None => None,
};
- let object = f_p(p, "object").map(|v| v.round() as u32);
+ let object = match p.get("object") {
+ None => None,
+ Some(value) => {
+ let value = value
+ .as_f64()
+ .filter(|v| v.is_finite() && (0.0..=rt::MAX_OBJECT_ID as f64).contains(v))
+ .ok_or_else(|| bad(cmd, format!("`object` is 0–{}", rt::MAX_OBJECT_ID)))?;
+ Some(value.round() as u32)
+ }
+ };
let n0 = d.strokes.len();
- d.strokes.retain(|st| !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k) && object.is_none_or(|o| st.object == o)));
+ d.strokes.retain(|st| {
+ !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k) && object.is_none_or(|o| st.kind.is_segmentation() && st.object == o))
+ });
if d.strokes.iter().all(|s| !s.kind.is_segmentation()) {
d.base = None;
} else if d.base.is_some_and(|b| !d.strokes.iter().any(|s| s.frame == b && s.kind.is_segmentation())) {
diff --git a/crates/engine/src/tests/roto.rs b/crates/engine/src/tests/roto.rs
index e686962e1..99c155265 100644
--- a/crates/engine/src/tests/roto.rs
+++ b/crates/engine/src/tests/roto.rs
@@ -1125,3 +1125,37 @@ fn strokes_belong_to_the_current_object() {
let json = s.project.to_json();
assert!(json.contains("\\\"object\\\":31"), "the project file carries the object");
}
+
+#[test]
+fn clearing_one_object_keeps_shared_refine_edge_strokes() {
+ let mut s = Session::default();
+ s.execute("comp.new", json!({"width": 64, "height": 64, "duration": 1})).unwrap();
+ let layer = s.execute("layer.newSolid", json!({})).unwrap()["layer"].as_u64().unwrap();
+ for (object, kind) in [(0, "fg"), (0, "bg"), (0, "refine"), (0, "refineErase"), (1, "fg")] {
+ s.execute("roto.stroke", json!({"layer": layer, "object": object, "kind": kind, "points": [[20, 20]], "radius": 4})).unwrap();
+ }
+ let before = s.project.to_json();
+ let result = s.execute("roto.clearStrokes", json!({"layer": layer, "object": 0})).unwrap();
+ assert_eq!(result["removed"], 2, "only the object's foreground/background strokes are owned by it");
+ assert_eq!(result["strokes"], 3);
+ assert!(s.undo());
+ assert_eq!(s.project.to_json(), before);
+ for kind in ["refine", "refineErase"] {
+ let result = s.execute("roto.clearStrokes", json!({"layer": layer, "kind": kind})).unwrap();
+ assert_eq!(result["removed"], 1, "explicitly clearing shared {kind} still works");
+ }
+}
+
+#[test]
+fn clearing_strokes_rejects_invalid_object_filters_without_editing() {
+ let mut s = Session::default();
+ s.execute("comp.new", json!({"width": 64, "height": 64, "duration": 1})).unwrap();
+ let layer = s.execute("layer.newSolid", json!({})).unwrap()["layer"].as_u64().unwrap();
+ s.execute("roto.stroke", json!({"layer": layer, "points": [[20, 20]], "radius": 4})).unwrap();
+ let before = s.project.to_json();
+ for object in [json!(-1), json!(-0.1), json!(32), json!(1e30), json!(null), json!("0")] {
+ let result = s.execute("roto.clearStrokes", json!({"layer": layer, "object": object}));
+ assert!(result.is_err(), "invalid object {object} must not delete object zero: {result:?}");
+ assert_eq!(s.project.to_json(), before);
+ }
+}
diff --git a/crates/sam2/src/session.rs b/crates/sam2/src/session.rs
index 48ec72a31..0524ee89e 100644
--- a/crates/sam2/src/session.rs
+++ b/crates/sam2/src/session.rs
@@ -96,8 +96,8 @@ pub enum Backend {
Auto,
}
-/// The GPU path: kernels for the backbone and the memory attention. Any failure switches it off for good
-/// and the CPU path takes over.
+/// The GPU path: kernels for the backbone and memory attention. Device failures disable it;
+/// memory-attention budget limits fall back to the CPU for that call only.
struct GpuParts {
encoder: GpuEncoder,
attention: GpuMemoryAttention,
@@ -229,6 +229,7 @@ impl VideoModel {
if let Some(g) = self.gpu.as_ref().filter(|g| g.ok.load(Ordering::Relaxed)) {
match g.attention.forward(current, current_pos, memory, memory_pos, pointers) {
Ok(t) => return Ok(t),
+ Err(Error::Budget(reason)) => log::info!("SAM 2 memory attention on the CPU for this call: {reason}"),
Err(e) => self.gpu_failed("memory attention", &e),
}
}