EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
#1539: validate benchmark inputs and preserve shared refinement strokes
Kristian Myrama4f5c516ed8c
diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rsindex 69a56c696..b1b53b488 100644--- a/apps/effectcraft-cli/src/main.rs+++ b/apps/effectcraft-cli/src/main.rs@@ -1035,7 +1035,7 @@ fn bench_weights<T>(fn bench_roto(args: &Args) -> Result<(), Failure> {use effectcraft_segment::{Loaded, MOBILE_SAM, MaskModel};use effectcraft_track::roto::bench;- let frames = args.num("--frames")?.unwrap_or(24.0).clamp(12.0, 240.0) as usize;+ let frames = args.num("--frames")?.unwrap_or(24.0).clamp(1.0, 240.0) as usize;let session = effectcraft_host::session();// `--mobilesam FILE` uses the weights at FILE (a file that cannot be used is an error);// otherwise the installed ones.@@ -1060,13 +1060,14 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {let fixes = args.flag("--fixes");let measure = |sc: &bench::Scenario, run: &dyn Fn(&bench::Scenario) -> Option<bench::Run>| if fixes { bench::with_fixes(sc, 0.9, 6, run) } else { run(sc) };let mut runs = vec![];- let mut scenarios = bench::scenarios(frames);- // `--clip FRAMES_DIR --truth MASKS_DIR`: the user's own clip (image sequences, same file order) with its true mattes.- if let Some(dir) = args.opt("--clip") {+ // User clips pair unique filename stems, allowing JPEG frames with PNG truth masks.+ let scenarios = if let Some(dir) = args.opt("--clip") {let truth = args.opt("--truth").ok_or_else(|| Failure::Error("--clip needs --truth MASKS_DIR".into()))?;let name = std::path::Path::new(dir).file_name().map_or_else(|| "clip".to_string(), |n| n.to_string_lossy().chars().take(10).collect());- scenarios = vec![load_bench_clip(&name, dir, truth, frames).map_err(Failure::Error)?];- }+ vec![load_bench_clip(&name, dir, truth, frames).map_err(Failure::Error)?]+ } else {+ bench::scenarios(frames)+ };for sc in scenarios {runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "classical", None))));if let Some(m) = &mobilesam {@@ -1085,14 +1086,14 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {Ok(())}-/// The image files of `dir` (png, jpg, jpeg), in file-name order.+/// The image files of `dir` (png, jpg, jpeg), ordered by stem then full file name.fn image_files(dir: &str) -> Result<Vec<std::path::PathBuf>, String> {let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(dir).map_err(|e| format!("cannot read {dir}: {e}"))?.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().and_then(|e| e.to_str()).is_some_and(|e| ["png", "jpg", "jpeg"].contains(&e.to_ascii_lowercase().as_str()))).collect();- files.sort();+ files.sort_by(|a, b| a.file_stem().cmp(&b.file_stem()).then_with(|| a.file_name().cmp(&b.file_name())));Ok(files)}@@ -1103,6 +1104,11 @@ fn load_bench_clip(name: &str, frames_dir: &str, truth_dir: &str, limit: usize)if frames.is_empty() || frames.len() != masks.len() {return Err(format!("{} frames in {frames_dir} but {} masks in {truth_dir}: they must match one to one", frames.len(), masks.len()));}+ if frames.iter().map(|p| p.file_stem()).ne(masks.iter().map(|p| p.file_stem()))+ || frames.windows(2).chain(masks.windows(2)).any(|pair| pair.first().and_then(|p| p.file_stem()) == pair.get(1).and_then(|p| p.file_stem()))+ {+ return Err("frame and truth file names must match one to one, with unique stems (extensions may differ)".into());+ }let n = frames.len().min(limit.clamp(1, 240));let (mut imgs, mut truth) = (vec![], vec![]);for (f, m) in frames.iter().zip(&masks).take(n) {diff --git a/apps/effectcraft-cli/tests/it/cli.rs b/apps/effectcraft-cli/tests/it/cli.rsindex 93c899839..b3279260a 100644--- a/apps/effectcraft-cli/tests/it/cli.rs+++ b/apps/effectcraft-cli/tests/it/cli.rs@@ -689,7 +689,7 @@ fn bench_roto_runs_on_a_users_clip() {std::fs::create_dir_all(d).unwrap();}let (w, h) = (96u32, 72u32);- for f in 0..6u32 {+ for f in 0..14u32 {let (cx, cy) = (30.0 + 4.0 * f as f32, 36.0f32);let (mut img, mut mask) = (image::RgbImage::new(w, h), image::GrayImage::new(w, h));for y in 0..h {@@ -700,15 +700,23 @@ fn bench_roto_runs_on_a_users_clip() {mask.put_pixel(x, y, image::Luma([if inside { 255 } else { 0 }]));}}- img.save(frames.join(format!("f{f:03}.png"))).unwrap();- mask.save(masks.join(format!("f{f:03}.png"))).unwrap();+ let extension = if f == 0 { "jpg" } else { "png" };+ let stem = match f {+ 0 => "a".into(),+ 1 => "a.jpg".into(),+ _ => format!("f{f:03}"),+ };+ img.save(frames.join(format!("{stem}.{extension}"))).unwrap();+ mask.save(masks.join(format!("{stem}.png"))).unwrap();}let (f, m) = (frames.to_str().unwrap(), masks.to_str().unwrap());- let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6"]).output().unwrap();+ let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6", "--json"]).output().unwrap();let text = String::from_utf8_lossy(&out.stdout).to_string();assert!(out.status.success(), "{text}\n{}", String::from_utf8_lossy(&out.stderr));- let row = text.lines().find(|l| l.contains("classical")).unwrap_or_else(|| panic!("no classical row: {text}"));- let mean: f64 = row.split_whitespace().nth(2).unwrap().parse().unwrap();+ let rows: Value = serde_json::from_str(&text).unwrap();+ let row = rows.as_array().unwrap().iter().find(|row| row["tracker"] == "classical").unwrap();+ assert_eq!(row["per_frame"].as_array().unwrap().len(), 6, "a user clip respects the requested frame limit");+ let mean = row["mean_iou"].as_f64().unwrap();assert!(mean > 0.8, "classical mean IoU on the user's clip: {mean}\n{text}");// one mask short: refusedstd::fs::remove_file(masks.join("f005.png")).unwrap();@@ -720,6 +728,40 @@ fn bench_roto_runs_on_a_users_clip() {assert!(!out.status.success());}+#[test]+fn bench_roto_rejects_mismatched_names_before_loading_images() {+ let root = tmp("bench-clip-names");+ let (frames, masks) = (root.join("frames"), root.join("masks"));+ let _ = std::fs::remove_dir_all(&root);+ for dir in [&frames, &masks] {+ std::fs::create_dir_all(dir).unwrap();+ std::fs::write(dir.join("f000.png"), []).unwrap();+ }+ std::fs::write(frames.join("f001.png"), []).unwrap();+ std::fs::write(masks.join("f002.png"), []).unwrap();+ let out = bin()+ .env("EFFECTCRAFT_CONFIG_DIR", root.join("config"))+ .args(["bench", "--roto", "--frames", "1", "--clip"])+ .arg(&frames)+ .arg("--truth")+ .arg(&masks)+ .output()+ .unwrap();+ let error = String::from_utf8_lossy(&out.stderr);+ assert!(!out.status.success());+ assert!(error.contains("names must match"), "validate the entire name set before decoding: {error}");+ std::fs::remove_file(frames.join("f001.png")).unwrap();+ std::fs::remove_file(masks.join("f002.png")).unwrap();+ for dir in [&frames, &masks] {+ std::fs::write(dir.join("f000.jpg"), []).unwrap();+ }+ let out =+ bin().env("EFFECTCRAFT_CONFIG_DIR", root.join("config")).args(["bench", "--roto", "--clip"]).arg(&frames).arg("--truth").arg(&masks).output().unwrap();+ assert!(!out.status.success());+ assert!(String::from_utf8_lossy(&out.stderr).contains("unique stems"));+ std::fs::remove_dir_all(root).unwrap();+}+/// `bench --roto --sam2 FILE` / `--mobilesam FILE` say why a file named on the command line cannot be/// used (a missing file, one that is not the published weights), instead of "not installed".#[test]diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rsindex 1a1ce105f..379c772c5 100644--- a/crates/effects/src/roto.rs+++ b/crates/effects/src/roto.rs@@ -694,6 +694,9 @@ impl Chain {let mut prompts = vec![];for f in self.data.stroke_frames().into_iter().filter(|f| self.data.in_span(*f)) {let strokes = self.data.strokes_of(f, obj);+ if !strokes.iter().any(|s| s.kind == StrokeKind::Fg) {+ continue;+ }let p = match frame(f) {Some(img) => rb::tracker_prompt_boxed(&strokes, self.opts.scale, &rb::segment(&img, &strokes, None, &self.opts, self.refine_radius)),None => return Err(format!("cannot read Roto Brush prompt frame {f}")),@@ -1429,6 +1432,30 @@ mod tests {}}+ #[test]+ fn prompt_segmentation_reads_only_frames_with_that_objects_foreground() {+ let mut data = RotoData::default();+ for (kind, frame, object) in [(StrokeKind::Fg, 0, 0), (StrokeKind::Refine, 1, 0), (StrokeKind::Fg, 2, 1)] {+ data.add(Stroke { kind, frame, radius: 3.0, points: vec![[35.0, 40.0]], object }, 2, [0, 2]);+ }+ data.span = [0, 2];+ let mut params = params_with(&data);+ params.values.insert(INPUT_KEY.into(), Value::Str("foreground-only-prompts".into()));+ let mut chain = Chain::new(¶ms, [100.0, 80.0], 25.0, 1.0);+ let fake = Arc::new(TwoObjects(Default::default()));+ chain.model = Some(fake.clone());+ let mut prompt_reads = vec![];+ let mut get = |f: i64| {+ if fake.0.lock().unwrap().is_empty() {+ prompt_reads.push(f);+ }+ Some(Arc::new(frame(f as f64 / 25.0)))+ };+ assert_eq!(chain.track(&mut get, &|| false, &mut |_| {}).unwrap().unwrap().frames, 3);+ assert_eq!(prompt_reads, [0, 2], "shared Refine Edge and other objects' frames need no prompt graph cut");+ assert_eq!(fake.0.lock().unwrap()[0], [1, 1]);+ }+#[test]fn separate_object_passes_cancel_at_the_pending_mask_budget() {use std::sync::atomic::Ordering::Relaxed;diff --git a/crates/engine/src/cmd/roto_cmds.rs b/crates/engine/src/cmd/roto_cmds.rsindex 5124d413d..792c60019 100644--- a/crates/engine/src/cmd/roto_cmds.rs+++ b/crates/engine/src/cmd/roto_cmds.rs@@ -378,9 +378,20 @@ fn clear_strokes(s: &mut Session, p: &Value) -> Result<Value> {Some(k) => Some(StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}`")))?),None => None,};- let object = f_p(p, "object").map(|v| v.round() as u32);+ let object = match p.get("object") {+ None => None,+ Some(value) => {+ let value = value+ .as_f64()+ .filter(|v| v.is_finite() && (0.0..=rt::MAX_OBJECT_ID as f64).contains(v))+ .ok_or_else(|| bad(cmd, format!("`object` is 0–{}", rt::MAX_OBJECT_ID)))?;+ Some(value.round() as u32)+ }+ };let n0 = d.strokes.len();- d.strokes.retain(|st| !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k) && object.is_none_or(|o| st.object == o)));+ d.strokes.retain(|st| {+ !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k) && object.is_none_or(|o| st.kind.is_segmentation() && st.object == o))+ });if d.strokes.iter().all(|s| !s.kind.is_segmentation()) {d.base = None;} else if d.base.is_some_and(|b| !d.strokes.iter().any(|s| s.frame == b && s.kind.is_segmentation())) {diff --git a/crates/engine/src/tests/roto.rs b/crates/engine/src/tests/roto.rsindex e686962e1..99c155265 100644--- a/crates/engine/src/tests/roto.rs+++ b/crates/engine/src/tests/roto.rs@@ -1125,3 +1125,37 @@ fn strokes_belong_to_the_current_object() {let json = s.project.to_json();assert!(json.contains("\\\"object\\\":31"), "the project file carries the object");}++#[test]+fn clearing_one_object_keeps_shared_refine_edge_strokes() {+ let mut s = Session::default();+ s.execute("comp.new", json!({"width": 64, "height": 64, "duration": 1})).unwrap();+ let layer = s.execute("layer.newSolid", json!({})).unwrap()["layer"].as_u64().unwrap();+ for (object, kind) in [(0, "fg"), (0, "bg"), (0, "refine"), (0, "refineErase"), (1, "fg")] {+ s.execute("roto.stroke", json!({"layer": layer, "object": object, "kind": kind, "points": [[20, 20]], "radius": 4})).unwrap();+ }+ let before = s.project.to_json();+ let result = s.execute("roto.clearStrokes", json!({"layer": layer, "object": 0})).unwrap();+ assert_eq!(result["removed"], 2, "only the object's foreground/background strokes are owned by it");+ assert_eq!(result["strokes"], 3);+ assert!(s.undo());+ assert_eq!(s.project.to_json(), before);+ for kind in ["refine", "refineErase"] {+ let result = s.execute("roto.clearStrokes", json!({"layer": layer, "kind": kind})).unwrap();+ assert_eq!(result["removed"], 1, "explicitly clearing shared {kind} still works");+ }+}++#[test]+fn clearing_strokes_rejects_invalid_object_filters_without_editing() {+ let mut s = Session::default();+ s.execute("comp.new", json!({"width": 64, "height": 64, "duration": 1})).unwrap();+ let layer = s.execute("layer.newSolid", json!({})).unwrap()["layer"].as_u64().unwrap();+ s.execute("roto.stroke", json!({"layer": layer, "points": [[20, 20]], "radius": 4})).unwrap();+ let before = s.project.to_json();+ for object in [json!(-1), json!(-0.1), json!(32), json!(1e30), json!(null), json!("0")] {+ let result = s.execute("roto.clearStrokes", json!({"layer": layer, "object": object}));+ assert!(result.is_err(), "invalid object {object} must not delete object zero: {result:?}");+ assert_eq!(s.project.to_json(), before);+ }+}diff --git a/crates/sam2/src/session.rs b/crates/sam2/src/session.rsindex 48ec72a31..0524ee89e 100644--- a/crates/sam2/src/session.rs+++ b/crates/sam2/src/session.rs@@ -96,8 +96,8 @@ pub enum Backend {Auto,}-/// The GPU path: kernels for the backbone and the memory attention. Any failure switches it off for good-/// and the CPU path takes over.+/// The GPU path: kernels for the backbone and memory attention. Device failures disable it;+/// memory-attention budget limits fall back to the CPU for that call only.struct GpuParts {encoder: GpuEncoder,attention: GpuMemoryAttention,@@ -229,6 +229,7 @@ impl VideoModel {if let Some(g) = self.gpu.as_ref().filter(|g| g.ok.load(Ordering::Relaxed)) {match g.attention.forward(current, current_pos, memory, memory_pos, pointers) {Ok(t) => return Ok(t),+ Err(Error::Budget(reason)) => log::info!("SAM 2 memory attention on the CPU for this call: {reason}"),Err(e) => self.gpu_failed("memory attention", &e),}}