EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
Roto Brush: model prompts follow the latest stroke (#1646)
tylerjenningsw818ce2ab427b
diff --git a/crates/track/src/roto.rs b/crates/track/src/roto.rsindex 9e5d8d6e1..0072ad223 100644--- a/crates/track/src/roto.rs+++ b/crates/track/src/roto.rs@@ -375,6 +375,12 @@ fn sweep(s: &Stroke, scale: f64, w: usize, h: usize, mut paint: impl FnMut(usize}}+/// Whether `s` covers the point `p` (matte pixels at `scale`).+fn covers(s: &Stroke, scale: f64, p: [f64; 2]) -> bool {+ let (r, segs) = segments(s, scale);+ segs.iter().any(|(a, b)| segment_distance(p, *a, *b) <= r)+}+/// Pixels covered by `strokes` (of the given kinds) at `scale`.pub fn rasterize(strokes: &[&Stroke], kinds: &[StrokeKind], scale: f64, w: usize, h: usize) -> Vec<bool> {let mut m = vec![false; w * h];@@ -894,15 +900,21 @@ fn points_along(s: &Stroke, scale: f64, max: usize) -> Vec<[f32; 2]> {out}-/// A model's prompt from the strokes: points along the foreground and background strokes, and-/// the box of a [`StrokeKind::Box`] stroke (matte pixels).+/// A model's prompt from the strokes: points along the foreground and background strokes (but+/// not where a later stroke of the other kind painted over them), and the box of a+/// [`StrokeKind::Box`] stroke (matte pixels).pub fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {+ let seg: Vec<&Stroke> = strokes.iter().copied().filter(|s| s.kind.is_segmentation()).collect();let mut points = vec![];for (kind, fg) in [(StrokeKind::Fg, true), (StrokeKind::Bg, false)] {- let ss: Vec<&&Stroke> = strokes.iter().filter(|s| s.kind == kind).collect();+ let ss: Vec<(usize, &Stroke)> = seg.iter().copied().enumerate().filter(|(_, s)| s.kind == kind).collect();let per = (16 / ss.len().max(1)).max(1);- for s in ss {- points.extend(points_along(s, scale, per).into_iter().map(|p| (p, fg)));+ for (k, s) in ss {+ let kept = |p: &[f32; 2]| {+ let p = [p[0] as f64, p[1] as f64];+ seg.iter().skip(k + 1).rev().find(|t| covers(t, scale, p)).is_none_or(|t| t.kind == kind)+ };+ points.extend(points_along(s, scale, per).into_iter().filter(kept).map(|p| (p, fg)));}}let bbox = strokes.iter().filter(|s| s.kind == StrokeKind::Box).find_map(|s| box_of(s, scale));@@ -918,8 +930,18 @@ fn box_of(s: &Stroke, scale: f64) -> Option<[f32; 4]> {}/// Add a prior matte to a prompt: its box (a little larger; unless the prompt has one), up to-/// four points spread through its core, and the matte itself.-fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bool {+/// four points spread through its core, and the matte itself, all with this frame's strokes+/// (`hard`, from [`resolve`]) painted over it.+fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], hard: &[u8], w: usize, h: usize) -> bool {+ let soft: Vec<f32> = soft+ .iter()+ .enumerate()+ .map(|(i, v)| match hard.get(i) {+ Some(&HARD_FG) => 1.0,+ Some(&HARD_BG) => 0.0,+ _ => *v,+ })+ .collect();let bin: Vec<u8> = soft.iter().map(|v| (*v >= 0.5) as u8).collect();let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize);for (i, v) in bin.iter().enumerate() {@@ -949,7 +971,7 @@ fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bo}}prompt.points.extend(picks.into_iter().map(|i| ([(i % w) as f32 + 0.5, (i / w) as f32 + 0.5], true)));- prompt.mask = Some(soft.to_vec());+ prompt.mask = Some(soft);true}@@ -960,7 +982,7 @@ pub fn segment_with(img: &Image, strokes: &[&Stroke], prior: Option<&Prior>, optlet Some(m) = model.filter(|_| w > 0 && h > 0) else { return segment(img, strokes, prior, opts, refine_radius) };let mut prompt = stroke_prompt(strokes, opts.scale);if let Some(p) = prior {- add_prior_prompt(&mut prompt, &p.soft, w, h);+ add_prior_prompt(&mut prompt, &p.soft, &resolve(strokes, StrokeKind::Fg, StrokeKind::Bg, opts.scale, w, h), w, h);}if !prompt.points.iter().any(|(_, fg)| *fg) {return segment(img, strokes, prior, opts, refine_radius);@@ -991,7 +1013,7 @@ pub fn propagate_with(let Some(m) = model.filter(|_| w > 0 && h > 0) else { return propagate(prev_img, prev, next_img, strokes, opts, refine_radius) };let prior = warp_prior(prev_img, prev, next_img, opts);let mut prompt = stroke_prompt(strokes, opts.scale);- if !add_prior_prompt(&mut prompt, &prior.soft, w, h) {+ if !add_prior_prompt(&mut prompt, &prior.soft, &resolve(strokes, StrokeKind::Fg, StrokeKind::Bg, opts.scale, w, h), w, h) {return propagate(prev_img, prev, next_img, strokes, opts, refine_radius);}match m.segment(&rgb_of(next_img), w, h, &prompt) {diff --git a/crates/track/src/roto/tests.rs b/crates/track/src/roto/tests.rsindex 1922dd0a2..f5170e754 100644--- a/crates/track/src/roto/tests.rs+++ b/crates/track/src/roto/tests.rs@@ -509,6 +509,27 @@ fn box_stroke_is_the_model_box_prompt() {assert!(rasterize(&[&bx], &[StrokeKind::Fg, StrokeKind::Bg], 1.0, 100, 80).iter().all(|v| !v));let mut q = stroke_prompt(&[&fg, &bx], 1.0);let soft: Vec<f32> = (0..100 * 80).map(|i| ((i % 100) < 10) as u8 as f32).collect();- assert!(add_prior_prompt(&mut q, &soft, 100, 80));+ assert!(add_prior_prompt(&mut q, &soft, &[], 100, 80));assert_eq!(q.bbox, Some([20.0, 10.0, 80.0, 60.0]));}++/// A model is never prompted against the latest correction: stroke points that a later stroke+/// of the other kind painted over are dropped, and the propagated matte's points and mask leave+/// out what this frame's background strokes removed.+#[test]+fn model_prompts_follow_the_latest_stroke() {+ let green = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 4.0, points: vec![[50.0, 40.0]] };+ let red = Stroke { kind: StrokeKind::Bg, ..green.clone() };+ assert_eq!(stroke_prompt(&[&green, &red], 1.0).points, vec![([50.0, 40.0], false)]);+ assert_eq!(stroke_prompt(&[&green, &red, &green], 1.0).points, vec![([50.0, 40.0], true); 2]);+ let (w, h) = (100, 80);+ let soft: Vec<f32> = (0..w * h).map(|i| ((i % w) < 50) as u8 as f32).collect();+ let erase = Stroke { kind: StrokeKind::Bg, frame: 0, radius: 15.0, points: vec![[25.0, 40.0]] };+ let hard = resolve(&[&erase], StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);+ let mut q = Prompt::default();+ assert!(add_prior_prompt(&mut q, &soft, &hard, w, h));+ assert!(!q.points.is_empty());+ assert!(q.points.iter().all(|(p, fg)| *fg && !covers(&erase, 1.0, [p[0] as f64, p[1] as f64])), "{:?}", q.points);+ let mask = q.mask.unwrap();+ assert!(hard.iter().zip(&mask).all(|(hd, m)| *hd != HARD_BG || *m == 0.0));+}