EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
S2.69: several objects in one Roto Brush instance, tracked in one SAM 2 pass
Kristian Myram08bf6e43fe5f
diff --git a/crates/effects/src/migrate.rs b/crates/effects/src/migrate.rsindex 35d0960e0..ecea0fe19 100644--- a/crates/effects/src/migrate.rs+++ b/crates/effects/src/migrate.rs@@ -765,7 +765,13 @@ not a stroke",let data = effectcraft_track::roto::RotoData {base: Some(0),span: [0, 10],- strokes: vec![effectcraft_track::roto::Stroke { kind: effectcraft_track::roto::StrokeKind::Fg, frame: 0, radius: 5.0, points: vec![[2.0, 2.0]] }],+ strokes: vec![effectcraft_track::roto::Stroke {+ kind: effectcraft_track::roto::StrokeKind::Fg,+ frame: 0,+ radius: 5.0,+ points: vec![[2.0, 2.0]],+ object: 0,+ }],};roto.get_mut(crate::roto::STROKES).unwrap().value = Value::Str(data.to_json());to_effect_space(spec, &mut roto, SIZE, D, &own);diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rsindex 4b5e1fd42..3a75bec1b 100644--- a/crates/effects/src/object_matte.rs+++ b/crates/effects/src/object_matte.rs@@ -144,7 +144,7 @@ pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_moif !inside(pt) {return None;}- let mut out = vec![Stroke { kind: StrokeKind::Fg, frame, radius: POINT_RADIUS, points: vec![pt] }];+ let mut out = vec![Stroke { kind: StrokeKind::Fg, frame, radius: POINT_RADIUS, points: vec![pt], object: 0 }];let rect = |x0: f64, y0: f64, x1: f64, y1: f64| vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]];let boxed = b(params, "useBox").then(|| Some((v2(params, "boxTopLeft")?, v2(params, "boxBottomRight")?))).flatten();match boxed {@@ -158,9 +158,9 @@ pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_mo}out.push(if with_model {// The model takes the box itself as its box prompt.- Stroke { kind: StrokeKind::Box, frame, radius: 0.0, points: vec![[x0, y0], [x1, y1]] }+ Stroke { kind: StrokeKind::Box, frame, radius: 0.0, points: vec![[x0, y0], [x1, y1]], object: 0 }} else {- Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) }+ Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1), object: 0 }});}None if !with_model => {@@ -168,7 +168,7 @@ pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_moif w <= 4.0 * i || h <= 4.0 * i {return None;}- out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(i, i, w - i, h - i) });+ out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(i, i, w - i, h - i), object: 0 });}None => {}}diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rsindex c31c4836b..ffa41f01e 100644--- a/crates/effects/src/roto.rs+++ b/crates/effects/src/roto.rs@@ -564,32 +564,72 @@ impl Chain {let (Some(&need_lo), Some(&need_hi)) = (needed.first(), needed.last()) else {return Ok(Some(TrackSummary::default()));};- // A prompt is a few clicks from the strokes and a box around what the classic engine makes of- // them on that frame (clicks alone pick a part of the object, not all of it).- let mut all_prompts: Vec<(i64, effectcraft_segment::Prompt)> = vec![];- for f in self.data.stroke_frames().into_iter().filter(|f| self.data.in_span(*f)) {- let strokes = self.data.strokes_at(f);- let p = match frame(f) {- Some(img) => rb::tracker_prompt_boxed(&strokes, self.opts.scale, &rb::segment(&img, &strokes, None, &self.opts, self.refine_radius)),- None => rb::tracker_prompt(&strokes, self.opts.scale),- };- if p.points.iter().any(|(_, fg)| *fg) {- all_prompts.push((f, p));+ // Each object has its own prompts: a few clicks from its strokes and a box around what the classic engine+ // makes of them on that frame (clicks alone pick a part of the object, not all of it).+ let objects = self.data.objects();+ let mut by_object: Vec<Vec<(i64, effectcraft_segment::Prompt)>> = vec![];+ for &obj in &objects {+ let mut prompts = vec![];+ for f in self.data.stroke_frames().into_iter().filter(|f| self.data.in_span(*f)) {+ let strokes = self.data.strokes_of(f, obj);+ let p = match frame(f) {+ Some(img) => rb::tracker_prompt_boxed(&strokes, self.opts.scale, &rb::segment(&img, &strokes, None, &self.opts, self.refine_radius)),+ None => rb::tracker_prompt(&strokes, self.opts.scale),+ };+ if p.points.iter().any(|(_, fg)| *fg) {+ prompts.push((f, p));+ }}+ by_object.push(prompts);}- if all_prompts.is_empty() {+ // objects without a foreground prompt cannot be followed+ let followed: Vec<(u32, Vec<(i64, effectcraft_segment::Prompt)>)> = objects.into_iter().zip(by_object).filter(|(_, p)| !p.is_empty()).collect();+ if followed.is_empty() {return Err("no foreground strokes to follow".into());}- // The window to track: from the nearest prompted frame at or before the first frame needed- // (a correction restarts the object there) to the last frame needed, reaching the first- // prompted frame when everything needed lies before it.- let first_prompt = all_prompts.first().map_or(need_lo, |p| p.0);- let lo = all_prompts.iter().map(|p| p.0).filter(|f| *f <= need_lo).max().unwrap_or(need_lo);- let hi = if need_hi < first_prompt { first_prompt } else { need_hi };- let prompts: Vec<(i64, effectcraft_segment::Prompt)> = all_prompts.into_iter().filter(|p| p.0 >= lo && p.0 <= hi).collect();- let req = TrackRequest::single(lo, (hi - lo + 1) as usize, prompts);+ // The window to track: every object restarts at the nearest of its prompted frames at or before the first+ // frame needed (a correction restarts the object there), so the window starts at the earliest restart; it+ // ends at the last frame needed, reaching the farthest first prompt when an object's needed frames lie+ // before its prompt.+ let lo = followed.iter().map(|(_, p)| p.iter().map(|q| q.0).filter(|f| *f <= need_lo).max().unwrap_or(need_lo)).min().unwrap_or(need_lo);+ let first_prompt = followed.iter().filter_map(|(_, p)| p.first().map(|q| q.0)).max().unwrap_or(need_hi);+ let hi = need_hi.max(first_prompt);+ let objects: Vec<u32> = followed.iter().map(|(o, _)| *o).collect();+ let prompts: Vec<Vec<(i64, effectcraft_segment::Prompt)>> =+ followed.into_iter().map(|(_, p)| p.into_iter().filter(|q| q.0 >= lo && q.0 <= hi).collect()).collect();+ let req = TrackRequest { first: lo, count: (hi - lo + 1) as usize, objects: prompts };let frame = std::cell::RefCell::new(frame);let mut summary = TrackSummary::default();+ // A frame is finished when every object has reported it (the objects' mattes are united).+ let mut pending: BTreeMap<i64, Vec<Option<effectcraft_segment::TrackedFrame>>> = BTreeMap::new();+ let finish = |g: i64, parts: Vec<Option<effectcraft_segment::TrackedFrame>>, summary: &mut TrackSummary, progress: &mut dyn FnMut(usize)| {+ let (Some(key), Some(img)) = (self.keys.get(&g).copied(), (frame.borrow_mut())(g)) else { return };+ let mut segs = vec![];+ let (mut confidence, mut present) = (f32::INFINITY, false);+ let mut worst = f32::INFINITY;+ for (obj, t) in objects.iter().zip(parts) {+ let Some(t) = t else { continue };+ let strokes = self.data.strokes_of(g, *obj);+ segs.push(rb::segment_tracked(&img, &strokes, t.prob, t.occluded, &self.opts, self.refine_radius));+ worst = worst.min(t.confidence);+ if !t.occluded {+ present = true;+ confidence = confidence.min(t.confidence);+ }+ }+ if segs.is_empty() {+ return;+ }+ // a frame is occluded when every object is absent; otherwise the doubtful object sets the confidence+ let (occluded, confidence) = (!present, if present { confidence } else { worst });+ store(tracked_key(key), self.opts.scale, Arc::new(effectcraft_track::roto::FrameSeg::union(segs)));+ trust_cache().lock().unwrap_or_else(|e| e.into_inner()).insert(tracked_key(key), FrameTrust { confidence, occluded });+ summary.frames += 1;+ summary.occluded += usize::from(occluded);+ summary.min_confidence = if summary.frames == 1 { confidence } else { summary.min_confidence.min(confidence) };+ progress(summary.frames);+ };+ let n_objects = objects.len();tracker.track(&req,&mut |g| {@@ -597,18 +637,21 @@ impl Chain {Some(Arc::new(ClipFrame { rgb: rb::rgb_of(&img), w: img.width as usize, h: img.height as usize }))},cancel,- &mut |_, g, t| {- let (Some(key), Some(img)) = (self.keys.get(&g).copied(), (frame.borrow_mut())(g)) else { return };- let strokes = self.data.strokes_at(g);- let seg = rb::segment_tracked(&img, &strokes, t.prob, t.occluded, &self.opts, self.refine_radius);- store(tracked_key(key), self.opts.scale, Arc::new(seg));- trust_cache().lock().unwrap_or_else(|e| e.into_inner()).insert(tracked_key(key), FrameTrust { confidence: t.confidence, occluded: t.occluded });- summary.frames += 1;- summary.occluded += usize::from(t.occluded);- summary.min_confidence = if summary.frames == 1 { t.confidence } else { summary.min_confidence.min(t.confidence) };- progress(summary.frames);+ &mut |o, g, t| {+ let parts = pending.entry(g).or_insert_with(|| vec![None; n_objects]);+ if let Some(slot) = parts.get_mut(o) {+ *slot = Some(t);+ }+ if parts.iter().all(Option::is_some) {+ let parts = pending.remove(&g).unwrap_or_default();+ finish(g, parts, &mut summary, progress);+ }},)?;+ // frames some object never reported (it was not followed there): united from the rest+ for (g, parts) in std::mem::take(&mut pending) {+ finish(g, parts, &mut summary, progress);+ }Ok(Some(summary))}}@@ -1038,8 +1081,8 @@ mod tests {#[test]fn renders_the_segmented_disk_on_demand() {let mut d = RotoData::default();- d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0], [45.0, 40.0]] }, 20, [0, 24]);- d.add(Stroke { kind: StrokeKind::Bg, frame: 0, radius: 3.0, points: vec![[5.0, 5.0], [95.0, 5.0], [95.0, 75.0]] }, 20, [0, 24]);+ d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0], [45.0, 40.0]], object: 0 }, 20, [0, 24]);+ d.add(Stroke { kind: StrokeKind::Bg, frame: 0, radius: 3.0, points: vec![[5.0, 5.0], [95.0, 5.0], [95.0, 75.0]], object: 0 }, 20, [0, 24]);let params = params_with(&d);let host = Seq;let run = |t: f64| {@@ -1083,6 +1126,91 @@ mod tests {assert!(o.data.iter().all(|p| p[3] == 0.0));}+ /// A stand-in tracker for two objects: object 0 is the moving disk, object 1 a fixed square. It records how many+ /// objects (and prompts per object) it was asked for.+ struct TwoObjects(std::sync::Mutex<Vec<Vec<usize>>>);+ impl effectcraft_segment::MaskModel for TwoObjects {+ fn info(&self) -> &'static effectcraft_segment::ModelInfo {+ &effectcraft_segment::SAM2_TINY+ }+ fn segment(&self, _: &[[f32; 3]], _: usize, _: usize, _: &effectcraft_segment::Prompt) -> effectcraft_segment::Result<Vec<f32>> {+ Err("no weights".into())+ }+ fn tracker(&self) -> Option<&dyn effectcraft_segment::ObjectTracker> {+ Some(self)+ }+ }+ impl effectcraft_segment::ObjectTracker for TwoObjects {+ fn track(+ &self,+ req: &effectcraft_segment::TrackRequest,+ frame: &mut dyn FnMut(i64) -> Option<Arc<ClipFrame>>,+ _: &dyn Fn() -> bool,+ out: &mut dyn FnMut(usize, i64, effectcraft_segment::TrackedFrame),+ ) -> effectcraft_segment::Result<()> {+ self.0.lock().unwrap().push(req.objects.iter().map(Vec::len).collect());+ for g in req.first..req.first + req.count as i64 {+ let img = frame(g).ok_or("no frame")?;+ let c = [40.0 + 2.0 * g as f64, 40.0];+ // the objects report in reverse order on odd frames: the union must not depend on it+ let order: Vec<usize> = if g % 2 == 1 { (0..req.objects.len()).rev().collect() } else { (0..req.objects.len()).collect() };+ for o in order {+ let prob = (0..img.w * img.h)+ .map(|i| {+ let (x, y) = ((i % img.w) as f64 + 0.5, (i / img.w) as f64 + 0.5);+ let inside = if o == 0 { (x - c[0]).hypot(y - c[1]) <= 15.0 } else { (8.0..22.0).contains(&x) && (8.0..22.0).contains(&y) };+ f32::from(inside)+ })+ .collect();+ out(o, g, effectcraft_segment::TrackedFrame { prob, w: img.w, h: img.h, confidence: if o == 0 { 0.9 } else { 0.4 }, occluded: false });+ }+ }+ Ok(())+ }+ }++ #[test]+ fn objects_are_followed_in_one_pass_and_their_mattes_united() {+ let mut d = RotoData::default();+ let stroke = |kind, points: Vec<[f64; 2]>, object| Stroke { kind, frame: 0, radius: 3.0, points, object };+ d.add(stroke(StrokeKind::Fg, vec![[35.0, 40.0], [45.0, 40.0]], 0), 8, [0, 24]);+ d.add(stroke(StrokeKind::Fg, vec![[12.0, 15.0], [18.0, 15.0]], 1), 8, [0, 24]);+ d.add(stroke(StrokeKind::Bg, vec![[85.0, 70.0], [95.0, 70.0]], 0), 8, [0, 24]);+ assert_eq!(d.objects(), vec![0, 1]);+ // the Refine Edge strokes belong to every object; the others only to their own+ assert_eq!(d.strokes_of(0, 1).len(), 1);+ assert_eq!(d.strokes_of(0, 0).len(), 2);+ let params = params_with(&d);+ let mut chain = Chain::new(¶ms, [100.0, 80.0], 25.0, 1.0);+ let fake = Arc::new(TwoObjects(Default::default()));+ chain.model = Some(fake.clone());+ // the clip: the moving disk and a fixed yellow square+ let mut get = |f: i64| {+ let mut img = frame(f as f64 / 25.0 + 0.001);+ for y in 8..22 {+ for x in 8..22 {+ img.set(x, y, [0.95, 0.9, 0.1 + ((x * 3 + y * 5) % 7) as f32 / 70.0, 1.0]);+ }+ }+ Some(Arc::new(img))+ };+ let mut seen = 0;+ let summary = chain.track(&mut get, &|| false, &mut |n| seen = n).unwrap().unwrap();+ // one request for both objects; object 0 has its foreground and background prompt on the frame, object 1 its own+ assert_eq!(fake.0.lock().unwrap().len(), 1);+ assert_eq!(fake.0.lock().unwrap()[0], vec![1, 1]);+ assert_eq!(summary.frames, 9);+ assert_eq!(seen, 9);+ // the least sure object sets the frame's confidence+ assert!((summary.min_confidence - 0.4).abs() < 1e-6, "{}", summary.min_confidence);+ assert_eq!(summary.occluded, 0);+ let seg = cached(tracked_key(chain.keys[&3]), 1.0).expect("frame 3 is tracked");+ let at = |x: usize, y: usize| seg.matte[y * 100 + x];+ assert!(at(46, 40) != 0, "the moving disk (frame 3 centre x = 46)");+ assert!(at(15, 15) != 0, "the fixed square");+ assert_eq!(at(60, 70), 0, "nothing elsewhere");+ }+/// Instances saved before the propagation controls got their own twirl-down (and motion/// blur / decontamination their nested ones) load with their values in the new places.#[test]diff --git a/crates/engine/src/analysis/roto.rs b/crates/engine/src/analysis/roto.rsindex 9cc950e07..626de3030 100644--- a/crates/engine/src/analysis/roto.rs+++ b/crates/engine/src/analysis/roto.rs@@ -51,6 +51,9 @@ pub struct RotoOptions {pub boundary_color: [f64; 3],/// Start propagating in the background after each stroke.pub auto_propagate: bool,+ /// The object new strokes belong to (0 is the first). With a tracker every object is followed on its own+ /// and their mattes are united; the classic engine treats all foreground strokes as one object.+ pub object: u32,}impl Default for RotoOptions {@@ -63,10 +66,14 @@ impl Default for RotoOptions {overlay_opacity: 50.0,boundary_color: [1.0, 0.25, 0.75],auto_propagate: true,+ object: 0,}}}+/// The highest object id (SAM 2 follows up to 32 objects in one pass).+pub const MAX_OBJECT_ID: u32 = 31;+pub const VIEWS: [&str; 4] = ["alphaBoundary", "alpha", "alphaOverlay", "none"];/// What a job does.diff --git a/crates/engine/src/cmd/roto_cmds.rs b/crates/engine/src/cmd/roto_cmds.rsindex 5849fa2c8..5124d413d 100644--- a/crates/engine/src/cmd/roto_cmds.rs+++ b/crates/engine/src/cmd/roto_cmds.rs@@ -27,7 +27,7 @@ pub fn specs() -> Vec<CommandSpec> {"Roto Brush Stroke",[],None,- "{layer?, kind?: fg|bg|refine|refineErase, points: [[x, y], …] (layer pixels), frame? (layer frame; default current), radius? (layer px; default the tool's diameter / 2), effect?}",+ "{layer?, kind?: fg|bg|refine|refineErase, points: [[x, y], …] (layer pixels), frame? (layer frame; default current), radius? (layer px; default the tool's diameter / 2), effect?, object? (0–31; default the current object: with a tracker each object is followed on its own and the mattes are united)}",has_comp,stroke),@@ -43,14 +43,22 @@ pub fn specs() -> Vec<CommandSpec> {cmd!("roto.span", "Set Segmentation Span", [], None, "{layer?, effect?, start?, end? (layer frames)}", has_comp, span),cmd!("roto.freeze", "Freeze", [], None, "{layer?, effect?, wait?}", has_comp, freeze),cmd!("roto.unfreeze", "Unfreeze", [], None, "{layer?, effect?}", has_comp, unfreeze),- cmd!("roto.clearStrokes", "Remove Roto Brush Strokes", [], None, "{layer?, effect?, frame? (only this layer frame), kind?}", has_comp, clear_strokes),+ cmd!(+ "roto.clearStrokes",+ "Remove Roto Brush Strokes",+ [],+ None,+ "{layer?, effect?, frame? (only this layer frame), kind?, object? (only this object's strokes)}",+ has_comp,+ clear_strokes+ ),cmd!("roto.cancel", "Stop Roto Brush", [], None, "{}", is_running, |s, _| Ok(json!({"stopped": s.stop_roto()}))),cmd!("roto.options","Roto Brush Options",[],None,- "{diameter?, refineDiameter?, view?: alphaBoundary|alpha|alphaOverlay|none, overlayColor?: [r,g,b], overlayOpacity?, boundaryColor?: [r,g,b], autoPropagate?}",+ "{diameter?, refineDiameter?, view?: alphaBoundary|alpha|alphaOverlay|none, overlayColor?: [r,g,b], overlayOpacity?, boundaryColor?: [r,g,b], autoPropagate?, object? (the object new strokes belong to, 0–31)}",always,options),@@ -232,7 +240,8 @@ fn stroke(s: &mut Session, p: &Value) -> Result<Value> {if existing.is_none() && !kind.is_segmentation() {return Err(bad(cmd, "draw a foreground stroke with the Roto Brush tool first"));}- let st = Stroke { kind, frame, radius, points };+ let object = f_p(p, "object").map_or(s.state.roto.object, |v| v.clamp(0.0, rt::MAX_OBJECT_ID as f64).round() as u32);+ let st = Stroke { kind, frame, radius, points, object };let label = if kind.is_segmentation() { "Roto Brush Stroke" } else { "Refine Edge Stroke" };let (uid, d) = s.edit(label, None, |proj, _| {let mut next = proj.next_id;@@ -263,7 +272,7 @@ fn stroke(s: &mut Session, p: &Value) -> Result<Value> {s.state.selected_layers = vec![lid];}Ok(json!({- "layer": lid.0, "effect": uid, "frame": frame, "kind": kind, "radius": radius,+ "layer": lid.0, "effect": uid, "frame": frame, "kind": kind, "radius": radius, "object": object,"base": d.base, "span": d.span, "strokes": d.strokes.len(),}))}@@ -369,8 +378,9 @@ fn clear_strokes(s: &mut Session, p: &Value) -> Result<Value> {Some(k) => Some(StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}`")))?),None => None,};+ let object = f_p(p, "object").map(|v| v.round() as u32);let n0 = d.strokes.len();- d.strokes.retain(|st| !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k)));+ d.strokes.retain(|st| !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k) && object.is_none_or(|o| st.object == o)));if d.strokes.iter().all(|s| !s.kind.is_segmentation()) {d.base = None;} else if d.base.is_some_and(|b| !d.strokes.iter().any(|s| s.frame == b && s.kind.is_segmentation())) {@@ -412,6 +422,9 @@ fn options(s: &mut Session, p: &Value) -> Result<Value> {if let Some(b) = b_p(p, "autoPropagate") {o.auto_propagate = b;}+ if let Some(v) = f_p(p, "object") {+ o.object = v.clamp(0.0, rt::MAX_OBJECT_ID as f64).round() as u32;+ }s.state.roto = o.clone();Ok(serde_json::to_value(o).unwrap_or(Value::Null))}@@ -430,6 +443,8 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {out["base"] = json!(d.base);out["span"] = json!(d.span);out["strokes"] = json!(d.strokes.len());+ out["objects"] = json!(d.objects().iter().map(|o| json!({"id": o, "strokes": d.strokes.iter().filter(|s| s.object == *o).count()})).collect::<Vec<_>>());+ out["currentObject"] = json!(s.state.roto.object);out["strokeFrames"] = json!(d.stroke_frames());out["frozen"] = json!(frozen_of(&g));out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid)));diff --git a/crates/engine/src/tests/roto.rs b/crates/engine/src/tests/roto.rsindex 66baee148..e4db67647 100644--- a/crates/engine/src/tests/roto.rs+++ b/crates/engine/src/tests/roto.rs@@ -1007,3 +1007,28 @@ fn a_correction_re_tracks_only_the_frames_it_changes() {assert_eq!(trust.len(), 21);assert!(trust.values().all(|t| t.confidence > 0.9 && !t.occluded));}++#[test]+fn strokes_belong_to_the_current_object() {+ let mut s = Session::default();+ s.execute("comp.new", json!({"width": 64, "height": 64, "frameRate": 25, "duration": 1})).unwrap();+ let layer = s.execute("layer.newSolid", json!({"width": 64, "height": 64})).unwrap()["layer"].as_u64().unwrap();+ s.execute("roto.stroke", json!({"layer": layer, "points": [[10, 10]], "radius": 4})).unwrap();+ // the current object is an option shared by the tool and the commands+ s.execute("roto.options", json!({"object": 1})).unwrap();+ s.execute("roto.stroke", json!({"layer": layer, "points": [[50, 50]], "radius": 4})).unwrap();+ s.execute("roto.stroke", json!({"layer": layer, "kind": "bg", "points": [[30, 30]], "radius": 4})).unwrap();+ // an explicit object wins over the option, and is clamped to the supported range+ s.execute("roto.stroke", json!({"layer": layer, "points": [[20, 50]], "radius": 4, "object": 99})).unwrap();+ let st = s.execute("roto.status", json!({"layer": layer})).unwrap();+ assert_eq!(st["currentObject"], 1);+ assert_eq!(st["objects"], json!([{"id": 0, "strokes": 1}, {"id": 1, "strokes": 2}, {"id": 31, "strokes": 1}]));+ // removing one object's strokes leaves the others+ let r = s.execute("roto.clearStrokes", json!({"layer": layer, "object": 1})).unwrap();+ assert_eq!(r["removed"], 2);+ let st = s.execute("roto.status", json!({"layer": layer})).unwrap();+ assert_eq!(st["objects"], json!([{"id": 0, "strokes": 1}, {"id": 31, "strokes": 1}]));+ // the object survives a save and a load+ let json = s.project.to_json();+ assert!(json.contains("\\\"object\\\":31"), "the project file carries the object");+}diff --git a/crates/track/src/roto.rs b/crates/track/src/roto.rsindex 000f7b55f..18c6eaa77 100644--- a/crates/track/src/roto.rs+++ b/crates/track/src/roto.rs@@ -88,6 +88,15 @@ pub struct Stroke {pub frame: i64,pub radius: f64,pub points: Vec<[f64; 2]>,+ /// Which object of the instance the stroke belongs to (0 for the first, and for every older project).+ /// With a tracker each object is followed on its own and the mattes are combined; the classic engine+ /// treats all foreground strokes as one object.+ #[serde(default, skip_serializing_if = "is_zero")]+ pub object: u32,+}++fn is_zero(v: &u32) -> bool {+ *v == 0}/// A Roto Brush instance's user data: strokes, the base frame and the segmentation span@@ -130,6 +139,18 @@ impl RotoData {v.dedup();v}+ /// The objects that have a foreground stroke, in order.+ pub fn objects(&self) -> Vec<u32> {+ let mut v: Vec<u32> = self.strokes.iter().filter(|s| s.kind == StrokeKind::Fg).map(|s| s.object).collect();+ v.sort_unstable();+ v.dedup();+ v+ }+ /// The strokes of `object` on `frame`: its own foreground, background and box strokes, and the Refine Edge+ /// strokes (which belong to the whole matte).+ pub fn strokes_of(&self, frame: i64, object: u32) -> Vec<&Stroke> {+ self.strokes.iter().filter(|s| s.frame == frame && (s.object == object || matches!(s.kind, StrokeKind::Refine | StrokeKind::RefineErase))).collect()+ }pub fn in_span(&self, f: i64) -> bool {self.base.is_some() && f >= self.span[0] && f <= self.span[1]}@@ -224,6 +245,26 @@ pub struct FrameSeg {}impl FrameSeg {+ /// The union of several objects' segmentations of one frame (the largest keeps its colour models). The+ /// parts must share one size; any that does not is ignored.+ pub fn union(parts: Vec<FrameSeg>) -> FrameSeg {+ let Some(first) = parts.first() else { return FrameSeg::default() };+ let (w, h) = (first.w, first.h);+ let mut parts: Vec<FrameSeg> = parts.into_iter().filter(|p| p.w == w && p.h == h && p.matte.len() == w * h).collect();+ parts.sort_by_key(|p| std::cmp::Reverse(p.area()));+ let mut it = parts.into_iter();+ let Some(mut out) = it.next() else { return FrameSeg::default() };+ for p in it {+ for (a, b) in out.matte.iter_mut().zip(&p.matte) {+ *a = (*a).max(*b);+ }+ for (a, b) in out.refine.iter_mut().zip(&p.refine) {+ *a = (*a).max(*b);+ }+ }+ out+ }+pub fn empty(w: usize, h: usize) -> FrameSeg {let Some(n) = w.checked_mul(h).filter(|n| *n <= rle::MAX_PLANE_BYTES) else {return FrameSeg::default();diff --git a/crates/track/src/roto/bench.rs b/crates/track/src/roto/bench.rsindex 2dafb9f3f..c086bfd57 100644--- a/crates/track/src/roto/bench.rs+++ b/crates/track/src/roto/bench.rs@@ -122,7 +122,7 @@ fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) ->let t = f64::from(k) / 10.0;[a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]};- Stroke { kind, frame, radius, points: (0..=10).map(pt).collect() }+ Stroke { kind, frame, radius, points: (0..=10).map(pt).collect(), object: 0 }}/// The strokes an artist would paint on the base frame: one through the disk, two around it.diff --git a/crates/track/src/roto/tests.rs b/crates/track/src/roto/tests.rsindex b7d8c7809..50ccaeaa4 100644--- a/crates/track/src/roto/tests.rs+++ b/crates/track/src/roto/tests.rs@@ -65,7 +65,13 @@ pub(crate) fn frame_at(c: [f64; 2]) -> (Image, Vec<u8>) {}fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) -> Stroke {- Stroke { kind, frame, radius, points: (0..=10).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 10.0, a[1] + (b[1] - a[1]) * k as f64 / 10.0]).collect() }+ Stroke {+ kind,+ frame,+ radius,+ points: (0..=10).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 10.0, a[1] + (b[1] - a[1]) * k as f64 / 10.0]).collect(),+ object: 0,+ }}/// The strokes an artist would paint on frame `f`: one through the disk, one around it.@@ -231,6 +237,7 @@ fn ring(frame: i64, c: [f64; 2], r: f64, radius: f64) -> Stroke {frame,radius,points: (0..=48).map(|k| k as f64 / 48.0 * std::f64::consts::TAU).map(|a| [c[0] + r * a.cos(), c[1] + r * a.sin()]).collect(),+ object: 0,}}@@ -295,7 +302,7 @@ fn later_refine_strokes_override_earlier_erases() {let (img, _) = soft_frame(c, 10.0, [0.9, 0.25, 0.1], [0.1, 0.3, 0.9]);let spot = [c[0] + R, c[1]];let i = spot[1] as usize * W as usize + spot[0] as usize;- let erase = Stroke { kind: StrokeKind::RefineErase, frame: 0, radius: 6.0, points: vec![spot] };+ let erase = Stroke { kind: StrokeKind::RefineErase, frame: 0, radius: 6.0, points: vec![spot], object: 0 };let refine = Stroke { kind: StrokeKind::Refine, ..erase.clone() };let band = |st: &[Stroke]| segment(&img, &st.iter().collect::<Vec<_>>(), None, &SegOpts::default(), 9.0).refine[i];let mut st = vec![@@ -389,7 +396,7 @@ fn roto_perf_1080p() {let mut seg2 = next.clone();let c = center(1);let ring: Vec<[f64; 2]> = (0..=64).map(|k| k as f64 / 64.0 * std::f64::consts::TAU).map(|t| [(c[0] + R * t.cos()) * s, (c[1] + R * t.sin()) * s]).collect();- let st = [Stroke { kind: StrokeKind::Refine, frame: 1, radius: 20.0, points: ring }];+ let st = [Stroke { kind: StrokeKind::Refine, frame: 1, radius: 20.0, points: ring, object: 0 }];let r = rasterize(&st.iter().collect::<Vec<_>>(), &[StrokeKind::Refine], 1.0, 1920, 1080);seg2.refine = r.iter().map(|v| *v as u8).collect();let rgb = rgb_of(&b);@@ -500,8 +507,8 @@ fn mobilesam_segments_and_propagates_the_disk() {/// ignores it; a prior's box does not replace it.#[test]fn box_stroke_is_the_model_box_prompt() {- let fg = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: vec![[50.0, 40.0]] };- let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[80.0, 60.0], [20.0, 10.0]] };+ let fg = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: vec![[50.0, 40.0]], object: 0 };+ let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[80.0, 60.0], [20.0, 10.0]], object: 0 };let p = stroke_prompt(&[&fg, &bx], &[], 0.5, 0, 0);assert_eq!(p.bbox, Some([10.0, 5.0, 40.0, 30.0]));assert_eq!(p.points, vec![([25.0, 20.0], true)]);@@ -523,14 +530,14 @@ fn box_stroke_is_the_model_box_prompt() {/// out what this frame's background strokes removed.#[test]fn model_prompts_follow_the_latest_stroke() {- let green = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 4.0, points: vec![[50.0, 40.0]] };+ let green = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 4.0, points: vec![[50.0, 40.0]], object: 0 };let red = Stroke { kind: StrokeKind::Bg, ..green.clone() };let (w, h) = (100, 80);let prompt = |st: &[&Stroke]| stroke_prompt(st, &resolve(st, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h), 1.0, w, h).points;assert_eq!(prompt(&[&green, &red]), vec![([50.0, 40.0], false)]);assert_eq!(prompt(&[&green, &red, &green]), vec![([50.0, 40.0], true); 2]);let soft: Vec<f32> = (0..w * h).map(|i| ((i % w) < 50) as u8 as f32).collect();- let erase = Stroke { kind: StrokeKind::Bg, frame: 0, radius: 15.0, points: vec![[25.0, 40.0]] };+ let erase = Stroke { kind: StrokeKind::Bg, frame: 0, radius: 15.0, points: vec![[25.0, 40.0]], object: 0 };let hard = resolve(&[&erase], StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);let mut q = Prompt::default();assert!(add_prior_prompt(&mut q, &soft, &hard, w, h));@@ -561,7 +568,7 @@ impl effectcraft_segment::MaskModel for CountingModel {fn a_correction_keeps_the_model_prompted_with_what_is_left() {let (img, gt, rect) = bar_frame();let (w, h) = (img.width as usize, img.height as usize);- let dot = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 15.0, points: vec![[80.0, 50.0]] };+ let dot = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 15.0, points: vec![[80.0, 50.0]], object: 0 };let tap = Stroke { kind: StrokeKind::Bg, radius: 2.0, ..dot.clone() };let refs = [&dot, &tap];let model = CountingModel(rect, Default::default());@@ -588,7 +595,7 @@ fn tracker_prompts_carry_a_handful_of_points() {let (fg, bg) = (p.points.iter().filter(|(_, f)| *f).count(), p.points.iter().filter(|(_, f)| !*f).count());assert!((1..=TRACKER_FG_POINTS).contains(&fg) && (1..=TRACKER_BG_POINTS).contains(&bg), "{fg} fg, {bg} bg");// a long, finely sampled stroke still gives at most the limit, and an empty one gives none- let long = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: (0..400).map(|i| [i as f64 * 0.4, 50.0]).collect() };+ let long = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: (0..400).map(|i| [i as f64 * 0.4, 50.0]).collect(), object: 0 };assert!(tracker_prompt(&[&long], 1.0).points.len() <= TRACKER_FG_POINTS);assert!(tracker_prompt(&[], 1.0).points.is_empty());}@@ -630,8 +637,55 @@ fn tracker_prompts_carry_the_box_of_the_classic_matte() {// no matte, no box; a box the user drew winslet none = tracker_prompt_boxed(&refs, 1.0, &FrameSeg::empty(W as usize, H as usize));assert_eq!(none.bbox, None);- let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[10.0, 10.0], [50.0, 60.0]] };+ let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[10.0, 10.0], [50.0, 60.0]], object: 0 };let mut with_box: Vec<&Stroke> = refs.clone();with_box.push(&bx);assert_eq!(tracker_prompt_boxed(&with_box, 1.0, &classic).bbox, Some([10.0, 10.0, 50.0, 60.0]));}++#[test]+fn strokes_carry_an_object_and_old_data_is_object_zero() {+ // saved before objects existed: no field, and the field is not written for object 0+ let old = r#"{"base":0,"span":[0,4],"strokes":[{"kind":"fg","frame":0,"radius":3.0,"points":[[1.0,2.0]]}]}"#;+ let d = RotoData::from_json(old);+ assert_eq!(d.strokes.first().map(|s| s.object), Some(0));+ assert!(!d.to_json().contains("object"), "{}", d.to_json());+ // the keys of existing projects do not change+ let same = RotoData::from_json(&d.to_json());+ assert_eq!(d.chain_keys(1), same.chain_keys(1));++ let mut d = d;+ d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[9.0, 9.0]], object: 2 }, 4, [0, 10]);+ d.add(Stroke { kind: StrokeKind::Bg, frame: 0, radius: 3.0, points: vec![[0.0, 0.0]], object: 2 }, 4, [0, 10]);+ d.add(Stroke { kind: StrokeKind::Refine, frame: 0, radius: 3.0, points: vec![[5.0, 5.0]], object: 0 }, 4, [0, 10]);+ assert_eq!(d.objects(), vec![0, 2]);+ assert!(d.to_json().contains("\"object\":2"));+ assert_eq!(RotoData::from_json(&d.to_json()), d);+ // an object's own strokes, plus the Refine Edge strokes that belong to the whole matte+ assert_eq!(d.strokes_of(0, 2).len(), 3);+ assert_eq!(d.strokes_of(0, 0).len(), 2);+ assert!(d.strokes_of(0, 1).len() == 1);+ // moving a stroke to another object changes the frame keys (the tracked frames are stale)+ let before = d.chain_keys(1);+ if let Some(s) = d.strokes.first_mut() {+ s.object = 1;+ }+ assert_ne!(before, d.chain_keys(1));+}++#[test]+fn the_union_of_objects_is_their_largest_extent() {+ let seg = |cells: &[usize]| {+ let mut s = FrameSeg::empty(4, 1);+ for c in cells {+ s.matte[*c] = 255;+ }+ s+ };+ let u = FrameSeg::union(vec![seg(&[0]), seg(&[2, 3]), seg(&[3])]);+ assert_eq!(u.matte, vec![255, 0, 255, 255]);+ assert_eq!(FrameSeg::union(vec![]).w, 0);+ // a part of another size is ignored+ let odd = FrameSeg::empty(2, 2);+ assert_eq!(FrameSeg::union(vec![seg(&[1]), odd]).matte, vec![0, 255, 0, 0]);+}diff --git a/crates/ui-egui/src/panels/roto_tool.rs b/crates/ui-egui/src/panels/roto_tool.rsindex f8cd383c6..bf47b558d 100644--- a/crates/ui-egui/src/panels/roto_tool.rs+++ b/crates/ui-egui/src/panels/roto_tool.rs@@ -8,7 +8,7 @@ use effectcraft_engine::render::{EvalCtx, RenderOpts, Renderer};use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2};use serde_json::json;-use crate::i18n::tr;+use crate::i18n::{tr, tr_args};use crate::theme::Tokens;use crate::{EffectcraftApp, widgets};@@ -373,6 +373,31 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte}app.auto.add("layerPanel.roto.freeze", r, label);x += 80.0;+ // Objects: strokes belong to the current object; with a tracker each is followed on its own and the mattes are united.+ let objects = fx::data(¶ms).objects();+ let current = app.session.state.roto.object;+ let known = objects.contains(¤t);+ let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(86.0, 20.0));+ let label = tr_args("Object {}", &[&(current + 1)]);+ if widgets::text_button(ui, r, &label, false, &t, egui::Id::new(("roto-object", uid))).clicked() {+ // cycle through the objects that have strokes (and the empty one being drawn)+ let mut all = objects.clone();+ if !known {+ all.push(current);+ all.sort_unstable();+ }+ let next = all.iter().copied().find(|o| *o > current).or_else(|| all.first().copied()).unwrap_or(0);+ let _ = app.session.execute("roto.options", json!({"object": next}));+ }+ app.auto.add("layerPanel.roto.object", r, &label);+ x += 90.0;+ let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));+ if widgets::text_button(ui, r, "+", false, &t, egui::Id::new(("roto-object-new", uid))).clicked() {+ let next = objects.iter().copied().chain([current]).max().map_or(0, |m| m + 1);+ let _ = app.session.execute("roto.options", json!({"object": next}));+ }+ app.auto.add("layerPanel.roto.newObject", r, "New object");+ x += 34.0;if running {let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(60.0, 20.0));if widgets::text_button(ui, r, tr("Stop"), false, &t, egui::Id::unique("roto-stop")).clicked() {diff --git a/crates/ui-egui/tests/ui_roto_span_confidence.rs b/crates/ui-egui/tests/ui_roto_span_confidence.rsindex de561014a..f3a8bcd9b 100644--- a/crates/ui-egui/tests/ui_roto_span_confidence.rs+++ b/crates/ui-egui/tests/ui_roto_span_confidence.rs@@ -82,6 +82,15 @@ fn the_span_bar_shows_the_trackers_confidence() {h.run_steps(4);assert_eq!(h.state().ui.focused, PanelKind::Layer);+ // the object picker is in the bar; "+" starts a new object, which the next stroke would belong to+ assert!(h.state().auto.find("layerPanel.roto.object").is_some(), "the object picker is on screen");+ assert!(h.state().auto.find("layerPanel.roto.newObject").is_some());+ assert_eq!(h.state().session.state.roto.object, 0);+ h.state_mut().session.execute("roto.options", json!({"object": 1})).unwrap();+ h.run_steps(2);+ assert_eq!(h.state().session.state.roto.object, 1);+ h.state_mut().session.execute("roto.options", json!({"object": 0})).unwrap();+ h.run_steps(2);let track = h.state().auto.find("layerPanel.rotoSpan").expect("the span bar is on screen").rect;let (comp, layer) = {let comp = h.state().session.active_comp().unwrap().clone();diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.mdindex 8b1931494..656574df6 100644--- a/docs/roto-brush-sam2.md+++ b/docs/roto-brush-sam2.md@@ -29,6 +29,23 @@ strokes and span they came from are unchanged. Changing a stroke or the span makChoosing another model, or the classic engine, leaves them in place and `roto.status` says so; a classic Propagatereplaces them with classic mattes. Object Matte saves nothing.+## Several objects in one instance+A Roto Brush instance can hold several objects. The **Object N** button in the Layer panel's bar picks the object new+strokes belong to (it cycles through the objects that have strokes), and **+** starts a new one. Agents: `roto.options+{object}` sets the current object, `roto.stroke` takes `object` (0–31; default the current one), `roto.clearStrokes`+takes `object` to remove one object's strokes, and `roto.status` lists `objects` (`{id, strokes}`) and `currentObject`.++With SAM 2, Propagate follows every object in **one pass** (the frame is encoded once, whatever the number of objects), each+from its own foreground and background strokes, and the mattes are united into the instance's one matte. Each object keeps+its own occlusion: the frame counts as absent only when every object is, and otherwise the least sure object sets the+frame's confidence on the span bar. A background stroke belongs to its object, so it separates that object from what+lies under it without cutting a hole in the others. Refine Edge strokes belong to the whole matte. The classic engine has+no objects: all foreground strokes make one object, and background strokes apply to it as before. Saved projects are+unchanged (strokes without an object are object 0, and nothing is written for object 0).++This is not several Roto Brush instances sharing a pass: each instance reads the layer after the effects above it, so+instances can see different pixels. Objects inside one instance always see the same ones.+## SizesFour checkpoints are in the registry (Settings ▸ Roto Brush); the size is detected from the file, and each ispinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bigger backbone costs time and memory.