EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
S2.70: each object keeps its own matte; Show Object outputs one of them
Kristian Myramf67a17e76555
diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rsindex 3a75bec1b..8a9750282 100644--- a/crates/effects/src/object_matte.rs+++ b/crates/effects/src/object_matte.rs@@ -227,7 +227,7 @@ pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec<i64> {pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option<Arc<dyn MaskModel>>) -> Option<Chain> {let data = roto_data(params, layer_size, f, model.is_some())?;let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref()));- Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model, saved: None })+ Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model, saved: None, show_object: 0 })}/// Frame `f`'s segmentation at full resolution, computing what is missing (the backgrounddiff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rsindex ffa41f01e..e3f6da171 100644--- a/crates/effects/src/roto.rs+++ b/crates/effects/src/roto.rs@@ -80,6 +80,7 @@ pub fn specs() -> Vec<EffectSpec> {p("rotoBrushMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),p("rotoBrushMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)),p("rotoBrushMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),+ p("rotoBrushMatte/object", "Show Object", num(0.0), slider(0.0, 32.0, 0.0, 32.0, 0)),p("refineEdgeMatte/smooth", "Smooth", num(1.0), slider(0.0, 100.0, 0.0, 10.0, 1)),p("refineEdgeMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)),p("refineEdgeMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),@@ -320,6 +321,11 @@ fn tracked_key(key: u64) -> u64 {key.rotate_left(17) ^ TRACK_SALT}+/// Where one object's matte of a tracked frame lives (kept only when the instance has several objects).+fn object_key(key: u64, object: u32) -> u64 {+ tracked_key(key).wrapping_add(0x9e37_79b9_7f4a_7c15u64.wrapping_mul(u64::from(object) + 1))+}+/// How far to trust a frame the tracker followed.#[derive(Clone, Copy, Debug, PartialEq)]pub struct FrameTrust {@@ -398,6 +404,8 @@ pub struct Chain {pub model: Option<Arc<dyn MaskModel>>,/// The saved tracking, while it applies (see [`TrackCache`]).pub(crate) saved: Option<SavedTracking>,+ /// Show Object: the matte of this object only (its id plus one; 0 shows all of them together).+ pub show_object: u32,}impl Chain {@@ -405,7 +413,8 @@ impl Chain {let data = data(params);let keys = data.chain_keys(seed(params, layer_size, fps));let saved = saved_for(params, &data, layer_size, fps);- Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale), model: model_for(params), saved }+ let show_object = f(params, "rotoBrushMatte/object").round().clamp(0.0, 32.0) as u32;+ Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale), model: model_for(params), saved, show_object }}/// Frame `f`'s segmentation at `scale` from the saved tracking or the caches; computes nothing.@@ -413,6 +422,49 @@ impl Chain {self.saved_seg(f, scale).or_else(|| self.keys.get(&f).and_then(|k| cached(*k, scale)))}+ /// Object `o`'s matte of frame `f` at this chain's scale (saved tracking, else this session's), when the instance+ /// has several objects and the tracker followed them there; computes nothing.+ pub fn object_seg(&self, f: i64, o: u32) -> Option<Arc<FrameSeg>> {+ let sc = self.opts.scale;+ let k = *self.keys.get(&f)?;+ let ok = object_key(k, o);+ if let Some(s) = cached(ok, sc) {+ return Some(s);+ }+ let full = match cached(ok, 1.0) {+ Some(s) => s,+ None => {+ let t = self.saved.as_ref()?;+ let s = Arc::new(decode_object(&t.cache, f, o)?);+ store(ok, 1.0, s.clone());+ s+ }+ };+ if sc == 1.0 {+ return Some(full);+ }+ let s = Arc::new(downscale(&full, sc));+ store(ok, sc, s.clone());+ Some(s)+ }++ /// What Show Object asks for at frame `f`: `None` shows the whole matte (all objects, a single object, or a frame+ /// the tracker did not follow); an object the instance does not have shows nothing.+ fn shown_object(&self, f: i64) -> Option<Arc<FrameSeg>> {+ let o = self.show_object.checked_sub(1)?;+ let objects = self.data.objects();+ if objects.len() < 2 {+ return None;+ }+ if !objects.contains(&o) {+ let sc = self.opts.scale;+ let [w, h] =+ self.saved.as_ref().map(|t| t.cache.size).or_else(|| self.keys.get(&f).and_then(|k| cached(tracked_key(*k), sc)).map(|s| [s.w, s.h]))?;+ return Some(Arc::new(FrameSeg::empty(w, h)));+ }+ self.object_seg(f, o)+ }+/// The tracker's trust in frame `f`: from the saved tracking, else from this session's tracking.pub fn trust_at(&self, f: i64) -> Option<FrameTrust> {if let Some(fr) = self.saved.as_ref().and_then(|t| t.cache.frames.get(&f)) {@@ -455,6 +507,10 @@ impl Chain {pub fn ensure(&self, f: i64, frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>, cancel: &dyn Fn() -> bool) -> Option<Arc<FrameSeg>> {let sc = self.opts.scale;let lookup = |g: i64| -> Option<Arc<FrameSeg>> {+ // Show Object: one object's matte, where the tracker followed the objects.+ if let Some(s) = self.shown_object(g) {+ return Some(s);+ }// A frame the tracker followed wins over the classic engine's: the saved tracking, else// this session's.if let Some(s) = self.saved_seg(g, sc) {@@ -604,13 +660,13 @@ impl Chain {let mut pending: BTreeMap<i64, Vec<Option<effectcraft_segment::TrackedFrame>>> = BTreeMap::new();let finish = |g: i64, parts: Vec<Option<effectcraft_segment::TrackedFrame>>, summary: &mut TrackSummary, progress: &mut dyn FnMut(usize)| {let (Some(key), Some(img)) = (self.keys.get(&g).copied(), (frame.borrow_mut())(g)) else { return };- let mut segs = vec![];+ let mut segs: Vec<(u32, effectcraft_track::roto::FrameSeg)> = vec![];let (mut confidence, mut present) = (f32::INFINITY, false);let mut worst = f32::INFINITY;for (obj, t) in objects.iter().zip(parts) {let Some(t) = t else { continue };let strokes = self.data.strokes_of(g, *obj);- segs.push(rb::segment_tracked(&img, &strokes, t.prob, t.occluded, &self.opts, self.refine_radius));+ segs.push((*obj, rb::segment_tracked(&img, &strokes, t.prob, t.occluded, &self.opts, self.refine_radius)));worst = worst.min(t.confidence);if !t.occluded {present = true;@@ -622,7 +678,16 @@ impl Chain {}// a frame is occluded when every object is absent; otherwise the doubtful object sets the confidencelet (occluded, confidence) = (!present, if present { confidence } else { worst });- store(tracked_key(key), self.opts.scale, Arc::new(effectcraft_track::roto::FrameSeg::union(segs)));+ // with several objects each keeps its own matte too (Show Object, the per-object overlay)+ let several = objects.len() > 1;+ let mut parts = Vec::with_capacity(segs.len());+ for (obj, seg) in segs {+ if several {+ store(object_key(key, obj), self.opts.scale, Arc::new(seg.clone()));+ }+ parts.push(seg);+ }+ store(tracked_key(key), self.opts.scale, Arc::new(effectcraft_track::roto::FrameSeg::union(parts)));trust_cache().lock().unwrap_or_else(|e| e.into_inner()).insert(tracked_key(key), FrameTrust { confidence, occluded });summary.frames += 1;summary.occluded += usize::from(occluded);@@ -656,13 +721,26 @@ impl Chain {}}+/// Each object's matte of layer frame `f` where the tracker followed several objects (full resolution; computes+/// nothing). Show Object limits it to the one chosen. Empty with fewer than two objects.+pub fn object_mattes(params: &Params, layer_size: [f64; 2], fps: f64, f: i64) -> Vec<(u32, Arc<FrameSeg>)> {+ let mut chain = Chain::new(params, layer_size, fps, 1.0);+ let objects = chain.data.objects();+ if objects.len() < 2 || !chain.data.in_span(f) {+ return vec![];+ }+ let shown = chain.show_object.checked_sub(1);+ chain.show_object = 0;+ objects.into_iter().filter(|o| shown.is_none_or(|s| s == *o)).filter_map(|o| chain.object_seg(f, o).map(|s| (o, s))).collect()+}+/// Nearest-neighbour resampling of a segmentation to another scale.pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg {let (w, h) = (((s.w as f64) * scale).round().max(1.0) as usize, ((s.h as f64) * scale).round().max(1.0) as usize);resample(s, w, h)}-pub(crate) fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {+pub fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {if s.w == w && s.h == h {return s.clone();}@@ -741,6 +819,9 @@ pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64) -> String {fnv(&mut h, &k.to_le_bytes());}fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes());+ if ch.show_object != 0 {+ fnv(&mut h, &ch.show_object.to_le_bytes());+ }format!("{h:016x}")}@@ -788,6 +869,17 @@ pub struct TrackCache {/// The mattes' width and height (the layer's size).pub size: [usize; 2],pub frames: BTreeMap<i64, TrackFrame>,+ /// With several objects: each one's matte of every tracked frame (object id, then frame).+ #[serde(skip_serializing_if = "BTreeMap::is_empty")]+ pub objects: BTreeMap<u32, BTreeMap<i64, ObjectFrame>>,+}++/// One object's matte and Refine Edge band of a tracked frame (RLE, base64).+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]+#[serde(default)]+pub struct ObjectFrame {+ pub m: String,+ pub r: String,}/// One tracked frame: its matte and Refine Edge band (RLE, base64), its colour models and the@@ -876,6 +968,16 @@ fn decode_tracked(cache: &TrackCache, f: i64) -> Option<FrameSeg> {(matte.len() == n && refine.len() == n).then(|| FrameSeg { w, h, matte, refine, fg: fr.fg.clone(), bg: fr.bg.clone() })}+/// Object `o`'s matte of a tracked frame of saved tracking (full resolution).+fn decode_object(cache: &TrackCache, f: i64, o: u32) -> Option<FrameSeg> {+ let fr = cache.objects.get(&o)?.get(&f)?;+ let [w, h] = cache.size;+ let n = w.checked_mul(h)?;+ let matte = rle::decode(&fr.m)?;+ let refine = rle::decode(&fr.r)?;+ (matte.len() == n && refine.len() == n).then(|| FrameSeg { w, h, matte, refine, ..Default::default() })+}+/// The tracking a chain has just made, to save with its instance: every frame of its span that the/// tracker followed, at full resolution, with its trust. `None` when it followed none.pub fn tracks_of(ch: &Chain) -> Option<TrackCache> {@@ -890,7 +992,21 @@ pub fn tracks_of(ch: &Chain) -> Option<TrackCache> {TrackFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine), fg: seg.fg.clone(), bg: seg.bg.clone(), c: t.confidence, o: t.occluded },);}- let cache = TrackCache { key: tracks_key(&ch.keys), model: model.to_string(), size: size?, frames };+ let mut objects: BTreeMap<u32, BTreeMap<i64, ObjectFrame>> = BTreeMap::new();+ let ids = ch.data.objects();+ if ids.len() > 1 {+ for (f, k) in &ch.keys {+ if !frames.contains_key(f) {+ continue;+ }+ for o in &ids {+ if let Some(seg) = cached(object_key(*k, *o), 1.0) {+ objects.entry(*o).or_default().insert(*f, ObjectFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine) });+ }+ }+ }+ }+ let cache = TrackCache { key: tracks_key(&ch.keys), model: model.to_string(), size: size?, frames, objects };(!cache.frames.is_empty()).then_some(cache)}@@ -1209,6 +1325,34 @@ mod tests {assert!(at(46, 40) != 0, "the moving disk (frame 3 centre x = 46)");assert!(at(15, 15) != 0, "the fixed square");assert_eq!(at(60, 70), 0, "nothing elsewhere");++ // each object keeps its own matte+ let (disk, square) = (chain.object_seg(3, 0).expect("object 0"), chain.object_seg(3, 1).expect("object 1"));+ assert!(disk.matte[40 * 100 + 46] != 0 && disk.matte[15 * 100 + 15] == 0);+ assert!(square.matte[15 * 100 + 15] != 0 && square.matte[40 * 100 + 46] == 0);+ // Show Object picks one (its id plus one); an object the instance does not have shows nothing; 0 shows all+ let mut shown = |c: &Chain| c.ensure(3, &mut get, &|| false).expect("frame 3");+ chain.show_object = 2;+ let one = shown(&chain);+ assert!(one.matte[15 * 100 + 15] != 0 && one.matte[40 * 100 + 46] == 0, "only the square");+ chain.show_object = 1;+ let zero = shown(&chain);+ assert!(zero.matte[40 * 100 + 46] != 0 && zero.matte[15 * 100 + 15] == 0, "only the disk");+ chain.show_object = 7;+ assert_eq!(shown(&chain).area(), 0, "no such object");+ chain.show_object = 0;+ assert!(shown(&chain).matte[15 * 100 + 15] != 0, "all together");++ // the objects are saved with the tracking and come back from the project file+ let saved = tracks_of(&chain).expect("tracking to save");+ assert_eq!(saved.objects.keys().copied().collect::<Vec<_>>(), vec![0, 1]);+ assert_eq!(saved.objects.get(&1).map(|f| f.len()), Some(9));+ assert_eq!(TrackCache::from_json(&saved.to_json()), Some(saved.clone()));+ // a tracking saved before objects existed has none+ let old = TrackCache { objects: Default::default(), ..saved.clone() };+ assert!(!old.to_json().contains("objects"));+ let rt = decode_object(&saved, 3, 1).expect("decoded");+ assert!(rt.matte[15 * 100 + 15] != 0 && rt.matte[40 * 100 + 46] == 0);}/// Instances saved before the propagation controls got their own twirl-down (and motiondiff --git a/docs/effects.md b/docs/effects.mdindex b3adb46ec..e233dd47f 100644--- a/docs/effects.md+++ b/docs/effects.md@@ -259,7 +259,7 @@ Effects are addressed by id (`ec.<category>.<name>`) or display name in commands| Object Matte | `ec.matte.objectmatte` | 13 | | 32 | Implemented || Refine Hard Matte | `ec.matte.refinehard` | 16 | GPU | 32 | Implemented || Refine Soft Matte | `ec.matte.refinesoft` | 18 | GPU | 32 | Implemented |-| Roto Brush & Refine Edge | `ec.matte.rotobrush` | 24 | | 32 | Implemented |+| Roto Brush & Refine Edge | `ec.matte.rotobrush` | 25 | | 32 | Implemented || Simple Choker | `ec.matte.simplechoker` | 2 | GPU | 32 | Implemented |## Noise & Grain