Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

S2.70: each object keeps its own matte; Show Object outputs one of them

Kristian Myramf67a17e76555
diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs
index 3a75bec1b..8a9750282 100644
--- a/crates/effects/src/object_matte.rs
+++ b/crates/effects/src/object_matte.rs
@@ -227,7 +227,7 @@ pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec<i64> {
pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option<Arc<dyn MaskModel>>) -> Option<Chain> {
let data = roto_data(params, layer_size, f, model.is_some())?;
let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref()));
- Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model, saved: None })
+ Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model, saved: None, show_object: 0 })
}
/// Frame `f`'s segmentation at full resolution, computing what is missing (the background
diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs
index ffa41f01e..e3f6da171 100644
--- a/crates/effects/src/roto.rs
+++ b/crates/effects/src/roto.rs
@@ -80,6 +80,7 @@ pub fn specs() -> Vec<EffectSpec> {
p("rotoBrushMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
p("rotoBrushMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)),
p("rotoBrushMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
+ p("rotoBrushMatte/object", "Show Object", num(0.0), slider(0.0, 32.0, 0.0, 32.0, 0)),
p("refineEdgeMatte/smooth", "Smooth", num(1.0), slider(0.0, 100.0, 0.0, 10.0, 1)),
p("refineEdgeMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)),
p("refineEdgeMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
@@ -320,6 +321,11 @@ fn tracked_key(key: u64) -> u64 {
key.rotate_left(17) ^ TRACK_SALT
}
+/// Where one object's matte of a tracked frame lives (kept only when the instance has several objects).
+fn object_key(key: u64, object: u32) -> u64 {
+ tracked_key(key).wrapping_add(0x9e37_79b9_7f4a_7c15u64.wrapping_mul(u64::from(object) + 1))
+}
+
/// How far to trust a frame the tracker followed.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct FrameTrust {
@@ -398,6 +404,8 @@ pub struct Chain {
pub model: Option<Arc<dyn MaskModel>>,
/// The saved tracking, while it applies (see [`TrackCache`]).
pub(crate) saved: Option<SavedTracking>,
+ /// Show Object: the matte of this object only (its id plus one; 0 shows all of them together).
+ pub show_object: u32,
}
impl Chain {
@@ -405,7 +413,8 @@ impl Chain {
let data = data(params);
let keys = data.chain_keys(seed(params, layer_size, fps));
let saved = saved_for(params, &data, layer_size, fps);
- Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale), model: model_for(params), saved }
+ let show_object = f(params, "rotoBrushMatte/object").round().clamp(0.0, 32.0) as u32;
+ Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale), model: model_for(params), saved, show_object }
}
/// Frame `f`'s segmentation at `scale` from the saved tracking or the caches; computes nothing.
@@ -413,6 +422,49 @@ impl Chain {
self.saved_seg(f, scale).or_else(|| self.keys.get(&f).and_then(|k| cached(*k, scale)))
}
+ /// Object `o`'s matte of frame `f` at this chain's scale (saved tracking, else this session's), when the instance
+ /// has several objects and the tracker followed them there; computes nothing.
+ pub fn object_seg(&self, f: i64, o: u32) -> Option<Arc<FrameSeg>> {
+ let sc = self.opts.scale;
+ let k = *self.keys.get(&f)?;
+ let ok = object_key(k, o);
+ if let Some(s) = cached(ok, sc) {
+ return Some(s);
+ }
+ let full = match cached(ok, 1.0) {
+ Some(s) => s,
+ None => {
+ let t = self.saved.as_ref()?;
+ let s = Arc::new(decode_object(&t.cache, f, o)?);
+ store(ok, 1.0, s.clone());
+ s
+ }
+ };
+ if sc == 1.0 {
+ return Some(full);
+ }
+ let s = Arc::new(downscale(&full, sc));
+ store(ok, sc, s.clone());
+ Some(s)
+ }
+
+ /// What Show Object asks for at frame `f`: `None` shows the whole matte (all objects, a single object, or a frame
+ /// the tracker did not follow); an object the instance does not have shows nothing.
+ fn shown_object(&self, f: i64) -> Option<Arc<FrameSeg>> {
+ let o = self.show_object.checked_sub(1)?;
+ let objects = self.data.objects();
+ if objects.len() < 2 {
+ return None;
+ }
+ if !objects.contains(&o) {
+ let sc = self.opts.scale;
+ let [w, h] =
+ self.saved.as_ref().map(|t| t.cache.size).or_else(|| self.keys.get(&f).and_then(|k| cached(tracked_key(*k), sc)).map(|s| [s.w, s.h]))?;
+ return Some(Arc::new(FrameSeg::empty(w, h)));
+ }
+ self.object_seg(f, o)
+ }
+
/// The tracker's trust in frame `f`: from the saved tracking, else from this session's tracking.
pub fn trust_at(&self, f: i64) -> Option<FrameTrust> {
if let Some(fr) = self.saved.as_ref().and_then(|t| t.cache.frames.get(&f)) {
@@ -455,6 +507,10 @@ impl Chain {
pub fn ensure(&self, f: i64, frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>, cancel: &dyn Fn() -> bool) -> Option<Arc<FrameSeg>> {
let sc = self.opts.scale;
let lookup = |g: i64| -> Option<Arc<FrameSeg>> {
+ // Show Object: one object's matte, where the tracker followed the objects.
+ if let Some(s) = self.shown_object(g) {
+ return Some(s);
+ }
// A frame the tracker followed wins over the classic engine's: the saved tracking, else
// this session's.
if let Some(s) = self.saved_seg(g, sc) {
@@ -604,13 +660,13 @@ impl Chain {
let mut pending: BTreeMap<i64, Vec<Option<effectcraft_segment::TrackedFrame>>> = BTreeMap::new();
let finish = |g: i64, parts: Vec<Option<effectcraft_segment::TrackedFrame>>, summary: &mut TrackSummary, progress: &mut dyn FnMut(usize)| {
let (Some(key), Some(img)) = (self.keys.get(&g).copied(), (frame.borrow_mut())(g)) else { return };
- let mut segs = vec![];
+ let mut segs: Vec<(u32, effectcraft_track::roto::FrameSeg)> = vec![];
let (mut confidence, mut present) = (f32::INFINITY, false);
let mut worst = f32::INFINITY;
for (obj, t) in objects.iter().zip(parts) {
let Some(t) = t else { continue };
let strokes = self.data.strokes_of(g, *obj);
- segs.push(rb::segment_tracked(&img, &strokes, t.prob, t.occluded, &self.opts, self.refine_radius));
+ segs.push((*obj, rb::segment_tracked(&img, &strokes, t.prob, t.occluded, &self.opts, self.refine_radius)));
worst = worst.min(t.confidence);
if !t.occluded {
present = true;
@@ -622,7 +678,16 @@ impl Chain {
}
// a frame is occluded when every object is absent; otherwise the doubtful object sets the confidence
let (occluded, confidence) = (!present, if present { confidence } else { worst });
- store(tracked_key(key), self.opts.scale, Arc::new(effectcraft_track::roto::FrameSeg::union(segs)));
+ // with several objects each keeps its own matte too (Show Object, the per-object overlay)
+ let several = objects.len() > 1;
+ let mut parts = Vec::with_capacity(segs.len());
+ for (obj, seg) in segs {
+ if several {
+ store(object_key(key, obj), self.opts.scale, Arc::new(seg.clone()));
+ }
+ parts.push(seg);
+ }
+ store(tracked_key(key), self.opts.scale, Arc::new(effectcraft_track::roto::FrameSeg::union(parts)));
trust_cache().lock().unwrap_or_else(|e| e.into_inner()).insert(tracked_key(key), FrameTrust { confidence, occluded });
summary.frames += 1;
summary.occluded += usize::from(occluded);
@@ -656,13 +721,26 @@ impl Chain {
}
}
+/// Each object's matte of layer frame `f` where the tracker followed several objects (full resolution; computes
+/// nothing). Show Object limits it to the one chosen. Empty with fewer than two objects.
+pub fn object_mattes(params: &Params, layer_size: [f64; 2], fps: f64, f: i64) -> Vec<(u32, Arc<FrameSeg>)> {
+ let mut chain = Chain::new(params, layer_size, fps, 1.0);
+ let objects = chain.data.objects();
+ if objects.len() < 2 || !chain.data.in_span(f) {
+ return vec![];
+ }
+ let shown = chain.show_object.checked_sub(1);
+ chain.show_object = 0;
+ objects.into_iter().filter(|o| shown.is_none_or(|s| s == *o)).filter_map(|o| chain.object_seg(f, o).map(|s| (o, s))).collect()
+}
+
/// Nearest-neighbour resampling of a segmentation to another scale.
pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg {
let (w, h) = (((s.w as f64) * scale).round().max(1.0) as usize, ((s.h as f64) * scale).round().max(1.0) as usize);
resample(s, w, h)
}
-pub(crate) fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {
+pub fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {
if s.w == w && s.h == h {
return s.clone();
}
@@ -741,6 +819,9 @@ pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64) -> String {
fnv(&mut h, &k.to_le_bytes());
}
fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes());
+ if ch.show_object != 0 {
+ fnv(&mut h, &ch.show_object.to_le_bytes());
+ }
format!("{h:016x}")
}
@@ -788,6 +869,17 @@ pub struct TrackCache {
/// The mattes' width and height (the layer's size).
pub size: [usize; 2],
pub frames: BTreeMap<i64, TrackFrame>,
+ /// With several objects: each one's matte of every tracked frame (object id, then frame).
+ #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+ pub objects: BTreeMap<u32, BTreeMap<i64, ObjectFrame>>,
+}
+
+/// One object's matte and Refine Edge band of a tracked frame (RLE, base64).
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+#[serde(default)]
+pub struct ObjectFrame {
+ pub m: String,
+ pub r: String,
}
/// One tracked frame: its matte and Refine Edge band (RLE, base64), its colour models and the
@@ -876,6 +968,16 @@ fn decode_tracked(cache: &TrackCache, f: i64) -> Option<FrameSeg> {
(matte.len() == n && refine.len() == n).then(|| FrameSeg { w, h, matte, refine, fg: fr.fg.clone(), bg: fr.bg.clone() })
}
+/// Object `o`'s matte of a tracked frame of saved tracking (full resolution).
+fn decode_object(cache: &TrackCache, f: i64, o: u32) -> Option<FrameSeg> {
+ let fr = cache.objects.get(&o)?.get(&f)?;
+ let [w, h] = cache.size;
+ let n = w.checked_mul(h)?;
+ let matte = rle::decode(&fr.m)?;
+ let refine = rle::decode(&fr.r)?;
+ (matte.len() == n && refine.len() == n).then(|| FrameSeg { w, h, matte, refine, ..Default::default() })
+}
+
/// The tracking a chain has just made, to save with its instance: every frame of its span that the
/// tracker followed, at full resolution, with its trust. `None` when it followed none.
pub fn tracks_of(ch: &Chain) -> Option<TrackCache> {
@@ -890,7 +992,21 @@ pub fn tracks_of(ch: &Chain) -> Option<TrackCache> {
TrackFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine), fg: seg.fg.clone(), bg: seg.bg.clone(), c: t.confidence, o: t.occluded },
);
}
- let cache = TrackCache { key: tracks_key(&ch.keys), model: model.to_string(), size: size?, frames };
+ let mut objects: BTreeMap<u32, BTreeMap<i64, ObjectFrame>> = BTreeMap::new();
+ let ids = ch.data.objects();
+ if ids.len() > 1 {
+ for (f, k) in &ch.keys {
+ if !frames.contains_key(f) {
+ continue;
+ }
+ for o in &ids {
+ if let Some(seg) = cached(object_key(*k, *o), 1.0) {
+ objects.entry(*o).or_default().insert(*f, ObjectFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine) });
+ }
+ }
+ }
+ }
+ let cache = TrackCache { key: tracks_key(&ch.keys), model: model.to_string(), size: size?, frames, objects };
(!cache.frames.is_empty()).then_some(cache)
}
@@ -1209,6 +1325,34 @@ mod tests {
assert!(at(46, 40) != 0, "the moving disk (frame 3 centre x = 46)");
assert!(at(15, 15) != 0, "the fixed square");
assert_eq!(at(60, 70), 0, "nothing elsewhere");
+
+ // each object keeps its own matte
+ let (disk, square) = (chain.object_seg(3, 0).expect("object 0"), chain.object_seg(3, 1).expect("object 1"));
+ assert!(disk.matte[40 * 100 + 46] != 0 && disk.matte[15 * 100 + 15] == 0);
+ assert!(square.matte[15 * 100 + 15] != 0 && square.matte[40 * 100 + 46] == 0);
+ // Show Object picks one (its id plus one); an object the instance does not have shows nothing; 0 shows all
+ let mut shown = |c: &Chain| c.ensure(3, &mut get, &|| false).expect("frame 3");
+ chain.show_object = 2;
+ let one = shown(&chain);
+ assert!(one.matte[15 * 100 + 15] != 0 && one.matte[40 * 100 + 46] == 0, "only the square");
+ chain.show_object = 1;
+ let zero = shown(&chain);
+ assert!(zero.matte[40 * 100 + 46] != 0 && zero.matte[15 * 100 + 15] == 0, "only the disk");
+ chain.show_object = 7;
+ assert_eq!(shown(&chain).area(), 0, "no such object");
+ chain.show_object = 0;
+ assert!(shown(&chain).matte[15 * 100 + 15] != 0, "all together");
+
+ // the objects are saved with the tracking and come back from the project file
+ let saved = tracks_of(&chain).expect("tracking to save");
+ assert_eq!(saved.objects.keys().copied().collect::<Vec<_>>(), vec![0, 1]);
+ assert_eq!(saved.objects.get(&1).map(|f| f.len()), Some(9));
+ assert_eq!(TrackCache::from_json(&saved.to_json()), Some(saved.clone()));
+ // a tracking saved before objects existed has none
+ let old = TrackCache { objects: Default::default(), ..saved.clone() };
+ assert!(!old.to_json().contains("objects"));
+ let rt = decode_object(&saved, 3, 1).expect("decoded");
+ assert!(rt.matte[15 * 100 + 15] != 0 && rt.matte[40 * 100 + 46] == 0);
}
/// Instances saved before the propagation controls got their own twirl-down (and motion
diff --git a/docs/effects.md b/docs/effects.md
index b3adb46ec..e233dd47f 100644
--- a/docs/effects.md
+++ b/docs/effects.md
@@ -259,7 +259,7 @@ Effects are addressed by id (`ec.<category>.<name>`) or display name in commands
| Object Matte | `ec.matte.objectmatte` | 13 | | 32 | Implemented |
| Refine Hard Matte | `ec.matte.refinehard` | 16 | GPU | 32 | Implemented |
| Refine Soft Matte | `ec.matte.refinesoft` | 18 | GPU | 32 | Implemented |
-| Roto Brush & Refine Edge | `ec.matte.rotobrush` | 24 | | 32 | Implemented |
+| Roto Brush & Refine Edge | `ec.matte.rotobrush` | 25 | | 32 | Implemented |
| Simple Choker | `ec.matte.simplechoker` | 2 | GPU | 32 | Implemented |
## Noise & Grain