Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

#783: Object Matte tracks in the background and freezes

vangel76560767dfe79e
diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs
index a7db562a9..32a196af2 100644
--- a/crates/effects/src/object_matte.rs
+++ b/crates/effects/src/object_matte.rs
@@ -16,8 +16,12 @@
//! ([`crate::roto::cached`]): the chain seed holds the engine-maintained Input Key (what the
//! frames are made of), the layer size, the frame rate, the mode and the model, and every
//! frame's key the prompt, so edits upstream or to the prompt recompute it and undo never sees a
-//! stale matte. **Feather**, **Contrast** and **Shift Edge** shape the matte, **Invert** keeps
-//! everything but the object and **View ▸ Matte** shows the matte in grey.
+//! stale matte. The engine fills that cache in a background job (`objectMatte.propagate`,
+//! outward from the reference frame over [`job_frames`]) and **Freeze** (`objectMatte.freeze`)
+//! stores the final mattes in the hidden **Frozen** parameter ([`FrozenCache`], as Roto Brush
+//! does): they are used, without segmenting, while [`frozen_key`] still matches. **Feather**,
+//! **Contrast** and **Shift Edge** shape the matte, **Invert** keeps everything but the object
+//! and **View ▸ Matte** shows the matte in grey.
use std::sync::Arc;
@@ -26,10 +30,10 @@ use effectcraft_project::ParamUi;
use effectcraft_raster::Image;
use effectcraft_segment::MaskModel;
use effectcraft_track::roto::refine::MatteParams;
-use effectcraft_track::roto::{RotoData, SegOpts, Stroke, StrokeKind};
+use effectcraft_track::roto::{FrameSeg, RotoData, SegOpts, Stroke, StrokeKind, rle};
use rayon::prelude::*;
-use crate::roto::{self, Chain, INPUT_KEY};
+use crate::roto::{self, Chain, FROZEN, FrozenCache, INPUT_KEY};
use crate::warp_stab::param;
use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider};
@@ -71,6 +75,7 @@ pub fn specs() -> Vec<EffectSpec> {
p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox),
p("view", "View", Value::Enum(0), popup(&VIEWS)),
p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden),
+ p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden),
],
render,
gpu: false,
@@ -197,6 +202,127 @@ pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn
h
}
+/// The layer frames a background job computes, outward from the reference frame: the tracked
+/// span (Frames Before / After around it), clipped to `limits` (the layer's frames). Without
+/// Track Object, the same frames, each segmented on its own.
+pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec<i64> {
+ let base = reference_frame(params);
+ let [before, after] = reach(params);
+ let (lo, hi) = (base.saturating_sub(before).max(limits[0]), base.saturating_add(after).min(limits[1]));
+ if base < limits[0] || base > limits[1] {
+ return vec![];
+ }
+ let mut v = vec![base];
+ v.extend(base.saturating_add(1)..=hi);
+ v.extend((lo..base).rev());
+ v
+}
+
+/// The segmentation chain the matte of layer frame `f` comes from (`None`: no usable prompt).
+pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option<Arc<dyn MaskModel>>) -> Option<Chain> {
+ let data = roto_data(params, layer_size, f, model.is_some())?;
+ let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref()));
+ Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model })
+}
+
+/// Frame `f`'s segmentation at full resolution, computing what is missing (the background
+/// job's step). `None` outside the tracked span, without a prompt, or when cancelled.
+pub fn ensure(
+ params: &Params,
+ layer_size: [f64; 2],
+ fps: f64,
+ f: i64,
+ model: Option<Arc<dyn MaskModel>>,
+ frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>,
+ cancel: &dyn Fn() -> bool,
+) -> Option<Arc<FrameSeg>> {
+ let ch = chain(params, layer_size, fps, f, 1.0, model)?;
+ if !ch.data.in_span(f) {
+ return None;
+ }
+ ch.ensure(f, frame, cancel)
+}
+
+/// The chain key of each of `frames` (what a computed segmentation is cached under).
+pub fn frame_keys(params: &Params, layer_size: [f64; 2], fps: f64, frames: &[i64], model: Option<Arc<dyn MaskModel>>) -> Vec<(i64, u64)> {
+ if tracks(params) {
+ let Some(ch) = chain(params, layer_size, fps, 0, 1.0, model) else { return vec![] };
+ return frames.iter().filter_map(|f| Some((*f, *ch.keys.get(f)?))).collect();
+ }
+ frames.iter().filter_map(|f| Some((*f, *chain(params, layer_size, fps, *f, 1.0, model.clone())?.keys.get(f)?))).collect()
+}
+
+/// What frozen mattes depend on: what the frames are (the seed: Input Key, size, rate, mode,
+/// model), the prompt, the tracked span and the matte settings.
+pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> String {
+ let mut h = seed(params, layer_size, fps, model);
+ let base = reference_frame(params);
+ if let Some(st) = prompt_strokes(params, layer_size, base, model.is_some()) {
+ fnv(&mut h, serde_json::to_string(&st).unwrap_or_default().as_bytes());
+ }
+ for v in [base, reach(params)[0], reach(params)[1]] {
+ fnv(&mut h, &v.to_le_bytes());
+ }
+ fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes());
+ format!("{h:016x}")
+}
+
+/// Whether the instance is frozen (has frozen mattes).
+pub fn is_frozen(params: &Params) -> bool {
+ !s(params, FROZEN).is_empty()
+}
+
+/// Frame `f`'s frozen matte on a `w`×`h` grid, when the instance is frozen, the mattes still
+/// match ([`frozen_key`]) and `f` is one of them.
+pub fn frozen_alpha(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, w: usize, h: usize, model: Option<&dyn MaskModel>) -> Option<Vec<f32>> {
+ let j = s(params, FROZEN);
+ if j.is_empty() {
+ return None;
+ }
+ let fc = frozen_cache(j)?;
+ if fc.key != frozen_key(params, layer_size, fps, model) {
+ return None;
+ }
+ let fr = fc.frames.get(&f)?;
+ let a = rle::decode(&fr.a)?;
+ let [fw, fh] = fc.size;
+ if fw == 0 || fh == 0 || a.len() != fw.checked_mul(fh)? {
+ return None;
+ }
+ let af: Vec<f32> = a.iter().map(|v| *v as f32 / 255.0).collect();
+ if fw == w && fh == h {
+ return Some(af);
+ }
+ let mut out = vec![0.0f32; w * h];
+ for y in 0..h {
+ for x in 0..w {
+ let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64);
+ out[y * w + x] = effectcraft_track::roto::matting::sample(&af, fw, fh, sx, sy);
+ }
+ }
+ Some(out)
+}
+
+/// The parsed Frozen parameter (parsed once per content).
+fn frozen_cache(j: &str) -> Option<Arc<FrozenCache>> {
+ use std::collections::HashMap;
+ use std::sync::{Mutex, OnceLock};
+ static C: OnceLock<Mutex<HashMap<u64, Arc<FrozenCache>>>> = OnceLock::new();
+ let mut h = 0xcbf2_9ce4_8422_2325;
+ fnv(&mut h, j.as_bytes());
+ let c = C.get_or_init(Default::default);
+ if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) {
+ return Some(d.clone());
+ }
+ let d = Arc::new(FrozenCache::from_json(j)?);
+ let mut m = c.lock().unwrap_or_else(|e| e.into_inner());
+ if m.len() > 8 {
+ m.clear();
+ }
+ m.insert(h, d.clone());
+ Some(d)
+}
+
/// Segmentation settings at `scale` (Roto Brush's defaults).
fn seg_opts(scale: f64) -> SegOpts {
SegOpts { scale, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false }
@@ -242,12 +368,10 @@ pub fn frame_matte(
frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>,
cancel: &dyn Fn() -> bool,
) -> FrameMatte {
- let Some(data) = roto_data(params, layer_size, f, model.is_some()) else { return FrameMatte::NoPrompt };
- if !data.in_span(f) {
+ let Some(chain) = chain(params, layer_size, fps, f, scale, model) else { return FrameMatte::NoPrompt };
+ if !chain.data.in_span(f) {
return FrameMatte::Outside;
}
- let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref()));
- let chain = Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model };
let Some(seg) = chain.ensure(f, frame, cancel) else { return FrameMatte::Outside };
let (w, h) = (img.width as usize, img.height as usize);
let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { roto::resample(&seg, w, h) };
@@ -275,7 +399,12 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf {
let im = roto::layer_grid(&other, layer_size);
(im.width as usize == w && im.height as usize == h).then(|| Arc::new(im))
};
- let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, roto::installed_model(), &mut frame, &|| false) {
+ let model = roto::installed_model();
+ // Frozen mattes are used as they are (no segmentation).
+ if let Some(a) = frozen_alpha(ctx.params, layer_size, fps, f, w, h, model.as_deref()) {
+ return compose(&buf, Some(&a), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1);
+ }
+ let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, model, &mut frame, &|| false) {
FrameMatte::Alpha(a) if a.len() == w * h => Some(a),
FrameMatte::NoPrompt => return buf,
_ => None,
@@ -513,4 +642,50 @@ mod tests {
let _ = frame_matte(&p, [100.0, 80.0], 25.0, 0, 0.5, &half, Some(stub.clone()), &mut frame, &|| false);
assert_eq!(*stub.bbox.lock().unwrap(), Some([37.0, 27.0, 50.0, 40.0]));
}
+
+ #[test]
+ fn job_frames_go_outward_from_the_reference_frame_inside_the_layer() {
+ let p = params(&[
+ ("objectTracking/referenceFrame", Value::Scalar(5.0)),
+ ("objectTracking/framesBefore", Value::Scalar(2.0)),
+ ("objectTracking/framesAfter", Value::Scalar(3.0)),
+ ]);
+ assert_eq!(job_frames(&p, [0, 100]), [5, 6, 7, 8, 4, 3]);
+ assert_eq!(job_frames(&p, [4, 6]), [5, 6, 4]);
+ assert!(job_frames(&p, [10, 20]).is_empty());
+ let huge = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesAfter", Value::Scalar(f64::INFINITY))]);
+ assert!(job_frames(&huge, [0, 100]).is_empty());
+ }
+
+ /// Frozen mattes are used as stored while their key matches (no segmentation: a stored
+ /// pattern the segmentation could never produce comes out), and ignored once it doesn't.
+ #[test]
+ fn frozen_mattes_are_used_while_their_key_matches() {
+ let mut p = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]);
+ p.values.insert(INPUT_KEY.into(), Value::Str("frozen-test".into()));
+ // Left half opaque on frame 2.
+ let alpha: Vec<f32> = (0..100 * 80).map(|i| ((i % 100) < 50) as u8 as f32).collect();
+ let mut fc = FrozenCache { key: frozen_key(&p, [100.0, 80.0], 25.0, None), size: [100, 80], ..Default::default() };
+ fc.frames.insert(2, roto::freeze_frame(&alpha, &[]));
+ p.values.insert(FROZEN.into(), Value::Str(fc.to_json()));
+ assert!(is_frozen(&p));
+ let out = run(&p, 2);
+ assert!(out.data[40 * 100 + 10][3] > 0.99 && out.data[40 * 100 + 90][3] < 0.01, "the frozen matte is used");
+ // At half resolution too.
+ assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 50, 40, None).map(|a| (a[20 * 50 + 5], a[20 * 50 + 45])), Some((1.0, 0.0)));
+ // A frame without a frozen matte is computed (the disk).
+ assert!(wrong(&run(&p, 1), 42.0) < 30);
+ // Another prompt: the frozen mattes no longer apply.
+ p.values.insert("objectPoint".into(), Value::Vec2([41.0, 40.0]));
+ assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
+ assert!(wrong(&run(&p, 2), 44.0) < 30);
+ // Damaged frozen data is ignored, not a crash.
+ p.values.insert(FROZEN.into(), Value::Str("{\"key\": 3, \"frames\": [".into()));
+ assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
+ let mut fc2 = fc.clone();
+ fc2.size = [usize::MAX, 2];
+ p.values.insert("objectPoint".into(), Value::Vec2([40.0, 40.0]));
+ p.values.insert(FROZEN.into(), Value::Str(fc2.to_json()));
+ assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
+ }
}
diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs
index f4f33d53a..ee9cd3a28 100644
--- a/crates/engine/src/commands/roto_cmds.rs
+++ b/crates/engine/src/commands/roto_cmds.rs
@@ -63,6 +63,23 @@ pub fn specs() -> Vec<CommandSpec> {
has_comp,
object_matte_apply
),
+ cmd!(
+ "objectMatte.propagate",
+ "Track Object Matte",
+ [],
+ None,
+ "{layer?, effect?, direction?: forward|backward|both, wait?: block until done} (a background job like Roto Brush propagation; roto.cancel stops it)",
+ has_comp,
+ object_matte_propagate
+ ),
+ cmd!("objectMatte.freeze", "Freeze Object Matte", [], None, "{layer?, effect?, wait?}", has_comp, object_matte_freeze),
+ cmd!("objectMatte.unfreeze", "Unfreeze Object Matte", [], None, "{layer?, effect?}", has_comp, object_matte_unfreeze),
+ query!(
+ "objectMatte.status",
+ "Object Matte Status",
+ "{layer?, effect?}: the job's progress, whether the instance is frozen, its frames and which are computed",
+ object_matte_status
+ ),
query!(
"roto.status",
"Roto Brush Status",
@@ -79,6 +96,16 @@ fn is_running(s: &Session) -> std::result::Result<(), String> {
/// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index),
/// else a selected one, else the first on the selected (or any) layer.
pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> {
+ instance_lookup(s, p, cmd, is_roto)?.ok_or_else(|| bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)"))
+}
+
+/// The Object Matte instance targeted by `p` (as [`roto_p`]).
+fn object_matte_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> {
+ instance_lookup(s, p, cmd, rt::is_object_matte)?.ok_or_else(|| bad(cmd, "no Object Matte effect found (click an object with the Object Matte tool first)"))
+}
+
+/// The instance of a kind (`is`) targeted by `p`; `None` when there is none.
+fn instance_lookup(s: &Session, p: &Value, cmd: &str, is: fn(&PropGroup) -> bool) -> Result<Option<(ItemId, LayerId, Uid)>> {
let cid = super::comp_id(s, p)?;
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let candidates: Vec<LayerId> = match p.get("layer") {
@@ -102,17 +129,17 @@ pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, Layer
.state
.selected_props
.iter()
- .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten())
- .or_else(|| fxg.groups().find(|g| is_roto(g))),
+ .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is(g))).flatten())
+ .or_else(|| fxg.groups().find(|g| is(g))),
};
- if let Some(g) = g.filter(|g| is_roto(g)) {
- return Ok((cid, lid, g.uid));
+ if let Some(g) = g.filter(|g| is(g)) {
+ return Ok(Some((cid, lid, g.uid)));
}
if p.get("layer").is_some() {
break;
}
}
- Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)"))
+ Ok(None)
}
fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> {
@@ -459,10 +486,13 @@ fn object_matte_apply(s: &mut Session, p: &Value) -> Result<Value> {
let existing = match p.get("effect").filter(|v| !v.is_null()) {
Some(v) => {
let uid = v.as_u64().ok_or_else(|| bad(cmd, "`effect` is an Object Matte effect's uid"))?;
- layer
+ let g = layer
.effects()
.and_then(|f| f.groups().find(|g| g.uid == uid && rt::is_object_matte(g)))
.ok_or_else(|| bad(cmd, format!("no Object Matte effect {uid} on this layer")))?;
+ if frozen_of(g) {
+ return Err(bad(cmd, "the Object Matte is frozen: Unfreeze it to change the prompt"));
+ }
Some(uid)
}
None => None,
@@ -556,3 +586,65 @@ fn object_matte_reach(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> (bool
let [before, after] = om::reach(&ps);
(om::tracks(&ps), before, after)
}
+
+fn object_matte_propagate(s: &mut Session, p: &Value) -> Result<Value> {
+ let cmd = "objectMatte.propagate";
+ let (cid, lid, uid) = object_matte_p(s, p, cmd)?;
+ let dir = match str_p(p, "direction") {
+ Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?,
+ None => Direction::Both,
+ };
+ let wait = b_p(p, "wait").unwrap_or(false);
+ let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?;
+ let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()});
+ if wait {
+ out["status"] = object_matte_status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?;
+ }
+ Ok(out)
+}
+
+fn object_matte_freeze(s: &mut Session, p: &Value) -> Result<Value> {
+ let cmd = "objectMatte.freeze";
+ let (cid, lid, uid) = object_matte_p(s, p, cmd)?;
+ let wait = b_p(p, "wait").unwrap_or(false);
+ let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?;
+ let (_, g) = group(s, cid, lid, uid)?;
+ Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()}))
+}
+
+fn object_matte_unfreeze(s: &mut Session, p: &Value) -> Result<Value> {
+ let cmd = "objectMatte.unfreeze";
+ let (cid, lid, uid) = object_matte_p(s, p, cmd)?;
+ let (_, g) = group(s, cid, lid, uid)?;
+ if frozen_of(g) {
+ s.edit("Unfreeze", None, |proj, _| {
+ let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?;
+ if let Some(pr) = g.get_mut(FROZEN) {
+ pr.value = KV::Str(String::new());
+ }
+ Ok(())
+ })?;
+ }
+ Ok(json!({"layer": lid.0, "effect": uid, "frozen": false}))
+}
+
+fn object_matte_status(s: &mut Session, p: &Value) -> Result<Value> {
+ s.poll_roto(false);
+ let running = s.is_roto_running();
+ let mut out = json!({"running": running, "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner())});
+ let Ok((cid, lid, uid)) = object_matte_p(s, p, "objectMatte.status") else { return Ok(out) };
+ let (_, g) = group(s, cid, lid, uid)?;
+ let frozen = frozen_of(g);
+ let (track, before, after) = object_matte_reach(s, cid, lid, uid);
+ let computed = s.roto_computed(cid, lid, uid);
+ out["layer"] = json!(lid.0);
+ out["effect"] = json!(uid);
+ out["frozen"] = json!(frozen);
+ out["track"] = json!(track);
+ out["reach"] = json!([before, after]);
+ out["mine"] = json!(running && s.roto_target().is_some_and(|(c, l, u, _)| (c, l, u) == (cid, lid, uid)));
+ out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid)));
+ out["frames"] = json!(computed.keys().collect::<Vec<_>>());
+ out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::<Vec<_>>());
+ Ok(out)
+}
diff --git a/crates/engine/src/offload.rs b/crates/engine/src/offload.rs
index 23fa2b38b..561ebe6c4 100644
--- a/crates/engine/src/offload.rs
+++ b/crates/engine/src/offload.rs
@@ -561,7 +561,8 @@ async fn fill(s: &mut Session, plan: crate::commands::content_fill::FillPlan, po
/// Roto Brush propagation in a worker: the segmentations stream back (at most every 250 ms)
/// as they are computed, so stopping the job keeps what was propagated.
async fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: crate::roto::Direction, post: Post) -> Result<(), String> {
- let keys: Vec<u64> = s.roto_chain(comp, layer, effect).map(|c| c.keys.values().copied().collect()).ok_or("no Roto Brush strokes")?;
+ let keys: Vec<u64> =
+ s.roto_keys(comp, layer, effect).map(|k| k.into_iter().map(|(_, k)| k).collect()).ok_or("no Roto Brush strokes or Object Matte prompt")?;
let sent = Rc::new(RefCell::new(std::collections::HashSet::new()));
let (p, k2, s2) = (post.clone(), keys.clone(), sent.clone());
let mut last = None::<web_time::Instant>;
diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs
index b8c2c649c..49f53d39b 100644
--- a/crates/engine/src/roto.rs
+++ b/crates/engine/src/roto.rs
@@ -10,12 +10,18 @@
//! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made
//! of (the layer's source and timing, its masks and the effects above it) and drops frozen
//! mattes made from other frames.
+//!
+//! Object Matte instances run through the same jobs (`objectMatte.propagate` / `.freeze`): the
+//! frames are [`om::job_frames`] around the reference frame, each segmented through
+//! [`om::ensure`]; Freeze stores [`om::frame_matte`]'s mattes under [`om::frozen_key`]. An edit
+//! to an instance's frames or prompt queues it for background tracking like a Roto Brush.
use crate::offload::JobKind;
use std::collections::{BTreeMap, HashMap};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
+use effectcraft_effects::object_matte as om;
use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY};
use effectcraft_effects::{Params, flatten_params};
use effectcraft_keyframe::Value;
@@ -104,14 +110,18 @@ pub struct RotoProgress {
pub finished: bool,
pub cancelled: bool,
pub error: Option<String>,
+ /// The job works on an Object Matte (else a Roto Brush & Refine Edge).
+ pub object_matte: bool,
}
impl RotoProgress {
pub fn banner(&self) -> String {
let pc = (100 * self.done).checked_div(self.total).unwrap_or(0);
- match self.task {
- Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"),
- _ => format!("Propagating Roto Brush: {pc}%"),
+ match (self.task, self.object_matte) {
+ (Some(RotoTask::Freeze), false) => format!("Freezing Roto Brush & Refine Edge: {pc}%"),
+ (Some(RotoTask::Freeze), true) => format!("Freezing Object Matte: {pc}%"),
+ (_, false) => format!("Propagating Roto Brush: {pc}%"),
+ (_, true) => format!("Tracking Object Matte: {pc}%"),
}
}
}
@@ -130,8 +140,11 @@ pub struct RotoJob {
pub layer: LayerId,
pub effect: Uid,
pub task: RotoTask,
- /// The strokes the job was started with (a Freeze result is dropped when they changed).
+ /// What the job was started from: the strokes (Roto Brush) or the prompt (Object Matte); a
+ /// Freeze result is dropped when it changed.
strokes: String,
+ /// The job works on an Object Matte.
+ pub object_matte: bool,
}
impl RotoJob {
@@ -153,6 +166,20 @@ pub(crate) fn is_object_matte(g: &PropGroup) -> bool {
matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID)
}
+/// What an Object Matte's segmentation is made from besides its frames: the prompt and tracking
+/// parameters (static values), for queueing it after edits and guarding a Freeze.
+pub(crate) fn om_prompt(g: &PropGroup) -> String {
+ let ps = params_static(g);
+ let mut keys: Vec<&String> = ps.values.keys().filter(|k| ["objectPoint", "useBox", "box", "objectTracking/"].iter().any(|p| k.starts_with(p))).collect();
+ keys.sort();
+ keys.iter().map(|k| format!("{k}={:?};", ps.values.get(*k))).collect()
+}
+
+/// The job-relevant guard of an instance: its strokes (Roto Brush) or prompt (Object Matte).
+fn guard_of(g: &PropGroup) -> String {
+ if is_object_matte(g) { om_prompt(g) } else { str_of(g, fx::STROKES) }
+}
+
fn str_of(g: &PropGroup, m: &str) -> String {
match g.get(m).map(|p| &p.value) {
Some(Value::Str(s)) => s.clone(),
@@ -195,8 +222,8 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String {
}
/// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from
-/// other frames are dropped. Returns the Roto Brush instances whose segmentation changed (to
-/// propagate again).
+/// other frames are dropped. Returns the instances whose segmentation changed (to propagate
+/// again): new frames, strokes (Roto Brush) or prompt (Object Matte).
pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
let mut todo = vec![];
let mut set = vec![];
@@ -206,14 +233,11 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI
let Some(fxg) = l.effects() else { continue };
for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) {
let key = input_key(after, *cid, l, g.uid);
- let roto = is_roto(g);
let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid));
if str_of(g, INPUT_KEY) != key {
set.push((*cid, l.id, g.uid, key));
- if roto {
- todo.push((*cid, l.id, g.uid));
- }
- } else if roto && old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) {
+ todo.push((*cid, l.id, g.uid));
+ } else if old.is_none_or(|o| guard_of(o) != guard_of(g)) {
todo.push((*cid, l.id, g.uid));
}
}
@@ -229,7 +253,11 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI
}
}
todo.retain(|(c, l, u)| {
- after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(&params_static(g)).is_empty())
+ after
+ .comp(*c)
+ .and_then(|c| c.layer(*l))
+ .and_then(|l| l.props.find_group(*u))
+ .is_some_and(|g| is_object_matte(g) || !fx::data(&params_static(g)).is_empty())
});
todo
}
@@ -284,6 +312,7 @@ struct Work {
fps: f64,
frames: Vec<i64>,
task: RotoTask,
+ object_matte: bool,
}
/// Renders the effect's input frames for a worker, keeping the last few. With a browser job
@@ -400,10 +429,13 @@ async fn run_work(w: Work, shared: &RotoShared) {
let mut s = lock(&shared.state);
s.task = Some(w.task);
s.total = w.frames.len() as u64;
+ s.object_matte = w.object_matte;
}
let cancel = || shared.cancel.load(Ordering::Relaxed);
let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0);
- let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() };
+ let model = fx::installed_model();
+ let key = if w.object_matte { om::frozen_key(&w.params, w.size, w.fps, model.as_deref()) } else { fx::frozen_key(&w.params, w.size, w.fps) };
+ let mut frozen = FrozenCache { key, ..Default::default() };
for (k, f) in w.frames.iter().enumerate() {
if cancel() {
let mut s = lock(&shared.state);
@@ -414,9 +446,21 @@ async fn run_work(w: Work, shared: &RotoShared) {
// The frame and its neighbours (what a step reads), on the GPU.
src.prefetch(&[*f, f - 1, f + 1]).await;
let mut get = |g: i64| src.get(g);
- let ok = match w.task {
- RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(),
- RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) {
+ let ok = match (w.task, w.object_matte) {
+ (RotoTask::Propagate, true) => om::ensure(&w.params, w.size, w.fps, *f, model.clone(), &mut get, &cancel).is_some(),
+ (RotoTask::Freeze, true) => {
+ let img = get(*f);
+ match img.map(|img| (om::frame_matte(&w.params, w.size, w.fps, *f, 1.0, &img, model.clone(), &mut get, &cancel), img)) {
+ Some((om::FrameMatte::Alpha(a), img)) => {
+ frozen.size = [img.width as usize, img.height as usize];
+ frozen.frames.insert(*f, fx::freeze_frame(&a, &[]));
+ true
+ }
+ _ => false,
+ }
+ }
+ (RotoTask::Propagate, false) => chain.ensure(*f, &mut get, &cancel).is_some(),
+ (RotoTask::Freeze, false) => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) {
Some((a, b)) => {
if let Some(img) = src.get(*f) {
frozen.size = [img.width as usize, img.height as usize];
@@ -471,12 +515,15 @@ impl Session {
self.roto_job.as_ref().map(|j| j.progress()).or_else(|| {
self.offloaded_roto().map(|(j, task)| {
let p = &j.progress;
+ let object_matte =
+ self.project.comp(j.comp).and_then(|c| c.layer(j.layer)).and_then(|l| l.props.find_group(j.uid)).is_some_and(is_object_matte);
RotoProgress {
task: Some(task),
done: p.done,
total: p.total,
elapsed: p.elapsed,
fps: crate::offload::rate(p.done, p.elapsed),
+ object_matte,
..Default::default()
}
})
@@ -488,7 +535,8 @@ impl Session {
self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)).or_else(|| self.offloaded_roto().map(|(j, task)| (j.comp, j.layer, j.uid, task)))
}
- /// The segmentation chain a propagation of `effect` computes (its frames' cache keys).
+ /// The segmentation chain a propagation of Roto Brush `effect` computes (its frames' cache
+ /// keys).
pub fn roto_chain(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option<fx::Chain> {
let c = self.project.comp(comp)?;
let l = c.layer(layer)?;
@@ -499,6 +547,27 @@ impl Session {
Some(fx::Chain::new(&params, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0))
}
+ /// The cache keys of the segmentations a propagation of `effect` (Roto Brush or Object
+ /// Matte) computes, by layer frame.
+ pub fn roto_keys(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option<Vec<(i64, u64)>> {
+ if let Some(ch) = self.roto_chain(comp, layer, effect) {
+ return Some(ch.keys.into_iter().collect());
+ }
+ let c = self.project.comp(comp)?;
+ let l = c.layer(layer)?;
+ let g = l.effects()?.groups().find(|g| g.uid == effect && is_object_matte(g))?;
+ let params = self.om_params(comp, l, g)?;
+ let frames = om::job_frames(&params, frame_limits(c, l));
+ Some(om::frame_keys(&params, layer_size(&self.project, c, l), c.frame_rate.as_f64(), &frames, fx::installed_model()))
+ }
+
+ /// An Object Matte's parameters at its reference frame (what its jobs segment with).
+ fn om_params(&self, comp: ItemId, l: &Layer, g: &PropGroup) -> Option<Params> {
+ let c = self.project.comp(comp)?;
+ let t = comp_time_of(c, l, om::reference_frame(&params_static(g)));
+ Some(params_at(&self.project, comp, l, g, t, self.expr.as_deref()))
+ }
+
pub fn stop_roto(&mut self) -> bool {
if self.cancel_offloaded(JobKind::RotoFreeze) || self.cancel_offloaded(JobKind::RotoPropagate) {
return true;
@@ -532,17 +601,36 @@ impl Session {
let l = c.layer(layer).ok_or("no layer")?;
let fxg = l.effects().ok_or("the layer has no effects")?;
let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?;
- let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?;
+ let g = fxg.groups().nth(index).filter(|g| is_roto(g) || is_object_matte(g)).ok_or("not a Roto Brush & Refine Edge or Object Matte effect")?;
+ let object_matte = is_object_matte(g);
let size = layer_size(&self.project, c, l);
let fps = c.frame_rate.as_f64();
- let pstat = params_static(g);
- let d = fx::data(&pstat);
- if d.is_empty() {
- return Err("draw a foreground stroke with the Roto Brush tool first".into());
- }
- let t = comp_time_of(c, l, d.base.unwrap_or(0));
- let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref());
- let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir });
+ let (params, frames) = if object_matte {
+ let params = self.om_params(comp, l, g).ok_or("no composition")?;
+ let base = om::reference_frame(&params);
+ if om::roto_data(&params, size, base, fx::installed_model().is_some()).is_none() {
+ return Err("set the object first: click it with the Object Matte tool (its point is off the layer or outside its box)".into());
+ }
+ let all = om::job_frames(&params, frame_limits(c, l));
+ if all.is_empty() {
+ return Err(format!("the reference frame {base} is outside the layer"));
+ }
+ // Forward / backward from the reference frame, as for a Roto Brush.
+ let frames = all
+ .into_iter()
+ .filter(|f| task == RotoTask::Freeze || dir == Direction::Both || *f == base || (*f > base) == (dir == Direction::Forward))
+ .collect();
+ (params, frames)
+ } else {
+ let pstat = params_static(g);
+ let d = fx::data(&pstat);
+ if d.is_empty() {
+ return Err("draw a foreground stroke with the Roto Brush tool first".into());
+ }
+ let t = comp_time_of(c, l, d.base.unwrap_or(0));
+ let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref());
+ (params, job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }))
+ };
let n = frames.len();
let work = Work {
project: self.project.clone(),
@@ -557,8 +645,9 @@ impl Session {
fps,
frames,
task,
+ object_matte,
};
- let strokes = str_of(g, fx::STROKES);
+ let strokes = guard_of(g);
if !wait && self.offloads() {
drop(work);
self.offload_analysis(match task {
@@ -569,6 +658,13 @@ impl Session {
}
let wait = wait || cfg!(target_arch = "wasm32");
let shared = Arc::new(RotoShared::default());
+ {
+ // Known before the worker starts (the progress shows at once).
+ let mut st = lock(&shared.state);
+ st.task = Some(task);
+ st.total = n as u64;
+ st.object_matte = object_matte;
+ }
let thread = if wait {
crate::offload::run_or_queue({
let sh = shared.clone();
@@ -584,7 +680,7 @@ impl Session {
.map_err(|e| e.to_string())?,
)
};
- self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes });
+ self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes, object_matte });
self.events.push(Event::ProjectChanged { revision: self.revision });
self.poll_roto_job();
Ok(n)
@@ -601,40 +697,43 @@ impl Session {
let _ = t.join();
}
let st = job.progress();
+ let name = if job.object_matte { "Object Matte" } else { "Roto Brush" };
if let Some(e) = &st.error {
- self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true });
+ self.events.push(Event::Toast { message: format!("{name}: {e}"), error: true });
self.bump();
return true;
}
if st.cancelled {
- self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false });
+ self.events.push(Event::Toast { message: format!("{name}: stopped"), error: false });
self.bump();
return true;
}
match job.task {
RotoTask::Propagate => {
- self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false });
+ let verb = if job.object_matte { "tracked" } else { "propagated" };
+ self.events.push(Event::Toast { message: format!("{name}: {verb} {} frame(s) in {:.1} s", st.done, st.elapsed), error: false });
// Rendered frames depend on the cache: redraw.
self.bump();
}
RotoTask::Freeze => {
let Some(fc) = lock(&job.shared.frozen).take() else { return true };
let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone());
+ let what = if job.object_matte { "prompt" } else { "strokes" };
let r = self.edit("Freeze", None, |p, _| {
let g = p
.comp_mut(cid)
.and_then(|c| c.layer_mut(lid))
.and_then(|l| l.props.find_group_mut(uid))
- .ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?;
- if str_of(g, fx::STROKES) != strokes {
- return Err(crate::EngineError::Other("the strokes changed while freezing".into()));
+ .ok_or_else(|| crate::EngineError::Other(format!("the {name} effect was removed")))?;
+ if guard_of(g) != strokes {
+ return Err(crate::EngineError::Other(format!("the {what} changed while freezing")));
}
set_str(g, FROZEN, fc.to_json());
Ok(())
});
let msg = match r {
- Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed),
- Err(e) => format!("Roto Brush: {e}"),
+ Ok(()) => format!("{name}: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed),
+ Err(e) => format!("{name}: {e}"),
};
self.events.push(Event::Toast { message: msg, error: false });
}
@@ -664,6 +763,12 @@ impl Session {
let Some(c) = self.project.comp(comp) else { return out };
let Some(l) = c.layer(layer) else { return out };
let Some(g) = l.props.find_group(effect) else { return out };
+ if is_object_matte(g) {
+ for (f, k) in self.roto_keys(comp, layer, effect).unwrap_or_default() {
+ out.insert(f, fx::cached(k, 1.0).is_some());
+ }
+ return out;
+ }
let ps = params_static(g);
let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0);
for f in ch.keys.keys() {
diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs
index 19eee1103..edcab5c1c 100644
--- a/crates/engine/src/tests_roto.rs
+++ b/crates/engine/src/tests_roto.rs
@@ -465,3 +465,84 @@ fn object_matte_reprompt_updates_the_instance() {
// An unknown or non-Object Matte effect is refused.
assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": 999_999, "point": c})).is_err());
}
+
+/// Object Matte runs in the background like Roto Brush propagation: queued by the edit, a job
+/// that returns at once and reports progress, stops when cancelled (keeping what it computed),
+/// computes the tracked frames when left to run, and Freeze stores mattes that renders then use
+/// without segmenting again.
+#[test]
+fn object_matte_background_job_cancel_and_freeze() {
+ hold_roto_cache_test_lock();
+ let (mut s, clip, solid) = setup(|f| draw(center(f)));
+ let cid = s.active_comp_id().unwrap();
+ s.edit("remove solid", None, |p, _| {
+ p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid);
+ for it in p.items.values_mut() {
+ if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
+ f.path = "object-matte-job.mov".into();
+ }
+ }
+ Ok(())
+ })
+ .unwrap();
+ let c = center(0);
+ let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "before": 0, "after": 24, "frame": 0})).unwrap();
+ let uid = r["effect"].as_u64().unwrap();
+ assert!(s.roto_pending.contains(&(cid, clip, uid)), "the new prompt is queued for tracking");
+ let wait_done = |s: &mut Session| {
+ let t = std::time::Instant::now();
+ while s.is_roto_running() {
+ assert!(t.elapsed().as_secs() < 120, "the job never finished");
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ s.poll_roto(false);
+ }
+ s.poll_roto(false);
+ };
+ let toast = |s: &Session, m: &str| s.events.iter().any(|e| matches!(e, crate::Event::Toast { message, .. } if message.starts_with(m)));
+ let status = |s: &mut Session| s.execute("objectMatte.status", json!({"layer": clip.0, "effect": uid})).unwrap();
+
+ // Cancelled at once: it stops early, and says so.
+ let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap();
+ assert_eq!(r["frames"], 25);
+ assert!(!s.roto_pending.contains(&(cid, clip, uid)));
+ let _ = s.execute("roto.cancel", json!({}));
+ wait_done(&mut s);
+ assert!(toast(&s, "Object Matte: stopped"), "{:?}", s.events);
+ let st = status(&mut s);
+ assert_eq!(st["frames"].as_array().unwrap().len(), 25, "{st}");
+ let early = st["computed"].as_array().unwrap().len();
+ assert!(early < 25, "{st}");
+
+ // Left to run: the call returns while the job works (progress visible), then every tracked
+ // frame is computed.
+ let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap();
+ assert_eq!(r["running"], true, "{r}");
+ let pr = s.roto_progress().unwrap();
+ assert!(pr.object_matte && pr.banner().starts_with("Tracking Object Matte"), "{pr:?}");
+ // One job at a time.
+ assert!(s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).is_err());
+ wait_done(&mut s);
+ assert!(toast(&s, "Object Matte: tracked"), "{:?}", s.events);
+ assert_eq!(status(&mut s)["computed"].as_array().unwrap().len(), 25);
+ let before = alpha_at(&s, 4);
+
+ // Freeze, then drop the segmentations: renders use the frozen mattes and compute nothing.
+ let r = s.execute_checked("objectMatte.freeze", json!({"layer": clip.0, "effect": uid, "wait": true})).unwrap();
+ assert_eq!(r["frozen"], true, "{r}");
+ let keys: Vec<u64> = s.roto_keys(cid, clip, uid).unwrap().into_iter().map(|(_, k)| k).collect();
+ assert_eq!(keys.len(), 25);
+ effectcraft_effects::roto::forget(keys.iter().copied());
+ let frozen = alpha_at(&s, 4);
+ let diff = before.iter().zip(&frozen).filter(|(a, b)| (**a - **b).abs() > 0.02).count();
+ assert!(diff < 30, "{diff} pixels differ from the computed matte");
+ assert!(status(&mut s)["computed"].as_array().unwrap().is_empty(), "a frozen frame was segmented again");
+ assert_eq!(status(&mut s)["frozen"], true);
+ // A frozen prompt is refused; Unfreeze computes again.
+ assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": uid, "point": c})).is_err());
+ s.execute_checked("objectMatte.unfreeze", json!({"layer": clip.0, "effect": uid})).unwrap();
+ let _ = alpha_at(&s, 4);
+ assert!(status(&mut s)["computed"].as_array().unwrap().iter().any(|f| f == 4));
+ // Undo brings the frozen mattes back.
+ s.execute("edit.undo", json!({})).unwrap();
+ assert_eq!(status(&mut s)["frozen"], true);
+}
diff --git a/crates/ui-egui/src/i18n/ui.rs b/crates/ui-egui/src/i18n/ui.rs
index f7c779053..117d4b708 100644
--- a/crates/ui-egui/src/i18n/ui.rs
+++ b/crates/ui-egui/src/i18n/ui.rs
@@ -508,6 +508,7 @@ const JAPANESE: &[(&str, &str)] = &[
("⌘/Ctrl-drag a pin to record its motion", "⌘/Ctrl+ドラッグでピンの動きを記録"),
("▣ {}", "▣ {}"),
("⚠ {} is used by:", "⚠ {}は次のコマンドで使用中:"),
+ ("Track", "トラック"),
];
const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[
@@ -1033,6 +1034,7 @@ const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[
("After Effects project", "After Effects 项目"),
("Lottie animation", "Lottie 动画"),
("Project", "项目"),
+ ("Track", "跟踪"),
];
const TRADITIONAL_CHINESE: &[(&str, &str)] = &[
@@ -1553,6 +1555,7 @@ const TRADITIONAL_CHINESE: &[(&str, &str)] = &[
("After Effects project", "After Effects 專案"),
("Lottie animation", "Lottie 動畫"),
("Project", "專案"),
+ ("Track", "追蹤"),
];
/// Ukrainian (uk): original translations of the same interface rows as the Chinese catalogs.
@@ -1988,6 +1991,7 @@ const UKRAINIAN: &[(&str, &str)] = &[
("⌘/Ctrl-drag a pin to record its motion", "Перетягніть опорну точку з ⌘/Ctrl, щоб записати її рух"),
("▣ {}", "▣ {}"),
("⚠ {} is used by:", "⚠ {} використовується в:"),
+ ("Track", "Відстежити"),
];
#[cfg(test)]
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index a0e05e007..a6b5501cd 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -1580,6 +1580,12 @@ impl EffectcraftApp {
self.session.poll_roto(true);
ctx.request_repaint_after(std::time::Duration::from_millis(100));
}
+ // Roto Brush propagation and Object Matte tracking / Freeze: finish jobs (a Freeze writes
+ // its mattes) and start queued ones in the background.
+ if self.session.roto_job.is_some() || !self.session.roto_pending.is_empty() {
+ self.session.poll_roto(true);
+ ctx.request_repaint_after(std::time::Duration::from_millis(100));
+ }
// 3D Camera Tracker: likewise (tracking, then solving).
if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() {
self.session.poll_camera(true);
diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs
index 501adf898..a63b5a832 100644
--- a/crates/ui-egui/src/panels/effect_controls.rs
+++ b/crates/ui-egui/src/panels/effect_controls.rs
@@ -787,6 +787,44 @@ fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l
p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim);
}
+/// Object Matte: Freeze / Unfreeze and Track / Stop buttons (the background job that computes
+/// the matte over the tracked frames) and its status.
+fn object_matte_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) {
+ use effectcraft_engine::effects::object_matte as om;
+ let t = app.tokens;
+ let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone());
+ let frozen = om::is_frozen(&params);
+ let mine = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid);
+ let running = mine && app.session.is_roto_running();
+ let busy = app.session.is_roto_running();
+ let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0));
+ let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0));
+ let label = if frozen { "Unfreeze" } else { "Freeze" };
+ if widgets::text_button(ui, b1, crate::i18n::tr(label), frozen, &t, egui::Id::new(("om-freeze-ec", g.uid))).clicked() && !busy {
+ let id = if frozen { "objectMatte.unfreeze" } else { "objectMatte.freeze" };
+ actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid})));
+ }
+ app.auto.add(&format!("effectControls.objectMatte.{}.freeze", g.uid), b1, label);
+ let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Track", "objectMatte.propagate") };
+ if widgets::text_button(ui, b2, crate::i18n::tr(l2), false, &t, egui::Id::new(("om-track-ec", g.uid))).clicked() && !frozen && (running || !busy) {
+ actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid})));
+ }
+ app.auto.add(&format!("effectControls.objectMatte.{}.track", g.uid), b2, l2);
+ let base = om::reference_frame(&params);
+ let status = match app.session.roto_progress() {
+ Some(pr) if running => pr.banner(),
+ _ if frozen => format!("Frozen from reference frame {base}"),
+ _ if app.session.roto_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to track".to_string(),
+ _ => format!("Reference frame {base}: Track computes the matte ahead of playback"),
+ };
+ p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, &status, Tokens::ui(11.5), t.text_dim);
+ app.auto.add(
+ &format!("effectControls.objectMatte.{}.status", g.uid),
+ Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 30.0), vec2(r.width() - 24.0, 16.0)),
+ &status,
+ );
+}
+
// ---------------------------------------------------------------------------------------------
// Curves / Levels editors
@@ -797,7 +835,7 @@ fn editor_height(effect: &str, g: &PropGroup, width: f32) -> f32 {
"ec.color.curves" => 34.0 + curves_size(width) + 26.0,
effectcraft_engine::effects::warp_stab::ID => 50.0,
effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT,
- effectcraft_engine::effects::roto::ID => 50.0,
+ effectcraft_engine::effects::roto::ID | effectcraft_engine::effects::object_matte::ID => 50.0,
"ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0,
"ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0,
_ => 0.0,
@@ -1207,6 +1245,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions),
effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions),
effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions),
+ effectcraft_engine::effects::object_matte::ID => object_matte_editor(app, ui, &bp, &layer, g, er, &mut actions),
"ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions),
super::fx_editors::GLOW | super::fx_editors::RESHAPE | super::fx_editors::EXTRACTOR => {
super::fx_editors::header_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions)
diff --git a/crates/ui-egui/tests/ui_object_matte.rs b/crates/ui-egui/tests/ui_object_matte.rs
index 483d271ea..5f30ba03b 100644
--- a/crates/ui-egui/tests/ui_object_matte.rs
+++ b/crates/ui-egui/tests/ui_object_matte.rs
@@ -8,6 +8,7 @@ use effectcraft_engine::effects::object_matte as om;
use effectcraft_engine::keyframe::Value;
use effectcraft_engine::project::{GroupKind, LayerId, PropGroup};
use effectcraft_ui_egui::EffectcraftApp;
+use effectcraft_ui_egui::dock::PanelKind;
use effectcraft_ui_egui::state::Tool;
use egui::{Event, Modifiers, Pos2, Rect, pos2, vec2};
use egui_kittest::Harness;
@@ -140,3 +141,45 @@ fn object_matte_tool_prompts_with_a_click_and_a_box() {
h.state_mut().session.execute("edit.undo", json!({})).unwrap();
assert_eq!(flatten_params(&mattes(&h)[0], &mut |p| p.value.clone()).get("useBox"), Some(&Value::Bool(false)));
}
+
+/// Effect Controls: the Object Matte's Track and Freeze buttons run the background job and its
+/// status line shows where it is; the UI keeps drawing frames while it runs and finishes the
+/// job itself (a Freeze writes its mattes without any other job running).
+#[test]
+fn object_matte_effect_controls_track_and_freeze() {
+ let mut h = harness();
+ h.state_mut().show_panel(PanelKind::EffectControls);
+ h.state_mut().ui.tool = Tool::ObjectMatte;
+ h.run_steps(2);
+ let center = screen(&h, [320.0, 180.0]);
+ click(&mut h, center, Modifiers::NONE);
+ let uid = mattes(&h)[0].uid;
+ h.run_steps(2);
+ let status = |h: &Harness<'_, EffectcraftApp>| h.state().auto.find(&format!("effectControls.objectMatte.{uid}.status")).map(|e| e.label.clone());
+ assert!(status(&h).is_some(), "no Object Matte status line");
+ // Freeze: a background job (the frames keep coming), then the effect is frozen.
+ let b = rect(&h, &format!("effectControls.objectMatte.{uid}.freeze"));
+ // Let any automatic tracking finish first (one job at a time).
+ let frozen = |h: &Harness<'_, EffectcraftApp>| om::is_frozen(&flatten_params(&mattes(h)[0], &mut |p| p.value.clone()));
+ let t = std::time::Instant::now();
+ while h.state().session.is_roto_running() {
+ assert!(t.elapsed().as_secs() < 120);
+ h.step();
+ }
+ click(&mut h, b.center(), Modifiers::NONE);
+ let t = std::time::Instant::now();
+ while !frozen(&h) {
+ assert!(
+ t.elapsed().as_secs() < 20,
+ "never frozen: {:?} / {} / running {} / {:?}",
+ status(&h),
+ h.state().ui.status,
+ h.state().session.is_roto_running(),
+ h.state().session.roto_progress()
+ );
+ h.step();
+ }
+ h.run_steps(2);
+ assert!(status(&h).is_some_and(|s| s.starts_with("Frozen")), "{:?}", status(&h));
+ assert_eq!(h.state().auto.find(&format!("effectControls.objectMatte.{uid}.freeze")).map(|e| e.label.as_str()), Some("Unfreeze"));
+}
diff --git a/docs/agents.md b/docs/agents.md
index 6a3b9e85e..d94d46bdc 100644
--- a/docs/agents.md
+++ b/docs/agents.md
@@ -471,6 +471,24 @@ effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[
roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save
```
+### Object Matte
+
+The Object Matte tool (third in the Alt+W slot) works in the Composition viewer: click an object,
+or drag a box around it; Shift adds another Object Matte instead of re-prompting the layer's.
+Agents use `objectMatte.apply {layer, point: [x, y], box?: [x0, y0, x1, y1], frame?, track?,
+before?, after?, effect?}` (layer pixels, layer frames; `effect` re-prompts that instance), one
+undo step each. With a trained model installed (Settings ▸ Roto Brush) the box is the model's box
+prompt. `objectMatte.propagate {layer, effect?, direction?, wait?}` tracks the matte over the
+frames around the reference frame in the background (`roto.cancel` stops it; edits to the prompt
+or the layer's frames queue it again), `objectMatte.freeze {wait?}` / `objectMatte.unfreeze`
+store or drop the final mattes, and `objectMatte.status {layer, effect?}` reports the job's
+progress, the frozen state, the tracked frames and which are computed.
+
+```sh
+effectcraft-cli run clip.ecproj objectMatte.apply '{"layer":"#1","point":[320,200],"box":[250,120,400,300]}' \
+ objectMatte.propagate '{"layer":"#1","wait":true}' objectMatte.freeze '{"layer":"#1","wait":true}' --save
+```
+
### Text: styles and editing
Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the
diff --git a/docs/effects.md b/docs/effects.md
index 84f16ef86..8a02b4892 100644
--- a/docs/effects.md
+++ b/docs/effects.md
@@ -3,7 +3,7 @@
<!-- Generated from the effect registry (crates/effects/src/catalog.rs); do not edit by hand.
Regenerate: UPDATE_DOCS=1 cargo test -p effectcraft-effects --lib effects_doc_is_current -->
-EffectCraft ships 307 effects with 3028 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
+EffectCraft ships 307 effects with 3029 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
- **GPU**: 280 effects also run on the GPU compositor with identical results.
- **32**: 307 effects process 32-bit float (HDR, overbright) pixels without clamping.