Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

#783: Object Matte effect from a point / box prompt, tracked

vangel76953fbbc730be
diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs
index 7e968297a..d32cfb2e1 100644
--- a/crates/effects/src/lib.rs
+++ b/crates/effects/src/lib.rs
@@ -40,6 +40,7 @@ pub mod mocha_shape;
mod noise;
mod noise2;
mod noise3;
+pub mod object_matte;
mod obsolete;
mod ocio;
pub mod ocio_config;
@@ -497,6 +498,7 @@ pub fn registry() -> &'static [EffectSpec] {
v.extend(camera_tracker::specs());
v.extend(face_track::specs());
v.extend(roto::specs());
+ v.extend(object_matte::specs());
v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name)));
for s in v.iter_mut() {
if GPU_EFFECTS.contains(&s.id) {
@@ -805,7 +807,15 @@ pub const PARAM_GROUPS: &[(&str, &str)] = &[
/// Display names of nested parameter groups by match id.
fn group_name(m: &str) -> &str {
- warp_stab::GROUPS.iter().chain(roto::GROUPS).chain(keylight::GROUPS).chain(PARAM_GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m)
+ warp_stab::GROUPS
+ .iter()
+ .chain(roto::GROUPS)
+ .chain(object_matte::GROUPS)
+ .chain(keylight::GROUPS)
+ .chain(PARAM_GROUPS)
+ .find(|(id, _)| *id == m)
+ .map(|(_, n)| *n)
+ .unwrap_or(m)
}
/// The nested group at `path` (`borders/autoScale`) under `g`, created on first use.
@@ -1180,6 +1190,7 @@ pub const TIME_DEPENDENT: &[&str] = &[
camera_tracker::ID,
// Each frame has its own segmentation.
roto::ID,
+ object_matte::ID,
];
/// See [`TIME_DEPENDENT`].
diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs
new file mode 100644
index 000000000..3b7b7c28e
--- /dev/null
+++ b/crates/effects/src/object_matte.rs
@@ -0,0 +1,468 @@
+//! Object Matte (Effect ▸ Matte): isolate one object from a click, and follow it through the
+//! shot (After Effects 26.2's Object Matte, from its public description: pick an object with a
+//! point, optionally a box around it, and get a matte that tracks it).
+//!
+//! The prompt is the **Object Point** (layer pixels) and, with **Use Box**, a box around the
+//! object. On the **Reference Frame** it is segmented by the same engine as Roto Brush: the
+//! trained model chosen in Settings ▸ Roto Brush (MobileSAM, `effectcraft_segment`) when one is
+//! installed, the classical graph cut otherwise (the point is a foreground stroke, the box's
+//! edges, or without a box the layer's border, background strokes). With **Track Object** the
+//! matte is then carried frame by frame up to **Frames Before** / **Frames After** away (optical
+//! flow, re-segmented by the model prompted with the carried matte); without it every frame is
+//! segmented on its own from the prompt (useful when the point is animated).
+//!
+//! Like Roto Brush, the segmentation is derived data in Roto Brush's content-keyed cache
+//! ([`crate::roto::cached`]): the chain seed holds the engine-maintained Input Key (what the
+//! frames are made of), the layer size, the frame rate, the mode and the model, and every
+//! frame's key the prompt, so edits upstream or to the prompt recompute it and undo never sees a
+//! stale matte. **Feather**, **Contrast** and **Shift Edge** shape the matte, **Invert** keeps
+//! everything but the object and **View ▸ Matte** shows the matte in grey.
+
+use std::sync::Arc;
+
+use effectcraft_keyframe::Value;
+use effectcraft_project::ParamUi;
+use effectcraft_raster::Image;
+use effectcraft_segment::MaskModel;
+use effectcraft_track::roto::refine::MatteParams;
+use effectcraft_track::roto::{RotoData, SegOpts, Stroke, StrokeKind};
+use rayon::prelude::*;
+
+use crate::roto::{self, Chain, INPUT_KEY};
+use crate::warp_stab::param;
+use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider};
+
+/// The effect's id.
+pub const ID: &str = "ec.matte.objectmatte";
+
+/// Display names of the instance's parameter groups.
+pub const GROUPS: &[(&str, &str)] = &[("objectTracking", "Tracking"), ("objectMatte", "Matte")];
+
+pub const VIEWS: [&str; 2] = ["Final Output", "Matte"];
+
+/// Most frames a matte is carried from the reference frame each way.
+pub const MAX_REACH: f64 = 600.0;
+/// Default reach each way (frames).
+pub const DEFAULT_REACH: f64 = 30.0;
+
+/// Radius (layer pixels) of the foreground stroke the Object Point becomes.
+const POINT_RADIUS: f64 = 3.0;
+/// Radius (layer pixels) of the background strokes along the box / the layer's border.
+const EDGE_RADIUS: f64 = 2.0;
+
+pub fn specs() -> Vec<EffectSpec> {
+ vec![EffectSpec {
+ id: ID,
+ name: "Object Matte",
+ category: "Matte",
+ params: vec![
+ p("objectPoint", "Object Point", Value::Vec2([0.5, 0.5]), ParamUi::Point),
+ p("useBox", "Use Box", Value::Bool(false), ParamUi::Checkbox),
+ p("boxTopLeft", "Box Top Left", Value::Vec2([0.25, 0.25]), ParamUi::Point),
+ p("boxBottomRight", "Box Bottom Right", Value::Vec2([0.75, 0.75]), ParamUi::Point),
+ p("objectTracking/trackObject", "Track Object", Value::Bool(true), ParamUi::Checkbox),
+ p("objectTracking/referenceFrame", "Reference Frame", num(0.0), ParamUi::Number),
+ p("objectTracking/framesBefore", "Frames Before", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)),
+ p("objectTracking/framesAfter", "Frames After", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)),
+ p("objectMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)),
+ p("objectMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
+ p("objectMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)),
+ p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox),
+ p("view", "View", Value::Enum(0), popup(&VIEWS)),
+ p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden),
+ ],
+ render,
+ gpu: false,
+ float: true,
+ }]
+}
+
+fn f(params: &Params, id: &str) -> f64 {
+ param(params, id).map(Value::as_f64).unwrap_or(0.0)
+}
+fn b(params: &Params, id: &str) -> bool {
+ param(params, id).map(Value::as_bool).unwrap_or(false)
+}
+fn v2(params: &Params, id: &str) -> Option<[f64; 2]> {
+ let v = param(params, id)?.as_vec2();
+ v.iter().all(|c| c.is_finite()).then_some(v)
+}
+fn s<'a>(params: &'a Params, id: &str) -> &'a str {
+ match param(params, id) {
+ Some(Value::Str(s)) => s,
+ _ => "",
+ }
+}
+
+fn fnv(h: &mut u64, bytes: &[u8]) {
+ for b in bytes {
+ *h ^= *b as u64;
+ *h = h.wrapping_mul(0x0100_0000_01b3);
+ }
+}
+
+/// A frame count parameter (finite, clamped to `lo..=hi`).
+fn frames(params: &Params, id: &str, lo: f64, hi: f64) -> i64 {
+ let v = f(params, id);
+ if v.is_finite() { v.round().clamp(lo, hi) as i64 } else { 0 }
+}
+
+/// The reference frame (layer frame).
+pub fn reference_frame(params: &Params) -> i64 {
+ frames(params, "objectTracking/referenceFrame", -1e9, 1e9)
+}
+
+/// Whether the matte is carried from the reference frame (else every frame is segmented on its
+/// own).
+pub fn tracks(params: &Params) -> bool {
+ b(params, "objectTracking/trackObject")
+}
+
+/// The prompt as Roto Brush strokes on layer frame `frame`: the Object Point as a foreground
+/// stroke; the box's edges as background strokes; without a box, the layer's border when there
+/// is no model (the graph cut needs background samples; a model does not, and a border stroke
+/// would cut objects touching the frame edge). `None` when the point is unusable.
+pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_model: bool) -> Option<Vec<Stroke>> {
+ let [w, h] = layer_size;
+ if !(w.is_finite() && h.is_finite() && w >= 1.0 && h >= 1.0) {
+ return None;
+ }
+ let pt = v2(params, "objectPoint")?;
+ let inside = |q: [f64; 2]| q[0] >= 0.0 && q[1] >= 0.0 && q[0] <= w && q[1] <= h;
+ if !inside(pt) {
+ return None;
+ }
+ let mut out = vec![Stroke { kind: StrokeKind::Fg, frame, radius: POINT_RADIUS, points: vec![pt] }];
+ let rect = |x0: f64, y0: f64, x1: f64, y1: f64| vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]];
+ let boxed = b(params, "useBox").then(|| Some((v2(params, "boxTopLeft")?, v2(params, "boxBottomRight")?))).flatten();
+ match boxed {
+ Some((a, c)) => {
+ let (x0, x1) = (a[0].min(c[0]).clamp(0.0, w), a[0].max(c[0]).clamp(0.0, w));
+ let (y0, y1) = (a[1].min(c[1]).clamp(0.0, h), a[1].max(c[1]).clamp(0.0, h));
+ // The point must lie inside the box, clear of its edges, for the box to mean anything.
+ let m = POINT_RADIUS + EDGE_RADIUS;
+ if pt[0] - m <= x0 || pt[0] + m >= x1 || pt[1] - m <= y0 || pt[1] + m >= y1 {
+ return None;
+ }
+ out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) });
+ }
+ None if !with_model => {
+ let i = EDGE_RADIUS;
+ if w <= 4.0 * i || h <= 4.0 * i {
+ return None;
+ }
+ out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(i, i, w - i, h - i) });
+ }
+ None => {}
+ }
+ Some(out)
+}
+
+/// The segmentation data the matte of layer frame `f` comes from: with Track Object, the prompt
+/// on the reference frame and the span it is carried over; otherwise the prompt on `f` alone.
+pub fn roto_data(params: &Params, layer_size: [f64; 2], f: i64, with_model: bool) -> Option<RotoData> {
+ if tracks(params) {
+ let base = reference_frame(params);
+ let before = frames(params, "objectTracking/framesBefore", 0.0, MAX_REACH);
+ let after = frames(params, "objectTracking/framesAfter", 0.0, MAX_REACH);
+ let strokes = prompt_strokes(params, layer_size, base, with_model)?;
+ Some(RotoData { base: Some(base), span: [base.saturating_sub(before), base.saturating_add(after)], strokes })
+ } else {
+ Some(RotoData { base: Some(f), span: [f, f], strokes: prompt_strokes(params, layer_size, f, with_model)? })
+ }
+}
+
+/// Seed of the chain keys: what the frames are (the Input Key), the layer size, the frame rate,
+/// the mode and the model. The prompt is in each frame's key.
+pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> u64 {
+ let mut h = 0xcbf2_9ce4_8422_2325;
+ fnv(&mut h, ID.as_bytes());
+ fnv(&mut h, s(params, INPUT_KEY).as_bytes());
+ for v in [layer_size[0], layer_size[1], fps] {
+ fnv(&mut h, &v.to_bits().to_le_bytes());
+ }
+ fnv(&mut h, &[tracks(params) as u8]);
+ fnv(&mut h, model.map_or(effectcraft_segment::CLASSICAL, |m| m.info().id).as_bytes());
+ h
+}
+
+/// Segmentation settings at `scale` (Roto Brush's defaults).
+fn seg_opts(scale: f64) -> SegOpts {
+ SegOpts { scale, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false }
+}
+
+/// Matte settings: Feather, Contrast, Shift Edge (no Refine Edge band).
+pub fn matte_params(params: &Params) -> MatteParams {
+ let num = |id: &str| {
+ let v = f(params, id);
+ if v.is_finite() { v } else { 0.0 }
+ };
+ MatteParams {
+ rb_feather: num("objectMatte/feather").clamp(0.0, 100.0),
+ rb_contrast: num("objectMatte/contrast").clamp(0.0, 100.0),
+ rb_shift: num("objectMatte/shiftEdge").clamp(-100.0, 100.0),
+ decontaminate: false,
+ ..MatteParams::default()
+ }
+}
+
+/// What a frame's matte is: the object's alpha on the layer grid, or why there is none.
+#[derive(Clone, Debug, PartialEq)]
+pub enum FrameMatte {
+ /// The object's alpha (0–1) on the frame's layer grid.
+ Alpha(Vec<f32>),
+ /// The prompt is unusable (point outside the layer, outside the box…): the layer passes.
+ NoPrompt,
+ /// The frame is beyond the tracked range, or its matte could not be computed.
+ Outside,
+}
+
+/// The matte of layer frame `f` on the layer grid of `img` (the effect's input on that frame at
+/// `scale`), computing what is missing. `frame(g)`: the input on frame g on the same grid.
+#[allow(clippy::too_many_arguments)]
+pub fn frame_matte(
+ params: &Params,
+ layer_size: [f64; 2],
+ fps: f64,
+ f: i64,
+ scale: f64,
+ img: &Image,
+ model: Option<Arc<dyn MaskModel>>,
+ frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>,
+ cancel: &dyn Fn() -> bool,
+) -> FrameMatte {
+ let Some(data) = roto_data(params, layer_size, f, model.is_some()) else { return FrameMatte::NoPrompt };
+ if !data.in_span(f) {
+ return FrameMatte::Outside;
+ }
+ let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref()));
+ let chain = Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model };
+ let Some(seg) = chain.ensure(f, frame, cancel) else { return FrameMatte::Outside };
+ let (w, h) = (img.width as usize, img.height as usize);
+ let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { roto::resample(&seg, w, h) };
+ FrameMatte::Alpha(roto::final_alpha(img, &seg, None, None, &matte_params(params), 0.0, scale))
+}
+
+fn render(ctx: &EffectCtx, buf: Buf) -> Buf {
+ if ctx.adjustment {
+ return buf;
+ }
+ let fps = ctx.fps();
+ let f = roto::frame_of(ctx.time, fps);
+ let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 };
+ let img = Arc::new(roto::layer_grid(&buf, ctx.layer_size));
+ let (w, h) = (img.width as usize, img.height as usize);
+ let host = ctx.env.host;
+ let effects = ctx.env.effect_index;
+ let layer_size = ctx.layer_size;
+ let me = img.clone();
+ let mut frame = |g: i64| -> Option<Arc<Image>> {
+ if g == f {
+ return Some(me.clone());
+ }
+ let other = host?.self_at(g as f64 / fps, effects)?;
+ let im = roto::layer_grid(&other, layer_size);
+ (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im))
+ };
+ let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, roto::installed_model(), &mut frame, &|| false) {
+ FrameMatte::Alpha(a) if a.len() == w * h => Some(a),
+ FrameMatte::NoPrompt => return buf,
+ _ => None,
+ };
+ compose(&buf, alpha.as_deref(), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1)
+}
+
+/// The output: the input keyed by the matte (`None`: no object, all zero), or the matte itself.
+fn compose(buf: &Buf, alpha: Option<&[f32]>, w: usize, h: usize, invert: bool, view_matte: bool) -> Buf {
+ let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64);
+ let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale };
+ out.img.rows_mut().for_each(|(by, row)| {
+ for (bx, px) in row.iter_mut().enumerate() {
+ let (x, y) = (bx as i64 - ox, by as i64 - oy);
+ let a = if x >= 0 && y >= 0 && (x as usize) < w && (y as usize) < h {
+ alpha.and_then(|a| a.get(y as usize * w + x as usize)).copied().unwrap_or(0.0)
+ } else {
+ 0.0
+ };
+ let a = if invert { 1.0 - a } else { a }.clamp(0.0, 1.0);
+ let src = buf.img.get(bx as i64, by as i64);
+ *px = if view_matte {
+ let v = a * src[3];
+ [v, v, v, 1.0]
+ } else {
+ [src[0] * a, src[1] * a, src[2] * a, src[3] * a]
+ };
+ }
+ });
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::{EffectEnv, EffectHost, LayerPixels};
+
+ /// A red disk of radius 15 moving 2 px a frame right over a textured blue background
+ /// (100×80, 25 fps), plus a still green square in the corner.
+ fn frame_at(fi: i64) -> Image {
+ let c = [40.0 + 2.0 * fi as f64, 40.0];
+ let mut img = Image::new(100, 80);
+ for y in 0..80 {
+ for x in 0..100 {
+ let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0;
+ let a = (0.5 - d).clamp(0.0, 1.0) as f32;
+ let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0;
+ let bg = if x >= 80 && y >= 60 { [0.1, 0.8, 0.2] } else { [0.1, 0.4 + tex, 0.7] };
+ let fg = [0.85 + tex, 0.2, 0.1];
+ img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]);
+ }
+ }
+ img
+ }
+ fn frame_of_time(t: f64) -> Image {
+ frame_at((t * 25.0 + 1e-6).floor() as i64)
+ }
+
+ struct Seq;
+ impl EffectHost for Seq {
+ fn layer(&self, _: u64, _: bool) -> Option<LayerPixels> {
+ None
+ }
+ fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option<Vec<f32>> {
+ None
+ }
+ fn self_at(&self, t: f64, _: usize) -> Option<Buf> {
+ Some(Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 })
+ }
+ }
+
+ fn params(vals: &[(&str, Value)]) -> Params {
+ let s = &specs()[0];
+ let mut p = Params { values: s.params.iter().map(|p| (p.id.to_string(), crate::default_value(p, [100.0, 80.0]))).collect() };
+ for (k, v) in vals {
+ p.values.insert(k.to_string(), v.clone());
+ }
+ p
+ }
+
+ fn run(p: &Params, fi: i64) -> Image {
+ let host = Seq;
+ let t = fi as f64 / 25.0 + 0.001;
+ let ctx = EffectCtx {
+ params: p,
+ time: t,
+ layer_size: [100.0, 80.0],
+ seed: 1,
+ adjustment: false,
+ env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() },
+ };
+ crate::apply(&specs()[0], &ctx, Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 }).img
+ }
+
+ /// Pixels whose alpha disagrees with the disk centred at `cx` (1 px of slack at the edge).
+ fn wrong(out: &Image, cx: f64) -> usize {
+ let mut n = 0;
+ for y in 0..80u32 {
+ for x in 0..100u32 {
+ let d = (x as f64 + 0.5 - cx).hypot(y as f64 + 0.5 - 40.0);
+ let a = out.data[(y * 100 + x) as usize][3];
+ if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) {
+ n += 1;
+ }
+ }
+ }
+ n
+ }
+
+ #[test]
+ fn a_click_isolates_the_object_and_tracks_it() {
+ let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]);
+ // The reference frame: the disk, nothing else.
+ let out = run(&p, 0);
+ assert!(wrong(&out, 40.0) < 20, "{} wrong pixels on the reference frame", wrong(&out, 40.0));
+ // Six frames on (carried through frames 1–5), the disk has moved 12 px and the matte with it.
+ let out = run(&p, 6);
+ assert!(wrong(&out, 52.0) < 30, "{} wrong pixels on frame 6", wrong(&out, 52.0));
+ // Beyond Frames After: no matte.
+ let p2 = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]);
+ assert!(run(&p2, 6).data.iter().all(|px| px[3] == 0.0));
+ // Invert keeps everything but the disk; View ▸ Matte shows it in grey, opaque.
+ let inv = run(&params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("invert", Value::Bool(true))]), 0);
+ assert!(inv.data[40 * 100 + 40][3] < 0.5 && inv.data[5 * 100 + 95][3] > 0.5);
+ let m = run(&params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("view", Value::Enum(1))]), 0);
+ assert!(m.data[40 * 100 + 40][0] > 0.5 && m.data[5 * 100 + 95][0] < 0.5 && m.data.iter().all(|px| px[3] == 1.0));
+ }
+
+ #[test]
+ fn without_tracking_every_frame_follows_the_prompt() {
+ // The point animated onto the disk at frame 9: frame 9 is segmented from it alone.
+ let p = params(&[("objectPoint", Value::Vec2([58.0, 40.0])), ("objectTracking/trackObject", Value::Bool(false))]);
+ let out = run(&p, 9);
+ assert!(wrong(&out, 58.0) < 20, "{} wrong pixels", wrong(&out, 58.0));
+ }
+
+ #[test]
+ fn a_box_and_bad_prompts() {
+ // A box around the green square, the point in it: the square, not the disk.
+ let p = params(&[
+ ("objectPoint", Value::Vec2([90.0, 70.0])),
+ ("useBox", Value::Bool(true)),
+ ("boxTopLeft", Value::Vec2([74.0, 54.0])),
+ ("boxBottomRight", Value::Vec2([100.0, 80.0])),
+ ]);
+ let out = run(&p, 0);
+ assert!(out.data[70 * 100 + 90][3] > 0.5 && out.data[40 * 100 + 40][3] < 0.5);
+ // Unusable prompts pass the layer through instead of failing: a point off the layer, a
+ // non-finite point, a point outside its box.
+ let src = frame_at(0);
+ for vals in [
+ vec![("objectPoint", Value::Vec2([-5.0, 40.0]))],
+ vec![("objectPoint", Value::Vec2([f64::NAN, 40.0]))],
+ vec![
+ ("objectPoint", Value::Vec2([40.0, 40.0])),
+ ("useBox", Value::Bool(true)),
+ ("boxTopLeft", Value::Vec2([60.0, 10.0])),
+ ("boxBottomRight", Value::Vec2([90.0, 30.0])),
+ ],
+ ] {
+ let p = params(&vals);
+ assert_eq!(run(&p, 0).data, src.data, "{vals:?}");
+ }
+ // A huge reference frame or reach doesn't overflow.
+ let p = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesBefore", Value::Scalar(f64::INFINITY))]);
+ assert!(run(&p, 0).data.iter().all(|px| px[3] == 0.0));
+ }
+
+ /// A stand-in model (the weights are an optional download): a disk around the first
+ /// foreground point, radius 15, so the test sees the model's answer in the matte.
+ #[derive(Default)]
+ struct Stub {
+ calls: std::sync::atomic::AtomicUsize,
+ }
+ impl MaskModel for Stub {
+ fn info(&self) -> &'static effectcraft_segment::ModelInfo {
+ &effectcraft_segment::MOBILE_SAM
+ }
+ fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result<Vec<f32>> {
+ self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ let Some(([cx, cy], _)) = prompt.points.iter().find(|(_, fg)| *fg) else { return Err("no point".into()) };
+ Ok((0..w * h).map(|i| (((i % w) as f32 + 0.5 - cx).hypot((i / w) as f32 + 0.5 - cy) < 15.0) as u8 as f32).collect())
+ }
+ }
+
+ #[test]
+ fn segments_with_the_installed_model_without_a_border_stroke() {
+ let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]);
+ // With a model, no border stroke: an object touching the frame edge stays whole.
+ assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, true).map(|s| s.len()), Some(1));
+ assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, false).map(|s| s.len()), Some(2));
+ // The model's answer is part of the cache key, and drives the matte.
+ let stub = Arc::new(Stub::default());
+ assert_ne!(seed(&p, [100.0, 80.0], 25.0, Some(stub.as_ref())), seed(&p, [100.0, 80.0], 25.0, None));
+ let img = frame_at(0);
+ let mut frame = |g: i64| Some(Arc::new(frame_at(g)));
+ let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") };
+ assert!(a[40 * 100 + 40] > 0.5 && a[5 * 100 + 5] < 0.5);
+ assert_eq!(stub.calls.load(std::sync::atomic::Ordering::Relaxed), 1);
+ }
+}
diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs
index d514e80a1..cd2d240e4 100644
--- a/crates/effects/src/roto.rs
+++ b/crates/effects/src/roto.rs
@@ -195,6 +195,12 @@ pub fn model_for(params: &Params) -> Option<Arc<dyn MaskModel>> {
if e(params, "version") == 0 {
return None;
}
+ installed_model()
+}
+
+/// The trained model chosen in Settings ▸ Roto Brush, if one is installed (Roto Brush 2.0 / 3.0
+/// and Object Matte segment with it).
+pub fn installed_model() -> Option<Arc<dyn MaskModel>> {
model_slot().read().ok().and_then(|m| m.clone())
}
@@ -392,7 +398,7 @@ pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg {
resample(s, w, h)
}
-fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {
+pub(crate) fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {
if s.w == w && s.h == h {
return s.clone();
}
diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs
index 95326b423..ff459d0bb 100644
--- a/crates/engine/src/commands/roto_cmds.rs
+++ b/crates/engine/src/commands/roto_cmds.rs
@@ -1,6 +1,9 @@
//! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer
//! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status.
//!
+//! `objectMatte.apply` adds an Object Matte effect (Effect ▸ Matte) from a point / box prompt on
+//! a frame; it segments with the same engine and model as Roto Brush.
+//!
//! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the
//! effect on the first stroke, as After Effects does); the first foreground stroke sets the base
//! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze
@@ -51,6 +54,15 @@ pub fn specs() -> Vec<CommandSpec> {
always,
options
),
+ cmd!(
+ "objectMatte.apply",
+ "Object Matte",
+ [],
+ None,
+ "{layer?, point: [x, y] (layer pixels: the object), box?: [x0, y0, x1, y1] (layer pixels, around the object), frame? (reference layer frame; default current), track?: follow the object from that frame (default true), before?, after? (frames to follow it; default 30, up to 600)}",
+ has_comp,
+ object_matte_apply
+ ),
query!(
"roto.status",
"Roto Brush Status",
@@ -422,3 +434,81 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::<Vec<_>>());
Ok(out)
}
+
+/// Two finite numbers.
+fn xy_p(p: &Value, k: &str, n: usize, cmd: &str) -> Result<Option<Vec<f64>>> {
+ let Some(v) = p.get(k).filter(|v| !v.is_null()) else { return Ok(None) };
+ let a = v.as_array().filter(|a| a.len() == n).ok_or_else(|| bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")))?;
+ let out: Vec<f64> = a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect();
+ if out.len() != n {
+ return Err(bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")));
+ }
+ Ok(Some(out))
+}
+
+fn object_matte_apply(s: &mut Session, p: &Value) -> Result<Value> {
+ use effectcraft_effects::object_matte as om;
+ let cmd = "objectMatte.apply";
+ let (cid, lid) = layer_p(s, p, cmd)?;
+ let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
+ let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?;
+ if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() {
+ return Err(bad(cmd, "Object Matte works on footage, solid and composition layers"));
+ }
+ let size = rt::layer_size(&s.project, comp, layer);
+ let point = xy_p(p, "point", 2, cmd)?.ok_or_else(|| bad(cmd, "missing `point` [x, y] (layer pixels)"))?;
+ let inside = |x: f64, y: f64| (0.0..=size[0]).contains(&x) && (0.0..=size[1]).contains(&y);
+ if !inside(point[0], point[1]) {
+ return Err(bad(cmd, format!("`point` is outside the layer ({}×{})", size[0], size[1])));
+ }
+ let bx = xy_p(p, "box", 4, cmd)?;
+ if let Some(b) = &bx {
+ let (x0, x1, y0, y1) = (b[0].min(b[2]), b[0].max(b[2]), b[1].min(b[3]), b[1].max(b[3]));
+ if !(point[0] > x0 && point[0] < x1 && point[1] > y0 && point[1] < y1) {
+ return Err(bad(cmd, "`point` must be inside `box`"));
+ }
+ }
+ let frame = frame_p(s, p, cid, layer);
+ let limits = rt::frame_limits(comp, layer);
+ if frame < limits[0] || frame > limits[1] {
+ return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1])));
+ }
+ let track = b_p(p, "track").unwrap_or(true);
+ let reach = |k: &str| f_p(p, k).filter(|v| v.is_finite()).unwrap_or(om::DEFAULT_REACH).round().clamp(0.0, om::MAX_REACH);
+ let (before, after) = (reach("before"), reach("after"));
+ let spec = effectcraft_effects::find(om::ID).ok_or_else(|| EngineError::Other("Object Matte effect missing".into()))?;
+ let uid = s.edit("Object Matte", None, |proj, _| {
+ let mut next = proj.next_id;
+ let l = layer_mut(proj, cid, lid)?;
+ let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?;
+ let n = fxg.groups().filter(|g| rt::is_object_matte(g)).count();
+ let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) };
+ let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size);
+ let mut set = |path: &str, v: KV| {
+ let (grp, leaf) = path.rsplit_once('/').map_or((None, path), |(a, b)| (Some(a), b));
+ let target = match grp {
+ Some(sub) => g.sub_mut(sub),
+ None => Some(&mut g),
+ };
+ if let Some(pr) = target.and_then(|t| t.get_mut(leaf)) {
+ pr.value = v;
+ }
+ };
+ set("objectPoint", KV::Vec2([point[0], point[1]]));
+ if let Some(b) = &bx {
+ set("useBox", KV::Bool(true));
+ set("boxTopLeft", KV::Vec2([b[0].min(b[2]), b[1].min(b[3])]));
+ set("boxBottomRight", KV::Vec2([b[0].max(b[2]), b[1].max(b[3])]));
+ }
+ set("objectTracking/trackObject", KV::Bool(track));
+ set("objectTracking/referenceFrame", KV::Scalar(frame as f64));
+ set("objectTracking/framesBefore", KV::Scalar(before));
+ set("objectTracking/framesAfter", KV::Scalar(after));
+ let u = g.uid;
+ fxg.children.push(g.into());
+ proj.next_id = proj.next_id.max(next);
+ Ok(u)
+ })?;
+ s.state.selected_layers = vec![lid];
+ Ok(json!({"layer": lid.0, "effect": uid, "frame": frame, "track": track, "span": [frame - before as i64, frame + after as i64]}))
+}
diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs
index ca92a74bc..b8c2c649c 100644
--- a/crates/engine/src/roto.rs
+++ b/crates/engine/src/roto.rs
@@ -147,6 +147,12 @@ pub(crate) fn is_roto(g: &PropGroup) -> bool {
matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID)
}
+/// An Object Matte instance (its Input Key is kept current like Roto Brush's; it segments on
+/// demand while rendering).
+pub(crate) fn is_object_matte(g: &PropGroup) -> bool {
+ matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID)
+}
+
fn str_of(g: &PropGroup, m: &str) -> String {
match g.get(m).map(|p| &p.value) {
Some(Value::Str(s)) => s.clone(),
@@ -188,8 +194,9 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String {
format!("{:016x}", fnv(&s))
}
-/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped.
-/// Returns the instances whose segmentation changed (to propagate again).
+/// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from
+/// other frames are dropped. Returns the Roto Brush instances whose segmentation changed (to
+/// propagate again).
pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
let mut todo = vec![];
let mut set = vec![];
@@ -197,13 +204,16 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI
let ItemKind::Comp(c) = &it.kind else { continue };
for l in &c.layers {
let Some(fxg) = l.effects() else { continue };
- for g in fxg.groups().filter(|g| is_roto(g)) {
+ for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) {
let key = input_key(after, *cid, l, g.uid);
+ let roto = is_roto(g);
let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid));
if str_of(g, INPUT_KEY) != key {
set.push((*cid, l.id, g.uid, key));
- todo.push((*cid, l.id, g.uid));
- } else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) {
+ if roto {
+ todo.push((*cid, l.id, g.uid));
+ }
+ } else if roto && old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) {
todo.push((*cid, l.id, g.uid));
}
}
diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs
index c1cbd2c8d..cdf585400 100644
--- a/crates/engine/src/tests_roto.rs
+++ b/crates/engine/src/tests_roto.rs
@@ -351,3 +351,78 @@ fn offloaded_propagation_streams_segmentations_back() {
s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap();
assert!(s.offloaded.is_empty());
}
+
+/// Object Matte: one click on the disk on frame 0 isolates it there and on the frames it is
+/// followed to; the Input Key follows the layer's frames; bad prompts are refused.
+#[test]
+fn object_matte_from_a_click() {
+ hold_roto_cache_test_lock();
+ let (mut s, clip, solid) = setup(|f| draw(center(f)));
+ let cid = s.active_comp_id().unwrap();
+ s.edit("remove solid", None, |p, _| {
+ p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid);
+ for it in p.items.values_mut() {
+ if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
+ f.path = "object-matte.mov".into();
+ }
+ }
+ Ok(())
+ })
+ .unwrap();
+ s.execute("time.set", json!({"time": 0})).unwrap();
+ let c = center(0);
+ let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "after": 6})).unwrap();
+ assert_eq!(r["frame"], 0);
+ assert_eq!(r["span"], json!([-30, 6]));
+ let uid = r["effect"].as_u64().unwrap();
+ let key = |s: &Session| {
+ let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone();
+ crate::roto::params_static(&g).get(effectcraft_effects::roto::INPUT_KEY).cloned()
+ };
+ let k0 = key(&s);
+ assert!(matches!(&k0, Some(effectcraft_keyframe::Value::Str(k)) if !k.is_empty()), "{k0:?}");
+ let wrong = |s: &Session, f: u32| {
+ let c = center(f);
+ alpha_at(s, f)
+ .iter()
+ .enumerate()
+ .filter(|(i, v)| {
+ let d = (((*i as u32 % W) as f64 + 0.5 - c[0]).hypot((*i as u32 / W) as f64 + 0.5 - c[1])) - R;
+ (d < -3.0 && **v < 0.5) || (d > 3.0 && **v > 0.5)
+ })
+ .count()
+ };
+ let w0 = wrong(&s, 0);
+ assert!(w0 < 40, "{w0} wrong pixels on the reference frame");
+ let w4 = wrong(&s, 4);
+ assert!(w4 < 60, "{w4} wrong pixels on frame 4");
+ // Past Frames After: transparent.
+ assert!(alpha_at(&s, 8).iter().all(|v| *v == 0.0));
+ // Other frames under the effect: a new Input Key (the mattes are recomputed).
+ s.edit("swap clip", None, |p, _| {
+ for it in p.items.values_mut() {
+ if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
+ f.path = "object-matte-2.mov".into();
+ }
+ }
+ Ok(())
+ })
+ .unwrap();
+ assert_ne!(key(&s), k0);
+ // Bad prompts are refused.
+ for bad in [
+ json!({"layer": clip.0}),
+ json!({"layer": clip.0, "point": [-10, 5]}),
+ json!({"layer": clip.0, "point": [10, "x"]}),
+ json!({"layer": clip.0, "point": c, "box": [0, 0, 10, 10]}),
+ json!({"layer": clip.0, "point": c, "frame": 999}),
+ ] {
+ assert!(s.execute_checked("objectMatte.apply", bad.clone()).is_err(), "{bad}");
+ }
+ // A box and a static prompt are accepted; the second instance is numbered.
+ let r = s
+ .execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "box": [c[0] + 50.0, c[1] + 50.0, c[0] - 50.0, c[1] - 50.0], "track": false}))
+ .unwrap();
+ let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(r["effect"].as_u64().unwrap()).unwrap().clone();
+ assert_eq!(g.name, "Object Matte 2");
+}
diff --git a/docs/effects.md b/docs/effects.md
index e498cf1ae..84f16ef86 100644
--- a/docs/effects.md
+++ b/docs/effects.md
@@ -3,11 +3,11 @@
<!-- Generated from the effect registry (crates/effects/src/catalog.rs); do not edit by hand.
Regenerate: UPDATE_DOCS=1 cargo test -p effectcraft-effects --lib effects_doc_is_current -->
-EffectCraft ships 306 effects with 3014 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
+EffectCraft ships 307 effects with 3028 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
- **GPU**: 280 effects also run on the GPU compositor with identical results.
-- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping.
-- **Status**: 306 implemented in full, 0 partial (what is missing is listed).
+- **32**: 307 effects process 32-bit float (HDR, overbright) pixels without clamping.
+- **Status**: 307 implemented in full, 0 partial (what is missing is listed).
Effects are addressed by id (`ec.<category>.<name>`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`).
@@ -255,6 +255,7 @@ Effects are addressed by id (`ec.<category>.<name>`) or display name in commands
|---|---|---:|:-:|:-:|---|
| Matte Choker | `ec.matte.mattechoker` | 7 | GPU | 32 | Implemented |
| Mocha shape | `ec.obsolete.mochashape` | 6 | | 32 | Implemented |
+| Object Matte | `ec.matte.objectmatte` | 13 | | 32 | Implemented |
| Refine Hard Matte | `ec.matte.refinehard` | 16 | GPU | 32 | Implemented |
| Refine Soft Matte | `ec.matte.refinesoft` | 18 | GPU | 32 | Implemented |
| Roto Brush & Refine Edge | `ec.matte.rotobrush` | 24 | | 32 | Implemented |
diff --git a/docs/gaps.md b/docs/gaps.md
index acb22a630..577f7bd6f 100644
--- a/docs/gaps.md
+++ b/docs/gaps.md
@@ -70,7 +70,7 @@ are grouped by area; the issue tracker has the rest.
| 11 | **Text rendering and layout** | #453 CJK text in comps, #316 paragraph alignment inverted, #483 squeezed at 4:3, #417 soft at Fit, #583 font search, #550 toolbar font options | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) |
| 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); Z position (#489); camera aperture (#260); reflections (#261); bevel options (#315); 3D issue reopened (#307) | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) |
| 13 | **3D model formats and materials** | After Effects 26 imports FBX, USD/USDZ, STL besides glTF and OBJ (`usd_plugins`), and Substance 3D materials with mesh displacement (26.0–26.2); we read glTF and OBJ | Product shots and 3D scenes come as FBX/USDZ | 25–45 h | — · [file-format-parity.md](file-format-parity.md) |
-| 14 | **Object Matte** (26.2, disk cache in 26.5) | Not present; MobileSAM is already in `effectcraft-segment` and could back it | The new one-click roto; Roto Brush covers part of it | 15–30 h | G7 · [target-app-parity.md](target-app-parity.md) |
+| 14 | **Object Matte** (26.2, disk cache in 26.5) | First slice (#783): the Object Matte effect (Effect ▸ Matte) and `objectMatte.apply` turn a point (and optional box) on a reference frame into a matte and follow the object up to 600 frames each way, or segment every frame on its own; Roto Brush's engine, cache and model (MobileSAM from Settings ▸ Roto Brush, classical graph cut without it). Missing: the click / hover tool in the viewer, a background propagation job, the disk cache, comparison with After Effects | The new one-click roto | 10–20 h | G7 · [target-app-parity.md](target-app-parity.md) |
| 15 | **Hardware video decode and encode** | No VideoToolbox, NVENC, Quick Sync or VA-API; #498 asks for it; FilmCraft has VideoToolbox / NVENC / VA-API paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) |
| 16 | **Localization depth** | Menus in 6 languages; panels in 4; effect names and ~1,650 effect parameter names untranslated; French, German, Korean, Italian, Russian (After Effects ships them) absent; #599 Hangul boxes on macOS | After Effects is used heavily in Japan, China, Korea, Brazil, Europe | 60–110 h + native review | G9 · [localization-parity.md](localization-parity.md) |
| 17 | **Media Browser lists files import refuses** | `media_browser.rs` lists TGA, DPX, HDR, AVI, MXF; `media::is_importable` has none of them; #431 any image imports as a sequence; #324 MP3 goes silent | Confusing failures on common VFX formats (DPX, TGA) | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) |
diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md
index 12d37245b..f0ed90b6e 100644
--- a/docs/target-app-parity.md
+++ b/docs/target-app-parity.md
@@ -122,7 +122,7 @@ compositing and effects dominate; audio is light). Hours are to full parity for
| Text and typography | 7% | 94% | 50% | 20–35 | CJK text in comps renders wrongly (#453); paragraph alignment inverted (#316); squeezed text at 4:3 (#483); font search (#583); soft text at Fit (#417); variable-font filter (26.3) |
| Effects and animation presets | 14% | 85% | 40% | 70–130 | All 306 effects exist, but rendered output is unmeasured against After Effects; 8 built-in presets against After Effects' 621; open effect bugs (#580 crash, #514, #526, #463, #369, #473); effect colour labels (26.5); see [effects-parity.md](effects-parity.md) |
| 3D (Classic and Advanced) | 6% | 82% | 35% | 35–65 | No FBX/USD/STL models, no Substance materials, mesh displacement unverified; no 3D transform gizmos (#392, #347); Z position (#489); reflections (#261); camera aperture (#260); bevel options (#315) |
-| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | No Object Matte (26.2); Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) |
+| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | Object Matte (26.2) is a first slice: an effect and command from a point / box prompt, no viewer tool or disk cache; Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) |
| Preview and cache | 7% | 92% | 40% | 25–50 | Playback stutters on cached frames (#595, #304); slow scrubbing (#329); late audio (#554, #504, #335); no benchmark against After Effects |
| Render queue and export | 7% | 85% | 55% | 25–45 | No hardware encoding; ProRes 4444 colour tag (#340); rendering over an existing file (#325); default macOS output location (#446); uncompressed single-file output (#372); HEVC/AV1 without B-frames |
| Audio | 2% | 85% | 45% | 6–12 | Audio sync and crackle (#335); audio to keyframes depth |
@@ -144,7 +144,7 @@ Feature depth (≈ 50%) feeds the overall *ready* number with the other dimensio
| Stability | 45% | 40–80 | Open crash reports #580, #591, #501, #426, #341, #306, #234, #511; no CI runs the tests on pull requests | [gaps.md](gaps.md#g3-stability-and-a-green-main) |
| Platforms | 65% | 30–60 | Releases for macOS, Windows x64/x86/ARM64, Linux, FreeBSD and the web (more than After Effects), but Linux and Windows users hit basic bugs | [gaps.md](gaps.md#g2-real-user-reliability-on-every-platform) |
| Ecosystem and plug-ins | 20% | 40–80 | After Effects scripts and expressions mostly carry over; no After Effects SDK plug-ins, `.ffx` presets or `.mogrt`; our WebAssembly plug-in API has no third-party plug-ins yet | [plugins.md](plugins.md) |
-| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; no Object Matte; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) |
+| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; Object Matte (point / box prompt, same model) without its viewer tool or disk cache; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) |
| Agent control (beyond After Effects) | ahead | — | Every command is drivable over MCP, the control channel and the CLI | [agents.md](agents.md) |
**Totals.** Summing the rows double-counts: AI overlaps Tracking, performance overlaps Preview,