Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

Roto Brush: save the tracked frames in the project

Claude1eee687ebc70
diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs
index 6804ea2c7..4b5e1fd42 100644
--- a/crates/effects/src/object_matte.rs
+++ b/crates/effects/src/object_matte.rs
@@ -227,7 +227,7 @@ pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec<i64> {
pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option<Arc<dyn MaskModel>>) -> Option<Chain> {
let data = roto_data(params, layer_size, f, model.is_some())?;
let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref()));
- Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model })
+ Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model, saved: None })
}
/// Frame `f`'s segmentation at full resolution, computing what is missing (the background
diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs
index ffc2be376..9596b6d50 100644
--- a/crates/effects/src/roto.rs
+++ b/crates/effects/src/roto.rs
@@ -31,7 +31,7 @@ use effectcraft_project::ParamUi;
use effectcraft_raster::Image;
use effectcraft_segment::{ClipFrame, MaskModel, TrackRequest};
use effectcraft_track::roto::refine::{self, MatteParams};
-use effectcraft_track::roto::{self as rb, RotoData, SegOpts, rle};
+use effectcraft_track::roto::{self as rb, Gmm, RotoData, SegOpts, rle};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
@@ -46,6 +46,8 @@ pub const ID: &str = "ec.matte.rotobrush";
pub const STROKES: &str = "strokes";
pub const INPUT_KEY: &str = "inputKey";
pub const FROZEN: &str = "frozen";
+/// The saved tracking (see [`TrackCache`]).
+pub const TRACKED: &str = "tracked";
/// Default segmentation span each side of the base frame (frames).
pub const DEFAULT_SPAN: i64 = 20;
@@ -95,6 +97,7 @@ pub fn specs() -> Vec<EffectSpec> {
p(STROKES, "Strokes", Value::Str(String::new()), ParamUi::Hidden),
p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden),
p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden),
+ p(TRACKED, "Tracked", Value::Str(String::new()), ParamUi::Hidden),
],
render,
gpu: false,
@@ -214,6 +217,11 @@ pub fn installed_model() -> Option<Arc<dyn MaskModel>> {
/// Seed of the chain keys: what the frames are (the engine-maintained Input Key), the layer
/// size, the frame rate, the segmentation settings and the model.
pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64) -> u64 {
+ seed_with(params, layer_size, fps, model_for(params).map_or(effectcraft_segment::CLASSICAL, |m| m.info().id))
+}
+
+/// [`seed`] with the segmenting model named by its id (saved tracking names the one that made it).
+fn seed_with(params: &Params, layer_size: [f64; 2], fps: f64, model: &str) -> u64 {
let mut h = 0xcbf2_9ce4_8422_2325;
fnv(&mut h, s(params, INPUT_KEY).as_bytes());
for v in [
@@ -227,7 +235,7 @@ pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64) -> u64 {
fnv(&mut h, &v.to_bits().to_le_bytes());
}
fnv(&mut h, &[e(params, "quality") as u8, e(params, "version") as u8]);
- fnv(&mut h, model_for(params).map_or(effectcraft_segment::CLASSICAL, |m| m.info().id).as_bytes());
+ fnv(&mut h, model.as_bytes());
h
}
@@ -388,13 +396,57 @@ pub struct Chain {
pub refine_radius: f64,
/// The trained model (Version 2.0 / 3.0 with one installed).
pub model: Option<Arc<dyn MaskModel>>,
+ /// The saved tracking, while it applies (see [`TrackCache`]).
+ pub(crate) saved: Option<SavedTracking>,
}
impl Chain {
pub fn new(params: &Params, layer_size: [f64; 2], fps: f64, scale: f64) -> Chain {
let data = data(params);
let keys = data.chain_keys(seed(params, layer_size, fps));
- Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale), model: model_for(params) }
+ let saved = saved_for(params, &data, layer_size, fps);
+ Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale), model: model_for(params), saved }
+ }
+
+ /// Frame `f`'s segmentation at `scale` from the saved tracking or the caches; computes nothing.
+ pub fn cached_seg(&self, f: i64, scale: f64) -> Option<Arc<FrameSeg>> {
+ self.saved_seg(f, scale).or_else(|| self.keys.get(&f).and_then(|k| cached(*k, scale)))
+ }
+
+ /// The tracker's trust in frame `f`: from the saved tracking, else from this session's tracking.
+ pub fn trust_at(&self, f: i64) -> Option<FrameTrust> {
+ if let Some(fr) = self.saved.as_ref().and_then(|t| t.cache.frames.get(&f)) {
+ return Some(FrameTrust { confidence: fr.c, occluded: fr.o });
+ }
+ trust(*self.keys.get(&f)?)
+ }
+
+ /// The model the saved tracking was made with, while it applies.
+ pub fn saved_model(&self) -> Option<&str> {
+ self.saved.as_ref().map(|t| t.cache.model.as_str())
+ }
+
+ /// Frame `f` of the saved tracking at `scale` (decoded once, then cached).
+ fn saved_seg(&self, f: i64, scale: f64) -> Option<Arc<FrameSeg>> {
+ let t = self.saved.as_ref()?;
+ let k = tracked_key(*t.keys.get(&f)?);
+ if let Some(s) = cached(k, scale) {
+ return Some(s);
+ }
+ let full = match cached(k, 1.0) {
+ Some(s) => s,
+ None => {
+ let s = Arc::new(decode_tracked(&t.cache, f)?);
+ store(k, 1.0, s.clone());
+ s
+ }
+ };
+ if scale == 1.0 {
+ return Some(full);
+ }
+ let s = Arc::new(downscale(&full, scale));
+ store(k, scale, s.clone());
+ Some(s)
}
/// Frame `f`'s segmentation at this chain's scale: cached, or computed from the nearest
@@ -403,9 +455,12 @@ impl Chain {
pub fn ensure(&self, f: i64, frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>, cancel: &dyn Fn() -> bool) -> Option<Arc<FrameSeg>> {
let sc = self.opts.scale;
let lookup = |g: i64| -> Option<Arc<FrameSeg>> {
+ // A frame the tracker followed wins over the classic engine's: the saved tracking, else
+ // this session's.
+ if let Some(s) = self.saved_seg(g, sc) {
+ return Some(s);
+ }
let k = *self.keys.get(&g)?;
- // A frame the tracker followed wins over the classic engine's (memory only: its
- // confidence is not persisted).
if let Some(s) = cached(tracked_key(k), sc) {
return Some(s);
}
@@ -654,6 +709,128 @@ fn frozen(params: &Params) -> Option<Arc<FrozenCache>> {
Some(d)
}
+// ------------------------------------------------------------------ saved tracking
+
+/// The tracker's frames of an instance, saved with the project (the hidden [`TRACKED`] parameter,
+/// as [`FrozenCache`] is). They apply while `key` matches the instance's inputs ([`tracks_key`]),
+/// whichever model Settings choose: a stroke or span change makes them stale, and the next Propagate
+/// replaces them (with the classic engine, that leaves none).
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+pub struct TrackCache {
+ /// What the frames were tracked from ([`tracks_key`]).
+ pub key: String,
+ /// The tracker's model id.
+ pub model: String,
+ /// The mattes' width and height (the layer's size).
+ pub size: [usize; 2],
+ pub frames: BTreeMap<i64, TrackFrame>,
+}
+
+/// One tracked frame: its matte and Refine Edge band (RLE, base64), its colour models and the
+/// tracker's trust in it.
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+#[serde(default)]
+pub struct TrackFrame {
+ /// Matte (RLE, base64).
+ pub m: String,
+ /// Refine Edge band (RLE, base64).
+ pub r: String,
+ pub fg: Gmm,
+ pub bg: Gmm,
+ /// 0–1: the tracker's confidence.
+ pub c: f32,
+ /// The object is not in the frame.
+ pub o: bool,
+}
+
+impl TrackCache {
+ pub fn from_json(s: &str) -> Option<TrackCache> {
+ if s.trim().is_empty() {
+ return None;
+ }
+ serde_json::from_str(s).ok()
+ }
+ pub fn to_json(&self) -> String {
+ serde_json::to_string(self).unwrap_or_default()
+ }
+}
+
+/// What saved tracking depends on: the chain keys of its frames (the inputs and the model).
+pub fn tracks_key(keys: &BTreeMap<i64, u64>) -> String {
+ let mut h = 0xcbf2_9ce4_8422_2325;
+ for (f, k) in keys {
+ fnv(&mut h, &f.to_le_bytes());
+ fnv(&mut h, &k.to_le_bytes());
+ }
+ format!("{h:016x}")
+}
+
+/// The saved tracking of `params`, parsed once per content.
+fn saved_tracks(params: &Params) -> Option<Arc<TrackCache>> {
+ static C: OnceLock<Mutex<HashMap<u64, Arc<TrackCache>>>> = OnceLock::new();
+ let j = s(params, TRACKED);
+ if j.is_empty() {
+ return None;
+ }
+ let mut h = 0xcbf2_9ce4_8422_2325;
+ fnv(&mut h, j.as_bytes());
+ let c = C.get_or_init(Default::default);
+ if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) {
+ return Some(d.clone());
+ }
+ let d = Arc::new(TrackCache::from_json(j)?);
+ let mut m = c.lock().unwrap_or_else(|e| e.into_inner());
+ if m.len() > 8 {
+ m.clear();
+ }
+ m.insert(h, d.clone());
+ Some(d)
+}
+
+/// A saved tracking that applies to a chain: its frames, keyed under the model that made them.
+pub(crate) struct SavedTracking {
+ cache: Arc<TrackCache>,
+ keys: BTreeMap<i64, u64>,
+}
+
+/// The saved tracking of `params` if it applies: the inputs are the instance's now. It does not depend
+/// on which model Settings choose or has loaded, so a project renders the same anywhere.
+fn saved_for(params: &Params, data: &RotoData, layer_size: [f64; 2], fps: f64) -> Option<SavedTracking> {
+ let cache = saved_tracks(params)?;
+ let keys = data.chain_keys(seed_with(params, layer_size, fps, &cache.model));
+ (tracks_key(&keys) == cache.key).then_some(SavedTracking { cache, keys })
+}
+
+/// Decode a tracked frame of saved tracking (full resolution).
+fn decode_tracked(cache: &TrackCache, f: i64) -> Option<FrameSeg> {
+ let fr = cache.frames.get(&f)?;
+ // The size comes from the project file: checked, so a corrupt one is ignored rather than overflowing.
+ let [w, h] = cache.size;
+ let n = w.checked_mul(h)?;
+ let matte = rle::decode(&fr.m)?;
+ let refine = rle::decode(&fr.r)?;
+ (matte.len() == n && refine.len() == n).then(|| FrameSeg { w, h, matte, refine, fg: fr.fg.clone(), bg: fr.bg.clone() })
+}
+
+/// The tracking a chain has just made, to save with its instance: every frame of its span that the
+/// tracker followed, at full resolution, with its trust. `None` when it followed none.
+pub fn tracks_of(ch: &Chain) -> Option<TrackCache> {
+ let model = ch.model.as_ref()?.info().id;
+ let mut size = None;
+ let mut frames = BTreeMap::new();
+ for (f, k) in &ch.keys {
+ let (Some(seg), Some(t)) = (cached(tracked_key(*k), 1.0), trust(*k)) else { continue };
+ size.get_or_insert([seg.w, seg.h]);
+ frames.insert(
+ *f,
+ TrackFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine), fg: seg.fg.clone(), bg: seg.bg.clone(), c: t.confidence, o: t.occluded },
+ );
+ }
+ let cache = TrackCache { key: tracks_key(&ch.keys), model: model.to_string(), size: size?, frames };
+ (!cache.frames.is_empty()).then_some(cache)
+}
+
/// Encode one frame for [`FrozenCache`].
pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame {
let a: Vec<u8> = alpha.iter().map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8).collect();
diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rs
index 11f856d5f..bad9fd5ff 100644
--- a/crates/engine/src/analysis/models.rs
+++ b/crates/engine/src/analysis/models.rs
@@ -240,6 +240,18 @@ impl Session {
}
}
+ /// What `roto.status` says when the saved tracking applies but Settings choose another model (`None`:
+ /// they agree, or nothing is saved).
+ pub fn saved_tracking_notice(&self, saved: Option<&str>) -> Option<String> {
+ let saved = saved?;
+ let chosen = self.chosen_model(Task::Mask);
+ if saved == chosen {
+ return None;
+ }
+ let name = |id: &str| seg::info(id).map_or(ROTO.classic_name, |m| m.name);
+ Some(format!("The saved tracking was made with {}; {} is chosen: Propagate again to replace it", name(saved), name(chosen)))
+ }
+
/// Make sure the Roto Brush model Settings chose is in use before a job that segments with it
/// starts, so the job never runs on the classic engine without saying so. With `wait`, a load
/// under way is waited for and one not started is loaded here; without it, a model that is not
diff --git a/crates/engine/src/analysis/roto.rs b/crates/engine/src/analysis/roto.rs
index 357e77b13..286b95095 100644
--- a/crates/engine/src/analysis/roto.rs
+++ b/crates/engine/src/analysis/roto.rs
@@ -22,7 +22,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use effectcraft_effects::object_matte as om;
-use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY};
+use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY, TRACKED};
use effectcraft_effects::{Params, flatten_params};
use effectcraft_keyframe::Value;
use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid};
@@ -135,6 +135,8 @@ pub struct RotoShared {
pub cancel: AtomicBool,
pub state: Mutex<RotoProgress>,
frozen: Mutex<Option<FrozenCache>>,
+ /// The tracker's frames, as the saved tracking (JSON) to store with the instance.
+ saved: Mutex<Option<String>>,
}
pub struct RotoJob {
@@ -253,6 +255,7 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI
set_str(g, INPUT_KEY, key);
if had {
set_str(g, FROZEN, String::new());
+ set_str(g, TRACKED, String::new());
}
}
}
@@ -459,11 +462,19 @@ async fn run_work(w: Work, shared: &RotoShared) {
s.finished = true;
return;
}
+ // The frames it followed replace the saved tracking (see `poll_roto_job`): none if it failed.
+ *lock(&shared.saved) = Some(match &result {
+ Ok(_) => fx::tracks_of(&chain).map(|t| t.to_json()).unwrap_or_default(),
+ Err(_) => String::new(),
+ });
let mut s = lock(&shared.state);
match result {
Ok(summary) => s.tracked = summary,
Err(e) => s.tracker_note = Some(format!("tracker unavailable, classic engine used: {e}")),
}
+ } else if w.task == RotoTask::Propagate && !w.object_matte {
+ // With the classic engine, Propagate leaves no saved tracking behind.
+ *lock(&shared.saved) = Some(String::new());
}
for (k, f) in w.frames.iter().enumerate() {
if cancel() {
@@ -738,6 +749,17 @@ impl Session {
Ok(n)
}
+ /// Save the tracker's frames in the instance, unless its strokes changed while it tracked. No undo
+ /// step: the frames are a cache of the strokes and the span (a stale copy is ignored).
+ fn save_tracks(&mut self, cid: ItemId, lid: LayerId, uid: Uid, strokes: &str, json: String) {
+ let p = Arc::make_mut(&mut self.project);
+ if let Some(g) = p.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid))
+ && guard_of(g) == strokes
+ {
+ set_str(g, TRACKED, json);
+ }
+ }
+
/// Finish a done job: Freeze writes the frozen mattes (one undo step).
fn poll_roto_job(&mut self) -> bool {
let Some(job) = &self.roto_job else { return false };
@@ -764,6 +786,10 @@ impl Session {
RotoTask::Propagate => {
let verb = if job.object_matte { "tracked" } else { "propagated" };
self.events.push(Event::Toast { message: format!("{name}: {verb} {} frame(s) in {:.1} s", st.done, st.elapsed), error: false });
+ // The frames the tracker followed are saved with the instance.
+ if let Some(json) = lock(&job.shared.saved).take() {
+ self.save_tracks(job.comp, job.layer, job.effect, &job.strokes, json);
+ }
// Rendered frames depend on the cache: redraw.
self.bump();
}
diff --git a/crates/engine/src/cmd/roto_cmds.rs b/crates/engine/src/cmd/roto_cmds.rs
index c7f190fde..0931236a2 100644
--- a/crates/engine/src/cmd/roto_cmds.rs
+++ b/crates/engine/src/cmd/roto_cmds.rs
@@ -433,7 +433,6 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid)));
// The model Settings chose and whether this layer segments with it (propagate refuses or waits otherwise).
out["model"] = s.roto_model();
- out["notice"] = json!(s.roto_model_notice(s.uses_roto_model(cid, lid, uid)));
out["limits"] = json!(rt::frame_limits(&comp, &layer));
let frame = frame_p(s, p, cid, &layer);
out["frame"] = json!(frame);
@@ -444,7 +443,7 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
let fps = comp.frame_rate.as_f64();
let ps = params_static(&g);
let chain = fx::Chain::new(&ps, size, fps, 1.0);
- let seg = match chain.keys.get(&frame).and_then(|k| fx::cached(*k, 1.0)) {
+ let seg = match chain.cached_seg(frame, 1.0) {
Some(sg) => Some(sg),
None if compute => {
let index = layer.effects().and_then(|f| f.groups().position(|x| x.uid == uid)).unwrap_or(0);
@@ -474,10 +473,12 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
let chain = fx::Chain::new(&params_static(&g), size, comp.frame_rate.as_f64(), 1.0);
let tracked: serde_json::Map<String, Value> = chain
.keys
- .iter()
- .filter_map(|(f, k)| fx::trust(*k).map(|t| (f.to_string(), json!({"confidence": t.confidence, "occluded": t.occluded}))))
+ .keys()
+ .filter_map(|f| chain.trust_at(*f).map(|t| (f.to_string(), json!({"confidence": t.confidence, "occluded": t.occluded}))))
.collect();
- out["tracker"] = json!({"model": s.models_roto_active(), "frames": tracked});
+ out["tracker"] = json!({"model": s.models_roto_active(), "saved": chain.saved_model(), "frames": tracked});
+ let notice = s.roto_model_notice(s.uses_roto_model(cid, lid, uid)).or_else(|| s.saved_tracking_notice(chain.saved_model()));
+ out["notice"] = json!(notice);
out["source"] = json!(if tracked.contains_key(&frame.to_string()) {
"tracker"
} else if chain.model.is_some() {
diff --git a/crates/engine/src/tests/roto.rs b/crates/engine/src/tests/roto.rs
index 9ef6a848f..b18cd386d 100644
--- a/crates/engine/src/tests/roto.rs
+++ b/crates/engine/src/tests/roto.rs
@@ -700,9 +700,14 @@ fn sam2_tracks_the_clip_through_roto_propagate() {
assert_eq!(frames.len(), 21);
assert!(frames.values().all(|t| t["occluded"] == false && t["confidence"].as_f64().unwrap() > 0.3), "{frames:?}");
assert_eq!(st["tracker"]["model"], "sam2.1-hiera-tiny");
- // Back to the classic engine: same strokes, its own segmentation, no tracker data.
+ // Settings choose the classic engine: the saved tracking stays until a Propagate replaces it.
s.execute_checked("roto.model.select", json!({"id": "classical", "wait": true})).unwrap();
let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+ assert_eq!(st["source"], "tracker", "{st}");
+ assert!(st["notice"].as_str().unwrap().contains("Propagate again"), "{st}");
+ // A classic Propagate: its own segmentation, no tracker data.
+ s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap();
+ let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
assert_eq!(st["source"], "classical", "{st}");
assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 0);
let _ = std::fs::remove_dir_all(dir);
@@ -751,6 +756,116 @@ fn sam2_selected_without_wait_is_loaded_before_propagate() {
let _ = std::fs::remove_dir_all(dir);
}
+/// A tracker that follows the disc exactly: its true mask on every frame of the request. A stand-in
+/// for SAM 2, so the saved tracking can be tested without the weights.
+struct Exact;
+
+impl effectcraft_segment::MaskModel for Exact {
+ fn info(&self) -> &'static effectcraft_segment::ModelInfo {
+ &effectcraft_segment::SAM2_TINY
+ }
+ fn segment(&self, _: &[[f32; 3]], _: usize, _: usize, _: &effectcraft_segment::Prompt) -> effectcraft_segment::Result<Vec<f32>> {
+ Err("no weights".into())
+ }
+ fn tracker(&self) -> Option<&dyn effectcraft_segment::ObjectTracker> {
+ Some(self)
+ }
+}
+
+impl effectcraft_segment::ObjectTracker for Exact {
+ fn track(
+ &self,
+ req: &effectcraft_segment::TrackRequest,
+ frame: &mut dyn FnMut(i64) -> Option<std::sync::Arc<effectcraft_segment::ClipFrame>>,
+ _: &dyn Fn() -> bool,
+ out: &mut dyn FnMut(i64, effectcraft_segment::TrackedFrame),
+ ) -> effectcraft_segment::Result<()> {
+ for g in req.first..req.first + req.count as i64 {
+ let Some(img) = frame(g) else { return Err(format!("no frame {g}")) };
+ let c = center(g as u32);
+ let prob = (0..img.w * img.h)
+ .map(|i| ((((i % img.w) as f64 + 0.5 - c[0]).hypot((i / img.w) as f64 + 0.5 - c[1])) <= R) as u8 as f32)
+ .collect::<Vec<f32>>();
+ out(g, effectcraft_segment::TrackedFrame { prob, w: img.w, h: img.h, confidence: 0.99, occluded: false });
+ }
+ Ok(())
+ }
+}
+
+/// Saved tracking is part of the project: written with the instance, it applies when the project
+/// opens, with or without the model loaded, and Settings choosing the classic engine leaves it in place
+/// until a Propagate replaces it.
+#[test]
+fn saved_tracking_is_kept_in_the_project_and_applies_when_it_reopens() {
+ let sam2 = "sam2.1-hiera-tiny";
+ let (s, clip, uid, path) = saved_project("saved");
+ // Reopened with SAM 2 chosen but not loaded (none is installed here): the saved frames apply.
+ forget(&s, clip, uid);
+ let mut s2 = Session::default();
+ s2.execute_checked("file.open", json!({"path": path})).unwrap();
+ s2.execute_checked("roto.model.select", json!({"id": sam2})).unwrap();
+ let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+ assert_eq!(st["source"], "tracker", "{st}");
+ assert_eq!(st["tracker"]["saved"], sam2, "{st}");
+ assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 21, "{st}");
+ let j = iou_at(&mut s2, clip, 12, center(12));
+ assert!(j > 0.9, "IoU of a saved frame: {j}");
+ // The render path (Chain::ensure) reads the same frame.
+ let comp = s2.active_comp().unwrap();
+ let l = comp.layer(clip).unwrap();
+ let g = l.props.find_group(uid).unwrap();
+ let size = crate::roto::layer_size(&s2.project, comp, l);
+ let ch = effectcraft_effects::roto::Chain::new(&crate::roto::params_static(g), size, comp.frame_rate.as_f64(), 1.0);
+ let seg = ch.ensure(12, &mut |_| None, &|| false).expect("the render path reads the saved frame");
+ let truth = rle::decode(&gt(center(12))).unwrap();
+ let inter = seg.matte.iter().zip(&truth).filter(|(a, b)| **a != 0 && **b != 0).count();
+ let uni = seg.matte.iter().zip(&truth).filter(|(a, b)| **a != 0 || **b != 0).count();
+ assert!(inter as f64 / uni as f64 > 0.9, "the render path's frame: IoU {}", inter as f64 / uni as f64);
+ // Settings choose the classic engine: the saved frames still apply, and the notice says so.
+ s2.execute_checked("roto.model.select", json!({"id": "classical"})).unwrap();
+ let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+ assert_eq!(st["source"], "tracker", "{st}");
+ assert!(st["notice"].as_str().unwrap().contains("Propagate again"), "{st}");
+ // A classic Propagate replaces the saved tracking with nothing.
+ s2.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap();
+ let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+ assert_eq!(st["source"], "classical", "{st}");
+ assert!(st["tracker"]["saved"].is_null() && st["notice"].is_null(), "{st}");
+ let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap());
+}
+
+/// A tracked clip saved to a project file: the session that tracked it, its layer and instance, and the
+/// file.
+fn saved_project(name: &str) -> (Session, LayerId, u64, String) {
+ let (mut s, clip, uid) = painted(center, &format!("disk-{name}.mov"));
+ effectcraft_effects::roto::set_model(Some(std::sync::Arc::new(Exact)));
+ let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true}));
+ effectcraft_effects::roto::set_model(None);
+ assert_eq!(r.unwrap()["frames"], 21);
+ let dir = std::env::temp_dir().join(format!("ec-roto-{name}-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("tracked.ecproj").to_string_lossy().to_string();
+ s.execute_checked("file.saveAs", json!({"path": path})).unwrap();
+ let saved = std::fs::read_to_string(&path).unwrap();
+ assert!(saved.contains("sam2.1-hiera-tiny"), "the project file carries the tracking and the model that made it");
+ (s, clip, uid, path)
+}
+
+/// A stroke change makes the saved tracking stale: the instance is classical until the next Propagate.
+#[test]
+fn a_stroke_change_makes_the_saved_tracking_stale() {
+ let (_s, clip, _uid, path) = saved_project("stale");
+ let mut s2 = Session::default();
+ s2.execute_checked("file.open", json!({"path": path})).unwrap();
+ s2.execute_checked("roto.model.select", json!({"id": "sam2.1-hiera-tiny"})).unwrap();
+ assert_eq!(s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap()["source"], "tracker");
+ s2.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "bg", "points": line([10.0, 10.0], [60.0, 12.0]), "radius": 4})).unwrap();
+ let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+ assert_eq!(st["source"], "classical", "{st}");
+ assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 0, "{st}");
+ let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap());
+}
+
/// A model whose tracker always fails: Roto Brush must carry on with the classic engine and say so.
#[test]
fn a_failing_tracker_falls_back_to_the_classic_engine() {
diff --git a/docs/agents.md b/docs/agents.md
index 54ba21afa..ddd730557 100644
--- a/docs/agents.md
+++ b/docs/agents.md
@@ -511,6 +511,9 @@ Trained model (Roto Brush 2.0 / 3.0):
- `roto.propagate` never falls back to the classic engine silently: a chosen model that is not installed, or failed
to load, is an error, and without `"wait": true` so is a model still loading. `roto.status` reports `model`
(`selected`, `installed`, `active`, `loading`, `error`) and a `notice` while the chosen model is not the one in use.
+ Tracked frames are saved in the project with the instance (`tracker.saved` names their model): they apply while the
+ strokes and span are unchanged, whichever model Settings choose (`notice` says when that differs), and a new
+ Propagate replaces them.
- `roto.model.remove {id}` deletes the installed weights.
- The `version` property (`effects/#1/version`: 0 = 1.0, 1 = 2.0, 2 = 3.0) picks between the
classic engine (1.0) and the chosen model (2.0, 3.0).
diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md
index f5946a98e..d5216e6d4 100644
--- a/docs/roto-brush-sam2.md
+++ b/docs/roto-brush-sam2.md
@@ -16,15 +16,18 @@ Apache-2.0). It is optional: nothing changes until you install the model and cho
`classical`) and, for tracked frames, `tracker.frames[f]`: `confidence` (0–1, the model's object score times its
predicted mask quality) and `occluded` (the object is not in the frame: an empty matte). `model` says what Settings
chose and whether it is installed, loaded and loading; `notice` says when it is not the one in use for the layer.
+ `tracker.saved` names the model the saved frames came from.
Propagate never falls back to the classic engine without saying so. With SAM 2 chosen but not installed, or failing to
load, it refuses with the reason. While it is still loading, `wait: true` (agents) loads it first; without `wait` it
refuses, and the load goes on in the background. Deselecting SAM 2 (choosing Classic) runs Propagate exactly as before,
and Roto Brush 1.0 never uses a model.
-Tracked frames live in the session that ran Propagate and are not saved in the project file. A separate render process
-(`effectcraft-cli render-frame`) does not have them, so render in the same session: in the app, or in one `run` that
-includes `roto.propagate` and then `comp.saveFrameAs`.
+Tracked frames are saved in the project with the instance, as Freeze's are, so a saved project renders the same when it
+reopens, in the app or with `effectcraft-cli`, without the model being loaded or chosen there. They apply while the
+strokes and span they came from are unchanged. Changing a stroke or the span makes them stale until the next Propagate.
+Choosing another model, or the classic engine, leaves them in place and `roto.status` says so; a classic Propagate
+replaces them with classic mattes. Object Matte saves nothing.
## How it works
`crates/sam2` is a Rust port (on [candle](https://github.com/huggingface/candle), native only) of the Hugging Face