EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
Roto Brush: discard failed tracks and bound SAM2 caches (#1539)
Kristian Myram365abf0f2d39
diff --git a/apps/effectcraft-cli/tests/it/cli.rs b/apps/effectcraft-cli/tests/it/cli.rsindex ed6d98086..8bdfb3c00 100644--- a/apps/effectcraft-cli/tests/it/cli.rs+++ b/apps/effectcraft-cli/tests/it/cli.rs@@ -683,6 +683,7 @@ fn render_bitrate_applies_to_vp9_webm() {#[test]fn bench_roto_runs_on_a_users_clip() {let (frames, masks) = (tmp("clip-frames"), tmp("clip-masks"));+ let config = frames.join("config");for d in [&frames, &masks] {let _ = std::fs::remove_dir_all(d);std::fs::create_dir_all(d).unwrap();@@ -703,7 +704,7 @@ fn bench_roto_runs_on_a_users_clip() {mask.save(masks.join(format!("f{f:03}.png"))).unwrap();}let (f, m) = (frames.to_str().unwrap(), masks.to_str().unwrap());- let out = bin().args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6"]).output().unwrap();+ let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6"]).output().unwrap();let text = String::from_utf8_lossy(&out.stdout).to_string();assert!(out.status.success(), "{text}\n{}", String::from_utf8_lossy(&out.stderr));let row = text.lines().find(|l| l.contains("classical")).unwrap_or_else(|| panic!("no classical row: {text}"));@@ -711,11 +712,11 @@ fn bench_roto_runs_on_a_users_clip() {assert!(mean > 0.8, "classical mean IoU on the user's clip: {mean}\n{text}");// one mask short: refusedstd::fs::remove_file(masks.join("f005.png")).unwrap();- let out = bin().args(["bench", "--roto", "--clip", f, "--truth", m]).output().unwrap();+ let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f, "--truth", m]).output().unwrap();assert!(!out.status.success());assert!(String::from_utf8_lossy(&out.stderr).contains("must match"), "{}", String::from_utf8_lossy(&out.stderr));// --clip without --truth- let out = bin().args(["bench", "--roto", "--clip", f]).output().unwrap();+ let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f]).output().unwrap();assert!(!out.status.success());}@@ -730,11 +731,11 @@ fn bench_roto_reports_why_a_named_model_file_failed() {std::fs::write(&bogus, b"not the weights").unwrap();for flag in ["--sam2", "--mobilesam"] {let missing = dir.join("missing.bin");- let out = bin().args(["bench", "--roto", flag, missing.to_str().unwrap()]).output().unwrap();+ let out = bin().env("EFFECTCRAFT_CONFIG_DIR", dir.join("config")).args(["bench", "--roto", flag, missing.to_str().unwrap()]).output().unwrap();let err = String::from_utf8_lossy(&out.stderr).to_string();assert!(!out.status.success(), "{flag}: {err}");assert!(err.contains(flag) && err.contains("missing.bin"), "{flag} names the file that could not be read: {err}");- let out = bin().args(["bench", "--roto", flag, bogus.to_str().unwrap()]).output().unwrap();+ let out = bin().env("EFFECTCRAFT_CONFIG_DIR", dir.join("config")).args(["bench", "--roto", flag, bogus.to_str().unwrap()]).output().unwrap();let err = String::from_utf8_lossy(&out.stderr).to_string();assert!(!out.status.success(), "{flag}: {err}");assert!(err.contains("does not match"), "{flag} says the file is not the published weights: {err}");diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rsindex 8046c4773..e66400416 100644--- a/crates/effects/src/roto.rs+++ b/crates/effects/src/roto.rs@@ -624,6 +624,20 @@ impl Chain {}}+ /// Discard a failed track completely before falling back to the classic engine.+ pub fn forget_tracking(&mut self) {+ let mut objects = self.data.objects();+ if let Some(saved) = &self.saved {+ objects.extend(saved.cache.objects.keys().copied());+ objects.sort_unstable();+ objects.dedup();+ }+ let keys = self.keys.values().chain(self.saved.iter().flat_map(|saved| saved.keys.values())).copied();+ forget(keys.flat_map(|key| std::iter::once(key).chain(objects.iter().copied().map(move |object| object_key(key, object)))));+ self.saved = None;+ self.model = None;+ }+/// Follow the object through the span with the model's tracker, from the strokes on their/// frames (only the frames without a tracked result yet: after a correction stroke, the frames it/// changes, tracked on from the nearest prompted frame), and cache every frame's segmentation (under the tracker's keys, with its@@ -1471,6 +1485,24 @@ mod tests {let seg = chain.object_seg(1, 1).expect("the saved object matte");assert_eq!(seg.matte.len(), 100 * 80);assert!(cached(object_key(other[&1], 1), 1.0).is_none(), "the other model's cache holds none of it");+ for key in [keys[&1], other[&1]] {+ for scale in [1.0, 0.5] {+ store(tracked_key(key), scale, seg.clone());+ store(object_key(key, 1), scale, seg.clone());+ }+ trust_cache().lock().unwrap().insert(tracked_key(key), FrameTrust { confidence: 0.9, occluded: false });+ }+ chain.forget_tracking();+ assert!(chain.model.is_none());+ assert!(!chain.saved_has(1));+ for key in [keys[&1], other[&1]] {+ for scale in [1.0, 0.5] {+ assert!(cached(tracked_key(key), scale).is_none());+ assert!(cached(object_key(key, 1), scale).is_none());+ }+ assert!(trust(key).is_none());+ }+ assert!(chain.ensure(1, &mut |_| None, &|| false).is_none(), "neither saved nor in-memory tracking survives fallback");}#[test]diff --git a/crates/engine/src/analysis/roto.rs b/crates/engine/src/analysis/roto.rsindex 03b287d6e..67b0afc68 100644--- a/crates/engine/src/analysis/roto.rs+++ b/crates/engine/src/analysis/roto.rs@@ -477,6 +477,9 @@ async fn run_work(w: Work, shared: &RotoShared) {Ok(_) => fx::tracks_of(&chain).map(|t| t.to_json()).unwrap_or_default(),Err(_) => String::new(),});+ if result.is_err() {+ chain.forget_tracking();+ }let mut s = lock(&shared.state);match result {Ok(summary) => s.tracked = summary,diff --git a/crates/engine/src/tests/roto.rs b/crates/engine/src/tests/roto.rsindex afc3adad9..e686962e1 100644--- a/crates/engine/src/tests/roto.rs+++ b/crates/engine/src/tests/roto.rs@@ -937,9 +937,9 @@ fn a_stroke_change_makes_the_saved_tracking_stale() {let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap());}-/// A model whose tracker always fails: Roto Brush must carry on with the classic engine and say so.+/// A tracker that emits a completed frame before failing must leave only classic results behind.#[test]-fn a_failing_tracker_falls_back_to_the_classic_engine() {+fn a_partially_failing_tracker_falls_back_to_the_classic_engine() {use effectcraft_segment::{ClipFrame, MaskModel, ModelInfo, ObjectTracker, Prompt, TrackRequest, TrackedFrame};use std::sync::Arc;struct Broken;@@ -957,29 +957,51 @@ fn a_failing_tracker_falls_back_to_the_classic_engine() {impl ObjectTracker for Broken {fn track(&self,- _: &TrackRequest,- _: &mut dyn FnMut(i64) -> Option<Arc<ClipFrame>>,+ req: &TrackRequest,+ frame: &mut dyn FnMut(i64) -> Option<Arc<ClipFrame>>,_: &dyn Fn() -> bool,- _: &mut dyn FnMut(usize, i64, TrackedFrame),+ out: &mut dyn FnMut(usize, i64, TrackedFrame),) -> effectcraft_segment::Result<()> {- Err("the tracker broke".into())+ let img = frame(req.first).ok_or("missing test frame")?;+ for object in 0..req.objects.len() {+ out(object, req.first, TrackedFrame { prob: vec![1.0; img.w * img.h], w: img.w, h: img.h, confidence: 0.9, occluded: false });+ }+ Err("the tracker broke after emitting a frame".into())}}hold_roto_cache_test_lock();- let (mut s, clip, _) = painted(center, "disk-broken-tracker.mov");+ let (mut s, clip, uid) = painted(center, "disk-partially-broken-tracker.mov");+ s.execute_checked("roto.stroke", json!({"layer": clip.0, "object": 1, "points": [center(0)], "radius": 4})).unwrap();effectcraft_effects::roto::set_model(Some(Arc::new(Broken)));let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true}));+ let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 0}));+ let comp = s.active_comp().unwrap();+ let layer = comp.layer(clip).unwrap();+ let params = crate::roto::params_static(layer.props.find_group(uid).unwrap());+ let size = crate::roto::layer_size(&s.project, comp, layer);+ let mut chain = effectcraft_effects::roto::Chain::new(¶ms, size, comp.frame_rate.as_f64(), 1.0);+ let index = layer.effects().unwrap().groups().position(|g| g.uid == uid).unwrap();+ let mut source =+ crate::roto::FrameSource::new(&s.project, s.footage.as_ref(), s.expr.as_deref(), &s.layer_cache, s.active_comp_id().unwrap(), comp, layer, index, size);+ let mut get = |f: i64| source.get(f);+ let actual: Option<Vec<_>> = (0..=20).map(|f| chain.ensure(f, &mut get, &|| false)).collect();+ let retained_object = chain.object_seg(0, 0).is_some() || chain.object_seg(0, 1).is_some();+ let retained_trust = chain.trust_at(0);+ // Build the reference only after capturing the fallback, with no model or cached union.+ effectcraft_effects::roto::forget(chain.keys.values().copied());+ chain.model = None;+ let expected: Option<Vec<_>> = (0..=20).map(|f| chain.ensure(f, &mut get, &|| false)).collect();effectcraft_effects::roto::set_model(None);let r = r.unwrap();assert_eq!(r["frames"], 21, "{r}");- let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 10})).unwrap();- assert_eq!(st["source"], "classical", "{st}");+ let st = st.unwrap();+ assert_ne!(st["source"], "tracker", "{st}");assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 0);- let mut worst: f64 = 1.0;- for f in 0..=20 {- worst = worst.min(iou_at(&mut s, clip, f, center(f)));+ assert!(!retained_object, "partial per-object mattes must be discarded");+ assert_eq!(retained_trust, None, "partial tracker confidence must be discarded");+ for (frame, (actual, expected)) in actual.unwrap().into_iter().zip(expected.unwrap()).enumerate() {+ assert!(actual == expected, "fallback frame {frame} differs from the classic engine");}- assert!(worst > 0.9, "the classic engine should have carried the clip: {worst}");}/// Without a tracking model the Roto Brush chain is exactly the classic engine: every framediff --git a/crates/sam2/src/mask.rs b/crates/sam2/src/mask.rsindex 911ff2e00..382f9b514 100644--- a/crates/sam2/src/mask.rs+++ b/crates/sam2/src/mask.rs@@ -19,10 +19,7 @@ const MAX_PIXELS: usize = 100_000_000;/// Resize `w x h` RGB (0..1) to the model input and normalise it: `[1, 3, 1024, 1024]`.pub fn preprocess(rgb: &[[f32; 3]], w: usize, h: usize) -> Result<Tensor> {- let pixels = w.checked_mul(h).filter(|n| *n > 0 && *n == rgb.len() && w <= MAX_SIDE && h <= MAX_SIDE && *n <= MAX_PIXELS);- let Some(pixels) = pixels else {- return Err(Error::Input(format!("a {w} x {h} frame needs {} pixels (at most {MAX_PIXELS}), got {}", w.saturating_mul(h), rgb.len())));- };+ let pixels = validate_frame_size(w, h, rgb.len())?;const MEAN: [f32; 3] = [0.485, 0.456, 0.406];const STD: [f32; 3] = [0.229, 0.224, 0.225];let mut planar = vec![0f32; 3 * pixels];@@ -40,6 +37,12 @@ pub fn preprocess(rgb: &[[f32; 3]], w: usize, h: usize) -> Result<Tensor> {Ok(x.broadcast_sub(&mean)?.broadcast_div(&std)?)}+pub(crate) fn validate_frame_size(w: usize, h: usize, len: usize) -> Result<usize> {+ w.checked_mul(h)+ .filter(|n| *n > 0 && *n == len && w <= MAX_SIDE && h <= MAX_SIDE && *n <= MAX_PIXELS)+ .ok_or_else(|| Error::Input(format!("a {w} x {h} frame needs {} pixels (at most {MAX_PIXELS}), got {len}", w.saturating_mul(h))))+}+fn fingerprint(rgb: &[[f32; 3]], w: usize, h: usize) -> u64 {let mut hash = 0xcbf2_9ce4_8422_2325u64 ^ ((w as u64) << 32 | h as u64);for px in rgb {diff --git a/crates/sam2/src/neck.rs b/crates/sam2/src/neck.rsindex c897e9818..3a80ede9e 100644--- a/crates/sam2/src/neck.rs+++ b/crates/sam2/src/neck.rs@@ -13,23 +13,47 @@ use crate::weights::Weights;use crate::{Error, Result};/// Sine position encoding `[1, 2 * num_pos_features, h, w]`, normalised (`Sam2SinePositionEmbedding`).-/// It depends only on the sizes, so each is built once (it was 150 ms of a 2.7 s frame).+/// Only small tables are retained globally; the neck owns its larger feature-pyramid tables.pub fn sine_position(num_pos_features: usize, h: usize, w: usize) -> Result<Tensor> {- static CACHE: OnceLock<Mutex<HashMap<(usize, usize, usize), Tensor>>> = OnceLock::new();- let cache = CACHE.get_or_init(Default::default);- if let Some(t) = cache.lock().unwrap_or_else(PoisonError::into_inner).get(&(num_pos_features, h, w)) {- return Ok(t.clone());+ static CACHE: OnceLock<PositionCache> = OnceLock::new();+ CACHE.get_or_init(|| PositionCache::new(1024 * 1024 / size_of::<f32>())).get(num_pos_features, h, w)+}++const MAX_POSITION_ELEMENTS: usize = 128 * 1024 * 1024 / size_of::<f32>();++struct PositionCache {+ entries: Mutex<HashMap<(usize, usize, usize), Tensor>>,+ max_elements: usize,+}++impl PositionCache {+ fn new(max_elements: usize) -> Self {+ Self { entries: Mutex::new(HashMap::new()), max_elements }}- let t = build_sine_position(num_pos_features, h, w)?;- let mut map = cache.lock().unwrap_or_else(PoisonError::into_inner);- if map.len() >= 16 {- map.clear();++ fn get(&self, features: usize, h: usize, w: usize) -> Result<Tensor> {+ let key = (features, h, w);+ if let Some(t) = self.entries.lock().unwrap_or_else(PoisonError::into_inner).get(&key) {+ return Ok(t.clone());+ }+ let t = build_sine_position(features, h, w)?;+ let count = t.elem_count();+ if count <= self.max_elements {+ let mut map = self.entries.lock().unwrap_or_else(PoisonError::into_inner);+ let retained = map.values().fold(0usize, |n, t| n.saturating_add(t.elem_count()));+ if map.len() >= 16 || retained.saturating_add(count) > self.max_elements {+ map.clear();+ }+ map.insert(key, t.clone());+ }+ Ok(t)}- map.insert((num_pos_features, h, w), t.clone());- Ok(t)}fn build_sine_position(num_pos_features: usize, h: usize, w: usize) -> Result<Tensor> {+ let count = num_pos_features.checked_mul(2).and_then(|n| n.checked_mul(h)).and_then(|n| n.checked_mul(w));+ let count =+ count.filter(|n| *n > 0 && *n <= MAX_POSITION_ELEMENTS).ok_or_else(|| Error::Input("sine position dimensions are empty or exceed 128 MiB".into()))?;let eps = 1e-6f32;let scale = 2.0 * std::f32::consts::PI;let dim_t: Vec<f32> = (0..num_pos_features).map(|i| 10000f32.powf(2.0 * (i / 2) as f32 / num_pos_features as f32)).collect();@@ -39,7 +63,7 @@ fn build_sine_position(num_pos_features: usize, h: usize, w: usize) -> Result<Tedim_t.iter().enumerate().map(|(i, d)| if i % 2 == 0 { (e / d).sin() } else { (e / d).cos() }).collect()};let (ys, xs): (Vec<_>, Vec<_>) = ((0..h).map(|k| axis(h, k)).collect(), (0..w).map(|k| axis(w, k)).collect());- let mut out = vec![0f32; 2 * num_pos_features * h * w];+ let mut out = vec![0f32; count];for y in 0..h {for x in 0..w {for f in 0..num_pos_features {@@ -75,6 +99,7 @@ pub struct Neck {convs: Vec<Conv1x1>,top_down: Vec<usize>,pos_features: usize,+ positions: PositionCache,}/// What the neck produces, both in the neck's own order (lowest resolution first, all four@@ -97,7 +122,7 @@ impl Neck {})}).collect::<Result<Vec<_>>>()?;- Ok(Neck { convs, top_down: top_down.to_vec(), pos_features: fpn / 2 })+ Ok(Neck { convs, top_down: top_down.to_vec(), pos_features: fpn / 2, positions: PositionCache::new(MAX_POSITION_ELEMENTS) })}/// `stages`: Hiera stage outputs, NHWC, highest resolution first.@@ -119,7 +144,7 @@ impl Neck {}_ => lateral,};- positions.push(sine_position(self.pos_features, cur.dim(2)?, cur.dim(3)?)?);+ positions.push(self.positions.get(self.pos_features, cur.dim(2)?, cur.dim(3)?)?);features.push(cur.clone());prev = Some(cur);}@@ -160,3 +185,31 @@ impl VisionEncoder {Ok(ImageFeatures { features: keep(out.features), positions: keep(out.positions) })}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn position_tables_reject_empty_overflowing_and_oversized_dimensions() {+ for (features, h, w) in [(0, 1, 1), (1, 0, 1), (1, 1, 0), (usize::MAX, 2, 2), (128, usize::MAX, 2), (128, 1024, 1024)] {+ assert!(matches!(sine_position(features, h, w), Err(Error::Input(_))), "{features}, {h}, {w}");+ }+ }++ #[test]+ fn position_cache_bounds_retained_elements_and_preserves_the_encoding() {+ let cache = PositionCache::new(64);+ let first = cache.get(2, 2, 4).unwrap();+ assert_eq!(first.dims(), &[1, 4, 2, 4]);+ assert_eq!(first.flatten_all().unwrap().to_vec1::<f32>().unwrap(), cache.get(2, 2, 4).unwrap().flatten_all().unwrap().to_vec1::<f32>().unwrap());+ cache.get(2, 3, 4).unwrap();+ let entries = cache.entries.lock().unwrap();+ assert_eq!(entries.len(), 1, "older tables are evicted to fit the budget");+ assert!(entries.values().map(Tensor::elem_count).sum::<usize>() <= 64);+ drop(entries);+ let large = cache.get(2, 8, 8).unwrap();+ assert_eq!(large.dims(), &[1, 4, 8, 8]);+ assert_eq!(cache.entries.lock().unwrap().len(), 1, "a table over budget is returned without retention");+ }+}diff --git a/crates/sam2/src/resize.rs b/crates/sam2/src/resize.rsindex 67e96e9af..fdbcbab48 100644--- a/crates/sam2/src/resize.rs+++ b/crates/sam2/src/resize.rs@@ -5,8 +5,9 @@use candle_core::{Device, Tensor};+use crate::Result;+use crate::mask::validate_frame_size;use crate::prompt::INPUT_SIZE;-use crate::{Error, Result};/// For each output index: the first input index and the normalised triangle-filter weights.fn axis_weights(n_in: usize, n_out: usize) -> Vec<(usize, Vec<f32>)> {@@ -35,13 +36,13 @@ fn axis_weights(n_in: usize, n_out: usize) -> Vec<(usize, Vec<f32>)> {/// Antialiased bilinear resize of a row-major `h x w` image to `oh x ow`.pub fn antialias_bilinear(src: &[f32], (h, w): (usize, usize), (oh, ow): (usize, usize)) -> Result<Vec<f32>> {- if h == 0 || w == 0 || oh == 0 || ow == 0 || src.len() != h * w {- return Err(Error::Input(format!("cannot resize {} values as {w}x{h} to {ow}x{oh}", src.len())));- }+ validate_frame_size(w, h, src.len())?;+ let out_len = validate_frame_size(ow, oh, oh.saturating_mul(ow))?;+ let mid_len = validate_frame_size(ow, h, h.saturating_mul(ow))?;let xs = axis_weights(w, ow);let ys = axis_weights(h, oh);// horizontal pass: h x ow- let mut mid = vec![0f32; h * ow];+ let mut mid = vec![0f32; mid_len];for y in 0..h {let row = src.get(y * w..(y + 1) * w).unwrap_or(&[]);for (x, (x0, wt)) in xs.iter().enumerate() {@@ -52,7 +53,7 @@ pub fn antialias_bilinear(src: &[f32], (h, w): (usize, usize), (oh, ow): (usize,}}// vertical pass: oh x ow- let mut out = vec![0f32; oh * ow];+ let mut out = vec![0f32; out_len];for (y, (y0, wt)) in ys.iter().enumerate() {for x in 0..ow {let acc: f32 = wt.iter().enumerate().map(|(k, a)| a * mid.get((y0 + k) * ow + x).copied().unwrap_or(0.0)).sum();@@ -67,6 +68,7 @@ pub fn antialias_bilinear(src: &[f32], (h, w): (usize, usize), (oh, ow): (usize,/// A binary mask of any size as the model's mask prompt: `[1, 1, 1024, 1024]` of 0 and 1 (resized/// like the reference: antialiased bilinear, then thresholded at one half).pub fn mask_to_model(mask: &[u8], w: usize, h: usize) -> Result<Tensor> {+ validate_frame_size(w, h, mask.len())?;let side = INPUT_SIZE;let f: Vec<f32> = mask.iter().map(|v| f32::from(*v != 0)).collect();let data = if (w, h) == (side, side) { f } else { antialias_bilinear(&f, (h, w), (side, side))?.into_iter().map(|v| f32::from(v >= 0.5)).collect() };@@ -77,6 +79,15 @@ pub fn mask_to_model(mask: &[u8], w: usize, h: usize) -> Result<Tensor> {mod tests {use super::*;+ #[test]+ fn resize_rejects_overflowing_oversized_and_mismatched_grids() {+ for (input, output) in [((usize::MAX, 2), (1, 1)), ((1, 1), (usize::MAX, 2)), ((1, 1), (16_385, 1)), ((0, 1), (1, 1))] {+ assert!(antialias_bilinear(&[1.0], input, output).is_err(), "{input:?} to {output:?}");+ }+ assert!(mask_to_model(&[1], usize::MAX, 2).is_err());+ assert!(mask_to_model(&[1], INPUT_SIZE, INPUT_SIZE).is_err());+ }+#[test]fn a_constant_stays_constant_and_weights_sum_to_one() {let out = antialias_bilinear(&[3.0; 64 * 64], (64, 64), (16, 16)).unwrap();diff --git a/crates/sam2/src/track.rs b/crates/sam2/src/track.rsindex ad920efe7..7e73a1c74 100644--- a/crates/sam2/src/track.rs+++ b/crates/sam2/src/track.rs@@ -11,7 +11,7 @@ use std::sync::Arc;use candle_core::Tensor;use effectcraft_segment::{ClipFrame, ObjectTracker, TrackRequest, TrackedFrame};-use crate::mask::{Sam2Mask, preprocess};+use crate::mask::{Sam2Mask, preprocess, validate_frame_size};use crate::prompt::{INPUT_SIZE, Prompt};use crate::resize::mask_to_model;use crate::session::{Cancel, Direction, FrameMask, FrameSource, ObjectId, VideoTracker};@@ -36,8 +36,12 @@ impl FrameSource for Source<'_> {}fn frame(&self, index: usize) -> Result<Tensor> {- let f = (self.fetch.borrow_mut())(self.first + index as i64)- .ok_or_else(|| Error::Input(format!("frame {} is not available", self.first + index as i64)))?;+ let number = i64::try_from(index)+ .ok()+ .filter(|_| index < self.count)+ .and_then(|offset| self.first.checked_add(offset))+ .ok_or_else(|| Error::Input(format!("frame index {index} is outside the clip")))?;+ let f = (self.fetch.borrow_mut())(number).ok_or_else(|| Error::Input(format!("frame {number} is not available")))?;self.sizes.borrow_mut().insert(index, (f.w, f.h));preprocess(&f.rgb, f.w, f.h)}@@ -59,6 +63,7 @@ impl Sam2Mask {if req.count == 0 || req.count > MAX_CLIP {return Err(Error::Input(format!("a tracking request needs 1 to {MAX_CLIP} frames, got {}", req.count)));}+ req.first.checked_add((req.count - 1) as i64).ok_or_else(|| Error::Input("the clip's last frame overflows its frame number".into()))?;if req.objects.is_empty() || req.objects.len() > MAX_OBJECTS || req.objects.iter().any(|o| o.is_empty()) {return Err(Error::Input(format!("tracking needs 1 to {MAX_OBJECTS} objects, each with a prompt on at least one frame")));}@@ -69,12 +74,14 @@ impl Sam2Mask {let mut prompt_frames = vec![];for (oi, prompts) in req.objects.iter().enumerate() {for (frame, p) in prompts {- let idx = usize::try_from(frame - req.first)- .ok()+ let idx = frame+ .checked_sub(req.first)+ .and_then(|offset| usize::try_from(offset).ok()).filter(|i| *i < req.count).ok_or_else(|| Error::Input(format!("prompt frame {frame} is outside the clip")))?;if !sizes.borrow().contains_key(&idx) {let f = (source.fetch.borrow_mut())(*frame).ok_or_else(|| Error::Input(format!("frame {frame} is not available")))?;+ validate_frame_size(f.w, f.h, f.rgb.len())?;sizes.borrow_mut().insert(idx, (f.w, f.h));}prompt_frames.push((oi, idx, p));@@ -159,3 +166,16 @@ impl ObjectTracker for Sam2Mask {r}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn source_rejects_overflowing_and_out_of_range_indices_before_fetching() {+ let mut fetch = |_| panic!("invalid indices must not fetch a frame");+ let source = Source { first: i64::MAX, count: 2, fetch: RefCell::new(&mut fetch), sizes: Default::default() };+ assert!(matches!(source.frame(1), Err(Error::Input(_))));+ assert!(matches!(source.frame(usize::MAX), Err(Error::Input(_))));+ }+}diff --git a/docs/architecture.md b/docs/architecture.mdindex 0bfe4023e..c4bebd9ed 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -1,6 +1,6 @@# Architecture-> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** minor (crate table checked against `crates/`: effect count 306, VP9 inter frames) · **Target:** Adobe After Effects 2026+> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-29 · **Change:** minor (SAM 2 crate added to the layer table) · **Target:** Adobe After Effects 2026EffectCraft is a stack of small Rust crates. The engine knows nothing about the user interface;the egui frontend, the command-line tool and the MCP server all sit on top of the same `Session`@@ -39,6 +39,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for| L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model || L3 | `export` | Render queue encoding: H.264, HEVC and AV1 MP4, ProRes, WebM (VP9 + alpha or AV1, Opus), PNG/JPEG/TIFF/EXR/Targa sequences, GIF, WAV/AIFF || L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer |+| L3 | `sam2` | SAM 2.1 image segmentation and video object tracking for Roto Brush, with candle CPU inference and optional wgpu acceleration; native only, empty on wasm32, with user-installed weights || L3 | `interchange` | Premiere Pro interop through timeline interchange (FCP7 XML, FCPXML, OTIO, EDL, AAF, OMF via FilmCraft's `filmcraft-interchange`): sequences ↔ compositions (§6a) || L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express || L3 | `plugin` | Effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1; the wasmi interpreter behind the `wasm` feature, on for native hosts) into the effect registry ([plugins.md](plugins.md)) |