EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
S2.23: video tracker session
Kristian Myram5b0cc57f8767
diff --git a/crates/sam2/src/lib.rs b/crates/sam2/src/lib.rsindex 53156c7ab..4cb6f6f3e 100644--- a/crates/sam2/src/lib.rs+++ b/crates/sam2/src/lib.rs@@ -19,6 +19,7 @@ pub mod model;pub mod neck;pub mod nn;pub mod prompt;+pub mod session;pub mod weights;/// Everything that can go wrong in this crate. Never a panic.diff --git a/crates/sam2/src/model.rs b/crates/sam2/src/model.rsindex 13481fdb9..c30cbb5b9 100644--- a/crates/sam2/src/model.rs+++ b/crates/sam2/src/model.rs@@ -71,6 +71,16 @@ impl ImageModel {self.decoder.forward(&image, &self.image_pos, &sparse, &dense, &enc.high, multimask)}+ /// Decode against an already memory-conditioned embedding `[1, 256, g, g]` (video tracking).+ pub fn decode_embedded(&self, image: &Tensor, high: &(Tensor, Tensor), sparse: &Tensor, dense: &Tensor, multimask: bool) -> Result<DecoderOutput> {+ self.decoder.forward(image, &self.image_pos, sparse, dense, high, multimask)+ }++ /// The learned embedding added to a frame's features when there is no memory yet.+ pub fn no_memory(&self) -> &Tensor {+ &self.no_memory+ }+pub fn device(&self) -> Device {Device::Cpu}diff --git a/crates/sam2/src/prompt.rs b/crates/sam2/src/prompt.rsindex 55dd901a2..51aade1c7 100644--- a/crates/sam2/src/prompt.rs+++ b/crates/sam2/src/prompt.rs@@ -146,6 +146,14 @@ impl PromptEncoder {Ok(Tensor::from_vec(tokens, (1, n, DIM), &Device::Cpu)?)}+ /// Tokens for "no prompt" on a tracked frame: the reference feeds one point labelled -1 and+ /// then pads another, so two `not_a_point` tokens, `[1, 2, 256]`.+ pub fn sparse_none(&self) -> Result<Tensor> {+ let mut v = self.not_a_point.clone();+ v.extend_from_slice(&self.not_a_point);+ Ok(Tensor::from_vec(v, (1, 2, DIM), &Device::Cpu)?)+ }+/// The dense embedding for "no mask prompt", `[1, 256, GRID, GRID]`.pub fn dense_empty(&self) -> Result<Tensor> {Ok(self.no_mask.reshape((1, DIM, 1, 1))?.broadcast_as((1, DIM, GRID, GRID))?.contiguous()?)diff --git a/crates/sam2/src/session.rs b/crates/sam2/src/session.rsnew file mode 100644index 000000000..45d1ffd0a--- /dev/null+++ b/crates/sam2/src/session.rs@@ -0,0 +1,363 @@+// Modified work: ported from Hugging Face `transformers` `models/sam2_video/modeling_sam2_video.py`+// (Sam2VideoModel and Sam2VideoInferenceSession: memory-conditioned single-frame inference,+// memory bank selection, object pointers, propagation; Apache-2.0, Copyright The HuggingFace+// Inc. team) to Rust on candle. See NOTICE.+//! The video tracker: per-object memory banks, prompts on any frame, propagation forwards and+//! backwards with streaming results and cancellation.++use std::collections::{BTreeMap, VecDeque};+use std::path::Path;+use std::sync::Arc;+use std::sync::atomic::{AtomicBool, Ordering};++use candle_core::{DType, Tensor};+use candle_nn::{Linear, ops::sigmoid};++use crate::decoder::FeedForward;+use crate::memattn::MemoryAttention;+use crate::memory::{Memory, MemoryEncoder};+use crate::model::ImageModel;+use crate::nn::{apply, linear};+use crate::prompt::{INPUT_SIZE, Prompt};+use crate::weights::Weights;+use crate::{Error, Result};++/// Mask logit used where the object is absent (the reference's `NO_OBJ_SCORE`).+const NO_OBJECT: f32 = -1024.0;+/// Frames of memory the attention looks back over.+const MEMORY_FRAMES: usize = 7;+/// Object pointers kept per query.+const MAX_POINTERS: usize = 16;+/// Largest clip the session accepts.+const MAX_FRAMES: usize = 1_000_000;+const GRID: usize = 64;++/// Frames of the clip, already resized to the model input and normalised: `[1, 3, 1024, 1024]`.+pub trait FrameSource: Send {+ fn num_frames(&self) -> usize;+ fn frame(&self, index: usize) -> Result<Tensor>;+}++/// Cooperative cancellation shared with the caller.+#[derive(Clone, Default)]+pub struct Cancel(Arc<AtomicBool>);++impl Cancel {+ pub fn new() -> Cancel {+ Cancel::default()+ }+ pub fn cancel(&self) {+ self.0.store(true, Ordering::Relaxed);+ }+ pub fn is_cancelled(&self) -> bool {+ self.0.load(Ordering::Relaxed)+ }+}++#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+pub struct ObjectId(pub u32);++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum Direction {+ Forward,+ Backward,+}++/// One object's mask on one frame.+#[derive(Clone, Debug, PartialEq)]+pub struct FrameMask {+ pub frame: usize,+ pub object: ObjectId,+ /// Mask logits at 256 x 256 (positive is inside the object); `-1024` everywhere when the+ /// object is absent.+ pub logits: Vec<f32>,+ /// The model's own estimate of the mask quality (0..1).+ pub iou: f32,+ /// Object-presence logit; at or below zero the object is not in the frame.+ pub object_score: f32,+ pub occluded: bool,+}++/// The weights of everything the video tracker needs.+pub struct VideoModel {+ pub image: ImageModel,+ memory_encoder: MemoryEncoder,+ memory_attention: MemoryAttention,+ temporal_pos: Vec<f32>,+ no_object_pointer: Tensor,+ pointer_proj: FeedForward,+ pointer_time_proj: Linear,+}++impl VideoModel {+ pub fn load(w: &Weights) -> Result<VideoModel> {+ Ok(VideoModel {+ image: ImageModel::load(w)?,+ memory_encoder: MemoryEncoder::load(w)?,+ memory_attention: MemoryAttention::load(w)?,+ temporal_pos: w.get("memory_temporal_positional_encoding", &[MEMORY_FRAMES, 1, 1, 64])?.flatten_all()?.to_vec1::<f32>()?,+ no_object_pointer: w.get("no_object_pointer", &[1, 256])?,+ pointer_proj: FeedForward::load(w, "object_pointer_proj", 256, 256, 256, 3, false)?,+ pointer_time_proj: linear(w, "temporal_positional_encoding_projection_layer", 256, 64)?,+ })+ }++ pub fn open(path: &Path) -> Result<VideoModel> {+ VideoModel::load(&Weights::open(path)?)+ }+}++/// What the tracker keeps for a frame it has processed.+struct FrameOut {+ memory: Memory,+ /// `[1, 256]`+ pointer: Tensor,+ /// The mask of a conditioning frame (returned again, unchanged, when propagation passes it).+ result: Option<FrameMask>,+}++#[derive(Default)]+struct ObjectState {+ cond: BTreeMap<usize, FrameOut>,+ non_cond: BTreeMap<usize, FrameOut>,+}++struct FrameFeatures {+ /// Lowest-resolution level `[1, 256, 64, 64]` and its position encoding.+ low: Tensor,+ pos: Tensor,+ high: (Tensor, Tensor),+}++pub struct VideoTracker {+ model: Arc<VideoModel>,+ frames: Box<dyn FrameSource>,+ objects: BTreeMap<ObjectId, ObjectState>,+ cache: VecDeque<(usize, Arc<FrameFeatures>)>,+}++fn sine_embedding_1d(positions: &[f32], dim: usize) -> Result<Tensor> {+ let half = dim / 2;+ let dim_t: Vec<f32> = (0..half).map(|i| 10000f32.powf(2.0 * (i / 2) as f32 / half as f32)).collect();+ let mut out = Vec::with_capacity(positions.len() * dim);+ for p in positions {+ out.extend(dim_t.iter().map(|d| (p / d).sin()));+ out.extend(dim_t.iter().map(|d| (p / d).cos()));+ }+ Ok(Tensor::from_vec(out, (positions.len(), dim), &candle_core::Device::Cpu)?)+}++fn scalar(t: &Tensor) -> Result<f32> {+ Ok(t.flatten_all()?.to_vec1::<f32>()?.first().copied().unwrap_or(0.0))+}++impl VideoTracker {+ pub fn new(model: Arc<VideoModel>, frames: Box<dyn FrameSource>) -> Result<VideoTracker> {+ let n = frames.num_frames();+ if n == 0 || n > MAX_FRAMES {+ return Err(Error::Input(format!("a clip needs between 1 and {MAX_FRAMES} frames, got {n}")));+ }+ Ok(VideoTracker { model, frames, objects: BTreeMap::new(), cache: VecDeque::new() })+ }++ pub fn num_frames(&self) -> usize {+ self.frames.num_frames()+ }++ /// Forget every object and cached frame (the clip stays).+ pub fn reset(&mut self) {+ self.objects.clear();+ self.cache.clear();+ }++ fn features(&mut self, frame: usize) -> Result<Arc<FrameFeatures>> {+ if let Some((_, f)) = self.cache.iter().find(|(i, _)| *i == frame) {+ return Ok(f.clone());+ }+ let pixels = self.frames.frame(frame)?;+ let enc = self.model.image.encode(&pixels)?;+ let (low, pos) = match (enc.features.features.get(2), enc.features.positions.get(2)) {+ (Some(l), Some(p)) => (l.clone(), p.clone()),+ _ => return Err(Error::Compute("vision encoder returned too few levels".into())),+ };+ let f = Arc::new(FrameFeatures { low, pos, high: enc.high });+ self.cache.push_back((frame, f.clone()));+ while self.cache.len() > 2 {+ self.cache.pop_front();+ }+ Ok(f)+ }++ /// The memory bank for `frame`: memory frames and object pointers with their position+ /// encodings, plus how many trailing tokens are pointers.+ fn memory_bank(&self, obj: &ObjectState, frame: usize, reverse: bool) -> Result<(Tensor, Tensor, usize)> {+ let (mut feats, mut poss) = (vec![], vec![]);+ let temporal = |off: usize| -> Result<Tensor> {+ // The reference indexes with `offset - 1`, so the conditioning frames (offset 0) use the last row.+ let row = if off == 0 { MEMORY_FRAMES - 1 } else { off - 1 };+ let v = self.model.temporal_pos.get(row * 64..(row + 1) * 64).ok_or_else(|| Error::Weights("temporal position table".into()))?;+ Ok(Tensor::from_slice(v, (1, 64), &candle_core::Device::Cpu)?)+ };+ for out in obj.cond.values() {+ feats.push(out.memory.features.clone());+ poss.push(out.memory.pos.broadcast_add(&temporal(0)?)?);+ }+ for off in (1..MEMORY_FRAMES).rev() {+ let prev = if reverse { frame.checked_add(off) } else { frame.checked_sub(off) };+ if let Some(out) = prev.and_then(|p| obj.non_cond.get(&p)) {+ feats.push(out.memory.features.clone());+ poss.push(out.memory.pos.broadcast_add(&temporal(off)?)?);+ }+ }+ // Object pointers: conditioning frames on the already-tracked side, then recent frames.+ let n_frames = self.frames.num_frames();+ let max_ptrs = n_frames.min(MAX_POINTERS);+ let (mut offsets, mut pointers): (Vec<f32>, Vec<Tensor>) = (vec![], vec![]);+ let sign = if reverse { -1.0f32 } else { 1.0 };+ for (t, out) in &obj.cond {+ if if reverse { *t >= frame } else { *t <= frame } {+ offsets.push((frame as f32 - *t as f32) * sign);+ pointers.push(out.pointer.clone());+ }+ }+ for off in 1..max_ptrs {+ let r = if reverse { frame.checked_add(off) } else { frame.checked_sub(off) };+ let Some(r) = r.filter(|r| *r < n_frames) else { break };+ if let Some(out) = obj.non_cond.get(&r) {+ offsets.push(off as f32);+ pointers.push(out.pointer.clone());+ }+ }+ let mut n_pointer_tokens = 0;+ if !pointers.is_empty() {+ let max_diff = (max_ptrs.saturating_sub(1)).max(1) as f32;+ let norm: Vec<f32> = offsets.iter().map(|o| o / max_diff).collect();+ let pe = apply(&self.model.pointer_time_proj, &sine_embedding_1d(&norm, 256)?)?;+ let ptrs = Tensor::cat(&pointers, 0)?; // [P, 256]+ let p = ptrs.dim(0)?;+ let tokens = ptrs.reshape((p * 4, 64))?;+ let pe = pe.unsqueeze(1)?.broadcast_as((p, 4, 64))?.reshape((p * 4, 64))?;+ n_pointer_tokens = p * 4;+ feats.push(tokens);+ poss.push(pe.contiguous()?);+ }+ if feats.is_empty() {+ return Err(Error::Input("the object has no conditioning frame; add a prompt first".into()));+ }+ Ok((Tensor::cat(&feats, 0)?, Tensor::cat(&poss, 0)?, n_pointer_tokens))+ }++ /// Run the model on one frame for one object and store the result.+ fn step(&mut self, id: ObjectId, frame: usize, prompt: Option<&Prompt>, reverse: bool, conditioning: bool) -> Result<FrameMask> {+ let feats = self.features(frame)?;+ let model = self.model.clone();+ let tokens = |t: &Tensor| -> Result<Tensor> { Ok(t.flatten_from(2)?.squeeze(0)?.t()?.contiguous()?) };+ let conditioned = if conditioning {+ feats.low.broadcast_add(model.image.no_memory())?+ } else {+ let obj = self.objects.get(&id).ok_or_else(|| Error::Input("unknown object".into()))?;+ let (mem, mem_pos, n_ptr) = self.memory_bank(obj, frame, reverse)?;+ let out = model.memory_attention.forward(&tokens(&feats.low)?, &tokens(&feats.pos)?, &mem, &mem_pos, n_ptr)?;+ out.t()?.contiguous()?.reshape((1, 256, GRID, GRID))?+ };+ let num_points = prompt.map_or(0, |p| p.points.len() + 2 * usize::from(p.bbox.is_some()));+ let multimask = num_points <= 1;+ let sparse = match prompt {+ Some(p) => model.image.prompt.sparse(p)?,+ None => model.image.prompt.sparse_none()?,+ };+ let dense = model.image.prompt.dense_empty()?;+ let out = model.image.decode_embedded(&conditioned, &feats.high, &sparse, &dense, multimask)?;++ let score = scalar(&out.object_score)?;+ let present = score > 0.0;+ let ious = out.iou.flatten_all()?.to_vec1::<f32>()?;+ let best = if multimask { ious.iter().enumerate().fold((0usize, f32::NEG_INFINITY), |b, (i, v)| if *v > b.1 { (i, *v) } else { b }).0 } else { 0 };+ let low = out.masks.narrow(1, best, 1)?;+ let low = if present { low } else { Tensor::full(NO_OBJECT, low.dims(), low.device())? };+ let token = if multimask { out.sam_tokens.narrow(1, best, 1)? } else { out.sam_tokens.narrow(1, 0, 1)? }.squeeze(1)?;+ let pointer = if present { model.pointer_proj.forward(&token)? } else { model.no_object_pointer.clone() };++ let high = low.upsample_bilinear2d(INPUT_SIZE, INPUT_SIZE, false)?;+ let for_memory = if prompt.is_some() { high.gt(0f32)?.to_dtype(DType::F32)? } else { sigmoid(&high)? };+ let for_memory = ((for_memory * 20.0)? - 10.0)?;+ let memory = model.memory_encoder.memory(&feats.low, &for_memory, present)?;++ let iou = ious.get(best).copied().unwrap_or(0.0);+ let result = FrameMask { frame, object: id, logits: low.flatten_all()?.to_vec1::<f32>()?, iou, object_score: score, occluded: !present };+ let state = self.objects.entry(id).or_default();+ let entry = FrameOut { memory, pointer, result: conditioning.then(|| result.clone()) };+ if conditioning {+ state.non_cond.remove(&frame);+ state.cond.insert(frame, entry);+ } else {+ state.non_cond.insert(frame, entry);+ Self::prune(state, frame);+ }+ Ok(result)+ }++ /// Drop memories that no later frame in either direction can still look at.+ fn prune(state: &mut ObjectState, frame: usize) {+ let keep = |f: usize, cond: &BTreeMap<usize, FrameOut>| f.abs_diff(frame) <= MAX_POINTERS || cond.keys().any(|c| f.abs_diff(*c) <= MEMORY_FRAMES);+ let doomed: Vec<usize> = state.non_cond.keys().copied().filter(|f| !keep(*f, &state.cond)).collect();+ for f in doomed {+ state.non_cond.remove(&f);+ }+ }++ fn check_frame(&self, frame: usize) -> Result<()> {+ if frame >= self.frames.num_frames() {+ return Err(Error::Input(format!("frame {frame} is outside the clip of {} frames", self.frames.num_frames())));+ }+ Ok(())+ }++ /// Prompt an object on a frame (clicks and/or a box). The frame becomes one of the object's+ /// conditioning frames when it has not been tracked yet; the mask comes back at once.+ pub fn prompt(&mut self, frame: usize, object: ObjectId, prompt: &Prompt) -> Result<FrameMask> {+ self.check_frame(frame)?;+ prompt.validate()?;+ if prompt.points.is_empty() && prompt.bbox.is_none() {+ return Err(Error::Input("a prompt needs at least one point or a box".into()));+ }+ let tracked = self.objects.get(&object).is_some_and(|o| o.non_cond.contains_key(&frame));+ if tracked {+ // A correction on a tracked frame is conditioned on the memory like any tracked frame.+ let reverse = false;+ return self.step(object, frame, Some(prompt), reverse, false);+ }+ self.objects.entry(object).or_default();+ self.step(object, frame, Some(prompt), false, true)+ }++ /// Track every object from `start` through `max_frames` frames (the rest of the clip when+ /// `None`) in `direction`, calling `out` for each object on each frame as it is finished.+ pub fn propagate(&mut self, start: usize, direction: Direction, max_frames: Option<usize>, cancel: &Cancel, out: &mut dyn FnMut(FrameMask)) -> Result<()> {+ self.check_frame(start)?;+ let n = self.frames.num_frames();+ let reverse = direction == Direction::Backward;+ let span = max_frames.unwrap_or(n).min(n);+ let ids: Vec<ObjectId> = self.objects.iter().filter(|(_, o)| !o.cond.is_empty()).map(|(id, _)| *id).collect();+ if ids.is_empty() {+ return Err(Error::Input("no object has been prompted".into()));+ }+ for k in 0..=span {+ let frame = if reverse { start.checked_sub(k) } else { Some(start + k).filter(|f| *f < n) };+ let Some(frame) = frame else { break };+ for id in &ids {+ if cancel.is_cancelled() {+ return Err(Error::Cancelled);+ }+ let cached = self.objects.get(id).is_some_and(|o| o.cond.contains_key(&frame));+ let stored = self.objects.get(id).and_then(|o| o.cond.get(&frame)).and_then(|o| o.result.clone());+ let mask = match stored {+ Some(m) if cached => m,+ _ => self.step(*id, frame, None, reverse, false)?,+ };+ out(mask);+ }+ }+ Ok(())+ }+}diff --git a/crates/sam2/tests/reference.rs b/crates/sam2/tests/reference.rsindex d6d457871..7e480af10 100644--- a/crates/sam2/tests/reference.rs+++ b/crates/sam2/tests/reference.rs@@ -194,3 +194,56 @@ fn memory_attention_matches_pytorch() {eprintln!("memory attention: max abs error {e:e} (pointers {pointers})");assert!(e < MEMORY_TOL, "memory attention error {e}");}++/// Frames of the video fixture, served one at a time.+struct Clip(Tensor);++impl effectcraft_sam2::session::FrameSource for Clip {+ fn num_frames(&self) -> usize {+ self.0.dim(0).unwrap()+ }+ fn frame(&self, i: usize) -> effectcraft_sam2::Result<Tensor> {+ Ok(self.0.narrow(0, i, 1)?.contiguous()?)+ }+}++#[test]+fn tracker_rejects_hostile_input_and_honours_cancel() {+ use effectcraft_sam2::Error;+ use effectcraft_sam2::prompt::Prompt;+ use effectcraft_sam2::session::{Cancel, Direction, ObjectId, VideoModel, VideoTracker};+ let Some((w, r)) = load_video() else { return };+ let model = std::sync::Arc::new(VideoModel::load(&w).unwrap());+ let clip = r.get("clip.pixel_values").unwrap().narrow(0, 0, 3).unwrap();+ let mut t = VideoTracker::new(model.clone(), Box::new(Clip(clip.clone()))).unwrap();+ let click = Prompt { points: vec![([416.0, 512.0], true)], bbox: None };+ let obj = ObjectId(7);+ // outside the clip, empty and non-finite prompts, nothing prompted yet+ assert!(matches!(t.prompt(3, obj, &click), Err(Error::Input(_))));+ assert!(matches!(t.prompt(0, obj, &Prompt::default()), Err(Error::Input(_))));+ assert!(matches!(t.prompt(0, obj, &Prompt { points: vec![([f32::NAN, 1.0], true)], bbox: None }), Err(Error::Input(_))));+ let mut sink = |_m| {};+ assert!(matches!(t.propagate(0, Direction::Forward, None, &Cancel::new(), &mut sink), Err(Error::Input(_))));+ assert!(matches!(t.propagate(9, Direction::Forward, None, &Cancel::new(), &mut sink), Err(Error::Input(_))));+ // an empty clip is refused; a cancelled run stops with Cancelled and keeps the tracker usable+ assert!(VideoTracker::new(model.clone(), Box::new(Clip(clip.narrow(0, 0, 0).unwrap()))).is_err());+ t.prompt(0, obj, &click).unwrap();+ let cancel = Cancel::new();+ cancel.cancel();+ assert!(matches!(t.propagate(0, Direction::Forward, None, &cancel, &mut sink), Err(Error::Cancelled)));+ let mut seen = vec![];+ t.propagate(0, Direction::Forward, Some(1), &Cancel::new(), &mut |m| seen.push(m.frame)).unwrap();+ assert_eq!(seen, vec![0, 1]);+ // a frame source that returns the wrong size is an error, not a crash+ struct Bad;+ impl effectcraft_sam2::session::FrameSource for Bad {+ fn num_frames(&self) -> usize {+ 2+ }+ fn frame(&self, _i: usize) -> effectcraft_sam2::Result<Tensor> {+ Ok(Tensor::zeros((1, 3, 64, 64), candle_core::DType::F32, &Device::Cpu)?)+ }+ }+ let mut bad = VideoTracker::new(model, Box::new(Bad)).unwrap();+ assert!(matches!(bad.prompt(0, obj, &click), Err(Error::Input(_))));+}