Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(vision): measure a camera's shake and work out the steadying correction (#253)

Jareer4edd466383cd
diff --git a/src/crates/concat-vision/src/lib.rs b/src/crates/concat-vision/src/lib.rs
index f2762d91..fa408042 100644
--- a/src/crates/concat-vision/src/lib.rs
+++ b/src/crates/concat-vision/src/lib.rs
@@ -31,6 +31,10 @@
//! host infers and writes them. [`onnx`] says whether this machine can run
//! the runtime at all, which an Intel Mac older than its runtime cannot.
//!
+//! Stabilization is [`stabilize`]: the camera's movement measured by
+//! tracking corners from frame to frame, cached beside the masks, and the
+//! correction worked out from it live. Plain arithmetic, no model.
+//!
//! A mask is whatever shape its model answers in - square for the object
//! model, the picture's own aspect for the person model - and every
//! position in one is a fraction of the source picture: `(0, 0)` its
@@ -51,6 +55,7 @@ pub mod onnx;
pub mod runtime;
#[cfg(feature = "infer")]
pub mod segment;
+pub mod stabilize;
pub mod store;
pub mod strokes;
diff --git a/src/crates/concat-vision/src/stabilize/fit.rs b/src/crates/concat-vision/src/stabilize/fit.rs
new file mode 100644
index 00000000..86f0dea6
--- /dev/null
+++ b/src/crates/concat-vision/src/stabilize/fit.rs
@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! One similarity fitted to many tracks, robustly.
+//!
+//! A similarity is a turn, a uniform zoom and a shift: four numbers, and
+//! the most a camera's shake needs. Written the way a complex number
+//! multiplies, `p' = z·p + t` with `z = a + bi`, it composes and inverts
+//! in a line each and fits by least squares in closed form.
+//!
+//! The fit is RANSAC: two tracks fix a similarity, so pairs are drawn at
+//! random, each one's answer is scored by how many tracks agree with it,
+//! and the best is fitted again over everything that agreed. A person
+//! walking through the frame is a cluster of tracks that agree with each
+//! other and with nothing else; as long as the background has more, the
+//! background wins.
+
+/// `p' = z·p + t`, `z = a + bi`: a turn by `atan2(b, a)`, a zoom by
+/// `|z|`, then a shift by `t`.
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct Similarity {
+ /// The real half of `z`: the zoom times the cosine of the turn.
+ pub a: f64,
+ /// The imaginary half: the zoom times its sine.
+ pub b: f64,
+ /// The shift across.
+ pub tx: f64,
+ /// The shift down.
+ pub ty: f64,
+}
+
+impl Default for Similarity {
+ fn default() -> Self {
+ Similarity::IDENTITY
+ }
+}
+
+impl Similarity {
+ /// Nothing moved.
+ pub const IDENTITY: Similarity = Similarity {
+ a: 1.0,
+ b: 0.0,
+ tx: 0.0,
+ ty: 0.0,
+ };
+
+ /// A turn by `angle` radians (clockwise on screen, `y` running down),
+ /// a zoom by `scale`, then a shift by `(tx, ty)`.
+ pub fn new(angle: f64, scale: f64, tx: f64, ty: f64) -> Similarity {
+ let (sin, cos) = angle.sin_cos();
+ Similarity {
+ a: scale * cos,
+ b: scale * sin,
+ tx,
+ ty,
+ }
+ }
+
+ /// A zoom about the origin.
+ pub fn scaling(scale: f64) -> Similarity {
+ Similarity {
+ a: scale,
+ b: 0.0,
+ tx: 0.0,
+ ty: 0.0,
+ }
+ }
+
+ /// Where `p` goes.
+ pub fn apply(&self, (x, y): (f64, f64)) -> (f64, f64) {
+ (
+ self.a * x - self.b * y + self.tx,
+ self.b * x + self.a * y + self.ty,
+ )
+ }
+
+ /// This, then `after`: `after(self(p))`.
+ pub fn then(&self, after: &Similarity) -> Similarity {
+ let a = after.a * self.a - after.b * self.b;
+ let b = after.a * self.b + after.b * self.a;
+ let (tx, ty) = after.apply((self.tx, self.ty));
+ Similarity { a, b, tx, ty }
+ }
+
+ /// The map back. A degenerate one - no size at all - is its own.
+ pub fn inverse(&self) -> Similarity {
+ let norm = self.a * self.a + self.b * self.b;
+ if norm <= f64::MIN_POSITIVE {
+ return *self;
+ }
+ let a = self.a / norm;
+ let b = -self.b / norm;
+ Similarity {
+ a,
+ b,
+ tx: -(a * self.tx - b * self.ty),
+ ty: -(b * self.tx + a * self.ty),
+ }
+ }
+
+ /// The zoom.
+ pub fn scale(&self) -> f64 {
+ self.a.hypot(self.b)
+ }
+
+ /// The turn, radians.
+ pub fn angle(&self) -> f64 {
+ self.b.atan2(self.a)
+ }
+
+ /// Whether every number is one.
+ pub fn is_finite(&self) -> bool {
+ self.a.is_finite() && self.b.is_finite() && self.tx.is_finite() && self.ty.is_finite()
+ }
+
+ /// Whether this moves nothing, to within `epsilon`.
+ pub fn is_identity(&self, epsilon: f64) -> bool {
+ (self.a - 1.0).abs() <= epsilon
+ && self.b.abs() <= epsilon
+ && self.tx.abs() <= epsilon
+ && self.ty.abs() <= epsilon
+ }
+}
+
+/// One track: where a point was, and where it went.
+pub type Pair = ((f64, f64), (f64, f64));
+
+/// The least-squares similarity taking each pair's first point to its
+/// second, or `None` with fewer than two distinct points.
+pub fn least_squares(pairs: &[Pair]) -> Option<Similarity> {
+ if pairs.len() < 2 {
+ return None;
+ }
+ let n = pairs.len() as f64;
+ let (mut px, mut py, mut qx, mut qy) = (0.0, 0.0, 0.0, 0.0);
+ for &((x, y), (u, v)) in pairs {
+ px += x;
+ py += y;
+ qx += u;
+ qy += v;
+ }
+ let (px, py, qx, qy) = (px / n, py / n, qx / n, qy / n);
+ let (mut dot, mut cross, mut norm) = (0.0, 0.0, 0.0);
+ for &((x, y), (u, v)) in pairs {
+ let (x, y, u, v) = (x - px, y - py, u - qx, v - qy);
+ dot += x * u + y * v;
+ cross += x * v - y * u;
+ norm += x * x + y * y;
+ }
+ if norm <= 1e-18 {
+ return None;
+ }
+ let a = dot / norm;
+ let b = cross / norm;
+ let similarity = Similarity {
+ a,
+ b,
+ tx: qx - (a * px - b * py),
+ ty: qy - (b * px + a * py),
+ };
+ similarity.is_finite().then_some(similarity)
+}
+
+/// The similarity two pairs fix exactly: `z = (q1 - q0) / (p1 - p0)`.
+fn through(first: &Pair, second: &Pair) -> Option<Similarity> {
+ let ((x0, y0), (u0, v0)) = *first;
+ let ((x1, y1), (u1, v1)) = *second;
+ let (dx, dy) = (x1 - x0, y1 - y0);
+ let (du, dv) = (u1 - u0, v1 - v0);
+ let norm = dx * dx + dy * dy;
+ if norm <= 1e-12 {
+ return None;
+ }
+ let a = (du * dx + dv * dy) / norm;
+ let b = (dv * dx - du * dy) / norm;
+ Some(Similarity {
+ a,
+ b,
+ tx: u0 - (a * x0 - b * y0),
+ ty: v0 - (b * x0 + a * y0),
+ })
+}
+
+/// What a robust fit found.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Robust {
+ /// The similarity, fitted over the tracks that agree with it.
+ pub similarity: Similarity,
+ /// Which tracks agree, in the order given.
+ pub inliers: Vec<bool>,
+}
+
+impl Robust {
+ /// How many tracks agree.
+ pub fn count(&self) -> usize {
+ self.inliers.iter().filter(|&&inlier| inlier).count()
+ }
+}
+
+/// A small, seeded generator: the same tracks always fit the same way,
+/// so an analysis run twice caches the same answer.
+struct XorShift(u64);
+
+impl XorShift {
+ fn next(&mut self) -> u64 {
+ let mut x = self.0;
+ x ^= x << 13;
+ x ^= x >> 7;
+ x ^= x << 17;
+ self.0 = x;
+ x
+ }
+
+ fn below(&mut self, n: usize) -> usize {
+ (self.next() % n as u64) as usize
+ }
+}
+
+/// The similarity most tracks agree on, a track agreeing when it lands
+/// within `threshold` of where the similarity sends it. `None` when no
+/// pair of tracks fixes one that at least `min_inliers` agree with, or
+/// the answer is no camera's: a zoom past a quarter either way between
+/// two frames.
+pub fn ransac(pairs: &[Pair], threshold: f64, min_inliers: usize) -> Option<Robust> {
+ let n = pairs.len();
+ if n < 2 || n < min_inliers {
+ return None;
+ }
+ let squared = threshold * threshold;
+ let agreeing = |similarity: &Similarity| -> Vec<bool> {
+ pairs
+ .iter()
+ .map(|&(p, (u, v))| {
+ let (x, y) = similarity.apply(p);
+ (x - u) * (x - u) + (y - v) * (y - v) <= squared
+ })
+ .collect()
+ };
+ let mut random = XorShift(0x9e37_79b9_7f4a_7c15 ^ n as u64);
+ let mut best: Option<(usize, Similarity)> = None;
+ let mut wanted = 500usize;
+ let mut tried = 0usize;
+ while tried < wanted {
+ tried += 1;
+ let i = random.below(n);
+ let mut j = random.below(n);
+ if i == j {
+ j = (j + 1) % n;
+ }
+ let Some(candidate) = through(&pairs[i], &pairs[j]) else {
+ continue;
+ };
+ if !(0.75..=1.333).contains(&candidate.scale()) {
+ continue;
+ }
+ let count = agreeing(&candidate).iter().filter(|&&inlier| inlier).count();
+ if best.is_none_or(|(most, _)| count > most) {
+ best = Some((count, candidate));
+ // Enough draws that a clean pair was very likely among them:
+ // 99.9% sure, at the share of tracks this one says agree.
+ let share = count as f64 / n as f64;
+ let clean = share * share;
+ if clean >= 1.0 {
+ break;
+ }
+ if clean > 0.0 {
+ let needed = ((1.0 - 0.999f64).ln() / (1.0 - clean).ln()).ceil();
+ wanted = wanted.min((needed as usize).max(16));
+ }
+ }
+ }
+ let (_, mut similarity) = best?;
+ // Fitted again over its own agreement, twice: the second pass picks up
+ // the tracks the sharper answer now agrees with.
+ let mut inliers = agreeing(&similarity);
+ for _ in 0..2 {
+ let kept: Vec<Pair> = pairs
+ .iter()
+ .zip(&inliers)
+ .filter(|(_, inlier)| **inlier)
+ .map(|(pair, _)| *pair)
+ .collect();
+ if let Some(refit) = least_squares(&kept) {
+ similarity = refit;
+ }
+ inliers = agreeing(&similarity);
+ }
+ let robust = Robust {
+ similarity,
+ inliers,
+ };
+ (robust.count() >= min_inliers && (0.75..=1.333).contains(&similarity.scale()))
+ .then_some(robust)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn close(a: f64, b: f64, tolerance: f64) -> bool {
+ (a - b).abs() <= tolerance
+ }
+
+ #[test]
+ fn a_similarity_composes_and_inverts() {
+ let s = Similarity::new(0.1, 1.05, 0.02, -0.03);
+ let t = Similarity::new(-0.3, 0.9, -0.1, 0.05);
+ let p = (0.3, -0.2);
+ let composed = s.then(&t).apply(p);
+ let stepwise = t.apply(s.apply(p));
+ assert!(close(composed.0, stepwise.0, 1e-12) && close(composed.1, stepwise.1, 1e-12));
+ let back = s.inverse().apply(s.apply(p));
+ assert!(close(back.0, p.0, 1e-12) && close(back.1, p.1, 1e-12));
+ assert!(close(s.angle(), 0.1, 1e-12) && close(s.scale(), 1.05, 1e-12));
+ }
+
+ #[test]
+ fn least_squares_recovers_an_exact_similarity() {
+ let truth = Similarity::new(0.02, 1.01, 0.005, -0.004);
+ let pairs: Vec<Pair> = (0..20)
+ .map(|i| {
+ let p = ((i % 5) as f64 * 0.2 - 0.4, (i / 5) as f64 * 0.2 - 0.3);
+ (p, truth.apply(p))
+ })
+ .collect();
+ let fitted = least_squares(&pairs).expect("fits");
+ assert!(fitted.then(&truth.inverse()).is_identity(1e-9));
+ }
+
+ /// A subject walking across a third of the tracks moves them its own
+ /// way; the fit follows the background's two thirds.
+ #[test]
+ fn ransac_rejects_a_moving_subject() {
+ let camera = Similarity::new(-0.01, 1.0, 0.004, 0.002);
+ let mut pairs: Vec<Pair> = Vec::new();
+ for i in 0..60 {
+ // A little noise, as real tracks have: a few thousandths of a
+ // pixel at 480 lines is ~1e-5 of the height.
+ let jitter = ((i * 37 % 11) as f64 - 5.0) * 2e-5;
+ let p = (
+ (i % 10) as f64 * 0.15 - 0.7 + jitter,
+ (i / 10) as f64 * 0.15 - 0.4,
+ );
+ let (u, v) = camera.apply(p);
+ pairs.push((p, (u + jitter, v - jitter)));
+ }
+ // The subject: a block of 30 tracks all shifted far to the right.
+ for i in 0..30 {
+ let p = (0.1 + (i % 6) as f64 * 0.02, 0.1 + (i / 6) as f64 * 0.02);
+ let (u, v) = camera.apply(p);
+ pairs.push((p, (u + 0.05, v + 0.01)));
+ }
+ let robust = ransac(&pairs, 2.0 / 480.0, 10).expect("fits");
+ assert!(
+ robust.similarity.then(&camera.inverse()).is_identity(2e-4),
+ "{:?}",
+ robust.similarity
+ );
+ assert!(robust.inliers[..60].iter().all(|&inlier| inlier));
+ assert!(robust.inliers[60..].iter().all(|&inlier| !inlier));
+ // And plain least squares, by contrast, is dragged off.
+ let dragged = least_squares(&pairs).expect("fits");
+ assert!(!dragged.then(&camera.inverse()).is_identity(2e-3));
+ }
+
+ #[test]
+ fn too_few_agreeing_tracks_is_no_answer() {
+ let pairs: Vec<Pair> = (0..8)
+ .map(|i| {
+ let p = (i as f64 * 0.1, (i * i) as f64 * 0.01);
+ (p, (p.1 * 3.0, -p.0 * 2.0 + i as f64))
+ })
+ .collect();
+ assert!(ransac(&pairs, 1e-3, 6).is_none());
+ }
+}
diff --git a/src/crates/concat-vision/src/stabilize/mod.rs b/src/crates/concat-vision/src/stabilize/mod.rs
new file mode 100644
index 00000000..19969abc
--- /dev/null
+++ b/src/crates/concat-vision/src/stabilize/mod.rs
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Taking the camera's shake out of a picture.
+//!
+//! Done the way Premiere's Warp Stabilizer and Concat's own cutout are:
+//! analysed once, cached, applied live. In four parts:
+//!
+//! - [`track`]: the camera's movement between two frames, measured. Good
+//! corners are found spread over a grid and followed to the next frame
+//! by pyramidal Lucas-Kanade optical flow, each track checked by
+//! following it back again.
+//! - [`fit`]: the movement as one similarity - a shift, a turn and a
+//! zoom - fitted to the tracks by RANSAC, so a subject moving across
+//! the frame is outvoted by the background rather than dragging the
+//! answer with it.
+//! - [`store`]: the measured movement of every frame of a media file,
+//! cached in the project folder and keyed by the file and the
+//! analysis's version. The host's job writes it; the renderer reads it.
+//! - [`path`]: the correction, from the cached movement and the clip's
+//! settings: the camera's path accumulated, smoothed, the difference
+//! taken, and the one zoom the whole clip needs to keep its edges out
+//! of view. Pure and fast, so it is worked out again whenever a setting
+//! changes rather than cached.
+//!
+//! Every position here is in the picture's own normalised units: the
+//! centre of the frame is the origin, `y` runs down, and the frame's
+//! height is one, so a turn is a turn whatever the picture's aspect and
+//! whatever size it was analysed or decoded at.
+
+pub mod fit;
+pub mod path;
+pub mod store;
+pub mod track;
+
+pub use fit::Similarity;
+pub use path::{Correction, Settings};
+pub use store::{FrameMotion, Motion, Step, motion_dir};
+
+/// Bumped whenever the analysis would measure differently, so a cache
+/// from an older one is measured again rather than trusted.
+pub const ANALYSIS_VERSION: u32 = 1;
+
+/// The longest side the analysis decodes at. Shake is a few pixels of a
+/// full frame; at this size it is still a pixel or more, and a frame
+/// costs milliseconds rather than tens of them.
+pub const ANALYSIS_SIDE: u32 = 480;
diff --git a/src/crates/concat-vision/src/stabilize/path.rs b/src/crates/concat-vision/src/stabilize/path.rs
new file mode 100644
index 00000000..8bdb9ab1
--- /dev/null
+++ b/src/crates/concat-vision/src/stabilize/path.rs
@@ -0,0 +1,599 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! The correction: from the measured movement to how each frame is moved.
+//!
+//! The movement between frames, composed frame after frame, is the
+//! camera's path: where the first frame's scene sits in every later one.
+//! The path is split into four curves - the shift across and down, the
+//! turn, and the zoom (as a logarithm, so smoothing it is fair to a zoom
+//! in and a zoom out) - and each is smoothed by a Gaussian whose width is
+//! the Smoothness setting, measured in seconds of the timeline: a clip at
+//! four times its speed shows four times the source in that time, and is
+//! smoothed over four times the source frames, so it looks as calm.
+//! Tripod takes the curves' mean instead: one still camera.
+//!
+//! A frame is then drawn as the smooth camera would have seen it: each
+//! point of the picture shown is looked for where the real camera put it,
+//! `real ∘ smooth⁻¹`. That leaves the frame's edges moving about inside
+//! the picture, so the picture is zoomed by the least that hides them for
+//! every frame of the clip - one zoom, so the picture does not breathe -
+//! up to the setting's most. Where that is not enough, the frames that
+//! would show an edge have their correction eased off just so far, and the
+//! easing is itself smoothed so it does not jerk.
+//!
+//! The curves are padded at either end by reflecting them through their
+//! end points, so a pan that runs to the end of the clip stays a pan
+//! rather than slowing to a stop the smoothing invented.
+
+use super::fit::Similarity;
+use super::store::Motion;
+
+/// What a clip's stabilization asks for.
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct Settings {
+ /// `0..=1`; see [`sigma_seconds`].
+ pub smoothness: f64,
+ /// The most the picture may be zoomed to hide the edges, `>= 1`.
+ pub max_zoom: f64,
+ /// One still camera for the whole clip.
+ pub tripod: bool,
+ /// Leave the camera's roll alone.
+ pub keep_rotation: bool,
+}
+
+/// The Gaussian's width, in seconds of timeline, for a smoothness: a
+/// tenth of a second at none - only the jitter goes - to three seconds at
+/// all, evenly in between on a log scale.
+pub fn sigma_seconds(smoothness: f64) -> f64 {
+ 0.1 * 30f64.powf(smoothness.clamp(0.0, 1.0))
+}
+
+/// The correction for one clip: for every frame it shows, the map from a
+/// point of the picture as drawn to the point of the frame as shot that
+/// is drawn there, in normalised units.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Correction {
+ /// The frames' times, seconds of source, ascending.
+ times: Vec<f64>,
+ /// Each frame's map, picture to source.
+ maps: Vec<Similarity>,
+ /// The usual time between frames.
+ interval: f64,
+ /// The zoom applied over the whole clip, `>= 1`.
+ pub zoom: f64,
+ /// The zoom the whole correction would need to keep every edge out
+ /// of view; more than `zoom` when the setting held it back.
+ pub needed: f64,
+ /// How shaky the clip is as shot: the root mean square of how far
+ /// each frame sits from the smooth path, as a fraction of the frame's
+ /// height.
+ pub shake: f64,
+}
+
+impl Correction {
+ /// A clip with nothing to correct.
+ pub fn none() -> Correction {
+ Correction {
+ times: Vec::new(),
+ maps: Vec::new(),
+ interval: 1.0 / 30.0,
+ zoom: 1.0,
+ needed: 1.0,
+ shake: 0.0,
+ }
+ }
+
+ /// Whether there is a correction at all.
+ pub fn is_empty(&self) -> bool {
+ self.maps.is_empty()
+ }
+
+ /// The map for the frame on screen at `seconds` of source: the last
+ /// frame shown at or before it. The identity outside the frames
+ /// measured, so a stretch not analysed shows as shot.
+ pub fn at(&self, seconds: f64) -> Similarity {
+ let index = self.times.partition_point(|&at| at <= seconds + 1e-4);
+ if index == 0 {
+ return Similarity::IDENTITY;
+ }
+ let index = index - 1;
+ // Past the last frame by more than a frame or two is past what was
+ // measured.
+ if seconds - self.times[index] > 2.5 * self.interval {
+ return Similarity::IDENTITY;
+ }
+ self.maps[index]
+ }
+}
+
+/// The four curves of a camera path.
+#[derive(Clone, Copy, Debug, Default, PartialEq)]
+struct Pose {
+ tx: f64,
+ ty: f64,
+ angle: f64,
+ log_scale: f64,
+}
+
+impl Pose {
+ fn of(similarity: &Similarity, previous_angle: Option<f64>) -> Pose {
+ let mut angle = similarity.angle();
+ // Unwrapped, so a slow turn past half a revolution is a curve and
+ // not a jump.
+ if let Some(previous) = previous_angle {
+ let tau = std::f64::consts::TAU;
+ angle += ((previous - angle) / tau).round() * tau;
+ }
+ Pose {
+ tx: similarity.tx,
+ ty: similarity.ty,
+ angle,
+ log_scale: similarity.scale().max(1e-9).ln(),
+ }
+ }
+
+ fn similarity(&self) -> Similarity {
+ Similarity::new(self.angle, self.log_scale.exp(), self.tx, self.ty)
+ }
+
+ fn lerp(&self, other: &Pose, t: f64) -> Pose {
+ Pose {
+ tx: self.tx + (other.tx - self.tx) * t,
+ ty: self.ty + (other.ty - self.ty) * t,
+ angle: self.angle + (other.angle - self.angle) * t,
+ log_scale: self.log_scale + (other.log_scale - self.log_scale) * t,
+ }
+ }
+}
+
+/// The least zoom about the centre at which `map` - picture to source -
+/// keeps the whole picture inside the source frame, `aspect` wide and one
+/// high. Infinite when no zoom does.
+pub fn zoom_needed(map: &Similarity, aspect: f64) -> f64 {
+ let (hw, hh) = (aspect / 2.0, 0.5);
+ let mut zoom: f64 = 1.0;
+ for corner in [(-hw, -hh), (hw, -hh), (-hw, hh), (hw, hh)] {
+ // The linear part of the map on the corner; the shift is apart.
+ let x = map.a * corner.0 - map.b * corner.1;
+ let y = map.b * corner.0 + map.a * corner.1;
+ for (reach, shift, half) in [(x, map.tx, hw), (y, map.ty, hh)] {
+ // reach / zoom + shift must stay within ±half.
+ let room = if reach > 0.0 {
+ half - shift
+ } else {
+ half + shift
+ };
+ if reach.abs() <= 1e-12 {
+ if shift.abs() > half {
+ return f64::INFINITY;
+ }
+ continue;
+ }
+ if room <= 0.0 {
+ return f64::INFINITY;
+ }
+ zoom = zoom.max(reach.abs() / room);
+ }
+ }
+ zoom
+}
+
+/// `values` smoothed by a Gaussian of `sigma` samples: three box passes,
+/// which come within a few percent of one and cost the same at any width.
+/// The ends are padded by reflecting through the end points.
+pub fn gaussian(values: &[f64], sigma: f64) -> Vec<f64> {
+ let n = values.len();
+ if n < 2 || !(sigma >= 0.5) {
+ return values.to_vec();
+ }
+ // Three boxes of width w have the variance of a Gaussian of sigma
+ // when 3 (w² - 1) / 12 = sigma².
+ let width = (4.0 * sigma * sigma + 1.0).sqrt();
+ let radius = ((width - 1.0) / 2.0).round().max(1.0) as usize;
+ let pad = 3 * radius + 1;
+ // Reflected through a line fitted to each end rather than through the
+ // end sample itself: a shaky first frame is not pinned in place.
+ let reach = (2 * radius + 1).min(n);
+ let pivots = (
+ line_at(&values[..reach], 0.0),
+ line_at(&values[n - reach..], (reach - 1) as f64),
+ );
+ let mut padded: Vec<f64> = (-(pad as isize)..(n + pad) as isize)
+ .map(|i| reflected(values, pivots, i))
+ .collect();
+ for _ in 0..3 {
+ padded = box_mean(&padded, radius);
+ }
+ padded[pad..pad + n].to_vec()
+}
+
+/// `values` at `i`, past either end reflected through that end's pivot -
+/// and past the far end of the reflection reflected again - so a straight
+/// line carries on straight however far it is read.
+fn reflected(values: &[f64], (start, end): (f64, f64), mut i: isize) -> f64 {
+ let last = values.len() as isize - 1;
+ if last <= 0 {
+ return values.first().copied().unwrap_or(0.0);
+ }
+ // Each reflection is `2·end − value`, so a run of them adds up to a
+ // signed sum of end values and one value inside.
+ let mut sign = 1.0;
+ let mut offset = 0.0;
+ loop {
+ if i < 0 {
+ offset += sign * 2.0 * start;
+ sign = -sign;
+ i = -i;
+ } else if i > last {
+ offset += sign * 2.0 * end;
+ sign = -sign;
+ i = 2 * last - i;
+ } else {
+ return offset + sign * values[i as usize];
+ }
+ }
+}
+
+/// The least-squares line through `values`, at index `x`.
+fn line_at(values: &[f64], x: f64) -> f64 {
+ let n = values.len() as f64;
+ if values.len() < 2 {
+ return values.first().copied().unwrap_or(0.0);
+ }
+ let mean_x = (n - 1.0) / 2.0;
+ let mean_y = values.iter().sum::<f64>() / n;
+ let (mut sxy, mut sxx) = (0.0, 0.0);
+ for (i, v) in values.iter().enumerate() {
+ let dx = i as f64 - mean_x;
+ sxy += dx * (v - mean_y);
+ sxx += dx * dx;
+ }
+ mean_y + sxy / sxx * (x - mean_x)
+}
+
+/// The mean over `2 radius + 1` samples around each, fewer at the ends.
+fn box_mean(values: &[f64], radius: usize) -> Vec<f64> {
+ let n = values.len();
+ let mut prefix = vec![0.0; n + 1];
+ for (i, v) in values.iter().enumerate() {
+ prefix[i + 1] = prefix[i] + v;
+ }
+ (0..n)
+ .map(|i| {
+ let from = i.saturating_sub(radius);
+ let to = (i + radius + 1).min(n);
+ (prefix[to] - prefix[from]) / (to - from) as f64
+ })
+ .collect()
+}
+
+/// The least over `2 radius + 1` samples around each.
+fn box_min(values: &[f64], radius: usize) -> Vec<f64> {
+ let n = values.len();
+ (0..n)
+ .map(|i| {
+ let from = i.saturating_sub(radius);
+ let to = (i + radius + 1).min(n);
+ values[from..to].iter().copied().fold(f64::INFINITY, f64::min)
+ })
+ .collect()
+}
+
+/// The correction for the clip showing `from..to` seconds of source of
+/// the media whose movement is `motion`, at `speed` source seconds per
+/// timeline second.
+pub fn correct(
+ motion: &Motion,
+ (from, to): (f64, f64),
+ speed: f64,
+ settings: &Settings,
+) -> Correction {
+ let interval = motion.interval();
+ let speed = if speed.is_finite() { speed.abs().max(1.0 / 16.0) } else { 1.0 };
+ let sigma_source = sigma_seconds(settings.smoothness) * speed;
+ let sigma_frames = sigma_source / interval;
+ // The frames the clip shows, and the smoothing's reach either side,
+ // which it reads but does not correct for.
+ let reach = 3.0 * sigma_source + interval;
+ let first = motion
+ .frames
+ .partition_point(|frame| frame.at < from - reach - 1e-6);
+ let end = motion
+ .frames
+ .partition_point(|frame| frame.at <= to + reach + 1e-6);
+ let frames = &motion.frames[first..end.max(first)];
+ if frames.len() < 2 {
+ return Correction::none();
+ }
+ let aspect = motion.aspect;
+
+ // The path: where the window's first frame's scene sits in each frame.
+ let mut path: Vec<Similarity> = Vec::with_capacity(frames.len());
+ let mut camera = Similarity::IDENTITY;
+ for (i, frame) in frames.iter().enumerate() {
+ if i > 0 {
+ camera = camera.then(&frame.movement());
+ }
+ path.push(camera);
+ }
+ let mut poses: Vec<Pose> = Vec::with_capacity(path.len());
+ for similarity in &path {
+ let previous = poses.last().map(|pose: &Pose| pose.angle);
+ poses.push(Pose::of(similarity, previous));
+ }
+
+ // The frames on screen: from the one showing at `from` to the one at
+ // `to`, which is where the zoom is decided and the shake measured.
+ let shown_from = frames
+ .partition_point(|frame| frame.at <= from + 1e-4)
+ .saturating_sub(1);
+ let shown_to = frames
+ .partition_point(|frame| frame.at <= to + 1e-4)
+ .max(shown_from + 1);
+ let shown = shown_from..shown_to;
+
+ let smooth: Vec<Pose> = if settings.tripod {
+ let count = shown.len() as f64;
+ let mut mean = Pose::default();
+ for pose in &poses[shown.clone()] {
+ mean.tx += pose.tx / count;
+ mean.ty += pose.ty / count;
+ mean.angle += pose.angle / count;
+ mean.log_scale += pose.log_scale / count;
+ }
+ poses
+ .iter()
+ .map(|pose| Pose {
+ angle: if settings.keep_rotation {
+ pose.angle
+ } else {
+ mean.angle
+ },
+ ..mean
+ })
+ .collect()
+ } else {
+ let curve = |pick: fn(&Pose) -> f64| {
+ gaussian(&poses.iter().map(pick).collect::<Vec<_>>(), sigma_frames)
+ };
+ let (tx, ty, angle, log_scale) = (
+ curve(|p| p.tx),
+ curve(|p| p.ty),
+ curve(|p| p.angle),
+ curve(|p| p.log_scale),
+ );
+ (0..poses.len())
+ .map(|i| Pose {
+ tx: tx[i],
+ ty: ty[i],
+ angle: if settings.keep_rotation {
+ poses[i].angle
+ } else {
+ angle[i]
+ },
+ log_scale: log_scale[i],
+ })
+ .collect()
+ };
+
+ // How far towards the smooth path each frame goes: all the way, unless
+ // the zoom allowed cannot hide the edges that shows.
+ let map_at = |i: usize, towards: f64| -> Similarity {
+ let target = poses[i].lerp(&smooth[i], towards).similarity();
+ target.inverse().then(&path[i])
+ };
+ let full: Vec<Similarity> = (0..poses.len()).map(|i| map_at(i, 1.0)).collect();
+ let needed = shown
+ .clone()
+ .map(|i| zoom_needed(&full[i], aspect))
+ .fold(1.0, f64::max);
+ let cap = if settings.max_zoom.is_finite() {
+ settings.max_zoom.max(1.0)
+ } else {
+ 1.0
+ };
+ let zoom = needed.min(cap);
+ let mut maps = full;
+ if needed > zoom + 1e-9 {
+ // Each frame's own furthest, by bisection: none of the way always
+ // fits, since it is the frame as shot.
+ let mut eased: Vec<f64> = (0..poses.len())
+ .map(|i| {
+ if zoom_needed(&maps[i], aspect) <= zoom {
+ return 1.0;
+ }
+ let (mut low, mut high) = (0.0, 1.0);
+ for _ in 0..14 {
+ let mid = 0.5 * (low + high);
+ if zoom_needed(&map_at(i, mid), aspect) <= zoom {
+ low = mid;
+ } else {
+ high = mid;
+ }
+ }
+ low
+ })
+ .collect();
+ // Smoothed without rising past any frame's own: the least over a
+ // window, then the mean over the same window, each at most what
+ // every frame in reach allows.
+ let radius = sigma_frames.round().max(1.0) as usize;
+ let smoothed = box_mean(&box_min(&eased, radius), radius);
+ for (i, value) in smoothed.into_iter().enumerate() {
+ if zoom_needed(&map_at(i, value), aspect) <= zoom + 1e-9 {
+ eased[i] = value;
+ }
+ }
+ maps = (0..poses.len()).map(|i| map_at(i, eased[i])).collect();
+ }
+
+ // The shake: how far the frame's middle is from where the smooth
+ // camera would have had it.
+ let mut squares = 0.0;
+ for i in shown.clone() {
+ let (x, y) = maps[i].apply((0.0, 0.0));
+ squares += x * x + y * y;
+ }
+ let shake = (squares / shown.len() as f64).sqrt();
+
+ let unzoom = Similarity::scaling(1.0 / zoom);
+ Correction {
+ times: frames.iter().map(|frame| frame.at).collect(),
+ maps: maps.iter().map(|map| unzoom.then(map)).collect(),
+ interval,
+ zoom,
+ needed,
+ shake,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::stabilize::store::{FrameMotion, Step};
+
+ /// A clip at thirty frames a second whose camera shook by `shake(i)`
+ /// between frames.
+ fn shot(frames: usize, shake: impl Fn(usize) -> Similarity) -> Motion {
+ Motion {
+ aspect: 16.0 / 9.0,
+ covered: vec![(0.0, frames as f64 / 30.0)],
+ frames: (0..frames)
+ .map(|i| FrameMotion {
+ at: i as f64 / 30.0,
+ step: if i == 0 { Step::Start } else { Step::Tracked },
+ motion: shake(i),
+ })
+ .collect(),
+ }
+ }
+
+ fn settings(smoothness: f64, max_zoom: f64) -> Settings {
+ Settings {
+ smoothness,
+ max_zoom,
+ tripod: false,
+ keep_rotation: false,
+ }
+ }
+
+ #[test]
+ fn a_gaussian_keeps_a_line_and_flattens_a_wobble() {
+ let line: Vec<f64> = (0..50).map(|i| 0.3 + 0.01 * i as f64).collect();
+ let smoothed = gaussian(&line, 6.0);
+ for (a, b) in line.iter().zip(&smoothed) {
+ assert!((a - b).abs() < 1e-9, "a line through the ends stays a line");
+ }
+ let wobble: Vec<f64> = (0..200).map(|i| if i % 2 == 0 { 1.0 } else { -1.0 }).collect();
+ let smoothed = gaussian(&wobble, 4.0);
+ assert!(smoothed[20..180].iter().all(|v| v.abs() < 0.02));
+ }
+
+ #[test]
+ fn the_zoom_needed_hides_a_shift_and_a_turn() {
+ let aspect = 16.0 / 9.0;
+ assert!((zoom_needed(&Similarity::IDENTITY, aspect) - 1.0).abs() < 1e-12);
+ // A shift of a tenth of the height down: the bottom edge has 0.4
+ // of room for 0.5 of reach, so the picture zooms by 1.25.
+ let down = Similarity::new(0.0, 1.0, 0.0, 0.1);
+ assert!((zoom_needed(&down, aspect) - 1.25).abs() < 1e-9);
+ // And that zoom does keep every corner inside.
+ let zoomed = Similarity::scaling(1.0 / 1.25).then(&down);
+ for corner in [(-aspect / 2.0, -0.5), (aspect / 2.0, 0.5)] {
+ let (x, y) = zoomed.apply(corner);
+ assert!(x.abs() <= aspect / 2.0 + 1e-9 && y.abs() <= 0.5 + 1e-9);
+ }
+ // A turn needs more than nothing, and a turn the other way as much.
+ let left = zoom_needed(&Similarity::new(0.02, 1.0, 0.0, 0.0), aspect);
+ let right = zoom_needed(&Similarity::new(-0.02, 1.0, 0.0, 0.0), aspect);
+ assert!(left > 1.01 && (left - right).abs() < 1e-9);
+ }
+
+ /// A camera shaking a few pixels back and forth is held still: each
+ /// frame's map undoes where the shake put it.
+ #[test]
+ fn a_shake_is_corrected_and_zoomed_over() {
+ let h = 1080.0;
+ // Frame k sits 4 px right on odd frames: the step alternates.
+ let motion = shot(90, |i| {
+ let dx = if i % 2 == 1 { 4.0 / h } else { -4.0 / h };
+ Similarity::new(0.0, 1.0, dx, 0.0)
+ });
+ let correction = correct(&motion, (0.0, 3.0), 1.0, &settings(0.5, 1.3));
+ assert!(correction.zoom > 1.0 && correction.zoom < 1.01);
+ assert_eq!(correction.zoom, correction.needed);
+ for frame in 10..80 {
+ let at = frame as f64 / 30.0;
+ // The camera's offset at this frame, relative to its mean.
+ let offset = if frame % 2 == 1 { 2.0 / h } else { -2.0 / h };
+ // The picture's middle is looked for where the shake put it.
+ let (x, _) = correction.at(at).apply((0.0, 0.0));
+ assert!((x - offset).abs() < 0.2 / h, "frame {frame}: {} px", x * h);
+ }
+ assert!((correction.shake * h - 2.0).abs() < 0.2, "{}", correction.shake * h);
+ }
+
+ /// A steady pan stays a pan: nothing to correct, nothing to zoom.
+ #[test]
+ fn a_steady_pan_is_left_alone() {
+ let motion = shot(120, |_| Similarity::new(0.0, 1.0, 0.005, 0.0));
+ let correction = correct(&motion, (0.5, 3.5), 1.0, &settings(0.8, 1.3));
+ assert!(correction.zoom < 1.0 + 1e-6, "{}", correction.zoom);
+ assert!(correction.at(2.0).is_identity(1e-9));
+ }
+
+ /// Tripod locks the frame to one place: every frame's map undoes the
+ /// whole of the pan, around its mean.
+ #[test]
+ fn tripod_locks_the_frame() {
+ let motion = shot(60, |_| Similarity::new(0.0, 1.0, 0.001, 0.0));
+ let tripod = Settings {
+ tripod: true,
+ ..settings(0.5, 1.5)
+ };
+ let correction = correct(&motion, (0.0, 2.0), 1.0, &tripod);
+ // The scene point at the middle of the frame at the mean pose
+ // stays put: frame k's map sends the middle where frame k has it.
+ let first = correction.at(0.0).apply((0.0, 0.0)).0;
+ let last = correction.at(59.0 / 30.0).apply((0.0, 0.0)).0;
+ assert!(((last - first) - 59.0 * 0.001).abs() < 1e-6);
+ }
+
+ /// Asked for more than the zoom allows, the correction is eased
+ /// rather than showing an edge: every frame's map stays inside.
+ #[test]
+ fn a_capped_zoom_eases_the_correction_rather_than_show_an_edge() {
+ // A big lurch half way through.
+ let motion = shot(90, |i| {
+ Similarity::new(0.0, 1.0, if i == 45 { 0.15 } else { 0.0 }, 0.0)
+ });
+ let correction = correct(&motion, (0.0, 3.0), 1.0, &settings(1.0, 1.05));
+ assert!(correction.needed > 1.05);
+ assert!((correction.zoom - 1.05).abs() < 1e-12);
+ for frame in 0..90 {
+ let map = correction.at(frame as f64 / 30.0);
+ assert!(zoom_needed(&map, 16.0 / 9.0) <= 1.0 + 1e-6, "frame {frame}");
+ }
+ }
+
+ /// At four times the speed the same timeline smoothness reaches over
+ /// four times the source frames.
+ #[test]
+ fn a_faster_clip_is_smoothed_over_more_source() {
+ let motion = shot(600, |i| {
+ Similarity::new(0.0, 1.0, 0.002 * ((i as f64) * 0.15).sin(), 0.0)
+ });
+ let slow = correct(&motion, (5.0, 15.0), 1.0, &settings(0.3, 1.5));
+ let fast = correct(&motion, (5.0, 15.0), 4.0, &settings(0.3, 1.5));
+ assert!(fast.needed > slow.needed);
+ }
+
+ #[test]
+ fn outside_what_was_measured_is_left_as_shot() {
+ let motion = shot(30, |_| Similarity::new(0.01, 1.0, 0.003, 0.0));
+ let correction = correct(&motion, (0.0, 5.0), 1.0, &settings(0.5, 1.2));
+ assert!(correction.at(4.0).is_identity(0.0));
+ assert!(correction.at(-1.0).is_identity(0.0));
+ }
+}
diff --git a/src/crates/concat-vision/src/stabilize/store.rs b/src/crates/concat-vision/src/stabilize/store.rs
new file mode 100644
index 00000000..c28ef1cd
--- /dev/null
+++ b/src/crates/concat-vision/src/stabilize/store.rs
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Where a media file's measured movement lives between runs.
+//!
+//! One directory per media file per analysis version, inside the project's
+//! `cache/` folder beside the masks, so it travels with the project and a
+//! newer analysis never reads an older one's numbers. In it, one small
+//! text file, `motion`: the stretches of source measured, and for every
+//! frame in them its time and how the camera moved since the frame before.
+//! A few dozen bytes a frame; an hour of footage is a few megabytes.
+//!
+//! The host's analysis writes it whole, by way of a file beside it, so a
+//! reader never sees half of one. The renderer reads it through a
+//! process-wide cache keyed by the file's length and modification time,
+//! since a preview builds its plan again on every edit and the file only
+//! changes when an analysis finishes.
+
+use std::collections::HashMap;
+use std::fmt::Write as _;
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex, OnceLock};
+use std::time::SystemTime;
+
+use super::ANALYSIS_VERSION;
+use super::fit::Similarity;
+
+/// The directory a media file's movement is kept in under `project`.
+/// Named like the masks' directories, by a hash of the path, with the
+/// analysis's version after it.
+pub fn motion_dir(project: &Path, media_path: &str) -> PathBuf {
+ project.join("cache").join("stabilize").join(format!(
+ "{:016x}-v{ANALYSIS_VERSION}",
+ crate::store::fnv1a(media_path.as_bytes())
+ ))
+}
+
+/// The file in it.
+fn motion_file(dir: &Path) -> PathBuf {
+ dir.join("motion")
+}
+
+/// How a frame's movement was found.
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum Step {
+ /// The first frame of a stretch: nothing before it was measured, so
+ /// nothing is known of how it got here.
+ Start,
+ /// Measured: the tracks agreed.
+ Tracked,
+ /// Too little texture to measure. Taken as no movement.
+ Flat,
+}
+
+impl Step {
+ fn letter(self) -> char {
+ match self {
+ Step::Start => 's',
+ Step::Tracked => 't',
+ Step::Flat => 'f',
+ }
+ }
+
+ fn of(letter: &str) -> Option<Step> {
+ match letter {
+ "s" => Some(Step::Start),
+ "t" => Some(Step::Tracked),
+ "f" => Some(Step::Flat),
+ _ => None,
+ }
+ }
+}
+
+/// One frame's measured movement.
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct FrameMotion {
+ /// When the frame is shown, seconds of source.
+ pub at: f64,
+ /// How the movement was found.
+ pub step: Step,
+ /// Where a still point of the scene in the frame before went in this
+ /// one, normalised. The identity unless the step is `Tracked`.
+ pub motion: Similarity,
+}
+
+impl FrameMotion {
+ /// The movement to accumulate: the measured one, or none.
+ pub fn movement(&self) -> Similarity {
+ match self.step {
+ Step::Tracked => self.motion,
+ Step::Start | Step::Flat => Similarity::IDENTITY,
+ }
+ }
+}
+
+/// A media file's measured movement: what was measured, and every frame
+/// of it, in time order.
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct Motion {
+ /// The picture's width over its height, as analysed: what the frame's
+ /// edges are in normalised units.
+ pub aspect: f64,
+ /// The stretches of source measured, seconds, in order and apart.
+ pub covered: Vec<(f64, f64)>,
+ /// The frames, in time order.
+ pub frames: Vec<FrameMotion>,
+}
+
+/// Two times this close are one frame's.
+const SAME_FRAME: f64 = 1e-4;
+/// A gap in the coverage narrower than this is no gap: under a frame at
+/// the slowest rate anyone shoots.
+const SEAM: f64 = 0.05;
+
+impl Motion {
+ /// The movement cached in `dir`, or `None` when nothing is, or what is
+ /// there cannot be read. Read once per change of the file.
+ pub fn read(dir: &Path) -> Option<Arc<Motion>> {
+ static CACHE: OnceLock<Mutex<HashMap<PathBuf, (SystemTime, u64, Arc<Motion>)>>> =
+ OnceLock::new();
+ let file = motion_file(dir);
+ let meta = std::fs::metadata(&file).ok()?;
+ let stamp = (meta.modified().unwrap_or(SystemTime::UNIX_EPOCH), meta.len());
+ let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new()));
+ if let Ok(cache) = cache.lock()
+ && let Some((modified, len, motion)) = cache.get(&file)
+ && (*modified, *len) == stamp
+ {
+ return Some(Arc::clone(motion));
+ }
+ let text = std::fs::read_to_string(&file).ok()?;
+ let motion = Arc::new(Motion::parse(&text)?);
+ if let Ok(mut cache) = cache.lock() {
+ // A handful of media are stabilized at once; a long session
+ // with many is still a few megabytes.
+ if cache.len() > 64 {
+ cache.clear();
+ }
+ cache.insert(file, (stamp.0, stamp.1, Arc::clone(&motion)));
+ }
+ Some(motion)
+ }
+
+ /// The text form, read back. `None` for a file of another version or
+ /// none at all.
+ pub fn parse(text: &str) -> Option<Motion> {
+ let mut lines = text.lines();
+ let header = lines.next()?;
+ if header.trim() != format!("concat-stabilize {ANALYSIS_VERSION}") {
+ return None;
+ }
+ let mut motion = Motion::default();
+ for line in lines {
+ let mut words = line.split_ascii_whitespace();
+ match words.next() {
+ Some("aspect") => motion.aspect = words.next()?.parse().ok()?,
+ Some("covered") => {
+ let from: f64 = words.next()?.parse().ok()?;
+ let to: f64 = words.next()?.parse().ok()?;
+ motion.covered.push((from, to));
+ }
+ Some("f") => {
+ let at: f64 = words.next()?.parse().ok()?;
+ let step = Step::of(words.next()?)?;
+ let mut numbers = [0.0f64; 4];
+ for number in &mut numbers {
+ *number = words.next()?.parse().ok()?;
+ }
+ let [a, b, tx, ty] = numbers;
+ motion.frames.push(FrameMotion {
+ at,
+ step,
+ motion: Similarity { a, b, tx, ty },
+ });
+ }
+ _ => {}
+ }
+ }
+ if !(motion.aspect.is_finite() && motion.aspect > 0.0) {
+ return None;
+ }
+ motion.frames.sort_by(|a, b| a.at.total_cmp(&b.at));
+ Some(motion)
+ }
+
+ /// The text form.
+ pub fn text(&self) -> String {
+ let mut out = format!("concat-stabilize {ANALYSIS_VERSION}\naspect {}\n", self.aspect);
+ for (from, to) in &self.covered {
+ let _ = writeln!(out, "covered {from:.6} {to:.6}");
+ }
+ for frame in &self.frames {
+ let m = frame.motion;
+ let _ = writeln!(
+ out,
+ "f {:.6} {} {:.9} {:.9} {:.9} {:.9}",
+ frame.at,
+ frame.step.letter(),
+ m.a,
+ m.b,
+ m.tx,
+ m.ty
+ );
+ }
+ out
+ }
+
+ /// Writes the movement to `dir`, whole or not at all.
+ pub fn write(&self, dir: &Path) -> Result<(), String> {
+ std::fs::create_dir_all(dir)
+ .map_err(|error| format!("could not create {}: {error}", dir.display()))?;
+ let file = motion_file(dir);
+ let partial = file.with_extension("part");
+ std::fs::write(&partial, self.text())
+ .map_err(|error| format!("could not write {}: {error}", partial.display()))?;
+ std::fs::rename(&partial, &file)
+ .map_err(|error| format!("could not finish {}: {error}", file.display()))
+ }
+
+ /// The parts of `ranges` not measured yet, in order, each a stretch of
+ /// source seconds.
+ pub fn missing(&self, ranges: &[(f64, f64)]) -> Vec<(f64, f64)> {
+ let mut wanted = merged(ranges.to_vec());
+ for &(from, to) in &self.covered {
+ wanted = wanted
+ .into_iter()
+ .flat_map(|(a, b)| {
+ if b <= from || a >= to {
+ return vec![(a, b)];
+ }
+ let mut left = Vec::new();
+ if a < from {
+ left.push((a, from));
+ }
+ if b > to {
+ left.push((to, b));
+ }
+ left
+ })
+ .filter(|(a, b)| b - a > SEAM)
+ .collect();
+ }
+ wanted
+ }
+
+ /// Takes in one measured stretch: its frames, and the span of source
+ /// it covers. A frame already here is kept, unless it began a stretch
+ /// and this one says how it was reached.
+ pub fn merge(&mut self, run: &[FrameMotion], span: (f64, f64)) {
+ for frame in run {
+ let index = self.frames.partition_point(|f| f.at < frame.at - SAME_FRAME);
+ match self.frames.get_mut(index) {
+ Some(existing) if (existing.at - frame.at).abs() <= SAME_FRAME => {
+ if existing.step == Step::Start && frame.step != Step::Start {
+ *existing = *frame;
+ }
+ }
+ _ => self.frames.insert(index, *frame),
+ }
+ }
+ self.covered.push(span);
+ self.covered = merged(std::mem::take(&mut self.covered));
+ }
+
+ /// The usual time between frames: the median, so a dropped frame or a
+ /// seam between stretches does not count. A thirtieth with too few
+ /// frames to say.
+ pub fn interval(&self) -> f64 {
+ let mut gaps: Vec<f64> = self
+ .frames
+ .windows(2)
+ .map(|pair| pair[1].at - pair[0].at)
+ .filter(|gap| *gap > SAME_FRAME)
+ .collect();
+ if gaps.is_empty() {
+ return 1.0 / 30.0;
+ }
+ let middle = gaps.len() / 2;
+ let (_, median, _) = gaps.select_nth_unstable_by(middle, f64::total_cmp);
+ *median
+ }
+}
+
+/// Stretches sorted and joined where they meet or overlap.
+fn merged(mut ranges: Vec<(f64, f64)>) -> Vec<(f64, f64)> {
+ ranges.retain(|(a, b)| a.is_finite() && b.is_finite() && b > a);
+ ranges.sort_by(|a, b| a.0.total_cmp(&b.0));
+ let mut out: Vec<(f64, f64)> = Vec::new();
+ for (a, b) in ranges {
+ match out.last_mut() {
+ Some(last) if a <= last.1 + SEAM => last.1 = last.1.max(b),
+ _ => out.push((a, b)),
+ }
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn frame(at: f64, step: Step) -> FrameMotion {
+ FrameMotion {
+ at,
+ step,
+ motion: Similarity::new(0.001 * at, 1.0, 0.002, -0.001),
+ }
+ }
+
+ #[test]
+ fn the_text_form_reads_back() {
+ let mut motion = Motion {
+ aspect: 16.0 / 9.0,
+ ..Motion::default()
+ };
+ motion.merge(
+ &[
+ frame(0.0, Step::Start),
+ frame(1.0 / 30.0, Step::Tracked),
+ frame(2.0 / 30.0, Step::Flat),
+ ],
+ (0.0, 0.1),
+ );
+ let back = Motion::parse(&motion.text()).expect("reads");
+ assert_eq!(back.covered, motion.covered);
+ assert_eq!(back.frames.len(), 3);
+ assert!((back.frames[1].motion.b - motion.frames[1].motion.b).abs() < 1e-9);
+ assert_eq!(back.frames[2].step, Step::Flat);
+ assert!(Motion::parse("concat-stabilize 0\naspect 1\n").is_none());
+ }
+
+ #[test]
+ fn what_is_missing_is_what_is_not_covered() {
+ let motion = Motion {
+ aspect: 1.0,
+ covered: vec![(2.0, 5.0), (8.0, 9.0)],
+ frames: Vec::new(),
+ };
+ assert_eq!(
+ motion.missing(&[(0.0, 10.0)]),
+ vec![(0.0, 2.0), (5.0, 8.0), (9.0, 10.0)]
+ );
+ assert!(motion.missing(&[(2.5, 4.0), (8.0, 9.02)]).is_empty());
+ }
+
+ /// A stretch measured later joins the one before it: the frame that
+ /// began the earlier stretch learns how it was reached.
+ #[test]
+ fn a_later_stretch_joins_an_earlier_one() {
+ let mut motion = Motion {
+ aspect: 1.0,
+ ..Motion::default()
+ };
+ motion.merge(&[frame(1.0, Step::Start), frame(1.1, Step::Tracked)], (1.0, 1.1));
+ motion.merge(
+ &[
+ frame(0.8, Step::Start),
+ frame(0.9, Step::Tracked),
+ frame(1.0, Step::Tracked),
+ ],
+ (0.8, 1.0),
+ );
+ let steps: Vec<Step> = motion.frames.iter().map(|f| f.step).collect();
+ assert_eq!(
+ steps,
+ vec![Step::Start, Step::Tracked, Step::Tracked, Step::Tracked]
+ );
+ assert_eq!(motion.covered, vec![(0.8, 1.1)]);
+ assert!((motion.interval() - 0.1).abs() < 1e-9);
+ }
+}
diff --git a/src/crates/concat-vision/src/stabilize/track.rs b/src/crates/concat-vision/src/stabilize/track.rs
new file mode 100644
index 00000000..bee2dc82
--- /dev/null
+++ b/src/crates/concat-vision/src/stabilize/track.rs
@@ -0,0 +1,713 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! The camera's movement between two frames, measured.
+//!
+//! Corners are where tracking works: a point on an edge can slide along
+//! it, a point on a flat wall can be anywhere. Shi-Tomasi's measure - the
+//! smaller eigenvalue of the gradients' structure around a pixel - says
+//! how cornery a pixel is, and the strongest are taken cell by cell over
+//! a grid so the tracks cover the frame rather than bunch on one busy
+//! patch. Each is followed into the next frame by Lucas-Kanade optical
+//! flow, coarse to fine over an image pyramid so a fast move is found at
+//! a size where it is small, and then followed back again: a track that
+//! does not return to where it started was lost, and is dropped.
+//!
+//! The tracks that survive are fitted by [`super::fit::ransac`]. Its
+//! agreeing tracks carry on into the next frame; when too few are left,
+//! the grid is topped up with fresh corners in the cells they left empty.
+//! A frame with too little texture to find any - a white wall, a black
+//! frame - is said to be flat, and the caller records no movement there.
+
+use concat_core::frame::Frame;
+
+use super::fit::{self, Pair, Similarity};
+
+/// One channel of a picture, as floats: brightness `0..=255`.
+#[derive(Clone, Debug)]
+pub struct Gray {
+ /// Pixels across.
+ pub width: usize,
+ /// Pixels down.
+ pub height: usize,
+ /// Row by row.
+ pub data: Vec<f32>,
+}
+
+impl Gray {
+ /// A picture of one brightness.
+ pub fn filled(width: usize, height: usize, value: f32) -> Gray {
+ Gray {
+ width,
+ height,
+ data: vec![value; width * height],
+ }
+ }
+
+ /// A frame's luma, Rec. 601 weights: what the eye calls detail.
+ pub fn of_frame(frame: &Frame) -> Gray {
+ let (width, height) = (frame.width() as usize, frame.height() as usize);
+ let data = frame
+ .pixels()
+ .chunks_exact(4)
+ .map(|px| 0.299 * f32::from(px[0]) + 0.587 * f32::from(px[1]) + 0.114 * f32::from(px[2]))
+ .collect();
+ Gray {
+ width,
+ height,
+ data,
+ }
+ }
+
+ #[inline]
+ fn at(&self, x: isize, y: isize) -> f32 {
+ let x = x.clamp(0, self.width as isize - 1) as usize;
+ let y = y.clamp(0, self.height as isize - 1) as usize;
+ self.data[y * self.width + x]
+ }
+
+ /// The brightness at `(x, y)`, pixel centres at whole numbers, read
+ /// bilinearly with the edges held.
+ #[inline]
+ pub fn sample(&self, x: f32, y: f32) -> f32 {
+ let x0 = x.floor();
+ let y0 = y.floor();
+ let (fx, fy) = (x - x0, y - y0);
+ let (x0, y0) = (x0 as isize, y0 as isize);
+ if x0 >= 0 && y0 >= 0 && (x0 as usize) + 1 < self.width && (y0 as usize) + 1 < self.height
+ {
+ let at = y0 as usize * self.width + x0 as usize;
+ let (a, b) = (self.data[at], self.data[at + 1]);
+ let (c, d) = (self.data[at + self.width], self.data[at + self.width + 1]);
+ let top = a + (b - a) * fx;
+ let bottom = c + (d - c) * fx;
+ return top + (bottom - top) * fy;
+ }
+ let top = self.at(x0, y0) * (1.0 - fx) + self.at(x0 + 1, y0) * fx;
+ let bottom = self.at(x0, y0 + 1) * (1.0 - fx) + self.at(x0 + 1, y0 + 1) * fx;
+ top * (1.0 - fy) + bottom * fy
+ }
+
+ /// Half the size each way, blurred first by the binomial five-tap so
+ /// what is dropped cannot alias into what is kept.
+ fn half(&self) -> Gray {
+ const TAPS: [f32; 5] = [1.0 / 16.0, 4.0 / 16.0, 6.0 / 16.0, 4.0 / 16.0, 1.0 / 16.0];
+ let (w, h) = (self.width, self.height);
+ let (hw, hh) = (w.div_ceil(2), h.div_ceil(2));
+ // Across, at every row and every other column.
+ let mut across = vec![0.0f32; hw * h];
+ for y in 0..h {
+ for x in 0..hw {
+ let cx = (2 * x) as isize;
+ let mut sum = 0.0;
+ for (k, tap) in TAPS.iter().enumerate() {
+ sum += tap * self.at(cx + k as isize - 2, y as isize);
+ }
+ across[y * hw + x] = sum;
+ }
+ }
+ let across = Gray {
+ width: hw,
+ height: h,
+ data: across,
+ };
+ let mut data = vec![0.0f32; hw * hh];
+ for y in 0..hh {
+ let cy = (2 * y) as isize;
+ for x in 0..hw {
+ let mut sum = 0.0;
+ for (k, tap) in TAPS.iter().enumerate() {
+ sum += tap * across.at(x as isize, cy + k as isize - 2);
+ }
+ data[y * hw + x] = sum;
+ }
+ }
+ Gray {
+ width: hw,
+ height: hh,
+ data,
+ }
+ }
+
+ /// Scharr's derivatives across and down, in brightness per pixel.
+ fn gradients(&self) -> (Gray, Gray) {
+ let (w, h) = (self.width, self.height);
+ let mut gx = vec![0.0f32; w * h];
+ let mut gy = vec![0.0f32; w * h];
+ for y in 0..h as isize {
+ for x in 0..w as isize {
+ let at = y as usize * w + x as usize;
+ gx[at] = (3.0 * (self.at(x + 1, y - 1) - self.at(x - 1, y - 1))
+ + 10.0 * (self.at(x + 1, y) - self.at(x - 1, y))
+ + 3.0 * (self.at(x + 1, y + 1) - self.at(x - 1, y + 1)))
+ / 32.0;
+ gy[at] = (3.0 * (self.at(x - 1, y + 1) - self.at(x - 1, y - 1))
+ + 10.0 * (self.at(x, y + 1) - self.at(x, y - 1))
+ + 3.0 * (self.at(x + 1, y + 1) - self.at(x + 1, y - 1)))
+ / 32.0;
+ }
+ }
+ (
+ Gray {
+ width: w,
+ height: h,
+ data: gx,
+ },
+ Gray {
+ width: w,
+ height: h,
+ data: gy,
+ },
+ )
+ }
+}
+
+/// One level of a pyramid: the picture and its derivatives.
+struct Level {
+ image: Gray,
+ gx: Gray,
+ gy: Gray,
+}
+
+/// A picture at full size and at every halving down to a few dozen
+/// pixels, each with its derivatives.
+pub struct Pyramid {
+ levels: Vec<Level>,
+}
+
+/// The tracking window's half-width: fifteen pixels square.
+const RADIUS: isize = 7;
+/// The most halvings: at 480 pixels, the top is 60 across, where a move of
+/// a hundred pixels at full size is a dozen.
+const LEVELS: usize = 4;
+/// Steps of the flow per level before it stops refining.
+const ITERATIONS: usize = 20;
+/// A step this small, in pixels, is converged.
+const CONVERGED: f32 = 0.01;
+/// A track whose way back lands further than this from its start, in
+/// full-size pixels, was lost.
+const ROUND_TRIP: f32 = 0.7;
+
+impl Pyramid {
+ /// The pyramid of `image`.
+ pub fn of(image: Gray) -> Pyramid {
+ let mut levels = Vec::with_capacity(LEVELS);
+ let mut current = image;
+ loop {
+ let (gx, gy) = current.gradients();
+ let small = current.width.min(current.height);
+ let next = (levels.len() + 1 < LEVELS && small >= 48).then(|| current.half());
+ levels.push(Level {
+ image: current,
+ gx,
+ gy,
+ });
+ match next {
+ Some(next) => current = next,
+ None => break,
+ }
+ }
+ Pyramid { levels }
+ }
+
+ /// The full-size picture.
+ pub fn base(&self) -> &Gray {
+ &self.levels[0].image
+ }
+
+ /// Where the point at `from` in this picture went in `next`, or `None`
+ /// when the flow lost it: out of the picture, or on texture too weak
+ /// to say. Positions in full-size pixels, centres at whole numbers.
+ pub fn flow(&self, next: &Pyramid, from: (f32, f32)) -> Option<(f32, f32)> {
+ let levels = self.levels.len().min(next.levels.len());
+ let side = (2 * RADIUS + 1) as usize;
+ let mut window_i = vec![0.0f32; side * side];
+ let mut window_x = vec![0.0f32; side * side];
+ let mut window_y = vec![0.0f32; side * side];
+ let mut guess = (0.0f32, 0.0f32);
+ for level in (0..levels).rev() {
+ let scale = 1.0 / (1u32 << level) as f32;
+ let here = &self.levels[level];
+ let there = &next.levels[level].image;
+ let (px, py) = (from.0 * scale, from.1 * scale);
+ let (w, h) = (here.image.width as f32, here.image.height as f32);
+ if px < 0.0 || py < 0.0 || px > w - 1.0 || py > h - 1.0 {
+ return None;
+ }
+ let (mut gxx, mut gxy, mut gyy) = (0.0f32, 0.0f32, 0.0f32);
+ let mut k = 0;
+ for dy in -RADIUS..=RADIUS {
+ for dx in -RADIUS..=RADIUS {
+ let (x, y) = (px + dx as f32, py + dy as f32);
+ let ix = here.gx.sample(x, y);
+ let iy = here.gy.sample(x, y);
+ window_i[k] = here.image.sample(x, y);
+ window_x[k] = ix;
+ window_y[k] = iy;
+ gxx += ix * ix;
+ gxy += ix * iy;
+ gyy += iy * iy;
+ k += 1;
+ }
+ }
+ let det = gxx * gyy - gxy * gxy;
+ let area = (side * side) as f32;
+ let min_eigen = 0.5 * (gxx + gyy - ((gxx - gyy).powi(2) + 4.0 * gxy * gxy).sqrt());
+ if det.abs() < 1e-6 || min_eigen / area < 0.25 {
+ if level == 0 {
+ return None;
+ }
+ guess = (guess.0 * 2.0, guess.1 * 2.0);
+ continue;
+ }
+ let mut step = (0.0f32, 0.0f32);
+ for _ in 0..ITERATIONS {
+ let (qx, qy) = (px + guess.0 + step.0, py + guess.1 + step.1);
+ if qx < -(RADIUS as f32)
+ || qy < -(RADIUS as f32)
+ || qx > w - 1.0 + RADIUS as f32
+ || qy > h - 1.0 + RADIUS as f32
+ {
+ return None;
+ }
+ let (mut bx, mut by) = (0.0f32, 0.0f32);
+ let mut k = 0;
+ for dy in -RADIUS..=RADIUS {
+ for dx in -RADIUS..=RADIUS {
+ let residual =
+ window_i[k] - there.sample(qx + dx as f32, qy + dy as f32);
+ bx += residual * window_x[k];
+ by += residual * window_y[k];
+ k += 1;
+ }
+ }
+ let ex = (gyy * bx - gxy * by) / det;
+ let ey = (gxx * by - gxy * bx) / det;
+ step.0 += ex;
+ step.1 += ey;
+ if ex * ex + ey * ey < CONVERGED * CONVERGED {
+ break;
+ }
+ }
+ guess = (guess.0 + step.0, guess.1 + step.1);
+ if level > 0 {
+ guess = (guess.0 * 2.0, guess.1 * 2.0);
+ }
+ }
+ let to = (from.0 + guess.0, from.1 + guess.1);
+ let base = &next.levels[0].image;
+ (to.0 >= 0.0
+ && to.1 >= 0.0
+ && to.0 <= base.width as f32 - 1.0
+ && to.1 <= base.height as f32 - 1.0)
+ .then_some(to)
+ }
+
+ /// The strongest corners, spread cell by cell over a grid of cells
+ /// about `cell` pixels square, at most `per_cell` in each, and none
+ /// nearer than `spacing` pixels to another or to one of `taken`.
+ pub fn corners(
+ &self,
+ cell: usize,
+ per_cell: usize,
+ spacing: f32,
+ taken: &[(f32, f32)],
+ ) -> Vec<(f32, f32)> {
+ let level = &self.levels[0];
+ let (w, h) = (level.image.width, level.image.height);
+ let margin = RADIUS as usize + 3;
+ if w <= 2 * margin + 2 || h <= 2 * margin + 2 {
+ return Vec::new();
+ }
+ // The structure tensor summed over five by five, by running sums.
+ let products: [Vec<f32>; 3] = [
+ level.gx.data.iter().map(|g| g * g).collect(),
+ level
+ .gx
+ .data
+ .iter()
+ .zip(&level.gy.data)
+ .map(|(a, b)| a * b)
+ .collect(),
+ level.gy.data.iter().map(|g| g * g).collect(),
+ ];
+ let boxed: Vec<Vec<f32>> = products.iter().map(|p| box5(p, w, h)).collect();
+ let mut score = vec![0.0f32; w * h];
+ let mut strongest = 0.0f32;
+ for y in margin..h - margin {
+ for x in margin..w - margin {
+ let at = y * w + x;
+ let (a, b, c) = (boxed[0][at], boxed[1][at], boxed[2][at]);
+ let lambda = 0.5 * (a + c - ((a - c) * (a - c) + 4.0 * b * b).sqrt());
+ score[at] = lambda;
+ strongest = strongest.max(lambda);
+ }
+ }
+ // Weaker than a hundredth of the best, or than an average gradient
+ // of about one level a pixel, is not a corner worth following.
+ let floor = (strongest * 0.01).max(25.0);
+ let columns = w.div_ceil(cell).max(1);
+ let rows = h.div_ceil(cell).max(1);
+ let mut cells: Vec<Vec<(f32, usize)>> = vec![Vec::new(); columns * rows];
+ for y in margin..h - margin {
+ for x in margin..w - margin {
+ let at = y * w + x;
+ let s = score[at];
+ if s < floor {
+ continue;
+ }
+ // A local maximum over its eight neighbours.
+ let peak = [
+ at - w - 1,
+ at - w,
+ at - w + 1,
+ at - 1,
+ at + 1,
+ at + w - 1,
+ at + w,
+ at + w + 1,
+ ]
+ .iter()
+ .all(|&n| score[n] <= s);
+ if peak {
+ cells[(y / cell) * columns + x / cell].push((s, at));
+ }
+ }
+ }
+ let mut chosen: Vec<(f32, f32)> = Vec::new();
+ let near = |p: (f32, f32), list: &[(f32, f32)]| {
+ list.iter()
+ .any(|q| (p.0 - q.0).powi(2) + (p.1 - q.1).powi(2) < spacing * spacing)
+ };
+ // A cell already holding tracks has that much less room.
+ let mut held = vec![0usize; columns * rows];
+ for p in taken {
+ let (cx, cy) = (p.0.max(0.0) as usize / cell, p.1.max(0.0) as usize / cell);
+ if cx < columns && cy < rows {
+ held[cy * columns + cx] += 1;
+ }
+ }
+ for (index, candidates) in cells.iter_mut().enumerate() {
+ candidates.sort_by(|a, b| b.0.total_cmp(&a.0));
+ let mut room = per_cell.saturating_sub(held[index]);
+ for &(_, at) in candidates.iter() {
+ if room == 0 {
+ break;
+ }
+ let p = ((at % w) as f32, (at / w) as f32);
+ if near(p, taken) || near(p, &chosen) {
+ continue;
+ }
+ chosen.push(p);
+ room -= 1;
+ }
+ }
+ chosen
+ }
+}
+
+/// `values` summed over the five by five around each pixel, the edges
+/// held.
+fn box5(values: &[f32], w: usize, h: usize) -> Vec<f32> {
+ let mut across = vec![0.0f32; w * h];
+ for y in 0..h {
+ let row = &values[y * w..(y + 1) * w];
+ for x in 0..w {
+ let mut sum = 0.0;
+ for d in -2isize..=2 {
+ sum += row[(x as isize + d).clamp(0, w as isize - 1) as usize];
+ }
+ across[y * w + x] = sum;
+ }
+ }
+ let mut out = vec![0.0f32; w * h];
+ for y in 0..h {
+ for x in 0..w {
+ let mut sum = 0.0;
+ for d in -2isize..=2 {
+ sum += across[(y as isize + d).clamp(0, h as isize - 1) as usize * w + x];
+ }
+ out[y * w + x] = sum;
+ }
+ }
+ out
+}
+
+/// What one frame told the tracker.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum Measured {
+ /// The first frame: nothing to measure against.
+ First,
+ /// The camera moved like this from the frame before, in normalised
+ /// units: where a still point of the scene was, to where it is now.
+ Moved(Similarity),
+ /// Too little texture, or too little agreement, to say.
+ Flat,
+}
+
+/// Cells about this many pixels square at the analysis size: eight across
+/// a 480-pixel frame.
+const CELL: usize = 60;
+/// Corners a cell may hold.
+const PER_CELL: usize = 5;
+/// The nearest two corners may be, in pixels.
+const SPACING: f32 = 8.0;
+/// A track agreeing with the camera lands within this many pixels of
+/// where the camera's movement sends it.
+const AGREE: f64 = 1.5;
+/// Fewer agreeing tracks than this is no answer.
+const MIN_AGREEING: usize = 10;
+
+/// Follows the camera through a run of frames, one at a time.
+pub struct Tracker {
+ previous: Option<Pyramid>,
+ points: Vec<(f32, f32)>,
+ threads: usize,
+}
+
+impl Default for Tracker {
+ fn default() -> Self {
+ Tracker::new()
+ }
+}
+
+impl Tracker {
+ /// A tracker that has seen nothing.
+ pub fn new() -> Tracker {
+ Tracker {
+ previous: None,
+ points: Vec::new(),
+ threads: std::thread::available_parallelism()
+ .map(|n| n.get())
+ .unwrap_or(1)
+ .clamp(1, 8),
+ }
+ }
+
+ /// Forgets the frames seen: the next one starts a new run.
+ pub fn reset(&mut self) {
+ self.previous = None;
+ self.points.clear();
+ }
+
+ /// Takes the next frame of the run and measures how the camera moved
+ /// from the one before.
+ pub fn push(&mut self, image: Gray) -> Measured {
+ let next = Pyramid::of(image);
+ let Some(previous) = self.previous.take() else {
+ self.points = next.corners(CELL, PER_CELL, SPACING, &[]);
+ self.previous = Some(next);
+ return Measured::First;
+ };
+ let (w, h) = (next.base().width, next.base().height);
+ let tracked = self.follow(&previous, &next);
+ let normal = |p: (f32, f32)| {
+ (
+ (f64::from(p.0) + 0.5 - w as f64 / 2.0) / h as f64,
+ (f64::from(p.1) + 0.5 - h as f64 / 2.0) / h as f64,
+ )
+ };
+ let pairs: Vec<Pair> = tracked
+ .iter()
+ .map(|&(from, to)| (normal(from), normal(to)))
+ .collect();
+ let robust = fit::ransac(&pairs, AGREE / h as f64, MIN_AGREEING);
+ let (measured, mut survivors) = match &robust {
+ Some(robust) => (
+ Measured::Moved(robust.similarity),
+ tracked
+ .iter()
+ .zip(&robust.inliers)
+ .filter(|(_, inlier)| **inlier)
+ .map(|((_, to), _)| *to)
+ .collect::<Vec<_>>(),
+ ),
+ None => (Measured::Flat, Vec::new()),
+ };
+ // Topped up when a good share of the grid has gone quiet.
+ let full = w.div_ceil(CELL) * h.div_ceil(CELL) * PER_CELL;
+ if survivors.len() < full * 3 / 5 {
+ let fresh = next.corners(CELL, PER_CELL, SPACING, &survivors);
+ survivors.extend(fresh);
+ }
+ self.points = survivors;
+ self.previous = Some(next);
+ measured
+ }
+
+ /// Every point followed from `previous` into `next` and back, the
+ /// ones that made the round trip paired with where they went. Split
+ /// over a few threads: the flow is most of an analysis's time.
+ fn follow(&self, previous: &Pyramid, next: &Pyramid) -> Vec<((f32, f32), (f32, f32))> {
+ let one = |from: (f32, f32)| -> Option<((f32, f32), (f32, f32))> {
+ let to = previous.flow(next, from)?;
+ let back = next.flow(previous, to)?;
+ ((back.0 - from.0).powi(2) + (back.1 - from.1).powi(2) <= ROUND_TRIP * ROUND_TRIP)
+ .then_some((from, to))
+ };
+ let chunk = self.points.len().div_ceil(self.threads).max(16);
+ if self.threads <= 1 || self.points.len() <= chunk {
+ return self.points.iter().filter_map(|&p| one(p)).collect();
+ }
+ std::thread::scope(|scope| {
+ let handles: Vec<_> = self
+ .points
+ .chunks(chunk)
+ .map(|part| scope.spawn(move || part.iter().filter_map(|&p| one(p)).collect::<Vec<_>>()))
+ .collect();
+ handles
+ .into_iter()
+ .flat_map(|handle| handle.join().unwrap_or_default())
+ .collect()
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// A picture with texture everywhere: a sum of waves of a few
+ /// periods and angles, not periodic over the frame, with corners
+ /// wherever two cross.
+ fn scene(x: f64, y: f64) -> f64 {
+ let mut v = 128.0;
+ for (k, (fx, fy, phase)) in [
+ (0.071, 0.023, 0.3),
+ (-0.031, 0.083, 1.1),
+ (0.11, 0.097, 2.0),
+ (0.017, -0.13, 0.7),
+ (0.29, 0.05, 1.7),
+ ]
+ .iter()
+ .enumerate()
+ {
+ v += (30.0 / (k as f64 + 1.0).sqrt()) * (x * fx + y * fy + phase).sin();
+ }
+ // Soft-edged squares, for corners proper: a lens never draws an
+ // edge sharper than a pixel.
+ v + 25.0 * (3.0 * (x * std::f64::consts::PI / 23.0).sin() * (y * std::f64::consts::PI / 19.0).sin()).tanh()
+ }
+
+ /// The scene seen through `camera` (normalised units): each pixel of
+ /// the frame shows the scene point the inverse movement brings it
+ /// from.
+ fn frame(width: usize, height: usize, camera: &Similarity) -> Gray {
+ let back = camera.inverse();
+ let h = height as f64;
+ let mut data = Vec::with_capacity(width * height);
+ for y in 0..height {
+ for x in 0..width {
+ let n = (
+ (x as f64 + 0.5 - width as f64 / 2.0) / h,
+ (y as f64 + 0.5 - h / 2.0) / h,
+ );
+ let (sx, sy) = back.apply(n);
+ let (px, py) = (sx * h + width as f64 / 2.0, sy * h + h / 2.0);
+ // Supersampled four times, as a lens integrates.
+ let mut sum = 0.0;
+ for (ox, oy) in [(-0.25, -0.25), (0.25, -0.25), (-0.25, 0.25), (0.25, 0.25)] {
+ sum += scene(px + ox, py + oy);
+ }
+ data.push((sum / 4.0) as f32);
+ }
+ }
+ Gray {
+ width,
+ height,
+ data,
+ }
+ }
+
+ fn moved(measured: Measured) -> Similarity {
+ match measured {
+ Measured::Moved(similarity) => similarity,
+ other => panic!("expected a movement, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn a_shift_is_recovered() {
+ let (w, h) = (320, 180);
+ let shift = Similarity::new(0.0, 1.0, 3.4 / h as f64, -2.2 / h as f64);
+ let mut tracker = Tracker::new();
+ assert_eq!(
+ tracker.push(frame(w, h, &Similarity::IDENTITY)),
+ Measured::First
+ );
+ let found = moved(tracker.push(frame(w, h, &shift)));
+ let px = |v: f64| v * h as f64;
+ assert!(
+ (px(found.tx) - 3.4).abs() < 0.05 && (px(found.ty) + 2.2).abs() < 0.05,
+ "{found:?}"
+ );
+ assert!(found.angle().abs() < 0.002 && (found.scale() - 1.0).abs() < 0.002);
+ }
+
+ #[test]
+ fn a_turn_and_a_zoom_are_recovered() {
+ let (w, h) = (320, 180);
+ let camera = Similarity::new(1.5f64.to_radians(), 1.02, -4.0 / h as f64, 1.0 / h as f64);
+ let mut tracker = Tracker::new();
+ tracker.push(frame(w, h, &Similarity::IDENTITY));
+ let found = moved(tracker.push(frame(w, h, &camera)));
+ assert!(
+ (found.angle().to_degrees() - 1.5).abs() < 0.05,
+ "angle {}",
+ found.angle().to_degrees()
+ );
+ assert!((found.scale() - 1.02).abs() < 0.002, "scale {}", found.scale());
+ assert!(((found.tx - camera.tx) * h as f64).abs() < 0.2);
+ assert!(((found.ty - camera.ty) * h as f64).abs() < 0.2);
+ }
+
+ /// A move of twenty pixels is found through the pyramid, and the
+ /// tracks carry on into a third frame.
+ #[test]
+ fn a_fast_move_is_found_and_tracks_carry_on() {
+ let (w, h) = (320, 180);
+ let first = Similarity::new(0.0, 1.0, 20.0 / h as f64, 6.0 / h as f64);
+ let second = Similarity::new(0.0, 1.0, 26.0 / h as f64, 2.0 / h as f64);
+ let mut tracker = Tracker::new();
+ tracker.push(frame(w, h, &Similarity::IDENTITY));
+ let found = moved(tracker.push(frame(w, h, &first)));
+ assert!(((found.tx * h as f64) - 20.0).abs() < 0.15, "{found:?}");
+ let found = moved(tracker.push(frame(w, h, &second)));
+ assert!(((found.tx * h as f64) - 6.0).abs() < 0.15, "{found:?}");
+ assert!(((found.ty * h as f64) + 4.0).abs() < 0.15, "{found:?}");
+ }
+
+ /// How long a frame takes at the analysis size; run with `--ignored
+ /// --nocapture` in release to see it.
+ #[test]
+ #[ignore]
+ fn time_a_frame_at_the_analysis_size() {
+ let (w, h) = (480, 270);
+ let frames: Vec<Gray> = (0..31)
+ .map(|i| {
+ let t = i as f64;
+ let camera = Similarity::new(
+ 0.002 * (t * 0.7).sin(),
+ 1.0,
+ (3.0 * (t * 1.3).sin() + t) / h as f64,
+ (2.0 * (t * 0.9).cos()) / h as f64,
+ );
+ frame(w, h, &camera)
+ })
+ .collect();
+ let mut tracker = Tracker::new();
+ let started = std::time::Instant::now();
+ for frame in frames {
+ tracker.push(frame);
+ }
+ let each = started.elapsed().as_secs_f64() * 1000.0 / 31.0;
+ println!("{w}x{h}: {each:.2} ms a frame, {} tracks", tracker.points.len());
+ }
+
+ #[test]
+ fn a_blank_frame_is_flat() {
+ let mut tracker = Tracker::new();
+ tracker.push(Gray::filled(200, 120, 90.0));
+ assert_eq!(tracker.push(Gray::filled(200, 120, 90.0)), Measured::Flat);
+ }
+}
diff --git a/src/crates/concat-vision/src/store.rs b/src/crates/concat-vision/src/store.rs
index 925af3ce..eb220bcd 100644
--- a/src/crates/concat-vision/src/store.rs
+++ b/src/crates/concat-vision/src/store.rs
@@ -67,7 +67,7 @@ pub fn mask_file(dir: &Path, millis: u64) -> PathBuf {
}
/// FNV-1a 64, for names that outlive the process.
-fn fnv1a(bytes: &[u8]) -> u64 {
+pub(crate) fn fnv1a(bytes: &[u8]) -> u64 {
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
for &byte in bytes {
hash ^= u64::from(byte);