The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(host): the stabilization analysis, one media at a time, only over the source clips show (#253)
Jareer50bea8dd51f7
diff --git a/src/crates/concat-host/src/lib.rs b/src/crates/concat-host/src/lib.rsindex d519e254..64dc3f5f 100644--- a/src/crates/concat-host/src/lib.rs+++ b/src/crates/concat-host/src/lib.rs@@ -8,7 +8,8 @@//! window talks to: it opens a project folder as a [`session::Session`],//! keeps the recents list, caches waveforms and filmstrips beside the//! project, composites the paused monitor's true frame, plays the audible-//! clips, finds the masks behind cutouts, packs templates, and holds the+//! clips, finds the masks behind cutouts, measures the camera shake stabilized+//! clips take out, packs templates, and holds the//! one-at-a-time slots for long jobs.//!//! The window calls these functions in-process, and the doctrine is that@@ -39,6 +40,7 @@ pub mod proxy;pub mod record;pub mod reverse;pub mod session;+pub mod stabilize;pub mod templates;pub mod titles;pub mod updates;@@ -51,6 +53,7 @@ pub use jobs::{Job, SingleFlight};pub use projects::ProjectInfo;pub use reverse::{ReverseRequest, Reversers};pub use session::{EditorView, Session, SettingsView};+pub use stabilize::{StabilizeRequest, Stabilizers};pub use titles::{TitleClip, Titles};/// The one scheduler every decode for the screen goes through: thediff --git a/src/crates/concat-host/src/preview.rs b/src/crates/concat-host/src/preview.rsindex 68fbec27..aae22a1b 100644--- a/src/crates/concat-host/src/preview.rs+++ b/src/crates/concat-host/src/preview.rs@@ -372,6 +372,17 @@ impl Monitor {);}+ /// Forgets the kept plan, so the next frame builds it again: for when+ /// something the plan read from the project folder - a stabilized+ /// clip's measured movement - has changed under a clip list that has+ /// not.+ pub fn forget_plan(&self) {+ *self+ .plan+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;+ }+/// Forgets every cached frame, reader and plan, for when the project/// closes.pub fn clear(&self) {diff --git a/src/crates/concat-host/src/stabilize.rs b/src/crates/concat-host/src/stabilize.rsnew file mode 100644index 00000000..bd4d3c35--- /dev/null+++ b/src/crates/concat-host/src/stabilize.rs@@ -0,0 +1,280 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! Measuring how the camera moved, for the clips that are stabilized.+//!+//! A stabilized clip needs its media's movement measured over the stretch+//! of source it shows, and a little either side for the smoothing to read.+//! This is the job that measures it: the media decoded small and grey+//! along each stretch not measured yet, every frame tracked against the one+//! before (see `concat_vision::stabilize::track`), and the movement merged+//! into the media's cache in the project folder. Only the stretches clips+//! use are read: a clip of twenty seconds from a twenty-minute recording+//! costs twenty seconds, not twenty minutes.+//!+//! Nothing here is the correction itself. That is worked out from the+//! cache whenever the timeline is built, from the clip's own settings, so+//! moving a slider never runs this again; trimming a clip longer does, for+//! the new part only.+//!+//! One job at a time through a [`SingleFlight`], like the cutout's, and+//! guarded by a [`Tripwire`] so footage the app died measuring is not+//! measured again by itself on the next open.++use std::path::{Path, PathBuf};+use std::sync::Arc;++use concat_core::time::Rational;+use concat_media::{DecodeOptions, Decoder, FrameSource};+use concat_project::model::{ClipKind, Project};+use concat_vision::stabilize::track::{Gray, Measured, Tracker};+use concat_vision::stabilize::{ANALYSIS_SIDE, FrameMotion, Motion, Similarity, Step, motion_dir};++use crate::jobs::{SingleFlight, Tripwire};++/// What one analysis covers: one media file, the stretches of it clips+/// show.+#[derive(Clone, Debug, PartialEq)]+pub struct StabilizeRequest {+ /// The project folder the movement is cached under.+ pub project: PathBuf,+ /// The media file to measure.+ pub media_path: String,+ /// The media's picture size, when the document knows it: what the+ /// analysis size is worked out from.+ pub media_size: Option<(u32, u32)>,+ /// The stretches of source, in seconds, to measure, margins included.+ pub ranges: Vec<(f64, f64)>,+}++/// Source read either side of what a clip shows, for the smoothing to+/// see where the camera was going: a second and a half at the clip's own+/// pace, more for a clip sped up.+fn margin(speed: f64) -> f64 {+ (1.5 * speed.max(1.0)).min(10.0)+}++/// The analysis service: the one-job slot.+pub struct Stabilizers {+ gate: Arc<SingleFlight>,+}++impl Default for Stabilizers {+ fn default() -> Self {+ Stabilizers::new()+ }+}++impl Stabilizers {+ /// A service with nothing running.+ pub fn new() -> Stabilizers {+ Stabilizers {+ gate: Arc::new(SingleFlight::new()),+ }+ }++ /// Whether an analysis is running.+ pub fn is_busy(&self) -> bool {+ self.gate.is_busy()+ }++ /// Asks the running analysis to stop after the frame in hand. What it+ /// measured so far is kept.+ pub fn cancel(&self) {+ self.gate.cancel();+ }++ /// What the active timeline's stabilized clips need measured: one+ /// request per media, covering every stretch its stabilized clips+ /// show, paired with the media's id. The window asks after every+ /// change and the API before an export; both get the same list.+ pub fn requests(project: &Project, project_dir: &Path) -> Vec<(String, StabilizeRequest)> {+ let mut wanted: Vec<(String, StabilizeRequest)> = Vec::new();+ for clip in &project.active().clips {+ if clip.stabilize.is_none() || clip.kind != ClipKind::Video {+ continue;+ }+ let Some(media) = project.media_by_id(&clip.media_id) else {+ continue;+ };+ let speed = clip.speed.max(0.0625);+ let pad = margin(speed);+ let range = (+ (clip.source_start - pad).max(0.0),+ clip.source_start + clip.duration * speed + pad,+ );+ match wanted.iter_mut().find(|(id, _)| *id == media.id) {+ Some((_, request)) => request.ranges.push(range),+ None => wanted.push((+ media.id.clone(),+ StabilizeRequest {+ project: project_dir.to_path_buf(),+ media_path: media.path.clone(),+ media_size: media.width.zip(media.height),+ ranges: vec![range],+ },+ )),+ }+ }+ wanted+ }++ /// Where `request`'s movement is kept.+ pub fn dir(request: &StabilizeRequest) -> PathBuf {+ motion_dir(&request.project, &request.media_path)+ }++ /// Whether the app died measuring `request` last time, so it is not to+ /// be measured again by itself. See [`Tripwire`].+ pub fn tripped(request: &StabilizeRequest) -> bool {+ Tripwire::tripped(&Self::dir(request))+ }++ /// The stretches `request` still needs measured, without measuring.+ pub fn outstanding(request: &StabilizeRequest) -> Vec<(f64, f64)> {+ match Motion::read(&Self::dir(request)) {+ Some(motion) => motion.missing(&request.ranges),+ None => Motion::default().missing(&request.ranges),+ }+ }++ /// Measures what `request` is missing. Blocks for the whole run, so+ /// run it on its own thread; `progress` is told `0..=1` of the source+ /// seconds wanted as it goes. Returns how many frames were measured.+ pub fn analyse(+ &self,+ request: &StabilizeRequest,+ progress: &mut dyn FnMut(f32),+ ) -> Result<usize, String> {+ let job = self.gate.begin("stabilization analysis")?;+ let dir = Self::dir(request);+ let wanted = Self::outstanding(request);+ let total: f64 = wanted.iter().map(|(a, b)| b - a).sum();+ if wanted.is_empty() || total <= 0.0 {+ return Ok(0);+ }+ let _wire = Tripwire::arm(&dir);+ let mut motion = Motion::read(&dir)+ .map(|motion| (*motion).clone())+ .unwrap_or_default();+ let size = analysis_size(request)?;+ if motion.aspect <= 0.0 {+ motion.aspect = f64::from(size.0) / f64::from(size.1);+ }+ progress(0.0);++ let mut done_seconds = 0.0;+ let mut measured = 0usize;+ let mut tracker = Tracker::new();+ for &(from, to) in &wanted {+ // From a little before the stretch, so its first frame is one+ // already measured where there is one, and the stretch joins+ // what is there rather than starting afresh.+ let lead = (from - 0.1).max(0.0);+ let options = DecodeOptions::default()+ .starting_at(Rational::approximate(lead).unwrap_or(Rational::ZERO))+ .scaled_to(size.0, size.1);+ let mut decoder = Decoder::open(&request.media_path, &options)+ .map_err(|error| format!("{}: {error}", request.media_path))?;+ tracker.reset();+ let mut run: Vec<FrameMotion> = Vec::new();+ let mut reached = from;+ let mut cancelled = false;+ loop {+ if job.cancelled() {+ cancelled = true;+ break;+ }+ let Some(frame) = decoder.next_frame().map_err(|error| error.to_string())? else {+ // The file ran out before its stated length: what was+ // measured is what there is, and the rest counts as+ // covered so it is not asked for again.+ reached = to;+ break;+ };+ let at = decoder+ .position()+ .map(|position| position.as_f64())+ .unwrap_or_else(|| run.last().map_or(lead, |last| last.at + 1.0 / 30.0));+ let (step, movement) = match tracker.push(Gray::of_frame(&frame)) {+ Measured::First => (Step::Start, Similarity::IDENTITY),+ Measured::Moved(similarity) => (Step::Tracked, similarity),+ Measured::Flat => (Step::Flat, Similarity::IDENTITY),+ };+ run.push(FrameMotion {+ at,+ step,+ motion: movement,+ });+ measured += 1;+ reached = at.min(to);+ progress(((done_seconds + (reached - from).max(0.0)) / total).min(1.0) as f32);+ if at >= to {+ break;+ }+ }+ if let Some(first) = run.first() {+ let span = (first.at.min(from), if cancelled { reached } else { to });+ motion.merge(&run, span);+ motion.write(&dir)?;+ } else if !cancelled {+ // Nothing to read in the stretch at all: past the end.+ motion.merge(&[], (from, to));+ motion.write(&dir)?;+ }+ if cancelled {+ return Err("stabilization analysis cancelled".to_owned());+ }+ done_seconds += to - from;+ }+ progress(1.0);+ Ok(measured)+ }+}++/// The size the analysis decodes `request`'s media at: its longest side+/// [`ANALYSIS_SIDE`], or its own when smaller, even on both sides. The+/// file is opened to ask when the document does not know.+fn analysis_size(request: &StabilizeRequest) -> Result<(u32, u32), String> {+ let (width, height) = match request.media_size.filter(|(w, h)| *w > 0 && *h > 0) {+ Some(size) => size,+ None => {+ let decoder = Decoder::open(&request.media_path, &DecodeOptions::default().limited_to(1))+ .map_err(|error| format!("{}: {error}", request.media_path))?;+ (decoder.width(), decoder.height())+ }+ };+ if width == 0 || height == 0 {+ return Err(format!("{}: no picture to measure", request.media_path));+ }+ let longest = width.max(height);+ let scale = (f64::from(ANALYSIS_SIDE) / f64::from(longest)).min(1.0);+ let even = |side: u32| (((f64::from(side) * scale).round() as u32).max(16)) & !1;+ Ok((even(width), even(height)))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_analysis_size_keeps_the_aspect_and_stays_even() {+ let request = |size| StabilizeRequest {+ project: PathBuf::new(),+ media_path: String::new(),+ media_size: Some(size),+ ranges: Vec::new(),+ };+ assert_eq!(analysis_size(&request((1920, 1080))), Ok((480, 270)));+ assert_eq!(analysis_size(&request((1080, 1920))), Ok((270, 480)));+ assert_eq!(analysis_size(&request((320, 240))), Ok((320, 240)));+ }++ #[test]+ fn a_faster_clip_reads_a_wider_margin() {+ assert_eq!(margin(1.0), 1.5);+ assert_eq!(margin(0.5), 1.5);+ assert_eq!(margin(4.0), 6.0);+ assert_eq!(margin(100.0), 10.0);+ }+}diff --git a/src/crates/concat-host/tests/stabilize.rs b/src/crates/concat-host/tests/stabilize.rsnew file mode 100644index 00000000..e6654260--- /dev/null+++ b/src/crates/concat-host/tests/stabilize.rs@@ -0,0 +1,300 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! Stabilization end to end: a shaky shot made here, its clip stabilized+//! by a command, the camera's movement measured by the host's job, and the+//! export and the monitor both checked against what the shake was.+//!+//! The shot is a still, textured scene filmed by a camera that wanders a+//! few pixels about one spot, by a known amount each frame, encoded with+//! H.264 as a phone would. So the measurement can be held to the numbers+//! the shot was made with, and a stabilized export to the scene keeping+//! still.++use std::path::{Path, PathBuf};+use std::sync::Arc;+use std::sync::atomic::AtomicBool;++use concat_core::frame::Frame;+use concat_core::time::FrameRate;+use concat_host::export::{self, ExportSpec};+use concat_host::preview::{FrameSpec, Monitor};+use concat_host::session::Session;+use concat_host::stabilize::Stabilizers;+use concat_host::{media, projects};+use concat_media::{+ DecodeOptions, Decoder, EncodeOptions, Encoder, FrameSink, FrameSource, RateMode, VideoCodec,+};+use concat_project::commands::Command;+use concat_project::model::{ColorSpace, Stabilize};+use concat_vision::stabilize::{Motion, Step};++const WIDTH: u32 = 320;+const HEIGHT: u32 = 180;+const FRAMES: usize = 60;++/// Where the camera sits at frame `k`, in pixels: a wander of a few+/// pixels about one spot, never the same twice running.+fn camera(k: usize) -> (f64, f64) {+ let k = k as f64;+ (+ 4.0 * (1.9 * k).sin() + 2.0 * (0.7 * k).sin(),+ 3.0 * (1.3 * k).cos() + 1.0 * (0.4 * k).sin(),+ )+}++/// The scene, a smooth texture with corners all over it.+fn scene(x: f64, y: f64) -> f64 {+ let mut v = 120.0;+ for (k, (fx, fy, phase)) in [+ (0.071, 0.023, 0.3),+ (-0.031, 0.083, 1.1),+ (0.11, 0.097, 2.0),+ (0.017, -0.13, 0.7),+ (0.21, 0.05, 1.7),+ ]+ .iter()+ .enumerate()+ {+ v += (28.0 / (k as f64 + 1.0).sqrt()) * (x * fx + y * fy + phase).sin();+ }+ v + 30.0+ * (3.0 * (x * std::f64::consts::PI / 17.0).sin() * (y * std::f64::consts::PI / 13.0).sin())+ .tanh()+}++/// The shaky shot: frame `k` shows the scene moved by `camera(k)`.+fn shoot(path: &Path) {+ let options = EncodeOptions {+ codec: VideoCodec::H264,+ preset: "ultrafast".to_owned(),+ crf: 14,+ rate_mode: RateMode::Vbr,+ bitrate_kbps: 0,+ ten_bit: false,+ color_range: concat_media::ColorRange::Limited,+ hardware: false,+ threads: 0,+ };+ let mut encoder = Encoder::create(path, WIDTH, HEIGHT, FrameRate::THIRTY, &options)+ .expect("the linked FFmpeg encodes");+ for k in 0..FRAMES {+ let (dx, dy) = camera(k);+ let mut frame = Frame::black(WIDTH, HEIGHT);+ for y in 0..HEIGHT {+ for x in 0..WIDTH {+ let v = scene(f64::from(x) - dx, f64::from(y) - dy).clamp(0.0, 255.0) as u8;+ frame.set_pixel(x, y, [v, v, v, 255]);+ }+ }+ encoder.write_frame(&frame).expect("writes a frame");+ }+ encoder.finish().expect("finishes the shot");+}++struct Scratch(PathBuf);++impl Scratch {+ fn new(name: &str) -> Scratch {+ let dir =+ std::env::temp_dir().join(format!("concat-stabilize-{name}-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).expect("makes the scratch folder");+ Scratch(dir)+ }+}++impl Drop for Scratch {+ fn drop(&mut self) {+ let _ = std::fs::remove_dir_all(&self.0);+ }+}++/// Every frame of a file.+fn frames_of(path: &str) -> Vec<Frame> {+ let mut decoder = Decoder::open(path, &DecodeOptions::default()).expect("opens");+ let mut frames = Vec::new();+ while let Some(frame) = decoder.next_frame().expect("decodes") {+ frames.push(frame);+ }+ frames+}++/// The mean difference in level between two frames over their middle+/// half, where no edge a correction moves in can reach.+fn difference(a: &[u8], b: &[u8]) -> f64 {+ let (w, h) = (WIDTH as usize, HEIGHT as usize);+ let mut sum = 0.0;+ let mut count = 0.0;+ for y in h / 4..3 * h / 4 {+ for x in w / 4..3 * w / 4 {+ let at = (y * w + x) * 4;+ sum += (f64::from(a[at]) - f64::from(b[at])).abs();+ count += 1.0;+ }+ }+ sum / count+}++/// How much the picture moves frame to frame over a run of frames.+fn wobble(frames: &[Vec<u8>]) -> f64 {+ let pairs: Vec<f64> = frames+ .windows(2)+ .map(|pair| difference(&pair[0], &pair[1]))+ .collect();+ pairs.iter().sum::<f64>() / pairs.len() as f64+}++fn export_to(session: &Session, output: &Path) -> String {+ let spec = ExportSpec {+ output: output.to_string_lossy().into_owned(),+ crf: 12,+ preset: "ultrafast".to_owned(),+ codec: VideoCodec::H264,+ ten_bit: false,+ rate_mode: RateMode::Vbr,+ bitrate_kbps: 0,+ color_range: concat_media::ColorRange::Limited,+ hdr: false,+ };+ let request = export::request(session, &spec, Vec::new());+ export::run(&request, &AtomicBool::new(false), |_| {}).expect("exports")+}++#[test]+fn a_shaky_shot_is_measured_and_held_still() {+ let scratch = Scratch::new("shot");+ let shot = scratch.0.join("shaky.mp4");+ shoot(&shot);++ let info = projects::create(+ scratch.0.to_str().expect("utf-8"),+ "Steady",+ WIDTH,+ HEIGHT,+ 30,+ 1,+ )+ .expect("creates the project");+ let mut session = Session::open_info(&info).expect("opens the project");+ let summary = media::probe(shot.to_str().expect("utf-8")).expect("probes");+ let media_id = session+ .apply(Command::AddMedia {+ item: summary.to_new_media(),+ })+ .expect("imports")+ .created_id+ .expect("an id");+ let clip_id = session+ .apply(Command::AddClipAtFirstFree {+ media_id,+ start: 0.0,+ })+ .expect("adds")+ .created_id+ .expect("an id");++ // Shown as shot first, for the comparison.+ let plain = export_to(&session, &scratch.0.join("plain.mp4"));++ session+ .apply(Command::SetClipStabilize {+ clip_id: clip_id.clone(),+ stabilize: Some(Stabilize {+ smoothness: 0.6,+ max_zoom: 1.3,+ ..Stabilize::default()+ }),+ })+ .expect("stabilizes");+ let project_dir = PathBuf::from(session.path());++ // Not measured yet: the clip shows as shot.+ let requests = Stabilizers::requests(session.project(), &project_dir);+ assert_eq!(requests.len(), 1);+ let (_, request) = &requests[0];+ assert!(!Stabilizers::outstanding(request).is_empty());++ let stabilizers = Stabilizers::new();+ let started = std::time::Instant::now();+ let mut last = 0.0f32;+ let measured = stabilizers+ .analyse(request, &mut |fraction| last = fraction)+ .expect("measures");+ let took = started.elapsed();+ eprintln!(+ "measured {measured} frames in {:.0} ms ({:.2} ms a frame, decode included)",+ took.as_secs_f64() * 1000.0,+ took.as_secs_f64() * 1000.0 / measured as f64+ );+ assert_eq!(measured, FRAMES);+ assert!((last - 1.0).abs() < 1e-6);+ assert!(Stabilizers::outstanding(request).is_empty(), "all measured");+ assert!(!Stabilizers::tripped(request), "the tripwire is cleared");++ // The measurement against the shot: each frame's movement is the+ // camera's step, to within a fifth of a pixel through H.264.+ let motion = Motion::read(&Stabilizers::dir(request)).expect("cached");+ assert_eq!(motion.frames.len(), FRAMES);+ assert_eq!(motion.frames[0].step, Step::Start);+ let h = f64::from(HEIGHT);+ let mut worst: f64 = 0.0;+ for k in 1..FRAMES {+ let frame = motion.frames[k];+ assert_eq!(frame.step, Step::Tracked, "frame {k}");+ let (x0, y0) = camera(k - 1);+ let (x1, y1) = camera(k);+ let (dx, dy) = (frame.motion.tx * h, frame.motion.ty * h);+ worst = worst.max((dx - (x1 - x0)).abs()).max((dy - (y1 - y0)).abs());+ assert!(frame.motion.angle().abs() < 0.002, "frame {k} turned");+ assert!((frame.motion.scale() - 1.0).abs() < 0.002, "frame {k} zoomed");+ }+ eprintln!("worst step error: {worst:.3} px");+ assert!(worst < 0.2, "a step measured {worst:.3} px off");++ // A second run has nothing to do.+ assert_eq!(stabilizers.analyse(request, &mut |_| {}), Ok(0));++ // Exported steadied: the scene holds still where the shot wobbled.+ let steady = export_to(&session, &scratch.0.join("steady.mp4"));+ let plain: Vec<Vec<u8>> = frames_of(&plain)+ .into_iter()+ .map(Frame::into_pixels)+ .collect();+ let steady: Vec<Vec<u8>> = frames_of(&steady)+ .into_iter()+ .map(Frame::into_pixels)+ .collect();+ assert_eq!(steady.len(), FRAMES);+ // Away from the ends, where the smoothing's reflection still lets a+ // little of the path through.+ let (shaky, held) = (wobble(&plain[10..50]), wobble(&steady[10..50]));+ eprintln!("frame-to-frame difference: as shot {shaky:.2}, stabilized {held:.2}");+ assert!(held < shaky * 0.35, "as shot {shaky:.2}, stabilized {held:.2}");++ // The monitor draws what the export wrote: the same frame, to within+ // what the encoder changed, and nowhere near the frame as shot.+ let monitor = Monitor::new();+ let clips = Arc::new(session.flattened_clips());+ for k in [12usize, 31, 47] {+ let pixels = monitor+ .frame(+ Arc::clone(&clips),+ &session.settings(),+ FrameSpec {+ time: k as f64 / 30.0,+ width: WIDTH,+ height: HEIGHT,+ moving: false,+ proxy: false,+ color_space: ColorSpace::default(),+ },+ )+ .expect("the monitor draws");+ let against_export = difference(&pixels, &steady[k]);+ let against_shot = difference(&pixels, &plain[k]);+ eprintln!("frame {k}: monitor vs export {against_export:.2}, vs as shot {against_shot:.2}");+ assert!(against_export < 2.0, "frame {k}: {against_export:.2}");+ assert!(against_shot > against_export * 2.0, "frame {k}");+ }+}