The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
perf(enhance): enhance only the span a clip shows, and reuse the answer for a frame repeated to the byte
Jareer7db37f2ab88d
diff --git a/src/crates/concat-host/src/enhance.rs b/src/crates/concat-host/src/enhance.rsindex be51d061..8323b118 100644--- a/src/crates/concat-host/src/enhance.rs+++ b/src/crates/concat-host/src/enhance.rs@@ -73,6 +73,11 @@ pub struct EnhanceRequest {pub strength: f32,/// Where the copy goes; see [`target_for`].pub target: PathBuf,+ /// Seconds into the file to start at and how many to cover: the span+ /// a clip shows, with a little either side, rather than the whole+ /// file - an hour's recording with ten seconds of it on the timeline+ /// is ten seconds' work. `None` for the whole file, and for a still.+ pub span: Option<(f64, f64)>,}/// What a preview covers: one frame of a file, at the size it would be@@ -145,17 +150,24 @@ pub struct Enhancers {/// made before it had a strength - keeps the old name, so a copy made/// then is found again rather than made twice. None for a file that/// cannot be stat'ed, which cannot be read either.+///+/// A copy of a span is named by the span too, to the millisecond, so a+/// clip's copy is not mistaken for another clip's of the same file.pub fn target_for(project: &Path,media: &str,factor: u32,strength: f32,still: bool,+ span: Option<(f64, f64)>,) -> Option<PathBuf> {let meta = std::fs::metadata(media).ok()?;let mut hasher = std::collections::hash_map::DefaultHasher::new();media.hash(&mut hasher);meta.len().hash(&mut hasher);+ if let Some((start, duration)) = span {+ ((start * 1000.0).round() as u64, (duration * 1000.0).round() as u64).hash(&mut hasher);+ }if let Ok(modified) = meta.modified()&& let Ok(since) = modified.duration_since(std::time::UNIX_EPOCH){@@ -362,11 +374,11 @@ fn footage(let rate = video.frame_rate;// About how many frames there are, for the readout; the count is not// known until the last one is read.- let expected = info- .duration- .map(|duration| duration.as_f64() * rate.fps().as_f64())- .unwrap_or(0.0)- .max(1.0);+ let seconds = match request.span {+ Some((_, duration)) => duration,+ None => info.duration.map_or(0.0, |duration| duration.as_f64()),+ };+ let expected = (seconds * rate.fps().as_f64()).max(1.0);// Even sides, which the encoder wants: a stray pixel at the right or// bottom is cropped rather than the whole copy refused.let width = (video.width * request.factor) & !1;@@ -377,7 +389,11 @@ fn footage(let partial_video = request.target.with_extension("part.mp4");let partial_muxed = request.target.with_extension("muxed.part.mp4");- let options = DecodeOptions::default().in_software().threaded(2);+ let mut options = DecodeOptions::default().in_software().threaded(2);+ if let Some((start, _)) = request.span {+ options = options.starting_at(Rational::approximate(start).unwrap_or(Rational::ZERO));+ }+ let end = request.span.map(|(start, duration)| start + duration);let mut decoder =Decoder::open(&request.media_path, &options).map_err(|error| error.to_string())?;let mut encoder = Encoder::create(@@ -414,16 +430,42 @@ fn footage(t[slot] += now - at;at = now;};+ // The frame before and what it came to: a frame the same to the+ // byte - a held shot, a slide, a still screen, a title card - is the+ // same answer, and the model is not asked twice.+ let mut last: Option<(Frame, Frame)> = None;+ let mut repeated = 0u64;while let Some(frame) = decoder.next_frame().map_err(|error| error.to_string())? {+ // The span's end by the frame's own clock where the container+ // keeps one, and by count where it does not.+ if let Some(end) = end {+ let past = match decoder.position() {+ Some(at) => at.as_f64() >= end,+ None => count as f64 >= expected,+ };+ if past {+ break;+ }+ }lap(0, &mut t);if job.cancelled() {return Err("enhance cancelled".to_owned());}- let model = enhancer.enhance(&frame, request.factor)?;- lap(1, &mut t);- let enhanced = vision::mix(model, &frame, request.strength);- lap(2, &mut t);- let enhanced = crop(&enhanced, width, height);+ let enhanced = match &last {+ Some((before, answer)) if before.pixels() == frame.pixels() => {+ repeated += 1;+ answer.clone()+ }+ _ => {+ let model = enhancer.enhance(&frame, request.factor)?;+ lap(1, &mut t);+ let enhanced = vision::mix(model, &frame, request.strength);+ lap(2, &mut t);+ let enhanced = crop(&enhanced, width, height);+ last = Some((frame, enhanced.clone()));+ enhanced+ }+ };encoder.write_frame(&enhanced).map_err(|error| error.to_string())?;@@ -435,7 +477,8 @@ fn footage(}let per = |d: std::time::Duration| d.as_secs_f64() / count.max(1) as f64;log::info!(- "enhance: {count} frames, a frame: decode {:.3}s, model {:.3}s, mix {:.3}s, encode {:.3}s",+ "enhance: {count} frames ({repeated} repeats reused), a frame: decode {:.3}s, \+ model {:.3}s, mix {:.3}s, encode {:.3}s",per(t[0]),per(t[1]),per(t[2]),@@ -452,11 +495,39 @@ fn footage(return Err(error);}let finished = if info.audio.is_some() {- let muxed = concat_media::audio::mux(- &partial_video,- Path::new(&request.media_path),- &partial_muxed,- )+ // A span's sound is mixed out for the same seconds, so the copy's+ // sound starts where its picture does; a whole file's is the+ // file's own, muxed in as it is.+ let muxed = match request.span {+ Some((start, duration)) => {+ let partial_audio = request.target.with_extension("part.m4a");+ let clip = concat_media::audio::AudioClip {+ path: PathBuf::from(&request.media_path),+ stream: None,+ start: 0.0,+ duration,+ source_start: start,+ speed: 1.0,+ preserve_pitch: true,+ volume: 1.0,+ volume_curve: concat_core::animate::Track::new(Vec::new()),+ fade_in: 0.0,+ fade_out: 0.0,+ filter_chain: String::new(),+ };+ let muxed = concat_media::audio::mix_to_file(&[clip], duration, &partial_audio)+ .and_then(|()| {+ concat_media::audio::mux(&partial_video, &partial_audio, &partial_muxed)+ });+ let _ = std::fs::remove_file(&partial_audio);+ muxed+ }+ None => concat_media::audio::mux(+ &partial_video,+ Path::new(&request.media_path),+ &partial_muxed,+ ),+ }.map_err(|error| error.to_string());let _ = std::fs::remove_file(&partial_video);muxed.and_then(|()| {@@ -560,6 +631,80 @@ impl vision::WindowRunner for GpuWindows {mod tests {use super::*;+ /// A span of a file enhanced on its own: the copy is the span long, its+ /// picture starts at the span's start, and its sound is there beside+ /// it. Needs the fetched model and ffmpeg on the path.+ #[test]+ fn a_span_is_enhanced_on_its_own() {+ let Some(data) = ["org.standalones.concat", "app.concat.editor"]+ .iter()+ .filter_map(|id| {+ let p = PathBuf::from(std::env::var_os("HOME")?)+ .join("Library/Application Support")+ .join(id);+ p.join("cutout-models/realesr-general-x4v3.onnx").is_file().then_some(p)+ })+ .next()+ else {+ eprintln!("no Enhance model here; skipped");+ return;+ };+ let dir = std::env::temp_dir().join(format!("concat-enhance-span-{}", std::process::id()));+ let _ = std::fs::create_dir_all(&dir);+ let clip = dir.join("clip.mp4");+ // Four seconds whose frame number is drawn on it, with a tone.+ let made = std::process::Command::new("ffmpeg")+ .args(["-y", "-v", "error"])+ .args(["-f", "lavfi", "-i", "testsrc2=size=160x96:rate=10:duration=4"])+ .args(["-f", "lavfi", "-i", "sine=frequency=440:duration=4"])+ .args(["-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest"])+ .arg(&clip)+ .status();+ if !made.is_ok_and(|status| status.success()) {+ eprintln!("no ffmpeg here; skipped");+ return;+ }+ let enhancers = Enhancers::new(&data);+ let target = dir.join("span.mp4");+ let request = EnhanceRequest {+ media_path: clip.to_string_lossy().into_owned(),+ still: false,+ factor: 2,+ strength: 1.0,+ target: target.clone(),+ span: Some((1.0, 1.5)),+ };+ enhancers.enhance(&request, &mut |_| {}).expect("enhances");+ let info = concat_media::probe(&target).expect("probes");+ let length = info.duration.expect("a length").as_f64();+ assert!((length - 1.5).abs() < 0.15, "{length}");+ assert!(info.audio.is_some(), "the span's sound is there");+ let video = info.video.expect("a picture");+ assert_eq!((video.width, video.height), (320, 192));+ // The copy's first frame is the original's frame at one second.+ let first = frame_at(&target.to_string_lossy(), 0.0).expect("reads");+ let at_one = frame_at(&clip.to_string_lossy(), 1.0).expect("reads");+ let plain = vision::resize(&at_one, first.width(), first.height());+ let off: f64 = first+ .pixels()+ .iter()+ .zip(plain.pixels())+ .map(|(a, b)| f64::from(a.abs_diff(*b)))+ .sum::<f64>()+ / first.pixels().len() as f64;+ let at_zero = frame_at(&clip.to_string_lossy(), 0.0).expect("reads");+ let wrong = vision::resize(&at_zero, first.width(), first.height());+ let off_wrong: f64 = first+ .pixels()+ .iter()+ .zip(wrong.pixels())+ .map(|(a, b)| f64::from(a.abs_diff(*b)))+ .sum::<f64>()+ / first.pixels().len() as f64;+ assert!(off < off_wrong, "starts at one second: {off} against {off_wrong}");+ let _ = std::fs::remove_dir_all(&dir);+ }+/// Times the stages of a real run, by hand.#[test]#[ignore = "a probe"]@@ -592,6 +737,7 @@ mod tests {factor: 2,strength: 0.8,target,+ span: None,};let started = std::time::Instant::now();enhancers.enhance(&request, &mut |_| {}).expect("enhances");@@ -659,18 +805,18 @@ mod tests {let project = dir.join("project");// Full strength keeps the name Enhance gave copies before it had// one, so those are found again.- let two = target_for(&project, &media, 2, 1.0, false).expect("named");+ let two = target_for(&project, &media, 2, 1.0, false, None).expect("named");assert!(two.starts_with(project.join("cache")));assert!(two.to_string_lossy().ends_with("-2x.mp4"), "{two:?}");assert_eq!(- target_for(&project, &media, 2, 1.0, false),+ target_for(&project, &media, 2, 1.0, false, None),Some(two.clone()));assert_ne!(- target_for(&project, &media, 1, 1.0, false),+ target_for(&project, &media, 1, 1.0, false, None),Some(two.clone()));- let still = target_for(&project, &media, 2, 1.0, true).expect("named");+ let still = target_for(&project, &media, 2, 1.0, true, None).expect("named");assert!(still.to_string_lossy().ends_with("-2x.png"), "{still:?}");assert_eq!(target_for(@@ -678,26 +824,27 @@ mod tests {&dir.join("gone.mp4").to_string_lossy(),2,1.0,- false+ false,+ None),None);// Another strength is another copy, keyed by its whole percent so// a slider's float noise is not.- let blended = target_for(&project, &media, 2, 0.75, false).expect("named");+ let blended = target_for(&project, &media, 2, 0.75, false, None).expect("named");assert!(blended.to_string_lossy().ends_with("-2x-75pc.mp4"),"{blended:?}");assert_ne!(blended, two);assert_eq!(- target_for(&project, &media, 2, 0.750_001, false),+ target_for(&project, &media, 2, 0.750_001, false, None),Some(blended.clone()));- assert_ne!(target_for(&project, &media, 4, 0.75, false), Some(blended));+ assert_ne!(target_for(&project, &media, 4, 0.75, false, None), Some(blended));// A file with different bytes is a different copy.std::fs::write(&media, b"not really, but longer").expect("write");- assert_ne!(target_for(&project, &media, 2, 1.0, false), Some(two));+ assert_ne!(target_for(&project, &media, 2, 1.0, false, None), Some(two));let _ = std::fs::remove_dir_all(&dir);}diff --git a/src/crates/concat-project/src/commands/clips.rs b/src/crates/concat-project/src/commands/clips.rsindex a72824ea..fd522b4a 100644--- a/src/crates/concat-project/src/commands/clips.rs+++ b/src/crates/concat-project/src/commands/clips.rs@@ -428,6 +428,7 @@ pub(super) fn apply(item,source_start,enhanced,+ offset,} => {if project.active().clip(&clip_id).is_none() {return Ok(Outcome::default());@@ -478,6 +479,7 @@ pub(super) fn apply(original,copy: media_id.clone(),on: true,+ offset,})} else {None@@ -509,7 +511,15 @@ pub(super) fn apply(} else {link.original.clone()};+ // The in-point moves with the media, from one's seconds to the+ // other's, so the frame on screen is the same frame.+ let start = match (link.on, on) {+ (false, true) => (clip.source_start - link.offset).max(0.0),+ (true, false) => clip.source_start + link.offset,+ _ => clip.source_start,+ };let applied = assign(&mut clip.media_id, media)+ | assign(&mut clip.source_start, start)| assign(&mut clip.enhanced, Some(Enhanced { on, ..link }));Ok(Outcome {created_id: None,diff --git a/src/crates/concat-project/src/commands/mod.rs b/src/crates/concat-project/src/commands/mod.rsindex 1eae24ca..54e82fbc 100644--- a/src/crates/concat-project/src/commands/mod.rs+++ b/src/crates/concat-project/src/commands/mod.rs@@ -621,6 +621,11 @@ pub enum Command {/// one - ends the clip's Enhance, which was of the media now gone.#[serde(default, skip_serializing_if = "std::ops::Not::not")]enhanced: bool,+ /// With `enhanced`, the seconds into the original the copy starts+ /// at (see [`crate::model::Enhanced::offset`]); `source_start` is+ /// then the clip's in-point in the copy.+ #[serde(default)]+ offset: f64,},/// Shows a clip's enhanced copy, or its original again. A no-op for a/// clip Enhance has not made a copy for.@@ -1136,8 +1141,17 @@ impl Command {Command::SplitClips { time, .. } => bad([*time]),Command::FreezeFrame { time, duration, .. } => bad([*time]) || bad(*duration),Command::ReplaceClipMedia {- item, source_start, ..- } => bad(item.duration) || bad(item.frame_rate) || bad(*source_start),+ item,+ source_start,+ offset,+ ..+ } => {+ bad(item.duration)+ || bad(item.frame_rate)+ || bad(*source_start)+ || bad([*offset])+ || *offset < 0.0+ }Command::UpdateClip { patch, .. } => {bad(patch.volume)|| bad(patch.fade_in)diff --git a/src/crates/concat-project/src/lib.rs b/src/crates/concat-project/src/lib.rsindex 38d02a7f..fdb8ea8e 100644--- a/src/crates/concat-project/src/lib.rs+++ b/src/crates/concat-project/src/lib.rs@@ -1788,6 +1788,7 @@ mod tests {item: copy,source_start: Some(0.0),enhanced: false,+ offset: 0.0,}).expect("replaces");assert!(outcome.applied);@@ -1821,6 +1822,7 @@ mod tests {item: copy.clone(),source_start: None,enhanced: false,+ offset: 0.0,}).expect("replaces");assert!(outcome.applied);@@ -1846,6 +1848,7 @@ mod tests {item: copy.clone(),source_start: None,enhanced: false,+ offset: 0.0,}).expect("no-op");assert!(!again.applied);@@ -1861,6 +1864,7 @@ mod tests {},source_start: None,enhanced: false,+ offset: 0.0,}).expect("no-op");assert!(!nobody.applied);@@ -1894,6 +1898,7 @@ mod tests {item: bad,source_start: None,enhanced: false,+ offset: 0.0,}).is_err());@@ -5557,6 +5562,7 @@ mod tests {item: item("/a-enhanced.mp4"),source_start: None,enhanced: true,+ offset: 0.0,}).expect("replaces").created_id@@ -5593,12 +5599,58 @@ mod tests {editor.undo();assert_eq!(shown(&editor).media_id, original, "one undo step");+ // A copy of a span: it starts 4 s into the original, so the clip's+ // in-point is 4 s less in it, and switching moves it back and forth+ // with the same moment on screen.+ // The head trimmed 6 s in, so the in-point is past the copy's start.+ editor+ .apply(Command::TrimClip {+ clip_id: clip.clone(),+ edge: crate::commands::TrimEdge::Start,+ delta: 6.0,+ ripple: false,+ })+ .expect("trims");+ editor+ .apply(Command::SetClipEnhanced {+ clip_id: clip.clone(),+ on: false,+ })+ .expect("off");+ let original_start = shown(&editor).source_start;+ assert!(original_start >= 4.0, "{original_start}");+ editor+ .apply(Command::ReplaceClipMedia {+ clip_id: clip.clone(),+ item: item("/a-span.mp4"),+ source_start: Some(original_start - 4.0),+ enhanced: true,+ offset: 4.0,+ })+ .expect("replaces");+ assert_eq!(shown(&editor).source_start, original_start - 4.0);+ editor+ .apply(Command::SetClipEnhanced {+ clip_id: clip.clone(),+ on: false,+ })+ .expect("off");+ assert_eq!(shown(&editor).source_start, original_start);+ editor+ .apply(Command::SetClipEnhanced {+ clip_id: clip.clone(),+ on: true,+ })+ .expect("on");+ assert_eq!(shown(&editor).source_start, original_start - 4.0);+editor.apply(Command::ReplaceClipMedia {clip_id: clip.clone(),item: item("/a-reversed.mp4"),source_start: None,enhanced: false,+ offset: 0.0,}).expect("replaces");assert_eq!(shown(&editor).enhanced, None);diff --git a/src/crates/concat-project/src/model.rs b/src/crates/concat-project/src/model.rsindex 63063238..06569346 100644--- a/src/crates/concat-project/src/model.rs+++ b/src/crates/concat-project/src/model.rs@@ -1325,6 +1325,18 @@ pub struct Enhanced {pub copy: String,/// Whether the clip shows the copy.pub on: bool,+ /// Seconds into the original the copy starts at: a copy of the span the+ /// clip covers, rather than of the whole file, starts partway in. The+ /// clip's in-point is in the copy's seconds while it shows the copy, so+ /// switching between the two moves it by this much and the same moment+ /// stays on screen. Zero, and left out of the document, for a copy of+ /// the whole file.+ #[serde(default, skip_serializing_if = "is_zero_seconds")]+ pub offset: f64,+}++fn is_zero_seconds(seconds: &f64) -> bool {+ *seconds == 0.0}/// A transition on the cut into a clip.diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rsindex c0daed1d..d61cd363 100644--- a/src/crates/concat/src/studio.rs+++ b/src/crates/concat/src/studio.rs@@ -817,6 +817,10 @@ pub fn sync<T: Clone + PartialEq + 'static>(model: &VecModel<T>, next: Vec<T>) {}}+/// Seconds of the original an enhanced copy covers beyond each end of the+/// clip it was made for, so a small trim after does not run off the copy.+const ENHANCE_MARGIN: f64 = 1.0;+/// The window's state. See the module docs for what is whose.pub struct Studio {pub host: Host,@@ -7365,10 +7369,13 @@ impl Studio {return;}let factor = self.enhance_factor(&media);+ // The moment under the playhead in the original's seconds, which is+ // the file the preview reads, the clip may be showing a copy of a+ // span that starts partway into it.let seconds = if clip.kind == model::ClipKind::Image {0.0} else {- self.source_at_playhead(&clip)+ self.source_at_playhead(&clip) + Self::original_start(&clip) - clip.source_start};let request = concat_host::enhance::PreviewRequest {media_path: media.path.clone(),@@ -7468,8 +7475,18 @@ impl Studio {let factor = self.enhance_factor(&media);let strength = self.enhance_strength;let still = clip.kind == model::ClipKind::Image;+ // The span the clip shows of the original, and a second either side+ // so a small trim after is still inside the copy: the work is the+ // clip's, not the whole file's.+ let span = (!still).then(|| {+ let start = Self::original_start(&clip);+ let covered = clip.duration * clip.speed;+ let from = (start - ENHANCE_MARGIN).max(0.0);+ let to = (start + covered + ENHANCE_MARGIN).min(media.duration.unwrap_or(f64::MAX));+ (from, (to - from).max(covered))+ });let Some(target) =- concat_host::enhance::target_for(&project, &media.path, factor, strength, still)+ concat_host::enhance::target_for(&project, &media.path, factor, strength, still, span)else {self.notify(&tf("studio.couldNotRead", &[&media.path]), true);return;@@ -7480,6 +7497,7 @@ impl Studio {factor,strength,target,+ span,};let percent = concat_host::enhance::strength_percent(strength);let clip_id = clip.id.clone();@@ -7526,7 +7544,8 @@ impl Studio {match result {Ok(path) if ticked => {studio.enhance_ticked.remove(&clip_id);- studio.adopt_enhanced(&clip_id, &path, factor, percent);+ let offset = span.map_or(0.0, |(start, _)| start);+ studio.adopt_enhanced(&clip_id, &path, factor, percent, offset);}Ok(_) => {}Err(error) if error.contains("cancelled") => {}@@ -7650,14 +7669,43 @@ impl Studio {item,source_start: Some(0.0),enhanced: false,+ offset: 0.0,});self.request_preview();self.notify(&t("studio.reversedClipNowShows"), false);}+ /// Clip `clip`'s in-point in its original's seconds: its own, or, while+ /// it shows an enhanced copy of a span, its own moved by where that+ /// copy starts.+ fn original_start(clip: &Clip) -> f64 {+ match &clip.enhanced {+ Some(link) if link.on => clip.source_start + link.offset,+ _ => clip.source_start,+ }+ }+/// Points clip `id` at the enhanced copy at `path`, probed the way an/// import is, and shows it.- fn adopt_enhanced(&mut self, id: &str, path: &std::path::Path, factor: u32, percent: u32) {+ fn adopt_enhanced(+ &mut self,+ id: &str,+ path: &std::path::Path,+ factor: u32,+ percent: u32,+ offset: f64,+ ) {+ let Some(clip) = self.clip(id).cloned() else {+ return;+ };+ // The clip's in-point in the copy, which starts `offset` into the+ // original; a clip moved off the copy's span while it was made is+ // left as it is rather than pointed past the copy's ends.+ let start = Self::original_start(&clip) - offset;+ if start < -1e-6 {+ self.notify(&t("studio.clipChangedWhileBeing"), true);+ return;+ }let summary = match media::probe(&path.to_string_lossy()) {Ok(summary) => summary,Err(error) => {@@ -7672,11 +7720,18 @@ impl Studio {if let Some(original) = self.clip(id).and_then(|clip| self.enhance_source(clip)) {item.name = format!("{} (enhanced {factor}x, {percent}%)", original.name);}+ if let Some(length) = item.duration+ && start + clip.duration * clip.speed > length + 1e-3+ {+ self.notify(&t("studio.clipChangedWhileBeing"), true);+ return;+ }self.apply(Command::ReplaceClipMedia {clip_id: id.to_owned(),item,- source_start: None,+ source_start: Some(start.max(0.0)),enhanced: true,+ offset,});self.request_preview();self.notify(&t("studio.enhancedClipNowShows"), false);