The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
fix(media): sound stays in sync with the picture in files whose streams start at different times (#457)
Jareer93c18b7270aa
diff --git a/src/crates/concat-media/fixtures/picture-starts-late.mov b/src/crates/concat-media/fixtures/picture-starts-late.movnew file mode 100644index 00000000..9c79cb32Binary files /dev/null and b/src/crates/concat-media/fixtures/picture-starts-late.mov differdiff --git a/src/crates/concat-media/fixtures/sound-starts-late.mov b/src/crates/concat-media/fixtures/sound-starts-late.movnew file mode 100644index 00000000..f96d66c2Binary files /dev/null and b/src/crates/concat-media/fixtures/sound-starts-late.mov differdiff --git a/src/crates/concat-media/src/audio.rs b/src/crates/concat-media/src/audio.rsindex a5eaa9f4..2858952d 100644--- a/src/crates/concat-media/src/audio.rs+++ b/src/crates/concat-media/src/audio.rs@@ -218,7 +218,9 @@ pub fn mix_graph(clips: &[AudioClip]) -> Result<String> {/// starts, which the graph cuts off the front. A container seek lands on/// the packet at or before the target - a whole AAC frame early, 21 ms -/// and `mix_to_file` measures the miss from the first decoded frame; on-/// the continuous clock it stamps, the trim is then exact.+/// the continuous clock it stamps, the trim is then exact. A lead below+/// zero is sound that starts after the in-point - a file whose picture+/// starts before its sound - and that much silence goes in front.fn mix_graph_from(clips: &[AudioClip], leads: &[f64]) -> Result<String> {let mut chains: Vec<String> = Vec::new();@@ -241,6 +243,8 @@ fn mix_graph_from(clips: &[AudioClip], leads: &[f64]) -> Result<String> {let lead = leads.get(index).copied().unwrap_or(0.0);let mut stage = if lead > 0.0 {format!("[{index}:a]atrim=start={lead:.6},asetpts=PTS-STARTPTS")+ } else if lead < -0.0005 {+ format!("[{index}:a]adelay=delays={:.3}:all=1", -lead * 1000.0)} else {format!("[{index}:a]anull")};@@ -397,10 +401,10 @@ pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Re})?;let stream = input.stream(stream_index).expect("just found");let time_base = stream.time_base();- // Where the stream's clock starts: taken off what is read, put- // back on the seek, so the in-point is measured from the sound's- // own first sample. See `ffi::start_of`.- let start = ffi::start_of(&stream).as_f64();+ // Where the file's clock starts: taken off what is read, put back+ // on the seek, so the in-point is measured from the same instant+ // the picture's is. See `ffi::origin`.+ let start = ffi::origin(&input).as_f64();let decoder = ffmpeg::codec::Context::from_parameters(stream.parameters()).and_then(|context| context.decoder().audio()).map_err(|error| ffi::fail("open decoder", path, error))?;@@ -428,13 +432,14 @@ pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Re})?;// How far before the in-point the seek landed, by the first frame's// own timestamp - the one place a source timestamp is read, and- // only to place the cut. A frame with none, or one past the- // in-point, means nothing to cut.+ // only to place the cut. A frame with none means nothing to cut;+ // one past the in-point is sound that starts after it, which is+ // put that far in with silence, so it stays with its picture.let landed = first.timestamp().and_then(|ticks| ffi::seconds(ticks, time_base)).map_or(clip.source_start, |at| at.as_f64() - start);- let lead = (clip.source_start - landed).max(0.0);+ let lead = clip.source_start - landed;leads.push(lead);// Our continuous PTS: sample count from the start of this clip.first.set_pts(Some(0));@@ -444,7 +449,7 @@ pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Re// the speed factor. The first frame goes into the graph separately// below, so it counts towards the total.mix_input.samples_sent = first.samples() as i64;- mix_input.clip_samples = ((clip.duration * clamp_speed(clip.speed) + lead)+ mix_input.clip_samples = ((clip.duration * clamp_speed(clip.speed) + lead).max(0.0)* f64::from(first.rate())).round() as i64;// One sample per time-base tick: the PTS we stamp is a plain@@ -1140,6 +1145,54 @@ mod tests {);}+ /// The export's mix keeps a file's sound with its picture whichever+ /// starts first: each fixture sounds a tone half a second after its+ /// first frame, and so does its clip in the mix, whole or cut 0.2 s in+ /// (#457).+ #[test]+ fn the_mix_keeps_sound_with_its_picture_whichever_starts_first() {+ use crate::{AudioDecoder, AudioOptions, SampleFormat};+ let output =+ std::env::temp_dir().join(format!("concat-mix-origin-{}.m4a", std::process::id()));+ for fixture in ["picture-starts-late.mov", "sound-starts-late.mov"] {+ for (source_start, want) in [(0.0, 0.5), (0.2, 0.3)] {+ let source = format!("{}/fixtures/{fixture}", env!("CARGO_MANIFEST_DIR"));+ let piece = AudioClip {+ duration: 1.0,+ source_start,+ ..clip(&source)+ };+ mix_to_file(&[piece], 1.0, &output).expect("mixes");+ let options = AudioOptions {+ rate: 8_000,+ channels: 1,+ format: SampleFormat::F32,+ ..AudioOptions::default()+ };+ let mut decoder = AudioDecoder::open(&output, &options).expect("opens the mix");+ let mut samples: Vec<f32> = Vec::new();+ while let Some(frame) = decoder.next_frame().expect("decodes") {+ let bytes = &frame.data(0)[..frame.samples() * 4];+ samples.extend(+ bytes+ .chunks_exact(4)+ .map(|b| f32::from_ne_bytes([b[0], b[1], b[2], b[3]])),+ );+ }+ let onset = samples+ .iter()+ .position(|sample| sample.abs() > 0.05)+ .expect("the tone is in the mix") as f64+ / 8_000.0;+ assert!(+ (onset - want).abs() < 0.015,+ "{fixture} cut {source_start} s in: the tone is at {onset:.3} s, not {want}"+ );+ }+ }+ let _ = std::fs::remove_file(&output);+ }+/// A trimmed clip's mix ends when the timeline does. Two pieces of one/// file, the second cut in at two seconds, mixed to sixteen: the mix/// must come out, and be sixteen seconds long.diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rsindex c253c386..61619e47 100644--- a/src/crates/concat-media/src/decode.rs+++ b/src/crates/concat-media/src/decode.rs@@ -619,9 +619,9 @@ pub struct Decoder {input: format::context::Input,stream: usize,time_base: ffmpeg::Rational,- /// Where the stream's timestamps start; every timestamp read has this+ /// Where the file's timestamps start; every timestamp read has this/// taken off and every seek has it put back, so the file reads from- /// its own first frame. See `ffi::start_of`.+ /// its own first frame, in step with its sound. See `ffi::origin`.start: Rational,decoder: decoder::Video,rotation: i64,@@ -697,7 +697,7 @@ impl Decoder {})?;let stream_index = stream.index();let time_base = stream.time_base();- let start = ffi::start_of(&stream);+ let start = ffi::origin(&input);let rotation = ffi::rotation(&stream);// Wide pixels are shown square: see `ffi::sample_aspect`.let aspect = ffi::sample_aspect(&stream);diff --git a/src/crates/concat-media/src/ffi.rs b/src/crates/concat-media/src/ffi.rsindex 5c22a84a..6b1ed854 100644--- a/src/crates/concat-media/src/ffi.rs+++ b/src/crates/concat-media/src/ffi.rs@@ -156,17 +156,52 @@ pub(crate) fn seconds(ticks: i64, time_base: ffmpeg::Rational) -> Option<Rationa)}-/// Where a stream's timestamps start, in seconds: what its frames are-/// measured from. Zero for a stream that starts at zero or does not say;-/// 1.4 s for MPEG-TS by default and the MTS files cameras write, which-/// read as 1.4 s into the picture unless this is taken off every-/// timestamp and put back on every seek (audit 2026-09-23, #9).-pub(crate) fn start_of(stream: &ffmpeg::format::stream::Stream<'_>) -> Rational {+/// Where a stream's timestamps start, in seconds, or `None` where it does+/// not say. Zero for a stream that starts at or before zero.+fn start_of(stream: &ffmpeg::format::stream::Stream<'_>) -> Option<Rational> {let ticks = stream.start_time();- if ticks == ffmpeg::sys::AV_NOPTS_VALUE || ticks <= 0 {- return Rational::from_int(0);+ if ticks == ffmpeg::sys::AV_NOPTS_VALUE {+ return None;}- seconds(ticks, stream.time_base()).unwrap_or_else(|| Rational::from_int(0))+ if ticks <= 0 {+ return Some(Rational::ZERO);+ }+ seconds(ticks, stream.time_base())+}++/// Where a file's picture and sound are measured from, in seconds: where+/// its picture starts, or for a file with none, its sound. Taken off every+/// timestamp read and put back on every seek, by every reader of the file.+///+/// 1.4 s for MPEG-TS by default and the MTS files cameras write, which+/// read as 1.4 s into the picture unless it is taken off (audit 2026-09-23,+/// #9). One origin for the whole file, not each stream its own: the+/// streams of a file seldom start together - a camera's MTS sound 5 ms+/// before its picture, an FLV's 23 ms, a clip cut with `-c copy` whose+/// picture waits half a second for its first keyframe - and measuring each+/// from its own first frame slid the picture against the sound by exactly+/// that gap, in the preview and the export alike (#457). The picture's+/// start rather than the earliest: its frames then sit on the timeline's+/// grid as they always did, and the sound carries the gap - cut where it+/// starts first, put in after silence where it starts later. A cover+/// picture is no part of the timeline, and a stream that does not say+/// where it starts says nothing.+pub(crate) fn origin(input: &ffmpeg::format::context::Input) -> Rational {+ use ffmpeg::format::stream::Disposition;+ let picture = input+ .streams()+ .best(ffmpeg::media::Type::Video)+ .filter(|stream| !stream.disposition().contains(Disposition::ATTACHED_PIC))+ .and_then(|stream| start_of(&stream));+ picture+ .or_else(|| {+ input+ .streams()+ .filter(|stream| stream.parameters().medium() == ffmpeg::media::Type::Audio)+ .filter_map(|stream| start_of(&stream))+ .min()+ })+ .unwrap_or(Rational::ZERO)}/// Seconds in `AV_TIME_BASE` units, which is what container-level seeks take.diff --git a/src/crates/concat-media/src/samples.rs b/src/crates/concat-media/src/samples.rsindex 6f81b8cd..95117f20 100644--- a/src/crates/concat-media/src/samples.rs+++ b/src/crates/concat-media/src/samples.rs@@ -84,11 +84,14 @@ impl Default for AudioOptions {}/// The graph between the codec and the caller, as one string. `origin` is-/// the stream's own start, in seconds: the decoded frames carry it on their+/// where the file starts, in seconds: the decoded frames carry it on their/// timestamps, so a cut `start` seconds into the sound is made at-/// `origin + start` (see `ffi::start_of`).-fn audio_filter(options: &AudioOptions, origin: f64) -> String {+/// `origin + start` (see `ffi::origin`). `late` is how long after that the+/// sound's first sample comes - a file whose picture starts first - and is+/// put in front as silence, so the sound stays with its picture.+fn audio_filter(options: &AudioOptions, origin: f64, late: f64) -> String {let mut parts: Vec<String> = Vec::new();+ let pad = (late > 0.0005).then(|| format!("adelay=delays={:.3}:all=1", late * 1000.0));match (options.start.map(|start| start + origin), options.duration) {(Some(start), Some(duration)) => {parts.push(format!("atrim=start={start:.6}:duration={duration:.6}"));@@ -98,11 +101,29 @@ fn audio_filter(options: &AudioOptions, origin: f64) -> String {parts.push(format!("atrim=start={start:.6}"));parts.push("asetpts=PTS-STARTPTS".to_owned());}+ // Sound that starts before the picture is cut where the picture+ // starts, as a cut would cut it.+ (None, Some(duration)) if late < -0.0005 => {+ parts.push(format!("atrim=start={origin:.6}:duration={duration:.6}"));+ parts.push("asetpts=PTS-STARTPTS".to_owned());+ }+ (None, None) if late < -0.0005 => {+ parts.push(format!("atrim=start={origin:.6}"));+ parts.push("asetpts=PTS-STARTPTS".to_owned());+ }(None, Some(duration)) => {parts.push(format!("atrim=duration={duration:.6}"));}(None, None) => {}}+ // After the cut, which restamps from its first sample; and cut again+ // to the length asked for, which the silence has pushed out.+ if let Some(pad) = pad {+ parts.push(pad);+ if let Some(duration) = options.duration {+ parts.push(format!("atrim=duration={duration:.6}"));+ }+ }parts.extend(options.filters@@ -173,10 +194,10 @@ impl AudioDecoder {let stream = input.stream(stream_index).expect("just found");let time_base = stream.time_base();// A file whose sound starts late - MPEG-TS at 1.4 s, the MTS files- // cameras write - is measured from its first sample, as the picture- // and the export mix are; without this the preview's sound and a- // transcript ran ahead of the picture (audit 2026-10-04, #3).- let origin = ffi::start_of(&stream).as_f64();+ // cameras write - is measured from where the file starts, as the+ // picture and the export mix are; without this the preview's sound+ // and a transcript ran ahead of the picture (audit 2026-10-04, #3).+ let origin = ffi::origin(&input).as_f64();let context = ffmpeg::codec::Context::from_parameters(stream.parameters()).map_err(|error| ffi::fail("codec parameters", path, error))?;let decoder = context@@ -221,6 +242,14 @@ impl AudioDecoder {}fn build_graph(&self, first: &Audio) -> Result<filter::Graph> {+ // How long after the cut the first sample comes: past zero only for+ // sound that starts after the file's origin, the picture first.+ let late = first+ .timestamp()+ .and_then(|ticks| ffi::seconds(ticks, self.time_base))+ .map_or(0.0, |at| {+ at.as_f64() - self.origin - self.options.start.unwrap_or(0.0)+ });let mut graph = filter::Graph::new();let args = format!("time_base={}/{}:sample_rate={}:sample_fmt={}:channel_layout={}",@@ -251,7 +280,7 @@ impl AudioDecoder {graph.output("in", 0).and_then(|parser| parser.input("out", 0))- .and_then(|parser| parser.parse(&audio_filter(&self.options, self.origin)))+ .and_then(|parser| parser.parse(&audio_filter(&self.options, self.origin, late))).map_err(|error| ffi::fail("filter graph", &self.path, error))?;graph.validate()@@ -388,7 +417,7 @@ mod tests {stream: None,};assert_eq!(- audio_filter(&options, 0.0),+ audio_filter(&options, 0.0, 0.0),"atrim=start=1.500000:duration=2.000000,asetpts=PTS-STARTPTS,atempo=2.000000,\aresample=16000,aformat=sample_fmts=flt:sample_rates=16000:channel_layouts=mono");@@ -396,7 +425,7 @@ mod tests {#[test]fn a_whole_file_has_no_trim() {- let graph = audio_filter(&AudioOptions::default(), 1.4);+ let graph = audio_filter(&AudioOptions::default(), 1.4, 0.0);assert!(graph.starts_with("aresample=48000,"), "{graph}");assert!(graph.ends_with("channel_layouts=stereo"), "{graph}");}@@ -408,7 +437,40 @@ mod tests {start: Some(0.5),..AudioOptions::default()};- assert!(audio_filter(&options, 1.4).starts_with("atrim=start=1.900000,"));+ assert!(audio_filter(&options, 1.4, 0.0).starts_with("atrim=start=1.900000,"));+ }++ /// Sound that starts before the picture is cut where the picture does,+ /// with no seek to have cut it.+ #[test]+ fn sound_that_starts_early_is_cut_where_the_picture_starts() {+ let whole = audio_filter(&AudioOptions::default(), 0.5, -0.5);+ assert!(+ whole.starts_with("atrim=start=0.500000,asetpts=PTS-STARTPTS,aresample"),+ "{whole}"+ );+ }++ /// Sound that starts after the file's origin - after its picture - is+ /// put in with silence in front, after any cut, so it plays with the+ /// picture rather than ahead of it.+ #[test]+ fn sound_that_starts_late_is_put_in_after_silence() {+ let whole = audio_filter(&AudioOptions::default(), 0.0, 0.5);+ assert!(whole.starts_with("adelay=delays=500.000:all=1,"), "{whole}");+ let cut = AudioOptions {+ start: Some(0.2),+ duration: Some(1.0),+ ..AudioOptions::default()+ };+ assert!(+ audio_filter(&cut, 0.0, 0.3).starts_with(+ "atrim=start=0.200000:duration=1.000000,asetpts=PTS-STARTPTS,\+ adelay=delays=300.000:all=1,atrim=duration=1.000000,"+ ),+ "{}",+ audio_filter(&cut, 0.0, 0.3)+ );}/// A window into a file whose sound starts at 1.4 s - silence, then a@@ -451,6 +513,56 @@ mod tests {assert!(loud(4_200..7_600) > 0.08, "the tone after half a second");}+ /// Where a tone first sounds in what a decoder of `path` hands back,+ /// in seconds, at 8 kHz mono.+ fn tone_onset(path: &str, start: Option<f64>) -> f64 {+ let options = AudioOptions {+ start,+ rate: 8_000,+ channels: 1,+ format: SampleFormat::F32,+ ..AudioOptions::default()+ };+ let mut decoder = AudioDecoder::open(path, &options).expect("opens");+ let mut samples: Vec<f32> = Vec::new();+ while let Some(frame) = decoder.next_frame().expect("decodes") {+ let bytes = &frame.data(0)[..frame.samples() * 4];+ samples.extend(+ bytes+ .chunks_exact(4)+ .map(|b| f32::from_ne_bytes([b[0], b[1], b[2], b[3]])),+ );+ }+ let first = samples+ .iter()+ .position(|sample| sample.abs() > 0.05)+ .expect("a tone");+ first as f64 / 8_000.0+ }++ /// A file's sound is measured from where its picture starts, so the two+ /// play together as they do in a player. Each fixture sounds a tone half+ /// a second after its first frame: one whose picture starts half a+ /// second after its sound, one whose sound starts after its picture.+ /// Each measured from its own first sample, the first played its tone a+ /// second in and the second a fifth of a second in (#457).+ #[test]+ fn sound_plays_with_the_picture_whichever_starts_first() {+ for fixture in ["picture-starts-late.mov", "sound-starts-late.mov"] {+ let path = format!("{}/fixtures/{fixture}", env!("CARGO_MANIFEST_DIR"));+ let whole = tone_onset(&path, None);+ assert!(+ (whole - 0.5).abs() < 0.01,+ "{fixture}: the tone sounds {whole:.3} s after the first frame"+ );+ let cut = tone_onset(&path, Some(0.2));+ assert!(+ (cut - 0.3).abs() < 0.01,+ "{fixture}: cut 0.2 s in, the tone sounds at {cut:.3} s"+ );+ }+ }+#[test]fn a_missing_file_is_an_error_not_a_panic() {assert!(AudioDecoder::open("does-not-exist.mp3", &AudioOptions::default()).is_err());