Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

fix(media): sound stays in sync with the picture in files whose streams start at different times (#457)

Jareer93c18b7270aa
diff --git a/src/crates/concat-media/fixtures/picture-starts-late.mov b/src/crates/concat-media/fixtures/picture-starts-late.mov
new file mode 100644
index 00000000..9c79cb32
Binary files /dev/null and b/src/crates/concat-media/fixtures/picture-starts-late.mov differ
diff --git a/src/crates/concat-media/fixtures/sound-starts-late.mov b/src/crates/concat-media/fixtures/sound-starts-late.mov
new file mode 100644
index 00000000..f96d66c2
Binary files /dev/null and b/src/crates/concat-media/fixtures/sound-starts-late.mov differ
diff --git a/src/crates/concat-media/src/audio.rs b/src/crates/concat-media/src/audio.rs
index a5eaa9f4..2858952d 100644
--- a/src/crates/concat-media/src/audio.rs
+++ b/src/crates/concat-media/src/audio.rs
@@ -218,7 +218,9 @@ pub fn mix_graph(clips: &[AudioClip]) -> Result<String> {
/// starts, which the graph cuts off the front. A container seek lands on
/// the packet at or before the target - a whole AAC frame early, 21 ms -
/// and `mix_to_file` measures the miss from the first decoded frame; on
-/// the continuous clock it stamps, the trim is then exact.
+/// the continuous clock it stamps, the trim is then exact. A lead below
+/// zero is sound that starts after the in-point - a file whose picture
+/// starts before its sound - and that much silence goes in front.
fn mix_graph_from(clips: &[AudioClip], leads: &[f64]) -> Result<String> {
let mut chains: Vec<String> = Vec::new();
@@ -241,6 +243,8 @@ fn mix_graph_from(clips: &[AudioClip], leads: &[f64]) -> Result<String> {
let lead = leads.get(index).copied().unwrap_or(0.0);
let mut stage = if lead > 0.0 {
format!("[{index}:a]atrim=start={lead:.6},asetpts=PTS-STARTPTS")
+ } else if lead < -0.0005 {
+ format!("[{index}:a]adelay=delays={:.3}:all=1", -lead * 1000.0)
} else {
format!("[{index}:a]anull")
};
@@ -397,10 +401,10 @@ pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Re
})?;
let stream = input.stream(stream_index).expect("just found");
let time_base = stream.time_base();
- // Where the stream's clock starts: taken off what is read, put
- // back on the seek, so the in-point is measured from the sound's
- // own first sample. See `ffi::start_of`.
- let start = ffi::start_of(&stream).as_f64();
+ // Where the file's clock starts: taken off what is read, put back
+ // on the seek, so the in-point is measured from the same instant
+ // the picture's is. See `ffi::origin`.
+ let start = ffi::origin(&input).as_f64();
let decoder = ffmpeg::codec::Context::from_parameters(stream.parameters())
.and_then(|context| context.decoder().audio())
.map_err(|error| ffi::fail("open decoder", path, error))?;
@@ -428,13 +432,14 @@ pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Re
})?;
// How far before the in-point the seek landed, by the first frame's
// own timestamp - the one place a source timestamp is read, and
- // only to place the cut. A frame with none, or one past the
- // in-point, means nothing to cut.
+ // only to place the cut. A frame with none means nothing to cut;
+ // one past the in-point is sound that starts after it, which is
+ // put that far in with silence, so it stays with its picture.
let landed = first
.timestamp()
.and_then(|ticks| ffi::seconds(ticks, time_base))
.map_or(clip.source_start, |at| at.as_f64() - start);
- let lead = (clip.source_start - landed).max(0.0);
+ let lead = clip.source_start - landed;
leads.push(lead);
// Our continuous PTS: sample count from the start of this clip.
first.set_pts(Some(0));
@@ -444,7 +449,7 @@ pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Re
// the speed factor. The first frame goes into the graph separately
// below, so it counts towards the total.
mix_input.samples_sent = first.samples() as i64;
- mix_input.clip_samples = ((clip.duration * clamp_speed(clip.speed) + lead)
+ mix_input.clip_samples = ((clip.duration * clamp_speed(clip.speed) + lead).max(0.0)
* f64::from(first.rate()))
.round() as i64;
// One sample per time-base tick: the PTS we stamp is a plain
@@ -1140,6 +1145,54 @@ mod tests {
);
}
+ /// The export's mix keeps a file's sound with its picture whichever
+ /// starts first: each fixture sounds a tone half a second after its
+ /// first frame, and so does its clip in the mix, whole or cut 0.2 s in
+ /// (#457).
+ #[test]
+ fn the_mix_keeps_sound_with_its_picture_whichever_starts_first() {
+ use crate::{AudioDecoder, AudioOptions, SampleFormat};
+ let output =
+ std::env::temp_dir().join(format!("concat-mix-origin-{}.m4a", std::process::id()));
+ for fixture in ["picture-starts-late.mov", "sound-starts-late.mov"] {
+ for (source_start, want) in [(0.0, 0.5), (0.2, 0.3)] {
+ let source = format!("{}/fixtures/{fixture}", env!("CARGO_MANIFEST_DIR"));
+ let piece = AudioClip {
+ duration: 1.0,
+ source_start,
+ ..clip(&source)
+ };
+ mix_to_file(&[piece], 1.0, &output).expect("mixes");
+ let options = AudioOptions {
+ rate: 8_000,
+ channels: 1,
+ format: SampleFormat::F32,
+ ..AudioOptions::default()
+ };
+ let mut decoder = AudioDecoder::open(&output, &options).expect("opens the mix");
+ let mut samples: Vec<f32> = Vec::new();
+ while let Some(frame) = decoder.next_frame().expect("decodes") {
+ let bytes = &frame.data(0)[..frame.samples() * 4];
+ samples.extend(
+ bytes
+ .chunks_exact(4)
+ .map(|b| f32::from_ne_bytes([b[0], b[1], b[2], b[3]])),
+ );
+ }
+ let onset = samples
+ .iter()
+ .position(|sample| sample.abs() > 0.05)
+ .expect("the tone is in the mix") as f64
+ / 8_000.0;
+ assert!(
+ (onset - want).abs() < 0.015,
+ "{fixture} cut {source_start} s in: the tone is at {onset:.3} s, not {want}"
+ );
+ }
+ }
+ let _ = std::fs::remove_file(&output);
+ }
+
/// A trimmed clip's mix ends when the timeline does. Two pieces of one
/// file, the second cut in at two seconds, mixed to sixteen: the mix
/// must come out, and be sixteen seconds long.
diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rs
index c253c386..61619e47 100644
--- a/src/crates/concat-media/src/decode.rs
+++ b/src/crates/concat-media/src/decode.rs
@@ -619,9 +619,9 @@ pub struct Decoder {
input: format::context::Input,
stream: usize,
time_base: ffmpeg::Rational,
- /// Where the stream's timestamps start; every timestamp read has this
+ /// Where the file's timestamps start; every timestamp read has this
/// taken off and every seek has it put back, so the file reads from
- /// its own first frame. See `ffi::start_of`.
+ /// its own first frame, in step with its sound. See `ffi::origin`.
start: Rational,
decoder: decoder::Video,
rotation: i64,
@@ -697,7 +697,7 @@ impl Decoder {
})?;
let stream_index = stream.index();
let time_base = stream.time_base();
- let start = ffi::start_of(&stream);
+ let start = ffi::origin(&input);
let rotation = ffi::rotation(&stream);
// Wide pixels are shown square: see `ffi::sample_aspect`.
let aspect = ffi::sample_aspect(&stream);
diff --git a/src/crates/concat-media/src/ffi.rs b/src/crates/concat-media/src/ffi.rs
index 5c22a84a..6b1ed854 100644
--- a/src/crates/concat-media/src/ffi.rs
+++ b/src/crates/concat-media/src/ffi.rs
@@ -156,17 +156,52 @@ pub(crate) fn seconds(ticks: i64, time_base: ffmpeg::Rational) -> Option<Rationa
)
}
-/// Where a stream's timestamps start, in seconds: what its frames are
-/// measured from. Zero for a stream that starts at zero or does not say;
-/// 1.4 s for MPEG-TS by default and the MTS files cameras write, which
-/// read as 1.4 s into the picture unless this is taken off every
-/// timestamp and put back on every seek (audit 2026-09-23, #9).
-pub(crate) fn start_of(stream: &ffmpeg::format::stream::Stream<'_>) -> Rational {
+/// Where a stream's timestamps start, in seconds, or `None` where it does
+/// not say. Zero for a stream that starts at or before zero.
+fn start_of(stream: &ffmpeg::format::stream::Stream<'_>) -> Option<Rational> {
let ticks = stream.start_time();
- if ticks == ffmpeg::sys::AV_NOPTS_VALUE || ticks <= 0 {
- return Rational::from_int(0);
+ if ticks == ffmpeg::sys::AV_NOPTS_VALUE {
+ return None;
}
- seconds(ticks, stream.time_base()).unwrap_or_else(|| Rational::from_int(0))
+ if ticks <= 0 {
+ return Some(Rational::ZERO);
+ }
+ seconds(ticks, stream.time_base())
+}
+
+/// Where a file's picture and sound are measured from, in seconds: where
+/// its picture starts, or for a file with none, its sound. Taken off every
+/// timestamp read and put back on every seek, by every reader of the file.
+///
+/// 1.4 s for MPEG-TS by default and the MTS files cameras write, which
+/// read as 1.4 s into the picture unless it is taken off (audit 2026-09-23,
+/// #9). One origin for the whole file, not each stream its own: the
+/// streams of a file seldom start together - a camera's MTS sound 5 ms
+/// before its picture, an FLV's 23 ms, a clip cut with `-c copy` whose
+/// picture waits half a second for its first keyframe - and measuring each
+/// from its own first frame slid the picture against the sound by exactly
+/// that gap, in the preview and the export alike (#457). The picture's
+/// start rather than the earliest: its frames then sit on the timeline's
+/// grid as they always did, and the sound carries the gap - cut where it
+/// starts first, put in after silence where it starts later. A cover
+/// picture is no part of the timeline, and a stream that does not say
+/// where it starts says nothing.
+pub(crate) fn origin(input: &ffmpeg::format::context::Input) -> Rational {
+ use ffmpeg::format::stream::Disposition;
+ let picture = input
+ .streams()
+ .best(ffmpeg::media::Type::Video)
+ .filter(|stream| !stream.disposition().contains(Disposition::ATTACHED_PIC))
+ .and_then(|stream| start_of(&stream));
+ picture
+ .or_else(|| {
+ input
+ .streams()
+ .filter(|stream| stream.parameters().medium() == ffmpeg::media::Type::Audio)
+ .filter_map(|stream| start_of(&stream))
+ .min()
+ })
+ .unwrap_or(Rational::ZERO)
}
/// Seconds in `AV_TIME_BASE` units, which is what container-level seeks take.
diff --git a/src/crates/concat-media/src/samples.rs b/src/crates/concat-media/src/samples.rs
index 6f81b8cd..95117f20 100644
--- a/src/crates/concat-media/src/samples.rs
+++ b/src/crates/concat-media/src/samples.rs
@@ -84,11 +84,14 @@ impl Default for AudioOptions {
}
/// The graph between the codec and the caller, as one string. `origin` is
-/// the stream's own start, in seconds: the decoded frames carry it on their
+/// where the file starts, in seconds: the decoded frames carry it on their
/// timestamps, so a cut `start` seconds into the sound is made at
-/// `origin + start` (see `ffi::start_of`).
-fn audio_filter(options: &AudioOptions, origin: f64) -> String {
+/// `origin + start` (see `ffi::origin`). `late` is how long after that the
+/// sound's first sample comes - a file whose picture starts first - and is
+/// put in front as silence, so the sound stays with its picture.
+fn audio_filter(options: &AudioOptions, origin: f64, late: f64) -> String {
let mut parts: Vec<String> = Vec::new();
+ let pad = (late > 0.0005).then(|| format!("adelay=delays={:.3}:all=1", late * 1000.0));
match (options.start.map(|start| start + origin), options.duration) {
(Some(start), Some(duration)) => {
parts.push(format!("atrim=start={start:.6}:duration={duration:.6}"));
@@ -98,11 +101,29 @@ fn audio_filter(options: &AudioOptions, origin: f64) -> String {
parts.push(format!("atrim=start={start:.6}"));
parts.push("asetpts=PTS-STARTPTS".to_owned());
}
+ // Sound that starts before the picture is cut where the picture
+ // starts, as a cut would cut it.
+ (None, Some(duration)) if late < -0.0005 => {
+ parts.push(format!("atrim=start={origin:.6}:duration={duration:.6}"));
+ parts.push("asetpts=PTS-STARTPTS".to_owned());
+ }
+ (None, None) if late < -0.0005 => {
+ parts.push(format!("atrim=start={origin:.6}"));
+ parts.push("asetpts=PTS-STARTPTS".to_owned());
+ }
(None, Some(duration)) => {
parts.push(format!("atrim=duration={duration:.6}"));
}
(None, None) => {}
}
+ // After the cut, which restamps from its first sample; and cut again
+ // to the length asked for, which the silence has pushed out.
+ if let Some(pad) = pad {
+ parts.push(pad);
+ if let Some(duration) = options.duration {
+ parts.push(format!("atrim=duration={duration:.6}"));
+ }
+ }
parts.extend(
options
.filters
@@ -173,10 +194,10 @@ impl AudioDecoder {
let stream = input.stream(stream_index).expect("just found");
let time_base = stream.time_base();
// A file whose sound starts late - MPEG-TS at 1.4 s, the MTS files
- // cameras write - is measured from its first sample, as the picture
- // and the export mix are; without this the preview's sound and a
- // transcript ran ahead of the picture (audit 2026-10-04, #3).
- let origin = ffi::start_of(&stream).as_f64();
+ // cameras write - is measured from where the file starts, as the
+ // picture and the export mix are; without this the preview's sound
+ // and a transcript ran ahead of the picture (audit 2026-10-04, #3).
+ let origin = ffi::origin(&input).as_f64();
let context = ffmpeg::codec::Context::from_parameters(stream.parameters())
.map_err(|error| ffi::fail("codec parameters", path, error))?;
let decoder = context
@@ -221,6 +242,14 @@ impl AudioDecoder {
}
fn build_graph(&self, first: &Audio) -> Result<filter::Graph> {
+ // How long after the cut the first sample comes: past zero only for
+ // sound that starts after the file's origin, the picture first.
+ let late = first
+ .timestamp()
+ .and_then(|ticks| ffi::seconds(ticks, self.time_base))
+ .map_or(0.0, |at| {
+ at.as_f64() - self.origin - self.options.start.unwrap_or(0.0)
+ });
let mut graph = filter::Graph::new();
let args = format!(
"time_base={}/{}:sample_rate={}:sample_fmt={}:channel_layout={}",
@@ -251,7 +280,7 @@ impl AudioDecoder {
graph
.output("in", 0)
.and_then(|parser| parser.input("out", 0))
- .and_then(|parser| parser.parse(&audio_filter(&self.options, self.origin)))
+ .and_then(|parser| parser.parse(&audio_filter(&self.options, self.origin, late)))
.map_err(|error| ffi::fail("filter graph", &self.path, error))?;
graph
.validate()
@@ -388,7 +417,7 @@ mod tests {
stream: None,
};
assert_eq!(
- audio_filter(&options, 0.0),
+ audio_filter(&options, 0.0, 0.0),
"atrim=start=1.500000:duration=2.000000,asetpts=PTS-STARTPTS,atempo=2.000000,\
aresample=16000,aformat=sample_fmts=flt:sample_rates=16000:channel_layouts=mono"
);
@@ -396,7 +425,7 @@ mod tests {
#[test]
fn a_whole_file_has_no_trim() {
- let graph = audio_filter(&AudioOptions::default(), 1.4);
+ let graph = audio_filter(&AudioOptions::default(), 1.4, 0.0);
assert!(graph.starts_with("aresample=48000,"), "{graph}");
assert!(graph.ends_with("channel_layouts=stereo"), "{graph}");
}
@@ -408,7 +437,40 @@ mod tests {
start: Some(0.5),
..AudioOptions::default()
};
- assert!(audio_filter(&options, 1.4).starts_with("atrim=start=1.900000,"));
+ assert!(audio_filter(&options, 1.4, 0.0).starts_with("atrim=start=1.900000,"));
+ }
+
+ /// Sound that starts before the picture is cut where the picture does,
+ /// with no seek to have cut it.
+ #[test]
+ fn sound_that_starts_early_is_cut_where_the_picture_starts() {
+ let whole = audio_filter(&AudioOptions::default(), 0.5, -0.5);
+ assert!(
+ whole.starts_with("atrim=start=0.500000,asetpts=PTS-STARTPTS,aresample"),
+ "{whole}"
+ );
+ }
+
+ /// Sound that starts after the file's origin - after its picture - is
+ /// put in with silence in front, after any cut, so it plays with the
+ /// picture rather than ahead of it.
+ #[test]
+ fn sound_that_starts_late_is_put_in_after_silence() {
+ let whole = audio_filter(&AudioOptions::default(), 0.0, 0.5);
+ assert!(whole.starts_with("adelay=delays=500.000:all=1,"), "{whole}");
+ let cut = AudioOptions {
+ start: Some(0.2),
+ duration: Some(1.0),
+ ..AudioOptions::default()
+ };
+ assert!(
+ audio_filter(&cut, 0.0, 0.3).starts_with(
+ "atrim=start=0.200000:duration=1.000000,asetpts=PTS-STARTPTS,\
+ adelay=delays=300.000:all=1,atrim=duration=1.000000,"
+ ),
+ "{}",
+ audio_filter(&cut, 0.0, 0.3)
+ );
}
/// A window into a file whose sound starts at 1.4 s - silence, then a
@@ -451,6 +513,56 @@ mod tests {
assert!(loud(4_200..7_600) > 0.08, "the tone after half a second");
}
+ /// Where a tone first sounds in what a decoder of `path` hands back,
+ /// in seconds, at 8 kHz mono.
+ fn tone_onset(path: &str, start: Option<f64>) -> f64 {
+ let options = AudioOptions {
+ start,
+ rate: 8_000,
+ channels: 1,
+ format: SampleFormat::F32,
+ ..AudioOptions::default()
+ };
+ let mut decoder = AudioDecoder::open(path, &options).expect("opens");
+ let mut samples: Vec<f32> = Vec::new();
+ while let Some(frame) = decoder.next_frame().expect("decodes") {
+ let bytes = &frame.data(0)[..frame.samples() * 4];
+ samples.extend(
+ bytes
+ .chunks_exact(4)
+ .map(|b| f32::from_ne_bytes([b[0], b[1], b[2], b[3]])),
+ );
+ }
+ let first = samples
+ .iter()
+ .position(|sample| sample.abs() > 0.05)
+ .expect("a tone");
+ first as f64 / 8_000.0
+ }
+
+ /// A file's sound is measured from where its picture starts, so the two
+ /// play together as they do in a player. Each fixture sounds a tone half
+ /// a second after its first frame: one whose picture starts half a
+ /// second after its sound, one whose sound starts after its picture.
+ /// Each measured from its own first sample, the first played its tone a
+ /// second in and the second a fifth of a second in (#457).
+ #[test]
+ fn sound_plays_with_the_picture_whichever_starts_first() {
+ for fixture in ["picture-starts-late.mov", "sound-starts-late.mov"] {
+ let path = format!("{}/fixtures/{fixture}", env!("CARGO_MANIFEST_DIR"));
+ let whole = tone_onset(&path, None);
+ assert!(
+ (whole - 0.5).abs() < 0.01,
+ "{fixture}: the tone sounds {whole:.3} s after the first frame"
+ );
+ let cut = tone_onset(&path, Some(0.2));
+ assert!(
+ (cut - 0.3).abs() < 0.01,
+ "{fixture}: cut 0.2 s in, the tone sounds at {cut:.3} s"
+ );
+ }
+ }
+
#[test]
fn a_missing_file_is_an_error_not_a_panic() {
assert!(AudioDecoder::open("does-not-exist.mp3", &AudioOptions::default()).is_err());