Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

fix(media): a trimmed clip's sound starts where its cut is, not behind it (#535)

Sireenb5042b5c8d17
diff --git a/src/crates/concat-media/fixtures/keyframes-3s-apart.ts b/src/crates/concat-media/fixtures/keyframes-3s-apart.ts
new file mode 100644
index 00000000..b14fb583
Binary files /dev/null and b/src/crates/concat-media/fixtures/keyframes-3s-apart.ts differ
diff --git a/src/crates/concat-media/fixtures/sound-behind-the-picture.mpg b/src/crates/concat-media/fixtures/sound-behind-the-picture.mpg
new file mode 100644
index 00000000..4d2ccd71
Binary files /dev/null and b/src/crates/concat-media/fixtures/sound-behind-the-picture.mpg differ
diff --git a/src/crates/concat-media/src/audio.rs b/src/crates/concat-media/src/audio.rs
index 4ff1aaaf..01641e4f 100644
--- a/src/crates/concat-media/src/audio.rs
+++ b/src/crates/concat-media/src/audio.rs
@@ -772,11 +772,11 @@ fn run_mix(
let decoder = ffmpeg::codec::Context::from_parameters(stream.parameters())
.and_then(|context| context.decoder().audio())
.map_err(|error| ffi::fail("open decoder", path, error))?;
- // Near the in-point: the packet at or before it. The graph cuts the
- // rest, by the lead measured off the first frame below.
+ // Near the in-point: the packet of the sound at or before it, on the
+ // sound's own timestamps. The graph cuts the rest, by the lead
+ // measured off the first frame below.
if in_point > 0.0 {
- let target = ((in_point + start) * f64::from(ffmpeg::sys::AV_TIME_BASE)) as i64;
- let _ = input.seek(target, ..=target);
+ let _ = ffi::seek_audio(&mut input, stream_index, in_point + start);
}
let label = format!("{index}:a");
let mut mix_input = MixInput {
@@ -1811,6 +1811,34 @@ mod tests {
assert!(difference <= 1, "the window differs by {difference}");
}
+ /// A clip trimmed into a Program Stream mixes from its first sample,
+ /// not from behind the in-point. The fixture's sound is interleaved
+ /// behind its picture, and the mix's seek used to be made on the
+ /// picture's timestamps, forward: it landed past the in-point and the
+ /// overshoot went in as silence in front of the clip. The tone has
+ /// sounded from half a second in, so a cut at 3.5 s is inside it.
+ #[test]
+ fn a_trimmed_program_stream_clip_mixes_from_its_first_sample() {
+ let path = format!(
+ "{}/fixtures/sound-behind-the-picture.mpg",
+ env!("CARGO_MANIFEST_DIR")
+ );
+ let mut piece = clip(&path);
+ piece.duration = 0.6;
+ piece.source_start = 3.5;
+ let mixed = left(&[piece], &[], 0.0, 0.6);
+ assert_eq!(mixed.len(), (0.6 * f64::from(MIX_RATE)) as usize);
+ let first = mixed
+ .iter()
+ .position(|sample| i32::from(*sample).abs() > 1_500)
+ .expect("the tone is in the mix");
+ assert!(
+ first < 48 * 30,
+ "the tone starts {} samples in, not at once",
+ first
+ );
+ }
+
#[test]
fn ordinary_chains_pass_validation() {
for good in [
diff --git a/src/crates/concat-media/src/ffi.rs b/src/crates/concat-media/src/ffi.rs
index c755228f..7b848595 100644
--- a/src/crates/concat-media/src/ffi.rs
+++ b/src/crates/concat-media/src/ffi.rs
@@ -227,6 +227,54 @@ pub(crate) fn av_ticks(seconds: Rational) -> i64 {
scaled.floor()
}
+/// Seeks a file's sound to `at` seconds on the file's clock: the packet of
+/// stream `stream` at or before it, on the sound's own timestamps, never
+/// one after it. The reader cuts exactly at `at`, so a landing short of it
+/// costs a little decoding and a landing past it is sound the cut can never
+/// reach.
+///
+/// The wrapper's `seek` cannot ask this. It seeks by the file's default
+/// stream - the picture, in a file with both - and for the first packet
+/// past the target; a demuxer without an index, MPEG-TS and MPEG-PS,
+/// then finds the packet on the picture's timestamps and hands the sound
+/// interleaved after it. On a Program Stream that is a frame or two of
+/// sound past the cut, which the reader padded as silence: every trimmed
+/// .mpg began its sound late (audit 2026-10-24, E-M2). Asking on the
+/// sound's own stream, and backwards, lands on the packet the cut itself
+/// will keep.
+///
+/// A file whose sound cannot be sought at all - a raw stream with no
+/// timestamps - is read from the top instead, so the cut still comes out
+/// right at the cost of decoding what comes before it; only when even the
+/// top cannot be sought does the error reach the caller.
+pub(crate) fn seek_audio(
+ input: &mut ffmpeg::format::context::Input,
+ stream: usize,
+ at: f64,
+) -> Result<(), ffmpeg::Error> {
+ let time_base = input
+ .stream(stream)
+ .map(|stream| stream.time_base())
+ .ok_or(ffmpeg::Error::StreamNotFound)?;
+ // SAFETY: the input is open for the reader's lifetime, so its format
+ // pointer is valid; the stream index was found in that same input, and
+ // `AVSEEK_FLAG_BACKWARD` is a plain integer.
+ let done = unsafe {
+ ffmpeg::sys::av_seek_frame(
+ input.as_mut_ptr(),
+ stream as libc::c_int,
+ (at * f64::from(time_base.denominator()) / f64::from(time_base.numerator())) as i64,
+ ffmpeg::sys::AVSEEK_FLAG_BACKWARD,
+ )
+ };
+ if done >= 0 {
+ return Ok(());
+ }
+ // The wrapper's fallback: the top of the file, which every demuxer that
+ // can seek at all can reach, and which no index is needed to find.
+ input.seek(0, ..=0).map_err(|_| ffmpeg::Error::from(done))
+}
+
/// The rotation a stream asks players to apply before showing its frames,
/// in whole degrees clockwise, normalised to `0..360`.
///
diff --git a/src/crates/concat-media/src/samples.rs b/src/crates/concat-media/src/samples.rs
index efd2e3e4..a7fbf64b 100644
--- a/src/crates/concat-media/src/samples.rs
+++ b/src/crates/concat-media/src/samples.rs
@@ -210,9 +210,7 @@ impl AudioDecoder {
if let Some(start) = options.start
&& start > 0.0
{
- let target = ((start + origin) * f64::from(ffmpeg::sys::AV_TIME_BASE)) as i64;
- input
- .seek(target, ..=target)
+ ffi::seek_audio(&mut input, stream_index, start + origin)
.map_err(|error| ffi::fail("seek", path, error))?;
}
@@ -632,4 +630,30 @@ mod tests {
fn a_missing_file_is_an_error_not_a_panic() {
assert!(AudioDecoder::open("does-not-exist.mp3", &AudioOptions::default()).is_err());
}
+
+ /// A window cut into a file with both sound and picture starts its
+ /// sound where the cut is, not behind it. The Program Stream
+ /// interleaves its sound behind its picture, and the seek used to be
+ /// made on the picture's timestamps, forward: it landed on the picture
+ /// at the cut and handed the sound from after it - a frame or two of
+ /// silence in front of every trimmed .mpg. The transport stream
+ /// interleaves tightly and landed either way; it is here to prove the
+ /// seek on the sound's own stream keeps that. Each fixture sounds a
+ /// tone from half a second in, so every cut here is inside it.
+ #[test]
+ fn a_cut_starts_its_sound_where_the_cut_is_whichever_way_the_file_interleaves() {
+ let fixtures = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/");
+ for (name, cut) in [
+ ("sound-behind-the-picture.mpg", 1.0),
+ ("sound-behind-the-picture.mpg", 3.5),
+ ("keyframes-3s-apart.ts", 1.0),
+ ("keyframes-3s-apart.ts", 4.0),
+ ] {
+ let onset = tone_onset(&format!("{fixtures}{name}"), Some(cut));
+ assert!(
+ onset < 0.03,
+ "{name} cut {cut} s in: the tone starts {onset:.3} s behind the cut"
+ );
+ }
+ }
}