Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(enc-ffmpeg): keep audio timestamps continuous when the input sample rate changes mid-stream

Richie McIlroy9fe24c7f46b8
diff --git a/crates/enc-ffmpeg/src/audio/base.rs b/crates/enc-ffmpeg/src/audio/base.rs
index fbfdbb72f..45b6358e4 100644
--- a/crates/enc-ffmpeg/src/audio/base.rs
+++ b/crates/enc-ffmpeg/src/audio/base.rs
@@ -42,10 +42,18 @@ impl AudioEncoderBase {
output: &mut S,
) -> Result<(), ffmpeg::Error> {
// Input frames are stamped in input-rate units; BufferedResampler
- // rescales them to the encoder's output rate.
- let input_rate = f64::from(self.resampler.input().rate);
+ // rescales them to the encoder's output rate. A source that changes
+ // rate mid-stream moves that unit, so the stored state moves with it.
+ let previous_rate = self.resampler.input().rate;
+ let input_rate = if frame.rate() > 0 {
+ frame.rate()
+ } else {
+ previous_rate
+ };
+ self.inner
+ .rescale_input_rate(f64::from(previous_rate), f64::from(input_rate));
self.inner
- .update_pts_with_rate(&mut frame, timestamp, input_rate);
+ .update_pts_with_rate(&mut frame, timestamp, f64::from(input_rate));
self.resampler.add_frame(frame);
diff --git a/crates/enc-ffmpeg/src/audio/buffered_resampler.rs b/crates/enc-ffmpeg/src/audio/buffered_resampler.rs
index 15cee7933..0680aa66a 100644
--- a/crates/enc-ffmpeg/src/audio/buffered_resampler.rs
+++ b/crates/enc-ffmpeg/src/audio/buffered_resampler.rs
@@ -78,6 +78,13 @@ impl BufferedResampler {
(output.format, output.channel_layout, output.rate),
) {
Ok(resampler) => {
+ let previous_rate = self.resampler.input().rate;
+ if previous_rate > 0 && frame.rate() > 0 {
+ self.min_next_pts = self.min_next_pts.map(|pts| {
+ (pts as f64 * f64::from(frame.rate()) / f64::from(previous_rate)).round()
+ as i64
+ });
+ }
self.resampler = resampler;
true
}
@@ -95,6 +102,10 @@ impl BufferedResampler {
}
pub fn add_frame(&mut self, mut frame: ffmpeg::frame::Audio) {
+ if !self.accepts(&frame) && !self.rebuild_for(&frame) {
+ return;
+ }
+
if let Some(min_next_pts) = self.min_next_pts
&& let Some(pts) = frame.pts()
{
@@ -103,10 +114,6 @@ impl BufferedResampler {
let pts = frame.pts().unwrap();
- if !self.accepts(&frame) && !self.rebuild_for(&frame) {
- return;
- }
-
let mut resampled_frame = ffmpeg::frame::Audio::empty();
if let Err(error) = self.resampler.run(&frame, &mut resampled_frame) {
@@ -502,6 +509,32 @@ mod test {
assert_eq!(samples, 400);
}
+ #[test]
+ fn a_rate_change_keeps_the_timeline_continuous() {
+ let packed_u8 = format::Sample::U8(cap_media_info::Type::Packed);
+ let mut bufferer = BufferedResampler::new(
+ AudioInfo::new_raw(packed_u8, 48_000, 1),
+ AudioInfo::new_raw(packed_u8, 48_000, 1),
+ )
+ .unwrap();
+ for k in 0..100i64 {
+ bufferer.add_frame(frame(packed_u8, 1, 48_000, 4_800, k * 4_800));
+ }
+ bufferer.add_frame(frame(packed_u8, 1, 8_000, 800, 80_000));
+
+ let (last, pts) = bufferer
+ .buffer
+ .iter()
+ .rev()
+ .find(|(frame, _)| frame.samples() > 0)
+ .unwrap();
+ let end_secs = (*pts as f64 + last.samples() as f64) / 48_000.0;
+ assert!(
+ (10.0..10.3).contains(&end_secs),
+ "the 8 kHz frame ends at {end_secs} s, not right after the 10 s at 48 kHz"
+ );
+ }
+
#[test]
fn frames_no_resampler_accepts_are_dropped() {
let mut bufferer = create_resampler(IN_RATE);
diff --git a/crates/enc-ffmpeg/src/base.rs b/crates/enc-ffmpeg/src/base.rs
index c2a5626df..5a017399a 100644
--- a/crates/enc-ffmpeg/src/base.rs
+++ b/crates/enc-ffmpeg/src/base.rs
@@ -153,6 +153,17 @@ impl EncoderBase {
/// whereas [`Self::update_pts`] uses the encoder's own (output) time
/// base. Mixing the two conventions plays non-48kHz microphones at the
/// wrong speed.
+ /// Re-expresses the stored timestamp state, kept in input-rate samples,
+ /// at a new input rate.
+ pub fn rescale_input_rate(&mut self, from: f64, to: f64) {
+ if from <= 0.0 || to <= 0.0 || from == to {
+ return;
+ }
+ let rescale = |pts: i64| (pts as f64 * to / from).round() as i64;
+ self.first_pts = self.first_pts.map(rescale);
+ self.last_frame_pts = self.last_frame_pts.map(rescale);
+ }
+
pub fn update_pts_with_rate(
&mut self,
frame: &mut frame::Frame,