Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(enc-ffmpeg): keep audio timestamps continuous when the input sample rate changes mid-stream
Richie McIlroy9fe24c7f46b8
diff --git a/crates/enc-ffmpeg/src/audio/base.rs b/crates/enc-ffmpeg/src/audio/base.rsindex fbfdbb72f..45b6358e4 100644--- a/crates/enc-ffmpeg/src/audio/base.rs+++ b/crates/enc-ffmpeg/src/audio/base.rs@@ -42,10 +42,18 @@ impl AudioEncoderBase {output: &mut S,) -> Result<(), ffmpeg::Error> {// Input frames are stamped in input-rate units; BufferedResampler- // rescales them to the encoder's output rate.- let input_rate = f64::from(self.resampler.input().rate);+ // rescales them to the encoder's output rate. A source that changes+ // rate mid-stream moves that unit, so the stored state moves with it.+ let previous_rate = self.resampler.input().rate;+ let input_rate = if frame.rate() > 0 {+ frame.rate()+ } else {+ previous_rate+ };+ self.inner+ .rescale_input_rate(f64::from(previous_rate), f64::from(input_rate));self.inner- .update_pts_with_rate(&mut frame, timestamp, input_rate);+ .update_pts_with_rate(&mut frame, timestamp, f64::from(input_rate));self.resampler.add_frame(frame);diff --git a/crates/enc-ffmpeg/src/audio/buffered_resampler.rs b/crates/enc-ffmpeg/src/audio/buffered_resampler.rsindex 15cee7933..0680aa66a 100644--- a/crates/enc-ffmpeg/src/audio/buffered_resampler.rs+++ b/crates/enc-ffmpeg/src/audio/buffered_resampler.rs@@ -78,6 +78,13 @@ impl BufferedResampler {(output.format, output.channel_layout, output.rate),) {Ok(resampler) => {+ let previous_rate = self.resampler.input().rate;+ if previous_rate > 0 && frame.rate() > 0 {+ self.min_next_pts = self.min_next_pts.map(|pts| {+ (pts as f64 * f64::from(frame.rate()) / f64::from(previous_rate)).round()+ as i64+ });+ }self.resampler = resampler;true}@@ -95,6 +102,10 @@ impl BufferedResampler {}pub fn add_frame(&mut self, mut frame: ffmpeg::frame::Audio) {+ if !self.accepts(&frame) && !self.rebuild_for(&frame) {+ return;+ }+if let Some(min_next_pts) = self.min_next_pts&& let Some(pts) = frame.pts(){@@ -103,10 +114,6 @@ impl BufferedResampler {let pts = frame.pts().unwrap();- if !self.accepts(&frame) && !self.rebuild_for(&frame) {- return;- }-let mut resampled_frame = ffmpeg::frame::Audio::empty();if let Err(error) = self.resampler.run(&frame, &mut resampled_frame) {@@ -502,6 +509,32 @@ mod test {assert_eq!(samples, 400);}+ #[test]+ fn a_rate_change_keeps_the_timeline_continuous() {+ let packed_u8 = format::Sample::U8(cap_media_info::Type::Packed);+ let mut bufferer = BufferedResampler::new(+ AudioInfo::new_raw(packed_u8, 48_000, 1),+ AudioInfo::new_raw(packed_u8, 48_000, 1),+ )+ .unwrap();+ for k in 0..100i64 {+ bufferer.add_frame(frame(packed_u8, 1, 48_000, 4_800, k * 4_800));+ }+ bufferer.add_frame(frame(packed_u8, 1, 8_000, 800, 80_000));++ let (last, pts) = bufferer+ .buffer+ .iter()+ .rev()+ .find(|(frame, _)| frame.samples() > 0)+ .unwrap();+ let end_secs = (*pts as f64 + last.samples() as f64) / 48_000.0;+ assert!(+ (10.0..10.3).contains(&end_secs),+ "the 8 kHz frame ends at {end_secs} s, not right after the 10 s at 48 kHz"+ );+ }+#[test]fn frames_no_resampler_accepts_are_dropped() {let mut bufferer = create_resampler(IN_RATE);diff --git a/crates/enc-ffmpeg/src/base.rs b/crates/enc-ffmpeg/src/base.rsindex c2a5626df..5a017399a 100644--- a/crates/enc-ffmpeg/src/base.rs+++ b/crates/enc-ffmpeg/src/base.rs@@ -153,6 +153,17 @@ impl EncoderBase {/// whereas [`Self::update_pts`] uses the encoder's own (output) time/// base. Mixing the two conventions plays non-48kHz microphones at the/// wrong speed.+ /// Re-expresses the stored timestamp state, kept in input-rate samples,+ /// at a new input rate.+ pub fn rescale_input_rate(&mut self, from: f64, to: f64) {+ if from <= 0.0 || to <= 0.0 || from == to {+ return;+ }+ let rescale = |pts: i64| (pts as f64 * to / from).round() as i64;+ self.first_pts = self.first_pts.map(rescale);+ self.last_frame_pts = self.last_frame_pts.map(rescale);+ }+pub fn update_pts_with_rate(&mut self,frame: &mut frame::Frame,