Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: align mixer inputs to their capture timestamps
Richie McIlroy2b47d7b2ea7a
diff --git a/crates/recording/src/sources/audio_mixer.rs b/crates/recording/src/sources/audio_mixer.rsindex f4d0b8cce..32ff5b7ca 100644--- a/crates/recording/src/sources/audio_mixer.rs+++ b/crates/recording/src/sources/audio_mixer.rs@@ -14,7 +14,9 @@ use std::{use tokio_util::sync::CancellationToken;use tracing::{debug, info, warn};-use crate::output_pipeline::{AudioFrame, HealthSender, PipelineHealthEvent, emit_health};+use crate::output_pipeline::{+ AudioFrame, HealthSender, PipelineHealthEvent, emit_health, trim_audio_frame_front,+};const DEFAULT_BUFFER_TIMEOUT: Duration = Duration::from_millis(100);const MIN_BUFFER_TIMEOUT_WIRED: Duration = Duration::from_millis(20);@@ -38,6 +40,7 @@ struct MixerSource {buffer_last: Option<(Timestamp, Duration)>,last_input_timestamp: Option<Timestamp>,clock_state: SourceClockState,+ samples_supplied: usize,}pub struct AudioMixerBuilder {@@ -101,6 +104,7 @@ impl AudioMixerBuilder {buffer_last: None,last_input_timestamp: None,clock_state: SourceClockState::new(clock_label),+ samples_supplied: 0,});}@@ -423,6 +427,11 @@ impl AudioMixer {self._aformat = graph.aformat;self.abuffersink = graph.abuffersink;self._filter_graph = graph.filter_graph;+ for source in &mut self.sources {+ source.samples_supplied =+ (self.samples_out as u128 * source.info.sample_rate as u128+ / Self::INFO.sample_rate as u128) as usize;+ }self.max_buffer_timeout = self.sources.iter()@@ -631,48 +640,6 @@ impl AudioMixer {}}- if let Some(start_timestamp) = self.start_timestamp- && let Some(elapsed_since_start) = now- .duration_since(self.timestamps)- .checked_sub(start_timestamp.duration_since(self.timestamps))- && elapsed_since_start > self.max_buffer_timeout- {- for source in &mut self.sources {- if source.buffer_last.is_none() {- let rate = source.info.rate();- let buffer_timeout = source.buffer_timeout;-- let mut remaining = elapsed_since_start;- while remaining > buffer_timeout {- let chunk_samples = samples_for_timeout(rate, buffer_timeout);- let frame_duration = duration_from_samples(chunk_samples, rate);-- let mut frame = ffmpeg::frame::Audio::new(- source.info.sample_format,- chunk_samples,- source.info.channel_layout(),- );-- for i in 0..frame.planes() {- frame.data_mut(i).fill(0);- }-- frame.set_rate(source.info.rate() as u32);-- let timestamp =- start_timestamp + elapsed_since_start.saturating_sub(remaining);- source.buffer_last = Some((timestamp, frame_duration));- source.buffer.push_front(AudioFrame::new(frame, timestamp));-- if frame_duration.is_zero() {- break;- }-- remaining = remaining.saturating_sub(frame_duration);- }- }- }- }Ok(())}@@ -709,12 +676,49 @@ impl AudioMixer {}for (i, source) in self.sources.iter_mut().enumerate() {+ let rate = source.info.rate();for buffer in source.buffer.drain(..) {- let result = self.abuffers[i].source().add(&buffer.inner);+ let position =+ sample_position(buffer.timestamp, start_timestamp, self.timestamps, rate);+ while position > source.samples_supplied as i128 {+ let samples = ((position - source.samples_supplied as i128) as usize)+ .min(samples_for_timeout(rate, source.buffer_timeout));+ let silence = silence_frame(source.info, samples);+ self.abuffers[i]+ .source()+ .add(&silence)+ .with_context(|| format!("Audio mixer source {i} silence input failed"))?;+ source.samples_supplied += samples;+ }+ let overlap = (source.samples_supplied as i128 - position).max(0) as usize;+ let frame = if overlap == 0 {+ buffer.inner+ } else if overlap >= buffer.inner.samples() {+ continue;+ } else {+ trim_audio_frame_front(&buffer.inner, overlap)+ .context("Audio mixer input overlap trim failed")?+ };+ let samples = frame.samples();+ let result = self.abuffers[i].source().add(&frame);if self.required_stop.is_some() {result.with_context(|| format!("Audio mixer source {i} filter input failed"))?;}+ source.samples_supplied += samples;+ }+ if source.last_input_timestamp.is_none() {+ let position = sample_position(now, start_timestamp, self.timestamps, rate)+ - samples_for_timeout(rate, source.buffer_timeout) as i128;+ while position > source.samples_supplied as i128 {+ let samples = ((position - source.samples_supplied as i128) as usize)+ .min(samples_for_timeout(rate, source.buffer_timeout));+ let silence = silence_frame(source.info, samples);+ self.abuffers[i].source().add(&silence).with_context(|| {+ format!("Audio mixer source {i} startup silence failed")+ })?;+ source.samples_supplied += samples;+ }}}@@ -771,6 +775,31 @@ impl AudioMixer {}}+fn sample_position(+ timestamp: Timestamp,+ start: Timestamp,+ timestamps: Timestamps,+ rate: i32,+) -> i128 {+ let nanos = timestamp.duration_since(timestamps).as_nanos() as i128+ - start.duration_since(timestamps).as_nanos() as i128;+ (nanos * rate as i128 + 500_000_000) / 1_000_000_000+}++fn silence_frame(info: AudioInfo, samples: usize) -> ffmpeg::frame::Audio {+ let mut frame = ffmpeg::frame::Audio::new(info.sample_format, samples, info.channel_layout());+ let silence = if matches!(info.sample_format, cap_media_info::Sample::U8(_)) {+ 128+ } else {+ 0+ };+ for plane in 0..frame.planes() {+ frame.data_mut(plane).fill(silence);+ }+ frame.set_rate(info.sample_rate);+ frame+}+fn detect_format_mismatch(current: &AudioInfo, frame: &ffmpeg::frame::Audio) -> Option<AudioInfo> {let frame_rate: u32 = frame.rate();let frame_channels: usize = frame.channels() as usize;@@ -965,6 +994,168 @@ mod test {const ONE_SECOND: Duration = Duration::from_secs(1);const SAMPLES_SECOND: usize = SOURCE_INFO.rate() as usize;+ struct PcmProbe {+ mixer: AudioMixer,+ inputs: Vec<mpsc::Sender<AudioFrame>>,+ output: mpsc::Receiver<AudioFrame>,+ pcm: Vec<[f32; 2]>,+ }++ impl PcmProbe {+ fn new(sources: usize) -> Self {+ let (sender, output) = mpsc::channel(128);+ let mut builder = AudioMixerBuilder::new();+ let inputs = (0..sources)+ .map(|_| {+ let (sender, receiver) = mpsc::channel(32);+ builder.add_source(AudioMixer::INFO, receiver);+ sender+ })+ .collect();+ Self {+ mixer: builder.build(sender).unwrap(),+ inputs,+ output,+ pcm: Vec::new(),+ }+ }++ fn at(&self, millis: u64) -> Timestamp {+ Timestamp::Instant(self.mixer.timestamps.instant() + Duration::from_millis(millis))+ }++ fn send(&mut self, source: usize, millis: u64, samples: usize, peak: Option<usize>) {+ let mut bytes = vec![0; samples * 8];+ if let Some(peak) = peak {+ let offset = peak * 8 + source * 4;+ bytes[offset..offset + 4].copy_from_slice(&0.5_f32.to_le_bytes());+ }+ let timestamp = self.at(millis);+ self.inputs[source]+ .try_send(AudioFrame::new(+ AudioMixer::INFO.wrap_frame(&bytes),+ timestamp,+ ))+ .unwrap();+ }++ fn tick(&mut self, millis: u64) {+ assert!(+ self.mixer+ .tick(self.mixer.timestamps, self.at(millis))+ .unwrap()+ );+ while let Ok(Some(frame)) = self.output.try_next() {+ for pair in frame.inner.data(0)[..frame.inner.samples() * 8].chunks_exact(8) {+ self.pcm.push([+ f32::from_le_bytes(pair[..4].try_into().unwrap()),+ f32::from_le_bytes(pair[4..].try_into().unwrap()),+ ]);+ }+ }+ }++ fn peaks(&self, channel: usize) -> Vec<usize> {+ self.pcm+ .iter()+ .enumerate()+ .filter_map(|(index, sample)| (sample[channel].abs() > 0.01).then_some(index))+ .collect()+ }+ }++ #[test]+ fn pcm_arrival_schedules_preserve_sixty_ms_source_offset() {+ for early_poll in [false, true] {+ let mut probe = PcmProbe::new(2);+ probe.send(1, 200, 24_000, Some(9_600));+ if early_poll {+ probe.tick(200);+ probe.tick(254);+ }+ probe.send(0, 260, 24_000, Some(9_600));+ probe.send(1, 700, 2_880, None);+ probe.tick(300);+ assert_eq!(probe.peaks(1), [9_600]);+ assert_eq!(probe.peaks(0), [12_480]);+ assert_eq!(probe.peaks(0)[0] - probe.peaks(1)[0], 2_880);+ }+ }++ #[test]+ fn pcm_arrival_schedules_align_simultaneous_capture_events() {+ for early_poll in [false, true] {+ let mut probe = PcmProbe::new(2);+ probe.send(1, 200, 24_000, Some(9_600));+ if early_poll {+ probe.tick(200);+ probe.tick(254);+ }+ probe.send(0, 260, 24_000, Some(6_720));+ probe.send(1, 700, 2_880, None);+ probe.tick(300);+ assert_eq!(probe.peaks(0), [9_600]);+ assert_eq!(probe.peaks(1), [9_600]);+ }+ }++ #[test]+ fn pcm_continuous_system_and_late_mic_follow_capture_time() {+ for early_poll in [false, true] {+ let mut probe = PcmProbe::new(2);+ probe.send(0, 0, 24_000, Some(14_400));+ if early_poll {+ probe.tick(0);+ probe.tick(254);+ }+ probe.send(1, 200, 14_400, Some(4_800));+ probe.tick(300);+ assert_eq!(probe.peaks(0), [14_400]);+ assert_eq!(probe.peaks(1), [14_400]);+ }+ }++ #[test]+ fn pcm_sparse_system_and_pause_resume_preserve_later_gaps() {+ let mut probe = PcmProbe::new(2);+ probe.send(1, 0, 48_000, Some(14_400));+ probe.send(0, 0, 960, None);+ probe.send(0, 300, 960, Some(0));+ probe.send(0, 600, 960, Some(0));+ probe.send(0, 900, 4_800, Some(0));+ probe.tick(1_000);+ assert_eq!(probe.peaks(0), [14_400, 28_800, 43_200]);+ assert_eq!(probe.peaks(1), [14_400]);+ probe.send(0, 1_500, 4_800, Some(0));+ probe.send(1, 1_500, 4_800, Some(0));+ probe.tick(1_600);+ assert_eq!(probe.peaks(0), [14_400, 28_800, 43_200, 72_000]);+ assert_eq!(probe.peaks(1), [14_400, 72_000]);+ }++ #[test]+ fn pcm_single_input_is_unchanged_and_reset_keeps_the_supplied_cursor() {+ let mut probe = PcmProbe::new(1);+ probe.send(0, 200, 4_800, Some(2_400));+ probe.tick(300);+ assert_eq!(probe.pcm.len(), 4_800);+ assert_eq!(probe.peaks(0), [2_400]);+ for (index, sample) in probe.pcm.iter().enumerate() {+ assert_eq!(+ *sample,+ if index == 2_400 {+ [0.5, 0.0]+ } else {+ [0.0, 0.0]+ }+ );+ }+ probe.mixer.force_reset_source(mixer_source_label(0), 2_000);+ probe.send(0, 400, 4_800, Some(0));+ probe.tick(500);+ assert_eq!(probe.peaks(0), [2_400, 9_600]);+ }+#[tokio::test]async fn mix_sources() {let (tx, mut output_rx) = mpsc::channel(4);