Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: drain captured audio when stopping the mixer

Richie McIlroy1da9787b2cc9
diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs
index 960574421..0c35351ae 100644
--- a/crates/recording/src/output_pipeline/core.rs
+++ b/crates/recording/src/output_pipeline/core.rs
@@ -1,5 +1,5 @@
use super::start_gate::{AudioAdmission, RecordingStartGate};
-use crate::sources::audio_mixer::AudioMixer;
+use crate::sources::audio_mixer::{AudioMixer, MIXER_STOP_TIMEOUT};
use anyhow::{Context, anyhow};
use cap_media_info::{AudioInfo, VideoInfo};
use cap_timestamp::{MasterClock, SourceClockOutcome, SourceClockState, Timestamp, Timestamps};
@@ -3805,6 +3805,7 @@ impl PreparedAudioSources {
audio_anchor: AudioAnchor,
) {
let audio_info = self.audio_info;
+ let mixed_audio = self.erased_audio_sources.len() > 1;
let has_wireless_source = self.has_wireless_source;
let health_tx = setup_ctx.health_tx().clone();
let master_clock = setup_ctx.master_clock().clone();
@@ -3905,16 +3906,49 @@ impl PreparedAudioSources {
None
};
+ let mixed_sources_stopped = was_cancelled && mixed_audio;
+ if mixed_sources_stopped {
+ let stopped = tokio::time::timeout(
+ Duration::from_secs(1),
+ stop_audio_sources(&mut self.erased_audio_sources, required_scope.as_ref()),
+ )
+ .await;
+ if stopped.is_err() {
+ let error = anyhow!("Mixed audio sources did not stop before the deadline");
+ if let Some(scope) = &required_scope {
+ scope.fail_required(error.to_string());
+ }
+ let _ = audio_error.get_or_insert(error);
+ }
+ self.erased_audio_sources.clear();
+ }
+
if was_cancelled && !audio_degraded {
let drain_start = std::time::Instant::now();
+ let drain_deadline = tokio::time::Instant::now() + MIXER_STOP_TIMEOUT;
let max_drain_frames = 500u64;
let mut drained = 0u64;
let mut skipped = 0u64;
let mut degraded_during_drain = false;
while drained < max_drain_frames {
- match self.audio_rx.try_next() {
- Ok(Some(frame)) => match process_audio_frame(
+ let next = if mixed_audio {
+ match tokio::time::timeout_at(drain_deadline, self.audio_rx.next()).await {
+ Ok(frame) => frame,
+ Err(_) => {
+ let error = anyhow!("Mixed audio output did not close before the drain deadline");
+ if let Some(scope) = &required_scope {
+ scope.fail_required(error.to_string());
+ }
+ let _ = audio_error.get_or_insert(error);
+ break;
+ }
+ }
+ } else {
+ self.audio_rx.try_next().ok().flatten()
+ };
+ match next {
+ Some(frame) => match process_audio_frame(
AudioFrameProcessContext {
audio_info: &audio_info,
sample_rate,
@@ -3965,7 +3999,7 @@ impl PreparedAudioSources {
skipped += 1;
}
},
- Ok(None) | Err(_) => break,
+ None => break,
}
}
@@ -3986,7 +4020,9 @@ impl PreparedAudioSources {
let final_pause_duration = shared_pause.total_pause_duration();
- if let Some(scope) = &required_scope {
+ if let Some(scope) = &required_scope
+ && !mixed_sources_stopped
+ {
stop_audio_sources(&mut self.erased_audio_sources, Some(scope)).await;
}
@@ -4104,7 +4140,7 @@ impl PreparedAudioSources {
let _ = gap_summary_slot.set(gap_tracker.gap_summary());
}
- if required_scope.is_none() {
+ if required_scope.is_none() && !mixed_sources_stopped {
stop_audio_sources(&mut self.erased_audio_sources, None).await;
}
@@ -9235,6 +9271,64 @@ mod tests {
}
}
+ struct StopFrameSource {
+ output: mpsc::Sender<AudioFrame>,
+ final_frame: Option<AudioFrame>,
+ }
+
+ impl AudioSource for StopFrameSource {
+ type Config = Option<AudioFrame>;
+
+ fn setup(
+ final_frame: Self::Config,
+ output: mpsc::Sender<AudioFrame>,
+ _: &mut SetupCtx,
+ ) -> impl Future<Output = anyhow::Result<Self>> + Send + 'static {
+ std::future::ready(Ok(Self {
+ output,
+ final_frame,
+ }))
+ }
+
+ fn audio_info(&self) -> AudioInfo {
+ AudioMixer::INFO
+ }
+
+ async fn stop(&mut self) -> anyhow::Result<()> {
+ if let Some(frame) = self.final_frame.take() {
+ self.output.send(frame).await?;
+ }
+ Ok(())
+ }
+ }
+
+ #[tokio::test]
+ async fn mixed_pipeline_stop_receives_audio_delivered_during_source_stop() {
+ let temp = tempfile::tempdir().unwrap();
+ let timestamps = Timestamps::now();
+ let sent = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let pipeline = OutputPipeline::builder(temp.path().join("mixed-stop.ogg"))
+ .with_timestamps(timestamps)
+ .with_audio_source::<StopFrameSource>(None)
+ .with_audio_source::<StopFrameSource>(Some(AudioFrame::new(
+ AudioMixer::INFO.empty_frame(4_800),
+ Timestamp::Instant(timestamps.instant()),
+ )))
+ .build::<RecordingAudioMuxer>(sent.clone())
+ .await
+ .unwrap();
+ tokio::time::timeout(Duration::from_secs(3), pipeline.stop())
+ .await
+ .unwrap()
+ .unwrap();
+ let frames = sent.lock().unwrap();
+ assert_eq!(
+ frames.iter().map(|frame| frame.samples).sum::<usize>(),
+ 4_800
+ );
+ assert_eq!(frames[0].timestamp, Duration::ZERO);
+ }
+
struct AudioTimelineHarness {
info: AudioInfo,
timestamps: Timestamps,
diff --git a/crates/recording/src/sources/audio_mixer.rs b/crates/recording/src/sources/audio_mixer.rs
index c21740634..19717f263 100644
--- a/crates/recording/src/sources/audio_mixer.rs
+++ b/crates/recording/src/sources/audio_mixer.rs
@@ -24,6 +24,7 @@ const MIN_BUFFER_TIMEOUT_WIRELESS: Duration = Duration::from_millis(90);
const MAX_BUFFER_TIMEOUT: Duration = Duration::from_millis(250);
const BUFFER_TIMEOUT_HEADROOM: f64 = 2.5;
const STARTUP_GRACE: Duration = Duration::from_secs(1);
+pub(crate) const MIXER_STOP_TIMEOUT: Duration = Duration::from_secs(2);
pub const MAX_BUFFERING_TICKS: u32 = 45;
pub const AUDIO_OUTPUT_FRAMES: u32 = cap_timestamp::AUDIO_OUTPUT_FRAMES as u32;
@@ -43,6 +44,7 @@ struct MixerSource {
clock_state: SourceClockState,
samples_supplied: usize,
late_samples_dropped: u64,
+ ended: bool,
}
pub struct AudioMixerBuilder {
@@ -108,6 +110,7 @@ impl AudioMixerBuilder {
clock_state: SourceClockState::new(clock_label),
samples_supplied: 0,
late_samples_dropped: 0,
+ ended: false,
});
}
@@ -146,6 +149,8 @@ impl AudioMixerBuilder {
buffering: MixerBufferingTracker::new(),
health_tx: self.health_tx,
required_stop: self.required_stop,
+ drain_deadline: None,
+ finishing: false,
})
}
@@ -177,16 +182,11 @@ impl AudioMixerBuilder {
.is_some_and(CancellationToken::is_cancelled)
{
info!("Mixer stop flag triggered");
+ mixer.finish()?;
break;
}
- #[cfg(target_os = "macos")]
- let now = Timestamp::MachAbsoluteTime(cap_timestamp::MachAbsoluteTimestamp::now());
- #[cfg(windows)]
- let now =
- Timestamp::PerformanceCounter(cap_timestamp::PerformanceCounterTimestamp::now());
- #[cfg(not(any(target_os = "macos", windows)))]
- let now = Timestamp::Instant(Instant::now());
+ let now = mixer_now();
if !mixer.tick(start, now)? {
info!("Mixer output closed");
@@ -227,6 +227,8 @@ pub struct AudioMixer {
buffering: MixerBufferingTracker,
health_tx: Option<HealthSender>,
required_stop: Option<CancellationToken>,
+ drain_deadline: Option<Instant>,
+ finishing: bool,
}
#[derive(Debug, Clone)]
@@ -522,12 +524,19 @@ impl AudioMixer {
let rate = source.info.rate();
loop {
+ if self
+ .drain_deadline
+ .is_some_and(|deadline| Instant::now() >= deadline)
+ {
+ source.rx.close();
+ }
let AudioFrame {
inner: frame,
timestamp: raw_timestamp,
} = match source.rx.try_next() {
Ok(Some(frame)) => frame,
Ok(None) => {
+ source.ended = true;
anyhow::ensure!(
self.required_stop
.as_ref()
@@ -624,6 +633,30 @@ impl AudioMixer {
self.tick_with_output(now, |sink, frame| sink.sink().frame(frame))
}
+ fn finish(&mut self) -> anyhow::Result<()> {
+ if let Some(stop) = &self.required_stop {
+ stop.cancel();
+ }
+ self.drain_deadline = Some(Instant::now() + STARTUP_GRACE);
+ loop {
+ self.buffer_sources(mixer_now())?;
+ if self.sources.iter().all(|source| source.ended) {
+ break;
+ }
+ std::thread::sleep(Duration::from_millis(1));
+ }
+ self.finishing = true;
+ self.drain_deadline = Some(Instant::now() + STARTUP_GRACE);
+ if !self.tick(self.timestamps, mixer_now())? {
+ return Ok(());
+ }
+ for input in &mut self.abuffers {
+ input.source().flush().context("Audio mixer flush failed")?;
+ }
+ self.tick(self.timestamps, mixer_now())?;
+ Ok(())
+ }
+
fn tick_with_output(
&mut self,
now: Timestamp,
@@ -639,7 +672,11 @@ impl AudioMixer {
};
let wall_now = Instant::now();
- let laggard = self.detect_laggard_source(now);
+ let laggard = if self.finishing {
+ None
+ } else {
+ self.detect_laggard_source(now)
+ };
let action = self
.buffering
.observe(wall_now, laggard, self.health_tx.as_ref());
@@ -662,11 +699,17 @@ impl AudioMixer {
if let Some((timestamp, duration)) = source.buffer_last {
let end = timestamp + duration;
if watermark.is_none_or(|previous: Timestamp| {
- end.duration_since(self.timestamps) < previous.duration_since(self.timestamps)
+ let end = end.duration_since(self.timestamps);
+ let previous = previous.duration_since(self.timestamps);
+ if self.finishing {
+ end > previous
+ } else {
+ end < previous
+ }
}) {
watermark = Some(end);
}
- } else if !grace_elapsed {
+ } else if !grace_elapsed && !self.finishing {
waiting_for_input = true;
}
}
@@ -727,7 +770,7 @@ impl AudioMixer {
}
source.samples_supplied += samples;
}
- if source.buffer_last.is_none() {
+ if source.buffer_last.is_none() || self.finishing {
supply_silence(source, &mut self.abuffers[i], i, limit)?;
}
}
@@ -765,6 +808,11 @@ impl AudioMixer {
match self.output.try_send(frame) {
Ok(()) => break,
Err(err) if err.is_full() => {
+ anyhow::ensure!(
+ self.drain_deadline
+ .is_none_or(|deadline| Instant::now() < deadline),
+ "Audio mixer output did not drain before the stop deadline"
+ );
frame = err.into_inner();
std::thread::sleep(Duration::from_millis(1));
}
@@ -786,6 +834,21 @@ impl AudioMixer {
}
}
+fn mixer_now() -> Timestamp {
+ #[cfg(target_os = "macos")]
+ {
+ Timestamp::MachAbsoluteTime(cap_timestamp::MachAbsoluteTimestamp::now())
+ }
+ #[cfg(windows)]
+ {
+ Timestamp::PerformanceCounter(cap_timestamp::PerformanceCounterTimestamp::now())
+ }
+ #[cfg(not(any(target_os = "macos", windows)))]
+ {
+ Timestamp::Instant(Instant::now())
+ }
+}
+
fn samples_duration(samples: usize, rate: u32) -> Duration {
Duration::from_nanos((samples as u128 * 1_000_000_000 / u128::from(rate)) as u64)
}
@@ -1083,6 +1146,10 @@ mod test {
.tick(self.mixer.timestamps, self.at(millis))
.unwrap()
);
+ self.collect(millis);
+ }
+
+ fn collect(&mut self, millis: u64) {
while let Ok(Some(frame)) = self.output.try_next() {
self.deliveries.push((self.pcm.len(), millis));
for pair in frame.inner.data(0)[..frame.inner.samples() * 8].chunks_exact(8) {
@@ -1320,6 +1387,72 @@ mod test {
assert_eq!(probe.peaks(0), [0, 1_440, 2_400]);
}
+ #[test]
+ fn pcm_stop_before_startup_grace_preserves_captured_audio() {
+ let mut probe = PcmProbe::new(2);
+ probe.send(1, 0, 4_800, Some(960));
+ probe.tick(100);
+ assert!(probe.pcm.is_empty());
+ probe.inputs.clear();
+ probe.mixer.finish().unwrap();
+ probe.collect(100);
+ assert_eq!(probe.peaks(1), [960]);
+ assert_eq!(probe.pcm.len(), 4_800);
+ }
+
+ #[test]
+ fn pcm_stop_drains_the_longest_input_and_pads_closed_sources() {
+ for millis in [100, 1_500] {
+ let mut probe = PcmProbe::new(2);
+ probe.send(0, 0, 960, None);
+ probe.send(1, 0, 4_800, Some(4_799));
+ probe.tick(millis);
+ probe.inputs.clear();
+ probe.mixer.finish().unwrap();
+ probe.collect(millis);
+ assert_eq!(probe.peaks(1), [4_799]);
+ assert_eq!(probe.pcm.len(), 4_800);
+ }
+ }
+
+ #[test]
+ fn pcm_stop_waits_for_queued_capture_before_closing_inputs() {
+ let mut probe = PcmProbe::new(2);
+ probe.send(1, 0, 960, None);
+ let mut input = probe.inputs.pop().unwrap();
+ probe.inputs.clear();
+ let timestamp = probe.at(20);
+ let worker = std::thread::spawn(move || {
+ std::thread::sleep(Duration::from_millis(20));
+ let mut bytes = vec![0; 3_840 * 8];
+ bytes[3_000 * 8 + 4..3_000 * 8 + 8].copy_from_slice(&0.5_f32.to_le_bytes());
+ input
+ .try_send(AudioFrame::new(
+ AudioMixer::INFO.wrap_frame(&bytes),
+ timestamp,
+ ))
+ .unwrap();
+ });
+ probe.mixer.finish().unwrap();
+ worker.join().unwrap();
+ probe.collect(100);
+ assert_eq!(probe.peaks(1), [3_960]);
+ assert_eq!(probe.pcm.len(), 4_800);
+ }
+
+ #[test]
+ fn pcm_stop_bounds_a_producer_that_never_closes() {
+ let mut probe = PcmProbe::new(2);
+ probe.send(1, 0, 4_800, Some(960));
+ let start = Instant::now();
+ probe.mixer.finish().unwrap();
+ assert!(start.elapsed() < MIXER_STOP_TIMEOUT);
+ probe.collect(100);
+ assert_eq!(probe.peaks(1), [960]);
+ assert_eq!(probe.pcm.len(), 4_800);
+ assert!(probe.inputs.iter_mut().all(|input| input.is_closed()));
+ }
+
#[test]
fn pcm_single_input_is_unchanged_and_reset_keeps_the_supplied_cursor() {
let mut probe = PcmProbe::new(1);