Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(recording): encode macOS studio audio with AudioToolbox AAC

Hridya Agrawal37bb220e7b94
diff --git a/Cargo.lock b/Cargo.lock
index f059b7eff..b8fc45e9b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1695,6 +1695,7 @@ dependencies = [
"cap-frame-converter",
"cap-media-info",
"ffmpeg-next",
+ "libc",
"serde",
"serde_json",
"sysinfo 0.32.1",
diff --git a/crates/enc-ffmpeg/Cargo.toml b/crates/enc-ffmpeg/Cargo.toml
index 878134e90..d5180ef5a 100644
--- a/crates/enc-ffmpeg/Cargo.toml
+++ b/crates/enc-ffmpeg/Cargo.toml
@@ -19,6 +19,7 @@ cap-frame-converter = { path = "../frame-converter" }
windows = { workspace = true, features = ["Win32_Storage_FileSystem"] }
[dev-dependencies]
+libc = "0.2"
tempfile = "3.20.0"
[lints]
diff --git a/crates/enc-ffmpeg/examples/recording-audio-benchmark.rs b/crates/enc-ffmpeg/examples/recording-audio-benchmark.rs
new file mode 100644
index 000000000..ecd0a0d7b
--- /dev/null
+++ b/crates/enc-ffmpeg/examples/recording-audio-benchmark.rs
@@ -0,0 +1,201 @@
+//! Encodes the same input through `FragmentedAudioFile` with FFmpeg's `aac`
+//! and AudioToolbox's `aac_at`, interleaving rounds, and reports process CPU
+//! time, wall time and output bitrate for mono and stereo.
+//!
+//! Run: cargo run --release -p cap-enc-ffmpeg --example recording-audio-benchmark -- [input-file] [rounds]
+//! With no input file a deterministic 60 s signal (220 Hz tone + filtered
+//! noise) is synthesized; any ffmpeg-decodable file is decoded to 48 kHz mono.
+
+use std::{
+ path::{Path, PathBuf},
+ time::{Duration, Instant},
+};
+
+use cap_enc_ffmpeg::fragmented_audio::FragmentedAudioFile;
+use cap_media_info::AudioInfo;
+use ffmpeg::{
+ ChannelLayout,
+ codec::context,
+ format::{self, Sample, sample::Type},
+ frame, media, software,
+};
+
+const RATE: u32 = 48_000;
+const FRAME_SAMPLES: usize = 960;
+
+fn cpu_secs() -> f64 {
+ let mut usage = std::mem::MaybeUninit::<libc::rusage>::zeroed();
+ unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) };
+ let usage = unsafe { usage.assume_init() };
+ let secs = |time: libc::timeval| time.tv_sec as f64 + time.tv_usec as f64 / 1_000_000.0;
+ secs(usage.ru_utime) + secs(usage.ru_stime)
+}
+
+fn decode_to_mono(path: &Path) -> Vec<f32> {
+ let mut input = format::input(path).unwrap();
+ let stream = input.streams().best(media::Type::Audio).unwrap();
+ let stream_index = stream.index();
+ let mut decoder = context::Context::from_parameters(stream.parameters())
+ .unwrap()
+ .decoder()
+ .audio()
+ .unwrap();
+ let input_layout = if decoder.channel_layout().is_empty() {
+ AudioInfo::new_raw(decoder.format(), decoder.rate(), decoder.channels()).channel_layout()
+ } else {
+ decoder.channel_layout()
+ };
+ let mut resampler = software::resampler(
+ (decoder.format(), input_layout, decoder.rate()),
+ (Sample::F32(Type::Packed), ChannelLayout::MONO, RATE),
+ )
+ .unwrap();
+
+ let mut samples = Vec::new();
+ let mut decoded = frame::Audio::empty();
+ let mut resampled = frame::Audio::empty();
+ for (stream, packet) in input.packets() {
+ if stream.index() != stream_index {
+ continue;
+ }
+ decoder.send_packet(&packet).unwrap();
+ while decoder.receive_frame(&mut decoded).is_ok() {
+ decoded.set_channel_layout(input_layout);
+ resampler.run(&decoded, &mut resampled).unwrap();
+ samples.extend_from_slice(resampled.plane::<f32>(0));
+ }
+ }
+ decoder.send_eof().unwrap();
+ while decoder.receive_frame(&mut decoded).is_ok() {
+ decoded.set_channel_layout(input_layout);
+ resampler.run(&decoded, &mut resampled).unwrap();
+ samples.extend_from_slice(resampled.plane::<f32>(0));
+ }
+ while resampler.delay().is_some() {
+ resampler.flush(&mut resampled).unwrap();
+ if resampled.samples() == 0 {
+ break;
+ }
+ samples.extend_from_slice(resampled.plane::<f32>(0));
+ }
+ samples
+}
+
+fn synthetic(seconds: usize) -> Vec<f32> {
+ let mut state = 0x9e37_79b9_u32;
+ let mut low = 0.0_f32;
+ (0..seconds * RATE as usize)
+ .map(|index| {
+ state ^= state << 13;
+ state ^= state >> 17;
+ state ^= state << 5;
+ let white = state as f32 / u32::MAX as f32 - 0.5;
+ low = 0.98f32.mul_add(low, 0.02 * white);
+ let time = index as f32 / RATE as f32;
+ 0.15 * (std::f32::consts::TAU * 220.0 * time).sin() + 0.4 * low
+ })
+ .collect()
+}
+
+fn encode_run(
+ path: &Path,
+ channels: u16,
+ mono: &[f32],
+ prefer_audiotoolbox: bool,
+) -> (f64, f64, u64) {
+ let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels);
+ let cpu_started = cpu_secs();
+ let wall_started = Instant::now();
+ let mut file =
+ FragmentedAudioFile::init_with_encoder_preference(path.into(), info, prefer_audiotoolbox)
+ .unwrap();
+ for (index, chunk) in mono.chunks(FRAME_SAMPLES).enumerate() {
+ let mut frame = frame::Audio::new(
+ Sample::F32(Type::Packed),
+ chunk.len(),
+ info.channel_layout(),
+ );
+ frame.set_rate(RATE);
+ for (i, bytes) in frame.data_mut(0)[..chunk.len() * usize::from(channels) * 4]
+ .chunks_exact_mut(4)
+ .enumerate()
+ {
+ bytes.copy_from_slice(&chunk[i / usize::from(channels)].to_le_bytes());
+ }
+ file.queue_frame(
+ frame,
+ Duration::from_secs_f64((index * FRAME_SAMPLES) as f64 / f64::from(RATE)),
+ )
+ .unwrap();
+ }
+ file.finish().unwrap().unwrap();
+ let wall = wall_started.elapsed().as_secs_f64();
+ let cpu = cpu_secs() - cpu_started;
+ let bytes = std::fs::metadata(path).map(|meta| meta.len()).unwrap_or(0);
+ (cpu, wall, bytes)
+}
+
+fn median(values: &[f64]) -> f64 {
+ let mut sorted = values.to_vec();
+ sorted.sort_by(f64::total_cmp);
+ sorted[sorted.len() / 2]
+}
+
+fn main() {
+ let mut arguments = std::env::args().skip(1);
+ let input = arguments.next().map(PathBuf::from);
+ let rounds = arguments
+ .next()
+ .and_then(|value| value.parse::<usize>().ok())
+ .unwrap_or(5)
+ .max(1);
+
+ let mono = input
+ .as_deref()
+ .map(decode_to_mono)
+ .unwrap_or_else(|| synthetic(60));
+ let audio_secs = mono.len() as f64 / f64::from(RATE);
+ println!(
+ "input: {} ({audio_secs:.1}s of 48 kHz mono), {rounds} rounds",
+ input
+ .map(|p| p.display().to_string())
+ .unwrap_or_else(|| "synthetic".to_string())
+ );
+
+ let dir = std::env::temp_dir().join("cap-recording-audio-benchmark");
+ std::fs::create_dir_all(&dir).unwrap();
+
+ let configs = [
+ ("aac", 1u16, false),
+ ("aac_at", 1, true),
+ ("aac", 2, false),
+ ("aac_at", 2, true),
+ ];
+ let mut runs = vec![(Vec::<f64>::new(), Vec::<f64>::new(), 0u64); configs.len()];
+ for round in 0..rounds {
+ for (index, &(name, channels, prefer)) in configs.iter().enumerate() {
+ let path = dir.join(format!("{name}-{channels}ch-{round}.m4a"));
+ let _ = std::fs::remove_file(&path);
+ let (cpu, wall, bytes) = encode_run(&path, channels, &mono, prefer);
+ runs[index].0.push(cpu);
+ runs[index].1.push(wall);
+ runs[index].2 = bytes;
+ }
+ }
+
+ println!("encoder ch cpu_ms/s wall_s kbps");
+ for (index, &(name, channels, _)) in configs.iter().enumerate() {
+ let (cpus, walls, bytes) = &runs[index];
+ let cpu_ms = median(cpus) * 1_000.0 / audio_secs;
+ let kbps = *bytes as f64 * 8.0 / audio_secs / 1_000.0;
+ println!(
+ "{name:<8} {channels} {cpu_ms:>8.1} {:>6.2} {kbps:>6.1}",
+ median(walls)
+ );
+ }
+ for channels in [1usize, 2] {
+ let aac = median(&runs[channels * 2 - 2].0);
+ let at = median(&runs[channels * 2 - 1].0);
+ println!("cpu ratio aac_at/aac ({channels}ch): {:.2}", at / aac);
+ }
+}
diff --git a/crates/enc-ffmpeg/src/audio/aac.rs b/crates/enc-ffmpeg/src/audio/aac.rs
index 89ce84fcd..ca58b2664 100644
--- a/crates/enc-ffmpeg/src/audio/aac.rs
+++ b/crates/enc-ffmpeg/src/audio/aac.rs
@@ -33,6 +33,7 @@ pub struct AACEncoder {
impl AACEncoder {
const OUTPUT_BITRATE: usize = 320 * 1000; // 128k
const SAMPLE_FORMAT: Sample = Sample::F32(Type::Planar);
+ const FFMPEG_AAC_PRIMING: i32 = 1_024;
pub fn factory(
input_config: AudioInfo,
@@ -44,7 +45,57 @@ impl AACEncoder {
input_config: AudioInfo,
output: &mut format::context::Output,
) -> Result<Self, AACEncoderError> {
- let codec = encoder::find_by_name("aac").ok_or(AACEncoderError::CodecNotFound)?;
+ Self::init_with_codec(
+ "aac",
+ ffmpeg::Dictionary::new(),
+ Self::OUTPUT_BITRATE,
+ input_config,
+ output,
+ )
+ }
+
+ pub fn init_for_recording(
+ input_config: AudioInfo,
+ output: &mut format::context::Output,
+ ) -> Result<Self, AACEncoderError> {
+ let prefer_audiotoolbox = cfg!(target_os = "macos")
+ && std::env::var_os("CAP_RECORDING_FFMPEG_AAC").is_none_or(|value| value.is_empty());
+ Self::init_for_recording_with(prefer_audiotoolbox, input_config, output)
+ }
+
+ pub(crate) fn init_for_recording_with(
+ prefer_audiotoolbox: bool,
+ input_config: AudioInfo,
+ output: &mut format::context::Output,
+ ) -> Result<Self, AACEncoderError> {
+ if prefer_audiotoolbox {
+ let mut options = ffmpeg::Dictionary::new();
+ options.set("aac_at_mode", "abr");
+ let bit_rate = if input_config.channels == 1 {
+ 192_000
+ } else {
+ 256_000
+ };
+ match Self::init_with_codec("aac_at", options, bit_rate, input_config, output) {
+ Ok(encoder) => return Ok(encoder),
+ Err(error) => {
+ tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's");
+ }
+ }
+ }
+
+ Self::init(input_config, output)
+ }
+
+ fn init_with_codec(
+ name: &str,
+ options: ffmpeg::Dictionary,
+ bit_rate: usize,
+ input_config: AudioInfo,
+ output: &mut format::context::Output,
+ ) -> Result<Self, AACEncoderError> {
+ let codec = encoder::find_by_name(name).ok_or(AACEncoderError::CodecNotFound)?;
+ let audio_codec = codec.audio()?;
let mut encoder_ctx = context::Context::new_with_codec(codec);
let thread_count = thread::available_parallelism()
.map(|v| v.get())
@@ -53,39 +104,62 @@ impl AACEncoder {
let mut encoder = encoder_ctx.encoder().audio()?;
let rate = {
- let mut rates = codec
- .audio()
- .unwrap()
+ let mut rates = audio_codec
.rates()
.into_iter()
.flatten()
.collect::<Vec<_>>();
rates.sort();
- let Some(&rate) = rates
- .iter()
- .find(|r| **r >= input_config.rate())
- .or(rates.first())
- else {
- return Err(AACEncoderError::RateNotSupported(input_config.rate()));
- };
- rate
+ if rates.is_empty() {
+ input_config.rate()
+ } else {
+ let Some(&rate) = rates
+ .iter()
+ .find(|r| **r >= input_config.rate())
+ .or(rates.first())
+ else {
+ return Err(AACEncoderError::RateNotSupported(input_config.rate()));
+ };
+ rate
+ }
+ };
+
+ let sample_format = {
+ let formats = audio_codec
+ .formats()
+ .into_iter()
+ .flatten()
+ .collect::<Vec<_>>();
+ if formats.contains(&Self::SAMPLE_FORMAT) {
+ Self::SAMPLE_FORMAT
+ } else {
+ formats.first().copied().unwrap_or(Self::SAMPLE_FORMAT)
+ }
};
let mut output_config = input_config;
- output_config.sample_format = Self::SAMPLE_FORMAT;
+ output_config.sample_format = sample_format;
output_config.sample_rate = rate as u32;
let resampler = BufferedResampler::new(input_config, output_config)
.map_err(AACEncoderError::Resampler)?;
- encoder.set_bit_rate(Self::OUTPUT_BITRATE);
+ encoder.set_bit_rate(bit_rate);
encoder.set_rate(rate);
encoder.set_format(output_config.sample_format);
encoder.set_channel_layout(output_config.channel_layout());
encoder.set_time_base(FFRational(1, output_config.rate()));
- let encoder = encoder.open()?;
+ let encoder = encoder.open_with(options)?;
+
+ // Fragmented outputs keep encoder priming in the decoded timeline;
+ // dropping the priming a codec adds beyond FFmpeg's aac (e.g. 2112 vs
+ // 1024 samples for aac_at) keeps every content sample where the native
+ // encoder put it.
+ let leading_trim = unsafe { (*encoder.as_ptr()).initial_padding }
+ .saturating_sub(Self::FFMPEG_AAC_PRIMING)
+ .max(0) as usize;
let mut output_stream = output.add_stream(codec)?;
output_stream.set_time_base(FFRational(1, output_config.rate()));
@@ -93,7 +167,8 @@ impl AACEncoder {
let stream_index = output_stream.index();
Ok(Self {
- base: AudioEncoderBase::new(encoder, resampler, stream_index),
+ base: AudioEncoderBase::new(encoder, resampler, stream_index)
+ .with_leading_trim(leading_trim),
stream_index,
})
}
@@ -102,6 +177,10 @@ impl AACEncoder {
self.stream_index
}
+ pub fn codec_name(&self) -> Option<String> {
+ self.base.codec_name()
+ }
+
pub fn send_frame(
&mut self,
frame: frame::Audio,
diff --git a/crates/enc-ffmpeg/src/audio/base.rs b/crates/enc-ffmpeg/src/audio/base.rs
index cd5197a7c..fbfdbb72f 100644
--- a/crates/enc-ffmpeg/src/audio/base.rs
+++ b/crates/enc-ffmpeg/src/audio/base.rs
@@ -1,12 +1,18 @@
use super::buffered_resampler::BufferedResampler;
use crate::base::{EncoderBase, PacketSink};
-use ffmpeg::{codec::encoder, frame};
+use ffmpeg::{
+ Packet, Rational, Rescale,
+ codec::{Capabilities, encoder},
+ frame,
+};
use std::time::Duration;
pub struct AudioEncoderBase {
inner: EncoderBase,
encoder: encoder::Audio,
resampler: BufferedResampler,
+ leading_trim: usize,
+ input_end: Option<i64>,
}
impl AudioEncoderBase {
@@ -15,9 +21,20 @@ impl AudioEncoderBase {
inner: EncoderBase::new(stream_index),
encoder,
resampler,
+ leading_trim: 0,
+ input_end: None,
}
}
+ pub fn with_leading_trim(mut self, samples: usize) -> Self {
+ self.leading_trim = samples;
+ self
+ }
+
+ pub fn codec_name(&self) -> Option<String> {
+ self.encoder.codec().map(|codec| codec.name().to_owned())
+ }
+
pub fn send_frame<S: PacketSink + ?Sized>(
&mut self,
mut frame: frame::Audio,
@@ -32,7 +49,15 @@ impl AudioEncoderBase {
self.resampler.add_frame(frame);
+ if self.leading_trim > 0 {
+ match self.resampler.get_frame(self.leading_trim) {
+ Some(_) => self.leading_trim = 0,
+ None => return Ok(()),
+ }
+ }
+
while let Some(frame) = self.resampler.get_frame(self.encoder.frame_size() as usize) {
+ self.track_input_end(&frame);
self.inner.send_frame(&frame, output, &mut self.encoder)?;
}
@@ -40,10 +65,72 @@ impl AudioEncoderBase {
}
pub fn flush<S: PacketSink + ?Sized>(&mut self, output: &mut S) -> Result<(), ffmpeg::Error> {
+ while self.leading_trim > 0 {
+ match self.resampler.flush(self.leading_trim) {
+ Some(frame) => self.leading_trim -= frame.samples(),
+ None => break,
+ }
+ }
+
+ let mut frames = Vec::new();
while let Some(frame) = self.resampler.flush(self.encoder.frame_size() as usize) {
- self.inner.send_frame(&frame, output, &mut self.encoder)?;
+ self.track_input_end(&frame);
+ frames.push(frame);
+ }
+
+ // Encoders that only take full frames (AudioToolbox) get the last one
+ // padded by libavcodec, and their packet durations count that padding,
+ // so the track would end up to a frame after the input did.
+ let pads_final_frame = self.encoder.codec().is_some_and(|codec| {
+ !codec
+ .capabilities()
+ .intersects(Capabilities::SMALL_LAST_FRAME | Capabilities::VARIABLE_FRAME_SIZE)
+ });
+ let mut output = EndTrimmedSink {
+ output,
+ end: self.input_end.filter(|_| pads_final_frame),
+ time_base: self.encoder.time_base(),
+ };
+ for frame in &frames {
+ self.inner
+ .send_frame(frame, &mut output, &mut self.encoder)?;
}
- self.inner.process_eof(output, &mut self.encoder)
+ self.inner.process_eof(&mut output, &mut self.encoder)
+ }
+
+ fn track_input_end(&mut self, frame: &frame::Audio) {
+ if let Some(pts) = frame.pts() {
+ let end = pts + frame.samples() as i64;
+ self.input_end = Some(self.input_end.map_or(end, |current| current.max(end)));
+ }
+ }
+}
+
+struct EndTrimmedSink<'a, S: PacketSink + ?Sized> {
+ output: &'a mut S,
+ end: Option<i64>,
+ time_base: Rational,
+}
+
+impl<S: PacketSink + ?Sized> PacketSink for EndTrimmedSink<'_, S> {
+ fn stream_time_base(&self, stream_index: usize) -> Rational {
+ self.output.stream_time_base(stream_index)
+ }
+
+ fn write_packet(&mut self, mut packet: Packet) -> Result<(), ffmpeg::Error> {
+ if let (Some(end), Some(pts)) = (self.end, packet.pts()) {
+ let end = end.rescale(
+ self.time_base,
+ self.output.stream_time_base(packet.stream()),
+ );
+ if pts >= end {
+ return Ok(());
+ }
+ if pts + packet.duration() > end {
+ packet.set_duration(end - pts);
+ }
+ }
+ self.output.write_packet(packet)
}
}
diff --git a/crates/enc-ffmpeg/src/mux/fragmented_audio.rs b/crates/enc-ffmpeg/src/mux/fragmented_audio.rs
index 1203ad72c..900d0b7fa 100644
--- a/crates/enc-ffmpeg/src/mux/fragmented_audio.rs
+++ b/crates/enc-ffmpeg/src/mux/fragmented_audio.rs
@@ -64,7 +64,29 @@ fn configure_fragmentation(output: &mut format::context::Output) -> Result<(), f
}
impl FragmentedAudioFile {
- pub fn init(mut output_path: PathBuf, audio_config: AudioInfo) -> Result<Self, InitError> {
+ pub fn init(output_path: PathBuf, audio_config: AudioInfo) -> Result<Self, InitError> {
+ Self::init_inner(output_path, audio_config, AACEncoder::init_for_recording)
+ }
+
+ #[doc(hidden)]
+ pub fn init_with_encoder_preference(
+ output_path: PathBuf,
+ audio_config: AudioInfo,
+ prefer_audiotoolbox: bool,
+ ) -> Result<Self, InitError> {
+ Self::init_inner(output_path, audio_config, |config, output| {
+ AACEncoder::init_for_recording_with(prefer_audiotoolbox, config, output)
+ })
+ }
+
+ fn init_inner(
+ mut output_path: PathBuf,
+ audio_config: AudioInfo,
+ encoder_init: impl FnOnce(
+ AudioInfo,
+ &mut format::context::Output,
+ ) -> Result<AACEncoder, AACEncoderError>,
+ ) -> Result<Self, InitError> {
output_path.set_extension("m4a");
if let Some(parent) = output_path.parent() {
@@ -75,7 +97,7 @@ impl FragmentedAudioFile {
configure_fragmentation(&mut output)?;
- let encoder = AACEncoder::init(audio_config, &mut output)?;
+ let encoder = encoder_init(audio_config, &mut output)?;
output.write_header()?;
@@ -324,3 +346,290 @@ mod publication_tests {
assert!(path.exists());
}
}
+
+#[cfg(test)]
+mod recording_encoder_tests {
+ use super::*;
+ use ffmpeg::{
+ Rescale,
+ codec::context,
+ format::{Sample, sample::Type},
+ };
+ use std::path::Path;
+
+ const RATE: u32 = 48_000;
+ const FRAME_SAMPLES: usize = 960;
+ const CLICK_SAMPLES: usize = 1;
+ const FFMPEG_AAC_PRIMING: usize = 1_024;
+
+ fn timeline_with_clicks(total: usize, clicks: &[usize]) -> Vec<f32> {
+ let mut timeline = vec![0.0_f32; total];
+ for &click in clicks {
+ for sample in timeline
+ .iter_mut()
+ .take((click + CLICK_SAMPLES).min(total))
+ .skip(click)
+ {
+ *sample = 0.9;
+ }
+ }
+ timeline
+ }
+
+ fn encode_recording(
+ prefer_audiotoolbox: bool,
+ channels: u16,
+ timeline: &[f32],
+ send_ranges: &[(usize, usize)],
+ ) -> (Vec<f32>, i64, u64) {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("audio.m4a");
+ let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels);
+ let mut file = FragmentedAudioFile::init_with_encoder_preference(
+ path.clone(),
+ info,
+ prefer_audiotoolbox,
+ )
+ .unwrap();
+
+ for &(start, len) in send_ranges {
+ let mut position = start;
+ while position < start + len {
+ let samples = FRAME_SAMPLES.min(start + len - position);
+ let mut frame =
+ frame::Audio::new(Sample::F32(Type::Packed), samples, info.channel_layout());
+ frame.set_rate(RATE);
+ for (index, bytes) in frame.data_mut(0)
+ [..samples * usize::from(channels) * size_of::<f32>()]
+ .chunks_exact_mut(size_of::<f32>())
+ .enumerate()
+ {
+ bytes.copy_from_slice(
+ &timeline[position + index / usize::from(channels)].to_le_bytes(),
+ );
+ }
+ file.queue_frame(
+ frame,
+ Duration::from_secs_f64(position as f64 / f64::from(RATE)),
+ )
+ .unwrap();
+ position += samples;
+ }
+ }
+
+ file.finish().unwrap().unwrap();
+ let bytes = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0);
+ let (samples, track_end) = decode_channel_zero(&path);
+ (samples, track_end, bytes)
+ }
+
+ fn encode_recording_variable(
+ prefer_audiotoolbox: bool,
+ channels: u16,
+ timeline: &[f32],
+ frame_sizes: &[usize],
+ ) -> (Vec<f32>, i64, u64) {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("audio.m4a");
+ let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels);
+ let mut file = FragmentedAudioFile::init_with_encoder_preference(
+ path.clone(),
+ info,
+ prefer_audiotoolbox,
+ )
+ .unwrap();
+
+ let mut position = 0;
+ let mut frame_index = 0;
+ while position < timeline.len() {
+ let samples =
+ frame_sizes[frame_index % frame_sizes.len()].min(timeline.len() - position);
+ let mut frame =
+ frame::Audio::new(Sample::F32(Type::Packed), samples, info.channel_layout());
+ frame.set_rate(RATE);
+ for (index, bytes) in frame.data_mut(0)
+ [..samples * usize::from(channels) * size_of::<f32>()]
+ .chunks_exact_mut(size_of::<f32>())
+ .enumerate()
+ {
+ bytes.copy_from_slice(
+ &timeline[position + index / usize::from(channels)].to_le_bytes(),
+ );
+ }
+ file.queue_frame(
+ frame,
+ Duration::from_nanos(position as u64 * 1_000_000_000 / u64::from(RATE)),
+ )
+ .unwrap();
+ position += samples;
+ frame_index += 1;
+ }
+
+ file.finish().unwrap().unwrap();
+ let bytes = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0);
+ let (samples, track_end) = decode_channel_zero(&path);
+ (samples, track_end, bytes)
+ }
+
+ fn decode_channel_zero(path: &Path) -> (Vec<f32>, i64) {
+ let mut input = format::input(path).unwrap();
+ let stream = input.streams().best(ffmpeg::media::Type::Audio).unwrap();
+ let stream_index = stream.index();
+ let time_base = stream.time_base();
+ let mut decoder = context::Context::from_parameters(stream.parameters())
+ .unwrap()
+ .decoder()
+ .audio()
+ .unwrap();
+ let mut decoded = frame::Audio::empty();
+ let mut samples = Vec::new();
+ let mut track_end = i64::MIN;
+
+ for (stream, packet) in input.packets() {
+ if stream.index() != stream_index {
+ continue;
+ }
+ if let Some(pts) = packet.pts() {
+ track_end =
+ track_end.max((pts + packet.duration()).rescale(time_base, (1, 48_000)));
+ }
+ decoder.send_packet(&packet).unwrap();
+ while decoder.receive_frame(&mut decoded).is_ok() {
+ assert_eq!(decoded.format(), Sample::F32(Type::Planar));
+ samples.extend_from_slice(decoded.plane::<f32>(0));
+ }
+ }
+ decoder.send_eof().unwrap();
+ while decoder.receive_frame(&mut decoded).is_ok() {
+ assert_eq!(decoded.format(), Sample::F32(Type::Planar));
+ samples.extend_from_slice(decoded.plane::<f32>(0));
+ }
+ (samples, track_end)
+ }
+
+ fn click_index(samples: &[f32]) -> usize {
+ samples
+ .iter()
+ .enumerate()
+ .max_by(|(_, a), (_, b)| a.abs().partial_cmp(&b.abs()).unwrap())
+ .map(|(index, _)| index)
+ .expect("no decoded samples")
+ }
+
+ fn click_positions(samples: &[f32]) -> Vec<usize> {
+ let mut positions = Vec::new();
+ let mut index = 0;
+ while index < samples.len() {
+ if samples[index].abs() > 0.2 {
+ let window_end = (index + 512).min(samples.len());
+ positions.push(index + click_index(&samples[index..window_end]));
+ index = window_end;
+ } else {
+ index += 1;
+ }
+ }
+ positions
+ }
+
+ #[test]
+ fn recording_audio_click_decodes_at_the_same_sample_with_both_encoders() {
+ for channels in [1u16, 2] {
+ let timeline = timeline_with_clicks(96_000, &[24_000]);
+ let ranges = [(0usize, 96_000usize)];
+
+ let (ffmpeg_samples, _, _) = encode_recording(false, channels, &timeline, &ranges);
+ let ffmpeg_click = click_index(&ffmpeg_samples);
+ assert_eq!(
+ ffmpeg_click,
+ 24_000 + FFMPEG_AAC_PRIMING,
+ "FFmpeg aac decoded the click at {ffmpeg_click} for {channels}ch"
+ );
+
+ let (at_samples, _, _) = encode_recording(true, channels, &timeline, &ranges);
+ let at_click = click_index(&at_samples);
+ assert!(
+ at_click.abs_diff(ffmpeg_click) <= 4,
+ "AudioToolbox decoded the click at {at_click}, FFmpeg at {ffmpeg_click} ({channels}ch)"
+ );
+ }
+ }
+
+ #[test]
+ fn recording_audio_silence_filled_gap_decodes_the_same_with_both_encoders() {
+ let frame_sizes = [441, 960, 1000, 512, 4800];
+ for channels in [1u16, 2] {
+ let timeline = timeline_with_clicks(168_000, &[24_000, 120_000]);
+
+ let (ffmpeg_samples, ffmpeg_end, _) =
+ encode_recording_variable(false, channels, &timeline, &frame_sizes);
+ let (at_samples, at_end, _) =
+ encode_recording_variable(true, channels, &timeline, &frame_sizes);
+
+ let ffmpeg_clicks = click_positions(&ffmpeg_samples);
+ let at_clicks = click_positions(&at_samples);
+ assert_eq!(ffmpeg_clicks.len(), 2, "FFmpeg clicks: {ffmpeg_clicks:?}");
+ assert_eq!(at_clicks.len(), 2, "AudioToolbox clicks: {at_clicks:?}");
+ for (index, (&ffmpeg_click, &at_click)) in
+ ffmpeg_clicks.iter().zip(&at_clicks).enumerate()
+ {
+ let expected = [24_000usize, 120_000][index] + FFMPEG_AAC_PRIMING;
+ assert_eq!(
+ ffmpeg_click, expected,
+ "FFmpeg aac decoded click {index} at {ffmpeg_click}, expected {expected} ({channels}ch)"
+ );
+ assert!(
+ at_click.abs_diff(ffmpeg_click) <= 4,
+ "click {index}: AudioToolbox {at_click} vs FFmpeg {ffmpeg_click} ({channels}ch)"
+ );
+ }
+ assert!(
+ at_samples.len().abs_diff(ffmpeg_samples.len()) <= 4,
+ "decoded {} vs {} samples ({channels}ch)",
+ at_samples.len(),
+ ffmpeg_samples.len()
+ );
+ assert!(
+ at_end.abs_diff(ffmpeg_end) <= 1,
+ "track end {at_end} vs {ffmpeg_end} ({channels}ch)"
+ );
+ }
+ }
+
+ #[test]
+ fn recording_audio_track_ends_where_the_ffmpeg_track_ends() {
+ let timeline = vec![0.0_f32; 47_123];
+ let ranges = [(0usize, 47_123usize)];
+
+ let (_, ffmpeg_end, _) = encode_recording(false, 1, &timeline, &ranges);
+ let (_, at_end, _) = encode_recording(true, 1, &timeline, &ranges);
+
+ assert!(
+ at_end.abs_diff(ffmpeg_end) <= 1,
+ "track end {at_end} vs {ffmpeg_end}"
+ );
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn recording_audio_uses_audiotoolbox_on_macos() {
+ if std::env::var_os("CAP_RECORDING_FFMPEG_AAC").is_some_and(|v| !v.is_empty()) {
+ return;
+ }
+ let directory = tempfile::tempdir().unwrap();
+ let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, 1);
+ let file = FragmentedAudioFile::init(directory.path().join("audio.m4a"), info).unwrap();
+ assert_eq!(file.encoder().codec_name().as_deref(), Some("aac_at"));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn recording_audio_silence_stays_small() {
+ let timeline = vec![0.0_f32; 480_000];
+ let ranges = [(0usize, 480_000usize)];
+ let (_, _, bytes) = encode_recording(true, 2, &timeline, &ranges);
+ assert!(
+ bytes < 64 * 1024,
+ "10s of stereo silence took {bytes} bytes"
+ );
+ }
+}