Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(recording): encode macOS studio audio with AudioToolbox AAC
Hridya Agrawal37bb220e7b94
diff --git a/Cargo.lock b/Cargo.lockindex f059b7eff..b8fc45e9b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1695,6 +1695,7 @@ dependencies = ["cap-frame-converter","cap-media-info","ffmpeg-next",+ "libc","serde","serde_json","sysinfo 0.32.1",diff --git a/crates/enc-ffmpeg/Cargo.toml b/crates/enc-ffmpeg/Cargo.tomlindex 878134e90..d5180ef5a 100644--- a/crates/enc-ffmpeg/Cargo.toml+++ b/crates/enc-ffmpeg/Cargo.toml@@ -19,6 +19,7 @@ cap-frame-converter = { path = "../frame-converter" }windows = { workspace = true, features = ["Win32_Storage_FileSystem"] }[dev-dependencies]+libc = "0.2"tempfile = "3.20.0"[lints]diff --git a/crates/enc-ffmpeg/examples/recording-audio-benchmark.rs b/crates/enc-ffmpeg/examples/recording-audio-benchmark.rsnew file mode 100644index 000000000..ecd0a0d7b--- /dev/null+++ b/crates/enc-ffmpeg/examples/recording-audio-benchmark.rs@@ -0,0 +1,201 @@+//! Encodes the same input through `FragmentedAudioFile` with FFmpeg's `aac`+//! and AudioToolbox's `aac_at`, interleaving rounds, and reports process CPU+//! time, wall time and output bitrate for mono and stereo.+//!+//! Run: cargo run --release -p cap-enc-ffmpeg --example recording-audio-benchmark -- [input-file] [rounds]+//! With no input file a deterministic 60 s signal (220 Hz tone + filtered+//! noise) is synthesized; any ffmpeg-decodable file is decoded to 48 kHz mono.++use std::{+ path::{Path, PathBuf},+ time::{Duration, Instant},+};++use cap_enc_ffmpeg::fragmented_audio::FragmentedAudioFile;+use cap_media_info::AudioInfo;+use ffmpeg::{+ ChannelLayout,+ codec::context,+ format::{self, Sample, sample::Type},+ frame, media, software,+};++const RATE: u32 = 48_000;+const FRAME_SAMPLES: usize = 960;++fn cpu_secs() -> f64 {+ let mut usage = std::mem::MaybeUninit::<libc::rusage>::zeroed();+ unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) };+ let usage = unsafe { usage.assume_init() };+ let secs = |time: libc::timeval| time.tv_sec as f64 + time.tv_usec as f64 / 1_000_000.0;+ secs(usage.ru_utime) + secs(usage.ru_stime)+}++fn decode_to_mono(path: &Path) -> Vec<f32> {+ let mut input = format::input(path).unwrap();+ let stream = input.streams().best(media::Type::Audio).unwrap();+ let stream_index = stream.index();+ let mut decoder = context::Context::from_parameters(stream.parameters())+ .unwrap()+ .decoder()+ .audio()+ .unwrap();+ let input_layout = if decoder.channel_layout().is_empty() {+ AudioInfo::new_raw(decoder.format(), decoder.rate(), decoder.channels()).channel_layout()+ } else {+ decoder.channel_layout()+ };+ let mut resampler = software::resampler(+ (decoder.format(), input_layout, decoder.rate()),+ (Sample::F32(Type::Packed), ChannelLayout::MONO, RATE),+ )+ .unwrap();++ let mut samples = Vec::new();+ let mut decoded = frame::Audio::empty();+ let mut resampled = frame::Audio::empty();+ for (stream, packet) in input.packets() {+ if stream.index() != stream_index {+ continue;+ }+ decoder.send_packet(&packet).unwrap();+ while decoder.receive_frame(&mut decoded).is_ok() {+ decoded.set_channel_layout(input_layout);+ resampler.run(&decoded, &mut resampled).unwrap();+ samples.extend_from_slice(resampled.plane::<f32>(0));+ }+ }+ decoder.send_eof().unwrap();+ while decoder.receive_frame(&mut decoded).is_ok() {+ decoded.set_channel_layout(input_layout);+ resampler.run(&decoded, &mut resampled).unwrap();+ samples.extend_from_slice(resampled.plane::<f32>(0));+ }+ while resampler.delay().is_some() {+ resampler.flush(&mut resampled).unwrap();+ if resampled.samples() == 0 {+ break;+ }+ samples.extend_from_slice(resampled.plane::<f32>(0));+ }+ samples+}++fn synthetic(seconds: usize) -> Vec<f32> {+ let mut state = 0x9e37_79b9_u32;+ let mut low = 0.0_f32;+ (0..seconds * RATE as usize)+ .map(|index| {+ state ^= state << 13;+ state ^= state >> 17;+ state ^= state << 5;+ let white = state as f32 / u32::MAX as f32 - 0.5;+ low = 0.98f32.mul_add(low, 0.02 * white);+ let time = index as f32 / RATE as f32;+ 0.15 * (std::f32::consts::TAU * 220.0 * time).sin() + 0.4 * low+ })+ .collect()+}++fn encode_run(+ path: &Path,+ channels: u16,+ mono: &[f32],+ prefer_audiotoolbox: bool,+) -> (f64, f64, u64) {+ let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels);+ let cpu_started = cpu_secs();+ let wall_started = Instant::now();+ let mut file =+ FragmentedAudioFile::init_with_encoder_preference(path.into(), info, prefer_audiotoolbox)+ .unwrap();+ for (index, chunk) in mono.chunks(FRAME_SAMPLES).enumerate() {+ let mut frame = frame::Audio::new(+ Sample::F32(Type::Packed),+ chunk.len(),+ info.channel_layout(),+ );+ frame.set_rate(RATE);+ for (i, bytes) in frame.data_mut(0)[..chunk.len() * usize::from(channels) * 4]+ .chunks_exact_mut(4)+ .enumerate()+ {+ bytes.copy_from_slice(&chunk[i / usize::from(channels)].to_le_bytes());+ }+ file.queue_frame(+ frame,+ Duration::from_secs_f64((index * FRAME_SAMPLES) as f64 / f64::from(RATE)),+ )+ .unwrap();+ }+ file.finish().unwrap().unwrap();+ let wall = wall_started.elapsed().as_secs_f64();+ let cpu = cpu_secs() - cpu_started;+ let bytes = std::fs::metadata(path).map(|meta| meta.len()).unwrap_or(0);+ (cpu, wall, bytes)+}++fn median(values: &[f64]) -> f64 {+ let mut sorted = values.to_vec();+ sorted.sort_by(f64::total_cmp);+ sorted[sorted.len() / 2]+}++fn main() {+ let mut arguments = std::env::args().skip(1);+ let input = arguments.next().map(PathBuf::from);+ let rounds = arguments+ .next()+ .and_then(|value| value.parse::<usize>().ok())+ .unwrap_or(5)+ .max(1);++ let mono = input+ .as_deref()+ .map(decode_to_mono)+ .unwrap_or_else(|| synthetic(60));+ let audio_secs = mono.len() as f64 / f64::from(RATE);+ println!(+ "input: {} ({audio_secs:.1}s of 48 kHz mono), {rounds} rounds",+ input+ .map(|p| p.display().to_string())+ .unwrap_or_else(|| "synthetic".to_string())+ );++ let dir = std::env::temp_dir().join("cap-recording-audio-benchmark");+ std::fs::create_dir_all(&dir).unwrap();++ let configs = [+ ("aac", 1u16, false),+ ("aac_at", 1, true),+ ("aac", 2, false),+ ("aac_at", 2, true),+ ];+ let mut runs = vec![(Vec::<f64>::new(), Vec::<f64>::new(), 0u64); configs.len()];+ for round in 0..rounds {+ for (index, &(name, channels, prefer)) in configs.iter().enumerate() {+ let path = dir.join(format!("{name}-{channels}ch-{round}.m4a"));+ let _ = std::fs::remove_file(&path);+ let (cpu, wall, bytes) = encode_run(&path, channels, &mono, prefer);+ runs[index].0.push(cpu);+ runs[index].1.push(wall);+ runs[index].2 = bytes;+ }+ }++ println!("encoder ch cpu_ms/s wall_s kbps");+ for (index, &(name, channels, _)) in configs.iter().enumerate() {+ let (cpus, walls, bytes) = &runs[index];+ let cpu_ms = median(cpus) * 1_000.0 / audio_secs;+ let kbps = *bytes as f64 * 8.0 / audio_secs / 1_000.0;+ println!(+ "{name:<8} {channels} {cpu_ms:>8.1} {:>6.2} {kbps:>6.1}",+ median(walls)+ );+ }+ for channels in [1usize, 2] {+ let aac = median(&runs[channels * 2 - 2].0);+ let at = median(&runs[channels * 2 - 1].0);+ println!("cpu ratio aac_at/aac ({channels}ch): {:.2}", at / aac);+ }+}diff --git a/crates/enc-ffmpeg/src/audio/aac.rs b/crates/enc-ffmpeg/src/audio/aac.rsindex 89ce84fcd..ca58b2664 100644--- a/crates/enc-ffmpeg/src/audio/aac.rs+++ b/crates/enc-ffmpeg/src/audio/aac.rs@@ -33,6 +33,7 @@ pub struct AACEncoder {impl AACEncoder {const OUTPUT_BITRATE: usize = 320 * 1000; // 128kconst SAMPLE_FORMAT: Sample = Sample::F32(Type::Planar);+ const FFMPEG_AAC_PRIMING: i32 = 1_024;pub fn factory(input_config: AudioInfo,@@ -44,7 +45,57 @@ impl AACEncoder {input_config: AudioInfo,output: &mut format::context::Output,) -> Result<Self, AACEncoderError> {- let codec = encoder::find_by_name("aac").ok_or(AACEncoderError::CodecNotFound)?;+ Self::init_with_codec(+ "aac",+ ffmpeg::Dictionary::new(),+ Self::OUTPUT_BITRATE,+ input_config,+ output,+ )+ }++ pub fn init_for_recording(+ input_config: AudioInfo,+ output: &mut format::context::Output,+ ) -> Result<Self, AACEncoderError> {+ let prefer_audiotoolbox = cfg!(target_os = "macos")+ && std::env::var_os("CAP_RECORDING_FFMPEG_AAC").is_none_or(|value| value.is_empty());+ Self::init_for_recording_with(prefer_audiotoolbox, input_config, output)+ }++ pub(crate) fn init_for_recording_with(+ prefer_audiotoolbox: bool,+ input_config: AudioInfo,+ output: &mut format::context::Output,+ ) -> Result<Self, AACEncoderError> {+ if prefer_audiotoolbox {+ let mut options = ffmpeg::Dictionary::new();+ options.set("aac_at_mode", "abr");+ let bit_rate = if input_config.channels == 1 {+ 192_000+ } else {+ 256_000+ };+ match Self::init_with_codec("aac_at", options, bit_rate, input_config, output) {+ Ok(encoder) => return Ok(encoder),+ Err(error) => {+ tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's");+ }+ }+ }++ Self::init(input_config, output)+ }++ fn init_with_codec(+ name: &str,+ options: ffmpeg::Dictionary,+ bit_rate: usize,+ input_config: AudioInfo,+ output: &mut format::context::Output,+ ) -> Result<Self, AACEncoderError> {+ let codec = encoder::find_by_name(name).ok_or(AACEncoderError::CodecNotFound)?;+ let audio_codec = codec.audio()?;let mut encoder_ctx = context::Context::new_with_codec(codec);let thread_count = thread::available_parallelism().map(|v| v.get())@@ -53,39 +104,62 @@ impl AACEncoder {let mut encoder = encoder_ctx.encoder().audio()?;let rate = {- let mut rates = codec- .audio()- .unwrap()+ let mut rates = audio_codec.rates().into_iter().flatten().collect::<Vec<_>>();rates.sort();- let Some(&rate) = rates- .iter()- .find(|r| **r >= input_config.rate())- .or(rates.first())- else {- return Err(AACEncoderError::RateNotSupported(input_config.rate()));- };- rate+ if rates.is_empty() {+ input_config.rate()+ } else {+ let Some(&rate) = rates+ .iter()+ .find(|r| **r >= input_config.rate())+ .or(rates.first())+ else {+ return Err(AACEncoderError::RateNotSupported(input_config.rate()));+ };+ rate+ }+ };++ let sample_format = {+ let formats = audio_codec+ .formats()+ .into_iter()+ .flatten()+ .collect::<Vec<_>>();+ if formats.contains(&Self::SAMPLE_FORMAT) {+ Self::SAMPLE_FORMAT+ } else {+ formats.first().copied().unwrap_or(Self::SAMPLE_FORMAT)+ }};let mut output_config = input_config;- output_config.sample_format = Self::SAMPLE_FORMAT;+ output_config.sample_format = sample_format;output_config.sample_rate = rate as u32;let resampler = BufferedResampler::new(input_config, output_config).map_err(AACEncoderError::Resampler)?;- encoder.set_bit_rate(Self::OUTPUT_BITRATE);+ encoder.set_bit_rate(bit_rate);encoder.set_rate(rate);encoder.set_format(output_config.sample_format);encoder.set_channel_layout(output_config.channel_layout());encoder.set_time_base(FFRational(1, output_config.rate()));- let encoder = encoder.open()?;+ let encoder = encoder.open_with(options)?;++ // Fragmented outputs keep encoder priming in the decoded timeline;+ // dropping the priming a codec adds beyond FFmpeg's aac (e.g. 2112 vs+ // 1024 samples for aac_at) keeps every content sample where the native+ // encoder put it.+ let leading_trim = unsafe { (*encoder.as_ptr()).initial_padding }+ .saturating_sub(Self::FFMPEG_AAC_PRIMING)+ .max(0) as usize;let mut output_stream = output.add_stream(codec)?;output_stream.set_time_base(FFRational(1, output_config.rate()));@@ -93,7 +167,8 @@ impl AACEncoder {let stream_index = output_stream.index();Ok(Self {- base: AudioEncoderBase::new(encoder, resampler, stream_index),+ base: AudioEncoderBase::new(encoder, resampler, stream_index)+ .with_leading_trim(leading_trim),stream_index,})}@@ -102,6 +177,10 @@ impl AACEncoder {self.stream_index}+ pub fn codec_name(&self) -> Option<String> {+ self.base.codec_name()+ }+pub fn send_frame(&mut self,frame: frame::Audio,diff --git a/crates/enc-ffmpeg/src/audio/base.rs b/crates/enc-ffmpeg/src/audio/base.rsindex cd5197a7c..fbfdbb72f 100644--- a/crates/enc-ffmpeg/src/audio/base.rs+++ b/crates/enc-ffmpeg/src/audio/base.rs@@ -1,12 +1,18 @@use super::buffered_resampler::BufferedResampler;use crate::base::{EncoderBase, PacketSink};-use ffmpeg::{codec::encoder, frame};+use ffmpeg::{+ Packet, Rational, Rescale,+ codec::{Capabilities, encoder},+ frame,+};use std::time::Duration;pub struct AudioEncoderBase {inner: EncoderBase,encoder: encoder::Audio,resampler: BufferedResampler,+ leading_trim: usize,+ input_end: Option<i64>,}impl AudioEncoderBase {@@ -15,9 +21,20 @@ impl AudioEncoderBase {inner: EncoderBase::new(stream_index),encoder,resampler,+ leading_trim: 0,+ input_end: None,}}+ pub fn with_leading_trim(mut self, samples: usize) -> Self {+ self.leading_trim = samples;+ self+ }++ pub fn codec_name(&self) -> Option<String> {+ self.encoder.codec().map(|codec| codec.name().to_owned())+ }+pub fn send_frame<S: PacketSink + ?Sized>(&mut self,mut frame: frame::Audio,@@ -32,7 +49,15 @@ impl AudioEncoderBase {self.resampler.add_frame(frame);+ if self.leading_trim > 0 {+ match self.resampler.get_frame(self.leading_trim) {+ Some(_) => self.leading_trim = 0,+ None => return Ok(()),+ }+ }+while let Some(frame) = self.resampler.get_frame(self.encoder.frame_size() as usize) {+ self.track_input_end(&frame);self.inner.send_frame(&frame, output, &mut self.encoder)?;}@@ -40,10 +65,72 @@ impl AudioEncoderBase {}pub fn flush<S: PacketSink + ?Sized>(&mut self, output: &mut S) -> Result<(), ffmpeg::Error> {+ while self.leading_trim > 0 {+ match self.resampler.flush(self.leading_trim) {+ Some(frame) => self.leading_trim -= frame.samples(),+ None => break,+ }+ }++ let mut frames = Vec::new();while let Some(frame) = self.resampler.flush(self.encoder.frame_size() as usize) {- self.inner.send_frame(&frame, output, &mut self.encoder)?;+ self.track_input_end(&frame);+ frames.push(frame);+ }++ // Encoders that only take full frames (AudioToolbox) get the last one+ // padded by libavcodec, and their packet durations count that padding,+ // so the track would end up to a frame after the input did.+ let pads_final_frame = self.encoder.codec().is_some_and(|codec| {+ !codec+ .capabilities()+ .intersects(Capabilities::SMALL_LAST_FRAME | Capabilities::VARIABLE_FRAME_SIZE)+ });+ let mut output = EndTrimmedSink {+ output,+ end: self.input_end.filter(|_| pads_final_frame),+ time_base: self.encoder.time_base(),+ };+ for frame in &frames {+ self.inner+ .send_frame(frame, &mut output, &mut self.encoder)?;}- self.inner.process_eof(output, &mut self.encoder)+ self.inner.process_eof(&mut output, &mut self.encoder)+ }++ fn track_input_end(&mut self, frame: &frame::Audio) {+ if let Some(pts) = frame.pts() {+ let end = pts + frame.samples() as i64;+ self.input_end = Some(self.input_end.map_or(end, |current| current.max(end)));+ }+ }+}++struct EndTrimmedSink<'a, S: PacketSink + ?Sized> {+ output: &'a mut S,+ end: Option<i64>,+ time_base: Rational,+}++impl<S: PacketSink + ?Sized> PacketSink for EndTrimmedSink<'_, S> {+ fn stream_time_base(&self, stream_index: usize) -> Rational {+ self.output.stream_time_base(stream_index)+ }++ fn write_packet(&mut self, mut packet: Packet) -> Result<(), ffmpeg::Error> {+ if let (Some(end), Some(pts)) = (self.end, packet.pts()) {+ let end = end.rescale(+ self.time_base,+ self.output.stream_time_base(packet.stream()),+ );+ if pts >= end {+ return Ok(());+ }+ if pts + packet.duration() > end {+ packet.set_duration(end - pts);+ }+ }+ self.output.write_packet(packet)}}diff --git a/crates/enc-ffmpeg/src/mux/fragmented_audio.rs b/crates/enc-ffmpeg/src/mux/fragmented_audio.rsindex 1203ad72c..900d0b7fa 100644--- a/crates/enc-ffmpeg/src/mux/fragmented_audio.rs+++ b/crates/enc-ffmpeg/src/mux/fragmented_audio.rs@@ -64,7 +64,29 @@ fn configure_fragmentation(output: &mut format::context::Output) -> Result<(), f}impl FragmentedAudioFile {- pub fn init(mut output_path: PathBuf, audio_config: AudioInfo) -> Result<Self, InitError> {+ pub fn init(output_path: PathBuf, audio_config: AudioInfo) -> Result<Self, InitError> {+ Self::init_inner(output_path, audio_config, AACEncoder::init_for_recording)+ }++ #[doc(hidden)]+ pub fn init_with_encoder_preference(+ output_path: PathBuf,+ audio_config: AudioInfo,+ prefer_audiotoolbox: bool,+ ) -> Result<Self, InitError> {+ Self::init_inner(output_path, audio_config, |config, output| {+ AACEncoder::init_for_recording_with(prefer_audiotoolbox, config, output)+ })+ }++ fn init_inner(+ mut output_path: PathBuf,+ audio_config: AudioInfo,+ encoder_init: impl FnOnce(+ AudioInfo,+ &mut format::context::Output,+ ) -> Result<AACEncoder, AACEncoderError>,+ ) -> Result<Self, InitError> {output_path.set_extension("m4a");if let Some(parent) = output_path.parent() {@@ -75,7 +97,7 @@ impl FragmentedAudioFile {configure_fragmentation(&mut output)?;- let encoder = AACEncoder::init(audio_config, &mut output)?;+ let encoder = encoder_init(audio_config, &mut output)?;output.write_header()?;@@ -324,3 +346,290 @@ mod publication_tests {assert!(path.exists());}}++#[cfg(test)]+mod recording_encoder_tests {+ use super::*;+ use ffmpeg::{+ Rescale,+ codec::context,+ format::{Sample, sample::Type},+ };+ use std::path::Path;++ const RATE: u32 = 48_000;+ const FRAME_SAMPLES: usize = 960;+ const CLICK_SAMPLES: usize = 1;+ const FFMPEG_AAC_PRIMING: usize = 1_024;++ fn timeline_with_clicks(total: usize, clicks: &[usize]) -> Vec<f32> {+ let mut timeline = vec![0.0_f32; total];+ for &click in clicks {+ for sample in timeline+ .iter_mut()+ .take((click + CLICK_SAMPLES).min(total))+ .skip(click)+ {+ *sample = 0.9;+ }+ }+ timeline+ }++ fn encode_recording(+ prefer_audiotoolbox: bool,+ channels: u16,+ timeline: &[f32],+ send_ranges: &[(usize, usize)],+ ) -> (Vec<f32>, i64, u64) {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("audio.m4a");+ let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels);+ let mut file = FragmentedAudioFile::init_with_encoder_preference(+ path.clone(),+ info,+ prefer_audiotoolbox,+ )+ .unwrap();++ for &(start, len) in send_ranges {+ let mut position = start;+ while position < start + len {+ let samples = FRAME_SAMPLES.min(start + len - position);+ let mut frame =+ frame::Audio::new(Sample::F32(Type::Packed), samples, info.channel_layout());+ frame.set_rate(RATE);+ for (index, bytes) in frame.data_mut(0)+ [..samples * usize::from(channels) * size_of::<f32>()]+ .chunks_exact_mut(size_of::<f32>())+ .enumerate()+ {+ bytes.copy_from_slice(+ &timeline[position + index / usize::from(channels)].to_le_bytes(),+ );+ }+ file.queue_frame(+ frame,+ Duration::from_secs_f64(position as f64 / f64::from(RATE)),+ )+ .unwrap();+ position += samples;+ }+ }++ file.finish().unwrap().unwrap();+ let bytes = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0);+ let (samples, track_end) = decode_channel_zero(&path);+ (samples, track_end, bytes)+ }++ fn encode_recording_variable(+ prefer_audiotoolbox: bool,+ channels: u16,+ timeline: &[f32],+ frame_sizes: &[usize],+ ) -> (Vec<f32>, i64, u64) {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("audio.m4a");+ let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels);+ let mut file = FragmentedAudioFile::init_with_encoder_preference(+ path.clone(),+ info,+ prefer_audiotoolbox,+ )+ .unwrap();++ let mut position = 0;+ let mut frame_index = 0;+ while position < timeline.len() {+ let samples =+ frame_sizes[frame_index % frame_sizes.len()].min(timeline.len() - position);+ let mut frame =+ frame::Audio::new(Sample::F32(Type::Packed), samples, info.channel_layout());+ frame.set_rate(RATE);+ for (index, bytes) in frame.data_mut(0)+ [..samples * usize::from(channels) * size_of::<f32>()]+ .chunks_exact_mut(size_of::<f32>())+ .enumerate()+ {+ bytes.copy_from_slice(+ &timeline[position + index / usize::from(channels)].to_le_bytes(),+ );+ }+ file.queue_frame(+ frame,+ Duration::from_nanos(position as u64 * 1_000_000_000 / u64::from(RATE)),+ )+ .unwrap();+ position += samples;+ frame_index += 1;+ }++ file.finish().unwrap().unwrap();+ let bytes = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0);+ let (samples, track_end) = decode_channel_zero(&path);+ (samples, track_end, bytes)+ }++ fn decode_channel_zero(path: &Path) -> (Vec<f32>, i64) {+ let mut input = format::input(path).unwrap();+ let stream = input.streams().best(ffmpeg::media::Type::Audio).unwrap();+ let stream_index = stream.index();+ let time_base = stream.time_base();+ let mut decoder = context::Context::from_parameters(stream.parameters())+ .unwrap()+ .decoder()+ .audio()+ .unwrap();+ let mut decoded = frame::Audio::empty();+ let mut samples = Vec::new();+ let mut track_end = i64::MIN;++ for (stream, packet) in input.packets() {+ if stream.index() != stream_index {+ continue;+ }+ if let Some(pts) = packet.pts() {+ track_end =+ track_end.max((pts + packet.duration()).rescale(time_base, (1, 48_000)));+ }+ decoder.send_packet(&packet).unwrap();+ while decoder.receive_frame(&mut decoded).is_ok() {+ assert_eq!(decoded.format(), Sample::F32(Type::Planar));+ samples.extend_from_slice(decoded.plane::<f32>(0));+ }+ }+ decoder.send_eof().unwrap();+ while decoder.receive_frame(&mut decoded).is_ok() {+ assert_eq!(decoded.format(), Sample::F32(Type::Planar));+ samples.extend_from_slice(decoded.plane::<f32>(0));+ }+ (samples, track_end)+ }++ fn click_index(samples: &[f32]) -> usize {+ samples+ .iter()+ .enumerate()+ .max_by(|(_, a), (_, b)| a.abs().partial_cmp(&b.abs()).unwrap())+ .map(|(index, _)| index)+ .expect("no decoded samples")+ }++ fn click_positions(samples: &[f32]) -> Vec<usize> {+ let mut positions = Vec::new();+ let mut index = 0;+ while index < samples.len() {+ if samples[index].abs() > 0.2 {+ let window_end = (index + 512).min(samples.len());+ positions.push(index + click_index(&samples[index..window_end]));+ index = window_end;+ } else {+ index += 1;+ }+ }+ positions+ }++ #[test]+ fn recording_audio_click_decodes_at_the_same_sample_with_both_encoders() {+ for channels in [1u16, 2] {+ let timeline = timeline_with_clicks(96_000, &[24_000]);+ let ranges = [(0usize, 96_000usize)];++ let (ffmpeg_samples, _, _) = encode_recording(false, channels, &timeline, &ranges);+ let ffmpeg_click = click_index(&ffmpeg_samples);+ assert_eq!(+ ffmpeg_click,+ 24_000 + FFMPEG_AAC_PRIMING,+ "FFmpeg aac decoded the click at {ffmpeg_click} for {channels}ch"+ );++ let (at_samples, _, _) = encode_recording(true, channels, &timeline, &ranges);+ let at_click = click_index(&at_samples);+ assert!(+ at_click.abs_diff(ffmpeg_click) <= 4,+ "AudioToolbox decoded the click at {at_click}, FFmpeg at {ffmpeg_click} ({channels}ch)"+ );+ }+ }++ #[test]+ fn recording_audio_silence_filled_gap_decodes_the_same_with_both_encoders() {+ let frame_sizes = [441, 960, 1000, 512, 4800];+ for channels in [1u16, 2] {+ let timeline = timeline_with_clicks(168_000, &[24_000, 120_000]);++ let (ffmpeg_samples, ffmpeg_end, _) =+ encode_recording_variable(false, channels, &timeline, &frame_sizes);+ let (at_samples, at_end, _) =+ encode_recording_variable(true, channels, &timeline, &frame_sizes);++ let ffmpeg_clicks = click_positions(&ffmpeg_samples);+ let at_clicks = click_positions(&at_samples);+ assert_eq!(ffmpeg_clicks.len(), 2, "FFmpeg clicks: {ffmpeg_clicks:?}");+ assert_eq!(at_clicks.len(), 2, "AudioToolbox clicks: {at_clicks:?}");+ for (index, (&ffmpeg_click, &at_click)) in+ ffmpeg_clicks.iter().zip(&at_clicks).enumerate()+ {+ let expected = [24_000usize, 120_000][index] + FFMPEG_AAC_PRIMING;+ assert_eq!(+ ffmpeg_click, expected,+ "FFmpeg aac decoded click {index} at {ffmpeg_click}, expected {expected} ({channels}ch)"+ );+ assert!(+ at_click.abs_diff(ffmpeg_click) <= 4,+ "click {index}: AudioToolbox {at_click} vs FFmpeg {ffmpeg_click} ({channels}ch)"+ );+ }+ assert!(+ at_samples.len().abs_diff(ffmpeg_samples.len()) <= 4,+ "decoded {} vs {} samples ({channels}ch)",+ at_samples.len(),+ ffmpeg_samples.len()+ );+ assert!(+ at_end.abs_diff(ffmpeg_end) <= 1,+ "track end {at_end} vs {ffmpeg_end} ({channels}ch)"+ );+ }+ }++ #[test]+ fn recording_audio_track_ends_where_the_ffmpeg_track_ends() {+ let timeline = vec![0.0_f32; 47_123];+ let ranges = [(0usize, 47_123usize)];++ let (_, ffmpeg_end, _) = encode_recording(false, 1, &timeline, &ranges);+ let (_, at_end, _) = encode_recording(true, 1, &timeline, &ranges);++ assert!(+ at_end.abs_diff(ffmpeg_end) <= 1,+ "track end {at_end} vs {ffmpeg_end}"+ );+ }++ #[cfg(target_os = "macos")]+ #[test]+ fn recording_audio_uses_audiotoolbox_on_macos() {+ if std::env::var_os("CAP_RECORDING_FFMPEG_AAC").is_some_and(|v| !v.is_empty()) {+ return;+ }+ let directory = tempfile::tempdir().unwrap();+ let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, 1);+ let file = FragmentedAudioFile::init(directory.path().join("audio.m4a"), info).unwrap();+ assert_eq!(file.encoder().codec_name().as_deref(), Some("aac_at"));+ }++ #[cfg(target_os = "macos")]+ #[test]+ fn recording_audio_silence_stays_small() {+ let timeline = vec![0.0_f32; 480_000];+ let ranges = [(0usize, 480_000usize)];+ let (_, _, bytes) = encode_recording(true, 2, &timeline, &ranges);+ assert!(+ bytes < 64 * 1024,+ "10s of stereo silence took {bytes} bytes"+ );+ }+}