Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
Merge remote-tracking branch 'origin/main' into perf/audiotoolbox-export-aac
Vagarthbca5f2a2814b
diff --cc crates/enc-ffmpeg/src/audio/aac.rsindex 77a9aef74,ca58b2664..e6291de13--- a/crates/enc-ffmpeg/src/audio/aac.rs+++ b/crates/enc-ffmpeg/src/audio/aac.rs@@@ -25,6 -25,6 +25,12 @@@ pub enum AACEncoderErrorResampler(ffmpeg::Error),}++#[derive(Clone, Copy)]++enum Priming {++ Keep,++ TrimToFfmpegAac,++}++pub struct AACEncoder {base: AudioEncoderBase,stream_index: usize,@@@ -44,23 -45,38 +51,76 @@@ impl AACEncoderinput_config: AudioInfo,output: &mut format::context::Output,) -> Result<Self, AACEncoderError> {- Self::init_with_codec("aac", ffmpeg::Dictionary::new(), input_config, output)+ Self::init_with_codec(+ "aac",+ ffmpeg::Dictionary::new(),+ Self::OUTPUT_BITRATE,++ Priming::Keep,+ input_config,+ output,+ )+ }++ pub fn init_for_recording(+ input_config: AudioInfo,+ output: &mut format::context::Output,+ ) -> Result<Self, AACEncoderError> {+ let prefer_audiotoolbox = cfg!(target_os = "macos")+ && std::env::var_os("CAP_RECORDING_FFMPEG_AAC").is_none_or(|value| value.is_empty());+ Self::init_for_recording_with(prefer_audiotoolbox, input_config, output)+ }++ pub(crate) fn init_for_recording_with(+ prefer_audiotoolbox: bool,+ input_config: AudioInfo,+ output: &mut format::context::Output,+ ) -> Result<Self, AACEncoderError> {+ if prefer_audiotoolbox {+ let mut options = ffmpeg::Dictionary::new();+ options.set("aac_at_mode", "abr");+ let bit_rate = if input_config.channels == 1 {+ 192_000+ } else {+ 256_000+ };- match Self::init_with_codec("aac_at", options, bit_rate, input_config, output) {++ match Self::init_with_codec(++ "aac_at",++ options,++ bit_rate,++ Priming::TrimToFfmpegAac,++ input_config,++ output,++ ) {++ Ok(encoder) => return Ok(encoder),++ Err(error) => {++ tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's");++ }++ }++ }++++ Self::init(input_config, output)+ }+- // A plain MP4's edit list trims AudioToolbox's 2112-sample encoder priming.- // Fragmented recording outputs keep priming in the timeline, so they stay on- // FFmpeg's encoder: AudioToolbox would start their audio ~23 ms later.++ // A plain MP4's edit list skips AudioToolbox's 2112-sample encoder priming,++ // so exports keep every input sample instead of trimming like recordings.+ pub fn init_for_export(+ input_config: AudioInfo,+ output: &mut format::context::Output,+ ) -> Result<Self, AACEncoderError> {+ #[cfg(target_os = "macos")]+ {+ // AudioToolbox defaults to constant bitrate, which fills silence up to+ // the full 320 kbps; average bitrate keeps silent stretches near-free.+ let mut options = ffmpeg::Dictionary::new();+ options.set("aac_at_mode", "abr");- match Self::init_with_codec("aac_at", options, input_config, output) {++ match Self::init_with_codec(++ "aac_at",++ options,++ Self::OUTPUT_BITRATE,++ Priming::Keep,++ input_config,++ output,++ ) {Ok(encoder) => return Ok(encoder),Err(error) => {tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's");@@@ -74,6 -90,7 +134,8 @@@fn init_with_codec(name: &str,options: ffmpeg::Dictionary,+ bit_rate: usize,++ priming: Priming,input_config: AudioInfo,output: &mut format::context::Output,) -> Result<Self, AACEncoderError> {@@@ -135,6 -153,14 +198,17 @@@let encoder = encoder.open_with(options)?;+ // Fragmented outputs keep encoder priming in the decoded timeline;+ // dropping the priming a codec adds beyond FFmpeg's aac (e.g. 2112 vs+ // 1024 samples for aac_at) keeps every content sample where the native+ // encoder put it.- let leading_trim = unsafe { (*encoder.as_ptr()).initial_padding }- .saturating_sub(Self::FFMPEG_AAC_PRIMING)- .max(0) as usize;++ let leading_trim = match priming {++ Priming::TrimToFfmpegAac => unsafe { (*encoder.as_ptr()).initial_padding }++ .saturating_sub(Self::FFMPEG_AAC_PRIMING)++ .max(0) as usize,++ Priming::Keep => 0,++ };+let mut output_stream = output.add_stream(codec)?;output_stream.set_time_base(FFRational(1, output_config.rate()));output_stream.set_parameters(&encoder);@@@ -335,233 -366,6 +414,236 @@@ mod testsassert_eq!(encode_audio(true), encode_audio_through_queue());}- const CLICK_AT: usize = 24_000;++ const CLICKS: [usize; 2] = [240, 24_000];++ fn click_frame(start: usize, samples: usize) -> frame::Audio {+ let mut audio =+ frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO);+ audio.set_rate(48_000);+ audio.set_pts(Some(start as i64));+ for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::<f32>()]+ .chunks_exact_mut(2 * size_of::<f32>())+ .enumerate()+ {+ let position = start + index;- let value = if (CLICK_AT..CLICK_AT + 48).contains(&position) {++ let value = if CLICKS++ .iter()++ .any(|click| (*click..click + 48).contains(&position))++ {+ 0.9_f32+ } else {+ 0.0+ };+ sample[..4].copy_from_slice(&value.to_le_bytes());+ sample[4..].copy_from_slice(&value.to_le_bytes());+ }+ audio+ }+- fn encode_click_for_export(path: &std::path::Path) -> i32 {++ fn encode_clicks_for_export(path: &std::path::Path) {+ let mut output = format::output(path).unwrap();+ let mut encoder = AACEncoder::init_for_export(+ AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),+ &mut output,+ )+ .unwrap();- let initial_padding = unsafe {- (*output- .stream(encoder.stream_index())- .unwrap()- .parameters()- .as_ptr())- .initial_padding- };+ output.write_header().unwrap();++ let mut position = 0;+ while position < 48_000 {+ let samples = 960.min(48_000 - position);+ encoder+ .send_frame(click_frame(position, samples), Duration::MAX, &mut output)+ .unwrap();+ position += samples;+ }++ encoder.flush(&mut output).unwrap();+ output.write_trailer().unwrap();- initial_padding+ }++ fn decode_first_channel(path: &std::path::Path) -> (Vec<f32>, i64) {+ use ffmpeg::Rescale;++ let mut input = format::input(path).unwrap();+ let stream = input.streams().best(ffmpeg::media::Type::Audio).unwrap();+ let stream_index = stream.index();+ let time_base = stream.time_base();+ let mut decoder = context::Context::from_parameters(stream.parameters())+ .unwrap()+ .decoder()+ .audio()+ .unwrap();+ let mut decoded = frame::Audio::empty();+ let mut samples = Vec::new();+ let mut track_end = i64::MIN;++ for (stream, packet) in input.packets() {+ if stream.index() != stream_index {+ continue;+ }+ if let Some(pts) = packet.pts() {+ track_end =+ track_end.max((pts + packet.duration()).rescale(time_base, (1, 48_000)));+ }+ decoder.send_packet(&packet).unwrap();+ while decoder.receive_frame(&mut decoded).is_ok() {+ samples.extend_from_slice(decoded.plane::<f32>(0));+ }+ }+ decoder.send_eof().unwrap();+ while decoder.receive_frame(&mut decoded).is_ok() {+ samples.extend_from_slice(decoded.plane::<f32>(0));+ }+ (samples, track_end)+ }++ #[test]+ fn export_encoder_keeps_audio_aligned_in_mp4() {+ let directory = tempfile::tempdir().unwrap();- let path = directory.path().join("click.mp4");- encode_click_for_export(&path);++ let path = directory.path().join("clicks.mp4");++ encode_clicks_for_export(&path);++ let (samples, track_end) = decode_first_channel(&path);- let click = samples- .iter()- .position(|sample| sample.abs() > 0.3)- .expect("the encoded click was not found");- assert!(- click.abs_diff(CLICK_AT) < 64,- "click decoded at sample {click}, encoded at {CLICK_AT}"- );++ for encoded in CLICKS {++ let search_from = encoded.saturating_sub(64);++ let decoded = samples[search_from..]++ .iter()++ .position(|sample| sample.abs() > 0.3)++ .map(|offset| search_from + offset)++ .expect("the encoded click was not found");++ assert!(++ decoded.abs_diff(encoded) < 64,++ "click decoded at sample {decoded}, encoded at {encoded}"++ );++ }+ assert_eq!(track_end, 48_000);+ }++ #[test]+ fn export_encoder_spends_almost_nothing_on_silence() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("silence.mp4");+ let mut output = format::output(&path).unwrap();+ let mut encoder = AACEncoder::init_for_export(+ AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),+ &mut output,+ )+ .unwrap();+ output.write_header().unwrap();+ for start in (0..480_000).step_by(960) {+ let mut audio =+ frame::Audio::new(Sample::F32(Type::Packed), 960, ChannelLayout::STEREO);+ audio.set_rate(48_000);+ audio.set_pts(Some(start));+ audio.data_mut(0)[..960 * 2 * size_of::<f32>()].fill(0);+ encoder+ .send_frame(audio, Duration::MAX, &mut output)+ .unwrap();+ }+ encoder.flush(&mut output).unwrap();+ output.write_trailer().unwrap();+ drop(encoder);+ drop(output);++ let mut input = format::input(&path).unwrap();+ let stream_index = input+ .streams()+ .best(ffmpeg::media::Type::Audio)+ .unwrap()+ .index();+ let bytes = input+ .packets()+ .filter(|(stream, _)| stream.index() == stream_index)+ .map(|(_, packet)| packet.size())+ .sum::<usize>();+ assert!(bytes < 40_000, "10 s of silence took {bytes} bytes");+ }++ #[cfg(target_os = "macos")]+ #[test]+ fn export_encoder_uses_audiotoolbox_on_macos() {+ let directory = tempfile::tempdir().unwrap();- let initial_padding = encode_click_for_export(&directory.path().join("click.mp4"));- assert_eq!(initial_padding, 2112);++ let mut output = format::output(&directory.path().join("audio.mp4")).unwrap();++ let encoder = AACEncoder::init_for_export(++ AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),++ &mut output,++ )++ .unwrap();++ assert_eq!(encoder.codec_name().as_deref(), Some("aac_at"));+ }++ #[test]+ #[ignore]+ fn benchmark_export_aac_encoder() {+ type Init =+ fn(AudioInfo, &mut format::context::Output) -> Result<AACEncoder, AACEncoderError>;++ fn voice_like_frame(start: usize, samples: usize, noise: &mut u32) -> frame::Audio {+ let mut audio =+ frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO);+ audio.set_rate(48_000);+ audio.set_pts(Some(start as i64));+ for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::<f32>()]+ .chunks_exact_mut(size_of::<f32>())+ .enumerate()+ {+ *noise ^= *noise << 13;+ *noise ^= *noise >> 17;+ *noise ^= *noise << 5;+ let time = (start + index / 2) as f32 / 48_000.0;+ let value = 0.3 * (std::f32::consts::TAU * 220.0 * time).sin()+ + 0.15 * (std::f32::consts::TAU * 1_330.0 * time).sin()+ + 0.1 * (*noise as f32 / u32::MAX as f32 - 0.5);+ sample.copy_from_slice(&value.to_le_bytes());+ }+ audio+ }++ let seconds = std::env::var("CAP_AAC_BENCH_SECONDS")+ .ok()+ .and_then(|value| value.parse().ok())+ .unwrap_or(60usize);+ let samples = seconds * 48_000;+ let directory = tempfile::tempdir().unwrap();+ let encoders: [(&str, Init); 2] = [+ ("init (FFmpeg aac)", AACEncoder::init),+ ("init_for_export", AACEncoder::init_for_export),+ ];+ for round in 0..3 {+ for (label, init) in encoders {+ let path = directory.path().join(format!("{round}.mp4"));+ let mut output = format::output(&path).unwrap();+ let mut encoder = init(+ AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),+ &mut output,+ )+ .unwrap();+ output.write_header().unwrap();+ let mut noise = 0x9e37_79b9_u32;+ let frames = (0..samples)+ .step_by(960)+ .map(|position| {+ voice_like_frame(position, 960.min(samples - position), &mut noise)+ })+ .collect::<Vec<_>>();+ let started = std::time::Instant::now();+ for frame in frames {+ encoder+ .send_frame(frame, Duration::MAX, &mut output)+ .unwrap();+ }+ encoder.flush(&mut output).unwrap();+ output.write_trailer().unwrap();+ let wall = started.elapsed().as_secs_f64();+ println!(+ "round {round} {label}: {seconds}s of 48 kHz stereo in {wall:.3}s ({:.0}x realtime)",+ seconds as f64 / wall+ );+ }+ }+ }+#[test]fn checked_audio_submission_reports_a_closed_encoder() {let directory = tempfile::tempdir().unwrap();