Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
use std::{thread, time::Duration};
use cap_media_info::{AudioInfo, FFRational};
use ffmpeg::{
codec::{context, encoder},
format::{self, Sample, sample::Type},
frame,
threading::Config,
};
use crate::{
AudioEncoder, PacketSink,
audio::{base::AudioEncoderBase, buffered_resampler::BufferedResampler},
};
#[derive(thiserror::Error, Debug)]
pub enum AACEncoderError {
#[error("{0:?}")]
FFmpeg(#[from] ffmpeg::Error),
#[error("AAC codec not found")]
CodecNotFound,
#[error("Sample rate not supported: {0}")]
RateNotSupported(i32),
#[error("Resampler: {0}")]
Resampler(ffmpeg::Error),
}
#[derive(Clone, Copy)]
enum Priming {
Keep,
TrimToFfmpegAac,
}
pub struct AACEncoder {
base: AudioEncoderBase,
stream_index: usize,
}
impl AACEncoder {
const OUTPUT_BITRATE: usize = 320 * 1000; // 128k
const SAMPLE_FORMAT: Sample = Sample::F32(Type::Planar);
const FFMPEG_AAC_PRIMING: i32 = 1_024;
pub fn factory(
input_config: AudioInfo,
) -> impl FnOnce(&mut format::context::Output) -> Result<Self, AACEncoderError> {
move |o| Self::init(input_config, o)
}
pub fn init(
input_config: AudioInfo,
output: &mut format::context::Output,
) -> Result<Self, AACEncoderError> {
Self::init_with_codec(
"aac",
ffmpeg::Dictionary::new(),
Self::OUTPUT_BITRATE,
Priming::Keep,
input_config,
output,
)
}
pub fn init_for_recording(
input_config: AudioInfo,
output: &mut format::context::Output,
) -> Result<Self, AACEncoderError> {
let prefer_audiotoolbox = cfg!(target_os = "macos")
&& std::env::var_os("CAP_RECORDING_FFMPEG_AAC").is_none_or(|value| value.is_empty());
Self::init_for_recording_with(prefer_audiotoolbox, input_config, output)
}
pub(crate) fn init_for_recording_with(
prefer_audiotoolbox: bool,
input_config: AudioInfo,
output: &mut format::context::Output,
) -> Result<Self, AACEncoderError> {
if prefer_audiotoolbox {
let mut options = ffmpeg::Dictionary::new();
options.set("aac_at_mode", "abr");
let bit_rate = if input_config.channels == 1 {
192_000
} else {
256_000
};
match Self::init_with_codec(
"aac_at",
options,
bit_rate,
Priming::TrimToFfmpegAac,
input_config,
output,
) {
Ok(encoder) => return Ok(encoder),
Err(error) => {
tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's");
}
}
}
Self::init(input_config, output)
}
// A plain MP4's edit list skips AudioToolbox's 2112-sample encoder priming,
// so exports keep every input sample instead of trimming like recordings.
pub fn init_for_export(
input_config: AudioInfo,
output: &mut format::context::Output,
) -> Result<Self, AACEncoderError> {
#[cfg(target_os = "macos")]
{
// AudioToolbox defaults to constant bitrate, which fills silence up to
// the full 320 kbps; average bitrate keeps silent stretches near-free.
let mut options = ffmpeg::Dictionary::new();
options.set("aac_at_mode", "abr");
match Self::init_with_codec(
"aac_at",
options,
Self::OUTPUT_BITRATE,
Priming::Keep,
input_config,
output,
) {
Ok(encoder) => return Ok(encoder),
Err(error) => {
tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's");
}
}
}
Self::init(input_config, output)
}
fn init_with_codec(
name: &str,
options: ffmpeg::Dictionary,
bit_rate: usize,
priming: Priming,
input_config: AudioInfo,
output: &mut format::context::Output,
) -> Result<Self, AACEncoderError> {
let codec = encoder::find_by_name(name).ok_or(AACEncoderError::CodecNotFound)?;
let audio_codec = codec.audio()?;
let mut encoder_ctx = context::Context::new_with_codec(codec);
let thread_count = thread::available_parallelism()
.map(|v| v.get())
.unwrap_or(1);
encoder_ctx.set_threading(Config::count(thread_count));
let mut encoder = encoder_ctx.encoder().audio()?;
let rate = {
let mut rates = audio_codec
.rates()
.into_iter()
.flatten()
.collect::<Vec<_>>();
rates.sort();
if rates.is_empty() {
input_config.rate()
} else {
let Some(&rate) = rates
.iter()
.find(|r| **r >= input_config.rate())
.or(rates.last())
else {
return Err(AACEncoderError::RateNotSupported(input_config.rate()));
};
rate
}
};
let sample_format = {
let formats = audio_codec
.formats()
.into_iter()
.flatten()
.collect::<Vec<_>>();
if formats.contains(&Self::SAMPLE_FORMAT) {
Self::SAMPLE_FORMAT
} else {
formats.first().copied().unwrap_or(Self::SAMPLE_FORMAT)
}
};
let mut output_config = input_config;
output_config.sample_format = sample_format;
output_config.sample_rate = rate as u32;
let resampler = BufferedResampler::new(input_config, output_config)
.map_err(AACEncoderError::Resampler)?;
encoder.set_bit_rate(bit_rate);
encoder.set_rate(rate);
encoder.set_format(output_config.sample_format);
encoder.set_channel_layout(output_config.channel_layout());
encoder.set_time_base(FFRational(1, output_config.rate()));
let encoder = encoder.open_with(options)?;
// Fragmented outputs keep encoder priming in the decoded timeline;
// dropping the priming a codec adds beyond FFmpeg's aac (e.g. 2112 vs
// 1024 samples for aac_at) keeps every content sample where the native
// encoder put it.
let leading_trim = match priming {
Priming::TrimToFfmpegAac => unsafe { (*encoder.as_ptr()).initial_padding }
.saturating_sub(Self::FFMPEG_AAC_PRIMING)
.max(0) as usize,
Priming::Keep => 0,
};
let mut output_stream = output.add_stream(codec)?;
output_stream.set_time_base(FFRational(1, output_config.rate()));
output_stream.set_parameters(&encoder);
let stream_index = output_stream.index();
Ok(Self {
base: AudioEncoderBase::new(encoder, resampler, stream_index)
.with_leading_trim(leading_trim),
stream_index,
})
}
pub fn stream_index(&self) -> usize {
self.stream_index
}
pub fn codec_name(&self) -> Option<String> {
self.base.codec_name()
}
pub fn send_frame(
&mut self,
frame: frame::Audio,
timestamp: Duration,
output: &mut format::context::Output,
) -> Result<(), ffmpeg::Error> {
self.base.send_frame(frame, timestamp, output)
}
pub fn flush(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> {
self.base.flush(output)
}
pub fn send_frame_to(
&mut self,
frame: frame::Audio,
timestamp: Duration,
sink: &mut impl PacketSink,
) -> Result<(), ffmpeg::Error> {
self.base.send_frame(frame, timestamp, sink)
}
pub fn flush_to(&mut self, sink: &mut impl PacketSink) -> Result<(), ffmpeg::Error> {
self.base.flush(sink)
}
}
impl AudioEncoder for AACEncoder {
fn send_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output) {
let _ = self.send_frame(frame, Duration::MAX, output);
}
fn try_send_frame(
&mut self,
frame: frame::Audio,
output: &mut format::context::Output,
) -> Result<(), ffmpeg::Error> {
self.send_frame(frame, Duration::MAX, output)
}
fn flush(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> {
self.flush(output)
}
}
#[cfg(test)]
mod tests {
use super::*;
use ffmpeg::ChannelLayout;
#[test]
fn sample_rate_selection_preserves_supported_rates_and_clamps_outliers() {
let directory = tempfile::tempdir().unwrap();
for (input_rate, expected_rate) in [
(4_000, 7_350),
(44_100, 44_100),
(45_000, 48_000),
(96_000, 96_000),
(192_000, 96_000),
] {
let mut output =
format::output(&directory.path().join(format!("{input_rate}.mp4"))).unwrap();
let encoder = AACEncoder::init(
AudioInfo::new_raw(Sample::F32(Type::Packed), input_rate, 1),
&mut output,
)
.unwrap();
let stream = output.stream(encoder.stream_index()).unwrap();
let decoder = context::Context::from_parameters(stream.parameters())
.unwrap()
.decoder()
.audio()
.unwrap();
assert_eq!(decoder.rate(), expected_rate, "input rate {input_rate}");
}
}
#[test]
fn high_rate_input_keeps_high_frequency_audio_through_encode_decode() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("high-rate.mp4");
let mut output = format::output(&path).unwrap();
let mut encoder = AACEncoder::init(
AudioInfo::new_raw(Sample::F32(Type::Packed), 192_000, 1),
&mut output,
)
.unwrap();
output.write_header().unwrap();
let mut audio = frame::Audio::new(Sample::F32(Type::Packed), 192_000, ChannelLayout::MONO);
audio.set_rate(192_000);
audio.set_pts(Some(0));
for (index, sample) in audio.plane_mut::<f32>(0).iter_mut().enumerate() {
*sample = 0.5 * (std::f32::consts::TAU * 12_000.0 * index as f32 / 192_000.0).sin();
}
encoder
.send_frame(audio, Duration::MAX, &mut output)
.unwrap();
encoder.flush(&mut output).unwrap();
output.write_trailer().unwrap();
drop(encoder);
drop(output);
let (samples, track_end) = decode_first_channel(&path);
assert_eq!(track_end, 48_000);
assert!(samples.len() >= 96_000, "decoded {} samples", samples.len());
let mut sine = 0.0;
let mut cosine = 0.0;
let signal = &samples[9_600..86_400];
for (index, sample) in signal.iter().enumerate() {
let phase = std::f64::consts::TAU * 12_000.0 * index as f64 / 96_000.0;
sine += f64::from(*sample) * phase.sin();
cosine += f64::from(*sample) * phase.cos();
}
let amplitude = 2.0 * sine.hypot(cosine) / signal.len() as f64;
assert!(amplitude > 0.4, "12 kHz tone amplitude {amplitude}");
}
fn input_frame(start: i64, samples: usize) -> frame::Audio {
let mut audio =
frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO);
audio.set_rate(48_000);
audio.set_pts(Some(start));
for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::<f32>()]
.chunks_exact_mut(size_of::<f32>())
.enumerate()
{
let value = ((index % 17) as f32 - 8.0) / 32.0;
sample.copy_from_slice(&value.to_le_bytes());
}
audio
}
fn encode_audio(checked: bool) -> Vec<u8> {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("audio.mp4");
let mut output = format::output(&path).unwrap();
let mut encoder = AACEncoder::init(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
output.write_header().unwrap();
let mut position = 0;
for samples in [1, 997, 4_096, 1_024, 37] {
let audio = input_frame(position, samples);
if checked {
AudioEncoder::try_send_frame(&mut encoder, audio, &mut output).unwrap();
} else {
AudioEncoder::send_frame(&mut encoder, audio, &mut output);
}
position += samples as i64;
}
encoder.flush(&mut output).unwrap();
output.write_trailer().unwrap();
drop(encoder);
drop(output);
std::fs::read(path).unwrap()
}
fn encode_audio_through_queue() -> Vec<u8> {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("audio.mp4");
let mut output = format::output(&path).unwrap();
let mut encoder = AACEncoder::init(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
output.write_header().unwrap();
let mut queue = crate::PacketQueue::new(output.stream_time_base(encoder.stream_index()));
let mut position = 0;
for samples in [1, 997, 4_096, 1_024, 37] {
encoder
.send_frame_to(input_frame(position, samples), Duration::MAX, &mut queue)
.unwrap();
for packet in queue.take() {
output.write_packet(packet).unwrap();
}
position += samples as i64;
}
encoder.flush(&mut output).unwrap();
output.write_trailer().unwrap();
drop(encoder);
drop(output);
std::fs::read(path).unwrap()
}
fn encode_two_streams(queued_delay_batches: usize) -> Vec<u8> {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("two-streams.mp4");
let mut output = format::output(&path).unwrap();
let mut direct = AACEncoder::init(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
let mut queued = AACEncoder::init(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
output.write_header().unwrap();
let mut queue = crate::PacketQueue::new(output.stream_time_base(queued.stream_index()));
let mut delayed = std::collections::VecDeque::new();
let mut position = 0;
for index in 0..240 {
let samples = [800, 801, 1_024, 997][index % 4];
direct
.send_frame(input_frame(position, samples), Duration::MAX, &mut output)
.unwrap();
queued
.send_frame_to(input_frame(position, samples), Duration::MAX, &mut queue)
.unwrap();
delayed.push_back(queue.take());
while delayed.len() > queued_delay_batches {
for packet in delayed.pop_front().unwrap() {
output.write_packet(packet).unwrap();
}
}
position += samples as i64;
}
for packet in delayed.into_iter().flatten() {
output.write_packet(packet).unwrap();
}
direct.flush(&mut output).unwrap();
queued.flush(&mut output).unwrap();
output.write_trailer().unwrap();
drop(direct);
drop(queued);
drop(output);
std::fs::read(path).unwrap()
}
#[test]
fn checked_audio_submission_preserves_encoded_bytes() {
assert_eq!(encode_audio(false), encode_audio(true));
}
#[test]
fn delayed_queued_packets_interleave_like_lockstep_writes() {
let lockstep = encode_two_streams(0);
assert_eq!(lockstep, encode_two_streams(1));
assert_eq!(lockstep, encode_two_streams(120));
}
#[test]
fn queued_audio_packets_match_direct_writes() {
assert_eq!(encode_audio(true), encode_audio_through_queue());
}
const CLICKS: [usize; 2] = [240, 24_000];
fn click_frame(start: usize, samples: usize) -> frame::Audio {
let mut audio =
frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO);
audio.set_rate(48_000);
audio.set_pts(Some(start as i64));
for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::<f32>()]
.chunks_exact_mut(2 * size_of::<f32>())
.enumerate()
{
let position = start + index;
let value = if CLICKS
.iter()
.any(|click| (*click..click + 48).contains(&position))
{
0.9_f32
} else {
0.0
};
sample[..4].copy_from_slice(&value.to_le_bytes());
sample[4..].copy_from_slice(&value.to_le_bytes());
}
audio
}
fn encode_clicks_for_export(path: &std::path::Path) {
let mut output = format::output(path).unwrap();
let mut encoder = AACEncoder::init_for_export(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
output.write_header().unwrap();
let mut position = 0;
while position < 48_000 {
let samples = 960.min(48_000 - position);
encoder
.send_frame(click_frame(position, samples), Duration::MAX, &mut output)
.unwrap();
position += samples;
}
encoder.flush(&mut output).unwrap();
output.write_trailer().unwrap();
}
fn decode_first_channel(path: &std::path::Path) -> (Vec<f32>, i64) {
use ffmpeg::Rescale;
let mut input = format::input(path).unwrap();
let stream = input.streams().best(ffmpeg::media::Type::Audio).unwrap();
let stream_index = stream.index();
let time_base = stream.time_base();
let mut decoder = context::Context::from_parameters(stream.parameters())
.unwrap()
.decoder()
.audio()
.unwrap();
let mut decoded = frame::Audio::empty();
let mut samples = Vec::new();
let mut track_end = i64::MIN;
for (stream, packet) in input.packets() {
if stream.index() != stream_index {
continue;
}
if let Some(pts) = packet.pts() {
track_end =
track_end.max((pts + packet.duration()).rescale(time_base, (1, 48_000)));
}
decoder.send_packet(&packet).unwrap();
while decoder.receive_frame(&mut decoded).is_ok() {
samples.extend_from_slice(decoded.plane::<f32>(0));
}
}
decoder.send_eof().unwrap();
while decoder.receive_frame(&mut decoded).is_ok() {
samples.extend_from_slice(decoded.plane::<f32>(0));
}
(samples, track_end)
}
#[test]
fn export_encoder_keeps_audio_aligned_in_mp4() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("clicks.mp4");
encode_clicks_for_export(&path);
let (samples, track_end) = decode_first_channel(&path);
for encoded in CLICKS {
let search_from = encoded.saturating_sub(64);
let decoded = samples[search_from..]
.iter()
.position(|sample| sample.abs() > 0.3)
.map(|offset| search_from + offset)
.expect("the encoded click was not found");
assert!(
decoded.abs_diff(encoded) < 64,
"click decoded at sample {decoded}, encoded at {encoded}"
);
}
assert_eq!(track_end, 48_000);
}
#[test]
fn export_encoder_spends_almost_nothing_on_silence() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("silence.mp4");
let mut output = format::output(&path).unwrap();
let mut encoder = AACEncoder::init_for_export(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
output.write_header().unwrap();
for start in (0..480_000).step_by(960) {
let mut audio =
frame::Audio::new(Sample::F32(Type::Packed), 960, ChannelLayout::STEREO);
audio.set_rate(48_000);
audio.set_pts(Some(start));
audio.data_mut(0)[..960 * 2 * size_of::<f32>()].fill(0);
encoder
.send_frame(audio, Duration::MAX, &mut output)
.unwrap();
}
encoder.flush(&mut output).unwrap();
output.write_trailer().unwrap();
drop(encoder);
drop(output);
let mut input = format::input(&path).unwrap();
let stream_index = input
.streams()
.best(ffmpeg::media::Type::Audio)
.unwrap()
.index();
let bytes = input
.packets()
.filter(|(stream, _)| stream.index() == stream_index)
.map(|(_, packet)| packet.size())
.sum::<usize>();
assert!(bytes < 40_000, "10 s of silence took {bytes} bytes");
}
#[cfg(target_os = "macos")]
#[test]
fn export_encoder_uses_audiotoolbox_on_macos() {
let directory = tempfile::tempdir().unwrap();
let mut output = format::output(&directory.path().join("audio.mp4")).unwrap();
let encoder = AACEncoder::init_for_export(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
assert_eq!(encoder.codec_name().as_deref(), Some("aac_at"));
}
#[test]
#[ignore]
fn benchmark_export_aac_encoder() {
type Init =
fn(AudioInfo, &mut format::context::Output) -> Result<AACEncoder, AACEncoderError>;
fn voice_like_frame(start: usize, samples: usize, noise: &mut u32) -> frame::Audio {
let mut audio =
frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO);
audio.set_rate(48_000);
audio.set_pts(Some(start as i64));
for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::<f32>()]
.chunks_exact_mut(size_of::<f32>())
.enumerate()
{
*noise ^= *noise << 13;
*noise ^= *noise >> 17;
*noise ^= *noise << 5;
let time = (start + index / 2) as f32 / 48_000.0;
let value = 0.3 * (std::f32::consts::TAU * 220.0 * time).sin()
+ 0.15 * (std::f32::consts::TAU * 1_330.0 * time).sin()
+ 0.1 * (*noise as f32 / u32::MAX as f32 - 0.5);
sample.copy_from_slice(&value.to_le_bytes());
}
audio
}
let seconds = std::env::var("CAP_AAC_BENCH_SECONDS")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(60usize);
let samples = seconds * 48_000;
let directory = tempfile::tempdir().unwrap();
let encoders: [(&str, Init); 2] = [
("init (FFmpeg aac)", AACEncoder::init),
("init_for_export", AACEncoder::init_for_export),
];
for round in 0..3 {
for (label, init) in encoders {
let path = directory.path().join(format!("{round}.mp4"));
let mut output = format::output(&path).unwrap();
let mut encoder = init(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
output.write_header().unwrap();
let mut noise = 0x9e37_79b9_u32;
let frames = (0..samples)
.step_by(960)
.map(|position| {
voice_like_frame(position, 960.min(samples - position), &mut noise)
})
.collect::<Vec<_>>();
let started = std::time::Instant::now();
for frame in frames {
encoder
.send_frame(frame, Duration::MAX, &mut output)
.unwrap();
}
encoder.flush(&mut output).unwrap();
output.write_trailer().unwrap();
let wall = started.elapsed().as_secs_f64();
println!(
"round {round} {label}: {seconds}s of 48 kHz stereo in {wall:.3}s ({:.0}x realtime)",
seconds as f64 / wall
);
}
}
}
#[test]
fn checked_audio_submission_reports_a_closed_encoder() {
let directory = tempfile::tempdir().unwrap();
let mut output = format::output(&directory.path().join("closed.mp4")).unwrap();
let mut encoder = AACEncoder::init(
AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2),
&mut output,
)
.unwrap();
output.write_header().unwrap();
encoder.flush(&mut output).unwrap();
assert_eq!(
AudioEncoder::try_send_frame(&mut encoder, input_frame(0, 1_024), &mut output),
Err(ffmpeg::Error::Eof)
);
}
}