Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
use cpal::{SampleFormat, SupportedBufferSize, SupportedStreamConfig};
use ffmpeg::frame;
pub use ffmpeg::{
format::{
pixel::Pixel,
sample::{Sample, Type},
},
util::{channel_layout::ChannelLayout, rational::Rational as FFRational},
};
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct AudioInfo {
pub sample_format: Sample,
pub sample_rate: u32,
pub channels: usize,
pub time_base: FFRational,
pub buffer_size: u32,
pub is_wireless_transport: bool,
}
#[derive(Debug, thiserror::Error)]
pub enum AudioInfoError {
#[error("Unsupported number of channels: {0}")]
ChannelLayout(u16),
}
impl AudioInfo {
/// Maximum number of audio channels supported by FFmpeg channel layouts.
/// Matches the highest channel count in `channel_layout_raw` (7.1 surround = 8 channels).
/// Note: Actual device channel counts may exceed this; we store the real count
/// but clamp to this value when creating FFmpeg frames.
pub const MAX_AUDIO_CHANNELS: u16 = 8;
pub const fn new(
sample_format: Sample,
sample_rate: u32,
channel_count: u16,
) -> Result<Self, AudioInfoError> {
if Self::channel_layout_raw(channel_count).is_none() {
return Err(AudioInfoError::ChannelLayout(channel_count));
}
Ok(Self {
sample_format,
sample_rate,
channels: channel_count as usize,
time_base: FFRational(1, 1_000_000),
buffer_size: 1024,
is_wireless_transport: false,
})
}
pub const fn new_raw(sample_format: Sample, sample_rate: u32, channel_count: u16) -> Self {
Self {
sample_format,
sample_rate,
channels: channel_count as usize,
time_base: FFRational(1, 1_000_000),
buffer_size: 1024,
is_wireless_transport: false,
}
}
pub fn from_stream_config(config: &SupportedStreamConfig) -> Self {
Self::from_stream_config_with_buffer(config, None)
}
pub fn from_stream_config_with_buffer(
config: &SupportedStreamConfig,
buffer_size_override: Option<u32>,
) -> Self {
let sample_format = ffmpeg_sample_format_for(config.sample_format()).unwrap();
let buffer_size = buffer_size_override.unwrap_or_else(|| match config.buffer_size() {
SupportedBufferSize::Range { max, .. } => *max,
// TODO: Different buffer sizes for different contexts?
SupportedBufferSize::Unknown => 1024,
});
// Store the actual channel count from the device, even if it exceeds
// MAX_AUDIO_CHANNELS. This ensures audio data parsing works correctly.
// The clamping to supported FFmpeg layouts happens in channel_layout()
// and wrap_frame_with_max_channels() when creating output frames.
let raw_channels = config.channels().max(1);
Self {
sample_format,
sample_rate: config.sample_rate().0,
channels: raw_channels.into(),
time_base: FFRational(1, 1_000_000),
buffer_size,
is_wireless_transport: false,
}
}
pub fn from_decoder(decoder: &ffmpeg::codec::decoder::Audio) -> Result<Self, AudioInfoError> {
Self::channel_layout_raw(decoder.channels())
.ok_or(AudioInfoError::ChannelLayout(decoder.channels()))?;
Ok(Self {
sample_format: decoder.format(),
sample_rate: decoder.rate(),
channels: usize::from(decoder.channels()),
time_base: decoder.time_base(),
buffer_size: decoder.frame_size(),
is_wireless_transport: false,
})
}
const fn channel_layout_raw(channels: u16) -> Option<ChannelLayout> {
Some(match channels {
1 => ChannelLayout::MONO,
2 => ChannelLayout::STEREO,
3 => ChannelLayout::SURROUND,
4 => ChannelLayout::QUAD,
5 => ChannelLayout::_5POINT0,
6 => ChannelLayout::_5POINT1,
7 => ChannelLayout::_6POINT1,
8 => ChannelLayout::_7POINT1,
_ => return None,
})
}
pub fn channel_layout(&self) -> ChannelLayout {
// Clamp channels to supported range and return appropriate layout.
// This prevents panics when audio devices report unusual channel counts
// (e.g., 0 channels or more than 8 channels).
let clamped_channels = (self.channels as u16).clamp(1, 8);
Self::channel_layout_raw(clamped_channels).unwrap_or(ChannelLayout::STEREO)
}
/// Channel count a packed frame from [`Self::wrap_frame_with_max_channels`] carries
/// for `max_channels`. The single source of truth for the wrapped count so a
/// resampler reading those frames cannot disagree on the channel count.
pub fn wrapped_frame_channels(&self, max_channels: usize) -> usize {
self.channels
.max(1)
.min(max_channels.max(1))
.min(Self::MAX_AUDIO_CHANNELS as usize)
}
/// Channel layout a packed frame from [`Self::wrap_frame_with_max_channels`] carries.
///
/// A resampler that consumes those frames MUST be configured with this exact
/// layout. `swr_convert_frame` revalidates every input frame's channel layout
/// against the context and silently rejects it (→ the whole frame is dropped →
/// silent track) when the masks differ. The named [`Self::channel_layout`] masks
/// diverge from `ChannelLayout::default` for 3/4/5/6-channel sources, so the two
/// must never be mixed across a frame builder and the resampler reading it.
pub fn wrapped_frame_layout(&self, max_channels: usize) -> ChannelLayout {
ChannelLayout::default(self.wrapped_frame_channels(max_channels) as i32)
}
/// Channel layout for an un-clamped packed frame of this info's channel count,
/// matching frames built with `ChannelLayout::default(channels)` (e.g.
/// `scap_ffmpeg::DataExt::as_ffmpeg`). See [`Self::wrapped_frame_layout`] for why
/// a resampler's input layout must match its frame builder's exactly.
pub fn packed_channel_layout(&self) -> ChannelLayout {
ChannelLayout::default(self.channels.max(1) as i32)
}
pub fn sample_size(&self) -> usize {
self.sample_format.bytes()
}
pub const fn rate(&self) -> i32 {
self.sample_rate as i32
}
pub fn empty_frame(&self, sample_count: usize) -> frame::Audio {
let mut frame = frame::Audio::new(self.sample_format, sample_count, self.channel_layout());
frame.set_rate(self.sample_rate);
frame
}
/// Always expects packed input data
pub fn wrap_frame_with_max_channels(
&self,
packed_data: &[u8],
max_channels: usize,
) -> frame::Audio {
// Use actual channel count for parsing input data (at least 1 to avoid div by zero)
let input_channels = self.channels.max(1);
// Clamp output channels to both max_channels and MAX_AUDIO_CHANNELS for FFmpeg
// compatibility. Shared with `wrapped_frame_layout` so a resampler configured
// for these frames is guaranteed the same channel count + layout.
let out_channels = self.wrapped_frame_channels(max_channels);
let sample_size = self.sample_size();
let packed_sample_size = sample_size * input_channels;
let samples = packed_data.len() / packed_sample_size;
// A device callback can end mid-sample or carry no whole sample at
// all (empty buffers around device changes); a frame with zero samples
// has no planes to copy into.
let packed_data = &packed_data[..samples * packed_sample_size];
let mut frame = frame::Audio::new(
self.sample_format,
samples,
ChannelLayout::default(out_channels as i32),
);
frame.set_rate(self.sample_rate);
if samples == 0 {
return frame;
}
if input_channels == 1 || (frame.is_packed() && input_channels <= out_channels) {
// frame is allocated with parameters derived from packed_data, so this is safe
frame.data_mut(0)[0..packed_data.len()].copy_from_slice(packed_data);
} else if frame.is_packed() && input_channels > out_channels {
let copy_len = sample_size * out_channels;
for (packed_chunk, frame_chunk) in packed_data
.chunks_exact(packed_sample_size)
.zip(frame.data_mut(0).chunks_exact_mut(copy_len))
{
frame_chunk.copy_from_slice(&packed_chunk[..copy_len]);
}
} else {
for (chunk_index, packed_chunk) in packed_data.chunks(packed_sample_size).enumerate() {
let start = chunk_index * sample_size;
for channel in 0..out_channels {
let channel_start = channel * sample_size;
let channel_end = channel_start + sample_size;
if let (Some(chunk_slice), Some(frame_slice)) = (
packed_chunk.get(channel_start..channel_end),
frame.data_mut(channel).get_mut(start..start + sample_size),
) {
frame_slice.copy_from_slice(chunk_slice);
}
}
}
}
frame
}
/// Always expects packed input data
pub fn wrap_frame(&self, data: &[u8]) -> frame::Audio {
self.wrap_frame_with_max_channels(data, self.channels)
}
pub fn with_wireless_transport(mut self, is_wireless: bool) -> Self {
self.is_wireless_transport = is_wireless;
self
}
pub fn with_max_channels(&self, channels: u16) -> Self {
let mut this = *self;
this.channels = this.channels.min(channels as usize);
this
}
pub fn with_sample_rate(&self, rate: u32) -> Self {
let mut this = *self;
this.sample_rate = rate;
this
}
pub fn with_sample_format(&self, format: Sample) -> Self {
let mut this = *self;
this.sample_format = format;
this
}
pub fn with_channels(&self, channels: usize) -> Self {
let mut this = *self;
this.channels = channels;
this
}
pub fn matches_format(&self, other: &Self) -> bool {
self.sample_rate == other.sample_rate
&& self.channels == other.channels
&& self.sample_format == other.sample_format
}
/// Returns a version of this AudioInfo with channels clamped for FFmpeg compatibility.
/// FFmpeg channel layouts only support up to 8 channels (7.1 surround).
pub fn for_ffmpeg_output(&self) -> Self {
self.with_max_channels(Self::MAX_AUDIO_CHANNELS)
}
}
pub enum RawVideoFormat {
Bgra,
Mjpeg,
Uyvy,
RawRgb,
Nv12,
Nv21,
Gray,
Gray16,
Yuyv422,
Yuv420p,
Rgba,
Rgb565,
P010,
}
#[derive(Debug, Copy, Clone)]
pub struct VideoInfo {
pub pixel_format: Pixel,
pub width: u32,
pub height: u32,
pub time_base: FFRational,
pub frame_rate: FFRational,
}
impl VideoInfo {
pub fn from_raw(format: RawVideoFormat, width: u32, height: u32, fps: u32) -> Self {
Self {
pixel_format: match format {
RawVideoFormat::Bgra => Pixel::BGRA,
RawVideoFormat::Mjpeg => Pixel::YUVJ422P,
RawVideoFormat::Uyvy => Pixel::UYVY422,
RawVideoFormat::RawRgb => Pixel::RGB24,
RawVideoFormat::Nv12 => Pixel::NV12,
RawVideoFormat::Nv21 => Pixel::NV21,
RawVideoFormat::Gray => Pixel::GRAY8,
RawVideoFormat::Gray16 => Pixel::GRAY16LE,
RawVideoFormat::Yuyv422 => Pixel::YUYV422,
RawVideoFormat::Yuv420p => Pixel::YUV420P,
RawVideoFormat::Rgba => Pixel::RGBA,
RawVideoFormat::Rgb565 => Pixel::RGB565LE,
RawVideoFormat::P010 => Pixel::P010LE,
},
width,
height,
time_base: FFRational(1, 1_000_000),
frame_rate: FFRational(fps as i32, 1),
}
}
pub fn from_raw_ffmpeg(pixel_format: Pixel, width: u32, height: u32, fps: u32) -> Self {
Self {
pixel_format,
width,
height,
time_base: FFRational(1, 1_000_000),
frame_rate: FFRational(fps as i32, 1),
}
}
pub fn fps(&self) -> u32 {
self.frame_rate.0 as u32
}
pub fn scaled(&self, width: u32, fps: u32) -> Self {
let (width, height) = match self.width <= width {
true => (self.width, self.height),
false => {
let new_width = width & !1;
let new_height = (((new_width as f32) * (self.height as f32) / (self.width as f32))
.round() as u32)
& !1;
(new_width, new_height)
}
};
Self {
pixel_format: self.pixel_format,
width,
height,
time_base: self.time_base,
frame_rate: FFRational(fps.try_into().unwrap(), 1),
}
}
pub fn wrap_frame(&self, data: &[u8], timestamp: i64, stride: usize) -> frame::Video {
let mut frame = frame::Video::new(self.pixel_format, self.width, self.height);
frame.set_pts(Some(timestamp));
let frame_stride = frame.stride(0);
let frame_height = self.height as usize;
// Ensure we don't try to copy more data than we have
if frame.stride(0) == self.width as usize {
let copy_len = std::cmp::min(data.len(), frame.data_mut(0).len());
frame.data_mut(0)[0..copy_len].copy_from_slice(&data[0..copy_len]);
} else {
for line in 0..frame_height {
if line * stride >= data.len() {
break; // Stop if we run out of source data
}
let src_start = line * stride;
let src_end = std::cmp::min(src_start + frame_stride, data.len());
if src_end <= src_start {
break; // Stop if we can't get any more source data
}
let dst_start = line * frame_stride;
let dst_end = dst_start + (src_end - src_start);
// Only copy if we have enough destination space
if dst_end <= frame.data_mut(0).len() {
frame.data_mut(0)[dst_start..dst_end]
.copy_from_slice(&data[src_start..src_end]);
}
}
}
frame
}
}
pub fn ensure_even(value: u32) -> u32 {
let adjusted = value - (value % 2);
if adjusted == 0 { 2 } else { adjusted }
}
pub fn ffmpeg_sample_format_for(sample_format: SampleFormat) -> Option<Sample> {
match sample_format {
SampleFormat::U8 => Some(Sample::U8(Type::Packed)),
SampleFormat::I16 => Some(Sample::I16(Type::Packed)),
SampleFormat::I32 => Some(Sample::I32(Type::Packed)),
SampleFormat::I64 => Some(Sample::I64(Type::Packed)),
SampleFormat::F32 => Some(Sample::F32(Type::Packed)),
SampleFormat::F64 => Some(Sample::F64(Type::Packed)),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
mod audio_info {
use super::*;
#[test]
fn wrap_packed_frame() {
let info = AudioInfo::new_raw(Sample::U8(Type::Packed), 2, 4);
let input = &[1, 2, 3, 4, 1, 2, 3, 4];
let frame = info.wrap_frame(input);
assert_eq!(&frame.data(0)[0..input.len()], input);
}
#[test]
fn wrap_planar_frame() {
let info = AudioInfo::new_raw(Sample::U8(Type::Planar), 2, 4);
let input = &[1, 2, 3, 4, 1, 2, 3, 4];
let frame = info.wrap_frame(input);
assert_eq!(frame.planes(), 4);
assert_eq!(&frame.data(0)[0..2], &[1, 1]);
assert_eq!(&frame.data(1)[0..2], &[2, 2]);
assert_eq!(&frame.data(2)[0..2], &[3, 3]);
assert_eq!(&frame.data(3)[0..2], &[4, 4]);
}
#[test]
fn wrap_empty_and_partial_callbacks() {
for format in [Sample::F32(Type::Packed), Sample::F32(Type::Planar)] {
for channels in [1, 2, 6] {
let info = AudioInfo::new_raw(format, 48_000, channels);
let sample_bytes = 4 * channels as usize;
for len in [
0,
1,
3,
sample_bytes - 1,
sample_bytes + 1,
3 * sample_bytes + 3,
] {
let input: Vec<u8> = (0..len).map(|i| i as u8).collect();
let frame = info.wrap_frame(&input);
assert_eq!(
frame.samples(),
len / sample_bytes,
"{format:?} {channels} {len}"
);
}
}
}
let info = AudioInfo::new_raw(Sample::U8(Type::Packed), 2, 2);
let frame = info.wrap_frame(&[1, 2, 3, 4, 5]);
assert_eq!(frame.samples(), 2);
assert_eq!(&frame.data(0)[0..4], &[1, 2, 3, 4]);
}
#[test]
fn packed_channel_limits_preserve_complete_frames_and_sample_bytes() {
for format in [
Sample::U8(Type::Packed),
Sample::I16(Type::Packed),
Sample::I32(Type::Packed),
Sample::F32(Type::Packed),
Sample::F64(Type::Packed),
] {
for channels in [0, 1, 2, 6, 8, 9, 16, 32] {
let info = AudioInfo::new_raw(format, 48_000, channels);
let input_stride = info.sample_size() * usize::from(channels.max(1));
for max in [0, 1, 2, 8, 64] {
let output_channels = usize::from(channels.max(1)).min(max.max(1)).min(8);
let output_stride = info.sample_size() * output_channels;
for samples in [0, 1, 3, 64] {
for tail in [0, input_stride - 1] {
let input = (0..samples * input_stride + tail)
.map(|index| (index * 97 + 31) as u8)
.collect::<Vec<_>>();
let expected = input
.chunks_exact(input_stride)
.flat_map(|frame| frame[..output_stride].iter().copied())
.collect::<Vec<_>>();
let frame = info.wrap_frame_with_max_channels(&input, max);
assert_eq!(frame.samples(), samples);
assert_eq!(frame.channels() as usize, output_channels);
assert_eq!(frame.rate(), 48_000);
assert_eq!(frame.format(), format);
if samples > 0 {
assert_eq!(&frame.data(0)[..expected.len()], expected);
}
}
}
}
}
}
}
#[test]
fn wrap_packed_frame_max_channels() {
let info = AudioInfo::new_raw(Sample::U8(Type::Packed), 2, 4);
let input = &[1, 2, 3, 4, 1, 2, 3, 4];
let frame = info.wrap_frame_with_max_channels(input, 2);
assert_eq!(&frame.data(0)[0..4], &[1, 2, 1, 2]);
}
#[test]
fn wrap_planar_frame_max_channels() {
let info = AudioInfo::new_raw(Sample::U8(Type::Planar), 2, 4);
let input = &[1, 2, 3, 4, 1, 2, 3, 4];
let frame = info.wrap_frame_with_max_channels(input, 2);
assert_eq!(frame.planes(), 2);
assert_eq!(&frame.data(0)[0..2], &[1, 1]);
assert_eq!(&frame.data(1)[0..2], &[2, 2]);
}
#[test]
fn channel_layout_returns_valid_layout_for_supported_counts() {
// Test all supported channel counts (1-8)
for channels in 1..=8u16 {
let info = AudioInfo::new_raw(Sample::F32(Type::Planar), 48000, channels);
// Should not panic and should return a valid layout
let layout = info.channel_layout();
assert!(!layout.is_empty());
}
}
#[test]
fn channel_layout_handles_zero_channels() {
// Zero channels should be clamped to 1 (MONO)
let info = AudioInfo::new_raw(Sample::F32(Type::Planar), 48000, 0);
let layout = info.channel_layout();
assert_eq!(layout, ChannelLayout::MONO);
}
#[test]
fn channel_layout_handles_excessive_channels() {
// More than 8 channels should be clamped to 8 (7.1 surround)
for channels in [9, 10, 16, 32, 64] {
let info = AudioInfo::new_raw(Sample::F32(Type::Planar), 48000, channels);
let layout = info.channel_layout();
assert_eq!(layout, ChannelLayout::_7POINT1);
}
}
#[test]
fn wrap_frame_handles_zero_channels() {
// Zero channels should be treated as mono to avoid division by zero
let info = AudioInfo::new_raw(Sample::U8(Type::Packed), 2, 0);
let input = &[1, 2, 3, 4];
// This should not panic
let frame = info.wrap_frame(input);
// With effective_channels = 1, all input should be copied as mono
assert_eq!(&frame.data(0)[0..input.len()], input);
}
#[test]
fn wrapped_frame_layout_matches_actual_wrapped_frame() {
// Regression for the silent 3/4/5/6-channel mic/system bug: a resampler is
// configured from `wrapped_frame_layout`, while the frames it consumes come
// from `wrap_frame_with_max_channels`. If those two layouts ever diverge,
// `swr_convert_frame` rejects every frame and the track goes silent.
let max = AudioInfo::MAX_AUDIO_CHANNELS as usize;
for channels in 1..=8u16 {
let info = AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, channels);
let data = vec![0u8; 16 * usize::from(channels) * 4];
let frame = info.wrap_frame_with_max_channels(&data, max);
assert_eq!(
frame.channel_layout(),
info.wrapped_frame_layout(max),
"wrapped frame layout must equal the resampler-config layout for {channels} channels"
);
assert_eq!(frame.channels() as usize, info.wrapped_frame_channels(max));
}
}
}
}