Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
use crate::{AudioSampleSource, VoiceProfile, voice_level::VoiceFilter};
use cap_rnnoise::{DELAY_SAMPLES, DenoiseState, FRAME_SIZE};
use std::{collections::VecDeque, ops::Range};
const FRAME: usize = FRAME_SIZE;
pub const VOICE_PREROLL_SAMPLES: usize = 24_000;
pub const VOICE_WINDOW_PADDING_SAMPLES: usize = FRAME * 9;
pub struct VoiceEnhancer {
states: Vec<DenoiseState>,
next_input: usize,
expected_start: Option<usize>,
warm_start: usize,
pending_start: usize,
pending: VecDeque<f32>,
previous: Vec<f32>,
frame: Vec<f32>,
wet: f32,
profile: Option<VoiceProfile>,
leveler: Option<VoiceFilter>,
speech_frames: usize,
}
impl VoiceEnhancer {
pub fn new(channels: u16) -> Self {
let channels = usize::from(channels.clamp(1, 2));
Self {
states: (0..channels).map(|_| DenoiseState::new()).collect(),
next_input: 0,
expected_start: None,
warm_start: 0,
pending_start: 0,
pending: VecDeque::new(),
previous: vec![0.0; DELAY_SAMPLES * channels],
frame: vec![0.0; FRAME * channels],
wet: 0.9,
profile: None,
leveler: None,
speech_frames: 0,
}
}
pub fn with_settings(channels: u16, wet: f32, profile: VoiceProfile) -> Self {
let mut enhancer = Self::new(channels);
enhancer.wet = if wet.is_finite() {
wet.clamp(0.0, 1.0)
} else {
0.9
};
enhancer.profile = Some(profile);
enhancer
}
pub fn speech_frames(&self) -> usize {
self.speech_frames
}
pub fn is_contiguous(&self, start: usize) -> bool {
self.expected_start == Some(start)
|| (self.pending_start..self.pending_start + self.pending.len() / self.states.len())
.contains(&start)
}
pub fn source_range(&self, start: usize, count: usize) -> Range<usize> {
if count == 0 {
return start..start;
}
let first = if self.is_contiguous(start) {
self.next_input
} else {
(start / FRAME * FRAME).saturating_sub(VOICE_PREROLL_SAMPLES)
};
let padding = if self.profile.is_some() { 7 } else { 3 };
let end = start
.saturating_add(count)
.div_ceil(FRAME)
.saturating_add(padding)
.saturating_mul(FRAME);
first..end.max(first)
}
pub fn initial_source_range(start: usize, count: usize) -> Range<usize> {
let first = (start / FRAME * FRAME).saturating_sub(VOICE_PREROLL_SAMPLES);
let end = start
.saturating_add(count)
.div_ceil(FRAME)
.saturating_add(7)
.saturating_mul(FRAME);
first..end.max(first)
}
pub fn render<T: AudioSampleSource>(
&mut self,
source: &T,
start: usize,
count: usize,
) -> VoiceAudio {
let channels = self.states.len();
let count = count.min(source.sample_count().saturating_sub(start));
if count == 0 {
return VoiceAudio {
samples: Vec::new(),
channels: channels as u16,
start,
total_samples: source.sample_count(),
};
}
if count > 0 && !self.is_contiguous(start) {
self.next_input = self.source_range(start, count).start;
self.warm_start = self.next_input;
self.pending_start = self.next_input;
self.pending.clear();
self.previous.fill(0.0);
if self.expected_start.is_some() {
self.states = (0..channels).map(|_| DenoiseState::new()).collect();
}
self.speech_frames = 0;
self.leveler = self.profile.and_then(|profile| {
VoiceFilter::new(channels as u16, &profile.filter(channels as u16))
.map_err(|error| {
tracing::warn!(%error, "Studio Sound leveling is unavailable");
})
.ok()
});
}
while count > 0 && self.pending_start + self.pending.len() / channels < start + count {
self.process_frame(source);
}
let first = (start - self.pending_start) * channels;
let samples = self
.pending
.iter()
.skip(first)
.take(count * channels)
.copied()
.collect();
// Fractional timeline mappings can repeat a source sample without rewinding the decoder.
let history_start = (start + count - 1) / FRAME * FRAME;
let discard = history_start
.saturating_sub(self.pending_start)
.min(self.pending.len() / channels);
drop(self.pending.drain(..discard * channels));
self.pending_start += discard;
self.expected_start = Some(start + count);
VoiceAudio {
samples,
channels: channels as u16,
start,
total_samples: source.sample_count(),
}
}
fn process_frame<T: AudioSampleSource>(&mut self, source: &T) {
let channels = self.states.len();
let mut input = [0.0; FRAME];
let mut output = [0.0; FRAME];
let delayed = (self.next_input / FRAME % 2) * FRAME * channels;
let mut speech = false;
for (channel, state) in self.states.iter_mut().enumerate() {
for (index, sample) in input.iter_mut().enumerate() {
let value = source
.sample((self.next_input + index) * channels + channel)
.copied()
.unwrap_or(0.0);
*sample = if value.is_finite() {
value.clamp(-1.0, 1.0) * 32_768.0
} else {
0.0
};
}
speech |= state.process_frame(&mut output, &input) > 0.6;
for (index, (&clean, &raw)) in output.iter().zip(&input).enumerate() {
let index = index * channels + channel;
self.frame[index] =
clean / 32_768.0 * self.wet + self.previous[delayed + index] * (1.0 - self.wet);
self.previous[delayed + index] = raw / 32_768.0;
}
}
self.speech_frames += usize::from(speech);
if self.next_input >= self.warm_start + DELAY_SAMPLES {
if let Some(leveler) = &mut self.leveler {
if let Err(error) = leveler.process(&self.frame, &mut self.pending) {
tracing::warn!(%error, "Studio Sound leveling failed");
self.leveler = None;
self.pending.extend(&self.frame);
}
} else {
self.pending.extend(&self.frame);
}
}
self.next_input += FRAME;
}
}
pub struct VoiceAudio {
samples: Vec<f32>,
channels: u16,
start: usize,
total_samples: usize,
}
impl VoiceAudio {
pub fn samples(&self) -> &[f32] {
&self.samples
}
}
impl AudioSampleSource for VoiceAudio {
fn channels(&self) -> u16 {
self.channels
}
fn sample_count(&self) -> usize {
self.total_samples
}
fn sample(&self, index: usize) -> Option<&f32> {
index
.checked_sub(self.start * usize::from(self.channels))
.and_then(|index| self.samples.get(index))
}
fn sample_slice(&self, range: Range<usize>) -> Option<&[f32]> {
let offset = self.start * usize::from(self.channels);
self.samples
.get(range.start.checked_sub(offset)?..range.end.checked_sub(offset)?)
}
}
pub enum VoiceSource<'a, T> {
Original(&'a T),
Enhanced(&'a VoiceAudio),
}
impl<T: AudioSampleSource> AudioSampleSource for VoiceSource<'_, T> {
fn channels(&self) -> u16 {
match self {
Self::Original(source) => source.channels(),
Self::Enhanced(source) => source.channels(),
}
}
fn sample_count(&self) -> usize {
match self {
Self::Original(source) => source.sample_count(),
Self::Enhanced(source) => source.sample_count(),
}
}
fn sample(&self, index: usize) -> Option<&f32> {
match self {
Self::Original(source) => source.sample(index),
Self::Enhanced(source) => source.sample(index),
}
}
fn sample_slice(&self, range: Range<usize>) -> Option<&[f32]> {
match self {
Self::Original(source) => source.sample_slice(range),
Self::Enhanced(source) => source.sample_slice(range),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
struct Samples {
data: Vec<f32>,
channels: u16,
}
impl AudioSampleSource for Samples {
fn channels(&self) -> u16 {
self.channels
}
fn sample_count(&self) -> usize {
self.data.len() / usize::from(self.channels)
}
fn sample(&self, index: usize) -> Option<&f32> {
self.data.get(index)
}
}
fn signal(frames: usize, channels: u16) -> Samples {
let mut seed = 42_u32;
Samples {
channels,
data: (0..frames * usize::from(channels))
.map(|_| {
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
(seed as f64 / u32::MAX as f64 * 0.16 - 0.08) as f32
})
.collect(),
}
}
/// Guard sentinel A09 (scripts/verify/guards.toml): a render that does
/// not continue the previous one must sound exactly like a render by a
/// new enhancer (the reset in `render`, and the editor cache that never
/// reuses an enhancer across a seek).
#[test]
fn guard_a09_render_after_a_seek_equals_a_fresh_enhancer() {
ffmpeg::init().unwrap();
// Voiced, syllable-modulated harmonics plus noise, so the render
// is not silence.
let mut source = signal(96_000 * 2, 2);
for (index, sample) in source.data.iter_mut().enumerate() {
let t = (index / 2) as f32 / 48_000.0;
let pitch = 140.0 + 30.0 * (t * 1.3).sin();
let voice: f32 = (1..6)
.map(|h| (std::f32::consts::TAU * pitch * h as f32 * t).sin() / h as f32)
.sum();
let syllables = (std::f32::consts::TAU * 4.0 * t).sin().max(0.0);
*sample += 0.25 * voice * syllables;
}
let mut reused = VoiceEnhancer::with_settings(2, 1.0, VoiceProfile::default());
let _ = reused.render(&source, 0, 9_600);
let _ = reused.render(&source, 9_600, 9_600);
let after_seek = reused.render(&source, 120_000, 9_600);
let fresh = VoiceEnhancer::with_settings(2, 1.0, VoiceProfile::default())
.render(&source, 120_000, 9_600);
assert!(
fresh.samples().iter().any(|v| v.abs() > 0.01),
"silent render"
);
assert_eq!(after_seek.samples(), fresh.samples());
}
#[test]
fn initial_state_reuse_matches_forced_reset_after_empty_reads_and_seeks() {
ffmpeg::init().unwrap();
for channels in [1, 2] {
let source = signal(96_001, channels);
for profile in [false, true] {
let create = || {
if profile {
VoiceEnhancer::with_settings(channels, 0.9, VoiceProfile::default())
} else {
VoiceEnhancer::new(channels)
}
};
for start in [0, 27_017] {
for count in [1, 4_096] {
let mut actual = create();
let mut reference = create();
reference.expected_start = Some(usize::MAX);
for empty_start in [0, 50_000] {
assert!(actual.render(&source, empty_start, 0).samples().is_empty());
assert!(
reference
.render(&source, empty_start, 0)
.samples()
.is_empty()
);
}
let first = actual.render(&source, start, count);
let expected = reference.render(&source, start, count);
assert_eq!(
first
.samples()
.iter()
.map(|value| value.to_bits())
.collect::<Vec<_>>(),
expected
.samples()
.iter()
.map(|value| value.to_bits())
.collect::<Vec<_>>()
);
assert_eq!(actual.speech_frames(), reference.speech_frames());
let seek = actual.render(&source, 48_017, 997);
let mut fresh = create();
fresh.expected_start = Some(usize::MAX);
let expected = fresh.render(&source, 48_017, 997);
assert_eq!(
seek.samples()
.iter()
.map(|value| value.to_bits())
.collect::<Vec<_>>(),
expected
.samples()
.iter()
.map(|value| value.to_bits())
.collect::<Vec<_>>()
);
assert_eq!(actual.speech_frames(), fresh.speech_frames());
}
}
}
}
}
#[test]
fn leveling_preserves_chunk_boundaries_timing_and_peak_headroom() {
ffmpeg::init().unwrap();
for channels in [1, 2] {
let mut source = signal(96_137, channels);
for (index, sample) in source.data.iter_mut().enumerate() {
let position = index / usize::from(channels);
*sample = if (8_000..80_000).contains(&position) {
(position as f32 * 0.037).sin() * 0.05
} else {
0.0
};
}
let profile = VoiceProfile {
gain_db: 12.0,
makeup_db: 2.0,
};
let expected = VoiceEnhancer::with_settings(channels, 0.9, profile).render(
&source,
0,
source.sample_count(),
);
let mut enhancer = VoiceEnhancer::with_settings(channels, 0.9, profile);
let mut actual = Vec::new();
let mut cursor = 0;
while cursor < source.sample_count() {
let audio = enhancer.render(&source, cursor, cursor % 5_007 + 1);
cursor += audio.samples.len() / usize::from(channels);
actual.extend(audio.samples);
assert!(
enhancer.pending.len() < VOICE_WINDOW_PADDING_SAMPLES * usize::from(channels)
);
}
assert_eq!(actual, expected.samples);
assert!(
actual
.iter()
.all(|value| value.is_finite() && value.abs() < 0.86)
);
assert!(actual.iter().any(|value| value.abs() > 0.05));
for remainder in 0..FRAME {
let range = enhancer.source_range(48_000 + remainder, 4_096);
assert!(
range.len() <= 4_096 + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES
);
}
let mut impulse = Samples {
data: vec![0.0; 12_017 * usize::from(channels)],
channels,
};
impulse.data[5_333 * usize::from(channels)] = 0.8;
let audio = VoiceEnhancer::with_settings(channels, 0.0, VoiceProfile::default())
.render(&impulse, 0, impulse.sample_count());
let peak = audio
.samples
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.abs().total_cmp(&b.abs()))
.unwrap()
.0;
assert_eq!(peak / usize::from(channels), 5_333);
}
}
#[test]
fn arbitrary_blocks_match_whole_processing_including_stereo_and_tail() {
for channels in [1, 2] {
let source = signal(20_137, channels);
let expected = VoiceEnhancer::new(channels).render(&source, 0, source.sample_count());
let mut enhancer = VoiceEnhancer::new(channels);
let mut actual = Vec::new();
let mut cursor = 0;
for count in [1, 479, 481, 4_096, 7, 11_999, 3_074] {
let audio = enhancer.render(&source, cursor, count);
cursor += audio.samples.len() / usize::from(channels);
actual.extend(audio.samples);
}
assert_eq!(cursor, source.sample_count());
assert_eq!(actual, expected.samples);
assert!(enhancer.render(&source, cursor, 512).samples.is_empty());
}
}
#[test]
fn seeks_prime_from_a_bounded_source_window() {
let source = signal(120_001, 1);
let mut enhancer = VoiceEnhancer::new(1);
enhancer.render(&source, 0, 4_096);
for start in [96_177, 13, 60_050] {
let range = enhancer.source_range(start, 4096);
assert!(range.len() <= 4096 + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES);
let actual = enhancer.render(&source, start, 4096);
let expected = VoiceEnhancer::new(1).render(&source, start, 4096);
assert_eq!(actual.samples, expected.samples);
}
}
#[test]
fn every_frame_alignment_fits_the_bounded_source_window() {
let enhancer = VoiceEnhancer::new(1);
for remainder in 0..FRAME {
let range = enhancer.source_range(48_000 + remainder, 4_096);
assert!(range.len() <= 4_096 + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES);
}
}
#[test]
fn silence_short_tracks_and_non_finite_samples_stay_safe() {
for frames in [0, 1, 479, 480, 481, 997] {
let source = Samples {
data: vec![0.0; frames],
channels: 1,
};
let output = VoiceEnhancer::new(1).render(&source, 0, frames + 480);
assert_eq!(output.samples, source.data);
}
let source = Samples {
data: vec![f32::NAN, f32::INFINITY, f32::NEG_INFINITY],
channels: 1,
};
let output = VoiceEnhancer::new(1).render(&source, 0, 3);
assert_eq!(output.samples, vec![0.0; 3]);
}
#[test]
fn lookahead_compensates_the_filter_delay() {
let mut source = Samples {
data: vec![0.0; 12_017],
channels: 1,
};
source.data[5_333] = 0.8;
let output = VoiceEnhancer::new(1).render(&source, 0, source.sample_count());
let peak = output
.samples
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.abs().total_cmp(&b.abs()))
.unwrap()
.0;
assert_eq!(peak, 5_333);
assert_eq!(output.samples.len(), source.data.len());
}
#[test]
fn fan_noise_is_reduced_without_clipping_or_accumulated_buffers() {
let mut source = signal(96_000, 1);
let mut previous = 0.0;
for sample in &mut source.data {
previous = previous * 0.95 + *sample * 0.05;
*sample = previous;
}
let mut enhancer = VoiceEnhancer::new(1);
let output = enhancer.render(&source, 0, source.sample_count());
let power = |samples: &[f32]| samples.iter().map(|sample| sample * sample).sum::<f32>();
assert!(power(&output.samples[48_000..]) < power(&source.data[48_000..]) * 0.1);
assert!(
output
.samples
.iter()
.all(|sample| sample.is_finite() && sample.abs() <= 1.0)
);
assert_eq!(enhancer.states.len(), 1);
assert_eq!(enhancer.frame.len(), FRAME);
assert_eq!(enhancer.previous.len(), DELAY_SAMPLES);
}
}