Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
use crate::{AudioRenderer, SegmentMedia};
use cap_audio::{
AudioData, AudioRendererTrack, AudioSampleSource, AudioStream, AudioStreamError, ChunkRead,
StereoMode, VOICE_PREROLL_SAMPLES, VOICE_PROFILE_SAMPLES, VOICE_WINDOW_PADDING_SAMPLES,
VoiceAudio, VoiceEnhancer, VoiceProfile, VoiceSource,
};
use cap_project::{ClipOffsets, ProjectConfiguration, RecordingMeta, StudioRecordingMeta};
use std::{
collections::HashMap,
fmt,
ops::Range,
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
pub(crate) const EXPORT_AUDIO_BLOCK_SAMPLES: usize = 4_096;
const MAX_PARALLEL_AUDIO_SOURCES: usize = 4;
const MAX_ENHANCED_WINDOW_SAMPLES: usize =
EXPORT_AUDIO_BLOCK_SAMPLES + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES;
const STUDIO_SOUND_HISTORY_SAMPLES: usize = VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES;
#[derive(Clone, Debug)]
pub enum ExportAudioError {
Cancelled,
Source {
source_index: usize,
path: PathBuf,
source: AudioStreamError,
},
InvalidWindow,
Sink(String),
Worker {
source_index: usize,
message: String,
},
}
impl fmt::Display for ExportAudioError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Cancelled => f.write_str("Export cancelled"),
Self::Source { path, source, .. } => write!(f, "Audio at {}: {source}", path.display()),
Self::InvalidWindow => f.write_str("Invalid sequential export audio window"),
Self::Sink(error) => f.write_str(error),
Self::Worker { message, .. } => write!(f, "Audio source worker failed: {message}"),
}
}
}
impl std::error::Error for ExportAudioError {}
impl ExportAudioError {
pub fn source_index(&self) -> Option<usize> {
match self {
Self::Source { source_index, .. } | Self::Worker { source_index, .. } => {
Some(*source_index)
}
_ => None,
}
}
fn is_cancelled(&self) -> bool {
matches!(self, Self::Cancelled)
|| matches!(self, Self::Source { source, .. } if source.is_cancelled())
}
}
fn source_worker_panic(
source_index: usize,
panic: Box<dyn std::any::Any + Send>,
) -> ExportAudioError {
ExportAudioError::Worker {
source_index,
message: panic
.downcast_ref::<String>()
.cloned()
.or_else(|| {
panic
.downcast_ref::<&str>()
.map(|message| (*message).into())
})
.unwrap_or_else(|| "worker panicked".into()),
}
}
fn run_source_job<T, R>(
source_index: usize,
input: &mut T,
operation: &impl Fn(usize, &mut T) -> Result<R, ExportAudioError>,
) -> Result<R, ExportAudioError> {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
operation(source_index, input)
}))
.unwrap_or_else(|panic| Err(source_worker_panic(source_index, panic)))
}
fn run_source_jobs<T: Send, R: Send>(
inputs: &mut [(usize, T)],
abort: &AtomicBool,
operation: impl Fn(usize, &mut T) -> Result<R, ExportAudioError> + Sync,
) -> Result<Vec<R>, ExportAudioError> {
if inputs.len() <= 1 {
let result = inputs
.iter_mut()
.map(|(index, input)| run_source_job(*index, input, &operation))
.collect::<Result<_, _>>();
if result.as_ref().is_err_and(|error| !error.is_cancelled()) {
abort.store(true, Ordering::Relaxed);
}
return result;
}
let mut results = Vec::with_capacity(inputs.len());
for batch in inputs.chunks_mut(MAX_PARALLEL_AUDIO_SOURCES) {
let joined = std::thread::scope(|scope| {
let operation = &operation;
let handles = batch
.iter_mut()
.map(|(index, input)| {
let index = *index;
std::thread::Builder::new()
.spawn_scoped(scope, move || run_source_job(index, input, operation))
.ok()
})
.collect::<Vec<_>>();
let mut earlier_source_not_started = false;
handles
.into_iter()
.map(|handle| {
let Some(handle) = handle else {
earlier_source_not_started = true;
return None;
};
let result = handle.join();
if !earlier_source_not_started
&& match &result {
Ok(Err(error)) => !error.is_cancelled(),
Err(_) => true,
_ => false,
}
{
abort.store(true, Ordering::Relaxed);
}
Some(result)
})
.collect::<Vec<_>>()
});
for ((index, input), result) in batch.iter_mut().zip(joined) {
let result = match result {
Some(Ok(result)) => result,
Some(Err(panic)) => Err(source_worker_panic(*index, panic)),
None => run_source_job(*index, input, &operation),
};
if result.as_ref().is_err_and(|error| !error.is_cancelled()) {
abort.store(true, Ordering::Relaxed);
}
results.push(result);
}
}
let mut values = Vec::with_capacity(results.len());
let mut first_error: Option<ExportAudioError> = None;
for result in results {
match result {
Ok(value) => values.push(value),
Err(error) => {
if first_error
.as_ref()
.is_none_or(|previous| previous.is_cancelled() && !error.is_cancelled())
{
first_error = Some(error);
}
}
}
}
match first_error {
Some(error) => Err(error),
None => Ok(values),
}
}
pub struct ExportAudioRenderer {
renderer: AudioRenderer,
sources: ExportAudioSources,
failure: Option<ExportAudioError>,
}
pub struct ExportAudioPreparation {
sources: ExportAudioSources,
}
impl ExportAudioPreparation {
fn segment_count(meta: &StudioRecordingMeta) -> usize {
match meta {
StudioRecordingMeta::SingleSegment { .. } => 1,
StudioRecordingMeta::MultipleSegments { inner, .. } => inner.segments.len(),
}
}
pub fn open(
recording: &RecordingMeta,
meta: &StudioRecordingMeta,
cancellation: Arc<AtomicBool>,
abort: Arc<AtomicBool>,
) -> Result<Self, ExportAudioError> {
let mut inputs = Vec::new();
match meta {
StudioRecordingMeta::SingleSegment { segment } => {
if let Some(audio) = &segment.audio {
inputs.push((0, recording.path(&audio.path), true));
}
}
StudioRecordingMeta::MultipleSegments { inner, .. } => {
for (index, segment) in inner.segments.iter().enumerate() {
if let Some(audio) = &segment.mic {
inputs.push((index, recording.path(&audio.path), true));
}
if let Some(audio) = segment.usable_system_audio() {
inputs.push((index, recording.path(&audio.path), false));
}
}
}
}
let mut inputs = inputs.into_iter().enumerate().collect::<Vec<_>>();
let opened = run_source_jobs(&mut inputs, &abort, |source_index, (index, path, mic)| {
ExportAudioTrack::open(path, source_index, *mic, 0.0, &cancellation, &abort)
.map(|track| (*index, track))
})?;
let mut tracks = (0..Self::segment_count(meta))
.map(|_| Vec::new())
.collect::<Vec<_>>();
for (index, track) in opened {
tracks[index].push(track);
}
Ok(Self {
sources: ExportAudioSources {
tracks,
cancellation,
abort,
},
})
}
pub fn finish(
self,
segments: &[SegmentMedia],
) -> Result<ExportAudioRenderer, ExportAudioError> {
self.finish_with_timing_repair(segments.iter().map(|segment| segment.audio_timing_repair))
}
fn finish_with_timing_repair(
mut self,
timing_repair: impl ExactSizeIterator<Item = crate::editor_instance::SegmentAudioTimingRepair>,
) -> Result<ExportAudioRenderer, ExportAudioError> {
if timing_repair.len() != self.sources.tracks.len() {
return Err(ExportAudioError::InvalidWindow);
}
for (tracks, repair) in self.sources.tracks.iter_mut().zip(timing_repair) {
for track in tracks {
track.timing_offset_secs = if track.mic {
repair.mic_offset_secs
} else {
repair.system_audio_offset_secs
};
}
}
Ok(ExportAudioRenderer {
renderer: AudioRenderer::new(Vec::new()),
sources: self.sources,
failure: None,
})
}
}
pub struct ExportAudioValidation {
sources: ExportAudioSources,
}
impl ExportAudioValidation {
pub fn validate_to_end(mut self) -> Result<(), ExportAudioError> {
self.sources.validate_to_end()
}
}
impl ExportAudioRenderer {
pub fn eligible(project: &ProjectConfiguration, meta: &StudioRecordingMeta) -> bool {
let source_count = match meta {
StudioRecordingMeta::SingleSegment { segment } => usize::from(segment.audio.is_some()),
StudioRecordingMeta::MultipleSegments { inner, .. } => inner
.segments
.iter()
.map(|segment| {
usize::from(segment.mic.is_some())
+ usize::from(segment.usable_system_audio().is_some())
})
.sum(),
};
Self::eligible_sources(project, source_count)
}
fn eligible_sources(project: &ProjectConfiguration, source_count: usize) -> bool {
if !(1..=MAX_PARALLEL_AUDIO_SOURCES).contains(&source_count)
|| project.clips.iter().any(|clip| {
[clip.offsets.mic, clip.offsets.system_audio]
.iter()
.any(|offset| {
!offset.is_finite() || (offset * 48_000.0).abs() >= isize::MAX as f32 / 2.0
})
})
{
return false;
}
let Some(timeline) = &project.timeline else {
return true;
};
if !timeline.audio_segments.is_empty() {
return false;
}
let holds = timeline.hold_windows();
if holds
.iter()
.any(|(start, end)| !start.is_finite() || !end.is_finite() || *start < 0.0)
{
return false;
}
let mut ends = HashMap::new();
let mut duration = holds.iter().map(|(start, end)| end - start).sum::<f64>();
if !duration.is_finite() || duration * 48_000.0 >= isize::MAX as f64 / 2.0 {
return false;
}
for segment in &timeline.segments {
if segment.timescale != 1.0
|| !segment.start.is_finite()
|| !segment.end.is_finite()
|| segment.start < 0.0
|| segment.end <= segment.start
|| segment.end * 48_000.0 >= isize::MAX as f64 / 2.0
{
return false;
}
duration += segment.duration();
if !duration.is_finite() || duration * 48_000.0 >= isize::MAX as f64 / 2.0 {
return false;
}
if let Some(previous_end) = ends.insert(segment.recording_clip, segment.end)
&& (!timeline.transitions.is_empty() || segment.start < previous_end)
{
return false;
}
}
true
}
pub fn open(
recording: &RecordingMeta,
meta: &StudioRecordingMeta,
segments: &[SegmentMedia],
cancellation: Arc<AtomicBool>,
abort: Arc<AtomicBool>,
) -> Result<Self, ExportAudioError> {
if segments.len() != ExportAudioPreparation::segment_count(meta) {
return Err(ExportAudioError::InvalidWindow);
}
ExportAudioPreparation::open(recording, meta, cancellation, abort)?.finish(segments)
}
pub fn take_unused_sources(
&mut self,
project: &ProjectConfiguration,
) -> Option<ExportAudioValidation> {
let timeline = project.timeline.as_ref()?;
if self.sources.tracks.iter().flatten().count() > MAX_PARALLEL_AUDIO_SOURCES {
return None;
}
let mut unused = Vec::new();
for (clip_index, tracks) in self.sources.tracks.iter_mut().enumerate() {
if !tracks.is_empty()
&& !timeline
.segments
.iter()
.any(|segment| segment.recording_clip as usize == clip_index)
{
unused.push(std::mem::take(tracks));
}
}
(!unused.is_empty()).then(|| ExportAudioValidation {
sources: ExportAudioSources {
tracks: unused,
cancellation: self.sources.cancellation.clone(),
abort: self.sources.abort.clone(),
},
})
}
pub fn render_chunks(
&mut self,
samples: usize,
project: &ProjectConfiguration,
emit: impl FnMut(usize, &[f32]) -> Result<(), ExportAudioError>,
) -> Result<Option<usize>, ExportAudioError> {
if let Some(error) = &self.failure {
return Err(error.clone());
}
let result = self
.renderer
.render_export_chunks(&mut self.sources, samples, project, emit);
if let Err(error) = &result {
self.failure = Some(error.clone());
}
result
}
pub fn validate_to_end(&mut self) -> Result<(), ExportAudioError> {
if let Some(error) = &self.failure {
return Err(error.clone());
}
let result = self.sources.validate_to_end();
if let Err(error) = &result {
self.failure = Some(error.clone());
}
result
}
}
pub(crate) struct ExportAudioSources {
tracks: Vec<Vec<ExportAudioTrack>>,
cancellation: Arc<AtomicBool>,
abort: Arc<AtomicBool>,
}
impl ExportAudioSources {
pub(crate) fn check_cancelled(&self) -> Result<(), ExportAudioError> {
if self.cancellation.load(Ordering::Relaxed) || self.abort.load(Ordering::Relaxed) {
Err(ExportAudioError::Cancelled)
} else {
Ok(())
}
}
fn validate_to_end(&mut self) -> Result<(), ExportAudioError> {
self.check_cancelled()?;
let mut tracks = self
.tracks
.iter_mut()
.flatten()
.map(|track| (track.source_index, track))
.collect::<Vec<_>>();
run_source_jobs(&mut tracks, &self.abort, |_, track| {
track
.source
.validate_to_end()
.map_err(|source| ExportAudioError::Source {
source_index: track.source_index,
path: track.path.clone(),
source,
})?;
track.samples.clear();
Ok(())
})?;
self.check_cancelled()
}
pub(crate) fn render(
&mut self,
project: &ProjectConfiguration,
clip_index: u32,
cursor: usize,
samples: usize,
output: &mut [f32],
) -> Result<usize, ExportAudioError> {
self.check_cancelled()?;
let Some(tracks) = self.tracks.get_mut(clip_index as usize) else {
return Ok(0);
};
let offsets = project
.clips
.iter()
.find(|clip| clip.index == clip_index)
.map(|clip| clip.offsets)
.unwrap_or_default();
for track in tracks.iter_mut() {
let start = cursor as i128 + track.offset(&offsets) as i128;
let end = start + samples as i128;
let improve = track.mic
&& project.audio.improve
&& !project.audio.mute
&& project.audio.mic_volume_db > -30.0
&& (1..=2).contains(&track.source.channels());
if improve {
let start =
usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;
let end =
usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;
if track.eof.is_some_and(|eof| start >= eof) {
track.enhanced = None;
continue;
}
if track.isolation != project.audio.isolation {
track.enhancer = None;
track.isolation = project.audio.isolation;
}
let profile = *track.voice_profile.get_or_insert_with(|| {
AudioData::from_file_range(&track.path, 0, VOICE_PROFILE_SAMPLES)
.map(|audio| VoiceProfile::analyze(&audio))
.unwrap_or_else(|error| {
tracing::warn!(%error, "Could not estimate export voice level");
VoiceProfile::default()
})
});
let mut enhancer = track.enhancer.take().unwrap_or_else(|| {
VoiceEnhancer::with_settings(
track.source.channels(),
project.audio.isolation.strength(),
profile,
)
});
let range = enhancer.source_range(start, end.saturating_sub(start));
track.prepare(
range.start as i128,
range.end as i128,
STUDIO_SOUND_HISTORY_SAMPLES,
)?;
track.enhanced =
Some(enhancer.render(&track.view(), start, end.saturating_sub(start)));
track.enhancer = Some(enhancer);
} else {
track.prepare(start, end, 0)?;
track.enhanced = None;
track.enhancer = None;
}
}
let views = tracks.iter().map(|track| track.view()).collect::<Vec<_>>();
let sources = tracks
.iter()
.zip(&views)
.map(|(track, view)| {
track
.enhanced
.as_ref()
.map_or(VoiceSource::Original(view), VoiceSource::Enhanced)
})
.collect::<Vec<_>>();
let max_samples = tracks
.iter()
.map(|track| (track.available_end as isize - track.offset(&offsets)).max(0) as usize)
.max()
.unwrap_or(0);
if cursor >= max_samples {
return Ok(0);
}
let tracks = tracks
.iter()
.zip(&sources)
.map(|(track, view)| {
let gain = if track.mic {
project.audio.mic_volume_db
} else {
project.audio.system_volume_db
};
AudioRendererTrack {
data: view,
gain: if project.audio.mute || gain < -30.0 {
f32::NEG_INFINITY
} else {
gain
},
stereo_mode: if !track.mic {
StereoMode::Stereo
} else {
match project.audio.mic_stereo_mode {
cap_project::StereoMode::Stereo => StereoMode::Stereo,
cap_project::StereoMode::MonoL => StereoMode::MonoL,
cap_project::StereoMode::MonoR => StereoMode::MonoR,
}
},
offset: track.offset(&offsets),
}
})
.collect::<Vec<_>>();
Ok(cap_audio::render_audio(
&tracks,
cursor,
samples.min(max_samples - cursor),
0,
output,
))
}
}
struct ExportAudioTrack {
source: AudioStream,
source_index: usize,
path: PathBuf,
mic: bool,
timing_offset_secs: f32,
samples: Vec<f32>,
source_start: usize,
available_end: usize,
eof: Option<usize>,
enhancer: Option<VoiceEnhancer>,
enhanced: Option<VoiceAudio>,
voice_profile: Option<VoiceProfile>,
isolation: cap_project::VoiceIsolation,
}
impl ExportAudioTrack {
fn open(
path: &Path,
source_index: usize,
mic: bool,
timing_offset_secs: f32,
cancellation: &Arc<AtomicBool>,
abort: &Arc<AtomicBool>,
) -> Result<Self, ExportAudioError> {
let source = AudioStream::open_with_abort(path, cancellation.clone(), abort.clone())
.map_err(|source| ExportAudioError::Source {
source_index,
path: path.to_path_buf(),
source,
})?;
Ok(Self {
source,
source_index,
path: path.to_path_buf(),
mic,
timing_offset_secs,
samples: Vec::new(),
source_start: 0,
available_end: 0,
eof: None,
enhancer: None,
enhanced: None,
voice_profile: None,
isolation: cap_project::VoiceIsolation::default(),
})
}
fn offset(&self, offsets: &ClipOffsets) -> isize {
let offset = if self.mic {
offsets.mic
} else {
offsets.system_audio
};
((offset + self.timing_offset_secs) * AudioRenderer::SAMPLE_RATE as f32).round() as isize
}
fn prepare(&mut self, start: i128, end: i128, history: usize) -> Result<(), ExportAudioError> {
let start = usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;
let end = usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;
if end < start || end - start > MAX_ENHANCED_WINDOW_SAMPLES || start < self.source_start {
return Err(ExportAudioError::InvalidWindow);
}
let channels = self.source.channels() as usize;
let retained_start = start.saturating_sub(history).max(self.source_start);
let discard = retained_start
.saturating_sub(self.source_start)
.min(self.samples.len() / channels)
* channels;
self.samples.copy_within(discard.., 0);
self.samples.truncate(self.samples.len() - discard);
self.source_start = retained_start;
while self.eof.is_none() && self.source.position() < end as u64 {
let wanted = (end as u64 - self.source.position()).min(12_000) as usize;
match self
.source
.read_chunk(wanted)
.map_err(|source| ExportAudioError::Source {
source_index: self.source_index,
path: self.path.clone(),
source,
})? {
ChunkRead::Chunk(chunk) => {
let chunk_start = usize::try_from(chunk.source_start_sample)
.map_err(|_| ExportAudioError::InvalidWindow)?;
let discard = retained_start
.saturating_sub(chunk_start)
.min(chunk.samples.len() / channels)
* channels;
self.samples.extend_from_slice(&chunk.samples[discard..]);
}
ChunkRead::Eof { next_sample } => {
self.eof = Some(
usize::try_from(next_sample)
.map_err(|_| ExportAudioError::InvalidWindow)?,
);
}
}
}
self.available_end = self.eof.unwrap_or(end.max(self.source.position() as usize));
Ok(())
}
fn view(&self) -> ExportAudioView<'_> {
ExportAudioView {
samples: &self.samples,
channels: self.source.channels(),
source_start: self.source_start,
available_end: self.available_end,
}
}
}
struct ExportAudioView<'a> {
samples: &'a [f32],
channels: u16,
source_start: usize,
available_end: usize,
}
impl AudioSampleSource for ExportAudioView<'_> {
fn channels(&self) -> u16 {
self.channels
}
fn sample_count(&self) -> usize {
self.available_end
}
fn sample(&self, index: usize) -> Option<&f32> {
index
.checked_sub(self.source_start * self.channels as usize)
.and_then(|index| self.samples.get(index))
}
fn sample_slice(&self, range: Range<usize>) -> Option<&[f32]> {
let base = self.source_start * self.channels as usize;
self.samples
.get(range.start.checked_sub(base)?..range.end.checked_sub(base)?)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::audio::{AudioSegment, AudioSegmentTrack};
use cap_audio::AudioData;
use cap_project::{
ClipConfiguration, ClipSpeedAudioMode, ClipTransition, ClipTransitionType,
TimelineConfiguration, TimelineSegment,
};
#[test]
fn source_jobs_are_bounded_joined_and_ordered() {
use std::{
sync::{Condvar, Mutex, atomic::AtomicUsize},
time::Duration,
};
let active = AtomicUsize::new(0);
let peak = AtomicUsize::new(0);
let completed = AtomicUsize::new(0);
let arrived = Mutex::new(0);
let changed = Condvar::new();
let abort = AtomicBool::new(false);
let mut inputs = (0..9).map(|value| (value, value)).collect::<Vec<_>>();
let results = run_source_jobs(&mut inputs, &abort, |_, input| {
let count = active.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(count, Ordering::SeqCst);
if *input < 8 {
let mut count = arrived.lock().unwrap();
*count += 1;
changed.notify_all();
let (_guard, timed_out) = changed
.wait_timeout_while(count, Duration::from_secs(2), |count| {
*count
< (*input / MAX_PARALLEL_AUDIO_SOURCES + 1) * MAX_PARALLEL_AUDIO_SOURCES
})
.unwrap();
assert!(!timed_out.timed_out());
}
active.fetch_sub(1, Ordering::SeqCst);
completed.fetch_add(1, Ordering::SeqCst);
Ok(*input)
})
.unwrap();
assert_eq!(results, (0..9).collect::<Vec<_>>());
assert_eq!(peak.load(Ordering::SeqCst), MAX_PARALLEL_AUDIO_SOURCES);
assert_eq!(active.load(Ordering::SeqCst), 0);
assert_eq!(completed.load(Ordering::SeqCst), inputs.len());
assert!(!abort.load(Ordering::Relaxed));
}
#[test]
fn source_jobs_preserve_first_real_error_and_join_later_sources() {
use std::sync::atomic::AtomicUsize;
let completed = AtomicUsize::new(0);
let abort = AtomicBool::new(false);
let mut inputs = [(0, 0), (1, 1), (2, 2), (3, 3)];
let error = run_source_jobs(&mut inputs, &abort, |_, input| {
completed.fetch_add(1, Ordering::SeqCst);
match input {
0 => Err(ExportAudioError::Cancelled),
1 => Err(ExportAudioError::Sink("first source error".into())),
2 => Err(ExportAudioError::Sink("later source error".into())),
_ => Ok(()),
}
})
.unwrap_err();
assert_eq!(error.to_string(), "first source error");
assert_eq!(completed.load(Ordering::SeqCst), inputs.len());
assert!(abort.load(Ordering::Relaxed));
}
#[test]
fn source_worker_panic_becomes_error_after_other_sources_join() {
use std::sync::atomic::AtomicUsize;
let completed = AtomicUsize::new(0);
let abort = AtomicBool::new(false);
let mut inputs = [(0, 0), (1, 1), (2, 2), (3, 3)];
let error = run_source_jobs(&mut inputs, &abort, |_, input| {
if *input == 1 {
panic!("source worker test");
}
completed.fetch_add(1, Ordering::SeqCst);
Ok(())
})
.unwrap_err();
assert_eq!(
error.to_string(),
"Audio source worker failed: source worker test"
);
assert_eq!(completed.load(Ordering::SeqCst), inputs.len() - 1);
assert_eq!(error.source_index(), Some(1));
assert!(abort.load(Ordering::Relaxed));
}
#[test]
fn failed_source_aborts_and_joins_waiting_peer_without_user_cancellation() {
use std::time::{Duration, Instant};
let user_cancellation = AtomicBool::new(false);
let abort = AtomicBool::new(false);
let peer_started = AtomicBool::new(false);
let peer_observed_abort = AtomicBool::new(false);
let peer_finished = AtomicBool::new(false);
let error = run_source_jobs(&mut [(0, 0), (1, 1)], &abort, |_, input| {
let started = Instant::now();
if *input == 0 {
while !peer_started.load(Ordering::Acquire)
&& started.elapsed() < Duration::from_secs(2)
{
std::thread::yield_now();
}
assert!(peer_started.load(Ordering::Acquire));
return Err(ExportAudioError::Sink("first source failed".into()));
}
peer_started.store(true, Ordering::Release);
while !abort.load(Ordering::Relaxed)
&& !user_cancellation.load(Ordering::Relaxed)
&& started.elapsed() < Duration::from_secs(2)
{
std::thread::yield_now();
}
peer_observed_abort.store(abort.load(Ordering::Relaxed), Ordering::Relaxed);
peer_finished.store(true, Ordering::Release);
Err::<(), _>(ExportAudioError::Cancelled)
})
.unwrap_err();
assert_eq!(error.to_string(), "first source failed");
assert!(peer_observed_abort.load(Ordering::Relaxed));
assert!(peer_finished.load(Ordering::Acquire));
assert!(!user_cancellation.load(Ordering::Relaxed));
}
#[test]
fn single_source_job_stays_on_current_thread() {
let current = std::thread::current().id();
let results = run_source_jobs(&mut [(7, 0)], &AtomicBool::new(false), |index, value| {
assert_eq!(std::thread::current().id(), current);
assert_eq!(index, 7);
*value += 1;
Ok(*value)
})
.unwrap();
assert_eq!(results, [1]);
}
#[test]
fn inline_source_panic_preserves_original_source_index() {
let abort = AtomicBool::new(false);
let error = run_source_jobs::<_, ()>(&mut [(7, ())], &abort, |_, _| {
panic!("inline source panic");
})
.unwrap_err();
assert_eq!(error.source_index(), Some(7));
assert_eq!(
error.to_string(),
"Audio source worker failed: inline source panic"
);
assert!(abort.load(Ordering::Relaxed));
}
fn write_wav(path: &Path, frames: usize, channels: u16) {
let data_size = frames as u32 * u32::from(channels) * 2;
let mut bytes = Vec::new();
bytes.extend_from_slice(b"RIFF");
bytes.extend_from_slice(&(36 + data_size).to_le_bytes());
bytes.extend_from_slice(b"WAVEfmt ");
bytes.extend_from_slice(&16u32.to_le_bytes());
bytes.extend_from_slice(&1u16.to_le_bytes());
bytes.extend_from_slice(&channels.to_le_bytes());
bytes.extend_from_slice(&48_000u32.to_le_bytes());
bytes.extend_from_slice(&(48_000u32 * u32::from(channels) * 2).to_le_bytes());
bytes.extend_from_slice(&(channels * 2).to_le_bytes());
bytes.extend_from_slice(&16u16.to_le_bytes());
bytes.extend_from_slice(b"data");
bytes.extend_from_slice(&data_size.to_le_bytes());
for index in 0..frames * channels as usize {
bytes.extend_from_slice(&(((index * 73) % 60_001) as i32 - 30_000).to_le_bytes()[..2]);
}
std::fs::write(path, bytes).unwrap();
}
fn preparation_recording(directory: &Path) -> RecordingMeta {
std::fs::write(
directory.join("recording-meta.json"),
r#"{
"pretty_name": "Audio preparation test",
"segments": [
{
"display": { "path": "unused.mp4" },
"mic": { "path": "mic.wav" },
"system_audio": { "path": "system.wav" }
},
{ "display": { "path": "unused.mp4" } },
{
"display": { "path": "unused.mp4" },
"mic": { "path": "incoming.wav" }
}
]
}"#,
)
.unwrap();
RecordingMeta::load_for_project(directory).unwrap()
}
#[test]
fn preparation_preserves_readers_positions_and_timing_repair_bits() {
use crate::editor_instance::SegmentAudioTimingRepair;
let directory = tempfile::tempdir().unwrap();
let recording = preparation_recording(directory.path());
for (name, channels) in [("mic.wav", 1), ("system.wav", 2), ("incoming.wav", 2)] {
write_wav(&directory.path().join(name), 17_003, channels);
}
let user = Arc::new(AtomicBool::new(false));
let abort = Arc::new(AtomicBool::new(false));
let prepared = ExportAudioPreparation::open(
&recording,
recording.studio_meta().unwrap(),
user.clone(),
abort.clone(),
)
.unwrap();
let identity = |track: &ExportAudioTrack| {
(
std::ptr::from_ref(&track.source) as usize,
track.source_index,
track.path.clone(),
track.mic,
track.source.position(),
)
};
let before = prepared
.sources
.tracks
.iter()
.flatten()
.map(identity)
.collect::<Vec<_>>();
assert!(before.iter().all(|track| track.4 == 0));
assert_eq!(
prepared
.sources
.tracks
.iter()
.map(Vec::len)
.collect::<Vec<_>>(),
[2, 0, 1]
);
let repairs = [
SegmentAudioTimingRepair {
mic_offset_secs: -0.0,
system_audio_offset_secs: f32::from_bits(0x7fc0_1234),
},
SegmentAudioTimingRepair::default(),
SegmentAudioTimingRepair {
mic_offset_secs: -0.137_125,
system_audio_offset_secs: 0.0,
},
];
let mut renderer = prepared
.finish_with_timing_repair(repairs.into_iter())
.unwrap();
let after = renderer
.sources
.tracks
.iter()
.flatten()
.map(identity)
.collect::<Vec<_>>();
assert_eq!(before, after);
assert_eq!(
renderer
.sources
.tracks
.iter()
.flatten()
.map(|track| track.timing_offset_secs.to_bits())
.collect::<Vec<_>>(),
[
repairs[0].mic_offset_secs.to_bits(),
repairs[0].system_audio_offset_secs.to_bits(),
repairs[2].mic_offset_secs.to_bits()
],
);
for track in renderer.sources.tracks.iter_mut().flatten() {
let reference = AudioData::from_file(&track.path).unwrap();
let ChunkRead::Chunk(chunk) = track.source.read_chunk(257).unwrap() else {
panic!("prepared source unexpectedly empty");
};
assert_eq!(chunk.source_start_sample, 0);
assert_eq!(
chunk
.samples
.iter()
.map(|value| value.to_bits())
.collect::<Vec<_>>(),
reference.samples()[..chunk.samples.len()]
.iter()
.map(|value| value.to_bits())
.collect::<Vec<_>>(),
);
}
assert!(!user.load(Ordering::Relaxed));
assert!(!abort.load(Ordering::Relaxed));
}
#[test]
fn invalid_finish_drops_all_prepared_readers() {
let directory = tempfile::tempdir().unwrap();
let recording = preparation_recording(directory.path());
for name in ["mic.wav", "system.wav", "incoming.wav"] {
write_wav(&directory.path().join(name), 17_003, 1);
}
let user = Arc::new(AtomicBool::new(false));
let abort = Arc::new(AtomicBool::new(false));
let prepared = ExportAudioPreparation::open(
&recording,
recording.studio_meta().unwrap(),
user.clone(),
abort.clone(),
)
.unwrap();
assert!(Arc::strong_count(&user) > 1);
assert!(matches!(
prepared.finish(&[]),
Err(ExportAudioError::InvalidWindow)
));
assert_eq!(Arc::strong_count(&user), 1);
assert_eq!(Arc::strong_count(&abort), 1);
assert!(!user.load(Ordering::Relaxed));
assert!(!abort.load(Ordering::Relaxed));
}
#[test]
fn open_rejects_segment_mismatch_before_missing_source_io() {
let directory = tempfile::tempdir().unwrap();
let recording = preparation_recording(directory.path());
let user = Arc::new(AtomicBool::new(false));
let abort = Arc::new(AtomicBool::new(false));
assert!(matches!(
ExportAudioRenderer::open(
&recording,
recording.studio_meta().unwrap(),
&[],
user.clone(),
abort.clone(),
),
Err(ExportAudioError::InvalidWindow),
));
assert_eq!(Arc::strong_count(&user), 1);
assert_eq!(Arc::strong_count(&abort), 1);
assert!(!abort.load(Ordering::Relaxed));
assert!(!directory.path().join("mic.wav").exists());
}
fn segment(index: u32, start: f64, end: f64) -> TimelineSegment {
TimelineSegment {
recording_clip: index,
start,
end,
timescale: 1.0,
name: None,
speed_audio_mode: None,
hide_cursor: None,
volume: None,
}
}
fn project() -> ProjectConfiguration {
ProjectConfiguration {
timeline: Some(TimelineConfiguration {
segments: vec![segment(0, 0.001_01, 0.33), segment(1, 0.002_01, 0.29)],
transitions: Vec::new(),
zoom_segments: Vec::new(),
scene_segments: Vec::new(),
style_segments: Vec::new(),
image_segments: Vec::new(),
mask_segments: Vec::new(),
text_segments: Vec::new(),
caption_segments: Vec::new(),
keyboard_segments: Vec::new(),
audio_segments: Vec::new(),
camera3d_segments: Vec::new(),
}),
clips: vec![
ClipConfiguration {
index: 0,
offsets: ClipOffsets {
mic: -0.003_13,
system_audio: 0.001_33,
..Default::default()
},
..Default::default()
},
ClipConfiguration {
index: 1,
..Default::default()
},
],
..Default::default()
}
}
fn fixture(paths: &[PathBuf], data: &[Arc<AudioData>]) -> (AudioRenderer, ExportAudioRenderer) {
let cancellation = Arc::new(AtomicBool::new(false));
let full = vec![
AudioSegment {
tracks: vec![
AudioSegmentTrack::new(
data[0].clone(),
|config| config.mic_volume_db,
|config| match config.mic_stereo_mode {
cap_project::StereoMode::Stereo => StereoMode::Stereo,
cap_project::StereoMode::MonoL => StereoMode::MonoL,
cap_project::StereoMode::MonoR => StereoMode::MonoR,
},
|offset| offset.mic,
)
.with_microphone_enhancement(),
AudioSegmentTrack::new(
data[1].clone(),
|config| config.system_volume_db,
|_| StereoMode::Stereo,
|offset| offset.system_audio,
),
],
},
AudioSegment {
tracks: vec![
AudioSegmentTrack::new(
data[2].clone(),
|config| config.mic_volume_db,
|config| match config.mic_stereo_mode {
cap_project::StereoMode::Stereo => StereoMode::Stereo,
cap_project::StereoMode::MonoL => StereoMode::MonoL,
cap_project::StereoMode::MonoR => StereoMode::MonoR,
},
|offset| offset.mic,
)
.with_microphone_enhancement(),
],
},
];
let tracks = vec![
vec![
ExportAudioTrack::open(&paths[0], 0, true, 0.0, &cancellation, &cancellation)
.unwrap(),
ExportAudioTrack::open(&paths[1], 1, false, 0.0, &cancellation, &cancellation)
.unwrap(),
],
vec![
ExportAudioTrack::open(&paths[2], 2, true, 0.0, &cancellation, &cancellation)
.unwrap(),
],
];
(
AudioRenderer::new(full),
ExportAudioRenderer {
renderer: AudioRenderer::new(Vec::new()),
sources: ExportAudioSources {
tracks,
abort: cancellation.clone(),
cancellation,
},
failure: None,
},
)
}
#[test]
fn unused_source_detachment_preserves_slots_identity_and_project() {
let directory = tempfile::tempdir().unwrap();
let paths =
["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name));
for (path, channels) in paths.iter().zip([1, 2, 2]) {
write_wav(path, 17_003, channels);
}
let data = paths
.iter()
.map(|path| Arc::new(AudioData::from_file(path).unwrap()))
.collect::<Vec<_>>();
for variant in 0..9 {
let (_, mut candidate) = fixture(&paths, &data);
candidate.sources.tracks[1].push(
ExportAudioTrack::open(
&paths[2],
3,
false,
0.0,
&candidate.sources.cancellation,
&candidate.sources.abort,
)
.unwrap(),
);
for track in candidate.sources.tracks.iter_mut().flatten() {
track.prepare(10, 100, 0).unwrap();
}
let identity = |track: &ExportAudioTrack| {
(
track.source_index,
track.path.clone(),
track.source.position(),
track.samples.as_ptr() as usize,
track.samples.len(),
)
};
let before = candidate
.sources
.tracks
.iter()
.flatten()
.map(identity)
.collect::<Vec<_>>();
let mut project = project();
let expected_unused = match variant {
0 => {
project.timeline = None;
vec![]
}
1 => vec![],
2 => {
project.audio.mute = true;
for segment in &mut project.timeline.as_mut().unwrap().segments {
segment.speed_audio_mode = Some(ClipSpeedAudioMode::Mute);
}
vec![]
}
3 => {
project
.timeline
.as_mut()
.unwrap()
.transitions
.push(ClipTransition {
segment_index: 1,
kind: ClipTransitionType::CrossFade,
duration: 0.13,
});
vec![]
}
4 => {
project.timeline.as_mut().unwrap().segments = vec![segment(0, 0.0, 0.1)];
vec![2, 3]
}
5 => {
project.timeline.as_mut().unwrap().segments = vec![segment(1, 0.0, 0.1)];
vec![0, 1]
}
6 => {
project.timeline.as_mut().unwrap().segments = vec![segment(9, 0.0, 0.1)];
vec![0, 1, 2, 3]
}
7 => {
project.timeline.as_mut().unwrap().segments.clear();
vec![0, 1, 2, 3]
}
8 => {
project.timeline.as_mut().unwrap().segments =
vec![segment(0, 0.0, 0.1), segment(0, 0.2, 0.3)];
vec![2, 3]
}
_ => unreachable!(),
};
let project_before = format!("{project:?}");
let detached = candidate.take_unused_sources(&project);
let unused_indices = detached
.iter()
.flat_map(|validation| validation.sources.tracks.iter().flatten())
.map(|track| track.source_index)
.collect::<Vec<_>>();
assert_eq!(unused_indices, expected_unused, "variant {variant}");
assert_eq!(candidate.sources.tracks.len(), 2);
for (clip_index, tracks) in candidate.sources.tracks.iter().enumerate() {
for track in tracks {
assert_eq!(track.source_index / 2, clip_index);
}
}
let mut after = candidate
.sources
.tracks
.iter()
.flatten()
.chain(
detached
.iter()
.flat_map(|validation| validation.sources.tracks.iter().flatten()),
)
.map(identity)
.collect::<Vec<_>>();
after.sort_by_key(|track| track.0);
assert_eq!(after, before, "variant {variant}");
assert_eq!(format!("{project:?}"), project_before);
assert!(candidate.take_unused_sources(&project).is_none());
if let Some(detached) = detached {
detached.validate_to_end().unwrap();
}
candidate.validate_to_end().unwrap();
}
}
#[test]
fn detachment_does_not_split_more_than_four_readers() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("source.wav");
write_wav(&path, 17_003, 2);
let paths = vec![path.clone(); 3];
let data = paths
.iter()
.map(|path| Arc::new(AudioData::from_file(path).unwrap()))
.collect::<Vec<_>>();
let (_, mut candidate) = fixture(&paths, &data);
for index in 3..5 {
candidate.sources.tracks[1].push(
ExportAudioTrack::open(
&path,
index,
false,
0.0,
&candidate.sources.cancellation,
&candidate.sources.abort,
)
.unwrap(),
);
}
let mut project = project();
project.timeline.as_mut().unwrap().segments.truncate(1);
assert!(candidate.take_unused_sources(&project).is_none());
assert_eq!(candidate.sources.tracks.iter().flatten().count(), 5);
}
#[test]
#[ignore = "requires CAP_STUDIO_SOUND_TEST_AUDIO pointing to a local microphone recording"]
fn studio_sound_real_voice_matches_preview_and_export_at_every_tier() {
ffmpeg::init().unwrap();
let path = PathBuf::from(std::env::var_os("CAP_STUDIO_SOUND_TEST_AUDIO").unwrap());
let audio = Arc::new(AudioData::from_file(&path).unwrap());
let paths = vec![path; 3];
let data = vec![audio.clone(); 3];
for isolation in [
cap_project::VoiceIsolation::Light,
cap_project::VoiceIsolation::Balanced,
cap_project::VoiceIsolation::Strong,
] {
let (mut preview, mut export) = fixture(&paths, &data);
let mut project = ProjectConfiguration::default();
project.audio.improve = true;
project.audio.isolation = isolation;
project.audio.system_volume_db = f32::NEG_INFINITY;
let mut count = 0;
while let Some((frames, expected)) = preview.render_frame_raw(4096, &project) {
let mut actual = Vec::new();
let written = export
.render_chunks(4096, &project, |_, samples| {
actual.extend_from_slice(samples);
Ok(())
})
.unwrap();
assert_eq!(written, Some(frames));
assert_eq!(actual, expected, "tier {isolation:?}, frame {count}");
count += frames;
}
assert_eq!(count, audio.sample_count());
assert_ne!(
export.sources.tracks[0][0].voice_profile.unwrap(),
VoiceProfile::default()
);
eprintln!("{isolation:?}: {count} microphone frames match exactly");
}
}
#[test]
#[ignore = "requires CAP_STUDIO_SOUND_BENCH_RECORDING pointing to a local Studio recording"]
fn studio_sound_streams_a_real_recording_with_bounded_memory() {
ffmpeg::init().unwrap();
let path = std::env::var_os("CAP_STUDIO_SOUND_BENCH_RECORDING").unwrap();
let recording = RecordingMeta::load_for_project(Path::new(&path)).unwrap();
let mut totals = Vec::new();
for improve in [false, true] {
let started = std::time::Instant::now();
let mut preparation = ExportAudioPreparation::open(
&recording,
recording.studio_meta().unwrap(),
Arc::new(AtomicBool::new(false)),
Arc::new(AtomicBool::new(false)),
)
.unwrap();
let mut project = ProjectConfiguration::default();
project.audio.improve = improve;
let mut count = 0_usize;
let mut peak_window_bytes = 0;
let mut output = [0.0; EXPORT_AUDIO_BLOCK_SAMPLES * 2];
for clip in 0..preparation.sources.tracks.len() {
let mut cursor = 0;
loop {
output.fill(0.0);
let rendered = preparation
.sources
.render(
&project,
clip as u32,
cursor,
EXPORT_AUDIO_BLOCK_SAMPLES,
&mut output,
)
.unwrap();
assert!(
output
.iter()
.all(|sample| sample.is_finite() && sample.abs() <= 1.0)
);
let bytes = preparation
.sources
.tracks
.iter()
.flatten()
.map(|track| {
assert!(
track.samples.len() / usize::from(track.source.channels())
<= MAX_ENHANCED_WINDOW_SAMPLES
);
track.samples.capacity() * size_of::<f32>()
})
.sum::<usize>();
peak_window_bytes = peak_window_bytes.max(bytes);
cursor += rendered;
if rendered == 0 {
break;
}
}
count += cursor;
}
assert!(count > 0);
totals.push(count);
eprintln!(
"studio_sound={improve}, audio_seconds={:.3}, elapsed_seconds={:.3}, source_window_bytes={peak_window_bytes}",
count as f64 / 48_000.0,
started.elapsed().as_secs_f64()
);
}
assert_eq!(totals[0], totals[1]);
}
#[test]
fn bounded_sink_matches_full_renderer_at_original_request_boundaries() {
assert_bounded_sink_variants(0..22, &[1, 7, 997, 4_096, 4_800, 48_001, 96_000, 384_000]);
}
#[test]
fn clip_volume_matches_preview_in_streaming_exports_and_transitions() {
assert_bounded_sink_variants(22..26, &[1, 997, 4_800, 48_001]);
}
#[test]
fn eligibility_accepts_studio_sound_short_forward_cuts() {
let mut short_cut = project();
short_cut
.timeline
.as_mut()
.unwrap()
.segments
.push(segment(0, 0.34, 0.36));
assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));
short_cut.audio.improve = true;
assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));
short_cut.audio.mute = true;
assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));
let mut long_cut = project();
long_cut.audio.improve = true;
long_cut
.timeline
.as_mut()
.unwrap()
.segments
.push(segment(0, 1.0, 1.5));
assert!(ExportAudioRenderer::eligible_sources(&long_cut, 3));
let mut exact_split = project();
exact_split.audio.improve = true;
exact_split
.timeline
.as_mut()
.unwrap()
.segments
.push(segment(0, 0.33, 0.9));
assert!(ExportAudioRenderer::eligible_sources(&exact_split, 3));
short_cut.timeline.as_mut().unwrap().segments[2].start = 0.32;
assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3));
}
#[test]
fn studio_sound_transition_with_single_sample_requests_keeps_lookahead() {
assert_bounded_sink_variants([19], &[1]);
}
#[test]
fn short_studio_sound_cuts_match_preview_after_history_rolls() {
ffmpeg::init().unwrap();
let directory = tempfile::tempdir().unwrap();
let paths =
["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name));
for ((path, frames), channels) in
paths.iter().zip([144_013, 96_007, 168_013]).zip([1, 2, 2])
{
write_wav(path, frames, channels);
}
let data = paths
.iter()
.map(|path| Arc::new(AudioData::from_file(path).unwrap()))
.collect::<Vec<_>>();
for (tier, isolation) in [
cap_project::VoiceIsolation::Light,
cap_project::VoiceIsolation::Balanced,
cap_project::VoiceIsolation::Strong,
]
.into_iter()
.enumerate()
{
for offset in [0, 1] {
let mut project = project();
project.audio.improve = true;
project.audio.isolation = isolation;
if offset == 1 {
project.clips[0].offsets.mic = 0.137_125;
project.clips[0].offsets.system_audio = -0.031_25;
project.clips[1].offsets.mic = -0.117_125;
project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR;
}
project.timeline.as_mut().unwrap().segments = vec![
segment(0, 0.001_01, 0.72),
segment(0, 0.73, 0.8),
segment(0, 0.800_02, 0.92),
segment(0, 1.0, 1.63),
segment(1, 0.35, 1.4),
segment(1, 1.5, 2.2),
segment(0, 1.8, 2.5),
segment(0, 2.500_02, 3.05),
];
let requests = [997, 4_096, 48_001]
.into_iter()
.chain((tier == 1 && offset == 0).then_some(1));
for request in requests {
let (mut reference, mut candidate) = fixture(&paths, &data);
reference.set_playhead(0.0, &project);
loop {
let expected = reference.render_frame_raw(request, &project);
let mut actual = Vec::new();
let written = candidate
.render_chunks(request, &project, |offset, samples| {
assert_eq!(offset * 2, actual.len());
assert!(samples.len() <= EXPORT_AUDIO_BLOCK_SAMPLES * 2);
actual.extend_from_slice(samples);
Ok(())
})
.unwrap_or_else(|error| {
panic!("{isolation:?}, offset {offset}, request {request}: {error}")
});
assert_eq!(written, expected.as_ref().map(|(count, _)| *count));
let expected = expected.as_ref().map_or(&[][..], |(_, data)| data);
assert_eq!(actual.len(), expected.len());
assert!(
actual
.iter()
.map(|sample| sample.to_bits())
.eq(expected.iter().map(|sample| sample.to_bits())),
"{isolation:?}, offset {offset}, request {request}"
);
for track in candidate.sources.tracks.iter().flatten() {
let limit = if track.enhancer.is_some() {
MAX_ENHANCED_WINDOW_SAMPLES
+ VOICE_PREROLL_SAMPLES
+ VOICE_WINDOW_PADDING_SAMPLES
} else {
EXPORT_AUDIO_BLOCK_SAMPLES
};
assert!(
track.samples.len() <= limit * track.source.channels() as usize
);
}
if written.is_none() {
break;
}
}
candidate.validate_to_end().unwrap();
}
}
}
}
fn write_voice_like_wav(path: &Path, frames: usize, channels: u16, seed: u64) {
let data_size = frames as u32 * u32::from(channels) * 2;
let mut bytes = Vec::new();
bytes.extend_from_slice(b"RIFF");
bytes.extend_from_slice(&(36 + data_size).to_le_bytes());
bytes.extend_from_slice(b"WAVEfmt ");
bytes.extend_from_slice(&16u32.to_le_bytes());
bytes.extend_from_slice(&1u16.to_le_bytes());
bytes.extend_from_slice(&channels.to_le_bytes());
bytes.extend_from_slice(&48_000u32.to_le_bytes());
bytes.extend_from_slice(&(48_000u32 * u32::from(channels) * 2).to_le_bytes());
bytes.extend_from_slice(&(channels * 2).to_le_bytes());
bytes.extend_from_slice(&16u16.to_le_bytes());
bytes.extend_from_slice(b"data");
bytes.extend_from_slice(&data_size.to_le_bytes());
let mut state = seed | 1;
for frame in 0..frames {
let t = frame as f64 / 48_000.0;
let envelope = ((t * 2.7).sin() * 0.5 + 0.5).powi(3)
* if (t * 0.9).fract() < 0.2 { 0.02 } else { 1.0 };
for _ in 0..channels {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
let noise = (state >> 40) as f64 / (1u64 << 24) as f64 - 0.5;
let voice =
(t * 2.0 * std::f64::consts::PI * (140.0 + 40.0 * (t * 3.1).sin())).sin();
let value = (voice * 0.6 + noise * 0.4) * envelope * 30_000.0;
bytes.extend_from_slice(&(value as i16).to_le_bytes());
}
}
std::fs::write(path, bytes).unwrap();
}
#[test]
fn studio_sound_random_cuts_never_reject_the_export_window() {
ffmpeg::init().unwrap();
let directory = tempfile::tempdir().unwrap();
let paths =
["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name));
for (((path, frames), channels), seed) in paths
.iter()
.zip([300_013, 250_007, 280_013])
.zip([1, 2, 2])
.zip([11, 23, 37])
{
write_voice_like_wav(path, frames, channels, seed);
}
let data = paths
.iter()
.map(|path| Arc::new(AudioData::from_file(path).unwrap()))
.collect::<Vec<_>>();
let iterations = std::env::var("CAP_AUDIO_FUZZ_ITERATIONS")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(12u64);
let mut state = std::env::var("CAP_AUDIO_FUZZ_SEED")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(0x9e37_79b9_7f4a_7c15u64);
let mut next = move |bound: u64| {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
state % bound.max(1)
};
let mut failures = Vec::new();
for iteration in 0..iterations {
let mut project = project();
project.audio.improve = true;
project.audio.isolation = [
cap_project::VoiceIsolation::Light,
cap_project::VoiceIsolation::Balanced,
cap_project::VoiceIsolation::Strong,
][next(3) as usize];
for clip in &mut project.clips {
clip.offsets.mic = if next(2) == 0 {
0.0
} else {
(next(600_001) as f32 - 300_000.0) / 1_000_000.0
};
clip.offsets.system_audio = if next(2) == 0 {
0.0
} else {
(next(600_001) as f32 - 300_000.0) / 1_000_000.0
};
}
let mut cursors = [
next(48_000) as f64 / 48_000.0,
next(48_000) as f64 / 48_000.0,
];
let mut segments = Vec::new();
while segments.len() < 14 {
let clip = next(2) as usize;
if cursors[clip] > 6.0 {
if cursors.iter().all(|cursor| *cursor > 6.0) {
break;
}
continue;
}
let gap = match next(5) {
0 => 0,
1 => 1 + next(480),
2 => 480 + next(9_600),
3 => 9_600 + next(40_000),
_ => 40_000 + next(100_000),
} as f64
/ 48_000.0;
let start = cursors[clip] + gap;
let end = start + (24 + next(60_000)) as f64 / 48_000.0;
segments.push(segment(clip as u32, start, end));
cursors[clip] = end;
}
project.timeline.as_mut().unwrap().segments = segments;
if !ExportAudioRenderer::eligible_sources(&project, 3) {
continue;
}
let request = [1, 480, 997, 4_096, 4_097, 48_001][next(6) as usize];
let (mut reference, mut candidate) = fixture(&paths, &data);
reference.set_playhead(0.0, &project);
let mut position = 0;
loop {
let expected = reference.render_frame_raw(request, &project);
let mut actual = Vec::new();
let written = match candidate.render_chunks(request, &project, |_, samples| {
actual.extend_from_slice(samples);
Ok(())
}) {
Ok(written) => written,
Err(error) => {
let segments = project
.timeline
.as_ref()
.unwrap()
.segments
.iter()
.map(|segment| (segment.recording_clip, segment.start, segment.end))
.collect::<Vec<_>>();
let offsets = project
.clips
.iter()
.map(|clip| (clip.offsets.mic, clip.offsets.system_audio))
.collect::<Vec<_>>();
failures.push(format!(
"iteration {iteration} request {request} at output sample {position}: {error}\n{:?} {offsets:?}\n{segments:?}",
project.audio.isolation
));
break;
}
};
assert_eq!(
written,
expected.as_ref().map(|(count, _)| *count),
"iteration {iteration} request {request} at output sample {position}"
);
assert_eq!(
actual.len(),
expected.as_ref().map_or(0, |(_, data)| data.len()),
"iteration {iteration} request {request} at output sample {position}"
);
let Some(written) = written else {
break;
};
position += written;
}
}
assert!(failures.is_empty(), "{}", failures.join("\n\n"));
}
#[test]
fn fullscreen_holds_match_full_renderer_without_retaining_whole_tracks() {
assert_bounded_sink_variants(26..32, &[1, 7, 997, 4_096, 48_001]);
}
#[test]
fn streaming_eligibility_accepts_valid_fullscreen_holds() {
let mut project = project();
let hold = serde_json::from_value(serde_json::json!({
"start": 0.1, "end": 0.2, "layout": "fullscreen"
}))
.unwrap();
project.timeline.as_mut().unwrap().text_segments.push(hold);
assert!(ExportAudioRenderer::eligible_sources(&project, 3));
for (start, end) in [
(f64::NEG_INFINITY, 0.2),
(-1.0, 0.2),
(0.1, f64::INFINITY),
(0.1, f64::MAX),
] {
let hold = &mut project.timeline.as_mut().unwrap().text_segments[0];
hold.start = start;
hold.end = end;
assert!(
!ExportAudioRenderer::eligible_sources(&project, 3),
"{start}..{end}"
);
}
}
#[test]
fn fullscreen_hold_emits_silence_without_reading_audio_and_cancels_between_blocks() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("source.wav");
write_wav(&path, 48_000, 1);
let data = Arc::new(AudioData::from_file(&path).unwrap());
let (_, mut candidate) = fixture(
&[path.clone(), path.clone(), path],
&[data.clone(), data.clone(), data],
);
let mut project = project();
project.timeline.as_mut().unwrap().text_segments.push(
serde_json::from_value(serde_json::json!({
"start": 0.0, "end": 1.0, "layout": "fullscreen"
}))
.unwrap(),
);
let mut samples = Vec::new();
assert_eq!(
candidate
.render_chunks(4_096, &project, |_, block| {
samples.extend_from_slice(block);
Ok(())
})
.unwrap(),
Some(4_096)
);
assert_eq!(samples, vec![0.0; 4_096 * 2]);
assert!(
candidate
.sources
.tracks
.iter()
.flatten()
.all(|track| track.source.position() == 0)
);
let cancellation = candidate.sources.cancellation.clone();
let mut blocks = 0;
assert!(matches!(
candidate.render_chunks(48_000, &project, |_, block| {
assert!(block.iter().all(|sample| *sample == 0.0));
blocks += 1;
cancellation.store(true, Ordering::Relaxed);
Ok(())
}),
Err(ExportAudioError::Cancelled)
));
assert_eq!(blocks, 1);
assert!(
candidate
.sources
.tracks
.iter()
.flatten()
.all(|track| track.source.position() == 0)
);
assert!(matches!(
candidate.validate_to_end(),
Err(ExportAudioError::Cancelled)
));
}
fn assert_bounded_sink_variants(variants: impl IntoIterator<Item = usize>, requests: &[usize]) {
ffmpeg::init().unwrap();
let directory = tempfile::tempdir().unwrap();
let paths =
["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name));
for ((path, frames), channels) in paths.iter().zip([17_003, 9_007, 23_013]).zip([1, 2, 2]) {
write_wav(path, frames, channels);
}
let data = paths
.iter()
.map(|path| Arc::new(AudioData::from_file(path).unwrap()))
.collect::<Vec<_>>();
for variant in variants {
let mut project = project();
match variant {
0 => project.timeline = None,
1 => {}
2 | 3 => project
.timeline
.as_mut()
.unwrap()
.transitions
.push(ClipTransition {
segment_index: 1,
kind: if variant == 2 {
ClipTransitionType::CrossFade
} else {
ClipTransitionType::FadeThroughBlack
},
duration: 0.131_234_567,
}),
4 => {
project.timeline.as_mut().unwrap().segments[0].speed_audio_mode =
Some(ClipSpeedAudioMode::Mute)
}
5 => project.audio.mute = true,
6 => project.audio.mic_stereo_mode = cap_project::StereoMode::MonoL,
7 => project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR,
8 => project.audio.mic_volume_db = f32::NAN,
9 => project.audio.mic_volume_db = f32::INFINITY,
10 => project.audio.system_volume_db = f32::NEG_INFINITY,
11 => project.clips[0].offsets.mic = 0.9,
12 => project.clips[0].offsets.system_audio = -0.9,
13 => project
.timeline
.as_mut()
.unwrap()
.segments
.push(segment(0, 0.34, 0.36)),
14 => project.timeline.as_mut().unwrap().segments[1].recording_clip = 9,
15 => {
project.audio.mic_volume_db = 4.0;
project.audio.system_volume_db = -29.9;
}
16 => project.audio.improve = true,
17 => {
project.audio.improve = true;
project.timeline = None;
}
18 => {
project.audio.improve = true;
project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR;
}
19 => {
project.audio.improve = true;
project
.timeline
.as_mut()
.unwrap()
.transitions
.push(ClipTransition {
segment_index: 1,
kind: ClipTransitionType::CrossFade,
duration: 0.131_234_567,
});
}
20 | 21 => {
project.audio.improve = true;
project.audio.isolation = if variant == 20 {
cap_project::VoiceIsolation::Light
} else {
cap_project::VoiceIsolation::Strong
};
}
22..=25 => {
let timeline = project.timeline.as_mut().unwrap();
timeline.segments[0].volume = Some(if variant == 22 { 0.0 } else { 0.5 });
timeline.segments[1].volume = Some(2.0);
if variant >= 24 {
timeline.transitions.push(ClipTransition {
segment_index: 1,
kind: if variant == 24 {
ClipTransitionType::CrossFade
} else {
ClipTransitionType::FadeThroughBlack
},
duration: 0.131_234_567,
});
}
}
26..=31 => {
project.audio.improve = variant == 30;
let timeline = project.timeline.as_mut().unwrap();
let windows: &[(f64, f64)] = match variant {
26 => &[(0.0, 0.071_234_567)],
27 | 30 => &[(0.113_456_789, 0.224_567_891)],
28 => &[(0.1, 0.2), (0.15, 0.25), (0.25, 0.3)],
29 => &[(0.25, 0.4)],
31 => &[(0.0, 0.1), (0.4, 0.5), (0.816_98, 0.9)],
_ => unreachable!(),
};
timeline.text_segments = windows
.iter()
.map(|&(start, end)| {
serde_json::from_value(serde_json::json!({
"start": start, "end": end, "layout": "fullscreen"
}))
.unwrap()
})
.collect();
if variant == 29 {
timeline.transitions.push(ClipTransition {
segment_index: 1,
kind: ClipTransitionType::CrossFade,
duration: 0.131_234_567,
});
}
}
_ => unreachable!(),
}
for &request in requests {
let (mut reference, mut candidate) = fixture(&paths, &data);
reference.set_playhead(0.0, &project);
loop {
let expected = reference.render_frame_raw(request, &project);
let mut actual = Vec::new();
let written = candidate
.render_chunks(request, &project, |offset, samples| {
assert!(samples.len() <= EXPORT_AUDIO_BLOCK_SAMPLES * 2);
assert_eq!(offset * 2, actual.len());
actual.extend_from_slice(samples);
Ok(())
})
.unwrap_or_else(|error| {
panic!("variant {variant}, request {request}: {error}")
});
assert_eq!(
written,
expected.as_ref().map(|(count, _)| *count),
"variant {variant}, request {request}"
);
assert_eq!(
actual
.iter()
.map(|value| value.to_bits())
.collect::<Vec<_>>(),
expected
.as_ref()
.map(|(_, values)| values
.iter()
.map(|value| value.to_bits())
.collect::<Vec<_>>())
.unwrap_or_default(),
"variant {variant}, request {request}"
);
assert_eq!(
reference.elapsed_samples_to_playhead().to_bits(),
candidate.renderer.elapsed_samples_to_playhead().to_bits()
);
for track in candidate.sources.tracks.iter().flatten() {
assert!(
track.samples.len()
<= (if track.enhancer.is_some() {
MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES
} else {
EXPORT_AUDIO_BLOCK_SAMPLES
}) * track.source.channels() as usize
);
}
if expected.is_none() {
break;
}
}
candidate.validate_to_end().unwrap();
assert_eq!(
candidate
.sources
.tracks
.iter()
.flatten()
.map(|track| track.source.position())
.collect::<Vec<_>>(),
data.iter()
.map(|data| data.sample_count() as u64)
.collect::<Vec<_>>()
);
}
}
}
#[test]
fn eligibility_rejects_unbounded_or_nonsequential_shapes() {
let original = project();
assert!(ExportAudioRenderer::eligible_sources(&original, 3));
assert!(!ExportAudioRenderer::eligible_sources(&original, 0));
assert!(!ExportAudioRenderer::eligible_sources(&original, 5));
for offset in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, f32::MAX] {
let mut project = original.clone();
project.clips[0].offsets.mic = offset;
assert!(!ExportAudioRenderer::eligible_sources(&project, 3));
}
for (start, end, speed) in [
(f64::NAN, 1.0, 1.0),
(-0.1, 1.0, 1.0),
(0.0, f64::INFINITY, 1.0),
(0.0, 0.0, 1.0),
(0.0, 1.0, 2.0),
] {
let mut project = original.clone();
let segment = &mut project.timeline.as_mut().unwrap().segments[0];
segment.start = start;
segment.end = end;
segment.timescale = speed;
assert!(!ExportAudioRenderer::eligible_sources(&project, 3));
}
let mut project = original.clone();
project
.timeline
.as_mut()
.unwrap()
.segments
.push(segment(0, 0.34, 0.4));
assert!(ExportAudioRenderer::eligible_sources(&project, 3));
project
.timeline
.as_mut()
.unwrap()
.transitions
.push(ClipTransition {
segment_index: 1,
kind: ClipTransitionType::CrossFade,
duration: 0.01,
});
assert!(!ExportAudioRenderer::eligible_sources(&project, 3));
let mut project = original;
project
.timeline
.as_mut()
.unwrap()
.segments
.push(segment(0, 0.01, 0.1));
assert!(!ExportAudioRenderer::eligible_sources(&project, 3));
}
#[test]
fn offset_view_slice_and_indexed_mixing_are_bit_identical() {
struct Indexed<'a>(&'a ExportAudioView<'a>);
impl AudioSampleSource for Indexed<'_> {
fn channels(&self) -> u16 {
self.0.channels()
}
fn sample_count(&self) -> usize {
self.0.sample_count()
}
fn sample(&self, index: usize) -> Option<&f32> {
self.0.sample(index)
}
}
fn track<'a, T: AudioSampleSource>(
data: &'a T,
channels: u16,
mono_right: bool,
) -> AudioRendererTrack<'a, T> {
AudioRendererTrack {
data,
gain: -6.0,
stereo_mode: if channels == 1 {
StereoMode::Stereo
} else if mono_right {
StereoMode::MonoR
} else {
StereoMode::Stereo
},
offset: 0,
}
}
for channels in [1u16, 2] {
for mono_right in [false, true] {
if channels == 1 && mono_right {
continue;
}
let source_start = 1_537usize;
let frames = source_start + 5_000;
let width = usize::from(channels);
let samples = (0..frames * width)
.map(|index| {
let raw = (index.wrapping_mul(73) % 60_001) as i32 - 30_000;
raw as f32 / 32_768.0
})
.collect::<Vec<_>>();
let window = &samples[source_start * width..];
let view = ExportAudioView {
samples: window,
channels,
source_start,
available_end: frames,
};
let short = ExportAudioView {
samples: &window[..100 * width],
channels,
source_start,
available_end: frames,
};
for index in 0..frames * width {
let indexed = view.sample(index).copied().map(f32::to_bits);
let sliced = view
.sample_slice(index..index + 1)
.map(|slice| slice[0].to_bits());
assert_eq!(indexed, sliced, "channels {channels} index {index}");
}
let aligned = source_start * width..(source_start + 128) * width;
let slice = view.sample_slice(aligned.clone()).unwrap();
assert_eq!(slice.len(), aligned.end - aligned.start);
assert!(view.sample_slice(0..width).is_none());
assert!(
view.sample_slice((frames - 1) * width..(frames + 1) * width)
.is_none()
);
assert!(view.sample((frames - 1) * width).is_some());
assert!(
short
.sample_slice(source_start * width..(source_start + 101) * width)
.is_none()
);
assert!(
short
.sample_slice(source_start * width..(source_start + 100) * width)
.is_some()
);
let indexed = Indexed(&view);
let indexed_short = Indexed(&short);
let mut slice_out = vec![13.0f32; 4_096 * 2];
let mut indexed_out = slice_out.clone();
for (cursor, request, source_view, source_indexed) in [
(0usize, 64usize, &view, &indexed),
(source_start - 3, 4_096, &view, &indexed),
(source_start, 4_096, &view, &indexed),
(source_start + 17, 1, &view, &indexed),
(source_start + 100, 7, &view, &indexed),
(frames - 10, 4_096, &view, &indexed),
(frames, 8, &view, &indexed),
(source_start, 4_096, &short, &indexed_short),
(source_start + 90, 4_096, &short, &indexed_short),
] {
let rendered_slice = cap_audio::render_audio(
&[track(source_view, channels, mono_right)],
cursor,
request,
0,
&mut slice_out,
);
let rendered_indexed = cap_audio::render_audio(
&[track(source_indexed, channels, mono_right)],
cursor,
request,
0,
&mut indexed_out,
);
assert_eq!(rendered_slice, rendered_indexed);
if cursor >= source_start
&& cursor + rendered_slice <= source_view.sample_count()
&& rendered_slice > 0
&& (cursor + rendered_slice).saturating_sub(source_start) * width
<= source_view.samples.len()
{
assert!(
source_view
.sample_slice(cursor * width..(cursor + rendered_slice) * width)
.is_some(),
"channels {channels} cursor {cursor}"
);
}
assert_eq!(
slice_out[..rendered_slice * 2]
.iter()
.copied()
.map(f32::to_bits)
.collect::<Vec<_>>(),
indexed_out[..rendered_indexed * 2]
.iter()
.copied()
.map(f32::to_bits)
.collect::<Vec<_>>(),
"channels {channels} cursor {cursor} mono_right {mono_right}"
);
}
}
}
}
#[test]
fn source_windows_reject_backward_reads_and_keep_failures_sticky() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("source.wav");
write_wav(&path, 17_003, 2);
let cancellation = Arc::new(AtomicBool::new(false));
let mut track =
ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap();
track.prepare(8_000, 9_000, 0).unwrap();
track.prepare(8_500, 9_501, 0).unwrap();
let full = AudioData::from_file(&path).unwrap();
assert_eq!(track.samples, full.samples()[8_500 * 2..9_501 * 2]);
assert!(matches!(
track.prepare(0, 1, 0),
Err(ExportAudioError::InvalidWindow)
));
let mut candidate = ExportAudioRenderer {
renderer: AudioRenderer::new(Vec::new()),
sources: ExportAudioSources {
tracks: vec![vec![track]],
cancellation: cancellation.clone(),
abort: cancellation.clone(),
},
failure: None,
};
let error = candidate
.render_chunks(1, &ProjectConfiguration::default(), |_, _| Ok(()))
.unwrap_err()
.to_string();
assert_eq!(candidate.validate_to_end().unwrap_err().to_string(), error);
cancellation.store(true, Ordering::Relaxed);
assert_eq!(
candidate
.render_chunks(1, &ProjectConfiguration::default(), |_, _| Ok(()))
.unwrap_err()
.to_string(),
error
);
}
#[test]
fn source_history_reuses_decoded_samples_and_stays_bounded() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("source.wav");
write_wav(&path, 144_013, 2);
let cancellation = Arc::new(AtomicBool::new(false));
let mut track =
ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap();
let full = AudioData::from_file(&path).unwrap();
track
.prepare(50_000, 52_000, STUDIO_SOUND_HISTORY_SAMPLES)
.unwrap();
let position = track.source.position();
let oldest = track.source_start;
track
.prepare(25_000, 27_000, STUDIO_SOUND_HISTORY_SAMPLES)
.unwrap();
assert_eq!(track.source.position(), position);
assert_eq!(track.source_start, oldest);
assert_eq!(
track.view().sample_slice(25_000 * 2..27_000 * 2).unwrap(),
&full.samples()[25_000 * 2..27_000 * 2]
);
assert!(matches!(
track.prepare(
oldest as i128 - 1,
oldest as i128,
STUDIO_SOUND_HISTORY_SAMPLES
),
Err(ExportAudioError::InvalidWindow)
));
for start in (60_000..150_000).step_by(997) {
track
.prepare(start, start + 1_632, STUDIO_SOUND_HISTORY_SAMPLES)
.unwrap();
let end = track.source.position() as usize;
assert_eq!(
track.samples,
full.samples()[track.source_start * 2..end * 2]
);
assert!(
track.samples.len()
<= (MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES) * 2
);
}
track.prepare(144_013, 144_014, 0).unwrap();
assert!(track.samples.is_empty());
}
#[test]
fn sink_failure_stops_before_later_source_reads() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("source.wav");
write_wav(&path, 17_003, 2);
let paths = vec![path; 3];
let data = paths
.iter()
.map(|path| Arc::new(AudioData::from_file(path).unwrap()))
.collect::<Vec<_>>();
let (_, mut candidate) = fixture(&paths, &data);
let error = candidate
.render_chunks(96_000, &project(), |_, _| {
Err(ExportAudioError::Sink("sink stopped".into()))
})
.unwrap_err();
assert_eq!(error.to_string(), "sink stopped");
assert_eq!(candidate.sources.tracks[1][0].source.position(), 0);
assert_eq!(
candidate.validate_to_end().unwrap_err().to_string(),
"sink stopped"
);
}
}