Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(editor): stream short Studio Sound cuts with bounded history
vedanthc62021c58fd4
diff --git a/crates/editor/src/export_audio.rs b/crates/editor/src/export_audio.rsindex 0a887bef0..fe3a1e0b5 100644--- a/crates/editor/src/export_audio.rs+++ b/crates/editor/src/export_audio.rs@@ -20,6 +20,7 @@ pub(crate) const EXPORT_AUDIO_BLOCK_SAMPLES: usize = 4_096;const MAX_PARALLEL_AUDIO_SOURCES: usize = 4;const MAX_ENHANCED_WINDOW_SAMPLES: usize =EXPORT_AUDIO_BLOCK_SAMPLES + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES;+const STUDIO_SOUND_HISTORY_SAMPLES: usize = VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES;#[derive(Clone, Debug)]pub enum ExportAudioError {@@ -325,17 +326,6 @@ impl ExportAudioRenderer {{return false;}- // Studio Sound re-reads up to VOICE_PREROLL_SAMPLES before each cut to warm up the- // enhancer. The streaming reader only moves forward, so a cut shorter than that- // within one clip has to use the in-memory export path instead. An exact split- // (nothing removed) keeps the enhancer contiguous and still streams.- let studio_sound =- project.audio.improve && !project.audio.mute && project.audio.mic_volume_db > -30.0;- let min_gap = if studio_sound {- (VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES) as f64 / 48_000.0- } else {- 0.0- };let mut ends = HashMap::new();let mut duration = holds.iter().map(|(start, end)| end - start).sum::<f64>();if !duration.is_finite() || duration * 48_000.0 >= isize::MAX as f64 / 2.0 {@@ -356,11 +346,7 @@ impl ExportAudioRenderer {return false;}if let Some(previous_end) = ends.insert(segment.recording_clip, segment.end)- && (!timeline.transitions.is_empty()- || segment.start < previous_end- || (min_gap > 0.0- && segment.start > previous_end- && segment.start < previous_end + min_gap))+ && (!timeline.transitions.is_empty() || segment.start < previous_end){return false;}@@ -532,12 +518,16 @@ impl ExportAudioSources {)});let range = enhancer.source_range(start, end.saturating_sub(start));- track.prepare(range.start as i128, range.end as i128)?;+ track.prepare(+ range.start as i128,+ range.end as i128,+ STUDIO_SOUND_HISTORY_SAMPLES,+ )?;track.enhanced =Some(enhancer.render(&track.view(), start, end.saturating_sub(start)));track.enhancer = Some(enhancer);} else {- track.prepare(start, end)?;+ track.prepare(start, end, 0)?;track.enhanced = None;track.enhancer = None;}@@ -657,20 +647,21 @@ impl ExportAudioTrack {((offset + self.timing_offset_secs) * AudioRenderer::SAMPLE_RATE as f32).round() as isize}- fn prepare(&mut self, start: i128, end: i128) -> Result<(), ExportAudioError> {+ fn prepare(&mut self, start: i128, end: i128, history: usize) -> Result<(), ExportAudioError> {let start = usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;let end = usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;if end < start || end - start > MAX_ENHANCED_WINDOW_SAMPLES || start < self.source_start {return Err(ExportAudioError::InvalidWindow);}let channels = self.source.channels() as usize;- let discard = start+ let retained_start = start.saturating_sub(history).max(self.source_start);+ let discard = retained_start.saturating_sub(self.source_start).min(self.samples.len() / channels)* channels;self.samples.copy_within(discard.., 0);self.samples.truncate(self.samples.len() - discard);- self.source_start = start;+ self.source_start = retained_start;while self.eof.is_none() && self.source.position() < end as u64 {let wanted = (end as u64 - self.source.position()).min(12_000) as usize;match self@@ -684,7 +675,7 @@ impl ExportAudioTrack {ChunkRead::Chunk(chunk) => {let chunk_start = usize::try_from(chunk.source_start_sample).map_err(|_| ExportAudioError::InvalidWindow)?;- let discard = start+ let discard = retained_start.saturating_sub(chunk_start).min(chunk.samples.len() / channels)* channels;@@ -1240,7 +1231,7 @@ mod tests {.unwrap(),);for track in candidate.sources.tracks.iter_mut().flatten() {- track.prepare(10, 100).unwrap();+ track.prepare(10, 100, 0).unwrap();}let identity = |track: &ExportAudioTrack| {(@@ -1498,7 +1489,7 @@ mod tests {}#[test]- fn eligibility_rejects_studio_sound_cuts_shorter_than_voice_preroll() {+ fn eligibility_accepts_studio_sound_short_forward_cuts() {let mut short_cut = project();short_cut.timeline@@ -1508,7 +1499,7 @@ mod tests {.push(segment(0, 0.34, 0.36));assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));short_cut.audio.improve = true;- assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3));+ assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));short_cut.audio.mute = true;assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));@@ -1531,6 +1522,9 @@ mod tests {.segments.push(segment(0, 0.33, 0.9));assert!(ExportAudioRenderer::eligible_sources(&exact_split, 3));++ short_cut.timeline.as_mut().unwrap().segments[2].start = 0.32;+ assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3));}#[test]@@ -1538,6 +1532,100 @@ mod tests {assert_bounded_sink_variants([19], &[1]);}+ #[test]+ fn short_studio_sound_cuts_match_preview_after_history_rolls() {+ ffmpeg::init().unwrap();+ let directory = tempfile::tempdir().unwrap();+ let paths =+ ["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name));+ for ((path, frames), channels) in+ paths.iter().zip([144_013, 96_007, 168_013]).zip([1, 2, 2])+ {+ write_wav(path, frames, channels);+ }+ let data = paths+ .iter()+ .map(|path| Arc::new(AudioData::from_file(path).unwrap()))+ .collect::<Vec<_>>();+ for (tier, isolation) in [+ cap_project::VoiceIsolation::Light,+ cap_project::VoiceIsolation::Balanced,+ cap_project::VoiceIsolation::Strong,+ ]+ .into_iter()+ .enumerate()+ {+ for offset in [0, 1] {+ let mut project = project();+ project.audio.improve = true;+ project.audio.isolation = isolation;+ if offset == 1 {+ project.clips[0].offsets.mic = 0.137_125;+ project.clips[0].offsets.system_audio = -0.031_25;+ project.clips[1].offsets.mic = -0.117_125;+ project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR;+ }+ project.timeline.as_mut().unwrap().segments = vec![+ segment(0, 0.001_01, 0.72),+ segment(0, 0.73, 0.8),+ segment(0, 0.800_02, 0.92),+ segment(0, 1.0, 1.63),+ segment(1, 0.35, 1.4),+ segment(1, 1.5, 2.2),+ segment(0, 1.8, 2.5),+ segment(0, 2.500_02, 3.05),+ ];+ let requests = [997, 4_096, 48_001]+ .into_iter()+ .chain((tier == 1 && offset == 0).then_some(1));+ for request in requests {+ let (mut reference, mut candidate) = fixture(&paths, &data);+ reference.set_playhead(0.0, &project);+ loop {+ let expected = reference.render_frame_raw(request, &project);+ let mut actual = Vec::new();+ let written = candidate+ .render_chunks(request, &project, |offset, samples| {+ assert_eq!(offset * 2, actual.len());+ assert!(samples.len() <= EXPORT_AUDIO_BLOCK_SAMPLES * 2);+ actual.extend_from_slice(samples);+ Ok(())+ })+ .unwrap_or_else(|error| {+ panic!("{isolation:?}, offset {offset}, request {request}: {error}")+ });+ assert_eq!(written, expected.as_ref().map(|(count, _)| *count));+ let expected = expected.as_ref().map_or(&[][..], |(_, data)| data);+ assert_eq!(actual.len(), expected.len());+ assert!(+ actual+ .iter()+ .map(|sample| sample.to_bits())+ .eq(expected.iter().map(|sample| sample.to_bits())),+ "{isolation:?}, offset {offset}, request {request}"+ );+ for track in candidate.sources.tracks.iter().flatten() {+ let limit = if track.enhancer.is_some() {+ MAX_ENHANCED_WINDOW_SAMPLES+ + VOICE_PREROLL_SAMPLES+ + VOICE_WINDOW_PADDING_SAMPLES+ } else {+ EXPORT_AUDIO_BLOCK_SAMPLES+ };+ assert!(+ track.samples.len() <= limit * track.source.channels() as usize+ );+ }+ if written.is_none() {+ break;+ }+ }+ candidate.validate_to_end().unwrap();+ }+ }+ }+ }+#[test]fn fullscreen_holds_match_full_renderer_without_retaining_whole_tracks() {assert_bounded_sink_variants(26..32, &[1, 7, 997, 4_096, 48_001]);@@ -1804,7 +1892,7 @@ mod tests {assert!(track.samples.len()<= (if track.enhancer.is_some() {- MAX_ENHANCED_WINDOW_SAMPLES+ MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES} else {EXPORT_AUDIO_BLOCK_SAMPLES}) * track.source.channels() as usize@@ -2045,12 +2133,12 @@ mod tests {let cancellation = Arc::new(AtomicBool::new(false));let mut track =ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap();- track.prepare(8_000, 9_000).unwrap();- track.prepare(8_500, 9_501).unwrap();+ track.prepare(8_000, 9_000, 0).unwrap();+ track.prepare(8_500, 9_501, 0).unwrap();let full = AudioData::from_file(&path).unwrap();assert_eq!(track.samples, full.samples()[8_500 * 2..9_501 * 2]);assert!(matches!(- track.prepare(0, 1),+ track.prepare(0, 1, 0),Err(ExportAudioError::InvalidWindow)));let mut candidate = ExportAudioRenderer {@@ -2077,6 +2165,55 @@ mod tests {);}+ #[test]+ fn source_history_reuses_decoded_samples_and_stays_bounded() {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("source.wav");+ write_wav(&path, 144_013, 2);+ let cancellation = Arc::new(AtomicBool::new(false));+ let mut track =+ ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap();+ let full = AudioData::from_file(&path).unwrap();+ track+ .prepare(50_000, 52_000, STUDIO_SOUND_HISTORY_SAMPLES)+ .unwrap();+ let position = track.source.position();+ let oldest = track.source_start;+ track+ .prepare(25_000, 27_000, STUDIO_SOUND_HISTORY_SAMPLES)+ .unwrap();+ assert_eq!(track.source.position(), position);+ assert_eq!(track.source_start, oldest);+ assert_eq!(+ track.view().sample_slice(25_000 * 2..27_000 * 2).unwrap(),+ &full.samples()[25_000 * 2..27_000 * 2]+ );+ assert!(matches!(+ track.prepare(+ oldest as i128 - 1,+ oldest as i128,+ STUDIO_SOUND_HISTORY_SAMPLES+ ),+ Err(ExportAudioError::InvalidWindow)+ ));+ for start in (60_000..150_000).step_by(997) {+ track+ .prepare(start, start + 1_632, STUDIO_SOUND_HISTORY_SAMPLES)+ .unwrap();+ let end = track.source.position() as usize;+ assert_eq!(+ track.samples,+ full.samples()[track.source_start * 2..end * 2]+ );+ assert!(+ track.samples.len()+ <= (MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES) * 2+ );+ }+ track.prepare(144_013, 144_014, 0).unwrap();+ assert!(track.samples.is_empty());+ }+#[test]fn sink_failure_stops_before_later_source_reads() {let directory = tempfile::tempdir().unwrap();diff --git a/crates/export/examples/studio-sound-cut-benchmark.rs b/crates/export/examples/studio-sound-cut-benchmark.rsnew file mode 100644index 000000000..8d4abab75--- /dev/null+++ b/crates/export/examples/studio-sound-cut-benchmark.rs@@ -0,0 +1,118 @@+use cap_export::{+ ExporterBase,+ mp4::{ExportCompression, Mp4ExportSettings},+};+use cap_project::{ClipConfiguration, ProjectConfiguration, RecordingMeta, TimelineSegment, XY};+use std::{+ path::PathBuf,+ sync::{+ Arc,+ atomic::{AtomicBool, AtomicU64, Ordering},+ },+ time::Instant,+};++#[tokio::main(flavor = "multi_thread", worker_threads = 2)]+async fn main() {+ let root = PathBuf::from(std::env::args_os().nth(1).expect("fixture directory"));+ let output = PathBuf::from(std::env::args_os().nth(2).expect("output MP4 path"));+ let gap: f64 = std::env::var("CAP_EXPORT_CUT_GAP").map_or(0.1, |v| v.parse().unwrap());+ let mut project = ProjectConfiguration {+ timeline: Some(+ serde_json::from_value(serde_json::json!({"segments":[],"zoomSegments":[]})).unwrap(),+ ),+ ..Default::default()+ };+ project.audio.improve = true;+ project.camera.hide = true;+ project.cursor.hide = true;+ project.cursor.raw = true;+ project.clips = vec![ClipConfiguration {+ index: 0,+ ..Default::default()+ }];+ project.timeline.as_mut().unwrap().segments = vec![+ TimelineSegment {+ recording_clip: 0,+ start: 0.0,+ end: 4.0,+ timescale: 1.0,+ ..Default::default()+ },+ TimelineSegment {+ recording_clip: 0,+ start: 4.0 + gap,+ end: 8.0 + gap,+ timescale: 1.0,+ ..Default::default()+ },+ ];+ project.write(&root).unwrap();+ let mut meta: RecordingMeta = serde_json::from_value(serde_json::json!({+ "platform":"Linux", "pretty_name":"Studio Sound cut benchmark", "sharing":null,+ "segments":[{+ "display":{"path":"display.mp4","fps":1,"start_time":0.0},+ "mic":{"path":"mic.wav","start_time":0.0}+ }],+ "cursors":{},"status":{"status":"Complete"}+ }))+ .unwrap();+ meta.project_path = root.clone();+ meta.save_for_project().unwrap();+ let first_progress = Arc::new(AtomicU64::new(u64::MAX));+ let recorded_progress = first_progress.clone();+ let start = Instant::now();+ let base = ExporterBase::builder(root)+ .with_output_path(output.clone())+ .with_force_ffmpeg_decoder(true)+ .build_for_mp4(Arc::new(AtomicBool::new(false)))+ .await+ .expect("export preparation");+ let build_seconds = start.elapsed().as_secs_f64();+ let streaming = base.uses_streaming_audio();+ let settings = Mp4ExportSettings {+ fps: 15,+ resolution_base: XY::new(640, 360),+ compression: ExportCompression::Web,+ custom_bpp: None,+ force_ffmpeg_decoder: true,+ optimize_filesize: false,+ };+ let export_start = Instant::now();+ settings+ .export_prepared(base, move |_| {+ recorded_progress+ .compare_exchange(+ u64::MAX,+ start.elapsed().as_micros() as u64,+ Ordering::Relaxed,+ Ordering::Relaxed,+ )+ .ok();+ true+ })+ .await+ .expect("complete export");+ let export_seconds = export_start.elapsed().as_secs_f64();+ let total_seconds = start.elapsed().as_secs_f64();+ let maxrss_kib = std::fs::read_to_string("/proc/self/status")+ .ok()+ .and_then(|status| {+ status.lines().find_map(|line| {+ line.strip_prefix("VmHWM:")?+ .split_whitespace()+ .next()?+ .parse::<u64>()+ .ok()+ })+ });+ println!(+ "EXPORT_BENCH {}",+ serde_json::json!({+ "gap":gap,"streaming_audio":streaming,"build_seconds":build_seconds,+ "first_progress_seconds":first_progress.load(Ordering::Relaxed) as f64 / 1_000_000.0,+ "export_seconds":export_seconds,"total_seconds":total_seconds,"maxrss_kib":maxrss_kib,+ "output":output+ })+ );+}