Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(editor): stream short Studio Sound cuts with bounded history

vedanthc62021c58fd4
diff --git a/crates/editor/src/export_audio.rs b/crates/editor/src/export_audio.rs
index 0a887bef0..fe3a1e0b5 100644
--- a/crates/editor/src/export_audio.rs
+++ b/crates/editor/src/export_audio.rs
@@ -20,6 +20,7 @@ pub(crate) const EXPORT_AUDIO_BLOCK_SAMPLES: usize = 4_096;
const MAX_PARALLEL_AUDIO_SOURCES: usize = 4;
const MAX_ENHANCED_WINDOW_SAMPLES: usize =
EXPORT_AUDIO_BLOCK_SAMPLES + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES;
+const STUDIO_SOUND_HISTORY_SAMPLES: usize = VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES;
#[derive(Clone, Debug)]
pub enum ExportAudioError {
@@ -325,17 +326,6 @@ impl ExportAudioRenderer {
{
return false;
}
- // Studio Sound re-reads up to VOICE_PREROLL_SAMPLES before each cut to warm up the
- // enhancer. The streaming reader only moves forward, so a cut shorter than that
- // within one clip has to use the in-memory export path instead. An exact split
- // (nothing removed) keeps the enhancer contiguous and still streams.
- let studio_sound =
- project.audio.improve && !project.audio.mute && project.audio.mic_volume_db > -30.0;
- let min_gap = if studio_sound {
- (VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES) as f64 / 48_000.0
- } else {
- 0.0
- };
let mut ends = HashMap::new();
let mut duration = holds.iter().map(|(start, end)| end - start).sum::<f64>();
if !duration.is_finite() || duration * 48_000.0 >= isize::MAX as f64 / 2.0 {
@@ -356,11 +346,7 @@ impl ExportAudioRenderer {
return false;
}
if let Some(previous_end) = ends.insert(segment.recording_clip, segment.end)
- && (!timeline.transitions.is_empty()
- || segment.start < previous_end
- || (min_gap > 0.0
- && segment.start > previous_end
- && segment.start < previous_end + min_gap))
+ && (!timeline.transitions.is_empty() || segment.start < previous_end)
{
return false;
}
@@ -532,12 +518,16 @@ impl ExportAudioSources {
)
});
let range = enhancer.source_range(start, end.saturating_sub(start));
- track.prepare(range.start as i128, range.end as i128)?;
+ track.prepare(
+ range.start as i128,
+ range.end as i128,
+ STUDIO_SOUND_HISTORY_SAMPLES,
+ )?;
track.enhanced =
Some(enhancer.render(&track.view(), start, end.saturating_sub(start)));
track.enhancer = Some(enhancer);
} else {
- track.prepare(start, end)?;
+ track.prepare(start, end, 0)?;
track.enhanced = None;
track.enhancer = None;
}
@@ -657,20 +647,21 @@ impl ExportAudioTrack {
((offset + self.timing_offset_secs) * AudioRenderer::SAMPLE_RATE as f32).round() as isize
}
- fn prepare(&mut self, start: i128, end: i128) -> Result<(), ExportAudioError> {
+ fn prepare(&mut self, start: i128, end: i128, history: usize) -> Result<(), ExportAudioError> {
let start = usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;
let end = usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;
if end < start || end - start > MAX_ENHANCED_WINDOW_SAMPLES || start < self.source_start {
return Err(ExportAudioError::InvalidWindow);
}
let channels = self.source.channels() as usize;
- let discard = start
+ let retained_start = start.saturating_sub(history).max(self.source_start);
+ let discard = retained_start
.saturating_sub(self.source_start)
.min(self.samples.len() / channels)
* channels;
self.samples.copy_within(discard.., 0);
self.samples.truncate(self.samples.len() - discard);
- self.source_start = start;
+ self.source_start = retained_start;
while self.eof.is_none() && self.source.position() < end as u64 {
let wanted = (end as u64 - self.source.position()).min(12_000) as usize;
match self
@@ -684,7 +675,7 @@ impl ExportAudioTrack {
ChunkRead::Chunk(chunk) => {
let chunk_start = usize::try_from(chunk.source_start_sample)
.map_err(|_| ExportAudioError::InvalidWindow)?;
- let discard = start
+ let discard = retained_start
.saturating_sub(chunk_start)
.min(chunk.samples.len() / channels)
* channels;
@@ -1240,7 +1231,7 @@ mod tests {
.unwrap(),
);
for track in candidate.sources.tracks.iter_mut().flatten() {
- track.prepare(10, 100).unwrap();
+ track.prepare(10, 100, 0).unwrap();
}
let identity = |track: &ExportAudioTrack| {
(
@@ -1498,7 +1489,7 @@ mod tests {
}
#[test]
- fn eligibility_rejects_studio_sound_cuts_shorter_than_voice_preroll() {
+ fn eligibility_accepts_studio_sound_short_forward_cuts() {
let mut short_cut = project();
short_cut
.timeline
@@ -1508,7 +1499,7 @@ mod tests {
.push(segment(0, 0.34, 0.36));
assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));
short_cut.audio.improve = true;
- assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3));
+ assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));
short_cut.audio.mute = true;
assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));
@@ -1531,6 +1522,9 @@ mod tests {
.segments
.push(segment(0, 0.33, 0.9));
assert!(ExportAudioRenderer::eligible_sources(&exact_split, 3));
+
+ short_cut.timeline.as_mut().unwrap().segments[2].start = 0.32;
+ assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3));
}
#[test]
@@ -1538,6 +1532,100 @@ mod tests {
assert_bounded_sink_variants([19], &[1]);
}
+ #[test]
+ fn short_studio_sound_cuts_match_preview_after_history_rolls() {
+ ffmpeg::init().unwrap();
+ let directory = tempfile::tempdir().unwrap();
+ let paths =
+ ["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name));
+ for ((path, frames), channels) in
+ paths.iter().zip([144_013, 96_007, 168_013]).zip([1, 2, 2])
+ {
+ write_wav(path, frames, channels);
+ }
+ let data = paths
+ .iter()
+ .map(|path| Arc::new(AudioData::from_file(path).unwrap()))
+ .collect::<Vec<_>>();
+ for (tier, isolation) in [
+ cap_project::VoiceIsolation::Light,
+ cap_project::VoiceIsolation::Balanced,
+ cap_project::VoiceIsolation::Strong,
+ ]
+ .into_iter()
+ .enumerate()
+ {
+ for offset in [0, 1] {
+ let mut project = project();
+ project.audio.improve = true;
+ project.audio.isolation = isolation;
+ if offset == 1 {
+ project.clips[0].offsets.mic = 0.137_125;
+ project.clips[0].offsets.system_audio = -0.031_25;
+ project.clips[1].offsets.mic = -0.117_125;
+ project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR;
+ }
+ project.timeline.as_mut().unwrap().segments = vec![
+ segment(0, 0.001_01, 0.72),
+ segment(0, 0.73, 0.8),
+ segment(0, 0.800_02, 0.92),
+ segment(0, 1.0, 1.63),
+ segment(1, 0.35, 1.4),
+ segment(1, 1.5, 2.2),
+ segment(0, 1.8, 2.5),
+ segment(0, 2.500_02, 3.05),
+ ];
+ let requests = [997, 4_096, 48_001]
+ .into_iter()
+ .chain((tier == 1 && offset == 0).then_some(1));
+ for request in requests {
+ let (mut reference, mut candidate) = fixture(&paths, &data);
+ reference.set_playhead(0.0, &project);
+ loop {
+ let expected = reference.render_frame_raw(request, &project);
+ let mut actual = Vec::new();
+ let written = candidate
+ .render_chunks(request, &project, |offset, samples| {
+ assert_eq!(offset * 2, actual.len());
+ assert!(samples.len() <= EXPORT_AUDIO_BLOCK_SAMPLES * 2);
+ actual.extend_from_slice(samples);
+ Ok(())
+ })
+ .unwrap_or_else(|error| {
+ panic!("{isolation:?}, offset {offset}, request {request}: {error}")
+ });
+ assert_eq!(written, expected.as_ref().map(|(count, _)| *count));
+ let expected = expected.as_ref().map_or(&[][..], |(_, data)| data);
+ assert_eq!(actual.len(), expected.len());
+ assert!(
+ actual
+ .iter()
+ .map(|sample| sample.to_bits())
+ .eq(expected.iter().map(|sample| sample.to_bits())),
+ "{isolation:?}, offset {offset}, request {request}"
+ );
+ for track in candidate.sources.tracks.iter().flatten() {
+ let limit = if track.enhancer.is_some() {
+ MAX_ENHANCED_WINDOW_SAMPLES
+ + VOICE_PREROLL_SAMPLES
+ + VOICE_WINDOW_PADDING_SAMPLES
+ } else {
+ EXPORT_AUDIO_BLOCK_SAMPLES
+ };
+ assert!(
+ track.samples.len() <= limit * track.source.channels() as usize
+ );
+ }
+ if written.is_none() {
+ break;
+ }
+ }
+ candidate.validate_to_end().unwrap();
+ }
+ }
+ }
+ }
+
#[test]
fn fullscreen_holds_match_full_renderer_without_retaining_whole_tracks() {
assert_bounded_sink_variants(26..32, &[1, 7, 997, 4_096, 48_001]);
@@ -1804,7 +1892,7 @@ mod tests {
assert!(
track.samples.len()
<= (if track.enhancer.is_some() {
- MAX_ENHANCED_WINDOW_SAMPLES
+ MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES
} else {
EXPORT_AUDIO_BLOCK_SAMPLES
}) * track.source.channels() as usize
@@ -2045,12 +2133,12 @@ mod tests {
let cancellation = Arc::new(AtomicBool::new(false));
let mut track =
ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap();
- track.prepare(8_000, 9_000).unwrap();
- track.prepare(8_500, 9_501).unwrap();
+ track.prepare(8_000, 9_000, 0).unwrap();
+ track.prepare(8_500, 9_501, 0).unwrap();
let full = AudioData::from_file(&path).unwrap();
assert_eq!(track.samples, full.samples()[8_500 * 2..9_501 * 2]);
assert!(matches!(
- track.prepare(0, 1),
+ track.prepare(0, 1, 0),
Err(ExportAudioError::InvalidWindow)
));
let mut candidate = ExportAudioRenderer {
@@ -2077,6 +2165,55 @@ mod tests {
);
}
+ #[test]
+ fn source_history_reuses_decoded_samples_and_stays_bounded() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("source.wav");
+ write_wav(&path, 144_013, 2);
+ let cancellation = Arc::new(AtomicBool::new(false));
+ let mut track =
+ ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap();
+ let full = AudioData::from_file(&path).unwrap();
+ track
+ .prepare(50_000, 52_000, STUDIO_SOUND_HISTORY_SAMPLES)
+ .unwrap();
+ let position = track.source.position();
+ let oldest = track.source_start;
+ track
+ .prepare(25_000, 27_000, STUDIO_SOUND_HISTORY_SAMPLES)
+ .unwrap();
+ assert_eq!(track.source.position(), position);
+ assert_eq!(track.source_start, oldest);
+ assert_eq!(
+ track.view().sample_slice(25_000 * 2..27_000 * 2).unwrap(),
+ &full.samples()[25_000 * 2..27_000 * 2]
+ );
+ assert!(matches!(
+ track.prepare(
+ oldest as i128 - 1,
+ oldest as i128,
+ STUDIO_SOUND_HISTORY_SAMPLES
+ ),
+ Err(ExportAudioError::InvalidWindow)
+ ));
+ for start in (60_000..150_000).step_by(997) {
+ track
+ .prepare(start, start + 1_632, STUDIO_SOUND_HISTORY_SAMPLES)
+ .unwrap();
+ let end = track.source.position() as usize;
+ assert_eq!(
+ track.samples,
+ full.samples()[track.source_start * 2..end * 2]
+ );
+ assert!(
+ track.samples.len()
+ <= (MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES) * 2
+ );
+ }
+ track.prepare(144_013, 144_014, 0).unwrap();
+ assert!(track.samples.is_empty());
+ }
+
#[test]
fn sink_failure_stops_before_later_source_reads() {
let directory = tempfile::tempdir().unwrap();
diff --git a/crates/export/examples/studio-sound-cut-benchmark.rs b/crates/export/examples/studio-sound-cut-benchmark.rs
new file mode 100644
index 000000000..8d4abab75
--- /dev/null
+++ b/crates/export/examples/studio-sound-cut-benchmark.rs
@@ -0,0 +1,118 @@
+use cap_export::{
+ ExporterBase,
+ mp4::{ExportCompression, Mp4ExportSettings},
+};
+use cap_project::{ClipConfiguration, ProjectConfiguration, RecordingMeta, TimelineSegment, XY};
+use std::{
+ path::PathBuf,
+ sync::{
+ Arc,
+ atomic::{AtomicBool, AtomicU64, Ordering},
+ },
+ time::Instant,
+};
+
+#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
+async fn main() {
+ let root = PathBuf::from(std::env::args_os().nth(1).expect("fixture directory"));
+ let output = PathBuf::from(std::env::args_os().nth(2).expect("output MP4 path"));
+ let gap: f64 = std::env::var("CAP_EXPORT_CUT_GAP").map_or(0.1, |v| v.parse().unwrap());
+ let mut project = ProjectConfiguration {
+ timeline: Some(
+ serde_json::from_value(serde_json::json!({"segments":[],"zoomSegments":[]})).unwrap(),
+ ),
+ ..Default::default()
+ };
+ project.audio.improve = true;
+ project.camera.hide = true;
+ project.cursor.hide = true;
+ project.cursor.raw = true;
+ project.clips = vec![ClipConfiguration {
+ index: 0,
+ ..Default::default()
+ }];
+ project.timeline.as_mut().unwrap().segments = vec![
+ TimelineSegment {
+ recording_clip: 0,
+ start: 0.0,
+ end: 4.0,
+ timescale: 1.0,
+ ..Default::default()
+ },
+ TimelineSegment {
+ recording_clip: 0,
+ start: 4.0 + gap,
+ end: 8.0 + gap,
+ timescale: 1.0,
+ ..Default::default()
+ },
+ ];
+ project.write(&root).unwrap();
+ let mut meta: RecordingMeta = serde_json::from_value(serde_json::json!({
+ "platform":"Linux", "pretty_name":"Studio Sound cut benchmark", "sharing":null,
+ "segments":[{
+ "display":{"path":"display.mp4","fps":1,"start_time":0.0},
+ "mic":{"path":"mic.wav","start_time":0.0}
+ }],
+ "cursors":{},"status":{"status":"Complete"}
+ }))
+ .unwrap();
+ meta.project_path = root.clone();
+ meta.save_for_project().unwrap();
+ let first_progress = Arc::new(AtomicU64::new(u64::MAX));
+ let recorded_progress = first_progress.clone();
+ let start = Instant::now();
+ let base = ExporterBase::builder(root)
+ .with_output_path(output.clone())
+ .with_force_ffmpeg_decoder(true)
+ .build_for_mp4(Arc::new(AtomicBool::new(false)))
+ .await
+ .expect("export preparation");
+ let build_seconds = start.elapsed().as_secs_f64();
+ let streaming = base.uses_streaming_audio();
+ let settings = Mp4ExportSettings {
+ fps: 15,
+ resolution_base: XY::new(640, 360),
+ compression: ExportCompression::Web,
+ custom_bpp: None,
+ force_ffmpeg_decoder: true,
+ optimize_filesize: false,
+ };
+ let export_start = Instant::now();
+ settings
+ .export_prepared(base, move |_| {
+ recorded_progress
+ .compare_exchange(
+ u64::MAX,
+ start.elapsed().as_micros() as u64,
+ Ordering::Relaxed,
+ Ordering::Relaxed,
+ )
+ .ok();
+ true
+ })
+ .await
+ .expect("complete export");
+ let export_seconds = export_start.elapsed().as_secs_f64();
+ let total_seconds = start.elapsed().as_secs_f64();
+ let maxrss_kib = std::fs::read_to_string("/proc/self/status")
+ .ok()
+ .and_then(|status| {
+ status.lines().find_map(|line| {
+ line.strip_prefix("VmHWM:")?
+ .split_whitespace()
+ .next()?
+ .parse::<u64>()
+ .ok()
+ })
+ });
+ println!(
+ "EXPORT_BENCH {}",
+ serde_json::json!({
+ "gap":gap,"streaming_audio":streaming,"build_seconds":build_seconds,
+ "first_progress_seconds":first_progress.load(Ordering::Relaxed) as f64 / 1_000_000.0,
+ "export_seconds":export_seconds,"total_seconds":total_seconds,"maxrss_kib":maxrss_kib,
+ "output":output
+ })
+ );
+}