Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(audio): replace per-sample transcription counts with source intervals

Dennis Khylkouski7606d1e6fcd6
diff --git a/crates/audio/examples/transcription-assembly-benchmark.rs b/crates/audio/examples/transcription-assembly-benchmark.rs
new file mode 100644
index 000000000..04beef561
--- /dev/null
+++ b/crates/audio/examples/transcription-assembly-benchmark.rs
@@ -0,0 +1,54 @@
+use cap_audio::{TranscriptionAudioSource, TranscriptionAudioTake, append_transcription_audio};
+use std::{error::Error, hint::black_box, io, time::Instant};
+
+fn main() -> Result<(), Box<dyn Error>> {
+ let mut args = std::env::args().skip(1);
+ let seconds: usize = args
+ .next()
+ .ok_or_else(|| io::Error::other("Expected duration in seconds and mic or mic-system"))?
+ .parse()?;
+ let mode = args.next().unwrap_or_else(|| "mic".to_string());
+ if !matches!(mode.as_str(), "mic" | "mic-system") {
+ return Err(io::Error::other("Expected mic or mic-system").into());
+ }
+ let rate = 48_000;
+ let frames = seconds * rate;
+ let samples = |channels| {
+ (0..frames * channels)
+ .map(|index| ((index % 4096) as f32 - 2048.0) / 2048.0)
+ .collect()
+ };
+ let mut sources = Vec::new();
+ if mode == "mic-system" {
+ sources.push(TranscriptionAudioSource {
+ samples: samples(2),
+ channels: 2,
+ offset_secs: 0.0,
+ });
+ }
+ sources.push(TranscriptionAudioSource {
+ samples: samples(1),
+ channels: 1,
+ offset_secs: if mode == "mic-system" { -0.125 } else { 0.0 },
+ });
+ let take = TranscriptionAudioTake {
+ display_duration_secs: seconds as f64,
+ sources,
+ };
+ let mut output = Vec::new();
+ let started = Instant::now();
+ append_transcription_audio(black_box(&mut output), black_box(&take), rate as u32)
+ .map_err(io::Error::other)?;
+ let elapsed = started.elapsed();
+ assert_eq!(output.len(), frames);
+ let checksum = black_box(&output).iter().fold(0_u64, |hash, sample| {
+ hash.wrapping_mul(31)
+ .wrapping_add(u64::from(sample.to_bits()))
+ });
+ println!(
+ "synthetic,{mode},{seconds}s,assembly_ms={:.3},samples={},checksum={checksum:016x}",
+ elapsed.as_secs_f64() * 1000.0,
+ output.len(),
+ );
+ Ok(())
+}
diff --git a/crates/audio/src/transcription_timing.rs b/crates/audio/src/transcription_timing.rs
index 1cbd50124..601a32573 100644
--- a/crates/audio/src/transcription_timing.rs
+++ b/crates/audio/src/transcription_timing.rs
@@ -37,15 +37,6 @@ fn offset_samples(offset_secs: f64, sample_rate: u32) -> Result<isize, String> {
Ok(samples.round() as isize)
}
-fn try_zeroed<T: Clone>(len: usize, value: T, label: &str) -> Result<Vec<T>, String> {
- let mut values = Vec::new();
- values
- .try_reserve_exact(len)
- .map_err(|_| format!("{label} is too large"))?;
- values.resize(len, value);
- Ok(values)
-}
-
pub fn assemble_transcription_audio(
takes: &[TranscriptionAudioTake],
sample_rate: u32,
@@ -80,7 +71,10 @@ pub fn append_transcription_audio(
output
.try_reserve_exact(frames)
.map_err(|_| "Transcription audio is too large".to_string())?;
- let mut counts = try_zeroed(frames, 0_u32, "Transcription audio")?;
+ let mut boundaries = Vec::new();
+ boundaries
+ .try_reserve_exact(take.sources.len() * 2)
+ .map_err(|_| "Transcription audio is too large".to_string())?;
let start = output.len();
output.resize(start + frames, 0.0);
let sums = &mut output[start..];
@@ -88,26 +82,40 @@ pub fn append_transcription_audio(
for source in &take.sources {
let offset = offset_samples(source.offset_secs, sample_rate)?;
let source_frames = source.samples.len() / source.channels;
- for source_frame in 0..source_frames {
- let destination = source_frame as i128 - offset as i128;
- if destination < 0 || destination >= frames as i128 {
- continue;
- }
-
- let first_sample = source_frame * source.channels;
- let mono = source.samples[first_sample..first_sample + source.channels]
- .iter()
- .copied()
- .sum::<f32>()
- / source.channels as f32;
- let destination = destination as usize;
- sums[destination] += mono;
- counts[destination] += 1;
+ let first = (-(offset as i128)).clamp(0, frames as i128) as usize;
+ let end = (source_frames as i128 - offset as i128).clamp(0, frames as i128) as usize;
+ if first == end {
+ continue;
+ }
+ boundaries.push((first, true));
+ boundaries.push((end, false));
+ let source_start = (first as i128 + offset as i128) as usize;
+ for (sum, frame) in sums[first..end].iter_mut().zip(
+ source
+ .samples
+ .chunks_exact(source.channels)
+ .skip(source_start),
+ ) {
+ *sum += frame.iter().copied().sum::<f32>() / source.channels as f32;
}
}
- for (sum, count) in sums.iter_mut().zip(counts) {
- *sum = if count == 0 { 0.0 } else { *sum / count as f32 };
+ // Each source covers one interval; its contribution count only changes at the ends.
+ boundaries.sort_unstable_by_key(|(position, _)| *position);
+ let mut count = 0_u32;
+ let mut start = 0;
+ for (end, starting) in boundaries {
+ if count > 0 {
+ for sample in &mut sums[start..end] {
+ *sample /= count as f32;
+ }
+ }
+ if starting {
+ count += 1;
+ } else {
+ count -= 1;
+ }
+ start = end;
}
Ok(())
}
@@ -116,7 +124,7 @@ pub fn append_transcription_audio(
mod tests {
use super::{
TranscriptionAudioSource, TranscriptionAudioTake, append_transcription_audio,
- assemble_transcription_audio,
+ assemble_transcription_audio, offset_samples, sample_count,
};
fn take(duration: f64, sources: Vec<TranscriptionAudioSource>) -> TranscriptionAudioTake {
@@ -233,4 +241,128 @@ mod tests {
assert!(assemble_transcription_audio(&invalid_duration, 48_000).is_err());
assert!(assemble_transcription_audio(&invalid_offset, 48_000).is_err());
}
+
+ fn reference(take: &TranscriptionAudioTake, rate: u32) -> Vec<f32> {
+ let frames = sample_count(take.display_duration_secs, rate, "Display duration").unwrap();
+ let mut sums = vec![0.0_f32; frames];
+ let mut counts = vec![0_u32; frames];
+ for source in &take.sources {
+ let offset = offset_samples(source.offset_secs, rate).unwrap();
+ for source_frame in 0..source.samples.len() / source.channels {
+ let destination = source_frame as i128 - offset as i128;
+ if destination < 0 || destination >= frames as i128 {
+ continue;
+ }
+ let first = source_frame * source.channels;
+ sums[destination as usize] += source.samples[first..first + source.channels]
+ .iter()
+ .copied()
+ .sum::<f32>()
+ / source.channels as f32;
+ counts[destination as usize] += 1;
+ }
+ }
+ sums.into_iter()
+ .zip(counts)
+ .map(|(sum, count)| if count == 0 { 0.0 } else { sum / count as f32 })
+ .collect()
+ }
+
+ fn bits(samples: &[f32]) -> Vec<u32> {
+ samples.iter().map(|sample| sample.to_bits()).collect()
+ }
+
+ #[test]
+ fn interval_counts_match_per_sample_counts_and_preserve_previous_takes() {
+ let values = [
+ 0.0,
+ -0.0,
+ 0.5,
+ -0.25,
+ f32::INFINITY,
+ f32::NEG_INFINITY,
+ f32::NAN,
+ ];
+ for channels in [1, 2, 3, 6] {
+ for frames in [0, 1, 9, 32] {
+ for offset in [-2.0, -0.5, 0.0, 0.25, 1.0, 4.0] {
+ for source_count in 0..=3 {
+ let take = take(
+ frames as f64 / 8.0,
+ (0..source_count)
+ .map(|index| {
+ source(
+ (0..(frames + index) * channels)
+ .map(|i| values[(i + index) % values.len()])
+ .collect(),
+ channels,
+ offset + index as f64 / 8.0,
+ )
+ })
+ .collect(),
+ );
+ let mut expected = vec![13.0, -0.0, f32::from_bits(0x7fc0_1234)];
+ let mut actual = expected.clone();
+ for _ in 0..3 {
+ expected.extend(reference(&take, 8));
+ append_transcription_audio(&mut actual, &take, 8).unwrap();
+ assert_eq!(actual.len(), expected.len());
+ assert_eq!(bits(&actual[..3]), bits(&expected[..3]));
+ for (actual, expected) in actual[3..].iter().zip(&expected[3..]) {
+ if expected.is_nan() {
+ assert!(actual.is_nan());
+ } else {
+ assert_eq!(actual.to_bits(), expected.to_bits());
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn adjacent_overlapping_and_disjoint_source_boundaries_keep_exact_averages() {
+ let take = take(
+ 10.0,
+ vec![
+ source(vec![1.0; 4], 1, 0.0),
+ source(vec![3.0; 4], 1, -2.0),
+ source(vec![5.0; 2], 1, -4.0),
+ source(vec![7.0; 1], 1, -8.0),
+ source(Vec::new(), 1, -4.0),
+ source(vec![9.0; 2], 1, 3.0),
+ ],
+ );
+ let expected = vec![1.0, 1.0, 2.0, 2.0, 4.0, 4.0, 0.0, 0.0, 7.0, 0.0];
+ assert_eq!(assemble_transcription_audio(&[take], 1).unwrap(), expected);
+ }
+
+ #[test]
+ fn invalid_append_leaves_existing_samples_unchanged() {
+ for (duration, channels, offset, rate, samples) in [
+ (1.0, 1, 0.0, 0, vec![1.0]),
+ (-1.0, 1, 0.0, 8, vec![1.0]),
+ (f64::NAN, 1, 0.0, 8, vec![1.0]),
+ (f64::MAX, 1, 0.0, 8, vec![1.0]),
+ (1.0, 0, 0.0, 8, vec![1.0]),
+ (1.0, 2, 0.0, 8, vec![1.0]),
+ (1.0, 1, f64::NAN, 8, vec![1.0]),
+ (1.0, 1, f64::INFINITY, 8, vec![1.0]),
+ (1.0, 1, f64::MAX, 8, vec![1.0]),
+ ] {
+ let mut output = vec![0.25, -0.0];
+ let expected = bits(&output);
+ let take = take(
+ duration,
+ vec![
+ source(vec![1.0; 8], 1, 0.0),
+ source(samples, channels, offset),
+ ],
+ );
+ assert!(append_transcription_audio(&mut output, &take, rate).is_err());
+ assert_eq!(bits(&output), expected);
+ }
+ }
}