Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(audio): transcribe short speech in long chunks by measuring speech ratio over a 5 s window (SCR-583) (#7621)

Ezra Ellettecac3530acb07
diff --git a/crates/screenpipe-audio/src/speaker/prepare_segments.rs b/crates/screenpipe-audio/src/speaker/prepare_segments.rs
index 56f1f4d9e..b815285be 100644
--- a/crates/screenpipe-audio/src/speaker/prepare_segments.rs
+++ b/crates/screenpipe-audio/src/speaker/prepare_segments.rs
@@ -54,6 +54,7 @@ pub async fn prepare_segments(
let mut audio_frames = Vec::new();
let mut total_frames = 0;
let mut speech_frame_count = 0;
+ let mut speech_frames = Vec::new();
let mut final_frame = vec![0.0; frame_size];
for chunk in audio_data.chunks(frame_size) {
@@ -69,11 +70,13 @@ pub async fn prepare_segments(
chunk
};
let status = vad_engine.lock().await.audio_type(vad_frame);
+ let mut is_speech = false;
match status {
Ok(VadStatus::Speech) => {
if let Ok(processed_audio) = spectral_subtraction(chunk, noise) {
new_chunk = processed_audio;
speech_frame_count += 1;
+ is_speech = true;
}
}
Ok(VadStatus::Unknown) => {
@@ -81,6 +84,7 @@ pub async fn prepare_segments(
}
_ => {}
}
+ speech_frames.push(is_speech);
audio_frames.extend(new_chunk);
}
@@ -89,7 +93,25 @@ pub async fn prepare_segments(
vad_engine.lock().await.set_speech_threshold(None);
}
- let speech_ratio = speech_frame_count as f32 / total_frames as f32;
+ // Measure density over a fixed window so a short word is admitted the same
+ // way whether it lands in a 5 s chunk or a 30 s one.
+ let window_frames = (5 * 16000usize).div_ceil(frame_size);
+ let chunk_ratio = speech_frame_count as f32 / total_frames as f32;
+ let speech_ratio = if speech_frames.len() <= window_frames {
+ chunk_ratio
+ } else {
+ let mut window_speech = speech_frames[..window_frames]
+ .iter()
+ .filter(|speech| **speech)
+ .count();
+ let mut densest = window_speech;
+ for frame in window_frames..speech_frames.len() {
+ window_speech += usize::from(speech_frames[frame]);
+ window_speech -= usize::from(speech_frames[frame - window_frames]);
+ densest = densest.max(window_speech);
+ }
+ chunk_ratio.max(densest as f32 / window_frames as f32)
+ };
let current_min_ratio = crate::vad::min_speech_ratio();
debug!(
"device: {}, speech ratio: {}, min_speech_ratio: {}, audio_frames: {}, speech_frames: {}",
diff --git a/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs b/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs
index ade1af73d..38813a4ce 100644
--- a/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs
+++ b/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs
@@ -64,6 +64,101 @@ async fn short_cue_between_native_windows_reaches_transcription() {
assert!(calls.iter().all(|call| call.len() == 512));
}
+fn burst_audio(total_frames: usize, speech_frames: &[usize]) -> Vec<f32> {
+ let mut audio = vec![0.0; total_frames * 512];
+ for &frame in speech_frames {
+ for (i, sample) in audio[frame * 512..(frame + 1) * 512].iter_mut().enumerate() {
+ *sample = if i % 2 == 0 { 0.5 } else { -0.5 };
+ }
+ }
+ audio
+}
+
+async fn admit(audio: &[f32]) -> (bool, usize, f32) {
+ let vad: Arc<tokio::sync::Mutex<Box<dyn VadEngine + Send>>> =
+ Arc::new(tokio::sync::Mutex::new(Box::new(WindowVad {
+ seen: Arc::new(Mutex::new(Vec::new())),
+ })));
+ let (mut segments, accepted, ratio) = prepare_segments(
+ audio,
+ vad,
+ None,
+ Arc::new(Mutex::new(EmbeddingManager::new(4))),
+ None,
+ "synthetic microphone",
+ false,
+ false,
+ )
+ .await
+ .unwrap();
+ let samples = match segments.recv().await {
+ Some(segment) => segment.samples.len(),
+ None => 0,
+ };
+ (accepted, samples, ratio)
+}
+
+#[tokio::test]
+async fn short_burst_in_a_30s_chunk_is_admitted_with_the_full_chunk() {
+ let audio = burst_audio(30 * 16000 / 512, &[400, 401, 402, 403, 404]);
+ let (accepted, samples, ratio) = admit(&audio).await;
+ assert!(
+ accepted,
+ "a short spoken word must not be diluted by chunk length, ratio={ratio}"
+ );
+ assert_eq!(samples, audio.len());
+}
+
+#[tokio::test]
+async fn short_burst_in_a_5s_chunk_stays_admitted() {
+ let audio = burst_audio(5 * 16000 / 512, &[60, 61, 62, 63, 64]);
+ let (accepted, samples, ratio) = admit(&audio).await;
+ assert!(accepted, "control chunk must stay admitted, ratio={ratio}");
+ assert_eq!(samples, audio.len());
+}
+
+#[tokio::test]
+async fn exact_5s_chunk_keeps_the_whole_chunk_ratio() {
+ let mut audio = burst_audio(157, &[0, 1, 2, 156]);
+ audio.truncate(5 * 16000);
+ let (accepted, samples, ratio) = admit(&audio).await;
+ assert!(accepted, "a 5 s chunk must behave as before, ratio={ratio}");
+ assert_eq!(ratio, 4.0 / 157.0);
+ assert_eq!(samples, audio.len());
+}
+
+#[tokio::test]
+async fn speech_at_both_ends_of_a_6s_chunk_stays_admitted() {
+ let audio = burst_audio(200, &[0, 1, 2, 197, 198, 199]);
+ let (accepted, samples, ratio) = admit(&audio).await;
+ assert!(
+ accepted,
+ "nothing main admits may be rejected, ratio={ratio}"
+ );
+ assert_eq!(ratio, 6.0 / 200.0);
+ assert_eq!(samples, audio.len());
+}
+
+#[tokio::test]
+async fn speech_too_thin_for_any_window_is_still_rejected() {
+ let total_frames = 30 * 16000 / 512;
+ let sparse: Vec<usize> = (0..total_frames).step_by(93).collect();
+ let (accepted, samples, ratio) = admit(&burst_audio(total_frames, &sparse)).await;
+ assert!(
+ !accepted,
+ "speech spread below the density threshold must stay rejected, ratio={ratio}"
+ );
+ assert_eq!(samples, 0);
+}
+
+#[tokio::test]
+async fn silent_30s_chunk_is_rejected() {
+ let (accepted, samples, ratio) = admit(&burst_audio(30 * 16000 / 512, &[])).await;
+ assert!(!accepted, "silence must stay rejected");
+ assert_eq!(ratio, 0.0);
+ assert_eq!(samples, 0);
+}
+
#[test]
fn webrtc_accepts_30ms_of_16khz_audio() {
let mut vad = WebRtcVad::new();
diff --git a/crates/screenpipe-audio/tests/vad_min_speech_ratio_gate_test.rs b/crates/screenpipe-audio/tests/vad_min_speech_ratio_gate_test.rs
new file mode 100644
index 000000000..002b41d47
--- /dev/null
+++ b/crates/screenpipe-audio/tests/vad_min_speech_ratio_gate_test.rs
@@ -0,0 +1,53 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+use screenpipe_audio::speaker::{embedding_manager::EmbeddingManager, prepare_segments};
+use screenpipe_audio::vad::{reset_min_speech_ratio, set_min_speech_ratio, VadEngine};
+use std::sync::{Arc, Mutex};
+use vad_rs::VadStatus;
+
+struct LoudIsSpeech;
+impl VadEngine for LoudIsSpeech {
+ fn is_voice_segment(&mut self, audio: &[f32]) -> anyhow::Result<bool> {
+ Ok(matches!(self.audio_type(audio)?, VadStatus::Speech))
+ }
+ fn audio_type(&mut self, audio: &[f32]) -> anyhow::Result<VadStatus> {
+ Ok(if audio.iter().any(|x| x.abs() > 0.1) {
+ VadStatus::Speech
+ } else {
+ VadStatus::Silence
+ })
+ }
+ fn set_speech_threshold(&mut self, _: Option<f32>) {}
+}
+
+/// Owns the process-global min speech ratio: keep it the only test in this binary.
+#[tokio::test]
+async fn ratio_of_one_blocks_even_a_fully_spoken_chunk() {
+ let audio: Vec<f32> = (0..30 * 16000)
+ .map(|i| if i % 2 == 0 { 0.5 } else { -0.5 })
+ .collect();
+ let vad: Arc<tokio::sync::Mutex<Box<dyn VadEngine + Send>>> =
+ Arc::new(tokio::sync::Mutex::new(Box::new(LoudIsSpeech)));
+
+ set_min_speech_ratio(1.0);
+ let result = prepare_segments(
+ &audio,
+ vad,
+ None,
+ Arc::new(Mutex::new(EmbeddingManager::new(4))),
+ None,
+ "synthetic microphone",
+ false,
+ false,
+ )
+ .await;
+ reset_min_speech_ratio();
+
+ let (mut segments, accepted, ratio) = result.unwrap();
+ assert!(
+ !accepted,
+ "audio_disabled must block all audio, ratio={ratio}"
+ );
+ assert!(segments.recv().await.is_none());
+}