YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(audio): transcribe short speech in long chunks by measuring speech ratio over a 5 s window (SCR-583) (#7621)
Ezra Ellettecac3530acb07
diff --git a/crates/screenpipe-audio/src/speaker/prepare_segments.rs b/crates/screenpipe-audio/src/speaker/prepare_segments.rsindex 56f1f4d9e..b815285be 100644--- a/crates/screenpipe-audio/src/speaker/prepare_segments.rs+++ b/crates/screenpipe-audio/src/speaker/prepare_segments.rs@@ -54,6 +54,7 @@ pub async fn prepare_segments(let mut audio_frames = Vec::new();let mut total_frames = 0;let mut speech_frame_count = 0;+ let mut speech_frames = Vec::new();let mut final_frame = vec![0.0; frame_size];for chunk in audio_data.chunks(frame_size) {@@ -69,11 +70,13 @@ pub async fn prepare_segments(chunk};let status = vad_engine.lock().await.audio_type(vad_frame);+ let mut is_speech = false;match status {Ok(VadStatus::Speech) => {if let Ok(processed_audio) = spectral_subtraction(chunk, noise) {new_chunk = processed_audio;speech_frame_count += 1;+ is_speech = true;}}Ok(VadStatus::Unknown) => {@@ -81,6 +84,7 @@ pub async fn prepare_segments(}_ => {}}+ speech_frames.push(is_speech);audio_frames.extend(new_chunk);}@@ -89,7 +93,25 @@ pub async fn prepare_segments(vad_engine.lock().await.set_speech_threshold(None);}- let speech_ratio = speech_frame_count as f32 / total_frames as f32;+ // Measure density over a fixed window so a short word is admitted the same+ // way whether it lands in a 5 s chunk or a 30 s one.+ let window_frames = (5 * 16000usize).div_ceil(frame_size);+ let chunk_ratio = speech_frame_count as f32 / total_frames as f32;+ let speech_ratio = if speech_frames.len() <= window_frames {+ chunk_ratio+ } else {+ let mut window_speech = speech_frames[..window_frames]+ .iter()+ .filter(|speech| **speech)+ .count();+ let mut densest = window_speech;+ for frame in window_frames..speech_frames.len() {+ window_speech += usize::from(speech_frames[frame]);+ window_speech -= usize::from(speech_frames[frame - window_frames]);+ densest = densest.max(window_speech);+ }+ chunk_ratio.max(densest as f32 / window_frames as f32)+ };let current_min_ratio = crate::vad::min_speech_ratio();debug!("device: {}, speech ratio: {}, min_speech_ratio: {}, audio_frames: {}, speech_frames: {}",diff --git a/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs b/crates/screenpipe-audio/tests/vad_frame_continuity_test.rsindex ade1af73d..38813a4ce 100644--- a/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs+++ b/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs@@ -64,6 +64,101 @@ async fn short_cue_between_native_windows_reaches_transcription() {assert!(calls.iter().all(|call| call.len() == 512));}+fn burst_audio(total_frames: usize, speech_frames: &[usize]) -> Vec<f32> {+ let mut audio = vec![0.0; total_frames * 512];+ for &frame in speech_frames {+ for (i, sample) in audio[frame * 512..(frame + 1) * 512].iter_mut().enumerate() {+ *sample = if i % 2 == 0 { 0.5 } else { -0.5 };+ }+ }+ audio+}++async fn admit(audio: &[f32]) -> (bool, usize, f32) {+ let vad: Arc<tokio::sync::Mutex<Box<dyn VadEngine + Send>>> =+ Arc::new(tokio::sync::Mutex::new(Box::new(WindowVad {+ seen: Arc::new(Mutex::new(Vec::new())),+ })));+ let (mut segments, accepted, ratio) = prepare_segments(+ audio,+ vad,+ None,+ Arc::new(Mutex::new(EmbeddingManager::new(4))),+ None,+ "synthetic microphone",+ false,+ false,+ )+ .await+ .unwrap();+ let samples = match segments.recv().await {+ Some(segment) => segment.samples.len(),+ None => 0,+ };+ (accepted, samples, ratio)+}++#[tokio::test]+async fn short_burst_in_a_30s_chunk_is_admitted_with_the_full_chunk() {+ let audio = burst_audio(30 * 16000 / 512, &[400, 401, 402, 403, 404]);+ let (accepted, samples, ratio) = admit(&audio).await;+ assert!(+ accepted,+ "a short spoken word must not be diluted by chunk length, ratio={ratio}"+ );+ assert_eq!(samples, audio.len());+}++#[tokio::test]+async fn short_burst_in_a_5s_chunk_stays_admitted() {+ let audio = burst_audio(5 * 16000 / 512, &[60, 61, 62, 63, 64]);+ let (accepted, samples, ratio) = admit(&audio).await;+ assert!(accepted, "control chunk must stay admitted, ratio={ratio}");+ assert_eq!(samples, audio.len());+}++#[tokio::test]+async fn exact_5s_chunk_keeps_the_whole_chunk_ratio() {+ let mut audio = burst_audio(157, &[0, 1, 2, 156]);+ audio.truncate(5 * 16000);+ let (accepted, samples, ratio) = admit(&audio).await;+ assert!(accepted, "a 5 s chunk must behave as before, ratio={ratio}");+ assert_eq!(ratio, 4.0 / 157.0);+ assert_eq!(samples, audio.len());+}++#[tokio::test]+async fn speech_at_both_ends_of_a_6s_chunk_stays_admitted() {+ let audio = burst_audio(200, &[0, 1, 2, 197, 198, 199]);+ let (accepted, samples, ratio) = admit(&audio).await;+ assert!(+ accepted,+ "nothing main admits may be rejected, ratio={ratio}"+ );+ assert_eq!(ratio, 6.0 / 200.0);+ assert_eq!(samples, audio.len());+}++#[tokio::test]+async fn speech_too_thin_for_any_window_is_still_rejected() {+ let total_frames = 30 * 16000 / 512;+ let sparse: Vec<usize> = (0..total_frames).step_by(93).collect();+ let (accepted, samples, ratio) = admit(&burst_audio(total_frames, &sparse)).await;+ assert!(+ !accepted,+ "speech spread below the density threshold must stay rejected, ratio={ratio}"+ );+ assert_eq!(samples, 0);+}++#[tokio::test]+async fn silent_30s_chunk_is_rejected() {+ let (accepted, samples, ratio) = admit(&burst_audio(30 * 16000 / 512, &[])).await;+ assert!(!accepted, "silence must stay rejected");+ assert_eq!(ratio, 0.0);+ assert_eq!(samples, 0);+}+#[test]fn webrtc_accepts_30ms_of_16khz_audio() {let mut vad = WebRtcVad::new();diff --git a/crates/screenpipe-audio/tests/vad_min_speech_ratio_gate_test.rs b/crates/screenpipe-audio/tests/vad_min_speech_ratio_gate_test.rsnew file mode 100644index 000000000..002b41d47--- /dev/null+++ b/crates/screenpipe-audio/tests/vad_min_speech_ratio_gate_test.rs@@ -0,0 +1,53 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_audio::speaker::{embedding_manager::EmbeddingManager, prepare_segments};+use screenpipe_audio::vad::{reset_min_speech_ratio, set_min_speech_ratio, VadEngine};+use std::sync::{Arc, Mutex};+use vad_rs::VadStatus;++struct LoudIsSpeech;+impl VadEngine for LoudIsSpeech {+ fn is_voice_segment(&mut self, audio: &[f32]) -> anyhow::Result<bool> {+ Ok(matches!(self.audio_type(audio)?, VadStatus::Speech))+ }+ fn audio_type(&mut self, audio: &[f32]) -> anyhow::Result<VadStatus> {+ Ok(if audio.iter().any(|x| x.abs() > 0.1) {+ VadStatus::Speech+ } else {+ VadStatus::Silence+ })+ }+ fn set_speech_threshold(&mut self, _: Option<f32>) {}+}++/// Owns the process-global min speech ratio: keep it the only test in this binary.+#[tokio::test]+async fn ratio_of_one_blocks_even_a_fully_spoken_chunk() {+ let audio: Vec<f32> = (0..30 * 16000)+ .map(|i| if i % 2 == 0 { 0.5 } else { -0.5 })+ .collect();+ let vad: Arc<tokio::sync::Mutex<Box<dyn VadEngine + Send>>> =+ Arc::new(tokio::sync::Mutex::new(Box::new(LoudIsSpeech)));++ set_min_speech_ratio(1.0);+ let result = prepare_segments(+ &audio,+ vad,+ None,+ Arc::new(Mutex::new(EmbeddingManager::new(4))),+ None,+ "synthetic microphone",+ false,+ false,+ )+ .await;+ reset_min_speech_ratio();++ let (mut segments, accepted, ratio) = result.unwrap();+ assert!(+ !accepted,+ "audio_disabled must block all audio, ratio={ratio}"+ );+ assert!(segments.recv().await.is_none());+}