YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(audio): feed contiguous native frames to speech detection (#7365)
Louis Beaumont6e531145bc85
diff --git a/crates/screenpipe-audio/src/speaker/prepare_segments.rs b/crates/screenpipe-audio/src/speaker/prepare_segments.rsindex 185dd715a..56f1f4d9e 100644--- a/crates/screenpipe-audio/src/speaker/prepare_segments.rs+++ b/crates/screenpipe-audio/src/speaker/prepare_segments.rs@@ -35,14 +35,9 @@ pub async fn prepare_segments(filter_music_frames(&mut audio_data);}- // Silero VAD v5 expects continuous 512-sample chunks at 16kHz (32ms).- // On Windows, WASAPI delivers lower audio levels than CoreAudio, so we- // must feed Silero at its native frame size to preserve its LSTM temporal- // state — using 1600 caused 68ms gaps that broke speech detection.- #[cfg(target_os = "windows")]- let frame_size = 512;- #[cfg(not(target_os = "windows"))]- let frame_size = 1600;+ // Feed every sample to the selected engine at its native window size.+ // Oversized Silero input is truncated internally, creating detection gaps.+ let frame_size = vad_engine.lock().await.frame_size();let vad_engine = vad_engine.clone();// Use a lower speech threshold for output/system audio devices.@@ -60,11 +55,20 @@ pub async fn prepare_segments(let mut total_frames = 0;let mut speech_frame_count = 0;+ let mut final_frame = vec![0.0; frame_size];for chunk in audio_data.chunks(frame_size) {total_frames += 1;let mut new_chunk = chunk.to_vec();- let status = vad_engine.lock().await.audio_type(chunk);+ // Pad only the final analysis window. Keep the original samples and+ // duration for persistence/transcription, including a short final cue.+ let vad_frame = if chunk.len() < frame_size {+ final_frame[..chunk.len()].copy_from_slice(chunk);+ &final_frame[..]+ } else {+ chunk+ };+ let status = vad_engine.lock().await.audio_type(vad_frame);match status {Ok(VadStatus::Speech) => {if let Ok(processed_audio) = spectral_subtraction(chunk, noise) {diff --git a/crates/screenpipe-audio/src/vad/mod.rs b/crates/screenpipe-audio/src/vad/mod.rsindex d387ee1ab..085c873ec 100644--- a/crates/screenpipe-audio/src/vad/mod.rs+++ b/crates/screenpipe-audio/src/vad/mod.rs@@ -52,6 +52,12 @@ pub enum VadEngineEnum {pub const OUTPUT_SPEECH_THRESHOLD: f32 = 0.15;pub trait VadEngine: Send {+ /// Native analysis window at the pipeline's 16 kHz sample rate.+ /// Silero requires 512 contiguous samples; other engines may override it.+ fn frame_size(&self) -> usize {+ 512+ }+fn is_voice_segment(&mut self, audio_chunk: &[f32]) -> anyhow::Result<bool>;fn audio_type(&mut self, audio_chunk: &[f32]) -> anyhow::Result<VadStatus>;/// Override the speech probability threshold. Call with `None` to reset to default.diff --git a/crates/screenpipe-audio/src/vad/webrtc.rs b/crates/screenpipe-audio/src/vad/webrtc.rsindex c67fef19a..aaee13975 100644--- a/crates/screenpipe-audio/src/vad/webrtc.rs+++ b/crates/screenpipe-audio/src/vad/webrtc.rs@@ -1,20 +1,32 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+use anyhow;use vad_rs::VadStatus;use super::VadEngine;-#[derive(Default)]pub struct WebRtcVad {vad: webrtc_vad::Vad,}impl WebRtcVad {pub fn new() -> Self {- let vad = webrtc_vad::Vad::new();+ let vad = webrtc_vad::Vad::new_with_rate(webrtc_vad::SampleRate::Rate16kHz);Self { vad }}}+impl Default for WebRtcVad {+ fn default() -> Self {+ Self::new()+ }+}impl VadEngine for WebRtcVad {+ fn frame_size(&self) -> usize {+ // WebRTC accepts 10, 20 or 30 ms frames, not Silero's 32 ms window.+ 480+ }+fn is_voice_segment(&mut self, audio_chunk: &[f32]) -> anyhow::Result<bool> {// Convert f32 to i16let i16_chunk: Vec<i16> = audio_chunk.iter().map(|&x| (x * 32767.0) as i16).collect();diff --git a/crates/screenpipe-audio/tests/audio_pipeline_benchmark/cross_device_benchmark.rs b/crates/screenpipe-audio/tests/audio_pipeline_benchmark/cross_device_benchmark.rsindex a2fa7fa2b..9618626e4 100644--- a/crates/screenpipe-audio/tests/audio_pipeline_benchmark/cross_device_benchmark.rs+++ b/crates/screenpipe-audio/tests/audio_pipeline_benchmark/cross_device_benchmark.rs@@ -17,15 +17,12 @@ use screenpipe_audio::vad::silero::SileroVad;use screenpipe_audio::vad::VadEngine;use vad_rs::VadStatus;-/// Frame size matching prepare_segments.rs.-const FRAME_SIZE: usize = 1600;-/// Compute speech_ratio for an audio segment.fn compute_speech_ratio(audio: &[f32], vad: &mut SileroVad) -> f32 {let mut total_frames = 0u32;let mut speech_frames = 0u32;- for chunk in audio.chunks(FRAME_SIZE) {+ for chunk in audio.chunks(vad.frame_size()) {total_frames += 1;if let Ok(VadStatus::Speech) = vad.audio_type(chunk) {speech_frames += 1;diff --git a/crates/screenpipe-audio/tests/audio_pipeline_benchmark/pipeline_benchmark.rs b/crates/screenpipe-audio/tests/audio_pipeline_benchmark/pipeline_benchmark.rsindex d761e0ed3..caf65d22e 100644--- a/crates/screenpipe-audio/tests/audio_pipeline_benchmark/pipeline_benchmark.rs+++ b/crates/screenpipe-audio/tests/audio_pipeline_benchmark/pipeline_benchmark.rs@@ -22,9 +22,6 @@ use vad_rs::VadStatus;use std::sync::Arc;-/// Frame size matching prepare_segments.rs.-const FRAME_SIZE: usize = 1600;-/// Chunk duration in seconds (matching production pipeline).const CHUNK_DURATION_SECS: f64 = 30.0;@@ -59,7 +56,7 @@ fn simulate_pipeline(let mut total_frames = 0u32;let mut speech_frames = 0u32;- for frame in normalized.chunks(FRAME_SIZE) {+ for frame in normalized.chunks(vad.frame_size()) {total_frames += 1;if let Ok(VadStatus::Speech) = vad.audio_type(frame) {speech_frames += 1;diff --git a/crates/screenpipe-audio/tests/audio_pipeline_benchmark/vad_benchmark.rs b/crates/screenpipe-audio/tests/audio_pipeline_benchmark/vad_benchmark.rsindex 2245f1e85..efc919985 100644--- a/crates/screenpipe-audio/tests/audio_pipeline_benchmark/vad_benchmark.rs+++ b/crates/screenpipe-audio/tests/audio_pipeline_benchmark/vad_benchmark.rs@@ -24,9 +24,6 @@ use vad_rs::VadStatus;/// The thresholds to sweep. Includes the current production value (0.05).const THRESHOLDS: &[f32] = &[0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.10, 0.20];-/// Frame size matching prepare_segments.rs (100ms at 16kHz).-const FRAME_SIZE: usize = 1600;-/// Chunk duration matching the audio pipeline (30 seconds).const CHUNK_DURATION_SECS: f64 = 30.0;@@ -52,7 +49,7 @@ fn compute_speech_ratio(audio_chunk: &[f32], vad: &mut SileroVad) -> f32 {let mut total_frames = 0u32;let mut speech_frame_count = 0u32;- for chunk in audio_chunk.chunks(FRAME_SIZE) {+ for chunk in audio_chunk.chunks(vad.frame_size()) {total_frames += 1;let status = vad.audio_type(chunk);if let Ok(VadStatus::Speech) = status {diff --git a/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs b/crates/screenpipe-audio/tests/vad_frame_continuity_test.rsnew file mode 100644index 000000000..ade1af73d--- /dev/null+++ b/crates/screenpipe-audio/tests/vad_frame_continuity_test.rs@@ -0,0 +1,186 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_audio::speaker::{embedding_manager::EmbeddingManager, prepare_segments};+use screenpipe_audio::vad::{webrtc::WebRtcVad, VadEngine};+use std::sync::{Arc, Mutex};+use vad_rs::VadStatus;++// Model the native Silero window: samples beyond its first 512 never reach+// inference. Put a short cue in the portion the old macOS frame loop skipped.+struct WindowVad {+ seen: Arc<Mutex<Vec<Vec<f32>>>>,+}+impl VadEngine for WindowVad {+ fn is_voice_segment(&mut self, audio: &[f32]) -> anyhow::Result<bool> {+ Ok(matches!(self.audio_type(audio)?, VadStatus::Speech))+ }+ fn audio_type(&mut self, audio: &[f32]) -> anyhow::Result<VadStatus> {+ self.seen.lock().unwrap().push(audio.to_vec());+ Ok(if audio.iter().take(512).any(|x| x.abs() > 0.1) {+ VadStatus::Speech+ } else {+ VadStatus::Silence+ })+ }+ fn set_speech_threshold(&mut self, _: Option<f32>) {}+}++#[tokio::test]+async fn short_cue_between_native_windows_reaches_transcription() {+ let seen = Arc::new(Mutex::new(Vec::new()));+ let vad: Arc<tokio::sync::Mutex<Box<dyn VadEngine + Send>>> =+ Arc::new(tokio::sync::Mutex::new(Box::new(WindowVad {+ seen: seen.clone(),+ })));+ let mut audio = vec![0.0; 1601];+ for (i, sample) in audio[600..1000].iter_mut().enumerate() {+ *sample = if i % 2 == 0 { 0.5 } else { -0.5 };+ }+ let (mut segments, accepted, _) = prepare_segments(+ &audio,+ vad,+ None,+ Arc::new(Mutex::new(EmbeddingManager::new(4))),+ None,+ "synthetic microphone",+ false,+ false,+ )+ .await+ .unwrap();+ assert!(+ accepted,+ "a cue outside the first 512 samples must not be skipped"+ );+ let segment = segments+ .recv()+ .await+ .expect("retain the full audio for transcription");+ assert_eq!(segment.samples.len(), audio.len());+ assert!(segments.recv().await.is_none());+ let calls = seen.lock().unwrap();+ assert_eq!(calls.len(), 4);+ assert!(calls.iter().all(|call| call.len() == 512));+}++#[test]+fn webrtc_accepts_30ms_of_16khz_audio() {+ let mut vad = WebRtcVad::new();+ assert!(+ vad.audio_type(&[0.0; 480]).is_ok(),+ "fallback must use the pipeline's 16 kHz sample rate"+ );+}++struct CheckedWebRtc {+ inner: WebRtcVad,+ calls: Arc<Mutex<usize>>,+}+impl VadEngine for CheckedWebRtc {+ fn frame_size(&self) -> usize {+ self.inner.frame_size()+ }+ fn is_voice_segment(&mut self, audio: &[f32]) -> anyhow::Result<bool> {+ self.inner.is_voice_segment(audio)+ }+ fn audio_type(&mut self, audio: &[f32]) -> anyhow::Result<VadStatus> {+ assert_eq!(audio.len(), 480);+ let result = self.inner.audio_type(audio);+ assert!(+ result.is_ok(),+ "fallback must receive a supported frame, including the tail"+ );+ *self.calls.lock().unwrap() += 1;+ result+ }+ fn set_speech_threshold(&mut self, value: Option<f32>) {+ self.inner.set_speech_threshold(value)+ }+}++#[tokio::test]+async fn fallback_processes_a_partial_final_frame_without_invalid_length_errors() {+ let calls = Arc::new(Mutex::new(0));+ let vad: Arc<tokio::sync::Mutex<Box<dyn VadEngine + Send>>> =+ Arc::new(tokio::sync::Mutex::new(Box::new(CheckedWebRtc {+ inner: WebRtcVad::default(),+ calls: calls.clone(),+ })));+ let (mut segments, accepted, ratio) = prepare_segments(+ &[0.0; 1601],+ vad,+ None,+ Arc::new(Mutex::new(EmbeddingManager::new(4))),+ None,+ "synthetic microphone",+ false,+ false,+ )+ .await+ .unwrap();+ assert!(!accepted);+ assert_eq!(ratio, 0.0);+ assert!(segments.recv().await.is_none());+ assert_eq!(*calls.lock().unwrap(), 4);+}++/// Bounded native-model measurement on a public repository speech fixture.+/// Set VAD_SPEECH_FIXTURE to the Git LFS contents of test_data/accuracy1.wav.+/// Reports timing without a machine-dependent latency assertion.+#[tokio::test]+#[ignore = "requires the Silero model and a decoded Git LFS speech fixture"]+async fn native_silero_speech_continuity_cost() {+ use screenpipe_audio::vad::silero::SileroVad;+ use std::time::Instant;+ let path = std::env::var("VAD_SPEECH_FIXTURE").expect("set VAD_SPEECH_FIXTURE");+ let (audio, rate) = screenpipe_audio::pcm_decode(std::path::Path::new(&path)).unwrap();+ let mut audio = screenpipe_audio::resample(&audio, rate, 16000).unwrap();+ audio.truncate(30 * 16000);+ assert!(+ audio.len() >= 16000,+ "need at least one second of real audio"+ );+ let audio = screenpipe_audio::utils::audio::normalize_v2(&audio);+ SileroVad::ensure_model_available().await.unwrap();+ for frame_size in [1600, 512] {+ let mut vad = SileroVad::new().await.unwrap();+ let start = Instant::now();+ let mut speech = 0;+ for chunk in audio.chunks(frame_size) {+ speech += usize::from(matches!(vad.audio_type(chunk).unwrap(), VadStatus::Speech));+ }+ println!(+ "frame_size={frame_size} frames={} speech={speech} elapsed_ms={:.3}",+ audio.chunks(frame_size).len(),+ start.elapsed().as_secs_f64() * 1000.0+ );+ if frame_size == 512 {+ assert!(speech > 0, "native detector must recognize fixture speech");+ }+ }+ let vad: Arc<tokio::sync::Mutex<Box<dyn VadEngine + Send>>> = Arc::new(+ tokio::sync::Mutex::new(Box::new(SileroVad::new().await.unwrap())),+ );+ let start = Instant::now();+ let (mut segments, accepted, ratio) = prepare_segments(+ &audio,+ vad,+ None,+ Arc::new(Mutex::new(EmbeddingManager::new(4))),+ None,+ "public speech fixture",+ false,+ false,+ )+ .await+ .unwrap();+ println!(+ "prepare_segments audio_seconds={:.3} elapsed_ms={:.3} speech_ratio={ratio}",+ audio.len() as f64 / 16000.0,+ start.elapsed().as_secs_f64() * 1000.0+ );+ assert!(accepted);+ let segment = segments.recv().await.unwrap();+ assert_eq!(segment.samples.len(), audio.len());+}diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex 3b8b84d6e..76e86668e 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine- Mapped suites: 36-- Mapped Rust files: 380-- Active test blocks: 3735-- Ignored/manual test blocks: 168-- Weighted coverage points: 3081.1+- Mapped Rust files: 381+- Active test blocks: 3738+- Ignored/manual test blocks: 169+- Weighted coverage points: 3082.6| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 32 | 3589 | 154 | 3014.9 | 21 | 11 | 100% |-| macos | 31 | 3617 | 129 | 3004.8 | 22 | 11 | 100% |-| linux | 28 | 3174 | 129 | 2639.6 | 20 | 11 | 100% |+| windows | 32 | 3592 | 155 | 3016.4 | 21 | 11 | 100% |+| macos | 31 | 3620 | 130 | 3006.3 | 22 | 11 | 100% |+| linux | 28 | 3177 | 130 | 2641.1 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex c6e0cd4db..a48225d9b 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -8,11 +8,11 @@ confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 36-- Mapped Rust files: 380-- Active test blocks: 3735-- Ignored/manual test blocks: 168-- Declared test blocks: 3903-- Weighted coverage points: 3081.1+- Mapped Rust files: 381+- Active test blocks: 3738+- Ignored/manual test blocks: 169+- Declared test blocks: 3907+- Weighted coverage points: 3082.6Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,9 +23,9 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 32 | 3589 | 154 | 3014.9 | 21 | 11 | 100% |-| macos | 31 | 3617 | 129 | 3004.8 | 22 | 11 | 100% |-| linux | 28 | 3174 | 129 | 2639.6 | 20 | 11 | 100% |+| windows | 32 | 3592 | 155 | 3016.4 | 21 | 11 | 100% |+| macos | 31 | 3620 | 130 | 3006.3 | 22 | 11 | 100% |+| linux | 28 | 3177 | 130 | 2641.1 | 20 | 11 | 100% |## Crate Summary@@ -34,7 +34,7 @@ are explicitly enabled in a runtime lane.| screenpipe-engine | 12 | 21 | 126 | 1796 | 52 | 1381.6 | 10 || screenpipe-db | 5 | 64 | 26 | 589 | 25 | 561.1 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |-| screenpipe-audio | 7 | 25 | 53 | 672 | 48 | 588.8 | 5 |+| screenpipe-audio | 7 | 26 | 53 | 675 | 49 | 590.3 | 5 || screenpipe-screen | 6 | 9 | 19 | 272 | 9 | 242.3 | 4 || screenpipe-a11y | 4 | 2 | 31 | 376 | 34 | 277.3 | 3 || screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |@@ -63,7 +63,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| Layer | windows | macos | linux || --- | --- | --- | --- || accessibility | 4 suites / 374 active / 35 ignored / 340.1 pts | 4 suites / 415 active / 10 ignored / 336.4 pts | 4 suites / 334 active / 7 ignored / 306.5 pts |-| audio | 9 suites / 794 active / 49 ignored / 710.8 pts | 8 suites / 788 active / 49 ignored / 706.6 pts | 7 suites / 709 active / 48 ignored / 651.3 pts |+| audio | 9 suites / 797 active / 50 ignored / 712.3 pts | 8 suites / 791 active / 50 ignored / 708.0 pts | 7 suites / 712 active / 49 ignored / 652.8 pts || audio-device | 2 suites / 234 active / 7 ignored / 210.3 pts | 2 suites / 234 active / 7 ignored / 210.3 pts | 1 suites / 155 active / 6 ignored / 155.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts || database | 7 suites / 542 active / 21 ignored / 514.1 pts | 7 suites / 542 active / 21 ignored / 514.1 pts | 7 suites / 542 active / 21 ignored / 514.1 pts |@@ -73,7 +73,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| meeting | 6 suites / 1693 active / 24 ignored / 1382.5 pts | 6 suites / 1693 active / 24 ignored / 1382.5 pts | 4 suites / 1344 active / 20 ignored / 1057.2 pts || ocr | 4 suites / 137 active / 7 ignored / 127.7 pts | 4 suites / 130 active / 7 ignored / 125.5 pts | 3 suites / 121 active / 6 ignored / 116.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1622 active / 78 ignored / 1435.0 pts | 14 suites / 1730 active / 82 ignored / 1478.2 pts | 13 suites / 1622 active / 78 ignored / 1435.0 pts |+| performance | 13 suites / 1625 active / 79 ignored / 1436.5 pts | 14 suites / 1733 active / 83 ignored / 1479.7 pts | 13 suites / 1625 active / 79 ignored / 1436.5 pts || pipes | 1 suites / 527 active / 3 ignored / 368.9 pts | 1 suites / 527 active / 3 ignored / 368.9 pts | 1 suites / 527 active / 3 ignored / 368.9 pts || privacy | 5 suites / 980 active / 42 ignored / 791.6 pts | 5 suites / 1021 active / 17 ignored / 787.9 pts | 5 suites / 940 active / 14 ignored / 757.9 pts || real-app | - | 1 suites / 108 active / 4 ignored / 43.2 pts | - |@@ -81,7 +81,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| storage | 4 suites / 667 active / 38 ignored / 553.9 pts | 4 suites / 667 active / 38 ignored / 553.9 pts | 4 suites / 667 active / 38 ignored / 553.9 pts || sync | 1 suites / 527 active / 3 ignored / 368.9 pts | 1 suites / 527 active / 3 ignored / 368.9 pts | 1 suites / 527 active / 3 ignored / 368.9 pts || timeline | 4 suites / 1214 active / 44 ignored / 994.4 pts | 4 suites / 1214 active / 44 ignored / 994.4 pts | 4 suites / 1214 active / 44 ignored / 994.4 pts |-| transcription | 6 suites / 867 active / 47 ignored / 678.8 pts | 5 suites / 861 active / 47 ignored / 674.6 pts | 5 suites / 861 active / 47 ignored / 674.6 pts |+| transcription | 6 suites / 870 active / 48 ignored / 680.2 pts | 5 suites / 864 active / 48 ignored / 676.0 pts | 5 suites / 864 active / 48 ignored / 676.0 pts || ui-events | 4 suites / 795 active / 34 ignored / 603.0 pts | 3 suites / 728 active / 5 ignored / 556.1 pts | 3 suites / 728 active / 5 ignored / 556.1 pts || vision-capture | 6 suites / 569 active / 32 ignored / 448.4 pts | 6 suites / 562 active / 32 ignored / 446.2 pts | 5 suites / 553 active / 31 ignored / 437.2 pts |@@ -126,7 +126,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 155 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). || audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 26 | 269 | 10 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. || audio-models-filtering | screenpipe-audio | windows, macos, linux | audio, transcription, privacy | audio-record-transcribe, privacy-and-redaction | medium | partial | mixed | 6 | 20 | 10 | Model-download/TLS guards, ONNX startup smoke, and music-versus-speech filtering. |-| audio-pipeline-benchmarks | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | medium | partial | benchmark | 8 | 22 | 12 | Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing. |+| audio-pipeline-benchmarks | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | medium | partial | benchmark | 9 | 25 | 13 | Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing. Native-window continuity, valid 16 kHz fallback frames and partial-tail regression tests; optional real-model speech-fixture timing is ignored by default. || audio-platform-output-capture | screenpipe-audio | windows, macos | audio-device, audio, meeting | audio-device-health, audio-record-transcribe, meeting-live-notes | high | partial | unit | 7 | 79 | 1 | OS-specific output/system-audio capture: CoreAudio process tap and SCK output watchdog plus VPIO health policy on macOS, per-process meeting audio taps on both platforms, and the Windows follow-the-audio output watchdog. Platform impl files are cfg-gated to their target OS. || audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 16 | 121 | 9 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. || audio-windows-parakeet-provider | screenpipe-audio | windows | audio, transcription | audio-record-transcribe | high | partial | unit | 1 | 6 | 0 | Registers existing CPU override, eligible adapter selection, DXGI ordinal, error fallback and adapter determinism tests. Windows-only; manifest registration does not verify execution on physical GPU hardware. |@@ -271,6 +271,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio-meetings-speakers-dedup | screenpipe-audio | tests/speaker_identification.rs | integration | 2 | 1 | 3 || audio-meetings-speakers-dedup | screenpipe-audio | tests/speaker_identity_gate_real_audio_test.rs | integration | 0 | 4 | 4 || audio-meetings-speakers-dedup | screenpipe-audio | tests/speaker_identity_gate_test.rs | integration | 11 | 0 | 11 |+| audio-pipeline-benchmarks | screenpipe-audio | tests/vad_frame_continuity_test.rs | integration | 3 | 1 | 4 || db-runtime-reliability | screenpipe-db | src/cancellable_query.rs | source | 2 | 0 | 2 || db-timeline-frames | screenpipe-db | src/db/activity_ledger.rs | source | 5 | 0 | 5 || db-accessibility-ui-events | screenpipe-db | src/db/elements.rs | source | 5 | 0 | 5 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex 6692c0ca4..8d613b0af 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -1050,7 +1050,8 @@"tests/audio_pipeline_benchmark/pipeline_benchmark.rs","tests/audio_pipeline_benchmark/quality_regression.rs","tests/audio_pipeline_benchmark/smart_mode_benchmark.rs",- "tests/audio_pipeline_benchmark/vad_benchmark.rs"+ "tests/audio_pipeline_benchmark/vad_benchmark.rs",+ "tests/vad_frame_continuity_test.rs"],"platforms": ["windows",@@ -1070,7 +1071,7 @@"criticality": "medium","confidence": "partial","kind": "benchmark",- "notes": "Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing."+ "notes": "Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing. Native-window continuity, valid 16 kHz fallback frames and partial-tail regression tests; optional real-model speech-fixture timing is ignored by default."},{"id": "audio-models-filtering",