YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Fix native AEC microphone recovery with supervised capture workers (#7658)
Louis Beaumont3ec2a7786001
diff --git a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock b/apps/screenpipe-app-tauri/src-tauri/Cargo.lockindex 84b3653e0..f7a095ade 100644--- a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock+++ b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock@@ -11418,6 +11418,7 @@ dependencies = ["hound","knf-rs","lazy_static",+ "libc","libpulse-binding","libpulse-simple-binding","libsamplerate-sys",diff --git a/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs b/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rsindex f79b8bda7..846667b50 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs@@ -15,8 +15,8 @@//!//! The mic side has exactly one event (`audio_capture_health_mic_capture_failed`),//! covering the meeting-piggyback mic follow: the meeting app's mic could not-//! be opened at all (no action button — capture already fell back to the-//! default mic). There are deliberately NO mic-silence events: the piggyback+//! be opened at all. The event does not prove a default-mic fallback.+//! There are deliberately NO mic-silence events: the piggyback//! never acts on silence (a silent meeting mic is the user's own in-meeting//! feedback loop — they fix it in the app and the piggyback follows).//!@@ -106,18 +106,14 @@ fn show_speaker_silent_notification(app: AppHandle, data: Value) {/// Show the in-app notification panel telling the user screenpipe couldn't/// open the meeting app's microphone (per-process capture failed to start),-/// but recording continues on the default mic. No action button — there's-/// nothing to restart; the device is unavailable, not stalled.+/// without claiming a default-mic fallback the event has not verified.fn show_mic_capture_failed_notification(app: AppHandle, data: Value) {let reason = data.get("reason").and_then(|v| v.as_str()).unwrap_or("the device may be in exclusive use");- let body = format!(- "{} — recording continues on your default microphone.",- reason- );+ let body = mic_capture_failed_body(reason);let payload = serde_json::json!({"id": "audio_capture_health_mic_capture_failed",@@ -134,3 +130,23 @@ fn show_mic_capture_failed_notification(app: AppHandle, data: Value) {}});}++fn mic_capture_failed_body(reason: &str) -> String {+ format!("{}. Your voice may be missing from the recording. If this persists, quit and reopen screenpipe.", reason.trim_end_matches('.'))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn mic_failure_does_not_claim_unverified_fallback() {+ let body = mic_capture_failed_body(+ "audio device MacBook Pro Microphone (input) teardown timed out",+ );+ assert!(body.contains("Your voice may be missing"));+ assert!(body.contains("quit and reopen screenpipe"));+ assert!(!body.contains("recording continues"));+ assert!(body.contains("teardown timed out."));+ }+}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rsindex e256fbb7c..af0ee5bbc 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs@@ -477,6 +477,11 @@ macro_rules! define_specta_builder {#[tokio::main]async fn main() {+ #[cfg(target_os = "macos")]+ if let Some(code) = screenpipe_audio::core::vpio_process::run_hidden_mode().await {+ std::process::exit(code);+ }+// Invoked by the scoped video tool, before any application side effects.let arguments: Vec<String> = std::env::args().collect();if arguments.get(1).is_some_and(|arg| arg == "--render-workflow-video") {diff --git a/crates/screenpipe-audio/Cargo.toml b/crates/screenpipe-audio/Cargo.tomlindex fc723593e..d5b228a26 100644--- a/crates/screenpipe-audio/Cargo.toml+++ b/crates/screenpipe-audio/Cargo.toml@@ -169,6 +169,7 @@ samplerate = { workspace = true }libsamplerate-sys = { workspace = true }[target.'cfg(target_os = "macos")'.dependencies]+libc = { workspace = true }cidre = { git = "https://github.com/yury/cidre.git", rev = "8481f3f0dc7dd39bb5be80d9424bfac0cad3801a" }once_cell = { workspace = true }objc = { workspace = true }diff --git a/crates/screenpipe-audio/src/audio_manager/manager.rs b/crates/screenpipe-audio/src/audio_manager/manager.rsindex a676a07e5..b69b889c4 100644--- a/crates/screenpipe-audio/src/audio_manager/manager.rs+++ b/crates/screenpipe-audio/src/audio_manager/manager.rs@@ -1911,6 +1911,24 @@ impl AudioManager {.collect::<Vec<AudioDevice>>()}+ /// Failed native teardown is persistent evidence, unlike the short-lived+ /// stream-timeout metric. Only report devices still selected for capture;+ /// stopped, user-paused and piggyback-suspended devices are not missing audio.+ pub async fn capture_failure_devices(&self) -> Vec<AudioDevice> {+ let mut selected = self.enabled_devices().await;+ selected.extend(self.session_devices());+ let disabled = self.user_disabled_devices().await;+ let suspended = self.suspended_devices();+ self.device_manager+ .teardown_failed_devices()+ .into_iter()+ .filter(|device| {+ let name = device.to_string();+ selected.contains(&name) && !disabled.contains(&name) && !suspended.contains(&name)+ })+ .collect()+ }+/// True when `device` has a live, non-disconnected stream. Thin wrapper over/// the device monitor's [`super::is_device_actively_streaming`] so callers/// that only hold an `AudioManager` (the piggyback sweep) can distinguish adiff --git a/crates/screenpipe-audio/src/core/mod.rs b/crates/screenpipe-audio/src/core/mod.rsindex cf6a09681..0c13c0ef5 100644--- a/crates/screenpipe-audio/src/core/mod.rs+++ b/crates/screenpipe-audio/src/core/mod.rs@@ -15,6 +15,8 @@ pub use run_record_and_transcribe::StreamDeath;pub mod sck_output_watchdog;pub mod source_buffer;pub mod stream;+#[cfg(target_os = "macos")]+pub mod vpio_process;use crate::AudioInput;use anyhow::Result;use dashmap::DashMap;diff --git a/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs b/crates/screenpipe-audio/src/core/run_record_and_transcribe.rsindex fad3ad7a7..ea6f7f591 100644--- a/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs+++ b/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs@@ -226,12 +226,13 @@ pub async fn run_record_and_transcribe(let mut collected_audio = Vec::new();let mut segment_start_time = now_epoch_secs();let mut last_diagnostics_time = Instant::now();- let mut _last_non_zero_at: Option<Instant> = None;- let mut _sck_watchdog = crate::core::sck_output_watchdog::SckOutputWatchdog::default();+ let mut mic_deadline = tokio::time::Instant::now()+ + Duration::from_secs(STREAM_STARTUP_GRACE_SECS + AUDIO_RECEIVE_TIMEOUT_SECS);+ let mut stream_error = None;let mut was_paused_for_lock = false;- while is_running.load(Ordering::Relaxed)+ 'capture: while is_running.load(Ordering::Relaxed)&& !audio_stream.is_disconnected.load(Ordering::Relaxed){// Skip recording while the screen is locked@@ -273,15 +274,28 @@ pub async fn run_record_and_transcribe(}}- while collected_audio.len() < max_samples && is_running.load(Ordering::Relaxed) {+ while collected_audio.len() < max_samples+ && is_running.load(Ordering::Relaxed)+ && !audio_stream.is_disconnected.load(Ordering::Relaxed)+ {+ if audio_stream.capture_failed.load(Ordering::Acquire) && receiver.is_empty() {+ stream_error = Some(anyhow!("native audio capture worker exited"));+ break 'capture;+ }tokio::select! {// Receive mic chunkmic_res = receiver.recv() => {match mic_res {Ok(chunk) => {+ if !chunk.is_empty() {+ // Silence is valid delivery. Only actual+ // mic samples extend this deadline, never+ // speaker traffic or the 50ms AEC tick.+ mic_deadline = tokio::time::Instant::now()+ + Duration::from_secs(AUDIO_RECEIVE_TIMEOUT_SECS);+ }metrics.update_audio_levels(&device_name, &chunk);if !chunk.is_empty() && chunk.iter().any(|&x| x.abs() >= SILENT_BUFFER_PEAK_THRESHOLD) {- _last_non_zero_at = Some(Instant::now());update_device_capture_time(&device_name);}@@ -305,7 +319,8 @@ pub async fn run_record_and_transcribe(}Err(e) => {error!("AEC: Error receiving mic data: {}", e);- return Err(anyhow!("Mic stream error: {}", e));+ stream_error = Some(anyhow!("Mic stream error: {}", e));+ break 'capture;}}}@@ -334,7 +349,17 @@ pub async fn run_record_and_transcribe(}speaker_sample_counter += chunk_len as u64;}- // Timeout to prevent hanging when silent+ _ = tokio::time::sleep_until(mic_deadline) => {+ if is_running.load(Ordering::Relaxed) {+ stream_error = Some(anyhow!(StreamDeath::ReceiveTimeout {+ secs: AUDIO_RECEIVE_TIMEOUT_SECS,+ }));+ }+ // Preserve captured samples via the final flush before+ // handing the typed failure to the recovery monitor.+ break 'capture;+ }+ // Drain AEC while waiting for additional callbacks._ = tokio::time::sleep(Duration::from_millis(50)) => {}}@@ -413,6 +438,19 @@ pub async fn run_record_and_transcribe(warn!("AEC: Final mic flush failed for {}: {}", device_name, e);}+ if let Some(error) = stream_error {+ if matches!(+ error.downcast_ref::<StreamDeath>(),+ Some(StreamDeath::ReceiveTimeout { .. })+ ) {+ // Publishing disconnected earlier lets the monitor abort this+ // task while its captured tail is still waiting to be queued.+ let _ = fail_stream_dead(&audio_stream, &device_name, &metrics);+ } else if audio_stream.capture_failed.load(Ordering::Acquire) {+ audio_stream.is_disconnected.store(true, Ordering::Relaxed);+ }+ return Err(error);+ }if audio_stream.is_disconnected.load(Ordering::Relaxed) {info!("AEC: Stopped recording for {} (disconnected)", device_name);Err(anyhow::anyhow!("device {} disconnected", device_name))@@ -451,10 +489,11 @@ pub async fn run_record_and_transcribe(let mut last_non_zero_at = last_non_zero_seed_after_restart(&device_name, Instant::now());let mut sck_watchdog = crate::core::sck_output_watchdog::SckOutputWatchdog::default();let mut segment_count: u64 = 0;+ let mut stream_error = None;let mut was_paused_for_lock = false;- while is_running.load(Ordering::Relaxed)+ 'capture: while is_running.load(Ordering::Relaxed)&& !audio_stream.is_disconnected.load(Ordering::Relaxed){if screenpipe_config::should_pause_audio_for_lock() {@@ -498,7 +537,10 @@ pub async fn run_record_and_transcribe(// stream is being torn down either way, so break out and// let the flushes below persist what we have.Err(_) if !is_running.load(Ordering::Relaxed) => break,- Err(e) => return Err(e),+ Err(e) => {+ stream_error = Some(e);+ break 'capture;+ }};match received {Some(chunk) => {@@ -560,6 +602,17 @@ pub async fn run_record_and_transcribe(warn!("final flush failed for {}: {}", device_name, e);}+ if let Some(error) = stream_error {+ if audio_stream.capture_failed.load(Ordering::Acquire)+ || error.downcast_ref::<StreamDeath>().is_some()+ {+ // The supervisor owns native cleanup. Only now, once+ // queued audio is safe, may the monitor replace this recorder.+ audio_stream.is_disconnected.store(true, Ordering::Relaxed);+ }+ return Err(error);+ }+if audio_stream.is_disconnected.load(Ordering::Relaxed) {info!("stopped recording for {} (disconnected)", device_name);Err(anyhow::anyhow!("device {} disconnected", device_name))@@ -579,6 +632,11 @@ async fn recv_audio_chunk(last_non_zero_at: &mut Option<Instant>,sck_watchdog: &mut crate::core::sck_output_watchdog::SckOutputWatchdog,) -> Result<Option<Vec<f32>>> {+ // Consume packets already queued before the terminal wake-up. A native+ // worker failure must not expose disconnection until the caller flushes.+ if audio_stream.capture_failed.load(Ordering::Acquire) && receiver.is_empty() {+ return Err(anyhow!("native audio capture worker exited"));+ }let recv_result = tokio::time::timeout(Duration::from_secs(AUDIO_RECEIVE_TIMEOUT_SECS),receiver.recv(),@@ -618,7 +676,6 @@ async fn recv_audio_chunk(device_name, INPUT_SILENT_BUFFER_TIMEOUT_SECS);metrics.record_stream_timeout(device_name);- audio_stream.is_disconnected.store(true, Ordering::Relaxed);return Err(anyhow!(StreamDeath::ZeroFill {device: device_name.to_string(),secs: INPUT_SILENT_BUFFER_TIMEOUT_SECS,@@ -672,7 +729,8 @@ async fn recv_audio_chunk(sck_watchdog,)} else {- fail_stream_dead(audio_stream, device_name, metrics)+ // The recorder publishes disconnection after flushing its tail.+ stream_dead_error(device_name, metrics)}}}@@ -705,13 +763,20 @@ fn fail_stream_dead(audio_stream: &Arc<AudioStream>,device_name: &str,metrics: &Arc<AudioPipelineMetrics>,+) -> Result<Option<Vec<f32>>> {+ audio_stream.is_disconnected.store(true, Ordering::Relaxed);+ stream_dead_error(device_name, metrics)+}++fn stream_dead_error(+ device_name: &str,+ metrics: &Arc<AudioPipelineMetrics>,) -> Result<Option<Vec<f32>>> {warn!("no audio received from {} for {}s - stream dead, triggering reconnect",device_name, AUDIO_RECEIVE_TIMEOUT_SECS);metrics.record_stream_timeout(device_name);- audio_stream.is_disconnected.store(true, Ordering::Relaxed);// Typed cause so the VPIO runtime-fallback policy can recognize this death// by `downcast_ref` rather than message text. Display is unchanged.Err(anyhow!(StreamDeath::ReceiveTimeout {@@ -885,6 +950,161 @@ mod tests {use super::*;use crate::core::device::AudioDevice;+ #[tokio::test]+ async fn native_worker_failure_flushes_audio_before_publishing_disconnection() {+ let device = Arc::new(AudioDevice::new(+ "native worker test".into(),+ DeviceType::Input,+ ));+ let (stream, sender) = AudioStream::from_sender_for_test(device, 16_000, 1);+ let stream = Arc::new(stream);+ let (tx, rx) = flume::bounded(0);+ let task = tokio::spawn(run_record_and_transcribe(+ stream.clone(),+ Duration::from_secs(30),+ Arc::new(tx),+ Arc::new(AtomicBool::new(true)),+ Arc::new(AudioPipelineMetrics::new()),+ None,+ None,+ false,+ ));+ while sender.receiver_count() == 0 {+ tokio::task::yield_now().await;+ }+ sender.send(vec![0.25; 3_200]).unwrap();+ stream.capture_failed.store(true, Ordering::Release);+ sender.send(Vec::new()).unwrap();+ tokio::time::sleep(Duration::from_millis(100)).await;+ assert!(!task.is_finished());+ assert!(+ !stream.is_disconnected(),+ "monitor must wait for the buffered tail"+ );+ let segment = tokio::time::timeout(Duration::from_secs(1), rx.recv_async()).await;+ if segment.is_err() {+ task.abort();+ }+ let segment = segment+ .expect("native worker failure must flush captured audio")+ .unwrap();+ assert_eq!(segment.data.len(), 3_200);+ let error = tokio::time::timeout(Duration::from_secs(1), task)+ .await+ .unwrap()+ .unwrap()+ .unwrap_err();+ assert!(error+ .to_string()+ .contains("native audio capture worker exited"));+ assert!(stream.is_disconnected());+ assert!(rx.is_empty());+ }++ #[tokio::test]+ async fn aec_mic_callback_stall_requests_recovery_and_flushes_captured_audio() {+ let device = Arc::new(AudioDevice::new(+ "AEC stalled input".into(),+ DeviceType::Input,+ ));+ let (stream, sender) = AudioStream::from_sender_for_test(device, 16_000, 1);+ let stream = Arc::new(stream);+ let (segments_tx, segments_rx) = flume::bounded(0);+ let metrics = Arc::new(AudioPipelineMetrics::new());+ let mut task = tokio::spawn(run_record_and_transcribe(+ stream.clone(),+ Duration::from_secs(30),+ Arc::new(segments_tx),+ Arc::new(AtomicBool::new(true)),+ metrics.clone(),+ None,+ None,+ true,+ ));+ while sender.receiver_count() == 0 {+ tokio::task::yield_now().await;+ }+ sender.send(vec![0.25; 3_200]).unwrap();+ // Keep the sender alive but stop callbacks, exactly like a native+ // worker stuck in CoreAudio. A timer tick is not a microphone sample.+ tokio::time::sleep(Duration::from_secs(AUDIO_RECEIVE_TIMEOUT_SECS + 1)).await;+ assert!(+ !task.is_finished(),+ "flush must wait for the persistence queue"+ );+ assert!(+ !stream.is_disconnected(),+ "recovery must not abort a pending final flush"+ );+ let segment = tokio::time::timeout(Duration::from_secs(1), segments_rx.recv_async())+ .await+ .expect("AEC must flush after mic callbacks stop")+ .unwrap();+ let result = tokio::time::timeout(+ Duration::from_secs(AUDIO_RECEIVE_TIMEOUT_SECS + 2),+ &mut task,+ )+ .await;+ if result.is_err() {+ task.abort();+ }+ let error = result+ .expect("AEC must not wait forever after mic callbacks stop")+ .unwrap()+ .unwrap_err();+ assert_eq!(+ error.downcast_ref::<StreamDeath>(),+ Some(&StreamDeath::ReceiveTimeout {+ secs: AUDIO_RECEIVE_TIMEOUT_SECS+ })+ );+ assert!(stream.is_disconnected());+ assert!(+ segments_rx.is_empty(),+ "captured audio must be emitted exactly once"+ );+ assert!(!segment.data.is_empty());+ }++ #[tokio::test]+ async fn aec_silent_callbacks_keep_recording_past_receive_timeout() {+ let device = Arc::new(AudioDevice::new(+ "AEC silent input".into(),+ DeviceType::Input,+ ));+ let (stream, sender) = AudioStream::from_sender_for_test(device, 16_000, 1);+ let stream = Arc::new(stream);+ let (segments_tx, segments_rx) = flume::bounded(8);+ let running = Arc::new(AtomicBool::new(true));+ let task = tokio::spawn(run_record_and_transcribe(+ stream.clone(),+ Duration::from_secs(30),+ Arc::new(segments_tx),+ running.clone(),+ Arc::new(AudioPipelineMetrics::new()),+ None,+ None,+ true,+ ));+ while sender.receiver_count() == 0 {+ tokio::task::yield_now().await;+ }+ for _ in 0..(AUDIO_RECEIVE_TIMEOUT_SECS + 1) * 10 {+ sender.send(vec![0.0; 1_600]).unwrap();+ tokio::time::sleep(Duration::from_millis(100)).await;+ assert!(!task.is_finished(), "silence is not missing callbacks");+ }+ running.store(false, Ordering::Relaxed);+ sender.send(Vec::new()).unwrap();+ tokio::time::timeout(Duration::from_secs(1), task)+ .await+ .unwrap()+ .unwrap()+ .unwrap();+ assert!(!stream.is_disconnected());+ assert!(!segments_rx.try_recv().unwrap().data.is_empty());+ }+async fn run_stopped_recorder(samples: Vec<f32>,segment_duration: Duration,diff --git a/crates/screenpipe-audio/src/core/stream.rs b/crates/screenpipe-audio/src/core/stream.rsindex 84f875b85..dc86971f0 100644--- a/crates/screenpipe-audio/src/core/stream.rs+++ b/crates/screenpipe-audio/src/core/stream.rs@@ -20,8 +20,10 @@ use std::sync::Arc;#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]use std::sync::{Mutex, OnceLock};use tokio::sync::{broadcast, oneshot};+#[cfg(target_os = "windows")]+use tracing::info;#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]-use tracing::{error, info, warn};+use tracing::{error, warn};#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]use crate::utils::audio::audio_to_mono;@@ -80,6 +82,8 @@ pub struct AudioStream {stream_control: mpsc::Sender<StreamControl>,stream_thread: Option<Arc<tokio::sync::Mutex<Option<tokio::task::JoinHandle<()>>>>>,pub is_disconnected: Arc<AtomicBool>,+ /// Native worker exited; recorder flushes its tail before publishing disconnection.+ pub(crate) capture_failed: Arc<AtomicBool>,/// True when this stream is backed by the cpal/ScreenCaptureKit path/// (the default), false when it is backed by the CoreAudio Process Tap./// The macOS System Audio liveness watchdog ([`super::sck_output_watchdog`])@@ -90,7 +94,7 @@ pub struct AudioStream {}#[derive(Clone, Copy, Debug, Eq, PartialEq)]-enum StopMode {+pub(super) enum StopMode {Immediate,#[cfg(any(not(all(target_os = "linux", feature = "pulseaudio")), test))]DeferredDeviceDisconnect,@@ -109,7 +113,7 @@ impl StopMode {}}-enum StreamControl {+pub(super) enum StreamControl {Stop {_response: oneshot::Sender<()>,_mode: StopMode,@@ -158,6 +162,7 @@ impl AudioStream {let (tx, _) = broadcast::channel::<Vec<f32>>(1000);let tx_clone = tx.clone();let is_disconnected = Arc::new(AtomicBool::new(false));+ let capture_failed = Arc::new(AtomicBool::new(false));let (stream_control_tx, stream_control_rx) = mpsc::channel();#[cfg(all(target_os = "linux", feature = "pulseaudio"))]@@ -260,6 +265,7 @@ impl AudioStream {&stream_control_tx,windows_input_aec,macos_input_vpio,+ &capture_failed,).await?;(config, thread, true)@@ -280,6 +286,7 @@ impl AudioStream {&stream_control_tx,windows_input_aec,macos_input_vpio,+ &capture_failed,).await?;(config, thread, true)@@ -293,6 +300,7 @@ impl AudioStream {stream_control: stream_control_tx,stream_thread: Some(Arc::new(tokio::sync::Mutex::new(Some(stream_thread)))),is_disconnected,+ capture_failed,is_sck_backed,})}@@ -310,56 +318,28 @@ impl AudioStream {stream_control_tx: &mpsc::Sender<StreamControl>,windows_input_aec: bool,macos_input_vpio: bool,+ #[cfg_attr(not(target_os = "macos"), allow(unused_variables))] capture_failed: &Arc<+ AtomicBool,+ >,) -> Result<(AudioStreamConfig, tokio::task::JoinHandle<()>)> {- // Only the macOS VoiceProcessingIO branch below reassigns `config`, so the- // `mut` is dead weight on every other target.- #[cfg_attr(not(target_os = "macos"), allow(unused_mut))]- let (cpal_audio_device, mut config) = get_cpal_device_and_config(device).await?;- let is_running_weak = Arc::downgrade(is_running);- let input_aec = windows_input_aec && device.device_type == super::device::DeviceType::Input;- let input_vpio = macos_input_vpio- && device.device_type == super::device::DeviceType::Input- && is_default_input_device(device);-#[cfg(target_os = "macos")]- {- if macos_input_vpio && input_vpio {- let original_rate = config.sample_rate().0;- info!(- device = %device,- "screenpipe-audio: enabling VoiceProcessingIO (AEC) on default microphone"- );- config = cpal_audio_device.default_input_config().map_err(|e| {- anyhow!(- "could not get default input config for VoiceProcessingIO on {}: {}",- device,- e- )- })?;- let vpio_rate = config.sample_rate().0;- if vpio_rate != original_rate {- info!(- device = %device,- original_rate,- vpio_rate,- "screenpipe-audio: VPIO config rate differs from originally-picked config; \- using hardware native rate"- );- }- info!(- device = %device,- sample_rate = vpio_rate,- channels = config.channels(),- "screenpipe-audio: using default input config for VoiceProcessingIO"- );- } else if macos_input_vpio && device.device_type == super::device::DeviceType::Input {- info!(- device = %device,- "screenpipe-audio: macOS VPIO requested but using HAL (only system default input supports VoiceProcessingIO)"- );- }+ if macos_input_vpio && is_default_input_device(device) {+ return super::vpio_process::start(+ device.clone(),+ tx,+ stream_control_rx,+ capture_failed.clone(),+ )+ .await;}+ let (cpal_audio_device, config) = get_cpal_device_and_config(device).await?;+ let is_running_weak = Arc::downgrade(is_running);+ let input_aec = windows_input_aec && device.device_type == super::device::DeviceType::Input;+ // Native AEC is isolated above. Other CPAL streams stay in-process.+ let _ = macos_input_vpio;+ let input_vpio = false;+let audio_config = AudioStreamConfig::from(&config);let channels = config.channels();@@ -394,8 +374,6 @@ impl AudioStream {macos_input_vpio: bool,) -> Result<tokio::task::JoinHandle<()>> {let device_name = device.name()?;- #[cfg(target_os = "macos")]- let mut use_vpio = macos_input_vpio;#[cfg(target_os = "windows")]let use_aec = windows_input_aec;@@ -516,43 +494,6 @@ impl AudioStream {}}}- #[cfg(target_os = "macos")]- Err(primary_err) if macos_input_vpio => {- // Report the backend actually running after fallback.- use_vpio = false;- warn!(- device = %device_name,- rate = config.sample_rate().0,- channels = channels,- format = ?config.sample_format(),- "VoiceProcessingIO stream creation failed ({}); retrying with HAL (VPIO disabled)",- primary_err,- );- let fallback_cb = create_error_callback(- device_name.clone(),- is_running_weak,- is_disconnected,- stream_control_tx,- );- match build_input_stream(- &device,- &config,- channels,- tx,- fallback_cb,- windows_input_aec,- false,- ) {- Ok(s) => Some(s),- Err(fallback_err) => {- error!(- "HAL fallback also failed for {} after VPIO error: {} (VPIO error: {})",- device_name, fallback_err, primary_err- );- None- }- }- }Err(e) => {if is_mic_access_denied(&e) {if first_access_denied_for_device(&device_name) {@@ -574,13 +515,6 @@ impl AudioStream {return;}- #[cfg(target_os = "macos")]- if use_vpio {- info!(- device = %device_name,- "screenpipe-audio: VoiceProcessingIO microphone capture running (AEC initialized)"- );- }#[cfg(target_os = "windows")]if use_aec {info!(@@ -681,11 +615,35 @@ impl AudioStream {stream_control: stream_control_tx,stream_thread: None,is_disconnected: Arc::new(AtomicBool::new(false)),+ capture_failed: Arc::new(AtomicBool::new(false)),is_sck_backed: false,};(stream, tx_arc)}+ #[cfg(all(test, not(all(target_os = "linux", feature = "pulseaudio"))))]+ pub(crate) fn with_blocked_teardown_for_test(+ device: Arc<AudioDevice>,+ ) -> (Self, mpsc::Sender<()>) {+ let (mut stream, _tx) = Self::from_sender_for_test(device, 48_000, 1);+ let (control_tx, control_rx) = mpsc::channel();+ let (release_tx, release_rx) = mpsc::channel();+ stream.stream_control = control_tx;+ stream.stream_thread = Some(Arc::new(tokio::sync::Mutex::new(Some(+ tokio::task::spawn_blocking(move || {+ let _ = release_rx.recv();+ if let Ok(StreamControl::Stop {+ _response: response,+ ..+ }) = control_rx.recv()+ {+ let _ = response.send(());+ }+ }),+ ))));+ (stream, release_tx)+ }+/// Build an AudioStream that plays back a wav (or any symphonia-decodable)/// file into the broadcast channel, mimicking what a real cpal device/// would produce. Lets the eval harness drive the full pipeline (VAD,@@ -760,6 +718,7 @@ impl AudioStream {// shape so `stop()` can abort the playback task uniformly.stream_thread: Some(Arc::new(tokio::sync::Mutex::new(Some(thread)))),is_disconnected,+ capture_failed: Arc::new(AtomicBool::new(false)),// wav playback fixture is an input device, never the SCK output path.is_sck_backed: false,})@@ -846,7 +805,7 @@ fn first_access_denied_for_device(device: &str) -> bool {}#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]-fn build_input_stream(+pub(super) fn build_input_stream(device: &cpal::Device,config: &cpal::SupportedStreamConfig,channels: u16,@@ -910,26 +869,18 @@ fn build_input_stream(}}-#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]+#[cfg(target_os = "macos")]fn is_default_input_device(device: &AudioDevice) -> bool {use super::device::{default_input_device, DeviceType};-- if device.device_type != DeviceType::Input {- return false;- }-- default_input_device()- .map(|default_device| default_device == *device)- .unwrap_or(false)+ device.device_type == DeviceType::Input+ && default_input_device()+ .map(|default| default == *device)+ .unwrap_or(false)}#[cfg(target_os = "macos")]fn macos_voice_processing_config(enabled: bool) -> Option<cpal::MacosVoiceProcessingInputConfig> {- if enabled {- Some(cpal::MacosVoiceProcessingInputConfig::screenpipe_aec())- } else {- None- }+ enabled.then(cpal::MacosVoiceProcessingInputConfig::screenpipe_aec)}#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]@@ -1124,6 +1075,79 @@ mod from_wav_tests {}}+ #[cfg(target_os = "macos")]+ #[test]+ fn native_aec_remains_enabled_for_capture_worker() {+ let config = macos_voice_processing_config(true).unwrap();+ assert!(config.enable_voice_processing);+ assert_eq!(config.voice_processing_bypass, Some(false));+ assert!(macos_voice_processing_config(false).is_none());+ }++ #[cfg(target_os = "macos")]+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ #[ignore = "opens the real microphone with native Apple AEC"]+ async fn live_native_aec_records_and_flushes_audio() -> Result<()> {+ use crate::core::run_record_and_transcribe::run_record_and_transcribe;+ use crate::metrics::AudioPipelineMetrics;+ use crate::utils::ffmpeg::{read_audio_from_file, write_audio_to_file};+ let device = Arc::new(super::super::device::default_input_device()?);+ let stream = Arc::new(+ AudioStream::from_device(+ device,+ Arc::new(AtomicBool::new(true)),+ false,+ false,+ true,+ None,+ )+ .await?,+ );+ let mut samples = stream.subscribe().await;+ let running = Arc::new(AtomicBool::new(true));+ let (tx, rx) = flume::bounded(8);+ let recorder = tokio::spawn(run_record_and_transcribe(+ stream.clone(),+ Duration::from_secs(30),+ Arc::new(tx),+ running.clone(),+ Arc::new(AudioPipelineMetrics::new()),+ None,+ None,+ false,+ ));+ tokio::time::timeout(Duration::from_secs(5), async {+ let mut count = 0;+ while count < stream.device_config.sample_rate().0 as usize {+ count += samples.recv().await?.len();+ }+ Ok::<_, anyhow::Error>(())+ })+ .await??;+ running.store(false, Ordering::Relaxed);+ tokio::time::timeout(Duration::from_secs(2), recorder).await???;+ stream.stop().await?;+ let segments: Vec<_> = rx.try_iter().collect();+ anyhow::ensure!(!segments.is_empty());+ let rate = segments[0].sample_rate;+ let captured: Vec<_> = segments+ .iter()+ .flat_map(|s| s.data.iter().copied())+ .collect();+ anyhow::ensure!(!captured.is_empty());+ let dir = tempfile::tempdir()?;+ let path = dir.path().join("native-aec.m4a");+ write_audio_to_file(&captured, rate, &path, false)?;+ let (decoded, decoded_rate) = read_audio_from_file(&path)?;+ anyhow::ensure!(decoded.len() >= captured.len() * decoded_rate as usize / rate as usize);+ println!(+ "native AEC actual recorder: captured={}, decoded={}, teardown=ok",+ captured.len(),+ decoded.len()+ );+ Ok(())+ }+/// 16 kHz mono sine wav round-trips through `from_wav`. The test counts/// every chunk that lands on the broadcast receiver — sample count must/// match the original signal exactly (resampling is bypassed for 16 kHz).diff --git a/crates/screenpipe-audio/src/core/vpio_process.rs b/crates/screenpipe-audio/src/core/vpio_process.rsnew file mode 100644index 000000000..cb0b20c06--- /dev/null+++ b/crates/screenpipe-audio/src/core/vpio_process.rs@@ -0,0 +1,541 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+//! Native AEC runs in a child of the same signed executable. CoreAudio calls+//! cannot be cancelled by Tokio; process ownership lets teardown kill and reap+//! a wedged unit before recovery starts another native-AEC capture session.+use super::device::{parse_audio_device, AudioDevice};+use super::stream::{AudioStreamConfig, StreamControl};+use anyhow::{anyhow, bail, Context, Result};+use cpal::traits::{DeviceTrait, StreamTrait};+use std::io::{BufRead, BufReader, BufWriter, Read, Write};+use std::process::{Child, Command, Stdio};+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{mpsc, Arc};+use std::time::{Duration, Instant};+use tokio::sync::{broadcast, oneshot};++const MODE: &str = "--screenpipe-native-aec-worker";+const MAGIC: &str = "SPVPIO1";+const START_TIMEOUT: Duration = Duration::from_secs(18);+const SAMPLE_TIMEOUT: Duration = Duration::from_secs(8);+const STOP_GRACE: Duration = Duration::from_millis(500);+const MAX_SAMPLES: usize = 65_536;++enum Packet {+ Ready(AudioStreamConfig),+ Audio(Vec<f32>),+}++/// Drop is the cancellation backstop, including startup errors and panics.+/// Never detach an owned child or leave a native capture worker behind.+struct OwnedChild(Child);+impl Drop for OwnedChild {+ fn drop(&mut self) {+ let _ = self.0.kill();+ let _ = self.0.wait();+ }+}++fn command(device: &AudioDevice) -> Result<Command> {+ let mut command = Command::new(std::env::current_exe()?);+ #[cfg(not(test))]+ command.arg(MODE).arg(device.to_string());+ #[cfg(test)]+ command+ .args([+ "--exact",+ "core::vpio_process::tests::child_entry",+ "--ignored",+ "--nocapture",+ ])+ .env("SCREENPIPE_TEST_AEC_DEVICE", device.to_string());+ Ok(command)+}++pub(super) async fn start(+ device: Arc<AudioDevice>,+ tx: broadcast::Sender<Vec<f32>>,+ control: mpsc::Receiver<StreamControl>,+ capture_failed: Arc<AtomicBool>,+) -> Result<(AudioStreamConfig, tokio::task::JoinHandle<()>)> {+ let command = command(&device)?;+ start_command(command, tx, control, capture_failed, START_TIMEOUT).await+}++async fn start_command(+ command: Command,+ tx: broadcast::Sender<Vec<f32>>,+ control: mpsc::Receiver<StreamControl>,+ capture_failed: Arc<AtomicBool>,+ startup_timeout: Duration,+) -> Result<(AudioStreamConfig, tokio::task::JoinHandle<()>)> {+ let (ready_tx, ready_rx) = oneshot::channel();+ let worker = tokio::task::spawn_blocking(move || {+ let wake_recorder = tx.clone();+ if let Err(error) = supervise(command, tx, control, ready_tx, startup_timeout) {+ tracing::warn!(%error, "native AEC capture worker stopped");+ capture_failed.store(true, Ordering::Release);+ // Wake the recorder after all preceding audio packets. It must+ // persist its tail before exposing disconnection to the monitor.+ let _ = wake_recorder.send(Vec::new());+ }+ });+ match ready_rx.await {+ Ok(Ok(config)) => Ok((config, worker)),+ result => {+ // Supervisor has already killed/reaped before reporting failure.+ worker.await?;+ Err(match result {+ Ok(Err(error)) => error,+ Err(_) => anyhow!("native AEC worker exited before startup"),+ _ => unreachable!(),+ })+ }+ }+}++fn supervise(+ mut command: Command,+ tx: broadcast::Sender<Vec<f32>>,+ control: mpsc::Receiver<StreamControl>,+ ready: oneshot::Sender<Result<AudioStreamConfig>>,+ startup_timeout: Duration,+) -> Result<()> {+ let mut ready = Some(ready);+ let child = command+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::inherit())+ .spawn();+ let mut child = match child {+ Ok(child) => OwnedChild(child),+ Err(error) => {+ let _ = ready.take().unwrap().send(Err(error.into()));+ return Ok(());+ }+ };+ let stdout = child.0.stdout.take().context("native AEC stdout missing")?;+ let (packets_tx, packets_rx) = mpsc::sync_channel(8);+ let reader = std::thread::spawn(move || {+ let result = read_packets(stdout, &packets_tx);+ if let Err(error) = result {+ let _ = packets_tx.send(Err(error));+ }+ });+ let mut deadline = Instant::now() + startup_timeout;+ let mut stop_response = None;+ let result = loop {+ match control.try_recv() {+ Ok(StreamControl::Stop { _response, .. }) => {+ stop_response = Some(_response);+ break Ok(());+ }+ Err(mpsc::TryRecvError::Disconnected) => break Ok(()),+ Err(mpsc::TryRecvError::Empty) => {}+ }+ if ready.as_ref().is_some_and(|ready| ready.is_closed()) {+ break Ok(()); // cancelled startup, no orphan+ }+ if Instant::now() >= deadline {+ break Err(anyhow!(+ "native AEC worker stopped delivering microphone samples"+ ));+ }+ match packets_rx.recv_timeout(Duration::from_millis(20)) {+ Ok(Ok(Packet::Ready(config))) => {+ if let Some(ready) = ready.take() {+ let _ = ready.send(Ok(config));+ deadline = Instant::now() + SAMPLE_TIMEOUT;+ } else {+ break Err(anyhow!("duplicate native AEC header"));+ }+ }+ Ok(Ok(Packet::Audio(samples))) => {+ deadline = Instant::now() + SAMPLE_TIMEOUT;+ let _ = tx.send(samples);+ }+ Ok(Err(error)) => break Err(error),+ Err(mpsc::RecvTimeoutError::Disconnected) => {+ break Err(anyhow!("native AEC worker closed audio pipe"))+ }+ Err(mpsc::RecvTimeoutError::Timeout) => {}+ }+ };+ // Ask for graceful native disposal first; a stalled native call cannot+ // hold the parent hostage. Keep stdin open until after kill/wait, so the+ // parent's own teardown path is what establishes completion.+ if let Some(stdin) = child.0.stdin.as_mut() {+ let _ = stdin.write_all(b"s");+ }+ let stop_deadline = Instant::now() + STOP_GRACE;+ while Instant::now() < stop_deadline {+ if child.0.try_wait().ok().flatten().is_some() {+ break;+ }+ std::thread::sleep(Duration::from_millis(10));+ }+ drop(child); // kill if necessary, wait, then close pipe handles+ drop(packets_rx); // release a reader blocked on the bounded queue+ let _ = reader.join();+ if let Some(ready) = ready {+ let message = result+ .as_ref()+ .err()+ .map(ToString::to_string)+ .unwrap_or_else(|| "native AEC startup cancelled".into());+ let _ = ready.send(Err(anyhow!(message)));+ }+ if let Some(response) = stop_response {+ let _ = response.send(());+ }+ result+}++fn read_packets(read: impl Read, packets: &mpsc::SyncSender<Result<Packet>>) -> Result<()> {+ let mut read = BufReader::new(read);+ let mut header = String::new();+ // The test executable emits a small harness preamble. Production emits+ // the header first. Bound parsing so a wrong executable cannot exhaust RAM.+ loop {+ let mut line = String::new();+ let size = read.by_ref().take(256).read_line(&mut line)?;+ if size == 0 {+ bail!("native AEC worker exited before header");+ }+ header.push_str(&line);+ if header.len() > 1024 {+ bail!("invalid native AEC header");+ }+ if line.starts_with(MAGIC) {+ let fields: Vec<_> = line.split_whitespace().collect();+ if fields.len() != 2 {+ bail!("invalid native AEC format");+ }+ let rate: u32 = fields[1].parse()?;+ if !(8_000..=384_000).contains(&rate) {+ bail!("invalid native AEC sample rate");+ }+ packets.send(Ok(Packet::Ready(AudioStreamConfig::new(rate, 1))))?;+ break;+ }+ }+ loop {+ let mut length = [0u8; 4];+ read.read_exact(&mut length)?;+ let count = u32::from_le_bytes(length) as usize;+ if count == 0 || count > MAX_SAMPLES {+ bail!("invalid native AEC packet size");+ }+ let mut bytes = vec![0; count * 4];+ read.read_exact(&mut bytes)?;+ let samples: Vec<f32> = bytes+ .chunks_exact(4)+ .map(|bytes| f32::from_le_bytes(bytes.try_into().unwrap()))+ .collect();+ if !samples.iter().all(|sample| sample.is_finite()) {+ bail!("invalid native AEC sample");+ }+ packets.send(Ok(Packet::Audio(samples)))?;+ }+}++/// Must run before desktop/engine initialization. The helper opens no database,+/// server, UI, or transcription engine and uses the same executable identity.+pub async fn run_hidden_mode() -> Option<i32> {+ let args: Vec<_> = std::env::args().collect();+ if args.get(1).map(String::as_str) != Some(MODE) {+ return None;+ }+ Some(match args.get(2).filter(|_| args.len() == 3) {+ Some(device) => match capture(device).await {+ Ok(()) => 0,+ Err(error) => {+ eprintln!("native AEC worker: {error:#}");+ 1+ }+ },+ None => 2,+ })+}++async fn capture(device_name: &str) -> Result<()> {+ let stopping = Arc::new(AtomicBool::new(false));+ let stop = stopping.clone();+ std::thread::spawn(move || {+ let mut byte = [0];+ loop {+ match std::io::stdin().read(&mut byte) {+ // Do not run atexit/native destructors: those may be the very+ // locks that wedged capture. The OS releases process resources.+ Ok(0) | Err(_) => unsafe { libc::_exit(0) },+ Ok(_) => stop.store(true, Ordering::Relaxed),+ }+ }+ });+ let device = parse_audio_device(device_name)?;+ let (native_device, _) = super::device::get_cpal_device_and_config(&device).await?;+ let config = native_device.default_input_config()?;+ let (tx, mut rx) = broadcast::channel(1000);+ let failed = Arc::new(AtomicBool::new(false));+ let failure = failed.clone();+ // Native AEC is mandatory here: no HAL/software substitution. Reuse the+ // existing callback; IPC and serialization occur outside CoreAudio's callback.+ let stream = super::stream::build_input_stream(+ &native_device,+ &config,+ config.channels(),+ tx,+ move |error| {+ // Render errors can be transient during route reconfiguration.+ // Preserve the existing policy: device loss stops immediately;+ // absent callbacks are handled by the parent's sample watchdog.+ if matches!(error, cpal::StreamError::DeviceNotAvailable) {+ failure.store(true, Ordering::Relaxed);+ }+ },+ false,+ true,+ )?;+ stream.play()?;+ let output = std::io::stdout();+ let mut stdout = BufWriter::with_capacity(MAX_SAMPLES * 4 + 4, output.lock());+ writeln!(stdout, "{MAGIC} {}", config.sample_rate().0)?;+ stdout.flush()?;+ while !stopping.load(Ordering::Relaxed) && !failed.load(Ordering::Relaxed) {+ tokio::select! {+ sample = rx.recv() => match sample {+ Ok(samples) if !samples.is_empty() => {+ if samples.len() > MAX_SAMPLES { bail!("native callback exceeds packet limit"); }+ stdout.write_all(&(samples.len() as u32).to_le_bytes())?;+ for sample in samples { stdout.write_all(&sample.to_le_bytes())?; }+ stdout.flush()?;+ }+ Ok(_) | Err(broadcast::error::RecvError::Lagged(_)) => {},+ Err(error) => return Err(error.into()),+ },+ _ = tokio::time::sleep(Duration::from_millis(20)) => {},+ }+ }+ stream.pause()?;+ drop(stream);+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::core::device::{default_input_device, DeviceType};++ #[test]+ #[ignore = "private subprocess entry; invoked by supervisor tests"]+ fn child_entry() {+ if let Ok(path) = std::env::var("SCREENPIPE_TEST_AEC_PID") {+ std::fs::write(path, std::process::id().to_string()).unwrap();+ }+ let device = std::env::var("SCREENPIPE_TEST_AEC_DEVICE").unwrap();+ if let Ok(fault) = std::env::var("SCREENPIPE_TEST_AEC_FAULT") {+ if fault != "open" {+ let mut out = std::io::stdout();+ writeln!(out, "{MAGIC} 16000").unwrap();+ out.write_all(&1u32.to_le_bytes()).unwrap();+ out.write_all(&0.25f32.to_le_bytes()).unwrap();+ out.flush().unwrap();+ }+ // Model a native open/dispose that never returns. The parent must+ // reap this actual process, not just abandon a task or JoinHandle.+ loop {+ std::thread::park();+ }+ }+ let runtime = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap();+ let result = runtime.block_on(capture(&device));+ if let Err(error) = &result {+ eprintln!("{error:#}");+ }+ std::process::exit(if result.is_ok() { 0 } else { 1 });+ }++ fn fixture_command(dir: &std::path::Path, fault: Option<&str>) -> Command {+ let device = AudioDevice::new("test mic".into(), DeviceType::Input);+ let mut cmd = command(&device).unwrap();+ cmd.env("SCREENPIPE_TEST_AEC_PID", dir.join("pid"));+ if let Some(fault) = fault {+ cmd.env("SCREENPIPE_TEST_AEC_FAULT", fault);+ }+ cmd+ }++ fn pid(dir: &std::path::Path) -> i32 {+ std::fs::read_to_string(dir.join("pid"))+ .unwrap()+ .parse()+ .unwrap()+ }++ fn assert_reaped(pid: i32) {+ assert_eq!(+ unsafe { libc::kill(pid, 0) },+ -1,+ "capture process must be gone"+ );+ assert_eq!(+ std::io::Error::last_os_error().raw_os_error(),+ Some(libc::ESRCH)+ );+ }++ async fn stop(control: &mpsc::Sender<StreamControl>, worker: tokio::task::JoinHandle<()>) {+ let (response, done) = oneshot::channel();+ control+ .send(StreamControl::Stop {+ _response: response,+ _mode: super::super::stream::StopMode::Immediate,+ })+ .unwrap();+ tokio::time::timeout(Duration::from_secs(3), done)+ .await+ .unwrap()+ .unwrap();+ tokio::time::timeout(Duration::from_secs(1), worker)+ .await+ .unwrap()+ .unwrap();+ }++ #[tokio::test]+ async fn blocked_native_disposal_is_reaped_before_stop_acknowledgement() {+ let dir = tempfile::tempdir().unwrap();+ let (tx, _) = broadcast::channel(16);+ let (control, controls) = mpsc::channel();+ let (_, worker) = start_command(+ fixture_command(dir.path(), Some("stop")),+ tx,+ controls,+ Arc::new(AtomicBool::new(false)),+ Duration::from_secs(3),+ )+ .await+ .unwrap();+ let child_pid = pid(dir.path());+ stop(&control, worker).await;+ assert_reaped(child_pid);+ }++ #[tokio::test]+ async fn blocked_native_construction_is_reaped_before_startup_error() {+ let dir = tempfile::tempdir().unwrap();+ let (tx, _) = broadcast::channel(16);+ let (_control, controls) = mpsc::channel();+ let result = start_command(+ fixture_command(dir.path(), Some("open")),+ tx,+ controls,+ Arc::new(AtomicBool::new(false)),+ Duration::from_secs(1),+ )+ .await;+ assert!(result.is_err());+ assert_reaped(pid(dir.path()));+ }++ #[tokio::test]+ async fn cancelling_startup_reaps_the_owned_process() {+ let dir = tempfile::tempdir().unwrap();+ let (tx, _) = broadcast::channel(16);+ let (control, controls) = mpsc::channel();+ let task = tokio::spawn(start_command(+ fixture_command(dir.path(), Some("open")),+ tx,+ controls,+ Arc::new(AtomicBool::new(false)),+ Duration::from_secs(30),+ ));+ tokio::time::timeout(Duration::from_secs(3), async {+ while !dir.path().join("pid").exists() {+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .unwrap();+ let child_pid = pid(dir.path());+ task.abort();+ let _ = task.await;+ drop(control);+ tokio::time::timeout(Duration::from_secs(3), async {+ while unsafe { libc::kill(child_pid, 0) } == 0 {+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ })+ .await+ .unwrap();+ assert_reaped(child_pid);+ }++ #[test]+ fn malformed_worker_packets_cannot_allocate_unbounded_audio() {+ let mut bytes = b"SPVPIO1 48000\n".to_vec();+ bytes.extend_from_slice(&u32::MAX.to_le_bytes());+ let (tx, rx) = mpsc::sync_channel(4);+ assert!(read_packets(bytes.as_slice(), &tx).is_err());+ assert!(matches!(rx.recv().unwrap().unwrap(), Packet::Ready(_)));+ assert!(rx.try_recv().is_err());+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ #[ignore = "opens native AEC microphone and freezes only its own child"]+ async fn live_native_aec_recovers_after_frozen_worker_and_persists_samples() -> Result<()> {+ use crate::utils::ffmpeg::{read_audio_from_file, write_audio_to_file};+ let device = default_input_device()?;+ let dir = tempfile::tempdir()?;+ for cycle in 0..3 {+ let cycle_dir = dir.path().join(cycle.to_string());+ std::fs::create_dir(&cycle_dir)?;+ let mut cmd = command(&device)?;+ cmd.env("SCREENPIPE_TEST_AEC_PID", cycle_dir.join("pid"));+ let (tx, mut audio) = broadcast::channel(100);+ let (control, controls) = mpsc::channel();+ let capture_failed = Arc::new(AtomicBool::new(false));+ let (config, worker) =+ start_command(cmd, tx, controls, capture_failed.clone(), START_TIMEOUT).await?;+ let samples = tokio::time::timeout(Duration::from_secs(5), async {+ let mut samples = Vec::new();+ while samples.len() < config.sample_rate().0 as usize / 2 {+ samples.extend(audio.recv().await?);+ }+ Ok::<_, anyhow::Error>(samples)+ })+ .await??;+ let path = cycle_dir.join("capture.m4a");+ write_audio_to_file(&samples, config.sample_rate().0, &path, false)?;+ let (decoded, rate) = read_audio_from_file(&path)?;+ let expected = samples.len() * rate as usize / config.sample_rate().0 as usize;+ assert!(decoded.len() >= expected);+ let child_pid = pid(&cycle_dir);+ // SIGSTOP freezes every thread, including any native stop call.+ // This deterministic failure cannot be recovered with a Rust timeout.+ assert_eq!(unsafe { libc::kill(child_pid, libc::SIGSTOP) }, 0);+ let started = Instant::now();+ if cycle == 1 {+ // No stop request: the callback watchdog itself must dispose+ // of the dead native worker so normal device recovery can run.+ tokio::time::timeout(SAMPLE_TIMEOUT + Duration::from_secs(3), worker)+ .await+ .context("missing callbacks did not trigger native recovery")??;+ assert!(capture_failed.load(Ordering::Relaxed));+ } else {+ stop(&control, worker).await;+ }+ assert_reaped(child_pid);+ println!(+ "native AEC cycle {cycle}: captured={}, decoded={}, frozen worker reaped in {:?}",+ samples.len(),+ decoded.len(),+ started.elapsed()+ );+ }+ Ok(())+ }+}diff --git a/crates/screenpipe-audio/src/device/device_manager.rs b/crates/screenpipe-audio/src/device/device_manager.rsindex 652a9b8ae..9a8865d4d 100644--- a/crates/screenpipe-audio/src/device/device_manager.rs+++ b/crates/screenpipe-audio/src/device/device_manager.rs@@ -32,6 +32,9 @@ pub struct DeviceManager {operations: DashMap<AudioDevice, Arc<Mutex<()>>>,streams: Arc<DashMap<AudioDevice, Arc<AudioStream>>>,states: Arc<DashMap<AudioDevice, Arc<AtomicBool>>>,+ /// Retained native workers whose stop failed. These must remain visible to+ /// health even after the recording task has been removed.+ teardown_failures: DashMap<AudioDevice, ()>,/// When true, System Audio (output) uses the CoreAudio Process Tap path/// on macOS 14.4+ instead of ScreenCaptureKit. Propagated to/// AudioStream::from_device at device-start time. Has no effect on@@ -60,6 +63,7 @@ impl DeviceManager {operations: DashMap::new(),streams,states,+ teardown_failures: DashMap::new(),use_coreaudio_tap: AtomicBool::new(use_coreaudio_tap),windows_input_aec: AtomicBool::new(windows_input_aec),macos_input_vpio: AtomicBool::new(macos_input_vpio),@@ -296,14 +300,25 @@ impl DeviceManager {// Never hold a DashMap guard across native teardown. Retain the stream// on timeout so a later start cannot orphan a still-running worker.if let Some(stream) = self.stream(device) {- stream.stop().await?;+ if let Err(error) = stream.stop().await {+ self.teardown_failures.insert(device.clone(), ());+ return Err(error);+ }}+ self.teardown_failures.remove(device);self.streams.remove(device);Ok(())}+ pub fn teardown_failed_devices(&self) -> Vec<AudioDevice> {+ self.teardown_failures+ .iter()+ .map(|entry| entry.key().clone())+ .collect()+ }+pub fn is_running_mut(&self, device: &AudioDevice) -> Option<Arc<AtomicBool>> {self.states.get(device).map(|s| s.value().clone())}@@ -315,6 +330,28 @@ mod tests {use crate::core::device::DeviceType;use crate::core::stream::AudioStream;+ #[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]+ #[tokio::test]+ async fn teardown_failure_survives_lost_recording_handle_until_native_cleanup() {+ let dm = DeviceManager::new(false, false, false).await.unwrap();+ let device = mic();+ let (stream, release) =+ AudioStream::with_blocked_teardown_for_test(Arc::new(device.clone()));+ dm.streams.insert(device.clone(), Arc::new(stream));+ dm.states+ .insert(device.clone(), Arc::new(AtomicBool::new(false)));+ assert!(dm.stop_device(&device).await.is_err());+ assert_eq!(dm.teardown_failed_devices(), vec![device.clone()]);+ assert!(+ dm.stream(&device).is_some(),+ "timeout must retain the native worker"+ );+ release.send(()).unwrap();+ dm.stop_device(&device).await.unwrap();+ assert!(dm.teardown_failed_devices().is_empty());+ assert!(dm.stream(&device).is_none());+ }+#[tokio::test]async fn published_stream_is_not_reopened_before_recording_handle_registration() {let dm = DeviceManager::new(false, false, false).await.unwrap();diff --git a/crates/screenpipe-config/src/recording.rs b/crates/screenpipe-config/src/recording.rsindex 1f8202b2f..30e55696d 100644--- a/crates/screenpipe-config/src/recording.rs+++ b/crates/screenpipe-config/src/recording.rs@@ -1303,6 +1303,13 @@ mod tests {assert_eq!(settings.effective_aec_flags(), (false, false, false));}+ #[test]+ fn native_macos_aec_selection_preserves_native_echo_cancellation() {+ let settings: RecordingSettings =+ serde_json::from_str(r#"{"aecMode":"macos","macosInputVpioEnabled":true}"#).unwrap();+ assert_eq!(settings.effective_aec_flags(), (false, false, true));+ }+#[test]fn explicit_screenpipe_aec_mode_enables_software_aec() {let settings: RecordingSettings = serde_json::from_str(diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rsindex 669d220e3..42f533054 100644--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs@@ -297,6 +297,12 @@ fn main() -> anyhow::Result<()> {let _profiler = dhat::Profiler::new_heap();let runtime = tokio::runtime::Runtime::new()?;+ #[cfg(target_os = "macos")]+ if let Some(code) = runtime.block_on(screenpipe_audio::core::vpio_process::run_hidden_mode())+ {+ std::process::exit(code);+ }+// Private ACP modes must run before public CLI parsing or AppKit startup.if let Some(exit_code) =runtime.block_on(screenpipe_core::agents::acp::run_hidden_mode_without_observer())diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rsindex 7fdbef589..ef57c8e02 100644--- a/crates/screenpipe-engine/src/routes/health.rs+++ b/crates/screenpipe-engine/src/routes/health.rs@@ -523,9 +523,9 @@ fn input_device_is_available(device_name: &str, clamshell_mode: bool) -> bool {}/// Classify the raw audio capture status from health signals. Pure so it can be-/// unit-tested in isolation. `stream_timeout_recent` must mean a recent timeout-/// that the same current device has not recovered from, NOT a cumulative count-/// or aggregate activity from some other stream.+/// unit-tested in isolation. Timeouts are correlated to the same current device.+/// A retained native teardown failure is separate: its recording handle may be+/// gone and its original timeout may have expired while the worker is stuck.#[allow(clippy::too_many_arguments)]fn classify_audio_status(audio_disabled: bool,@@ -539,6 +539,7 @@ fn classify_audio_status(last_audio_ts: u64,now_ts: u64,threshold_secs: u64,+ native_teardown_failed: bool,) -> &'static str {if audio_disabled {"disabled"@@ -552,6 +553,11 @@ fn classify_audio_status("meeting_detector_unavailable"} else if audio_waiting_for_meeting {"waiting_for_meeting"+ } else if native_teardown_failed {+ // Includes a selected stream whose native teardown is still stuck.+ // Its recording handle may already be gone. Neither that absence nor+ // a healthy output stream turns the capture failure into benign idle.+ "active_no_data"} else if !has_input_device {// Audio is on but there is no microphone to capture from — expected// idle, not a failure. Distinct from "not_started" so /health stays 200@@ -562,9 +568,7 @@ fn classify_audio_status(} else if audio_never_captured {"not_started"} else if stream_timeout_recent {- // The watchdog fired on a current device and that same device has not- // produced usable audio since. This remains a failure even if another- // microphone or system-output stream is healthy.+ // Healthy system output cannot clear a timeout on a current microphone."active_no_data"} else if global_audio_active {"ok"@@ -1598,6 +1602,12 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {// from. The old global timestamp let a healthy output stream mask a dead// mic, while the older cumulative counter pinned recovered devices forever.let now_ts = now.timestamp().max(0) as u64;+ let mut failed_capture_devices = state.audio_manager.capture_failure_devices().await;+ // A built-in microphone is intentionally unavailable in clamshell mode.+ failed_capture_devices.retain(|device| {+ let name = device.to_string();+ !name.contains("(input)") || input_device_is_available(&name, clamshell_mode)+ });let stream_timeout_recent = has_unrecovered_recent_stream_timeout(&state.audio_metrics.per_device_stream_timeouts_snapshot(),¤t_device_capture_at,@@ -1622,6 +1632,7 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {last_audio_ts,now_ts,threshold_secs,+ !failed_capture_devices.is_empty(),).to_string();@@ -1662,9 +1673,10 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {);// Format device statuses as a string for a more detailed view- let device_status_details = if !device_statuses.is_empty() {+ let device_status_details = if !device_statuses.is_empty() || !failed_capture_devices.is_empty()+ {let now_secs = now.timestamp() as u64;- let device_details: Vec<String> = device_statuses+ let mut device_details: Vec<String> = device_statuses.iter().map(|(name, active, last_capture)| {format!(@@ -1676,6 +1688,12 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {}).collect();+ device_details.extend(failed_capture_devices.iter().map(|device| {+ format!(+ "{}: capture failed (native teardown did not finish)",+ device+ )+ }));Some(device_details.join(", "))} else {None@@ -3219,9 +3237,29 @@ mod tests {1_000, // last_audio_ts1_010, // now_ts60, // threshold_secs+ false,)}+ #[test]+ fn failed_mic_with_live_output_is_not_healthy() {+ // The recording handle was removed, but the selected native worker is+ // still stuck. Fresh system output must not make the mic loss benign.+ assert_eq!(+ classify_audio_status(+ false, false, false, false, false, false, false, true, 999, 1000, 60, true,+ ),+ "active_no_data"+ );+ // A real machine without a mic and without a failure remains healthy.+ assert_eq!(+ classify_audio_status(+ false, false, false, false, false, false, false, true, 999, 1000, 60, false,+ ),+ "no_input_device"+ );+ }+#[test]fn timeout_recovery_is_correlated_to_the_same_current_device() {let timeouts = HashMap::from([("mic".to_string(), 100)]);@@ -3401,29 +3439,35 @@ mod tests {// Guard the unrelated branches against accidental regressions.assert_eq!(classify_audio_status(- true, true, false, false, false, true, true, true, 1000, 1010, 60+ true, true, false, false, false, true, true, true, 1000, 1010, 60, false,),"disabled");// intentional lock pause wins over stale/not-started signalsassert_eq!(- classify_audio_status(false, true, false, false, true, true, false, false, 0, 1010, 60),+ classify_audio_status(+ false, true, false, false, true, true, false, false, 0, 1010, 60, false+ ),"ok");// a missing prerequisite fails closed and is visible, not benign idleassert_eq!(- classify_audio_status(false, false, true, false, true, true, false, false, 0, 1010, 60),+ classify_audio_status(+ false, false, true, false, true, true, false, false, 0, 1010, 60, false+ ),"meeting_detector_unavailable");// intentional meetings-only idle is distinct and benignassert_eq!(- classify_audio_status(false, false, false, true, true, true, false, false, 0, 1010, 60),+ classify_audio_status(+ false, false, false, true, true, true, false, false, 0, 1010, 60, false+ ),"waiting_for_meeting");// never captured + no mic -> benign no_input_device (stays 200)assert_eq!(classify_audio_status(- false, false, false, false, true, false, false, false, 0, 1010, 60+ false, false, false, false, true, false, false, false, 0, 1010, 60, false,),"no_input_device");@@ -3431,28 +3475,28 @@ mod tests {// remains the same benign state instead of degrading to stale.assert_eq!(classify_audio_status(- false, false, false, false, false, false, true, false, 1000, 2000, 60+ false, false, false, false, false, false, true, false, 1000, 2000, 60, false,),"no_input_device");// never captured but a mic exists -> not_startedassert_eq!(classify_audio_status(- false, false, false, false, true, true, false, false, 0, 1010, 60+ false, false, false, false, true, true, false, false, 0, 1010, 60, false,),"not_started");// not active, last audio within threshold -> okassert_eq!(classify_audio_status(- false, false, false, false, false, true, false, false, 1000, 1030, 60+ false, false, false, false, false, true, false, false, 1000, 1030, 60, false,),"ok");// not active, last audio stale -> staleassert_eq!(classify_audio_status(- false, false, false, false, false, true, false, false, 1000, 2000, 60+ false, false, false, false, false, true, false, false, 1000, 2000, 60, false,),"stale");diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex aabd81aa0..6942286bf 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine- Mapped suites: 37-- Mapped Rust files: 401-- Active test blocks: 3905-- Ignored/manual test blocks: 185-- Weighted coverage points: 3216.2+- Mapped Rust files: 402+- Active test blocks: 3915+- Ignored/manual test blocks: 188+- Weighted coverage points: 3225.0| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3742 | 167 | 3143.2 | 21 | 11 | 100% |-| macos | 32 | 3785 | 146 | 3138.5 | 22 | 11 | 100% |-| linux | 29 | 3320 | 142 | 2761.5 | 20 | 11 | 100% |+| windows | 33 | 3752 | 170 | 3152.0 | 21 | 11 | 100% |+| macos | 32 | 3795 | 149 | 3147.3 | 22 | 11 | 100% |+| linux | 29 | 3330 | 145 | 2770.3 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 355a6ae75..340c73490 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -8,11 +8,11 @@ confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 37-- Mapped Rust files: 401-- Active test blocks: 3905-- Ignored/manual test blocks: 185-- Declared test blocks: 4090-- Weighted coverage points: 3216.2+- Mapped Rust files: 402+- Active test blocks: 3915+- Ignored/manual test blocks: 188+- Declared test blocks: 4103+- Weighted coverage points: 3225.0Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,18 +23,18 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3742 | 167 | 3143.2 | 21 | 11 | 100% |-| macos | 32 | 3785 | 146 | 3138.5 | 22 | 11 | 100% |-| linux | 29 | 3320 | 142 | 2761.5 | 20 | 11 | 100% |+| windows | 33 | 3752 | 170 | 3152.0 | 21 | 11 | 100% |+| macos | 32 | 3795 | 149 | 3147.3 | 22 | 11 | 100% |+| linux | 29 | 3330 | 145 | 2770.3 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- |-| screenpipe-engine | 12 | 22 | 131 | 1869 | 53 | 1439.0 | 10 |+| screenpipe-engine | 12 | 22 | 131 | 1870 | 53 | 1439.7 | 10 || screenpipe-db | 5 | 69 | 29 | 631 | 32 | 601.4 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |-| screenpipe-audio | 8 | 29 | 53 | 696 | 53 | 607.9 | 5 |+| screenpipe-audio | 8 | 29 | 54 | 705 | 56 | 616.0 | 5 || screenpipe-screen | 6 | 9 | 19 | 278 | 9 | 246.4 | 4 || screenpipe-a11y | 4 | 2 | 34 | 401 | 38 | 291.5 | 3 || screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |@@ -63,25 +63,25 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| Layer | windows | macos | linux || --- | --- | --- | --- || accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 440 active / 14 ignored / 350.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts |-| audio | 10 suites / 834 active / 57 ignored / 745.9 pts | 9 suites / 828 active / 57 ignored / 741.7 pts | 8 suites / 749 active / 56 ignored / 686.4 pts |-| audio-device | 2 suites / 236 active / 7 ignored / 212.3 pts | 2 suites / 236 active / 7 ignored / 212.3 pts | 1 suites / 157 active / 6 ignored / 157.0 pts |+| audio | 10 suites / 843 active / 60 ignored / 754.0 pts | 9 suites / 837 active / 60 ignored / 749.8 pts | 8 suites / 758 active / 59 ignored / 694.5 pts |+| audio-device | 2 suites / 242 active / 10 ignored / 218.3 pts | 2 suites / 242 active / 10 ignored / 218.3 pts | 1 suites / 163 active / 9 ignored / 163.0 pts || configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts || database | 7 suites / 587 active / 28 ignored / 557.4 pts | 7 suites / 587 active / 28 ignored / 557.4 pts | 7 suites / 587 active / 28 ignored / 557.4 pts || db-search | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts || engine-lifecycle | 6 suites / 272 active / 1 ignored / 243.4 pts | 6 suites / 272 active / 1 ignored / 243.4 pts | 6 suites / 268 active / 9 ignored / 242.5 pts |-| local-api | 3 suites / 489 active / 12 ignored / 346.2 pts | 3 suites / 489 active / 12 ignored / 346.2 pts | 3 suites / 489 active / 12 ignored / 346.2 pts |-| meeting | 6 suites / 1757 active / 32 ignored / 1437.2 pts | 6 suites / 1757 active / 32 ignored / 1437.2 pts | 4 suites / 1403 active / 28 ignored / 1106.9 pts |+| local-api | 3 suites / 490 active / 12 ignored / 346.9 pts | 3 suites / 490 active / 12 ignored / 346.9 pts | 3 suites / 490 active / 12 ignored / 346.9 pts |+| meeting | 6 suites / 1758 active / 32 ignored / 1437.9 pts | 6 suites / 1758 active / 32 ignored / 1437.9 pts | 4 suites / 1404 active / 28 ignored / 1107.6 pts || ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1691 active / 83 ignored / 1492.9 pts | 14 suites / 1816 active / 91 ignored / 1542.9 pts | 13 suites / 1691 active / 83 ignored / 1492.9 pts |+| performance | 13 suites / 1700 active / 86 ignored / 1501.0 pts | 14 suites / 1825 active / 94 ignored / 1551.0 pts | 13 suites / 1700 active / 86 ignored / 1501.0 pts || pipes | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts || privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1069 active / 21 ignored / 818.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts || real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - || speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts || storage | 4 suites / 698 active / 41 ignored / 581.6 pts | 4 suites / 698 active / 41 ignored / 581.6 pts | 4 suites / 698 active / 41 ignored / 581.6 pts || sync | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |-| timeline | 4 suites / 1246 active / 48 ignored / 1023.1 pts | 4 suites / 1246 active / 48 ignored / 1023.1 pts | 4 suites / 1246 active / 48 ignored / 1023.1 pts |-| transcription | 7 suites / 896 active / 53 ignored / 700.8 pts | 6 suites / 890 active / 53 ignored / 696.6 pts | 6 suites / 890 active / 53 ignored / 696.6 pts |+| timeline | 4 suites / 1247 active / 48 ignored / 1023.8 pts | 4 suites / 1247 active / 48 ignored / 1023.8 pts | 4 suites / 1247 active / 48 ignored / 1023.8 pts |+| transcription | 7 suites / 900 active / 53 ignored / 703.6 pts | 6 suites / 894 active / 53 ignored / 699.4 pts | 6 suites / 894 active / 53 ignored / 699.4 pts || ui-events | 4 suites / 828 active / 34 ignored / 627.7 pts | 3 suites / 759 active / 5 ignored / 579.4 pts | 3 suites / 759 active / 5 ignored / 579.4 pts || vision-capture | 6 suites / 579 active / 32 ignored / 455.3 pts | 6 suites / 572 active / 32 ignored / 453.1 pts | 5 suites / 563 active / 31 ignored / 444.1 pts |@@ -123,20 +123,20 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 27 | 1 | Linux-specific accessibility/incognito normalization tests. || a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 11 | 125 | 8 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. || a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 7 | 69 | 29 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. |-| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 157 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |+| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 15 | 163 | 9 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). || audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 28 | 281 | 14 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. || audio-min-speech-ratio-gate | screenpipe-audio | windows, macos, linux | audio, transcription | audio-record-transcribe | high | partial | integration | 1 | 1 | 0 | Synthetic VAD input verifies that a minimum speech ratio of one rejects even a fully spoken chunk and emits no segments. Does not exercise physical devices or live model inference. || audio-models-filtering | screenpipe-audio | windows, macos, linux | audio, transcription, privacy | audio-record-transcribe, privacy-and-redaction | medium | partial | mixed | 6 | 20 | 10 | Model-download/TLS guards, ONNX startup smoke, and music-versus-speech filtering. || audio-pipeline-benchmarks | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | medium | partial | benchmark | 9 | 31 | 13 | Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing. Native-window continuity, valid 16 kHz fallback frames and partial-tail regression tests; optional real-model speech-fixture timing is ignored by default. || audio-platform-output-capture | screenpipe-audio | windows, macos | audio-device, audio, meeting | audio-device-health, audio-record-transcribe, meeting-live-notes | high | partial | unit | 7 | 79 | 1 | OS-specific output/system-audio capture: CoreAudio process tap and SCK output watchdog plus VPIO health policy on macOS, per-process meeting audio taps on both platforms, and the Windows follow-the-audio output watchdog. Platform impl files are cfg-gated to their target OS. |-| audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 16 | 121 | 9 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. |+| audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 16 | 124 | 9 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. || audio-windows-parakeet-provider | screenpipe-audio | windows | audio, transcription | audio-record-transcribe | high | partial | unit | 1 | 6 | 0 | Registers existing CPU override, eligible adapter selection, DXGI ordinal, error fallback and adapter determinism tests. Windows-only; manifest registration does not verify execution on physical GPU hardware. || db-accessibility-ui-events | screenpipe-db | windows, macos, linux | database, configuration, accessibility, ui-events, performance | settings-to-engine-config, accessibility-ui-events, performance-liveness | medium | partial | integration | 7 | 28 | 2 | Elements bulk insert, on-screen filtering, UI event batching, DB tier config, and ignored heavy-read real-DB probes. || db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 19 | 135 | 4 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 18 | 51 | 7 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 14 | 107 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. || db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 40 | 310 | 15 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |-| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 436 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |+| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 437 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 307 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 13 | 125 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. || engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 33 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |@@ -213,11 +213,12 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio-device-stream-health | screenpipe-audio | src/core/process_tap/counters.rs | source | 2 | 0 | 2 || audio-platform-output-capture | screenpipe-audio | src/core/process_tap/macos.rs | source | 28 | 1 | 29 || audio-platform-output-capture | screenpipe-audio | src/core/process_tap/windows.rs | source | 12 | 0 | 12 |-| audio-transcription-pipeline | screenpipe-audio | src/core/run_record_and_transcribe.rs | source | 16 | 0 | 16 |+| audio-transcription-pipeline | screenpipe-audio | src/core/run_record_and_transcribe.rs | source | 19 | 0 | 19 || audio-platform-output-capture | screenpipe-audio | src/core/sck_output_watchdog.rs | source | 12 | 0 | 12 || audio-device-stream-health | screenpipe-audio | src/core/source_buffer.rs | source | 7 | 0 | 7 |-| audio-device-stream-health | screenpipe-audio | src/core/stream.rs | source | 13 | 0 | 13 |-| audio-device-stream-health | screenpipe-audio | src/device/device_manager.rs | source | 9 | 0 | 9 |+| audio-device-stream-health | screenpipe-audio | src/core/stream.rs | source | 14 | 1 | 15 |+| audio-device-stream-health | screenpipe-audio | src/core/vpio_process.rs | source | 4 | 2 | 6 |+| audio-device-stream-health | screenpipe-audio | src/device/device_manager.rs | source | 10 | 0 | 10 || audio-platform-output-capture | screenpipe-audio | src/device/vpio_health.rs | source | 5 | 0 | 5 || audio-device-stream-health | screenpipe-audio | src/idle_detector.rs | source | 4 | 0 | 4 || audio-device-stream-health | screenpipe-audio | src/lib.rs | source | 4 | 0 | 4 |@@ -467,7 +468,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-api-routes | screenpipe-engine | src/routes/data.rs | source | 2 | 0 | 2 || engine-api-routes | screenpipe-engine | src/routes/elements.rs | source | 25 | 1 | 26 || engine-api-routes | screenpipe-engine | src/routes/frames.rs | source | 12 | 1 | 13 |-| engine-api-routes | screenpipe-engine | src/routes/health.rs | source | 57 | 0 | 57 |+| engine-api-routes | screenpipe-engine | src/routes/health.rs | source | 58 | 0 | 58 || engine-api-routes | screenpipe-engine | src/routes/live_views.rs | source | 1 | 0 | 1 || engine-api-routes | screenpipe-engine | src/routes/meeting_summary_status.rs | source | 11 | 0 | 11 || engine-api-routes | screenpipe-engine | src/routes/meetings.rs | source | 7 | 0 | 7 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex c24c1a482..40369c767 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -940,6 +940,7 @@"src/core/process_tap/counters.rs","src/core/source_buffer.rs","src/core/stream.rs",+ "src/core/vpio_process.rs","src/device/device_manager.rs","src/idle_detector.rs","src/lib.rs",