Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Fix native AEC microphone recovery with supervised capture workers (#7658)

Louis Beaumont3ec2a7786001
diff --git a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
index 84b3653e0..f7a095ade 100644
--- a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
+++ b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
@@ -11418,6 +11418,7 @@ dependencies = [
"hound",
"knf-rs",
"lazy_static",
+ "libc",
"libpulse-binding",
"libpulse-simple-binding",
"libsamplerate-sys",
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs b/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs
index f79b8bda7..846667b50 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs
@@ -15,8 +15,8 @@
//!
//! The mic side has exactly one event (`audio_capture_health_mic_capture_failed`),
//! covering the meeting-piggyback mic follow: the meeting app's mic could not
-//! be opened at all (no action button — capture already fell back to the
-//! default mic). There are deliberately NO mic-silence events: the piggyback
+//! be opened at all. The event does not prove a default-mic fallback.
+//! There are deliberately NO mic-silence events: the piggyback
//! never acts on silence (a silent meeting mic is the user's own in-meeting
//! feedback loop — they fix it in the app and the piggyback follows).
//!
@@ -106,18 +106,14 @@ fn show_speaker_silent_notification(app: AppHandle, data: Value) {
/// Show the in-app notification panel telling the user screenpipe couldn't
/// open the meeting app's microphone (per-process capture failed to start),
-/// but recording continues on the default mic. No action button — there's
-/// nothing to restart; the device is unavailable, not stalled.
+/// without claiming a default-mic fallback the event has not verified.
fn show_mic_capture_failed_notification(app: AppHandle, data: Value) {
let reason = data
.get("reason")
.and_then(|v| v.as_str())
.unwrap_or("the device may be in exclusive use");
- let body = format!(
- "{} — recording continues on your default microphone.",
- reason
- );
+ let body = mic_capture_failed_body(reason);
let payload = serde_json::json!({
"id": "audio_capture_health_mic_capture_failed",
@@ -134,3 +130,23 @@ fn show_mic_capture_failed_notification(app: AppHandle, data: Value) {
}
});
}
+
+fn mic_capture_failed_body(reason: &str) -> String {
+ format!("{}. Your voice may be missing from the recording. If this persists, quit and reopen screenpipe.", reason.trim_end_matches('.'))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn mic_failure_does_not_claim_unverified_fallback() {
+ let body = mic_capture_failed_body(
+ "audio device MacBook Pro Microphone (input) teardown timed out",
+ );
+ assert!(body.contains("Your voice may be missing"));
+ assert!(body.contains("quit and reopen screenpipe"));
+ assert!(!body.contains("recording continues"));
+ assert!(body.contains("teardown timed out."));
+ }
+}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
index e256fbb7c..af0ee5bbc 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
@@ -477,6 +477,11 @@ macro_rules! define_specta_builder {
#[tokio::main]
async fn main() {
+ #[cfg(target_os = "macos")]
+ if let Some(code) = screenpipe_audio::core::vpio_process::run_hidden_mode().await {
+ std::process::exit(code);
+ }
+
// Invoked by the scoped video tool, before any application side effects.
let arguments: Vec<String> = std::env::args().collect();
if arguments.get(1).is_some_and(|arg| arg == "--render-workflow-video") {
diff --git a/crates/screenpipe-audio/Cargo.toml b/crates/screenpipe-audio/Cargo.toml
index fc723593e..d5b228a26 100644
--- a/crates/screenpipe-audio/Cargo.toml
+++ b/crates/screenpipe-audio/Cargo.toml
@@ -169,6 +169,7 @@ samplerate = { workspace = true }
libsamplerate-sys = { workspace = true }
[target.'cfg(target_os = "macos")'.dependencies]
+libc = { workspace = true }
cidre = { git = "https://github.com/yury/cidre.git", rev = "8481f3f0dc7dd39bb5be80d9424bfac0cad3801a" }
once_cell = { workspace = true }
objc = { workspace = true }
diff --git a/crates/screenpipe-audio/src/audio_manager/manager.rs b/crates/screenpipe-audio/src/audio_manager/manager.rs
index a676a07e5..b69b889c4 100644
--- a/crates/screenpipe-audio/src/audio_manager/manager.rs
+++ b/crates/screenpipe-audio/src/audio_manager/manager.rs
@@ -1911,6 +1911,24 @@ impl AudioManager {
.collect::<Vec<AudioDevice>>()
}
+ /// Failed native teardown is persistent evidence, unlike the short-lived
+ /// stream-timeout metric. Only report devices still selected for capture;
+ /// stopped, user-paused and piggyback-suspended devices are not missing audio.
+ pub async fn capture_failure_devices(&self) -> Vec<AudioDevice> {
+ let mut selected = self.enabled_devices().await;
+ selected.extend(self.session_devices());
+ let disabled = self.user_disabled_devices().await;
+ let suspended = self.suspended_devices();
+ self.device_manager
+ .teardown_failed_devices()
+ .into_iter()
+ .filter(|device| {
+ let name = device.to_string();
+ selected.contains(&name) && !disabled.contains(&name) && !suspended.contains(&name)
+ })
+ .collect()
+ }
+
/// True when `device` has a live, non-disconnected stream. Thin wrapper over
/// the device monitor's [`super::is_device_actively_streaming`] so callers
/// that only hold an `AudioManager` (the piggyback sweep) can distinguish a
diff --git a/crates/screenpipe-audio/src/core/mod.rs b/crates/screenpipe-audio/src/core/mod.rs
index cf6a09681..0c13c0ef5 100644
--- a/crates/screenpipe-audio/src/core/mod.rs
+++ b/crates/screenpipe-audio/src/core/mod.rs
@@ -15,6 +15,8 @@ pub use run_record_and_transcribe::StreamDeath;
pub mod sck_output_watchdog;
pub mod source_buffer;
pub mod stream;
+#[cfg(target_os = "macos")]
+pub mod vpio_process;
use crate::AudioInput;
use anyhow::Result;
use dashmap::DashMap;
diff --git a/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs b/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs
index fad3ad7a7..ea6f7f591 100644
--- a/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs
+++ b/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs
@@ -226,12 +226,13 @@ pub async fn run_record_and_transcribe(
let mut collected_audio = Vec::new();
let mut segment_start_time = now_epoch_secs();
let mut last_diagnostics_time = Instant::now();
- let mut _last_non_zero_at: Option<Instant> = None;
- let mut _sck_watchdog = crate::core::sck_output_watchdog::SckOutputWatchdog::default();
+ let mut mic_deadline = tokio::time::Instant::now()
+ + Duration::from_secs(STREAM_STARTUP_GRACE_SECS + AUDIO_RECEIVE_TIMEOUT_SECS);
+ let mut stream_error = None;
let mut was_paused_for_lock = false;
- while is_running.load(Ordering::Relaxed)
+ 'capture: while is_running.load(Ordering::Relaxed)
&& !audio_stream.is_disconnected.load(Ordering::Relaxed)
{
// Skip recording while the screen is locked
@@ -273,15 +274,28 @@ pub async fn run_record_and_transcribe(
}
}
- while collected_audio.len() < max_samples && is_running.load(Ordering::Relaxed) {
+ while collected_audio.len() < max_samples
+ && is_running.load(Ordering::Relaxed)
+ && !audio_stream.is_disconnected.load(Ordering::Relaxed)
+ {
+ if audio_stream.capture_failed.load(Ordering::Acquire) && receiver.is_empty() {
+ stream_error = Some(anyhow!("native audio capture worker exited"));
+ break 'capture;
+ }
tokio::select! {
// Receive mic chunk
mic_res = receiver.recv() => {
match mic_res {
Ok(chunk) => {
+ if !chunk.is_empty() {
+ // Silence is valid delivery. Only actual
+ // mic samples extend this deadline, never
+ // speaker traffic or the 50ms AEC tick.
+ mic_deadline = tokio::time::Instant::now()
+ + Duration::from_secs(AUDIO_RECEIVE_TIMEOUT_SECS);
+ }
metrics.update_audio_levels(&device_name, &chunk);
if !chunk.is_empty() && chunk.iter().any(|&x| x.abs() >= SILENT_BUFFER_PEAK_THRESHOLD) {
- _last_non_zero_at = Some(Instant::now());
update_device_capture_time(&device_name);
}
@@ -305,7 +319,8 @@ pub async fn run_record_and_transcribe(
}
Err(e) => {
error!("AEC: Error receiving mic data: {}", e);
- return Err(anyhow!("Mic stream error: {}", e));
+ stream_error = Some(anyhow!("Mic stream error: {}", e));
+ break 'capture;
}
}
}
@@ -334,7 +349,17 @@ pub async fn run_record_and_transcribe(
}
speaker_sample_counter += chunk_len as u64;
}
- // Timeout to prevent hanging when silent
+ _ = tokio::time::sleep_until(mic_deadline) => {
+ if is_running.load(Ordering::Relaxed) {
+ stream_error = Some(anyhow!(StreamDeath::ReceiveTimeout {
+ secs: AUDIO_RECEIVE_TIMEOUT_SECS,
+ }));
+ }
+ // Preserve captured samples via the final flush before
+ // handing the typed failure to the recovery monitor.
+ break 'capture;
+ }
+ // Drain AEC while waiting for additional callbacks.
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
}
@@ -413,6 +438,19 @@ pub async fn run_record_and_transcribe(
warn!("AEC: Final mic flush failed for {}: {}", device_name, e);
}
+ if let Some(error) = stream_error {
+ if matches!(
+ error.downcast_ref::<StreamDeath>(),
+ Some(StreamDeath::ReceiveTimeout { .. })
+ ) {
+ // Publishing disconnected earlier lets the monitor abort this
+ // task while its captured tail is still waiting to be queued.
+ let _ = fail_stream_dead(&audio_stream, &device_name, &metrics);
+ } else if audio_stream.capture_failed.load(Ordering::Acquire) {
+ audio_stream.is_disconnected.store(true, Ordering::Relaxed);
+ }
+ return Err(error);
+ }
if audio_stream.is_disconnected.load(Ordering::Relaxed) {
info!("AEC: Stopped recording for {} (disconnected)", device_name);
Err(anyhow::anyhow!("device {} disconnected", device_name))
@@ -451,10 +489,11 @@ pub async fn run_record_and_transcribe(
let mut last_non_zero_at = last_non_zero_seed_after_restart(&device_name, Instant::now());
let mut sck_watchdog = crate::core::sck_output_watchdog::SckOutputWatchdog::default();
let mut segment_count: u64 = 0;
+ let mut stream_error = None;
let mut was_paused_for_lock = false;
- while is_running.load(Ordering::Relaxed)
+ 'capture: while is_running.load(Ordering::Relaxed)
&& !audio_stream.is_disconnected.load(Ordering::Relaxed)
{
if screenpipe_config::should_pause_audio_for_lock() {
@@ -498,7 +537,10 @@ pub async fn run_record_and_transcribe(
// stream is being torn down either way, so break out and
// let the flushes below persist what we have.
Err(_) if !is_running.load(Ordering::Relaxed) => break,
- Err(e) => return Err(e),
+ Err(e) => {
+ stream_error = Some(e);
+ break 'capture;
+ }
};
match received {
Some(chunk) => {
@@ -560,6 +602,17 @@ pub async fn run_record_and_transcribe(
warn!("final flush failed for {}: {}", device_name, e);
}
+ if let Some(error) = stream_error {
+ if audio_stream.capture_failed.load(Ordering::Acquire)
+ || error.downcast_ref::<StreamDeath>().is_some()
+ {
+ // The supervisor owns native cleanup. Only now, once
+ // queued audio is safe, may the monitor replace this recorder.
+ audio_stream.is_disconnected.store(true, Ordering::Relaxed);
+ }
+ return Err(error);
+ }
+
if audio_stream.is_disconnected.load(Ordering::Relaxed) {
info!("stopped recording for {} (disconnected)", device_name);
Err(anyhow::anyhow!("device {} disconnected", device_name))
@@ -579,6 +632,11 @@ async fn recv_audio_chunk(
last_non_zero_at: &mut Option<Instant>,
sck_watchdog: &mut crate::core::sck_output_watchdog::SckOutputWatchdog,
) -> Result<Option<Vec<f32>>> {
+ // Consume packets already queued before the terminal wake-up. A native
+ // worker failure must not expose disconnection until the caller flushes.
+ if audio_stream.capture_failed.load(Ordering::Acquire) && receiver.is_empty() {
+ return Err(anyhow!("native audio capture worker exited"));
+ }
let recv_result = tokio::time::timeout(
Duration::from_secs(AUDIO_RECEIVE_TIMEOUT_SECS),
receiver.recv(),
@@ -618,7 +676,6 @@ async fn recv_audio_chunk(
device_name, INPUT_SILENT_BUFFER_TIMEOUT_SECS
);
metrics.record_stream_timeout(device_name);
- audio_stream.is_disconnected.store(true, Ordering::Relaxed);
return Err(anyhow!(StreamDeath::ZeroFill {
device: device_name.to_string(),
secs: INPUT_SILENT_BUFFER_TIMEOUT_SECS,
@@ -672,7 +729,8 @@ async fn recv_audio_chunk(
sck_watchdog,
)
} else {
- fail_stream_dead(audio_stream, device_name, metrics)
+ // The recorder publishes disconnection after flushing its tail.
+ stream_dead_error(device_name, metrics)
}
}
}
@@ -705,13 +763,20 @@ fn fail_stream_dead(
audio_stream: &Arc<AudioStream>,
device_name: &str,
metrics: &Arc<AudioPipelineMetrics>,
+) -> Result<Option<Vec<f32>>> {
+ audio_stream.is_disconnected.store(true, Ordering::Relaxed);
+ stream_dead_error(device_name, metrics)
+}
+
+fn stream_dead_error(
+ device_name: &str,
+ metrics: &Arc<AudioPipelineMetrics>,
) -> Result<Option<Vec<f32>>> {
warn!(
"no audio received from {} for {}s - stream dead, triggering reconnect",
device_name, AUDIO_RECEIVE_TIMEOUT_SECS
);
metrics.record_stream_timeout(device_name);
- audio_stream.is_disconnected.store(true, Ordering::Relaxed);
// Typed cause so the VPIO runtime-fallback policy can recognize this death
// by `downcast_ref` rather than message text. Display is unchanged.
Err(anyhow!(StreamDeath::ReceiveTimeout {
@@ -885,6 +950,161 @@ mod tests {
use super::*;
use crate::core::device::AudioDevice;
+ #[tokio::test]
+ async fn native_worker_failure_flushes_audio_before_publishing_disconnection() {
+ let device = Arc::new(AudioDevice::new(
+ "native worker test".into(),
+ DeviceType::Input,
+ ));
+ let (stream, sender) = AudioStream::from_sender_for_test(device, 16_000, 1);
+ let stream = Arc::new(stream);
+ let (tx, rx) = flume::bounded(0);
+ let task = tokio::spawn(run_record_and_transcribe(
+ stream.clone(),
+ Duration::from_secs(30),
+ Arc::new(tx),
+ Arc::new(AtomicBool::new(true)),
+ Arc::new(AudioPipelineMetrics::new()),
+ None,
+ None,
+ false,
+ ));
+ while sender.receiver_count() == 0 {
+ tokio::task::yield_now().await;
+ }
+ sender.send(vec![0.25; 3_200]).unwrap();
+ stream.capture_failed.store(true, Ordering::Release);
+ sender.send(Vec::new()).unwrap();
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ assert!(!task.is_finished());
+ assert!(
+ !stream.is_disconnected(),
+ "monitor must wait for the buffered tail"
+ );
+ let segment = tokio::time::timeout(Duration::from_secs(1), rx.recv_async()).await;
+ if segment.is_err() {
+ task.abort();
+ }
+ let segment = segment
+ .expect("native worker failure must flush captured audio")
+ .unwrap();
+ assert_eq!(segment.data.len(), 3_200);
+ let error = tokio::time::timeout(Duration::from_secs(1), task)
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap_err();
+ assert!(error
+ .to_string()
+ .contains("native audio capture worker exited"));
+ assert!(stream.is_disconnected());
+ assert!(rx.is_empty());
+ }
+
+ #[tokio::test]
+ async fn aec_mic_callback_stall_requests_recovery_and_flushes_captured_audio() {
+ let device = Arc::new(AudioDevice::new(
+ "AEC stalled input".into(),
+ DeviceType::Input,
+ ));
+ let (stream, sender) = AudioStream::from_sender_for_test(device, 16_000, 1);
+ let stream = Arc::new(stream);
+ let (segments_tx, segments_rx) = flume::bounded(0);
+ let metrics = Arc::new(AudioPipelineMetrics::new());
+ let mut task = tokio::spawn(run_record_and_transcribe(
+ stream.clone(),
+ Duration::from_secs(30),
+ Arc::new(segments_tx),
+ Arc::new(AtomicBool::new(true)),
+ metrics.clone(),
+ None,
+ None,
+ true,
+ ));
+ while sender.receiver_count() == 0 {
+ tokio::task::yield_now().await;
+ }
+ sender.send(vec![0.25; 3_200]).unwrap();
+ // Keep the sender alive but stop callbacks, exactly like a native
+ // worker stuck in CoreAudio. A timer tick is not a microphone sample.
+ tokio::time::sleep(Duration::from_secs(AUDIO_RECEIVE_TIMEOUT_SECS + 1)).await;
+ assert!(
+ !task.is_finished(),
+ "flush must wait for the persistence queue"
+ );
+ assert!(
+ !stream.is_disconnected(),
+ "recovery must not abort a pending final flush"
+ );
+ let segment = tokio::time::timeout(Duration::from_secs(1), segments_rx.recv_async())
+ .await
+ .expect("AEC must flush after mic callbacks stop")
+ .unwrap();
+ let result = tokio::time::timeout(
+ Duration::from_secs(AUDIO_RECEIVE_TIMEOUT_SECS + 2),
+ &mut task,
+ )
+ .await;
+ if result.is_err() {
+ task.abort();
+ }
+ let error = result
+ .expect("AEC must not wait forever after mic callbacks stop")
+ .unwrap()
+ .unwrap_err();
+ assert_eq!(
+ error.downcast_ref::<StreamDeath>(),
+ Some(&StreamDeath::ReceiveTimeout {
+ secs: AUDIO_RECEIVE_TIMEOUT_SECS
+ })
+ );
+ assert!(stream.is_disconnected());
+ assert!(
+ segments_rx.is_empty(),
+ "captured audio must be emitted exactly once"
+ );
+ assert!(!segment.data.is_empty());
+ }
+
+ #[tokio::test]
+ async fn aec_silent_callbacks_keep_recording_past_receive_timeout() {
+ let device = Arc::new(AudioDevice::new(
+ "AEC silent input".into(),
+ DeviceType::Input,
+ ));
+ let (stream, sender) = AudioStream::from_sender_for_test(device, 16_000, 1);
+ let stream = Arc::new(stream);
+ let (segments_tx, segments_rx) = flume::bounded(8);
+ let running = Arc::new(AtomicBool::new(true));
+ let task = tokio::spawn(run_record_and_transcribe(
+ stream.clone(),
+ Duration::from_secs(30),
+ Arc::new(segments_tx),
+ running.clone(),
+ Arc::new(AudioPipelineMetrics::new()),
+ None,
+ None,
+ true,
+ ));
+ while sender.receiver_count() == 0 {
+ tokio::task::yield_now().await;
+ }
+ for _ in 0..(AUDIO_RECEIVE_TIMEOUT_SECS + 1) * 10 {
+ sender.send(vec![0.0; 1_600]).unwrap();
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ assert!(!task.is_finished(), "silence is not missing callbacks");
+ }
+ running.store(false, Ordering::Relaxed);
+ sender.send(Vec::new()).unwrap();
+ tokio::time::timeout(Duration::from_secs(1), task)
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap();
+ assert!(!stream.is_disconnected());
+ assert!(!segments_rx.try_recv().unwrap().data.is_empty());
+ }
+
async fn run_stopped_recorder(
samples: Vec<f32>,
segment_duration: Duration,
diff --git a/crates/screenpipe-audio/src/core/stream.rs b/crates/screenpipe-audio/src/core/stream.rs
index 84f875b85..dc86971f0 100644
--- a/crates/screenpipe-audio/src/core/stream.rs
+++ b/crates/screenpipe-audio/src/core/stream.rs
@@ -20,8 +20,10 @@ use std::sync::Arc;
#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]
use std::sync::{Mutex, OnceLock};
use tokio::sync::{broadcast, oneshot};
+#[cfg(target_os = "windows")]
+use tracing::info;
#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]
-use tracing::{error, info, warn};
+use tracing::{error, warn};
#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]
use crate::utils::audio::audio_to_mono;
@@ -80,6 +82,8 @@ pub struct AudioStream {
stream_control: mpsc::Sender<StreamControl>,
stream_thread: Option<Arc<tokio::sync::Mutex<Option<tokio::task::JoinHandle<()>>>>>,
pub is_disconnected: Arc<AtomicBool>,
+ /// Native worker exited; recorder flushes its tail before publishing disconnection.
+ pub(crate) capture_failed: Arc<AtomicBool>,
/// True when this stream is backed by the cpal/ScreenCaptureKit path
/// (the default), false when it is backed by the CoreAudio Process Tap.
/// The macOS System Audio liveness watchdog ([`super::sck_output_watchdog`])
@@ -90,7 +94,7 @@ pub struct AudioStream {
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-enum StopMode {
+pub(super) enum StopMode {
Immediate,
#[cfg(any(not(all(target_os = "linux", feature = "pulseaudio")), test))]
DeferredDeviceDisconnect,
@@ -109,7 +113,7 @@ impl StopMode {
}
}
-enum StreamControl {
+pub(super) enum StreamControl {
Stop {
_response: oneshot::Sender<()>,
_mode: StopMode,
@@ -158,6 +162,7 @@ impl AudioStream {
let (tx, _) = broadcast::channel::<Vec<f32>>(1000);
let tx_clone = tx.clone();
let is_disconnected = Arc::new(AtomicBool::new(false));
+ let capture_failed = Arc::new(AtomicBool::new(false));
let (stream_control_tx, stream_control_rx) = mpsc::channel();
#[cfg(all(target_os = "linux", feature = "pulseaudio"))]
@@ -260,6 +265,7 @@ impl AudioStream {
&stream_control_tx,
windows_input_aec,
macos_input_vpio,
+ &capture_failed,
)
.await?;
(config, thread, true)
@@ -280,6 +286,7 @@ impl AudioStream {
&stream_control_tx,
windows_input_aec,
macos_input_vpio,
+ &capture_failed,
)
.await?;
(config, thread, true)
@@ -293,6 +300,7 @@ impl AudioStream {
stream_control: stream_control_tx,
stream_thread: Some(Arc::new(tokio::sync::Mutex::new(Some(stream_thread)))),
is_disconnected,
+ capture_failed,
is_sck_backed,
})
}
@@ -310,56 +318,28 @@ impl AudioStream {
stream_control_tx: &mpsc::Sender<StreamControl>,
windows_input_aec: bool,
macos_input_vpio: bool,
+ #[cfg_attr(not(target_os = "macos"), allow(unused_variables))] capture_failed: &Arc<
+ AtomicBool,
+ >,
) -> Result<(AudioStreamConfig, tokio::task::JoinHandle<()>)> {
- // Only the macOS VoiceProcessingIO branch below reassigns `config`, so the
- // `mut` is dead weight on every other target.
- #[cfg_attr(not(target_os = "macos"), allow(unused_mut))]
- let (cpal_audio_device, mut config) = get_cpal_device_and_config(device).await?;
- let is_running_weak = Arc::downgrade(is_running);
- let input_aec = windows_input_aec && device.device_type == super::device::DeviceType::Input;
- let input_vpio = macos_input_vpio
- && device.device_type == super::device::DeviceType::Input
- && is_default_input_device(device);
-
#[cfg(target_os = "macos")]
- {
- if macos_input_vpio && input_vpio {
- let original_rate = config.sample_rate().0;
- info!(
- device = %device,
- "screenpipe-audio: enabling VoiceProcessingIO (AEC) on default microphone"
- );
- config = cpal_audio_device.default_input_config().map_err(|e| {
- anyhow!(
- "could not get default input config for VoiceProcessingIO on {}: {}",
- device,
- e
- )
- })?;
- let vpio_rate = config.sample_rate().0;
- if vpio_rate != original_rate {
- info!(
- device = %device,
- original_rate,
- vpio_rate,
- "screenpipe-audio: VPIO config rate differs from originally-picked config; \
- using hardware native rate"
- );
- }
- info!(
- device = %device,
- sample_rate = vpio_rate,
- channels = config.channels(),
- "screenpipe-audio: using default input config for VoiceProcessingIO"
- );
- } else if macos_input_vpio && device.device_type == super::device::DeviceType::Input {
- info!(
- device = %device,
- "screenpipe-audio: macOS VPIO requested but using HAL (only system default input supports VoiceProcessingIO)"
- );
- }
+ if macos_input_vpio && is_default_input_device(device) {
+ return super::vpio_process::start(
+ device.clone(),
+ tx,
+ stream_control_rx,
+ capture_failed.clone(),
+ )
+ .await;
}
+ let (cpal_audio_device, config) = get_cpal_device_and_config(device).await?;
+ let is_running_weak = Arc::downgrade(is_running);
+ let input_aec = windows_input_aec && device.device_type == super::device::DeviceType::Input;
+ // Native AEC is isolated above. Other CPAL streams stay in-process.
+ let _ = macos_input_vpio;
+ let input_vpio = false;
+
let audio_config = AudioStreamConfig::from(&config);
let channels = config.channels();
@@ -394,8 +374,6 @@ impl AudioStream {
macos_input_vpio: bool,
) -> Result<tokio::task::JoinHandle<()>> {
let device_name = device.name()?;
- #[cfg(target_os = "macos")]
- let mut use_vpio = macos_input_vpio;
#[cfg(target_os = "windows")]
let use_aec = windows_input_aec;
@@ -516,43 +494,6 @@ impl AudioStream {
}
}
}
- #[cfg(target_os = "macos")]
- Err(primary_err) if macos_input_vpio => {
- // Report the backend actually running after fallback.
- use_vpio = false;
- warn!(
- device = %device_name,
- rate = config.sample_rate().0,
- channels = channels,
- format = ?config.sample_format(),
- "VoiceProcessingIO stream creation failed ({}); retrying with HAL (VPIO disabled)",
- primary_err,
- );
- let fallback_cb = create_error_callback(
- device_name.clone(),
- is_running_weak,
- is_disconnected,
- stream_control_tx,
- );
- match build_input_stream(
- &device,
- &config,
- channels,
- tx,
- fallback_cb,
- windows_input_aec,
- false,
- ) {
- Ok(s) => Some(s),
- Err(fallback_err) => {
- error!(
- "HAL fallback also failed for {} after VPIO error: {} (VPIO error: {})",
- device_name, fallback_err, primary_err
- );
- None
- }
- }
- }
Err(e) => {
if is_mic_access_denied(&e) {
if first_access_denied_for_device(&device_name) {
@@ -574,13 +515,6 @@ impl AudioStream {
return;
}
- #[cfg(target_os = "macos")]
- if use_vpio {
- info!(
- device = %device_name,
- "screenpipe-audio: VoiceProcessingIO microphone capture running (AEC initialized)"
- );
- }
#[cfg(target_os = "windows")]
if use_aec {
info!(
@@ -681,11 +615,35 @@ impl AudioStream {
stream_control: stream_control_tx,
stream_thread: None,
is_disconnected: Arc::new(AtomicBool::new(false)),
+ capture_failed: Arc::new(AtomicBool::new(false)),
is_sck_backed: false,
};
(stream, tx_arc)
}
+ #[cfg(all(test, not(all(target_os = "linux", feature = "pulseaudio"))))]
+ pub(crate) fn with_blocked_teardown_for_test(
+ device: Arc<AudioDevice>,
+ ) -> (Self, mpsc::Sender<()>) {
+ let (mut stream, _tx) = Self::from_sender_for_test(device, 48_000, 1);
+ let (control_tx, control_rx) = mpsc::channel();
+ let (release_tx, release_rx) = mpsc::channel();
+ stream.stream_control = control_tx;
+ stream.stream_thread = Some(Arc::new(tokio::sync::Mutex::new(Some(
+ tokio::task::spawn_blocking(move || {
+ let _ = release_rx.recv();
+ if let Ok(StreamControl::Stop {
+ _response: response,
+ ..
+ }) = control_rx.recv()
+ {
+ let _ = response.send(());
+ }
+ }),
+ ))));
+ (stream, release_tx)
+ }
+
/// Build an AudioStream that plays back a wav (or any symphonia-decodable)
/// file into the broadcast channel, mimicking what a real cpal device
/// would produce. Lets the eval harness drive the full pipeline (VAD,
@@ -760,6 +718,7 @@ impl AudioStream {
// shape so `stop()` can abort the playback task uniformly.
stream_thread: Some(Arc::new(tokio::sync::Mutex::new(Some(thread)))),
is_disconnected,
+ capture_failed: Arc::new(AtomicBool::new(false)),
// wav playback fixture is an input device, never the SCK output path.
is_sck_backed: false,
})
@@ -846,7 +805,7 @@ fn first_access_denied_for_device(device: &str) -> bool {
}
#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]
-fn build_input_stream(
+pub(super) fn build_input_stream(
device: &cpal::Device,
config: &cpal::SupportedStreamConfig,
channels: u16,
@@ -910,26 +869,18 @@ fn build_input_stream(
}
}
-#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]
+#[cfg(target_os = "macos")]
fn is_default_input_device(device: &AudioDevice) -> bool {
use super::device::{default_input_device, DeviceType};
-
- if device.device_type != DeviceType::Input {
- return false;
- }
-
- default_input_device()
- .map(|default_device| default_device == *device)
- .unwrap_or(false)
+ device.device_type == DeviceType::Input
+ && default_input_device()
+ .map(|default| default == *device)
+ .unwrap_or(false)
}
#[cfg(target_os = "macos")]
fn macos_voice_processing_config(enabled: bool) -> Option<cpal::MacosVoiceProcessingInputConfig> {
- if enabled {
- Some(cpal::MacosVoiceProcessingInputConfig::screenpipe_aec())
- } else {
- None
- }
+ enabled.then(cpal::MacosVoiceProcessingInputConfig::screenpipe_aec)
}
#[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]
@@ -1124,6 +1075,79 @@ mod from_wav_tests {
}
}
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn native_aec_remains_enabled_for_capture_worker() {
+ let config = macos_voice_processing_config(true).unwrap();
+ assert!(config.enable_voice_processing);
+ assert_eq!(config.voice_processing_bypass, Some(false));
+ assert!(macos_voice_processing_config(false).is_none());
+ }
+
+ #[cfg(target_os = "macos")]
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ #[ignore = "opens the real microphone with native Apple AEC"]
+ async fn live_native_aec_records_and_flushes_audio() -> Result<()> {
+ use crate::core::run_record_and_transcribe::run_record_and_transcribe;
+ use crate::metrics::AudioPipelineMetrics;
+ use crate::utils::ffmpeg::{read_audio_from_file, write_audio_to_file};
+ let device = Arc::new(super::super::device::default_input_device()?);
+ let stream = Arc::new(
+ AudioStream::from_device(
+ device,
+ Arc::new(AtomicBool::new(true)),
+ false,
+ false,
+ true,
+ None,
+ )
+ .await?,
+ );
+ let mut samples = stream.subscribe().await;
+ let running = Arc::new(AtomicBool::new(true));
+ let (tx, rx) = flume::bounded(8);
+ let recorder = tokio::spawn(run_record_and_transcribe(
+ stream.clone(),
+ Duration::from_secs(30),
+ Arc::new(tx),
+ running.clone(),
+ Arc::new(AudioPipelineMetrics::new()),
+ None,
+ None,
+ false,
+ ));
+ tokio::time::timeout(Duration::from_secs(5), async {
+ let mut count = 0;
+ while count < stream.device_config.sample_rate().0 as usize {
+ count += samples.recv().await?.len();
+ }
+ Ok::<_, anyhow::Error>(())
+ })
+ .await??;
+ running.store(false, Ordering::Relaxed);
+ tokio::time::timeout(Duration::from_secs(2), recorder).await???;
+ stream.stop().await?;
+ let segments: Vec<_> = rx.try_iter().collect();
+ anyhow::ensure!(!segments.is_empty());
+ let rate = segments[0].sample_rate;
+ let captured: Vec<_> = segments
+ .iter()
+ .flat_map(|s| s.data.iter().copied())
+ .collect();
+ anyhow::ensure!(!captured.is_empty());
+ let dir = tempfile::tempdir()?;
+ let path = dir.path().join("native-aec.m4a");
+ write_audio_to_file(&captured, rate, &path, false)?;
+ let (decoded, decoded_rate) = read_audio_from_file(&path)?;
+ anyhow::ensure!(decoded.len() >= captured.len() * decoded_rate as usize / rate as usize);
+ println!(
+ "native AEC actual recorder: captured={}, decoded={}, teardown=ok",
+ captured.len(),
+ decoded.len()
+ );
+ Ok(())
+ }
+
/// 16 kHz mono sine wav round-trips through `from_wav`. The test counts
/// every chunk that lands on the broadcast receiver — sample count must
/// match the original signal exactly (resampling is bypassed for 16 kHz).
diff --git a/crates/screenpipe-audio/src/core/vpio_process.rs b/crates/screenpipe-audio/src/core/vpio_process.rs
new file mode 100644
index 000000000..cb0b20c06
--- /dev/null
+++ b/crates/screenpipe-audio/src/core/vpio_process.rs
@@ -0,0 +1,541 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+//! Native AEC runs in a child of the same signed executable. CoreAudio calls
+//! cannot be cancelled by Tokio; process ownership lets teardown kill and reap
+//! a wedged unit before recovery starts another native-AEC capture session.
+use super::device::{parse_audio_device, AudioDevice};
+use super::stream::{AudioStreamConfig, StreamControl};
+use anyhow::{anyhow, bail, Context, Result};
+use cpal::traits::{DeviceTrait, StreamTrait};
+use std::io::{BufRead, BufReader, BufWriter, Read, Write};
+use std::process::{Child, Command, Stdio};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{mpsc, Arc};
+use std::time::{Duration, Instant};
+use tokio::sync::{broadcast, oneshot};
+
+const MODE: &str = "--screenpipe-native-aec-worker";
+const MAGIC: &str = "SPVPIO1";
+const START_TIMEOUT: Duration = Duration::from_secs(18);
+const SAMPLE_TIMEOUT: Duration = Duration::from_secs(8);
+const STOP_GRACE: Duration = Duration::from_millis(500);
+const MAX_SAMPLES: usize = 65_536;
+
+enum Packet {
+ Ready(AudioStreamConfig),
+ Audio(Vec<f32>),
+}
+
+/// Drop is the cancellation backstop, including startup errors and panics.
+/// Never detach an owned child or leave a native capture worker behind.
+struct OwnedChild(Child);
+impl Drop for OwnedChild {
+ fn drop(&mut self) {
+ let _ = self.0.kill();
+ let _ = self.0.wait();
+ }
+}
+
+fn command(device: &AudioDevice) -> Result<Command> {
+ let mut command = Command::new(std::env::current_exe()?);
+ #[cfg(not(test))]
+ command.arg(MODE).arg(device.to_string());
+ #[cfg(test)]
+ command
+ .args([
+ "--exact",
+ "core::vpio_process::tests::child_entry",
+ "--ignored",
+ "--nocapture",
+ ])
+ .env("SCREENPIPE_TEST_AEC_DEVICE", device.to_string());
+ Ok(command)
+}
+
+pub(super) async fn start(
+ device: Arc<AudioDevice>,
+ tx: broadcast::Sender<Vec<f32>>,
+ control: mpsc::Receiver<StreamControl>,
+ capture_failed: Arc<AtomicBool>,
+) -> Result<(AudioStreamConfig, tokio::task::JoinHandle<()>)> {
+ let command = command(&device)?;
+ start_command(command, tx, control, capture_failed, START_TIMEOUT).await
+}
+
+async fn start_command(
+ command: Command,
+ tx: broadcast::Sender<Vec<f32>>,
+ control: mpsc::Receiver<StreamControl>,
+ capture_failed: Arc<AtomicBool>,
+ startup_timeout: Duration,
+) -> Result<(AudioStreamConfig, tokio::task::JoinHandle<()>)> {
+ let (ready_tx, ready_rx) = oneshot::channel();
+ let worker = tokio::task::spawn_blocking(move || {
+ let wake_recorder = tx.clone();
+ if let Err(error) = supervise(command, tx, control, ready_tx, startup_timeout) {
+ tracing::warn!(%error, "native AEC capture worker stopped");
+ capture_failed.store(true, Ordering::Release);
+ // Wake the recorder after all preceding audio packets. It must
+ // persist its tail before exposing disconnection to the monitor.
+ let _ = wake_recorder.send(Vec::new());
+ }
+ });
+ match ready_rx.await {
+ Ok(Ok(config)) => Ok((config, worker)),
+ result => {
+ // Supervisor has already killed/reaped before reporting failure.
+ worker.await?;
+ Err(match result {
+ Ok(Err(error)) => error,
+ Err(_) => anyhow!("native AEC worker exited before startup"),
+ _ => unreachable!(),
+ })
+ }
+ }
+}
+
+fn supervise(
+ mut command: Command,
+ tx: broadcast::Sender<Vec<f32>>,
+ control: mpsc::Receiver<StreamControl>,
+ ready: oneshot::Sender<Result<AudioStreamConfig>>,
+ startup_timeout: Duration,
+) -> Result<()> {
+ let mut ready = Some(ready);
+ let child = command
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::inherit())
+ .spawn();
+ let mut child = match child {
+ Ok(child) => OwnedChild(child),
+ Err(error) => {
+ let _ = ready.take().unwrap().send(Err(error.into()));
+ return Ok(());
+ }
+ };
+ let stdout = child.0.stdout.take().context("native AEC stdout missing")?;
+ let (packets_tx, packets_rx) = mpsc::sync_channel(8);
+ let reader = std::thread::spawn(move || {
+ let result = read_packets(stdout, &packets_tx);
+ if let Err(error) = result {
+ let _ = packets_tx.send(Err(error));
+ }
+ });
+ let mut deadline = Instant::now() + startup_timeout;
+ let mut stop_response = None;
+ let result = loop {
+ match control.try_recv() {
+ Ok(StreamControl::Stop { _response, .. }) => {
+ stop_response = Some(_response);
+ break Ok(());
+ }
+ Err(mpsc::TryRecvError::Disconnected) => break Ok(()),
+ Err(mpsc::TryRecvError::Empty) => {}
+ }
+ if ready.as_ref().is_some_and(|ready| ready.is_closed()) {
+ break Ok(()); // cancelled startup, no orphan
+ }
+ if Instant::now() >= deadline {
+ break Err(anyhow!(
+ "native AEC worker stopped delivering microphone samples"
+ ));
+ }
+ match packets_rx.recv_timeout(Duration::from_millis(20)) {
+ Ok(Ok(Packet::Ready(config))) => {
+ if let Some(ready) = ready.take() {
+ let _ = ready.send(Ok(config));
+ deadline = Instant::now() + SAMPLE_TIMEOUT;
+ } else {
+ break Err(anyhow!("duplicate native AEC header"));
+ }
+ }
+ Ok(Ok(Packet::Audio(samples))) => {
+ deadline = Instant::now() + SAMPLE_TIMEOUT;
+ let _ = tx.send(samples);
+ }
+ Ok(Err(error)) => break Err(error),
+ Err(mpsc::RecvTimeoutError::Disconnected) => {
+ break Err(anyhow!("native AEC worker closed audio pipe"))
+ }
+ Err(mpsc::RecvTimeoutError::Timeout) => {}
+ }
+ };
+ // Ask for graceful native disposal first; a stalled native call cannot
+ // hold the parent hostage. Keep stdin open until after kill/wait, so the
+ // parent's own teardown path is what establishes completion.
+ if let Some(stdin) = child.0.stdin.as_mut() {
+ let _ = stdin.write_all(b"s");
+ }
+ let stop_deadline = Instant::now() + STOP_GRACE;
+ while Instant::now() < stop_deadline {
+ if child.0.try_wait().ok().flatten().is_some() {
+ break;
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ drop(child); // kill if necessary, wait, then close pipe handles
+ drop(packets_rx); // release a reader blocked on the bounded queue
+ let _ = reader.join();
+ if let Some(ready) = ready {
+ let message = result
+ .as_ref()
+ .err()
+ .map(ToString::to_string)
+ .unwrap_or_else(|| "native AEC startup cancelled".into());
+ let _ = ready.send(Err(anyhow!(message)));
+ }
+ if let Some(response) = stop_response {
+ let _ = response.send(());
+ }
+ result
+}
+
+fn read_packets(read: impl Read, packets: &mpsc::SyncSender<Result<Packet>>) -> Result<()> {
+ let mut read = BufReader::new(read);
+ let mut header = String::new();
+ // The test executable emits a small harness preamble. Production emits
+ // the header first. Bound parsing so a wrong executable cannot exhaust RAM.
+ loop {
+ let mut line = String::new();
+ let size = read.by_ref().take(256).read_line(&mut line)?;
+ if size == 0 {
+ bail!("native AEC worker exited before header");
+ }
+ header.push_str(&line);
+ if header.len() > 1024 {
+ bail!("invalid native AEC header");
+ }
+ if line.starts_with(MAGIC) {
+ let fields: Vec<_> = line.split_whitespace().collect();
+ if fields.len() != 2 {
+ bail!("invalid native AEC format");
+ }
+ let rate: u32 = fields[1].parse()?;
+ if !(8_000..=384_000).contains(&rate) {
+ bail!("invalid native AEC sample rate");
+ }
+ packets.send(Ok(Packet::Ready(AudioStreamConfig::new(rate, 1))))?;
+ break;
+ }
+ }
+ loop {
+ let mut length = [0u8; 4];
+ read.read_exact(&mut length)?;
+ let count = u32::from_le_bytes(length) as usize;
+ if count == 0 || count > MAX_SAMPLES {
+ bail!("invalid native AEC packet size");
+ }
+ let mut bytes = vec![0; count * 4];
+ read.read_exact(&mut bytes)?;
+ let samples: Vec<f32> = bytes
+ .chunks_exact(4)
+ .map(|bytes| f32::from_le_bytes(bytes.try_into().unwrap()))
+ .collect();
+ if !samples.iter().all(|sample| sample.is_finite()) {
+ bail!("invalid native AEC sample");
+ }
+ packets.send(Ok(Packet::Audio(samples)))?;
+ }
+}
+
+/// Must run before desktop/engine initialization. The helper opens no database,
+/// server, UI, or transcription engine and uses the same executable identity.
+pub async fn run_hidden_mode() -> Option<i32> {
+ let args: Vec<_> = std::env::args().collect();
+ if args.get(1).map(String::as_str) != Some(MODE) {
+ return None;
+ }
+ Some(match args.get(2).filter(|_| args.len() == 3) {
+ Some(device) => match capture(device).await {
+ Ok(()) => 0,
+ Err(error) => {
+ eprintln!("native AEC worker: {error:#}");
+ 1
+ }
+ },
+ None => 2,
+ })
+}
+
+async fn capture(device_name: &str) -> Result<()> {
+ let stopping = Arc::new(AtomicBool::new(false));
+ let stop = stopping.clone();
+ std::thread::spawn(move || {
+ let mut byte = [0];
+ loop {
+ match std::io::stdin().read(&mut byte) {
+ // Do not run atexit/native destructors: those may be the very
+ // locks that wedged capture. The OS releases process resources.
+ Ok(0) | Err(_) => unsafe { libc::_exit(0) },
+ Ok(_) => stop.store(true, Ordering::Relaxed),
+ }
+ }
+ });
+ let device = parse_audio_device(device_name)?;
+ let (native_device, _) = super::device::get_cpal_device_and_config(&device).await?;
+ let config = native_device.default_input_config()?;
+ let (tx, mut rx) = broadcast::channel(1000);
+ let failed = Arc::new(AtomicBool::new(false));
+ let failure = failed.clone();
+ // Native AEC is mandatory here: no HAL/software substitution. Reuse the
+ // existing callback; IPC and serialization occur outside CoreAudio's callback.
+ let stream = super::stream::build_input_stream(
+ &native_device,
+ &config,
+ config.channels(),
+ tx,
+ move |error| {
+ // Render errors can be transient during route reconfiguration.
+ // Preserve the existing policy: device loss stops immediately;
+ // absent callbacks are handled by the parent's sample watchdog.
+ if matches!(error, cpal::StreamError::DeviceNotAvailable) {
+ failure.store(true, Ordering::Relaxed);
+ }
+ },
+ false,
+ true,
+ )?;
+ stream.play()?;
+ let output = std::io::stdout();
+ let mut stdout = BufWriter::with_capacity(MAX_SAMPLES * 4 + 4, output.lock());
+ writeln!(stdout, "{MAGIC} {}", config.sample_rate().0)?;
+ stdout.flush()?;
+ while !stopping.load(Ordering::Relaxed) && !failed.load(Ordering::Relaxed) {
+ tokio::select! {
+ sample = rx.recv() => match sample {
+ Ok(samples) if !samples.is_empty() => {
+ if samples.len() > MAX_SAMPLES { bail!("native callback exceeds packet limit"); }
+ stdout.write_all(&(samples.len() as u32).to_le_bytes())?;
+ for sample in samples { stdout.write_all(&sample.to_le_bytes())?; }
+ stdout.flush()?;
+ }
+ Ok(_) | Err(broadcast::error::RecvError::Lagged(_)) => {},
+ Err(error) => return Err(error.into()),
+ },
+ _ = tokio::time::sleep(Duration::from_millis(20)) => {},
+ }
+ }
+ stream.pause()?;
+ drop(stream);
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::core::device::{default_input_device, DeviceType};
+
+ #[test]
+ #[ignore = "private subprocess entry; invoked by supervisor tests"]
+ fn child_entry() {
+ if let Ok(path) = std::env::var("SCREENPIPE_TEST_AEC_PID") {
+ std::fs::write(path, std::process::id().to_string()).unwrap();
+ }
+ let device = std::env::var("SCREENPIPE_TEST_AEC_DEVICE").unwrap();
+ if let Ok(fault) = std::env::var("SCREENPIPE_TEST_AEC_FAULT") {
+ if fault != "open" {
+ let mut out = std::io::stdout();
+ writeln!(out, "{MAGIC} 16000").unwrap();
+ out.write_all(&1u32.to_le_bytes()).unwrap();
+ out.write_all(&0.25f32.to_le_bytes()).unwrap();
+ out.flush().unwrap();
+ }
+ // Model a native open/dispose that never returns. The parent must
+ // reap this actual process, not just abandon a task or JoinHandle.
+ loop {
+ std::thread::park();
+ }
+ }
+ let runtime = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ let result = runtime.block_on(capture(&device));
+ if let Err(error) = &result {
+ eprintln!("{error:#}");
+ }
+ std::process::exit(if result.is_ok() { 0 } else { 1 });
+ }
+
+ fn fixture_command(dir: &std::path::Path, fault: Option<&str>) -> Command {
+ let device = AudioDevice::new("test mic".into(), DeviceType::Input);
+ let mut cmd = command(&device).unwrap();
+ cmd.env("SCREENPIPE_TEST_AEC_PID", dir.join("pid"));
+ if let Some(fault) = fault {
+ cmd.env("SCREENPIPE_TEST_AEC_FAULT", fault);
+ }
+ cmd
+ }
+
+ fn pid(dir: &std::path::Path) -> i32 {
+ std::fs::read_to_string(dir.join("pid"))
+ .unwrap()
+ .parse()
+ .unwrap()
+ }
+
+ fn assert_reaped(pid: i32) {
+ assert_eq!(
+ unsafe { libc::kill(pid, 0) },
+ -1,
+ "capture process must be gone"
+ );
+ assert_eq!(
+ std::io::Error::last_os_error().raw_os_error(),
+ Some(libc::ESRCH)
+ );
+ }
+
+ async fn stop(control: &mpsc::Sender<StreamControl>, worker: tokio::task::JoinHandle<()>) {
+ let (response, done) = oneshot::channel();
+ control
+ .send(StreamControl::Stop {
+ _response: response,
+ _mode: super::super::stream::StopMode::Immediate,
+ })
+ .unwrap();
+ tokio::time::timeout(Duration::from_secs(3), done)
+ .await
+ .unwrap()
+ .unwrap();
+ tokio::time::timeout(Duration::from_secs(1), worker)
+ .await
+ .unwrap()
+ .unwrap();
+ }
+
+ #[tokio::test]
+ async fn blocked_native_disposal_is_reaped_before_stop_acknowledgement() {
+ let dir = tempfile::tempdir().unwrap();
+ let (tx, _) = broadcast::channel(16);
+ let (control, controls) = mpsc::channel();
+ let (_, worker) = start_command(
+ fixture_command(dir.path(), Some("stop")),
+ tx,
+ controls,
+ Arc::new(AtomicBool::new(false)),
+ Duration::from_secs(3),
+ )
+ .await
+ .unwrap();
+ let child_pid = pid(dir.path());
+ stop(&control, worker).await;
+ assert_reaped(child_pid);
+ }
+
+ #[tokio::test]
+ async fn blocked_native_construction_is_reaped_before_startup_error() {
+ let dir = tempfile::tempdir().unwrap();
+ let (tx, _) = broadcast::channel(16);
+ let (_control, controls) = mpsc::channel();
+ let result = start_command(
+ fixture_command(dir.path(), Some("open")),
+ tx,
+ controls,
+ Arc::new(AtomicBool::new(false)),
+ Duration::from_secs(1),
+ )
+ .await;
+ assert!(result.is_err());
+ assert_reaped(pid(dir.path()));
+ }
+
+ #[tokio::test]
+ async fn cancelling_startup_reaps_the_owned_process() {
+ let dir = tempfile::tempdir().unwrap();
+ let (tx, _) = broadcast::channel(16);
+ let (control, controls) = mpsc::channel();
+ let task = tokio::spawn(start_command(
+ fixture_command(dir.path(), Some("open")),
+ tx,
+ controls,
+ Arc::new(AtomicBool::new(false)),
+ Duration::from_secs(30),
+ ));
+ tokio::time::timeout(Duration::from_secs(3), async {
+ while !dir.path().join("pid").exists() {
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+ let child_pid = pid(dir.path());
+ task.abort();
+ let _ = task.await;
+ drop(control);
+ tokio::time::timeout(Duration::from_secs(3), async {
+ while unsafe { libc::kill(child_pid, 0) } == 0 {
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ })
+ .await
+ .unwrap();
+ assert_reaped(child_pid);
+ }
+
+ #[test]
+ fn malformed_worker_packets_cannot_allocate_unbounded_audio() {
+ let mut bytes = b"SPVPIO1 48000\n".to_vec();
+ bytes.extend_from_slice(&u32::MAX.to_le_bytes());
+ let (tx, rx) = mpsc::sync_channel(4);
+ assert!(read_packets(bytes.as_slice(), &tx).is_err());
+ assert!(matches!(rx.recv().unwrap().unwrap(), Packet::Ready(_)));
+ assert!(rx.try_recv().is_err());
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ #[ignore = "opens native AEC microphone and freezes only its own child"]
+ async fn live_native_aec_recovers_after_frozen_worker_and_persists_samples() -> Result<()> {
+ use crate::utils::ffmpeg::{read_audio_from_file, write_audio_to_file};
+ let device = default_input_device()?;
+ let dir = tempfile::tempdir()?;
+ for cycle in 0..3 {
+ let cycle_dir = dir.path().join(cycle.to_string());
+ std::fs::create_dir(&cycle_dir)?;
+ let mut cmd = command(&device)?;
+ cmd.env("SCREENPIPE_TEST_AEC_PID", cycle_dir.join("pid"));
+ let (tx, mut audio) = broadcast::channel(100);
+ let (control, controls) = mpsc::channel();
+ let capture_failed = Arc::new(AtomicBool::new(false));
+ let (config, worker) =
+ start_command(cmd, tx, controls, capture_failed.clone(), START_TIMEOUT).await?;
+ let samples = tokio::time::timeout(Duration::from_secs(5), async {
+ let mut samples = Vec::new();
+ while samples.len() < config.sample_rate().0 as usize / 2 {
+ samples.extend(audio.recv().await?);
+ }
+ Ok::<_, anyhow::Error>(samples)
+ })
+ .await??;
+ let path = cycle_dir.join("capture.m4a");
+ write_audio_to_file(&samples, config.sample_rate().0, &path, false)?;
+ let (decoded, rate) = read_audio_from_file(&path)?;
+ let expected = samples.len() * rate as usize / config.sample_rate().0 as usize;
+ assert!(decoded.len() >= expected);
+ let child_pid = pid(&cycle_dir);
+ // SIGSTOP freezes every thread, including any native stop call.
+ // This deterministic failure cannot be recovered with a Rust timeout.
+ assert_eq!(unsafe { libc::kill(child_pid, libc::SIGSTOP) }, 0);
+ let started = Instant::now();
+ if cycle == 1 {
+ // No stop request: the callback watchdog itself must dispose
+ // of the dead native worker so normal device recovery can run.
+ tokio::time::timeout(SAMPLE_TIMEOUT + Duration::from_secs(3), worker)
+ .await
+ .context("missing callbacks did not trigger native recovery")??;
+ assert!(capture_failed.load(Ordering::Relaxed));
+ } else {
+ stop(&control, worker).await;
+ }
+ assert_reaped(child_pid);
+ println!(
+ "native AEC cycle {cycle}: captured={}, decoded={}, frozen worker reaped in {:?}",
+ samples.len(),
+ decoded.len(),
+ started.elapsed()
+ );
+ }
+ Ok(())
+ }
+}
diff --git a/crates/screenpipe-audio/src/device/device_manager.rs b/crates/screenpipe-audio/src/device/device_manager.rs
index 652a9b8ae..9a8865d4d 100644
--- a/crates/screenpipe-audio/src/device/device_manager.rs
+++ b/crates/screenpipe-audio/src/device/device_manager.rs
@@ -32,6 +32,9 @@ pub struct DeviceManager {
operations: DashMap<AudioDevice, Arc<Mutex<()>>>,
streams: Arc<DashMap<AudioDevice, Arc<AudioStream>>>,
states: Arc<DashMap<AudioDevice, Arc<AtomicBool>>>,
+ /// Retained native workers whose stop failed. These must remain visible to
+ /// health even after the recording task has been removed.
+ teardown_failures: DashMap<AudioDevice, ()>,
/// When true, System Audio (output) uses the CoreAudio Process Tap path
/// on macOS 14.4+ instead of ScreenCaptureKit. Propagated to
/// AudioStream::from_device at device-start time. Has no effect on
@@ -60,6 +63,7 @@ impl DeviceManager {
operations: DashMap::new(),
streams,
states,
+ teardown_failures: DashMap::new(),
use_coreaudio_tap: AtomicBool::new(use_coreaudio_tap),
windows_input_aec: AtomicBool::new(windows_input_aec),
macos_input_vpio: AtomicBool::new(macos_input_vpio),
@@ -296,14 +300,25 @@ impl DeviceManager {
// Never hold a DashMap guard across native teardown. Retain the stream
// on timeout so a later start cannot orphan a still-running worker.
if let Some(stream) = self.stream(device) {
- stream.stop().await?;
+ if let Err(error) = stream.stop().await {
+ self.teardown_failures.insert(device.clone(), ());
+ return Err(error);
+ }
}
+ self.teardown_failures.remove(device);
self.streams.remove(device);
Ok(())
}
+ pub fn teardown_failed_devices(&self) -> Vec<AudioDevice> {
+ self.teardown_failures
+ .iter()
+ .map(|entry| entry.key().clone())
+ .collect()
+ }
+
pub fn is_running_mut(&self, device: &AudioDevice) -> Option<Arc<AtomicBool>> {
self.states.get(device).map(|s| s.value().clone())
}
@@ -315,6 +330,28 @@ mod tests {
use crate::core::device::DeviceType;
use crate::core::stream::AudioStream;
+ #[cfg(not(all(target_os = "linux", feature = "pulseaudio")))]
+ #[tokio::test]
+ async fn teardown_failure_survives_lost_recording_handle_until_native_cleanup() {
+ let dm = DeviceManager::new(false, false, false).await.unwrap();
+ let device = mic();
+ let (stream, release) =
+ AudioStream::with_blocked_teardown_for_test(Arc::new(device.clone()));
+ dm.streams.insert(device.clone(), Arc::new(stream));
+ dm.states
+ .insert(device.clone(), Arc::new(AtomicBool::new(false)));
+ assert!(dm.stop_device(&device).await.is_err());
+ assert_eq!(dm.teardown_failed_devices(), vec![device.clone()]);
+ assert!(
+ dm.stream(&device).is_some(),
+ "timeout must retain the native worker"
+ );
+ release.send(()).unwrap();
+ dm.stop_device(&device).await.unwrap();
+ assert!(dm.teardown_failed_devices().is_empty());
+ assert!(dm.stream(&device).is_none());
+ }
+
#[tokio::test]
async fn published_stream_is_not_reopened_before_recording_handle_registration() {
let dm = DeviceManager::new(false, false, false).await.unwrap();
diff --git a/crates/screenpipe-config/src/recording.rs b/crates/screenpipe-config/src/recording.rs
index 1f8202b2f..30e55696d 100644
--- a/crates/screenpipe-config/src/recording.rs
+++ b/crates/screenpipe-config/src/recording.rs
@@ -1303,6 +1303,13 @@ mod tests {
assert_eq!(settings.effective_aec_flags(), (false, false, false));
}
+ #[test]
+ fn native_macos_aec_selection_preserves_native_echo_cancellation() {
+ let settings: RecordingSettings =
+ serde_json::from_str(r#"{"aecMode":"macos","macosInputVpioEnabled":true}"#).unwrap();
+ assert_eq!(settings.effective_aec_flags(), (false, false, true));
+ }
+
#[test]
fn explicit_screenpipe_aec_mode_enables_software_aec() {
let settings: RecordingSettings = serde_json::from_str(
diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
index 669d220e3..42f533054 100644
--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
@@ -297,6 +297,12 @@ fn main() -> anyhow::Result<()> {
let _profiler = dhat::Profiler::new_heap();
let runtime = tokio::runtime::Runtime::new()?;
+ #[cfg(target_os = "macos")]
+ if let Some(code) = runtime.block_on(screenpipe_audio::core::vpio_process::run_hidden_mode())
+ {
+ std::process::exit(code);
+ }
+
// Private ACP modes must run before public CLI parsing or AppKit startup.
if let Some(exit_code) =
runtime.block_on(screenpipe_core::agents::acp::run_hidden_mode_without_observer())
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index 7fdbef589..ef57c8e02 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -523,9 +523,9 @@ fn input_device_is_available(device_name: &str, clamshell_mode: bool) -> bool {
}
/// Classify the raw audio capture status from health signals. Pure so it can be
-/// unit-tested in isolation. `stream_timeout_recent` must mean a recent timeout
-/// that the same current device has not recovered from, NOT a cumulative count
-/// or aggregate activity from some other stream.
+/// unit-tested in isolation. Timeouts are correlated to the same current device.
+/// A retained native teardown failure is separate: its recording handle may be
+/// gone and its original timeout may have expired while the worker is stuck.
#[allow(clippy::too_many_arguments)]
fn classify_audio_status(
audio_disabled: bool,
@@ -539,6 +539,7 @@ fn classify_audio_status(
last_audio_ts: u64,
now_ts: u64,
threshold_secs: u64,
+ native_teardown_failed: bool,
) -> &'static str {
if audio_disabled {
"disabled"
@@ -552,6 +553,11 @@ fn classify_audio_status(
"meeting_detector_unavailable"
} else if audio_waiting_for_meeting {
"waiting_for_meeting"
+ } else if native_teardown_failed {
+ // Includes a selected stream whose native teardown is still stuck.
+ // Its recording handle may already be gone. Neither that absence nor
+ // a healthy output stream turns the capture failure into benign idle.
+ "active_no_data"
} else if !has_input_device {
// Audio is on but there is no microphone to capture from — expected
// idle, not a failure. Distinct from "not_started" so /health stays 200
@@ -562,9 +568,7 @@ fn classify_audio_status(
} else if audio_never_captured {
"not_started"
} else if stream_timeout_recent {
- // The watchdog fired on a current device and that same device has not
- // produced usable audio since. This remains a failure even if another
- // microphone or system-output stream is healthy.
+ // Healthy system output cannot clear a timeout on a current microphone.
"active_no_data"
} else if global_audio_active {
"ok"
@@ -1598,6 +1602,12 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
// from. The old global timestamp let a healthy output stream mask a dead
// mic, while the older cumulative counter pinned recovered devices forever.
let now_ts = now.timestamp().max(0) as u64;
+ let mut failed_capture_devices = state.audio_manager.capture_failure_devices().await;
+ // A built-in microphone is intentionally unavailable in clamshell mode.
+ failed_capture_devices.retain(|device| {
+ let name = device.to_string();
+ !name.contains("(input)") || input_device_is_available(&name, clamshell_mode)
+ });
let stream_timeout_recent = has_unrecovered_recent_stream_timeout(
&state.audio_metrics.per_device_stream_timeouts_snapshot(),
&current_device_capture_at,
@@ -1622,6 +1632,7 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
last_audio_ts,
now_ts,
threshold_secs,
+ !failed_capture_devices.is_empty(),
)
.to_string();
@@ -1662,9 +1673,10 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
);
// Format device statuses as a string for a more detailed view
- let device_status_details = if !device_statuses.is_empty() {
+ let device_status_details = if !device_statuses.is_empty() || !failed_capture_devices.is_empty()
+ {
let now_secs = now.timestamp() as u64;
- let device_details: Vec<String> = device_statuses
+ let mut device_details: Vec<String> = device_statuses
.iter()
.map(|(name, active, last_capture)| {
format!(
@@ -1676,6 +1688,12 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
})
.collect();
+ device_details.extend(failed_capture_devices.iter().map(|device| {
+ format!(
+ "{}: capture failed (native teardown did not finish)",
+ device
+ )
+ }));
Some(device_details.join(", "))
} else {
None
@@ -3219,9 +3237,29 @@ mod tests {
1_000, // last_audio_ts
1_010, // now_ts
60, // threshold_secs
+ false,
)
}
+ #[test]
+ fn failed_mic_with_live_output_is_not_healthy() {
+ // The recording handle was removed, but the selected native worker is
+ // still stuck. Fresh system output must not make the mic loss benign.
+ assert_eq!(
+ classify_audio_status(
+ false, false, false, false, false, false, false, true, 999, 1000, 60, true,
+ ),
+ "active_no_data"
+ );
+ // A real machine without a mic and without a failure remains healthy.
+ assert_eq!(
+ classify_audio_status(
+ false, false, false, false, false, false, false, true, 999, 1000, 60, false,
+ ),
+ "no_input_device"
+ );
+ }
+
#[test]
fn timeout_recovery_is_correlated_to_the_same_current_device() {
let timeouts = HashMap::from([("mic".to_string(), 100)]);
@@ -3401,29 +3439,35 @@ mod tests {
// Guard the unrelated branches against accidental regressions.
assert_eq!(
classify_audio_status(
- true, true, false, false, false, true, true, true, 1000, 1010, 60
+ true, true, false, false, false, true, true, true, 1000, 1010, 60, false,
),
"disabled"
);
// intentional lock pause wins over stale/not-started signals
assert_eq!(
- classify_audio_status(false, true, false, false, true, true, false, false, 0, 1010, 60),
+ classify_audio_status(
+ false, true, false, false, true, true, false, false, 0, 1010, 60, false
+ ),
"ok"
);
// a missing prerequisite fails closed and is visible, not benign idle
assert_eq!(
- classify_audio_status(false, false, true, false, true, true, false, false, 0, 1010, 60),
+ classify_audio_status(
+ false, false, true, false, true, true, false, false, 0, 1010, 60, false
+ ),
"meeting_detector_unavailable"
);
// intentional meetings-only idle is distinct and benign
assert_eq!(
- classify_audio_status(false, false, false, true, true, true, false, false, 0, 1010, 60),
+ classify_audio_status(
+ false, false, false, true, true, true, false, false, 0, 1010, 60, false
+ ),
"waiting_for_meeting"
);
// never captured + no mic -> benign no_input_device (stays 200)
assert_eq!(
classify_audio_status(
- false, false, false, false, true, false, false, false, 0, 1010, 60
+ false, false, false, false, true, false, false, false, 0, 1010, 60, false,
),
"no_input_device"
);
@@ -3431,28 +3475,28 @@ mod tests {
// remains the same benign state instead of degrading to stale.
assert_eq!(
classify_audio_status(
- false, false, false, false, false, false, true, false, 1000, 2000, 60
+ false, false, false, false, false, false, true, false, 1000, 2000, 60, false,
),
"no_input_device"
);
// never captured but a mic exists -> not_started
assert_eq!(
classify_audio_status(
- false, false, false, false, true, true, false, false, 0, 1010, 60
+ false, false, false, false, true, true, false, false, 0, 1010, 60, false,
),
"not_started"
);
// not active, last audio within threshold -> ok
assert_eq!(
classify_audio_status(
- false, false, false, false, false, true, false, false, 1000, 1030, 60
+ false, false, false, false, false, true, false, false, 1000, 1030, 60, false,
),
"ok"
);
// not active, last audio stale -> stale
assert_eq!(
classify_audio_status(
- false, false, false, false, false, true, false, false, 1000, 2000, 60
+ false, false, false, false, false, true, false, false, 1000, 2000, 60, false,
),
"stale"
);
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index aabd81aa0..6942286bf 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 37
-- Mapped Rust files: 401
-- Active test blocks: 3905
-- Ignored/manual test blocks: 185
-- Weighted coverage points: 3216.2
+- Mapped Rust files: 402
+- Active test blocks: 3915
+- Ignored/manual test blocks: 188
+- Weighted coverage points: 3225.0
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3742 | 167 | 3143.2 | 21 | 11 | 100% |
-| macos | 32 | 3785 | 146 | 3138.5 | 22 | 11 | 100% |
-| linux | 29 | 3320 | 142 | 2761.5 | 20 | 11 | 100% |
+| windows | 33 | 3752 | 170 | 3152.0 | 21 | 11 | 100% |
+| macos | 32 | 3795 | 149 | 3147.3 | 22 | 11 | 100% |
+| linux | 29 | 3330 | 145 | 2770.3 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 355a6ae75..340c73490 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 37
-- Mapped Rust files: 401
-- Active test blocks: 3905
-- Ignored/manual test blocks: 185
-- Declared test blocks: 4090
-- Weighted coverage points: 3216.2
+- Mapped Rust files: 402
+- Active test blocks: 3915
+- Ignored/manual test blocks: 188
+- Declared test blocks: 4103
+- Weighted coverage points: 3225.0
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,18 +23,18 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3742 | 167 | 3143.2 | 21 | 11 | 100% |
-| macos | 32 | 3785 | 146 | 3138.5 | 22 | 11 | 100% |
-| linux | 29 | 3320 | 142 | 2761.5 | 20 | 11 | 100% |
+| windows | 33 | 3752 | 170 | 3152.0 | 21 | 11 | 100% |
+| macos | 32 | 3795 | 149 | 3147.3 | 22 | 11 | 100% |
+| linux | 29 | 3330 | 145 | 2770.3 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 12 | 22 | 131 | 1869 | 53 | 1439.0 | 10 |
+| screenpipe-engine | 12 | 22 | 131 | 1870 | 53 | 1439.7 | 10 |
| screenpipe-db | 5 | 69 | 29 | 631 | 32 | 601.4 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
-| screenpipe-audio | 8 | 29 | 53 | 696 | 53 | 607.9 | 5 |
+| screenpipe-audio | 8 | 29 | 54 | 705 | 56 | 616.0 | 5 |
| screenpipe-screen | 6 | 9 | 19 | 278 | 9 | 246.4 | 4 |
| screenpipe-a11y | 4 | 2 | 34 | 401 | 38 | 291.5 | 3 |
| screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |
@@ -63,25 +63,25 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Layer | windows | macos | linux |
| --- | --- | --- | --- |
| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 440 active / 14 ignored / 350.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts |
-| audio | 10 suites / 834 active / 57 ignored / 745.9 pts | 9 suites / 828 active / 57 ignored / 741.7 pts | 8 suites / 749 active / 56 ignored / 686.4 pts |
-| audio-device | 2 suites / 236 active / 7 ignored / 212.3 pts | 2 suites / 236 active / 7 ignored / 212.3 pts | 1 suites / 157 active / 6 ignored / 157.0 pts |
+| audio | 10 suites / 843 active / 60 ignored / 754.0 pts | 9 suites / 837 active / 60 ignored / 749.8 pts | 8 suites / 758 active / 59 ignored / 694.5 pts |
+| audio-device | 2 suites / 242 active / 10 ignored / 218.3 pts | 2 suites / 242 active / 10 ignored / 218.3 pts | 1 suites / 163 active / 9 ignored / 163.0 pts |
| configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |
| database | 7 suites / 587 active / 28 ignored / 557.4 pts | 7 suites / 587 active / 28 ignored / 557.4 pts | 7 suites / 587 active / 28 ignored / 557.4 pts |
| db-search | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts |
| engine-lifecycle | 6 suites / 272 active / 1 ignored / 243.4 pts | 6 suites / 272 active / 1 ignored / 243.4 pts | 6 suites / 268 active / 9 ignored / 242.5 pts |
-| local-api | 3 suites / 489 active / 12 ignored / 346.2 pts | 3 suites / 489 active / 12 ignored / 346.2 pts | 3 suites / 489 active / 12 ignored / 346.2 pts |
-| meeting | 6 suites / 1757 active / 32 ignored / 1437.2 pts | 6 suites / 1757 active / 32 ignored / 1437.2 pts | 4 suites / 1403 active / 28 ignored / 1106.9 pts |
+| local-api | 3 suites / 490 active / 12 ignored / 346.9 pts | 3 suites / 490 active / 12 ignored / 346.9 pts | 3 suites / 490 active / 12 ignored / 346.9 pts |
+| meeting | 6 suites / 1758 active / 32 ignored / 1437.9 pts | 6 suites / 1758 active / 32 ignored / 1437.9 pts | 4 suites / 1404 active / 28 ignored / 1107.6 pts |
| ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1691 active / 83 ignored / 1492.9 pts | 14 suites / 1816 active / 91 ignored / 1542.9 pts | 13 suites / 1691 active / 83 ignored / 1492.9 pts |
+| performance | 13 suites / 1700 active / 86 ignored / 1501.0 pts | 14 suites / 1825 active / 94 ignored / 1551.0 pts | 13 suites / 1700 active / 86 ignored / 1501.0 pts |
| pipes | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |
| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1069 active / 21 ignored / 818.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |
| real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - |
| speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts |
| storage | 4 suites / 698 active / 41 ignored / 581.6 pts | 4 suites / 698 active / 41 ignored / 581.6 pts | 4 suites / 698 active / 41 ignored / 581.6 pts |
| sync | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |
-| timeline | 4 suites / 1246 active / 48 ignored / 1023.1 pts | 4 suites / 1246 active / 48 ignored / 1023.1 pts | 4 suites / 1246 active / 48 ignored / 1023.1 pts |
-| transcription | 7 suites / 896 active / 53 ignored / 700.8 pts | 6 suites / 890 active / 53 ignored / 696.6 pts | 6 suites / 890 active / 53 ignored / 696.6 pts |
+| timeline | 4 suites / 1247 active / 48 ignored / 1023.8 pts | 4 suites / 1247 active / 48 ignored / 1023.8 pts | 4 suites / 1247 active / 48 ignored / 1023.8 pts |
+| transcription | 7 suites / 900 active / 53 ignored / 703.6 pts | 6 suites / 894 active / 53 ignored / 699.4 pts | 6 suites / 894 active / 53 ignored / 699.4 pts |
| ui-events | 4 suites / 828 active / 34 ignored / 627.7 pts | 3 suites / 759 active / 5 ignored / 579.4 pts | 3 suites / 759 active / 5 ignored / 579.4 pts |
| vision-capture | 6 suites / 579 active / 32 ignored / 455.3 pts | 6 suites / 572 active / 32 ignored / 453.1 pts | 5 suites / 563 active / 31 ignored / 444.1 pts |
@@ -123,20 +123,20 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 27 | 1 | Linux-specific accessibility/incognito normalization tests. |
| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 11 | 125 | 8 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |
| a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 7 | 69 | 29 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. |
-| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 157 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |
+| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 15 | 163 | 9 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |
| audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 28 | 281 | 14 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |
| audio-min-speech-ratio-gate | screenpipe-audio | windows, macos, linux | audio, transcription | audio-record-transcribe | high | partial | integration | 1 | 1 | 0 | Synthetic VAD input verifies that a minimum speech ratio of one rejects even a fully spoken chunk and emits no segments. Does not exercise physical devices or live model inference. |
| audio-models-filtering | screenpipe-audio | windows, macos, linux | audio, transcription, privacy | audio-record-transcribe, privacy-and-redaction | medium | partial | mixed | 6 | 20 | 10 | Model-download/TLS guards, ONNX startup smoke, and music-versus-speech filtering. |
| audio-pipeline-benchmarks | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | medium | partial | benchmark | 9 | 31 | 13 | Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing. Native-window continuity, valid 16 kHz fallback frames and partial-tail regression tests; optional real-model speech-fixture timing is ignored by default. |
| audio-platform-output-capture | screenpipe-audio | windows, macos | audio-device, audio, meeting | audio-device-health, audio-record-transcribe, meeting-live-notes | high | partial | unit | 7 | 79 | 1 | OS-specific output/system-audio capture: CoreAudio process tap and SCK output watchdog plus VPIO health policy on macOS, per-process meeting audio taps on both platforms, and the Windows follow-the-audio output watchdog. Platform impl files are cfg-gated to their target OS. |
-| audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 16 | 121 | 9 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. |
+| audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 16 | 124 | 9 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. |
| audio-windows-parakeet-provider | screenpipe-audio | windows | audio, transcription | audio-record-transcribe | high | partial | unit | 1 | 6 | 0 | Registers existing CPU override, eligible adapter selection, DXGI ordinal, error fallback and adapter determinism tests. Windows-only; manifest registration does not verify execution on physical GPU hardware. |
| db-accessibility-ui-events | screenpipe-db | windows, macos, linux | database, configuration, accessibility, ui-events, performance | settings-to-engine-config, accessibility-ui-events, performance-liveness | medium | partial | integration | 7 | 28 | 2 | Elements bulk insert, on-screen filtering, UI event batching, DB tier config, and ignored heavy-read real-DB probes. |
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 19 | 135 | 4 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 18 | 51 | 7 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 14 | 107 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 40 | 310 | 15 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
-| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 436 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
+| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 437 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 307 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 13 | 125 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 33 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |
@@ -213,11 +213,12 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio-device-stream-health | screenpipe-audio | src/core/process_tap/counters.rs | source | 2 | 0 | 2 |
| audio-platform-output-capture | screenpipe-audio | src/core/process_tap/macos.rs | source | 28 | 1 | 29 |
| audio-platform-output-capture | screenpipe-audio | src/core/process_tap/windows.rs | source | 12 | 0 | 12 |
-| audio-transcription-pipeline | screenpipe-audio | src/core/run_record_and_transcribe.rs | source | 16 | 0 | 16 |
+| audio-transcription-pipeline | screenpipe-audio | src/core/run_record_and_transcribe.rs | source | 19 | 0 | 19 |
| audio-platform-output-capture | screenpipe-audio | src/core/sck_output_watchdog.rs | source | 12 | 0 | 12 |
| audio-device-stream-health | screenpipe-audio | src/core/source_buffer.rs | source | 7 | 0 | 7 |
-| audio-device-stream-health | screenpipe-audio | src/core/stream.rs | source | 13 | 0 | 13 |
-| audio-device-stream-health | screenpipe-audio | src/device/device_manager.rs | source | 9 | 0 | 9 |
+| audio-device-stream-health | screenpipe-audio | src/core/stream.rs | source | 14 | 1 | 15 |
+| audio-device-stream-health | screenpipe-audio | src/core/vpio_process.rs | source | 4 | 2 | 6 |
+| audio-device-stream-health | screenpipe-audio | src/device/device_manager.rs | source | 10 | 0 | 10 |
| audio-platform-output-capture | screenpipe-audio | src/device/vpio_health.rs | source | 5 | 0 | 5 |
| audio-device-stream-health | screenpipe-audio | src/idle_detector.rs | source | 4 | 0 | 4 |
| audio-device-stream-health | screenpipe-audio | src/lib.rs | source | 4 | 0 | 4 |
@@ -467,7 +468,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-api-routes | screenpipe-engine | src/routes/data.rs | source | 2 | 0 | 2 |
| engine-api-routes | screenpipe-engine | src/routes/elements.rs | source | 25 | 1 | 26 |
| engine-api-routes | screenpipe-engine | src/routes/frames.rs | source | 12 | 1 | 13 |
-| engine-api-routes | screenpipe-engine | src/routes/health.rs | source | 57 | 0 | 57 |
+| engine-api-routes | screenpipe-engine | src/routes/health.rs | source | 58 | 0 | 58 |
| engine-api-routes | screenpipe-engine | src/routes/live_views.rs | source | 1 | 0 | 1 |
| engine-api-routes | screenpipe-engine | src/routes/meeting_summary_status.rs | source | 11 | 0 | 11 |
| engine-api-routes | screenpipe-engine | src/routes/meetings.rs | source | 7 | 0 | 7 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index c24c1a482..40369c767 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -940,6 +940,7 @@
"src/core/process_tap/counters.rs",
"src/core/source_buffer.rs",
"src/core/stream.rs",
+ "src/core/vpio_process.rs",
"src/device/device_manager.rs",
"src/idle_detector.rs",
"src/lib.rs",