Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(audio): correlate timeout recovery per device (#5672)

Louis Beaumontcd74538805e6
diff --git a/TESTING.md b/TESTING.md
index f6fe62932..3e157d523 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -134,6 +134,7 @@ commits: `28e5c247`
- [ ] **bluetooth device connect/disconnect** — AirPods connect mid-recording. audio continues without gap.
- [ ] **no audio device available** — unplug all audio. app continues (vision still works). log shows warning, not crash.
- [ ] **audio stream timeout recovery** — if audio stream times out (30s no data), it should reconnect automatically.
+- [ ] **per-device audio-timeout recovery** — Force one microphone timeout, then prove that only later usable microphone audio clears `active_no_data`; healthy system output alone must not clear it. Repeat with system output timed out and microphone live. Verify a one-shot timeout expires before the native 90-tick alert, repeated zero-fill/receive timeouts keep the failure active, a recovered device clears immediately, an unresolved current device still raises `recording needs help`, and removing/deselecting the failed device stops it from degrading current capture. Run this after a packaged macOS display/wake transition.
- [ ] **multiple audio devices simultaneously** — input (mic) + output (speakers) both recording. both show in device list.
- [ ] **disable audio setting** — toggling "disable audio" stops all audio recording. re-enabling restarts it.
- [ ] **Metal GPU for whisper** — transcription uses GPU acceleration on macOS (`f882caef`). verify with Activity Monitor GPU tab.
diff --git a/crates/screenpipe-audio/src/core/mod.rs b/crates/screenpipe-audio/src/core/mod.rs
index fb99fcb45..fdc70c197 100644
--- a/crates/screenpipe-audio/src/core/mod.rs
+++ b/crates/screenpipe-audio/src/core/mod.rs
@@ -62,6 +62,15 @@ pub fn get_device_capture_time(device_name: &str) -> u64 {
.unwrap_or_else(|| LAST_AUDIO_CAPTURE.load(Ordering::Relaxed))
}
+/// Exact per-device capture timestamp. Unlike `get_device_capture_time`, this
+/// never substitutes another stream's global activity for a device that has
+/// not produced usable audio.
+pub fn get_device_capture_time_exact(device_name: &str) -> Option<u64> {
+ DEVICE_AUDIO_CAPTURES
+ .get(device_name)
+ .map(|atomic| atomic.load(Ordering::Relaxed))
+}
+
#[cfg(all(test, target_os = "macos"))]
mod e2e_ghost_word_silent_room;
diff --git a/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs b/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs
index 81b9ed8e6..1b1290a91 100644
--- a/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs
+++ b/crates/screenpipe-audio/src/core/run_record_and_transcribe.rs
@@ -14,7 +14,7 @@ use tokio::sync::broadcast;
use tracing::{debug, error, info, warn};
use crate::{
- core::{device::DeviceType, update_device_capture_time},
+ core::{device::DeviceType, get_device_capture_time_exact, update_device_capture_time},
meeting_streaming::{MeetingAudioFrame, MeetingAudioTap},
metrics::AudioPipelineMetrics,
utils::audio::StreamResampler,
@@ -158,6 +158,12 @@ fn should_reconnect_after_silent_input(
zero_fill_reconnect_enabled_for_platform()
}
+fn last_non_zero_seed_after_restart(device_name: &str, restarted_at: Instant) -> Option<Instant> {
+ get_device_capture_time_exact(device_name)
+ .filter(|captured_at| *captured_at > 0)
+ .map(|_| restarted_at)
+}
+
fn meeting_frame_from_recorder_output(
samples: Vec<f32>,
audio_stream: &AudioStream,
@@ -430,7 +436,13 @@ pub async fn run_record_and_transcribe(
let mut collected_audio = Vec::new();
let mut segment_start_time = now_epoch_secs();
let stream_start = Instant::now();
- let mut last_non_zero_at: Option<Instant> = None;
+ // Preserve the fact that this device produced usable audio before a
+ // watchdog-driven rebuild. If the replacement stream only emits
+ // zero-fill, it must trip the watchdog again after 30s instead of
+ // silently clearing a persistent failure merely because the task was
+ // recreated. A device that has never produced usable audio retains the
+ // existing fail-open behavior for genuinely digital-silent hardware.
+ let mut last_non_zero_at = last_non_zero_seed_after_restart(&device_name, Instant::now());
let mut sck_watchdog = crate::core::sck_output_watchdog::SckOutputWatchdog::default();
let mut segment_count: u64 = 0;
@@ -593,7 +605,7 @@ async fn recv_audio_chunk(
(likely OS device hijack by another app), triggering reconnect",
device_name, INPUT_SILENT_BUFFER_TIMEOUT_SECS
);
- metrics.record_stream_timeout();
+ metrics.record_stream_timeout(device_name);
audio_stream.is_disconnected.store(true, Ordering::Relaxed);
return Err(anyhow!(StreamDeath::ZeroFill {
device: device_name.to_string(),
@@ -685,7 +697,7 @@ fn fail_stream_dead(
"no audio received from {} for {}s - stream dead, triggering reconnect",
device_name, AUDIO_RECEIVE_TIMEOUT_SECS
);
- metrics.record_stream_timeout();
+ metrics.record_stream_timeout(device_name);
audio_stream.is_disconnected.store(true, Ordering::Relaxed);
// Typed cause so the VPIO runtime-fallback policy can recognize this death
// by `downcast_ref` rather than message text. Display is unchanged.
@@ -732,7 +744,7 @@ fn classify_output_recv_timeout(
re-anchoring via device_monitor",
device_name, healthy, current
);
- metrics.record_stream_timeout();
+ metrics.record_stream_timeout(device_name);
audio_stream.is_disconnected.store(true, Ordering::Relaxed);
Err(anyhow!(
"SCK System Audio stream dead — display invalidation (#3901)"
@@ -924,6 +936,19 @@ mod tests {
));
}
+ #[test]
+ fn restarted_stream_keeps_prior_usable_audio_as_zero_fill_baseline() {
+ const DEVICE: &str = "restart-zero-fill-regression (input)";
+ let restarted_at = Instant::now();
+
+ assert_eq!(last_non_zero_seed_after_restart(DEVICE, restarted_at), None);
+ update_device_capture_time(DEVICE);
+ assert_eq!(
+ last_non_zero_seed_after_restart(DEVICE, restarted_at),
+ Some(restarted_at)
+ );
+ }
+
#[cfg(target_os = "windows")]
#[test]
fn windows_sustained_zero_fill_is_treated_as_silence_not_disconnect() {
diff --git a/crates/screenpipe-audio/src/metrics.rs b/crates/screenpipe-audio/src/metrics.rs
index d5eba20db..7820f29fd 100644
--- a/crates/screenpipe-audio/src/metrics.rs
+++ b/crates/screenpipe-audio/src/metrics.rs
@@ -24,6 +24,10 @@ pub struct AudioPipelineMetrics {
/// recovered long ago. Without it, a single historical timeout pins the
/// audio status to "active_no_data" forever — even while audio flows again.
pub last_stream_timeout_at: AtomicU64,
+ /// Most recent timeout per device. Recovery must be proven by later usable
+ /// audio from the same device; aggregate audio from another stream cannot
+ /// clear a microphone or system-output failure.
+ per_device_stream_timeout_at: RwLock<HashMap<String, u64>>,
/// Audio buffers skipped because the recorder consumer fell behind the
/// capture broadcast channel. This is otherwise-silent audio loss under
/// CPU contention — previously invisible to telemetry.
@@ -94,6 +98,7 @@ impl AudioPipelineMetrics {
chunks_channel_full: AtomicU64::new(0),
stream_timeouts: AtomicU64::new(0),
last_stream_timeout_at: AtomicU64::new(0),
+ per_device_stream_timeout_at: RwLock::new(HashMap::new()),
chunks_lagged: AtomicU64::new(0),
vad_passed: AtomicU64::new(0),
vad_rejected: AtomicU64::new(0),
@@ -129,13 +134,30 @@ impl AudioPipelineMetrics {
self.chunks_channel_full.fetch_add(1, Ordering::Relaxed);
}
- pub fn record_stream_timeout(&self) {
- self.stream_timeouts.fetch_add(1, Ordering::Relaxed);
+ pub fn record_stream_timeout(&self, device_name: &str) {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
- .unwrap()
+ .unwrap_or_default()
.as_secs();
+ self.record_stream_timeout_at(device_name, now);
+ }
+
+ fn record_stream_timeout_at(&self, device_name: &str, now: u64) {
+ self.stream_timeouts.fetch_add(1, Ordering::Relaxed);
self.last_stream_timeout_at.store(now, Ordering::Relaxed);
+ self.per_device_stream_timeout_at
+ .write()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .insert(device_name.to_string(), now);
+ }
+
+ /// Point-in-time timeout map used by `/health` to correlate recovery with
+ /// the device that actually failed.
+ pub fn per_device_stream_timeouts_snapshot(&self) -> HashMap<String, u64> {
+ self.per_device_stream_timeout_at
+ .read()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .clone()
}
/// Record `n` audio buffers skipped because the recorder consumer fell
@@ -439,4 +461,20 @@ mod tests {
assert_eq!(per_device.len(), 1);
assert_eq!(per_device["mic"], 0.0);
}
+
+ #[test]
+ fn stream_timeouts_keep_latest_timestamp_per_device() {
+ let metrics = AudioPipelineMetrics::new();
+
+ metrics.record_stream_timeout_at("mic", 100);
+ metrics.record_stream_timeout_at("speakers", 110);
+ metrics.record_stream_timeout_at("mic", 120);
+
+ let per_device = metrics.per_device_stream_timeouts_snapshot();
+ assert_eq!(per_device.len(), 2);
+ assert_eq!(per_device["mic"], 120);
+ assert_eq!(per_device["speakers"], 110);
+ assert_eq!(metrics.stream_timeouts.load(Ordering::Relaxed), 3);
+ assert_eq!(metrics.last_stream_timeout_at.load(Ordering::Relaxed), 120);
+ }
}
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index 4e4e10f4a..4846d87de 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -9,6 +9,7 @@ use chrono::{DateTime, TimeZone, Utc};
use serde::{Deserialize, Serialize};
use serde_json::json;
+use std::collections::HashMap;
use std::sync::{
atomic::{AtomicU64, Ordering},
Arc,
@@ -119,7 +120,30 @@ const SILENT_AUDIO_RMS_THRESHOLD: f64 = 0.001;
/// timeout while a stream is dead, so a window comfortably larger than one cycle
/// keeps a genuinely dead stream flagged, while a stream that recovered (no new
/// timeouts) clears back to "ok" instead of sticking forever on a stale count.
-const STREAM_TIMEOUT_RECENCY_SECS: u64 = 90;
+/// Keep this below the desktop's 90-tick incident debounce: a one-shot timeout
+/// from a successfully rebuilt but silent stream must expire before it can
+/// raise `recording needs help`. A genuinely dead stream re-fires its 8s/30s
+/// watchdog and refreshes this window.
+const STREAM_TIMEOUT_RECENCY_SECS: u64 = 60;
+
+/// A timeout remains actionable only while its device is still selected, the
+/// timeout is recent, and that same device has not produced usable audio after
+/// the timeout. Activity from another microphone or output cannot clear it.
+fn has_unrecovered_recent_stream_timeout(
+ per_device_timeout_at: &HashMap<String, u64>,
+ current_device_capture_at: &HashMap<String, u64>,
+ now_ts: u64,
+) -> bool {
+ per_device_timeout_at.iter().any(|(device, timeout_at)| {
+ let Some(capture_at) = current_device_capture_at.get(device) else {
+ // Removed/deselected devices are no longer part of current capture.
+ return false;
+ };
+ *timeout_at > 0
+ && now_ts.saturating_sub(*timeout_at) < STREAM_TIMEOUT_RECENCY_SECS
+ && *capture_at <= *timeout_at
+ })
+}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct MeetingsOnlyAudioIdleState {
@@ -143,9 +167,9 @@ fn meetings_only_audio_idle_state(
}
/// Classify the raw audio capture status from health signals. Pure so it can be
-/// unit-tested in isolation. `stream_timeout_recent` must reflect a *recent*
-/// stream timeout (see `STREAM_TIMEOUT_RECENCY_SECS`), NOT a cumulative count —
-/// passing "ever had a timeout" here is exactly the bug this extraction fixes.
+/// unit-tested in isolation. `stream_timeout_recent` must mean a recent timeout
+/// that the same current device has not recovered from, NOT a cumulative count
+/// or aggregate activity from some other stream.
#[allow(clippy::too_many_arguments)]
fn classify_audio_status(
audio_disabled: bool,
@@ -180,9 +204,10 @@ fn classify_audio_status(
"no_input_device"
} else if audio_never_captured {
"not_started"
- } else if stream_timeout_recent && global_audio_active {
- // Device active but the watchdog fired recently — hijack/dead-stream
- // recovery in progress. Clears automatically once timeouts stop.
+ } else if stream_timeout_recent {
+ // The watchdog fired on a current device and that same device has not
+ // produced usable audio since. This remains a failure even if another
+ // microphone or system-output stream is healthy.
"active_no_data"
} else if global_audio_active {
"ok"
@@ -256,7 +281,7 @@ fn capture_status(
"warning",
"audio capture has not produced data yet",
)
- } else if audio_status == "stale" || (audio_status == "active_no_data" && !audio_recent) {
+ } else if audio_status == "stale" || audio_status == "active_no_data" {
(
"audio_stalled",
"warning",
@@ -748,6 +773,7 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
std::collections::HashSet::new()
};
let mut device_statuses = Vec::new();
+ let mut current_device_capture_at = HashMap::new();
let mut global_audio_active = false;
let mut most_recent_audio_timestamp = 0; // Track the most recent timestamp
@@ -755,6 +781,9 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
for device in &audio_devices {
let device_name = device.to_string();
let last_capture = screenpipe_audio::core::get_device_capture_time(&device_name);
+ let exact_last_capture =
+ screenpipe_audio::core::get_device_capture_time_exact(&device_name).unwrap_or(0);
+ current_device_capture_at.insert(device_name.clone(), exact_last_capture);
// Update the most recent timestamp
most_recent_audio_timestamp = most_recent_audio_timestamp.max(last_capture);
@@ -1047,16 +1076,15 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
// watchdog reconnects after 30s of no real audio and keeps re-firing while
// the stream stays dead.
//
- // Gate on the RECENCY of the last timeout, not the cumulative count. The old
- // `stream_timeouts > 0` check pinned the status to "active_no_data" forever
- // after a single historical timeout (a wake/display invalidation, a device
- // switch, a transient glitch) — so a fully recovered mic with chunks flowing
- // again still read as broken. A healthy-but-silent room never trips this:
- // raw chunks keep arriving so the watchdog never fires; only a genuinely
- // dead/hijacked stream keeps refreshing `last_stream_timeout_at`.
+ // Gate on a recent timeout that the SAME current device has not recovered
+ // from. The old global timestamp let a healthy output stream mask a dead
+ // mic, while the older cumulative counter pinned recovered devices forever.
let now_ts = now.timestamp().max(0) as u64;
- let stream_timeout_recent = audio_snap.last_stream_timeout_at > 0
- && now_ts.saturating_sub(audio_snap.last_stream_timeout_at) < STREAM_TIMEOUT_RECENCY_SECS;
+ let stream_timeout_recent = has_unrecovered_recent_stream_timeout(
+ &state.audio_metrics.per_device_stream_timeouts_snapshot(),
+ &current_device_capture_at,
+ now_ts,
+ );
// Only report the intentional pause after the manager has actually
// released every stream. During the short teardown transition (or if a
@@ -1701,7 +1729,7 @@ mod tests {
}
#[test]
- fn capture_status_does_not_show_stalled_for_recovered_active_no_data() {
+ fn capture_status_does_not_let_other_fresh_audio_mask_active_no_data() {
let state = capture_status(
false,
false,
@@ -1720,8 +1748,8 @@ mod tests {
121,
);
- assert_eq!(state.status, "waiting_for_voice");
- assert_eq!(state.severity, "waiting");
+ assert_eq!(state.status, "audio_stalled");
+ assert_eq!(state.severity, "warning");
}
#[test]
@@ -1748,6 +1776,16 @@ mod tests {
assert_eq!(state.severity, "warning");
}
+ #[test]
+ fn capture_status_recovers_after_raw_status_clears() {
+ let state = capture_status(
+ false, false, false, false, "ok", 1, 1, 0, 0, false, None, 0.0, 4, 120, 121,
+ );
+
+ assert_eq!(state.status, "waiting_for_voice");
+ assert_eq!(state.severity, "waiting");
+ }
+
#[test]
fn capture_status_reports_intentional_screen_lock_pause() {
let state = capture_status(
@@ -2046,11 +2084,142 @@ mod tests {
)
}
+ #[test]
+ fn timeout_recovery_is_correlated_to_the_same_current_device() {
+ let timeouts = HashMap::from([("mic".to_string(), 100)]);
+ let output_live_mic_dead =
+ HashMap::from([("mic".to_string(), 99), ("output".to_string(), 120)]);
+ assert!(has_unrecovered_recent_stream_timeout(
+ &timeouts,
+ &output_live_mic_dead,
+ 120
+ ));
+
+ let output_live_mic_recovered =
+ HashMap::from([("mic".to_string(), 101), ("output".to_string(), 120)]);
+ assert!(!has_unrecovered_recent_stream_timeout(
+ &timeouts,
+ &output_live_mic_recovered,
+ 120
+ ));
+
+ let output_timeouts = HashMap::from([("output".to_string(), 100)]);
+ let mic_live_output_dead =
+ HashMap::from([("mic".to_string(), 120), ("output".to_string(), 99)]);
+ assert!(has_unrecovered_recent_stream_timeout(
+ &output_timeouts,
+ &mic_live_output_dead,
+ 120
+ ));
+ }
+
+ #[test]
+ fn timeout_recovery_boundaries_fail_closed_without_sticking_forever() {
+ let timeouts = HashMap::from([("mic".to_string(), 100)]);
+
+ for (capture_at, now_ts, expected) in [
+ (0, 100, true),
+ (99, 120, true),
+ (100, 120, true),
+ (101, 120, false),
+ (99, 159, true),
+ (99, 160, false),
+ ] {
+ let captures = HashMap::from([("mic".to_string(), capture_at)]);
+ assert_eq!(
+ has_unrecovered_recent_stream_timeout(&timeouts, &captures, now_ts),
+ expected,
+ "capture_at={capture_at}, now_ts={now_ts}"
+ );
+ }
+
+ assert!(has_unrecovered_recent_stream_timeout(
+ &timeouts,
+ &HashMap::from([("mic".to_string(), 0)]),
+ 99
+ ));
+ assert!(!has_unrecovered_recent_stream_timeout(
+ &timeouts,
+ &HashMap::from([("output".to_string(), 120)]),
+ 120
+ ));
+ }
+
+ #[test]
+ fn one_unrecovered_device_wins_over_other_recovered_devices() {
+ let timeouts = HashMap::from([("mic".to_string(), 100), ("output".to_string(), 110)]);
+ let captures = HashMap::from([("mic".to_string(), 101), ("output".to_string(), 109)]);
+ assert!(has_unrecovered_recent_stream_timeout(
+ &timeouts, &captures, 120
+ ));
+
+ let all_recovered = HashMap::from([("mic".to_string(), 101), ("output".to_string(), 111)]);
+ assert!(!has_unrecovered_recent_stream_timeout(
+ &timeouts,
+ &all_recovered,
+ 120
+ ));
+ }
+
+ #[test]
+ fn a_repeated_timeout_refreshes_the_incident_window() {
+ let captures = HashMap::from([("mic".to_string(), 99)]);
+
+ assert!(!has_unrecovered_recent_stream_timeout(
+ &HashMap::from([("mic".to_string(), 100)]),
+ &captures,
+ 160
+ ));
+ assert!(has_unrecovered_recent_stream_timeout(
+ &HashMap::from([("mic".to_string(), 130)]),
+ &captures,
+ 160
+ ));
+ }
+
+ #[test]
+ fn timeout_correlation_exhaustively_ignores_other_device_activity() {
+ let timeout_at = 100;
+ for failed_device in ["mic", "output"] {
+ let other_device = if failed_device == "mic" {
+ "output"
+ } else {
+ "mic"
+ };
+ for failed_device_present in [false, true] {
+ for own_capture_at in [99, 100, 101] {
+ for other_capture_at in [0, 120] {
+ for now_ts in [159, 160] {
+ let timeouts = HashMap::from([(failed_device.to_string(), timeout_at)]);
+ let mut captures =
+ HashMap::from([(other_device.to_string(), other_capture_at)]);
+ if failed_device_present {
+ captures.insert(failed_device.to_string(), own_capture_at);
+ }
+
+ let expected = failed_device_present
+ && own_capture_at <= timeout_at
+ && now_ts - timeout_at < STREAM_TIMEOUT_RECENCY_SECS;
+ assert_eq!(
+ has_unrecovered_recent_stream_timeout(
+ &timeouts, &captures, now_ts
+ ),
+ expected,
+ "failed={failed_device}, present={failed_device_present}, own={own_capture_at}, other={other_capture_at}, now={now_ts}"
+ );
+ }
+ }
+ }
+ }
+ }
+ }
+
#[test]
fn audio_status_active_no_data_only_while_timeout_is_recent() {
// Issue #3144: an active device whose zero-fill watchdog fired *recently*
// is "active_no_data" (hijacked / dead stream, recovery in progress).
assert_eq!(audio_status_for(true, true), "active_no_data");
+ assert_eq!(audio_status_for(true, false), "active_no_data");
}
#[test]