Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Throttle stale audio device recovery (#4782)

Louis Beaumontdea26d15e7d1
diff --git a/crates/screenpipe-audio/src/audio_manager/device_monitor.rs b/crates/screenpipe-audio/src/audio_manager/device_monitor.rs
index 35eddc7df..5bd87539a 100644
--- a/crates/screenpipe-audio/src/audio_manager/device_monitor.rs
+++ b/crates/screenpipe-audio/src/audio_manager/device_monitor.rs
@@ -91,6 +91,13 @@ impl DeviceRecoveryBackoff {
Self::new(Self::TRANSIENT_MAX_SECS, Self::OUTPUT_PERMANENT_MAX_SECS)
}
+ fn for_device_type(device_type: &DeviceType) -> Self {
+ match device_type {
+ DeviceType::Input => Self::for_input(),
+ DeviceType::Output => Self::for_output(),
+ }
+ }
+
fn new(transient_max_secs: u64, permanent_max_secs: u64) -> Self {
Self {
attempts: 0,
@@ -116,6 +123,15 @@ impl DeviceRecoveryBackoff {
self.is_permanent = false;
}
+ fn should_attempt(&self, now: Instant) -> bool {
+ now.saturating_duration_since(self.last_attempt)
+ >= Duration::from_secs(self.next_delay_secs())
+ }
+
+ fn record_attempt_at(&mut self, now: Instant) {
+ self.last_attempt = now;
+ }
+
fn next_delay_secs(&self) -> u64 {
if self.attempts == 0 {
return 0;
@@ -145,6 +161,47 @@ fn is_permanent_input_error(err: &anyhow::Error) -> bool {
msg.contains("No default input device detected")
}
+fn is_permanent_device_start_error(err: &anyhow::Error) -> bool {
+ let msg = err.to_string().to_lowercase();
+ msg.contains("not found")
+ || msg.contains("no default input device detected")
+ || msg.contains("no display audio device found")
+}
+
+fn should_log_recovery_attempt(attempts: u32) -> bool {
+ attempts <= 3 || attempts.is_multiple_of(30)
+}
+
+fn should_attempt_disconnected_device_recovery(
+ backoffs: &mut HashMap<String, DeviceRecoveryBackoff>,
+ device_name: &str,
+ device_type: &DeviceType,
+ now: Instant,
+) -> bool {
+ let backoff = backoffs
+ .entry(device_name.to_string())
+ .or_insert_with(|| DeviceRecoveryBackoff::for_device_type(device_type));
+ if backoff.should_attempt(now) {
+ backoff.record_attempt_at(now);
+ true
+ } else {
+ false
+ }
+}
+
+fn record_disconnected_device_failure(
+ backoffs: &mut HashMap<String, DeviceRecoveryBackoff>,
+ device_name: &str,
+ device_type: &DeviceType,
+ permanent: bool,
+) -> (u32, u64) {
+ let backoff = backoffs
+ .entry(device_name.to_string())
+ .or_insert_with(|| DeviceRecoveryBackoff::for_device_type(device_type));
+ backoff.record_failure(permanent);
+ (backoff.attempts, backoff.next_delay_secs())
+}
+
/// Heuristic: does this input device name look like an on-board mic? Used to
/// prefer the built-in mic over a virtual/aggregate input when failing over a
/// disconnected pinned device, so we don't grab some random loopback input
@@ -437,6 +494,8 @@ pub async fn start_device_monitor(
let mut failed_devices: HashMap<String, (u32, Instant)> = HashMap::new();
let mut input_recovery_backoff = DeviceRecoveryBackoff::for_input();
let mut output_recovery_backoff = DeviceRecoveryBackoff::for_output();
+ let mut disconnected_device_backoffs: HashMap<String, DeviceRecoveryBackoff> =
+ HashMap::new();
// Central handler restart cooldown: max 3 restarts in a 5-minute window
let mut central_restart_times: Vec<Instant> = Vec::new();
@@ -1195,6 +1254,7 @@ pub async fn start_device_monitor(
// Skip user-disabled devices — they're intentionally stopped
if user_disabled_for_reconnect.contains(&device_name) {
disconnected_devices.remove(&device_name);
+ disconnected_device_backoffs.remove(&device_name);
continue;
}
@@ -1202,24 +1262,133 @@ pub async fn start_device_monitor(
Ok(device) => device,
Err(e) => {
debug!("Device name {} invalid: {}", device_name, e);
+ disconnected_devices.remove(&device_name);
+ disconnected_device_backoffs.remove(&device_name);
continue;
}
};
+ let now = Instant::now();
+ if !should_attempt_disconnected_device_recovery(
+ &mut disconnected_device_backoffs,
+ &device_name,
+ &device.device_type,
+ now,
+ ) {
+ continue;
+ }
+
// In system default mode, try to restart with current default instead
if audio_manager.use_system_default_audio().await {
- let current_default = match device.device_type {
- DeviceType::Input => default_input_device().ok(),
- DeviceType::Output => default_output_device().await.ok(),
- };
-
- if let Some(default_device) = current_default {
- if audio_manager.start_device(&default_device).await.is_ok() {
- info!("restarted with system default device: {}", default_device);
- disconnected_devices.remove(&device_name);
- continue;
- }
+ match &device.device_type {
+ DeviceType::Input => match default_input_device() {
+ Ok(default_device) => {
+ match audio_manager.start_device(&default_device).await {
+ Ok(()) => {
+ info!(
+ "restarted with system default device: {}",
+ default_device
+ );
+ disconnected_devices.remove(&device_name);
+ disconnected_device_backoffs.remove(&device_name);
+ }
+ Err(e) => {
+ let permanent = is_permanent_device_start_error(&e);
+ let (attempts, delay) =
+ record_disconnected_device_failure(
+ &mut disconnected_device_backoffs,
+ &device_name,
+ &device.device_type,
+ permanent,
+ );
+ if should_log_recovery_attempt(attempts) {
+ warn!(
+ "[DEVICE_RECOVERY] failed to restart current system default {} for stale {} (attempt {}, next retry in {}s): {}",
+ default_device,
+ device_name,
+ attempts,
+ delay,
+ e
+ );
+ }
+ }
+ }
+ }
+ Err(e) => {
+ let permanent = is_permanent_input_error(&e);
+ let (attempts, delay) = record_disconnected_device_failure(
+ &mut disconnected_device_backoffs,
+ &device_name,
+ &device.device_type,
+ permanent,
+ );
+ if should_log_recovery_attempt(attempts) {
+ warn!(
+ "[DEVICE_RECOVERY] no system default input while recovering stale {} (attempt {}, {}, next retry in {}s): {}",
+ device_name,
+ attempts,
+ if permanent { "permanent" } else { "transient" },
+ delay,
+ e
+ );
+ }
+ }
+ },
+ DeviceType::Output => match default_output_device().await {
+ Ok(default_device) => {
+ match audio_manager.start_device(&default_device).await {
+ Ok(()) => {
+ info!(
+ "restarted with system default device: {}",
+ default_device
+ );
+ disconnected_devices.remove(&device_name);
+ disconnected_device_backoffs.remove(&device_name);
+ }
+ Err(e) => {
+ let permanent = is_permanent_device_start_error(&e);
+ let (attempts, delay) =
+ record_disconnected_device_failure(
+ &mut disconnected_device_backoffs,
+ &device_name,
+ &device.device_type,
+ permanent,
+ );
+ if should_log_recovery_attempt(attempts) {
+ warn!(
+ "[DEVICE_RECOVERY] failed to restart current system default {} for stale {} (attempt {}, next retry in {}s): {}",
+ default_device,
+ device_name,
+ attempts,
+ delay,
+ e
+ );
+ }
+ }
+ }
+ }
+ Err(e) => {
+ let permanent = is_permanent_output_error(&e);
+ let (attempts, delay) = record_disconnected_device_failure(
+ &mut disconnected_device_backoffs,
+ &device_name,
+ &device.device_type,
+ permanent,
+ );
+ if should_log_recovery_attempt(attempts) {
+ warn!(
+ "[DEVICE_RECOVERY] no system default output while recovering stale {} (attempt {}, {}, next retry in {}s): {}",
+ device_name,
+ attempts,
+ if permanent { "permanent" } else { "transient" },
+ delay,
+ e
+ );
+ }
+ }
+ },
}
+ continue;
}
match audio_manager.start_device(&device).await {
@@ -1229,12 +1398,32 @@ pub async fn start_device_monitor(
device_name
);
disconnected_devices.remove(&device_name);
+ disconnected_device_backoffs.remove(&device_name);
}
Err(e) => {
- warn!(
- "[DEVICE_RECOVERY] failed to restart device {}: {}",
- device_name, e
+ let e_str = e.to_string();
+ if e_str.contains("already running") {
+ disconnected_devices.remove(&device_name);
+ disconnected_device_backoffs.remove(&device_name);
+ continue;
+ }
+ let permanent = is_permanent_device_start_error(&e);
+ let (attempts, delay) = record_disconnected_device_failure(
+ &mut disconnected_device_backoffs,
+ &device_name,
+ &device.device_type,
+ permanent,
);
+ if should_log_recovery_attempt(attempts) {
+ warn!(
+ "[DEVICE_RECOVERY] failed to restart device {} (attempt {}, {}, next retry in {}s): {}",
+ device_name,
+ attempts,
+ if permanent { "permanent" } else { "transient" },
+ delay,
+ e
+ );
+ }
}
}
}
@@ -1728,6 +1917,75 @@ mod tests {
assert_eq!(b.next_delay_secs(), 0);
}
+ #[test]
+ fn disconnected_input_backoff_suppresses_two_second_retry_storm() {
+ let mut backoffs = HashMap::new();
+ let device_name = "Microphone (USB Composite Device) (input)";
+ let start = Instant::now();
+ let mut attempted_at_secs = Vec::new();
+
+ // Reproduce the monitor's 2s cadence for one minute while Windows has
+ // no default input device and the stale USB mic cannot be restarted.
+ // Before this backoff, the monitor attempted all 30 ticks.
+ for tick in 0_u64..30 {
+ let now = start + Duration::from_secs(tick * 2);
+ if should_attempt_disconnected_device_recovery(
+ &mut backoffs,
+ device_name,
+ &DeviceType::Input,
+ now,
+ ) {
+ attempted_at_secs.push(tick * 2);
+ record_disconnected_device_failure(
+ &mut backoffs,
+ device_name,
+ &DeviceType::Input,
+ true,
+ );
+ }
+ }
+
+ assert_eq!(attempted_at_secs, vec![0, 2, 6, 14, 30]);
+ let backoff = backoffs.get(device_name).unwrap();
+ assert_eq!(backoff.attempts, attempted_at_secs.len() as u32);
+ assert_eq!(backoff.next_delay_secs(), 30);
+ }
+
+ #[test]
+ fn disconnected_transient_backoff_keeps_recovery_responsive() {
+ let mut backoff = DeviceRecoveryBackoff::for_input();
+ let start = Instant::now();
+ let mut attempted_at_secs = Vec::new();
+
+ // Transient failures stay capped at 8s so a briefly flapping device can
+ // still recover quickly without retrying every monitor cycle.
+ for tick in 0_u64..10 {
+ let now = start + Duration::from_secs(tick * 2);
+ if backoff.should_attempt(now) {
+ attempted_at_secs.push(tick * 2);
+ backoff.record_attempt_at(now);
+ backoff.record_failure(false);
+ }
+ }
+
+ assert_eq!(attempted_at_secs, vec![0, 2, 6, 14]);
+ assert_eq!(backoff.next_delay_secs(), 8);
+ }
+
+ #[test]
+ fn disconnected_backoff_allows_immediate_attempt_after_success_reset() {
+ let mut backoff = DeviceRecoveryBackoff::for_input();
+ let start = Instant::now();
+
+ assert!(backoff.should_attempt(start));
+ backoff.record_attempt_at(start);
+ backoff.record_failure(true);
+ assert!(!backoff.should_attempt(start + Duration::from_secs(1)));
+
+ backoff.reset();
+ assert!(backoff.should_attempt(start + Duration::from_secs(1)));
+ }
+
#[test]
fn test_backoff_transient_then_permanent_escalates() {
let mut b = DeviceRecoveryBackoff::for_output();
@@ -1766,6 +2024,31 @@ mod tests {
assert!(!is_permanent_input_error(&transient));
}
+ #[test]
+ fn test_permanent_device_start_error_detection() {
+ assert!(is_permanent_device_start_error(&anyhow::anyhow!(
+ "device Microphone (USB Composite Device) (input) not found"
+ )));
+ assert!(is_permanent_device_start_error(&anyhow::anyhow!(
+ "No default input device detected"
+ )));
+ assert!(is_permanent_device_start_error(&anyhow::anyhow!(
+ "no display audio device found"
+ )));
+ assert!(!is_permanent_device_start_error(&anyhow::anyhow!(
+ "callback never fired"
+ )));
+ }
+
+ #[test]
+ fn test_recovery_attempt_log_throttling() {
+ assert!(should_log_recovery_attempt(1));
+ assert!(should_log_recovery_attempt(3));
+ assert!(!should_log_recovery_attempt(4));
+ assert!(should_log_recovery_attempt(30));
+ assert!(!should_log_recovery_attempt(31));
+ }
+
#[test]
fn test_cooldown_evicts_old_entries() {
let mut cd = RestartCooldown::new(3, Duration::from_secs(0)); // 0s window: everything expires instantly