YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Throttle stale audio device recovery (#4782)
Louis Beaumontdea26d15e7d1
diff --git a/crates/screenpipe-audio/src/audio_manager/device_monitor.rs b/crates/screenpipe-audio/src/audio_manager/device_monitor.rsindex 35eddc7df..5bd87539a 100644--- a/crates/screenpipe-audio/src/audio_manager/device_monitor.rs+++ b/crates/screenpipe-audio/src/audio_manager/device_monitor.rs@@ -91,6 +91,13 @@ impl DeviceRecoveryBackoff {Self::new(Self::TRANSIENT_MAX_SECS, Self::OUTPUT_PERMANENT_MAX_SECS)}+ fn for_device_type(device_type: &DeviceType) -> Self {+ match device_type {+ DeviceType::Input => Self::for_input(),+ DeviceType::Output => Self::for_output(),+ }+ }+fn new(transient_max_secs: u64, permanent_max_secs: u64) -> Self {Self {attempts: 0,@@ -116,6 +123,15 @@ impl DeviceRecoveryBackoff {self.is_permanent = false;}+ fn should_attempt(&self, now: Instant) -> bool {+ now.saturating_duration_since(self.last_attempt)+ >= Duration::from_secs(self.next_delay_secs())+ }++ fn record_attempt_at(&mut self, now: Instant) {+ self.last_attempt = now;+ }+fn next_delay_secs(&self) -> u64 {if self.attempts == 0 {return 0;@@ -145,6 +161,47 @@ fn is_permanent_input_error(err: &anyhow::Error) -> bool {msg.contains("No default input device detected")}+fn is_permanent_device_start_error(err: &anyhow::Error) -> bool {+ let msg = err.to_string().to_lowercase();+ msg.contains("not found")+ || msg.contains("no default input device detected")+ || msg.contains("no display audio device found")+}++fn should_log_recovery_attempt(attempts: u32) -> bool {+ attempts <= 3 || attempts.is_multiple_of(30)+}++fn should_attempt_disconnected_device_recovery(+ backoffs: &mut HashMap<String, DeviceRecoveryBackoff>,+ device_name: &str,+ device_type: &DeviceType,+ now: Instant,+) -> bool {+ let backoff = backoffs+ .entry(device_name.to_string())+ .or_insert_with(|| DeviceRecoveryBackoff::for_device_type(device_type));+ if backoff.should_attempt(now) {+ backoff.record_attempt_at(now);+ true+ } else {+ false+ }+}++fn record_disconnected_device_failure(+ backoffs: &mut HashMap<String, DeviceRecoveryBackoff>,+ device_name: &str,+ device_type: &DeviceType,+ permanent: bool,+) -> (u32, u64) {+ let backoff = backoffs+ .entry(device_name.to_string())+ .or_insert_with(|| DeviceRecoveryBackoff::for_device_type(device_type));+ backoff.record_failure(permanent);+ (backoff.attempts, backoff.next_delay_secs())+}+/// Heuristic: does this input device name look like an on-board mic? Used to/// prefer the built-in mic over a virtual/aggregate input when failing over a/// disconnected pinned device, so we don't grab some random loopback input@@ -437,6 +494,8 @@ pub async fn start_device_monitor(let mut failed_devices: HashMap<String, (u32, Instant)> = HashMap::new();let mut input_recovery_backoff = DeviceRecoveryBackoff::for_input();let mut output_recovery_backoff = DeviceRecoveryBackoff::for_output();+ let mut disconnected_device_backoffs: HashMap<String, DeviceRecoveryBackoff> =+ HashMap::new();// Central handler restart cooldown: max 3 restarts in a 5-minute windowlet mut central_restart_times: Vec<Instant> = Vec::new();@@ -1195,6 +1254,7 @@ pub async fn start_device_monitor(// Skip user-disabled devices — they're intentionally stoppedif user_disabled_for_reconnect.contains(&device_name) {disconnected_devices.remove(&device_name);+ disconnected_device_backoffs.remove(&device_name);continue;}@@ -1202,24 +1262,133 @@ pub async fn start_device_monitor(Ok(device) => device,Err(e) => {debug!("Device name {} invalid: {}", device_name, e);+ disconnected_devices.remove(&device_name);+ disconnected_device_backoffs.remove(&device_name);continue;}};+ let now = Instant::now();+ if !should_attempt_disconnected_device_recovery(+ &mut disconnected_device_backoffs,+ &device_name,+ &device.device_type,+ now,+ ) {+ continue;+ }+// In system default mode, try to restart with current default insteadif audio_manager.use_system_default_audio().await {- let current_default = match device.device_type {- DeviceType::Input => default_input_device().ok(),- DeviceType::Output => default_output_device().await.ok(),- };-- if let Some(default_device) = current_default {- if audio_manager.start_device(&default_device).await.is_ok() {- info!("restarted with system default device: {}", default_device);- disconnected_devices.remove(&device_name);- continue;- }+ match &device.device_type {+ DeviceType::Input => match default_input_device() {+ Ok(default_device) => {+ match audio_manager.start_device(&default_device).await {+ Ok(()) => {+ info!(+ "restarted with system default device: {}",+ default_device+ );+ disconnected_devices.remove(&device_name);+ disconnected_device_backoffs.remove(&device_name);+ }+ Err(e) => {+ let permanent = is_permanent_device_start_error(&e);+ let (attempts, delay) =+ record_disconnected_device_failure(+ &mut disconnected_device_backoffs,+ &device_name,+ &device.device_type,+ permanent,+ );+ if should_log_recovery_attempt(attempts) {+ warn!(+ "[DEVICE_RECOVERY] failed to restart current system default {} for stale {} (attempt {}, next retry in {}s): {}",+ default_device,+ device_name,+ attempts,+ delay,+ e+ );+ }+ }+ }+ }+ Err(e) => {+ let permanent = is_permanent_input_error(&e);+ let (attempts, delay) = record_disconnected_device_failure(+ &mut disconnected_device_backoffs,+ &device_name,+ &device.device_type,+ permanent,+ );+ if should_log_recovery_attempt(attempts) {+ warn!(+ "[DEVICE_RECOVERY] no system default input while recovering stale {} (attempt {}, {}, next retry in {}s): {}",+ device_name,+ attempts,+ if permanent { "permanent" } else { "transient" },+ delay,+ e+ );+ }+ }+ },+ DeviceType::Output => match default_output_device().await {+ Ok(default_device) => {+ match audio_manager.start_device(&default_device).await {+ Ok(()) => {+ info!(+ "restarted with system default device: {}",+ default_device+ );+ disconnected_devices.remove(&device_name);+ disconnected_device_backoffs.remove(&device_name);+ }+ Err(e) => {+ let permanent = is_permanent_device_start_error(&e);+ let (attempts, delay) =+ record_disconnected_device_failure(+ &mut disconnected_device_backoffs,+ &device_name,+ &device.device_type,+ permanent,+ );+ if should_log_recovery_attempt(attempts) {+ warn!(+ "[DEVICE_RECOVERY] failed to restart current system default {} for stale {} (attempt {}, next retry in {}s): {}",+ default_device,+ device_name,+ attempts,+ delay,+ e+ );+ }+ }+ }+ }+ Err(e) => {+ let permanent = is_permanent_output_error(&e);+ let (attempts, delay) = record_disconnected_device_failure(+ &mut disconnected_device_backoffs,+ &device_name,+ &device.device_type,+ permanent,+ );+ if should_log_recovery_attempt(attempts) {+ warn!(+ "[DEVICE_RECOVERY] no system default output while recovering stale {} (attempt {}, {}, next retry in {}s): {}",+ device_name,+ attempts,+ if permanent { "permanent" } else { "transient" },+ delay,+ e+ );+ }+ }+ },}+ continue;}match audio_manager.start_device(&device).await {@@ -1229,12 +1398,32 @@ pub async fn start_device_monitor(device_name);disconnected_devices.remove(&device_name);+ disconnected_device_backoffs.remove(&device_name);}Err(e) => {- warn!(- "[DEVICE_RECOVERY] failed to restart device {}: {}",- device_name, e+ let e_str = e.to_string();+ if e_str.contains("already running") {+ disconnected_devices.remove(&device_name);+ disconnected_device_backoffs.remove(&device_name);+ continue;+ }+ let permanent = is_permanent_device_start_error(&e);+ let (attempts, delay) = record_disconnected_device_failure(+ &mut disconnected_device_backoffs,+ &device_name,+ &device.device_type,+ permanent,);+ if should_log_recovery_attempt(attempts) {+ warn!(+ "[DEVICE_RECOVERY] failed to restart device {} (attempt {}, {}, next retry in {}s): {}",+ device_name,+ attempts,+ if permanent { "permanent" } else { "transient" },+ delay,+ e+ );+ }}}}@@ -1728,6 +1917,75 @@ mod tests {assert_eq!(b.next_delay_secs(), 0);}+ #[test]+ fn disconnected_input_backoff_suppresses_two_second_retry_storm() {+ let mut backoffs = HashMap::new();+ let device_name = "Microphone (USB Composite Device) (input)";+ let start = Instant::now();+ let mut attempted_at_secs = Vec::new();++ // Reproduce the monitor's 2s cadence for one minute while Windows has+ // no default input device and the stale USB mic cannot be restarted.+ // Before this backoff, the monitor attempted all 30 ticks.+ for tick in 0_u64..30 {+ let now = start + Duration::from_secs(tick * 2);+ if should_attempt_disconnected_device_recovery(+ &mut backoffs,+ device_name,+ &DeviceType::Input,+ now,+ ) {+ attempted_at_secs.push(tick * 2);+ record_disconnected_device_failure(+ &mut backoffs,+ device_name,+ &DeviceType::Input,+ true,+ );+ }+ }++ assert_eq!(attempted_at_secs, vec![0, 2, 6, 14, 30]);+ let backoff = backoffs.get(device_name).unwrap();+ assert_eq!(backoff.attempts, attempted_at_secs.len() as u32);+ assert_eq!(backoff.next_delay_secs(), 30);+ }++ #[test]+ fn disconnected_transient_backoff_keeps_recovery_responsive() {+ let mut backoff = DeviceRecoveryBackoff::for_input();+ let start = Instant::now();+ let mut attempted_at_secs = Vec::new();++ // Transient failures stay capped at 8s so a briefly flapping device can+ // still recover quickly without retrying every monitor cycle.+ for tick in 0_u64..10 {+ let now = start + Duration::from_secs(tick * 2);+ if backoff.should_attempt(now) {+ attempted_at_secs.push(tick * 2);+ backoff.record_attempt_at(now);+ backoff.record_failure(false);+ }+ }++ assert_eq!(attempted_at_secs, vec![0, 2, 6, 14]);+ assert_eq!(backoff.next_delay_secs(), 8);+ }++ #[test]+ fn disconnected_backoff_allows_immediate_attempt_after_success_reset() {+ let mut backoff = DeviceRecoveryBackoff::for_input();+ let start = Instant::now();++ assert!(backoff.should_attempt(start));+ backoff.record_attempt_at(start);+ backoff.record_failure(true);+ assert!(!backoff.should_attempt(start + Duration::from_secs(1)));++ backoff.reset();+ assert!(backoff.should_attempt(start + Duration::from_secs(1)));+ }+#[test]fn test_backoff_transient_then_permanent_escalates() {let mut b = DeviceRecoveryBackoff::for_output();@@ -1766,6 +2024,31 @@ mod tests {assert!(!is_permanent_input_error(&transient));}+ #[test]+ fn test_permanent_device_start_error_detection() {+ assert!(is_permanent_device_start_error(&anyhow::anyhow!(+ "device Microphone (USB Composite Device) (input) not found"+ )));+ assert!(is_permanent_device_start_error(&anyhow::anyhow!(+ "No default input device detected"+ )));+ assert!(is_permanent_device_start_error(&anyhow::anyhow!(+ "no display audio device found"+ )));+ assert!(!is_permanent_device_start_error(&anyhow::anyhow!(+ "callback never fired"+ )));+ }++ #[test]+ fn test_recovery_attempt_log_throttling() {+ assert!(should_log_recovery_attempt(1));+ assert!(should_log_recovery_attempt(3));+ assert!(!should_log_recovery_attempt(4));+ assert!(should_log_recovery_attempt(30));+ assert!(!should_log_recovery_attempt(31));+ }+#[test]fn test_cooldown_evicts_old_entries() {let mut cd = RestartCooldown::new(3, Duration::from_secs(0)); // 0s window: everything expires instantly