YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(vision): restart capture when it silently wedges while status stays Running (#3939) (#3952)
Louis Beaumont8c982230cd05
diff --git a/crates/screenpipe-engine/src/vision_manager/manager.rs b/crates/screenpipe-engine/src/vision_manager/manager.rsindex aac991f4f..79e26c417 100644--- a/crates/screenpipe-engine/src/vision_manager/manager.rs+++ b/crates/screenpipe-engine/src/vision_manager/manager.rs@@ -206,6 +206,14 @@ impl VisionManager {*self.status.read().await}+ /// Shared capture-pipeline metrics. The monitor watcher's silent-wedge+ /// watchdog reads this to tell a real stall (attempts firing but nothing+ /// persisted) apart from a healthy static screen (which still ticks+ /// `last_db_write_ts` via dedup-skip). Mirrors what /health reports.+ pub fn vision_metrics(&self) -> &Arc<PipelineMetrics> {+ &self.config.vision_metrics+ }+/// Check whether a monitor is allowed by the user's monitor filter settings./// Uses prefix matching (name + resolution) so that position changes after/// reconnect don't break the filter.diff --git a/crates/screenpipe-engine/src/vision_manager/monitor_watcher.rs b/crates/screenpipe-engine/src/vision_manager/monitor_watcher.rsindex 8dcdf12ea..44f69b804 100644--- a/crates/screenpipe-engine/src/vision_manager/monitor_watcher.rs+++ b/crates/screenpipe-engine/src/vision_manager/monitor_watcher.rs@@ -7,7 +7,7 @@use once_cell::sync::Lazy;use std::collections::{HashMap, HashSet};use std::sync::Arc;-use std::time::Duration;+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use tokio::sync::Mutex;use tokio::task::JoinHandle;use tracing::{debug, info, warn};@@ -21,6 +21,66 @@ use crate::permission_monitor;static MONITOR_WATCHER: Lazy<Mutex<Option<JoinHandle<()>>>> = Lazy::new(|| Mutex::new(None));+// ── Silent-wedge watchdog (#3939) ───────────────────────────────────────────+//+// Symptom: VisionManager status stays `Running` and the tray shows "Recording",+// but the capture loop has stopped persisting frames — for ~1h in the original+// report — with no auto-recovery. The existing recovery paths all miss it: the+// monitor watcher only restarts when status != Running, and the /health stall+// flag only colors the response, nothing acts on it.+//+// Detection: a healthy pipeline ticks `last_db_write_ts` on every DB write AND+// on every dedup-skip (a static screen: idle user, slide deck, video call). So+// "capture attempts still fresh while `last_db_write_ts` has been frozen for a+// long time" cannot happen on a healthy idle screen — it only happens when+// captures are erroring/timing out (DB write pool saturated after a disk-I/O+// stall: "writes fail with PoolTimedOut forever until restart"). That makes it+// a low-false-positive trigger for a recovery restart.+//+// Scope: this covers the "still attempting" wedge only. The "gone silent" wedge+// (loop blocked inside a non-cancellable sync SQLite call) also stops ticking+// `last_capture_attempt_ts`, and a restart can't preempt sync FFI; that needs+// cancellable DB I/O and is tracked separately.++/// The capture loop must still be issuing attempts this recently for the wedge+/// to be the restartable "still-attempting" kind.+const WEDGE_ATTEMPT_FRESH_SECS: u64 = 60;+/// No frame persisted (write or dedup-skip) for at least this long while+/// attempts stay fresh → wedged. Deliberately looser than the /health stall+/// threshold (60s) because the action here (a capture restart) is disruptive.+const WEDGE_DB_STALE_SECS: u64 = 120;+/// Ignore the warm-up window so a pipeline that hasn't written its first frame+/// yet is never mistaken for a stall.+const WEDGE_MIN_UPTIME_SECS: f64 = 120.0;+/// Never restart more than once per this window, so a wedge a restart can't fix+/// (e.g. a disk still stalled) can't turn into a restart storm.+const WEDGE_RESTART_COOLDOWN: Duration = Duration::from_secs(300);++fn now_epoch_secs() -> u64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map(|d| d.as_secs())+ .unwrap_or(0)+}++/// Pure decision: is vision capture in the restartable "still-attempting" wedge?+/// Kept free of clocks and I/O so it can be unit-tested with synthetic inputs.+fn vision_capture_wedged(+ uptime_secs: f64,+ last_capture_attempt_ts: u64,+ last_db_write_ts: u64,+ now_ts: u64,+) -> bool {+ uptime_secs > WEDGE_MIN_UPTIME_SECS+ // loop is alive and still issuing capture attempts (heartbeat fresh)+ && last_capture_attempt_ts > 0+ && now_ts.saturating_sub(last_capture_attempt_ts) < WEDGE_ATTEMPT_FRESH_SECS+ // ...but nothing has persisted (write or dedup-skip) for a long while.+ // last_db_write_ts == 0 means "never written yet" (warming up), not a stall.+ && last_db_write_ts > 0+ && now_ts.saturating_sub(last_db_write_ts) > WEDGE_DB_STALE_SECS+}+/// Start the monitor watcher that polls for monitor changes./// When `audio_manager` is provided, SCK-based (output) audio devices are also/// stopped/restarted alongside vision during DRM pause/resume.@@ -56,6 +116,8 @@ pub async fn start_monitor_watcher(let mut suppress_next_topology_event = false;// Warn once per recovery episode, then keep repeated retries quiet.let mut recovery_retry_warned = false;+ // Last time the silent-wedge watchdog restarted capture (cooldown gate).+ let mut last_vision_restart: Option<Instant> = None;// Initialize with current monitorsmatch list_monitors_detailed().await {@@ -224,6 +286,63 @@ pub async fn start_monitor_watcher(continue;}+ // ── Silent-wedge watchdog (#3939) ───────────────────────────────+ // status == Running, but is the loop actually persisting frames? If+ // it has been attempting while writing nothing for a sustained+ // window, the write path is wedged — restart capture to recover+ // (cooldown-gated so a still-stalled disk can't cause a restart+ // storm). See the module-level comment for why this won't fire on a+ // healthy static screen.+ {+ let now_ts = now_epoch_secs();+ let snap = vision_manager.vision_metrics().snapshot();+ let wedged = vision_capture_wedged(+ snap.uptime_secs,+ snap.last_capture_attempt_ts,+ snap.last_db_write_ts,+ now_ts,+ );+ let cooldown_ok = last_vision_restart+ .map(|t| t.elapsed() >= WEDGE_RESTART_COOLDOWN)+ .unwrap_or(true);+ if wedged && cooldown_ok {+ let db_stale = now_ts.saturating_sub(snap.last_db_write_ts);+ let attempt_age = now_ts.saturating_sub(snap.last_capture_attempt_ts);+ warn!(+ "vision capture wedged: status=Running, attempts fresh ({}s ago) but \+ no frame persisted for {}s — restarting VisionManager (#3939)",+ attempt_age, db_stale+ );+ let _ = screenpipe_events::send_event(+ "vision_capture_wedge_restart",+ serde_json::json!({+ "db_stale_secs": db_stale,+ "attempt_age_secs": attempt_age,+ "uptime_secs": snap.uptime_secs,+ }),+ );+ if let Err(e) = vision_manager.stop().await {+ warn!("wedge watchdog: vision stop failed: {:?}", e);+ }+ match vision_manager.start().await {+ Ok(()) => {+ info!("wedge watchdog: VisionManager restarted after silent wedge")+ }+ Err(e) => warn!("wedge watchdog: vision restart failed: {:?}", e),+ }+ last_vision_restart = Some(Instant::now());+ // Re-populate known_monitors after the restart, then re-loop.+ if let Ok(monitors) = list_monitors_detailed().await {+ known_monitors = monitors+ .iter()+ .map(|m| (m.id(), m.name().to_string()))+ .collect();+ }+ tokio::time::sleep(Duration::from_secs(2)).await;+ continue;+ }+ }+// Get currently connected monitors with detailed error infolet current_monitors = match list_monitors_detailed().await {Ok(monitors) => {@@ -400,3 +519,58 @@ pub async fn stop_monitor_watcher() -> anyhow::Result<()> {}Ok(())}++#[cfg(test)]+mod tests {+ use super::*;++ // Fixed "now" so deltas are exact and the tests never depend on wall clock.+ const NOW: u64 = 2_000_000_000;++ #[test]+ fn healthy_recent_write_is_not_wedged() {+ // Wrote 1s ago, attempting 1s ago → clearly healthy.+ assert!(!vision_capture_wedged(600.0, NOW - 1, NOW - 1, NOW));+ }++ #[test]+ fn static_screen_is_not_wedged() {+ // A static screen (idle user, slide deck, video call) still ticks+ // last_db_write_ts via dedup-skip, so even after an hour of uptime it is+ // never flagged: attempts fresh AND write fresh (via dedup).+ assert!(!vision_capture_wedged(3600.0, NOW - 2, NOW - 3, NOW));+ }++ #[test]+ fn warming_up_is_not_wedged() {+ // Below the warm-up uptime floor.+ assert!(!vision_capture_wedged(30.0, NOW - 1, NOW - 1, NOW));+ // Never written a frame yet (last_db_write_ts == 0) is "warming up",+ // not "writes stopped".+ assert!(!vision_capture_wedged(600.0, NOW - 1, 0, NOW));+ }++ #[test]+ fn gone_silent_or_idle_is_not_restartable_here() {+ // Attempts stale (loop idle, or blocked in sync FFI) → NOT the+ // still-attempting wedge a restart can fix; handled separately.+ assert!(!vision_capture_wedged(600.0, NOW - 300, NOW - 300, NOW));+ }++ #[test]+ fn still_attempting_wedge_is_detected() {+ // Uptime fine, attempting 3s ago, but nothing persisted for 200s. This+ // is the #3939 wedge: the loop is alive but every capture is failing.+ assert!(vision_capture_wedged(600.0, NOW - 3, NOW - 200, NOW));+ }++ #[test]+ fn thresholds_are_respected() {+ // db stale 119s (< 120s) → hold off a little longer.+ assert!(!vision_capture_wedged(600.0, NOW - 3, NOW - 119, NOW));+ // db stale 121s (> 120s) → trip.+ assert!(vision_capture_wedged(600.0, NOW - 3, NOW - 121, NOW));+ // attempt 60s ago is not "< 60s fresh" → treat as not actively attempting.+ assert!(!vision_capture_wedged(600.0, NOW - 60, NOW - 200, NOW));+ }+}