YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(health): name a wedged capture backend instead of guessing (#6124)
Louis Beaumont6b1130f82335
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rsindex 64ad08448..d1a0b41e4 100644--- a/crates/screenpipe-engine/src/routes/health.rs+++ b/crates/screenpipe-engine/src/routes/health.rs@@ -138,18 +138,41 @@ pub(crate) enum VisionStallCause {DbWritesNotLanding,}+/// A fallback older than this is history, not an explanation for the stall in+/// progress.+const CAPTURE_BACKEND_FALLBACK_RECENT_SECS: u64 = 300;+impl VisionStallCause {- pub(crate) fn detail(self, stalled_secs: u64, silent_loss: u64) -> String {+ /// `backend_fallback_secs` is how long ago capture last dropped off its+ /// primary backend, from+ /// [`screenpipe_screen::monitor::secs_since_capture_backend_fallback`].+ /// When that is recent, `CapturePaused` can name the wedged backend instead+ /// of leaving the reader to guess between TCC, display sleep and a hung+ /// daemon.+ pub(crate) fn detail(+ self,+ stalled_secs: u64,+ silent_loss: u64,+ backend_fallback_secs: Option<u64>,+ ) -> String {match self {Self::SilentLoss => format!("no vision frame written for {stalled_secs}s — capture is attempting but frames \are not reaching the writer ({silent_loss} attempts unaccounted); the database \is idle"),- Self::CapturePaused => format!(- "no vision frame written for {stalled_secs}s — the capture backend has stopped \- delivering frames; the database is idle"- ),+ Self::CapturePaused => match backend_fallback_secs {+ Some(ago) if ago <= CAPTURE_BACKEND_FALLBACK_RECENT_SECS => format!(+ "no vision frame written for {stalled_secs}s — the capture backend has \+ stopped delivering frames; ScreenCaptureKit last failed over to the \+ CoreGraphics fallback {ago}s ago, so the primary backend is wedged rather \+ than the screen being idle; the database is idle"+ ),+ _ => format!(+ "no vision frame written for {stalled_secs}s — the capture backend has \+ stopped delivering frames; the database is idle"+ ),+ },Self::DbWritesNotLanding => format!("vision DB writes stalled for {stalled_secs}s — capture running but DB writes not \landing"@@ -1481,6 +1504,7 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {.detail(now_ts.saturating_sub(vision_snap.last_db_write_ts),vision_snap.silent_loss,+ screenpipe_screen::monitor::secs_since_capture_backend_fallback(),),);}@@ -1846,7 +1870,7 @@ mod vision_stall_classification_tests {fn attempts_climbing_with_an_idle_writer_is_silent_loss_not_a_db_stall() {let cause = classify_vision_stall(45, 0, 119.0, 3);assert_eq!(cause, VisionStallCause::SilentLoss);- let detail = cause.detail(307, 43);+ let detail = cause.detail(307, 43, None);assert!(detail.contains("not reaching the writer") && detail.contains("database is idle"),"must not blame the database: {detail}"@@ -1865,7 +1889,53 @@ mod vision_stall_classification_tests {fn no_capture_attempts_means_the_backend_stopped_not_the_writer() {let cause = classify_vision_stall(0, 0, 119.0, 3);assert_eq!(cause, VisionStallCause::CapturePaused);- assert!(cause.detail(300, 0).contains("capture backend has stopped"));+ assert!(cause+ .detail(300, 0, None)+ .contains("capture backend has stopped"));+ }++ /// A wedged ScreenCaptureKit daemon keeps serving frames through the+ /// CoreGraphics fallback, so `/health` used to describe a stall without+ /// ever naming the backend that had actually failed. Support then had to+ /// ask for logs to distinguish it from TCC revoke or display sleep.+ #[test]+ fn a_recent_backend_failover_is_named_as_the_stall_cause() {+ let detail = VisionStallCause::CapturePaused.detail(300, 0, Some(12));+ assert!(+ detail.contains("CoreGraphics fallback")+ && detail.contains("primary backend is wedged"),+ "must name the wedged backend: {detail}"+ );+ assert!(+ !detail.contains("screen being idle") || detail.contains("rather than"),+ "must not leave an idle screen as the reading: {detail}"+ );+ }++ /// An old failover is history. Attributing an unrelated stall to it would+ /// be the same guessing this classifier exists to remove.+ #[test]+ fn a_stale_backend_failover_is_not_blamed_for_a_later_stall() {+ let stale = CAPTURE_BACKEND_FALLBACK_RECENT_SECS + 1;+ let detail = VisionStallCause::CapturePaused.detail(300, 0, Some(stale));+ assert!(!detail.contains("CoreGraphics fallback"), "{detail}");+ assert_eq!(detail, VisionStallCause::CapturePaused.detail(300, 0, None));+ }++ /// The failover only explains a paused backend; it must not rewrite the+ /// silent-loss or writer verdicts.+ #[test]+ fn a_failover_does_not_change_the_other_stall_causes() {+ for cause in [+ VisionStallCause::SilentLoss,+ VisionStallCause::DbWritesNotLanding,+ ] {+ assert_eq!(+ cause.detail(300, 7, Some(5)),+ cause.detail(300, 7, None),+ "{cause:?} must ignore the backend failover"+ );+ }}#[test]@@ -1886,7 +1956,7 @@ mod vision_stall_classification_tests {VisionStallCause::DbWritesNotLanding);assert!(classify_vision_stall(10, 4, 119.0, 3)- .detail(120, 0)+ .detail(120, 0, None).contains("DB writes stalled"));}diff --git a/crates/screenpipe-screen/src/monitor.rs b/crates/screenpipe-screen/src/monitor.rsindex 9fb78623a..237ab6112 100644--- a/crates/screenpipe-screen/src/monitor.rs+++ b/crates/screenpipe-screen/src/monitor.rs@@ -30,8 +30,17 @@ pub use linux::{pub use macos::{e2e_arm_sck_lookup_hang_fault, get_capture_backend, get_default_monitor, get_monitor_by_id,is_screen_capture_supported, list_monitors, list_monitors_detailed, macos_version,- set_sck_capture_max_width, HdCapture,+ secs_since_capture_backend_fallback, set_sck_capture_max_width, HdCapture,};++/// Seconds since capture last fell back off its primary backend, if ever.+///+/// Only macOS has a degraded fallback path (ScreenCaptureKit to CoreGraphics);+/// everywhere else there is nothing to fall back from.+#[cfg(not(target_os = "macos"))]+pub fn secs_since_capture_backend_fallback() -> Option<u64> {+ None+}#[cfg(target_os = "windows")]pub use windows::{get_capture_backend, get_default_monitor, get_monitor_by_id, is_screen_capture_supported,diff --git a/crates/screenpipe-screen/src/monitor/macos.rs b/crates/screenpipe-screen/src/monitor/macos.rsindex 19541056c..37c026f57 100644--- a/crates/screenpipe-screen/src/monitor/macos.rs+++ b/crates/screenpipe-screen/src/monitor/macos.rs@@ -736,6 +736,7 @@ async fn fallback_after_sck_monitor_error("ScreenCaptureKit monitor enumeration failed ({}); trying bounded CoreGraphics fallback",sck_error);+ note_capture_backend_fallback();enumerate_xcap_monitors_bounded().await.map_err(|cg_error| {MonitorListError::Other(format!("ScreenCaptureKit enumeration failed ({sck_error}); CoreGraphics fallback failed ({cg_error})"@@ -743,6 +744,36 @@ async fn fallback_after_sck_monitor_error(})}+/// Unix seconds of the last ScreenCaptureKit-to-CoreGraphics fallback, or 0.+///+/// Capture degrading to the CoreGraphics fallback is invisible to `/health`:+/// frames keep arriving, so `frame_status` stays healthy while the primary+/// backend is wedged and frames are being silently lost. Recording the fact+/// lets the stall detail name the real cause instead of listing candidates.+static LAST_CAPTURE_BACKEND_FALLBACK: std::sync::atomic::AtomicU64 =+ std::sync::atomic::AtomicU64::new(0);++fn note_capture_backend_fallback() {+ let now = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .map(|d| d.as_secs())+ .unwrap_or(0);+ LAST_CAPTURE_BACKEND_FALLBACK.store(now, Ordering::Release);+}++/// Seconds since capture last fell back off ScreenCaptureKit, if ever.+pub fn secs_since_capture_backend_fallback() -> Option<u64> {+ let at = LAST_CAPTURE_BACKEND_FALLBACK.load(Ordering::Acquire);+ if at == 0 {+ return None;+ }+ let now = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .map(|d| d.as_secs())+ .unwrap_or(0);+ Some(now.saturating_sub(at))+}+fn sck_monitor_error_allows_fallback(error: &MonitorListError) -> bool {// CoreGraphics display enumeration does not prove Screen Recording// permission. In addition to explicit PermissionDenied, SCK can return an