Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(health): name a wedged capture backend instead of guessing (#6124)

Louis Beaumont6b1130f82335
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index 64ad08448..d1a0b41e4 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -138,18 +138,41 @@ pub(crate) enum VisionStallCause {
DbWritesNotLanding,
}
+/// A fallback older than this is history, not an explanation for the stall in
+/// progress.
+const CAPTURE_BACKEND_FALLBACK_RECENT_SECS: u64 = 300;
+
impl VisionStallCause {
- pub(crate) fn detail(self, stalled_secs: u64, silent_loss: u64) -> String {
+ /// `backend_fallback_secs` is how long ago capture last dropped off its
+ /// primary backend, from
+ /// [`screenpipe_screen::monitor::secs_since_capture_backend_fallback`].
+ /// When that is recent, `CapturePaused` can name the wedged backend instead
+ /// of leaving the reader to guess between TCC, display sleep and a hung
+ /// daemon.
+ pub(crate) fn detail(
+ self,
+ stalled_secs: u64,
+ silent_loss: u64,
+ backend_fallback_secs: Option<u64>,
+ ) -> String {
match self {
Self::SilentLoss => format!(
"no vision frame written for {stalled_secs}s — capture is attempting but frames \
are not reaching the writer ({silent_loss} attempts unaccounted); the database \
is idle"
),
- Self::CapturePaused => format!(
- "no vision frame written for {stalled_secs}s — the capture backend has stopped \
- delivering frames; the database is idle"
- ),
+ Self::CapturePaused => match backend_fallback_secs {
+ Some(ago) if ago <= CAPTURE_BACKEND_FALLBACK_RECENT_SECS => format!(
+ "no vision frame written for {stalled_secs}s — the capture backend has \
+ stopped delivering frames; ScreenCaptureKit last failed over to the \
+ CoreGraphics fallback {ago}s ago, so the primary backend is wedged rather \
+ than the screen being idle; the database is idle"
+ ),
+ _ => format!(
+ "no vision frame written for {stalled_secs}s — the capture backend has \
+ stopped delivering frames; the database is idle"
+ ),
+ },
Self::DbWritesNotLanding => format!(
"vision DB writes stalled for {stalled_secs}s — capture running but DB writes not \
landing"
@@ -1481,6 +1504,7 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
.detail(
now_ts.saturating_sub(vision_snap.last_db_write_ts),
vision_snap.silent_loss,
+ screenpipe_screen::monitor::secs_since_capture_backend_fallback(),
),
);
}
@@ -1846,7 +1870,7 @@ mod vision_stall_classification_tests {
fn attempts_climbing_with_an_idle_writer_is_silent_loss_not_a_db_stall() {
let cause = classify_vision_stall(45, 0, 119.0, 3);
assert_eq!(cause, VisionStallCause::SilentLoss);
- let detail = cause.detail(307, 43);
+ let detail = cause.detail(307, 43, None);
assert!(
detail.contains("not reaching the writer") && detail.contains("database is idle"),
"must not blame the database: {detail}"
@@ -1865,7 +1889,53 @@ mod vision_stall_classification_tests {
fn no_capture_attempts_means_the_backend_stopped_not_the_writer() {
let cause = classify_vision_stall(0, 0, 119.0, 3);
assert_eq!(cause, VisionStallCause::CapturePaused);
- assert!(cause.detail(300, 0).contains("capture backend has stopped"));
+ assert!(cause
+ .detail(300, 0, None)
+ .contains("capture backend has stopped"));
+ }
+
+ /// A wedged ScreenCaptureKit daemon keeps serving frames through the
+ /// CoreGraphics fallback, so `/health` used to describe a stall without
+ /// ever naming the backend that had actually failed. Support then had to
+ /// ask for logs to distinguish it from TCC revoke or display sleep.
+ #[test]
+ fn a_recent_backend_failover_is_named_as_the_stall_cause() {
+ let detail = VisionStallCause::CapturePaused.detail(300, 0, Some(12));
+ assert!(
+ detail.contains("CoreGraphics fallback")
+ && detail.contains("primary backend is wedged"),
+ "must name the wedged backend: {detail}"
+ );
+ assert!(
+ !detail.contains("screen being idle") || detail.contains("rather than"),
+ "must not leave an idle screen as the reading: {detail}"
+ );
+ }
+
+ /// An old failover is history. Attributing an unrelated stall to it would
+ /// be the same guessing this classifier exists to remove.
+ #[test]
+ fn a_stale_backend_failover_is_not_blamed_for_a_later_stall() {
+ let stale = CAPTURE_BACKEND_FALLBACK_RECENT_SECS + 1;
+ let detail = VisionStallCause::CapturePaused.detail(300, 0, Some(stale));
+ assert!(!detail.contains("CoreGraphics fallback"), "{detail}");
+ assert_eq!(detail, VisionStallCause::CapturePaused.detail(300, 0, None));
+ }
+
+ /// The failover only explains a paused backend; it must not rewrite the
+ /// silent-loss or writer verdicts.
+ #[test]
+ fn a_failover_does_not_change_the_other_stall_causes() {
+ for cause in [
+ VisionStallCause::SilentLoss,
+ VisionStallCause::DbWritesNotLanding,
+ ] {
+ assert_eq!(
+ cause.detail(300, 7, Some(5)),
+ cause.detail(300, 7, None),
+ "{cause:?} must ignore the backend failover"
+ );
+ }
}
#[test]
@@ -1886,7 +1956,7 @@ mod vision_stall_classification_tests {
VisionStallCause::DbWritesNotLanding
);
assert!(classify_vision_stall(10, 4, 119.0, 3)
- .detail(120, 0)
+ .detail(120, 0, None)
.contains("DB writes stalled"));
}
diff --git a/crates/screenpipe-screen/src/monitor.rs b/crates/screenpipe-screen/src/monitor.rs
index 9fb78623a..237ab6112 100644
--- a/crates/screenpipe-screen/src/monitor.rs
+++ b/crates/screenpipe-screen/src/monitor.rs
@@ -30,8 +30,17 @@ pub use linux::{
pub use macos::{
e2e_arm_sck_lookup_hang_fault, get_capture_backend, get_default_monitor, get_monitor_by_id,
is_screen_capture_supported, list_monitors, list_monitors_detailed, macos_version,
- set_sck_capture_max_width, HdCapture,
+ secs_since_capture_backend_fallback, set_sck_capture_max_width, HdCapture,
};
+
+/// Seconds since capture last fell back off its primary backend, if ever.
+///
+/// Only macOS has a degraded fallback path (ScreenCaptureKit to CoreGraphics);
+/// everywhere else there is nothing to fall back from.
+#[cfg(not(target_os = "macos"))]
+pub fn secs_since_capture_backend_fallback() -> Option<u64> {
+ None
+}
#[cfg(target_os = "windows")]
pub use windows::{
get_capture_backend, get_default_monitor, get_monitor_by_id, is_screen_capture_supported,
diff --git a/crates/screenpipe-screen/src/monitor/macos.rs b/crates/screenpipe-screen/src/monitor/macos.rs
index 19541056c..37c026f57 100644
--- a/crates/screenpipe-screen/src/monitor/macos.rs
+++ b/crates/screenpipe-screen/src/monitor/macos.rs
@@ -736,6 +736,7 @@ async fn fallback_after_sck_monitor_error(
"ScreenCaptureKit monitor enumeration failed ({}); trying bounded CoreGraphics fallback",
sck_error
);
+ note_capture_backend_fallback();
enumerate_xcap_monitors_bounded().await.map_err(|cg_error| {
MonitorListError::Other(format!(
"ScreenCaptureKit enumeration failed ({sck_error}); CoreGraphics fallback failed ({cg_error})"
@@ -743,6 +744,36 @@ async fn fallback_after_sck_monitor_error(
})
}
+/// Unix seconds of the last ScreenCaptureKit-to-CoreGraphics fallback, or 0.
+///
+/// Capture degrading to the CoreGraphics fallback is invisible to `/health`:
+/// frames keep arriving, so `frame_status` stays healthy while the primary
+/// backend is wedged and frames are being silently lost. Recording the fact
+/// lets the stall detail name the real cause instead of listing candidates.
+static LAST_CAPTURE_BACKEND_FALLBACK: std::sync::atomic::AtomicU64 =
+ std::sync::atomic::AtomicU64::new(0);
+
+fn note_capture_backend_fallback() {
+ let now = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map(|d| d.as_secs())
+ .unwrap_or(0);
+ LAST_CAPTURE_BACKEND_FALLBACK.store(now, Ordering::Release);
+}
+
+/// Seconds since capture last fell back off ScreenCaptureKit, if ever.
+pub fn secs_since_capture_backend_fallback() -> Option<u64> {
+ let at = LAST_CAPTURE_BACKEND_FALLBACK.load(Ordering::Acquire);
+ if at == 0 {
+ return None;
+ }
+ let now = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map(|d| d.as_secs())
+ .unwrap_or(0);
+ Some(now.saturating_sub(at))
+}
+
fn sck_monitor_error_allows_fallback(error: &MonitorListError) -> bool {
// CoreGraphics display enumeration does not prove Screen Recording
// permission. In addition to explicit PermissionDenied, SCK can return an