Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(health): name the vision stall cause instead of guessing it twice (#5945)

Louis Beaumont47a329d7fe93
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index 03df99f25..1f969da19 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize};
use serde_json::json;
use std::collections::HashMap;
use std::sync::{
- atomic::{AtomicU64, Ordering},
+ atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering},
Arc,
};
use tokio::sync::{Mutex, RwLock};
@@ -113,6 +113,99 @@ fn suspected_stall_cause(read_idle: u32, write_idle: u32) -> &'static str {
}
}
+/// Why vision stopped producing frames, decided from the counters rather than
+/// guessed.
+///
+/// A stale `last_db_write_ts` says only that nothing landed. It does not say
+/// which stage failed, and the two consumers of that flag used to guess in
+/// opposite directions on the same evidence: the `/health` detail claimed
+/// "capture running but DB writes not landing" while this module's own WARN
+/// claimed "usually means a static screen / idle user, not a pipeline stall".
+/// During the 2026-08-06 macOS capture outage both fired at once and both were
+/// wrong — the database was idle (119ms average write latency, every pool
+/// connection free) and the screen was not static.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub(crate) enum VisionStallCause {
+ /// Capture attempts resolved to nothing: not persisted, not deduped, not
+ /// dropped. Frames are being lost between the attempt and the writer.
+ SilentLoss,
+ /// Attempts stopped entirely — capture is paused or the backend is
+ /// refusing to hand back frames (TCC revoke, display sleep, wedged
+ /// ScreenCaptureKit daemon).
+ CapturePaused,
+ /// Captures reached the writer and the writer is the problem.
+ DbWritesNotLanding,
+}
+
+impl VisionStallCause {
+ pub(crate) fn detail(self, stalled_secs: u64, silent_loss: u64) -> String {
+ match self {
+ Self::SilentLoss => format!(
+ "no vision frame written for {stalled_secs}s — capture is attempting but frames \
+ are not reaching the writer ({silent_loss} attempts unaccounted); the database \
+ is idle"
+ ),
+ Self::CapturePaused => format!(
+ "no vision frame written for {stalled_secs}s — the capture backend has stopped \
+ delivering frames; the database is idle"
+ ),
+ Self::DbWritesNotLanding => format!(
+ "vision DB writes stalled for {stalled_secs}s — capture running but DB writes not \
+ landing"
+ ),
+ }
+ }
+}
+
+/// Threshold above which the writer is slow enough to be the prime suspect.
+const VISION_DB_SLOW_MS: f64 = 10_000.0;
+
+/// Sentinel for "no stall in progress" in [`VISION_STALL_CAUSE`].
+const NO_VISION_STALL: u8 = u8::MAX;
+
+/// Last classification, shared from the detector to the `/health` detail
+/// builder so the response text and the log line can never disagree again.
+static VISION_STALL_CAUSE: AtomicU8 = AtomicU8::new(NO_VISION_STALL);
+
+fn current_vision_stall_cause() -> Option<VisionStallCause> {
+ match VISION_STALL_CAUSE.load(Ordering::Relaxed) {
+ x if x == VisionStallCause::SilentLoss as u8 => Some(VisionStallCause::SilentLoss),
+ x if x == VisionStallCause::CapturePaused as u8 => Some(VisionStallCause::CapturePaused),
+ x if x == VisionStallCause::DbWritesNotLanding as u8 => {
+ Some(VisionStallCause::DbWritesNotLanding)
+ }
+ _ => None,
+ }
+}
+
+/// Classify a confirmed vision stall.
+///
+/// Only called once `last_db_write_ts` has gone stale, which is itself
+/// load-bearing: `record_dedup_skip` and `record_corrupt_skip` both advance
+/// that timestamp, so a stale value proves there were *zero* dedups and zero
+/// corrupt skips in the window. A static screen therefore cannot produce this
+/// warning, and the old "idle user, not a pipeline stall" wording was wrong
+/// every single time it printed.
+pub(crate) fn classify_vision_stall(
+ capture_attempts_delta: u64,
+ frames_dropped_delta: u64,
+ avg_db_latency_ms: f64,
+ write_pool_idle: u32,
+) -> VisionStallCause {
+ if capture_attempts_delta == 0 {
+ return VisionStallCause::CapturePaused;
+ }
+ // Drops are the writer/capture boundary failing loudly (timeout or error);
+ // a saturated write pool or a slow writer implicates the DB directly.
+ // Anything else means the attempts simply evaporated.
+ if frames_dropped_delta > 0 || avg_db_latency_ms > VISION_DB_SLOW_MS || write_pool_idle == 0 {
+ VisionStallCause::DbWritesNotLanding
+ } else {
+ VisionStallCause::SilentLoss
+ }
+}
+
const SILENT_AUDIO_RMS_THRESHOLD: f64 = 0.001;
/// How recently the audio stream-timeout watchdog must have fired for the audio
@@ -969,7 +1062,38 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
let db_stale = vision_snap.last_db_write_ts > 0
&& now_ts.saturating_sub(vision_snap.last_db_write_ts) > threshold_secs;
let stalled = capture_fresh && db_stale;
+
+ // Baseline captured on the transition into a stall so the classifier
+ // sees what moved *during* it. Lifetime totals cannot answer
+ // "climbing or flat" — the question the triage rule below turns on.
+ static VISION_STALL_ACTIVE: AtomicBool = AtomicBool::new(false);
+ static VISION_STALL_BASE_ATTEMPTS: AtomicU64 = AtomicU64::new(0);
+ static VISION_STALL_BASE_DROPPED: AtomicU64 = AtomicU64::new(0);
if stalled {
+ if !VISION_STALL_ACTIVE.swap(true, Ordering::Relaxed) {
+ VISION_STALL_BASE_ATTEMPTS.store(vision_snap.capture_attempts, Ordering::Relaxed);
+ VISION_STALL_BASE_DROPPED.store(vision_snap.frames_dropped, Ordering::Relaxed);
+ }
+ } else {
+ VISION_STALL_ACTIVE.store(false, Ordering::Relaxed);
+ VISION_STALL_CAUSE.store(NO_VISION_STALL, Ordering::Relaxed);
+ }
+
+ if stalled {
+ let (_, _, ws, wi) = state.db.pool_stats();
+ let cause = classify_vision_stall(
+ vision_snap
+ .capture_attempts
+ .saturating_sub(VISION_STALL_BASE_ATTEMPTS.load(Ordering::Relaxed)),
+ vision_snap
+ .frames_dropped
+ .saturating_sub(VISION_STALL_BASE_DROPPED.load(Ordering::Relaxed)),
+ vision_snap.avg_db_latency_ms,
+ wi,
+ );
+ let _ = ws;
+ VISION_STALL_CAUSE.store(cause as u8, Ordering::Relaxed);
+
// throttle to once per 60s to avoid log spam (health runs every ~1s)
static LAST_VISION_STALL_LOG: AtomicU64 = AtomicU64::new(0);
let prev = LAST_VISION_STALL_LOG.load(Ordering::Relaxed);
@@ -1007,9 +1131,17 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
.saturating_sub(vision_snap.frames_db_written)
.saturating_sub(vision_snap.dedup_skips);
warn!(
- "health_check: no unique vision frame in {}s (capture heartbeat {}s ago — usually means a static screen / idle user, not a pipeline stall) | lifetime: attempts={}, persisted={}, dedup={}, silent_loss={} | pool: read={}/{} idle, write={}/{} idle | suspected: {}",
+ "health_check: no unique vision frame in {}s (capture heartbeat {}s ago — {}) | lifetime: attempts={}, persisted={}, dedup={}, silent_loss={} | pool: read={}/{} idle, write={}/{} idle | suspected: {}",
now_ts.saturating_sub(vision_snap.last_db_write_ts),
now_ts.saturating_sub(vision_snap.last_capture_attempt_ts),
+ match cause {
+ VisionStallCause::SilentLoss =>
+ "capture is attempting but frames are not reaching the writer",
+ VisionStallCause::CapturePaused =>
+ "capture stopped attempting — backend paused or refusing frames",
+ VisionStallCause::DbWritesNotLanding =>
+ "captures reached the writer and the writer is not landing them",
+ },
vision_snap.capture_attempts,
vision_snap.frames_db_written,
vision_snap.dedup_skips,
@@ -1326,10 +1458,16 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
));
}
if vision_db_write_stalled {
- detail_parts.push(format!(
- "vision DB writes stalled for {}s — capture running but DB writes not landing",
- now_ts.saturating_sub(vision_snap.last_db_write_ts)
- ));
+ // Named by the classifier, not assumed. Falls back to the
+ // historical wording only if the cause slot is somehow unset.
+ detail_parts.push(
+ current_vision_stall_cause()
+ .unwrap_or(VisionStallCause::DbWritesNotLanding)
+ .detail(
+ now_ts.saturating_sub(vision_snap.last_db_write_ts),
+ vision_snap.silent_loss,
+ ),
+ );
}
}
if audio_degraded || audio_status == "active_no_data" {
@@ -1677,6 +1815,80 @@ pub async fn api_vision_status() -> JsonResponse<serde_json::Value> {
}
}
+#[cfg(test)]
+mod vision_stall_classification_tests {
+ use super::*;
+
+ /// The 2026-08-06 macOS outage, replayed from its own `/health` payload:
+ /// attempts climbing (748 -> 793), nothing persisted, nothing deduped,
+ /// drops flat, 119ms average write latency, every write connection idle.
+ /// Both shipped messages were wrong — the overlay blamed the database and
+ /// the engine log called it an idle user.
+ #[test]
+ fn attempts_climbing_with_an_idle_writer_is_silent_loss_not_a_db_stall() {
+ let cause = classify_vision_stall(45, 0, 119.0, 3);
+ assert_eq!(cause, VisionStallCause::SilentLoss);
+ let detail = cause.detail(307, 43);
+ assert!(
+ detail.contains("not reaching the writer") && detail.contains("database is idle"),
+ "must not blame the database: {detail}"
+ );
+ assert!(
+ !detail.contains("DB writes stalled"),
+ "regression: still reporting a DB stall: {detail}"
+ );
+ }
+
+ /// A stale `last_db_write_ts` proves zero dedups in the window, because
+ /// `record_dedup_skip` advances it. So a static screen can never be the
+ /// explanation once this classifier runs — the removed "idle user"
+ /// wording was wrong every time it printed.
+ #[test]
+ fn no_capture_attempts_means_the_backend_stopped_not_the_writer() {
+ let cause = classify_vision_stall(0, 0, 119.0, 3);
+ assert_eq!(cause, VisionStallCause::CapturePaused);
+ assert!(cause.detail(300, 0).contains("capture backend has stopped"));
+ }
+
+ #[test]
+ fn drops_or_a_slow_or_saturated_writer_still_implicate_the_database() {
+ // Frames reached the writer and were dropped there.
+ assert_eq!(
+ classify_vision_stall(10, 4, 119.0, 3),
+ VisionStallCause::DbWritesNotLanding
+ );
+ // Writer is slow.
+ assert_eq!(
+ classify_vision_stall(10, 0, VISION_DB_SLOW_MS + 1.0, 3),
+ VisionStallCause::DbWritesNotLanding
+ );
+ // No write connection free.
+ assert_eq!(
+ classify_vision_stall(10, 0, 119.0, 0),
+ VisionStallCause::DbWritesNotLanding
+ );
+ assert!(classify_vision_stall(10, 4, 119.0, 3)
+ .detail(120, 0)
+ .contains("DB writes stalled"));
+ }
+
+ /// The detector and the `/health` detail builder read one slot, so they
+ /// cannot contradict each other the way the two hardcoded strings did.
+ #[test]
+ fn cause_slot_round_trips_and_clears() {
+ for cause in [
+ VisionStallCause::SilentLoss,
+ VisionStallCause::CapturePaused,
+ VisionStallCause::DbWritesNotLanding,
+ ] {
+ VISION_STALL_CAUSE.store(cause as u8, Ordering::Relaxed);
+ assert_eq!(current_vision_stall_cause(), Some(cause));
+ }
+ VISION_STALL_CAUSE.store(NO_VISION_STALL, Ordering::Relaxed);
+ assert_eq!(current_vision_stall_cause(), None);
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;