YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(health): name the vision stall cause instead of guessing it twice (#5945)
Louis Beaumont47a329d7fe93
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rsindex 03df99f25..1f969da19 100644--- a/crates/screenpipe-engine/src/routes/health.rs+++ b/crates/screenpipe-engine/src/routes/health.rs@@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize};use serde_json::json;use std::collections::HashMap;use std::sync::{- atomic::{AtomicU64, Ordering},+ atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering},Arc,};use tokio::sync::{Mutex, RwLock};@@ -113,6 +113,99 @@ fn suspected_stall_cause(read_idle: u32, write_idle: u32) -> &'static str {}}+/// Why vision stopped producing frames, decided from the counters rather than+/// guessed.+///+/// A stale `last_db_write_ts` says only that nothing landed. It does not say+/// which stage failed, and the two consumers of that flag used to guess in+/// opposite directions on the same evidence: the `/health` detail claimed+/// "capture running but DB writes not landing" while this module's own WARN+/// claimed "usually means a static screen / idle user, not a pipeline stall".+/// During the 2026-08-06 macOS capture outage both fired at once and both were+/// wrong — the database was idle (119ms average write latency, every pool+/// connection free) and the screen was not static.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+#[repr(u8)]+pub(crate) enum VisionStallCause {+ /// Capture attempts resolved to nothing: not persisted, not deduped, not+ /// dropped. Frames are being lost between the attempt and the writer.+ SilentLoss,+ /// Attempts stopped entirely — capture is paused or the backend is+ /// refusing to hand back frames (TCC revoke, display sleep, wedged+ /// ScreenCaptureKit daemon).+ CapturePaused,+ /// Captures reached the writer and the writer is the problem.+ DbWritesNotLanding,+}++impl VisionStallCause {+ pub(crate) fn detail(self, stalled_secs: u64, silent_loss: u64) -> String {+ match self {+ Self::SilentLoss => format!(+ "no vision frame written for {stalled_secs}s — capture is attempting but frames \+ are not reaching the writer ({silent_loss} attempts unaccounted); the database \+ is idle"+ ),+ Self::CapturePaused => format!(+ "no vision frame written for {stalled_secs}s — the capture backend has stopped \+ delivering frames; the database is idle"+ ),+ Self::DbWritesNotLanding => format!(+ "vision DB writes stalled for {stalled_secs}s — capture running but DB writes not \+ landing"+ ),+ }+ }+}++/// Threshold above which the writer is slow enough to be the prime suspect.+const VISION_DB_SLOW_MS: f64 = 10_000.0;++/// Sentinel for "no stall in progress" in [`VISION_STALL_CAUSE`].+const NO_VISION_STALL: u8 = u8::MAX;++/// Last classification, shared from the detector to the `/health` detail+/// builder so the response text and the log line can never disagree again.+static VISION_STALL_CAUSE: AtomicU8 = AtomicU8::new(NO_VISION_STALL);++fn current_vision_stall_cause() -> Option<VisionStallCause> {+ match VISION_STALL_CAUSE.load(Ordering::Relaxed) {+ x if x == VisionStallCause::SilentLoss as u8 => Some(VisionStallCause::SilentLoss),+ x if x == VisionStallCause::CapturePaused as u8 => Some(VisionStallCause::CapturePaused),+ x if x == VisionStallCause::DbWritesNotLanding as u8 => {+ Some(VisionStallCause::DbWritesNotLanding)+ }+ _ => None,+ }+}++/// Classify a confirmed vision stall.+///+/// Only called once `last_db_write_ts` has gone stale, which is itself+/// load-bearing: `record_dedup_skip` and `record_corrupt_skip` both advance+/// that timestamp, so a stale value proves there were *zero* dedups and zero+/// corrupt skips in the window. A static screen therefore cannot produce this+/// warning, and the old "idle user, not a pipeline stall" wording was wrong+/// every single time it printed.+pub(crate) fn classify_vision_stall(+ capture_attempts_delta: u64,+ frames_dropped_delta: u64,+ avg_db_latency_ms: f64,+ write_pool_idle: u32,+) -> VisionStallCause {+ if capture_attempts_delta == 0 {+ return VisionStallCause::CapturePaused;+ }+ // Drops are the writer/capture boundary failing loudly (timeout or error);+ // a saturated write pool or a slow writer implicates the DB directly.+ // Anything else means the attempts simply evaporated.+ if frames_dropped_delta > 0 || avg_db_latency_ms > VISION_DB_SLOW_MS || write_pool_idle == 0 {+ VisionStallCause::DbWritesNotLanding+ } else {+ VisionStallCause::SilentLoss+ }+}+const SILENT_AUDIO_RMS_THRESHOLD: f64 = 0.001;/// How recently the audio stream-timeout watchdog must have fired for the audio@@ -969,7 +1062,38 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {let db_stale = vision_snap.last_db_write_ts > 0&& now_ts.saturating_sub(vision_snap.last_db_write_ts) > threshold_secs;let stalled = capture_fresh && db_stale;++ // Baseline captured on the transition into a stall so the classifier+ // sees what moved *during* it. Lifetime totals cannot answer+ // "climbing or flat" — the question the triage rule below turns on.+ static VISION_STALL_ACTIVE: AtomicBool = AtomicBool::new(false);+ static VISION_STALL_BASE_ATTEMPTS: AtomicU64 = AtomicU64::new(0);+ static VISION_STALL_BASE_DROPPED: AtomicU64 = AtomicU64::new(0);if stalled {+ if !VISION_STALL_ACTIVE.swap(true, Ordering::Relaxed) {+ VISION_STALL_BASE_ATTEMPTS.store(vision_snap.capture_attempts, Ordering::Relaxed);+ VISION_STALL_BASE_DROPPED.store(vision_snap.frames_dropped, Ordering::Relaxed);+ }+ } else {+ VISION_STALL_ACTIVE.store(false, Ordering::Relaxed);+ VISION_STALL_CAUSE.store(NO_VISION_STALL, Ordering::Relaxed);+ }++ if stalled {+ let (_, _, ws, wi) = state.db.pool_stats();+ let cause = classify_vision_stall(+ vision_snap+ .capture_attempts+ .saturating_sub(VISION_STALL_BASE_ATTEMPTS.load(Ordering::Relaxed)),+ vision_snap+ .frames_dropped+ .saturating_sub(VISION_STALL_BASE_DROPPED.load(Ordering::Relaxed)),+ vision_snap.avg_db_latency_ms,+ wi,+ );+ let _ = ws;+ VISION_STALL_CAUSE.store(cause as u8, Ordering::Relaxed);+// throttle to once per 60s to avoid log spam (health runs every ~1s)static LAST_VISION_STALL_LOG: AtomicU64 = AtomicU64::new(0);let prev = LAST_VISION_STALL_LOG.load(Ordering::Relaxed);@@ -1007,9 +1131,17 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {.saturating_sub(vision_snap.frames_db_written).saturating_sub(vision_snap.dedup_skips);warn!(- "health_check: no unique vision frame in {}s (capture heartbeat {}s ago — usually means a static screen / idle user, not a pipeline stall) | lifetime: attempts={}, persisted={}, dedup={}, silent_loss={} | pool: read={}/{} idle, write={}/{} idle | suspected: {}",+ "health_check: no unique vision frame in {}s (capture heartbeat {}s ago — {}) | lifetime: attempts={}, persisted={}, dedup={}, silent_loss={} | pool: read={}/{} idle, write={}/{} idle | suspected: {}",now_ts.saturating_sub(vision_snap.last_db_write_ts),now_ts.saturating_sub(vision_snap.last_capture_attempt_ts),+ match cause {+ VisionStallCause::SilentLoss =>+ "capture is attempting but frames are not reaching the writer",+ VisionStallCause::CapturePaused =>+ "capture stopped attempting — backend paused or refusing frames",+ VisionStallCause::DbWritesNotLanding =>+ "captures reached the writer and the writer is not landing them",+ },vision_snap.capture_attempts,vision_snap.frames_db_written,vision_snap.dedup_skips,@@ -1326,10 +1458,16 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {));}if vision_db_write_stalled {- detail_parts.push(format!(- "vision DB writes stalled for {}s — capture running but DB writes not landing",- now_ts.saturating_sub(vision_snap.last_db_write_ts)- ));+ // Named by the classifier, not assumed. Falls back to the+ // historical wording only if the cause slot is somehow unset.+ detail_parts.push(+ current_vision_stall_cause()+ .unwrap_or(VisionStallCause::DbWritesNotLanding)+ .detail(+ now_ts.saturating_sub(vision_snap.last_db_write_ts),+ vision_snap.silent_loss,+ ),+ );}}if audio_degraded || audio_status == "active_no_data" {@@ -1677,6 +1815,80 @@ pub async fn api_vision_status() -> JsonResponse<serde_json::Value> {}}+#[cfg(test)]+mod vision_stall_classification_tests {+ use super::*;++ /// The 2026-08-06 macOS outage, replayed from its own `/health` payload:+ /// attempts climbing (748 -> 793), nothing persisted, nothing deduped,+ /// drops flat, 119ms average write latency, every write connection idle.+ /// Both shipped messages were wrong — the overlay blamed the database and+ /// the engine log called it an idle user.+ #[test]+ fn attempts_climbing_with_an_idle_writer_is_silent_loss_not_a_db_stall() {+ let cause = classify_vision_stall(45, 0, 119.0, 3);+ assert_eq!(cause, VisionStallCause::SilentLoss);+ let detail = cause.detail(307, 43);+ assert!(+ detail.contains("not reaching the writer") && detail.contains("database is idle"),+ "must not blame the database: {detail}"+ );+ assert!(+ !detail.contains("DB writes stalled"),+ "regression: still reporting a DB stall: {detail}"+ );+ }++ /// A stale `last_db_write_ts` proves zero dedups in the window, because+ /// `record_dedup_skip` advances it. So a static screen can never be the+ /// explanation once this classifier runs — the removed "idle user"+ /// wording was wrong every time it printed.+ #[test]+ fn no_capture_attempts_means_the_backend_stopped_not_the_writer() {+ let cause = classify_vision_stall(0, 0, 119.0, 3);+ assert_eq!(cause, VisionStallCause::CapturePaused);+ assert!(cause.detail(300, 0).contains("capture backend has stopped"));+ }++ #[test]+ fn drops_or_a_slow_or_saturated_writer_still_implicate_the_database() {+ // Frames reached the writer and were dropped there.+ assert_eq!(+ classify_vision_stall(10, 4, 119.0, 3),+ VisionStallCause::DbWritesNotLanding+ );+ // Writer is slow.+ assert_eq!(+ classify_vision_stall(10, 0, VISION_DB_SLOW_MS + 1.0, 3),+ VisionStallCause::DbWritesNotLanding+ );+ // No write connection free.+ assert_eq!(+ classify_vision_stall(10, 0, 119.0, 0),+ VisionStallCause::DbWritesNotLanding+ );+ assert!(classify_vision_stall(10, 4, 119.0, 3)+ .detail(120, 0)+ .contains("DB writes stalled"));+ }++ /// The detector and the `/health` detail builder read one slot, so they+ /// cannot contradict each other the way the two hardcoded strings did.+ #[test]+ fn cause_slot_round_trips_and_clears() {+ for cause in [+ VisionStallCause::SilentLoss,+ VisionStallCause::CapturePaused,+ VisionStallCause::DbWritesNotLanding,+ ] {+ VISION_STALL_CAUSE.store(cause as u8, Ordering::Relaxed);+ assert_eq!(current_vision_stall_cause(), Some(cause));+ }+ VISION_STALL_CAUSE.store(NO_VISION_STALL, Ordering::Relaxed);+ assert_eq!(current_vision_stall_cause(), None);+ }+}+#[cfg(test)]mod tests {use super::*;