Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(overlay): stop false recording failure alerts (#5350)

Louis Beaumont8c8234823a33
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/health.rs b/apps/screenpipe-app-tauri/src-tauri/src/health.rs
index 027ae5932..d26cf9350 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/health.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/health.rs
@@ -336,18 +336,76 @@ fn set_recording_info(status: RecordingStatus, devices: Vec<DeviceInfo>) {
info.devices = devices;
}
-/// Minimal audio pipeline info for stall detection (subset of server's full struct)
-#[derive(Debug, Deserialize, Default)]
-#[allow(dead_code)]
-struct AudioPipelineInfo {
+/// Vision progress counters used to distinguish a real capture failure from
+/// expected no-write cycles (content dedup on a static screen).
+#[derive(Debug, Deserialize, Default, Clone, Copy, PartialEq, Eq)]
+struct VisionPipelineInfo {
+ #[serde(default)]
+ frames_db_written: u64,
+ #[serde(default)]
+ frames_dropped_timeout: u64,
#[serde(default)]
- uptime_secs: f64,
+ frames_dropped_error: u64,
#[serde(default)]
- chunks_sent: u64,
+ dedup_skips: u64,
#[serde(default)]
- transcription_paused: Option<bool>,
+ frames_corrupt_black: u64,
#[serde(default)]
- meeting_detected: Option<bool>,
+ frames_corrupt_green: u64,
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+struct VisionProgressTracker {
+ previous: Option<VisionPipelineInfo>,
+ unrecovered_failure: bool,
+}
+
+impl VisionProgressTracker {
+ /// Observe one cumulative metrics sample and report whether a capture
+ /// failure has occurred without any later successful/dedup cycle.
+ fn observe(&mut self, current: Option<&VisionPipelineInfo>) -> bool {
+ let Some(current) = current.copied() else {
+ self.previous = None;
+ self.unrecovered_failure = false;
+ return false;
+ };
+
+ let Some(previous) = self.previous.replace(current) else {
+ // The first sample is a baseline. Historical counters must never
+ // become a fresh incident when the app starts polling mid-session.
+ return self.unrecovered_failure;
+ };
+
+ let counters_reset = current.frames_db_written < previous.frames_db_written
+ || current.frames_dropped_timeout < previous.frames_dropped_timeout
+ || current.frames_dropped_error < previous.frames_dropped_error
+ || current.dedup_skips < previous.dedup_skips
+ || current.frames_corrupt_black < previous.frames_corrupt_black
+ || current.frames_corrupt_green < previous.frames_corrupt_green;
+ if counters_reset {
+ // New engine generation after a restart. Re-baseline rather than
+ // comparing unrelated counter epochs.
+ self.unrecovered_failure = false;
+ return false;
+ }
+
+ let successful_cycle = current.frames_db_written > previous.frames_db_written
+ || current.dedup_skips > previous.dedup_skips;
+ let failed_cycle = current.frames_dropped_timeout > previous.frames_dropped_timeout
+ || current.frames_dropped_error > previous.frames_dropped_error
+ || current.frames_corrupt_green > previous.frames_corrupt_green;
+
+ if successful_cycle {
+ self.unrecovered_failure = false;
+ }
+ // Failure wins when different monitors report success and failure in
+ // the same aggregate tick. A later successful cycle clears the latch.
+ if failed_cycle {
+ self.unrecovered_failure = true;
+ }
+
+ self.unrecovered_failure
+ }
}
#[derive(Debug, Deserialize)]
@@ -377,15 +435,29 @@ struct HealthCheckResponse {
/// Monitor names from the server
#[serde(default)]
monitors: Option<Vec<String>>,
- /// Audio pipeline metrics for stall detection
+ /// Vision pipeline counters for capture-failure confirmation.
#[serde(default)]
- audio_pipeline: Option<AudioPipelineInfo>,
- /// Vision capture alive but DB writes stopped (pool exhaustion)
+ pipeline: Option<VisionPipelineInfo>,
+ /// Legacy vision no-write heuristic. This can also be true while a static
+ /// screen is being deduplicated, so it requires corroborating failure
+ /// progress before it can drive user-visible recording health.
#[serde(default)]
vision_db_write_stalled: bool,
- /// Audio devices active but DB writes stopped (pool exhaustion)
+ /// Transcription reconciliation is not making progress. Raw audio has
+ /// already been persisted; this must never mean "recording is broken".
#[serde(default)]
audio_db_write_stalled: bool,
+ /// The shared SQLite write queue has observed a failed write path.
+ #[serde(default)]
+ write_queue_degraded: bool,
+ /// Current consecutive fatal write batches. The queue declares itself
+ /// degraded after three; older engines omit this and deserialize to zero.
+ #[serde(default)]
+ write_queue_consecutive_fatal: u64,
+ /// Current consecutive batches that exhausted the SQLite contention retry
+ /// budget. One isolated failed batch must not flash the recording overlay.
+ #[serde(default)]
+ write_queue_consecutive_contention: u64,
/// DRM streaming content detected — capture should be fully stopped
#[serde(default)]
drm_content_paused: bool,
@@ -397,6 +469,135 @@ struct HealthCheckResponse {
schedule_paused: bool,
}
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+struct CaptureFailureSignals {
+ audio: bool,
+ vision: bool,
+ persistence: bool,
+}
+
+fn capture_failure_signals(
+ health: &HealthCheckResponse,
+ vision_progress: &mut VisionProgressTracker,
+) -> CaptureFailureSignals {
+ let audio = matches!(
+ health.audio_status.as_deref(),
+ Some("stale")
+ | Some("not_started")
+ | Some("active_no_data")
+ | Some("permission_denied")
+ | Some("capture_stalled")
+ | Some("error")
+ );
+ let vision_status_failed = matches!(
+ health.frame_status.as_deref(),
+ Some("stale")
+ | Some("not_started")
+ | Some("permission_denied")
+ | Some("capture_stalled")
+ | Some("error")
+ );
+ let unrecovered_vision_failure = vision_progress.observe(health.pipeline.as_ref());
+
+ CaptureFailureSignals {
+ audio,
+ vision: vision_status_failed
+ || (health.vision_db_write_stalled && unrecovered_vision_failure),
+ persistence: health.write_queue_degraded
+ && (health.write_queue_consecutive_fatal >= WRITE_QUEUE_FAILURE_THRESHOLD
+ || health.write_queue_consecutive_contention >= WRITE_QUEUE_FAILURE_THRESHOLD),
+ }
+}
+
+/// Recovery requires positive proof that both enabled raw-capture paths are
+/// healthy. A merely parseable `/health` response is not enough: the endpoint's
+/// timeout fallback reports `unknown`, and a first stale tick has not yet
+/// reached the user-visible failure debounce.
+fn health_confirms_recording(health: &HealthCheckResponse) -> bool {
+ let vision_healthy = matches!(health.frame_status.as_deref(), Some("ok") | Some("disabled"));
+ let audio_healthy = matches!(
+ health.audio_status.as_deref(),
+ Some("ok") | Some("disabled") | Some("no_input_device")
+ );
+
+ vision_healthy && audio_healthy && !health.write_queue_degraded
+}
+
+/// Advance one debounced capture-failure counter. On a healthy tick, return the
+/// length of the run that just recovered so callers can log it exactly once.
+fn advance_stall_counter(counter: &mut u32, failed: bool) -> Option<u32> {
+ if failed {
+ *counter = counter.saturating_add(1);
+ None
+ } else {
+ let recovered_after = (*counter > 0).then_some(*counter);
+ *counter = 0;
+ recovered_after
+ }
+}
+
+fn stall_confirmed(counter: u32) -> bool {
+ counter >= CAPTURE_STALL_THRESHOLD
+}
+
+fn should_track_capture_stalls(
+ status: RecordingStatus,
+ elapsed_since_start: Duration,
+ in_restart_grace: bool,
+) -> bool {
+ status == RecordingStatus::Recording
+ && elapsed_since_start > NOTIFICATION_COOLDOWN
+ && !in_restart_grace
+}
+
+fn should_notify_capture_stall(counter: u32, enabled: bool, cooldown_ok: bool) -> bool {
+ enabled && cooldown_ok && counter == CAPTURE_STALL_THRESHOLD
+}
+
+fn notification_cooldown_ok(last_notification: Option<Instant>, now: Instant) -> bool {
+ last_notification
+ .map(|last| now.saturating_duration_since(last) >= NOTIFICATION_COOLDOWN)
+ .unwrap_or(true)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct OverlayTickDecision {
+ broken: bool,
+ healthy: bool,
+}
+
+#[allow(clippy::too_many_arguments)]
+fn overlay_tick_decision(
+ status: RecordingStatus,
+ health_response_received: bool,
+ health_confirms_recording: bool,
+ capture_intended: bool,
+ start_in_progress: bool,
+ ever_connected: bool,
+ recently_woke: bool,
+ intentionally_paused: bool,
+ failures: CaptureFailureSignals,
+ simulated_break: bool,
+) -> OverlayTickDecision {
+ let engine_down = capture_intended
+ && !start_in_progress
+ && !recently_woke
+ && (status == RecordingStatus::Error
+ || (status == RecordingStatus::Stopped && ever_connected));
+ let confirmed_failure = failures.audio || failures.vision || failures.persistence;
+ let failure_suppressed = start_in_progress || recently_woke || intentionally_paused;
+ let broken = simulated_break || (!failure_suppressed && (engine_down || confirmed_failure));
+ let healthy = !intentionally_paused
+ && !recently_woke
+ && health_response_received
+ && health_confirms_recording
+ && status == RecordingStatus::Recording
+ && !confirmed_failure
+ && !simulated_break;
+
+ OverlayTickDecision { broken, healthy }
+}
+
/// Decide recording status based on health check result and time since startup.
///
/// During the grace period, connection errors are treated as "starting up"
@@ -861,6 +1062,11 @@ fn parse_devices_from_health(health_result: &Result<HealthCheckResponse>) -> Vec
/// At 1-second polling, 90 = 90 seconds of sustained failure.
const CAPTURE_STALL_THRESHOLD: u32 = 90;
+/// A single failed shared-queue batch is data loss, but not proof that ongoing
+/// recording is broken. Match the queue's three-batch fatal threshold and use
+/// the same threshold for repeated lock-contention failures.
+const WRITE_QUEUE_FAILURE_THRESHOLD: u64 = 3;
+
/// Suppress re-notification for this long after showing one.
const NOTIFICATION_COOLDOWN: Duration = Duration::from_secs(300); // 5 minutes
@@ -879,12 +1085,13 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {
// Capture stall detection state
let mut consecutive_audio_stall: u32 = 0;
let mut consecutive_vision_stall: u32 = 0;
+ let mut vision_progress = VisionProgressTracker::default();
let mut last_audio_notification: Option<Instant> = None;
let mut last_vision_notification: Option<Instant> = None;
let mut wake_reset_done = false;
// Grace period after ANY restart (manual, notification-triggered, or
- // settings-triggered): suppress stall detection for 120s, giving the
- // new pipeline time to load models and produce its first DB write.
+ // settings-triggered): suppress stall detection for five minutes, giving
+ // the new pipeline time to load models and produce its first DB write.
let mut last_restart_triggered: Option<Instant> = None;
let mut last_port_conflict_notified: Option<Instant> = None;
// Track last known spawn epoch to detect user-initiated restarts
@@ -1274,67 +1481,50 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {
let in_restart_grace = last_restart_triggered
.map(|t| t.elapsed() < NOTIFICATION_COOLDOWN)
.unwrap_or(false);
- if status == RecordingStatus::Recording
- && start_time.elapsed() > NOTIFICATION_COOLDOWN
- && !in_restart_grace
- {
+ let failure_signals = match &health_result {
+ Ok(health) => capture_failure_signals(health, &mut vision_progress),
+ Err(_) => {
+ vision_progress.observe(None);
+ CaptureFailureSignals::default()
+ }
+ };
+ let capture_health_confirmed = match &health_result {
+ Ok(health) => health_confirms_recording(health),
+ Err(_) => false,
+ };
+ if should_track_capture_stalls(status, start_time.elapsed(), in_restart_grace) {
if let Ok(ref health) = health_result {
- let audio_bad = matches!(
- health.audio_status.as_deref(),
- Some("stale") | Some("not_started")
- );
- let vision_bad = matches!(
- health.frame_status.as_deref(),
- Some("stale") | Some("not_started")
- );
-
- // Skip stall detection if transcription is intentionally paused or in a meeting
- let audio_excused = health
- .audio_pipeline
- .as_ref()
- .map(|ap| {
- ap.transcription_paused.unwrap_or(false)
- || ap.meeting_detected.unwrap_or(false)
- })
- .unwrap_or(false);
-
- // Audio stall tracking:
- // - audio_bad (capture stale/not_started): always counts
- // - audio_db_write_stalled: only counts as a stall signal.
- // Change #1 (engine side) ensures this flag only fires after
- // at least one successful DB write, so silent environments
- // (last_db_write_ts == 0) won't trigger false positives.
- let audio_db_stalled = health.audio_db_write_stalled;
- if (audio_bad || audio_db_stalled) && !audio_excused {
- consecutive_audio_stall = consecutive_audio_stall.saturating_add(1);
- } else {
- if consecutive_audio_stall >= CAPTURE_STALL_THRESHOLD {
+ // Only raw capture health drives the recording overlay.
+ // `audio_db_write_stalled` is a transcription-reconciliation
+ // signal: the audio files are already safe on disk, so a
+ // post-meeting catch-up must not claim recording stopped.
+ if let Some(recovered_after) =
+ advance_stall_counter(&mut consecutive_audio_stall, failure_signals.audio)
+ {
+ if stall_confirmed(recovered_after) {
info!(
"audio capture recovered after {} stale checks",
- consecutive_audio_stall
+ recovered_after
);
}
- consecutive_audio_stall = 0;
}
- // Vision stall tracking — also trigger on DB write stalls
- // (capture loop alive but pool exhaustion blocking writes).
- // A DRM pause stops all vision monitors on purpose, which
- // reads as frame_status "stale" — excuse it like audio
- // excuses meetings, or every long Netflix session becomes
- // a fake incident.
+ // Vision no-write heuristics need failure-counter
+ // corroboration: a healthy static screen also produces no
+ // new unique frame while dedup keeps capture alive.
+ // A DRM pause remains an intentional stop, never a stall.
let vision_excused = health.drm_content_paused;
let vision_db_stalled = health.vision_db_write_stalled;
- if (vision_bad || vision_db_stalled) && !vision_excused {
- consecutive_vision_stall = consecutive_vision_stall.saturating_add(1);
- } else {
- if consecutive_vision_stall >= CAPTURE_STALL_THRESHOLD {
+ if let Some(recovered_after) = advance_stall_counter(
+ &mut consecutive_vision_stall,
+ failure_signals.vision && !vision_excused,
+ ) {
+ if stall_confirmed(recovered_after) {
info!(
"vision capture recovered after {} stale checks",
- consecutive_vision_stall
+ recovered_after
);
}
- consecutive_vision_stall = 0;
}
// After wake from sleep, reset stall counters and notification
@@ -1361,40 +1551,39 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {
.unwrap_or(false);
let now_instant = Instant::now();
- if consecutive_audio_stall == CAPTURE_STALL_THRESHOLD && notifications_enabled {
- let cooldown_ok = last_audio_notification
- .map(|t| now_instant.duration_since(t) >= NOTIFICATION_COOLDOWN)
- .unwrap_or(true);
- if cooldown_ok {
- warn!(
- "audio capture stalled for {}s, showing restart notification",
- CAPTURE_STALL_THRESHOLD
- );
- last_audio_notification = Some(now_instant);
- last_restart_triggered = Some(now_instant);
- let _ = show_capture_stall_notification(&app, "audio").await;
- }
+ let audio_cooldown_ok =
+ notification_cooldown_ok(last_audio_notification, now_instant);
+ if should_notify_capture_stall(
+ consecutive_audio_stall,
+ notifications_enabled,
+ audio_cooldown_ok,
+ ) {
+ warn!(
+ "audio capture stalled for {}s, showing restart notification",
+ CAPTURE_STALL_THRESHOLD
+ );
+ last_audio_notification = Some(now_instant);
+ let _ = show_capture_stall_notification(&app, "audio").await;
}
- if consecutive_vision_stall == CAPTURE_STALL_THRESHOLD && notifications_enabled
- {
- let cooldown_ok = last_vision_notification
- .map(|t| now_instant.duration_since(t) >= NOTIFICATION_COOLDOWN)
- .unwrap_or(true);
- if cooldown_ok {
- let reason = if vision_db_stalled {
- "db write stall"
- } else {
- "capture stall"
- };
- warn!(
- "vision {} for {}s, showing restart notification",
- reason, CAPTURE_STALL_THRESHOLD
- );
- last_vision_notification = Some(now_instant);
- last_restart_triggered = Some(now_instant);
- let _ = show_capture_stall_notification(&app, "screen").await;
- }
+ let vision_cooldown_ok =
+ notification_cooldown_ok(last_vision_notification, now_instant);
+ if should_notify_capture_stall(
+ consecutive_vision_stall,
+ notifications_enabled,
+ vision_cooldown_ok,
+ ) {
+ let reason = if vision_db_stalled {
+ "db write stall"
+ } else {
+ "capture stall"
+ };
+ warn!(
+ "vision {} for {}s, showing restart notification",
+ reason, CAPTURE_STALL_THRESHOLD
+ );
+ last_vision_notification = Some(now_instant);
+ let _ = show_capture_stall_notification(&app, "screen").await;
}
}
} else {
@@ -1416,23 +1605,33 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {
// the server was ever up. RecordingStatus::Error (a spawn that
// definitively failed: TCC denied, port conflict) alerts even on
// first boot.
- let stall_active = consecutive_audio_stall >= CAPTURE_STALL_THRESHOLD
- || consecutive_vision_stall >= CAPTURE_STALL_THRESHOLD;
- let engine_down = capture_intended
- && !start_in_progress
- && !screenpipe_engine::sleep_monitor::recently_woke_from_sleep()
- && (status == RecordingStatus::Error
- || (status == RecordingStatus::Stopped && ever_connected));
let sim_break = crate::overlay_health::simulated_break_active();
- let overlay_broken = engine_down || stall_active || sim_break;
+ let recently_woke = screenpipe_engine::sleep_monitor::recently_woke_from_sleep();
+ let intentionally_paused = !capture_intended
+ || schedule_paused
+ || matches!(&health_result, Ok(health) if health.drm_content_paused)
+ || screenpipe_config::should_pause_audio_for_lock()
+ || crate::engine_events::is_power_capture_paused();
+ let decision = overlay_tick_decision(
+ status,
+ health_result.is_ok(),
+ capture_health_confirmed,
+ capture_intended,
+ start_in_progress,
+ ever_connected,
+ recently_woke,
+ intentionally_paused,
+ CaptureFailureSignals {
+ audio: stall_confirmed(consecutive_audio_stall),
+ vision: stall_confirmed(consecutive_vision_stall),
+ persistence: failure_signals.persistence,
+ },
+ sim_break,
+ );
// Recovery needs a real successful health response this tick:
// decide_status holds "Recording" through up to 30 failed checks
// (anti-flicker), which must not read as "recovered" mid-restart.
- let overlay_healthy = health_result.is_ok()
- && status == RecordingStatus::Recording
- && !stall_active
- && !sim_break;
- crate::overlay_health::on_tick(&app, overlay_broken, overlay_healthy).await;
+ crate::overlay_health::on_tick(&app, decision.broken, decision.healthy).await;
}
});
@@ -1553,9 +1752,12 @@ mod tests {
verbose_instructions: None,
device_status_details: None,
monitors: None,
- audio_pipeline: None,
+ pipeline: None,
vision_db_write_stalled: false,
audio_db_write_stalled: false,
+ write_queue_degraded: false,
+ write_queue_consecutive_fatal: 0,
+ write_queue_consecutive_contention: 0,
drm_content_paused: false,
schedule_paused: false,
})
@@ -1575,9 +1777,12 @@ mod tests {
verbose_instructions: None,
device_status_details: None,
monitors: None,
- audio_pipeline: None,
+ pipeline: None,
vision_db_write_stalled: false,
audio_db_write_stalled: false,
+ write_queue_degraded: false,
+ write_queue_consecutive_fatal: 0,
+ write_queue_consecutive_contention: 0,
drm_content_paused: false,
schedule_paused: false,
})
@@ -1587,6 +1792,683 @@ mod tests {
Err(anyhow::anyhow!("connection refused"))
}
+ fn healthy_health() -> HealthCheckResponse {
+ make_healthy_response().expect("healthy fixture")
+ }
+
+ #[test]
+ fn health_json_keeps_new_overlay_fields_backward_compatible() {
+ let legacy: HealthCheckResponse = serde_json::from_value(serde_json::json!({
+ "status": "healthy",
+ "frame_status": "ok",
+ "audio_status": "ok"
+ }))
+ .expect("older health payload should still deserialize");
+ assert_eq!(legacy.write_queue_consecutive_fatal, 0);
+ assert_eq!(legacy.write_queue_consecutive_contention, 0);
+ assert!(health_confirms_recording(&legacy));
+
+ let current: HealthCheckResponse = serde_json::from_value(serde_json::json!({
+ "status": "degraded",
+ "frame_status": "ok",
+ "audio_status": "ok",
+ "write_queue_degraded": true,
+ "write_queue_consecutive_fatal": 3,
+ "write_queue_consecutive_contention": 2,
+ "pipeline": {
+ "frames_db_written": 10,
+ "frames_dropped_timeout": 1,
+ "frames_dropped_error": 2,
+ "dedup_skips": 20,
+ "frames_corrupt_black": 3,
+ "frames_corrupt_green": 4
+ }
+ }))
+ .expect("current health payload should deserialize");
+ assert_eq!(current.write_queue_consecutive_fatal, 3);
+ assert_eq!(current.write_queue_consecutive_contention, 2);
+ assert_eq!(
+ current.pipeline,
+ Some(VisionPipelineInfo {
+ frames_db_written: 10,
+ frames_dropped_timeout: 1,
+ frames_dropped_error: 2,
+ dedup_skips: 20,
+ frames_corrupt_black: 3,
+ frames_corrupt_green: 4,
+ })
+ );
+ assert!(!health_confirms_recording(&current));
+ }
+
+ #[test]
+ fn stall_counter_has_exact_threshold_recovery_and_saturation_semantics() {
+ let mut counter = 0;
+ for expected in 1..CAPTURE_STALL_THRESHOLD {
+ assert_eq!(advance_stall_counter(&mut counter, true), None);
+ assert_eq!(counter, expected);
+ assert!(!stall_confirmed(counter));
+ }
+
+ assert_eq!(advance_stall_counter(&mut counter, true), None);
+ assert_eq!(counter, CAPTURE_STALL_THRESHOLD);
+ assert!(stall_confirmed(counter));
+ assert_eq!(advance_stall_counter(&mut counter, true), None);
+ assert_eq!(counter, CAPTURE_STALL_THRESHOLD + 1);
+ assert!(stall_confirmed(counter));
+ assert_eq!(
+ advance_stall_counter(&mut counter, false),
+ Some(CAPTURE_STALL_THRESHOLD + 1)
+ );
+ assert_eq!(counter, 0);
+ assert_eq!(advance_stall_counter(&mut counter, false), None);
+
+ counter = u32::MAX;
+ assert_eq!(advance_stall_counter(&mut counter, true), None);
+ assert_eq!(counter, u32::MAX, "a long outage must saturate, not wrap");
+ }
+
+ #[test]
+ fn stall_tracking_startup_and_restart_grace_boundaries_are_exact() {
+ for status in [
+ RecordingStatus::Starting,
+ RecordingStatus::Paused,
+ RecordingStatus::ScheduledPause,
+ RecordingStatus::Stopped,
+ RecordingStatus::Error,
+ ] {
+ assert!(!should_track_capture_stalls(
+ status,
+ NOTIFICATION_COOLDOWN + Duration::from_secs(1),
+ false,
+ ));
+ }
+
+ assert!(!should_track_capture_stalls(
+ RecordingStatus::Recording,
+ NOTIFICATION_COOLDOWN - Duration::from_millis(1),
+ false,
+ ));
+ assert!(!should_track_capture_stalls(
+ RecordingStatus::Recording,
+ NOTIFICATION_COOLDOWN,
+ false,
+ ));
+ assert!(should_track_capture_stalls(
+ RecordingStatus::Recording,
+ NOTIFICATION_COOLDOWN + Duration::from_millis(1),
+ false,
+ ));
+ assert!(!should_track_capture_stalls(
+ RecordingStatus::Recording,
+ NOTIFICATION_COOLDOWN + Duration::from_secs(1),
+ true,
+ ));
+ }
+
+ #[test]
+ fn notification_fires_once_at_threshold_and_respects_exact_cooldown_boundary() {
+ for counter in [
+ 0,
+ CAPTURE_STALL_THRESHOLD - 1,
+ CAPTURE_STALL_THRESHOLD + 1,
+ u32::MAX,
+ ] {
+ for enabled in [false, true] {
+ for cooldown_ok in [false, true] {
+ assert!(!should_notify_capture_stall(
+ counter,
+ enabled,
+ cooldown_ok,
+ ));
+ }
+ }
+ }
+ assert!(!should_notify_capture_stall(
+ CAPTURE_STALL_THRESHOLD,
+ false,
+ true,
+ ));
+ assert!(!should_notify_capture_stall(
+ CAPTURE_STALL_THRESHOLD,
+ true,
+ false,
+ ));
+ assert!(should_notify_capture_stall(
+ CAPTURE_STALL_THRESHOLD,
+ true,
+ true,
+ ));
+
+ let now = Instant::now();
+ assert!(notification_cooldown_ok(None, now));
+ assert!(!notification_cooldown_ok(
+ Some(now - NOTIFICATION_COOLDOWN + Duration::from_millis(1)),
+ now,
+ ));
+ assert!(notification_cooldown_ok(
+ Some(now - NOTIFICATION_COOLDOWN),
+ now,
+ ));
+ assert!(notification_cooldown_ok(
+ Some(now - NOTIFICATION_COOLDOWN - Duration::from_millis(1)),
+ now,
+ ));
+ assert!(
+ !notification_cooldown_ok(Some(now + Duration::from_secs(1)), now),
+ "a clock anomaly must fail closed instead of panicking or re-notifying"
+ );
+ }
+
+ #[test]
+ fn intermittent_failures_never_accumulate_into_a_false_incident() {
+ let mut counter = 0;
+ for _ in 0..CAPTURE_STALL_THRESHOLD * 4 {
+ advance_stall_counter(&mut counter, true);
+ assert_eq!(counter, 1);
+ assert!(!stall_confirmed(counter));
+ assert_eq!(advance_stall_counter(&mut counter, false), Some(1));
+ assert_eq!(counter, 0);
+ }
+ }
+
+ #[test]
+ fn temporal_overlay_boundary_is_quiet_at_89_broken_at_90_then_recovers() {
+ let mut counter = 0;
+ for _ in 0..CAPTURE_STALL_THRESHOLD - 1 {
+ advance_stall_counter(&mut counter, true);
+ }
+ let before_threshold = overlay_tick_decision(
+ RecordingStatus::Recording,
+ true,
+ false,
+ true,
+ false,
+ true,
+ false,
+ false,
+ CaptureFailureSignals {
+ audio: stall_confirmed(counter),
+ ..CaptureFailureSignals::default()
+ },
+ false,
+ );
+ assert_eq!(
+ before_threshold,
+ OverlayTickDecision {
+ broken: false,
+ healthy: false,
+ },
+ "a stale raw status blocks green recovery but stays quiet before debounce"
+ );
+
+ advance_stall_counter(&mut counter, true);
+ let at_threshold = overlay_tick_decision(
+ RecordingStatus::Recording,
+ true,
+ false,
+ true,
+ false,
+ true,
+ false,
+ false,
+ CaptureFailureSignals {
+ audio: stall_confirmed(counter),
+ ..CaptureFailureSignals::default()
+ },
+ false,
+ );
+ assert!(at_threshold.broken);
+ assert!(!at_threshold.healthy);
+
+ assert_eq!(
+ advance_stall_counter(&mut counter, false),
+ Some(CAPTURE_STALL_THRESHOLD)
+ );
+ let recovered = overlay_tick_decision(
+ RecordingStatus::Recording,
+ true,
+ true,
+ true,
+ false,
+ true,
+ false,
+ false,
+ CaptureFailureSignals::default(),
+ false,
+ );
+ assert_eq!(
+ recovered,
+ OverlayTickDecision {
+ broken: false,
+ healthy: true,
+ }
+ );
+ }
+
+ fn vision_sample(
+ frames_db_written: u64,
+ dedup_skips: u64,
+ frames_dropped_timeout: u64,
+ frames_dropped_error: u64,
+ ) -> VisionPipelineInfo {
+ VisionPipelineInfo {
+ frames_db_written,
+ frames_dropped_timeout,
+ frames_dropped_error,
+ dedup_skips,
+ frames_corrupt_black: 0,
+ frames_corrupt_green: 0,
+ }
+ }
+
+ // ==================== recording-overlay signal tests ====================
+
+ #[test]
+ fn transcription_backlog_is_not_a_recording_failure() {
+ let mut health = healthy_health();
+ health.frame_status = Some("ok".to_string());
+ health.audio_status = Some("ok".to_string());
+ health.audio_db_write_stalled = true;
+
+ let signals =
+ capture_failure_signals(&health, &mut VisionProgressTracker::default());
+
+ assert_eq!(signals, CaptureFailureSignals::default());
+ }
+
+ #[test]
+ fn static_screen_dedup_does_not_confirm_a_vision_failure() {
+ let mut tracker = VisionProgressTracker::default();
+ let mut health = healthy_health();
+ health.frame_status = Some("ok".to_string());
+ health.audio_status = Some("ok".to_string());
+ health.vision_db_write_stalled = true;
+ health.pipeline = Some(vision_sample(10, 20, 0, 0));
+ assert!(!capture_failure_signals(&health, &mut tracker).vision);
+
+ health.pipeline = Some(vision_sample(10, 21, 0, 0));
+ assert!(
+ !capture_failure_signals(&health, &mut tracker).vision,
+ "a dedup-skipped static frame proves the capture loop is healthy"
+ );
+ }
+
+ #[test]
+ fn dropped_vision_frame_confirms_stall_until_later_success() {
+ let mut tracker = VisionProgressTracker::default();
+ let mut health = healthy_health();
+ health.frame_status = Some("ok".to_string());
+ health.audio_status = Some("ok".to_string());
+ health.vision_db_write_stalled = true;
+ health.pipeline = Some(vision_sample(10, 20, 0, 0));
+ assert!(!capture_failure_signals(&health, &mut tracker).vision);
+
+ health.pipeline = Some(vision_sample(10, 20, 0, 1));
+ assert!(capture_failure_signals(&health, &mut tracker).vision);
+
+ // The error latch survives quiet/backoff ticks; otherwise a capture
+ // loop that sleeps between repeated permission failures never reaches
+ // the 90-second confirmation threshold.
+ assert!(capture_failure_signals(&health, &mut tracker).vision);
+
+ health.pipeline = Some(vision_sample(11, 20, 0, 1));
+ assert!(
+ !capture_failure_signals(&health, &mut tracker).vision,
+ "a later persisted frame is recovery proof"
+ );
+ }
+
+ #[test]
+ fn corrupt_vision_frames_never_masquerade_as_recovery() {
+ let mut tracker = VisionProgressTracker::default();
+ let mut health = healthy_health();
+ health.frame_status = Some("ok".to_string());
+ health.audio_status = Some("ok".to_string());
+ health.vision_db_write_stalled = true;
+ health.pipeline = Some(vision_sample(10, 20, 0, 0));
+ assert!(!capture_failure_signals(&health, &mut tracker).vision);
+
+ let mut green = vision_sample(10, 20, 0, 0);
+ green.frames_corrupt_green = 1;
+ health.pipeline = Some(green);
+ assert!(
+ capture_failure_signals(&health, &mut tracker).vision,
+ "green decode garbage is capture failure evidence, not recovery"
+ );
+
+ let mut black = green;
+ black.frames_corrupt_black = 1;
+ health.pipeline = Some(black);
+ assert!(
+ capture_failure_signals(&health, &mut tracker).vision,
+ "black/excluded frames are neutral and must not clear an existing failure"
+ );
+
+ black.dedup_skips += 1;
+ health.pipeline = Some(black);
+ assert!(
+ !capture_failure_signals(&health, &mut tracker).vision,
+ "a later valid dedup cycle proves the capture loop recovered"
+ );
+ }
+
+ #[test]
+ fn vision_counter_reset_rebaselines_after_engine_restart() {
+ let mut tracker = VisionProgressTracker::default();
+ let mut health = healthy_health();
+ health.frame_status = Some("ok".to_string());
+ health.audio_status = Some("ok".to_string());
+ health.vision_db_write_stalled = true;
+ health.pipeline = Some(vision_sample(50, 100, 2, 3));
+ assert!(!capture_failure_signals(&health, &mut tracker).vision);
+
+ health.pipeline = Some(vision_sample(0, 0, 0, 0));
+ assert!(!capture_failure_signals(&health, &mut tracker).vision);
+ }
+
+ #[test]
+ fn explicit_audio_capture_failures_are_detected() {
+ for status in [
+ "stale",
+ "not_started",
+ "active_no_data",
+ "permission_denied",
+ "capture_stalled",
+ "error",
+ ] {
+ let mut health = healthy_health();
+ health.audio_status = Some(status.to_string());
+ assert!(
+ capture_failure_signals(&health, &mut VisionProgressTracker::default()).audio,
+ "{status} must be treated as a capture failure"
+ );
+ }
+
+ for status in ["ok", "disabled", "no_input_device"] {
+ let mut health = healthy_health();
+ health.audio_status = Some(status.to_string());
+ assert!(
+ !capture_failure_signals(&health, &mut VisionProgressTracker::default()).audio,
+ "{status} must not be treated as a capture failure"
+ );
+ }
+ }
+
+ #[test]
+ fn explicit_vision_capture_failures_are_detected() {
+ for status in [
+ "stale",
+ "not_started",
+ "permission_denied",
+ "capture_stalled",
+ "error",
+ ] {
+ let mut health = healthy_health();
+ health.frame_status = Some(status.to_string());
+ assert!(
+ capture_failure_signals(&health, &mut VisionProgressTracker::default()).vision,
+ "{status} must be treated as a capture failure"
+ );
+ }
+
+ for status in ["ok", "disabled"] {
+ let mut health = healthy_health();
+ health.frame_status = Some(status.to_string());
+ assert!(
+ !capture_failure_signals(&health, &mut VisionProgressTracker::default()).vision,
+ "{status} must not be treated as a capture failure"
+ );
+ }
+ }
+
+ #[test]
+ fn repeated_write_queue_failure_is_user_visible_without_flashing_on_one_batch() {
+ let mut health = healthy_health();
+ health.write_queue_degraded = true;
+ health.write_queue_consecutive_contention = 1;
+ assert!(
+ !capture_failure_signals(&health, &mut VisionProgressTracker::default()).persistence,
+ "one failed shared-queue batch is not proof ongoing recording is broken"
+ );
+
+ health.write_queue_consecutive_contention = WRITE_QUEUE_FAILURE_THRESHOLD;
+ assert!(
+ capture_failure_signals(&health, &mut VisionProgressTracker::default()).persistence
+ );
+
+ health.write_queue_consecutive_contention = 0;
+ health.write_queue_consecutive_fatal = WRITE_QUEUE_FAILURE_THRESHOLD;
+ assert!(
+ capture_failure_signals(&health, &mut VisionProgressTracker::default()).persistence
+ );
+ }
+
+ #[test]
+ fn only_explicit_good_capture_statuses_confirm_recovery() {
+ for frame_status in ["ok", "disabled"] {
+ for audio_status in ["ok", "disabled", "no_input_device"] {
+ let mut health = healthy_health();
+ health.frame_status = Some(frame_status.to_string());
+ health.audio_status = Some(audio_status.to_string());
+ assert!(health_confirms_recording(&health));
+ }
+ }
+
+ for (frame_status, audio_status) in [
+ ("unknown", "ok"),
+ ("stale", "ok"),
+ ("ok", "unknown"),
+ ("ok", "active_no_data"),
+ ] {
+ let mut health = healthy_health();
+ health.frame_status = Some(frame_status.to_string());
+ health.audio_status = Some(audio_status.to_string());
+ assert!(
+ !health_confirms_recording(&health),
+ "frame={frame_status}, audio={audio_status} must not produce a green recovery"
+ );
+ }
+
+ let mut degraded = healthy_health();
+ degraded.frame_status = Some("ok".to_string());
+ degraded.audio_status = Some("ok".to_string());
+ degraded.write_queue_degraded = true;
+ assert!(!health_confirms_recording(&degraded));
+ }
+
+ #[test]
+ fn overlay_only_recovers_during_real_healthy_recording() {
+ let healthy = overlay_tick_decision(
+ RecordingStatus::Recording,
+ true,
+ true,
+ true,
+ false,
+ true,
+ false,
+ false,
+ CaptureFailureSignals::default(),
+ false,
+ );
+ assert!(!healthy.broken);
+ assert!(healthy.healthy);
+
+ let paused = overlay_tick_decision(
+ RecordingStatus::Recording,
+ true,
+ true,
+ false,
+ false,
+ true,
+ false,
+ true,
+ CaptureFailureSignals {
+ vision: true,
+ ..CaptureFailureSignals::default()
+ },
+ false,
+ );
+ assert!(!paused.broken);
+ assert!(
+ !paused.healthy,
+ "DRM, schedule, lock, power, and user pauses must stand down quietly"
+ );
+ }
+
+ #[test]
+ fn overlay_preserves_real_failure_and_transient_guards() {
+ let failures = CaptureFailureSignals {
+ vision: true,
+ ..CaptureFailureSignals::default()
+ };
+ let broken = overlay_tick_decision(
+ RecordingStatus::Recording,
+ true,
+ true,
+ true,
+ false,
+ true,
+ false,
+ false,
+ failures,
+ false,
+ );
+ assert!(broken.broken);
+ assert!(!broken.healthy);
+
+ for (start_in_progress, recently_woke) in [(true, false), (false, true)] {
+ let guarded = overlay_tick_decision(
+ RecordingStatus::Recording,
+ true,
+ true,
+ true,
+ start_in_progress,
+ true,
+ recently_woke,
+ false,
+ failures,
+ false,
+ );
+ assert!(!guarded.broken);
+ assert!(!guarded.healthy);
+ }
+
+ let engine_down = overlay_tick_decision(
+ RecordingStatus::Stopped,
+ false,
+ false,
+ true,
+ false,
+ true,
+ false,
+ false,
+ CaptureFailureSignals::default(),
+ false,
+ );
+ assert!(engine_down.broken);
+
+ let simulated = overlay_tick_decision(
+ RecordingStatus::Paused,
+ true,
+ true,
+ false,
+ false,
+ true,
+ false,
+ true,
+ CaptureFailureSignals::default(),
+ true,
+ );
+ assert!(simulated.broken, "debug simulation must remain testable");
+ }
+
+ #[test]
+ fn overlay_decision_exhaustively_checks_12288_state_combinations() {
+ let statuses = [
+ RecordingStatus::Starting,
+ RecordingStatus::Recording,
+ RecordingStatus::Paused,
+ RecordingStatus::ScheduledPause,
+ RecordingStatus::Stopped,
+ RecordingStatus::Error,
+ ];
+ let mut checked = 0usize;
+
+ // Eleven independent booleans x six recording states. This covers
+ // 12,288 combinations of response quality, capture intent, lifecycle
+ // guards, three failure classes, and the QA simulation path.
+ for status in statuses {
+ for bits in 0u16..(1 << 11) {
+ let health_response_received = bits & (1 << 0) != 0;
+ let health_confirmed = bits & (1 << 1) != 0;
+ let capture_intended = bits & (1 << 2) != 0;
+ let start_in_progress = bits & (1 << 3) != 0;
+ let ever_connected = bits & (1 << 4) != 0;
+ let recently_woke = bits & (1 << 5) != 0;
+ let external_pause = bits & (1 << 6) != 0;
+ let intentionally_paused = !capture_intended || external_pause;
+ let failures = CaptureFailureSignals {
+ audio: bits & (1 << 7) != 0,
+ vision: bits & (1 << 8) != 0,
+ persistence: bits & (1 << 9) != 0,
+ };
+ let simulated_break = bits & (1 << 10) != 0;
+
+ let decision = overlay_tick_decision(
+ status,
+ health_response_received,
+ health_confirmed,
+ capture_intended,
+ start_in_progress,
+ ever_connected,
+ recently_woke,
+ intentionally_paused,
+ failures,
+ simulated_break,
+ );
+
+ let failure_suppressed =
+ start_in_progress || recently_woke || intentionally_paused;
+ let confirmed_failure = failures.audio || failures.vision || failures.persistence;
+ let engine_down = capture_intended
+ && !start_in_progress
+ && !recently_woke
+ && (status == RecordingStatus::Error
+ || (status == RecordingStatus::Stopped && ever_connected));
+ let expected = OverlayTickDecision {
+ broken: simulated_break
+ || (!failure_suppressed && (engine_down || confirmed_failure)),
+ healthy: !intentionally_paused
+ && !recently_woke
+ && health_response_received
+ && health_confirmed
+ && status == RecordingStatus::Recording
+ && !confirmed_failure
+ && !simulated_break,
+ };
+
+ assert_eq!(decision, expected, "status={status:?}, bits={bits:011b}");
+ assert!(
+ !(decision.broken && decision.healthy),
+ "failure and recovery must be mutually exclusive"
+ );
+ if simulated_break {
+ assert!(decision.broken && !decision.healthy);
+ }
+ if !simulated_break && failure_suppressed {
+ assert!(!decision.broken);
+ }
+ if !health_response_received || !health_confirmed {
+ assert!(!decision.healthy);
+ }
+ checked += 1;
+ }
+ }
+
+ assert_eq!(checked, 12_288);
+ }
+
// Helper: call decide_status with thresholds exceeded (no debouncing active)
// Used for tests that don't care about debouncing behavior
fn decide_no_debounce(
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/overlay_health.rs b/apps/screenpipe-app-tauri/src-tauri/src/overlay_health.rs
index f31ea8633..9a7ec2950 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/overlay_health.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/overlay_health.rs
@@ -6,7 +6,7 @@
//!
//! The 1Hz health loop (health.rs) is the single authority: it feeds
//! `on_tick` with a debounced "recording is broken" signal that already
-//! excludes deliberate stops (capture intent OFF), scheduled/DRM/meeting
+//! excludes deliberate stops (capture intent OFF), scheduled/DRM/lock/power
//! pauses, wake-from-sleep and post-restart grace. This module runs the
//! overlay-facing state machine
//!
@@ -83,6 +83,187 @@ static INNER: Mutex<Inner> = Mutex::new(Inner {
last_detail: String::new(),
});
+#[derive(Debug, PartialEq, Eq)]
+enum TickEffect {
+ None,
+ Push(OverlayHealthState, Option<String>),
+ PushAndReveal(OverlayHealthState),
+ PushAndUnreveal(OverlayHealthState),
+}
+
+fn begin_fixing(inner: &mut Inner, now: Instant) -> bool {
+ if inner.state == OverlayHealthState::Fixing {
+ return false;
+ }
+ inner.state = OverlayHealthState::Fixing;
+ inner.fixing_since = Some(now);
+ inner.fixing_seen_down = false;
+ inner.healthy_ticks = 0;
+ inner.not_broken_ticks = 0;
+ inner.last_detail.clear();
+ true
+}
+
+fn fixing_failed(inner: &mut Inner) {
+ inner.state = OverlayHealthState::Failure;
+ inner.fixing_since = None;
+ inner.last_detail.clear();
+}
+
+fn dismiss_state(inner: &mut Inner) -> bool {
+ inner.dismissed = true;
+ inner.state = OverlayHealthState::Normal;
+ inner.fixing_since = None;
+ inner.recovered_at = None;
+ inner.not_broken_ticks = 0;
+ inner.last_detail.clear();
+ let was_auto_revealed = inner.auto_revealed;
+ inner.auto_revealed = false;
+ was_auto_revealed
+}
+
+/// Pure overlay state transition. All Tauri/Swift side effects stay in
+/// `on_tick`; keeping the reducer independent lets tests drive long temporal
+/// sequences with an injected clock and boot phase.
+fn transition_tick(
+ inner: &mut Inner,
+ broken: bool,
+ healthy: bool,
+ now: Instant,
+ boot_detail: &str,
+) -> TickEffect {
+ match inner.state {
+ OverlayHealthState::Normal => {
+ if broken && !inner.dismissed {
+ inner.state = OverlayHealthState::Failure;
+ inner.not_broken_ticks = 0;
+ info!("overlay health: recording incident confirmed — showing failure state");
+ TickEffect::PushAndReveal(OverlayHealthState::Failure)
+ } else {
+ if !broken && inner.dismissed {
+ // Incident resolved while dismissed — re-arm for the next one.
+ inner.dismissed = false;
+ }
+ TickEffect::None
+ }
+ }
+ OverlayHealthState::Failure => {
+ if healthy {
+ // Recovered without our restart (auto-respawn watchdog, user
+ // fixed it elsewhere) — confirm and collapse.
+ inner.state = OverlayHealthState::Recovered;
+ inner.recovered_at = Some(now);
+ inner.not_broken_ticks = 0;
+ info!("overlay health: recording recovered");
+ TickEffect::Push(OverlayHealthState::Recovered, None)
+ } else if !broken {
+ // The incident evaporated without a healthy engine — the
+ // user deliberately stopped recording, a scheduled/DRM
+ // pause took over, etc. Nothing recovered, so no green
+ // confirmation: stand down quietly (debounced so a flap
+ // between broken-reasons doesn't flicker the pill).
+ inner.not_broken_ticks += 1;
+ if inner.not_broken_ticks >= 3 {
+ inner.state = OverlayHealthState::Normal;
+ inner.not_broken_ticks = 0;
+ inner.dismissed = false;
+ info!("overlay health: incident no longer applies — standing down");
+ if inner.auto_revealed {
+ inner.auto_revealed = false;
+ TickEffect::PushAndUnreveal(OverlayHealthState::Normal)
+ } else {
+ TickEffect::Push(OverlayHealthState::Normal, None)
+ }
+ } else {
+ TickEffect::None
+ }
+ } else {
+ inner.not_broken_ticks = 0;
+ TickEffect::None
+ }
+ }
+ OverlayHealthState::Fixing => {
+ if !healthy {
+ inner.fixing_seen_down = true;
+ }
+ // Until the restart's stop has actually taken the server down
+ // (or 15s passed — covers odd cases), healthy responses come
+ // from the OLD engine and must not confirm the recovery.
+ let counting = inner.fixing_seen_down
+ || inner
+ .fixing_since
+ .map(|t| now.saturating_duration_since(t) > Duration::from_secs(15))
+ .unwrap_or(true);
+ if healthy && counting {
+ inner.healthy_ticks += 1;
+ if inner.healthy_ticks >= FIXING_CONFIRM_TICKS {
+ inner.state = OverlayHealthState::Recovered;
+ inner.recovered_at = Some(now);
+ inner.fixing_since = None;
+ inner.last_detail.clear();
+ info!("overlay health: restart confirmed healthy");
+ TickEffect::Push(OverlayHealthState::Recovered, None)
+ } else {
+ TickEffect::None
+ }
+ } else if healthy {
+ TickEffect::None
+ } else {
+ inner.healthy_ticks = 0;
+ // Long restarts look hung behind a bare spinner — surface
+ // the engine's boot phase ("fixing — updating database...")
+ // and don't run the timeout clock during a DB migration,
+ // which legitimately takes minutes on large installs.
+ let migrating = boot_detail == "updating database";
+ let timed_out = !migrating
+ && inner
+ .fixing_since
+ .map(|t| now.saturating_duration_since(t) > FIXING_TIMEOUT)
+ .unwrap_or(true);
+ if timed_out {
+ inner.state = OverlayHealthState::Failure;
+ inner.fixing_since = None;
+ inner.last_detail.clear();
+ warn!("overlay health: restart did not recover in time — back to failure");
+ TickEffect::Push(OverlayHealthState::Failure, None)
+ } else if boot_detail != inner.last_detail {
+ inner.last_detail = boot_detail.to_string();
+ TickEffect::Push(
+ OverlayHealthState::Fixing,
+ Some(boot_detail.to_string()),
+ )
+ } else {
+ TickEffect::None
+ }
+ }
+ }
+ OverlayHealthState::Recovered => {
+ if broken {
+ // Relapse inside the confirmation window.
+ inner.state = OverlayHealthState::Failure;
+ inner.not_broken_ticks = 0;
+ TickEffect::Push(OverlayHealthState::Failure, None)
+ } else if inner
+ .recovered_at
+ .map(|t| now.saturating_duration_since(t) > RECOVERED_HOLD)
+ .unwrap_or(true)
+ {
+ inner.state = OverlayHealthState::Normal;
+ inner.recovered_at = None;
+ inner.dismissed = false;
+ if inner.auto_revealed {
+ inner.auto_revealed = false;
+ TickEffect::PushAndUnreveal(OverlayHealthState::Normal)
+ } else {
+ TickEffect::Push(OverlayHealthState::Normal, None)
+ }
+ } else {
+ TickEffect::None
+ }
+ }
+ }
+}
+
/// Current state as the wire payload: "state" or "state|detail" (detail is
/// the boot-phase label while fixing). Both surfaces split on the first '|'.
pub fn current_state_payload() -> String {
@@ -147,162 +328,33 @@ fn track(app: &tauri::AppHandle, event: &'static str) {
/// signal that confirms a recovery.
/// The two are not complements: during a restart both are false.
pub async fn on_tick(app: &tauri::AppHandle, broken: bool, healthy: bool) {
- enum Effect {
- None,
- Push(OverlayHealthState, Option<String>),
- PushAndReveal(OverlayHealthState),
- PushAndUnreveal(OverlayHealthState),
- }
-
let effect = {
let mut inner = match INNER.lock() {
Ok(i) => i,
Err(_) => return,
};
- match inner.state {
- OverlayHealthState::Normal => {
- if broken && !inner.dismissed {
- inner.state = OverlayHealthState::Failure;
- inner.not_broken_ticks = 0;
- info!("overlay health: recording incident confirmed — showing failure state");
- Effect::PushAndReveal(OverlayHealthState::Failure)
- } else {
- if !broken && inner.dismissed {
- // Incident resolved while dismissed — re-arm for the next one.
- inner.dismissed = false;
- }
- Effect::None
- }
- }
- OverlayHealthState::Failure => {
- if healthy {
- // Recovered without our restart (auto-respawn watchdog, user
- // fixed it elsewhere) — confirm and collapse.
- inner.state = OverlayHealthState::Recovered;
- inner.recovered_at = Some(Instant::now());
- inner.not_broken_ticks = 0;
- info!("overlay health: recording recovered");
- Effect::Push(OverlayHealthState::Recovered, None)
- } else if !broken {
- // The incident evaporated without a healthy engine — the
- // user deliberately stopped recording, a scheduled/DRM
- // pause took over, etc. Nothing recovered, so no green
- // confirmation: stand down quietly (debounced so a flap
- // between broken-reasons doesn't flicker the pill).
- inner.not_broken_ticks += 1;
- if inner.not_broken_ticks >= 3 {
- inner.state = OverlayHealthState::Normal;
- inner.not_broken_ticks = 0;
- inner.dismissed = false;
- info!("overlay health: incident no longer applies — standing down");
- if inner.auto_revealed {
- inner.auto_revealed = false;
- Effect::PushAndUnreveal(OverlayHealthState::Normal)
- } else {
- Effect::Push(OverlayHealthState::Normal, None)
- }
- } else {
- Effect::None
- }
- } else {
- inner.not_broken_ticks = 0;
- Effect::None
- }
- }
- OverlayHealthState::Fixing => {
- if !healthy {
- inner.fixing_seen_down = true;
- }
- // Until the restart's stop has actually taken the server down
- // (or 15s passed — covers odd cases), healthy responses come
- // from the OLD engine and must not confirm the recovery.
- let counting = inner.fixing_seen_down
- || inner
- .fixing_since
- .map(|t| t.elapsed() > Duration::from_secs(15))
- .unwrap_or(true);
- if healthy && counting {
- inner.healthy_ticks += 1;
- if inner.healthy_ticks >= FIXING_CONFIRM_TICKS {
- inner.state = OverlayHealthState::Recovered;
- inner.recovered_at = Some(Instant::now());
- inner.fixing_since = None;
- inner.last_detail.clear();
- info!("overlay health: restart confirmed healthy");
- Effect::Push(OverlayHealthState::Recovered, None)
- } else {
- Effect::None
- }
- } else if healthy {
- Effect::None
- } else {
- inner.healthy_ticks = 0;
- // Long restarts look hung behind a bare spinner — surface
- // the engine's boot phase ("fixing — updating database...")
- // and don't run the timeout clock during a DB migration,
- // which legitimately takes minutes on large installs.
- let detail = boot_phase_detail();
- let migrating = detail == "updating database";
- let timed_out = !migrating
- && inner
- .fixing_since
- .map(|t| t.elapsed() > FIXING_TIMEOUT)
- .unwrap_or(true);
- if timed_out {
- inner.state = OverlayHealthState::Failure;
- inner.fixing_since = None;
- inner.last_detail.clear();
- warn!("overlay health: restart did not recover in time — back to failure");
- Effect::Push(OverlayHealthState::Failure, None)
- } else if detail != inner.last_detail {
- inner.last_detail = detail.to_string();
- Effect::Push(OverlayHealthState::Fixing, Some(detail.to_string()))
- } else {
- Effect::None
- }
- }
- }
- OverlayHealthState::Recovered => {
- if broken {
- // Relapse inside the confirmation window.
- inner.state = OverlayHealthState::Failure;
- inner.not_broken_ticks = 0;
- Effect::Push(OverlayHealthState::Failure, None)
- } else if inner
- .recovered_at
- .map(|t| t.elapsed() > RECOVERED_HOLD)
- .unwrap_or(true)
- {
- inner.state = OverlayHealthState::Normal;
- inner.recovered_at = None;
- inner.dismissed = false;
- if inner.auto_revealed {
- inner.auto_revealed = false;
- Effect::PushAndUnreveal(OverlayHealthState::Normal)
- } else {
- Effect::Push(OverlayHealthState::Normal, None)
- }
- } else {
- Effect::None
- }
- }
- }
+ let boot_detail = if inner.state == OverlayHealthState::Fixing && !healthy {
+ boot_phase_detail()
+ } else {
+ ""
+ };
+ transition_tick(&mut inner, broken, healthy, Instant::now(), boot_detail)
};
match effect {
- Effect::None => {}
- Effect::Push(s, detail) => {
+ TickEffect::None => {}
+ TickEffect::Push(s, detail) => {
if s == OverlayHealthState::Recovered {
track(app, "recording_incident_recovered");
}
push_state(app, s, detail.as_deref());
}
- Effect::PushAndReveal(s) => {
+ TickEffect::PushAndReveal(s) => {
track(app, "recording_incident_shown");
push_state(app, s, None);
reveal_overlay_if_hidden(app).await;
}
- Effect::PushAndUnreveal(s) => {
+ TickEffect::PushAndUnreveal(s) => {
push_state(app, s, None);
// The overlay was only on screen for this incident — put it back.
let _ = crate::commands::hide_shortcut_reminder(app.clone()).await;
@@ -356,17 +408,11 @@ async fn reveal_overlay_if_hidden(app: &tauri::AppHandle) {
pub async fn restart_recording(app: tauri::AppHandle) {
{
let Ok(mut inner) = INNER.lock() else { return };
- if inner.state == OverlayHealthState::Fixing {
+ if !begin_fixing(&mut inner, Instant::now()) {
// Double-click / both surfaces racing — one restart is enough.
info!("overlay health: restart already in progress — ignoring duplicate");
return;
}
- inner.state = OverlayHealthState::Fixing;
- inner.fixing_since = Some(Instant::now());
- inner.fixing_seen_down = false;
- inner.healthy_ticks = 0;
- inner.not_broken_ticks = 0;
- inner.last_detail.clear();
}
track(&app, "recording_incident_restart_clicked");
push_state(&app, OverlayHealthState::Fixing, None);
@@ -380,9 +426,7 @@ pub async fn restart_recording(app: tauri::AppHandle) {
if let Err(e) = crate::recording::spawn_screenpipe(app.state(), app.clone(), None).await {
warn!("overlay health: spawn during restart failed: {}", e);
if let Ok(mut inner) = INNER.lock() {
- inner.state = OverlayHealthState::Failure;
- inner.fixing_since = None;
- inner.last_detail.clear();
+ fixing_failed(&mut inner);
}
push_state(&app, OverlayHealthState::Failure, None);
}
@@ -394,17 +438,7 @@ pub async fn restart_recording(app: tauri::AppHandle) {
pub async fn dismiss_incident(app: tauri::AppHandle) {
let was_auto_revealed = {
match INNER.lock() {
- Ok(mut inner) => {
- inner.dismissed = true;
- inner.state = OverlayHealthState::Normal;
- inner.fixing_since = None;
- inner.recovered_at = None;
- inner.not_broken_ticks = 0;
- inner.last_detail.clear();
- let was = inner.auto_revealed;
- inner.auto_revealed = false;
- was
- }
+ Ok(mut inner) => dismiss_state(&mut inner),
Err(_) => false,
}
};
@@ -441,3 +475,404 @@ fn clear_simulated_break() {
#[cfg(not(debug_assertions))]
fn clear_simulated_break() {}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn test_inner(state: OverlayHealthState) -> Inner {
+ Inner {
+ state,
+ dismissed: false,
+ auto_revealed: false,
+ fixing_since: None,
+ fixing_seen_down: false,
+ recovered_at: None,
+ healthy_ticks: 0,
+ not_broken_ticks: 0,
+ last_detail: String::new(),
+ }
+ }
+
+ #[test]
+ fn failure_recovers_then_collapses_only_after_hold_boundary() {
+ let start = Instant::now();
+ let mut inner = test_inner(OverlayHealthState::Normal);
+
+ assert_eq!(
+ transition_tick(&mut inner, true, false, start, ""),
+ TickEffect::PushAndReveal(OverlayHealthState::Failure)
+ );
+ inner.auto_revealed = true;
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(1),
+ "",
+ ),
+ TickEffect::Push(OverlayHealthState::Recovered, None)
+ );
+
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(1) + RECOVERED_HOLD,
+ "",
+ ),
+ TickEffect::None,
+ "the recovered pill must remain at the exact hold boundary"
+ );
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(1) + RECOVERED_HOLD + Duration::from_millis(1),
+ "",
+ ),
+ TickEffect::PushAndUnreveal(OverlayHealthState::Normal)
+ );
+ assert_eq!(inner.state, OverlayHealthState::Normal);
+ assert!(!inner.auto_revealed);
+ }
+
+ #[test]
+ fn failure_stands_down_after_three_neutral_ticks_and_broken_resets_debounce() {
+ let start = Instant::now();
+ let mut inner = test_inner(OverlayHealthState::Normal);
+ transition_tick(&mut inner, true, false, start, "");
+ inner.auto_revealed = true;
+
+ assert_eq!(
+ transition_tick(&mut inner, false, false, start, ""),
+ TickEffect::None
+ );
+ assert_eq!(
+ transition_tick(&mut inner, false, false, start, ""),
+ TickEffect::None
+ );
+ assert_eq!(inner.not_broken_ticks, 2);
+ assert_eq!(
+ transition_tick(&mut inner, true, false, start, ""),
+ TickEffect::None
+ );
+ assert_eq!(inner.not_broken_ticks, 0);
+
+ for _ in 0..2 {
+ assert_eq!(
+ transition_tick(&mut inner, false, false, start, ""),
+ TickEffect::None
+ );
+ }
+ assert_eq!(
+ transition_tick(&mut inner, false, false, start, ""),
+ TickEffect::PushAndUnreveal(OverlayHealthState::Normal)
+ );
+ }
+
+ #[test]
+ fn dismissal_suppresses_only_the_current_incident_then_rearms() {
+ let start = Instant::now();
+ let mut inner = test_inner(OverlayHealthState::Normal);
+ transition_tick(&mut inner, true, false, start, "");
+ inner.auto_revealed = true;
+
+ assert!(dismiss_state(&mut inner));
+ assert_eq!(inner.state, OverlayHealthState::Normal);
+ assert!(inner.dismissed);
+ assert!(!inner.auto_revealed);
+ assert_eq!(
+ transition_tick(&mut inner, true, false, start, ""),
+ TickEffect::None,
+ "the same still-broken incident must remain dismissed"
+ );
+ assert_eq!(
+ transition_tick(&mut inner, false, false, start, ""),
+ TickEffect::None
+ );
+ assert!(!inner.dismissed);
+ assert_eq!(
+ transition_tick(&mut inner, true, false, start, ""),
+ TickEffect::PushAndReveal(OverlayHealthState::Failure),
+ "a distinct later incident must alert again"
+ );
+ }
+
+ #[test]
+ fn fixing_ignores_old_engine_health_until_down_or_fifteen_seconds() {
+ let start = Instant::now();
+ let mut inner = test_inner(OverlayHealthState::Failure);
+ assert!(begin_fixing(&mut inner, start));
+ assert!(!begin_fixing(&mut inner, start), "double restart must be ignored");
+
+ for offset in [1, 2, 15] {
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(offset),
+ "",
+ ),
+ TickEffect::None
+ );
+ assert_eq!(inner.healthy_ticks, 0);
+ }
+
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(15) + Duration::from_millis(1),
+ "",
+ ),
+ TickEffect::None
+ );
+ assert_eq!(inner.healthy_ticks, 1);
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(16),
+ "",
+ ),
+ TickEffect::Push(OverlayHealthState::Recovered, None)
+ );
+ }
+
+ #[test]
+ fn fixing_requires_two_consecutive_healthy_ticks_after_engine_goes_down() {
+ let start = Instant::now();
+ let mut inner = test_inner(OverlayHealthState::Failure);
+ begin_fixing(&mut inner, start);
+
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ false,
+ start + Duration::from_secs(1),
+ "starting audio",
+ ),
+ TickEffect::Push(
+ OverlayHealthState::Fixing,
+ Some("starting audio".to_string())
+ )
+ );
+ assert!(inner.fixing_seen_down);
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ false,
+ start + Duration::from_secs(2),
+ "starting audio",
+ ),
+ TickEffect::None,
+ "unchanged boot detail must not spam the surfaces"
+ );
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(3),
+ "",
+ );
+ assert_eq!(inner.healthy_ticks, 1);
+ transition_tick(
+ &mut inner,
+ false,
+ false,
+ start + Duration::from_secs(4),
+ "",
+ );
+ assert_eq!(inner.healthy_ticks, 0, "an unhealthy tick resets confirmation");
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(5),
+ "",
+ );
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ true,
+ start + Duration::from_secs(6),
+ "",
+ ),
+ TickEffect::Push(OverlayHealthState::Recovered, None)
+ );
+ }
+
+ #[test]
+ fn fixing_timeout_is_strict_and_database_migration_pauses_it() {
+ let start = Instant::now();
+ let mut inner = test_inner(OverlayHealthState::Failure);
+ begin_fixing(&mut inner, start);
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ false,
+ start + FIXING_TIMEOUT,
+ "",
+ ),
+ TickEffect::None
+ );
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ false,
+ start + FIXING_TIMEOUT + Duration::from_millis(1),
+ "",
+ ),
+ TickEffect::Push(OverlayHealthState::Failure, None)
+ );
+
+ begin_fixing(&mut inner, start);
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ false,
+ start + Duration::from_secs(600),
+ "updating database",
+ ),
+ TickEffect::Push(
+ OverlayHealthState::Fixing,
+ Some("updating database".to_string())
+ )
+ );
+ assert_eq!(inner.state, OverlayHealthState::Fixing);
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ false,
+ false,
+ start + Duration::from_secs(601),
+ "starting engine",
+ ),
+ TickEffect::Push(OverlayHealthState::Failure, None),
+ "timeout resumes as soon as migration is no longer active"
+ );
+ }
+
+ #[test]
+ fn recovered_relapse_returns_to_failure_without_a_second_reveal() {
+ let start = Instant::now();
+ let mut inner = test_inner(OverlayHealthState::Failure);
+ transition_tick(&mut inner, false, true, start, "");
+ assert_eq!(inner.state, OverlayHealthState::Recovered);
+ assert_eq!(
+ transition_tick(
+ &mut inner,
+ true,
+ false,
+ start + Duration::from_secs(1),
+ "",
+ ),
+ TickEffect::Push(OverlayHealthState::Failure, None)
+ );
+ }
+
+ #[test]
+ fn restart_failure_and_dismissal_clear_transient_state() {
+ let start = Instant::now();
+ let mut inner = test_inner(OverlayHealthState::Recovered);
+ inner.recovered_at = Some(start);
+ inner.auto_revealed = true;
+ begin_fixing(&mut inner, start);
+ inner.last_detail = "starting audio".to_string();
+ fixing_failed(&mut inner);
+ assert_eq!(inner.state, OverlayHealthState::Failure);
+ assert!(inner.fixing_since.is_none());
+ assert!(inner.last_detail.is_empty());
+
+ assert!(dismiss_state(&mut inner));
+ assert_eq!(inner.state, OverlayHealthState::Normal);
+ assert!(inner.recovered_at.is_none());
+ assert_eq!(inner.not_broken_ticks, 0);
+ }
+
+ #[test]
+ fn overlay_state_machine_exhaustively_checks_65536_operation_sequences() {
+ let start = Instant::now();
+ let mut transitions_seen = [[false; 4]; 4];
+ let state_index = |state: OverlayHealthState| match state {
+ OverlayHealthState::Normal => 0,
+ OverlayHealthState::Failure => 1,
+ OverlayHealthState::Fixing => 2,
+ OverlayHealthState::Recovered => 3,
+ };
+
+ // Operations are neutral tick, broken tick, healthy tick, and restart.
+ // Eight steps gives 4^8 = 65,536 temporal sequences against the real
+ // production reducer, including repeated/double actions.
+ for encoded in 0u32..65_536 {
+ let mut operations = encoded;
+ let mut inner = test_inner(OverlayHealthState::Normal);
+ for step in 0..8 {
+ let before = inner.state;
+ let now = start + Duration::from_secs(step);
+ let effect = match operations & 0b11 {
+ 0 => transition_tick(&mut inner, false, false, now, ""),
+ 1 => transition_tick(&mut inner, true, false, now, ""),
+ 2 => transition_tick(&mut inner, false, true, now, ""),
+ 3 => {
+ begin_fixing(&mut inner, now);
+ TickEffect::None
+ }
+ _ => unreachable!(),
+ };
+ operations >>= 2;
+ transitions_seen[state_index(before)][state_index(inner.state)] = true;
+
+ match effect {
+ TickEffect::None => {}
+ TickEffect::Push(state, _) => assert_eq!(state, inner.state),
+ TickEffect::PushAndReveal(state) => {
+ assert_eq!(state, OverlayHealthState::Failure);
+ assert_eq!(inner.state, OverlayHealthState::Failure);
+ }
+ TickEffect::PushAndUnreveal(state) => {
+ assert_eq!(state, OverlayHealthState::Normal);
+ assert_eq!(inner.state, OverlayHealthState::Normal);
+ }
+ }
+ if inner.state == OverlayHealthState::Fixing {
+ assert!(inner.fixing_since.is_some());
+ }
+ if inner.state == OverlayHealthState::Recovered {
+ assert!(inner.recovered_at.is_some());
+ }
+ }
+ }
+
+ for (from, to) in [
+ (0, 0),
+ (0, 1),
+ (0, 2),
+ (1, 0),
+ (1, 1),
+ (1, 2),
+ (1, 3),
+ (2, 2),
+ (2, 3),
+ (3, 0),
+ (3, 1),
+ (3, 2),
+ (3, 3),
+ ] {
+ assert!(transitions_seen[from][to], "missing transition {from} -> {to}");
+ }
+ }
+}