YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Fix stale DB wedge recovery after healthy writes (#5087)
Louis Beaumont830d5c0fea55
diff --git a/apps/screenpipe-app-tauri/e2e/specs/api-search-stress.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/api-search-stress.spec.tsindex 4d56b261d..f9967e4a1 100644--- a/apps/screenpipe-app-tauri/e2e/specs/api-search-stress.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/api-search-stress.spec.ts@@ -25,9 +25,13 @@ describe("Local API search and stability", function () {before(async () => {await waitForAppReady();await openHomeWindow();- await waitForLocalApi();-+ // Use the explicitly configured port while the Tauri command may still be+ // on its cold-start fallback. This keeps isolated E2E runs from ever+ // probing a developer's live default-port server.+ const expectedPort = Number(process.env.SCREENPIPE_PORT ?? "3030");+ await waitForLocalApi(expectedPort);const cfg = await getLocalApiConfig();+ expect(cfg.port).toBe(expectedPort);port = cfg.port;key = cfg.key;});@@ -459,6 +463,27 @@ describe("Local API search and stability", function () {expect(results.filter((r) => !r.ok)).toHaveLength(0);});+ it("handles health bursts across repeated cache refresh boundaries", async () => {+ for (let round = 0; round < 3; round += 1) {+ // Cross the one-second cache TTL so every burst begins on the refresh+ // boundary that previously allowed duplicate full health computations.+ await new Promise((resolve) => setTimeout(resolve, 1_100));+ const startedAt = Date.now();+ const results = await Promise.all(+ Array.from({ length: 30 }, () => fetchJson(apiUrl("/health"))),+ );+ expect(Date.now() - startedAt).toBeLessThan(5_000);+ expect(results.filter((r) => !r.ok)).toHaveLength(0);+ for (const result of results) {+ expect(result.body).not.toBeNull();+ expect(typeof result.body).toBe("object");+ const body = result.body as { status_code?: number; status?: string };+ expect(body.status_code).toBe(200);+ expect(body.status).not.toBe("degraded");+ }+ }+ });+it("handles mixed readonly API load while the UI stays responsive", async function () {if (!key) this.skip();const endpoints = [diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rsindex c9c293f7b..a34409085 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs@@ -804,6 +804,7 @@ async fn main() {.await;let recording_state = RecordingState {+ server_lifecycle: Arc::new(tokio::sync::Mutex::new(())),server: Arc::new(tokio::sync::Mutex::new(None)),capture: Arc::new(tokio::sync::Mutex::new(None)),is_starting: Arc::new(AtomicBool::new(false)),@@ -1559,9 +1560,31 @@ async fn main() {break 'start_server;}let recording_state = app_handle.state::<RecordingState>();+ // Native auto-start has the same intent semantics as the+ // spawn_screenpipe command. DB-wedge recovery consults this+ // shared flag so it can rebuild the server without silently+ // leaving a normally auto-started recording paused.+ recording_state.set_capture_intent(true);+ // Reserve the lifecycle slot before publishing is_starting or+ // spawning the OS thread. Otherwise a frontend spawn can win+ // the scheduling gap, hold this lock while waiting on+ // is_starting, and deadlock the native thread that must clear+ // that flag.+ let lifecycle_guard = match recording_state+ .server_lifecycle+ .clone()+ .try_lock_owned()+ {+ Ok(guard) => guard,+ Err(_) => {+ warn!("Server lifecycle already active; skipping duplicate native auto-start");+ break 'start_server;+ }+ };recording_state.is_starting.store(true, std::sync::atomic::Ordering::SeqCst);let server_arc = recording_state.server.clone();let capture_arc = recording_state.capture.clone();+ let wants_recording = recording_state.wants_recording.clone();let is_starting_clone = recording_state.is_starting.clone();let cloud_token_arc = recording_state.cloud_token.clone();// DB-wedge auto-recovery hook wiring — captured into the server@@ -1717,10 +1740,12 @@ async fn main() {// Wire the persistent-failure hook so a wedged DB// auto-restarts recording (rebuilding every pool +// the shared WAL-index).+ let db_health = server.db.write_queue_health();server.db.set_persistent_failure_hook(crate::recording::make_db_wedge_recovery_hook(app_for_db_wedge.clone(),db_wedge_breaker.clone(),+ db_health,),);@@ -1731,29 +1756,42 @@ async fn main() {crate::e2e_seed::seed_search_fixture(&server.db).await;}- // Phase 2: Start capture session- let capture = match capture_session::CaptureSession::start(&server, &config, true).await {- Ok(c) => c,- Err(e) => {- error!("Failed to start capture: {}", e);- // Store server anyway so pipes/search work- let mut guard = server_arc.lock().await;- *guard = Some(server);- drop(guard);- is_starting_clone.store(false, std::sync::atomic::Ordering::SeqCst);- return;+ // Phase 2: use the latest capture intent, not the+ // value from app launch. Hold the slot across+ // check/start/assign so a racing stop_capture wins.+ let mut capture_guard = capture_arc.lock().await;+ let capture = if wants_recording+ .load(std::sync::atomic::Ordering::SeqCst)+ {+ match capture_session::CaptureSession::start(+ &server, &config, true,+ )+ .await+ {+ Ok(c) => Some(c),+ Err(e) => {+ error!("Failed to start capture: {}", e);+ None+ }}+ } else {+ None};- info!("Server + capture started successfully on dedicated runtime");{let mut guard = server_arc.lock().await;*guard = Some(server);}- {- let mut guard = capture_arc.lock().await;- *guard = Some(capture);+ if let Some(capture) = capture {+ *capture_guard = Some(capture);+ info!("Server + capture started successfully on dedicated runtime");+ } else {+ info!("Server started without capture");}+ drop(capture_guard);+ is_starting_clone+ .store(false, std::sync::atomic::Ordering::SeqCst);+ drop(lifecycle_guard);// Keep runtime alive as long as server existsloop {diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rsindex 679835200..bcec66186 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs@@ -186,6 +186,32 @@ const DB_WEDGE_BREAKER_WINDOW: Duration = Duration::from_secs(600);/// Coalesce a burst of persistent-failure signals before acting.const DB_WEDGE_DEBOUNCE: Duration = Duration::from_secs(15);+/// Result of revalidating a persistent-failure signal after the debounce.+#[derive(Debug, PartialEq, Eq)]+enum DbWedgeRecoveryDecision {+ Restart,+ SkipNoServer,+ SkipSupersededGeneration,+ SkipRecovered,+}++fn db_wedge_recovery_decision(+ signaled_health: &screenpipe_db::WriteQueueHealth,+ signaled_recovery_epoch: u64,+ current_health: Option<&screenpipe_db::WriteQueueHealth>,+) -> DbWedgeRecoveryDecision {+ let Some(current_health) = current_health else {+ return DbWedgeRecoveryDecision::SkipNoServer;+ };+ if !signaled_health.is_same_instance(current_health) {+ return DbWedgeRecoveryDecision::SkipSupersededGeneration;+ }+ if current_health.fatal_run_recovery_epoch() != signaled_recovery_epoch {+ return DbWedgeRecoveryDecision::SkipRecovered;+ }+ DbWedgeRecoveryDecision::Restart+}+/// Build the `PersistentFailureHook` the DB layer fires when writes wedge/// persistently. The hook itself is sync (`Fn()`), so it spawns the async/// restart. Captures an `AppHandle` (cheap clone, Send+Sync) and the shared@@ -193,22 +219,66 @@ const DB_WEDGE_DEBOUNCE: Duration = Duration::from_secs(15);pub fn make_db_wedge_recovery_hook(app: tauri::AppHandle,breaker: DbWedgeBreaker,+ health: screenpipe_db::WriteQueueHealth,) -> screenpipe_db::PersistentFailureHook {std::sync::Arc::new(move || {let app = app.clone();let breaker = breaker.clone();- tokio::spawn(async move {- recover_from_db_wedge(app, breaker).await;+ let health = health.clone();+ let recovery_epoch = health.fatal_run_recovery_epoch();+ // The hook fires on the dedicated *server* runtime. Recovery removes+ // that server from state, which intentionally lets its runtime exit;+ // running this task there would cancel it halfway through respawn.+ // Dispatch onto Tauri's process-lifetime runtime instead.+ tauri::async_runtime::spawn(async move {+ recover_from_db_wedge(app, breaker, health, recovery_epoch).await;});})}-async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {+async fn recover_from_db_wedge(+ app: tauri::AppHandle,+ breaker: DbWedgeBreaker,+ signaled_health: screenpipe_db::WriteQueueHealth,+ signaled_recovery_epoch: u64,+) {// Debounce: let a burst of signals coalesce and any in-flight work settle.tokio::time::sleep(DB_WEDGE_DEBOUNCE).await;+ let recording_state = app.state::<RecordingState>();++ // Serialize the exact-generation claim, teardown, and respawn with every+ // explicit full stop/start. This closes the debounce TOCTOU where a stale+ // task could validate server A, a manual restart could install server B,+ // and the stale task would then tear B down.+ let _lifecycle_guard = recording_state.server_lifecycle.lock().await;++ // Lock in the documented order and atomically remove only the server+ // generation that raised this signal. Once it is removed, no other full+ // lifecycle path can replace it until this recovery releases the outer+ // lifecycle guard.+ let mut capture_guard = recording_state.capture.lock().await;+ let mut server_guard = recording_state.server.lock().await;+ let current_health = server_guard+ .as_ref()+ .map(|core| core.db.write_queue_health());+ let decision = db_wedge_recovery_decision(+ &signaled_health,+ signaled_recovery_epoch,+ current_health.as_ref(),+ );+ if decision != DbWedgeRecoveryDecision::Restart {+ info!(+ "db wedge auto-recovery: restart cancelled after debounce ({:?})",+ decision+ );+ return;+ }+// Circuit breaker: a DB that stays broken after a restart is on-disk// corruption a restart can't repair, so cap auto-restarts per window.+ // Decide while the exact generation is still claimed; a skipped stale+ // signal must not consume restart budget.let action = {let mut state = breaker.lock().unwrap();state.decide(@@ -218,6 +288,8 @@ async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {)};if let WedgeAction::GiveUp { notify } = action {+ drop(server_guard);+ drop(capture_guard);error!("db wedge auto-recovery: {} restarts within {:?} did not clear the write wedge — \in-process restarts can't fix this (poisoned WAL-index pinned by a leaked \@@ -236,18 +308,31 @@ async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {return;}+ let capture = capture_guard.take();+ let server = server_guard+ .take()+ .expect("restart decision requires a current server generation");+warn!("db wedge auto-recovery: persistent write failure detected — restarting recording to \rebuild all DB pools + the shared WAL-index");- if let Err(e) = stop_screenpipe(app.state::<RecordingState>(), app.clone()).await {- warn!(- "db wedge auto-recovery: stop_screenpipe failed (continuing to spawn): {}",- e- );++ *recording_state.interrupted_meeting.lock().await = None;+ if let Some(session) = capture {+ session.stop().await;}+ server.shutdown().await;+ // Keep the state guards until shutdown completes. The dedicated server+ // runtime exits when it can lock `server` and observe None; releasing the+ // guard earlier can drop that runtime mid-shutdown and cancel the pool/task+ // cleanup this recovery depends on.+ drop(server_guard);+ drop(capture_guard);+ recording_state.is_starting.store(false, Ordering::SeqCst);+ recording_state.last_spawn_epoch.store(0, Ordering::SeqCst);- // stop_screenpipe rebuilds the engine's read/write pools on respawn, but the+ // The teardown above rebuilds the engine's read/write pools on respawn, but the// secret-store pool is a process-lifetime cache (min_connections=1, no idle// reaping) that would otherwise keep a connection — and the poisoned `-shm`// WAL-index — open across the restart. SQLite only rebuilds `-shm` once the@@ -256,7 +341,10 @@ async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {// Pools recreate lazily on the next secret access after spawn reopens.screenpipe_secrets::close_all_secret_pools().await;- if let Err(e) = spawn_screenpipe(app.state::<RecordingState>(), app.clone(), None).await {+ // Preserve the latest user capture intent. In particular, stop_capture can+ // run during the debounce/teardown: the server still needs rebuilding, but+ // the new server must come back without resurrecting recording.+ if let Err(e) = spawn_screenpipe_inner(&recording_state, app.clone()).await {// The restart failed to bring the engine back up (e.g. the port never// rebound). Nothing else will retry until the DB layer fires the hook// again — and if the server is fully down it never will — so recording@@ -285,10 +373,14 @@ pub(crate) struct InterruptedMeeting {/// Two-phase state: server (long-lived) + capture (togglable).///-/// **Lock ordering**: `capture` may be locked independently (it's self-contained).-/// When both locks are needed (e.g. `start_capture`), always lock `capture` first,-/// then `server`. Never hold `server` while waiting on `capture`.+/// **Lock ordering**: acquire `server_lifecycle` first for a full stop/start,+/// then `capture`, then `server`. `capture` may be locked independently (it's+/// self-contained). Never hold `server` while waiting on `capture`.pub struct RecordingState {+ /// Serializes full server stop/start cycles. DB-wedge recovery holds this+ /// across its generation check, teardown, and respawn so a delayed hook+ /// cannot tear down a server that a manual restart just replaced.+ pub server_lifecycle: Arc<Mutex<()>>,/// Long-lived server core (DB, HTTP, pipes). None until first start.pub server: Arc<Mutex<Option<ServerCore>>>,/// Current capture session. None when recording is stopped/paused.@@ -344,10 +436,14 @@ impl RecordingState {/// Whether capture is currently intended to be running.pub fn capture_intended(&self) -> bool {- self.wants_recording.load(Ordering::SeqCst)+ capture_intended_now(&self.wants_recording)}}+fn capture_intended_now(wants_recording: &AtomicBool) -> bool {+ wants_recording.load(Ordering::SeqCst)+}+// ---------------------------------------------------------------------------// Device listing (unchanged)// ---------------------------------------------------------------------------@@ -732,12 +828,17 @@ pub async fn stop_screenpipe(state: State<'_, RecordingState>,_app: tauri::AppHandle,) -> Result<(), String> {- info!("stop_screenpipe: stopping capture and server");-// Deliberate stop → clear the intent so the health watchdog leaves the// server down instead of auto-respawning it.state.set_capture_intent(false);+ let _lifecycle_guard = state.server_lifecycle.lock().await;+ stop_screenpipe_inner(&state).await+}++async fn stop_screenpipe_inner(state: &RecordingState) -> Result<(), String> {+ info!("stop_screenpipe: stopping capture and server");+// Stop capture first{*state.interrupted_meeting.lock().await = None;@@ -819,13 +920,24 @@ pub async fn spawn_screenpipe(app: tauri::AppHandle,_override_args: Option<Vec<String>>,) -> Result<(), String> {- info!("spawn_screenpipe: starting server + capture");-// Mark recording as intended-ON up front (even if the start below fails or// is deferred by cooldown) so the health watchdog will keep trying to bring// a crashed/failed server back instead of treating it as a user stop.state.set_capture_intent(true);+ let _lifecycle_guard = state.server_lifecycle.lock().await;+ spawn_screenpipe_inner(&state, app).await+}++async fn spawn_screenpipe_inner(+ state: &RecordingState,+ app: tauri::AppHandle,+) -> Result<(), String> {+ info!(+ "spawn_screenpipe: starting server (capture intended: {})",+ state.capture_intended()+ );+// --- Cooldown enforcement ---let now_epoch = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)@@ -1003,6 +1115,11 @@ pub async fn spawn_screenpipe(.and_then(|core| core.local_api_key.clone());if probe_server_health(&health_url, api_key.as_deref()).await {info!("Server already running and healthy on port {}", port);+ if !state.capture_intended() {+ info!("Capture is deliberately stopped; leaving healthy server running");+ state.is_starting.store(false, Ordering::SeqCst);+ return Ok(());+ }// Server is fine — just ensure capture is runningdrop(server_guard);let capture_guard = state.capture.lock().await;@@ -1078,7 +1195,7 @@ pub async fn spawn_screenpipe(let permissions_check = do_permissions_check(false);let disable_audio = store.recording.disable_audio;- if !permissions_check.screen_recording.permitted() {+ if state.capture_intended() && !permissions_check.screen_recording.permitted() {warn!("Screen recording permission not granted: {:?}. Cannot start server.",permissions_check.screen_recording@@ -1095,7 +1212,7 @@ pub async fn spawn_screenpipe();}- if !disable_audio && !permissions_check.microphone.permitted() {+ if state.capture_intended() && !disable_audio && !permissions_check.microphone.permitted() {warn!("Microphone permission not granted: {:?}. Audio recording will not work.",permissions_check.microphone@@ -1103,8 +1220,8 @@ pub async fn spawn_screenpipe(}info!(- "Permissions OK. Starting server + capture. Audio disabled: {}, mic: {:?}",- disable_audio, permissions_check.microphone+ "Permissions OK. Starting server. Capture intended: {}, audio disabled: {}, mic: {:?}",+ state.capture_intended(), disable_audio, permissions_check.microphone);let (data_dir, fell_back) = config::resolve_data_dir(&store.data_dir);@@ -1143,6 +1260,7 @@ pub async fn spawn_screenpipe(let server_arc = state.server.clone();let capture_arc = state.capture.clone();+ let wants_recording = state.wants_recording.clone();let cloud_token_arc = state.cloud_token.clone();// Wire the DB-wedge auto-recovery hook onto every (re)created DB. Captured into// the dedicated server thread so the freshly-built `ServerCore` gets the hook@@ -1218,38 +1336,51 @@ pub async fn spawn_screenpipe(// Wire the persistent-failure hook so a wedged DB auto-restarts// recording (rebuilding every pool + the shared WAL-index).+ let db_health = server.db.write_queue_health();server.db.set_persistent_failure_hook(make_db_wedge_recovery_hook(app_for_db_wedge.clone(),db_wedge_breaker.clone(),+ db_health,));- // Phase 2: Start capture- let capture = match CaptureSession::start(&server, &recording_config, true).await {- Ok(c) => c,- Err(e) => {- error!("Failed to start capture session: {}", e);- // Server started but capture failed — store server anyway- // so pipes/search still work- {- let mut guard = server_arc.lock().await;- *guard = Some(server);+ // Phase 2: Start capture only if it is still intended. Hold+ // the capture slot across the check + construction + assign:+ // stop_capture clears the intent before waiting on this lock,+ // so a stop racing a full server spawn either prevents capture+ // from starting or waits and then tears down the new session.+ let mut capture_guard = capture_arc.lock().await;+ let capture = if capture_intended_now(&wants_recording) {+ match CaptureSession::start(&server, &recording_config, true).await {+ Ok(c) => Some(c),+ Err(e) => {+ error!("Failed to start capture session: {}", e);+ // Server started but capture failed — store server anyway+ // so pipes/search still work.+ {+ let mut guard = server_arc.lock().await;+ *guard = Some(server);+ }+ let _ = result_tx.send(Err(e));+ return;}- let _ = result_tx.send(Err(e));- return;}+ } else {+ None};- info!("Server + capture started successfully on dedicated runtime");{let mut guard = server_arc.lock().await;*guard = Some(server);}- {- let mut guard = capture_arc.lock().await;- *guard = Some(capture);+ if let Some(capture) = capture {+ *capture_guard = Some(capture);+ info!("Server + capture started successfully on dedicated runtime");+ } else {+ info!("Server started with capture deliberately stopped");}+ drop(capture_guard);let _ = result_tx.send(Ok(()));// Keep runtime alive as long as server exists@@ -1446,7 +1577,12 @@ async fn kill_process_on_port(port: u16) {#[cfg(test)]mod db_wedge_tests {- use super::{DbWedgeState, WedgeAction};+ use super::{+ capture_intended_now, db_wedge_recovery_decision, DbWedgeRecoveryDecision, DbWedgeState,+ WedgeAction,+ };+ use screenpipe_db::WriteQueueHealth;+ use std::sync::atomic::{AtomicBool, Ordering};use std::time::{Duration, Instant};const WINDOW: Duration = Duration::from_secs(600);@@ -1511,6 +1647,50 @@ mod db_wedge_tests {t += WINDOW + Duration::from_secs(1);}}++ #[test]+ fn debounce_recheck_restarts_only_the_same_unrecovered_generation() {+ let signaled = WriteQueueHealth::default();+ let same_generation = signaled.clone();+ let epoch = signaled.fatal_run_recovery_epoch();++ assert_eq!(+ db_wedge_recovery_decision(&signaled, epoch, Some(&same_generation)),+ DbWedgeRecoveryDecision::Restart+ );+ assert_eq!(+ db_wedge_recovery_decision(&signaled, epoch.wrapping_add(1), Some(&same_generation)),+ DbWedgeRecoveryDecision::SkipRecovered+ );+ }++ #[test]+ fn debounce_recheck_ignores_stale_generation_signals() {+ let signaled = WriteQueueHealth::default();+ let replacement = WriteQueueHealth::default();+ let epoch = signaled.fatal_run_recovery_epoch();++ assert_eq!(+ db_wedge_recovery_decision(&signaled, epoch, None),+ DbWedgeRecoveryDecision::SkipNoServer+ );+ assert_eq!(+ db_wedge_recovery_decision(&signaled, epoch, Some(&replacement)),+ DbWedgeRecoveryDecision::SkipSupersededGeneration+ );+ }++ #[test]+ fn stop_during_debounce_is_honored_when_recovery_respawns() {+ let wants_recording = AtomicBool::new(true);++ // The hook does not cache capture intent when it fires. A tray stop+ // during the debounce clears the shared flag, and the server thread+ // reads that latest value immediately before constructing capture.+ wants_recording.store(false, Ordering::SeqCst);++ assert!(!capture_intended_now(&wants_recording));+ }}#[cfg(test)]diff --git a/crates/screenpipe-db/src/failpoint_vfs.rs b/crates/screenpipe-db/src/failpoint_vfs.rsindex 3363d9b19..8aa78fbb3 100644--- a/crates/screenpipe-db/src/failpoint_vfs.rs+++ b/crates/screenpipe-db/src/failpoint_vfs.rs@@ -458,23 +458,41 @@ mod tests {"persistent-failure hook (the engine-restart request) must fire");assert!(health.persistent_failure_signals() >= 1);+ let recovery_epoch = health.fatal_run_recovery_epoch();// --- Simulate the cure the hook requests (an engine restart clears the fault).disarm();- // The queue heals in-process now the condition cleared: the next write- // succeeds and health returns to healthy. (The OLD code stayed wedged.)- let recovered = queue- .submit(WriteOp::InsertAudioChunk {- file_path: "/post/ok".into(),- timestamp: None,- })- .await;- assert!(- recovered.is_ok(),- "write must recover once the fault clears: {:?}",- recovered.as_ref().err()- );+ // The queue heals in-process now the condition cleared. Operational+ // health becomes green after the first write, but the recovery epoch+ // advances only after three consecutive healthy batches. That preserves+ // the July 2 protection against one lucky commit cancelling recovery.+ for (index, path) in ["/post/ok", "/post/ok-2", "/post/ok-3"]+ .into_iter()+ .enumerate()+ {+ let recovered = queue+ .submit(WriteOp::InsertAudioChunk {+ file_path: path.into(),+ timestamp: None,+ })+ .await;+ assert!(+ recovered.is_ok(),+ "healthy write {index} must recover once the fault clears: {:?}",+ recovered.as_ref().err()+ );+ let expected_epoch = if index < 2 {+ recovery_epoch+ } else {+ recovery_epoch + 1+ };+ assert_eq!(+ health.fatal_run_recovery_epoch(),+ expected_epoch,+ "recovery epoch must advance only after the full healthy streak"+ );+ }assert!(!health.is_degraded(),"health recovers after a successful write"diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rsindex a8de1fd9f..c79570ee0 100644--- a/crates/screenpipe-db/src/write_queue.rs+++ b/crates/screenpipe-db/src/write_queue.rs@@ -217,6 +217,10 @@ struct WriteQueueHealthInner {total_contention_batches: std::sync::atomic::AtomicU64,write_pool_reopens: std::sync::atomic::AtomicU64,persistent_failure_signals: std::sync::atomic::AtomicU64,+ /// Advances only after the escalation state observes the full healthy+ /// streak required to end a fatal run. A recovery hook snapshots this+ /// value before its debounce and cancels a stale restart when it changes.+ fatal_run_recovery_epoch: std::sync::atomic::AtomicU64,degraded: AtomicBool,last_success_unix_ms: std::sync::atomic::AtomicI64,}@@ -242,6 +246,20 @@ impl WriteQueueHealth {pub fn persistent_failure_signals(&self) -> u64 {self.inner.persistent_failure_signals.load(Ordering::SeqCst)}+ /// Generation of completed fatal-run recoveries for this write queue.+ ///+ /// Unlike `is_degraded` and `consecutive_fatal_batches`, this does not+ /// change after one lucky successful batch. It advances only once the+ /// existing three-healthy-batch rule has cleared the fatal run.+ pub fn fatal_run_recovery_epoch(&self) -> u64 {+ self.inner.fatal_run_recovery_epoch.load(Ordering::SeqCst)+ }+ /// True when both handles observe the same write-queue generation.+ /// A new `DatabaseManager` gets a fresh health instance, so this is a+ /// lightweight generation check that does not keep SQLite pools alive.+ pub fn is_same_instance(&self, other: &Self) -> bool {+ Arc::ptr_eq(&self.inner, &other.inner)+ }/// Unix-ms timestamp of the last healthy batch (0 if never).pub fn last_success_unix_ms(&self) -> i64 {self.inner.last_success_unix_ms.load(Ordering::SeqCst)@@ -284,6 +302,11 @@ impl WriteQueueHealth {.persistent_failure_signals.fetch_add(1, Ordering::SeqCst);}+ fn note_fatal_run_recovered(&self) {+ self.inner+ .fatal_run_recovery_epoch+ .fetch_add(1, Ordering::SeqCst);+ }}/// Hook invoked once when writes have failed persistently — the seam the app uses@@ -801,6 +824,7 @@ async fn drain_loop(}health.record_success();if escalation.on_healthy() {+ health.note_fatal_run_recovered();info!("write_queue: fatal run cleared");}}@@ -2214,6 +2238,36 @@ mod tests {);}+ #[test]+ fn recovery_epoch_requires_the_full_healthy_streak() {+ let health = WriteQueueHealth::default();+ let cloned = health.clone();+ let fresh = WriteQueueHealth::default();+ assert!(health.is_same_instance(&cloned));+ assert!(!health.is_same_instance(&fresh));++ let mut escalation = esc();+ let now = std::time::Instant::now();+ escalation.on_fatal(now);+ health.record_fatal();+ let epoch = health.fatal_run_recovery_epoch();++ for healthy in 1..FATAL_RUN_CLEAR_AFTER {+ health.record_success();+ assert!(!escalation.on_healthy());+ assert_eq!(+ health.fatal_run_recovery_epoch(),+ epoch,+ "healthy batch {healthy} must not cancel a pending recovery"+ );+ }++ health.record_success();+ assert!(escalation.on_healthy());+ health.note_fatal_run_recovered();+ assert_eq!(health.fatal_run_recovery_epoch(), epoch + 1);+ }+#[test]fn escalation_refires_while_run_persists() {// The requested engine restart may not cure the wedge (leakeddiff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rsindex 3ffab9e3c..73df3e2a7 100644--- a/crates/screenpipe-engine/src/routes/health.rs+++ b/crates/screenpipe-engine/src/routes/health.rs@@ -13,7 +13,7 @@ use std::sync::{atomic::{AtomicU64, Ordering},Arc,};-use tokio::sync::RwLock;+use tokio::sync::{Mutex, RwLock};use tracing::{debug, warn};use screenpipe_audio::audio_manager::builder::TranscriptionMode;@@ -30,6 +30,9 @@ use crate::ui_recorder::{/// times per second. The response only changes meaningfully every ~1s.static HEALTH_CACHE: std::sync::LazyLock<RwLock<(u64, Option<HealthCheckResponse>)>> =std::sync::LazyLock::new(|| RwLock::new((0, None)));+/// Single-flight gate for full health recomputation. Cache misses crossing the+/// same one-second boundary must not all run the DB-backed backlog query.+static HEALTH_REFRESH: std::sync::LazyLock<Mutex<()>> = std::sync::LazyLock::new(|| Mutex::new(()));type AudioReconciliationBacklogCache = (i64, Option<(u64, Option<DateTime<Utc>>)>);static AUDIO_RECONCILIATION_BACKLOG_CACHE: std::sync::LazyLock<RwLock<AudioReconciliationBacklogCache>,@@ -452,51 +455,105 @@ pub struct AudioPipelineHealthInfo {/// killed by a watchdog).const HEALTH_RESPONSE_BUDGET: std::time::Duration = std::time::Duration::from_secs(2);-#[oasgen]-pub async fn health_check(State(state): State<Arc<AppState>>) -> JsonResponse<HealthCheckResponse> {- let now_ts = std::time::SystemTime::now()+fn unix_now_secs() -> u64 {+ std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)- .unwrap()- .as_secs();+ .unwrap_or_default()+ .as_secs()+}- // Return cached response if still fresh. This prevents thundering-herd- // scenarios where dozens of WebSocket clients + HTTP polls recompute the- // full health response simultaneously.+async fn cached_health_or_refresh<F, Fut>(+ cache: &RwLock<(u64, Option<HealthCheckResponse>)>,+ refresh: &Mutex<()>,+ ttl_secs: u64,+ compute: F,+) -> HealthCheckResponse+where+ F: FnOnce() -> Fut,+ Fut: std::future::Future<Output = Option<HealthCheckResponse>>,+{+ let now = unix_now_secs();{- let cache = HEALTH_CACHE.read().await;- if now_ts.saturating_sub(cache.0) < HEALTH_CACHE_TTL_SECS {- if let Some(ref cached) = cache.1 {- return JsonResponse(cached.clone());+ let cached = cache.read().await;+ if now.saturating_sub(cached.0) < ttl_secs {+ if let Some(response) = cached.1.as_ref() {+ return response.clone();}}}- let response = match tokio::time::timeout(HEALTH_RESPONSE_BUDGET, health_check_inner(&state))- .await- {- Ok(r) => r,+ // Prefer stale-while-refresh over queuing every tray/WebSocket/HTTP poll+ // behind a potentially two-second health computation. Cold start has no+ // stale value, so those callers wait for the single refresh to finish.+ let _refresh_guard = match refresh.try_lock() {+ Ok(guard) => guard,Err(_) => {- // Inner computation exceeded the budget. Serve the last cached- // snapshot (even if past TTL) so callers see continuity. If no- // snapshot exists yet, return a minimal "degraded" response —- // never block forever.- warn!(- "health_check: inner computation exceeded {:?} budget — serving last cached snapshot",- HEALTH_RESPONSE_BUDGET- );- let cached = HEALTH_CACHE.read().await.1.clone();- // Don't refresh the cache timestamp here — the next caller- // should re-attempt rather than amortize the stale entry.- return JsonResponse(cached.unwrap_or_else(degraded_response));+ {+ let cached = cache.read().await;+ if let Some(response) = cached.1.as_ref() {+ return response.clone();+ }+ }+ refresh.lock().await}};- // Cache the result+ // Another cold-start caller may have populated the cache while this one+ // waited for the refresh gate.+ let now = unix_now_secs();{- let mut cache = HEALTH_CACHE.write().await;- *cache = (now_ts, Some(response.clone()));+ let cached = cache.read().await;+ if now.saturating_sub(cached.0) < ttl_secs {+ if let Some(response) = cached.1.as_ref() {+ return response.clone();+ }+ }}+ let response = match compute().await {+ Some(response) => response,+ None => cache+ .read()+ .await+ .1+ .clone()+ .unwrap_or_else(degraded_response),+ };++ // Publish timeout results too. Without this, a cold-cache burst queues on+ // the refresh mutex and every waiter performs its own full two-second+ // computation after the first timeout. The normal one-second TTL makes+ // this a short backoff, while all callers in the same burst share one+ // bounded attempt (or the same stale snapshot).+ let mut cached = cache.write().await;+ *cached = (unix_now_secs(), Some(response.clone()));+ response+}++#[oasgen]+pub async fn health_check(State(state): State<Arc<AppState>>) -> JsonResponse<HealthCheckResponse> {+ let response = cached_health_or_refresh(+ &HEALTH_CACHE,+ &HEALTH_REFRESH,+ HEALTH_CACHE_TTL_SECS,+ || async {+ match tokio::time::timeout(HEALTH_RESPONSE_BUDGET, health_check_inner(&state)).await {+ Ok(response) => Some(response),+ Err(_) => {+ // The shared refresh helper publishes this failed attempt+ // for one short TTL so cold-cache peers do not serialize+ // another full computation each.+ warn!(+ "health_check: inner computation exceeded {:?} budget — serving last cached snapshot",+ HEALTH_RESPONSE_BUDGET+ );+ None+ }+ }+ },+ )+ .await;+JsonResponse(response)}@@ -1624,6 +1681,100 @@ mod tests {}}+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn concurrent_stale_health_requests_share_one_refresh() {+ const CALLERS: usize = 24;+ let cache = Arc::new(RwLock::new((0, None)));+ let refresh = Arc::new(Mutex::new(()));+ let barrier = Arc::new(tokio::sync::Barrier::new(CALLERS));+ let computes = Arc::new(AtomicU64::new(0));+ let mut tasks = Vec::with_capacity(CALLERS);++ for _ in 0..CALLERS {+ let cache = cache.clone();+ let refresh = refresh.clone();+ let barrier = barrier.clone();+ let computes = computes.clone();+ tasks.push(tokio::spawn(async move {+ barrier.wait().await;+ cached_health_or_refresh(&cache, &refresh, 60, || async move {+ computes.fetch_add(1, Ordering::SeqCst);+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;+ Some(dummy_response("healthy"))+ })+ .await+ }));+ }++ for task in tasks {+ assert_eq!(task.await.unwrap().status, "healthy");+ }+ assert_eq!(+ computes.load(Ordering::SeqCst),+ 1,+ "a stale-cache burst must perform exactly one full health refresh"+ );+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn concurrent_cold_timeout_is_published_to_all_waiters() {+ const CALLERS: usize = 24;+ let cache = Arc::new(RwLock::new((0, None)));+ let refresh = Arc::new(Mutex::new(()));+ let barrier = Arc::new(tokio::sync::Barrier::new(CALLERS));+ let computes = Arc::new(AtomicU64::new(0));+ let mut tasks = Vec::with_capacity(CALLERS);++ for _ in 0..CALLERS {+ let cache = cache.clone();+ let refresh = refresh.clone();+ let barrier = barrier.clone();+ let computes = computes.clone();+ tasks.push(tokio::spawn(async move {+ barrier.wait().await;+ cached_health_or_refresh(&cache, &refresh, 60, || async move {+ computes.fetch_add(1, Ordering::SeqCst);+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;+ None+ })+ .await+ }));+ }++ for task in tasks {+ let response = task.await.unwrap();+ assert_eq!(response.status, "degraded");+ assert_eq!(response.status_code, 503);+ }+ assert_eq!(+ computes.load(Ordering::SeqCst),+ 1,+ "a cold timeout burst must share one bounded refresh attempt"+ );+ }++ #[tokio::test]+ async fn concurrent_refresh_serves_an_existing_stale_snapshot() {+ let cache = RwLock::new((0, Some(dummy_response("stale"))));+ let refresh = Mutex::new(());+ let in_flight = refresh.lock().await;+ let computes = AtomicU64::new(0);++ let response = tokio::time::timeout(+ std::time::Duration::from_millis(100),+ cached_health_or_refresh(&cache, &refresh, 60, || async {+ computes.fetch_add(1, Ordering::SeqCst);+ Some(dummy_response("unexpected"))+ }),+ )+ .await+ .expect("stale-while-refresh must not queue behind the active refresh");++ assert_eq!(response.status, "stale");+ assert_eq!(computes.load(Ordering::SeqCst), 0);+ drop(in_flight);+ }+#[test]fn health_response_is_cloneable() {let resp = dummy_response("healthy");