Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Fix stale DB wedge recovery after healthy writes (#5087)

Louis Beaumont830d5c0fea55
diff --git a/apps/screenpipe-app-tauri/e2e/specs/api-search-stress.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/api-search-stress.spec.ts
index 4d56b261d..f9967e4a1 100644
--- a/apps/screenpipe-app-tauri/e2e/specs/api-search-stress.spec.ts
+++ b/apps/screenpipe-app-tauri/e2e/specs/api-search-stress.spec.ts
@@ -25,9 +25,13 @@ describe("Local API search and stability", function () {
before(async () => {
await waitForAppReady();
await openHomeWindow();
- await waitForLocalApi();
-
+ // Use the explicitly configured port while the Tauri command may still be
+ // on its cold-start fallback. This keeps isolated E2E runs from ever
+ // probing a developer's live default-port server.
+ const expectedPort = Number(process.env.SCREENPIPE_PORT ?? "3030");
+ await waitForLocalApi(expectedPort);
const cfg = await getLocalApiConfig();
+ expect(cfg.port).toBe(expectedPort);
port = cfg.port;
key = cfg.key;
});
@@ -459,6 +463,27 @@ describe("Local API search and stability", function () {
expect(results.filter((r) => !r.ok)).toHaveLength(0);
});
+ it("handles health bursts across repeated cache refresh boundaries", async () => {
+ for (let round = 0; round < 3; round += 1) {
+ // Cross the one-second cache TTL so every burst begins on the refresh
+ // boundary that previously allowed duplicate full health computations.
+ await new Promise((resolve) => setTimeout(resolve, 1_100));
+ const startedAt = Date.now();
+ const results = await Promise.all(
+ Array.from({ length: 30 }, () => fetchJson(apiUrl("/health"))),
+ );
+ expect(Date.now() - startedAt).toBeLessThan(5_000);
+ expect(results.filter((r) => !r.ok)).toHaveLength(0);
+ for (const result of results) {
+ expect(result.body).not.toBeNull();
+ expect(typeof result.body).toBe("object");
+ const body = result.body as { status_code?: number; status?: string };
+ expect(body.status_code).toBe(200);
+ expect(body.status).not.toBe("degraded");
+ }
+ }
+ });
+
it("handles mixed readonly API load while the UI stays responsive", async function () {
if (!key) this.skip();
const endpoints = [
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
index c9c293f7b..a34409085 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
@@ -804,6 +804,7 @@ async fn main() {
.await;
let recording_state = RecordingState {
+ server_lifecycle: Arc::new(tokio::sync::Mutex::new(())),
server: Arc::new(tokio::sync::Mutex::new(None)),
capture: Arc::new(tokio::sync::Mutex::new(None)),
is_starting: Arc::new(AtomicBool::new(false)),
@@ -1559,9 +1560,31 @@ async fn main() {
break 'start_server;
}
let recording_state = app_handle.state::<RecordingState>();
+ // Native auto-start has the same intent semantics as the
+ // spawn_screenpipe command. DB-wedge recovery consults this
+ // shared flag so it can rebuild the server without silently
+ // leaving a normally auto-started recording paused.
+ recording_state.set_capture_intent(true);
+ // Reserve the lifecycle slot before publishing is_starting or
+ // spawning the OS thread. Otherwise a frontend spawn can win
+ // the scheduling gap, hold this lock while waiting on
+ // is_starting, and deadlock the native thread that must clear
+ // that flag.
+ let lifecycle_guard = match recording_state
+ .server_lifecycle
+ .clone()
+ .try_lock_owned()
+ {
+ Ok(guard) => guard,
+ Err(_) => {
+ warn!("Server lifecycle already active; skipping duplicate native auto-start");
+ break 'start_server;
+ }
+ };
recording_state.is_starting.store(true, std::sync::atomic::Ordering::SeqCst);
let server_arc = recording_state.server.clone();
let capture_arc = recording_state.capture.clone();
+ let wants_recording = recording_state.wants_recording.clone();
let is_starting_clone = recording_state.is_starting.clone();
let cloud_token_arc = recording_state.cloud_token.clone();
// DB-wedge auto-recovery hook wiring — captured into the server
@@ -1717,10 +1740,12 @@ async fn main() {
// Wire the persistent-failure hook so a wedged DB
// auto-restarts recording (rebuilding every pool +
// the shared WAL-index).
+ let db_health = server.db.write_queue_health();
server.db.set_persistent_failure_hook(
crate::recording::make_db_wedge_recovery_hook(
app_for_db_wedge.clone(),
db_wedge_breaker.clone(),
+ db_health,
),
);
@@ -1731,29 +1756,42 @@ async fn main() {
crate::e2e_seed::seed_search_fixture(&server.db).await;
}
- // Phase 2: Start capture session
- let capture = match capture_session::CaptureSession::start(&server, &config, true).await {
- Ok(c) => c,
- Err(e) => {
- error!("Failed to start capture: {}", e);
- // Store server anyway so pipes/search work
- let mut guard = server_arc.lock().await;
- *guard = Some(server);
- drop(guard);
- is_starting_clone.store(false, std::sync::atomic::Ordering::SeqCst);
- return;
+ // Phase 2: use the latest capture intent, not the
+ // value from app launch. Hold the slot across
+ // check/start/assign so a racing stop_capture wins.
+ let mut capture_guard = capture_arc.lock().await;
+ let capture = if wants_recording
+ .load(std::sync::atomic::Ordering::SeqCst)
+ {
+ match capture_session::CaptureSession::start(
+ &server, &config, true,
+ )
+ .await
+ {
+ Ok(c) => Some(c),
+ Err(e) => {
+ error!("Failed to start capture: {}", e);
+ None
+ }
}
+ } else {
+ None
};
- info!("Server + capture started successfully on dedicated runtime");
{
let mut guard = server_arc.lock().await;
*guard = Some(server);
}
- {
- let mut guard = capture_arc.lock().await;
- *guard = Some(capture);
+ if let Some(capture) = capture {
+ *capture_guard = Some(capture);
+ info!("Server + capture started successfully on dedicated runtime");
+ } else {
+ info!("Server started without capture");
}
+ drop(capture_guard);
+ is_starting_clone
+ .store(false, std::sync::atomic::Ordering::SeqCst);
+ drop(lifecycle_guard);
// Keep runtime alive as long as server exists
loop {
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
index 679835200..bcec66186 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
@@ -186,6 +186,32 @@ const DB_WEDGE_BREAKER_WINDOW: Duration = Duration::from_secs(600);
/// Coalesce a burst of persistent-failure signals before acting.
const DB_WEDGE_DEBOUNCE: Duration = Duration::from_secs(15);
+/// Result of revalidating a persistent-failure signal after the debounce.
+#[derive(Debug, PartialEq, Eq)]
+enum DbWedgeRecoveryDecision {
+ Restart,
+ SkipNoServer,
+ SkipSupersededGeneration,
+ SkipRecovered,
+}
+
+fn db_wedge_recovery_decision(
+ signaled_health: &screenpipe_db::WriteQueueHealth,
+ signaled_recovery_epoch: u64,
+ current_health: Option<&screenpipe_db::WriteQueueHealth>,
+) -> DbWedgeRecoveryDecision {
+ let Some(current_health) = current_health else {
+ return DbWedgeRecoveryDecision::SkipNoServer;
+ };
+ if !signaled_health.is_same_instance(current_health) {
+ return DbWedgeRecoveryDecision::SkipSupersededGeneration;
+ }
+ if current_health.fatal_run_recovery_epoch() != signaled_recovery_epoch {
+ return DbWedgeRecoveryDecision::SkipRecovered;
+ }
+ DbWedgeRecoveryDecision::Restart
+}
+
/// Build the `PersistentFailureHook` the DB layer fires when writes wedge
/// persistently. The hook itself is sync (`Fn()`), so it spawns the async
/// restart. Captures an `AppHandle` (cheap clone, Send+Sync) and the shared
@@ -193,22 +219,66 @@ const DB_WEDGE_DEBOUNCE: Duration = Duration::from_secs(15);
pub fn make_db_wedge_recovery_hook(
app: tauri::AppHandle,
breaker: DbWedgeBreaker,
+ health: screenpipe_db::WriteQueueHealth,
) -> screenpipe_db::PersistentFailureHook {
std::sync::Arc::new(move || {
let app = app.clone();
let breaker = breaker.clone();
- tokio::spawn(async move {
- recover_from_db_wedge(app, breaker).await;
+ let health = health.clone();
+ let recovery_epoch = health.fatal_run_recovery_epoch();
+ // The hook fires on the dedicated *server* runtime. Recovery removes
+ // that server from state, which intentionally lets its runtime exit;
+ // running this task there would cancel it halfway through respawn.
+ // Dispatch onto Tauri's process-lifetime runtime instead.
+ tauri::async_runtime::spawn(async move {
+ recover_from_db_wedge(app, breaker, health, recovery_epoch).await;
});
})
}
-async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {
+async fn recover_from_db_wedge(
+ app: tauri::AppHandle,
+ breaker: DbWedgeBreaker,
+ signaled_health: screenpipe_db::WriteQueueHealth,
+ signaled_recovery_epoch: u64,
+) {
// Debounce: let a burst of signals coalesce and any in-flight work settle.
tokio::time::sleep(DB_WEDGE_DEBOUNCE).await;
+ let recording_state = app.state::<RecordingState>();
+
+ // Serialize the exact-generation claim, teardown, and respawn with every
+ // explicit full stop/start. This closes the debounce TOCTOU where a stale
+ // task could validate server A, a manual restart could install server B,
+ // and the stale task would then tear B down.
+ let _lifecycle_guard = recording_state.server_lifecycle.lock().await;
+
+ // Lock in the documented order and atomically remove only the server
+ // generation that raised this signal. Once it is removed, no other full
+ // lifecycle path can replace it until this recovery releases the outer
+ // lifecycle guard.
+ let mut capture_guard = recording_state.capture.lock().await;
+ let mut server_guard = recording_state.server.lock().await;
+ let current_health = server_guard
+ .as_ref()
+ .map(|core| core.db.write_queue_health());
+ let decision = db_wedge_recovery_decision(
+ &signaled_health,
+ signaled_recovery_epoch,
+ current_health.as_ref(),
+ );
+ if decision != DbWedgeRecoveryDecision::Restart {
+ info!(
+ "db wedge auto-recovery: restart cancelled after debounce ({:?})",
+ decision
+ );
+ return;
+ }
+
// Circuit breaker: a DB that stays broken after a restart is on-disk
// corruption a restart can't repair, so cap auto-restarts per window.
+ // Decide while the exact generation is still claimed; a skipped stale
+ // signal must not consume restart budget.
let action = {
let mut state = breaker.lock().unwrap();
state.decide(
@@ -218,6 +288,8 @@ async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {
)
};
if let WedgeAction::GiveUp { notify } = action {
+ drop(server_guard);
+ drop(capture_guard);
error!(
"db wedge auto-recovery: {} restarts within {:?} did not clear the write wedge — \
in-process restarts can't fix this (poisoned WAL-index pinned by a leaked \
@@ -236,18 +308,31 @@ async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {
return;
}
+ let capture = capture_guard.take();
+ let server = server_guard
+ .take()
+ .expect("restart decision requires a current server generation");
+
warn!(
"db wedge auto-recovery: persistent write failure detected — restarting recording to \
rebuild all DB pools + the shared WAL-index"
);
- if let Err(e) = stop_screenpipe(app.state::<RecordingState>(), app.clone()).await {
- warn!(
- "db wedge auto-recovery: stop_screenpipe failed (continuing to spawn): {}",
- e
- );
+
+ *recording_state.interrupted_meeting.lock().await = None;
+ if let Some(session) = capture {
+ session.stop().await;
}
+ server.shutdown().await;
+ // Keep the state guards until shutdown completes. The dedicated server
+ // runtime exits when it can lock `server` and observe None; releasing the
+ // guard earlier can drop that runtime mid-shutdown and cancel the pool/task
+ // cleanup this recovery depends on.
+ drop(server_guard);
+ drop(capture_guard);
+ recording_state.is_starting.store(false, Ordering::SeqCst);
+ recording_state.last_spawn_epoch.store(0, Ordering::SeqCst);
- // stop_screenpipe rebuilds the engine's read/write pools on respawn, but the
+ // The teardown above rebuilds the engine's read/write pools on respawn, but the
// secret-store pool is a process-lifetime cache (min_connections=1, no idle
// reaping) that would otherwise keep a connection — and the poisoned `-shm`
// WAL-index — open across the restart. SQLite only rebuilds `-shm` once the
@@ -256,7 +341,10 @@ async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {
// Pools recreate lazily on the next secret access after spawn reopens.
screenpipe_secrets::close_all_secret_pools().await;
- if let Err(e) = spawn_screenpipe(app.state::<RecordingState>(), app.clone(), None).await {
+ // Preserve the latest user capture intent. In particular, stop_capture can
+ // run during the debounce/teardown: the server still needs rebuilding, but
+ // the new server must come back without resurrecting recording.
+ if let Err(e) = spawn_screenpipe_inner(&recording_state, app.clone()).await {
// The restart failed to bring the engine back up (e.g. the port never
// rebound). Nothing else will retry until the DB layer fires the hook
// again — and if the server is fully down it never will — so recording
@@ -285,10 +373,14 @@ pub(crate) struct InterruptedMeeting {
/// Two-phase state: server (long-lived) + capture (togglable).
///
-/// **Lock ordering**: `capture` may be locked independently (it's self-contained).
-/// When both locks are needed (e.g. `start_capture`), always lock `capture` first,
-/// then `server`. Never hold `server` while waiting on `capture`.
+/// **Lock ordering**: acquire `server_lifecycle` first for a full stop/start,
+/// then `capture`, then `server`. `capture` may be locked independently (it's
+/// self-contained). Never hold `server` while waiting on `capture`.
pub struct RecordingState {
+ /// Serializes full server stop/start cycles. DB-wedge recovery holds this
+ /// across its generation check, teardown, and respawn so a delayed hook
+ /// cannot tear down a server that a manual restart just replaced.
+ pub server_lifecycle: Arc<Mutex<()>>,
/// Long-lived server core (DB, HTTP, pipes). None until first start.
pub server: Arc<Mutex<Option<ServerCore>>>,
/// Current capture session. None when recording is stopped/paused.
@@ -344,10 +436,14 @@ impl RecordingState {
/// Whether capture is currently intended to be running.
pub fn capture_intended(&self) -> bool {
- self.wants_recording.load(Ordering::SeqCst)
+ capture_intended_now(&self.wants_recording)
}
}
+fn capture_intended_now(wants_recording: &AtomicBool) -> bool {
+ wants_recording.load(Ordering::SeqCst)
+}
+
// ---------------------------------------------------------------------------
// Device listing (unchanged)
// ---------------------------------------------------------------------------
@@ -732,12 +828,17 @@ pub async fn stop_screenpipe(
state: State<'_, RecordingState>,
_app: tauri::AppHandle,
) -> Result<(), String> {
- info!("stop_screenpipe: stopping capture and server");
-
// Deliberate stop → clear the intent so the health watchdog leaves the
// server down instead of auto-respawning it.
state.set_capture_intent(false);
+ let _lifecycle_guard = state.server_lifecycle.lock().await;
+ stop_screenpipe_inner(&state).await
+}
+
+async fn stop_screenpipe_inner(state: &RecordingState) -> Result<(), String> {
+ info!("stop_screenpipe: stopping capture and server");
+
// Stop capture first
{
*state.interrupted_meeting.lock().await = None;
@@ -819,13 +920,24 @@ pub async fn spawn_screenpipe(
app: tauri::AppHandle,
_override_args: Option<Vec<String>>,
) -> Result<(), String> {
- info!("spawn_screenpipe: starting server + capture");
-
// Mark recording as intended-ON up front (even if the start below fails or
// is deferred by cooldown) so the health watchdog will keep trying to bring
// a crashed/failed server back instead of treating it as a user stop.
state.set_capture_intent(true);
+ let _lifecycle_guard = state.server_lifecycle.lock().await;
+ spawn_screenpipe_inner(&state, app).await
+}
+
+async fn spawn_screenpipe_inner(
+ state: &RecordingState,
+ app: tauri::AppHandle,
+) -> Result<(), String> {
+ info!(
+ "spawn_screenpipe: starting server (capture intended: {})",
+ state.capture_intended()
+ );
+
// --- Cooldown enforcement ---
let now_epoch = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -1003,6 +1115,11 @@ pub async fn spawn_screenpipe(
.and_then(|core| core.local_api_key.clone());
if probe_server_health(&health_url, api_key.as_deref()).await {
info!("Server already running and healthy on port {}", port);
+ if !state.capture_intended() {
+ info!("Capture is deliberately stopped; leaving healthy server running");
+ state.is_starting.store(false, Ordering::SeqCst);
+ return Ok(());
+ }
// Server is fine — just ensure capture is running
drop(server_guard);
let capture_guard = state.capture.lock().await;
@@ -1078,7 +1195,7 @@ pub async fn spawn_screenpipe(
let permissions_check = do_permissions_check(false);
let disable_audio = store.recording.disable_audio;
- if !permissions_check.screen_recording.permitted() {
+ if state.capture_intended() && !permissions_check.screen_recording.permitted() {
warn!(
"Screen recording permission not granted: {:?}. Cannot start server.",
permissions_check.screen_recording
@@ -1095,7 +1212,7 @@ pub async fn spawn_screenpipe(
);
}
- if !disable_audio && !permissions_check.microphone.permitted() {
+ if state.capture_intended() && !disable_audio && !permissions_check.microphone.permitted() {
warn!(
"Microphone permission not granted: {:?}. Audio recording will not work.",
permissions_check.microphone
@@ -1103,8 +1220,8 @@ pub async fn spawn_screenpipe(
}
info!(
- "Permissions OK. Starting server + capture. Audio disabled: {}, mic: {:?}",
- disable_audio, permissions_check.microphone
+ "Permissions OK. Starting server. Capture intended: {}, audio disabled: {}, mic: {:?}",
+ state.capture_intended(), disable_audio, permissions_check.microphone
);
let (data_dir, fell_back) = config::resolve_data_dir(&store.data_dir);
@@ -1143,6 +1260,7 @@ pub async fn spawn_screenpipe(
let server_arc = state.server.clone();
let capture_arc = state.capture.clone();
+ let wants_recording = state.wants_recording.clone();
let cloud_token_arc = state.cloud_token.clone();
// Wire the DB-wedge auto-recovery hook onto every (re)created DB. Captured into
// the dedicated server thread so the freshly-built `ServerCore` gets the hook
@@ -1218,38 +1336,51 @@ pub async fn spawn_screenpipe(
// Wire the persistent-failure hook so a wedged DB auto-restarts
// recording (rebuilding every pool + the shared WAL-index).
+ let db_health = server.db.write_queue_health();
server
.db
.set_persistent_failure_hook(make_db_wedge_recovery_hook(
app_for_db_wedge.clone(),
db_wedge_breaker.clone(),
+ db_health,
));
- // Phase 2: Start capture
- let capture = match CaptureSession::start(&server, &recording_config, true).await {
- Ok(c) => c,
- Err(e) => {
- error!("Failed to start capture session: {}", e);
- // Server started but capture failed — store server anyway
- // so pipes/search still work
- {
- let mut guard = server_arc.lock().await;
- *guard = Some(server);
+ // Phase 2: Start capture only if it is still intended. Hold
+ // the capture slot across the check + construction + assign:
+ // stop_capture clears the intent before waiting on this lock,
+ // so a stop racing a full server spawn either prevents capture
+ // from starting or waits and then tears down the new session.
+ let mut capture_guard = capture_arc.lock().await;
+ let capture = if capture_intended_now(&wants_recording) {
+ match CaptureSession::start(&server, &recording_config, true).await {
+ Ok(c) => Some(c),
+ Err(e) => {
+ error!("Failed to start capture session: {}", e);
+ // Server started but capture failed — store server anyway
+ // so pipes/search still work.
+ {
+ let mut guard = server_arc.lock().await;
+ *guard = Some(server);
+ }
+ let _ = result_tx.send(Err(e));
+ return;
}
- let _ = result_tx.send(Err(e));
- return;
}
+ } else {
+ None
};
- info!("Server + capture started successfully on dedicated runtime");
{
let mut guard = server_arc.lock().await;
*guard = Some(server);
}
- {
- let mut guard = capture_arc.lock().await;
- *guard = Some(capture);
+ if let Some(capture) = capture {
+ *capture_guard = Some(capture);
+ info!("Server + capture started successfully on dedicated runtime");
+ } else {
+ info!("Server started with capture deliberately stopped");
}
+ drop(capture_guard);
let _ = result_tx.send(Ok(()));
// Keep runtime alive as long as server exists
@@ -1446,7 +1577,12 @@ async fn kill_process_on_port(port: u16) {
#[cfg(test)]
mod db_wedge_tests {
- use super::{DbWedgeState, WedgeAction};
+ use super::{
+ capture_intended_now, db_wedge_recovery_decision, DbWedgeRecoveryDecision, DbWedgeState,
+ WedgeAction,
+ };
+ use screenpipe_db::WriteQueueHealth;
+ use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
const WINDOW: Duration = Duration::from_secs(600);
@@ -1511,6 +1647,50 @@ mod db_wedge_tests {
t += WINDOW + Duration::from_secs(1);
}
}
+
+ #[test]
+ fn debounce_recheck_restarts_only_the_same_unrecovered_generation() {
+ let signaled = WriteQueueHealth::default();
+ let same_generation = signaled.clone();
+ let epoch = signaled.fatal_run_recovery_epoch();
+
+ assert_eq!(
+ db_wedge_recovery_decision(&signaled, epoch, Some(&same_generation)),
+ DbWedgeRecoveryDecision::Restart
+ );
+ assert_eq!(
+ db_wedge_recovery_decision(&signaled, epoch.wrapping_add(1), Some(&same_generation)),
+ DbWedgeRecoveryDecision::SkipRecovered
+ );
+ }
+
+ #[test]
+ fn debounce_recheck_ignores_stale_generation_signals() {
+ let signaled = WriteQueueHealth::default();
+ let replacement = WriteQueueHealth::default();
+ let epoch = signaled.fatal_run_recovery_epoch();
+
+ assert_eq!(
+ db_wedge_recovery_decision(&signaled, epoch, None),
+ DbWedgeRecoveryDecision::SkipNoServer
+ );
+ assert_eq!(
+ db_wedge_recovery_decision(&signaled, epoch, Some(&replacement)),
+ DbWedgeRecoveryDecision::SkipSupersededGeneration
+ );
+ }
+
+ #[test]
+ fn stop_during_debounce_is_honored_when_recovery_respawns() {
+ let wants_recording = AtomicBool::new(true);
+
+ // The hook does not cache capture intent when it fires. A tray stop
+ // during the debounce clears the shared flag, and the server thread
+ // reads that latest value immediately before constructing capture.
+ wants_recording.store(false, Ordering::SeqCst);
+
+ assert!(!capture_intended_now(&wants_recording));
+ }
}
#[cfg(test)]
diff --git a/crates/screenpipe-db/src/failpoint_vfs.rs b/crates/screenpipe-db/src/failpoint_vfs.rs
index 3363d9b19..8aa78fbb3 100644
--- a/crates/screenpipe-db/src/failpoint_vfs.rs
+++ b/crates/screenpipe-db/src/failpoint_vfs.rs
@@ -458,23 +458,41 @@ mod tests {
"persistent-failure hook (the engine-restart request) must fire"
);
assert!(health.persistent_failure_signals() >= 1);
+ let recovery_epoch = health.fatal_run_recovery_epoch();
// --- Simulate the cure the hook requests (an engine restart clears the fault).
disarm();
- // The queue heals in-process now the condition cleared: the next write
- // succeeds and health returns to healthy. (The OLD code stayed wedged.)
- let recovered = queue
- .submit(WriteOp::InsertAudioChunk {
- file_path: "/post/ok".into(),
- timestamp: None,
- })
- .await;
- assert!(
- recovered.is_ok(),
- "write must recover once the fault clears: {:?}",
- recovered.as_ref().err()
- );
+ // The queue heals in-process now the condition cleared. Operational
+ // health becomes green after the first write, but the recovery epoch
+ // advances only after three consecutive healthy batches. That preserves
+ // the July 2 protection against one lucky commit cancelling recovery.
+ for (index, path) in ["/post/ok", "/post/ok-2", "/post/ok-3"]
+ .into_iter()
+ .enumerate()
+ {
+ let recovered = queue
+ .submit(WriteOp::InsertAudioChunk {
+ file_path: path.into(),
+ timestamp: None,
+ })
+ .await;
+ assert!(
+ recovered.is_ok(),
+ "healthy write {index} must recover once the fault clears: {:?}",
+ recovered.as_ref().err()
+ );
+ let expected_epoch = if index < 2 {
+ recovery_epoch
+ } else {
+ recovery_epoch + 1
+ };
+ assert_eq!(
+ health.fatal_run_recovery_epoch(),
+ expected_epoch,
+ "recovery epoch must advance only after the full healthy streak"
+ );
+ }
assert!(
!health.is_degraded(),
"health recovers after a successful write"
diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rs
index a8de1fd9f..c79570ee0 100644
--- a/crates/screenpipe-db/src/write_queue.rs
+++ b/crates/screenpipe-db/src/write_queue.rs
@@ -217,6 +217,10 @@ struct WriteQueueHealthInner {
total_contention_batches: std::sync::atomic::AtomicU64,
write_pool_reopens: std::sync::atomic::AtomicU64,
persistent_failure_signals: std::sync::atomic::AtomicU64,
+ /// Advances only after the escalation state observes the full healthy
+ /// streak required to end a fatal run. A recovery hook snapshots this
+ /// value before its debounce and cancels a stale restart when it changes.
+ fatal_run_recovery_epoch: std::sync::atomic::AtomicU64,
degraded: AtomicBool,
last_success_unix_ms: std::sync::atomic::AtomicI64,
}
@@ -242,6 +246,20 @@ impl WriteQueueHealth {
pub fn persistent_failure_signals(&self) -> u64 {
self.inner.persistent_failure_signals.load(Ordering::SeqCst)
}
+ /// Generation of completed fatal-run recoveries for this write queue.
+ ///
+ /// Unlike `is_degraded` and `consecutive_fatal_batches`, this does not
+ /// change after one lucky successful batch. It advances only once the
+ /// existing three-healthy-batch rule has cleared the fatal run.
+ pub fn fatal_run_recovery_epoch(&self) -> u64 {
+ self.inner.fatal_run_recovery_epoch.load(Ordering::SeqCst)
+ }
+ /// True when both handles observe the same write-queue generation.
+ /// A new `DatabaseManager` gets a fresh health instance, so this is a
+ /// lightweight generation check that does not keep SQLite pools alive.
+ pub fn is_same_instance(&self, other: &Self) -> bool {
+ Arc::ptr_eq(&self.inner, &other.inner)
+ }
/// Unix-ms timestamp of the last healthy batch (0 if never).
pub fn last_success_unix_ms(&self) -> i64 {
self.inner.last_success_unix_ms.load(Ordering::SeqCst)
@@ -284,6 +302,11 @@ impl WriteQueueHealth {
.persistent_failure_signals
.fetch_add(1, Ordering::SeqCst);
}
+ fn note_fatal_run_recovered(&self) {
+ self.inner
+ .fatal_run_recovery_epoch
+ .fetch_add(1, Ordering::SeqCst);
+ }
}
/// Hook invoked once when writes have failed persistently — the seam the app uses
@@ -801,6 +824,7 @@ async fn drain_loop(
}
health.record_success();
if escalation.on_healthy() {
+ health.note_fatal_run_recovered();
info!("write_queue: fatal run cleared");
}
}
@@ -2214,6 +2238,36 @@ mod tests {
);
}
+ #[test]
+ fn recovery_epoch_requires_the_full_healthy_streak() {
+ let health = WriteQueueHealth::default();
+ let cloned = health.clone();
+ let fresh = WriteQueueHealth::default();
+ assert!(health.is_same_instance(&cloned));
+ assert!(!health.is_same_instance(&fresh));
+
+ let mut escalation = esc();
+ let now = std::time::Instant::now();
+ escalation.on_fatal(now);
+ health.record_fatal();
+ let epoch = health.fatal_run_recovery_epoch();
+
+ for healthy in 1..FATAL_RUN_CLEAR_AFTER {
+ health.record_success();
+ assert!(!escalation.on_healthy());
+ assert_eq!(
+ health.fatal_run_recovery_epoch(),
+ epoch,
+ "healthy batch {healthy} must not cancel a pending recovery"
+ );
+ }
+
+ health.record_success();
+ assert!(escalation.on_healthy());
+ health.note_fatal_run_recovered();
+ assert_eq!(health.fatal_run_recovery_epoch(), epoch + 1);
+ }
+
#[test]
fn escalation_refires_while_run_persists() {
// The requested engine restart may not cure the wedge (leaked
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index 3ffab9e3c..73df3e2a7 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -13,7 +13,7 @@ use std::sync::{
atomic::{AtomicU64, Ordering},
Arc,
};
-use tokio::sync::RwLock;
+use tokio::sync::{Mutex, RwLock};
use tracing::{debug, warn};
use screenpipe_audio::audio_manager::builder::TranscriptionMode;
@@ -30,6 +30,9 @@ use crate::ui_recorder::{
/// times per second. The response only changes meaningfully every ~1s.
static HEALTH_CACHE: std::sync::LazyLock<RwLock<(u64, Option<HealthCheckResponse>)>> =
std::sync::LazyLock::new(|| RwLock::new((0, None)));
+/// Single-flight gate for full health recomputation. Cache misses crossing the
+/// same one-second boundary must not all run the DB-backed backlog query.
+static HEALTH_REFRESH: std::sync::LazyLock<Mutex<()>> = std::sync::LazyLock::new(|| Mutex::new(()));
type AudioReconciliationBacklogCache = (i64, Option<(u64, Option<DateTime<Utc>>)>);
static AUDIO_RECONCILIATION_BACKLOG_CACHE: std::sync::LazyLock<
RwLock<AudioReconciliationBacklogCache>,
@@ -452,51 +455,105 @@ pub struct AudioPipelineHealthInfo {
/// killed by a watchdog).
const HEALTH_RESPONSE_BUDGET: std::time::Duration = std::time::Duration::from_secs(2);
-#[oasgen]
-pub async fn health_check(State(state): State<Arc<AppState>>) -> JsonResponse<HealthCheckResponse> {
- let now_ts = std::time::SystemTime::now()
+fn unix_now_secs() -> u64 {
+ std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
- .unwrap()
- .as_secs();
+ .unwrap_or_default()
+ .as_secs()
+}
- // Return cached response if still fresh. This prevents thundering-herd
- // scenarios where dozens of WebSocket clients + HTTP polls recompute the
- // full health response simultaneously.
+async fn cached_health_or_refresh<F, Fut>(
+ cache: &RwLock<(u64, Option<HealthCheckResponse>)>,
+ refresh: &Mutex<()>,
+ ttl_secs: u64,
+ compute: F,
+) -> HealthCheckResponse
+where
+ F: FnOnce() -> Fut,
+ Fut: std::future::Future<Output = Option<HealthCheckResponse>>,
+{
+ let now = unix_now_secs();
{
- let cache = HEALTH_CACHE.read().await;
- if now_ts.saturating_sub(cache.0) < HEALTH_CACHE_TTL_SECS {
- if let Some(ref cached) = cache.1 {
- return JsonResponse(cached.clone());
+ let cached = cache.read().await;
+ if now.saturating_sub(cached.0) < ttl_secs {
+ if let Some(response) = cached.1.as_ref() {
+ return response.clone();
}
}
}
- let response = match tokio::time::timeout(HEALTH_RESPONSE_BUDGET, health_check_inner(&state))
- .await
- {
- Ok(r) => r,
+ // Prefer stale-while-refresh over queuing every tray/WebSocket/HTTP poll
+ // behind a potentially two-second health computation. Cold start has no
+ // stale value, so those callers wait for the single refresh to finish.
+ let _refresh_guard = match refresh.try_lock() {
+ Ok(guard) => guard,
Err(_) => {
- // Inner computation exceeded the budget. Serve the last cached
- // snapshot (even if past TTL) so callers see continuity. If no
- // snapshot exists yet, return a minimal "degraded" response —
- // never block forever.
- warn!(
- "health_check: inner computation exceeded {:?} budget — serving last cached snapshot",
- HEALTH_RESPONSE_BUDGET
- );
- let cached = HEALTH_CACHE.read().await.1.clone();
- // Don't refresh the cache timestamp here — the next caller
- // should re-attempt rather than amortize the stale entry.
- return JsonResponse(cached.unwrap_or_else(degraded_response));
+ {
+ let cached = cache.read().await;
+ if let Some(response) = cached.1.as_ref() {
+ return response.clone();
+ }
+ }
+ refresh.lock().await
}
};
- // Cache the result
+ // Another cold-start caller may have populated the cache while this one
+ // waited for the refresh gate.
+ let now = unix_now_secs();
{
- let mut cache = HEALTH_CACHE.write().await;
- *cache = (now_ts, Some(response.clone()));
+ let cached = cache.read().await;
+ if now.saturating_sub(cached.0) < ttl_secs {
+ if let Some(response) = cached.1.as_ref() {
+ return response.clone();
+ }
+ }
}
+ let response = match compute().await {
+ Some(response) => response,
+ None => cache
+ .read()
+ .await
+ .1
+ .clone()
+ .unwrap_or_else(degraded_response),
+ };
+
+ // Publish timeout results too. Without this, a cold-cache burst queues on
+ // the refresh mutex and every waiter performs its own full two-second
+ // computation after the first timeout. The normal one-second TTL makes
+ // this a short backoff, while all callers in the same burst share one
+ // bounded attempt (or the same stale snapshot).
+ let mut cached = cache.write().await;
+ *cached = (unix_now_secs(), Some(response.clone()));
+ response
+}
+
+#[oasgen]
+pub async fn health_check(State(state): State<Arc<AppState>>) -> JsonResponse<HealthCheckResponse> {
+ let response = cached_health_or_refresh(
+ &HEALTH_CACHE,
+ &HEALTH_REFRESH,
+ HEALTH_CACHE_TTL_SECS,
+ || async {
+ match tokio::time::timeout(HEALTH_RESPONSE_BUDGET, health_check_inner(&state)).await {
+ Ok(response) => Some(response),
+ Err(_) => {
+ // The shared refresh helper publishes this failed attempt
+ // for one short TTL so cold-cache peers do not serialize
+ // another full computation each.
+ warn!(
+ "health_check: inner computation exceeded {:?} budget — serving last cached snapshot",
+ HEALTH_RESPONSE_BUDGET
+ );
+ None
+ }
+ }
+ },
+ )
+ .await;
+
JsonResponse(response)
}
@@ -1624,6 +1681,100 @@ mod tests {
}
}
+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+ async fn concurrent_stale_health_requests_share_one_refresh() {
+ const CALLERS: usize = 24;
+ let cache = Arc::new(RwLock::new((0, None)));
+ let refresh = Arc::new(Mutex::new(()));
+ let barrier = Arc::new(tokio::sync::Barrier::new(CALLERS));
+ let computes = Arc::new(AtomicU64::new(0));
+ let mut tasks = Vec::with_capacity(CALLERS);
+
+ for _ in 0..CALLERS {
+ let cache = cache.clone();
+ let refresh = refresh.clone();
+ let barrier = barrier.clone();
+ let computes = computes.clone();
+ tasks.push(tokio::spawn(async move {
+ barrier.wait().await;
+ cached_health_or_refresh(&cache, &refresh, 60, || async move {
+ computes.fetch_add(1, Ordering::SeqCst);
+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+ Some(dummy_response("healthy"))
+ })
+ .await
+ }));
+ }
+
+ for task in tasks {
+ assert_eq!(task.await.unwrap().status, "healthy");
+ }
+ assert_eq!(
+ computes.load(Ordering::SeqCst),
+ 1,
+ "a stale-cache burst must perform exactly one full health refresh"
+ );
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+ async fn concurrent_cold_timeout_is_published_to_all_waiters() {
+ const CALLERS: usize = 24;
+ let cache = Arc::new(RwLock::new((0, None)));
+ let refresh = Arc::new(Mutex::new(()));
+ let barrier = Arc::new(tokio::sync::Barrier::new(CALLERS));
+ let computes = Arc::new(AtomicU64::new(0));
+ let mut tasks = Vec::with_capacity(CALLERS);
+
+ for _ in 0..CALLERS {
+ let cache = cache.clone();
+ let refresh = refresh.clone();
+ let barrier = barrier.clone();
+ let computes = computes.clone();
+ tasks.push(tokio::spawn(async move {
+ barrier.wait().await;
+ cached_health_or_refresh(&cache, &refresh, 60, || async move {
+ computes.fetch_add(1, Ordering::SeqCst);
+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+ None
+ })
+ .await
+ }));
+ }
+
+ for task in tasks {
+ let response = task.await.unwrap();
+ assert_eq!(response.status, "degraded");
+ assert_eq!(response.status_code, 503);
+ }
+ assert_eq!(
+ computes.load(Ordering::SeqCst),
+ 1,
+ "a cold timeout burst must share one bounded refresh attempt"
+ );
+ }
+
+ #[tokio::test]
+ async fn concurrent_refresh_serves_an_existing_stale_snapshot() {
+ let cache = RwLock::new((0, Some(dummy_response("stale"))));
+ let refresh = Mutex::new(());
+ let in_flight = refresh.lock().await;
+ let computes = AtomicU64::new(0);
+
+ let response = tokio::time::timeout(
+ std::time::Duration::from_millis(100),
+ cached_health_or_refresh(&cache, &refresh, 60, || async {
+ computes.fetch_add(1, Ordering::SeqCst);
+ Some(dummy_response("unexpected"))
+ }),
+ )
+ .await
+ .expect("stale-while-refresh must not queue behind the active refresh");
+
+ assert_eq!(response.status, "stale");
+ assert_eq!(computes.load(Ordering::SeqCst), 0);
+ drop(in_flight);
+ }
+
#[test]
fn health_response_is_cloneable() {
let resp = dummy_response("healthy");