YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): quiesce pools before WAL restart (#6914)
Ezra Ellette1a645b0ba4d0
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/e2e/seeds.rs b/apps/screenpipe-app-tauri/src-tauri/src/e2e/seeds.rsindex b4a13e0fc..eef9a8519 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/e2e/seeds.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/e2e/seeds.rs@@ -574,10 +574,10 @@ async fn checkpoint_hard_fault_fixture(database: &DatabaseManager) -> Result<(),.await.map_err(|error| format!("failed to checkpoint corruption fixture: {error}"))?;last_checkpoint = Some((busy, log_pages, checkpointed_pages));- // Production checkpoints deliberately use RESTART instead of TRUNCATE:- // the WAL can remain physically allocated even after every logical- // frame is durable in the main database. Requiring zero frames here- // confuses that safe allocation reuse with an incomplete checkpoint.+ // Production checkpoints deliberately use PASSIVE: the WAL can remain+ // physically allocated even after every logical frame is durable in+ // the main database. Requiring zero frames here confuses safe+ // allocation reuse with an incomplete checkpoint.if busy == 0 && checkpointed_pages == log_pages {return Ok(());}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rsindex 356199943..e19c72cc9 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs@@ -2114,8 +2114,8 @@ async fn main() {// auto-restarts recording (rebuilding every pool +// the shared WAL-index).let db_health = server.db.write_queue_health();- server.db.set_persistent_failure_hook(- crate::recording::make_db_wedge_recovery_hook(+ server.db.set_database_restart_hook(+ crate::recording::make_database_restart_hook(app_for_db_wedge.clone(),db_wedge_breaker.clone(),db_health,diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rsindex 5df244372..8b7705b2f 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs@@ -299,7 +299,7 @@ const CAPTURE_RESTART_MEETING_REATTACH_WINDOW: Duration = Duration::from_secs(12mod db_wedge;pub use db_wedge::{- make_db_wedge_recovery_hook, new_db_wedge_breaker, DbWedgeBreaker, DbWedgeState,+ make_database_restart_hook, new_db_wedge_breaker, DbWedgeBreaker, DbWedgeState,};#[derive(Clone, Debug)]@@ -1466,7 +1466,7 @@ async fn spawn_screenpipe_inner(let db_health = server.db.write_queue_health();server.db- .set_persistent_failure_hook(make_db_wedge_recovery_hook(+ .set_database_restart_hook(make_database_restart_hook(app_for_db_wedge.clone(),db_wedge_breaker.clone(),db_health,diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rsindex da94efd05..4013be8a5 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs@@ -7,6 +7,7 @@use super::{bounded_teardown, spawn_screenpipe_inner, RecordingState, TeardownOutcome,};+use screenpipe_db::DatabaseRestartReason;use std::sync::atomic::Ordering;use std::sync::Arc;use std::time::Duration;@@ -116,7 +117,7 @@ enum DbWedgeRecoveryDecision {fn db_wedge_recovery_decision(signaled_health: &screenpipe_db::WriteQueueHealth,- signaled_recovery_epoch: u64,+ signaled_recovery_epoch: Option<u64>,current_health: Option<&screenpipe_db::WriteQueueHealth>,) -> DbWedgeRecoveryDecision {let Some(current_health) = current_health else {@@ -125,32 +126,35 @@ fn db_wedge_recovery_decision(if !signaled_health.is_same_instance(current_health) {return DbWedgeRecoveryDecision::SkipSupersededGeneration;}- if current_health.fatal_run_recovery_epoch() != signaled_recovery_epoch {+ if signaled_recovery_epoch+ .is_some_and(|epoch| current_health.fatal_run_recovery_epoch() != epoch)+ {return DbWedgeRecoveryDecision::SkipRecovered;}DbWedgeRecoveryDecision::Restart}-/// Build the `PersistentFailureHook` the DB layer fires when writes wedge-/// persistently. The hook itself is sync (`Fn()`), so it spawns the async-/// restart. Captures an `AppHandle` (cheap clone, Send+Sync) and the shared-/// breaker so restart-storm protection persists across restarts.-pub fn make_db_wedge_recovery_hook(+/// Build the lifecycle hook the DB layer fires when the current SQLite+/// generation must be replaced. The hook itself is synchronous, so it spawns+/// the async restart. Captures an `AppHandle` (cheap clone, Send+Sync) and the+/// shared breaker so restart-storm protection persists across restarts.+pub fn make_database_restart_hook(app: tauri::AppHandle,breaker: DbWedgeBreaker,health: screenpipe_db::WriteQueueHealth,-) -> screenpipe_db::PersistentFailureHook {- std::sync::Arc::new(move || {+) -> screenpipe_db::DatabaseRestartHook {+ std::sync::Arc::new(move |reason| {let app = app.clone();let breaker = breaker.clone();let health = health.clone();- let recovery_epoch = health.fatal_run_recovery_epoch();+ let recovery_epoch = matches!(reason, DatabaseRestartReason::PersistentWriteFailure)+ .then(|| health.fatal_run_recovery_epoch());// The hook fires on the dedicated *server* runtime. Recovery removes// that server from state, which intentionally lets its runtime exit;// running this task there would cancel it halfway through respawn.// Dispatch onto Tauri's process-lifetime runtime instead.tauri::async_runtime::spawn(async move {- recover_from_db_wedge(app, breaker, health, recovery_epoch).await;+ recover_from_db_wedge(app, breaker, health, recovery_epoch, reason).await;});})}@@ -159,7 +163,8 @@ async fn recover_from_db_wedge(app: tauri::AppHandle,breaker: DbWedgeBreaker,signaled_health: screenpipe_db::WriteQueueHealth,- signaled_recovery_epoch: u64,+ signaled_recovery_epoch: Option<u64>,+ reason: DatabaseRestartReason,) {// Debounce: let a burst of signals coalesce and any in-flight work settle.tokio::time::sleep(DB_WEDGE_DEBOUNCE).await;@@ -236,8 +241,8 @@ async fn recover_from_db_wedge(.expect("restart decision requires a current server generation");warn!(- "db wedge auto-recovery: persistent write failure detected — restarting recording to \- rebuild all DB pools + the shared WAL-index"+ ?reason,+ "database lifecycle recovery requested — restarting recording to rebuild all DB pools + the shared WAL-index");*recording_state.interrupted_meeting.lock().await = None;@@ -429,11 +434,20 @@ mod tests {let epoch = signaled.fatal_run_recovery_epoch();assert_eq!(- db_wedge_recovery_decision(&signaled, epoch, Some(&same_generation)),+ db_wedge_recovery_decision(&signaled, Some(epoch), Some(&same_generation)),DbWedgeRecoveryDecision::Restart);assert_eq!(- db_wedge_recovery_decision(&signaled, epoch.wrapping_add(1), Some(&same_generation)),+ db_wedge_recovery_decision(&signaled, None, Some(&same_generation)),+ DbWedgeRecoveryDecision::Restart,+ "non-write-wedge lifecycle requests are generation-bound, not fatal-run-bound"+ );+ assert_eq!(+ db_wedge_recovery_decision(+ &signaled,+ Some(epoch.wrapping_add(1)),+ Some(&same_generation)+ ),DbWedgeRecoveryDecision::SkipRecovered);}@@ -445,11 +459,11 @@ mod tests {let epoch = signaled.fatal_run_recovery_epoch();assert_eq!(- db_wedge_recovery_decision(&signaled, epoch, None),+ db_wedge_recovery_decision(&signaled, Some(epoch), None),DbWedgeRecoveryDecision::SkipNoServer);assert_eq!(- db_wedge_recovery_decision(&signaled, epoch, Some(&replacement)),+ db_wedge_recovery_decision(&signaled, Some(epoch), Some(&replacement)),DbWedgeRecoveryDecision::SkipSupersededGeneration);}diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rsindex 097e21212..c136739f7 100644--- a/crates/screenpipe-db/src/db/maintenance.rs+++ b/crates/screenpipe-db/src/db/maintenance.rs@@ -6,6 +6,20 @@ use super::*;const WAL_HARD_CAP_PAGES: i32 = 40_000;+fn wal_backlog_restart_reason(+ log_pages: i32,+ checkpointed_pages: i32,+) -> Option<crate::write_queue::DatabaseRestartReason> {+ let pending_pages = log_pages.saturating_sub(checkpointed_pages);+ (pending_pages > WAL_HARD_CAP_PAGES).then_some(+ crate::write_queue::DatabaseRestartReason::WalBacklog {+ pending_pages,+ log_pages,+ checkpointed_pages,+ },+ )+}+async fn run_routine_wal_checkpoint(pool: &SqlitePool) -> Result<(i32, i32, i32), sqlx::Error> {// Routine maintenance must never shorten the live WAL file. A TRUNCATE// checkpoint can make an already-open connection short-read the old WAL@@ -29,19 +43,6 @@ async fn run_guarded_routine_wal_checkpoint(run_routine_wal_checkpoint(pool).await.map(Some)}-async fn run_wal_restart_checkpoint(- connection: &mut sqlx::SqliteConnection,-) -> Result<(i32, i32, i32), sqlx::Error> {- // RESTART has the same completion guarantee as the former hard-cap- // TRUNCATE checkpoint, but deliberately leaves the WAL file allocated.- // Once existing readers drain, SQLite can reuse the file from its start- // without changing its physical length underneath another connection.- let row = sqlx::query("PRAGMA wal_checkpoint(RESTART)")- .fetch_one(connection)- .await?;- Ok((row.get(0), row.get(1), row.get(2)))-}-impl DatabaseManager {/// Execute trusted, row-returning dynamic SQL on the query pool.pub async fn query_raw_sql(&self, query: &str) -> Result<serde_json::Value, sqlx::Error> {@@ -1355,6 +1356,7 @@ impl DatabaseManager {let shutdown = self.close_token.clone();let write_queue_health = self.write_queue_health.clone();let write_semaphore = std::sync::Arc::clone(&self.write_semaphore);+ let persistent_failure_hook = self.persistent_failure_hook.clone();tokio::spawn(async move {// 60s (not 300s): with inline auto-checkpoint off, the WAL grows for// the whole interval between ticks, so check more often to keep it@@ -1399,57 +1401,23 @@ impl DatabaseManager {match run_guarded_routine_wal_checkpoint(&pool, &write_queue_health).await {Ok(Some((busy, log_pages, checkpointed))) => {let backlog_pages = log_pages.saturating_sub(checkpointed);- if backlog_pages > WAL_HARD_CAP_PAGES {+ if let Some(reason) = wal_backlog_restart_reason(log_pages, checkpointed) {warn!(- "passive wal checkpoint left {} pages pending; waiting for a non-truncating restart checkpoint (log={}, checkpointed={}, busy={})",+ "passive wal checkpoint left {} pages pending; requesting a full engine restart before WAL reuse (log={}, checkpointed={}, busy={})",backlog_pages, log_pages, checkpointed, busy);- // PASSIVE must remain the common path, but a reader- // can otherwise pin the WAL forever. Preserve the- // hard growth ceiling with RESTART: it waits for old- // readers and makes the next writer reuse the WAL,- // without physically shortening the live file.- match pool.acquire().await {- Ok(mut conn) => {- if let Err(e) = sqlx::query("PRAGMA busy_timeout = 60000")- .execute(&mut *conn)- .await- {- warn!("failed to set checkpoint busy timeout: {}", e);- }- match run_wal_restart_checkpoint(&mut conn).await {- Ok((restart_busy, restart_log, restart_checkpointed)) => {- let restart_backlog =- restart_log.saturating_sub(restart_checkpointed);- if restart_busy == 1 || restart_backlog > 0 {- warn!(- "restart wal checkpoint left {} pages pending (log={}, checkpointed={}, busy={})",- restart_backlog,- restart_log,- restart_checkpointed,- restart_busy- );- } else {- info!(- "restart wal checkpoint completed without truncation (checkpointed {}/{})",- restart_checkpointed, restart_log- );- }- }- Err(e) => warn!("restart wal checkpoint failed: {}", e),- }- if let Err(e) = sqlx::query("PRAGMA busy_timeout = 5000")- .execute(&mut *conn)- .await- {- warn!("failed to restore checkpoint busy timeout: {}", e);- }- }- Err(e) => warn!(- "failed to acquire connection for restart checkpoint: {}",- e- ),+ // The writer coordinator cannot quiesce query-pool+ // readers or an external opener, so a serialized+ // RESTART is not a complete lifecycle boundary. Ask+ // the owner to stop capture, drain readers, close+ // both pools, and respawn; startup checkpoints before+ // any traffic is admitted.+ if persistent_failure_hook.signal_wal_backlog_restart(reason) {+ return;}+ warn!(+ "wal hard cap exceeded but no lifecycle restart hook is installed; staying on safe PASSIVE checkpoints"+ );} else {debug!("passive wal checkpoint: busy={}, checkpointed {}/{} pages ({} pending)",@@ -1524,7 +1492,7 @@ impl DatabaseManager {);if first_for_manager {if let Some(hook) = persistent_failure_hook.take_hard_fault_hook() {- hook();+ hook(crate::write_queue::DatabaseRestartReason::SqliteHardFault);}}}@@ -1536,7 +1504,7 @@ impl DatabaseManager {shutdown.cancel();if first_for_manager {if let Some(hook) = persistent_failure_hook.take_hard_fault_hook() {- hook();+ hook(crate::write_queue::DatabaseRestartReason::SqliteHardFault);}}}@@ -1562,11 +1530,7 @@ impl DatabaseManager {if self.write_queue_health.is_hard_faulted() {return Err(SqlxError::PoolClosed);}- let mut conn = self.write_pool.acquire().await?;- let row = sqlx::query("PRAGMA wal_checkpoint(RESTART)")- .fetch_one(&mut *conn)- .await?;- Ok((row.get(0), row.get(1), row.get(2)))+ run_routine_wal_checkpoint(&self.write_pool).await}/// Create an atomic backup of the database using `VACUUM INTO`.@@ -1611,9 +1575,6 @@ impl DatabaseManager {let _ = sqlx::query("PRAGMA busy_timeout = 60000").execute(&mut *conn).await;- let _ = sqlx::query("PRAGMA wal_checkpoint(RESTART)")- .execute(&mut *conn)- .await;let result = sqlx::query("VACUUM").execute(&mut *conn).await.map(|_| ());// Restore the default busy_timeout on this pooled connection.let _ = sqlx::query("PRAGMA busy_timeout = 5000")@@ -1626,12 +1587,14 @@ impl DatabaseManager {#[cfg(test)]mod wal_maintenance_tests {use super::{- run_guarded_routine_wal_checkpoint, run_routine_wal_checkpoint, run_wal_restart_checkpoint,- DatabaseManager,+ run_guarded_routine_wal_checkpoint, run_routine_wal_checkpoint, wal_backlog_restart_reason,+ DatabaseManager, WAL_HARD_CAP_PAGES,};+ use crate::write_queue::{persistent_failure_slot, DatabaseRestartReason};use screenpipe_config::{DbConfig, DeviceTier};use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};use sqlx::Row;+ use std::sync::{Arc, Mutex};use std::time::Duration;#[tokio::test]@@ -1791,7 +1754,7 @@ mod wal_maintenance_tests {}#[tokio::test]- async fn restart_checkpoint_drains_reader_backlog_without_truncating_wal() {+ async fn reader_pinned_backlog_requests_one_lifecycle_restart_without_resetting_wal() {let dir = tempfile::tempdir().expect("temp dir");let db_path = dir.path().join("db.sqlite");let wal_path = dir.path().join("db.sqlite-wal");@@ -1841,50 +1804,66 @@ mod wal_maintenance_tests {"reader must leave frames pending for the escalation test");- let mut checkpoint_connection = pool.acquire().await.expect("checkpoint connection");- let checkpoint = tokio::spawn(async move {- let result = run_wal_restart_checkpoint(&mut checkpoint_connection).await;- (result, checkpoint_connection)- });- tokio::time::sleep(std::time::Duration::from_millis(50)).await;- assert!(- !checkpoint.is_finished(),- "restart checkpoint must wait while the old reader is active"+ let observed = Arc::new(Mutex::new(Vec::new()));+ let observed_from_hook = Arc::clone(&observed);+ let restart_hook = persistent_failure_slot(Some(Arc::new(move |reason| {+ observed_from_hook.lock().unwrap().push(reason);+ })));+ let reason = DatabaseRestartReason::WalBacklog {+ pending_pages: log_pages.saturating_sub(checkpointed),+ log_pages,+ checkpointed_pages: checkpointed,+ };+ assert!(restart_hook.signal_wal_backlog_restart(reason));+ assert!(restart_hook.signal_wal_backlog_restart(reason));+ assert_eq!(+ observed.lock().unwrap().as_slice(),+ &[reason],+ "one manager generation must request at most one lifecycle restart");- reader.rollback().await.expect("release reader snapshot");- let (checkpoint_result, checkpoint_connection) = checkpoint- .await- .expect("restart checkpoint task must not panic");- let (busy, restart_log, restart_checkpointed) =- checkpoint_result.expect("restart checkpoint");- assert_eq!(busy, 0, "restart checkpoint should complete");- assert_eq!(restart_checkpointed, restart_log, "all frames must drain");let wal_size_after = std::fs::metadata(&wal_path)- .expect("WAL remains allocated after restart checkpoint")+ .expect("WAL remains allocated after restart request").len();assert_eq!(wal_size_after, wal_size_before,- "restart escalation must not physically truncate the live WAL"+ "requesting lifecycle recovery must not reset the live WAL");- sqlx::query("INSERT INTO events (body) VALUES ('after-restart')")+ sqlx::query("INSERT INTO events (body) VALUES ('before-lifecycle-handoff')").execute(&pool).await- .expect("write after restart");+ .expect("restart request itself must not mutate or block SQLite");+ reader.rollback().await.expect("release reader snapshot");let integrity: String = sqlx::query_scalar("PRAGMA integrity_check").fetch_one(&pool).await.expect("integrity check");assert_eq!(integrity, "ok");- drop(checkpoint_connection);pool.close().await;}+ #[test]+ fn wal_backlog_policy_preserves_the_existing_hard_cap() {+ assert_eq!(+ wal_backlog_restart_reason(WAL_HARD_CAP_PAGES, 0),+ None,+ "the ceiling is exclusive"+ );+ assert_eq!(+ wal_backlog_restart_reason(WAL_HARD_CAP_PAGES + 1, 0),+ Some(DatabaseRestartReason::WalBacklog {+ pending_pages: WAL_HARD_CAP_PAGES + 1,+ log_pages: WAL_HARD_CAP_PAGES + 1,+ checkpointed_pages: 0,+ })+ );+ }+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]#[ignore = "manual 60-second production scheduler chaos test with a ~170 MB WAL"]- async fn production_scheduler_restarts_oversized_reader_pinned_wal_without_truncation() {+ async fn production_scheduler_requests_lifecycle_for_oversized_reader_pinned_wal() {let dir = tempfile::tempdir().expect("temp dir");let db_path = dir.path().join("db.sqlite");let wal_path = dir.path().join("db.sqlite-wal");@@ -1908,7 +1887,7 @@ mod wal_maintenance_tests {// One transaction creates more than WAL_HARD_CAP_PAGES real pages while// the old reader pins the first frame. This exercises the exact- // production timer + threshold + RESTART branch after 60 seconds.+ // production timer + threshold + lifecycle-handoff branch after 60 seconds.sqlx::query("WITH RECURSIVE rows(n) AS (\VALUES(1) UNION ALL SELECT n + 1 FROM rows WHERE n < 42000\@@ -1933,43 +1912,31 @@ mod wal_maintenance_tests {"WAL was not realistically large");- // `start_wal_maintenance()` consumed its immediate tick at manager- // construction. At 60s it must enter RESTART and remain blocked on our- // reader until we release it here.- tokio::time::sleep(Duration::from_secs(62)).await;- reader.rollback().await.expect("release pinned reader");-- // The coordinator is held until RESTART completes. A production write- // through the coordinator therefore proves the scheduled pass drained.- let mut tx = tokio::time::timeout(Duration::from_secs(10), db.begin_immediate_with_retry())- .await- .expect("scheduled restart did not release coordinator")- .expect("begin write after scheduled restart");- sqlx::query("INSERT INTO wal_cap_chaos(payload) VALUES (x'01')")- .execute(&mut **tx.conn())+ let (restart_tx, restart_rx) = tokio::sync::oneshot::channel();+ let restart_tx = Arc::new(Mutex::new(Some(restart_tx)));+ db.set_database_restart_hook(Arc::new(move |reason| {+ if let Some(tx) = restart_tx.lock().unwrap().take() {+ let _ = tx.send(reason);+ }+ }));+ let reason = tokio::time::timeout(Duration::from_secs(65), restart_rx).await- .expect("write after scheduled restart");- tx.commit().await.expect("commit after scheduled restart");+ .expect("scheduled lifecycle request timed out")+ .expect("scheduled lifecycle sender dropped");+ assert!(matches!(+ reason,+ DatabaseRestartReason::WalBacklog { pending_pages, .. }+ if pending_pages > WAL_HARD_CAP_PAGES+ ));let wal_size_after = std::fs::metadata(&wal_path).expect("WAL remains allocated").len();assert_eq!(wal_size_after, wal_size_before,- "scheduled hard-cap escalation physically truncated the WAL"- );- let (_, reused_log_pages, _) = run_routine_wal_checkpoint(&db.pool)- .await- .expect("inspect reused WAL");- assert!(- reused_log_pages < 100,- "next writer did not reuse WAL from its start: {reused_log_pages} pages"+ "scheduled hard-cap handling reset the live WAL");- let integrity: String = sqlx::query_scalar("PRAGMA integrity_check")- .fetch_one(&db.pool)- .await- .expect("integrity after scheduled restart");- assert_eq!(integrity, "ok");+ reader.rollback().await.expect("release pinned reader");db.close().await;}}diff --git a/crates/screenpipe-db/src/db/mod.rs b/crates/screenpipe-db/src/db/mod.rsindex 5783e719a..4030107b6 100644--- a/crates/screenpipe-db/src/db/mod.rs+++ b/crates/screenpipe-db/src/db/mod.rs@@ -158,7 +158,7 @@ impl HardFaultReporter {self.close_token.cancel();if first_for_manager {if let Some(hook) = self.persistent_failure_hook.take_hard_fault_hook() {- hook();+ hook(crate::write_queue::DatabaseRestartReason::SqliteHardFault);}}truediff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex 4a1803c76..0b8e0e28f 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -896,18 +896,23 @@ impl DatabaseManager {self.write_queue_health.clone()}- /// Set the hook fired when writes fail persistently (a process-wide WAL-index- /// desync that only a full engine restart can clear). The app wires this to a- /// recording restart. Safe to call after construction and to overwrite.- pub fn set_persistent_failure_hook(&self, hook: crate::write_queue::PersistentFailureHook) {+ /// Set the hook fired when this database generation needs every live pool+ /// rebuilt. The app wires this to a recording restart. Safe to call after+ /// construction and to overwrite.+ pub fn set_database_restart_hook(&self, hook: crate::write_queue::DatabaseRestartHook) {self.persistent_failure_hook.set_hook(hook);if self.write_queue_health.is_hard_faulted() {if let Some(hook) = self.persistent_failure_hook.take_hard_fault_hook() {- hook();+ hook(crate::write_queue::DatabaseRestartReason::SqliteHardFault);}}}+ /// Compatibility wrapper for callers using the former write-wedge name.+ pub fn set_persistent_failure_hook(&self, hook: crate::write_queue::PersistentFailureHook) {+ self.set_database_restart_hook(Arc::new(move |_| hook()));+ }+/// Route a hard SQLite error observed outside the coalescing queue (for/// example a read worker or direct transaction) through the same/// process-wide quarantine and app recovery hook. Returns true only for@@ -1214,7 +1219,7 @@ mod shutdown_tests {assert!(database.report_sqlite_error(&full));assert!(database.report_sqlite_error(&full));assert_eq!(hook_calls.load(Ordering::SeqCst), 0);- database.set_persistent_failure_hook(Arc::new(move || {+ database.set_database_restart_hook(Arc::new(move |_| {hook_counter.fetch_add(1, Ordering::SeqCst);}));assert!(database.write_queue_health().is_hard_faulted());diff --git a/crates/screenpipe-db/src/failpoint_vfs.rs b/crates/screenpipe-db/src/failpoint_vfs.rsindex a15002d25..ead87c704 100644--- a/crates/screenpipe-db/src/failpoint_vfs.rs+++ b/crates/screenpipe-db/src/failpoint_vfs.rs@@ -464,7 +464,7 @@ mod tests {Arc::from(format!("{}", db.display()).as_str()),WriteDrainOpts {on_persistent_failure: crate::write_queue::persistent_failure_slot(Some(Arc::new(- move || {+ move |_| {fired_hook.store(true, AtomicOrdering::SeqCst);},))),@@ -784,7 +784,7 @@ mod tests {);let hook_calls = Arc::new(AtomicUsize::new(0));let hook_counter = Arc::clone(&hook_calls);- let hook = crate::write_queue::persistent_failure_slot(Some(Arc::new(move || {+ let hook = crate::write_queue::persistent_failure_slot(Some(Arc::new(move |_| {hook_counter.fetch_add(1, AtomicOrdering::SeqCst);})));let mut tx = crate::ImmediateTx::for_test(conn, permit, health.clone(), hook);diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rsindex 5410c03e0..7ee8ee984 100644--- a/crates/screenpipe-db/src/lib.rs+++ b/crates/screenpipe-db/src/lib.rs@@ -68,6 +68,6 @@ pub use text_normalizer::{expand_search_query, sanitize_fts5_query};pub use types::*;pub use write_queue::{is_retryable_write_stall, is_write_lock_contention, is_write_pool_starved, request_write_pause,- request_write_resume, PersistentFailureHook, SyncTable, WriteQueueHealth,- WRITE_LOCK_HELD_MESSAGE, WRITE_POOL_STARVED_MESSAGE,+ request_write_resume, DatabaseRestartHook, DatabaseRestartReason, PersistentFailureHook,+ SyncTable, WriteQueueHealth, WRITE_LOCK_HELD_MESSAGE, WRITE_POOL_STARVED_MESSAGE,};diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rsindex 2cfdca206..3ea2b5b21 100644--- a/crates/screenpipe-db/src/write_queue.rs+++ b/crates/screenpipe-db/src/write_queue.rs@@ -479,47 +479,87 @@ impl WriteQueueHealth {}}-/// Hook invoked once when writes have failed persistently — the seam the app uses-/// to restart the engine (rebuilding every pool + the shared WAL-index).+/// Why the database layer needs the owning process to rebuild every live pool.+///+/// A WAL backlog is intentionally handled at the same lifecycle boundary as a+/// write wedge: only the owner can stop capture, drain readers, close every+/// connection, and reopen SQLite against a fresh WAL-index generation.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum DatabaseRestartReason {+ PersistentWriteFailure,+ SqliteHardFault,+ WalBacklog {+ pending_pages: i32,+ log_pages: i32,+ checkpointed_pages: i32,+ },+}++/// Hook used by the database layer to ask its owner for a full lifecycle+/// restart (rebuilding every pool + the shared WAL-index).+pub type DatabaseRestartHook = Arc<dyn Fn(DatabaseRestartReason) + Send + Sync>;++/// Legacy write-wedge hook retained for source compatibility. New owners should+/// install a [`DatabaseRestartHook`] so they can observe every restart reason.pub type PersistentFailureHook = Arc<dyn Fn() + Send + Sync>;/// A slot the app fills (after `DatabaseManager` is built) with the/// persistent-failure hook. Shared so the drain loop reads whatever the app/// last set; empty until wired.pub(crate) struct PersistentFailureState {- hook: std::sync::Mutex<Option<PersistentFailureHook>>,+ hook: std::sync::Mutex<Option<DatabaseRestartHook>>,hard_fault_signaled: AtomicBool,+ wal_backlog_signaled: AtomicBool,}pub(crate) type PersistentFailureSlot = Arc<PersistentFailureState>;impl PersistentFailureState {- pub(crate) fn hook(&self) -> Option<PersistentFailureHook> {+ pub(crate) fn hook(&self) -> Option<DatabaseRestartHook> {self.hook.lock().unwrap().clone()}- pub(crate) fn set_hook(&self, hook: PersistentFailureHook) {+ pub(crate) fn set_hook(&self, hook: DatabaseRestartHook) {*self.hook.lock().unwrap() = Some(hook);}/// Return the installed hook exactly once for a hard-fault generation./// If no hook is installed yet, leave the gate open so late app wiring can/// deliver the already-observed startup fault.- pub(crate) fn take_hard_fault_hook(&self) -> Option<PersistentFailureHook> {+ pub(crate) fn take_hard_fault_hook(&self) -> Option<DatabaseRestartHook> {let hook = self.hook()?;self.hard_fault_signaled.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst).ok().map(|_| hook)}++ /// Deliver at most one WAL-backlog restart request for this manager.+ /// Leave the gate open until a hook is installed so embedders that wire the+ /// lifecycle after construction can still receive the request on a later+ /// maintenance pass.+ pub(crate) fn signal_wal_backlog_restart(&self, reason: DatabaseRestartReason) -> bool {+ debug_assert!(matches!(reason, DatabaseRestartReason::WalBacklog { .. }));+ let Some(hook) = self.hook() else {+ return false;+ };+ if self+ .wal_backlog_signaled+ .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)+ .is_err()+ {+ return true;+ }+ hook(reason);+ true+ }}-pub(crate) fn persistent_failure_slot(- hook: Option<PersistentFailureHook>,-) -> PersistentFailureSlot {+pub(crate) fn persistent_failure_slot(hook: Option<DatabaseRestartHook>) -> PersistentFailureSlot {Arc::new(PersistentFailureState {hook: std::sync::Mutex::new(hook),hard_fault_signaled: AtomicBool::new(false),+ wal_backlog_signaled: AtomicBool::new(false),})}@@ -1125,7 +1165,7 @@ async fn drain_loop();let hook = on_persistent_failure.hook();if let Some(hook) = hook {- hook();+ hook(DatabaseRestartReason::PersistentWriteFailure);}}}@@ -1180,7 +1220,7 @@ async fn drain_loop();let hook = on_persistent_failure.hook();if let Some(hook) = hook {- hook();+ hook(DatabaseRestartReason::PersistentWriteFailure);}}}@@ -1205,7 +1245,7 @@ async fn drain_loop(shutdown.cancel();let hook = on_persistent_failure.take_hard_fault_hook();if let Some(hook) = hook {- hook();+ hook(DatabaseRestartReason::SqliteHardFault);}write_pool.close().await;return;@@ -2769,7 +2809,7 @@ mod tests {Arc::from("sqlite::memory:"),WriteDrainOpts {health: health.clone(),- on_persistent_failure: persistent_failure_slot(Some(Arc::new(move || {+ on_persistent_failure: persistent_failure_slot(Some(Arc::new(move |_| {hook_flag.store(true, AtomicOrdering::SeqCst);}))),// Escalate on the count rule so the test does not have to waitdiff --git a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rsindex daac4a582..8ac5c2a3f 100644--- a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs+++ b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs@@ -40,6 +40,7 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {rust_files(&repository.join("crates"), &mut files);let mut live_truncation = Vec::new();+ let mut live_capture_restart = Vec::new();let mut legacy_secret_open = Vec::new();let mut capture_pool_secret_store = Vec::new();for path in files {@@ -47,6 +48,12 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {if source.contains("\"PRAGMA wal_checkpoint(TRUNCATE)") {live_truncation.push(path.clone());}+ if path.starts_with(crate_dir.join("src"))+ && !path.ends_with("screenpipe-db/src/db/setup.rs")+ && source.contains("wal_checkpoint(RESTART)")+ {+ live_capture_restart.push(path.clone());+ }if !path.ends_with("screenpipe-secrets/src/store.rs")&& source.contains("SecretStore::open("){@@ -63,6 +70,10 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {live_truncation.is_empty(),"live WAL truncation is forbidden; use non-truncating checkpoints or offline recovery: {live_truncation:?}");+ assert!(+ live_capture_restart.is_empty(),+ "live capture code must hand off to a full pool teardown before resetting WAL state: {live_capture_restart:?}"+ );assert!(legacy_secret_open.is_empty(),"production credential callers must use open_for_data_dir and secrets.sqlite: {legacy_secret_open:?}"@@ -75,12 +86,21 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {let maintenance = production_source(&crate_dir.join("src/db/maintenance.rs"));assert!(!maintenance.contains("PRAGMA locking_mode = EXCLUSIVE"));assert!(!maintenance.contains("PRAGMA synchronous = OFF"));+ assert!(+ !maintenance.contains("wal_checkpoint(RESTART)"),+ "live maintenance must hand off to a full pool teardown before resetting WAL state"+ );assert!(maintenance.contains("online SQLite repair is disabled"));let setup = production_source(&crate_dir.join("src/db/setup.rs"));assert!(setup.contains("acquire_sqlite_manager_lease"));assert!(!setup.contains("sqlx::Sqlite::database_exists("));assert!(!setup.contains("sqlx::Sqlite::create_database("));+ assert_eq!(+ setup.matches("wal_checkpoint(RESTART)").count(),+ 1,+ "RESTART is reserved for the startup boundary before application traffic"+ );assert_eq!(setup.matches("capture_pool_options()").count(),2,diff --git a/crates/screenpipe-engine/src/cli/backup.rs b/crates/screenpipe-engine/src/cli/backup.rsindex bd6c6c374..e3003dd5d 100644--- a/crates/screenpipe-engine/src/cli/backup.rs+++ b/crates/screenpipe-engine/src/cli/backup.rs@@ -31,8 +31,8 @@ pub async fn handle_backup_command(let (busy, log_pages, checkpointed) = db.wal_checkpoint().await?;- if busy != 0 {- eprintln!("warning: checkpoint was busy (another process holds the database)");+ if busy != 0 || checkpointed != log_pages {+ eprintln!("warning: checkpoint was incomplete (an active reader may hold the WAL)");eprintln!(" wal pages: {}, checkpointed: {}", log_pages, checkpointed);std::process::exit(1);}diff --git a/crates/screenpipe-engine/src/routes/data.rs b/crates/screenpipe-engine/src/routes/data.rsindex 8c8f3596d..2fb028c7c 100644--- a/crates/screenpipe-engine/src/routes/data.rs+++ b/crates/screenpipe-engine/src/routes/data.rs@@ -406,10 +406,11 @@ pub(crate) async fn checkpoint_handler("manual wal checkpoint: busy={}, log_pages={}, checkpointed={}",busy, log_pages, checkpointed);+ let complete = busy == 0 && checkpointed == log_pages;Ok(JsonResponse(CheckpointResponse {- success: busy == 0,- busy: busy != 0,+ success: complete,+ busy: !complete,wal_pages: log_pages,checkpointed_pages: checkpointed,}))