Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): quiesce pools before WAL restart (#6914)

Ezra Ellette1a645b0ba4d0
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/e2e/seeds.rs b/apps/screenpipe-app-tauri/src-tauri/src/e2e/seeds.rs
index b4a13e0fc..eef9a8519 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/e2e/seeds.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/e2e/seeds.rs
@@ -574,10 +574,10 @@ async fn checkpoint_hard_fault_fixture(database: &DatabaseManager) -> Result<(),
.await
.map_err(|error| format!("failed to checkpoint corruption fixture: {error}"))?;
last_checkpoint = Some((busy, log_pages, checkpointed_pages));
- // Production checkpoints deliberately use RESTART instead of TRUNCATE:
- // the WAL can remain physically allocated even after every logical
- // frame is durable in the main database. Requiring zero frames here
- // confuses that safe allocation reuse with an incomplete checkpoint.
+ // Production checkpoints deliberately use PASSIVE: the WAL can remain
+ // physically allocated even after every logical frame is durable in
+ // the main database. Requiring zero frames here confuses safe
+ // allocation reuse with an incomplete checkpoint.
if busy == 0 && checkpointed_pages == log_pages {
return Ok(());
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
index 356199943..e19c72cc9 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
@@ -2114,8 +2114,8 @@ async fn main() {
// auto-restarts recording (rebuilding every pool +
// the shared WAL-index).
let db_health = server.db.write_queue_health();
- server.db.set_persistent_failure_hook(
- crate::recording::make_db_wedge_recovery_hook(
+ server.db.set_database_restart_hook(
+ crate::recording::make_database_restart_hook(
app_for_db_wedge.clone(),
db_wedge_breaker.clone(),
db_health,
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
index 5df244372..8b7705b2f 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
@@ -299,7 +299,7 @@ const CAPTURE_RESTART_MEETING_REATTACH_WINDOW: Duration = Duration::from_secs(12
mod db_wedge;
pub use db_wedge::{
- make_db_wedge_recovery_hook, new_db_wedge_breaker, DbWedgeBreaker, DbWedgeState,
+ make_database_restart_hook, new_db_wedge_breaker, DbWedgeBreaker, DbWedgeState,
};
#[derive(Clone, Debug)]
@@ -1466,7 +1466,7 @@ async fn spawn_screenpipe_inner(
let db_health = server.db.write_queue_health();
server
.db
- .set_persistent_failure_hook(make_db_wedge_recovery_hook(
+ .set_database_restart_hook(make_database_restart_hook(
app_for_db_wedge.clone(),
db_wedge_breaker.clone(),
db_health,
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs
index da94efd05..4013be8a5 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs
@@ -7,6 +7,7 @@
use super::{
bounded_teardown, spawn_screenpipe_inner, RecordingState, TeardownOutcome,
};
+use screenpipe_db::DatabaseRestartReason;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
@@ -116,7 +117,7 @@ enum DbWedgeRecoveryDecision {
fn db_wedge_recovery_decision(
signaled_health: &screenpipe_db::WriteQueueHealth,
- signaled_recovery_epoch: u64,
+ signaled_recovery_epoch: Option<u64>,
current_health: Option<&screenpipe_db::WriteQueueHealth>,
) -> DbWedgeRecoveryDecision {
let Some(current_health) = current_health else {
@@ -125,32 +126,35 @@ fn db_wedge_recovery_decision(
if !signaled_health.is_same_instance(current_health) {
return DbWedgeRecoveryDecision::SkipSupersededGeneration;
}
- if current_health.fatal_run_recovery_epoch() != signaled_recovery_epoch {
+ if signaled_recovery_epoch
+ .is_some_and(|epoch| current_health.fatal_run_recovery_epoch() != epoch)
+ {
return DbWedgeRecoveryDecision::SkipRecovered;
}
DbWedgeRecoveryDecision::Restart
}
-/// Build the `PersistentFailureHook` the DB layer fires when writes wedge
-/// persistently. The hook itself is sync (`Fn()`), so it spawns the async
-/// restart. Captures an `AppHandle` (cheap clone, Send+Sync) and the shared
-/// breaker so restart-storm protection persists across restarts.
-pub fn make_db_wedge_recovery_hook(
+/// Build the lifecycle hook the DB layer fires when the current SQLite
+/// generation must be replaced. The hook itself is synchronous, so it spawns
+/// the async restart. Captures an `AppHandle` (cheap clone, Send+Sync) and the
+/// shared breaker so restart-storm protection persists across restarts.
+pub fn make_database_restart_hook(
app: tauri::AppHandle,
breaker: DbWedgeBreaker,
health: screenpipe_db::WriteQueueHealth,
-) -> screenpipe_db::PersistentFailureHook {
- std::sync::Arc::new(move || {
+) -> screenpipe_db::DatabaseRestartHook {
+ std::sync::Arc::new(move |reason| {
let app = app.clone();
let breaker = breaker.clone();
let health = health.clone();
- let recovery_epoch = health.fatal_run_recovery_epoch();
+ let recovery_epoch = matches!(reason, DatabaseRestartReason::PersistentWriteFailure)
+ .then(|| health.fatal_run_recovery_epoch());
// The hook fires on the dedicated *server* runtime. Recovery removes
// that server from state, which intentionally lets its runtime exit;
// running this task there would cancel it halfway through respawn.
// Dispatch onto Tauri's process-lifetime runtime instead.
tauri::async_runtime::spawn(async move {
- recover_from_db_wedge(app, breaker, health, recovery_epoch).await;
+ recover_from_db_wedge(app, breaker, health, recovery_epoch, reason).await;
});
})
}
@@ -159,7 +163,8 @@ async fn recover_from_db_wedge(
app: tauri::AppHandle,
breaker: DbWedgeBreaker,
signaled_health: screenpipe_db::WriteQueueHealth,
- signaled_recovery_epoch: u64,
+ signaled_recovery_epoch: Option<u64>,
+ reason: DatabaseRestartReason,
) {
// Debounce: let a burst of signals coalesce and any in-flight work settle.
tokio::time::sleep(DB_WEDGE_DEBOUNCE).await;
@@ -236,8 +241,8 @@ async fn recover_from_db_wedge(
.expect("restart decision requires a current server generation");
warn!(
- "db wedge auto-recovery: persistent write failure detected — restarting recording to \
- rebuild all DB pools + the shared WAL-index"
+ ?reason,
+ "database lifecycle recovery requested — restarting recording to rebuild all DB pools + the shared WAL-index"
);
*recording_state.interrupted_meeting.lock().await = None;
@@ -429,11 +434,20 @@ mod tests {
let epoch = signaled.fatal_run_recovery_epoch();
assert_eq!(
- db_wedge_recovery_decision(&signaled, epoch, Some(&same_generation)),
+ db_wedge_recovery_decision(&signaled, Some(epoch), Some(&same_generation)),
DbWedgeRecoveryDecision::Restart
);
assert_eq!(
- db_wedge_recovery_decision(&signaled, epoch.wrapping_add(1), Some(&same_generation)),
+ db_wedge_recovery_decision(&signaled, None, Some(&same_generation)),
+ DbWedgeRecoveryDecision::Restart,
+ "non-write-wedge lifecycle requests are generation-bound, not fatal-run-bound"
+ );
+ assert_eq!(
+ db_wedge_recovery_decision(
+ &signaled,
+ Some(epoch.wrapping_add(1)),
+ Some(&same_generation)
+ ),
DbWedgeRecoveryDecision::SkipRecovered
);
}
@@ -445,11 +459,11 @@ mod tests {
let epoch = signaled.fatal_run_recovery_epoch();
assert_eq!(
- db_wedge_recovery_decision(&signaled, epoch, None),
+ db_wedge_recovery_decision(&signaled, Some(epoch), None),
DbWedgeRecoveryDecision::SkipNoServer
);
assert_eq!(
- db_wedge_recovery_decision(&signaled, epoch, Some(&replacement)),
+ db_wedge_recovery_decision(&signaled, Some(epoch), Some(&replacement)),
DbWedgeRecoveryDecision::SkipSupersededGeneration
);
}
diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rs
index 097e21212..c136739f7 100644
--- a/crates/screenpipe-db/src/db/maintenance.rs
+++ b/crates/screenpipe-db/src/db/maintenance.rs
@@ -6,6 +6,20 @@ use super::*;
const WAL_HARD_CAP_PAGES: i32 = 40_000;
+fn wal_backlog_restart_reason(
+ log_pages: i32,
+ checkpointed_pages: i32,
+) -> Option<crate::write_queue::DatabaseRestartReason> {
+ let pending_pages = log_pages.saturating_sub(checkpointed_pages);
+ (pending_pages > WAL_HARD_CAP_PAGES).then_some(
+ crate::write_queue::DatabaseRestartReason::WalBacklog {
+ pending_pages,
+ log_pages,
+ checkpointed_pages,
+ },
+ )
+}
+
async fn run_routine_wal_checkpoint(pool: &SqlitePool) -> Result<(i32, i32, i32), sqlx::Error> {
// Routine maintenance must never shorten the live WAL file. A TRUNCATE
// checkpoint can make an already-open connection short-read the old WAL
@@ -29,19 +43,6 @@ async fn run_guarded_routine_wal_checkpoint(
run_routine_wal_checkpoint(pool).await.map(Some)
}
-async fn run_wal_restart_checkpoint(
- connection: &mut sqlx::SqliteConnection,
-) -> Result<(i32, i32, i32), sqlx::Error> {
- // RESTART has the same completion guarantee as the former hard-cap
- // TRUNCATE checkpoint, but deliberately leaves the WAL file allocated.
- // Once existing readers drain, SQLite can reuse the file from its start
- // without changing its physical length underneath another connection.
- let row = sqlx::query("PRAGMA wal_checkpoint(RESTART)")
- .fetch_one(connection)
- .await?;
- Ok((row.get(0), row.get(1), row.get(2)))
-}
-
impl DatabaseManager {
/// Execute trusted, row-returning dynamic SQL on the query pool.
pub async fn query_raw_sql(&self, query: &str) -> Result<serde_json::Value, sqlx::Error> {
@@ -1355,6 +1356,7 @@ impl DatabaseManager {
let shutdown = self.close_token.clone();
let write_queue_health = self.write_queue_health.clone();
let write_semaphore = std::sync::Arc::clone(&self.write_semaphore);
+ let persistent_failure_hook = self.persistent_failure_hook.clone();
tokio::spawn(async move {
// 60s (not 300s): with inline auto-checkpoint off, the WAL grows for
// the whole interval between ticks, so check more often to keep it
@@ -1399,57 +1401,23 @@ impl DatabaseManager {
match run_guarded_routine_wal_checkpoint(&pool, &write_queue_health).await {
Ok(Some((busy, log_pages, checkpointed))) => {
let backlog_pages = log_pages.saturating_sub(checkpointed);
- if backlog_pages > WAL_HARD_CAP_PAGES {
+ if let Some(reason) = wal_backlog_restart_reason(log_pages, checkpointed) {
warn!(
- "passive wal checkpoint left {} pages pending; waiting for a non-truncating restart checkpoint (log={}, checkpointed={}, busy={})",
+ "passive wal checkpoint left {} pages pending; requesting a full engine restart before WAL reuse (log={}, checkpointed={}, busy={})",
backlog_pages, log_pages, checkpointed, busy
);
- // PASSIVE must remain the common path, but a reader
- // can otherwise pin the WAL forever. Preserve the
- // hard growth ceiling with RESTART: it waits for old
- // readers and makes the next writer reuse the WAL,
- // without physically shortening the live file.
- match pool.acquire().await {
- Ok(mut conn) => {
- if let Err(e) = sqlx::query("PRAGMA busy_timeout = 60000")
- .execute(&mut *conn)
- .await
- {
- warn!("failed to set checkpoint busy timeout: {}", e);
- }
- match run_wal_restart_checkpoint(&mut conn).await {
- Ok((restart_busy, restart_log, restart_checkpointed)) => {
- let restart_backlog =
- restart_log.saturating_sub(restart_checkpointed);
- if restart_busy == 1 || restart_backlog > 0 {
- warn!(
- "restart wal checkpoint left {} pages pending (log={}, checkpointed={}, busy={})",
- restart_backlog,
- restart_log,
- restart_checkpointed,
- restart_busy
- );
- } else {
- info!(
- "restart wal checkpoint completed without truncation (checkpointed {}/{})",
- restart_checkpointed, restart_log
- );
- }
- }
- Err(e) => warn!("restart wal checkpoint failed: {}", e),
- }
- if let Err(e) = sqlx::query("PRAGMA busy_timeout = 5000")
- .execute(&mut *conn)
- .await
- {
- warn!("failed to restore checkpoint busy timeout: {}", e);
- }
- }
- Err(e) => warn!(
- "failed to acquire connection for restart checkpoint: {}",
- e
- ),
+ // The writer coordinator cannot quiesce query-pool
+ // readers or an external opener, so a serialized
+ // RESTART is not a complete lifecycle boundary. Ask
+ // the owner to stop capture, drain readers, close
+ // both pools, and respawn; startup checkpoints before
+ // any traffic is admitted.
+ if persistent_failure_hook.signal_wal_backlog_restart(reason) {
+ return;
}
+ warn!(
+ "wal hard cap exceeded but no lifecycle restart hook is installed; staying on safe PASSIVE checkpoints"
+ );
} else {
debug!(
"passive wal checkpoint: busy={}, checkpointed {}/{} pages ({} pending)",
@@ -1524,7 +1492,7 @@ impl DatabaseManager {
);
if first_for_manager {
if let Some(hook) = persistent_failure_hook.take_hard_fault_hook() {
- hook();
+ hook(crate::write_queue::DatabaseRestartReason::SqliteHardFault);
}
}
}
@@ -1536,7 +1504,7 @@ impl DatabaseManager {
shutdown.cancel();
if first_for_manager {
if let Some(hook) = persistent_failure_hook.take_hard_fault_hook() {
- hook();
+ hook(crate::write_queue::DatabaseRestartReason::SqliteHardFault);
}
}
}
@@ -1562,11 +1530,7 @@ impl DatabaseManager {
if self.write_queue_health.is_hard_faulted() {
return Err(SqlxError::PoolClosed);
}
- let mut conn = self.write_pool.acquire().await?;
- let row = sqlx::query("PRAGMA wal_checkpoint(RESTART)")
- .fetch_one(&mut *conn)
- .await?;
- Ok((row.get(0), row.get(1), row.get(2)))
+ run_routine_wal_checkpoint(&self.write_pool).await
}
/// Create an atomic backup of the database using `VACUUM INTO`.
@@ -1611,9 +1575,6 @@ impl DatabaseManager {
let _ = sqlx::query("PRAGMA busy_timeout = 60000")
.execute(&mut *conn)
.await;
- let _ = sqlx::query("PRAGMA wal_checkpoint(RESTART)")
- .execute(&mut *conn)
- .await;
let result = sqlx::query("VACUUM").execute(&mut *conn).await.map(|_| ());
// Restore the default busy_timeout on this pooled connection.
let _ = sqlx::query("PRAGMA busy_timeout = 5000")
@@ -1626,12 +1587,14 @@ impl DatabaseManager {
#[cfg(test)]
mod wal_maintenance_tests {
use super::{
- run_guarded_routine_wal_checkpoint, run_routine_wal_checkpoint, run_wal_restart_checkpoint,
- DatabaseManager,
+ run_guarded_routine_wal_checkpoint, run_routine_wal_checkpoint, wal_backlog_restart_reason,
+ DatabaseManager, WAL_HARD_CAP_PAGES,
};
+ use crate::write_queue::{persistent_failure_slot, DatabaseRestartReason};
use screenpipe_config::{DbConfig, DeviceTier};
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
use sqlx::Row;
+ use std::sync::{Arc, Mutex};
use std::time::Duration;
#[tokio::test]
@@ -1791,7 +1754,7 @@ mod wal_maintenance_tests {
}
#[tokio::test]
- async fn restart_checkpoint_drains_reader_backlog_without_truncating_wal() {
+ async fn reader_pinned_backlog_requests_one_lifecycle_restart_without_resetting_wal() {
let dir = tempfile::tempdir().expect("temp dir");
let db_path = dir.path().join("db.sqlite");
let wal_path = dir.path().join("db.sqlite-wal");
@@ -1841,50 +1804,66 @@ mod wal_maintenance_tests {
"reader must leave frames pending for the escalation test"
);
- let mut checkpoint_connection = pool.acquire().await.expect("checkpoint connection");
- let checkpoint = tokio::spawn(async move {
- let result = run_wal_restart_checkpoint(&mut checkpoint_connection).await;
- (result, checkpoint_connection)
- });
- tokio::time::sleep(std::time::Duration::from_millis(50)).await;
- assert!(
- !checkpoint.is_finished(),
- "restart checkpoint must wait while the old reader is active"
+ let observed = Arc::new(Mutex::new(Vec::new()));
+ let observed_from_hook = Arc::clone(&observed);
+ let restart_hook = persistent_failure_slot(Some(Arc::new(move |reason| {
+ observed_from_hook.lock().unwrap().push(reason);
+ })));
+ let reason = DatabaseRestartReason::WalBacklog {
+ pending_pages: log_pages.saturating_sub(checkpointed),
+ log_pages,
+ checkpointed_pages: checkpointed,
+ };
+ assert!(restart_hook.signal_wal_backlog_restart(reason));
+ assert!(restart_hook.signal_wal_backlog_restart(reason));
+ assert_eq!(
+ observed.lock().unwrap().as_slice(),
+ &[reason],
+ "one manager generation must request at most one lifecycle restart"
);
- reader.rollback().await.expect("release reader snapshot");
- let (checkpoint_result, checkpoint_connection) = checkpoint
- .await
- .expect("restart checkpoint task must not panic");
- let (busy, restart_log, restart_checkpointed) =
- checkpoint_result.expect("restart checkpoint");
- assert_eq!(busy, 0, "restart checkpoint should complete");
- assert_eq!(restart_checkpointed, restart_log, "all frames must drain");
let wal_size_after = std::fs::metadata(&wal_path)
- .expect("WAL remains allocated after restart checkpoint")
+ .expect("WAL remains allocated after restart request")
.len();
assert_eq!(
wal_size_after, wal_size_before,
- "restart escalation must not physically truncate the live WAL"
+ "requesting lifecycle recovery must not reset the live WAL"
);
- sqlx::query("INSERT INTO events (body) VALUES ('after-restart')")
+ sqlx::query("INSERT INTO events (body) VALUES ('before-lifecycle-handoff')")
.execute(&pool)
.await
- .expect("write after restart");
+ .expect("restart request itself must not mutate or block SQLite");
+ reader.rollback().await.expect("release reader snapshot");
let integrity: String = sqlx::query_scalar("PRAGMA integrity_check")
.fetch_one(&pool)
.await
.expect("integrity check");
assert_eq!(integrity, "ok");
- drop(checkpoint_connection);
pool.close().await;
}
+ #[test]
+ fn wal_backlog_policy_preserves_the_existing_hard_cap() {
+ assert_eq!(
+ wal_backlog_restart_reason(WAL_HARD_CAP_PAGES, 0),
+ None,
+ "the ceiling is exclusive"
+ );
+ assert_eq!(
+ wal_backlog_restart_reason(WAL_HARD_CAP_PAGES + 1, 0),
+ Some(DatabaseRestartReason::WalBacklog {
+ pending_pages: WAL_HARD_CAP_PAGES + 1,
+ log_pages: WAL_HARD_CAP_PAGES + 1,
+ checkpointed_pages: 0,
+ })
+ );
+ }
+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "manual 60-second production scheduler chaos test with a ~170 MB WAL"]
- async fn production_scheduler_restarts_oversized_reader_pinned_wal_without_truncation() {
+ async fn production_scheduler_requests_lifecycle_for_oversized_reader_pinned_wal() {
let dir = tempfile::tempdir().expect("temp dir");
let db_path = dir.path().join("db.sqlite");
let wal_path = dir.path().join("db.sqlite-wal");
@@ -1908,7 +1887,7 @@ mod wal_maintenance_tests {
// One transaction creates more than WAL_HARD_CAP_PAGES real pages while
// the old reader pins the first frame. This exercises the exact
- // production timer + threshold + RESTART branch after 60 seconds.
+ // production timer + threshold + lifecycle-handoff branch after 60 seconds.
sqlx::query(
"WITH RECURSIVE rows(n) AS (\
VALUES(1) UNION ALL SELECT n + 1 FROM rows WHERE n < 42000\
@@ -1933,43 +1912,31 @@ mod wal_maintenance_tests {
"WAL was not realistically large"
);
- // `start_wal_maintenance()` consumed its immediate tick at manager
- // construction. At 60s it must enter RESTART and remain blocked on our
- // reader until we release it here.
- tokio::time::sleep(Duration::from_secs(62)).await;
- reader.rollback().await.expect("release pinned reader");
-
- // The coordinator is held until RESTART completes. A production write
- // through the coordinator therefore proves the scheduled pass drained.
- let mut tx = tokio::time::timeout(Duration::from_secs(10), db.begin_immediate_with_retry())
- .await
- .expect("scheduled restart did not release coordinator")
- .expect("begin write after scheduled restart");
- sqlx::query("INSERT INTO wal_cap_chaos(payload) VALUES (x'01')")
- .execute(&mut **tx.conn())
+ let (restart_tx, restart_rx) = tokio::sync::oneshot::channel();
+ let restart_tx = Arc::new(Mutex::new(Some(restart_tx)));
+ db.set_database_restart_hook(Arc::new(move |reason| {
+ if let Some(tx) = restart_tx.lock().unwrap().take() {
+ let _ = tx.send(reason);
+ }
+ }));
+ let reason = tokio::time::timeout(Duration::from_secs(65), restart_rx)
.await
- .expect("write after scheduled restart");
- tx.commit().await.expect("commit after scheduled restart");
+ .expect("scheduled lifecycle request timed out")
+ .expect("scheduled lifecycle sender dropped");
+ assert!(matches!(
+ reason,
+ DatabaseRestartReason::WalBacklog { pending_pages, .. }
+ if pending_pages > WAL_HARD_CAP_PAGES
+ ));
let wal_size_after = std::fs::metadata(&wal_path)
.expect("WAL remains allocated")
.len();
assert_eq!(
wal_size_after, wal_size_before,
- "scheduled hard-cap escalation physically truncated the WAL"
- );
- let (_, reused_log_pages, _) = run_routine_wal_checkpoint(&db.pool)
- .await
- .expect("inspect reused WAL");
- assert!(
- reused_log_pages < 100,
- "next writer did not reuse WAL from its start: {reused_log_pages} pages"
+ "scheduled hard-cap handling reset the live WAL"
);
- let integrity: String = sqlx::query_scalar("PRAGMA integrity_check")
- .fetch_one(&db.pool)
- .await
- .expect("integrity after scheduled restart");
- assert_eq!(integrity, "ok");
+ reader.rollback().await.expect("release pinned reader");
db.close().await;
}
}
diff --git a/crates/screenpipe-db/src/db/mod.rs b/crates/screenpipe-db/src/db/mod.rs
index 5783e719a..4030107b6 100644
--- a/crates/screenpipe-db/src/db/mod.rs
+++ b/crates/screenpipe-db/src/db/mod.rs
@@ -158,7 +158,7 @@ impl HardFaultReporter {
self.close_token.cancel();
if first_for_manager {
if let Some(hook) = self.persistent_failure_hook.take_hard_fault_hook() {
- hook();
+ hook(crate::write_queue::DatabaseRestartReason::SqliteHardFault);
}
}
true
diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rs
index 4a1803c76..0b8e0e28f 100644
--- a/crates/screenpipe-db/src/db/setup.rs
+++ b/crates/screenpipe-db/src/db/setup.rs
@@ -896,18 +896,23 @@ impl DatabaseManager {
self.write_queue_health.clone()
}
- /// Set the hook fired when writes fail persistently (a process-wide WAL-index
- /// desync that only a full engine restart can clear). The app wires this to a
- /// recording restart. Safe to call after construction and to overwrite.
- pub fn set_persistent_failure_hook(&self, hook: crate::write_queue::PersistentFailureHook) {
+ /// Set the hook fired when this database generation needs every live pool
+ /// rebuilt. The app wires this to a recording restart. Safe to call after
+ /// construction and to overwrite.
+ pub fn set_database_restart_hook(&self, hook: crate::write_queue::DatabaseRestartHook) {
self.persistent_failure_hook.set_hook(hook);
if self.write_queue_health.is_hard_faulted() {
if let Some(hook) = self.persistent_failure_hook.take_hard_fault_hook() {
- hook();
+ hook(crate::write_queue::DatabaseRestartReason::SqliteHardFault);
}
}
}
+ /// Compatibility wrapper for callers using the former write-wedge name.
+ pub fn set_persistent_failure_hook(&self, hook: crate::write_queue::PersistentFailureHook) {
+ self.set_database_restart_hook(Arc::new(move |_| hook()));
+ }
+
/// Route a hard SQLite error observed outside the coalescing queue (for
/// example a read worker or direct transaction) through the same
/// process-wide quarantine and app recovery hook. Returns true only for
@@ -1214,7 +1219,7 @@ mod shutdown_tests {
assert!(database.report_sqlite_error(&full));
assert!(database.report_sqlite_error(&full));
assert_eq!(hook_calls.load(Ordering::SeqCst), 0);
- database.set_persistent_failure_hook(Arc::new(move || {
+ database.set_database_restart_hook(Arc::new(move |_| {
hook_counter.fetch_add(1, Ordering::SeqCst);
}));
assert!(database.write_queue_health().is_hard_faulted());
diff --git a/crates/screenpipe-db/src/failpoint_vfs.rs b/crates/screenpipe-db/src/failpoint_vfs.rs
index a15002d25..ead87c704 100644
--- a/crates/screenpipe-db/src/failpoint_vfs.rs
+++ b/crates/screenpipe-db/src/failpoint_vfs.rs
@@ -464,7 +464,7 @@ mod tests {
Arc::from(format!("{}", db.display()).as_str()),
WriteDrainOpts {
on_persistent_failure: crate::write_queue::persistent_failure_slot(Some(Arc::new(
- move || {
+ move |_| {
fired_hook.store(true, AtomicOrdering::SeqCst);
},
))),
@@ -784,7 +784,7 @@ mod tests {
);
let hook_calls = Arc::new(AtomicUsize::new(0));
let hook_counter = Arc::clone(&hook_calls);
- let hook = crate::write_queue::persistent_failure_slot(Some(Arc::new(move || {
+ let hook = crate::write_queue::persistent_failure_slot(Some(Arc::new(move |_| {
hook_counter.fetch_add(1, AtomicOrdering::SeqCst);
})));
let mut tx = crate::ImmediateTx::for_test(conn, permit, health.clone(), hook);
diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rs
index 5410c03e0..7ee8ee984 100644
--- a/crates/screenpipe-db/src/lib.rs
+++ b/crates/screenpipe-db/src/lib.rs
@@ -68,6 +68,6 @@ pub use text_normalizer::{expand_search_query, sanitize_fts5_query};
pub use types::*;
pub use write_queue::{
is_retryable_write_stall, is_write_lock_contention, is_write_pool_starved, request_write_pause,
- request_write_resume, PersistentFailureHook, SyncTable, WriteQueueHealth,
- WRITE_LOCK_HELD_MESSAGE, WRITE_POOL_STARVED_MESSAGE,
+ request_write_resume, DatabaseRestartHook, DatabaseRestartReason, PersistentFailureHook,
+ SyncTable, WriteQueueHealth, WRITE_LOCK_HELD_MESSAGE, WRITE_POOL_STARVED_MESSAGE,
};
diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rs
index 2cfdca206..3ea2b5b21 100644
--- a/crates/screenpipe-db/src/write_queue.rs
+++ b/crates/screenpipe-db/src/write_queue.rs
@@ -479,47 +479,87 @@ impl WriteQueueHealth {
}
}
-/// Hook invoked once when writes have failed persistently — the seam the app uses
-/// to restart the engine (rebuilding every pool + the shared WAL-index).
+/// Why the database layer needs the owning process to rebuild every live pool.
+///
+/// A WAL backlog is intentionally handled at the same lifecycle boundary as a
+/// write wedge: only the owner can stop capture, drain readers, close every
+/// connection, and reopen SQLite against a fresh WAL-index generation.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum DatabaseRestartReason {
+ PersistentWriteFailure,
+ SqliteHardFault,
+ WalBacklog {
+ pending_pages: i32,
+ log_pages: i32,
+ checkpointed_pages: i32,
+ },
+}
+
+/// Hook used by the database layer to ask its owner for a full lifecycle
+/// restart (rebuilding every pool + the shared WAL-index).
+pub type DatabaseRestartHook = Arc<dyn Fn(DatabaseRestartReason) + Send + Sync>;
+
+/// Legacy write-wedge hook retained for source compatibility. New owners should
+/// install a [`DatabaseRestartHook`] so they can observe every restart reason.
pub type PersistentFailureHook = Arc<dyn Fn() + Send + Sync>;
/// A slot the app fills (after `DatabaseManager` is built) with the
/// persistent-failure hook. Shared so the drain loop reads whatever the app
/// last set; empty until wired.
pub(crate) struct PersistentFailureState {
- hook: std::sync::Mutex<Option<PersistentFailureHook>>,
+ hook: std::sync::Mutex<Option<DatabaseRestartHook>>,
hard_fault_signaled: AtomicBool,
+ wal_backlog_signaled: AtomicBool,
}
pub(crate) type PersistentFailureSlot = Arc<PersistentFailureState>;
impl PersistentFailureState {
- pub(crate) fn hook(&self) -> Option<PersistentFailureHook> {
+ pub(crate) fn hook(&self) -> Option<DatabaseRestartHook> {
self.hook.lock().unwrap().clone()
}
- pub(crate) fn set_hook(&self, hook: PersistentFailureHook) {
+ pub(crate) fn set_hook(&self, hook: DatabaseRestartHook) {
*self.hook.lock().unwrap() = Some(hook);
}
/// Return the installed hook exactly once for a hard-fault generation.
/// If no hook is installed yet, leave the gate open so late app wiring can
/// deliver the already-observed startup fault.
- pub(crate) fn take_hard_fault_hook(&self) -> Option<PersistentFailureHook> {
+ pub(crate) fn take_hard_fault_hook(&self) -> Option<DatabaseRestartHook> {
let hook = self.hook()?;
self.hard_fault_signaled
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.ok()
.map(|_| hook)
}
+
+ /// Deliver at most one WAL-backlog restart request for this manager.
+ /// Leave the gate open until a hook is installed so embedders that wire the
+ /// lifecycle after construction can still receive the request on a later
+ /// maintenance pass.
+ pub(crate) fn signal_wal_backlog_restart(&self, reason: DatabaseRestartReason) -> bool {
+ debug_assert!(matches!(reason, DatabaseRestartReason::WalBacklog { .. }));
+ let Some(hook) = self.hook() else {
+ return false;
+ };
+ if self
+ .wal_backlog_signaled
+ .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
+ .is_err()
+ {
+ return true;
+ }
+ hook(reason);
+ true
+ }
}
-pub(crate) fn persistent_failure_slot(
- hook: Option<PersistentFailureHook>,
-) -> PersistentFailureSlot {
+pub(crate) fn persistent_failure_slot(hook: Option<DatabaseRestartHook>) -> PersistentFailureSlot {
Arc::new(PersistentFailureState {
hook: std::sync::Mutex::new(hook),
hard_fault_signaled: AtomicBool::new(false),
+ wal_backlog_signaled: AtomicBool::new(false),
})
}
@@ -1125,7 +1165,7 @@ async fn drain_loop(
);
let hook = on_persistent_failure.hook();
if let Some(hook) = hook {
- hook();
+ hook(DatabaseRestartReason::PersistentWriteFailure);
}
}
}
@@ -1180,7 +1220,7 @@ async fn drain_loop(
);
let hook = on_persistent_failure.hook();
if let Some(hook) = hook {
- hook();
+ hook(DatabaseRestartReason::PersistentWriteFailure);
}
}
}
@@ -1205,7 +1245,7 @@ async fn drain_loop(
shutdown.cancel();
let hook = on_persistent_failure.take_hard_fault_hook();
if let Some(hook) = hook {
- hook();
+ hook(DatabaseRestartReason::SqliteHardFault);
}
write_pool.close().await;
return;
@@ -2769,7 +2809,7 @@ mod tests {
Arc::from("sqlite::memory:"),
WriteDrainOpts {
health: health.clone(),
- on_persistent_failure: persistent_failure_slot(Some(Arc::new(move || {
+ on_persistent_failure: persistent_failure_slot(Some(Arc::new(move |_| {
hook_flag.store(true, AtomicOrdering::SeqCst);
}))),
// Escalate on the count rule so the test does not have to wait
diff --git a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
index daac4a582..8ac5c2a3f 100644
--- a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
+++ b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
@@ -40,6 +40,7 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {
rust_files(&repository.join("crates"), &mut files);
let mut live_truncation = Vec::new();
+ let mut live_capture_restart = Vec::new();
let mut legacy_secret_open = Vec::new();
let mut capture_pool_secret_store = Vec::new();
for path in files {
@@ -47,6 +48,12 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {
if source.contains("\"PRAGMA wal_checkpoint(TRUNCATE)") {
live_truncation.push(path.clone());
}
+ if path.starts_with(crate_dir.join("src"))
+ && !path.ends_with("screenpipe-db/src/db/setup.rs")
+ && source.contains("wal_checkpoint(RESTART)")
+ {
+ live_capture_restart.push(path.clone());
+ }
if !path.ends_with("screenpipe-secrets/src/store.rs")
&& source.contains("SecretStore::open(")
{
@@ -63,6 +70,10 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {
live_truncation.is_empty(),
"live WAL truncation is forbidden; use non-truncating checkpoints or offline recovery: {live_truncation:?}"
);
+ assert!(
+ live_capture_restart.is_empty(),
+ "live capture code must hand off to a full pool teardown before resetting WAL state: {live_capture_restart:?}"
+ );
assert!(
legacy_secret_open.is_empty(),
"production credential callers must use open_for_data_dir and secrets.sqlite: {legacy_secret_open:?}"
@@ -75,12 +86,21 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {
let maintenance = production_source(&crate_dir.join("src/db/maintenance.rs"));
assert!(!maintenance.contains("PRAGMA locking_mode = EXCLUSIVE"));
assert!(!maintenance.contains("PRAGMA synchronous = OFF"));
+ assert!(
+ !maintenance.contains("wal_checkpoint(RESTART)"),
+ "live maintenance must hand off to a full pool teardown before resetting WAL state"
+ );
assert!(maintenance.contains("online SQLite repair is disabled"));
let setup = production_source(&crate_dir.join("src/db/setup.rs"));
assert!(setup.contains("acquire_sqlite_manager_lease"));
assert!(!setup.contains("sqlx::Sqlite::database_exists("));
assert!(!setup.contains("sqlx::Sqlite::create_database("));
+ assert_eq!(
+ setup.matches("wal_checkpoint(RESTART)").count(),
+ 1,
+ "RESTART is reserved for the startup boundary before application traffic"
+ );
assert_eq!(
setup.matches("capture_pool_options()").count(),
2,
diff --git a/crates/screenpipe-engine/src/cli/backup.rs b/crates/screenpipe-engine/src/cli/backup.rs
index bd6c6c374..e3003dd5d 100644
--- a/crates/screenpipe-engine/src/cli/backup.rs
+++ b/crates/screenpipe-engine/src/cli/backup.rs
@@ -31,8 +31,8 @@ pub async fn handle_backup_command(
let (busy, log_pages, checkpointed) = db.wal_checkpoint().await?;
- if busy != 0 {
- eprintln!("warning: checkpoint was busy (another process holds the database)");
+ if busy != 0 || checkpointed != log_pages {
+ eprintln!("warning: checkpoint was incomplete (an active reader may hold the WAL)");
eprintln!(" wal pages: {}, checkpointed: {}", log_pages, checkpointed);
std::process::exit(1);
}
diff --git a/crates/screenpipe-engine/src/routes/data.rs b/crates/screenpipe-engine/src/routes/data.rs
index 8c8f3596d..2fb028c7c 100644
--- a/crates/screenpipe-engine/src/routes/data.rs
+++ b/crates/screenpipe-engine/src/routes/data.rs
@@ -406,10 +406,11 @@ pub(crate) async fn checkpoint_handler(
"manual wal checkpoint: busy={}, log_pages={}, checkpointed={}",
busy, log_pages, checkpointed
);
+ let complete = busy == 0 && checkpointed == log_pages;
Ok(JsonResponse(CheckpointResponse {
- success: busy == 0,
- busy: busy != 0,
+ success: complete,
+ busy: !complete,
wal_pages: log_pages,
checkpointed_pages: checkpointed,
}))