Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): disable inline WAL auto-checkpoint to stop header-page corruption (#4946)

Louis Beaumont646cb3624e29
diff --git a/crates/screenpipe-config/src/defaults.rs b/crates/screenpipe-config/src/defaults.rs
index 8b33d5841..5c80e4035 100644
--- a/crates/screenpipe-config/src/defaults.rs
+++ b/crates/screenpipe-config/src/defaults.rs
@@ -160,6 +160,19 @@ impl Default for DbConfig {
/// checkpointed the same WAL on different thresholds. Apply EXACTLY these on
/// every pool; never set any of them to a different value on a side pool.
///
+/// `wal_autocheckpoint = 0` disables SQLite's INLINE auto-checkpoint entirely.
+/// With a non-zero threshold, whichever connection's COMMIT pushes the WAL past
+/// the threshold runs the checkpoint on *that* connection — under heavy load
+/// (large DB, many concurrent writers) a short read can leave the shared `-shm`
+/// WAL-index desynced, and the next inline checkpoint then copies a WAL frame to
+/// the *wrong* main-DB page (observed: a b-tree page written over page 1, the
+/// header). Setting it to `0` removes that under-load write path; ALL
+/// checkpointing is owned by the single serialized maintenance task
+/// (`start_wal_maintenance`), which also enforces a hard WAL-size ceiling so
+/// disabling auto-checkpoint cannot trade the corruption cliff for an
+/// unbounded-WAL cliff. Every pool still shares this one value (0), so the
+/// pool-parity invariant is preserved.
+///
/// `mmap_size` (must stay `0`, see [`DbConfig`]) and `cache_size` are applied
/// per-pool because they're tier-configurable; correctness only needs
/// `mmap_size == 0` everywhere, which the pool-parity test asserts separately.
@@ -169,10 +182,11 @@ pub const WAL_SAFETY_PRAGMAS: [(&str, &str); 4] = [
// the main DB) and cuts commit latency vs the FULL default.
("synchronous", "NORMAL"),
("temp_store", "MEMORY"),
- // Checkpoint after ~4000 pages (~16MB) not SQLite's default 1000 (~4MB).
- // CRITICAL: every pool over this file must use the SAME threshold or they
- // race checkpoints on the shared -shm WAL-index (the corruption above).
- ("wal_autocheckpoint", "4000"),
+ // 0 = no inline auto-checkpoint on committing connections. The single
+ // maintenance task is the sole checkpointer (see doc above). CRITICAL:
+ // every pool over this file must use the SAME value or they race
+ // checkpoints on the shared -shm WAL-index (the corruption above).
+ ("wal_autocheckpoint", "0"),
];
/// Audio/transcription channel capacities tuned per device tier.
diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rs
index 5daca15e5..89a32ce0d 100644
--- a/crates/screenpipe-db/src/db/maintenance.rs
+++ b/crates/screenpipe-db/src/db/maintenance.rs
@@ -1318,7 +1318,10 @@ impl DatabaseManager {
let restore_steps = [
"PRAGMA synchronous = NORMAL;",
"PRAGMA journal_mode = WAL;",
- "PRAGMA wal_autocheckpoint = 1000;",
+ // 0 = no inline auto-checkpoint (matches WAL_SAFETY_PRAGMAS); the
+ // maintenance task owns checkpointing. Must NOT re-enable inline
+ // auto-checkpoint here or a repaired DB re-opens the corruption path.
+ "PRAGMA wal_autocheckpoint = 0;",
"PRAGMA cache_size = -2000;", // Back to 2MB cache
"PRAGMA locking_mode = NORMAL;",
"PRAGMA busy_timeout = 5000;", // Back to 5s timeout
@@ -1352,13 +1355,33 @@ impl DatabaseManager {
}
}
- /// Spawn a background task that runs `PRAGMA wal_checkpoint(TRUNCATE)` every 5 minutes.
- /// This prevents unbounded WAL growth when long-running readers block auto-checkpoint.
+ /// Spawn the background task that owns ALL WAL checkpointing.
+ ///
+ /// Since `wal_autocheckpoint = 0` (see [`WAL_SAFETY_PRAGMAS`]) no committing
+ /// connection ever checkpoints inline — that under-load path could copy a
+ /// desynced `-shm` frame onto the wrong main-DB page. This task is therefore
+ /// the SOLE checkpointer, and it must (a) run often enough to keep the WAL
+ /// small and (b) never let the WAL grow without bound when readers keep a
+ /// plain `TRUNCATE` busy. It does a normal `TRUNCATE` each tick, and if the
+ /// WAL is over a hard page cap while still busy it escalates to the
+ /// serialized exclusive checkpoint (hold the single write permit so writers
+ /// queue, bump `busy_timeout` to wait out short-lived readers) — the same
+ /// reliable mechanism `compact()` uses. That escalation is the ceiling that
+ /// keeps `autocheckpoint = 0` from trading the corruption cliff for an
+ /// unbounded-WAL cliff on the heaviest install.
pub fn start_wal_maintenance(&self) {
let pool = self.pool.clone();
let shutdown = self.close_token.clone();
+ let write_semaphore = std::sync::Arc::clone(&self.write_semaphore);
tokio::spawn(async move {
- let mut interval = tokio::time::interval(Duration::from_secs(300));
+ // 60s (not 300s): with inline auto-checkpoint off, the WAL grows for
+ // the whole interval between ticks, so check more often to keep it
+ // small under sustained write load.
+ const INTERVAL: Duration = Duration::from_secs(60);
+ // ~40k pages * 4KB ≈ 160MB. Above this we force the checkpoint
+ // through rather than tolerate more growth.
+ const WAL_HARD_CAP_PAGES: i32 = 40_000;
+ let mut interval = tokio::time::interval(INTERVAL);
loop {
tokio::select! {
_ = interval.tick() => {}
@@ -1378,8 +1401,45 @@ impl DatabaseManager {
let busy: i32 = row.get(0);
let log_pages: i32 = row.get(1);
let checkpointed: i32 = row.get(2);
- if busy == 1 {
+ if busy == 1 && log_pages > WAL_HARD_CAP_PAGES {
+ // Readers kept the plain TRUNCATE busy and the WAL is
+ // over the cap. Force it: hold the single write
+ // permit (writers queue — a brief pause, like the
+ // compact path) and wait out short-lived readers.
warn!(
+ "wal checkpoint: busy with {} pages (> {} cap) — forcing exclusive checkpoint",
+ log_pages, WAL_HARD_CAP_PAGES
+ );
+ let _write_guard = write_semaphore.acquire().await.ok();
+ match pool.acquire().await {
+ Ok(mut conn) => {
+ let _ = sqlx::query("PRAGMA busy_timeout = 60000")
+ .execute(&mut *conn)
+ .await;
+ match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
+ .fetch_one(&mut *conn)
+ .await
+ {
+ Ok(r2) => {
+ let b2: i32 = r2.get(0);
+ let lp2: i32 = r2.get(1);
+ warn!(
+ "forced wal checkpoint done: busy={}, {} pages remain",
+ b2, lp2
+ );
+ }
+ Err(e) => warn!("forced wal checkpoint failed: {}", e),
+ }
+ // Restore the default busy_timeout before the
+ // connection returns to the pool.
+ let _ = sqlx::query("PRAGMA busy_timeout = 5000")
+ .execute(&mut *conn)
+ .await;
+ }
+ Err(e) => warn!("forced wal checkpoint: acquire failed: {}", e),
+ }
+ } else if busy == 1 {
+ debug!(
"wal checkpoint: busy (could not truncate), {} pages in WAL",
log_pages
);
diff --git a/crates/screenpipe-db/tests/multi_pool_wal_parity_test.rs b/crates/screenpipe-db/tests/multi_pool_wal_parity_test.rs
index 2ced0925c..209f2d109 100644
--- a/crates/screenpipe-db/tests/multi_pool_wal_parity_test.rs
+++ b/crates/screenpipe-db/tests/multi_pool_wal_parity_test.rs
@@ -59,8 +59,10 @@ async fn pragma_text(pool: &SqlitePool, name: &str) -> String {
/// THE GUARANTEE. The engine pool and the secret-store pool, opened over the same
/// file, must report byte-for-byte identical WAL-affecting pragmas. Any drift is a
-/// `-shm` WAL-index desync waiting to happen (the code-11 corruption). Without the
-/// fix this fails on `wal_autocheckpoint` (engine 4000 vs secret 1000).
+/// `-shm` WAL-index desync waiting to happen (the code-11 corruption). Both must
+/// now report `wal_autocheckpoint = 0` (inline auto-checkpoint disabled; the
+/// maintenance task owns checkpointing). The original bug was one side inheriting
+/// SQLite's default 1000 while the engine used a non-zero threshold.
#[tokio::test]
async fn engine_and_secret_pools_agree_on_wal_safety_pragmas() {
let (_guard, path) = temp_db();
@@ -97,12 +99,18 @@ async fn engine_and_secret_pools_agree_on_wal_safety_pragmas() {
);
}
- // Pin the exact value that was the historical bug so a future change to the
- // engine default can't silently re-open the gap on only one side.
+ // Pin the exact value so a future change can't silently re-open the gap on
+ // only one side. 0 = inline auto-checkpoint DISABLED on every pool; the
+ // single maintenance task owns all checkpointing (inline auto-checkpoint on
+ // a committing connection under load could copy a desynced -shm frame onto
+ // the wrong page). The historical bug was one side inheriting SQLite's
+ // default 1000 while the other used a different value; pinning 0 on both
+ // keeps them from ever racing the shared -shm WAL-index.
assert_eq!(
pragma_i64(&secret_pool, "wal_autocheckpoint").await,
- 4000,
- "secret pool must use wal_autocheckpoint=4000, not SQLite's default 1000",
+ 0,
+ "secret pool must use wal_autocheckpoint=0 (inline auto-checkpoint disabled), \
+ not SQLite's default 1000",
);
assert_eq!(
pragma_i64(&db.pool, "mmap_size").await,