Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): refresh stale SQLite planner statistics in background (#7551)

Ezra Ellettefb6b168e0314
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index 0edffda9d..b0baa3bf7 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -346,6 +346,10 @@ async fn collect_migration_diagnostics_from_root(root: std::path::PathBuf) -> St
// if the snapshot directory is missing or unreadable.
json!({
"attempts": attempts,
+ "sqlite_planner": match screenpipe_db::planner_stats::report(&root) {
+ Ok(report) => json!(report),
+ Err(_) => json!({"error": "Could not read SQLite planner maintenance diagnostics"}),
+ },
"retry_block": crate::storage_migration::saved_migration_error(&root),
})
}),
@@ -1361,6 +1365,35 @@ mod tests {
.unwrap();
}
+ #[tokio::test]
+ async fn planner_maintenance_failure_survives_restart_and_reaches_redacted_support() {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("db.sqlite");
+ let db = screenpipe_db::DatabaseManager::new(
+ path.to_str().unwrap(),
+ screenpipe_config::DbConfig { write_pool_max: 1, ..Default::default() },
+ )
+ .await
+ .unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(timestamp,full_text) VALUES('2026-10-08','private history password=hunter2')").await.unwrap();
+ db.execute_raw_sql_write("PRAGMA query_only=ON").await.unwrap();
+ // A genuine SQLite failure, not a fabricated diagnostic record.
+ let attempt = db.maintain_planner_statistics().await;
+ assert_eq!(attempt.sqlite_code.as_deref(), Some("8"));
+ db.execute_raw_sql_write("PRAGMA query_only=OFF").await.unwrap();
+ db.close().await;
+ // The originating engine and its rolling logs are gone. Collect and
+ // redact the persisted attempt, then inspect the mock support upload.
+ assert_migration_report_uploaded(
+ root.path(),
+ &[
+ "sqlite_planner", "duration_ms", "last_failure", "deferred",
+ "attempt to write a readonly database", "sqlite_code",
+ ],
+ )
+ .await;
+ }
+
#[tokio::test]
async fn migration_retry_block_reaches_support_without_attempt_snapshots() {
let root = tempfile::tempdir().unwrap();
diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rs
index 7b247302c..11d530b90 100644
--- a/crates/screenpipe-db/src/db/maintenance.rs
+++ b/crates/screenpipe-db/src/db/maintenance.rs
@@ -1382,6 +1382,35 @@ impl DatabaseManager {
)
}
+ fn planner_diagnostics_root(&self) -> Option<std::path::PathBuf> {
+ let options = self.write_pool.connect_options();
+ if options.get_filename() == std::path::Path::new(":memory:")
+ || options.get_filename().to_string_lossy().contains("mode=memory")
+ {
+ return None;
+ }
+ self.storage.as_ref().map(|s| s.root.clone()).or_else(|| {
+ options
+ .get_filename()
+ .parent()
+ .map(std::path::Path::to_path_buf)
+ })
+ }
+
+ /// Attempt the same sampled, coordinated maintenance used by the hourly
+ /// background task. Start only with an idle writer and yield between tables;
+ /// an analysis already running finishes before waiting writes proceed.
+ pub async fn maintain_planner_statistics(&self) -> crate::planner_stats::Attempt {
+ crate::planner_stats::run_and_report(
+ &self.write_pool,
+ &self.write_semaphore,
+ &self.write_queue_health,
+ &self.close_token,
+ self.planner_diagnostics_root().as_deref(),
+ )
+ .await
+ }
+
/// Spawn the background task that owns routine WAL checkpointing.
///
/// Since `wal_autocheckpoint = 0` (see [`WAL_SAFETY_PRAGMAS`]), committing
@@ -1391,6 +1420,7 @@ impl DatabaseManager {
/// finish; backlog alone must not interrupt capture.
pub fn start_wal_maintenance(&self) {
let pool = self.write_pool.clone();
+ let diagnostics_root = self.planner_diagnostics_root();
let shutdown = self.close_token.clone();
let write_queue_health = self.write_queue_health.clone();
let write_semaphore = std::sync::Arc::clone(&self.write_semaphore);
@@ -1401,6 +1431,7 @@ impl DatabaseManager {
const INTERVAL: Duration = Duration::from_secs(60);
let mut interval = tokio::time::interval(INTERVAL);
let mut backlog_since = None;
+ let mut next_planner_run = tokio::time::Instant::now();
// `interval()` yields its first tick immediately. Startup has just
// run a serialized checkpoint, so consume that tick and wait a
// full interval instead of racing callers' first transactions.
@@ -1449,16 +1480,22 @@ impl DatabaseManager {
Err(e) => warn!("wal checkpoint failed: {}", e),
}
- // Nothing else refreshes SQLite's query-planner statistics on a
- // 24/7 recorder — ANALYZE otherwise only runs inside the
- // emergency `repair_database()` path. `PRAGMA optimize` is
- // designed to be cheap to call on every tick: it only does
- // real work on tables whose content has changed enough since
- // the last run to make stale stats likely, so piggybacking it
- // on this existing 60s tick keeps planner stats fresh without
- // a dedicated schedule.
- if let Err(e) = sqlx::query("PRAGMA optimize").execute(&pool).await {
- warn!("pragma optimize failed: {}", e);
+ drop(_write_guard);
+ // Repair existing histories on the first background tick;
+ // check hourly thereafter, retrying deferred work next tick.
+ if tokio::time::Instant::now() >= next_planner_run {
+ let attempt = crate::planner_stats::run_and_report(
+ &pool,
+ &write_semaphore,
+ &write_queue_health,
+ &shutdown,
+ diagnostics_root.as_deref(),
+ )
+ .await;
+ if attempt.status == "complete" {
+ next_planner_run =
+ tokio::time::Instant::now() + crate::planner_stats::INTERVAL;
+ }
}
}
});
diff --git a/crates/screenpipe-db/src/db/mod.rs b/crates/screenpipe-db/src/db/mod.rs
index 37575fcb2..348e528d5 100644
--- a/crates/screenpipe-db/src/db/mod.rs
+++ b/crates/screenpipe-db/src/db/mod.rs
@@ -466,6 +466,8 @@ mod elements;
mod feedback;
mod frames;
mod maintenance;
+#[cfg(test)]
+mod planner_stats_tests;
mod meetings;
mod memories;
mod starred;
diff --git a/crates/screenpipe-db/src/db/planner_stats_tests.rs b/crates/screenpipe-db/src/db/planner_stats_tests.rs
new file mode 100644
index 000000000..19b27fe34
--- /dev/null
+++ b/crates/screenpipe-db/src/db/planner_stats_tests.rs
@@ -0,0 +1,683 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+use super::*;
+use crate::planner_stats::{self, Attempt};
+use std::time::Instant;
+
+const LOOKUP: &str = "SELECT id FROM _bulk_element_rows WHERE id IN (SELECT value FROM json_each('[1]')) ORDER BY id";
+const PLAN: &str = "EXPLAIN QUERY PLAN SELECT id FROM _bulk_element_rows WHERE id IN (SELECT value FROM json_each('[1]')) ORDER BY id";
+
+// Delay the actual SQLite ANALYZE preparation, after maintenance owns the
+// writer. This deterministically reproduces a slow operation without depending
+// on host disk speed or adding a production timing knob.
+struct AnalyzeHook {
+ entered: tokio::sync::Notify,
+ delay: Duration,
+ cancel: Option<tokio_util::sync::CancellationToken>,
+}
+
+unsafe extern "C" fn analyze_hook(
+ context: *mut std::ffi::c_void,
+ action: std::ffi::c_int,
+ table: *const std::ffi::c_char,
+ _: *const std::ffi::c_char,
+ _: *const std::ffi::c_char,
+ _: *const std::ffi::c_char,
+) -> std::ffi::c_int {
+ if action == libsqlite3_sys::SQLITE_ANALYZE
+ && !table.is_null()
+ && unsafe { std::ffi::CStr::from_ptr(table).to_bytes() } == b"_bulk_element_rows"
+ {
+ // SAFETY: registration retains an Arc until the callback is removed.
+ let hook = unsafe { &*context.cast::<AnalyzeHook>() };
+ hook.entered.notify_one();
+ if let Some(cancel) = &hook.cancel {
+ cancel.cancel();
+ }
+ std::thread::sleep(hook.delay);
+ }
+ libsqlite3_sys::SQLITE_OK
+}
+
+async fn install_analyze_hook(db: &DatabaseManager, hook: &Arc<AnalyzeHook>) {
+ let mut conn = db.write_pool.acquire().await.unwrap();
+ {
+ let mut handle = conn.lock_handle().await.unwrap();
+ // SAFETY: exclusive connection; this fixture has no permanent authorizer.
+ // Retain ownership even if the test panics before explicit removal.
+ let result = unsafe {
+ libsqlite3_sys::sqlite3_set_authorizer(
+ handle.as_raw_handle().as_ptr(),
+ Some(analyze_hook),
+ Arc::into_raw(Arc::clone(hook)).cast_mut().cast(),
+ )
+ };
+ assert_eq!(result, libsqlite3_sys::SQLITE_OK);
+ }
+ conn.return_to_pool().await;
+}
+
+async fn remove_analyze_hook(db: &DatabaseManager, hook: &Arc<AnalyzeHook>) {
+ let mut conn = db.write_pool.acquire().await.unwrap();
+ {
+ let mut handle = conn.lock_handle().await.unwrap();
+ // SAFETY: remove callback under exclusive access before releasing its Arc.
+ unsafe {
+ assert_eq!(
+ libsqlite3_sys::sqlite3_set_authorizer(
+ handle.as_raw_handle().as_ptr(),
+ None,
+ std::ptr::null_mut()
+ ),
+ libsqlite3_sys::SQLITE_OK
+ );
+ drop(Arc::from_raw(Arc::as_ptr(hook)));
+ }
+ }
+ conn.return_to_pool().await;
+}
+
+async fn plan(conn: &mut sqlx::SqliteConnection) -> String {
+ sqlx::query(PLAN)
+ .persistent(false)
+ .fetch_all(conn)
+ .await
+ .unwrap()
+ .iter()
+ .map(|r| r.get::<String, _>(3))
+ .collect::<Vec<_>>()
+ .join("; ")
+}
+
+async fn scan_steps(conn: &mut sqlx::SqliteConnection, reset: bool) -> i32 {
+ let mut handle = conn.lock_handle().await.unwrap();
+ // SAFETY: exclusive handle guard; statement pointers are borrowed only
+ // while the guard prevents the SQLx worker from using this connection.
+ unsafe {
+ let db = handle.as_raw_handle().as_ptr();
+ let mut stmt = libsqlite3_sys::sqlite3_next_stmt(db, std::ptr::null_mut());
+ while !stmt.is_null() {
+ let sql = std::ffi::CStr::from_ptr(libsqlite3_sys::sqlite3_sql(stmt))
+ .to_str()
+ .unwrap();
+ if sql == LOOKUP {
+ return libsqlite3_sys::sqlite3_stmt_status(
+ stmt,
+ libsqlite3_sys::SQLITE_STMTSTATUS_FULLSCAN_STEP,
+ i32::from(reset),
+ );
+ }
+ stmt = libsqlite3_sys::sqlite3_next_stmt(db, stmt);
+ }
+ }
+ panic!("cached lookup missing");
+}
+
+async fn fixture() -> (tempfile::TempDir, DatabaseManager) {
+ let root = tempfile::tempdir().unwrap();
+ let config = DbConfig {
+ read_pool_max: 4,
+ read_pool_min: 1,
+ write_pool_max: 1,
+ ..Default::default()
+ };
+ let initial = DatabaseManager::new_hybrid(root.path(), config.clone(), Default::default())
+ .await
+ .unwrap();
+ let storage = initial.storage.clone();
+ let path = initial
+ .write_pool
+ .connect_options()
+ .get_filename()
+ .to_owned();
+ initial.close().await;
+ let db =
+ DatabaseManager::new_with_storage(path.to_str().unwrap(), config, storage, false, false)
+ .await
+ .unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-08','private history');
+ WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<30)
+ INSERT INTO _bulk_element_rows(id,frame_id,text,_archive_generation) SELECT x,1,'private history',1 FROM n;
+ ANALYZE _bulk_element_rows;
+ CREATE TABLE planner_other(id INTEGER PRIMARY KEY, group_id INTEGER);
+ CREATE INDEX planner_other_group ON planner_other(group_id);
+ INSERT INTO planner_other SELECT id,1 FROM _bulk_element_rows;
+ ANALYZE planner_other;
+ INSERT INTO upload_bindings(destination,checkpoint) VALUES('test','{\"last_id\":42}');")
+ .await.unwrap();
+ let storage = db.storage.clone();
+ let config = DbConfig {
+ read_pool_max: 4,
+ read_pool_min: 1,
+ write_pool_max: 1,
+ ..Default::default()
+ };
+ db.close().await;
+ let db =
+ DatabaseManager::new_with_storage(path.to_str().unwrap(), config, storage, false, false)
+ .await
+ .unwrap();
+ (root, db)
+}
+
+async fn grow(db: &DatabaseManager, end: i64) {
+ db.execute_raw_sql_write(&format!("WITH RECURSIVE n(x) AS (VALUES(31) UNION ALL SELECT x+1 FROM n WHERE x<{end})
+ INSERT INTO _bulk_element_rows(id,frame_id,text,_archive_generation) SELECT x,1,'private history',1 FROM n;
+ INSERT INTO planner_other SELECT id,1 FROM _bulk_element_rows WHERE id>30;"))
+ .await.unwrap();
+}
+
+async fn maintain(db: &DatabaseManager) -> Attempt {
+ planner_stats::run(
+ &db.write_pool,
+ &db.write_semaphore,
+ &db.write_queue_health,
+ &db.close_token,
+ Attempt::new(),
+ )
+ .await
+}
+
+async fn finish(db: &DatabaseManager) -> usize {
+ let mut analyzed = 0;
+ for _ in 0..100 {
+ let attempt = maintain(db).await;
+ println!("maintenance: {attempt:?}");
+ analyzed += attempt.analyzed_tables;
+ if attempt.status == "complete" {
+ return analyzed;
+ }
+ assert_eq!(attempt.status, "deferred");
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ panic!("maintenance failed to complete");
+}
+
+#[tokio::test]
+async fn stale_statistics_repair_replans_existing_pools_and_preserves_storage() {
+ let (_root, db) = fixture().await;
+ // Keep two application readers alive, including their cached statements.
+ let mut a = db.pool.acquire().await.unwrap();
+ let mut b = db.pool.acquire().await.unwrap();
+ let before = plan(&mut a).await;
+ assert!(before.contains("SCAN _bulk_element_rows"), "{before}");
+ assert!(plan(&mut b).await.contains("SCAN _bulk_element_rows"));
+ grow(&db, 300_000).await;
+ let stats: String =
+ sqlx::query_scalar("SELECT stat FROM sqlite_stat1 WHERE idx='_bulk_element_staged_frame'")
+ .fetch_one(&mut *a)
+ .await
+ .unwrap();
+ assert_eq!(stats, "30 30");
+ // The old maintenance path runs on the writer, whose SELECT history does
+ // not contain these reader lookups. It leaves the 30-row estimate stale.
+ {
+ let permit = db.coordinated_writer().lock().await.unwrap();
+ sqlx::query("PRAGMA optimize")
+ .execute(permit.pool())
+ .await
+ .unwrap();
+ }
+ let stats: String =
+ sqlx::query_scalar("SELECT stat FROM sqlite_stat1 WHERE idx='_bulk_element_staged_frame'")
+ .fetch_one(&mut *a)
+ .await
+ .unwrap();
+ assert_eq!(stats, "30 30");
+ let rows_before: (i64, i64, i64) =
+ sqlx::query_as("SELECT count(*),sum(id),sum(length(text)) FROM _bulk_element_rows")
+ .fetch_one(&mut *a)
+ .await
+ .unwrap();
+ let revisions: (i64, i64) = sqlx::query_as(
+ "SELECT revision,(SELECT revision FROM _storage_revocation) FROM storage_metadata",
+ )
+ .fetch_one(&mut *a)
+ .await
+ .unwrap();
+ let schema: Vec<(String, Option<String>)> = sqlx::query_as(
+ "SELECT name,sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_stat%' ORDER BY name",
+ )
+ .fetch_all(&mut *a)
+ .await
+ .unwrap();
+ let start = Instant::now();
+ let ids: Vec<i64> = sqlx::query_scalar(LOOKUP).fetch_all(&mut *a).await.unwrap();
+ let slow = start.elapsed();
+ assert_eq!(ids, vec![1]);
+ assert!(scan_steps(&mut a, true).await > 200_000);
+ sqlx::query_scalar::<_, i64>(LOOKUP)
+ .fetch_all(&mut *b)
+ .await
+ .unwrap();
+ assert!(scan_steps(&mut b, true).await > 200_000);
+ assert!(finish(&db).await >= 2);
+ for conn in [&mut *a, &mut *b] {
+ // EXPLAIN does not step the program's schema-cookie check. Execute
+ // the cached application query first, as real callers do.
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(LOOKUP)
+ .fetch_all(&mut *conn)
+ .await
+ .unwrap(),
+ vec![1]
+ );
+ assert_eq!(
+ scan_steps(conn, false).await,
+ 0,
+ "cached lookup must not scan"
+ );
+ let after = plan(conn).await;
+ println!("before={before}; after={after}; scan_duration={slow:?}");
+ assert!(
+ after.contains("SEARCH _bulk_element_rows USING INTEGER PRIMARY KEY"),
+ "{after}"
+ );
+ let start = Instant::now();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(LOOKUP)
+ .fetch_all(&mut *conn)
+ .await
+ .unwrap(),
+ vec![1]
+ );
+ println!("pk_duration={:?}", start.elapsed());
+ let other: Vec<(i64,i64,i64,String)> = sqlx::query_as("EXPLAIN QUERY PLAN SELECT id FROM planner_other WHERE id IN (SELECT value FROM json_each('[1]')) ORDER BY id").fetch_all(&mut *conn).await.unwrap();
+ assert!(
+ other
+ .iter()
+ .any(|r| r.3.contains("USING INTEGER PRIMARY KEY")),
+ "{other:?}"
+ );
+ }
+ assert_eq!(
+ sqlx::query_as::<_, (i64, i64, i64)>(
+ "SELECT count(*),sum(id),sum(length(text)) FROM _bulk_element_rows"
+ )
+ .fetch_one(&mut *a)
+ .await
+ .unwrap(),
+ rows_before
+ );
+ assert_eq!(
+ sqlx::query_as::<_, (i64, i64)>(
+ "SELECT revision,(SELECT revision FROM _storage_revocation) FROM storage_metadata"
+ )
+ .fetch_one(&mut *a)
+ .await
+ .unwrap(),
+ revisions
+ );
+ assert_eq!(
+ sqlx::query_as::<_, (String, Option<String>)>(
+ "SELECT name,sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_stat%' ORDER BY name"
+ )
+ .fetch_all(&mut *a)
+ .await
+ .unwrap(),
+ schema
+ );
+ let selected = sqlx::query_scalar::<_, String>("PRAGMA main.optimize=0x10003")
+ .fetch_all(&db.write_pool)
+ .await
+ .unwrap();
+ assert!(
+ !selected
+ .iter()
+ .any(|s| s.contains("_bulk_element_rows") || s.contains("planner_other")),
+ "unchanged large tables need no reanalysis: {selected:?}"
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, String>(
+ "SELECT checkpoint FROM upload_bindings WHERE destination='test'"
+ )
+ .fetch_one(&mut *a)
+ .await
+ .unwrap(),
+ r#"{"last_id":42}"#
+ );
+ drop(a);
+ drop(b);
+ db.close().await;
+}
+
+#[tokio::test]
+async fn maintenance_defers_to_writer_and_recording_remains_durable() {
+ let (root, db) = fixture().await;
+ grow(&db, 100_000).await;
+ let tx = db.begin_immediate_with_retry().await.unwrap();
+ let start = Instant::now();
+ let deferred = maintain(&db).await;
+ assert_eq!(deferred.status, "deferred");
+ assert!(start.elapsed() < Duration::from_millis(100));
+ planner_stats::save(root.path(), &deferred).await;
+ tx.commit().await.unwrap();
+ let writes = async {
+ let mut max_latency = Duration::ZERO;
+ for id in 2..52 {
+ let start = Instant::now();
+ let _ = id;
+ db.insert_snapshot_frame(
+ "test",
+ chrono::Utc::now(),
+ "",
+ None,
+ None,
+ None,
+ true,
+ Some("test"),
+ Some("private history"),
+ Some("accessibility"),
+ None,
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ max_latency = max_latency.max(start.elapsed());
+ tokio::time::sleep(Duration::from_millis(2)).await;
+ }
+ println!("maximum capture writer latency={max_latency:?}");
+ assert!(max_latency < Duration::from_secs(1));
+ };
+ let (_, _) = tokio::join!(writes, finish(&db));
+ let complete = maintain(&db).await;
+ planner_stats::save(root.path(), &complete).await;
+ let path = db.write_pool.connect_options().get_filename().to_owned();
+ let config = DbConfig::default();
+ db.close().await;
+ let reopened = DatabaseManager::new_with_storage(
+ path.to_str().unwrap(),
+ config,
+ crate::storage::HybridStorage::for_index(&path).unwrap(),
+ false,
+ false,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames")
+ .fetch_one(&reopened.pool)
+ .await
+ .unwrap(),
+ 51
+ );
+ let report = planner_stats::report(root.path()).unwrap();
+ assert_eq!(
+ report.last_failure.unwrap().detail.unwrap(),
+ "writer busy; retry on next maintenance tick"
+ );
+ assert_eq!(report.last_success.unwrap().status, "complete");
+ reopened.close().await;
+}
+
+#[tokio::test]
+async fn interrupted_analysis_rolls_back_and_returns_a_usable_writer() {
+ let (_root, db) = fixture().await;
+ grow(&db, 100_000).await;
+ let original: String =
+ sqlx::query_scalar("SELECT stat FROM sqlite_stat1 WHERE idx='_bulk_element_staged_frame'")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ let mut conn = db.write_pool.acquire().await.unwrap();
+ let busy: i64 = sqlx::query_scalar("PRAGMA busy_timeout")
+ .fetch_one(&mut *conn)
+ .await
+ .unwrap();
+ let limit: i64 = sqlx::query_scalar("PRAGMA analysis_limit")
+ .fetch_one(&mut *conn)
+ .await
+ .unwrap();
+ conn.return_to_pool().await;
+ let cancel = tokio_util::sync::CancellationToken::new();
+ let hook = Arc::new(AnalyzeHook {
+ entered: tokio::sync::Notify::new(),
+ delay: Duration::from_millis(10),
+ cancel: Some(cancel.clone()),
+ });
+ install_analyze_hook(&db, &hook).await;
+ let error = planner_stats::slice(
+ &db.write_pool,
+ &db.write_semaphore,
+ &db.write_queue_health,
+ &cancel,
+ Some("ANALYZE main._bulk_element_rows"),
+ )
+ .await
+ .unwrap_err();
+ remove_analyze_hook(&db, &hook).await;
+ assert!(crate::is_sqlite_interrupt(&error), "{error}");
+ let mut conn = db.write_pool.acquire().await.unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("PRAGMA busy_timeout")
+ .fetch_one(&mut *conn)
+ .await
+ .unwrap(),
+ busy
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("PRAGMA analysis_limit")
+ .fetch_one(&mut *conn)
+ .await
+ .unwrap(),
+ limit
+ );
+ conn.return_to_pool().await;
+ assert_eq!(
+ sqlx::query_scalar::<_, String>(
+ "SELECT stat FROM sqlite_stat1 WHERE idx='_bulk_element_staged_frame'"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ original
+ );
+ db.insert_snapshot_frame(
+ "test",
+ chrono::Utc::now(),
+ "",
+ None,
+ None,
+ None,
+ true,
+ Some("test"),
+ Some("private history"),
+ Some("accessibility"),
+ None,
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ assert!(!db.write_queue_health.is_hard_faulted());
+ finish(&db).await;
+ db.close().await;
+}
+
+#[tokio::test]
+async fn background_tick_repairs_reopened_history_without_capture_trigger() {
+ let (root, db) = fixture().await;
+ grow(&db, 100_000).await;
+ db.start_wal_maintenance();
+ tokio::time::timeout(Duration::from_secs(70), async {
+ loop {
+ if planner_stats::report(root.path())
+ .unwrap()
+ .last_success
+ .is_some()
+ {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ })
+ .await
+ .unwrap();
+ let mut reader = db.pool.acquire().await.unwrap();
+ sqlx::query_scalar::<_, i64>(LOOKUP)
+ .fetch_all(&mut *reader)
+ .await
+ .unwrap();
+ assert!(plan(&mut reader)
+ .await
+ .contains("USING INTEGER PRIMARY KEY"));
+ drop(reader);
+ db.close().await;
+}
+
+#[tokio::test]
+async fn dropped_maintenance_keeps_writer_until_sqlite_cleanup_finishes() {
+ let (_root, db) = fixture().await;
+ let db = Arc::new(db);
+ // Startup returns pooled connections asynchronously; make this writer
+ // available before testing cancellation rather than incidental deferral.
+ let mut ready = db.write_pool.acquire().await.unwrap();
+ ready.return_to_pool().await;
+ let copy = Arc::clone(&db);
+ let cancel = tokio_util::sync::CancellationToken::new();
+ let shutdown = cancel.clone();
+ // Deliberately expensive SQL at the same shutdown cancellation boundary.
+ // Aborting the caller must not abandon its active worker/transaction.
+ let task = tokio::spawn(async move {
+ planner_stats::slice(&copy.write_pool, &copy.write_semaphore, &copy.write_queue_health,
+ &shutdown, Some("WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<100000000) SELECT sum(x) FROM n")).await
+ });
+ tokio::time::timeout(Duration::from_secs(1), async {
+ while db.write_semaphore.available_permits() != 0 {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ task.abort();
+ let _ = task.await;
+ cancel.cancel();
+ tokio::time::timeout(
+ Duration::from_secs(1),
+ db.insert_snapshot_frame(
+ "test",
+ chrono::Utc::now(),
+ "",
+ None,
+ None,
+ None,
+ true,
+ Some("test"),
+ Some("private history"),
+ Some("accessibility"),
+ None,
+ None,
+ None,
+ ),
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ // The shutdown watcher must not interrupt the next borrower.
+ finish(&db).await;
+ db.close().await;
+}
+
+#[tokio::test]
+async fn slow_analysis_finishes_and_waiting_recording_is_preserved() {
+ let (root, db) = fixture().await;
+ grow(&db, 300_000).await;
+ let mut reader = db.pool.acquire().await.unwrap();
+ assert!(plan(&mut reader).await.contains("SCAN _bulk_element_rows"));
+ sqlx::query_scalar::<_, i64>(LOOKUP)
+ .fetch_all(&mut *reader)
+ .await
+ .unwrap();
+ assert!(scan_steps(&mut reader, true).await > 200_000);
+ let hook = Arc::new(AnalyzeHook {
+ entered: tokio::sync::Notify::new(),
+ delay: Duration::from_millis(250),
+ cancel: None,
+ });
+ install_analyze_hook(&db, &hook).await;
+ let record = async {
+ hook.entered.notified().await;
+ let start = Instant::now();
+ db.insert_snapshot_frame(
+ "test",
+ chrono::Utc::now(),
+ "",
+ None,
+ None,
+ None,
+ true,
+ Some("test"),
+ Some("recorded while analysis was running"),
+ Some("accessibility"),
+ None,
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ start.elapsed()
+ };
+ let (attempt, recording_wait) = tokio::time::timeout(Duration::from_secs(10), async {
+ tokio::join!(db.maintain_planner_statistics(), record)
+ })
+ .await
+ .unwrap();
+ remove_analyze_hook(&db, &hook).await;
+ println!("slow analysis attempt={attempt:?}; recording wait={recording_wait:?}");
+ assert!(attempt.analyzed_tables >= 1, "{attempt:?}");
+ assert!(attempt.duration_ms >= 250, "{attempt:?}");
+ assert_eq!(
+ attempt.sqlite_code, None,
+ "slow analysis must not be interrupted"
+ );
+ // The slow table must be repaired in this attempt, even when maintenance
+ // defers other tables to let the waiting recording take the writer.
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(LOOKUP)
+ .fetch_all(&mut *reader)
+ .await
+ .unwrap(),
+ vec![1]
+ );
+ assert_eq!(scan_steps(&mut reader, false).await, 0);
+ assert!(plan(&mut reader)
+ .await
+ .contains("USING INTEGER PRIMARY KEY"));
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows")
+ .fetch_one(&mut *reader)
+ .await
+ .unwrap(),
+ 300_000
+ );
+ drop(reader);
+ let path = db.write_pool.connect_options().get_filename().to_owned();
+ db.close().await;
+ let reopened = DatabaseManager::new_with_storage(
+ path.to_str().unwrap(),
+ DbConfig::default(),
+ crate::storage::HybridStorage::for_index(&path).unwrap(),
+ false,
+ false,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames")
+ .fetch_one(&reopened.pool)
+ .await
+ .unwrap(),
+ 2
+ );
+ assert!(
+ planner_stats::report(root.path())
+ .unwrap()
+ .latest
+ .unwrap()
+ .duration_ms
+ >= 250
+ );
+ reopened.close().await;
+}
diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rs
index fedb7e3d2..96dd1c18c 100644
--- a/crates/screenpipe-db/src/lib.rs
+++ b/crates/screenpipe-db/src/lib.rs
@@ -31,6 +31,7 @@ mod cancellable_query;
mod db;
#[cfg(test)]
mod failpoint_vfs;
+pub mod planner_stats;
mod recovery;
mod sqlite_error;
pub mod storage;
diff --git a/crates/screenpipe-db/src/planner_stats.rs b/crates/screenpipe-db/src/planner_stats.rs
new file mode 100644
index 000000000..0146a11bb
--- /dev/null
+++ b/crates/screenpipe-db/src/planner_stats.rs
@@ -0,0 +1,406 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Sampled planner maintenance, independent of the writer's query history.
+//! No capture-path hooks, connection recycling, or recording-data mutations.
+
+use serde::{Deserialize, Serialize};
+use sqlx::{SqliteConnection, SqlitePool};
+use std::{
+ path::Path,
+ sync::{Arc, Mutex},
+ time::{Duration, Instant},
+};
+use tokio::sync::Semaphore;
+use tokio_util::sync::CancellationToken;
+
+pub(crate) const INTERVAL: Duration = Duration::from_secs(3600);
+// SQLite recommends approximate ANALYZE with 100–1000 sampled rows per index;
+// 1000 also repaired the reported production plan. This bounds sampling, not
+// elapsed time: counting index pages may take longer and must be allowed to finish.
+const ANALYSIS_LIMIT: i64 = 1000;
+const FILE: &str = "sqlite-planner-maintenance.json";
+
+/// Allowlisted metadata only: never SQL, table contents, paths, or raw errors.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Attempt {
+ pub started_at: String,
+ pub status: String,
+ pub duration_ms: u64,
+ pub analyzed_tables: usize,
+ pub stage: String,
+ pub sqlite_code: Option<String>,
+ pub detail: Option<String>,
+}
+
+#[derive(Clone, Default, Debug, Serialize, Deserialize)]
+pub struct Report {
+ pub latest: Option<Attempt>,
+ pub last_success: Option<Attempt>,
+ pub last_failure: Option<Attempt>,
+}
+
+/// Read durable evidence even after engine restart or rolling-log rotation.
+pub fn report(root: &Path) -> Result<Report, std::io::Error> {
+ let file = match std::fs::File::open(root.join(FILE)) {
+ Ok(file) => file,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Report::default()),
+ Err(e) => return Err(e),
+ };
+ serde_json::from_reader(file).map_err(std::io::Error::other)
+}
+
+pub(crate) async fn save(root: &Path, attempt: &Attempt) {
+ let root = root.to_owned();
+ let attempt = attempt.clone();
+ let result = tokio::task::spawn_blocking(move || {
+ let mut saved = report(&root).map_err(sqlx::Error::Io)?;
+ if attempt.status == "complete" {
+ saved.last_success = Some(attempt.clone());
+ } else if attempt.status != "running" {
+ saved.last_failure = Some(attempt.clone());
+ }
+ saved.latest = Some(attempt);
+ crate::storage::durable_json(&root.join(FILE), &saved)
+ })
+ .await;
+ if !matches!(result, Ok(Ok(()))) {
+ tracing::warn!("sqlite planner maintenance: could not persist support diagnostics");
+ }
+}
+
+impl Attempt {
+ pub(crate) fn new() -> Self {
+ Self {
+ started_at: chrono::Utc::now().to_rfc3339(),
+ status: "running".into(),
+ duration_ms: 0,
+ analyzed_tables: 0,
+ stage: "selecting_tables".into(),
+ sqlite_code: None,
+ detail: None,
+ }
+ }
+
+ fn failure(&mut self, error: &sqlx::Error) {
+ self.status = "deferred".into();
+ self.sqlite_code = error
+ .as_database_error()
+ .and_then(|e| e.code())
+ .map(|s| s.into_owned());
+ // sqlite3_errstr returns a static generic description, unlike errmsg,
+ // which can embed SQL, captured values or filesystem paths.
+ self.detail = Some(
+ if let Some(code) = self
+ .sqlite_code
+ .as_ref()
+ .and_then(|s| s.parse::<i32>().ok())
+ {
+ // SAFETY: SQLite owns this immutable, NUL-terminated string.
+ unsafe { std::ffi::CStr::from_ptr(libsqlite3_sys::sqlite3_errstr(code)) }
+ .to_string_lossy()
+ .into_owned()
+ } else {
+ match error {
+ sqlx::Error::PoolClosed => "database pool closed",
+ sqlx::Error::PoolTimedOut => "database pool unavailable",
+ sqlx::Error::Io(_) => "maintenance I/O failure",
+ _ => "maintenance failed; inspect SQLite health diagnostics",
+ }
+ .into()
+ },
+ );
+ if crate::sqlite_error::is_sqlite_hard_fault(error) {
+ self.status = "failed".into();
+ }
+ }
+}
+
+/// One slice owns the existing writer lane. Do not queue ahead of capture or
+/// wait for another SQLite writer. Await SQLite completion before releasing
+/// the permit; a Tokio timeout alone would leave its worker running.
+pub(crate) async fn slice(
+ pool: &SqlitePool,
+ semaphore: &Arc<Semaphore>,
+ health: &crate::write_queue::WriteQueueHealth,
+ shutdown: &CancellationToken,
+ analyze: Option<&str>,
+) -> Result<Option<Vec<String>>, sqlx::Error> {
+ let Ok(permit) = Arc::clone(semaphore).try_acquire_owned() else {
+ return Ok(None);
+ };
+ if health.is_hard_faulted() || shutdown.is_cancelled() {
+ return Err(sqlx::Error::PoolClosed);
+ }
+ let Some(mut conn) = pool.try_acquire() else {
+ return Ok(None);
+ };
+ let health = health.clone();
+ let shutdown = shutdown.clone();
+ let analyze = analyze.map(str::to_owned);
+ // Own both the connection and writer permit until SQLite has stopped and
+ // cleanup has finished, even if the caller drops/aborts its async future.
+ tokio::spawn(async move {
+ let _permit = permit;
+ let result = on_connection(&mut conn, &health, &shutdown, analyze.as_deref()).await;
+ if let Err(error) = &result {
+ if crate::sqlite_error::is_sqlite_hard_fault(error) {
+ health.latch_hard_fault(error);
+ }
+ }
+ conn.return_to_pool().await;
+ result.map(Some)
+ })
+ .await
+ .map_err(|_| sqlx::Error::WorkerCrashed)?
+}
+
+// SQLite's B-tree count can run inside a single VM opcode. A progress
+// handler alone cannot interrupt it during shutdown. sqlite3_interrupt is thread-safe;
+// this guard synchronizes its lifetime with the owned connection, including
+// unwinding, so cancellation cannot interrupt a later borrower or freed handle.
+struct InterruptOnShutdown {
+ target: Arc<Mutex<Option<usize>>>,
+ watcher: tokio::task::JoinHandle<()>,
+}
+
+impl InterruptOnShutdown {
+ fn new(raw: usize, shutdown: CancellationToken) -> Self {
+ let target = Arc::new(Mutex::new(Some(raw)));
+ let watcher_target = Arc::clone(&target);
+ let watcher = tokio::spawn(async move {
+ shutdown.cancelled().await;
+ let guard = watcher_target.lock().unwrap_or_else(|e| e.into_inner());
+ if let Some(raw) = *guard {
+ // SAFETY: Drop clears the pointer under this same lock before
+ // the owning task can return/close the connection.
+ unsafe {
+ libsqlite3_sys::sqlite3_interrupt(raw as *mut libsqlite3_sys::sqlite3);
+ }
+ }
+ });
+ Self { target, watcher }
+ }
+}
+
+impl Drop for InterruptOnShutdown {
+ fn drop(&mut self) {
+ *self.target.lock().unwrap_or_else(|e| e.into_inner()) = None;
+ self.watcher.abort();
+ }
+}
+
+async fn on_connection(
+ conn: &mut SqliteConnection,
+ health: &crate::write_queue::WriteQueueHealth,
+ shutdown: &CancellationToken,
+ analyze: Option<&str>,
+) -> Result<Vec<String>, sqlx::Error> {
+ let busy: i64 = sqlx::query_scalar("PRAGMA busy_timeout")
+ .fetch_one(&mut *conn)
+ .await?;
+ let limit: i64 = sqlx::query_scalar("PRAGMA analysis_limit")
+ .fetch_one(&mut *conn)
+ .await?;
+ sqlx::query("PRAGMA busy_timeout=0")
+ .execute(&mut *conn)
+ .await?;
+ let cancel = shutdown.clone();
+ let raw = {
+ let mut handle = conn.lock_handle().await?;
+ handle.set_progress_handler(1000, move || !cancel.is_cancelled());
+ handle.as_raw_handle().as_ptr() as usize
+ };
+ let watcher = InterruptOnShutdown::new(raw, shutdown.clone());
+ let result = work(conn, analyze).await;
+ drop(watcher);
+ if let Err(error) = &result {
+ if crate::sqlite_error::is_sqlite_hard_fault(error) {
+ health.latch_hard_fault(error);
+ }
+ }
+ let cleanup = async {
+ let in_transaction = {
+ let mut handle = conn.lock_handle().await?;
+ handle.remove_progress_handler();
+ // SAFETY: the handle guard excludes SQLx. INTERRUPT can already have
+ // rolled back the transaction; SQLx's transaction-depth counter does
+ // not track that, so maintenance uses SQLite's actual transaction state.
+ unsafe { libsqlite3_sys::sqlite3_get_autocommit(handle.as_raw_handle().as_ptr()) == 0 }
+ };
+ if in_transaction {
+ sqlx::query("ROLLBACK").execute(&mut *conn).await?;
+ }
+ // Always restore connection-local settings before normal writes reuse it.
+ sqlx::query(sqlx::AssertSqlSafe(format!(
+ "PRAGMA analysis_limit={limit}"
+ )))
+ .execute(&mut *conn)
+ .await?;
+ sqlx::query(sqlx::AssertSqlSafe(format!("PRAGMA busy_timeout={busy}")))
+ .execute(&mut *conn)
+ .await?;
+ if result.is_err()
+ && !result
+ .as_ref()
+ .err()
+ .is_some_and(crate::sqlite_error::is_sqlite_hard_fault)
+ {
+ // ANALYZE may have loaded uncommitted estimates before a later error
+ // rolled back the transaction. Restore this writer's in-memory view.
+ sqlx::query("ANALYZE main.sqlite_schema")
+ .execute(&mut *conn)
+ .await?;
+ }
+ Ok::<_, sqlx::Error>(())
+ }
+ .await;
+ if let Err(error) = &cleanup {
+ if crate::sqlite_error::is_sqlite_hard_fault(error) {
+ health.latch_hard_fault(error);
+ }
+ }
+ // A cleanup error must not overwrite the originating SQLite failure.
+ match result {
+ Err(error) => Err(error),
+ Ok(rows) => cleanup.map(|()| rows),
+ }
+}
+
+async fn work(
+ conn: &mut SqliteConnection,
+ analyze: Option<&str>,
+) -> Result<Vec<String>, sqlx::Error> {
+ if let Some(statement) = analyze {
+ sqlx::query(sqlx::AssertSqlSafe(format!(
+ "PRAGMA analysis_limit={ANALYSIS_LIMIT}"
+ )))
+ .execute(&mut *conn)
+ .await?;
+ sqlx::query("BEGIN IMMEDIATE").execute(&mut *conn).await?;
+ // The statement comes exclusively from SQLite's optimize dry run.
+ sqlx::query(sqlx::AssertSqlSafe(statement))
+ .execute(&mut *conn)
+ .await?;
+ // ANALYZE reloads only this connection's in-memory statistics. A real,
+ // transactional schema change makes ALL existing connections reload
+ // their schema/statistics and reprepare cached statements at their next
+ // snapshot, without closing pools or writing schema_version by hand.
+ // No persistent object or storage revision is changed.
+ sqlx::raw_sql("CREATE TABLE main._screenpipe_planner_refresh(dummy); DROP TABLE main._screenpipe_planner_refresh;")
+ .execute(&mut *conn).await?;
+ sqlx::query("COMMIT").execute(conn).await?;
+ Ok(Vec::new())
+ } else {
+ // Bundled SQLite >=3.51.3: debug + ANALYZE + all-table growth checks.
+ // Selection uses b-tree estimates, not COUNT(*) scans. Analyze only
+ // missing stats or ~10x growth/shrinkage, including reader-only tables.
+ let candidates: Vec<String> = sqlx::query_scalar("PRAGMA main.optimize=0x10003")
+ .fetch_all(&mut *conn)
+ .await?;
+ let mut needed = Vec::new();
+ for statement in candidates {
+ // Empty indexless tables have no stat1 row even after ANALYZE,
+ // so SQLite can nominate them on every pass. Skip them in O(1)
+ // rather than repeatedly taking writer slices ahead of real work.
+ let quoted_table = statement
+ .strip_prefix("ANALYZE ")
+ .ok_or_else(|| sqlx::Error::Protocol("unexpected optimize operation".into()))?;
+ let populated: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
+ "SELECT EXISTS(SELECT 1 FROM {quoted_table} LIMIT 1)"
+ )))
+ .fetch_one(&mut *conn)
+ .await?;
+ if populated {
+ needed.push(statement);
+ }
+ }
+ Ok(needed)
+ }
+}
+
+pub(crate) async fn run_and_report(
+ pool: &SqlitePool,
+ semaphore: &Arc<Semaphore>,
+ health: &crate::write_queue::WriteQueueHealth,
+ shutdown: &CancellationToken,
+ root: Option<&Path>,
+) -> Attempt {
+ let attempt = Attempt::new();
+ if let Some(root) = root {
+ save(root, &attempt).await;
+ }
+ let attempt = run(pool, semaphore, health, shutdown, attempt).await;
+ tracing::info!(diagnostic = %serde_json::to_string(&attempt).unwrap_or_default(), "sqlite planner maintenance");
+ if let Some(root) = root {
+ save(root, &attempt).await;
+ }
+ attempt
+}
+
+pub(crate) async fn run(
+ pool: &SqlitePool,
+ semaphore: &Arc<Semaphore>,
+ health: &crate::write_queue::WriteQueueHealth,
+ shutdown: &CancellationToken,
+ mut attempt: Attempt,
+) -> Attempt {
+ let started = Instant::now();
+ let result = async {
+ let Some(statements) = slice(pool, semaphore, health, shutdown, None).await? else {
+ return Ok(false);
+ };
+ let mut first_error = None;
+ for statement in statements {
+ attempt.stage = "analyzing_table".into();
+ // Each table commits independently so a deferred table cannot
+ // discard completed work. Let queued captures take the writer.
+ tokio::task::yield_now().await;
+ match slice(pool, semaphore, health, shutdown, Some(&statement)).await {
+ Ok(Some(_)) => attempt.analyzed_tables += 1,
+ Ok(None) => return first_error.map_or(Ok(false), Err),
+ Err(error) => {
+ if health.is_hard_faulted() || shutdown.is_cancelled() {
+ return Err(error);
+ }
+ // A failed table must not prevent repair of other tables.
+ // Keep the first originating failure for support and retry
+ // unfinished work on the next tick.
+ if first_error.is_none() {
+ first_error = Some(error);
+ }
+ }
+ }
+ }
+ first_error.map_or(Ok(true), Err)
+ }
+ .await;
+ match result {
+ Ok(true) => {
+ attempt.status = "complete".into();
+ attempt.stage = "complete".into();
+ }
+ Ok(false) => {
+ attempt.status = "deferred".into();
+ attempt.detail = Some("writer busy; retry on next maintenance tick".into());
+ }
+ Err(error) => {
+ if crate::sqlite_error::is_sqlite_hard_fault(&error) {
+ health.latch_hard_fault(&error);
+ }
+ attempt.failure(&error);
+ if crate::is_sqlite_interrupt(&error) {
+ attempt.detail = Some(
+ if shutdown.is_cancelled() {
+ "shutdown interrupted maintenance; retry after restart"
+ } else {
+ "SQLite interrupted maintenance; retry on next maintenance tick"
+ }
+ .into(),
+ );
+ }
+ }
+ }
+ attempt.duration_ms = started.elapsed().as_millis() as u64;
+ attempt
+}