YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): refresh stale SQLite planner statistics in background (#7551)
Ezra Ellettefb6b168e0314
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex 0edffda9d..b0baa3bf7 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -346,6 +346,10 @@ async fn collect_migration_diagnostics_from_root(root: std::path::PathBuf) -> St// if the snapshot directory is missing or unreadable.json!({"attempts": attempts,+ "sqlite_planner": match screenpipe_db::planner_stats::report(&root) {+ Ok(report) => json!(report),+ Err(_) => json!({"error": "Could not read SQLite planner maintenance diagnostics"}),+ },"retry_block": crate::storage_migration::saved_migration_error(&root),})}),@@ -1361,6 +1365,35 @@ mod tests {.unwrap();}+ #[tokio::test]+ async fn planner_maintenance_failure_survives_restart_and_reaches_redacted_support() {+ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("db.sqlite");+ let db = screenpipe_db::DatabaseManager::new(+ path.to_str().unwrap(),+ screenpipe_config::DbConfig { write_pool_max: 1, ..Default::default() },+ )+ .await+ .unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(timestamp,full_text) VALUES('2026-10-08','private history password=hunter2')").await.unwrap();+ db.execute_raw_sql_write("PRAGMA query_only=ON").await.unwrap();+ // A genuine SQLite failure, not a fabricated diagnostic record.+ let attempt = db.maintain_planner_statistics().await;+ assert_eq!(attempt.sqlite_code.as_deref(), Some("8"));+ db.execute_raw_sql_write("PRAGMA query_only=OFF").await.unwrap();+ db.close().await;+ // The originating engine and its rolling logs are gone. Collect and+ // redact the persisted attempt, then inspect the mock support upload.+ assert_migration_report_uploaded(+ root.path(),+ &[+ "sqlite_planner", "duration_ms", "last_failure", "deferred",+ "attempt to write a readonly database", "sqlite_code",+ ],+ )+ .await;+ }+#[tokio::test]async fn migration_retry_block_reaches_support_without_attempt_snapshots() {let root = tempfile::tempdir().unwrap();diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rsindex 7b247302c..11d530b90 100644--- a/crates/screenpipe-db/src/db/maintenance.rs+++ b/crates/screenpipe-db/src/db/maintenance.rs@@ -1382,6 +1382,35 @@ impl DatabaseManager {)}+ fn planner_diagnostics_root(&self) -> Option<std::path::PathBuf> {+ let options = self.write_pool.connect_options();+ if options.get_filename() == std::path::Path::new(":memory:")+ || options.get_filename().to_string_lossy().contains("mode=memory")+ {+ return None;+ }+ self.storage.as_ref().map(|s| s.root.clone()).or_else(|| {+ options+ .get_filename()+ .parent()+ .map(std::path::Path::to_path_buf)+ })+ }++ /// Attempt the same sampled, coordinated maintenance used by the hourly+ /// background task. Start only with an idle writer and yield between tables;+ /// an analysis already running finishes before waiting writes proceed.+ pub async fn maintain_planner_statistics(&self) -> crate::planner_stats::Attempt {+ crate::planner_stats::run_and_report(+ &self.write_pool,+ &self.write_semaphore,+ &self.write_queue_health,+ &self.close_token,+ self.planner_diagnostics_root().as_deref(),+ )+ .await+ }+/// Spawn the background task that owns routine WAL checkpointing.////// Since `wal_autocheckpoint = 0` (see [`WAL_SAFETY_PRAGMAS`]), committing@@ -1391,6 +1420,7 @@ impl DatabaseManager {/// finish; backlog alone must not interrupt capture.pub fn start_wal_maintenance(&self) {let pool = self.write_pool.clone();+ let diagnostics_root = self.planner_diagnostics_root();let shutdown = self.close_token.clone();let write_queue_health = self.write_queue_health.clone();let write_semaphore = std::sync::Arc::clone(&self.write_semaphore);@@ -1401,6 +1431,7 @@ impl DatabaseManager {const INTERVAL: Duration = Duration::from_secs(60);let mut interval = tokio::time::interval(INTERVAL);let mut backlog_since = None;+ let mut next_planner_run = tokio::time::Instant::now();// `interval()` yields its first tick immediately. Startup has just// run a serialized checkpoint, so consume that tick and wait a// full interval instead of racing callers' first transactions.@@ -1449,16 +1480,22 @@ impl DatabaseManager {Err(e) => warn!("wal checkpoint failed: {}", e),}- // Nothing else refreshes SQLite's query-planner statistics on a- // 24/7 recorder — ANALYZE otherwise only runs inside the- // emergency `repair_database()` path. `PRAGMA optimize` is- // designed to be cheap to call on every tick: it only does- // real work on tables whose content has changed enough since- // the last run to make stale stats likely, so piggybacking it- // on this existing 60s tick keeps planner stats fresh without- // a dedicated schedule.- if let Err(e) = sqlx::query("PRAGMA optimize").execute(&pool).await {- warn!("pragma optimize failed: {}", e);+ drop(_write_guard);+ // Repair existing histories on the first background tick;+ // check hourly thereafter, retrying deferred work next tick.+ if tokio::time::Instant::now() >= next_planner_run {+ let attempt = crate::planner_stats::run_and_report(+ &pool,+ &write_semaphore,+ &write_queue_health,+ &shutdown,+ diagnostics_root.as_deref(),+ )+ .await;+ if attempt.status == "complete" {+ next_planner_run =+ tokio::time::Instant::now() + crate::planner_stats::INTERVAL;+ }}}});diff --git a/crates/screenpipe-db/src/db/mod.rs b/crates/screenpipe-db/src/db/mod.rsindex 37575fcb2..348e528d5 100644--- a/crates/screenpipe-db/src/db/mod.rs+++ b/crates/screenpipe-db/src/db/mod.rs@@ -466,6 +466,8 @@ mod elements;mod feedback;mod frames;mod maintenance;+#[cfg(test)]+mod planner_stats_tests;mod meetings;mod memories;mod starred;diff --git a/crates/screenpipe-db/src/db/planner_stats_tests.rs b/crates/screenpipe-db/src/db/planner_stats_tests.rsnew file mode 100644index 000000000..19b27fe34--- /dev/null+++ b/crates/screenpipe-db/src/db/planner_stats_tests.rs@@ -0,0 +1,683 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use super::*;+use crate::planner_stats::{self, Attempt};+use std::time::Instant;++const LOOKUP: &str = "SELECT id FROM _bulk_element_rows WHERE id IN (SELECT value FROM json_each('[1]')) ORDER BY id";+const PLAN: &str = "EXPLAIN QUERY PLAN SELECT id FROM _bulk_element_rows WHERE id IN (SELECT value FROM json_each('[1]')) ORDER BY id";++// Delay the actual SQLite ANALYZE preparation, after maintenance owns the+// writer. This deterministically reproduces a slow operation without depending+// on host disk speed or adding a production timing knob.+struct AnalyzeHook {+ entered: tokio::sync::Notify,+ delay: Duration,+ cancel: Option<tokio_util::sync::CancellationToken>,+}++unsafe extern "C" fn analyze_hook(+ context: *mut std::ffi::c_void,+ action: std::ffi::c_int,+ table: *const std::ffi::c_char,+ _: *const std::ffi::c_char,+ _: *const std::ffi::c_char,+ _: *const std::ffi::c_char,+) -> std::ffi::c_int {+ if action == libsqlite3_sys::SQLITE_ANALYZE+ && !table.is_null()+ && unsafe { std::ffi::CStr::from_ptr(table).to_bytes() } == b"_bulk_element_rows"+ {+ // SAFETY: registration retains an Arc until the callback is removed.+ let hook = unsafe { &*context.cast::<AnalyzeHook>() };+ hook.entered.notify_one();+ if let Some(cancel) = &hook.cancel {+ cancel.cancel();+ }+ std::thread::sleep(hook.delay);+ }+ libsqlite3_sys::SQLITE_OK+}++async fn install_analyze_hook(db: &DatabaseManager, hook: &Arc<AnalyzeHook>) {+ let mut conn = db.write_pool.acquire().await.unwrap();+ {+ let mut handle = conn.lock_handle().await.unwrap();+ // SAFETY: exclusive connection; this fixture has no permanent authorizer.+ // Retain ownership even if the test panics before explicit removal.+ let result = unsafe {+ libsqlite3_sys::sqlite3_set_authorizer(+ handle.as_raw_handle().as_ptr(),+ Some(analyze_hook),+ Arc::into_raw(Arc::clone(hook)).cast_mut().cast(),+ )+ };+ assert_eq!(result, libsqlite3_sys::SQLITE_OK);+ }+ conn.return_to_pool().await;+}++async fn remove_analyze_hook(db: &DatabaseManager, hook: &Arc<AnalyzeHook>) {+ let mut conn = db.write_pool.acquire().await.unwrap();+ {+ let mut handle = conn.lock_handle().await.unwrap();+ // SAFETY: remove callback under exclusive access before releasing its Arc.+ unsafe {+ assert_eq!(+ libsqlite3_sys::sqlite3_set_authorizer(+ handle.as_raw_handle().as_ptr(),+ None,+ std::ptr::null_mut()+ ),+ libsqlite3_sys::SQLITE_OK+ );+ drop(Arc::from_raw(Arc::as_ptr(hook)));+ }+ }+ conn.return_to_pool().await;+}++async fn plan(conn: &mut sqlx::SqliteConnection) -> String {+ sqlx::query(PLAN)+ .persistent(false)+ .fetch_all(conn)+ .await+ .unwrap()+ .iter()+ .map(|r| r.get::<String, _>(3))+ .collect::<Vec<_>>()+ .join("; ")+}++async fn scan_steps(conn: &mut sqlx::SqliteConnection, reset: bool) -> i32 {+ let mut handle = conn.lock_handle().await.unwrap();+ // SAFETY: exclusive handle guard; statement pointers are borrowed only+ // while the guard prevents the SQLx worker from using this connection.+ unsafe {+ let db = handle.as_raw_handle().as_ptr();+ let mut stmt = libsqlite3_sys::sqlite3_next_stmt(db, std::ptr::null_mut());+ while !stmt.is_null() {+ let sql = std::ffi::CStr::from_ptr(libsqlite3_sys::sqlite3_sql(stmt))+ .to_str()+ .unwrap();+ if sql == LOOKUP {+ return libsqlite3_sys::sqlite3_stmt_status(+ stmt,+ libsqlite3_sys::SQLITE_STMTSTATUS_FULLSCAN_STEP,+ i32::from(reset),+ );+ }+ stmt = libsqlite3_sys::sqlite3_next_stmt(db, stmt);+ }+ }+ panic!("cached lookup missing");+}++async fn fixture() -> (tempfile::TempDir, DatabaseManager) {+ let root = tempfile::tempdir().unwrap();+ let config = DbConfig {+ read_pool_max: 4,+ read_pool_min: 1,+ write_pool_max: 1,+ ..Default::default()+ };+ let initial = DatabaseManager::new_hybrid(root.path(), config.clone(), Default::default())+ .await+ .unwrap();+ let storage = initial.storage.clone();+ let path = initial+ .write_pool+ .connect_options()+ .get_filename()+ .to_owned();+ initial.close().await;+ let db =+ DatabaseManager::new_with_storage(path.to_str().unwrap(), config, storage, false, false)+ .await+ .unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-08','private history');+ WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<30)+ INSERT INTO _bulk_element_rows(id,frame_id,text,_archive_generation) SELECT x,1,'private history',1 FROM n;+ ANALYZE _bulk_element_rows;+ CREATE TABLE planner_other(id INTEGER PRIMARY KEY, group_id INTEGER);+ CREATE INDEX planner_other_group ON planner_other(group_id);+ INSERT INTO planner_other SELECT id,1 FROM _bulk_element_rows;+ ANALYZE planner_other;+ INSERT INTO upload_bindings(destination,checkpoint) VALUES('test','{\"last_id\":42}');")+ .await.unwrap();+ let storage = db.storage.clone();+ let config = DbConfig {+ read_pool_max: 4,+ read_pool_min: 1,+ write_pool_max: 1,+ ..Default::default()+ };+ db.close().await;+ let db =+ DatabaseManager::new_with_storage(path.to_str().unwrap(), config, storage, false, false)+ .await+ .unwrap();+ (root, db)+}++async fn grow(db: &DatabaseManager, end: i64) {+ db.execute_raw_sql_write(&format!("WITH RECURSIVE n(x) AS (VALUES(31) UNION ALL SELECT x+1 FROM n WHERE x<{end})+ INSERT INTO _bulk_element_rows(id,frame_id,text,_archive_generation) SELECT x,1,'private history',1 FROM n;+ INSERT INTO planner_other SELECT id,1 FROM _bulk_element_rows WHERE id>30;"))+ .await.unwrap();+}++async fn maintain(db: &DatabaseManager) -> Attempt {+ planner_stats::run(+ &db.write_pool,+ &db.write_semaphore,+ &db.write_queue_health,+ &db.close_token,+ Attempt::new(),+ )+ .await+}++async fn finish(db: &DatabaseManager) -> usize {+ let mut analyzed = 0;+ for _ in 0..100 {+ let attempt = maintain(db).await;+ println!("maintenance: {attempt:?}");+ analyzed += attempt.analyzed_tables;+ if attempt.status == "complete" {+ return analyzed;+ }+ assert_eq!(attempt.status, "deferred");+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ panic!("maintenance failed to complete");+}++#[tokio::test]+async fn stale_statistics_repair_replans_existing_pools_and_preserves_storage() {+ let (_root, db) = fixture().await;+ // Keep two application readers alive, including their cached statements.+ let mut a = db.pool.acquire().await.unwrap();+ let mut b = db.pool.acquire().await.unwrap();+ let before = plan(&mut a).await;+ assert!(before.contains("SCAN _bulk_element_rows"), "{before}");+ assert!(plan(&mut b).await.contains("SCAN _bulk_element_rows"));+ grow(&db, 300_000).await;+ let stats: String =+ sqlx::query_scalar("SELECT stat FROM sqlite_stat1 WHERE idx='_bulk_element_staged_frame'")+ .fetch_one(&mut *a)+ .await+ .unwrap();+ assert_eq!(stats, "30 30");+ // The old maintenance path runs on the writer, whose SELECT history does+ // not contain these reader lookups. It leaves the 30-row estimate stale.+ {+ let permit = db.coordinated_writer().lock().await.unwrap();+ sqlx::query("PRAGMA optimize")+ .execute(permit.pool())+ .await+ .unwrap();+ }+ let stats: String =+ sqlx::query_scalar("SELECT stat FROM sqlite_stat1 WHERE idx='_bulk_element_staged_frame'")+ .fetch_one(&mut *a)+ .await+ .unwrap();+ assert_eq!(stats, "30 30");+ let rows_before: (i64, i64, i64) =+ sqlx::query_as("SELECT count(*),sum(id),sum(length(text)) FROM _bulk_element_rows")+ .fetch_one(&mut *a)+ .await+ .unwrap();+ let revisions: (i64, i64) = sqlx::query_as(+ "SELECT revision,(SELECT revision FROM _storage_revocation) FROM storage_metadata",+ )+ .fetch_one(&mut *a)+ .await+ .unwrap();+ let schema: Vec<(String, Option<String>)> = sqlx::query_as(+ "SELECT name,sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_stat%' ORDER BY name",+ )+ .fetch_all(&mut *a)+ .await+ .unwrap();+ let start = Instant::now();+ let ids: Vec<i64> = sqlx::query_scalar(LOOKUP).fetch_all(&mut *a).await.unwrap();+ let slow = start.elapsed();+ assert_eq!(ids, vec![1]);+ assert!(scan_steps(&mut a, true).await > 200_000);+ sqlx::query_scalar::<_, i64>(LOOKUP)+ .fetch_all(&mut *b)+ .await+ .unwrap();+ assert!(scan_steps(&mut b, true).await > 200_000);+ assert!(finish(&db).await >= 2);+ for conn in [&mut *a, &mut *b] {+ // EXPLAIN does not step the program's schema-cookie check. Execute+ // the cached application query first, as real callers do.+ assert_eq!(+ sqlx::query_scalar::<_, i64>(LOOKUP)+ .fetch_all(&mut *conn)+ .await+ .unwrap(),+ vec![1]+ );+ assert_eq!(+ scan_steps(conn, false).await,+ 0,+ "cached lookup must not scan"+ );+ let after = plan(conn).await;+ println!("before={before}; after={after}; scan_duration={slow:?}");+ assert!(+ after.contains("SEARCH _bulk_element_rows USING INTEGER PRIMARY KEY"),+ "{after}"+ );+ let start = Instant::now();+ assert_eq!(+ sqlx::query_scalar::<_, i64>(LOOKUP)+ .fetch_all(&mut *conn)+ .await+ .unwrap(),+ vec![1]+ );+ println!("pk_duration={:?}", start.elapsed());+ let other: Vec<(i64,i64,i64,String)> = sqlx::query_as("EXPLAIN QUERY PLAN SELECT id FROM planner_other WHERE id IN (SELECT value FROM json_each('[1]')) ORDER BY id").fetch_all(&mut *conn).await.unwrap();+ assert!(+ other+ .iter()+ .any(|r| r.3.contains("USING INTEGER PRIMARY KEY")),+ "{other:?}"+ );+ }+ assert_eq!(+ sqlx::query_as::<_, (i64, i64, i64)>(+ "SELECT count(*),sum(id),sum(length(text)) FROM _bulk_element_rows"+ )+ .fetch_one(&mut *a)+ .await+ .unwrap(),+ rows_before+ );+ assert_eq!(+ sqlx::query_as::<_, (i64, i64)>(+ "SELECT revision,(SELECT revision FROM _storage_revocation) FROM storage_metadata"+ )+ .fetch_one(&mut *a)+ .await+ .unwrap(),+ revisions+ );+ assert_eq!(+ sqlx::query_as::<_, (String, Option<String>)>(+ "SELECT name,sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_stat%' ORDER BY name"+ )+ .fetch_all(&mut *a)+ .await+ .unwrap(),+ schema+ );+ let selected = sqlx::query_scalar::<_, String>("PRAGMA main.optimize=0x10003")+ .fetch_all(&db.write_pool)+ .await+ .unwrap();+ assert!(+ !selected+ .iter()+ .any(|s| s.contains("_bulk_element_rows") || s.contains("planner_other")),+ "unchanged large tables need no reanalysis: {selected:?}"+ );+ assert_eq!(+ sqlx::query_scalar::<_, String>(+ "SELECT checkpoint FROM upload_bindings WHERE destination='test'"+ )+ .fetch_one(&mut *a)+ .await+ .unwrap(),+ r#"{"last_id":42}"#+ );+ drop(a);+ drop(b);+ db.close().await;+}++#[tokio::test]+async fn maintenance_defers_to_writer_and_recording_remains_durable() {+ let (root, db) = fixture().await;+ grow(&db, 100_000).await;+ let tx = db.begin_immediate_with_retry().await.unwrap();+ let start = Instant::now();+ let deferred = maintain(&db).await;+ assert_eq!(deferred.status, "deferred");+ assert!(start.elapsed() < Duration::from_millis(100));+ planner_stats::save(root.path(), &deferred).await;+ tx.commit().await.unwrap();+ let writes = async {+ let mut max_latency = Duration::ZERO;+ for id in 2..52 {+ let start = Instant::now();+ let _ = id;+ db.insert_snapshot_frame(+ "test",+ chrono::Utc::now(),+ "",+ None,+ None,+ None,+ true,+ Some("test"),+ Some("private history"),+ Some("accessibility"),+ None,+ None,+ None,+ )+ .await+ .unwrap();+ max_latency = max_latency.max(start.elapsed());+ tokio::time::sleep(Duration::from_millis(2)).await;+ }+ println!("maximum capture writer latency={max_latency:?}");+ assert!(max_latency < Duration::from_secs(1));+ };+ let (_, _) = tokio::join!(writes, finish(&db));+ let complete = maintain(&db).await;+ planner_stats::save(root.path(), &complete).await;+ let path = db.write_pool.connect_options().get_filename().to_owned();+ let config = DbConfig::default();+ db.close().await;+ let reopened = DatabaseManager::new_with_storage(+ path.to_str().unwrap(),+ config,+ crate::storage::HybridStorage::for_index(&path).unwrap(),+ false,+ false,+ )+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames")+ .fetch_one(&reopened.pool)+ .await+ .unwrap(),+ 51+ );+ let report = planner_stats::report(root.path()).unwrap();+ assert_eq!(+ report.last_failure.unwrap().detail.unwrap(),+ "writer busy; retry on next maintenance tick"+ );+ assert_eq!(report.last_success.unwrap().status, "complete");+ reopened.close().await;+}++#[tokio::test]+async fn interrupted_analysis_rolls_back_and_returns_a_usable_writer() {+ let (_root, db) = fixture().await;+ grow(&db, 100_000).await;+ let original: String =+ sqlx::query_scalar("SELECT stat FROM sqlite_stat1 WHERE idx='_bulk_element_staged_frame'")+ .fetch_one(&db.pool)+ .await+ .unwrap();+ let mut conn = db.write_pool.acquire().await.unwrap();+ let busy: i64 = sqlx::query_scalar("PRAGMA busy_timeout")+ .fetch_one(&mut *conn)+ .await+ .unwrap();+ let limit: i64 = sqlx::query_scalar("PRAGMA analysis_limit")+ .fetch_one(&mut *conn)+ .await+ .unwrap();+ conn.return_to_pool().await;+ let cancel = tokio_util::sync::CancellationToken::new();+ let hook = Arc::new(AnalyzeHook {+ entered: tokio::sync::Notify::new(),+ delay: Duration::from_millis(10),+ cancel: Some(cancel.clone()),+ });+ install_analyze_hook(&db, &hook).await;+ let error = planner_stats::slice(+ &db.write_pool,+ &db.write_semaphore,+ &db.write_queue_health,+ &cancel,+ Some("ANALYZE main._bulk_element_rows"),+ )+ .await+ .unwrap_err();+ remove_analyze_hook(&db, &hook).await;+ assert!(crate::is_sqlite_interrupt(&error), "{error}");+ let mut conn = db.write_pool.acquire().await.unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("PRAGMA busy_timeout")+ .fetch_one(&mut *conn)+ .await+ .unwrap(),+ busy+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>("PRAGMA analysis_limit")+ .fetch_one(&mut *conn)+ .await+ .unwrap(),+ limit+ );+ conn.return_to_pool().await;+ assert_eq!(+ sqlx::query_scalar::<_, String>(+ "SELECT stat FROM sqlite_stat1 WHERE idx='_bulk_element_staged_frame'"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ original+ );+ db.insert_snapshot_frame(+ "test",+ chrono::Utc::now(),+ "",+ None,+ None,+ None,+ true,+ Some("test"),+ Some("private history"),+ Some("accessibility"),+ None,+ None,+ None,+ )+ .await+ .unwrap();+ assert!(!db.write_queue_health.is_hard_faulted());+ finish(&db).await;+ db.close().await;+}++#[tokio::test]+async fn background_tick_repairs_reopened_history_without_capture_trigger() {+ let (root, db) = fixture().await;+ grow(&db, 100_000).await;+ db.start_wal_maintenance();+ tokio::time::timeout(Duration::from_secs(70), async {+ loop {+ if planner_stats::report(root.path())+ .unwrap()+ .last_success+ .is_some()+ {+ break;+ }+ tokio::time::sleep(Duration::from_millis(100)).await;+ }+ })+ .await+ .unwrap();+ let mut reader = db.pool.acquire().await.unwrap();+ sqlx::query_scalar::<_, i64>(LOOKUP)+ .fetch_all(&mut *reader)+ .await+ .unwrap();+ assert!(plan(&mut reader)+ .await+ .contains("USING INTEGER PRIMARY KEY"));+ drop(reader);+ db.close().await;+}++#[tokio::test]+async fn dropped_maintenance_keeps_writer_until_sqlite_cleanup_finishes() {+ let (_root, db) = fixture().await;+ let db = Arc::new(db);+ // Startup returns pooled connections asynchronously; make this writer+ // available before testing cancellation rather than incidental deferral.+ let mut ready = db.write_pool.acquire().await.unwrap();+ ready.return_to_pool().await;+ let copy = Arc::clone(&db);+ let cancel = tokio_util::sync::CancellationToken::new();+ let shutdown = cancel.clone();+ // Deliberately expensive SQL at the same shutdown cancellation boundary.+ // Aborting the caller must not abandon its active worker/transaction.+ let task = tokio::spawn(async move {+ planner_stats::slice(©.write_pool, ©.write_semaphore, ©.write_queue_health,+ &shutdown, Some("WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<100000000) SELECT sum(x) FROM n")).await+ });+ tokio::time::timeout(Duration::from_secs(1), async {+ while db.write_semaphore.available_permits() != 0 {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ task.abort();+ let _ = task.await;+ cancel.cancel();+ tokio::time::timeout(+ Duration::from_secs(1),+ db.insert_snapshot_frame(+ "test",+ chrono::Utc::now(),+ "",+ None,+ None,+ None,+ true,+ Some("test"),+ Some("private history"),+ Some("accessibility"),+ None,+ None,+ None,+ ),+ )+ .await+ .unwrap()+ .unwrap();+ // The shutdown watcher must not interrupt the next borrower.+ finish(&db).await;+ db.close().await;+}++#[tokio::test]+async fn slow_analysis_finishes_and_waiting_recording_is_preserved() {+ let (root, db) = fixture().await;+ grow(&db, 300_000).await;+ let mut reader = db.pool.acquire().await.unwrap();+ assert!(plan(&mut reader).await.contains("SCAN _bulk_element_rows"));+ sqlx::query_scalar::<_, i64>(LOOKUP)+ .fetch_all(&mut *reader)+ .await+ .unwrap();+ assert!(scan_steps(&mut reader, true).await > 200_000);+ let hook = Arc::new(AnalyzeHook {+ entered: tokio::sync::Notify::new(),+ delay: Duration::from_millis(250),+ cancel: None,+ });+ install_analyze_hook(&db, &hook).await;+ let record = async {+ hook.entered.notified().await;+ let start = Instant::now();+ db.insert_snapshot_frame(+ "test",+ chrono::Utc::now(),+ "",+ None,+ None,+ None,+ true,+ Some("test"),+ Some("recorded while analysis was running"),+ Some("accessibility"),+ None,+ None,+ None,+ )+ .await+ .unwrap();+ start.elapsed()+ };+ let (attempt, recording_wait) = tokio::time::timeout(Duration::from_secs(10), async {+ tokio::join!(db.maintain_planner_statistics(), record)+ })+ .await+ .unwrap();+ remove_analyze_hook(&db, &hook).await;+ println!("slow analysis attempt={attempt:?}; recording wait={recording_wait:?}");+ assert!(attempt.analyzed_tables >= 1, "{attempt:?}");+ assert!(attempt.duration_ms >= 250, "{attempt:?}");+ assert_eq!(+ attempt.sqlite_code, None,+ "slow analysis must not be interrupted"+ );+ // The slow table must be repaired in this attempt, even when maintenance+ // defers other tables to let the waiting recording take the writer.+ assert_eq!(+ sqlx::query_scalar::<_, i64>(LOOKUP)+ .fetch_all(&mut *reader)+ .await+ .unwrap(),+ vec![1]+ );+ assert_eq!(scan_steps(&mut reader, false).await, 0);+ assert!(plan(&mut reader)+ .await+ .contains("USING INTEGER PRIMARY KEY"));+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows")+ .fetch_one(&mut *reader)+ .await+ .unwrap(),+ 300_000+ );+ drop(reader);+ let path = db.write_pool.connect_options().get_filename().to_owned();+ db.close().await;+ let reopened = DatabaseManager::new_with_storage(+ path.to_str().unwrap(),+ DbConfig::default(),+ crate::storage::HybridStorage::for_index(&path).unwrap(),+ false,+ false,+ )+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames")+ .fetch_one(&reopened.pool)+ .await+ .unwrap(),+ 2+ );+ assert!(+ planner_stats::report(root.path())+ .unwrap()+ .latest+ .unwrap()+ .duration_ms+ >= 250+ );+ reopened.close().await;+}diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rsindex fedb7e3d2..96dd1c18c 100644--- a/crates/screenpipe-db/src/lib.rs+++ b/crates/screenpipe-db/src/lib.rs@@ -31,6 +31,7 @@ mod cancellable_query;mod db;#[cfg(test)]mod failpoint_vfs;+pub mod planner_stats;mod recovery;mod sqlite_error;pub mod storage;diff --git a/crates/screenpipe-db/src/planner_stats.rs b/crates/screenpipe-db/src/planner_stats.rsnew file mode 100644index 000000000..0146a11bb--- /dev/null+++ b/crates/screenpipe-db/src/planner_stats.rs@@ -0,0 +1,406 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++//! Sampled planner maintenance, independent of the writer's query history.+//! No capture-path hooks, connection recycling, or recording-data mutations.++use serde::{Deserialize, Serialize};+use sqlx::{SqliteConnection, SqlitePool};+use std::{+ path::Path,+ sync::{Arc, Mutex},+ time::{Duration, Instant},+};+use tokio::sync::Semaphore;+use tokio_util::sync::CancellationToken;++pub(crate) const INTERVAL: Duration = Duration::from_secs(3600);+// SQLite recommends approximate ANALYZE with 100–1000 sampled rows per index;+// 1000 also repaired the reported production plan. This bounds sampling, not+// elapsed time: counting index pages may take longer and must be allowed to finish.+const ANALYSIS_LIMIT: i64 = 1000;+const FILE: &str = "sqlite-planner-maintenance.json";++/// Allowlisted metadata only: never SQL, table contents, paths, or raw errors.+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct Attempt {+ pub started_at: String,+ pub status: String,+ pub duration_ms: u64,+ pub analyzed_tables: usize,+ pub stage: String,+ pub sqlite_code: Option<String>,+ pub detail: Option<String>,+}++#[derive(Clone, Default, Debug, Serialize, Deserialize)]+pub struct Report {+ pub latest: Option<Attempt>,+ pub last_success: Option<Attempt>,+ pub last_failure: Option<Attempt>,+}++/// Read durable evidence even after engine restart or rolling-log rotation.+pub fn report(root: &Path) -> Result<Report, std::io::Error> {+ let file = match std::fs::File::open(root.join(FILE)) {+ Ok(file) => file,+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Report::default()),+ Err(e) => return Err(e),+ };+ serde_json::from_reader(file).map_err(std::io::Error::other)+}++pub(crate) async fn save(root: &Path, attempt: &Attempt) {+ let root = root.to_owned();+ let attempt = attempt.clone();+ let result = tokio::task::spawn_blocking(move || {+ let mut saved = report(&root).map_err(sqlx::Error::Io)?;+ if attempt.status == "complete" {+ saved.last_success = Some(attempt.clone());+ } else if attempt.status != "running" {+ saved.last_failure = Some(attempt.clone());+ }+ saved.latest = Some(attempt);+ crate::storage::durable_json(&root.join(FILE), &saved)+ })+ .await;+ if !matches!(result, Ok(Ok(()))) {+ tracing::warn!("sqlite planner maintenance: could not persist support diagnostics");+ }+}++impl Attempt {+ pub(crate) fn new() -> Self {+ Self {+ started_at: chrono::Utc::now().to_rfc3339(),+ status: "running".into(),+ duration_ms: 0,+ analyzed_tables: 0,+ stage: "selecting_tables".into(),+ sqlite_code: None,+ detail: None,+ }+ }++ fn failure(&mut self, error: &sqlx::Error) {+ self.status = "deferred".into();+ self.sqlite_code = error+ .as_database_error()+ .and_then(|e| e.code())+ .map(|s| s.into_owned());+ // sqlite3_errstr returns a static generic description, unlike errmsg,+ // which can embed SQL, captured values or filesystem paths.+ self.detail = Some(+ if let Some(code) = self+ .sqlite_code+ .as_ref()+ .and_then(|s| s.parse::<i32>().ok())+ {+ // SAFETY: SQLite owns this immutable, NUL-terminated string.+ unsafe { std::ffi::CStr::from_ptr(libsqlite3_sys::sqlite3_errstr(code)) }+ .to_string_lossy()+ .into_owned()+ } else {+ match error {+ sqlx::Error::PoolClosed => "database pool closed",+ sqlx::Error::PoolTimedOut => "database pool unavailable",+ sqlx::Error::Io(_) => "maintenance I/O failure",+ _ => "maintenance failed; inspect SQLite health diagnostics",+ }+ .into()+ },+ );+ if crate::sqlite_error::is_sqlite_hard_fault(error) {+ self.status = "failed".into();+ }+ }+}++/// One slice owns the existing writer lane. Do not queue ahead of capture or+/// wait for another SQLite writer. Await SQLite completion before releasing+/// the permit; a Tokio timeout alone would leave its worker running.+pub(crate) async fn slice(+ pool: &SqlitePool,+ semaphore: &Arc<Semaphore>,+ health: &crate::write_queue::WriteQueueHealth,+ shutdown: &CancellationToken,+ analyze: Option<&str>,+) -> Result<Option<Vec<String>>, sqlx::Error> {+ let Ok(permit) = Arc::clone(semaphore).try_acquire_owned() else {+ return Ok(None);+ };+ if health.is_hard_faulted() || shutdown.is_cancelled() {+ return Err(sqlx::Error::PoolClosed);+ }+ let Some(mut conn) = pool.try_acquire() else {+ return Ok(None);+ };+ let health = health.clone();+ let shutdown = shutdown.clone();+ let analyze = analyze.map(str::to_owned);+ // Own both the connection and writer permit until SQLite has stopped and+ // cleanup has finished, even if the caller drops/aborts its async future.+ tokio::spawn(async move {+ let _permit = permit;+ let result = on_connection(&mut conn, &health, &shutdown, analyze.as_deref()).await;+ if let Err(error) = &result {+ if crate::sqlite_error::is_sqlite_hard_fault(error) {+ health.latch_hard_fault(error);+ }+ }+ conn.return_to_pool().await;+ result.map(Some)+ })+ .await+ .map_err(|_| sqlx::Error::WorkerCrashed)?+}++// SQLite's B-tree count can run inside a single VM opcode. A progress+// handler alone cannot interrupt it during shutdown. sqlite3_interrupt is thread-safe;+// this guard synchronizes its lifetime with the owned connection, including+// unwinding, so cancellation cannot interrupt a later borrower or freed handle.+struct InterruptOnShutdown {+ target: Arc<Mutex<Option<usize>>>,+ watcher: tokio::task::JoinHandle<()>,+}++impl InterruptOnShutdown {+ fn new(raw: usize, shutdown: CancellationToken) -> Self {+ let target = Arc::new(Mutex::new(Some(raw)));+ let watcher_target = Arc::clone(&target);+ let watcher = tokio::spawn(async move {+ shutdown.cancelled().await;+ let guard = watcher_target.lock().unwrap_or_else(|e| e.into_inner());+ if let Some(raw) = *guard {+ // SAFETY: Drop clears the pointer under this same lock before+ // the owning task can return/close the connection.+ unsafe {+ libsqlite3_sys::sqlite3_interrupt(raw as *mut libsqlite3_sys::sqlite3);+ }+ }+ });+ Self { target, watcher }+ }+}++impl Drop for InterruptOnShutdown {+ fn drop(&mut self) {+ *self.target.lock().unwrap_or_else(|e| e.into_inner()) = None;+ self.watcher.abort();+ }+}++async fn on_connection(+ conn: &mut SqliteConnection,+ health: &crate::write_queue::WriteQueueHealth,+ shutdown: &CancellationToken,+ analyze: Option<&str>,+) -> Result<Vec<String>, sqlx::Error> {+ let busy: i64 = sqlx::query_scalar("PRAGMA busy_timeout")+ .fetch_one(&mut *conn)+ .await?;+ let limit: i64 = sqlx::query_scalar("PRAGMA analysis_limit")+ .fetch_one(&mut *conn)+ .await?;+ sqlx::query("PRAGMA busy_timeout=0")+ .execute(&mut *conn)+ .await?;+ let cancel = shutdown.clone();+ let raw = {+ let mut handle = conn.lock_handle().await?;+ handle.set_progress_handler(1000, move || !cancel.is_cancelled());+ handle.as_raw_handle().as_ptr() as usize+ };+ let watcher = InterruptOnShutdown::new(raw, shutdown.clone());+ let result = work(conn, analyze).await;+ drop(watcher);+ if let Err(error) = &result {+ if crate::sqlite_error::is_sqlite_hard_fault(error) {+ health.latch_hard_fault(error);+ }+ }+ let cleanup = async {+ let in_transaction = {+ let mut handle = conn.lock_handle().await?;+ handle.remove_progress_handler();+ // SAFETY: the handle guard excludes SQLx. INTERRUPT can already have+ // rolled back the transaction; SQLx's transaction-depth counter does+ // not track that, so maintenance uses SQLite's actual transaction state.+ unsafe { libsqlite3_sys::sqlite3_get_autocommit(handle.as_raw_handle().as_ptr()) == 0 }+ };+ if in_transaction {+ sqlx::query("ROLLBACK").execute(&mut *conn).await?;+ }+ // Always restore connection-local settings before normal writes reuse it.+ sqlx::query(sqlx::AssertSqlSafe(format!(+ "PRAGMA analysis_limit={limit}"+ )))+ .execute(&mut *conn)+ .await?;+ sqlx::query(sqlx::AssertSqlSafe(format!("PRAGMA busy_timeout={busy}")))+ .execute(&mut *conn)+ .await?;+ if result.is_err()+ && !result+ .as_ref()+ .err()+ .is_some_and(crate::sqlite_error::is_sqlite_hard_fault)+ {+ // ANALYZE may have loaded uncommitted estimates before a later error+ // rolled back the transaction. Restore this writer's in-memory view.+ sqlx::query("ANALYZE main.sqlite_schema")+ .execute(&mut *conn)+ .await?;+ }+ Ok::<_, sqlx::Error>(())+ }+ .await;+ if let Err(error) = &cleanup {+ if crate::sqlite_error::is_sqlite_hard_fault(error) {+ health.latch_hard_fault(error);+ }+ }+ // A cleanup error must not overwrite the originating SQLite failure.+ match result {+ Err(error) => Err(error),+ Ok(rows) => cleanup.map(|()| rows),+ }+}++async fn work(+ conn: &mut SqliteConnection,+ analyze: Option<&str>,+) -> Result<Vec<String>, sqlx::Error> {+ if let Some(statement) = analyze {+ sqlx::query(sqlx::AssertSqlSafe(format!(+ "PRAGMA analysis_limit={ANALYSIS_LIMIT}"+ )))+ .execute(&mut *conn)+ .await?;+ sqlx::query("BEGIN IMMEDIATE").execute(&mut *conn).await?;+ // The statement comes exclusively from SQLite's optimize dry run.+ sqlx::query(sqlx::AssertSqlSafe(statement))+ .execute(&mut *conn)+ .await?;+ // ANALYZE reloads only this connection's in-memory statistics. A real,+ // transactional schema change makes ALL existing connections reload+ // their schema/statistics and reprepare cached statements at their next+ // snapshot, without closing pools or writing schema_version by hand.+ // No persistent object or storage revision is changed.+ sqlx::raw_sql("CREATE TABLE main._screenpipe_planner_refresh(dummy); DROP TABLE main._screenpipe_planner_refresh;")+ .execute(&mut *conn).await?;+ sqlx::query("COMMIT").execute(conn).await?;+ Ok(Vec::new())+ } else {+ // Bundled SQLite >=3.51.3: debug + ANALYZE + all-table growth checks.+ // Selection uses b-tree estimates, not COUNT(*) scans. Analyze only+ // missing stats or ~10x growth/shrinkage, including reader-only tables.+ let candidates: Vec<String> = sqlx::query_scalar("PRAGMA main.optimize=0x10003")+ .fetch_all(&mut *conn)+ .await?;+ let mut needed = Vec::new();+ for statement in candidates {+ // Empty indexless tables have no stat1 row even after ANALYZE,+ // so SQLite can nominate them on every pass. Skip them in O(1)+ // rather than repeatedly taking writer slices ahead of real work.+ let quoted_table = statement+ .strip_prefix("ANALYZE ")+ .ok_or_else(|| sqlx::Error::Protocol("unexpected optimize operation".into()))?;+ let populated: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(+ "SELECT EXISTS(SELECT 1 FROM {quoted_table} LIMIT 1)"+ )))+ .fetch_one(&mut *conn)+ .await?;+ if populated {+ needed.push(statement);+ }+ }+ Ok(needed)+ }+}++pub(crate) async fn run_and_report(+ pool: &SqlitePool,+ semaphore: &Arc<Semaphore>,+ health: &crate::write_queue::WriteQueueHealth,+ shutdown: &CancellationToken,+ root: Option<&Path>,+) -> Attempt {+ let attempt = Attempt::new();+ if let Some(root) = root {+ save(root, &attempt).await;+ }+ let attempt = run(pool, semaphore, health, shutdown, attempt).await;+ tracing::info!(diagnostic = %serde_json::to_string(&attempt).unwrap_or_default(), "sqlite planner maintenance");+ if let Some(root) = root {+ save(root, &attempt).await;+ }+ attempt+}++pub(crate) async fn run(+ pool: &SqlitePool,+ semaphore: &Arc<Semaphore>,+ health: &crate::write_queue::WriteQueueHealth,+ shutdown: &CancellationToken,+ mut attempt: Attempt,+) -> Attempt {+ let started = Instant::now();+ let result = async {+ let Some(statements) = slice(pool, semaphore, health, shutdown, None).await? else {+ return Ok(false);+ };+ let mut first_error = None;+ for statement in statements {+ attempt.stage = "analyzing_table".into();+ // Each table commits independently so a deferred table cannot+ // discard completed work. Let queued captures take the writer.+ tokio::task::yield_now().await;+ match slice(pool, semaphore, health, shutdown, Some(&statement)).await {+ Ok(Some(_)) => attempt.analyzed_tables += 1,+ Ok(None) => return first_error.map_or(Ok(false), Err),+ Err(error) => {+ if health.is_hard_faulted() || shutdown.is_cancelled() {+ return Err(error);+ }+ // A failed table must not prevent repair of other tables.+ // Keep the first originating failure for support and retry+ // unfinished work on the next tick.+ if first_error.is_none() {+ first_error = Some(error);+ }+ }+ }+ }+ first_error.map_or(Ok(true), Err)+ }+ .await;+ match result {+ Ok(true) => {+ attempt.status = "complete".into();+ attempt.stage = "complete".into();+ }+ Ok(false) => {+ attempt.status = "deferred".into();+ attempt.detail = Some("writer busy; retry on next maintenance tick".into());+ }+ Err(error) => {+ if crate::sqlite_error::is_sqlite_hard_fault(&error) {+ health.latch_hard_fault(&error);+ }+ attempt.failure(&error);+ if crate::is_sqlite_interrupt(&error) {+ attempt.detail = Some(+ if shutdown.is_cancelled() {+ "shutdown interrupted maintenance; retry after restart"+ } else {+ "SQLite interrupted maintenance; retry on next maintenance tick"+ }+ .into(),+ );+ }+ }+ }+ attempt.duration_ms = started.elapsed().as_millis() as u64;+ attempt+}