YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): avoid scanning retained elements during cleanup (#7666)
Ezra Ellette30b50bb2449a
diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex 50d6b4584..c3cee3399 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -549,6 +549,11 @@ impl DatabaseManager {let upgraded =crate::storage::schema::upgrade_recording(&mut conn, storage.has_bulk()).await?;+ // Keep this independent of versioned recording migrations:+ // installed hybrid databases may already have migration v6.+ let cleanup_indexes_changed =+ crate::storage::schema::ensure_cleanup_indexes(&mut conn, storage.has_bulk())+ .await?;crate::storage::read_schema::upgrade(&mut conn, storage).await?;let resident_hooks_changed = crate::storage::schema::ensure_resident_table_hooks(&mut conn,@@ -557,7 +562,7 @@ impl DatabaseManager {.await?;storage.verify_catalog(&db_manager.pool).await?;drop(conn);- if upgraded || resident_hooks_changed {+ if upgraded || cleanup_indexes_changed || resident_hooks_changed {// Connections opened before the trigger migration need a// schema read before their first DML preparation. Refresh// every existing writer while startup still owns admission.diff --git a/crates/screenpipe-db/src/storage/bulk/elements.rs b/crates/screenpipe-db/src/storage/bulk/elements.rsindex f09d4225c..af038eb0c 100644--- a/crates/screenpipe-db/src/storage/bulk/elements.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements.rs@@ -28,6 +28,42 @@ pub(super) const TABLE: Table = Table {fts_condition: "text IS NOT NULL AND text!=''",};+pub(super) const TOMBSTONE_INDEX: &str = "_bulk_element_resident_tombstones";+pub(super) const TOMBSTONE_PREDICATE: &str = "_archive_deleted=1 AND _archive_file IS NULL";+pub(super) fn tombstone_index_sql() -> String {+ format!(+ "CREATE INDEX IF NOT EXISTS {TOMBSTONE_INDEX} \+ ON _bulk_element_rows(id) WHERE {TOMBSTONE_PREDICATE}"+ )+}++pub(super) fn cleanup_sql() -> String {+ format!(+ "DELETE FROM _bulk_element_rows+ INDEXED BY {TOMBSTONE_INDEX}+ WHERE {TOMBSTONE_PREDICATE} AND id IN (+ SELECT id FROM _bulk_element_rows+ INDEXED BY {TOMBSTONE_INDEX}+ WHERE {TOMBSTONE_PREDICATE} LIMIT {})",+ super::FILE_ROWS+ )+}++pub(super) async fn ensure_cleanup_index(conn: &mut SqliteConnection) -> Result<bool, sqlx::Error> {+ let existed: bool = sqlx::query_scalar(+ "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='index' AND name=?)",+ )+ .bind(TOMBSTONE_INDEX)+ .fetch_one(&mut *conn)+ .await?;+ // Normal startup can share a WAL with active readers. Requiring the+ // offline construction checkpoint here would reject a valid startup.+ sqlx::query(sqlx::AssertSqlSafe(tombstone_index_sql()))+ .execute(&mut *conn)+ .await?;+ Ok(!existed)+}+pub(super) fn names() -> String {TABLE.columns@@ -175,6 +211,7 @@ pub(super) async fn bootstrap_mode(INSERT INTO elements_fts(rowid,text,role,frame_id) SELECT id,text,role,frame_id FROM _bulk_element_rows WHERE text IS NOT NULL AND text!='';UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE((SELECT SUM({bytes}) FROM _bulk_element_rows),0);",bytes=TABLE.all_bytes(""))).await?;+ crate::storage::schema::construction_sql(conn, &tombstone_index_sql()).await?;if in_place {sqlx::query("UPDATE sqlite_sequence SET name='elements' WHERE name='_bulk_elements_source'",diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rsindex f00f99d50..da8daec5e 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs@@ -445,11 +445,13 @@ pub(crate) async fn reclaim(rewrite(storage, pool, writer, range.0, Some(range)).await?;}let permit = writer.lock().await?;- sqlx::query(- "DELETE FROM _bulk_element_rows WHERE _archive_deleted=1 AND _archive_file IS NULL",- )- .execute(permit.pool())- .await?;+ // Force the narrow partial index: installed databases can have stale+ // sqlite_stat1 rows that otherwise make SQLite rescan all resident history.+ // One established file-sized batch bounds writer occupancy; later+ // maintenance turns durably drain the remainder.+ sqlx::query(sqlx::AssertSqlSafe(super::cleanup_sql()))+ .execute(permit.pool())+ .await?;sqlx::query("UPDATE _bulk_files SET state='retired' WHERE table_name='elements' AND state='encoding'",)diff --git a/crates/screenpipe-db/src/storage/bulk/mod.rs b/crates/screenpipe-db/src/storage/bulk/mod.rsindex 80dc717a2..3698e8b9e 100644--- a/crates/screenpipe-db/src/storage/bulk/mod.rs+++ b/crates/screenpipe-db/src/storage/bulk/mod.rs@@ -18,7 +18,9 @@ use super::{storage_error, HybridStorage};pub(crate) use connection::pool_options;pub(crate) use connection::{register_hash, register_hash_extension};pub(super) use lifecycle::export;-pub(super) use schema::{bootstrap, bootstrap_in_place, finish_indexes, upgrade_recording};+pub(super) use schema::{+ bootstrap, bootstrap_in_place, ensure_cleanup_indexes, finish_indexes, upgrade_recording,+};use std::sync::{atomic::AtomicUsize, Arc, Mutex};pub const CAPABILITY: &str = "parquet-bulk-v1";diff --git a/crates/screenpipe-db/src/storage/bulk/schema.rs b/crates/screenpipe-db/src/storage/bulk/schema.rsindex 73e4925d2..35d4b95da 100644--- a/crates/screenpipe-db/src/storage/bulk/schema.rs+++ b/crates/screenpipe-db/src/storage/bulk/schema.rs@@ -178,3 +178,9 @@ pub(crate) async fn upgrade_recording(conn: &mut SqliteConnection) -> Result<(),}Ok(())}++pub(crate) async fn ensure_cleanup_indexes(+ conn: &mut SqliteConnection,+) -> Result<bool, sqlx::Error> {+ super::elements::ensure_cleanup_index(conn).await+}diff --git a/crates/screenpipe-db/src/storage/bulk/tests.rs b/crates/screenpipe-db/src/storage/bulk/tests.rsindex 0259f4a40..f4ad53832 100644--- a/crates/screenpipe-db/src/storage/bulk/tests.rs+++ b/crates/screenpipe-db/src/storage/bulk/tests.rs@@ -3,6 +3,7 @@use super::*;use crate::DatabaseManager;+use sqlx::Row;use std::sync::{atomic::{AtomicUsize, Ordering},Condvar,@@ -32,6 +33,304 @@ impl Drop for Pause {}}+#[tokio::test]+async fn resident_tombstone_cleanup_is_indexed_batched_and_upgraded_on_reopen() {+ use std::sync::atomic::AtomicU64;++ unsafe extern "C" fn count_steps(context: *mut std::ffi::c_void) -> i32 {+ let steps = unsafe { &*context.cast::<AtomicU64>() };+ steps.fetch_add(1000, Ordering::Relaxed);+ 0+ }++ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("db.sqlite");+ let mut options = crate::storage::MigrationOptions::default();+ options.budget.record_bytes = 1024 * 1024;+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)+ .await+ .unwrap();+ db.execute_raw_sql_write(+ "WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<1000000)+ INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,_archive_generation)+ SELECT id,1,'accessibility','AXText','retained',1 FROM n;+ UPDATE _bulk_element_rows SET text=hex(zeroblob(1048576)) WHERE id=1000000;+ ANALYZE;+ DELETE FROM sqlite_stat1;+ ANALYZE sqlite_schema",+ )+ .await+ .unwrap();++ let mut baseline_tx = db.begin_immediate_with_retry().await.unwrap();+ sqlx::query("DROP INDEX _bulk_element_resident_tombstones")+ .execute(&mut **baseline_tx.conn())+ .await+ .unwrap();+ let baseline_plan: Vec<String> = sqlx::query(+ "EXPLAIN QUERY PLAN DELETE FROM _bulk_element_rows+ WHERE _archive_deleted=1 AND _archive_file IS NULL",+ )+ .fetch_all(&mut **baseline_tx.conn())+ .await+ .unwrap()+ .into_iter()+ .map(|row| row.try_get(3).unwrap())+ .collect();+ baseline_tx.commit().await.unwrap();+ assert!(baseline_plan+ .iter()+ .any(|line| line == "SCAN _bulk_element_rows"));+ let baseline_steps = Box::new(AtomicU64::new(0));+ let mut baseline_tx = db.begin_immediate_with_retry().await.unwrap();+ {+ let mut handle = baseline_tx.conn().lock_handle().await.unwrap();+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ handle.as_raw_handle().as_ptr(),+ 1000,+ Some(count_steps),+ (&*baseline_steps as *const AtomicU64).cast_mut().cast(),+ );+ }+ }+ let baseline_started = std::time::Instant::now();+ sqlx::query(+ "DELETE FROM _bulk_element_rows+ WHERE _archive_deleted=1 AND _archive_file IS NULL",+ )+ .execute(&mut **baseline_tx.conn())+ .await+ .unwrap();+ let baseline_elapsed = baseline_started.elapsed();+ let baseline_steps = baseline_steps.load(Ordering::Relaxed);+ {+ let mut handle = baseline_tx.conn().lock_handle().await.unwrap();+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ handle.as_raw_handle().as_ptr(),+ 0,+ None,+ std::ptr::null_mut(),+ );+ }+ }+ baseline_tx.rollback().await.unwrap();+ assert!(+ baseline_steps >= 2_000_000,+ "baseline unexpectedly avoided its full scan: {baseline_steps}"+ );+ db.close().await;++ // Startup must add the index even though this installed database already+ // has the current versioned recording migration, and repeated reopen is+ // idempotent.+ let db = Arc::new(+ DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap(),+ );+ assert!(sqlx::query_scalar::<_, bool>(+ "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='_bulk_element_resident_tombstones')"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap());+ let missing_stats_plan: Vec<String> = sqlx::query(sqlx::AssertSqlSafe(format!(+ "EXPLAIN QUERY PLAN {}",+ elements::cleanup_sql()+ )))+ .fetch_all(&db.pool)+ .await+ .unwrap()+ .into_iter()+ .map(|row| row.try_get(3).unwrap())+ .collect();+ assert!(missing_stats_plan.iter().all(|line| {+ !line.contains("SCAN _bulk_element_rows") || line.contains(elements::TOMBSTONE_INDEX)+ }));+ db.execute_raw_sql_write(&format!(+ "ANALYZE;+ UPDATE sqlite_stat1 SET stat='30 30' WHERE tbl='_bulk_element_rows';+ UPDATE sqlite_stat1 SET stat='1000000000 1' WHERE idx='{}';+ ANALYZE sqlite_schema;+ INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-16','retained');+ INSERT INTO _bulk_files(id,table_name,path,checksum,state,rows)+ VALUES(1,'elements','preserved.parquet','test','published',1);+ INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,_archive_generation,_archive_deleted,_archive_file)+ VALUES(1000001,1,'accessibility','AXText','archived tombstone',1,1,1);+ WITH RECURSIVE n(id) AS (VALUES(1000002) UNION ALL SELECT id+1 FROM n WHERE id<{})+ INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,_archive_generation,_archive_deleted)+ SELECT id,1,'accessibility','AXText','resident tombstone',1,1 FROM n",+ elements::TOMBSTONE_INDEX,+ 1_000_001 + FILE_ROWS as i64 + 1+ )).await.unwrap();++ let fixed_plan: Vec<String> = sqlx::query(sqlx::AssertSqlSafe(format!(+ "EXPLAIN QUERY PLAN {}",+ elements::cleanup_sql()+ )))+ .fetch_all(&db.pool)+ .await+ .unwrap()+ .into_iter()+ .map(|row| row.try_get(3).unwrap())+ .collect();+ assert!(fixed_plan.iter().all(|line| {+ !line.contains("SCAN _bulk_element_rows") || line.contains(elements::TOMBSTONE_INDEX)+ }));+ assert!(fixed_plan+ .iter()+ .filter(|line| line.contains("_bulk_element_rows"))+ .all(|line| line.contains(elements::TOMBSTONE_INDEX)));++ let storage = db.storage.as_ref().unwrap().clone();+ assert!(storage.descriptor.budget.record_bytes < 2_097_152);+ let index_path = storage.root.join(&storage.descriptor.index);+ let pool = pool_options(Some(storage.clone()), false)+ .max_connections(1)+ .connect_with(sqlx::sqlite::SqliteConnectOptions::new().filename(&index_path))+ .await+ .unwrap();+ let writer = screenpipe_sqlite_coordinator::SqliteWritePool::new(+ pool.clone(),+ screenpipe_sqlite_coordinator::sqlite_write_lock(&index_path),+ );+ let steps = Box::new(AtomicU64::new(0));+ {+ let mut conn = pool.acquire().await.unwrap();+ let mut handle = conn.lock_handle().await.unwrap();+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ handle.as_raw_handle().as_ptr(),+ 1000,+ Some(count_steps),+ (&*steps as *const AtomicU64).cast_mut().cast(),+ );+ }+ }+ let fixed_backlog_started = std::time::Instant::now();+ let (cleanup, captured) = tokio::join!(+ elements::reclaim(&storage, &pool, &writer),+ db.insert_snapshot_frame(+ "display",+ chrono::Utc::now(),+ "",+ Some("test"),+ None,+ None,+ true,+ Some("test"),+ Some("durable capture"),+ Some("accessibility"),+ None,+ None,+ None,+ )+ );+ cleanup.unwrap();+ let fixed_backlog_elapsed = fixed_backlog_started.elapsed();+ let captured = captured.unwrap();+ let fixed_steps = steps.load(Ordering::Relaxed);+ assert_eq!(sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows WHERE _archive_deleted=1 AND _archive_file IS NULL").fetch_one(&pool).await.unwrap(), 1);+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM _bulk_element_rows WHERE id=1000001 AND _archive_file=1"+ )+ .fetch_one(&pool)+ .await+ .unwrap(),+ 1+ );+ // The archive row is only a predicate fixture. Remove its fake catalog+ // metadata before startup validation, while leaving the resident backlog.+ db.execute_raw_sql_write(+ "DELETE FROM _bulk_element_rows WHERE id=1000001; DELETE FROM _bulk_files WHERE id=1",+ )+ .await+ .unwrap();+ pool.close().await;+ drop(writer);+ drop(storage);+ db.close().await;++ let reopened = Arc::new(+ DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap(),+ );+ let payload = reopened+ .frame_payloads(&[captured], crate::storage::Projection::All)+ .await+ .unwrap()+ .remove(&captured)+ .unwrap();+ assert_eq!(+ payload.accessibility_text.as_deref(),+ Some("durable capture")+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames WHERE id=?")+ .bind(captured)+ .fetch_one(&reopened.pool)+ .await+ .unwrap(),+ 1+ );+ let storage = reopened.storage.as_ref().unwrap().clone();+ let index_path = storage.root.join(&storage.descriptor.index);+ let pool = pool_options(Some(storage.clone()), false)+ .max_connections(1)+ .connect_with(sqlx::sqlite::SqliteConnectOptions::new().filename(&index_path))+ .await+ .unwrap();+ let writer = screenpipe_sqlite_coordinator::SqliteWritePool::new(+ pool.clone(),+ screenpipe_sqlite_coordinator::sqlite_write_lock(&index_path),+ );+ elements::reclaim(&storage, &pool, &writer).await.unwrap();+ assert_eq!(sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows WHERE _archive_deleted=1 AND _archive_file IS NULL").fetch_one(&pool).await.unwrap(), 0);+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM _bulk_element_rows WHERE _archive_deleted=0"+ )+ .fetch_one(&pool)+ .await+ .unwrap(),+ 1_000_000+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT length(text) FROM _bulk_element_rows WHERE id=1000000"+ )+ .fetch_one(&pool)+ .await+ .unwrap(),+ 2_097_152+ );+ let fixed_zero_steps = Box::new(AtomicU64::new(0));+ {+ let mut conn = pool.acquire().await.unwrap();+ let mut handle = conn.lock_handle().await.unwrap();+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ handle.as_raw_handle().as_ptr(),+ 1000,+ Some(count_steps),+ (&*fixed_zero_steps as *const AtomicU64).cast_mut().cast(),+ );+ }+ }+ let fixed_zero_started = std::time::Instant::now();+ elements::reclaim(&storage, &pool, &writer).await.unwrap();+ let fixed_zero_elapsed = fixed_zero_started.elapsed();+ let fixed_zero_steps = fixed_zero_steps.load(Ordering::Relaxed);+ assert!(fixed_zero_steps < baseline_steps / 10);+ eprintln!("resident cleanup: baseline_plan={baseline_plan:?}; baseline_steps~={baseline_steps}; baseline_elapsed={baseline_elapsed:?}; missing_stats_plan={missing_stats_plan:?}; fixed_plan={fixed_plan:?}; fixed_backlog_steps~={fixed_steps}; fixed_backlog_elapsed={fixed_backlog_elapsed:?}; fixed_zero_steps~={fixed_zero_steps}; fixed_zero_elapsed={fixed_zero_elapsed:?}");+ pool.close().await;+ reopened.close().await;+}+#[tokio::test]async fn offline_selection_seeks_past_retained_history() {use std::sync::atomic::AtomicU64;@@ -635,3 +934,68 @@ async fn shutdown_releases_sql_workers_waiting_for_decoder_admission() {.unwrap();assert!(!storage.sql_readers_active());}++#[tokio::test]+async fn resident_tombstone_cleanup_is_indexed_batched_and_upgraded_on_reopen_allows_pinned_readers(+) {+ // Ordinary startup may share a WAL with an existing read snapshot. Index+ // installation is safe there; an offline FULL checkpoint is not a startup+ // prerequisite, including when the index is already installed.+ for already_installed in [false, true] {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ if !already_installed {+ db.execute_raw_sql_write(&format!("DROP INDEX {}", elements::TOMBSTONE_INDEX))+ .await+ .unwrap();+ }+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-16','first durable frame')")+ .await.unwrap();+ let mut reader = db.pool.begin().await.unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames")+ .fetch_one(&mut *reader)+ .await+ .unwrap(),+ 1+ );+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-10-16','second durable frame')")+ .await.unwrap();+ let result = {+ let writer = db.coordinated_writer();+ let permit = writer.lock().await.unwrap();+ let mut conn = permit.pool().acquire().await.unwrap();+ let original_timeout: i64 = sqlx::query_scalar("PRAGMA busy_timeout")+ .fetch_one(&mut *conn)+ .await+ .unwrap();+ // A zero test-only busy timeout observes the pinned-reader result+ // immediately; the fixture retains the reader until after the call.+ sqlx::query("PRAGMA busy_timeout=0")+ .execute(&mut *conn)+ .await+ .unwrap();+ let result = elements::ensure_cleanup_index(&mut conn).await;+ sqlx::query(sqlx::AssertSqlSafe(format!(+ "PRAGMA busy_timeout={original_timeout}"+ )))+ .execute(&mut *conn)+ .await+ .unwrap();+ result+ };+ reader.rollback().await.unwrap();+ assert!(result.is_ok(),+ "normal startup index installation must not require an offline FULL checkpoint while readers pin the WAL; already_installed={already_installed}, result={result:?}; retain construction checkpoints only in offline/bootstrap callers");+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 2+ );+ db.close().await;+ }+}diff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rsindex 79fa11f35..e1249c28e 100644--- a/crates/screenpipe-db/src/storage/schema.rs+++ b/crates/screenpipe-db/src/storage/schema.rs@@ -425,6 +425,17 @@ pub(crate) async fn upgrade_recording(Ok(true)}+pub(crate) async fn ensure_cleanup_indexes(+ conn: &mut SqliteConnection,+ has_bulk: bool,+) -> Result<bool, sqlx::Error> {+ if has_bulk {+ super::bulk::ensure_cleanup_indexes(conn).await+ } else {+ Ok(false)+ }+}+/// Give tables added by ordinary SQLx migrations the same read tracking as/// tables present during conversion. Inspect schema metadata only; existing/// capture/archive tables keep their specialized triggers.