YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: stop migration from rescanning retained history (#7129)
Ezra Ellette97b9fa3f720f
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex c94439f1a..184a700fd 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -742,6 +742,37 @@ mod tests {}}+ #[tokio::test]+ async fn element_selection_failure_reaches_support_without_rolling_logs() {+ use screenpipe_db::{storage, DatabaseManager};+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ // A broken resident source makes the real element selector fail. The+ // support report must retain that cause even after the pool is closed.+ db.execute_raw_sql_write("DROP TABLE _bulk_element_rows")+ .await+ .unwrap();+ let error =+ storage::diagnostics::observe(root.path(), "conversion", |_, _| {}, db.seal_payloads())+ .await+ .unwrap_err();+ assert!(error.to_string().contains("no such table"));+ db.close().await;+ assert_migration_failure_uploaded(+ root.path(),+ &[+ "selecting_staged_elements",+ "no such table",+ "_bulk_element_rows",+ "\"status\": \"failed\"",+ "\"table\": \"elements\"",+ ],+ )+ .await;+ }+#[tokio::test]async fn migration_conflict_reaches_support_after_log_rotation_and_redaction() {use screenpipe_db::{storage, DatabaseManager};@@ -784,9 +815,22 @@ mod tests {assert!(error.to_string().contains("source contains recorded history"));+ assert_migration_failure_uploaded(+ root.path(),+ &[+ "source contains recorded history",+ "validating_migration_source",+ "\"source_exists\": true",+ "\"index_exists\": true",+ ],+ )+ .await;+ }++ async fn assert_migration_failure_uploaded(root: &std::path::Path, expected: &[&str]) {timeout(Duration::from_secs(3), async {loop {- if storage::diagnostics::recent(root.path())+ if screenpipe_db::storage::diagnostics::recent(root).unwrap_or_default().iter().any(|snapshot| snapshot.status == "failed")@@ -799,16 +843,18 @@ mod tests {.await.unwrap();// A later support submission has no original rolling log in memory or on disk.- let diagnostics = collect_migration_diagnostics_from_root(root.path().to_owned()).await;+ let diagnostics = collect_migration_diagnostics_from_root(root.to_owned()).await;let raw =format!("[no log files found]\n\n=== Storage Migration Diagnostics ===\n{diagnostics}");let redacted = crate::feedback_redact::redact_pii_for_feedback(raw, "{}".into()).await.unwrap();- assert!(redacted.contains("source contains recorded history"));- assert!(redacted.contains("validating_migration_source"));- assert!(redacted.contains("\"source_exists\": true"));- assert!(redacted.contains("\"index_exists\": true"));+ for expected in expected {+ assert!(+ redacted.contains(expected),+ "missing {expected}: {redacted}"+ );+ }assert!(!redacted.contains("private history"));let server = MockServer::start().await;diff --git a/crates/screenpipe-db/src/storage/bulk/elements.rs b/crates/screenpipe-db/src/storage/bulk/elements.rsindex 1aba3a96a..e5f8d4790 100644--- a/crates/screenpipe-db/src/storage/bulk/elements.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements.rs@@ -10,8 +10,8 @@ mod vtab;use super::{integer, real, text, Column, Kind, Table};use sqlx::{Row, SqliteConnection};-pub(crate) use lifecycle::seal;pub(super) use lifecycle::{export, reclaim, verify};+pub(crate) use lifecycle::{seal, seal_after};pub(crate) use vtab::register;pub(super) const TABLE: Table = Table {diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rsindex 2b5bf79c2..4d73caca9 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs@@ -50,12 +50,28 @@ pub(crate) async fn seal(pool: &SqlitePool,writer: &SqliteWritePool,) -> Result<usize, sqlx::Error> {- crate::storage::diagnostics::batch("elements", None, None, None, None);- crate::storage::diagnostics::stage("selecting_staged_elements");+ Ok(seal_after(storage, pool, writer, None).await?.0)+}++// Only the exclusive offline owner carries a cursor across calls. Live+// recording can edit older rows and therefore always starts a fresh search.+pub(crate) async fn seal_after(+ storage: &Arc<HybridStorage>,+ pool: &SqlitePool,+ writer: &SqliteWritePool,+ mut after: Option<i64>,+) -> Result<(usize, Option<i64>), sqlx::Error> {// A dirty archive must be rewritten as a whole. If privacy processing// blocks that range, continue after it instead of starving newer work.- let mut after = None;loop {+ crate::storage::diagnostics::batch(+ "elements",+ after.and_then(|id| id.checked_add(1)),+ None,+ None,+ None,+ );+ crate::storage::diagnostics::stage("selecting_staged_elements");let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!("SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND {lower} AND ({}) ORDER BY id LIMIT 1",@@ -63,7 +79,7 @@ pub(crate) async fn seal())).fetch_optional(pool).await?;- let Some(id) = id else { return Ok(0) };+ let Some(id) = id else { return Ok((0, after)) };let range: Option<(i64, i64, i64)> = sqlx::query_as("SELECT first_id,last_id,file_id FROM _bulk_element_ranges WHERE first_id=(SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?1) AND last_id>=?1").bind(id).fetch_optional(pool).await?;if let Some((first, last, _)) = range {@@ -195,7 +211,7 @@ async fn rewrite(writer: &SqliteWritePool,id: i64,range: Option<(i64, i64, i64)>,-) -> Result<usize, sqlx::Error> {+) -> Result<(usize, Option<i64>), sqlx::Error> {let _token = storage.read_token(pool).await?;let version: i64 = sqlx::query_scalar("SELECT version FROM _bulk_element_state").fetch_one(pool)@@ -206,7 +222,7 @@ async fn rewrite(.await?;let (first, last) = if let Some((first, last, _)) = range {if range_blocked(pool, first, last).await? {- return Ok(0);+ return Ok((0, None));}(first, last)} else {@@ -234,8 +250,21 @@ async fn rewrite(None,);crate::storage::diagnostics::stage("selecting_element_batch");+ // A new range contains only resident rows. Reading its logical virtual+ // table would materialize every private/oversized row while looking+ // for enough eligible records, including a huge recovered backlog.+ let source = if range.is_some() {+ "elements"+ } else {+ "_bulk_element_rows"+ };+ let visible = if range.is_some() {+ "1"+ } else {+ "_archive_deleted=0"+ };let candidates: Vec<(i64, i64)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(- "SELECT id,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id LIMIT {FILE_ROWS}",+ "SELECT id,{} FROM {source} WHERE id BETWEEN ? AND ? AND {lower} AND {visible} AND ({}) ORDER BY id LIMIT {FILE_ROWS}",TABLE.all_bytes(""), TABLE.eligible))).bind(first).bind(last).fetch_all(pool).await?;@@ -263,7 +292,7 @@ async fn rewrite();crate::storage::diagnostics::stage("reading_elements_to_seal");let rows = sqlx::query(sqlx::AssertSqlSafe(format!(- "SELECT id,_archive_generation,{} FROM elements WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id", names()+ "SELECT id,_archive_generation,{} FROM {source} WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id", names()))).bind(serde_json::to_string(&ids).map_err(storage_error)?).fetch_all(pool).await?@@ -280,7 +309,7 @@ async fn rewrite(} else if let Some(file) = files.last() {file.last} else {- return Ok(0);+ return Ok((0, None));};crate::storage::diagnostics::stage("publishing_element_archive");#[cfg(test)]@@ -314,7 +343,7 @@ async fn rewrite(.await?;}tx.commit().await?;- return Ok(0);+ return Ok((0, None));}let selection = if range.is_some() {"1".into()@@ -367,7 +396,7 @@ async fn rewrite(crate::storage::faults::checkpoint("bulk_committed");let count = files.iter().map(|f| f.rows).sum();tracing::info!(rows = count, files = files.len(), "sealed element records");- Ok(count)+ Ok((count, Some(last)))}async fn resident_range_unchanged(diff --git a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/lifecycle.rsindex 5f93f35c7..2527e6f9f 100644--- a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs@@ -20,7 +20,7 @@ impl HybridStorage {if table.name == "elements" {return super::elements::seal(self, pool, writer).await;}- let rows = self.select_bulk(pool, table, None).await?;+ let rows = self.select_bulk(pool, table, None, None).await?;if !rows.is_empty() {return self.publish_bulk(pool, writer, table, rows, None).await;}@@ -32,15 +32,25 @@ impl HybridStorage {pool: &SqlitePool,table: &Table,file: Option<i64>,+ after: Option<i64>,) -> Result<Vec<Record>, sqlx::Error> {- crate::storage::diagnostics::batch(table.name, None, None, None, None);+ crate::storage::diagnostics::batch(+ table.name,+ after.and_then(|id| id.checked_add(1)),+ None,+ None,+ None,+ );crate::storage::diagnostics::stage("selecting_bulk_records");let _token = self.read_token(pool).await?;- let condition = if let Some(file) = file {+ let mut condition = if let Some(file) = file {format!("pending._archive_file={file}")} else {"pending._archive_mask!=0".into()};+ if let Some(after) = after {+ condition.push_str(&format!(" AND pending.id>{after}"));+ }// Keep the pending index as the outer loop so LIMIT bounds work before// materializing IDs from a large history.let candidates=sqlx::query(sqlx::AssertSqlSafe(format!("SELECT v.id,({bytes}) AS bytes FROM main.{t} pending CROSS JOIN (SELECT * FROM {view} WHERE {eligible}) v ON v.id=pending.id WHERE {condition} AND ({bytes})<=? ORDER BY pending.id LIMIT {FILE_ROWS}",bytes=table.all_bytes("v."),view=table.view(),t=table.name,eligible=table.eligible))).bind(self.descriptor.budget.record_bytes as i64).fetch_all(pool).await?;@@ -275,7 +285,7 @@ impl HybridStorage {.iter().find(|t| t.name == name).ok_or_else(|| storage_error("unknown bulk table"))?;- let rows = self.select_bulk(pool, table, Some(id)).await?;+ let rows = self.select_bulk(pool, table, Some(id), None).await?;if !rows.is_empty() {self.publish_bulk(pool, writer, table, rows, Some(id)).await?;diff --git a/crates/screenpipe-db/src/storage/bulk/tests.rs b/crates/screenpipe-db/src/storage/bulk/tests.rsindex addf82e54..9c1d6db3b 100644--- a/crates/screenpipe-db/src/storage/bulk/tests.rs+++ b/crates/screenpipe-db/src/storage/bulk/tests.rs@@ -32,6 +32,113 @@ impl Drop for Pause {}}+#[tokio::test]+async fn offline_selection_seeks_past_retained_history() {+ use std::sync::atomic::AtomicU64;+ unsafe extern "C" fn count_steps(context: *mut std::ffi::c_void) -> i32 {+ let steps = unsafe { &*context.cast::<AtomicU64>() };+ // Abort a regressed scan instead of letting large fixtures hang CI.+ i32::from(steps.fetch_add(1000, Ordering::Relaxed) > 100_000)+ }++ let root = tempfile::tempdir().unwrap();+ let mut options = crate::storage::MigrationOptions::default();+ options.privacy.identity = "migration-cursor-test".into();+ options.privacy.required_surfaces = 1;+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)+ .await+ .unwrap();+ db.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-19','private frame')",+ )+ .await+ .unwrap();+ let count = 200_000_i64;+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ // Keep a large privacy-pending prefix. This models an interrupted migration+ // after recording recovery has restored all elements to the resident table.+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,_archive_generation) SELECT id,1,'accessibility','AXText','private',1 FROM n")+ .bind(count).execute(&mut **tx.conn()).await.unwrap();+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO ui_events(id,timestamp,event_type,text_content) SELECT id,'2026-09-19','text','private' FROM n")+ .bind(count).execute(&mut **tx.conn()).await.unwrap();+ // In-place migration deliberately postpones these indexes until completion.+ sqlx::query("DROP INDEX _bulk_ui_events_pending")+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ tx.commit().await.unwrap();+ db.execute_raw_sql_write("INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(200001,1,'accessibility','AXText','eligible',1); INSERT INTO ui_events(id,timestamp,event_type,text_content,redacted_at) VALUES(200001,'2026-09-19','text','eligible',1)").await.unwrap();++ let storage = db.storage.as_ref().unwrap();+ let index = storage.root.join(&storage.descriptor.index);+ let pool = pool_options(Some(storage.clone()), false)+ .max_connections(1)+ .connect_with(sqlx::sqlite::SqliteConnectOptions::new().filename(&index))+ .await+ .unwrap();+ let writer = screenpipe_sqlite_coordinator::SqliteWritePool::new(+ pool.clone(),+ screenpipe_sqlite_coordinator::sqlite_write_lock(&index),+ );+ let steps = Box::new(AtomicU64::new(0));+ {+ let mut conn = pool.acquire().await.unwrap();+ let mut handle = conn.lock_handle().await.unwrap();+ // Context remains alive until the callback is removed below.+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ handle.as_raw_handle().as_ptr(),+ 1000,+ Some(count_steps),+ (&*steps as *const AtomicU64).cast_mut().cast(),+ );+ }+ }+ let bulk = storage+ .select_bulk(&pool, &TABLES[2], None, Some(count))+ .await;+ let elements = elements::seal_after(storage, &pool, &writer, Some(count)).await;+ let work = steps.load(Ordering::Relaxed);+ {+ let mut conn = pool.acquire().await.unwrap();+ let mut handle = conn.lock_handle().await.unwrap();+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ handle.as_raw_handle().as_ptr(),+ 0,+ None,+ std::ptr::null_mut(),+ );+ }+ }+ pool.close().await;+ let bulk = bulk+ .unwrap_or_else(|error| panic!("bulk scan exceeded work budget at {work} steps: {error}"));+ assert_eq!(+ bulk.iter().map(|r| r.id).collect::<Vec<_>>(),+ vec![count + 1]+ );+ assert_eq!(+ elements.unwrap_or_else(|error| panic!(+ "element scan exceeded work budget at {work} steps: {error}"+ )),+ (1, Some(count + 1))+ );+ assert!(+ work < 100_000,+ "selection revisited retained history: {work}"+ );+ eprintln!("offline cursor: retained_rows_per_table={count}, sqlite_steps={work}");+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ count+ );+ db.close().await;+}+#[tokio::test]async fn element_publication_ignores_new_captures_but_rejects_changed_range_members() {for archived in [false, true] {diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rsindex d67cc69e3..bfaee6df6 100644--- a/crates/screenpipe-db/src/storage/in_place.rs+++ b/crates/screenpipe-db/src/storage/in_place.rs@@ -175,7 +175,10 @@ async fn run(if !archive { progress(MigrationProgress::phase("restoring saved screen records")); }loop {if archive { reserve(&storage)?; }- if archive && storage.seal_once(&pool, &writer).await? != 0 {+ // The offline owner visits each table after building its search+ // entries. The live round-robin sealer would repeatedly scan other+ // unfinished tables and archive them before that backfill.+ if archive && storage.seal_frames(&pool, &writer).await? != 0 {let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;continue;@@ -201,16 +204,25 @@ async fn run(super::faults::checkpoint("migration_batch_staged");}let source_exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='_bulk_elements_source')").fetch_one(&pool).await?;- if source_exists {+ {if !archive { progress(MigrationProgress::phase("restoring saved accessibility records")); }- let columns = super::import::columns(&pool, "_bulk_elements_source").await?;+ let columns = if source_exists { super::import::columns(&pool, "_bulk_elements_source").await? } else { Vec::new() };+ let mut after = None;loop {if archive { reserve(&storage)?; }- if archive && bulk::elements::seal(&storage, &pool, &writer).await? != 0 {- let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;- report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;- continue;+ if archive {+ let (sealed, last) = bulk::elements::seal_after(&storage, &pool, &writer, after).await?;+ if sealed != 0 {+ after = last;+ let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;+ report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;+ continue;+ }}+ if !source_exists { break; }+ // Everything currently staged was visited. Retained private or+ // oversized rows must not be rescanned for every source batch.+ after = sqlx::query_scalar("SELECT max(id) FROM _bulk_element_rows").fetch_one(&pool).await?;// Move existing records inside SQLite, including records larger// than any encoder/decoder batch. Sealing can leave them resident.let range = resident_range(&pool, "_bulk_elements_source", None,@@ -237,31 +249,38 @@ async fn run(drop(permit);if archive { reclaim(&storage, &writer, file, &mut reclamation, false).await?; }}- let permit = writer.lock().await?;- super::diagnostics::stage("dropping_empty_element_source");- sqlx::query("DROP TABLE _bulk_elements_source").execute(permit.pool()).await?;+ if source_exists {+ let permit = writer.lock().await?;+ super::diagnostics::stage("dropping_empty_element_source");+ sqlx::query("DROP TABLE _bulk_elements_source").execute(permit.pool()).await?;+ }}if !archive { progress(MigrationProgress::phase("restoring history search indexes")); }for table in bulk::TABLES.iter().filter(|t| t.name != "elements") {+ let mut after = None;while archive {reserve(&storage)?;- let rows = storage.select_bulk(&pool, table, None).await?;+ let rows = storage.select_bulk(&pool, table, None, after).await?;if rows.is_empty() { break; }if !table.fts.is_empty() {let ids = serde_json::to_string(&rows.iter().map(|r| r.id).collect::<Vec<_>>()).map_err(storage_error)?;let permit = writer.lock().await?;- let mut conn = permit.pool().acquire().await?;- let mut tx = conn.begin().await?;- for sql in [format!("DELETE FROM {t}_fts WHERE rowid IN (SELECT value FROM json_each(?))",t=table.name), format!("INSERT INTO {t}_fts(rowid,{columns}) SELECT id,{columns} FROM {view} WHERE id IN (SELECT value FROM json_each(?)) AND ({condition})",t=table.name,columns=table.fts_columns(),view=table.view(),condition=table.fts_condition)] {- sqlx::query(sqlx::AssertSqlSafe(sql)).bind(&ids).execute(&mut *tx).await?;- }+ let mut tx = permit.pool().begin().await?;+ index_bulk_fts(&mut tx, table, &ids).await?;tx.commit().await?;}- storage.publish_bulk(&pool, &writer, table, rows, None).await?;+ let last = rows.last().unwrap().id;+ let expected = rows.len();+ if storage.publish_bulk(&pool, &writer, table, rows, None).await? != expected {+ return Err(storage_error("offline bulk batch changed before publication"));+ }+ // Exclusive ownership makes this cursor safe. A retry starts+ // from committed resident state, never from an uncommitted ID.+ after = Some(last);let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;}- backfill_resident_fts(&storage, &pool, &writer, table, file, &mut reclamation, archive).await?;+ backfill_bulk_fts(&storage, &pool, &writer, table, file, &mut reclamation, archive).await?;}if !archive { progress(MigrationProgress::phase("finishing recording recovery")); }{@@ -364,9 +383,22 @@ async fn reclaim(Ok(reclaimed)}-// Privacy-ineligible records remain resident. Their search entries still need-// the same bounded, committed backfill as records that were sealed above.-async fn backfill_resident_fts(+async fn index_bulk_fts(+ conn: &mut sqlx::SqliteConnection,+ table: &bulk::Table,+ ids: &str,+) -> Result<(), sqlx::Error> {+ super::diagnostics::stage("indexing_bulk_search_batch");+ for sql in [format!("DELETE FROM {t}_fts WHERE rowid IN (SELECT value FROM json_each(?))",t=table.name),format!("INSERT INTO {t}_fts(rowid,{columns}) SELECT id,{columns} FROM {view} WHERE id IN (SELECT value FROM json_each(?)) AND ({condition})",t=table.name,view=table.view(),columns=table.fts_columns(),condition=table.fts_condition)] {+ sqlx::query(sqlx::AssertSqlSafe(sql)).bind(ids).execute(&mut *conn).await?;+ }+ Ok(())+}++// Fill missing search entries for resident/private records and archives+// committed by older migration attempts that used the live sealer too early.+// Normal conversion still indexes, seals and reclaims one batch at a time.+async fn backfill_bulk_fts(storage: &HybridStorage,pool: &SqlitePool,writer: &SqliteWritePool,@@ -380,7 +412,7 @@ async fn backfill_resident_fts(return Ok(());}let step = format!(- "{}-fts-{}",+ "{}-all-fts-{}",if archive { "resident" } else { "recovery" },table.name);@@ -389,7 +421,7 @@ async fn backfill_resident_fts(.iter().map(|c| {format!(- "COALESCE(length(CAST({} AS BLOB)),0)",+ "COALESCE(length(CAST(v.{} AS BLOB)),0)",c.split_whitespace().next().unwrap())})@@ -405,7 +437,21 @@ async fn backfill_resident_fts(.fetch_optional(pool).await?.flatten();- let sql = format!("SELECT id,{size} FROM {t} WHERE _archive_file IS NULL AND id {comparison} ? ORDER BY id LIMIT {rows}",t=table.name,comparison=if after.is_some(){">"}else{">="},rows=bulk::FILE_ROWS);+ super::diagnostics::batch(+ table.name,+ after.and_then(|id| id.checked_add(1)),+ None,+ None,+ None,+ );+ super::diagnostics::stage("selecting_bulk_search_batch");+ let sql = format!(+ "SELECT v.id,{size} FROM {view} v WHERE v.id {comparison} ? AND NOT EXISTS(SELECT 1 FROM {t}_fts WHERE rowid=v.id) ORDER BY v.id LIMIT {rows}",+ view = table.view(),+ t = table.name,+ comparison = if after.is_some() { ">" } else { ">=" },+ rows = bulk::FILE_ROWS+ );let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(after.unwrap_or(i64::MIN)).fetch(pool);@@ -427,9 +473,7 @@ async fn backfill_resident_fts(let permit = writer.lock().await?;let mut conn = permit.pool().acquire().await?;let mut tx = conn.begin().await?;- for sql in [format!("DELETE FROM {t}_fts WHERE rowid IN (SELECT value FROM json_each(?))",t=table.name),format!("INSERT INTO {t}_fts(rowid,{columns}) SELECT id,{columns} FROM {t} WHERE id IN (SELECT value FROM json_each(?)) AND ({condition})",t=table.name,columns=table.fts_columns(),condition=table.fts_condition)] {- sqlx::query(sqlx::AssertSqlSafe(sql)).bind(&ids).execute(&mut *tx).await?;- }+ index_bulk_fts(&mut tx, table, &ids).await?;sqlx::query("INSERT INTO _storage_conversion_steps(step,last_id) VALUES(?,?) ON CONFLICT(step) DO UPDATE SET last_id=excluded.last_id").bind(&step).bind(last).execute(&mut *tx).await?;tx.commit().await?;if archive {diff --git a/crates/screenpipe-db/src/storage/sealing.rs b/crates/screenpipe-db/src/storage/sealing.rsindex 45e85965e..625a6b45d 100644--- a/crates/screenpipe-db/src/storage/sealing.rs+++ b/crates/screenpipe-db/src/storage/sealing.rs@@ -294,7 +294,7 @@ impl HybridStorage {Ok(0)}- async fn seal_frames(+ pub(super) async fn seal_frames(self: &Arc<Self>,pool: &SqlitePool,writer: &SqliteWritePool,diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rsindex 09ef15df5..579240b87 100644--- a/crates/screenpipe-db/tests/in_place_migration.rs+++ b/crates/screenpipe-db/tests/in_place_migration.rs@@ -224,6 +224,73 @@ async fn migration_diagnostics_preserve_table_totals_and_verified_completion() {assert!(!json.contains("capture detail"));}+#[tokio::test]+async fn migration_preserves_search_when_frames_and_bulk_history_are_both_present() {+ assert_migration_search(false).await;+}++#[tokio::test]+#[cfg(feature = "storage-fault-injection")]+async fn retry_repairs_search_for_already_archived_bulk_history() {+ assert_migration_search(true).await;+}++async fn assert_migration_search(resume: bool) {+ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("db.sqlite");+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-19','frame history'),(2,'2026-09-19','other frame'); INSERT INTO audio_chunks(id,file_path) VALUES(1,'test.wav'); INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,device) VALUES(1,1,0,'2026-09-19','migrationneedle spoken history','test'); INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(1,'2026-09-19','text','migrationneedle typed history'); INSERT INTO semantic_items(id,entity_fingerprint,version_fingerprint,kind,item_key,identity_quality,title,body,metadata_json) VALUES(1,randomblob(32),randomblob(32),'document','migration','stable','title','migrationneedle document','{}');").await.unwrap();+ db.close().await;+ if resume {+ let crashed = std::process::Command::new(env!("CARGO_BIN_EXE_screenpipe-storage"))+ .arg("migrate")+ .arg(root.path())+ .env("SCREENPIPE_STORAGE_CRASH_AT", "bulk_committed")+ .output()+ .unwrap();+ assert_eq!(crashed.status.code(), Some(86));+ screenpipe_db::storage::recover_interrupted_migration(root.path(), Default::default())+ .await+ .unwrap();+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM main.audio_transcriptions WHERE _archive_file IS NOT NULL"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 1+ );+ // Older attempts could publish this archive before populating FTS.+ db.execute_raw_sql_write("DELETE FROM audio_transcriptions_fts; DELETE FROM _storage_conversion_steps WHERE step LIKE '%-all-fts-%'")+ .await.unwrap();+ db.close().await;+ }+ let report = migrate(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ assert_eq!(report.frames, 2);+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ for table in ["audio_transcriptions", "ui_events", "semantic_items"] {+ let found: i64 = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(+ "SELECT count(*) FROM {table}_fts WHERE {table}_fts MATCH 'migrationneedle'"+ )))+ .fetch_one(&db.pool)+ .await+ .unwrap();+ assert_eq!(found, 1, "migrated {table} must remain searchable");+ }+ db.verify_storage().await.unwrap();+ db.close().await;+}+#[tokio::test]#[ignore = "migration throughput benchmark; generates an isolated recording history"]async fn migration_throughput() {@@ -1256,6 +1323,12 @@ async fn recording_recovery_keeps_committed_payloads_and_survives_restarts() {).await.unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows")+ .fetch_one(&db.pool).await.unwrap(),+ 0,+ "explicit retry must archive restored elements even after recovery removed the source table"+ );assert_eq!(db.frame_payloads(&[49, 50], Projection::Search).awaitdiff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex 15e0aba60..82ab08822 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 34- Mapped Rust files: 362-- Active test blocks: 3549+- Active test blocks: 3552- Ignored/manual test blocks: 149-- Weighted coverage points: 2933.3+- Weighted coverage points: 2936.3| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 31 | 3406 | 143 | 2868.2 | 21 | 11 | 100% |-| macos | 31 | 3468 | 123 | 2882.2 | 22 | 11 | 100% |-| linux | 27 | 3028 | 115 | 2519.5 | 20 | 11 | 100% |+| windows | 31 | 3409 | 143 | 2871.2 | 21 | 11 | 100% |+| macos | 31 | 3471 | 123 | 2885.2 | 22 | 11 | 100% |+| linux | 27 | 3031 | 115 | 2522.5 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 8d85414e9..5aabd841d 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 34- Mapped Rust files: 362-- Active test blocks: 3549+- Active test blocks: 3552- Ignored/manual test blocks: 149-- Declared test blocks: 3698-- Weighted coverage points: 2933.3+- Declared test blocks: 3701+- Weighted coverage points: 2936.3Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 31 | 3406 | 143 | 2868.2 | 21 | 11 | 100% |-| macos | 31 | 3468 | 123 | 2882.2 | 22 | 11 | 100% |-| linux | 27 | 3028 | 115 | 2519.5 | 20 | 11 | 100% |+| windows | 31 | 3409 | 143 | 2871.2 | 21 | 11 | 100% |+| macos | 31 | 3471 | 123 | 2885.2 | 22 | 11 | 100% |+| linux | 27 | 3031 | 115 | 2522.5 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 11 | 20 | 120 | 1732 | 43 | 1335.9 | 10 |-| screenpipe-db | 5 | 60 | 23 | 555 | 24 | 528.3 | 9 |+| screenpipe-db | 5 | 60 | 23 | 558 | 24 | 531.3 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 621 | 44 | 545.6 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |@@ -66,21 +66,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 8 suites / 737 active / 45 ignored / 661.6 pts | 8 suites / 737 active / 45 ignored / 661.6 pts | 7 suites / 662 active / 44 ignored / 609.1 pts || audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |-| database | 7 suites / 506 active / 20 ignored / 479.3 pts | 7 suites / 506 active / 20 ignored / 479.3 pts | 7 suites / 506 active / 20 ignored / 479.3 pts |+| database | 7 suites / 509 active / 20 ignored / 482.3 pts | 7 suites / 509 active / 20 ignored / 482.3 pts | 7 suites / 509 active / 20 ignored / 482.3 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts || engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts || local-api | 3 suites / 433 active / 10 ignored / 305.5 pts | 3 suites / 433 active / 10 ignored / 305.5 pts | 3 suites / 433 active / 10 ignored / 305.5 pts || meeting | 6 suites / 1636 active / 21 ignored / 1336.9 pts | 6 suites / 1636 active / 21 ignored / 1336.9 pts | 4 suites / 1291 active / 17 ignored / 1014.4 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1560 active / 75 ignored / 1379.6 pts | 14 suites / 1667 active / 79 ignored / 1422.4 pts | 13 suites / 1560 active / 75 ignored / 1379.6 pts |+| performance | 13 suites / 1563 active / 75 ignored / 1382.6 pts | 14 suites / 1670 active / 79 ignored / 1425.4 pts | 13 suites / 1563 active / 75 ignored / 1382.6 pts || pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts || privacy | 5 suites / 942 active / 37 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts || real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - || speaker | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts |-| storage | 4 suites / 621 active / 37 ignored / 514.8 pts | 4 suites / 621 active / 37 ignored / 514.8 pts | 4 suites / 621 active / 37 ignored / 514.8 pts |+| storage | 4 suites / 624 active / 37 ignored / 517.8 pts | 4 suites / 624 active / 37 ignored / 517.8 pts | 4 suites / 624 active / 37 ignored / 517.8 pts || sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |-| timeline | 4 suites / 1170 active / 42 ignored / 956.4 pts | 4 suites / 1170 active / 42 ignored / 956.4 pts | 4 suites / 1170 active / 42 ignored / 956.4 pts |+| timeline | 4 suites / 1173 active / 42 ignored / 959.4 pts | 4 suites / 1173 active / 42 ignored / 959.4 pts | 4 suites / 1173 active / 42 ignored / 959.4 pts || transcription | 5 suites / 814 active / 42 ignored / 637.8 pts | 5 suites / 814 active / 42 ignored / 637.8 pts | 5 suites / 814 active / 42 ignored / 637.8 pts || ui-events | 4 suites / 758 active / 29 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts || vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 33 | 267 | 11 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 33 | 270 | 11 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 411 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |@@ -278,7 +278,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-runtime-reliability | screenpipe-db | src/failpoint_vfs.rs | source | 4 | 0 | 4 || db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 || db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |-| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 7 | 0 | 7 |+| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 8 | 0 | 8 || db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 || db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 || db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |@@ -307,7 +307,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 || db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 || db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 16 | 5 | 21 |+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 18 | 5 | 23 || db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 || db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 || db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |