Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: stop migration from rescanning retained history (#7129)

Ezra Ellette97b9fa3f720f
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index c94439f1a..184a700fd 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -742,6 +742,37 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn element_selection_failure_reaches_support_without_rolling_logs() {
+ use screenpipe_db::{storage, DatabaseManager};
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ // A broken resident source makes the real element selector fail. The
+ // support report must retain that cause even after the pool is closed.
+ db.execute_raw_sql_write("DROP TABLE _bulk_element_rows")
+ .await
+ .unwrap();
+ let error =
+ storage::diagnostics::observe(root.path(), "conversion", |_, _| {}, db.seal_payloads())
+ .await
+ .unwrap_err();
+ assert!(error.to_string().contains("no such table"));
+ db.close().await;
+ assert_migration_failure_uploaded(
+ root.path(),
+ &[
+ "selecting_staged_elements",
+ "no such table",
+ "_bulk_element_rows",
+ "\"status\": \"failed\"",
+ "\"table\": \"elements\"",
+ ],
+ )
+ .await;
+ }
+
#[tokio::test]
async fn migration_conflict_reaches_support_after_log_rotation_and_redaction() {
use screenpipe_db::{storage, DatabaseManager};
@@ -784,9 +815,22 @@ mod tests {
assert!(error
.to_string()
.contains("source contains recorded history"));
+ assert_migration_failure_uploaded(
+ root.path(),
+ &[
+ "source contains recorded history",
+ "validating_migration_source",
+ "\"source_exists\": true",
+ "\"index_exists\": true",
+ ],
+ )
+ .await;
+ }
+
+ async fn assert_migration_failure_uploaded(root: &std::path::Path, expected: &[&str]) {
timeout(Duration::from_secs(3), async {
loop {
- if storage::diagnostics::recent(root.path())
+ if screenpipe_db::storage::diagnostics::recent(root)
.unwrap_or_default()
.iter()
.any(|snapshot| snapshot.status == "failed")
@@ -799,16 +843,18 @@ mod tests {
.await
.unwrap();
// A later support submission has no original rolling log in memory or on disk.
- let diagnostics = collect_migration_diagnostics_from_root(root.path().to_owned()).await;
+ let diagnostics = collect_migration_diagnostics_from_root(root.to_owned()).await;
let raw =
format!("[no log files found]\n\n=== Storage Migration Diagnostics ===\n{diagnostics}");
let redacted = crate::feedback_redact::redact_pii_for_feedback(raw, "{}".into())
.await
.unwrap();
- assert!(redacted.contains("source contains recorded history"));
- assert!(redacted.contains("validating_migration_source"));
- assert!(redacted.contains("\"source_exists\": true"));
- assert!(redacted.contains("\"index_exists\": true"));
+ for expected in expected {
+ assert!(
+ redacted.contains(expected),
+ "missing {expected}: {redacted}"
+ );
+ }
assert!(!redacted.contains("private history"));
let server = MockServer::start().await;
diff --git a/crates/screenpipe-db/src/storage/bulk/elements.rs b/crates/screenpipe-db/src/storage/bulk/elements.rs
index 1aba3a96a..e5f8d4790 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements.rs
@@ -10,8 +10,8 @@ mod vtab;
use super::{integer, real, text, Column, Kind, Table};
use sqlx::{Row, SqliteConnection};
-pub(crate) use lifecycle::seal;
pub(super) use lifecycle::{export, reclaim, verify};
+pub(crate) use lifecycle::{seal, seal_after};
pub(crate) use vtab::register;
pub(super) const TABLE: Table = Table {
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
index 2b5bf79c2..4d73caca9 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
@@ -50,12 +50,28 @@ pub(crate) async fn seal(
pool: &SqlitePool,
writer: &SqliteWritePool,
) -> Result<usize, sqlx::Error> {
- crate::storage::diagnostics::batch("elements", None, None, None, None);
- crate::storage::diagnostics::stage("selecting_staged_elements");
+ Ok(seal_after(storage, pool, writer, None).await?.0)
+}
+
+// Only the exclusive offline owner carries a cursor across calls. Live
+// recording can edit older rows and therefore always starts a fresh search.
+pub(crate) async fn seal_after(
+ storage: &Arc<HybridStorage>,
+ pool: &SqlitePool,
+ writer: &SqliteWritePool,
+ mut after: Option<i64>,
+) -> Result<(usize, Option<i64>), sqlx::Error> {
// A dirty archive must be rewritten as a whole. If privacy processing
// blocks that range, continue after it instead of starving newer work.
- let mut after = None;
loop {
+ crate::storage::diagnostics::batch(
+ "elements",
+ after.and_then(|id| id.checked_add(1)),
+ None,
+ None,
+ None,
+ );
+ crate::storage::diagnostics::stage("selecting_staged_elements");
let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));
let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
"SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND {lower} AND ({}) ORDER BY id LIMIT 1",
@@ -63,7 +79,7 @@ pub(crate) async fn seal(
)))
.fetch_optional(pool)
.await?;
- let Some(id) = id else { return Ok(0) };
+ let Some(id) = id else { return Ok((0, after)) };
let range: Option<(i64, i64, i64)> = sqlx::query_as("SELECT first_id,last_id,file_id FROM _bulk_element_ranges WHERE first_id=(SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?1) AND last_id>=?1")
.bind(id).fetch_optional(pool).await?;
if let Some((first, last, _)) = range {
@@ -195,7 +211,7 @@ async fn rewrite(
writer: &SqliteWritePool,
id: i64,
range: Option<(i64, i64, i64)>,
-) -> Result<usize, sqlx::Error> {
+) -> Result<(usize, Option<i64>), sqlx::Error> {
let _token = storage.read_token(pool).await?;
let version: i64 = sqlx::query_scalar("SELECT version FROM _bulk_element_state")
.fetch_one(pool)
@@ -206,7 +222,7 @@ async fn rewrite(
.await?;
let (first, last) = if let Some((first, last, _)) = range {
if range_blocked(pool, first, last).await? {
- return Ok(0);
+ return Ok((0, None));
}
(first, last)
} else {
@@ -234,8 +250,21 @@ async fn rewrite(
None,
);
crate::storage::diagnostics::stage("selecting_element_batch");
+ // A new range contains only resident rows. Reading its logical virtual
+ // table would materialize every private/oversized row while looking
+ // for enough eligible records, including a huge recovered backlog.
+ let source = if range.is_some() {
+ "elements"
+ } else {
+ "_bulk_element_rows"
+ };
+ let visible = if range.is_some() {
+ "1"
+ } else {
+ "_archive_deleted=0"
+ };
let candidates: Vec<(i64, i64)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(
- "SELECT id,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id LIMIT {FILE_ROWS}",
+ "SELECT id,{} FROM {source} WHERE id BETWEEN ? AND ? AND {lower} AND {visible} AND ({}) ORDER BY id LIMIT {FILE_ROWS}",
TABLE.all_bytes(""), TABLE.eligible
)))
.bind(first).bind(last).fetch_all(pool).await?;
@@ -263,7 +292,7 @@ async fn rewrite(
);
crate::storage::diagnostics::stage("reading_elements_to_seal");
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
- "SELECT id,_archive_generation,{} FROM elements WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id", names()
+ "SELECT id,_archive_generation,{} FROM {source} WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id", names()
)))
.bind(serde_json::to_string(&ids).map_err(storage_error)?)
.fetch_all(pool).await?
@@ -280,7 +309,7 @@ async fn rewrite(
} else if let Some(file) = files.last() {
file.last
} else {
- return Ok(0);
+ return Ok((0, None));
};
crate::storage::diagnostics::stage("publishing_element_archive");
#[cfg(test)]
@@ -314,7 +343,7 @@ async fn rewrite(
.await?;
}
tx.commit().await?;
- return Ok(0);
+ return Ok((0, None));
}
let selection = if range.is_some() {
"1".into()
@@ -367,7 +396,7 @@ async fn rewrite(
crate::storage::faults::checkpoint("bulk_committed");
let count = files.iter().map(|f| f.rows).sum();
tracing::info!(rows = count, files = files.len(), "sealed element records");
- Ok(count)
+ Ok((count, Some(last)))
}
async fn resident_range_unchanged(
diff --git a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
index 5f93f35c7..2527e6f9f 100644
--- a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
@@ -20,7 +20,7 @@ impl HybridStorage {
if table.name == "elements" {
return super::elements::seal(self, pool, writer).await;
}
- let rows = self.select_bulk(pool, table, None).await?;
+ let rows = self.select_bulk(pool, table, None, None).await?;
if !rows.is_empty() {
return self.publish_bulk(pool, writer, table, rows, None).await;
}
@@ -32,15 +32,25 @@ impl HybridStorage {
pool: &SqlitePool,
table: &Table,
file: Option<i64>,
+ after: Option<i64>,
) -> Result<Vec<Record>, sqlx::Error> {
- crate::storage::diagnostics::batch(table.name, None, None, None, None);
+ crate::storage::diagnostics::batch(
+ table.name,
+ after.and_then(|id| id.checked_add(1)),
+ None,
+ None,
+ None,
+ );
crate::storage::diagnostics::stage("selecting_bulk_records");
let _token = self.read_token(pool).await?;
- let condition = if let Some(file) = file {
+ let mut condition = if let Some(file) = file {
format!("pending._archive_file={file}")
} else {
"pending._archive_mask!=0".into()
};
+ if let Some(after) = after {
+ condition.push_str(&format!(" AND pending.id>{after}"));
+ }
// Keep the pending index as the outer loop so LIMIT bounds work before
// materializing IDs from a large history.
let candidates=sqlx::query(sqlx::AssertSqlSafe(format!("SELECT v.id,({bytes}) AS bytes FROM main.{t} pending CROSS JOIN (SELECT * FROM {view} WHERE {eligible}) v ON v.id=pending.id WHERE {condition} AND ({bytes})<=? ORDER BY pending.id LIMIT {FILE_ROWS}",bytes=table.all_bytes("v."),view=table.view(),t=table.name,eligible=table.eligible))).bind(self.descriptor.budget.record_bytes as i64).fetch_all(pool).await?;
@@ -275,7 +285,7 @@ impl HybridStorage {
.iter()
.find(|t| t.name == name)
.ok_or_else(|| storage_error("unknown bulk table"))?;
- let rows = self.select_bulk(pool, table, Some(id)).await?;
+ let rows = self.select_bulk(pool, table, Some(id), None).await?;
if !rows.is_empty() {
self.publish_bulk(pool, writer, table, rows, Some(id))
.await?;
diff --git a/crates/screenpipe-db/src/storage/bulk/tests.rs b/crates/screenpipe-db/src/storage/bulk/tests.rs
index addf82e54..9c1d6db3b 100644
--- a/crates/screenpipe-db/src/storage/bulk/tests.rs
+++ b/crates/screenpipe-db/src/storage/bulk/tests.rs
@@ -32,6 +32,113 @@ impl Drop for Pause {
}
}
+#[tokio::test]
+async fn offline_selection_seeks_past_retained_history() {
+ use std::sync::atomic::AtomicU64;
+ unsafe extern "C" fn count_steps(context: *mut std::ffi::c_void) -> i32 {
+ let steps = unsafe { &*context.cast::<AtomicU64>() };
+ // Abort a regressed scan instead of letting large fixtures hang CI.
+ i32::from(steps.fetch_add(1000, Ordering::Relaxed) > 100_000)
+ }
+
+ let root = tempfile::tempdir().unwrap();
+ let mut options = crate::storage::MigrationOptions::default();
+ options.privacy.identity = "migration-cursor-test".into();
+ options.privacy.required_surfaces = 1;
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)
+ .await
+ .unwrap();
+ db.execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-19','private frame')",
+ )
+ .await
+ .unwrap();
+ let count = 200_000_i64;
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ // Keep a large privacy-pending prefix. This models an interrupted migration
+ // after recording recovery has restored all elements to the resident table.
+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,_archive_generation) SELECT id,1,'accessibility','AXText','private',1 FROM n")
+ .bind(count).execute(&mut **tx.conn()).await.unwrap();
+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO ui_events(id,timestamp,event_type,text_content) SELECT id,'2026-09-19','text','private' FROM n")
+ .bind(count).execute(&mut **tx.conn()).await.unwrap();
+ // In-place migration deliberately postpones these indexes until completion.
+ sqlx::query("DROP INDEX _bulk_ui_events_pending")
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ db.execute_raw_sql_write("INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(200001,1,'accessibility','AXText','eligible',1); INSERT INTO ui_events(id,timestamp,event_type,text_content,redacted_at) VALUES(200001,'2026-09-19','text','eligible',1)").await.unwrap();
+
+ let storage = db.storage.as_ref().unwrap();
+ let index = storage.root.join(&storage.descriptor.index);
+ let pool = pool_options(Some(storage.clone()), false)
+ .max_connections(1)
+ .connect_with(sqlx::sqlite::SqliteConnectOptions::new().filename(&index))
+ .await
+ .unwrap();
+ let writer = screenpipe_sqlite_coordinator::SqliteWritePool::new(
+ pool.clone(),
+ screenpipe_sqlite_coordinator::sqlite_write_lock(&index),
+ );
+ let steps = Box::new(AtomicU64::new(0));
+ {
+ let mut conn = pool.acquire().await.unwrap();
+ let mut handle = conn.lock_handle().await.unwrap();
+ // Context remains alive until the callback is removed below.
+ unsafe {
+ libsqlite3_sys::sqlite3_progress_handler(
+ handle.as_raw_handle().as_ptr(),
+ 1000,
+ Some(count_steps),
+ (&*steps as *const AtomicU64).cast_mut().cast(),
+ );
+ }
+ }
+ let bulk = storage
+ .select_bulk(&pool, &TABLES[2], None, Some(count))
+ .await;
+ let elements = elements::seal_after(storage, &pool, &writer, Some(count)).await;
+ let work = steps.load(Ordering::Relaxed);
+ {
+ let mut conn = pool.acquire().await.unwrap();
+ let mut handle = conn.lock_handle().await.unwrap();
+ unsafe {
+ libsqlite3_sys::sqlite3_progress_handler(
+ handle.as_raw_handle().as_ptr(),
+ 0,
+ None,
+ std::ptr::null_mut(),
+ );
+ }
+ }
+ pool.close().await;
+ let bulk = bulk
+ .unwrap_or_else(|error| panic!("bulk scan exceeded work budget at {work} steps: {error}"));
+ assert_eq!(
+ bulk.iter().map(|r| r.id).collect::<Vec<_>>(),
+ vec![count + 1]
+ );
+ assert_eq!(
+ elements.unwrap_or_else(|error| panic!(
+ "element scan exceeded work budget at {work} steps: {error}"
+ )),
+ (1, Some(count + 1))
+ );
+ assert!(
+ work < 100_000,
+ "selection revisited retained history: {work}"
+ );
+ eprintln!("offline cursor: retained_rows_per_table={count}, sqlite_steps={work}");
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ count
+ );
+ db.close().await;
+}
+
#[tokio::test]
async fn element_publication_ignores_new_captures_but_rejects_changed_range_members() {
for archived in [false, true] {
diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rs
index d67cc69e3..bfaee6df6 100644
--- a/crates/screenpipe-db/src/storage/in_place.rs
+++ b/crates/screenpipe-db/src/storage/in_place.rs
@@ -175,7 +175,10 @@ async fn run(
if !archive { progress(MigrationProgress::phase("restoring saved screen records")); }
loop {
if archive { reserve(&storage)?; }
- if archive && storage.seal_once(&pool, &writer).await? != 0 {
+ // The offline owner visits each table after building its search
+ // entries. The live round-robin sealer would repeatedly scan other
+ // unfinished tables and archive them before that backfill.
+ if archive && storage.seal_frames(&pool, &writer).await? != 0 {
let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;
report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
continue;
@@ -201,16 +204,25 @@ async fn run(
super::faults::checkpoint("migration_batch_staged");
}
let source_exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='_bulk_elements_source')").fetch_one(&pool).await?;
- if source_exists {
+ {
if !archive { progress(MigrationProgress::phase("restoring saved accessibility records")); }
- let columns = super::import::columns(&pool, "_bulk_elements_source").await?;
+ let columns = if source_exists { super::import::columns(&pool, "_bulk_elements_source").await? } else { Vec::new() };
+ let mut after = None;
loop {
if archive { reserve(&storage)?; }
- if archive && bulk::elements::seal(&storage, &pool, &writer).await? != 0 {
- let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;
- report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
- continue;
+ if archive {
+ let (sealed, last) = bulk::elements::seal_after(&storage, &pool, &writer, after).await?;
+ if sealed != 0 {
+ after = last;
+ let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;
+ report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
+ continue;
+ }
}
+ if !source_exists { break; }
+ // Everything currently staged was visited. Retained private or
+ // oversized rows must not be rescanned for every source batch.
+ after = sqlx::query_scalar("SELECT max(id) FROM _bulk_element_rows").fetch_one(&pool).await?;
// Move existing records inside SQLite, including records larger
// than any encoder/decoder batch. Sealing can leave them resident.
let range = resident_range(&pool, "_bulk_elements_source", None,
@@ -237,31 +249,38 @@ async fn run(
drop(permit);
if archive { reclaim(&storage, &writer, file, &mut reclamation, false).await?; }
}
- let permit = writer.lock().await?;
- super::diagnostics::stage("dropping_empty_element_source");
- sqlx::query("DROP TABLE _bulk_elements_source").execute(permit.pool()).await?;
+ if source_exists {
+ let permit = writer.lock().await?;
+ super::diagnostics::stage("dropping_empty_element_source");
+ sqlx::query("DROP TABLE _bulk_elements_source").execute(permit.pool()).await?;
+ }
}
if !archive { progress(MigrationProgress::phase("restoring history search indexes")); }
for table in bulk::TABLES.iter().filter(|t| t.name != "elements") {
+ let mut after = None;
while archive {
reserve(&storage)?;
- let rows = storage.select_bulk(&pool, table, None).await?;
+ let rows = storage.select_bulk(&pool, table, None, after).await?;
if rows.is_empty() { break; }
if !table.fts.is_empty() {
let ids = serde_json::to_string(&rows.iter().map(|r| r.id).collect::<Vec<_>>()).map_err(storage_error)?;
let permit = writer.lock().await?;
- let mut conn = permit.pool().acquire().await?;
- let mut tx = conn.begin().await?;
- for sql in [format!("DELETE FROM {t}_fts WHERE rowid IN (SELECT value FROM json_each(?))",t=table.name), format!("INSERT INTO {t}_fts(rowid,{columns}) SELECT id,{columns} FROM {view} WHERE id IN (SELECT value FROM json_each(?)) AND ({condition})",t=table.name,columns=table.fts_columns(),view=table.view(),condition=table.fts_condition)] {
- sqlx::query(sqlx::AssertSqlSafe(sql)).bind(&ids).execute(&mut *tx).await?;
- }
+ let mut tx = permit.pool().begin().await?;
+ index_bulk_fts(&mut tx, table, &ids).await?;
tx.commit().await?;
}
- storage.publish_bulk(&pool, &writer, table, rows, None).await?;
+ let last = rows.last().unwrap().id;
+ let expected = rows.len();
+ if storage.publish_bulk(&pool, &writer, table, rows, None).await? != expected {
+ return Err(storage_error("offline bulk batch changed before publication"));
+ }
+ // Exclusive ownership makes this cursor safe. A retry starts
+ // from committed resident state, never from an uncommitted ID.
+ after = Some(last);
let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;
report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
}
- backfill_resident_fts(&storage, &pool, &writer, table, file, &mut reclamation, archive).await?;
+ backfill_bulk_fts(&storage, &pool, &writer, table, file, &mut reclamation, archive).await?;
}
if !archive { progress(MigrationProgress::phase("finishing recording recovery")); }
{
@@ -364,9 +383,22 @@ async fn reclaim(
Ok(reclaimed)
}
-// Privacy-ineligible records remain resident. Their search entries still need
-// the same bounded, committed backfill as records that were sealed above.
-async fn backfill_resident_fts(
+async fn index_bulk_fts(
+ conn: &mut sqlx::SqliteConnection,
+ table: &bulk::Table,
+ ids: &str,
+) -> Result<(), sqlx::Error> {
+ super::diagnostics::stage("indexing_bulk_search_batch");
+ for sql in [format!("DELETE FROM {t}_fts WHERE rowid IN (SELECT value FROM json_each(?))",t=table.name),format!("INSERT INTO {t}_fts(rowid,{columns}) SELECT id,{columns} FROM {view} WHERE id IN (SELECT value FROM json_each(?)) AND ({condition})",t=table.name,view=table.view(),columns=table.fts_columns(),condition=table.fts_condition)] {
+ sqlx::query(sqlx::AssertSqlSafe(sql)).bind(ids).execute(&mut *conn).await?;
+ }
+ Ok(())
+}
+
+// Fill missing search entries for resident/private records and archives
+// committed by older migration attempts that used the live sealer too early.
+// Normal conversion still indexes, seals and reclaims one batch at a time.
+async fn backfill_bulk_fts(
storage: &HybridStorage,
pool: &SqlitePool,
writer: &SqliteWritePool,
@@ -380,7 +412,7 @@ async fn backfill_resident_fts(
return Ok(());
}
let step = format!(
- "{}-fts-{}",
+ "{}-all-fts-{}",
if archive { "resident" } else { "recovery" },
table.name
);
@@ -389,7 +421,7 @@ async fn backfill_resident_fts(
.iter()
.map(|c| {
format!(
- "COALESCE(length(CAST({} AS BLOB)),0)",
+ "COALESCE(length(CAST(v.{} AS BLOB)),0)",
c.split_whitespace().next().unwrap()
)
})
@@ -405,7 +437,21 @@ async fn backfill_resident_fts(
.fetch_optional(pool)
.await?
.flatten();
- let sql = format!("SELECT id,{size} FROM {t} WHERE _archive_file IS NULL AND id {comparison} ? ORDER BY id LIMIT {rows}",t=table.name,comparison=if after.is_some(){">"}else{">="},rows=bulk::FILE_ROWS);
+ super::diagnostics::batch(
+ table.name,
+ after.and_then(|id| id.checked_add(1)),
+ None,
+ None,
+ None,
+ );
+ super::diagnostics::stage("selecting_bulk_search_batch");
+ let sql = format!(
+ "SELECT v.id,{size} FROM {view} v WHERE v.id {comparison} ? AND NOT EXISTS(SELECT 1 FROM {t}_fts WHERE rowid=v.id) ORDER BY v.id LIMIT {rows}",
+ view = table.view(),
+ t = table.name,
+ comparison = if after.is_some() { ">" } else { ">=" },
+ rows = bulk::FILE_ROWS
+ );
let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql))
.bind(after.unwrap_or(i64::MIN))
.fetch(pool);
@@ -427,9 +473,7 @@ async fn backfill_resident_fts(
let permit = writer.lock().await?;
let mut conn = permit.pool().acquire().await?;
let mut tx = conn.begin().await?;
- for sql in [format!("DELETE FROM {t}_fts WHERE rowid IN (SELECT value FROM json_each(?))",t=table.name),format!("INSERT INTO {t}_fts(rowid,{columns}) SELECT id,{columns} FROM {t} WHERE id IN (SELECT value FROM json_each(?)) AND ({condition})",t=table.name,columns=table.fts_columns(),condition=table.fts_condition)] {
- sqlx::query(sqlx::AssertSqlSafe(sql)).bind(&ids).execute(&mut *tx).await?;
- }
+ index_bulk_fts(&mut tx, table, &ids).await?;
sqlx::query("INSERT INTO _storage_conversion_steps(step,last_id) VALUES(?,?) ON CONFLICT(step) DO UPDATE SET last_id=excluded.last_id").bind(&step).bind(last).execute(&mut *tx).await?;
tx.commit().await?;
if archive {
diff --git a/crates/screenpipe-db/src/storage/sealing.rs b/crates/screenpipe-db/src/storage/sealing.rs
index 45e85965e..625a6b45d 100644
--- a/crates/screenpipe-db/src/storage/sealing.rs
+++ b/crates/screenpipe-db/src/storage/sealing.rs
@@ -294,7 +294,7 @@ impl HybridStorage {
Ok(0)
}
- async fn seal_frames(
+ pub(super) async fn seal_frames(
self: &Arc<Self>,
pool: &SqlitePool,
writer: &SqliteWritePool,
diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rs
index 09ef15df5..579240b87 100644
--- a/crates/screenpipe-db/tests/in_place_migration.rs
+++ b/crates/screenpipe-db/tests/in_place_migration.rs
@@ -224,6 +224,73 @@ async fn migration_diagnostics_preserve_table_totals_and_verified_completion() {
assert!(!json.contains("capture detail"));
}
+#[tokio::test]
+async fn migration_preserves_search_when_frames_and_bulk_history_are_both_present() {
+ assert_migration_search(false).await;
+}
+
+#[tokio::test]
+#[cfg(feature = "storage-fault-injection")]
+async fn retry_repairs_search_for_already_archived_bulk_history() {
+ assert_migration_search(true).await;
+}
+
+async fn assert_migration_search(resume: bool) {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("db.sqlite");
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-19','frame history'),(2,'2026-09-19','other frame'); INSERT INTO audio_chunks(id,file_path) VALUES(1,'test.wav'); INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,device) VALUES(1,1,0,'2026-09-19','migrationneedle spoken history','test'); INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(1,'2026-09-19','text','migrationneedle typed history'); INSERT INTO semantic_items(id,entity_fingerprint,version_fingerprint,kind,item_key,identity_quality,title,body,metadata_json) VALUES(1,randomblob(32),randomblob(32),'document','migration','stable','title','migrationneedle document','{}');").await.unwrap();
+ db.close().await;
+ if resume {
+ let crashed = std::process::Command::new(env!("CARGO_BIN_EXE_screenpipe-storage"))
+ .arg("migrate")
+ .arg(root.path())
+ .env("SCREENPIPE_STORAGE_CRASH_AT", "bulk_committed")
+ .output()
+ .unwrap();
+ assert_eq!(crashed.status.code(), Some(86));
+ screenpipe_db::storage::recover_interrupted_migration(root.path(), Default::default())
+ .await
+ .unwrap();
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(
+ "SELECT count(*) FROM main.audio_transcriptions WHERE _archive_file IS NOT NULL"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 1
+ );
+ // Older attempts could publish this archive before populating FTS.
+ db.execute_raw_sql_write("DELETE FROM audio_transcriptions_fts; DELETE FROM _storage_conversion_steps WHERE step LIKE '%-all-fts-%'")
+ .await.unwrap();
+ db.close().await;
+ }
+ let report = migrate(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ assert_eq!(report.frames, 2);
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ for table in ["audio_transcriptions", "ui_events", "semantic_items"] {
+ let found: i64 = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
+ "SELECT count(*) FROM {table}_fts WHERE {table}_fts MATCH 'migrationneedle'"
+ )))
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ assert_eq!(found, 1, "migrated {table} must remain searchable");
+ }
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
#[tokio::test]
#[ignore = "migration throughput benchmark; generates an isolated recording history"]
async fn migration_throughput() {
@@ -1256,6 +1323,12 @@ async fn recording_recovery_keeps_committed_payloads_and_survives_restarts() {
)
.await
.unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows")
+ .fetch_one(&db.pool).await.unwrap(),
+ 0,
+ "explicit retry must archive restored elements even after recovery removed the source table"
+ );
assert_eq!(
db.frame_payloads(&[49, 50], Projection::Search)
.await
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 15e0aba60..82ab08822 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 34
- Mapped Rust files: 362
-- Active test blocks: 3549
+- Active test blocks: 3552
- Ignored/manual test blocks: 149
-- Weighted coverage points: 2933.3
+- Weighted coverage points: 2936.3
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 31 | 3406 | 143 | 2868.2 | 21 | 11 | 100% |
-| macos | 31 | 3468 | 123 | 2882.2 | 22 | 11 | 100% |
-| linux | 27 | 3028 | 115 | 2519.5 | 20 | 11 | 100% |
+| windows | 31 | 3409 | 143 | 2871.2 | 21 | 11 | 100% |
+| macos | 31 | 3471 | 123 | 2885.2 | 22 | 11 | 100% |
+| linux | 27 | 3031 | 115 | 2522.5 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 8d85414e9..5aabd841d 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 34
- Mapped Rust files: 362
-- Active test blocks: 3549
+- Active test blocks: 3552
- Ignored/manual test blocks: 149
-- Declared test blocks: 3698
-- Weighted coverage points: 2933.3
+- Declared test blocks: 3701
+- Weighted coverage points: 2936.3
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 31 | 3406 | 143 | 2868.2 | 21 | 11 | 100% |
-| macos | 31 | 3468 | 123 | 2882.2 | 22 | 11 | 100% |
-| linux | 27 | 3028 | 115 | 2519.5 | 20 | 11 | 100% |
+| windows | 31 | 3409 | 143 | 2871.2 | 21 | 11 | 100% |
+| macos | 31 | 3471 | 123 | 2885.2 | 22 | 11 | 100% |
+| linux | 27 | 3031 | 115 | 2522.5 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 11 | 20 | 120 | 1732 | 43 | 1335.9 | 10 |
-| screenpipe-db | 5 | 60 | 23 | 555 | 24 | 528.3 | 9 |
+| screenpipe-db | 5 | 60 | 23 | 558 | 24 | 531.3 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 621 | 44 | 545.6 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -66,21 +66,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 8 suites / 737 active / 45 ignored / 661.6 pts | 8 suites / 737 active / 45 ignored / 661.6 pts | 7 suites / 662 active / 44 ignored / 609.1 pts |
| audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 7 suites / 506 active / 20 ignored / 479.3 pts | 7 suites / 506 active / 20 ignored / 479.3 pts | 7 suites / 506 active / 20 ignored / 479.3 pts |
+| database | 7 suites / 509 active / 20 ignored / 482.3 pts | 7 suites / 509 active / 20 ignored / 482.3 pts | 7 suites / 509 active / 20 ignored / 482.3 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts |
| local-api | 3 suites / 433 active / 10 ignored / 305.5 pts | 3 suites / 433 active / 10 ignored / 305.5 pts | 3 suites / 433 active / 10 ignored / 305.5 pts |
| meeting | 6 suites / 1636 active / 21 ignored / 1336.9 pts | 6 suites / 1636 active / 21 ignored / 1336.9 pts | 4 suites / 1291 active / 17 ignored / 1014.4 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1560 active / 75 ignored / 1379.6 pts | 14 suites / 1667 active / 79 ignored / 1422.4 pts | 13 suites / 1560 active / 75 ignored / 1379.6 pts |
+| performance | 13 suites / 1563 active / 75 ignored / 1382.6 pts | 14 suites / 1670 active / 79 ignored / 1425.4 pts | 13 suites / 1563 active / 75 ignored / 1382.6 pts |
| pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
| privacy | 5 suites / 942 active / 37 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts |
| real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - |
| speaker | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts |
-| storage | 4 suites / 621 active / 37 ignored / 514.8 pts | 4 suites / 621 active / 37 ignored / 514.8 pts | 4 suites / 621 active / 37 ignored / 514.8 pts |
+| storage | 4 suites / 624 active / 37 ignored / 517.8 pts | 4 suites / 624 active / 37 ignored / 517.8 pts | 4 suites / 624 active / 37 ignored / 517.8 pts |
| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
-| timeline | 4 suites / 1170 active / 42 ignored / 956.4 pts | 4 suites / 1170 active / 42 ignored / 956.4 pts | 4 suites / 1170 active / 42 ignored / 956.4 pts |
+| timeline | 4 suites / 1173 active / 42 ignored / 959.4 pts | 4 suites / 1173 active / 42 ignored / 959.4 pts | 4 suites / 1173 active / 42 ignored / 959.4 pts |
| transcription | 5 suites / 814 active / 42 ignored / 637.8 pts | 5 suites / 814 active / 42 ignored / 637.8 pts | 5 suites / 814 active / 42 ignored / 637.8 pts |
| ui-events | 4 suites / 758 active / 29 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |
| vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |
@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 33 | 267 | 11 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 33 | 270 | 11 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 411 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -278,7 +278,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | src/failpoint_vfs.rs | source | 4 | 0 | 4 |
| db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 |
| db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |
-| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 7 | 0 | 7 |
+| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 8 | 0 | 8 |
| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 |
| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 |
| db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |
@@ -307,7 +307,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 |
| db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 |
| db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |
-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 16 | 5 | 21 |
+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 18 | 5 | 23 |
| db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 |
| db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 |
| db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |