Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): prevent quadratic element migration scans (#7094)

Ezra Ellettecd773485d127
diff --git a/COVERAGE.md b/COVERAGE.md
index a6059e12c..8f84df623 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -35,16 +35,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 33
-- Mapped Rust files: 358
-- Active test blocks: 3520
+- Mapped Rust files: 359
+- Active test blocks: 3523
- Ignored/manual test blocks: 147
-- Weighted coverage points: 2906.1
+- Weighted coverage points: 2909.1
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3377 | 141 | 2841.0 | 21 | 11 | 100% |
-| macos | 30 | 3439 | 121 | 2855.0 | 22 | 11 | 100% |
-| linux | 26 | 3007 | 113 | 2500.3 | 20 | 11 | 100% |
+| windows | 30 | 3380 | 141 | 2844.0 | 21 | 11 | 100% |
+| macos | 30 | 3442 | 121 | 2858.0 | 22 | 11 | 100% |
+| linux | 26 | 3010 | 113 | 2503.3 | 20 | 11 | 100% |
## Refresh
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
index 3368a331b..2a91aa057 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
@@ -909,19 +909,27 @@ fn track_migration_diagnostic(
return;
};
let mut properties = serde_json::to_value(snapshot).unwrap_or_default();
- // Keep the bounded error locally for feedback redaction. Exception strings
- // can contain paths; remote counters/stages never include those strings.
- if let Some(properties) = properties.as_object_mut() {
- properties.remove("error");
- }
let analytics = Arc::clone(&analytics);
tauri::async_runtime::spawn(async move {
+ redact_migration_diagnostic_error(&mut properties);
if let Err(error) = analytics.send_event(event, Some(properties)).await {
tracing::warn!(%error, event, "migration diagnostic telemetry failed");
}
});
}
+fn redact_migration_diagnostic_error(properties: &mut serde_json::Value) {
+ if let Some(error) = properties.get_mut("error") {
+ if let Some(message) = error.as_str() {
+ // Keep diagnostic context while scrubbing recognizable secrets/PII
+ // locally. The snapshot already bounds the error to 512 characters.
+ *error = screenpipe_redact::adapters::regex::redact_one(message)
+ .redacted
+ .into();
+ }
+ }
+}
+
fn report_migration_failure(app: &tauri::AppHandle, root: &Path, error: &str) {
let state = app.state::<StorageMigrationState>();
let mut operation = state.0.lock().unwrap_or_else(|e| e.into_inner());
@@ -1064,6 +1072,33 @@ pub async fn delete_original_storage_database(
mod tests {
use super::*;
+ #[test]
+ fn migration_diagnostics_keep_useful_errors_and_scrub_sensitive_values() {
+ let mut properties = serde_json::json!({
+ "attempt_id": "attempt-42",
+ "stage": "selecting_element_batch",
+ "first_id": 1_400_786,
+ "last_id": 158_475_583,
+ "completed_records": 1_400_785,
+ "error": "SQLite error (code: 5): database is locked; [email protected] api_key=deadbeef",
+ });
+ let original = properties.clone();
+ redact_migration_diagnostic_error(&mut properties);
+ let error = properties["error"]
+ .as_str()
+ .expect("error must be reported");
+ assert!(error.contains("SQLite error (code: 5): database is locked"));
+ assert!(!error.contains("[email protected]"));
+ assert!(!error.contains("deadbeef"));
+ properties["error"] = original["error"].clone();
+ assert_eq!(properties, original, "diagnostic context must be preserved");
+
+ // A stalled query need not have returned an error yet.
+ properties["error"] = serde_json::Value::Null;
+ redact_migration_diagnostic_error(&mut properties);
+ assert!(properties["error"].is_null());
+ }
+
#[tokio::test]
async fn explicit_retry_waits_for_restart_then_owns_storage_before_later_restarts() {
let lifecycle = Arc::new(tokio::sync::Mutex::new(()));
diff --git a/crates/screenpipe-db/examples/migration_element_probe.rs b/crates/screenpipe-db/examples/migration_element_probe.rs
new file mode 100644
index 000000000..05535a2e5
--- /dev/null
+++ b/crates/screenpipe-db/examples/migration_element_probe.rs
@@ -0,0 +1,116 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Isolated probe for migration selection over privacy-pending element history.
+use screenpipe_db::{storage::MigrationOptions, DatabaseManager};
+use std::{
+ sync::atomic::{AtomicU64, Ordering},
+ time::Instant,
+};
+
+unsafe extern "C" fn progress(context: *mut std::ffi::c_void) -> i32 {
+ let steps = unsafe { &*(context as *const AtomicU64) };
+ steps.fetch_add(1000, Ordering::Relaxed);
+ 0
+}
+
+async fn measure(db: &DatabaseManager, label: &str, sql: &str, first: i64) -> anyhow::Result<()> {
+ let steps = Box::new(AtomicU64::new(0));
+ let mut conn = db.pool.acquire().await?;
+ {
+ let mut handle = conn.lock_handle().await?;
+ unsafe {
+ libsqlite3_sys::sqlite3_progress_handler(
+ handle.as_raw_handle().as_ptr(),
+ 1000,
+ Some(progress),
+ (&*steps as *const AtomicU64).cast_mut().cast(),
+ );
+ }
+ }
+ let started = Instant::now();
+ let result = sqlx::query_as::<_, (i64, i64)>(sqlx::AssertSqlSafe(sql.to_owned()))
+ .bind(first)
+ .bind(i64::MAX)
+ .fetch_all(&mut *conn)
+ .await;
+ let elapsed = started.elapsed();
+ {
+ let mut handle = conn.lock_handle().await?;
+ unsafe {
+ libsqlite3_sys::sqlite3_progress_handler(
+ handle.as_raw_handle().as_ptr(),
+ 0,
+ None,
+ std::ptr::null_mut(),
+ );
+ }
+ }
+ let rows = result?;
+ assert_eq!(rows.len(), 1);
+ println!(
+ "{label}: rows={} first={} elapsed_ms={} sqlite_steps={}",
+ rows.len(),
+ rows[0].0,
+ elapsed.as_millis(),
+ steps.load(Ordering::Relaxed)
+ );
+ Ok(())
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+ let count: i64 = std::env::args().nth(1).unwrap_or("100000".into()).parse()?;
+ let root = tempfile::tempdir()?;
+ let mut options = MigrationOptions::default();
+ options.privacy.identity = "sparse-redaction-probe".into();
+ options.privacy.required_surfaces = 1;
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options).await?;
+ let ready = count / 2;
+ // Seed only the resident row source needed by the read-only selection. This
+ // does not exercise ingestion, publication, or migration completion.
+ let mut tx = db.begin_immediate_with_retry().await?;
+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,properties,redacted_at,_archive_generation) SELECT id,1,'accessibility','AXText','pending privacy',?,CASE WHEN id=? THEN 1 ELSE NULL END,1 FROM n")
+ .bind(count).bind("properties ".repeat(32)).bind(ready).execute(&mut **tx.conn()).await?;
+ tx.commit().await?;
+ let text = ["source", "role", "text", "properties"];
+ let numeric = [
+ "frame_id",
+ "parent_id",
+ "depth",
+ "left_bound",
+ "top_bound",
+ "width_bound",
+ "height_bound",
+ "confidence",
+ "sort_order",
+ "on_screen",
+ "redacted_at",
+ ];
+ let size = text
+ .iter()
+ .map(|c| format!("COALESCE(length(CAST({c} AS BLOB)),0)"))
+ .chain(
+ numeric
+ .iter()
+ .map(|c| format!("CASE WHEN {c} IS NULL THEN 0 ELSE 8 END")),
+ )
+ .collect::<Vec<_>>()
+ .join("+");
+ let eligible = "redacted_at IS NOT NULL OR (SELECT required_surfaces=0 FROM storage_metadata) OR (COALESCE(text,'')='' AND properties IS NULL)";
+ println!(
+ "probe: elements={count} eligible=1 required_surfaces=1 sqlite={}",
+ sqlx::query_scalar::<_, String>("SELECT sqlite_version()")
+ .fetch_one(&db.pool)
+ .await?
+ );
+ for (label, source, extra) in [
+ ("virtual_table", "elements", "1"),
+ ("resident_table", "_bulk_element_rows", "_archive_deleted=0"),
+ ] {
+ let sql = format!("SELECT id,{size} FROM {source} WHERE id BETWEEN ? AND ? AND {extra} AND ({eligible}) ORDER BY id LIMIT 32768");
+ measure(&db, label, &sql, ready).await?;
+ }
+ db.close().await;
+ Ok(())
+}
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
index 2d76307ea..2b5bf79c2 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
@@ -224,6 +224,15 @@ async fn rewrite(
let mut files = Vec::new();
loop {
let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));
+ // Report the query bounds before awaiting it; actual batch sizes and
+ // row IDs are only available once selection returns.
+ crate::storage::diagnostics::batch(
+ "elements",
+ Some(after.map_or(first, |id: i64| id.saturating_add(1))),
+ Some(last),
+ None,
+ None,
+ );
crate::storage::diagnostics::stage("selecting_element_batch");
let candidates: Vec<(i64, i64)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(
"SELECT id,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id LIMIT {FILE_ROWS}",
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
index 2b341db28..e2a202540 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
@@ -262,6 +262,7 @@ unsafe impl<'v> VTab<'v> for Elements {
pending: VecDeque::new(),
pending_sql: String::new(),
pending_args: Vec::new(),
+ pending_seek_args: Vec::new(),
pending_after: None,
pending_done: false,
descending: false,
@@ -540,6 +541,7 @@ struct Cursor {
pending: VecDeque<(Record, bool)>,
pending_sql: String,
pending_args: Vec<SqlValue>,
+ pending_seek_args: Vec<usize>,
pending_after: Option<i64>,
pending_done: bool,
descending: bool,
@@ -567,6 +569,13 @@ impl Cursor {
});
let mut args = self.pending_args.clone();
if let Some(id) = self.pending_after {
+ // This row already satisfied every bound in the scan direction.
+ // Advance those bounds too: SQLite may seek using an original
+ // >=/<= parameter instead of the appended strict cursor predicate,
+ // otherwise rescanning the growing prefix on every 128-row page.
+ for &index in &self.pending_seek_args {
+ args[index] = SqlValue::Integer(id);
+ }
args.push(SqlValue::Integer(id));
}
let sql = format!(
@@ -716,6 +725,7 @@ unsafe impl VTabCursor for Cursor {
self.pending.clear();
self.pending_after = None;
self.pending_done = false;
+ self.pending_seek_args.clear();
self.files_after = None;
self.files_done = false;
self.archive = None;
@@ -769,6 +779,12 @@ unsafe impl VTabCursor for Cursor {
} else if matches!(op, ">" | ">=") {
range_args.extend([value.clone(), value.clone()]);
}
+ if matches!(
+ (self.descending, op),
+ (false, ">" | ">=") | (true, "<" | "<=")
+ ) {
+ self.pending_seek_args.push(pending_args.len());
+ }
pending.push(format!("id{op}?"));
pending_args.push(value);
}
diff --git a/crates/screenpipe-db/tests/element_pagination.rs b/crates/screenpipe-db/tests/element_pagination.rs
new file mode 100644
index 000000000..ad69561e1
--- /dev/null
+++ b/crates/screenpipe-db/tests/element_pagination.rs
@@ -0,0 +1,170 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+use rusqlite::{params_from_iter, types::Value, Connection};
+use screenpipe_db::{storage::MigrationOptions, DatabaseManager};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+struct Work {
+ steps: AtomicU64,
+ limit: u64,
+}
+
+unsafe extern "C" fn count_steps(context: *mut std::ffi::c_void) -> i32 {
+ let work = unsafe { &*(context as *const Work) };
+ i32::from(work.steps.fetch_add(1000, Ordering::Relaxed) + 1000 > work.limit)
+}
+
+struct ProgressGuard<'a> {
+ db: &'a Connection,
+ work: Box<Work>,
+}
+
+impl Drop for ProgressGuard<'_> {
+ fn drop(&mut self) {
+ // Remove the callback before its context or borrowed connection dies.
+ unsafe {
+ libsqlite3_sys::sqlite3_progress_handler(
+ self.db.handle(),
+ 0,
+ None,
+ std::ptr::null_mut(),
+ );
+ }
+ }
+}
+
+fn ids(db: &Connection, sql: &str, args: &[Value]) -> rusqlite::Result<Vec<i64>> {
+ db.prepare(sql)?
+ .query_map(params_from_iter(args), |r| r.get(0))?
+ .collect()
+}
+
+#[tokio::test]
+async fn sparse_element_selection_has_linear_work_in_both_directions() {
+ let root = tempfile::tempdir().unwrap();
+ let mut options = MigrationOptions::default();
+ options.privacy.identity = "pagination-regression".into();
+ options.privacy.required_surfaces = 1;
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)
+ .await
+ .unwrap();
+ let count = 100_000_i64;
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ // Isolate the read cursor: one eligible row in a large resident backlog.
+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,redacted_at,_archive_generation) SELECT id,1,'accessibility','AXText','pending privacy',CASE WHEN id=? THEN 1 ELSE NULL END,1 FROM n")
+ .bind(count).bind(count / 2).execute(&mut **tx.conn()).await.unwrap();
+ tx.commit().await.unwrap();
+ {
+ let mut conn = db.pool.acquire().await.unwrap();
+ let mut handle = conn.lock_handle().await.unwrap();
+ // The SQLx worker is paused while this borrowed handle is used.
+ let sqlite = unsafe { Connection::from_handle(handle.as_raw_handle().as_ptr()) }.unwrap();
+ for direction in ["ASC", "DESC"] {
+ // Count SQLite VM work, including nested virtual-table statements,
+ // instead of asserting machine-dependent timing. Interrupt a
+ // quadratic regression before it can hang CI on a larger fixture.
+ let guard = ProgressGuard {
+ db: &sqlite,
+ work: Box::new(Work {
+ steps: AtomicU64::new(0),
+ limit: count as u64 * 200,
+ }),
+ };
+ unsafe {
+ libsqlite3_sys::sqlite3_progress_handler(
+ sqlite.handle(),
+ 1000,
+ Some(count_steps),
+ (&*guard.work as *const Work).cast_mut().cast(),
+ );
+ }
+ let selected = ids(&sqlite, &format!("SELECT id FROM elements WHERE id BETWEEN ? AND ? AND (redacted_at IS NOT NULL OR (SELECT required_surfaces=0 FROM storage_metadata) OR (COALESCE(text,'')='' AND properties IS NULL)) ORDER BY id {direction} LIMIT 32768"), &[1.into(), count.into()]);
+ let work = guard.work.steps.load(Ordering::Relaxed);
+ drop(guard);
+ assert_eq!(
+ selected.unwrap_or_else(|error| {
+ panic!("{direction} selection failed at {work} steps: {error}")
+ }),
+ vec![count / 2]
+ );
+ }
+ }
+ db.close().await;
+}
+
+async fn assert_bounds(db: &DatabaseManager, reference: &Connection) {
+ let mut conn = db.pool.acquire().await.unwrap();
+ let mut handle = conn.lock_handle().await.unwrap();
+ let sqlite = unsafe { Connection::from_handle(handle.as_raw_handle().as_ptr()) }.unwrap();
+ let cases: Vec<(&str, Vec<Value>)> = vec![
+ ("1", vec![]),
+ ("id>=? AND id<=?", vec![(-250).into(), 350.into()]),
+ (
+ "id>=? AND id>? AND id<=? AND id<?",
+ vec![(-500).into(), (-250).into(), 400.into(), 350.into()],
+ ),
+ (
+ "id>=? AND id<=?",
+ vec![Value::Text("-250".into()), Value::Text("350".into())],
+ ),
+ (
+ "id>? AND id<?",
+ vec![Value::Real(-250.5), Value::Real(350.5)],
+ ),
+ ("id<?", vec![Value::Text("not a number".into())]),
+ ("id>?", vec![Value::Text("not a number".into())]),
+ ("id>=?", vec![Value::Null]),
+ ("id=?", vec![Value::Text("7".into())]),
+ ("id>?", vec![i64::MAX.into()]),
+ ("id<?", vec![i64::MIN.into()]),
+ (
+ "id>=? AND id<=? AND frame_id=?",
+ vec![(-400).into(), 400.into(), 1.into()],
+ ),
+ ];
+ for (condition, args) in cases {
+ for direction in ["ASC", "DESC"] {
+ let sql = format!("SELECT id FROM elements WHERE {condition} ORDER BY id {direction}");
+ assert_eq!(
+ ids(&sqlite, &sql, &args).unwrap(),
+ ids(reference, &sql, &args).unwrap(),
+ "{sql}, {args:?}"
+ );
+ }
+ }
+}
+
+#[tokio::test]
+async fn advancing_pages_preserve_bounds_affinity_and_mixed_archives() {
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ db.execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp) VALUES(1,'2026-09-18'),(2,'2026-09-18')",
+ )
+ .await
+ .unwrap();
+ let reference = Connection::open_in_memory().unwrap();
+ reference.execute_batch("CREATE TABLE elements(id INTEGER PRIMARY KEY,frame_id INTEGER,source TEXT,role TEXT,text TEXT)").unwrap();
+ let seed = "WITH RECURSIVE n(id) AS (VALUES(-512) UNION ALL SELECT id+1 FROM n WHERE id<512) INSERT INTO elements(id,frame_id,source,role,text) SELECT id,1+abs(id%2),'accessibility','AXText','page test' FROM n; INSERT INTO elements(id,frame_id,source,role,text) VALUES(-9223372036854775808,1,'accessibility','AXText','minimum'),(9223372036854775807,1,'accessibility','AXText','maximum');";
+ db.execute_raw_sql_write(seed).await.unwrap();
+ reference.execute_batch(seed).unwrap();
+ assert_bounds(&db, &reference).await;
+ while db.seal_payloads().await.unwrap() != 0 {}
+ let changes = "UPDATE elements SET text='edited' WHERE id%3=0; DELETE FROM elements WHERE id%7=0; INSERT INTO elements(id,frame_id,source,role,text) VALUES(777,2,'accessibility','AXText','new capture');";
+ db.execute_raw_sql_write(changes).await.unwrap();
+ reference.execute_batch(changes).unwrap();
+ assert_bounds(&db, &reference).await;
+ db.verify_storage().await.unwrap();
+ db.close().await;
+ let reopened = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ assert_bounds(&reopened, &reference).await;
+ reopened.close().await;
+}
diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rs
index c927318de..f2888a940 100644
--- a/crates/screenpipe-db/tests/in_place_migration.rs
+++ b/crates/screenpipe-db/tests/in_place_migration.rs
@@ -626,6 +626,92 @@ async fn privacy_pending_payloads_remain_resident_and_searchable() {
db.close().await;
}
+#[tokio::test]
+async fn sparse_privacy_backlog_completes_migration_and_preserves_recording() {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("db.sqlite");
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ let count = 100_000_i64;
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-18T12:00:00Z','pending frame privacy')").await.unwrap();
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO elements(id,frame_id,source,role,text,redacted_at) SELECT id,1,'accessibility','AXText','pending element history '||id,CASE WHEN id=? THEN 1 ELSE NULL END FROM n")
+ .bind(count).bind(count / 2).execute(&mut **tx.conn()).await.unwrap();
+ tx.commit().await.unwrap();
+ db.close().await;
+ let mut options = MigrationOptions::default();
+ options.privacy.identity = "sparse-privacy-migration".into();
+ options.privacy.required_surfaces = 1;
+ let report = migrate(root.path(), Default::default(), options)
+ .await
+ .unwrap();
+ assert_eq!(
+ report
+ .tables
+ .iter()
+ .find(|t| t.table == "elements")
+ .unwrap()
+ .rows,
+ count as u64
+ );
+ assert!(root
+ .path()
+ .join("storage-migration-complete.json")
+ .is_file());
+ assert!(!root.path().join("storage-migration.json").exists());
+
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ count - 1
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(
+ "SELECT sum(rows) FROM _bulk_files WHERE table_name='elements' AND state='published'"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 1
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(
+ "SELECT count(*) FROM elements_fts WHERE elements_fts MATCH 'history'"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ count
+ );
+ db.verify_storage().await.unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-18T13:00:00Z','new durable capture'); INSERT INTO elements(id,frame_id,source,role,text) VALUES(100001,2,'accessibility','AXText','new capture element')").await.unwrap();
+ db.close().await;
+ let reopened = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM elements")
+ .fetch_one(&reopened.pool)
+ .await
+ .unwrap(),
+ count + 1
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, String>("SELECT text FROM elements WHERE id=100001")
+ .fetch_one(&reopened.pool)
+ .await
+ .unwrap(),
+ "new capture element"
+ );
+ reopened.close().await;
+}
+
#[tokio::test]
#[ignore = "requires a marked disposable filesystem without hole punching"]
async fn unsupported_volume_fails_before_conversion() {
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 108c44929..29c415321 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 33
-- Mapped Rust files: 358
-- Active test blocks: 3520
+- Mapped Rust files: 359
+- Active test blocks: 3523
- Ignored/manual test blocks: 147
-- Declared test blocks: 3667
-- Weighted coverage points: 2906.1
+- Declared test blocks: 3670
+- Weighted coverage points: 2909.1
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3377 | 141 | 2841.0 | 21 | 11 | 100% |
-| macos | 30 | 3439 | 121 | 2855.0 | 22 | 11 | 100% |
-| linux | 26 | 3007 | 113 | 2500.3 | 20 | 11 | 100% |
+| windows | 30 | 3380 | 141 | 2844.0 | 21 | 11 | 100% |
+| macos | 30 | 3442 | 121 | 2858.0 | 22 | 11 | 100% |
+| linux | 26 | 3010 | 113 | 2503.3 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |
-| screenpipe-db | 5 | 58 | 23 | 544 | 23 | 517.3 | 9 |
+| screenpipe-db | 5 | 59 | 23 | 547 | 23 | 520.3 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 618 | 44 | 543.2 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 731 active / 45 ignored / 656.2 pts | 7 suites / 731 active / 45 ignored / 656.2 pts | 6 suites / 656 active / 44 ignored / 603.7 pts |
| audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 6 suites / 492 active / 19 ignored / 465.3 pts | 6 suites / 492 active / 19 ignored / 465.3 pts | 6 suites / 492 active / 19 ignored / 465.3 pts |
+| database | 6 suites / 495 active / 19 ignored / 468.3 pts | 6 suites / 495 active / 19 ignored / 468.3 pts | 6 suites / 495 active / 19 ignored / 468.3 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts |
| local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts |
| meeting | 6 suites / 1624 active / 20 ignored / 1325.8 pts | 6 suites / 1624 active / 20 ignored / 1325.8 pts | 4 suites / 1287 active / 16 ignored / 1011.3 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1547 active / 74 ignored / 1367.2 pts | 14 suites / 1654 active / 78 ignored / 1410.0 pts | 13 suites / 1547 active / 74 ignored / 1367.2 pts |
+| performance | 13 suites / 1550 active / 74 ignored / 1370.2 pts | 14 suites / 1657 active / 78 ignored / 1413.0 pts | 13 suites / 1550 active / 74 ignored / 1370.2 pts |
| pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
| privacy | 5 suites / 942 active / 37 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts |
| real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - |
| speaker | 2 suites / 368 active / 9 ignored / 368.0 pts | 2 suites / 368 active / 9 ignored / 368.0 pts | 2 suites / 368 active / 9 ignored / 368.0 pts |
-| storage | 4 suites / 609 active / 36 ignored / 503.1 pts | 4 suites / 609 active / 36 ignored / 503.1 pts | 4 suites / 609 active / 36 ignored / 503.1 pts |
+| storage | 4 suites / 612 active / 36 ignored / 506.1 pts | 4 suites / 612 active / 36 ignored / 506.1 pts | 4 suites / 612 active / 36 ignored / 506.1 pts |
| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
-| timeline | 4 suites / 1156 active / 40 ignored / 943.3 pts | 4 suites / 1156 active / 40 ignored / 943.3 pts | 4 suites / 1156 active / 40 ignored / 943.3 pts |
+| timeline | 4 suites / 1159 active / 40 ignored / 946.3 pts | 4 suites / 1159 active / 40 ignored / 946.3 pts | 4 suites / 1159 active / 40 ignored / 946.3 pts |
| transcription | 5 suites / 808 active / 41 ignored / 633.3 pts | 5 suites / 808 active / 41 ignored / 633.3 pts | 5 suites / 808 active / 41 ignored / 633.3 pts |
| ui-events | 4 suites / 758 active / 29 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |
| vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |
@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 31 | 256 | 10 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 32 | 259 | 10 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -298,13 +298,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-accessibility-ui-events | screenpipe-db | tests/db_config_test.rs | integration | 5 | 0 | 5 |
| db-timeline-frames | screenpipe-db | tests/db.rs | integration | 42 | 0 | 42 |
| db-accessibility-ui-events | screenpipe-db | tests/display_layout_test.rs | integration | 4 | 0 | 4 |
+| db-timeline-frames | screenpipe-db | tests/element_pagination.rs | integration | 2 | 0 | 2 |
| db-timeline-frames | screenpipe-db | tests/evict_media_null_chunk_test.rs | integration | 2 | 0 | 2 |
| db-timeline-frames | screenpipe-db | tests/frame_offset_sync_test.rs | integration | 6 | 0 | 6 |
| db-search-indexing | screenpipe-db | tests/fts_contention_test.rs | integration | 4 | 0 | 4 |
| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 |
| db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 |
| db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |
-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 15 | 4 | 19 |
+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 16 | 4 | 20 |
| db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 |
| db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 |
| db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index 527fafa25..0007b2eec 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -677,6 +677,7 @@
"src/write_queue.rs",
"tests/bulk_storage.rs",
"tests/db.rs",
+ "tests/element_pagination.rs",
"tests/evict_media_null_chunk_test.rs",
"tests/frame_offset_sync_test.rs",
"tests/hybrid_storage.rs",