YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): prevent quadratic element migration scans (#7094)
Ezra Ellettecd773485d127
diff --git a/COVERAGE.md b/COVERAGE.mdindex a6059e12c..8f84df623 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -35,16 +35,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine- Mapped suites: 33-- Mapped Rust files: 358-- Active test blocks: 3520+- Mapped Rust files: 359+- Active test blocks: 3523- Ignored/manual test blocks: 147-- Weighted coverage points: 2906.1+- Weighted coverage points: 2909.1| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3377 | 141 | 2841.0 | 21 | 11 | 100% |-| macos | 30 | 3439 | 121 | 2855.0 | 22 | 11 | 100% |-| linux | 26 | 3007 | 113 | 2500.3 | 20 | 11 | 100% |+| windows | 30 | 3380 | 141 | 2844.0 | 21 | 11 | 100% |+| macos | 30 | 3442 | 121 | 2858.0 | 22 | 11 | 100% |+| linux | 26 | 3010 | 113 | 2503.3 | 20 | 11 | 100% |## Refreshdiff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rsindex 3368a331b..2a91aa057 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs@@ -909,19 +909,27 @@ fn track_migration_diagnostic(return;};let mut properties = serde_json::to_value(snapshot).unwrap_or_default();- // Keep the bounded error locally for feedback redaction. Exception strings- // can contain paths; remote counters/stages never include those strings.- if let Some(properties) = properties.as_object_mut() {- properties.remove("error");- }let analytics = Arc::clone(&analytics);tauri::async_runtime::spawn(async move {+ redact_migration_diagnostic_error(&mut properties);if let Err(error) = analytics.send_event(event, Some(properties)).await {tracing::warn!(%error, event, "migration diagnostic telemetry failed");}});}+fn redact_migration_diagnostic_error(properties: &mut serde_json::Value) {+ if let Some(error) = properties.get_mut("error") {+ if let Some(message) = error.as_str() {+ // Keep diagnostic context while scrubbing recognizable secrets/PII+ // locally. The snapshot already bounds the error to 512 characters.+ *error = screenpipe_redact::adapters::regex::redact_one(message)+ .redacted+ .into();+ }+ }+}+fn report_migration_failure(app: &tauri::AppHandle, root: &Path, error: &str) {let state = app.state::<StorageMigrationState>();let mut operation = state.0.lock().unwrap_or_else(|e| e.into_inner());@@ -1064,6 +1072,33 @@ pub async fn delete_original_storage_database(mod tests {use super::*;+ #[test]+ fn migration_diagnostics_keep_useful_errors_and_scrub_sensitive_values() {+ let mut properties = serde_json::json!({+ "attempt_id": "attempt-42",+ "stage": "selecting_element_batch",+ "first_id": 1_400_786,+ "last_id": 158_475_583,+ "completed_records": 1_400_785,+ "error": "SQLite error (code: 5): database is locked; [email protected] api_key=deadbeef",+ });+ let original = properties.clone();+ redact_migration_diagnostic_error(&mut properties);+ let error = properties["error"]+ .as_str()+ .expect("error must be reported");+ assert!(error.contains("SQLite error (code: 5): database is locked"));+ assert!(!error.contains("[email protected]"));+ assert!(!error.contains("deadbeef"));+ properties["error"] = original["error"].clone();+ assert_eq!(properties, original, "diagnostic context must be preserved");++ // A stalled query need not have returned an error yet.+ properties["error"] = serde_json::Value::Null;+ redact_migration_diagnostic_error(&mut properties);+ assert!(properties["error"].is_null());+ }+#[tokio::test]async fn explicit_retry_waits_for_restart_then_owns_storage_before_later_restarts() {let lifecycle = Arc::new(tokio::sync::Mutex::new(()));diff --git a/crates/screenpipe-db/examples/migration_element_probe.rs b/crates/screenpipe-db/examples/migration_element_probe.rsnew file mode 100644index 000000000..05535a2e5--- /dev/null+++ b/crates/screenpipe-db/examples/migration_element_probe.rs@@ -0,0 +1,116 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++//! Isolated probe for migration selection over privacy-pending element history.+use screenpipe_db::{storage::MigrationOptions, DatabaseManager};+use std::{+ sync::atomic::{AtomicU64, Ordering},+ time::Instant,+};++unsafe extern "C" fn progress(context: *mut std::ffi::c_void) -> i32 {+ let steps = unsafe { &*(context as *const AtomicU64) };+ steps.fetch_add(1000, Ordering::Relaxed);+ 0+}++async fn measure(db: &DatabaseManager, label: &str, sql: &str, first: i64) -> anyhow::Result<()> {+ let steps = Box::new(AtomicU64::new(0));+ let mut conn = db.pool.acquire().await?;+ {+ let mut handle = conn.lock_handle().await?;+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ handle.as_raw_handle().as_ptr(),+ 1000,+ Some(progress),+ (&*steps as *const AtomicU64).cast_mut().cast(),+ );+ }+ }+ let started = Instant::now();+ let result = sqlx::query_as::<_, (i64, i64)>(sqlx::AssertSqlSafe(sql.to_owned()))+ .bind(first)+ .bind(i64::MAX)+ .fetch_all(&mut *conn)+ .await;+ let elapsed = started.elapsed();+ {+ let mut handle = conn.lock_handle().await?;+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ handle.as_raw_handle().as_ptr(),+ 0,+ None,+ std::ptr::null_mut(),+ );+ }+ }+ let rows = result?;+ assert_eq!(rows.len(), 1);+ println!(+ "{label}: rows={} first={} elapsed_ms={} sqlite_steps={}",+ rows.len(),+ rows[0].0,+ elapsed.as_millis(),+ steps.load(Ordering::Relaxed)+ );+ Ok(())+}++#[tokio::main]+async fn main() -> anyhow::Result<()> {+ let count: i64 = std::env::args().nth(1).unwrap_or("100000".into()).parse()?;+ let root = tempfile::tempdir()?;+ let mut options = MigrationOptions::default();+ options.privacy.identity = "sparse-redaction-probe".into();+ options.privacy.required_surfaces = 1;+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options).await?;+ let ready = count / 2;+ // Seed only the resident row source needed by the read-only selection. This+ // does not exercise ingestion, publication, or migration completion.+ let mut tx = db.begin_immediate_with_retry().await?;+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,properties,redacted_at,_archive_generation) SELECT id,1,'accessibility','AXText','pending privacy',?,CASE WHEN id=? THEN 1 ELSE NULL END,1 FROM n")+ .bind(count).bind("properties ".repeat(32)).bind(ready).execute(&mut **tx.conn()).await?;+ tx.commit().await?;+ let text = ["source", "role", "text", "properties"];+ let numeric = [+ "frame_id",+ "parent_id",+ "depth",+ "left_bound",+ "top_bound",+ "width_bound",+ "height_bound",+ "confidence",+ "sort_order",+ "on_screen",+ "redacted_at",+ ];+ let size = text+ .iter()+ .map(|c| format!("COALESCE(length(CAST({c} AS BLOB)),0)"))+ .chain(+ numeric+ .iter()+ .map(|c| format!("CASE WHEN {c} IS NULL THEN 0 ELSE 8 END")),+ )+ .collect::<Vec<_>>()+ .join("+");+ let eligible = "redacted_at IS NOT NULL OR (SELECT required_surfaces=0 FROM storage_metadata) OR (COALESCE(text,'')='' AND properties IS NULL)";+ println!(+ "probe: elements={count} eligible=1 required_surfaces=1 sqlite={}",+ sqlx::query_scalar::<_, String>("SELECT sqlite_version()")+ .fetch_one(&db.pool)+ .await?+ );+ for (label, source, extra) in [+ ("virtual_table", "elements", "1"),+ ("resident_table", "_bulk_element_rows", "_archive_deleted=0"),+ ] {+ let sql = format!("SELECT id,{size} FROM {source} WHERE id BETWEEN ? AND ? AND {extra} AND ({eligible}) ORDER BY id LIMIT 32768");+ measure(&db, label, &sql, ready).await?;+ }+ db.close().await;+ Ok(())+}diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rsindex 2d76307ea..2b5bf79c2 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs@@ -224,6 +224,15 @@ async fn rewrite(let mut files = Vec::new();loop {let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));+ // Report the query bounds before awaiting it; actual batch sizes and+ // row IDs are only available once selection returns.+ crate::storage::diagnostics::batch(+ "elements",+ Some(after.map_or(first, |id: i64| id.saturating_add(1))),+ Some(last),+ None,+ None,+ );crate::storage::diagnostics::stage("selecting_element_batch");let candidates: Vec<(i64, i64)> = sqlx::query_as(sqlx::AssertSqlSafe(format!("SELECT id,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id LIMIT {FILE_ROWS}",diff --git a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rsindex 2b341db28..e2a202540 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs@@ -262,6 +262,7 @@ unsafe impl<'v> VTab<'v> for Elements {pending: VecDeque::new(),pending_sql: String::new(),pending_args: Vec::new(),+ pending_seek_args: Vec::new(),pending_after: None,pending_done: false,descending: false,@@ -540,6 +541,7 @@ struct Cursor {pending: VecDeque<(Record, bool)>,pending_sql: String,pending_args: Vec<SqlValue>,+ pending_seek_args: Vec<usize>,pending_after: Option<i64>,pending_done: bool,descending: bool,@@ -567,6 +569,13 @@ impl Cursor {});let mut args = self.pending_args.clone();if let Some(id) = self.pending_after {+ // This row already satisfied every bound in the scan direction.+ // Advance those bounds too: SQLite may seek using an original+ // >=/<= parameter instead of the appended strict cursor predicate,+ // otherwise rescanning the growing prefix on every 128-row page.+ for &index in &self.pending_seek_args {+ args[index] = SqlValue::Integer(id);+ }args.push(SqlValue::Integer(id));}let sql = format!(@@ -716,6 +725,7 @@ unsafe impl VTabCursor for Cursor {self.pending.clear();self.pending_after = None;self.pending_done = false;+ self.pending_seek_args.clear();self.files_after = None;self.files_done = false;self.archive = None;@@ -769,6 +779,12 @@ unsafe impl VTabCursor for Cursor {} else if matches!(op, ">" | ">=") {range_args.extend([value.clone(), value.clone()]);}+ if matches!(+ (self.descending, op),+ (false, ">" | ">=") | (true, "<" | "<=")+ ) {+ self.pending_seek_args.push(pending_args.len());+ }pending.push(format!("id{op}?"));pending_args.push(value);}diff --git a/crates/screenpipe-db/tests/element_pagination.rs b/crates/screenpipe-db/tests/element_pagination.rsnew file mode 100644index 000000000..ad69561e1--- /dev/null+++ b/crates/screenpipe-db/tests/element_pagination.rs@@ -0,0 +1,170 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use rusqlite::{params_from_iter, types::Value, Connection};+use screenpipe_db::{storage::MigrationOptions, DatabaseManager};+use std::sync::atomic::{AtomicU64, Ordering};++struct Work {+ steps: AtomicU64,+ limit: u64,+}++unsafe extern "C" fn count_steps(context: *mut std::ffi::c_void) -> i32 {+ let work = unsafe { &*(context as *const Work) };+ i32::from(work.steps.fetch_add(1000, Ordering::Relaxed) + 1000 > work.limit)+}++struct ProgressGuard<'a> {+ db: &'a Connection,+ work: Box<Work>,+}++impl Drop for ProgressGuard<'_> {+ fn drop(&mut self) {+ // Remove the callback before its context or borrowed connection dies.+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ self.db.handle(),+ 0,+ None,+ std::ptr::null_mut(),+ );+ }+ }+}++fn ids(db: &Connection, sql: &str, args: &[Value]) -> rusqlite::Result<Vec<i64>> {+ db.prepare(sql)?+ .query_map(params_from_iter(args), |r| r.get(0))?+ .collect()+}++#[tokio::test]+async fn sparse_element_selection_has_linear_work_in_both_directions() {+ let root = tempfile::tempdir().unwrap();+ let mut options = MigrationOptions::default();+ options.privacy.identity = "pagination-regression".into();+ options.privacy.required_surfaces = 1;+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)+ .await+ .unwrap();+ let count = 100_000_i64;+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ // Isolate the read cursor: one eligible row in a large resident backlog.+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO _bulk_element_rows(id,frame_id,source,role,text,redacted_at,_archive_generation) SELECT id,1,'accessibility','AXText','pending privacy',CASE WHEN id=? THEN 1 ELSE NULL END,1 FROM n")+ .bind(count).bind(count / 2).execute(&mut **tx.conn()).await.unwrap();+ tx.commit().await.unwrap();+ {+ let mut conn = db.pool.acquire().await.unwrap();+ let mut handle = conn.lock_handle().await.unwrap();+ // The SQLx worker is paused while this borrowed handle is used.+ let sqlite = unsafe { Connection::from_handle(handle.as_raw_handle().as_ptr()) }.unwrap();+ for direction in ["ASC", "DESC"] {+ // Count SQLite VM work, including nested virtual-table statements,+ // instead of asserting machine-dependent timing. Interrupt a+ // quadratic regression before it can hang CI on a larger fixture.+ let guard = ProgressGuard {+ db: &sqlite,+ work: Box::new(Work {+ steps: AtomicU64::new(0),+ limit: count as u64 * 200,+ }),+ };+ unsafe {+ libsqlite3_sys::sqlite3_progress_handler(+ sqlite.handle(),+ 1000,+ Some(count_steps),+ (&*guard.work as *const Work).cast_mut().cast(),+ );+ }+ let selected = ids(&sqlite, &format!("SELECT id FROM elements WHERE id BETWEEN ? AND ? AND (redacted_at IS NOT NULL OR (SELECT required_surfaces=0 FROM storage_metadata) OR (COALESCE(text,'')='' AND properties IS NULL)) ORDER BY id {direction} LIMIT 32768"), &[1.into(), count.into()]);+ let work = guard.work.steps.load(Ordering::Relaxed);+ drop(guard);+ assert_eq!(+ selected.unwrap_or_else(|error| {+ panic!("{direction} selection failed at {work} steps: {error}")+ }),+ vec![count / 2]+ );+ }+ }+ db.close().await;+}++async fn assert_bounds(db: &DatabaseManager, reference: &Connection) {+ let mut conn = db.pool.acquire().await.unwrap();+ let mut handle = conn.lock_handle().await.unwrap();+ let sqlite = unsafe { Connection::from_handle(handle.as_raw_handle().as_ptr()) }.unwrap();+ let cases: Vec<(&str, Vec<Value>)> = vec![+ ("1", vec![]),+ ("id>=? AND id<=?", vec![(-250).into(), 350.into()]),+ (+ "id>=? AND id>? AND id<=? AND id<?",+ vec![(-500).into(), (-250).into(), 400.into(), 350.into()],+ ),+ (+ "id>=? AND id<=?",+ vec![Value::Text("-250".into()), Value::Text("350".into())],+ ),+ (+ "id>? AND id<?",+ vec![Value::Real(-250.5), Value::Real(350.5)],+ ),+ ("id<?", vec![Value::Text("not a number".into())]),+ ("id>?", vec![Value::Text("not a number".into())]),+ ("id>=?", vec![Value::Null]),+ ("id=?", vec![Value::Text("7".into())]),+ ("id>?", vec![i64::MAX.into()]),+ ("id<?", vec![i64::MIN.into()]),+ (+ "id>=? AND id<=? AND frame_id=?",+ vec![(-400).into(), 400.into(), 1.into()],+ ),+ ];+ for (condition, args) in cases {+ for direction in ["ASC", "DESC"] {+ let sql = format!("SELECT id FROM elements WHERE {condition} ORDER BY id {direction}");+ assert_eq!(+ ids(&sqlite, &sql, &args).unwrap(),+ ids(reference, &sql, &args).unwrap(),+ "{sql}, {args:?}"+ );+ }+ }+}++#[tokio::test]+async fn advancing_pages_preserve_bounds_affinity_and_mixed_archives() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ db.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp) VALUES(1,'2026-09-18'),(2,'2026-09-18')",+ )+ .await+ .unwrap();+ let reference = Connection::open_in_memory().unwrap();+ reference.execute_batch("CREATE TABLE elements(id INTEGER PRIMARY KEY,frame_id INTEGER,source TEXT,role TEXT,text TEXT)").unwrap();+ let seed = "WITH RECURSIVE n(id) AS (VALUES(-512) UNION ALL SELECT id+1 FROM n WHERE id<512) INSERT INTO elements(id,frame_id,source,role,text) SELECT id,1+abs(id%2),'accessibility','AXText','page test' FROM n; INSERT INTO elements(id,frame_id,source,role,text) VALUES(-9223372036854775808,1,'accessibility','AXText','minimum'),(9223372036854775807,1,'accessibility','AXText','maximum');";+ db.execute_raw_sql_write(seed).await.unwrap();+ reference.execute_batch(seed).unwrap();+ assert_bounds(&db, &reference).await;+ while db.seal_payloads().await.unwrap() != 0 {}+ let changes = "UPDATE elements SET text='edited' WHERE id%3=0; DELETE FROM elements WHERE id%7=0; INSERT INTO elements(id,frame_id,source,role,text) VALUES(777,2,'accessibility','AXText','new capture');";+ db.execute_raw_sql_write(changes).await.unwrap();+ reference.execute_batch(changes).unwrap();+ assert_bounds(&db, &reference).await;+ db.verify_storage().await.unwrap();+ db.close().await;+ let reopened = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ assert_bounds(&reopened, &reference).await;+ reopened.close().await;+}diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rsindex c927318de..f2888a940 100644--- a/crates/screenpipe-db/tests/in_place_migration.rs+++ b/crates/screenpipe-db/tests/in_place_migration.rs@@ -626,6 +626,92 @@ async fn privacy_pending_payloads_remain_resident_and_searchable() {db.close().await;}+#[tokio::test]+async fn sparse_privacy_backlog_completes_migration_and_preserves_recording() {+ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("db.sqlite");+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ let count = 100_000_i64;+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-18T12:00:00Z','pending frame privacy')").await.unwrap();+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO elements(id,frame_id,source,role,text,redacted_at) SELECT id,1,'accessibility','AXText','pending element history '||id,CASE WHEN id=? THEN 1 ELSE NULL END FROM n")+ .bind(count).bind(count / 2).execute(&mut **tx.conn()).await.unwrap();+ tx.commit().await.unwrap();+ db.close().await;+ let mut options = MigrationOptions::default();+ options.privacy.identity = "sparse-privacy-migration".into();+ options.privacy.required_surfaces = 1;+ let report = migrate(root.path(), Default::default(), options)+ .await+ .unwrap();+ assert_eq!(+ report+ .tables+ .iter()+ .find(|t| t.table == "elements")+ .unwrap()+ .rows,+ count as u64+ );+ assert!(root+ .path()+ .join("storage-migration-complete.json")+ .is_file());+ assert!(!root.path().join("storage-migration.json").exists());++ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM _bulk_element_rows")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ count - 1+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT sum(rows) FROM _bulk_files WHERE table_name='elements' AND state='published'"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 1+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM elements_fts WHERE elements_fts MATCH 'history'"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ count+ );+ db.verify_storage().await.unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-18T13:00:00Z','new durable capture'); INSERT INTO elements(id,frame_id,source,role,text) VALUES(100001,2,'accessibility','AXText','new capture element')").await.unwrap();+ db.close().await;+ let reopened = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM elements")+ .fetch_one(&reopened.pool)+ .await+ .unwrap(),+ count + 1+ );+ assert_eq!(+ sqlx::query_scalar::<_, String>("SELECT text FROM elements WHERE id=100001")+ .fetch_one(&reopened.pool)+ .await+ .unwrap(),+ "new capture element"+ );+ reopened.close().await;+}+#[tokio::test]#[ignore = "requires a marked disposable filesystem without hole punching"]async fn unsupported_volume_fails_before_conversion() {diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 108c44929..29c415321 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -8,11 +8,11 @@ confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y- Mapped suites: 33-- Mapped Rust files: 358-- Active test blocks: 3520+- Mapped Rust files: 359+- Active test blocks: 3523- Ignored/manual test blocks: 147-- Declared test blocks: 3667-- Weighted coverage points: 2906.1+- Declared test blocks: 3670+- Weighted coverage points: 2909.1Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3377 | 141 | 2841.0 | 21 | 11 | 100% |-| macos | 30 | 3439 | 121 | 2855.0 | 22 | 11 | 100% |-| linux | 26 | 3007 | 113 | 2500.3 | 20 | 11 | 100% |+| windows | 30 | 3380 | 141 | 2844.0 | 21 | 11 | 100% |+| macos | 30 | 3442 | 121 | 2858.0 | 22 | 11 | 100% |+| linux | 26 | 3010 | 113 | 2503.3 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |-| screenpipe-db | 5 | 58 | 23 | 544 | 23 | 517.3 | 9 |+| screenpipe-db | 5 | 59 | 23 | 547 | 23 | 520.3 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 618 | 44 | 543.2 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 7 suites / 731 active / 45 ignored / 656.2 pts | 7 suites / 731 active / 45 ignored / 656.2 pts | 6 suites / 656 active / 44 ignored / 603.7 pts || audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |-| database | 6 suites / 492 active / 19 ignored / 465.3 pts | 6 suites / 492 active / 19 ignored / 465.3 pts | 6 suites / 492 active / 19 ignored / 465.3 pts |+| database | 6 suites / 495 active / 19 ignored / 468.3 pts | 6 suites / 495 active / 19 ignored / 468.3 pts | 6 suites / 495 active / 19 ignored / 468.3 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts || engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts || local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts || meeting | 6 suites / 1624 active / 20 ignored / 1325.8 pts | 6 suites / 1624 active / 20 ignored / 1325.8 pts | 4 suites / 1287 active / 16 ignored / 1011.3 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1547 active / 74 ignored / 1367.2 pts | 14 suites / 1654 active / 78 ignored / 1410.0 pts | 13 suites / 1547 active / 74 ignored / 1367.2 pts |+| performance | 13 suites / 1550 active / 74 ignored / 1370.2 pts | 14 suites / 1657 active / 78 ignored / 1413.0 pts | 13 suites / 1550 active / 74 ignored / 1370.2 pts || pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts || privacy | 5 suites / 942 active / 37 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts || real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - || speaker | 2 suites / 368 active / 9 ignored / 368.0 pts | 2 suites / 368 active / 9 ignored / 368.0 pts | 2 suites / 368 active / 9 ignored / 368.0 pts |-| storage | 4 suites / 609 active / 36 ignored / 503.1 pts | 4 suites / 609 active / 36 ignored / 503.1 pts | 4 suites / 609 active / 36 ignored / 503.1 pts |+| storage | 4 suites / 612 active / 36 ignored / 506.1 pts | 4 suites / 612 active / 36 ignored / 506.1 pts | 4 suites / 612 active / 36 ignored / 506.1 pts || sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |-| timeline | 4 suites / 1156 active / 40 ignored / 943.3 pts | 4 suites / 1156 active / 40 ignored / 943.3 pts | 4 suites / 1156 active / 40 ignored / 943.3 pts |+| timeline | 4 suites / 1159 active / 40 ignored / 946.3 pts | 4 suites / 1159 active / 40 ignored / 946.3 pts | 4 suites / 1159 active / 40 ignored / 946.3 pts || transcription | 5 suites / 808 active / 41 ignored / 633.3 pts | 5 suites / 808 active / 41 ignored / 633.3 pts | 5 suites / 808 active / 41 ignored / 633.3 pts || ui-events | 4 suites / 758 active / 29 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts || vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 31 | 256 | 10 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 32 | 259 | 10 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |@@ -298,13 +298,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-accessibility-ui-events | screenpipe-db | tests/db_config_test.rs | integration | 5 | 0 | 5 || db-timeline-frames | screenpipe-db | tests/db.rs | integration | 42 | 0 | 42 || db-accessibility-ui-events | screenpipe-db | tests/display_layout_test.rs | integration | 4 | 0 | 4 |+| db-timeline-frames | screenpipe-db | tests/element_pagination.rs | integration | 2 | 0 | 2 || db-timeline-frames | screenpipe-db | tests/evict_media_null_chunk_test.rs | integration | 2 | 0 | 2 || db-timeline-frames | screenpipe-db | tests/frame_offset_sync_test.rs | integration | 6 | 0 | 6 || db-search-indexing | screenpipe-db | tests/fts_contention_test.rs | integration | 4 | 0 | 4 || db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 || db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 || db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 15 | 4 | 19 |+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 16 | 4 | 20 || db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 || db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 || db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex 527fafa25..0007b2eec 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -677,6 +677,7 @@"src/write_queue.rs","tests/bulk_storage.rs","tests/db.rs",+ "tests/element_pagination.rs","tests/evict_media_null_chunk_test.rs","tests/frame_offset_sync_test.rs","tests/hybrid_storage.rs",