Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

perf: speed up full-history migration verification (#7131)

Ezra Ellettee4c96aee00de
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index 184a700fd..177e41350 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -742,6 +742,37 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn parity_scan_failure_reaches_support_without_rolling_logs() {
+ use screenpipe_db::{storage, DatabaseManager};
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ // Valid SQLite with a source shape the existing migration cannot read:
+ // the actual receipt SELECT fails after integrity verification passes.
+ db.execute_raw_sql_write("CREATE TABLE parity_probe(key TEXT PRIMARY KEY,payload TEXT) WITHOUT ROWID; INSERT INTO parity_probe VALUES('key','private history')")
+ .await.unwrap();
+ db.close().await;
+ let error = storage::migrate(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap_err();
+ assert!(error.to_string().contains("no such column: rowid"));
+ assert_migration_failure_uploaded(
+ root.path(),
+ &[
+ "scanning_parity_rows",
+ "no such column: rowid",
+ "\"table\": \"parity_probe\"",
+ "\"status\": \"failed\"",
+ ],
+ )
+ .await;
+ }
+
#[tokio::test]
async fn element_selection_failure_reaches_support_without_rolling_logs() {
use screenpipe_db::{storage, DatabaseManager};
diff --git a/crates/screenpipe-db/src/storage/diagnostics.rs b/crates/screenpipe-db/src/storage/diagnostics.rs
index 0782cf6ff..88e87a787 100644
--- a/crates/screenpipe-db/src/storage/diagnostics.rs
+++ b/crates/screenpipe-db/src/storage/diagnostics.rs
@@ -110,6 +110,16 @@ pub fn app_version(version: String) {
});
}
+/// Native SQLite scans run on a blocking worker, outside Tokio's task-local
+/// scope. Keep their progress and originating failure in the same attempt.
+pub(super) fn blocking_scope<T>(work: impl FnOnce() -> T + Send) -> impl FnOnce() -> T + Send {
+ let run = ACTIVE.try_with(Arc::clone).ok();
+ move || match run {
+ Some(run) => ACTIVE.sync_scope(run, work),
+ None => work(),
+ }
+}
+
pub(super) fn source_state(state: SourceState) {
let _ = ACTIVE.try_with(|run| {
run.lock()
diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rs
index 2e798dc79..bfff87a7d 100644
--- a/crates/screenpipe-db/src/storage/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/lifecycle.rs
@@ -16,6 +16,151 @@ use std::path::{Path, PathBuf};
mod tests {
use super::*;
+ // Frozen pre-optimization receipt reader/codec: an independent oracle for
+ // old journals and the paired benchmark, never used by production.
+ async fn legacy_receipt(db: &DatabaseManager, table: &str) -> TableParity {
+ let mut columns: Vec<String> =
+ sqlx::query_scalar("SELECT name FROM pragma_table_info(?) ORDER BY cid")
+ .bind(table)
+ .fetch_all(&db.pool)
+ .await
+ .unwrap();
+ columns.retain(|column| !column.starts_with("_archive_"));
+ let mut hash = Sha256::new();
+ let mut count = 0;
+ let mut last = i64::MIN;
+ let mut first = true;
+ loop {
+ let sql = format!(
+ "SELECT rowid,{} FROM {} WHERE rowid{}? AND rowid<=? ORDER BY rowid LIMIT 128",
+ columns
+ .iter()
+ .map(|c| quote(c))
+ .collect::<Vec<_>>()
+ .join(","),
+ quote(table),
+ if first { ">=" } else { ">" }
+ );
+ super::super::diagnostics::batch(table, Some(last), None, Some(count), None);
+ super::super::diagnostics::stage("reading_parity_rows");
+ let rows = sqlx::query(sqlx::AssertSqlSafe(sql))
+ .bind(last)
+ .bind(i64::MAX)
+ .fetch_all(&db.pool)
+ .await
+ .unwrap();
+ if rows.is_empty() {
+ break;
+ }
+ first = false;
+ super::super::diagnostics::stage("hashing_parity_rows");
+ for row in rows {
+ last = row.get(0);
+ for index in 0..columns.len() {
+ legacy_hash_value(&mut hash, &row, index + 1, None).unwrap();
+ }
+ hash.update(b"E");
+ count += 1;
+ }
+ }
+ TableParity {
+ table: table.into(),
+ rows: count,
+ sha256: format!("{:x}", hash.finalize()),
+ }
+ }
+
+ fn legacy_hash_value(
+ hash: &mut Sha256,
+ row: &sqlx::sqlite::SqliteRow,
+ index: usize,
+ replacement: Option<&Option<String>>,
+ ) -> Result<(), sqlx::Error> {
+ if let Some(value) = replacement {
+ if let Some(value) = value {
+ hash.update(b"T");
+ hash.update((value.len() as u64).to_le_bytes());
+ hash.update(value.as_bytes());
+ } else {
+ hash.update(b"N");
+ }
+ return Ok(());
+ }
+ let raw = row.try_get_raw(index)?;
+ if raw.is_null() {
+ hash.update(b"N");
+ return Ok(());
+ }
+ match raw.type_info().name() {
+ "INTEGER" => {
+ hash.update(b"I");
+ hash.update(row.try_get::<i64, _>(index)?.to_le_bytes());
+ }
+ "REAL" => {
+ hash.update(b"R");
+ hash.update(row.try_get::<f64, _>(index)?.to_bits().to_le_bytes());
+ }
+ "TEXT" => {
+ let bytes: Vec<u8> = row.try_get(index)?;
+ hash.update(b"T");
+ hash.update((bytes.len() as u64).to_le_bytes());
+ hash.update(bytes);
+ }
+ _ => {
+ let bytes: Vec<u8> = row.try_get(index)?;
+ hash.update(b"B");
+ hash.update((bytes.len() as u64).to_le_bytes());
+ hash.update(bytes);
+ }
+ }
+ Ok(())
+ }
+
+ #[tokio::test]
+ #[ignore = "paired migration verification throughput benchmark on an isolated million-row history"]
+ async fn parity_throughput() {
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ // Use the actual legacy element column layout, without capture/FTS
+ // triggers in fixture setup. Both implementations hash the same cells.
+ db.execute_raw_sql_write("CREATE TABLE parity_benchmark AS SELECT * FROM elements WHERE 0; CREATE UNIQUE INDEX parity_benchmark_id ON parity_benchmark(id);").await.unwrap();
+ let rows = 1_000_000_i64;
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO parity_benchmark(id,frame_id,source,role,text,properties) SELECT id,id/100,'accessibility','AXText','searchable historical element '||id,'{\"label\":\"東京 history\",\"enabled\":true}' FROM n")
+ .bind(rows).execute(&mut **tx.conn()).await.unwrap();
+ tx.commit().await.unwrap();
+ let tables = [TableParity {
+ table: "parity_benchmark".into(),
+ rows: 0,
+ sha256: String::new(),
+ }];
+ for run in 1..=3 {
+ let mut receipts = Vec::new();
+ for legacy in if run % 2 == 1 {
+ [true, false]
+ } else {
+ [false, true]
+ } {
+ let started = std::time::Instant::now();
+ let receipt = if legacy {
+ legacy_receipt(&db, "parity_benchmark").await
+ } else {
+ table_receipts(&db, Some(&tables)).await.unwrap().remove(0)
+ };
+ assert_eq!(receipt.rows, rows as u64);
+ eprintln!("parity benchmark: run={run} legacy={legacy} rows={rows} elapsed_ms={:.3} sha256={}", started.elapsed().as_secs_f64()*1000.0, receipt.sha256);
+ receipts.push(receipt);
+ }
+ assert_eq!(receipts[0], receipts[1]);
+ }
+ db.close().await;
+ }
+
#[tokio::test]
async fn parity_includes_domain_payload_columns_and_minimum_rowids() {
let root = tempfile::tempdir().unwrap();
@@ -43,6 +188,43 @@ mod tests {
);
db.close().await;
}
+
+ #[tokio::test]
+ async fn streaming_parity_matches_existing_receipts_for_every_storage_class() {
+ for encoding in ["UTF-8", "UTF-16le"] {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("db.sqlite");
+ {
+ let db = rusqlite::Connection::open(&path).unwrap();
+ db.execute_batch(&format!(
+ "PRAGMA encoding='{encoding}'; CREATE TABLE encoding_probe(n INTEGER);"
+ ))
+ .unwrap();
+ }
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ let table = "receipt \"probe";
+ db.execute_raw_sql_write(&format!("CREATE TABLE {}(id INTEGER PRIMARY KEY,untyped,\"text value\" TEXT,bytes BLOB,number REAL,flag BOOLEAN,_archive_ignored TEXT); CREATE TABLE empty_receipt(id INTEGER PRIMARY KEY,payload TEXT);",quote(table))).await.unwrap();
+ db.execute_raw_sql_write(&format!("INSERT INTO {} VALUES(-9223372036854775808,NULL,'',X'',NULL,0,'ignored'),(-2,1,'東京'||char(0)||'tail',X'0001FF',1.25,1,'ignored'),(7,'1',CAST(X'80FF' AS TEXT),NULL,-1e200,NULL,'ignored'),(9223372036854775807,X'31',NULL,X'00',0.0,1,'ignored');",quote(table))).await.unwrap();
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query(sqlx::AssertSqlSafe(format!("WITH RECURSIVE n(id) AS (VALUES(10) UNION ALL SELECT id+1 FROM n WHERE id<9000) INSERT INTO {}(id,untyped,\"text value\",bytes) SELECT id,id,'retained history '||id,zeroblob(128) FROM n", quote(table))))
+ .execute(&mut **tx.conn()).await.unwrap();
+ tx.commit().await.unwrap();
+ for table in [table, "empty_receipt"] {
+ let expected = legacy_receipt(&db, table).await;
+ let actual = table_receipts(&db, Some(std::slice::from_ref(&expected)))
+ .await
+ .unwrap();
+ assert_eq!(
+ actual,
+ vec![expected],
+ "{encoding}: old journals must remain usable"
+ );
+ }
+ db.close().await;
+ }
+ }
}
#[derive(Debug, Clone, Default)]
@@ -1181,43 +1363,25 @@ pub(super) fn hash_value(
index: usize,
replacement: Option<&Option<String>>,
) -> Result<(), sqlx::Error> {
- if let Some(value) = replacement {
- if let Some(value) = value {
- hash.update(b"T");
- hash.update((value.len() as u64).to_le_bytes());
- hash.update(value.as_bytes());
+ use rusqlite::types::ValueRef as Cell;
+ let value = if let Some(value) = replacement {
+ value
+ .as_ref()
+ .map_or(Cell::Null, |text| Cell::Text(text.as_bytes()))
+ } else {
+ let raw = row.try_get_raw(index)?;
+ if raw.is_null() {
+ Cell::Null
} else {
- hash.update(b"N");
- }
- return Ok(());
- }
- let raw = row.try_get_raw(index)?;
- if raw.is_null() {
- hash.update(b"N");
- return Ok(());
- }
- match raw.type_info().name() {
- "INTEGER" => {
- hash.update(b"I");
- hash.update(row.try_get::<i64, _>(index)?.to_le_bytes());
- }
- "REAL" => {
- hash.update(b"R");
- hash.update(row.try_get::<f64, _>(index)?.to_bits().to_le_bytes());
- }
- "TEXT" => {
- let bytes: Vec<u8> = row.try_get(index)?;
- hash.update(b"T");
- hash.update((bytes.len() as u64).to_le_bytes());
- hash.update(bytes);
- }
- _ => {
- let bytes: Vec<u8> = row.try_get(index)?;
- hash.update(b"B");
- hash.update((bytes.len() as u64).to_le_bytes());
- hash.update(bytes);
+ match raw.type_info().name() {
+ "INTEGER" => Cell::Integer(row.try_get(index)?),
+ "REAL" => Cell::Real(row.try_get(index)?),
+ "TEXT" => Cell::Text(row.try_get(index)?),
+ _ => Cell::Blob(row.try_get(index)?),
+ }
}
- }
+ };
+ super::parity::hash_cell(hash, value);
Ok(())
}
@@ -1314,6 +1478,22 @@ pub(super) async fn table_receipts(
} else {
"rowid"
};
+ if table != "frames" || db.storage.is_none() {
+ let sql = format!(
+ "SELECT {key},{} FROM {} ORDER BY {key}",
+ columns
+ .iter()
+ .map(|column| quote(column))
+ .collect::<Vec<_>>()
+ .join(","),
+ quote(&table),
+ );
+ let receipt = super::parity::scan(db.pool.clone(), table.clone(), sql).await?;
+ tracing::info!(table=%table, rows=receipt.rows, "storage parity receipt complete");
+ result.push(receipt);
+ continue;
+ }
+ // Hybrid frames still hydrate sealed payloads in byte-bounded batches.
let mut hash = Sha256::new();
let mut count = 0;
let mut last = i64::MIN;
diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rs
index f25c650e4..56dd11f31 100644
--- a/crates/screenpipe-db/src/storage/mod.rs
+++ b/crates/screenpipe-db/src/storage/mod.rs
@@ -16,6 +16,7 @@ mod in_place;
mod inventory;
mod lifecycle;
mod maintenance;
+mod parity;
pub(crate) mod read_schema;
mod reader;
mod reclaim;
diff --git a/crates/screenpipe-db/src/storage/parity.rs b/crates/screenpipe-db/src/storage/parity.rs
new file mode 100644
index 000000000..1358e1297
--- /dev/null
+++ b/crates/screenpipe-db/src/storage/parity.rs
@@ -0,0 +1,160 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Offline logical receipts borrow SQLite values instead of copying every cell
+//! through SQLx. The existing read connection, statement snapshot and typed
+//! SHA-256 format are retained; memory does not grow with the history length.
+
+use super::{diagnostics, lifecycle::TableParity, storage_error};
+use rusqlite::types::ValueRef;
+use sha2::{Digest, Sha256};
+use sqlx::{sqlite::LockedSqliteHandle, SqlitePool};
+use tokio_util::sync::CancellationToken;
+
+// Bounds diagnostic overhead, not which records are verified.
+const PROGRESS_ROWS: u64 = 4096;
+
+struct ScanHandle<'a>(LockedSqliteHandle<'a>);
+
+impl Drop for ScanHandle<'_> {
+ fn drop(&mut self) {
+ // Also runs on query errors and unwinding. Never return a cancelled
+ // progress handler to another borrower of the pool connection.
+ self.0.remove_progress_handler();
+ }
+}
+
+pub(super) async fn scan(
+ pool: SqlitePool,
+ table: String,
+ sql: String,
+) -> Result<TableParity, sqlx::Error> {
+ let cancelled = CancellationToken::new();
+ let _cancel_on_drop = cancelled.clone().drop_guard();
+ tokio::task::spawn_blocking(diagnostics::blocking_scope(move || {
+ tokio::runtime::Handle::current().block_on(async move {
+ let mut conn = tokio::select! {
+ connection = pool.acquire() => connection?,
+ _ = cancelled.cancelled() => return Err(storage_error("parity scan cancelled")),
+ };
+ let mut handle = ScanHandle(conn.lock_handle().await?);
+ handle.0.set_progress_handler(
+ crate::cancellable_query::SQLITE_PROGRESS_CHECK_OPS,
+ move || !cancelled.is_cancelled(),
+ );
+ diagnostics::stage("scanning_parity_rows");
+ // SAFETY: SQLx's handle lock excludes its worker for the entire
+ // scan. from_handle borrows the database and never closes it. The
+ // statement/values are dropped before the lock and pool connection.
+ let database =
+ unsafe { rusqlite::Connection::from_handle(handle.0.as_raw_handle().as_ptr()) }
+ .map_err(sqlite_error)?;
+ let mut statement = database.prepare(&sql).map_err(sqlite_error)?;
+ let columns = statement.column_count();
+ let mut rows = statement.query([]).map_err(sqlite_error)?;
+ let mut hash = Sha256::new();
+ let mut count = 0;
+ let mut last = None;
+ while let Some(row) = rows.next().map_err(sqlite_error)? {
+ last = Some(row.get::<_, i64>(0).map_err(sqlite_error)?);
+ for column in 1..columns {
+ hash_cell(&mut hash, row.get_ref(column).map_err(sqlite_error)?);
+ }
+ hash.update(b"E");
+ count += 1;
+ if count % PROGRESS_ROWS == 0 {
+ diagnostics::batch(&table, last, None, Some(count), None);
+ }
+ }
+ diagnostics::batch(&table, last, None, Some(count), None);
+ Ok(TableParity {
+ table,
+ rows: count,
+ sha256: format!("{:x}", hash.finalize()),
+ })
+ })
+ }))
+ .await
+ .map_err(storage_error)?
+}
+
+fn sqlite_error(error: rusqlite::Error) -> sqlx::Error {
+ if let rusqlite::Error::SqliteFailure(code, _) = &error {
+ // Preserve extended SQLite codes for the normal hard-fault classifier.
+ storage_error(format!("(code: {}) {error}", code.extended_code))
+ } else {
+ storage_error(error)
+ }
+}
+
+/// This encoding is persisted in existing migration journals. Keep NULL,
+/// storage class, byte length, floating-point bits and row order identical.
+pub(super) fn hash_cell(hash: &mut Sha256, value: ValueRef<'_>) {
+ match value {
+ ValueRef::Null => hash.update(b"N"),
+ ValueRef::Integer(value) => {
+ hash.update(b"I");
+ hash.update(value.to_le_bytes());
+ }
+ ValueRef::Real(value) => {
+ hash.update(b"R");
+ hash.update(value.to_bits().to_le_bytes());
+ }
+ ValueRef::Text(bytes) | ValueRef::Blob(bytes) => {
+ hash.update(if matches!(value, ValueRef::Text(_)) {
+ b"T"
+ } else {
+ b"B"
+ });
+ hash.update((bytes.len() as u64).to_le_bytes());
+ hash.update(bytes);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::time::Duration;
+
+ #[tokio::test]
+ async fn cancelled_scan_stops_sqlite_and_returns_a_clean_connection() {
+ let pool = sqlx::sqlite::SqlitePoolOptions::new()
+ .max_connections(1)
+ .connect("sqlite::memory:")
+ .await
+ .unwrap();
+ let task = tokio::spawn(scan(pool.clone(), "cancel_probe".into(),
+ "WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<100000000) SELECT id,id FROM n ORDER BY id DESC".into()));
+ tokio::time::timeout(Duration::from_secs(3), async {
+ while pool.num_idle() != 0 {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ // The sort is still inside sqlite3_step before yielding its first row.
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ task.abort();
+ let _ = task.await;
+ let answer: i64 = tokio::time::timeout(Duration::from_secs(3),
+ sqlx::query_scalar("WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<10000) SELECT sum(x) FROM n").fetch_one(&pool))
+ .await.unwrap().unwrap();
+ assert_eq!(answer, 50_005_000);
+ pool.close().await;
+ }
+
+ #[test]
+ fn read_failure_keeps_its_sqlite_hard_fault_code() {
+ let code = libsqlite3_sys::SQLITE_IOERR_READ;
+ let error = sqlite_error(rusqlite::Error::SqliteFailure(
+ libsqlite3_sys::Error::new(code),
+ Some("source read failed".into()),
+ ));
+ assert_eq!(
+ crate::sqlite_error::sqlite_hard_fault_code(&error),
+ Some(code)
+ );
+ assert!(error.to_string().contains("source read failed"));
+ }
+}
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 176566bc1..0f177d34c 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -37,6 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 34
+- Mapped Rust files: 363
+- Active test blocks: 3555
+- Ignored/manual test blocks: 150
+- Weighted coverage points: 2939.3
+
+| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
+| --- | --- | --- | --- | --- | --- | --- | --- |
+| windows | 31 | 3412 | 144 | 2874.2 | 21 | 11 | 100% |
+| macos | 31 | 3474 | 124 | 2888.2 | 22 | 11 | 100% |
+| linux | 27 | 3034 | 116 | 2525.5 | 20 | 11 | 100% |
- Mapped Rust files: 362
- Active test blocks: 3556
- Ignored/manual test blocks: 149
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index df120119f..b9dbd2022 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -32,7 +32,7 @@ are explicitly enabled in a runtime lane.
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 11 | 20 | 120 | 1732 | 43 | 1335.9 | 10 |
-| screenpipe-db | 5 | 60 | 23 | 558 | 24 | 531.3 | 9 |
+| screenpipe-db | 5 | 60 | 24 | 561 | 25 | 534.3 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 625 | 44 | 549.6 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -66,7 +66,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 8 suites / 741 active / 45 ignored / 665.6 pts | 8 suites / 741 active / 45 ignored / 665.6 pts | 7 suites / 666 active / 44 ignored / 613.1 pts |
| audio-device | 2 suites / 222 active / 7 ignored / 199.5 pts | 2 suites / 222 active / 7 ignored / 199.5 pts | 1 suites / 147 active / 6 ignored / 147.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 7 suites / 509 active / 20 ignored / 482.3 pts | 7 suites / 509 active / 20 ignored / 482.3 pts | 7 suites / 509 active / 20 ignored / 482.3 pts |
+| database | 7 suites / 512 active / 21 ignored / 485.3 pts | 7 suites / 512 active / 21 ignored / 485.3 pts | 7 suites / 512 active / 21 ignored / 485.3 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts |
| local-api | 3 suites / 433 active / 10 ignored / 305.5 pts | 3 suites / 433 active / 10 ignored / 305.5 pts | 3 suites / 433 active / 10 ignored / 305.5 pts |
@@ -78,9 +78,9 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| privacy | 5 suites / 942 active / 37 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts |
| real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - |
| speaker | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts |
-| storage | 4 suites / 624 active / 37 ignored / 517.8 pts | 4 suites / 624 active / 37 ignored / 517.8 pts | 4 suites / 624 active / 37 ignored / 517.8 pts |
+| storage | 4 suites / 627 active / 38 ignored / 520.8 pts | 4 suites / 627 active / 38 ignored / 520.8 pts | 4 suites / 627 active / 38 ignored / 520.8 pts |
| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
-| timeline | 4 suites / 1173 active / 42 ignored / 959.4 pts | 4 suites / 1173 active / 42 ignored / 959.4 pts | 4 suites / 1173 active / 42 ignored / 959.4 pts |
+| timeline | 4 suites / 1176 active / 43 ignored / 962.4 pts | 4 suites / 1176 active / 43 ignored / 962.4 pts | 4 suites / 1176 active / 43 ignored / 962.4 pts |
| transcription | 5 suites / 814 active / 42 ignored / 637.8 pts | 5 suites / 814 active / 42 ignored / 637.8 pts | 5 suites / 814 active / 42 ignored / 637.8 pts |
| ui-events | 4 suites / 758 active / 29 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |
| vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |
@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 33 | 270 | 11 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 34 | 273 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 411 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -280,7 +280,8 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |
| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 8 | 0 | 8 |
| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 |
-| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 |
+| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 2 | 1 | 3 |
+| db-timeline-frames | screenpipe-db | src/storage/parity.rs | source | 2 | 0 | 2 |
| db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |
| db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 3 | 1 | 4 |
| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 3 | 1 | 4 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index 0ed1f2f39..808f9e4fe 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -680,6 +680,7 @@
"src/storage/bulk/tests.rs",
"src/storage/diagnostics.rs",
"src/storage/lifecycle.rs",
+ "src/storage/parity.rs",
"src/storage/reclaim.rs",
"src/storage/schema.rs",
"src/storage/sealing.rs",