Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): avoid repeated scans during Parquet migration (#7027)

Ezra Ellettea4be5ca79d71
diff --git a/COVERAGE.md b/COVERAGE.md
index 1b79507af..ca35dbcfb 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -36,15 +36,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 32
- Mapped Rust files: 351
-- Active test blocks: 3476
-- Ignored/manual test blocks: 142
-- Weighted coverage points: 2866.3
+- Active test blocks: 3478
+- Ignored/manual test blocks: 144
+- Weighted coverage points: 2868.3
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 3333 | 136 | 2801.2 | 21 | 11 | 100% |
-| macos | 29 | 3395 | 117 | 2815.2 | 22 | 11 | 100% |
-| linux | 25 | 2963 | 109 | 2460.5 | 20 | 11 | 100% |
+| windows | 29 | 3335 | 138 | 2803.2 | 21 | 11 | 100% |
+| macos | 29 | 3397 | 119 | 2817.2 | 22 | 11 | 100% |
+| linux | 25 | 2965 | 111 | 2462.5 | 20 | 11 | 100% |
## Refresh
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
index 09fb714de..fd940947f 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
@@ -825,10 +825,17 @@ unsafe impl VTabCursor for Cursor {
_ => (),
}
let (path, hash) = self.archive_location.as_ref().unwrap();
- full = self
- .storage
- .bulk_frame_records(path, hash, row.frame, archive)
- .map_err(|_| failure("element archive unavailable"))?;
+ // A history scan visits many frames in the same file. Decode
+ // that bounded file once instead of decompressing its pages
+ // again for each frame. Point/frame reads keep their selective
+ // projection; metadata-only scans still use the index above.
+ full = if self.frame.is_some() || self.lower == self.upper {
+ self.storage
+ .bulk_frame_records(path, hash, row.frame, archive)
+ } else {
+ self.storage.bulk_records(path, hash, 0)
+ }
+ .map_err(|_| failure("element archive unavailable"))?;
let position = full
.binary_search_by_key(&row.id, |r| r.id)
.map_err(|_| failure("element row is absent"))?;
diff --git a/crates/screenpipe-db/src/storage/bulk/tests.rs b/crates/screenpipe-db/src/storage/bulk/tests.rs
index f4ef8d2d0..ca2ae038d 100644
--- a/crates/screenpipe-db/src/storage/bulk/tests.rs
+++ b/crates/screenpipe-db/src/storage/bulk/tests.rs
@@ -57,6 +57,63 @@ async fn fixture() -> (tempfile::TempDir, Arc<DatabaseManager>) {
fixture_mode(true).await
}
+#[tokio::test]
+async fn element_scans_decode_each_file_once_and_point_reads_stay_selective() {
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp) VALUES(1,'2026-09-15'),(2,'2026-09-15'),(3,'2026-09-15'); INSERT INTO elements(id,frame_id,source,role,text) VALUES(1,1,'accessibility','AXText','one'),(2,2,'accessibility','AXText','two'),(3,3,'accessibility','AXText','three');").await.unwrap();
+ while db.seal_payloads().await.unwrap() != 0 {}
+ let storage = db.storage.as_ref().unwrap();
+ let paths: Vec<String> =
+ sqlx::query_scalar("SELECT path FROM _bulk_files WHERE table_name='elements'")
+ .fetch_all(&db.pool)
+ .await
+ .unwrap();
+ assert_eq!(paths.len(), 1);
+ storage.bulk.cache.retire(&paths).unwrap();
+ let decoded = Arc::new(Mutex::new(Vec::<String>::new()));
+ let observed = decoded.clone();
+ *storage.bulk.decode_hook.lock().unwrap() = Some(Arc::new(move |key| {
+ observed.lock().unwrap().push(key.to_owned());
+ }));
+ // Migration receipts use ID windows across all frames, including multiple
+ // windows in one file. A frame projection must not be decoded per window.
+ for (id, expected) in [(1, "one"), (2, "two"), (3, "three")] {
+ let actual: String =
+ sqlx::query_scalar("SELECT text FROM elements WHERE id>=? ORDER BY id LIMIT 1")
+ .bind(id)
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ assert_eq!(actual, expected);
+ }
+ {
+ let keys = decoded.lock().unwrap();
+ assert_eq!(keys.iter().filter(|k| k.starts_with("records:")).count(), 1);
+ assert!(!keys.iter().any(|k| k.starts_with("element-frame:")));
+ }
+ for sql in [
+ "SELECT text FROM elements WHERE id=2",
+ "SELECT text FROM elements WHERE frame_id=2",
+ ] {
+ storage.bulk.cache.retire(&paths).unwrap();
+ decoded.lock().unwrap().clear();
+ let actual: String = sqlx::query_scalar(sql).fetch_one(&db.pool).await.unwrap();
+ assert_eq!(actual, "two");
+ let keys = decoded.lock().unwrap();
+ assert_eq!(
+ keys.iter()
+ .filter(|k| k.starts_with("element-frame:"))
+ .count(),
+ 1
+ );
+ assert!(!keys.iter().any(|k| k.starts_with("records:")));
+ }
+ db.close().await;
+}
+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn cache_hits_and_unrelated_readers_progress_during_a_decode() {
let (_root, db) = fixture().await;
diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rs
index 7ac1530b0..6d9f7af1c 100644
--- a/crates/screenpipe-db/src/storage/in_place.rs
+++ b/crates/screenpipe-db/src/storage/in_place.rs
@@ -102,6 +102,8 @@ async fn run(
pool.clone(),
screenpipe_sqlite_coordinator::sqlite_write_lock(&index),
);
+ let mut reclamation = super::reclaim::Reclaimer::default();
+ let mut last_report = None;
let result = async {
if archive { reserve(&storage)?; }
{
@@ -143,14 +145,14 @@ async fn run(
if archive {
storage.reclaim_once(&pool, &writer).await?;
storage.reclaim_bulk(&pool, &writer).await?;
- reclaim(&storage, &writer, file).await?;
- report(&storage, &pool, original_allocated, total_records, progress).await?;
+ let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, true).await?;
+ report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
}
loop {
if archive { reserve(&storage)?; }
if archive && storage.seal_once(&pool, &writer).await? != 0 {
- reclaim(&storage, &writer, file).await?;
- report(&storage, &pool, original_allocated, total_records, progress).await?;
+ let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;
+ report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
continue;
}
let last: Option<i64> = sqlx::query_scalar("SELECT max(frame_id) FROM frame_payloads").fetch_one(&pool).await?;
@@ -177,8 +179,8 @@ async fn run(
loop {
if archive { reserve(&storage)?; }
if archive && bulk::elements::seal(&storage, &pool, &writer).await? != 0 {
- reclaim(&storage, &writer, file).await?;
- report(&storage, &pool, original_allocated, total_records, progress).await?;
+ let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;
+ report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
continue;
}
let rows = if archive {
@@ -209,7 +211,7 @@ async fn run(
super::faults::checkpoint("migration_batch_staged");
drop(conn);
drop(permit);
- if archive { reclaim(&storage, &writer, file).await?; }
+ if archive { reclaim(&storage, &writer, file, &mut reclamation, false).await?; }
}
let permit = writer.lock().await?;
sqlx::query("DROP TABLE _bulk_elements_source").execute(permit.pool()).await?;
@@ -230,10 +232,10 @@ async fn run(
tx.commit().await?;
}
storage.publish_bulk(&pool, &writer, table, rows, None).await?;
- reclaim(&storage, &writer, file).await?;
- report(&storage, &pool, original_allocated, total_records, progress).await?;
+ let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, false).await?;
+ report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
}
- backfill_resident_fts(&storage, &pool, &writer, table, file, archive).await?;
+ backfill_resident_fts(&storage, &pool, &writer, table, file, &mut reclamation, archive).await?;
}
{
let permit = writer.lock().await?;
@@ -252,7 +254,7 @@ async fn run(
tx.commit().await?;
}
if archive {
- reclaim(&storage, &writer, file).await?;
+ reclaim(&storage, &writer, file, &mut reclamation, true).await?;
progress(MigrationProgress { message: "conversion complete", completed_records: Some(total_records), total_records: Some(total_records),
bytes_saved: Some(original_allocated.saturating_sub(super::reclaim::footprint(&storage.root)?)), available_bytes: Some(fs2::available_space(&storage.root)?) });
}
@@ -304,12 +306,22 @@ async fn reclaim(
storage: &HybridStorage,
writer: &SqliteWritePool,
file: &mut std::fs::File,
-) -> Result<(), sqlx::Error> {
+ reclamation: &mut super::reclaim::Reclaimer,
+ force: bool,
+) -> Result<bool, sqlx::Error> {
let permit = writer.lock().await?;
let mut conn = permit.pool().acquire().await?;
- super::reclaim::free_leaves(&mut conn, file, storage.descriptor.budget.decode_bytes).await?;
+ let reclaimed = reclamation
+ .run(
+ &mut conn,
+ file,
+ &storage.root,
+ &storage.descriptor.budget,
+ force,
+ )
+ .await?;
super::faults::checkpoint("migration_batch_sealed");
- Ok(())
+ Ok(reclaimed)
}
// Privacy-ineligible records remain resident. Their search entries still need
@@ -320,6 +332,7 @@ async fn backfill_resident_fts(
writer: &SqliteWritePool,
table: &bulk::Table,
file: &mut std::fs::File,
+ reclamation: &mut super::reclaim::Reclaimer,
archive: bool,
) -> Result<(), sqlx::Error> {
use futures::TryStreamExt;
@@ -385,7 +398,14 @@ async fn backfill_resident_fts(
sqlx::query("INSERT INTO _storage_conversion_steps(step,last_id) VALUES(?,?) ON CONFLICT(step) DO UPDATE SET last_id=excluded.last_id").bind(&step).bind(last).execute(&mut *tx).await?;
tx.commit().await?;
if archive {
- super::reclaim::free_leaves(&mut conn, file, storage.descriptor.budget.decode_bytes)
+ reclamation
+ .run(
+ &mut conn,
+ file,
+ &storage.root,
+ &storage.descriptor.budget,
+ false,
+ )
.await?;
} else {
schema::construction_checkpoint(&mut conn).await?;
@@ -400,7 +420,17 @@ async fn report(
original: u64,
total: u64,
progress: &(impl Fn(MigrationProgress) + Send + Sync),
+ last_report: &mut Option<std::time::Instant>,
+ reclaimed: bool,
) -> Result<(), sqlx::Error> {
+ // Measuring savings inventories every archive file. Bound that work by
+ // time, and refresh immediately after a physical reclamation pass. The
+ // final completion report always measures exact totals.
+ if !reclaimed
+ && last_report.is_some_and(|last| last.elapsed() < std::time::Duration::from_secs(1))
+ {
+ return Ok(());
+ }
let frames: i64 =
sqlx::query_scalar("SELECT count(*) FROM frame_payloads WHERE state='sealed'")
.fetch_one(pool)
@@ -416,5 +446,6 @@ async fn report(
bytes_saved: Some(original.saturating_sub(super::reclaim::footprint(&storage.root)?)),
available_bytes: Some(fs2::available_space(&storage.root)?),
});
+ *last_report = Some(std::time::Instant::now());
Ok(())
}
diff --git a/crates/screenpipe-db/src/storage/reclaim.rs b/crates/screenpipe-db/src/storage/reclaim.rs
index 23c1e7cc9..1f5ee5d80 100644
--- a/crates/screenpipe-db/src/storage/reclaim.rs
+++ b/crates/screenpipe-db/src/storage/reclaim.rs
@@ -255,11 +255,165 @@ pub(super) async fn free_leaves(
Ok(())
}
+/// Scanning the growing freelist after every batch makes conversion quadratic.
+/// With spare disk, scan at geometrically increasing free-page totals. The
+/// offline caller forces a scan at startup and completion. Disk pressure always
+/// triggers a scan before the next batch's reserve check.
+/// This is only a scheduling hint: every scan still validates the current map,
+/// and every skipped scan still checkpoints the batch's WAL.
+#[derive(Default)]
+pub(super) struct Reclaimer {
+ next_free_pages: i64,
+}
+
+impl Reclaimer {
+ pub(super) async fn run(
+ &mut self,
+ conn: &mut SqliteConnection,
+ file: &mut File,
+ root: &Path,
+ budget: &super::StorageBudget,
+ force: bool,
+ ) -> Result<bool, sqlx::Error> {
+ // Check space after flushing: checkpointing can reallocate holes that
+ // SQLite reused, even when the free-page total did not change.
+ super::schema::construction_checkpoint(conn).await?;
+ let free: i64 = sqlx::query_scalar("PRAGMA freelist_count")
+ .fetch_one(&mut *conn)
+ .await?;
+ // Keep the original per-batch reclamation cadence on tight volumes.
+ // The next-batch minimum alone leaves too little allocation headroom
+ // for filesystem metadata and SQLite's staging/checkpoint writes.
+ // Defer only with room for the configured staging window plus reserve.
+ let headroom = super::in_place::working_space(budget).max(
+ budget
+ .disk_reserve_bytes
+ .saturating_add(budget.staging_bytes),
+ );
+ let needs_space = fs2::available_space(root)? < headroom;
+ if force || needs_space || free >= self.next_free_pages {
+ free_leaves(conn, file, budget.decode_bytes).await?;
+ self.next_free_pages = free.max(1).saturating_mul(2);
+ Ok(true)
+ } else {
+ Ok(false)
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
use sqlx::Connection;
+ async fn reclamation_workload(
+ batches: i64,
+ payload_bytes: i64,
+ scheduled: bool,
+ ) -> (usize, f64) {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("index.sqlite");
+ let mut file = File::options()
+ .read(true)
+ .write(true)
+ .create_new(true)
+ .open(&path)
+ .unwrap();
+ let mut conn = SqliteConnection::connect_with(
+ &sqlx::sqlite::SqliteConnectOptions::new()
+ .filename(&path)
+ .pragma("journal_mode", "WAL")
+ .pragma("locking_mode", "EXCLUSIVE")
+ .pragma("secure_delete", "OFF"),
+ )
+ .await
+ .unwrap();
+ sqlx::query("CREATE TABLE cells(id INTEGER PRIMARY KEY,payload BLOB)")
+ .execute(&mut conn)
+ .await
+ .unwrap();
+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO cells SELECT id,zeroblob(?) FROM n")
+ .bind(batches + 1).bind(payload_bytes).execute(&mut conn).await.unwrap();
+ super::super::schema::construction_checkpoint(&mut conn)
+ .await
+ .unwrap();
+ let before = allocated(&path).unwrap();
+ let mut reclaimer = Reclaimer::default();
+ let budget = super::super::StorageBudget::default();
+ let mut scans = 0;
+ let mut seconds = 0.0;
+ for id in 1..=batches {
+ sqlx::query("DELETE FROM cells WHERE id=?")
+ .bind(id)
+ .execute(&mut conn)
+ .await
+ .unwrap();
+ let start = std::time::Instant::now();
+ if scheduled {
+ scans += usize::from(
+ reclaimer
+ .run(&mut conn, &mut file, root.path(), &budget, false)
+ .await
+ .unwrap(),
+ );
+ } else {
+ free_leaves(&mut conn, &mut file, 128 * 1024 * 1024)
+ .await
+ .unwrap();
+ scans += 1;
+ }
+ seconds += start.elapsed().as_secs_f64();
+ }
+ // Disk pressure must reclaim accumulated pages below the next scan
+ // threshold. An impossible reserve exercises that path without filling
+ // the host volume; the unscheduled comparison forces its final sweep.
+ let mut final_budget = budget.clone();
+ if scheduled {
+ final_budget.disk_reserve_bytes = u64::MAX;
+ }
+ let start = std::time::Instant::now();
+ assert!(reclaimer
+ .run(&mut conn, &mut file, root.path(), &final_budget, !scheduled)
+ .await
+ .unwrap());
+ seconds += start.elapsed().as_secs_f64();
+ assert!(allocated(&path).unwrap() < before / 2);
+ assert_eq!(
+ sqlx::query_scalar::<_, String>("PRAGMA integrity_check")
+ .fetch_one(&mut conn)
+ .await
+ .unwrap(),
+ "ok"
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT length(payload) FROM cells")
+ .fetch_one(&mut conn)
+ .await
+ .unwrap(),
+ payload_bytes
+ );
+ conn.close().await.unwrap();
+ drop(file);
+ (scans, seconds)
+ }
+
+ #[tokio::test]
+ async fn reclamation_scans_grow_geometrically_and_pressure_releases_remaining_pages() {
+ let (scans, _) = reclamation_workload(64, 128 * 1024, true).await;
+ assert!(
+ scans <= 7,
+ "64 batches should need at most seven growing freelist scans, got {scans}"
+ );
+ }
+
+ #[tokio::test]
+ #[ignore = "compares repeated and scheduled reclamation on an isolated 256 MiB database"]
+ async fn reclamation_throughput() {
+ let (before_scans, before) = reclamation_workload(256, 1024 * 1024, false).await;
+ let (after_scans, after) = reclamation_workload(256, 1024 * 1024, true).await;
+ eprintln!("reclamation benchmark: before={before:.3}s ({before_scans} scans), after={after:.3}s ({after_scans} scans), speedup={:.2}x", before / after);
+ }
+
#[tokio::test]
async fn fragmented_freelist_preserves_live_rows_and_file_length() {
let root = tempfile::tempdir().unwrap();
diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rs
index 83402c4b9..d1336f2e4 100644
--- a/crates/screenpipe-db/tests/in_place_migration.rs
+++ b/crates/screenpipe-db/tests/in_place_migration.rs
@@ -24,6 +24,72 @@ async fn fixture(root: &std::path::Path) {
db.close().await;
}
+#[tokio::test]
+#[ignore = "migration throughput benchmark; generates an isolated recording history"]
+async fn migration_throughput() {
+ use std::{sync::Mutex, time::Instant};
+ let _ = tracing_subscriber::fmt()
+ .with_env_filter("screenpipe_db::storage=info")
+ .with_writer(std::io::stderr)
+ .try_init();
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ let frames = 4096_i64;
+ let elements = 262144_i64;
+ let detail = "capture detail ".repeat(4096);
+ let properties = "element properties ".repeat(32);
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ for id in 1..=frames {
+ sqlx::query("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(?,'2026-09-15T12:00:00Z','searchable migration history',?)")
+ .bind(id).bind(&detail).execute(&mut **tx.conn()).await.unwrap();
+ }
+ sqlx::query("WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<?) INSERT INTO elements(id,frame_id,source,role,text,properties) SELECT id,1+(id-1)/?,'accessibility','AXText','searchable element',? FROM n")
+ .bind(elements).bind(elements / frames).bind(&properties).execute(&mut **tx.conn()).await.unwrap();
+ tx.commit().await.unwrap();
+ db.wal_checkpoint().await.unwrap();
+ db.close().await;
+ let started = Instant::now();
+ let previous = Mutex::new(("", started, 0_u64));
+ let report = screenpipe_db::storage::migrate_with_progress(
+ root.path(),
+ Default::default(),
+ Default::default(),
+ |p| {
+ let mut previous = previous.lock().unwrap();
+ let completed = p.completed_records.unwrap_or(previous.2);
+ if p.message != previous.0 || completed >= previous.2 + 32768 {
+ eprintln!(
+ "migration benchmark: elapsed={:.3}s phase={} records={} interval={:.3}s",
+ started.elapsed().as_secs_f64(),
+ p.message,
+ completed,
+ previous.1.elapsed().as_secs_f64()
+ );
+ *previous = (p.message, Instant::now(), completed);
+ }
+ },
+ )
+ .await
+ .unwrap();
+ eprintln!("migration benchmark: total={:.3}s frames={frames} elements={elements} source={} payloads={}",
+ started.elapsed().as_secs_f64(), report.source_bytes, report.payload_bytes);
+ assert_eq!(report.frames, frames as u64);
+ assert_eq!(
+ report
+ .tables
+ .iter()
+ .find(|t| t.table == "elements")
+ .unwrap()
+ .rows,
+ elements as u64
+ );
+}
+
#[tokio::test]
#[ignore = "requires a marked disposable volume; optional production-default run uses up to 8 GiB"]
async fn migration_completes_with_less_free_space_than_its_final_payloads() {
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 593e9cc0d..8dd33957d 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 32
- Mapped Rust files: 351
-- Active test blocks: 3476
-- Ignored/manual test blocks: 142
-- Declared test blocks: 3618
-- Weighted coverage points: 2866.3
+- Active test blocks: 3478
+- Ignored/manual test blocks: 144
+- Declared test blocks: 3622
+- Weighted coverage points: 2868.3
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 3333 | 136 | 2801.2 | 21 | 11 | 100% |
-| macos | 29 | 3395 | 117 | 2815.2 | 22 | 11 | 100% |
-| linux | 25 | 2963 | 109 | 2460.5 | 20 | 11 | 100% |
+| windows | 29 | 3335 | 138 | 2803.2 | 21 | 11 | 100% |
+| macos | 29 | 3397 | 119 | 2817.2 | 22 | 11 | 100% |
+| linux | 25 | 2965 | 111 | 2462.5 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 10 | 19 | 116 | 1706 | 42 | 1315.0 | 10 |
-| screenpipe-db | 5 | 56 | 22 | 521 | 19 | 494.3 | 9 |
+| screenpipe-db | 5 | 56 | 22 | 523 | 21 | 496.3 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts |
| audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 6 suites / 468 active / 15 ignored / 441.3 pts | 6 suites / 468 active / 15 ignored / 441.3 pts | 6 suites / 468 active / 15 ignored / 441.3 pts |
+| database | 6 suites / 470 active / 17 ignored / 443.3 pts | 6 suites / 470 active / 17 ignored / 443.3 pts | 6 suites / 470 active / 17 ignored / 443.3 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 243 active / 1 ignored / 218.0 pts | 6 suites / 243 active / 1 ignored / 218.0 pts | 5 suites / 237 active / 1 ignored / 216.3 pts |
| local-api | 2 suites / 416 active / 9 ignored / 293.6 pts | 2 suites / 416 active / 9 ignored / 293.6 pts | 2 suites / 416 active / 9 ignored / 293.6 pts |
| meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1523 active / 70 ignored / 1343.5 pts | 14 suites / 1630 active / 74 ignored / 1386.3 pts | 13 suites / 1523 active / 70 ignored / 1343.5 pts |
+| performance | 13 suites / 1525 active / 72 ignored / 1345.5 pts | 14 suites / 1632 active / 76 ignored / 1388.3 pts | 13 suites / 1525 active / 72 ignored / 1345.5 pts |
| pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
| privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts |
| real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - |
| speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |
-| storage | 3 suites / 573 active / 32 ignored / 471.0 pts | 3 suites / 573 active / 32 ignored / 471.0 pts | 3 suites / 573 active / 32 ignored / 471.0 pts |
+| storage | 3 suites / 575 active / 34 ignored / 473.0 pts | 3 suites / 575 active / 34 ignored / 473.0 pts | 3 suites / 575 active / 34 ignored / 473.0 pts |
| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
-| timeline | 4 suites / 1133 active / 36 ignored / 920.3 pts | 4 suites / 1133 active / 36 ignored / 920.3 pts | 4 suites / 1133 active / 36 ignored / 920.3 pts |
+| timeline | 4 suites / 1135 active / 38 ignored / 922.3 pts | 4 suites / 1135 active / 38 ignored / 922.3 pts | 4 suites / 1135 active / 38 ignored / 922.3 pts |
| transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts |
| ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |
| vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |
@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 28 | 233 | 6 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 28 | 235 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -275,10 +275,10 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | src/failpoint_vfs.rs | source | 4 | 0 | 4 |
| db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 |
| db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |
-| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 5 | 0 | 5 |
+| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 6 | 0 | 6 |
| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 |
| db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |
-| db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 1 | 0 | 1 |
+| db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 2 | 1 | 3 |
| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 1 | 0 | 1 |
| db-timeline-frames | screenpipe-db | src/storage/sealing.rs | source | 1 | 0 | 1 |
| db-search-indexing | screenpipe-db | src/text_normalizer.rs | source | 19 | 0 | 19 |
@@ -302,7 +302,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 |
| db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 |
| db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |
-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 11 | 3 | 14 |
+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 11 | 4 | 15 |
| db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 |
| db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 |
| db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |