Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): finish migration without rejecting oversized history (#7056)

Ezra Ellettecd14a077c012
diff --git a/COVERAGE.md b/COVERAGE.md
index f6db5d25e..a955e061e 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -36,15 +36,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 33
- Mapped Rust files: 356
-- Active test blocks: 3502
+- Active test blocks: 3503
- Ignored/manual test blocks: 144
-- Weighted coverage points: 2888.4
+- Weighted coverage points: 2889.4
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3359 | 138 | 2823.3 | 21 | 11 | 100% |
-| macos | 30 | 3421 | 119 | 2837.3 | 22 | 11 | 100% |
-| linux | 26 | 2989 | 111 | 2482.6 | 20 | 11 | 100% |
+| windows | 30 | 3360 | 138 | 2824.3 | 21 | 11 | 100% |
+| macos | 30 | 3422 | 119 | 2838.3 | 22 | 11 | 100% |
+| linux | 26 | 2990 | 111 | 2483.6 | 20 | 11 | 100% |
## Refresh
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
index 139466c3f..4fbfa31e3 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
@@ -53,7 +53,7 @@ pub(crate) async fn seal(
crate::storage::diagnostics::stage("selecting_staged_elements");
let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
"SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND ({}) ORDER BY id LIMIT 1",
- TABLE.eligible
+ TABLE.sealable()
)))
.fetch_optional(pool)
.await?;
@@ -178,10 +178,10 @@ async fn rewrite(
sqlx::query_as("SELECT policy,required_surfaces FROM storage_metadata")
.fetch_one(pool)
.await?;
- let (first, last, limit) = if let Some((first, last, _)) = range {
+ let (first, last) = if let Some((first, last, _)) = range {
let blocked: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
"SELECT EXISTS(SELECT 1 FROM elements WHERE id BETWEEN ? AND ? AND NOT ({}))",
- TABLE.eligible
+ TABLE.sealable()
)))
.bind(first)
.bind(last)
@@ -190,18 +190,14 @@ async fn rewrite(
if blocked {
return Ok(0);
}
- (first, last, String::new())
+ (first, last)
} else {
let next: Option<i64> =
sqlx::query_scalar("SELECT min(first_id) FROM _bulk_element_ranges WHERE first_id>?")
.bind(id)
.fetch_one(pool)
.await?;
- (
- id,
- next.map_or(i64::MAX, |n| n - 1),
- format!(" LIMIT {FILE_ROWS}"),
- )
+ (id, next.map_or(i64::MAX, |n| n - 1))
};
// Release each read cursor before encoding takes the writer. Offline
// migration has one connection; a recovered resident range can exceed one
@@ -210,43 +206,41 @@ async fn rewrite(
let mut files = Vec::new();
loop {
let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));
- let sql=format!("SELECT id,_archive_generation,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id{limit}",names(),TABLE.eligible);
- crate::storage::diagnostics::batch(
- "elements",
- Some(after.unwrap_or(first)),
- Some(last),
- None,
- None,
- );
- crate::storage::diagnostics::stage("reading_elements_to_seal");
- let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql))
- .bind(first)
- .bind(last)
- .fetch(pool);
- let mut rows = Vec::new();
- let mut bytes = 0;
- while let Some(row) = stream.try_next().await? {
- let row = record(&row)?;
- if row.bytes() > storage.descriptor.budget.record_bytes {
- crate::storage::diagnostics::batch(
- "elements",
- Some(row.id),
- Some(row.id),
- Some(1),
- Some(row.bytes() as u64),
- );
- return Err(storage_error("element record exceeds sealing budget"));
- }
- if !rows.is_empty()
- && (rows.len() == FILE_ROWS
- || bytes + row.bytes() > storage.descriptor.budget.file_bytes)
+ crate::storage::diagnostics::stage("selecting_element_batch");
+ let candidates: Vec<(i64, i64)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(
+ "SELECT id,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id LIMIT {FILE_ROWS}",
+ TABLE.all_bytes(""), TABLE.eligible
+ )))
+ .bind(first).bind(last).fetch_all(pool).await?;
+ let mut ids = Vec::new();
+ let mut bytes = 0usize;
+ for (id, size) in candidates {
+ let size = usize::try_from(size).map_err(storage_error)?;
+ // End the range before a retained record so future rewrites never
+ // need to decode that record or remove its SQLite contents.
+ if size > storage.descriptor.budget.record_bytes
+ || (!ids.is_empty()
+ && bytes.saturating_add(size) > storage.descriptor.budget.file_bytes)
{
break;
}
- bytes += row.bytes();
- rows.push(row);
+ ids.push(id);
+ bytes += size;
}
- drop(stream);
+ crate::storage::diagnostics::batch(
+ "elements",
+ ids.first().copied(),
+ ids.last().copied(),
+ Some(ids.len() as u64),
+ Some(bytes as u64),
+ );
+ crate::storage::diagnostics::stage("reading_elements_to_seal");
+ let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
+ "SELECT id,_archive_generation,{} FROM elements WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id", names()
+ )))
+ .bind(serde_json::to_string(&ids).map_err(storage_error)?)
+ .fetch_all(pool).await?
+ .iter().map(record).collect::<Result<Vec<_>, _>>()?;
let Some(last_row) = rows.last() else { break };
after = Some(last_row.id);
files.push(encode(storage, writer, rows).await?);
@@ -284,7 +278,7 @@ async fn rewrite(
let selection = if range.is_some() {
"1".into()
} else {
- format!("_archive_deleted=1 OR ({})", TABLE.eligible)
+ format!("_archive_deleted=1 OR ({})", TABLE.sealable())
};
let freed:i64=sqlx::query_scalar(sqlx::AssertSqlSafe(format!("SELECT COALESCE(SUM({}),0) FROM _bulk_element_rows WHERE id BETWEEN ? AND ? AND ({selection})",TABLE.all_bytes("")))).bind(first).bind(last).fetch_one(&mut *tx).await?;
sqlx::query(sqlx::AssertSqlSafe(format!(
diff --git a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
index 3ebfe2d5f..fdd3293cb 100644
--- a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
@@ -48,31 +48,16 @@ impl HybridStorage {
};
// Keep the pending index as the outer loop so LIMIT bounds work before
// materializing IDs from a large history.
- let candidates=sqlx::query(sqlx::AssertSqlSafe(format!("SELECT v.id,({bytes}) AS bytes FROM main.{t} pending CROSS JOIN (SELECT * FROM {view} WHERE {eligible}) v ON v.id=pending.id WHERE {condition} ORDER BY pending.id LIMIT {FILE_ROWS}",bytes=table.all_bytes("v."),view=table.view(),t=table.name,eligible=table.eligible))).fetch_all(pool).await?;
+ let candidates=sqlx::query(sqlx::AssertSqlSafe(format!("SELECT v.id,({bytes}) AS bytes FROM main.{t} pending CROSS JOIN (SELECT * FROM {view} WHERE {eligible}) v ON v.id=pending.id WHERE {condition} AND ({bytes})<=? ORDER BY pending.id LIMIT {FILE_ROWS}",bytes=table.all_bytes("v."),view=table.view(),t=table.name,eligible=table.eligible))).bind(self.descriptor.budget.record_bytes as i64).fetch_all(pool).await?;
let mut ids = Vec::new();
let mut bytes = 0;
for candidate in candidates {
let size = candidate.try_get::<i64, _>("bytes")? as usize;
- if size > self.descriptor.budget.record_bytes {
- let id = candidate.try_get::<i64, _>("id")?;
- crate::storage::diagnostics::batch(
- table.name,
- Some(id),
- Some(id),
- Some(1),
- Some(size as u64),
- );
- return Err(storage_error("bulk record exceeds sealing budget"));
- }
- if file.is_none() && !ids.is_empty() && bytes + size > self.descriptor.budget.file_bytes
- {
+ if !ids.is_empty() && bytes + size > self.descriptor.budget.file_bytes {
break;
}
ids.push(candidate.try_get::<i64, _>("id")?);
bytes += size;
- if bytes > self.descriptor.budget.decode_bytes / 2 {
- return Err(storage_error("bulk batch exceeds sealing budget"));
- }
}
if ids.is_empty() {
return Ok(Vec::new());
diff --git a/crates/screenpipe-db/src/storage/bulk/mod.rs b/crates/screenpipe-db/src/storage/bulk/mod.rs
index 55ba6ae36..3078b7a38 100644
--- a/crates/screenpipe-db/src/storage/bulk/mod.rs
+++ b/crates/screenpipe-db/src/storage/bulk/mod.rs
@@ -110,6 +110,15 @@ pub(super) static TABLES: &[Table] = &[
];
impl Table {
+ // An encoder bound selects work; it cannot reject valid resident history.
+ fn sealable(&self) -> String {
+ format!(
+ "({}) AND ({})<=(SELECT record_limit FROM storage_metadata)",
+ self.eligible,
+ self.all_bytes("")
+ )
+ }
+
fn mask(&self) -> i64 {
(1 << self.columns.len()) - 1
}
diff --git a/crates/screenpipe-db/src/storage/import.rs b/crates/screenpipe-db/src/storage/import.rs
index 81f01a7ef..f78298a27 100644
--- a/crates/screenpipe-db/src/storage/import.rs
+++ b/crates/screenpipe-db/src/storage/import.rs
@@ -1,15 +1,7 @@
// screenpipe — AI that knows everything you've seen, said, or heard
// https://screenpipe.com
-//! Bounded resident-row batches shared by the in-place frame and element conversion.
-
-use super::{storage_error, StorageBudget};
-use futures::TryStreamExt;
-use sqlx::{sqlite::SqliteRow, Row, SqliteConnection, TypeInfo, ValueRef};
-
-fn quote(name: &str) -> String {
- format!("\"{}\"", name.replace('"', "\"\""))
-}
+//! Resident table columns used by SQL-to-SQL migration batches.
pub(super) async fn columns(
source: &sqlx::SqlitePool,
@@ -20,148 +12,3 @@ pub(super) async fn columns(
.fetch_all(source)
.await
}
-
-pub(super) async fn batch(
- source: &sqlx::SqlitePool,
- table: &str,
- columns: &[String],
- after: Option<i64>,
- budget: &StorageBudget,
-) -> Result<Vec<SqliteRow>, sqlx::Error> {
- let sql = format!(
- "SELECT rowid,{} FROM {} WHERE rowid{}? ORDER BY rowid LIMIT {}",
- columns
- .iter()
- .map(|c| quote(c))
- .collect::<Vec<_>>()
- .join(","),
- quote(table),
- if after.is_some() { ">" } else { ">=" },
- if table == "frames" {
- budget.file_rows
- } else {
- super::bulk::FILE_ROWS
- }
- );
- super::diagnostics::batch(table, after, None, None, None);
- super::diagnostics::stage("reading_resident_batch");
- let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql))
- .bind(after.unwrap_or(i64::MIN))
- .fetch(source);
- let mut rows = Vec::new();
- let mut bytes = 0;
- while let Some(row) = stream.try_next().await? {
- let mut size = 0;
- for i in 0..row.len() {
- let raw = row.try_get_raw(i)?;
- size += if raw.is_null() {
- 0
- } else {
- match raw.type_info().name() {
- "INTEGER" | "REAL" => 8,
- _ => row.try_get::<Vec<u8>, _>(i)?.len(),
- }
- };
- }
- if size > budget.decode_bytes / 2 {
- super::diagnostics::batch(
- table,
- Some(row.get(0)),
- Some(row.get(0)),
- Some(1),
- Some(size as u64),
- );
- return Err(storage_error(format!(
- "migration record in {table} exceeds decode budget"
- )));
- }
- if !rows.is_empty() && bytes + size > budget.file_bytes {
- break;
- }
- bytes += size;
- rows.push(row);
- }
- super::diagnostics::batch(
- table,
- rows.first().map(|r| r.get(0)),
- rows.last().map(|r| r.get(0)),
- Some(rows.len() as u64),
- Some(bytes as u64),
- );
- Ok(rows)
-}
-
-pub(super) async fn insert(
- conn: &mut SqliteConnection,
- table: &str,
- columns: &[String],
- rows: &[SqliteRow],
- elements: bool,
-) -> Result<(), sqlx::Error> {
- let destination = if elements {
- "_bulk_element_rows"
- } else {
- table
- };
- let prefix = format!(
- "INSERT INTO {}(rowid,{}{}) ",
- quote(destination),
- columns
- .iter()
- .map(|c| quote(c))
- .collect::<Vec<_>>()
- .join(","),
- if elements { ",_archive_generation" } else { "" }
- );
- // SQLite's parameter limit bounds statement size independently of payload
- // bytes. Values retain their original SQLite storage class.
- for chunk in rows.chunks(32766 / (columns.len() + 2)) {
- let mut query = sqlx::QueryBuilder::<sqlx::Sqlite>::new(&prefix);
- let mut error = None;
- query.push_values(chunk, |mut values, row| {
- for i in 0..row.len() {
- let value = (|| -> Result<(), sqlx::Error> {
- let raw = row.try_get_raw(i)?;
- if raw.is_null() {
- values.push_bind(None::<i64>);
- } else {
- match raw.type_info().name() {
- "INTEGER" => {
- values.push_bind(row.try_get::<i64, _>(i)?);
- }
- "REAL" => {
- values.push_bind(row.try_get::<f64, _>(i)?);
- }
- "TEXT" => {
- values.push_bind(row.try_get::<String, _>(i)?);
- }
- _ => {
- values.push_bind(row.try_get::<Vec<u8>, _>(i)?);
- }
- }
- }
- Ok(())
- })();
- if let Err(e) = value {
- error = Some(e);
- }
- }
- if elements {
- values.push_bind(1_i64);
- }
- });
- if let Some(error) = error {
- return Err(error);
- }
- query.build().execute(&mut *conn).await?;
- }
- if elements {
- super::bulk::elements::import_batch(
- conn,
- rows.first().unwrap().try_get(0)?,
- rows.last().unwrap().try_get(0)?,
- )
- .await?;
- }
- Ok(())
-}
diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rs
index e350b76dd..106840530 100644
--- a/crates/screenpipe-db/src/storage/in_place.rs
+++ b/crates/screenpipe-db/src/storage/in_place.rs
@@ -140,13 +140,13 @@ async fn run(
schema::finish_step(&mut tx,"suspend-original-triggers").await?;
tx.commit().await?;
}
+ if schema::converted_step(&mut conn,"conversion-complete").await? { return Ok(()); }
// A paused migration records resident payloads like the original
// SQLite store. Its disabled sealer must not impose a staging cap
// that eventually stops recording. Explicit completion restores it.
sqlx::query("UPDATE storage_metadata SET staging_limit=?")
.bind(if archive { storage.descriptor.budget.staging_bytes as i64 } else { i64::MAX })
.execute(&mut *conn).await?;
- if schema::converted_step(&mut conn,"conversion-complete").await? { return Ok(()); }
schema::construction_checkpoint(&mut conn).await?;
}
// Encoding reservations left by an interrupted attempt are handled by
@@ -196,32 +196,22 @@ async fn run(
report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
continue;
}
- let rows = if archive {
- super::import::batch(&pool, "_bulk_elements_source", &columns, None, &storage.descriptor.budget).await?
- } else { Vec::new() };
- let range = if archive {
- rows.first().zip(rows.last()).map(|(first,last)| (first.get::<i64,_>(0),last.get::<i64,_>(0)))
- } else {
- resident_range(&pool, "_bulk_elements_source", None, storage.descriptor.budget.file_bytes, None).await?
- };
+ // Move existing records inside SQLite, including records larger
+ // than any encoder/decoder batch. Sealing can leave them resident.
+ let range = resident_range(&pool, "_bulk_elements_source", None,
+ storage.descriptor.budget.file_bytes, archive.then_some(bulk::FILE_ROWS)).await?;
let Some((first, last)) = range else { break };
- super::diagnostics::batch("elements", Some(first), Some(last), Some(rows.len() as u64), None);
+ super::diagnostics::batch("elements", Some(first), Some(last), None, None);
super::diagnostics::stage("waiting_for_element_writer");
let permit = writer.lock().await?;
super::diagnostics::stage("waiting_for_element_connection");
let mut conn = permit.pool().acquire().await?;
let mut tx = conn.begin().await?;
super::diagnostics::stage("staging_element_rows");
- if archive {
- super::import::insert(&mut tx, "elements", &columns, &rows, true).await?;
- } else {
- // SQL-to-SQL batches also recover pre-existing records that
- // exceed the archive record budget, without decoding them.
- let names = columns.iter().map(|c| format!("\"{}\"", c.replace('"', "\"\""))).collect::<Vec<_>>().join(",");
- sqlx::query(sqlx::AssertSqlSafe(format!("INSERT INTO _bulk_element_rows({names},_archive_generation) SELECT {names},1 FROM _bulk_elements_source WHERE id BETWEEN ? AND ?")))
- .bind(first).bind(last).execute(&mut *tx).await?;
- bulk::elements::import_batch(&mut tx, first, last).await?;
- }
+ let names = columns.iter().map(|c| format!("\"{}\"", c.replace('"', "\"\""))).collect::<Vec<_>>().join(",");
+ sqlx::query(sqlx::AssertSqlSafe(format!("INSERT INTO _bulk_element_rows({names},_archive_generation) SELECT {names},1 FROM _bulk_elements_source WHERE id BETWEEN ? AND ?")))
+ .bind(first).bind(last).execute(&mut *tx).await?;
+ bulk::elements::import_batch(&mut tx, first, last).await?;
super::diagnostics::stage("deleting_staged_source_elements");
sqlx::query("DELETE FROM _bulk_elements_source WHERE id BETWEEN ? AND ?").bind(first).bind(last).execute(&mut *tx).await?;
super::diagnostics::stage("committing_element_staging");
@@ -268,6 +258,11 @@ async fn run(
let original: Vec<String> = sqlx::query_scalar("SELECT sql FROM _storage_conversion_triggers WHERE restore=1").fetch_all(&mut *tx).await?;
for sql in original { sqlx::raw_sql(sqlx::AssertSqlSafe(sql)).execute(&mut *tx).await?; }
if archive {
+ // Existing resident history occupies disk, not the capture work
+ // queue. Preserve a full configured staging allowance above it.
+ sqlx::query("UPDATE storage_metadata SET staging_limit=staging_bytes+?")
+ .bind(storage.descriptor.budget.staging_bytes as i64)
+ .execute(&mut *tx).await?;
schema::finish_step(&mut tx,"conversion-complete").await?;
} else {
// Restored application triggers must be suspended again only
@@ -395,7 +390,7 @@ async fn backfill_resident_fts(
.fetch_optional(pool)
.await?
.flatten();
- let sql = format!("SELECT id,{size} FROM {t} WHERE _archive_file IS NULL AND NOT ({eligible}) AND id {comparison} ? ORDER BY id LIMIT {rows}",t=table.name,eligible=if archive { table.eligible } else { "0" },comparison=if after.is_some(){">"}else{">="},rows=bulk::FILE_ROWS);
+ let sql = format!("SELECT id,{size} FROM {t} WHERE _archive_file IS NULL AND id {comparison} ? ORDER BY id LIMIT {rows}",t=table.name,comparison=if after.is_some(){">"}else{">="},rows=bulk::FILE_ROWS);
let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql))
.bind(after.unwrap_or(i64::MIN))
.fetch(pool);
@@ -403,11 +398,6 @@ async fn backfill_resident_fts(
let mut bytes = 0_usize;
while let Some(row) = stream.try_next().await? {
let size = row.get::<i64, _>(1) as usize;
- if archive && size > storage.descriptor.budget.record_bytes {
- return Err(storage_error(
- "resident search record exceeds migration budget",
- ));
- }
if !ids.is_empty() && bytes + size > storage.descriptor.budget.file_bytes {
break;
}
diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rs
index fc5285535..6c5ce81ef 100644
--- a/crates/screenpipe-db/src/storage/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/lifecycle.rs
@@ -427,10 +427,6 @@ async fn migrate_observed(
let original = async {
let frozen = source.begin_immediate_with_retry().await?;
verify_integrity(&source.pool).await?;
- let pages: i64 = sqlx::query_scalar("PRAGMA page_count").fetch_one(&source.pool).await?;
- if (pages as usize).div_ceil(8) > options.budget.decode_bytes / 2 {
- return Err(storage_error("freelist map exceeds migration memory budget; database unchanged"));
- }
let receipts = table_receipts(&source, None).await?;
let terms: Vec<String> = sqlx::query_scalar("SELECT full_text FROM frames WHERE full_text IS NOT NULL AND id IN ((SELECT min(id) FROM frames),(SELECT max(id) FROM frames))").fetch_all(&source.pool).await?;
let mut searches = Vec::new();
@@ -1074,6 +1070,56 @@ pub(super) async fn logical_tables(db: &DatabaseManager) -> Result<Vec<String>,
.fetch_all(&db.pool).await
}
+// Verification consumes history incrementally; the public response limit must
+// never decide whether that history can migrate. Resident frames are hashed
+// from SQLite, and only sealed frames need hydration through the payload reader.
+async fn verification_frame_batch(
+ db: &DatabaseManager,
+ after: Option<i64>,
+) -> Result<Option<(i64, Vec<i64>)>, sqlx::Error> {
+ let comparison = if after.is_some() { ">" } else { ">=" };
+ let (size, state, join, limit) = if let Some(storage) = &db.storage {
+ (
+ "p.bytes".to_owned(),
+ "p.state='sealed'",
+ "LEFT JOIN frame_payloads p ON p.frame_id=f.id",
+ storage
+ .descriptor
+ .budget
+ .file_bytes
+ .min(storage.descriptor.budget.response_bytes),
+ )
+ } else {
+ (
+ super::schema::BYTES.replace("NEW.", "f."),
+ "0",
+ "",
+ StorageBudget::default().file_bytes,
+ )
+ };
+ let rows: Vec<(i64, i64, bool)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(
+ "SELECT f.id,{size},{state} FROM frames f {join} WHERE f.id{comparison}? ORDER BY f.id LIMIT 128"
+ )))
+ .bind(after.unwrap_or(i64::MIN))
+ .fetch_all(&db.pool)
+ .await?;
+ let mut last = None;
+ let mut ids = Vec::new();
+ let mut bytes = 0usize;
+ for (id, size, sealed) in rows {
+ let size = usize::try_from(size).map_err(storage_error)?;
+ if last.is_some() && bytes.saturating_add(size) > limit {
+ break;
+ }
+ last = Some(id);
+ bytes = bytes.saturating_add(size);
+ if sealed {
+ ids.push(id);
+ }
+ }
+ Ok(last.map(|last| (last, ids)))
+}
+
pub(super) async fn table_receipts(
db: &DatabaseManager,
source: Option<&[TableParity]>,
@@ -1117,8 +1163,17 @@ pub(super) async fn table_receipts(
let mut last = i64::MIN;
let mut first = true;
loop {
+ let (through, sealed) = if table == "frames" {
+ let Some(batch) = verification_frame_batch(db, (!first).then_some(last)).await?
+ else {
+ break;
+ };
+ batch
+ } else {
+ (i64::MAX, Vec::new())
+ };
let sql = format!(
- "SELECT {key} AS __storage_rowid,{} FROM {} WHERE {key}{}? ORDER BY {key} LIMIT 128",
+ "SELECT {key} AS __storage_rowid,{} FROM {} WHERE {key}{}? AND {key}<=? ORDER BY {key} LIMIT 128",
columns
.iter()
.map(|c| quote(c))
@@ -1131,6 +1186,7 @@ pub(super) async fn table_receipts(
super::diagnostics::stage("reading_parity_rows");
let rows = sqlx::query(sqlx::AssertSqlSafe(sql))
.bind(last)
+ .bind(through)
.fetch_all(&db.pool)
.await?;
if rows.is_empty() {
@@ -1138,9 +1194,8 @@ pub(super) async fn table_receipts(
}
first = false;
super::diagnostics::stage("hashing_parity_rows");
- let payloads = if table == "frames" && db.storage.is_some() {
- let ids: Vec<i64> = rows.iter().map(|r| r.get("id")).collect();
- db.frame_payloads(&ids, Projection::All).await?
+ let payloads = if !sealed.is_empty() {
+ db.frame_payloads(&sealed, Projection::All).await?
} else {
Default::default()
};
@@ -1272,17 +1327,9 @@ impl DatabaseManager {
if let Some(storage) = &self.storage {
storage.verify_catalog(&self.pool).await?;
storage.verify_bulk(&self.pool).await?;
- let mut after = i64::MIN;
- loop {
- let ids: Vec<i64> =
- sqlx::query_scalar("SELECT id FROM frames WHERE id>? ORDER BY id LIMIT 128")
- .bind(after)
- .fetch_all(&self.pool)
- .await?;
- let Some(last) = ids.last() else {
- break;
- };
- after = *last;
+ let mut after = None;
+ while let Some((last, ids)) = verification_frame_batch(self, after).await? {
+ after = Some(last);
self.frame_payloads(&ids, Projection::All).await?;
}
}
diff --git a/crates/screenpipe-db/src/storage/reclaim.rs b/crates/screenpipe-db/src/storage/reclaim.rs
index 052d1fbad..a756a1655 100644
--- a/crates/screenpipe-db/src/storage/reclaim.rs
+++ b/crates/screenpipe-db/src/storage/reclaim.rs
@@ -181,10 +181,18 @@ pub(super) async fn free_leaves(
return Ok(());
}
let bitmap_bytes = (pages as usize).div_ceil(8);
- if pages <= 0 || bitmap_bytes > memory_limit / 2 {
- return Err(storage_error(
- "freelist map exceeds migration memory budget",
- ));
+ if pages <= 0 {
+ return Err(storage_error("invalid SQLite page count"));
+ }
+ if bitmap_bytes > memory_limit / 2 {
+ // Reclamation is an optimization. Keep these valid free pages in
+ // SQLite when its validation map would exceed the working budget.
+ tracing::debug!(
+ bitmap_bytes,
+ memory_limit,
+ "deferring SQLite space reclamation"
+ );
+ return Ok(());
}
let mut header = [0_u8; 100];
file.seek(SeekFrom::Start(0))?;
@@ -447,11 +455,11 @@ mod tests {
.unwrap();
let length = file.metadata().unwrap().len();
let before = allocated(&path).unwrap();
- assert!(free_leaves(&mut conn, &mut file, 0).await.is_err());
+ free_leaves(&mut conn, &mut file, 0).await.unwrap();
assert_eq!(
allocated(&path).unwrap(),
before,
- "rejected scan must not punch anything"
+ "deferred scan must not punch anything"
);
free_leaves(&mut conn, &mut file, 1024 * 1024)
.await
diff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rs
index 509d63a25..c18fdb5bc 100644
--- a/crates/screenpipe-db/src/storage/schema.rs
+++ b/crates/screenpipe-db/src/storage/schema.rs
@@ -234,7 +234,7 @@ UPDATE frames SET payload_full_text_length=length(COALESCE(full_text,'')),
payload_detail_present=accessibility_tree_json IS NOT NULL OR text_json IS NOT NULL;
"#;
-const BYTES: &str = "COALESCE(length(CAST(NEW.full_text AS BLOB)),0)+COALESCE(length(CAST(NEW.accessibility_text AS BLOB)),0)+COALESCE(length(CAST(NEW.accessibility_tree_json AS BLOB)),0)+COALESCE(length(CAST(NEW.text_json AS BLOB)),0)";
+pub(super) const BYTES: &str = "COALESCE(length(CAST(NEW.full_text AS BLOB)),0)+COALESCE(length(CAST(NEW.accessibility_text AS BLOB)),0)+COALESCE(length(CAST(NEW.accessibility_tree_json AS BLOB)),0)+COALESCE(length(CAST(NEW.text_json AS BLOB)),0)";
const EMPTY_SURFACES: &str = "(CASE WHEN COALESCE(NEW.full_text,'')='' THEN 1 ELSE 0 END | CASE WHEN COALESCE(NEW.accessibility_text,'')='' THEN 2 ELSE 0 END | CASE WHEN COALESCE(NEW.accessibility_tree_json,'')='' THEN 4 ELSE 0 END | CASE WHEN COALESCE(NEW.text_json,'')='' THEN 8 ELSE 0 END | CASE WHEN COALESCE(NEW.window_name,'')='' THEN 16 ELSE 0 END | CASE WHEN COALESCE(NEW.browser_url,'')='' THEN 32 ELSE 0 END)";
fn triggers() -> String {
diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rs
index c043a96e7..c927318de 100644
--- a/crates/screenpipe-db/tests/in_place_migration.rs
+++ b/crates/screenpipe-db/tests/in_place_migration.rs
@@ -25,57 +25,163 @@ async fn fixture(root: &std::path::Path) {
}
#[tokio::test]
-async fn migration_diagnostics_identify_the_record_that_exceeds_the_budget() {
- use screenpipe_db::storage::diagnostics;
- use std::time::Duration;
+async fn oversized_legacy_bulk_records_do_not_block_migration_or_recording() {
let root = tempfile::tempdir().unwrap();
- let db = DatabaseManager::new(
- root.path().join("db.sqlite").to_str().unwrap(),
- Default::default(),
+ let path = root.path().join("db.sqlite");
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-16','history'); INSERT INTO audio_chunks(id,file_path) VALUES(1,'test.wav');").await.unwrap();
+ db.execute_raw_sql_write(
+ "INSERT INTO meetings(id,meeting_start,meeting_app) VALUES(1,'2026-09-16','test')",
)
.await
.unwrap();
+ let other_payloads = [
+ ("ui_events", "text_content", "INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(?1,'2026-09-16','text',?2)"),
+ ("semantic_items", "body", "INSERT INTO semantic_items(id,entity_fingerprint,version_fingerprint,kind,item_key,identity_quality,title,body,metadata_json) VALUES(?1,randomblob(32),randomblob(32),'document','doc-'||?1,'stable','title',?2,'{}')"),
+ ("pipe_executions", "stdout", "INSERT INTO pipe_executions(id,pipe_name,status,finished_at,stdout) VALUES(?1,'test','completed','2026-09-16',?2)"),
+ ("meeting_transcript_segments", "transcript", "INSERT INTO meeting_transcript_segments(id,meeting_id,provider,item_id,transcript,captured_at) VALUES(?1,1,'test','item-'||?1,?2,'2026-09-16')"),
+ ("outputs", "preview", "INSERT INTO outputs(id,source,title,output_path,preview) VALUES(?1,'test','output','test-'||?1,?2)"),
+ ];
+ let large = "oversized searchable 東京 ".repeat(256);
let mut tx = db.begin_immediate_with_retry().await.unwrap();
- sqlx::query("INSERT INTO frames(id,timestamp) VALUES(1,'2026-09-16')")
+ for id in 1..=4 {
+ let text = if id == 2 || id == 4 {
+ large.as_str()
+ } else {
+ "ordinary searchable"
+ };
+ sqlx::query("INSERT INTO elements(id,frame_id,source,role,text,properties) VALUES(?,1,'accessibility','AXText',?,?)")
+ .bind(id).bind(text).bind(text).execute(&mut **tx.conn()).await.unwrap();
+ sqlx::query("INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,device) VALUES(?,1,?,'2026-09-16',?,'test')")
+ .bind(id).bind(id).bind(format!("{text} {id}")).execute(&mut **tx.conn()).await.unwrap();
+ }
+ for (_, _, sql) in other_payloads {
+ for id in 1..=4 {
+ let text = if id == 2 || id == 4 {
+ large.as_str()
+ } else {
+ "ordinary searchable"
+ };
+ sqlx::query(sql)
+ .bind(id)
+ .bind(text)
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ }
+ }
+ for table in ["elements", "audio_transcriptions", "ui_events"] {
+ sqlx::query(sqlx::AssertSqlSafe(format!(
+ "UPDATE {table} SET redacted_at=1 WHERE id<4"
+ )))
.execute(&mut **tx.conn())
.await
.unwrap();
- sqlx::query("INSERT INTO elements(id,frame_id,source,role,properties) VALUES(7,1,'accessibility','AXText',?)")
- .bind("p".repeat(2048)).execute(&mut **tx.conn()).await.unwrap();
+ }
tx.commit().await.unwrap();
db.close().await;
let mut options = MigrationOptions::default();
+ options.privacy.identity = "migration-regression-policy".into();
+ options.privacy.required_surfaces = 1;
options.budget.record_bytes = 1024;
options.budget.file_bytes = 1024;
- let error = migrate(root.path(), Default::default(), options)
+ options.budget.decode_bytes = 2048;
+ options.budget.response_bytes = 1024;
+ options.budget.staging_bytes = 1024;
+ migrate(root.path(), Default::default(), options)
.await
- .unwrap_err();
- assert!(error
- .to_string()
- .contains("element record exceeds sealing budget"));
- let snapshot = tokio::time::timeout(Duration::from_secs(3), async {
- loop {
- if let Some(snapshot) = diagnostics::recent(root.path())
- .unwrap()
- .into_iter()
- .find(|s| s.status == "failed")
- {
- break snapshot;
- }
- tokio::time::sleep(Duration::from_millis(10)).await;
+ .unwrap();
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.verify_storage().await.unwrap();
+ for id in [2, 4] {
+ let text: String = sqlx::query_scalar("SELECT text FROM elements WHERE id=?")
+ .bind(id)
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ assert_eq!(text, large);
+ let text: String =
+ sqlx::query_scalar("SELECT transcription FROM audio_transcriptions WHERE id=?")
+ .bind(id)
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ assert_eq!(text, format!("{large} {id}"));
+ }
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(
+ "SELECT count(*) FROM _bulk_element_rows WHERE _archive_deleted=0"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 2
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(
+ "SELECT count(*) FROM main.audio_transcriptions WHERE _archive_file IS NOT NULL"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 2
+ );
+ for (table, column, _) in other_payloads {
+ for id in [2, 4] {
+ let actual: String = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
+ "SELECT {column} FROM {table} WHERE id=?"
+ )))
+ .bind(id)
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ assert_eq!(actual, large);
}
- })
- .await
- .unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(sqlx::AssertSqlSafe(format!(
+ "SELECT count(*) FROM main.{table} WHERE _archive_file IS NOT NULL"
+ )))
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 2
+ );
+ }
+ for table in [
+ "elements",
+ "audio_transcriptions",
+ "ui_events",
+ "semantic_items",
+ ] {
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(sqlx::AssertSqlSafe(format!(
+ "SELECT count(*) FROM {table}_fts WHERE {table}_fts MATCH 'oversized'"
+ )))
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 2
+ );
+ }
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-16','new recording'); INSERT INTO elements(id,frame_id,source,role,text) VALUES(5,2,'accessibility','AXText','new recording');").await.unwrap();
+ while db.seal_payloads().await.unwrap() != 0 {}
+ db.close().await;
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.verify_storage().await.unwrap();
assert_eq!(
- snapshot.failure_stage.as_deref(),
- Some("reading_elements_to_seal")
+ sqlx::query_scalar::<_, String>("SELECT text FROM elements WHERE id=5")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ "new recording"
);
- assert_eq!(snapshot.table.as_deref(), Some("elements"));
- assert_eq!((snapshot.first_id, snapshot.last_id), (Some(7), Some(7)));
- assert!(snapshot.batch_bytes.unwrap() > 1024);
- assert_eq!(snapshot.error.as_deref(), Some(error.to_string().as_str()));
- assert!(!root.path().join("storage-migration-complete.json").exists());
+ db.close().await;
}
#[tokio::test]
@@ -979,6 +1085,67 @@ async fn recording_recovery_is_durable_when_interrupted_before_or_after_activati
}
}
+#[tokio::test]
+async fn migration_verification_does_not_use_the_response_payload_budget() {
+ for retain_oversized in [false, true] {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("db.sqlite");
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ let ordinary = "detail 東京\0".repeat(40);
+ let oversized = ordinary.repeat(4);
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ for id in 1..=132 {
+ let detail = if retain_oversized && id == 1 {
+ &oversized
+ } else {
+ &ordinary
+ };
+ sqlx::query("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(?,'2026-09-16T12:00:00Z','history',?)")
+ .bind(id).bind(detail).execute(&mut **tx.conn()).await.unwrap();
+ }
+ tx.commit().await.unwrap();
+ db.close().await;
+ let mut options = MigrationOptions::default();
+ options.budget.row_group_rows = 1;
+ options.budget.file_rows = 1;
+ options.budget.file_bytes = 1024;
+ options.budget.record_bytes = 1024;
+ options.budget.response_bytes = 1024;
+ assert!(ordinary.len() < options.budget.record_bytes);
+ assert!(oversized.len() > options.budget.response_bytes);
+ let report = migrate(root.path(), Default::default(), options)
+ .await
+ .unwrap();
+ assert_eq!(report.frames, 132);
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.verify_storage().await.unwrap();
+ assert!(db
+ .frame_payloads(&[2, 3], Projection::All)
+ .await
+ .unwrap_err()
+ .to_string()
+ .contains("response payload budget exceeded"));
+ if retain_oversized {
+ let (state, detail): (String, String) = sqlx::query_as("SELECT p.state,f.accessibility_tree_json FROM frames f JOIN frame_payloads p ON p.frame_id=f.id WHERE f.id=1")
+ .fetch_one(&db.pool).await.unwrap();
+ assert_eq!(state, "staged");
+ assert_eq!(detail, oversized);
+ }
+ assert_eq!(
+ db.frame_payloads(&[132], Projection::All).await.unwrap()[&132]
+ .accessibility_tree_json
+ .as_deref(),
+ Some(ordinary.as_str())
+ );
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(133,'2026-09-16T12:01:00Z','recording after verification')").await.unwrap();
+ db.close().await;
+ }
+}
+
#[tokio::test]
async fn oversized_legacy_frames_remain_readable_without_blocking_migration_or_capture() {
let root = tempfile::tempdir().unwrap();
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index a72116215..95689beb0 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 33
- Mapped Rust files: 356
-- Active test blocks: 3502
+- Active test blocks: 3503
- Ignored/manual test blocks: 144
-- Declared test blocks: 3646
-- Weighted coverage points: 2888.4
+- Declared test blocks: 3647
+- Weighted coverage points: 2889.4
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3359 | 138 | 2823.3 | 21 | 11 | 100% |
-| macos | 30 | 3421 | 119 | 2837.3 | 22 | 11 | 100% |
-| linux | 26 | 2989 | 111 | 2482.6 | 20 | 11 | 100% |
+| windows | 30 | 3360 | 138 | 2824.3 | 21 | 11 | 100% |
+| macos | 30 | 3422 | 119 | 2838.3 | 22 | 11 | 100% |
+| linux | 26 | 2990 | 111 | 2483.6 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |
-| screenpipe-db | 5 | 56 | 23 | 533 | 21 | 506.3 | 9 |
+| screenpipe-db | 5 | 56 | 23 | 534 | 21 | 507.3 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts |
| audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 6 suites / 481 active / 17 ignored / 454.3 pts | 6 suites / 481 active / 17 ignored / 454.3 pts | 6 suites / 481 active / 17 ignored / 454.3 pts |
+| database | 6 suites / 482 active / 17 ignored / 455.3 pts | 6 suites / 482 active / 17 ignored / 455.3 pts | 6 suites / 482 active / 17 ignored / 455.3 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts |
| local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts |
| meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1535 active / 72 ignored / 1355.5 pts | 14 suites / 1642 active / 76 ignored / 1398.3 pts | 13 suites / 1535 active / 72 ignored / 1355.5 pts |
+| performance | 13 suites / 1536 active / 72 ignored / 1356.5 pts | 14 suites / 1643 active / 76 ignored / 1399.3 pts | 13 suites / 1536 active / 72 ignored / 1356.5 pts |
| pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
| privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts |
| real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - |
| speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |
-| storage | 4 suites / 598 active / 34 ignored / 492.1 pts | 4 suites / 598 active / 34 ignored / 492.1 pts | 4 suites / 598 active / 34 ignored / 492.1 pts |
+| storage | 4 suites / 599 active / 34 ignored / 493.1 pts | 4 suites / 599 active / 34 ignored / 493.1 pts | 4 suites / 599 active / 34 ignored / 493.1 pts |
| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
-| timeline | 4 suites / 1145 active / 38 ignored / 932.3 pts | 4 suites / 1145 active / 38 ignored / 932.3 pts | 4 suites / 1145 active / 38 ignored / 932.3 pts |
+| timeline | 4 suites / 1146 active / 38 ignored / 933.3 pts | 4 suites / 1146 active / 38 ignored / 933.3 pts | 4 suites / 1146 active / 38 ignored / 933.3 pts |
| transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts |
| ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |
| vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |
@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 29 | 245 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 29 | 246 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -304,7 +304,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 |
| db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 |
| db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |
-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 14 | 4 | 18 |
+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 15 | 4 | 19 |
| db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 |
| db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 |
| db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |