YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): finish migration without rejecting oversized history (#7056)
Ezra Ellettecd14a077c012
diff --git a/COVERAGE.md b/COVERAGE.mdindex f6db5d25e..a955e061e 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -36,15 +36,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 33- Mapped Rust files: 356-- Active test blocks: 3502+- Active test blocks: 3503- Ignored/manual test blocks: 144-- Weighted coverage points: 2888.4+- Weighted coverage points: 2889.4| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3359 | 138 | 2823.3 | 21 | 11 | 100% |-| macos | 30 | 3421 | 119 | 2837.3 | 22 | 11 | 100% |-| linux | 26 | 2989 | 111 | 2482.6 | 20 | 11 | 100% |+| windows | 30 | 3360 | 138 | 2824.3 | 21 | 11 | 100% |+| macos | 30 | 3422 | 119 | 2838.3 | 22 | 11 | 100% |+| linux | 26 | 2990 | 111 | 2483.6 | 20 | 11 | 100% |## Refreshdiff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rsindex 139466c3f..4fbfa31e3 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs@@ -53,7 +53,7 @@ pub(crate) async fn seal(crate::storage::diagnostics::stage("selecting_staged_elements");let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!("SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND ({}) ORDER BY id LIMIT 1",- TABLE.eligible+ TABLE.sealable()))).fetch_optional(pool).await?;@@ -178,10 +178,10 @@ async fn rewrite(sqlx::query_as("SELECT policy,required_surfaces FROM storage_metadata").fetch_one(pool).await?;- let (first, last, limit) = if let Some((first, last, _)) = range {+ let (first, last) = if let Some((first, last, _)) = range {let blocked: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(format!("SELECT EXISTS(SELECT 1 FROM elements WHERE id BETWEEN ? AND ? AND NOT ({}))",- TABLE.eligible+ TABLE.sealable()))).bind(first).bind(last)@@ -190,18 +190,14 @@ async fn rewrite(if blocked {return Ok(0);}- (first, last, String::new())+ (first, last)} else {let next: Option<i64> =sqlx::query_scalar("SELECT min(first_id) FROM _bulk_element_ranges WHERE first_id>?").bind(id).fetch_one(pool).await?;- (- id,- next.map_or(i64::MAX, |n| n - 1),- format!(" LIMIT {FILE_ROWS}"),- )+ (id, next.map_or(i64::MAX, |n| n - 1))};// Release each read cursor before encoding takes the writer. Offline// migration has one connection; a recovered resident range can exceed one@@ -210,43 +206,41 @@ async fn rewrite(let mut files = Vec::new();loop {let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));- let sql=format!("SELECT id,_archive_generation,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id{limit}",names(),TABLE.eligible);- crate::storage::diagnostics::batch(- "elements",- Some(after.unwrap_or(first)),- Some(last),- None,- None,- );- crate::storage::diagnostics::stage("reading_elements_to_seal");- let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql))- .bind(first)- .bind(last)- .fetch(pool);- let mut rows = Vec::new();- let mut bytes = 0;- while let Some(row) = stream.try_next().await? {- let row = record(&row)?;- if row.bytes() > storage.descriptor.budget.record_bytes {- crate::storage::diagnostics::batch(- "elements",- Some(row.id),- Some(row.id),- Some(1),- Some(row.bytes() as u64),- );- return Err(storage_error("element record exceeds sealing budget"));- }- if !rows.is_empty()- && (rows.len() == FILE_ROWS- || bytes + row.bytes() > storage.descriptor.budget.file_bytes)+ crate::storage::diagnostics::stage("selecting_element_batch");+ let candidates: Vec<(i64, i64)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(+ "SELECT id,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id LIMIT {FILE_ROWS}",+ TABLE.all_bytes(""), TABLE.eligible+ )))+ .bind(first).bind(last).fetch_all(pool).await?;+ let mut ids = Vec::new();+ let mut bytes = 0usize;+ for (id, size) in candidates {+ let size = usize::try_from(size).map_err(storage_error)?;+ // End the range before a retained record so future rewrites never+ // need to decode that record or remove its SQLite contents.+ if size > storage.descriptor.budget.record_bytes+ || (!ids.is_empty()+ && bytes.saturating_add(size) > storage.descriptor.budget.file_bytes){break;}- bytes += row.bytes();- rows.push(row);+ ids.push(id);+ bytes += size;}- drop(stream);+ crate::storage::diagnostics::batch(+ "elements",+ ids.first().copied(),+ ids.last().copied(),+ Some(ids.len() as u64),+ Some(bytes as u64),+ );+ crate::storage::diagnostics::stage("reading_elements_to_seal");+ let rows = sqlx::query(sqlx::AssertSqlSafe(format!(+ "SELECT id,_archive_generation,{} FROM elements WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id", names()+ )))+ .bind(serde_json::to_string(&ids).map_err(storage_error)?)+ .fetch_all(pool).await?+ .iter().map(record).collect::<Result<Vec<_>, _>>()?;let Some(last_row) = rows.last() else { break };after = Some(last_row.id);files.push(encode(storage, writer, rows).await?);@@ -284,7 +278,7 @@ async fn rewrite(let selection = if range.is_some() {"1".into()} else {- format!("_archive_deleted=1 OR ({})", TABLE.eligible)+ format!("_archive_deleted=1 OR ({})", TABLE.sealable())};let freed:i64=sqlx::query_scalar(sqlx::AssertSqlSafe(format!("SELECT COALESCE(SUM({}),0) FROM _bulk_element_rows WHERE id BETWEEN ? AND ? AND ({selection})",TABLE.all_bytes("")))).bind(first).bind(last).fetch_one(&mut *tx).await?;sqlx::query(sqlx::AssertSqlSafe(format!(diff --git a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/lifecycle.rsindex 3ebfe2d5f..fdd3293cb 100644--- a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs@@ -48,31 +48,16 @@ impl HybridStorage {};// Keep the pending index as the outer loop so LIMIT bounds work before// materializing IDs from a large history.- let candidates=sqlx::query(sqlx::AssertSqlSafe(format!("SELECT v.id,({bytes}) AS bytes FROM main.{t} pending CROSS JOIN (SELECT * FROM {view} WHERE {eligible}) v ON v.id=pending.id WHERE {condition} ORDER BY pending.id LIMIT {FILE_ROWS}",bytes=table.all_bytes("v."),view=table.view(),t=table.name,eligible=table.eligible))).fetch_all(pool).await?;+ let candidates=sqlx::query(sqlx::AssertSqlSafe(format!("SELECT v.id,({bytes}) AS bytes FROM main.{t} pending CROSS JOIN (SELECT * FROM {view} WHERE {eligible}) v ON v.id=pending.id WHERE {condition} AND ({bytes})<=? ORDER BY pending.id LIMIT {FILE_ROWS}",bytes=table.all_bytes("v."),view=table.view(),t=table.name,eligible=table.eligible))).bind(self.descriptor.budget.record_bytes as i64).fetch_all(pool).await?;let mut ids = Vec::new();let mut bytes = 0;for candidate in candidates {let size = candidate.try_get::<i64, _>("bytes")? as usize;- if size > self.descriptor.budget.record_bytes {- let id = candidate.try_get::<i64, _>("id")?;- crate::storage::diagnostics::batch(- table.name,- Some(id),- Some(id),- Some(1),- Some(size as u64),- );- return Err(storage_error("bulk record exceeds sealing budget"));- }- if file.is_none() && !ids.is_empty() && bytes + size > self.descriptor.budget.file_bytes- {+ if !ids.is_empty() && bytes + size > self.descriptor.budget.file_bytes {break;}ids.push(candidate.try_get::<i64, _>("id")?);bytes += size;- if bytes > self.descriptor.budget.decode_bytes / 2 {- return Err(storage_error("bulk batch exceeds sealing budget"));- }}if ids.is_empty() {return Ok(Vec::new());diff --git a/crates/screenpipe-db/src/storage/bulk/mod.rs b/crates/screenpipe-db/src/storage/bulk/mod.rsindex 55ba6ae36..3078b7a38 100644--- a/crates/screenpipe-db/src/storage/bulk/mod.rs+++ b/crates/screenpipe-db/src/storage/bulk/mod.rs@@ -110,6 +110,15 @@ pub(super) static TABLES: &[Table] = &[];impl Table {+ // An encoder bound selects work; it cannot reject valid resident history.+ fn sealable(&self) -> String {+ format!(+ "({}) AND ({})<=(SELECT record_limit FROM storage_metadata)",+ self.eligible,+ self.all_bytes("")+ )+ }+fn mask(&self) -> i64 {(1 << self.columns.len()) - 1}diff --git a/crates/screenpipe-db/src/storage/import.rs b/crates/screenpipe-db/src/storage/import.rsindex 81f01a7ef..f78298a27 100644--- a/crates/screenpipe-db/src/storage/import.rs+++ b/crates/screenpipe-db/src/storage/import.rs@@ -1,15 +1,7 @@// screenpipe — AI that knows everything you've seen, said, or heard// https://screenpipe.com-//! Bounded resident-row batches shared by the in-place frame and element conversion.--use super::{storage_error, StorageBudget};-use futures::TryStreamExt;-use sqlx::{sqlite::SqliteRow, Row, SqliteConnection, TypeInfo, ValueRef};--fn quote(name: &str) -> String {- format!("\"{}\"", name.replace('"', "\"\""))-}+//! Resident table columns used by SQL-to-SQL migration batches.pub(super) async fn columns(source: &sqlx::SqlitePool,@@ -20,148 +12,3 @@ pub(super) async fn columns(.fetch_all(source).await}--pub(super) async fn batch(- source: &sqlx::SqlitePool,- table: &str,- columns: &[String],- after: Option<i64>,- budget: &StorageBudget,-) -> Result<Vec<SqliteRow>, sqlx::Error> {- let sql = format!(- "SELECT rowid,{} FROM {} WHERE rowid{}? ORDER BY rowid LIMIT {}",- columns- .iter()- .map(|c| quote(c))- .collect::<Vec<_>>()- .join(","),- quote(table),- if after.is_some() { ">" } else { ">=" },- if table == "frames" {- budget.file_rows- } else {- super::bulk::FILE_ROWS- }- );- super::diagnostics::batch(table, after, None, None, None);- super::diagnostics::stage("reading_resident_batch");- let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql))- .bind(after.unwrap_or(i64::MIN))- .fetch(source);- let mut rows = Vec::new();- let mut bytes = 0;- while let Some(row) = stream.try_next().await? {- let mut size = 0;- for i in 0..row.len() {- let raw = row.try_get_raw(i)?;- size += if raw.is_null() {- 0- } else {- match raw.type_info().name() {- "INTEGER" | "REAL" => 8,- _ => row.try_get::<Vec<u8>, _>(i)?.len(),- }- };- }- if size > budget.decode_bytes / 2 {- super::diagnostics::batch(- table,- Some(row.get(0)),- Some(row.get(0)),- Some(1),- Some(size as u64),- );- return Err(storage_error(format!(- "migration record in {table} exceeds decode budget"- )));- }- if !rows.is_empty() && bytes + size > budget.file_bytes {- break;- }- bytes += size;- rows.push(row);- }- super::diagnostics::batch(- table,- rows.first().map(|r| r.get(0)),- rows.last().map(|r| r.get(0)),- Some(rows.len() as u64),- Some(bytes as u64),- );- Ok(rows)-}--pub(super) async fn insert(- conn: &mut SqliteConnection,- table: &str,- columns: &[String],- rows: &[SqliteRow],- elements: bool,-) -> Result<(), sqlx::Error> {- let destination = if elements {- "_bulk_element_rows"- } else {- table- };- let prefix = format!(- "INSERT INTO {}(rowid,{}{}) ",- quote(destination),- columns- .iter()- .map(|c| quote(c))- .collect::<Vec<_>>()- .join(","),- if elements { ",_archive_generation" } else { "" }- );- // SQLite's parameter limit bounds statement size independently of payload- // bytes. Values retain their original SQLite storage class.- for chunk in rows.chunks(32766 / (columns.len() + 2)) {- let mut query = sqlx::QueryBuilder::<sqlx::Sqlite>::new(&prefix);- let mut error = None;- query.push_values(chunk, |mut values, row| {- for i in 0..row.len() {- let value = (|| -> Result<(), sqlx::Error> {- let raw = row.try_get_raw(i)?;- if raw.is_null() {- values.push_bind(None::<i64>);- } else {- match raw.type_info().name() {- "INTEGER" => {- values.push_bind(row.try_get::<i64, _>(i)?);- }- "REAL" => {- values.push_bind(row.try_get::<f64, _>(i)?);- }- "TEXT" => {- values.push_bind(row.try_get::<String, _>(i)?);- }- _ => {- values.push_bind(row.try_get::<Vec<u8>, _>(i)?);- }- }- }- Ok(())- })();- if let Err(e) = value {- error = Some(e);- }- }- if elements {- values.push_bind(1_i64);- }- });- if let Some(error) = error {- return Err(error);- }- query.build().execute(&mut *conn).await?;- }- if elements {- super::bulk::elements::import_batch(- conn,- rows.first().unwrap().try_get(0)?,- rows.last().unwrap().try_get(0)?,- )- .await?;- }- Ok(())-}diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rsindex e350b76dd..106840530 100644--- a/crates/screenpipe-db/src/storage/in_place.rs+++ b/crates/screenpipe-db/src/storage/in_place.rs@@ -140,13 +140,13 @@ async fn run(schema::finish_step(&mut tx,"suspend-original-triggers").await?;tx.commit().await?;}+ if schema::converted_step(&mut conn,"conversion-complete").await? { return Ok(()); }// A paused migration records resident payloads like the original// SQLite store. Its disabled sealer must not impose a staging cap// that eventually stops recording. Explicit completion restores it.sqlx::query("UPDATE storage_metadata SET staging_limit=?").bind(if archive { storage.descriptor.budget.staging_bytes as i64 } else { i64::MAX }).execute(&mut *conn).await?;- if schema::converted_step(&mut conn,"conversion-complete").await? { return Ok(()); }schema::construction_checkpoint(&mut conn).await?;}// Encoding reservations left by an interrupted attempt are handled by@@ -196,32 +196,22 @@ async fn run(report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;continue;}- let rows = if archive {- super::import::batch(&pool, "_bulk_elements_source", &columns, None, &storage.descriptor.budget).await?- } else { Vec::new() };- let range = if archive {- rows.first().zip(rows.last()).map(|(first,last)| (first.get::<i64,_>(0),last.get::<i64,_>(0)))- } else {- resident_range(&pool, "_bulk_elements_source", None, storage.descriptor.budget.file_bytes, None).await?- };+ // Move existing records inside SQLite, including records larger+ // than any encoder/decoder batch. Sealing can leave them resident.+ let range = resident_range(&pool, "_bulk_elements_source", None,+ storage.descriptor.budget.file_bytes, archive.then_some(bulk::FILE_ROWS)).await?;let Some((first, last)) = range else { break };- super::diagnostics::batch("elements", Some(first), Some(last), Some(rows.len() as u64), None);+ super::diagnostics::batch("elements", Some(first), Some(last), None, None);super::diagnostics::stage("waiting_for_element_writer");let permit = writer.lock().await?;super::diagnostics::stage("waiting_for_element_connection");let mut conn = permit.pool().acquire().await?;let mut tx = conn.begin().await?;super::diagnostics::stage("staging_element_rows");- if archive {- super::import::insert(&mut tx, "elements", &columns, &rows, true).await?;- } else {- // SQL-to-SQL batches also recover pre-existing records that- // exceed the archive record budget, without decoding them.- let names = columns.iter().map(|c| format!("\"{}\"", c.replace('"', "\"\""))).collect::<Vec<_>>().join(",");- sqlx::query(sqlx::AssertSqlSafe(format!("INSERT INTO _bulk_element_rows({names},_archive_generation) SELECT {names},1 FROM _bulk_elements_source WHERE id BETWEEN ? AND ?")))- .bind(first).bind(last).execute(&mut *tx).await?;- bulk::elements::import_batch(&mut tx, first, last).await?;- }+ let names = columns.iter().map(|c| format!("\"{}\"", c.replace('"', "\"\""))).collect::<Vec<_>>().join(",");+ sqlx::query(sqlx::AssertSqlSafe(format!("INSERT INTO _bulk_element_rows({names},_archive_generation) SELECT {names},1 FROM _bulk_elements_source WHERE id BETWEEN ? AND ?")))+ .bind(first).bind(last).execute(&mut *tx).await?;+ bulk::elements::import_batch(&mut tx, first, last).await?;super::diagnostics::stage("deleting_staged_source_elements");sqlx::query("DELETE FROM _bulk_elements_source WHERE id BETWEEN ? AND ?").bind(first).bind(last).execute(&mut *tx).await?;super::diagnostics::stage("committing_element_staging");@@ -268,6 +258,11 @@ async fn run(let original: Vec<String> = sqlx::query_scalar("SELECT sql FROM _storage_conversion_triggers WHERE restore=1").fetch_all(&mut *tx).await?;for sql in original { sqlx::raw_sql(sqlx::AssertSqlSafe(sql)).execute(&mut *tx).await?; }if archive {+ // Existing resident history occupies disk, not the capture work+ // queue. Preserve a full configured staging allowance above it.+ sqlx::query("UPDATE storage_metadata SET staging_limit=staging_bytes+?")+ .bind(storage.descriptor.budget.staging_bytes as i64)+ .execute(&mut *tx).await?;schema::finish_step(&mut tx,"conversion-complete").await?;} else {// Restored application triggers must be suspended again only@@ -395,7 +390,7 @@ async fn backfill_resident_fts(.fetch_optional(pool).await?.flatten();- let sql = format!("SELECT id,{size} FROM {t} WHERE _archive_file IS NULL AND NOT ({eligible}) AND id {comparison} ? ORDER BY id LIMIT {rows}",t=table.name,eligible=if archive { table.eligible } else { "0" },comparison=if after.is_some(){">"}else{">="},rows=bulk::FILE_ROWS);+ let sql = format!("SELECT id,{size} FROM {t} WHERE _archive_file IS NULL AND id {comparison} ? ORDER BY id LIMIT {rows}",t=table.name,comparison=if after.is_some(){">"}else{">="},rows=bulk::FILE_ROWS);let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(after.unwrap_or(i64::MIN)).fetch(pool);@@ -403,11 +398,6 @@ async fn backfill_resident_fts(let mut bytes = 0_usize;while let Some(row) = stream.try_next().await? {let size = row.get::<i64, _>(1) as usize;- if archive && size > storage.descriptor.budget.record_bytes {- return Err(storage_error(- "resident search record exceeds migration budget",- ));- }if !ids.is_empty() && bytes + size > storage.descriptor.budget.file_bytes {break;}diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rsindex fc5285535..6c5ce81ef 100644--- a/crates/screenpipe-db/src/storage/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/lifecycle.rs@@ -427,10 +427,6 @@ async fn migrate_observed(let original = async {let frozen = source.begin_immediate_with_retry().await?;verify_integrity(&source.pool).await?;- let pages: i64 = sqlx::query_scalar("PRAGMA page_count").fetch_one(&source.pool).await?;- if (pages as usize).div_ceil(8) > options.budget.decode_bytes / 2 {- return Err(storage_error("freelist map exceeds migration memory budget; database unchanged"));- }let receipts = table_receipts(&source, None).await?;let terms: Vec<String> = sqlx::query_scalar("SELECT full_text FROM frames WHERE full_text IS NOT NULL AND id IN ((SELECT min(id) FROM frames),(SELECT max(id) FROM frames))").fetch_all(&source.pool).await?;let mut searches = Vec::new();@@ -1074,6 +1070,56 @@ pub(super) async fn logical_tables(db: &DatabaseManager) -> Result<Vec<String>,.fetch_all(&db.pool).await}+// Verification consumes history incrementally; the public response limit must+// never decide whether that history can migrate. Resident frames are hashed+// from SQLite, and only sealed frames need hydration through the payload reader.+async fn verification_frame_batch(+ db: &DatabaseManager,+ after: Option<i64>,+) -> Result<Option<(i64, Vec<i64>)>, sqlx::Error> {+ let comparison = if after.is_some() { ">" } else { ">=" };+ let (size, state, join, limit) = if let Some(storage) = &db.storage {+ (+ "p.bytes".to_owned(),+ "p.state='sealed'",+ "LEFT JOIN frame_payloads p ON p.frame_id=f.id",+ storage+ .descriptor+ .budget+ .file_bytes+ .min(storage.descriptor.budget.response_bytes),+ )+ } else {+ (+ super::schema::BYTES.replace("NEW.", "f."),+ "0",+ "",+ StorageBudget::default().file_bytes,+ )+ };+ let rows: Vec<(i64, i64, bool)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(+ "SELECT f.id,{size},{state} FROM frames f {join} WHERE f.id{comparison}? ORDER BY f.id LIMIT 128"+ )))+ .bind(after.unwrap_or(i64::MIN))+ .fetch_all(&db.pool)+ .await?;+ let mut last = None;+ let mut ids = Vec::new();+ let mut bytes = 0usize;+ for (id, size, sealed) in rows {+ let size = usize::try_from(size).map_err(storage_error)?;+ if last.is_some() && bytes.saturating_add(size) > limit {+ break;+ }+ last = Some(id);+ bytes = bytes.saturating_add(size);+ if sealed {+ ids.push(id);+ }+ }+ Ok(last.map(|last| (last, ids)))+}+pub(super) async fn table_receipts(db: &DatabaseManager,source: Option<&[TableParity]>,@@ -1117,8 +1163,17 @@ pub(super) async fn table_receipts(let mut last = i64::MIN;let mut first = true;loop {+ let (through, sealed) = if table == "frames" {+ let Some(batch) = verification_frame_batch(db, (!first).then_some(last)).await?+ else {+ break;+ };+ batch+ } else {+ (i64::MAX, Vec::new())+ };let sql = format!(- "SELECT {key} AS __storage_rowid,{} FROM {} WHERE {key}{}? ORDER BY {key} LIMIT 128",+ "SELECT {key} AS __storage_rowid,{} FROM {} WHERE {key}{}? AND {key}<=? ORDER BY {key} LIMIT 128",columns.iter().map(|c| quote(c))@@ -1131,6 +1186,7 @@ pub(super) async fn table_receipts(super::diagnostics::stage("reading_parity_rows");let rows = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(last)+ .bind(through).fetch_all(&db.pool).await?;if rows.is_empty() {@@ -1138,9 +1194,8 @@ pub(super) async fn table_receipts(}first = false;super::diagnostics::stage("hashing_parity_rows");- let payloads = if table == "frames" && db.storage.is_some() {- let ids: Vec<i64> = rows.iter().map(|r| r.get("id")).collect();- db.frame_payloads(&ids, Projection::All).await?+ let payloads = if !sealed.is_empty() {+ db.frame_payloads(&sealed, Projection::All).await?} else {Default::default()};@@ -1272,17 +1327,9 @@ impl DatabaseManager {if let Some(storage) = &self.storage {storage.verify_catalog(&self.pool).await?;storage.verify_bulk(&self.pool).await?;- let mut after = i64::MIN;- loop {- let ids: Vec<i64> =- sqlx::query_scalar("SELECT id FROM frames WHERE id>? ORDER BY id LIMIT 128")- .bind(after)- .fetch_all(&self.pool)- .await?;- let Some(last) = ids.last() else {- break;- };- after = *last;+ let mut after = None;+ while let Some((last, ids)) = verification_frame_batch(self, after).await? {+ after = Some(last);self.frame_payloads(&ids, Projection::All).await?;}}diff --git a/crates/screenpipe-db/src/storage/reclaim.rs b/crates/screenpipe-db/src/storage/reclaim.rsindex 052d1fbad..a756a1655 100644--- a/crates/screenpipe-db/src/storage/reclaim.rs+++ b/crates/screenpipe-db/src/storage/reclaim.rs@@ -181,10 +181,18 @@ pub(super) async fn free_leaves(return Ok(());}let bitmap_bytes = (pages as usize).div_ceil(8);- if pages <= 0 || bitmap_bytes > memory_limit / 2 {- return Err(storage_error(- "freelist map exceeds migration memory budget",- ));+ if pages <= 0 {+ return Err(storage_error("invalid SQLite page count"));+ }+ if bitmap_bytes > memory_limit / 2 {+ // Reclamation is an optimization. Keep these valid free pages in+ // SQLite when its validation map would exceed the working budget.+ tracing::debug!(+ bitmap_bytes,+ memory_limit,+ "deferring SQLite space reclamation"+ );+ return Ok(());}let mut header = [0_u8; 100];file.seek(SeekFrom::Start(0))?;@@ -447,11 +455,11 @@ mod tests {.unwrap();let length = file.metadata().unwrap().len();let before = allocated(&path).unwrap();- assert!(free_leaves(&mut conn, &mut file, 0).await.is_err());+ free_leaves(&mut conn, &mut file, 0).await.unwrap();assert_eq!(allocated(&path).unwrap(),before,- "rejected scan must not punch anything"+ "deferred scan must not punch anything");free_leaves(&mut conn, &mut file, 1024 * 1024).awaitdiff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rsindex 509d63a25..c18fdb5bc 100644--- a/crates/screenpipe-db/src/storage/schema.rs+++ b/crates/screenpipe-db/src/storage/schema.rs@@ -234,7 +234,7 @@ UPDATE frames SET payload_full_text_length=length(COALESCE(full_text,'')),payload_detail_present=accessibility_tree_json IS NOT NULL OR text_json IS NOT NULL;"#;-const BYTES: &str = "COALESCE(length(CAST(NEW.full_text AS BLOB)),0)+COALESCE(length(CAST(NEW.accessibility_text AS BLOB)),0)+COALESCE(length(CAST(NEW.accessibility_tree_json AS BLOB)),0)+COALESCE(length(CAST(NEW.text_json AS BLOB)),0)";+pub(super) const BYTES: &str = "COALESCE(length(CAST(NEW.full_text AS BLOB)),0)+COALESCE(length(CAST(NEW.accessibility_text AS BLOB)),0)+COALESCE(length(CAST(NEW.accessibility_tree_json AS BLOB)),0)+COALESCE(length(CAST(NEW.text_json AS BLOB)),0)";const EMPTY_SURFACES: &str = "(CASE WHEN COALESCE(NEW.full_text,'')='' THEN 1 ELSE 0 END | CASE WHEN COALESCE(NEW.accessibility_text,'')='' THEN 2 ELSE 0 END | CASE WHEN COALESCE(NEW.accessibility_tree_json,'')='' THEN 4 ELSE 0 END | CASE WHEN COALESCE(NEW.text_json,'')='' THEN 8 ELSE 0 END | CASE WHEN COALESCE(NEW.window_name,'')='' THEN 16 ELSE 0 END | CASE WHEN COALESCE(NEW.browser_url,'')='' THEN 32 ELSE 0 END)";fn triggers() -> String {diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rsindex c043a96e7..c927318de 100644--- a/crates/screenpipe-db/tests/in_place_migration.rs+++ b/crates/screenpipe-db/tests/in_place_migration.rs@@ -25,57 +25,163 @@ async fn fixture(root: &std::path::Path) {}#[tokio::test]-async fn migration_diagnostics_identify_the_record_that_exceeds_the_budget() {- use screenpipe_db::storage::diagnostics;- use std::time::Duration;+async fn oversized_legacy_bulk_records_do_not_block_migration_or_recording() {let root = tempfile::tempdir().unwrap();- let db = DatabaseManager::new(- root.path().join("db.sqlite").to_str().unwrap(),- Default::default(),+ let path = root.path().join("db.sqlite");+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-16','history'); INSERT INTO audio_chunks(id,file_path) VALUES(1,'test.wav');").await.unwrap();+ db.execute_raw_sql_write(+ "INSERT INTO meetings(id,meeting_start,meeting_app) VALUES(1,'2026-09-16','test')",).await.unwrap();+ let other_payloads = [+ ("ui_events", "text_content", "INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(?1,'2026-09-16','text',?2)"),+ ("semantic_items", "body", "INSERT INTO semantic_items(id,entity_fingerprint,version_fingerprint,kind,item_key,identity_quality,title,body,metadata_json) VALUES(?1,randomblob(32),randomblob(32),'document','doc-'||?1,'stable','title',?2,'{}')"),+ ("pipe_executions", "stdout", "INSERT INTO pipe_executions(id,pipe_name,status,finished_at,stdout) VALUES(?1,'test','completed','2026-09-16',?2)"),+ ("meeting_transcript_segments", "transcript", "INSERT INTO meeting_transcript_segments(id,meeting_id,provider,item_id,transcript,captured_at) VALUES(?1,1,'test','item-'||?1,?2,'2026-09-16')"),+ ("outputs", "preview", "INSERT INTO outputs(id,source,title,output_path,preview) VALUES(?1,'test','output','test-'||?1,?2)"),+ ];+ let large = "oversized searchable 東京 ".repeat(256);let mut tx = db.begin_immediate_with_retry().await.unwrap();- sqlx::query("INSERT INTO frames(id,timestamp) VALUES(1,'2026-09-16')")+ for id in 1..=4 {+ let text = if id == 2 || id == 4 {+ large.as_str()+ } else {+ "ordinary searchable"+ };+ sqlx::query("INSERT INTO elements(id,frame_id,source,role,text,properties) VALUES(?,1,'accessibility','AXText',?,?)")+ .bind(id).bind(text).bind(text).execute(&mut **tx.conn()).await.unwrap();+ sqlx::query("INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,device) VALUES(?,1,?,'2026-09-16',?,'test')")+ .bind(id).bind(id).bind(format!("{text} {id}")).execute(&mut **tx.conn()).await.unwrap();+ }+ for (_, _, sql) in other_payloads {+ for id in 1..=4 {+ let text = if id == 2 || id == 4 {+ large.as_str()+ } else {+ "ordinary searchable"+ };+ sqlx::query(sql)+ .bind(id)+ .bind(text)+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ }+ }+ for table in ["elements", "audio_transcriptions", "ui_events"] {+ sqlx::query(sqlx::AssertSqlSafe(format!(+ "UPDATE {table} SET redacted_at=1 WHERE id<4"+ ))).execute(&mut **tx.conn()).await.unwrap();- sqlx::query("INSERT INTO elements(id,frame_id,source,role,properties) VALUES(7,1,'accessibility','AXText',?)")- .bind("p".repeat(2048)).execute(&mut **tx.conn()).await.unwrap();+ }tx.commit().await.unwrap();db.close().await;let mut options = MigrationOptions::default();+ options.privacy.identity = "migration-regression-policy".into();+ options.privacy.required_surfaces = 1;options.budget.record_bytes = 1024;options.budget.file_bytes = 1024;- let error = migrate(root.path(), Default::default(), options)+ options.budget.decode_bytes = 2048;+ options.budget.response_bytes = 1024;+ options.budget.staging_bytes = 1024;+ migrate(root.path(), Default::default(), options).await- .unwrap_err();- assert!(error- .to_string()- .contains("element record exceeds sealing budget"));- let snapshot = tokio::time::timeout(Duration::from_secs(3), async {- loop {- if let Some(snapshot) = diagnostics::recent(root.path())- .unwrap()- .into_iter()- .find(|s| s.status == "failed")- {- break snapshot;- }- tokio::time::sleep(Duration::from_millis(10)).await;+ .unwrap();+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.verify_storage().await.unwrap();+ for id in [2, 4] {+ let text: String = sqlx::query_scalar("SELECT text FROM elements WHERE id=?")+ .bind(id)+ .fetch_one(&db.pool)+ .await+ .unwrap();+ assert_eq!(text, large);+ let text: String =+ sqlx::query_scalar("SELECT transcription FROM audio_transcriptions WHERE id=?")+ .bind(id)+ .fetch_one(&db.pool)+ .await+ .unwrap();+ assert_eq!(text, format!("{large} {id}"));+ }+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM _bulk_element_rows WHERE _archive_deleted=0"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 2+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM main.audio_transcriptions WHERE _archive_file IS NOT NULL"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 2+ );+ for (table, column, _) in other_payloads {+ for id in [2, 4] {+ let actual: String = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(+ "SELECT {column} FROM {table} WHERE id=?"+ )))+ .bind(id)+ .fetch_one(&db.pool)+ .await+ .unwrap();+ assert_eq!(actual, large);}- })- .await- .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>(sqlx::AssertSqlSafe(format!(+ "SELECT count(*) FROM main.{table} WHERE _archive_file IS NOT NULL"+ )))+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 2+ );+ }+ for table in [+ "elements",+ "audio_transcriptions",+ "ui_events",+ "semantic_items",+ ] {+ assert_eq!(+ sqlx::query_scalar::<_, i64>(sqlx::AssertSqlSafe(format!(+ "SELECT count(*) FROM {table}_fts WHERE {table}_fts MATCH 'oversized'"+ )))+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 2+ );+ }+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-16','new recording'); INSERT INTO elements(id,frame_id,source,role,text) VALUES(5,2,'accessibility','AXText','new recording');").await.unwrap();+ while db.seal_payloads().await.unwrap() != 0 {}+ db.close().await;+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.verify_storage().await.unwrap();assert_eq!(- snapshot.failure_stage.as_deref(),- Some("reading_elements_to_seal")+ sqlx::query_scalar::<_, String>("SELECT text FROM elements WHERE id=5")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ "new recording");- assert_eq!(snapshot.table.as_deref(), Some("elements"));- assert_eq!((snapshot.first_id, snapshot.last_id), (Some(7), Some(7)));- assert!(snapshot.batch_bytes.unwrap() > 1024);- assert_eq!(snapshot.error.as_deref(), Some(error.to_string().as_str()));- assert!(!root.path().join("storage-migration-complete.json").exists());+ db.close().await;}#[tokio::test]@@ -979,6 +1085,67 @@ async fn recording_recovery_is_durable_when_interrupted_before_or_after_activati}}+#[tokio::test]+async fn migration_verification_does_not_use_the_response_payload_budget() {+ for retain_oversized in [false, true] {+ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("db.sqlite");+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ let ordinary = "detail 東京\0".repeat(40);+ let oversized = ordinary.repeat(4);+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ for id in 1..=132 {+ let detail = if retain_oversized && id == 1 {+ &oversized+ } else {+ &ordinary+ };+ sqlx::query("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(?,'2026-09-16T12:00:00Z','history',?)")+ .bind(id).bind(detail).execute(&mut **tx.conn()).await.unwrap();+ }+ tx.commit().await.unwrap();+ db.close().await;+ let mut options = MigrationOptions::default();+ options.budget.row_group_rows = 1;+ options.budget.file_rows = 1;+ options.budget.file_bytes = 1024;+ options.budget.record_bytes = 1024;+ options.budget.response_bytes = 1024;+ assert!(ordinary.len() < options.budget.record_bytes);+ assert!(oversized.len() > options.budget.response_bytes);+ let report = migrate(root.path(), Default::default(), options)+ .await+ .unwrap();+ assert_eq!(report.frames, 132);+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.verify_storage().await.unwrap();+ assert!(db+ .frame_payloads(&[2, 3], Projection::All)+ .await+ .unwrap_err()+ .to_string()+ .contains("response payload budget exceeded"));+ if retain_oversized {+ let (state, detail): (String, String) = sqlx::query_as("SELECT p.state,f.accessibility_tree_json FROM frames f JOIN frame_payloads p ON p.frame_id=f.id WHERE f.id=1")+ .fetch_one(&db.pool).await.unwrap();+ assert_eq!(state, "staged");+ assert_eq!(detail, oversized);+ }+ assert_eq!(+ db.frame_payloads(&[132], Projection::All).await.unwrap()[&132]+ .accessibility_tree_json+ .as_deref(),+ Some(ordinary.as_str())+ );+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(133,'2026-09-16T12:01:00Z','recording after verification')").await.unwrap();+ db.close().await;+ }+}+#[tokio::test]async fn oversized_legacy_frames_remain_readable_without_blocking_migration_or_capture() {let root = tempfile::tempdir().unwrap();diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex a72116215..95689beb0 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y- Mapped suites: 33- Mapped Rust files: 356-- Active test blocks: 3502+- Active test blocks: 3503- Ignored/manual test blocks: 144-- Declared test blocks: 3646-- Weighted coverage points: 2888.4+- Declared test blocks: 3647+- Weighted coverage points: 2889.4Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3359 | 138 | 2823.3 | 21 | 11 | 100% |-| macos | 30 | 3421 | 119 | 2837.3 | 22 | 11 | 100% |-| linux | 26 | 2989 | 111 | 2482.6 | 20 | 11 | 100% |+| windows | 30 | 3360 | 138 | 2824.3 | 21 | 11 | 100% |+| macos | 30 | 3422 | 119 | 2838.3 | 22 | 11 | 100% |+| linux | 26 | 2990 | 111 | 2483.6 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |-| screenpipe-db | 5 | 56 | 23 | 533 | 21 | 506.3 | 9 |+| screenpipe-db | 5 | 56 | 23 | 534 | 21 | 507.3 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts || audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |-| database | 6 suites / 481 active / 17 ignored / 454.3 pts | 6 suites / 481 active / 17 ignored / 454.3 pts | 6 suites / 481 active / 17 ignored / 454.3 pts |+| database | 6 suites / 482 active / 17 ignored / 455.3 pts | 6 suites / 482 active / 17 ignored / 455.3 pts | 6 suites / 482 active / 17 ignored / 455.3 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts || engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts || local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts || meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1535 active / 72 ignored / 1355.5 pts | 14 suites / 1642 active / 76 ignored / 1398.3 pts | 13 suites / 1535 active / 72 ignored / 1355.5 pts |+| performance | 13 suites / 1536 active / 72 ignored / 1356.5 pts | 14 suites / 1643 active / 76 ignored / 1399.3 pts | 13 suites / 1536 active / 72 ignored / 1356.5 pts || pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts || privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts || real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - || speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |-| storage | 4 suites / 598 active / 34 ignored / 492.1 pts | 4 suites / 598 active / 34 ignored / 492.1 pts | 4 suites / 598 active / 34 ignored / 492.1 pts |+| storage | 4 suites / 599 active / 34 ignored / 493.1 pts | 4 suites / 599 active / 34 ignored / 493.1 pts | 4 suites / 599 active / 34 ignored / 493.1 pts || sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |-| timeline | 4 suites / 1145 active / 38 ignored / 932.3 pts | 4 suites / 1145 active / 38 ignored / 932.3 pts | 4 suites / 1145 active / 38 ignored / 932.3 pts |+| timeline | 4 suites / 1146 active / 38 ignored / 933.3 pts | 4 suites / 1146 active / 38 ignored / 933.3 pts | 4 suites / 1146 active / 38 ignored / 933.3 pts || transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts || ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts || vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 29 | 245 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 29 | 246 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |@@ -304,7 +304,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 || db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 || db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 14 | 4 | 18 |+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 15 | 4 | 19 || db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 || db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 || db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |