YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): preserve oversized legacy frames during migration (#7046)
Ezra Ellette3cf2f10551b8
diff --git a/COVERAGE.md b/COVERAGE.mdindex 3e9896d3f..a7661ad65 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -36,15 +36,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 33- Mapped Rust files: 355-- Active test blocks: 3491+- Active test blocks: 3494- Ignored/manual test blocks: 144-- Weighted coverage points: 2877.4+- Weighted coverage points: 2880.4| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3348 | 138 | 2812.3 | 21 | 11 | 100% |-| macos | 30 | 3410 | 119 | 2826.3 | 22 | 11 | 100% |-| linux | 26 | 2978 | 111 | 2471.6 | 20 | 11 | 100% |+| windows | 30 | 3351 | 138 | 2815.3 | 21 | 11 | 100% |+| macos | 30 | 3413 | 119 | 2829.3 | 22 | 11 | 100% |+| linux | 26 | 2981 | 111 | 2474.6 | 20 | 11 | 100% |## Refreshdiff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex 372538552..be885ce6d 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -520,6 +520,7 @@ impl DatabaseManager {construction?;}crate::storage::schema::verify(&mut conn, &storage.descriptor).await?;+ crate::storage::schema::upgrade_resident_frames(&mut conn).await?;crate::storage::read_schema::upgrade(&mut conn, storage).await?;storage.verify_catalog(&db_manager.pool).await?;}diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rsindex 61ba33835..5941e5f11 100644--- a/crates/screenpipe-db/src/storage/in_place.rs+++ b/crates/screenpipe-db/src/storage/in_place.rs@@ -164,14 +164,10 @@ async fn run(continue;}let last: Option<i64> = sqlx::query_scalar("SELECT max(frame_id) FROM frame_payloads").fetch_one(&pool).await?;- let range = if archive {- let columns = super::import::columns(&pool, "frames").await?;- let columns: Vec<_> = columns.into_iter().filter(|c| !c.starts_with("payload_")).collect();- let rows = super::import::batch(&pool, "frames", &columns, last, &storage.descriptor.budget).await?;- rows.first().zip(rows.last()).map(|(first,last)| (first.get::<i64,_>(0),last.get::<i64,_>(0)))- } else {- resident_range(&pool, "frames", last, storage.descriptor.budget.file_bytes).await?- };+ // Staging is SQL-to-SQL. Only read IDs and lengths here so even a+ // legacy frame larger than the decoder budget stays in SQLite.+ let range = resident_range(&pool, "frames", last, storage.descriptor.budget.file_bytes,+ archive.then_some(storage.descriptor.budget.file_rows)).await?;let Some((first, last)) = range else { break };let permit = writer.lock().await?;let mut conn = permit.pool().acquire().await?;@@ -198,7 +194,7 @@ async fn run(let range = if archive {rows.first().zip(rows.last()).map(|(first,last)| (first.get::<i64,_>(0),last.get::<i64,_>(0)))} else {- resident_range(&pool, "_bulk_elements_source", None, storage.descriptor.budget.file_bytes).await?+ resident_range(&pool, "_bulk_elements_source", None, storage.descriptor.budget.file_bytes, None).await?};let Some((first, last)) = range else { break };let permit = writer.lock().await?;@@ -289,14 +285,16 @@ async fn resident_range(table: &str,after: Option<i64>,budget_bytes: usize,+ row_limit: Option<usize>,) -> Result<Option<(i64, i64)>, sqlx::Error> {let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));let size = super::import::columns(pool, table).await?.iter().map(|name| {let quoted = format!("\"{}\"", name.replace('"', "\"\""));format!("CASE WHEN typeof({quoted}) IN ('integer','real') THEN 8 ELSE COALESCE(length(CAST({quoted} AS BLOB)),0) END")}).collect::<Vec<_>>().join("+");+ let row_limit = row_limit.unwrap_or(128);let rows: Vec<(i64, i64)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(- "SELECT id,{size} FROM {table} WHERE {lower} ORDER BY id LIMIT 128"+ "SELECT id,{size} FROM {table} WHERE {lower} ORDER BY id LIMIT {row_limit}"))).fetch_all(pool).await?;diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rsindex 4d0864ea9..972578ecd 100644--- a/crates/screenpipe-db/src/storage/mod.rs+++ b/crates/screenpipe-db/src/storage/mod.rs@@ -105,6 +105,15 @@ impl Default for StorageBudget {}impl StorageBudget {+ /// Oversized legacy frames are resident history, not work for the sealer.+ pub(super) fn staged_frame_bytes(&self, bytes: usize) -> usize {+ if bytes <= self.record_bytes {+ bytes+ } else {+ 0+ }+ }+pub fn validate(&self) -> Result<(), sqlx::Error> {if self.row_group_rows == 0|| self.file_rows < self.row_group_rowsdiff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rsindex 4d282bda7..69c5a9925 100644--- a/crates/screenpipe-db/src/storage/schema.rs+++ b/crates/screenpipe-db/src/storage/schema.rs@@ -117,7 +117,8 @@ pub(super) async fn bootstrap_in_place(construction_checkpoint(conn).await?;super::faults::checkpoint("migration_schema_step");}- super::bulk::bootstrap_in_place(conn).await+ super::bulk::bootstrap_in_place(conn).await?;+ upgrade_resident_frames(conn).await}pub(super) async fn stage_frames(@@ -146,7 +147,7 @@ pub(super) async fn stage_frames(.bind(first).bind(last).execute(&mut *conn).await?;sqlx::query("INSERT INTO frames_fts(rowid,full_text,app_name,window_name,browser_url) SELECT id,full_text,COALESCE(app_name,''),COALESCE(window_name,''),COALESCE(browser_url,'') FROM frames WHERE id BETWEEN ? AND ? AND full_text IS NOT NULL AND full_text!=''").bind(first).bind(last).execute(&mut *conn).await?;- sqlx::query("UPDATE storage_metadata SET maintenance=0,staging_bytes=staging_bytes+COALESCE((SELECT sum(bytes) FROM frame_payloads WHERE frame_id BETWEEN ? AND ?),0)").bind(first).bind(last).execute(conn).await?;+ sqlx::query("UPDATE storage_metadata SET maintenance=0,staging_bytes=staging_bytes+COALESCE((SELECT sum(bytes) FROM frame_payloads WHERE frame_id BETWEEN ? AND ? AND bytes<=record_limit),0)").bind(first).bind(last).execute(conn).await?;Ok(())}@@ -236,6 +237,25 @@ const BYTES: &str = "COALESCE(length(CAST(NEW.full_text AS BLOB)),0)+COALESCE(leconst EMPTY_SURFACES: &str = "(CASE WHEN COALESCE(NEW.full_text,'')='' THEN 1 ELSE 0 END | CASE WHEN COALESCE(NEW.accessibility_text,'')='' THEN 2 ELSE 0 END | CASE WHEN COALESCE(NEW.accessibility_tree_json,'')='' THEN 4 ELSE 0 END | CASE WHEN COALESCE(NEW.text_json,'')='' THEN 8 ELSE 0 END | CASE WHEN COALESCE(NEW.window_name,'')='' THEN 16 ELSE 0 END | CASE WHEN COALESCE(NEW.browser_url,'')='' THEN 32 ELSE 0 END)";fn triggers() -> String {+ // Version 1's checksum is part of existing generation identities.+ frame_triggers(false)+}++fn frame_triggers(retain_oversized: bool) -> String {+ let charge = |bytes: &str| {+ if retain_oversized {+ format!("CASE WHEN ({bytes}) <= (SELECT record_limit FROM storage_metadata) THEN ({bytes}) ELSE 0 END")+ } else {+ bytes.to_owned()+ }+ };+ let new_staging_bytes = charge(BYTES);+ let prior_staging_bytes = charge("bytes");+ let growth_guard = if retain_oversized {+ format!(" AND ({BYTES}) > ({})", BYTES.replace("NEW.", "OLD."))+ } else {+ String::new()+ };format!(r#"CREATE TRIGGER hybrid_frame_insert AFTER INSERT ON frames@@ -265,10 +285,10 @@ BEGIN SELECT RAISE(ABORT,'frame storage: sealed frame requires coordinated payloCREATE TRIGGER hybrid_frame_update AFTER UPDATE OF full_text,accessibility_text,accessibility_tree_json,text_json,app_name,window_name,browser_url ON framesWHEN (SELECT maintenance=0 FROM storage_metadata)BEGIN- SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata)+ SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata){growth_guard}THEN RAISE(ABORT,'frame storage: record budget exceeded') END;UPDATE storage_metadata SET revision=revision+1,- staging_bytes=staging_bytes+({BYTES})-(SELECT bytes FROM frame_payloads WHERE frame_id=NEW.id);+ staging_bytes=staging_bytes+({new_staging_bytes})-(SELECT {prior_staging_bytes} FROM frame_payloads WHERE frame_id=NEW.id);SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata)THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;UPDATE frame_payloads SET generation=generation+1,bytes=({BYTES}),@@ -293,13 +313,57 @@ BEGINDELETE FROM frames_fts WHERE rowid=OLD.id;UPDATE payload_files SET state='dirty' WHERE id=(SELECT file_id FROM frame_payloads WHERE frame_id=OLD.id);UPDATE storage_metadata SET revision=revision+1,staging_bytes=staging_bytes-- COALESCE((SELECT bytes FROM frame_payloads WHERE frame_id=OLD.id AND state='staged'),0);+ COALESCE((SELECT {prior_staging_bytes} FROM frame_payloads WHERE frame_id=OLD.id AND state='staged'),0);DELETE FROM frame_payloads WHERE frame_id=OLD.id;END;"#)}+/// Keep legacy frames above the encoder budget resident without consuming the+/// capture backlog. Runs under the existing startup/offline writer, including+/// for generations left midway through migration by an older app.+pub(crate) async fn upgrade_resident_frames(+ conn: &mut SqliteConnection,+) -> Result<(), sqlx::Error> {+ let sql = frame_triggers(true);+ let checksum = format!("{:x}", Sha256::digest(format!("{sql}:resident-frames-v1")));+ let installed: Option<String> =+ sqlx::query_scalar("SELECT checksum FROM _hybrid_migrations WHERE version=4")+ .fetch_optional(&mut *conn)+ .await?;+ if let Some(installed) = installed {+ return if installed == checksum {+ Ok(())+ } else {+ Err(storage_error("resident frame schema checksum mismatch"))+ };+ }+ let mut tx = conn.begin().await?;+ sqlx::query("UPDATE storage_metadata SET staging_bytes=staging_bytes-COALESCE((SELECT sum(bytes) FROM frame_payloads WHERE state='staged' AND bytes>record_limit),0)")+ .execute(&mut *tx).await?;+ for name in [+ "hybrid_frame_insert",+ "hybrid_frame_update",+ "hybrid_frame_delete",+ "hybrid_frame_sealed_guard",+ "hybrid_frame_metadata",+ ] {+ sqlx::query(sqlx::AssertSqlSafe(format!("DROP TRIGGER {name}")))+ .execute(&mut *tx)+ .await?;+ }+ sqlx::raw_sql(sqlx::AssertSqlSafe(sql))+ .execute(&mut *tx)+ .await?;+ sqlx::query("INSERT INTO _hybrid_migrations VALUES(4,?)")+ .bind(checksum)+ .execute(&mut *tx)+ .await?;+ tx.commit().await?;+ Ok(())+}+pub(crate) async fn bootstrap(conn: &mut SqliteConnection,descriptor: &StorageDescriptor,@@ -395,6 +459,78 @@ mod checkpoint_tests {use sqlx::Connection;use std::time::Duration;+ #[tokio::test]+ async fn upgrades_existing_oversized_frame_accounting_once_without_changing_v1_identity() {+ use crate::storage::{MigrationOptions, Projection, StorageBudget};+ use crate::DatabaseManager;++ // Check against the shipped schema, not a checksum generated by this test.+ assert_eq!(+ format!("{:x}", Sha256::digest(format!("{CATALOG}{}", triggers()))),+ "f6e17a76ded55aeb312b6c7a15fb6498289f38881f3f860629931790802ed81d"+ );+ let root = tempfile::tempdir().unwrap();+ let options = MigrationOptions {+ budget: StorageBudget {+ record_bytes: 1024 * 1024,+ staging_bytes: 1024 * 1024,+ ..Default::default()+ },+ ..Default::default()+ };+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)+ .await+ .unwrap();+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ for name in ["insert", "update", "delete", "sealed_guard", "metadata"] {+ sqlx::query(sqlx::AssertSqlSafe(format!(+ "DROP TRIGGER hybrid_frame_{name}"+ )))+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ }+ sqlx::raw_sql(sqlx::AssertSqlSafe(triggers()))+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ sqlx::raw_sql("DELETE FROM _hybrid_migrations WHERE version=4; UPDATE storage_metadata SET maintenance=1; INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','legacy history',printf('%.*c',2097152,'x')),(2,'2026-09-14T12:00:01Z','pending capture',NULL);")+ .execute(&mut **tx.conn()).await.unwrap();+ stage_frames(tx.conn(), 1, 2).await.unwrap();+ sqlx::query(+ "UPDATE storage_metadata SET staging_bytes=(SELECT sum(bytes) FROM frame_payloads)",+ )+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ tx.commit().await.unwrap();+ db.close().await;++ for _ in 0..2 {+ let db = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ "pending capture".len() as i64+ );+ assert_eq!(+ db.frame_payloads(&[1], Projection::All).await.unwrap()[&1]+ .accessibility_tree_json+ .as_deref(),+ Some("x".repeat(2097152).as_str())+ );+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-09-14T12:01:00Z','new capture'); DELETE FROM frames WHERE id=3;").await.unwrap();+ db.close().await;+ }+ }+#[tokio::test]async fn construction_waits_for_finishing_readers_and_rejects_pinned_snapshots() {let root = tempfile::tempdir().unwrap();diff --git a/crates/screenpipe-db/src/storage/sealing.rs b/crates/screenpipe-db/src/storage/sealing.rsindex 54a01732a..7ba6208ba 100644--- a/crates/screenpipe-db/src/storage/sealing.rs+++ b/crates/screenpipe-db/src/storage/sealing.rs@@ -120,9 +120,6 @@ impl DatabaseManager {let Some(storage) = &self.storage else {return Err(storage_error("replacement requires hybrid storage"));};- if payload.bytes() > storage.descriptor.budget.record_bytes {- return Err(storage_error("record budget exceeded"));- }let mut tx = self.begin_immediate_with_retry().await?;let row = sqlx::query("SELECT p.generation,p.state,p.file_id,p.bytes,m.policy,m.staging_bytes,m.staging_limit FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.frame_id=?").bind(payload.id).fetch_optional(&mut **tx.conn()).await?;@@ -136,12 +133,26 @@ impl DatabaseManager {tx.rollback().await?;return Ok(false);}+ // Retained legacy frames can still be redacted or have detail removed.+ // New oversized payloads remain subject to normal capture admission.+ if payload.bytes() > storage.descriptor.budget.record_bytes+ && payload.bytes() > row.get::<i64, _>("bytes") as usize+ {+ return Err(storage_error("record budget exceeded"));+ }let prior_bytes = if row.get::<&str, _>("state") == "staged" {- row.get::<i64, _>("bytes")+ storage+ .descriptor+ .budget+ .staged_frame_bytes(row.get::<i64, _>("bytes") as usize) as i64} else {0};- let next_bytes = row.get::<i64, _>("staging_bytes") - prior_bytes + payload.bytes() as i64;+ let next_bytes = row.get::<i64, _>("staging_bytes") - prior_bytes+ + storage+ .descriptor+ .budget+ .staged_frame_bytes(payload.bytes()) as i64;if next_bytes > row.get::<i64, _>("staging_limit") {return Err(storage_error("staging budget reached"));}@@ -219,15 +230,15 @@ impl HybridStorage {writer: &SqliteWritePool,) -> Result<usize, sqlx::Error> {let _job = self.file_job.lock().await;- let candidates = sqlx::query("SELECT p.frame_id,p.bytes FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.state='staged' AND p.policy=m.policy AND (p.completed_surfaces & m.required_surfaces)=m.required_surfaces ORDER BY p.frame_id LIMIT ?")+ // Existing oversized frames remain readable through the staged SQLite+ // path. Filter before LIMIT so they cannot starve later sealable frames.+ let candidates = sqlx::query("SELECT p.frame_id,p.bytes FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.state='staged' AND p.bytes<=? AND p.policy=m.policy AND (p.completed_surfaces & m.required_surfaces)=m.required_surfaces ORDER BY p.frame_id LIMIT ?")+ .bind(self.descriptor.budget.record_bytes as i64).bind(self.descriptor.budget.file_rows as i64).fetch_all(pool).await?;let mut ids = Vec::new();let mut bytes = 0;for row in candidates {let size = row.get::<i64, _>("bytes") as usize;- if size > self.descriptor.budget.record_bytes {- return Err(storage_error("record exceeds sealing budget"));- }if !ids.is_empty() && bytes + size > self.descriptor.budget.file_bytes {break;}diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rsindex 516cf68f4..561b2d8a5 100644--- a/crates/screenpipe-db/tests/in_place_migration.rs+++ b/crates/screenpipe-db/tests/in_place_migration.rs@@ -886,33 +886,193 @@ async fn recording_recovery_is_durable_when_interrupted_before_or_after_activati}#[tokio::test]-async fn oversized_legacy_record_does_not_prevent_recording_recovery() {+async fn oversized_legacy_frames_remain_readable_without_blocking_migration_or_capture() {let root = tempfile::tempdir().unwrap();let path = root.path().join("db.sqlite");let db = DatabaseManager::new(path.to_str().unwrap(), Default::default()).await.unwrap();- db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','oversized history',printf('%.*c',2097152,'x'))").await.unwrap();+ let detail = "legacy detail 東京 ".repeat(160_000);+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ for id in [1, 2] {+ sqlx::query("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(?,'2026-09-14T12:00:00Z','oversized history',?)")+ .bind(id).bind(&detail).execute(&mut **tx.conn()).await.unwrap();+ }+ tx.commit().await.unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-09-14T12:00:00Z','ordinary history')").await.unwrap();db.close().await;let mut options = MigrationOptions::default();+ options.budget.row_group_rows = 1;+ options.budget.file_rows = 1;+ options.budget.file_bytes = 64 * 1024;options.budget.record_bytes = 1024 * 1024;- assert!(migrate(root.path(), Default::default(), options)+ options.budget.decode_bytes = 2 * 1024 * 1024;+ options.budget.staging_bytes = 1024 * 1024;+ // Each old record exceeds both decode and staging budgets. Migration must+ // keep it in SQLite rather than loading it or growing the capture backlog.+ assert!(detail.len() > options.budget.decode_bytes);+ let report = migrate(root.path(), Default::default(), options).await- .is_err());- screenpipe_db::storage::recover_interrupted_migration(root.path(), Default::default())+ .unwrap();+ assert_eq!(report.frames, 3);+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.verify_storage().await.unwrap();+ let mut payloads = db+ .frame_payloads(&[1, 2, 3], Projection::All)+ .await+ .unwrap();+ for id in [1, 2] {+ assert_eq!(+ payloads[&id].accessibility_tree_json.as_deref(),+ Some(detail.as_str())+ );+ assert_eq!(+ payloads[&id].full_text.as_deref(),+ Some("oversized history")+ );+ }+ assert_eq!(payloads[&3].full_text.as_deref(), Some("ordinary history"));+ assert_eq!(+ sqlx::query_as::<_, (i64, i64)>("SELECT staging_bytes,(SELECT count(*) FROM frame_payloads WHERE state='sealed') FROM storage_metadata")+ .fetch_one(&db.pool).await.unwrap(),+ (0, 1)+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM frames_fts WHERE frames_fts MATCH 'oversized'"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 2+ );+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 0);++ // Metadata and privacy updates must work on retained history. Once reduced+ // below the encoder limit, it joins the ordinary staging/sealing path.+ db.execute_raw_sql_write("UPDATE frames SET window_name='renamed window' WHERE id=2")+ .await+ .unwrap();+ let mut payload = payloads.remove(&1).unwrap();+ payload.accessibility_tree_json = Some("x".repeat(2 * 1024 * 1024));+ assert!(db+ .replace_frame_payload(&payload, "", 15, None, None)+ .await+ .unwrap());+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 0+ );+ let mut payload = db+ .frame_payloads(&[1], Projection::All)+ .await+ .unwrap()+ .remove(&1)+ .unwrap();+ payload.accessibility_tree_json = Some("redacted detail".into());+ assert!(db+ .replace_frame_payload(&payload, "", 15, None, None)+ .await+ .unwrap());+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ payload.bytes() as i64+ );+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 1);+ db.execute_raw_sql_write("DELETE FROM frames WHERE id=2; INSERT INTO frames(id,timestamp,full_text) VALUES(4,'2026-09-14T12:01:00Z','new recording')").await.unwrap();+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 1);+ db.close().await;++ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 0+ );+ assert_eq!(+ db.frame_payloads(&[4], Projection::Search).await.unwrap()[&4]+ .full_text+ .as_deref(),+ Some("new recording")+ );+ db.verify_storage().await.unwrap();+ db.close().await;+}++#[cfg(feature = "storage-fault-injection")]+#[tokio::test]+async fn oversized_legacy_frame_resumes_after_staging_without_rebuilding_sealed_files() {+ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("db.sqlite");+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','first history',NULL),(2,'2026-09-14T12:00:01Z','large history',printf('%.*c',41943040,'x')),(3,'2026-09-14T12:00:02Z','later history',NULL)").await.unwrap();+ db.close().await;+ let stopped = std::process::Command::new(env!("CARGO_BIN_EXE_screenpipe-storage"))+ .arg("migrate")+ .arg(root.path())+ .env("SCREENPIPE_STORAGE_CRASH_AT", "migration_batch_staged")+ .env("SCREENPIPE_STORAGE_CRASH_HIT", "2")+ .output()+ .unwrap();+ assert_eq!(+ stopped.status.code(),+ Some(86),+ "{}",+ String::from_utf8_lossy(&stopped.stderr)+ );+ let published: Vec<_> = screenpipe_db::storage::inventory(root.path())+ .unwrap()+ .into_iter()+ .filter(|p| p.extension().is_some_and(|ext| ext == "parquet"))+ .map(|p| {+ let bytes = std::fs::read(&p).unwrap();+ (p, bytes)+ })+ .collect();+ assert!(!published.is_empty());+ migrate(root.path(), Default::default(), Default::default()).await.unwrap();+ for (path, bytes) in published {+ assert_eq!(std::fs::read(path).unwrap(), bytes);+ }let db = DatabaseManager::new(path.to_str().unwrap(), Default::default()).await.unwrap();assert_eq!(- db.frame_payloads(&[1], Projection::All).await.unwrap()[&1]+ db.frame_payloads(&[2], Projection::All).await.unwrap()[&2].accessibility_tree_json- .as_ref()- .unwrap()- .len(),- 2097152+ .as_deref(),+ Some("x".repeat(41943040).as_str()));- db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-14T12:01:00Z','new recording')").await.unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frame_payloads WHERE state='sealed'")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 2+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 0+ );+ db.verify_storage().await.unwrap();db.close().await;}diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex d1808cb5b..111bed255 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y- Mapped suites: 33- Mapped Rust files: 355-- Active test blocks: 3491+- Active test blocks: 3494- Ignored/manual test blocks: 144-- Declared test blocks: 3635-- Weighted coverage points: 2877.4+- Declared test blocks: 3638+- Weighted coverage points: 2880.4Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3348 | 138 | 2812.3 | 21 | 11 | 100% |-| macos | 30 | 3410 | 119 | 2826.3 | 22 | 11 | 100% |-| linux | 26 | 2978 | 111 | 2471.6 | 20 | 11 | 100% |+| windows | 30 | 3351 | 138 | 2815.3 | 21 | 11 | 100% |+| macos | 30 | 3413 | 119 | 2829.3 | 22 | 11 | 100% |+| linux | 26 | 2981 | 111 | 2474.6 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- |-| screenpipe-engine | 11 | 20 | 119 | 1719 | 42 | 1324.1 | 10 |-| screenpipe-db | 5 | 56 | 22 | 523 | 21 | 496.3 | 9 |+| screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |+| screenpipe-db | 5 | 56 | 22 | 525 | 21 | 498.3 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts || audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |-| database | 6 suites / 470 active / 17 ignored / 443.3 pts | 6 suites / 470 active / 17 ignored / 443.3 pts | 6 suites / 470 active / 17 ignored / 443.3 pts |+| database | 6 suites / 473 active / 17 ignored / 446.3 pts | 6 suites / 473 active / 17 ignored / 446.3 pts | 6 suites / 473 active / 17 ignored / 446.3 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |-| engine-lifecycle | 6 suites / 243 active / 1 ignored / 218.0 pts | 6 suites / 243 active / 1 ignored / 218.0 pts | 5 suites / 237 active / 1 ignored / 216.3 pts |+| engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts || local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts || meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1525 active / 72 ignored / 1345.5 pts | 14 suites / 1632 active / 76 ignored / 1388.3 pts | 13 suites / 1525 active / 72 ignored / 1345.5 pts |+| performance | 13 suites / 1527 active / 72 ignored / 1347.5 pts | 14 suites / 1634 active / 76 ignored / 1390.3 pts | 13 suites / 1527 active / 72 ignored / 1347.5 pts || pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts || privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts || real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - || speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |-| storage | 4 suites / 588 active / 34 ignored / 482.1 pts | 4 suites / 588 active / 34 ignored / 482.1 pts | 4 suites / 588 active / 34 ignored / 482.1 pts |+| storage | 4 suites / 590 active / 34 ignored / 484.1 pts | 4 suites / 590 active / 34 ignored / 484.1 pts | 4 suites / 590 active / 34 ignored / 484.1 pts || sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |-| timeline | 4 suites / 1135 active / 38 ignored / 922.3 pts | 4 suites / 1135 active / 38 ignored / 922.3 pts | 4 suites / 1135 active / 38 ignored / 922.3 pts |+| timeline | 4 suites / 1137 active / 38 ignored / 924.3 pts | 4 suites / 1137 active / 38 ignored / 924.3 pts | 4 suites / 1137 active / 38 ignored / 924.3 pts || transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts || ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts || vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |@@ -132,11 +132,11 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 28 | 235 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 28 | 237 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |-| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 25 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |+| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 26 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. || engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. || engine-local-api-search-integration | screenpipe-engine | windows, macos, linux | local-api, db-search | local-api-search | high | strong | integration | 1 | 8 | 5 | Active /search route test builds an audio-disabled router, seeds captured-screen-shaped OCR data into an in-memory DB, and asserts the HTTP response and pagination. || engine-meeting-privacy-sync | screenpipe-engine | windows, macos, linux | meeting, privacy, ui-events, pipes, sync | meeting-live-notes, privacy-and-redaction, accessibility-ui-events, performance-liveness | medium | strong | unit | 32 | 511 | 3 | Unit-heavy coverage for privacy filter policy, capture exclusions, UI recorder safety, pipes/live-view/structured-output helpers, sync helpers, and CLI parsing. Meeting detection moved to the engine-meeting-watcher suite. |@@ -280,7 +280,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 || db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 || db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 2 | 1 | 3 |-| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 1 | 0 | 1 |+| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 2 | 0 | 2 || db-timeline-frames | screenpipe-db | src/storage/sealing.rs | source | 1 | 0 | 1 || db-search-indexing | screenpipe-db | src/text_normalizer.rs | source | 19 | 0 | 19 || db-search-indexing | screenpipe-db | src/text_similarity.rs | source | 20 | 0 | 20 |@@ -303,7 +303,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 || db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 || db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 11 | 4 | 15 |+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 12 | 4 | 16 || db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 || db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 || db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |@@ -353,7 +353,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-meeting-privacy-sync | screenpipe-engine | src/calendar_speaker_id.rs | source | 41 | 0 | 41 || engine-meeting-privacy-sync | screenpipe-engine | src/capture_exclusions.rs | source | 7 | 0 | 7 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/agent.rs | source | 38 | 0 | 38 |-| engine-db-recovery-cli | screenpipe-engine | src/cli/db.rs | source | 25 | 0 | 25 |+| engine-db-recovery-cli | screenpipe-engine | src/cli/db.rs | source | 26 | 0 | 26 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/install.rs | source | 4 | 0 | 4 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/login.rs | source | 1 | 0 | 1 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/mod.rs | source | 48 | 0 | 48 |