Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): preserve oversized legacy frames during migration (#7046)

Ezra Ellette3cf2f10551b8
diff --git a/COVERAGE.md b/COVERAGE.md
index 3e9896d3f..a7661ad65 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -36,15 +36,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 33
- Mapped Rust files: 355
-- Active test blocks: 3491
+- Active test blocks: 3494
- Ignored/manual test blocks: 144
-- Weighted coverage points: 2877.4
+- Weighted coverage points: 2880.4
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3348 | 138 | 2812.3 | 21 | 11 | 100% |
-| macos | 30 | 3410 | 119 | 2826.3 | 22 | 11 | 100% |
-| linux | 26 | 2978 | 111 | 2471.6 | 20 | 11 | 100% |
+| windows | 30 | 3351 | 138 | 2815.3 | 21 | 11 | 100% |
+| macos | 30 | 3413 | 119 | 2829.3 | 22 | 11 | 100% |
+| linux | 26 | 2981 | 111 | 2474.6 | 20 | 11 | 100% |
## Refresh
diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rs
index 372538552..be885ce6d 100644
--- a/crates/screenpipe-db/src/db/setup.rs
+++ b/crates/screenpipe-db/src/db/setup.rs
@@ -520,6 +520,7 @@ impl DatabaseManager {
construction?;
}
crate::storage::schema::verify(&mut conn, &storage.descriptor).await?;
+ crate::storage::schema::upgrade_resident_frames(&mut conn).await?;
crate::storage::read_schema::upgrade(&mut conn, storage).await?;
storage.verify_catalog(&db_manager.pool).await?;
}
diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rs
index 61ba33835..5941e5f11 100644
--- a/crates/screenpipe-db/src/storage/in_place.rs
+++ b/crates/screenpipe-db/src/storage/in_place.rs
@@ -164,14 +164,10 @@ async fn run(
continue;
}
let last: Option<i64> = sqlx::query_scalar("SELECT max(frame_id) FROM frame_payloads").fetch_one(&pool).await?;
- let range = if archive {
- let columns = super::import::columns(&pool, "frames").await?;
- let columns: Vec<_> = columns.into_iter().filter(|c| !c.starts_with("payload_")).collect();
- let rows = super::import::batch(&pool, "frames", &columns, last, &storage.descriptor.budget).await?;
- rows.first().zip(rows.last()).map(|(first,last)| (first.get::<i64,_>(0),last.get::<i64,_>(0)))
- } else {
- resident_range(&pool, "frames", last, storage.descriptor.budget.file_bytes).await?
- };
+ // Staging is SQL-to-SQL. Only read IDs and lengths here so even a
+ // legacy frame larger than the decoder budget stays in SQLite.
+ let range = resident_range(&pool, "frames", last, storage.descriptor.budget.file_bytes,
+ archive.then_some(storage.descriptor.budget.file_rows)).await?;
let Some((first, last)) = range else { break };
let permit = writer.lock().await?;
let mut conn = permit.pool().acquire().await?;
@@ -198,7 +194,7 @@ async fn run(
let range = if archive {
rows.first().zip(rows.last()).map(|(first,last)| (first.get::<i64,_>(0),last.get::<i64,_>(0)))
} else {
- resident_range(&pool, "_bulk_elements_source", None, storage.descriptor.budget.file_bytes).await?
+ resident_range(&pool, "_bulk_elements_source", None, storage.descriptor.budget.file_bytes, None).await?
};
let Some((first, last)) = range else { break };
let permit = writer.lock().await?;
@@ -289,14 +285,16 @@ async fn resident_range(
table: &str,
after: Option<i64>,
budget_bytes: usize,
+ row_limit: Option<usize>,
) -> Result<Option<(i64, i64)>, sqlx::Error> {
let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));
let size = super::import::columns(pool, table).await?.iter().map(|name| {
let quoted = format!("\"{}\"", name.replace('"', "\"\""));
format!("CASE WHEN typeof({quoted}) IN ('integer','real') THEN 8 ELSE COALESCE(length(CAST({quoted} AS BLOB)),0) END")
}).collect::<Vec<_>>().join("+");
+ let row_limit = row_limit.unwrap_or(128);
let rows: Vec<(i64, i64)> = sqlx::query_as(sqlx::AssertSqlSafe(format!(
- "SELECT id,{size} FROM {table} WHERE {lower} ORDER BY id LIMIT 128"
+ "SELECT id,{size} FROM {table} WHERE {lower} ORDER BY id LIMIT {row_limit}"
)))
.fetch_all(pool)
.await?;
diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rs
index 4d0864ea9..972578ecd 100644
--- a/crates/screenpipe-db/src/storage/mod.rs
+++ b/crates/screenpipe-db/src/storage/mod.rs
@@ -105,6 +105,15 @@ impl Default for StorageBudget {
}
impl StorageBudget {
+ /// Oversized legacy frames are resident history, not work for the sealer.
+ pub(super) fn staged_frame_bytes(&self, bytes: usize) -> usize {
+ if bytes <= self.record_bytes {
+ bytes
+ } else {
+ 0
+ }
+ }
+
pub fn validate(&self) -> Result<(), sqlx::Error> {
if self.row_group_rows == 0
|| self.file_rows < self.row_group_rows
diff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rs
index 4d282bda7..69c5a9925 100644
--- a/crates/screenpipe-db/src/storage/schema.rs
+++ b/crates/screenpipe-db/src/storage/schema.rs
@@ -117,7 +117,8 @@ pub(super) async fn bootstrap_in_place(
construction_checkpoint(conn).await?;
super::faults::checkpoint("migration_schema_step");
}
- super::bulk::bootstrap_in_place(conn).await
+ super::bulk::bootstrap_in_place(conn).await?;
+ upgrade_resident_frames(conn).await
}
pub(super) async fn stage_frames(
@@ -146,7 +147,7 @@ pub(super) async fn stage_frames(
.bind(first).bind(last).execute(&mut *conn).await?;
sqlx::query("INSERT INTO frames_fts(rowid,full_text,app_name,window_name,browser_url) SELECT id,full_text,COALESCE(app_name,''),COALESCE(window_name,''),COALESCE(browser_url,'') FROM frames WHERE id BETWEEN ? AND ? AND full_text IS NOT NULL AND full_text!=''")
.bind(first).bind(last).execute(&mut *conn).await?;
- sqlx::query("UPDATE storage_metadata SET maintenance=0,staging_bytes=staging_bytes+COALESCE((SELECT sum(bytes) FROM frame_payloads WHERE frame_id BETWEEN ? AND ?),0)").bind(first).bind(last).execute(conn).await?;
+ sqlx::query("UPDATE storage_metadata SET maintenance=0,staging_bytes=staging_bytes+COALESCE((SELECT sum(bytes) FROM frame_payloads WHERE frame_id BETWEEN ? AND ? AND bytes<=record_limit),0)").bind(first).bind(last).execute(conn).await?;
Ok(())
}
@@ -236,6 +237,25 @@ const BYTES: &str = "COALESCE(length(CAST(NEW.full_text AS BLOB)),0)+COALESCE(le
const EMPTY_SURFACES: &str = "(CASE WHEN COALESCE(NEW.full_text,'')='' THEN 1 ELSE 0 END | CASE WHEN COALESCE(NEW.accessibility_text,'')='' THEN 2 ELSE 0 END | CASE WHEN COALESCE(NEW.accessibility_tree_json,'')='' THEN 4 ELSE 0 END | CASE WHEN COALESCE(NEW.text_json,'')='' THEN 8 ELSE 0 END | CASE WHEN COALESCE(NEW.window_name,'')='' THEN 16 ELSE 0 END | CASE WHEN COALESCE(NEW.browser_url,'')='' THEN 32 ELSE 0 END)";
fn triggers() -> String {
+ // Version 1's checksum is part of existing generation identities.
+ frame_triggers(false)
+}
+
+fn frame_triggers(retain_oversized: bool) -> String {
+ let charge = |bytes: &str| {
+ if retain_oversized {
+ format!("CASE WHEN ({bytes}) <= (SELECT record_limit FROM storage_metadata) THEN ({bytes}) ELSE 0 END")
+ } else {
+ bytes.to_owned()
+ }
+ };
+ let new_staging_bytes = charge(BYTES);
+ let prior_staging_bytes = charge("bytes");
+ let growth_guard = if retain_oversized {
+ format!(" AND ({BYTES}) > ({})", BYTES.replace("NEW.", "OLD."))
+ } else {
+ String::new()
+ };
format!(
r#"
CREATE TRIGGER hybrid_frame_insert AFTER INSERT ON frames
@@ -265,10 +285,10 @@ BEGIN SELECT RAISE(ABORT,'frame storage: sealed frame requires coordinated paylo
CREATE TRIGGER hybrid_frame_update AFTER UPDATE OF full_text,accessibility_text,accessibility_tree_json,text_json,app_name,window_name,browser_url ON frames
WHEN (SELECT maintenance=0 FROM storage_metadata)
BEGIN
- SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata)
+ SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata){growth_guard}
THEN RAISE(ABORT,'frame storage: record budget exceeded') END;
UPDATE storage_metadata SET revision=revision+1,
- staging_bytes=staging_bytes+({BYTES})-(SELECT bytes FROM frame_payloads WHERE frame_id=NEW.id);
+ staging_bytes=staging_bytes+({new_staging_bytes})-(SELECT {prior_staging_bytes} FROM frame_payloads WHERE frame_id=NEW.id);
SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata)
THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;
UPDATE frame_payloads SET generation=generation+1,bytes=({BYTES}),
@@ -293,13 +313,57 @@ BEGIN
DELETE FROM frames_fts WHERE rowid=OLD.id;
UPDATE payload_files SET state='dirty' WHERE id=(SELECT file_id FROM frame_payloads WHERE frame_id=OLD.id);
UPDATE storage_metadata SET revision=revision+1,staging_bytes=staging_bytes-
- COALESCE((SELECT bytes FROM frame_payloads WHERE frame_id=OLD.id AND state='staged'),0);
+ COALESCE((SELECT {prior_staging_bytes} FROM frame_payloads WHERE frame_id=OLD.id AND state='staged'),0);
DELETE FROM frame_payloads WHERE frame_id=OLD.id;
END;
"#
)
}
+/// Keep legacy frames above the encoder budget resident without consuming the
+/// capture backlog. Runs under the existing startup/offline writer, including
+/// for generations left midway through migration by an older app.
+pub(crate) async fn upgrade_resident_frames(
+ conn: &mut SqliteConnection,
+) -> Result<(), sqlx::Error> {
+ let sql = frame_triggers(true);
+ let checksum = format!("{:x}", Sha256::digest(format!("{sql}:resident-frames-v1")));
+ let installed: Option<String> =
+ sqlx::query_scalar("SELECT checksum FROM _hybrid_migrations WHERE version=4")
+ .fetch_optional(&mut *conn)
+ .await?;
+ if let Some(installed) = installed {
+ return if installed == checksum {
+ Ok(())
+ } else {
+ Err(storage_error("resident frame schema checksum mismatch"))
+ };
+ }
+ let mut tx = conn.begin().await?;
+ sqlx::query("UPDATE storage_metadata SET staging_bytes=staging_bytes-COALESCE((SELECT sum(bytes) FROM frame_payloads WHERE state='staged' AND bytes>record_limit),0)")
+ .execute(&mut *tx).await?;
+ for name in [
+ "hybrid_frame_insert",
+ "hybrid_frame_update",
+ "hybrid_frame_delete",
+ "hybrid_frame_sealed_guard",
+ "hybrid_frame_metadata",
+ ] {
+ sqlx::query(sqlx::AssertSqlSafe(format!("DROP TRIGGER {name}")))
+ .execute(&mut *tx)
+ .await?;
+ }
+ sqlx::raw_sql(sqlx::AssertSqlSafe(sql))
+ .execute(&mut *tx)
+ .await?;
+ sqlx::query("INSERT INTO _hybrid_migrations VALUES(4,?)")
+ .bind(checksum)
+ .execute(&mut *tx)
+ .await?;
+ tx.commit().await?;
+ Ok(())
+}
+
pub(crate) async fn bootstrap(
conn: &mut SqliteConnection,
descriptor: &StorageDescriptor,
@@ -395,6 +459,78 @@ mod checkpoint_tests {
use sqlx::Connection;
use std::time::Duration;
+ #[tokio::test]
+ async fn upgrades_existing_oversized_frame_accounting_once_without_changing_v1_identity() {
+ use crate::storage::{MigrationOptions, Projection, StorageBudget};
+ use crate::DatabaseManager;
+
+ // Check against the shipped schema, not a checksum generated by this test.
+ assert_eq!(
+ format!("{:x}", Sha256::digest(format!("{CATALOG}{}", triggers()))),
+ "f6e17a76ded55aeb312b6c7a15fb6498289f38881f3f860629931790802ed81d"
+ );
+ let root = tempfile::tempdir().unwrap();
+ let options = MigrationOptions {
+ budget: StorageBudget {
+ record_bytes: 1024 * 1024,
+ staging_bytes: 1024 * 1024,
+ ..Default::default()
+ },
+ ..Default::default()
+ };
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)
+ .await
+ .unwrap();
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ for name in ["insert", "update", "delete", "sealed_guard", "metadata"] {
+ sqlx::query(sqlx::AssertSqlSafe(format!(
+ "DROP TRIGGER hybrid_frame_{name}"
+ )))
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ }
+ sqlx::raw_sql(sqlx::AssertSqlSafe(triggers()))
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ sqlx::raw_sql("DELETE FROM _hybrid_migrations WHERE version=4; UPDATE storage_metadata SET maintenance=1; INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','legacy history',printf('%.*c',2097152,'x')),(2,'2026-09-14T12:00:01Z','pending capture',NULL);")
+ .execute(&mut **tx.conn()).await.unwrap();
+ stage_frames(tx.conn(), 1, 2).await.unwrap();
+ sqlx::query(
+ "UPDATE storage_metadata SET staging_bytes=(SELECT sum(bytes) FROM frame_payloads)",
+ )
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ db.close().await;
+
+ for _ in 0..2 {
+ let db = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ "pending capture".len() as i64
+ );
+ assert_eq!(
+ db.frame_payloads(&[1], Projection::All).await.unwrap()[&1]
+ .accessibility_tree_json
+ .as_deref(),
+ Some("x".repeat(2097152).as_str())
+ );
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-09-14T12:01:00Z','new capture'); DELETE FROM frames WHERE id=3;").await.unwrap();
+ db.close().await;
+ }
+ }
+
#[tokio::test]
async fn construction_waits_for_finishing_readers_and_rejects_pinned_snapshots() {
let root = tempfile::tempdir().unwrap();
diff --git a/crates/screenpipe-db/src/storage/sealing.rs b/crates/screenpipe-db/src/storage/sealing.rs
index 54a01732a..7ba6208ba 100644
--- a/crates/screenpipe-db/src/storage/sealing.rs
+++ b/crates/screenpipe-db/src/storage/sealing.rs
@@ -120,9 +120,6 @@ impl DatabaseManager {
let Some(storage) = &self.storage else {
return Err(storage_error("replacement requires hybrid storage"));
};
- if payload.bytes() > storage.descriptor.budget.record_bytes {
- return Err(storage_error("record budget exceeded"));
- }
let mut tx = self.begin_immediate_with_retry().await?;
let row = sqlx::query("SELECT p.generation,p.state,p.file_id,p.bytes,m.policy,m.staging_bytes,m.staging_limit FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.frame_id=?")
.bind(payload.id).fetch_optional(&mut **tx.conn()).await?;
@@ -136,12 +133,26 @@ impl DatabaseManager {
tx.rollback().await?;
return Ok(false);
}
+ // Retained legacy frames can still be redacted or have detail removed.
+ // New oversized payloads remain subject to normal capture admission.
+ if payload.bytes() > storage.descriptor.budget.record_bytes
+ && payload.bytes() > row.get::<i64, _>("bytes") as usize
+ {
+ return Err(storage_error("record budget exceeded"));
+ }
let prior_bytes = if row.get::<&str, _>("state") == "staged" {
- row.get::<i64, _>("bytes")
+ storage
+ .descriptor
+ .budget
+ .staged_frame_bytes(row.get::<i64, _>("bytes") as usize) as i64
} else {
0
};
- let next_bytes = row.get::<i64, _>("staging_bytes") - prior_bytes + payload.bytes() as i64;
+ let next_bytes = row.get::<i64, _>("staging_bytes") - prior_bytes
+ + storage
+ .descriptor
+ .budget
+ .staged_frame_bytes(payload.bytes()) as i64;
if next_bytes > row.get::<i64, _>("staging_limit") {
return Err(storage_error("staging budget reached"));
}
@@ -219,15 +230,15 @@ impl HybridStorage {
writer: &SqliteWritePool,
) -> Result<usize, sqlx::Error> {
let _job = self.file_job.lock().await;
- let candidates = sqlx::query("SELECT p.frame_id,p.bytes FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.state='staged' AND p.policy=m.policy AND (p.completed_surfaces & m.required_surfaces)=m.required_surfaces ORDER BY p.frame_id LIMIT ?")
+ // Existing oversized frames remain readable through the staged SQLite
+ // path. Filter before LIMIT so they cannot starve later sealable frames.
+ let candidates = sqlx::query("SELECT p.frame_id,p.bytes FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.state='staged' AND p.bytes<=? AND p.policy=m.policy AND (p.completed_surfaces & m.required_surfaces)=m.required_surfaces ORDER BY p.frame_id LIMIT ?")
+ .bind(self.descriptor.budget.record_bytes as i64)
.bind(self.descriptor.budget.file_rows as i64).fetch_all(pool).await?;
let mut ids = Vec::new();
let mut bytes = 0;
for row in candidates {
let size = row.get::<i64, _>("bytes") as usize;
- if size > self.descriptor.budget.record_bytes {
- return Err(storage_error("record exceeds sealing budget"));
- }
if !ids.is_empty() && bytes + size > self.descriptor.budget.file_bytes {
break;
}
diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rs
index 516cf68f4..561b2d8a5 100644
--- a/crates/screenpipe-db/tests/in_place_migration.rs
+++ b/crates/screenpipe-db/tests/in_place_migration.rs
@@ -886,33 +886,193 @@ async fn recording_recovery_is_durable_when_interrupted_before_or_after_activati
}
#[tokio::test]
-async fn oversized_legacy_record_does_not_prevent_recording_recovery() {
+async fn oversized_legacy_frames_remain_readable_without_blocking_migration_or_capture() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join("db.sqlite");
let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
.await
.unwrap();
- db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','oversized history',printf('%.*c',2097152,'x'))").await.unwrap();
+ let detail = "legacy detail 東京 ".repeat(160_000);
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ for id in [1, 2] {
+ sqlx::query("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(?,'2026-09-14T12:00:00Z','oversized history',?)")
+ .bind(id).bind(&detail).execute(&mut **tx.conn()).await.unwrap();
+ }
+ tx.commit().await.unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-09-14T12:00:00Z','ordinary history')").await.unwrap();
db.close().await;
let mut options = MigrationOptions::default();
+ options.budget.row_group_rows = 1;
+ options.budget.file_rows = 1;
+ options.budget.file_bytes = 64 * 1024;
options.budget.record_bytes = 1024 * 1024;
- assert!(migrate(root.path(), Default::default(), options)
+ options.budget.decode_bytes = 2 * 1024 * 1024;
+ options.budget.staging_bytes = 1024 * 1024;
+ // Each old record exceeds both decode and staging budgets. Migration must
+ // keep it in SQLite rather than loading it or growing the capture backlog.
+ assert!(detail.len() > options.budget.decode_bytes);
+ let report = migrate(root.path(), Default::default(), options)
.await
- .is_err());
- screenpipe_db::storage::recover_interrupted_migration(root.path(), Default::default())
+ .unwrap();
+ assert_eq!(report.frames, 3);
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.verify_storage().await.unwrap();
+ let mut payloads = db
+ .frame_payloads(&[1, 2, 3], Projection::All)
+ .await
+ .unwrap();
+ for id in [1, 2] {
+ assert_eq!(
+ payloads[&id].accessibility_tree_json.as_deref(),
+ Some(detail.as_str())
+ );
+ assert_eq!(
+ payloads[&id].full_text.as_deref(),
+ Some("oversized history")
+ );
+ }
+ assert_eq!(payloads[&3].full_text.as_deref(), Some("ordinary history"));
+ assert_eq!(
+ sqlx::query_as::<_, (i64, i64)>("SELECT staging_bytes,(SELECT count(*) FROM frame_payloads WHERE state='sealed') FROM storage_metadata")
+ .fetch_one(&db.pool).await.unwrap(),
+ (0, 1)
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(
+ "SELECT count(*) FROM frames_fts WHERE frames_fts MATCH 'oversized'"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 2
+ );
+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 0);
+
+ // Metadata and privacy updates must work on retained history. Once reduced
+ // below the encoder limit, it joins the ordinary staging/sealing path.
+ db.execute_raw_sql_write("UPDATE frames SET window_name='renamed window' WHERE id=2")
+ .await
+ .unwrap();
+ let mut payload = payloads.remove(&1).unwrap();
+ payload.accessibility_tree_json = Some("x".repeat(2 * 1024 * 1024));
+ assert!(db
+ .replace_frame_payload(&payload, "", 15, None, None)
+ .await
+ .unwrap());
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 0
+ );
+ let mut payload = db
+ .frame_payloads(&[1], Projection::All)
+ .await
+ .unwrap()
+ .remove(&1)
+ .unwrap();
+ payload.accessibility_tree_json = Some("redacted detail".into());
+ assert!(db
+ .replace_frame_payload(&payload, "", 15, None, None)
+ .await
+ .unwrap());
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ payload.bytes() as i64
+ );
+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 1);
+ db.execute_raw_sql_write("DELETE FROM frames WHERE id=2; INSERT INTO frames(id,timestamp,full_text) VALUES(4,'2026-09-14T12:01:00Z','new recording')").await.unwrap();
+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 1);
+ db.close().await;
+
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 0
+ );
+ assert_eq!(
+ db.frame_payloads(&[4], Projection::Search).await.unwrap()[&4]
+ .full_text
+ .as_deref(),
+ Some("new recording")
+ );
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
+#[cfg(feature = "storage-fault-injection")]
+#[tokio::test]
+async fn oversized_legacy_frame_resumes_after_staging_without_rebuilding_sealed_files() {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("db.sqlite");
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','first history',NULL),(2,'2026-09-14T12:00:01Z','large history',printf('%.*c',41943040,'x')),(3,'2026-09-14T12:00:02Z','later history',NULL)").await.unwrap();
+ db.close().await;
+ let stopped = std::process::Command::new(env!("CARGO_BIN_EXE_screenpipe-storage"))
+ .arg("migrate")
+ .arg(root.path())
+ .env("SCREENPIPE_STORAGE_CRASH_AT", "migration_batch_staged")
+ .env("SCREENPIPE_STORAGE_CRASH_HIT", "2")
+ .output()
+ .unwrap();
+ assert_eq!(
+ stopped.status.code(),
+ Some(86),
+ "{}",
+ String::from_utf8_lossy(&stopped.stderr)
+ );
+ let published: Vec<_> = screenpipe_db::storage::inventory(root.path())
+ .unwrap()
+ .into_iter()
+ .filter(|p| p.extension().is_some_and(|ext| ext == "parquet"))
+ .map(|p| {
+ let bytes = std::fs::read(&p).unwrap();
+ (p, bytes)
+ })
+ .collect();
+ assert!(!published.is_empty());
+ migrate(root.path(), Default::default(), Default::default())
.await
.unwrap();
+ for (path, bytes) in published {
+ assert_eq!(std::fs::read(path).unwrap(), bytes);
+ }
let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
.await
.unwrap();
assert_eq!(
- db.frame_payloads(&[1], Projection::All).await.unwrap()[&1]
+ db.frame_payloads(&[2], Projection::All).await.unwrap()[&2]
.accessibility_tree_json
- .as_ref()
- .unwrap()
- .len(),
- 2097152
+ .as_deref(),
+ Some("x".repeat(41943040).as_str())
);
- db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-14T12:01:00Z','new recording')").await.unwrap();
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frame_payloads WHERE state='sealed'")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 2
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 0
+ );
+ db.verify_storage().await.unwrap();
db.close().await;
}
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index d1808cb5b..111bed255 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 33
- Mapped Rust files: 355
-- Active test blocks: 3491
+- Active test blocks: 3494
- Ignored/manual test blocks: 144
-- Declared test blocks: 3635
-- Weighted coverage points: 2877.4
+- Declared test blocks: 3638
+- Weighted coverage points: 2880.4
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3348 | 138 | 2812.3 | 21 | 11 | 100% |
-| macos | 30 | 3410 | 119 | 2826.3 | 22 | 11 | 100% |
-| linux | 26 | 2978 | 111 | 2471.6 | 20 | 11 | 100% |
+| windows | 30 | 3351 | 138 | 2815.3 | 21 | 11 | 100% |
+| macos | 30 | 3413 | 119 | 2829.3 | 22 | 11 | 100% |
+| linux | 26 | 2981 | 111 | 2474.6 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 11 | 20 | 119 | 1719 | 42 | 1324.1 | 10 |
-| screenpipe-db | 5 | 56 | 22 | 523 | 21 | 496.3 | 9 |
+| screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |
+| screenpipe-db | 5 | 56 | 22 | 525 | 21 | 498.3 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts |
| audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 6 suites / 470 active / 17 ignored / 443.3 pts | 6 suites / 470 active / 17 ignored / 443.3 pts | 6 suites / 470 active / 17 ignored / 443.3 pts |
+| database | 6 suites / 473 active / 17 ignored / 446.3 pts | 6 suites / 473 active / 17 ignored / 446.3 pts | 6 suites / 473 active / 17 ignored / 446.3 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
-| engine-lifecycle | 6 suites / 243 active / 1 ignored / 218.0 pts | 6 suites / 243 active / 1 ignored / 218.0 pts | 5 suites / 237 active / 1 ignored / 216.3 pts |
+| engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts |
| local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts |
| meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1525 active / 72 ignored / 1345.5 pts | 14 suites / 1632 active / 76 ignored / 1388.3 pts | 13 suites / 1525 active / 72 ignored / 1345.5 pts |
+| performance | 13 suites / 1527 active / 72 ignored / 1347.5 pts | 14 suites / 1634 active / 76 ignored / 1390.3 pts | 13 suites / 1527 active / 72 ignored / 1347.5 pts |
| pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
| privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts |
| real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - |
| speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |
-| storage | 4 suites / 588 active / 34 ignored / 482.1 pts | 4 suites / 588 active / 34 ignored / 482.1 pts | 4 suites / 588 active / 34 ignored / 482.1 pts |
+| storage | 4 suites / 590 active / 34 ignored / 484.1 pts | 4 suites / 590 active / 34 ignored / 484.1 pts | 4 suites / 590 active / 34 ignored / 484.1 pts |
| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
-| timeline | 4 suites / 1135 active / 38 ignored / 922.3 pts | 4 suites / 1135 active / 38 ignored / 922.3 pts | 4 suites / 1135 active / 38 ignored / 922.3 pts |
+| timeline | 4 suites / 1137 active / 38 ignored / 924.3 pts | 4 suites / 1137 active / 38 ignored / 924.3 pts | 4 suites / 1137 active / 38 ignored / 924.3 pts |
| transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts |
| ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |
| vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |
@@ -132,11 +132,11 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 28 | 235 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 28 | 237 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
-| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 25 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |
+| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 26 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |
| engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. |
| engine-local-api-search-integration | screenpipe-engine | windows, macos, linux | local-api, db-search | local-api-search | high | strong | integration | 1 | 8 | 5 | Active /search route test builds an audio-disabled router, seeds captured-screen-shaped OCR data into an in-memory DB, and asserts the HTTP response and pagination. |
| engine-meeting-privacy-sync | screenpipe-engine | windows, macos, linux | meeting, privacy, ui-events, pipes, sync | meeting-live-notes, privacy-and-redaction, accessibility-ui-events, performance-liveness | medium | strong | unit | 32 | 511 | 3 | Unit-heavy coverage for privacy filter policy, capture exclusions, UI recorder safety, pipes/live-view/structured-output helpers, sync helpers, and CLI parsing. Meeting detection moved to the engine-meeting-watcher suite. |
@@ -280,7 +280,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 |
| db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |
| db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 2 | 1 | 3 |
-| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 1 | 0 | 1 |
+| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 2 | 0 | 2 |
| db-timeline-frames | screenpipe-db | src/storage/sealing.rs | source | 1 | 0 | 1 |
| db-search-indexing | screenpipe-db | src/text_normalizer.rs | source | 19 | 0 | 19 |
| db-search-indexing | screenpipe-db | src/text_similarity.rs | source | 20 | 0 | 20 |
@@ -303,7 +303,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 |
| db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 |
| db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |
-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 11 | 4 | 15 |
+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 12 | 4 | 16 |
| db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 |
| db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 |
| db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |
@@ -353,7 +353,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-meeting-privacy-sync | screenpipe-engine | src/calendar_speaker_id.rs | source | 41 | 0 | 41 |
| engine-meeting-privacy-sync | screenpipe-engine | src/capture_exclusions.rs | source | 7 | 0 | 7 |
| engine-meeting-privacy-sync | screenpipe-engine | src/cli/agent.rs | source | 38 | 0 | 38 |
-| engine-db-recovery-cli | screenpipe-engine | src/cli/db.rs | source | 25 | 0 | 25 |
+| engine-db-recovery-cli | screenpipe-engine | src/cli/db.rs | source | 26 | 0 | 26 |
| engine-meeting-privacy-sync | screenpipe-engine | src/cli/install.rs | source | 4 | 0 | 4 |
| engine-meeting-privacy-sync | screenpipe-engine | src/cli/login.rs | source | 1 | 0 | 1 |
| engine-meeting-privacy-sync | screenpipe-engine | src/cli/mod.rs | source | 48 | 0 | 48 |