Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): keep recording and archival progressing together (#7063)

Louis Beaumont0ffa1916092d
diff --git a/AGENTS.md b/AGENTS.md
index a228b5ab4..3fd237054 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -66,6 +66,15 @@ signed `.app` path used only when persistent macOS TCC identity is required.
## Hot paths
+Recording is the critical path. Never sacrifice an authorized recording to
+archival, compression, indexing, redaction backlog, sync, or other background
+work. Persist captures durably through the existing writer; processing budgets
+bound background work, never capture admission. When processing cannot keep up,
+retain the recording and defer that work. Preserve privacy, user pause controls,
+and genuine disk/data-integrity protections. Tests must prove continued durable
+recording and recovery under processing failure, backlog, and restart; replace
+expectations that require recordings to be rejected or dropped for those reasons.
+
Capture and encode per frame (`screenpipe-screen`, `-capture`, `-a11y`), audio
device callbacks (`screenpipe-audio`), and SQLite writes (`screenpipe-db` via
`-sqlite-coordinator`) run continuously on every user's machine. No per-frame
diff --git a/COVERAGE.md b/COVERAGE.md
index 810612878..28c4f1588 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -35,16 +35,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 33
-- Mapped Rust files: 357
-- Active test blocks: 3504
-- Ignored/manual test blocks: 144
-- Weighted coverage points: 2890.4
+- Mapped Rust files: 358
+- Active test blocks: 3513
+- Ignored/manual test blocks: 146
+- Weighted coverage points: 2899.4
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3361 | 138 | 2825.3 | 21 | 11 | 100% |
-| macos | 30 | 3423 | 119 | 2839.3 | 22 | 11 | 100% |
-| linux | 26 | 2991 | 111 | 2484.6 | 20 | 11 | 100% |
+| windows | 30 | 3370 | 140 | 2834.3 | 21 | 11 | 100% |
+| macos | 30 | 3432 | 121 | 2848.3 | 22 | 11 | 100% |
+| linux | 26 | 3000 | 113 | 2493.6 | 20 | 11 | 100% |
## Refresh
diff --git a/Cargo.lock b/Cargo.lock
index 3ea6dc455..c76886590 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8860,6 +8860,7 @@ dependencies = [
"rusqlite",
"screenpipe-config",
"screenpipe-core",
+ "screenpipe-resource",
"screenpipe-secrets",
"screenpipe-semantic",
"screenpipe-sqlite-coordinator",
diff --git a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
index eef160dfe..9c9c467d8 100644
--- a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
+++ b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
@@ -11535,6 +11535,7 @@ dependencies = [
"regex",
"rusqlite",
"screenpipe-config",
+ "screenpipe-resource",
"screenpipe-semantic",
"screenpipe-sqlite-coordinator",
"serde",
diff --git a/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock b/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock
index 2c8887d41..f86fedb03 100644
--- a/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock
+++ b/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock
@@ -11534,6 +11534,7 @@ dependencies = [
"regex",
"rusqlite",
"screenpipe-config",
+ "screenpipe-resource",
"screenpipe-semantic",
"screenpipe-sqlite-coordinator",
"serde",
diff --git a/crates/screenpipe-db/Cargo.toml b/crates/screenpipe-db/Cargo.toml
index ee82e2ce4..3f91e3429 100644
--- a/crates/screenpipe-db/Cargo.toml
+++ b/crates/screenpipe-db/Cargo.toml
@@ -31,6 +31,7 @@ futures = { workspace = true, features = ["std"] }
screenpipe-config = { path = "../screenpipe-config" }
screenpipe-semantic = { path = "../screenpipe-semantic" }
+screenpipe-resource = { path = "../screenpipe-resource" }
screenpipe-sqlite-coordinator = { path = "../screenpipe-sqlite-coordinator" }
zerocopy = { version = "0.7.32" }
diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rs
index be885ce6d..a953a48ed 100644
--- a/crates/screenpipe-db/src/db/setup.rs
+++ b/crates/screenpipe-db/src/db/setup.rs
@@ -521,8 +521,25 @@ impl DatabaseManager {
}
crate::storage::schema::verify(&mut conn, &storage.descriptor).await?;
crate::storage::schema::upgrade_resident_frames(&mut conn).await?;
+ let upgraded =
+ crate::storage::schema::upgrade_recording(&mut conn, storage.has_bulk())
+ .await?;
crate::storage::read_schema::upgrade(&mut conn, storage).await?;
storage.verify_catalog(&db_manager.pool).await?;
+ drop(conn);
+ if upgraded {
+ // Connections opened before the trigger migration need a
+ // schema read before their first DML preparation. Refresh
+ // every existing writer while startup still owns admission.
+ let mut writers = Vec::new();
+ for _ in 0..db_manager.write_pool.size() {
+ let mut conn = db_manager.write_pool.acquire().await?;
+ sqlx::query("SELECT name FROM main.sqlite_schema LIMIT 1")
+ .fetch_optional(&mut *conn)
+ .await?;
+ writers.push(conn);
+ }
+ }
}
Ok::<(), sqlx::Error>(())
}
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
index 4fbfa31e3..2d76307ea 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
@@ -19,6 +19,7 @@ struct Encoded {
first: i64,
last: i64,
rows: usize,
+ generations: Vec<(i64, i64)>,
}
fn record(row: &sqlx::sqlite::SqliteRow) -> Result<Record, sqlx::Error> {
@@ -51,17 +52,41 @@ pub(crate) async fn seal(
) -> Result<usize, sqlx::Error> {
crate::storage::diagnostics::batch("elements", None, None, None, None);
crate::storage::diagnostics::stage("selecting_staged_elements");
- let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
- "SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND ({}) ORDER BY id LIMIT 1",
+ // A dirty archive must be rewritten as a whole. If privacy processing
+ // blocks that range, continue after it instead of starving newer work.
+ let mut after = None;
+ loop {
+ let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));
+ let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
+ "SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND {lower} AND ({}) ORDER BY id LIMIT 1",
+ TABLE.sealable()
+ )))
+ .fetch_optional(pool)
+ .await?;
+ let Some(id) = id else { return Ok(0) };
+ let range: Option<(i64, i64, i64)> = sqlx::query_as("SELECT first_id,last_id,file_id FROM _bulk_element_ranges WHERE first_id=(SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?1) AND last_id>=?1")
+ .bind(id).fetch_optional(pool).await?;
+ if let Some((first, last, _)) = range {
+ if range_blocked(pool, first, last).await? {
+ after = Some(last);
+ continue;
+ }
+ }
+ // A concurrent edit invalidating publication is retried next tick,
+ // rather than spinning over a stream of newly arriving records.
+ return rewrite(storage, pool, writer, id, range).await;
+ }
+}
+
+async fn range_blocked(pool: &SqlitePool, first: i64, last: i64) -> Result<bool, sqlx::Error> {
+ sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
+ "SELECT EXISTS(SELECT 1 FROM elements WHERE id BETWEEN ? AND ? AND NOT ({}))",
TABLE.sealable()
)))
- .fetch_optional(pool)
- .await?;
- let Some(id) = id else {
- return Ok(0);
- };
- let range:Option<(i64,i64,i64)>=sqlx::query_as("SELECT first_id,last_id,file_id FROM _bulk_element_ranges WHERE first_id=(SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?1) AND last_id>=?1").bind(id).fetch_optional(pool).await?;
- rewrite(storage, pool, writer, id, range).await
+ .bind(first)
+ .bind(last)
+ .fetch_one(pool)
+ .await
}
async fn encode(
@@ -114,6 +139,7 @@ async fn encode(
first: rows.first().unwrap().id,
last: rows.last().unwrap().id,
rows: rows.len(),
+ generations: rows.iter().map(|row| (row.id, row.generation)).collect(),
};
let budget = storage.descriptor.budget.clone();
let file = path.clone();
@@ -179,15 +205,7 @@ async fn rewrite(
.fetch_one(pool)
.await?;
let (first, last) = if let Some((first, last, _)) = range {
- let blocked: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
- "SELECT EXISTS(SELECT 1 FROM elements WHERE id BETWEEN ? AND ? AND NOT ({}))",
- TABLE.sealable()
- )))
- .bind(first)
- .bind(last)
- .fetch_one(pool)
- .await?;
- if blocked {
+ if range_blocked(pool, first, last).await? {
return Ok(0);
}
(first, last)
@@ -256,6 +274,13 @@ async fn rewrite(
return Ok(0);
};
crate::storage::diagnostics::stage("publishing_element_archive");
+ #[cfg(test)]
+ {
+ let hook = storage.bulk.element_publish_hook.lock().unwrap().clone();
+ if let Some(hook) = hook {
+ tokio::task::spawn_blocking(move || hook()).await.unwrap();
+ }
+ }
let permit = writer.lock().await?;
let mut tx = permit.pool().begin().await?;
let current: i64 = sqlx::query_scalar("SELECT version FROM _bulk_element_state")
@@ -265,7 +290,14 @@ async fn rewrite(
sqlx::query_as("SELECT policy,required_surfaces FROM storage_metadata")
.fetch_one(&mut *tx)
.await?;
- if current != version || current_policy != policy {
+ // New captures change the table-wide version while this batch encodes.
+ // Only mutations inside the range being published can invalidate it.
+ // Retain the O(1) check when nothing changed; otherwise compare resident
+ // generations, including tombstones and newly eligible gap inserts.
+ let valid = current_policy == policy
+ && (current == version
+ || resident_range_unchanged(&mut tx, first, last, &files, range.is_some()).await?);
+ if !valid {
for file in files {
sqlx::query("UPDATE _bulk_files SET state='retired' WHERE id=?")
.bind(file.id)
@@ -329,6 +361,42 @@ async fn rewrite(
Ok(count)
}
+async fn resident_range_unchanged(
+ conn: &mut sqlx::SqliteConnection,
+ first: i64,
+ last: i64,
+ files: &[Encoded],
+ whole_range: bool,
+) -> Result<bool, sqlx::Error> {
+ let mut rows = sqlx::query(sqlx::AssertSqlSafe(format!(
+ "SELECT id,_archive_generation,_archive_deleted,({}) AS sealable FROM _bulk_element_rows WHERE id BETWEEN ? AND ? ORDER BY id",
+ TABLE.sealable()
+ )))
+ .bind(first).bind(last).fetch(&mut *conn);
+ while let Some(row) = rows.try_next().await? {
+ let id: i64 = row.try_get("id")?;
+ let generation: i64 = row.try_get("_archive_generation")?;
+ let deleted: bool = row.try_get("_archive_deleted")?;
+ let sealable: bool = row.try_get("sealable")?;
+ let file = files.partition_point(|file| file.last < id);
+ let encoded = files.get(file).and_then(|file| {
+ file.generations
+ .binary_search_by_key(&id, |(id, _)| *id)
+ .ok()
+ .map(|index| file.generations[index].1)
+ });
+ match encoded {
+ Some(expected) if deleted || !sealable || generation != expected => return Ok(false),
+ // A newly eligible record in a gap must not be deleted without
+ // being encoded. Unchanged private gaps and old tombstones stay
+ // governed by the existing publication selection.
+ None if !deleted && (whole_range || sealable) => return Ok(false),
+ _ => {}
+ }
+ }
+ Ok(true)
+}
+
pub(crate) async fn reclaim(
storage: &Arc<HybridStorage>,
pool: &SqlitePool,
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
index fd940947f..2b341db28 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs
@@ -111,9 +111,16 @@ struct RowSql {
static ROW_SQL: LazyLock<RowSql> = LazyLock::new(|| {
let columns = names();
let projection = format!("id,_archive_generation,{columns},_archive_deleted");
+ // Preserve the tombstone's generation when an ID is deleted and reused.
+ // An encoder holding the old generation must never accept the new row.
+ let updates = format!("_archive_deleted,_archive_file,{columns}")
+ .split(',')
+ .map(|column| format!("{column}=excluded.{column}"))
+ .collect::<Vec<_>>()
+ .join(",");
RowSql {
lookup: format!("SELECT {projection} FROM _bulk_element_rows WHERE id=?"),
- upsert: format!("INSERT OR REPLACE INTO _bulk_element_rows(id,_archive_generation,_archive_deleted,_archive_file,{columns}) VALUES({})", vec!["?";TABLE.columns.len()+4].join(",")),
+ upsert: format!("INSERT INTO _bulk_element_rows(id,_archive_generation,_archive_deleted,_archive_file,{columns}) VALUES({}) ON CONFLICT(id) DO UPDATE SET _archive_generation=max(_bulk_element_rows._archive_generation+1,excluded._archive_generation),{updates}", vec!["?";TABLE.columns.len()+4].join(",")),
resident: format!("SELECT COALESCE((SELECT {} FROM _bulk_element_rows WHERE id=?),0)", TABLE.all_bytes("")),
projection,
}
@@ -323,9 +330,6 @@ impl Elements {
return Err(constraint("required element column is NULL"));
}
}
- if new.bytes() > self.storage.descriptor.budget.record_bytes {
- return Err(constraint("element record budget exceeded"));
- }
let Value::Integer(frame) = new.values[0] else {
return Err(constraint("invalid element frame"));
};
diff --git a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
index fdd3293cb..5f93f35c7 100644
--- a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
@@ -11,23 +11,18 @@ use sqlx::{Row, SqlitePool, ValueRef};
use std::{path::Path, sync::Arc};
impl HybridStorage {
- pub(crate) async fn seal_bulk(
+ pub(in crate::storage) async fn seal_bulk_table(
self: &Arc<Self>,
pool: &SqlitePool,
writer: &SqliteWritePool,
+ table: &'static Table,
) -> Result<usize, sqlx::Error> {
- if !self.has_bulk() {
- return Ok(0);
- }
- let elements = super::elements::seal(self, pool, writer).await?;
- if elements != 0 {
- return Ok(elements);
+ if table.name == "elements" {
+ return super::elements::seal(self, pool, writer).await;
}
- for table in TABLES.iter().filter(|t| t.name != "elements") {
- let rows = self.select_bulk(pool, table, None).await?;
- if !rows.is_empty() {
- return self.publish_bulk(pool, writer, table, rows, None).await;
- }
+ let rows = self.select_bulk(pool, table, None).await?;
+ if !rows.is_empty() {
+ return self.publish_bulk(pool, writer, table, rows, None).await;
}
Ok(0)
}
diff --git a/crates/screenpipe-db/src/storage/bulk/mod.rs b/crates/screenpipe-db/src/storage/bulk/mod.rs
index 3078b7a38..9c03f2ee8 100644
--- a/crates/screenpipe-db/src/storage/bulk/mod.rs
+++ b/crates/screenpipe-db/src/storage/bulk/mod.rs
@@ -18,7 +18,7 @@ use super::{storage_error, HybridStorage};
pub(crate) use connection::pool_options;
pub(crate) use connection::register_hash;
pub(super) use lifecycle::export;
-pub(super) use schema::{bootstrap, bootstrap_in_place, finish_indexes};
+pub(super) use schema::{bootstrap, bootstrap_in_place, finish_indexes, upgrade_recording};
use std::sync::{atomic::AtomicUsize, Arc, Mutex};
pub const CAPABILITY: &str = "parquet-bulk-v1";
@@ -209,6 +209,8 @@ pub(crate) struct Runtime {
element_frames: Mutex<Option<(String, Arc<std::collections::HashMap<i64, Vec<usize>>>)>>,
#[cfg(test)]
decode_hook: Mutex<Option<Arc<dyn Fn(&str) + Send + Sync>>>,
+ #[cfg(test)]
+ element_publish_hook: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
}
#[derive(Clone)]
diff --git a/crates/screenpipe-db/src/storage/bulk/schema.rs b/crates/screenpipe-db/src/storage/bulk/schema.rs
index 6f6d8839f..73e4925d2 100644
--- a/crates/screenpipe-db/src/storage/bulk/schema.rs
+++ b/crates/screenpipe-db/src/storage/bulk/schema.rs
@@ -127,20 +127,18 @@ fn triggers(table: &Table, original_columns: &[String]) -> String {
let t = table.name;
let new_bytes = table.all_bytes("NEW.");
let old_bytes = table.local_bytes("OLD.");
- let check=format!("SELECT CASE WHEN ({new_bytes})>(SELECT record_limit FROM storage_metadata) THEN RAISE(ABORT,'bulk record budget exceeded') END;");
- let staging_check = "SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata) THEN RAISE(ABORT,'storage staging budget reached') END;";
let fts = reindex(table);
let fts_delete = if table.fts.is_empty() {
String::new()
} else {
format!("DELETE FROM {t}_fts WHERE rowid=OLD.id;")
};
- let mut sql=format!("CREATE TRIGGER hybrid_bulk_{t}_id BEFORE UPDATE OF id ON {t} WHEN OLD._archive_file IS NOT NULL AND NEW.id!=OLD.id BEGIN SELECT RAISE(ABORT,'archived record IDs are immutable'); END; CREATE TRIGGER hybrid_bulk_{t}_insert AFTER INSERT ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN {check} UPDATE storage_metadata SET staging_bytes=staging_bytes+({new_bytes}),revision=revision+1; {staging_check} {fts} END; CREATE TRIGGER hybrid_bulk_{t}_delete BEFORE DELETE ON {t} BEGIN {fts_delete} UPDATE _bulk_files SET state='dirty' WHERE id=OLD._archive_file; UPDATE storage_metadata SET staging_bytes=staging_bytes-({old_bytes}),revision=revision+1; END;");
+ let mut sql=format!("CREATE TRIGGER hybrid_bulk_{t}_id BEFORE UPDATE OF id ON {t} WHEN OLD._archive_file IS NOT NULL AND NEW.id!=OLD.id BEGIN SELECT RAISE(ABORT,'archived record IDs are immutable'); END; CREATE TRIGGER hybrid_bulk_{t}_insert AFTER INSERT ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN UPDATE storage_metadata SET staging_bytes=staging_bytes+({new_bytes}),revision=revision+1; {fts} END; CREATE TRIGGER hybrid_bulk_{t}_delete BEFORE DELETE ON {t} BEGIN {fts_delete} UPDATE _bulk_files SET state='dirty' WHERE id=OLD._archive_file; UPDATE storage_metadata SET staging_bytes=staging_bytes-({old_bytes}),revision=revision+1; END;");
sql.push_str(&format!("CREATE TRIGGER hybrid_bulk_{t}_revision AFTER UPDATE OF {} ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN UPDATE storage_metadata SET revision=revision+1; END;", original_columns.join(",")));
for (index, column) in table.columns.iter().enumerate() {
let c = column.name;
let bit = 1 << index;
- sql.push_str(&format!("CREATE TRIGGER hybrid_bulk_{t}_{c} AFTER UPDATE OF {c} ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN {check} UPDATE {t} SET _archive_mask=_archive_mask|{bit},_archive_generation=_archive_generation+1 WHERE id=NEW.id; UPDATE _bulk_files SET state='dirty' WHERE id=OLD._archive_file; UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE(length(CAST(NEW.{c} AS BLOB)),0)-CASE WHEN (OLD._archive_mask & {bit})!=0 THEN COALESCE(length(CAST(OLD.{c} AS BLOB)),0) ELSE 0 END; {staging_check} {fts} END;"));
+ sql.push_str(&format!("CREATE TRIGGER hybrid_bulk_{t}_{c} AFTER UPDATE OF {c} ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN UPDATE {t} SET _archive_mask=_archive_mask|{bit},_archive_generation=_archive_generation+1 WHERE id=NEW.id; UPDATE _bulk_files SET state='dirty' WHERE id=OLD._archive_file; UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE(length(CAST(NEW.{c} AS BLOB)),0)-CASE WHEN (OLD._archive_mask & {bit})!=0 THEN COALESCE(length(CAST(OLD.{c} AS BLOB)),0) ELSE 0 END; {fts} END;"));
}
let metadata = table
.fts
@@ -153,3 +151,30 @@ fn triggers(table: &Table, original_columns: &[String]) -> String {
}
sql
}
+
+// Called inside the startup schema transaction; preserve v2's identity while
+// upgrading the installed triggers for existing as well as new generations.
+pub(crate) async fn upgrade_recording(conn: &mut SqliteConnection) -> Result<(), sqlx::Error> {
+ for table in TABLES.iter().filter(|t| t.name != "elements") {
+ let columns: Vec<String> = sqlx::query_scalar(
+ "SELECT name FROM pragma_table_info(?) WHERE name NOT LIKE '_archive_%' ORDER BY cid",
+ )
+ .bind(table.name)
+ .fetch_all(&mut *conn)
+ .await?;
+ let names: Vec<String> = sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=? AND name GLOB 'hybrid_bulk_*'")
+ .bind(table.name).fetch_all(&mut *conn).await?;
+ for name in names {
+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
+ "DROP TRIGGER \"{}\"",
+ name.replace('"', "\"\"")
+ )))
+ .execute(&mut *conn)
+ .await?;
+ }
+ sqlx::raw_sql(sqlx::AssertSqlSafe(triggers(table, &columns)))
+ .execute(&mut *conn)
+ .await?;
+ }
+ Ok(())
+}
diff --git a/crates/screenpipe-db/src/storage/bulk/tests.rs b/crates/screenpipe-db/src/storage/bulk/tests.rs
index ca2ae038d..addf82e54 100644
--- a/crates/screenpipe-db/src/storage/bulk/tests.rs
+++ b/crates/screenpipe-db/src/storage/bulk/tests.rs
@@ -32,6 +32,57 @@ impl Drop for Pause {
}
}
+#[tokio::test]
+async fn element_publication_ignores_new_captures_but_rejects_changed_range_members() {
+ for archived in [false, true] {
+ for mutation in [
+ "INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(30,1,'accessibility','AXText','new capture',1)",
+ "UPDATE elements SET text='edited while encoding' WHERE id=10",
+ "DELETE FROM elements WHERE id=10",
+ "INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(15,1,'accessibility','AXText','inserted gap',1)",
+ "INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(15,1,'accessibility','AXText','private gap',NULL)",
+ "UPDATE elements SET redacted_at=NULL WHERE id=10",
+ "DELETE FROM elements WHERE id=10; INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(10,1,'accessibility','AXText','reused ID',1)",
+ ] {
+ let root = tempfile::tempdir().unwrap();
+ let db = Arc::new(DatabaseManager::new_hybrid(root.path(), Default::default(),
+ crate::storage::MigrationOptions {
+ privacy: crate::storage::PrivacyPolicy { identity: "private".into(), required_surfaces: 1 },
+ ..Default::default()
+ }).await.unwrap());
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-17','pending frame'); INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(10,1,'accessibility','AXText','ten',1),(20,1,'accessibility','AXText','twenty',1)").await.unwrap();
+ if archived {
+ assert_eq!(db.seal_payloads().await.unwrap(), 2);
+ db.execute_raw_sql_write("UPDATE elements SET text='restaged ten' WHERE id=10").await.unwrap();
+ }
+ let storage = db.storage.as_ref().unwrap();
+ let encoded = Arc::new(tokio::sync::Notify::new());
+ let announced = encoded.clone();
+ let pause = Pause::new();
+ let held = Pause(pause.0.clone());
+ *storage.bulk.element_publish_hook.lock().unwrap() = Some(Arc::new(move || {
+ announced.notify_one();
+ held.wait();
+ }));
+ let sealing = db.clone();
+ let task = tokio::spawn(async move { sealing.seal_payloads().await });
+ tokio::time::timeout(Duration::from_secs(10), encoded.notified()).await.unwrap();
+ // The archive is fully encoded while the writer remains available.
+ db.execute_raw_sql_write(mutation).await.unwrap();
+ let expected = db.query_raw_sql("SELECT id,text,redacted_at FROM elements ORDER BY id").await.unwrap();
+ pause.release();
+ let count = task.await.unwrap().unwrap();
+ assert_eq!(count, if mutation.contains("VALUES(30,") || (!archived && mutation.contains("private gap")) { 2 } else { 0 }, "archived={archived}; {mutation}");
+ *storage.bulk.element_publish_hook.lock().unwrap() = None;
+ assert_eq!(db.query_raw_sql("SELECT id,text,redacted_at FROM elements ORDER BY id").await.unwrap(), expected);
+ while db.seal_payloads().await.unwrap() != 0 {}
+ db.verify_storage().await.unwrap();
+ assert_eq!(db.query_raw_sql("SELECT id,text,redacted_at FROM elements ORDER BY id").await.unwrap(), expected);
+ db.close().await;
+ }
+ }
+}
+
async fn fixture_mode(hybrid: bool) -> (tempfile::TempDir, Arc<DatabaseManager>) {
let root = tempfile::tempdir().unwrap();
let db = Arc::new(
diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rs
index 106840530..8335886f2 100644
--- a/crates/screenpipe-db/src/storage/in_place.rs
+++ b/crates/screenpipe-db/src/storage/in_place.rs
@@ -141,9 +141,8 @@ async fn run(
tx.commit().await?;
}
if schema::converted_step(&mut conn,"conversion-complete").await? { return Ok(()); }
- // A paused migration records resident payloads like the original
- // SQLite store. Its disabled sealer must not impose a staging cap
- // that eventually stops recording. Explicit completion restores it.
+ // Compatibility with pre-v6 triggers during interrupted conversion.
+ // The v6 startup upgrade removes capture admission limits entirely.
sqlx::query("UPDATE storage_metadata SET staging_limit=?")
.bind(if archive { storage.descriptor.budget.staging_bytes as i64 } else { i64::MAX })
.execute(&mut *conn).await?;
@@ -258,8 +257,8 @@ async fn run(
let original: Vec<String> = sqlx::query_scalar("SELECT sql FROM _storage_conversion_triggers WHERE restore=1").fetch_all(&mut *tx).await?;
for sql in original { sqlx::raw_sql(sqlx::AssertSqlSafe(sql)).execute(&mut *tx).await?; }
if archive {
- // Existing resident history occupies disk, not the capture work
- // queue. Preserve a full configured staging allowance above it.
+ // Preserve the legacy allowance for older schema readers. The
+ // current recording triggers do not enforce this as a cap.
sqlx::query("UPDATE storage_metadata SET staging_limit=staging_bytes+?")
.bind(storage.descriptor.budget.staging_bytes as i64)
.execute(&mut *tx).await?;
diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rs
index bfd2f92cd..b3648256e 100644
--- a/crates/screenpipe-db/src/storage/mod.rs
+++ b/crates/screenpipe-db/src/storage/mod.rs
@@ -75,10 +75,13 @@ pub struct StorageBudget {
pub row_group_rows: usize,
pub file_rows: usize,
pub file_bytes: usize,
+ /// Maximum record the archiver processes; larger captures remain in SQLite.
pub record_bytes: usize,
pub decode_bytes: usize,
pub response_bytes: usize,
pub concurrent_decodes: usize,
+ /// Background reclamation headroom and legacy staging allowance. It must
+ /// never limit durable recording; memory is bounded by processing batches.
pub staging_bytes: u64,
pub disk_reserve_bytes: u64,
pub operation_timeout_secs: u64,
@@ -106,7 +109,7 @@ impl Default for StorageBudget {
}
impl StorageBudget {
- /// Oversized legacy frames are resident history, not work for the sealer.
+ /// Oversized frames are resident history, not work for the sealer.
pub(super) fn staged_frame_bytes(&self, bytes: usize) -> usize {
if bytes <= self.record_bytes {
bytes
@@ -195,7 +198,8 @@ pub(crate) struct HybridStorage {
pub gate: Arc<Mutex<()>>,
pub leases: Arc<RwLock<()>>,
pub decoder: Arc<Semaphore>,
- pub file_job: Mutex<()>,
+ /// Serializes file jobs; the value is the next table's archival turn.
+ pub file_job: Mutex<usize>,
pub read_lanes: std::sync::OnceLock<Arc<Semaphore>>,
pub closing: tokio_util::sync::CancellationToken,
pub privacy_ready: std::sync::atomic::AtomicBool,
@@ -471,7 +475,7 @@ impl HybridStorage {
descriptor,
gate: Arc::new(Mutex::new(())),
leases: Arc::new(RwLock::new(())),
- file_job: Mutex::new(()),
+ file_job: Mutex::new(0),
read_lanes: std::sync::OnceLock::new(),
closing: tokio_util::sync::CancellationToken::new(),
privacy_ready: std::sync::atomic::AtomicBool::new(false),
diff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rs
index c18fdb5bc..44b2e617c 100644
--- a/crates/screenpipe-db/src/storage/schema.rs
+++ b/crates/screenpipe-db/src/storage/schema.rs
@@ -239,10 +239,11 @@ const EMPTY_SURFACES: &str = "(CASE WHEN COALESCE(NEW.full_text,'')='' THEN 1 EL
fn triggers() -> String {
// Version 1's checksum is part of existing generation identities.
- frame_triggers(false)
+ frame_triggers(false, false)
}
-fn frame_triggers(retain_oversized: bool) -> String {
+// Keep the v1/v4 SQL byte-for-byte stable for installed schema checksums.
+fn frame_triggers(retain_oversized: bool, recording: bool) -> String {
let charge = |bytes: &str| {
if retain_oversized {
format!("CASE WHEN ({bytes}) <= (SELECT record_limit FROM storage_metadata) THEN ({bytes}) ELSE 0 END")
@@ -257,18 +258,40 @@ fn frame_triggers(retain_oversized: bool) -> String {
} else {
String::new()
};
+ let insert_admission = if recording {
+ String::new()
+ } else {
+ format!(
+ r#"
+ SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata)
+ THEN RAISE(ABORT,'frame storage: record budget exceeded') END;
+ SELECT CASE WHEN (SELECT staging_bytes+({BYTES})>staging_limit FROM storage_metadata)
+ THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;"#
+ )
+ };
+ let update_admission = if recording {
+ String::new()
+ } else {
+ format!(
+ r#"
+ SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata){growth_guard}
+ THEN RAISE(ABORT,'frame storage: record budget exceeded') END;"#
+ )
+ };
+ let staging_admission = if recording {
+ ""
+ } else {
+ "\n SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata)\n THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;"
+ };
+ let inserted_bytes = if recording { &new_staging_bytes } else { BYTES };
format!(
r#"
CREATE TRIGGER hybrid_frame_insert AFTER INSERT ON frames
WHEN (SELECT maintenance=0 FROM storage_metadata WHERE singleton=1)
-BEGIN
- SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata)
- THEN RAISE(ABORT,'frame storage: record budget exceeded') END;
- SELECT CASE WHEN (SELECT staging_bytes+({BYTES})>staging_limit FROM storage_metadata)
- THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;
+BEGIN{insert_admission}
INSERT INTO frame_payloads(frame_id,generation,state,bytes,policy,completed_surfaces,capture_version)
SELECT NEW.id,1,'staged',({BYTES}),policy,{EMPTY_SURFACES},writer_version FROM storage_metadata;
- UPDATE storage_metadata SET staging_bytes=staging_bytes+({BYTES}),revision=revision+1;
+ UPDATE storage_metadata SET staging_bytes=staging_bytes+({inserted_bytes}),revision=revision+1;
UPDATE frames SET payload_full_text_length=length(COALESCE(NEW.full_text,'')),
payload_accessibility_length=length(COALESCE(NEW.accessibility_text,'')),
payload_full_text_present=NEW.full_text IS NOT NULL,
@@ -285,13 +308,9 @@ WHEN (SELECT maintenance=0 FROM storage_metadata) AND
BEGIN SELECT RAISE(ABORT,'frame storage: sealed frame requires coordinated payload replacement'); END;
CREATE TRIGGER hybrid_frame_update AFTER UPDATE OF full_text,accessibility_text,accessibility_tree_json,text_json,app_name,window_name,browser_url ON frames
WHEN (SELECT maintenance=0 FROM storage_metadata)
-BEGIN
- SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata){growth_guard}
- THEN RAISE(ABORT,'frame storage: record budget exceeded') END;
+BEGIN{update_admission}
UPDATE storage_metadata SET revision=revision+1,
- staging_bytes=staging_bytes+({new_staging_bytes})-(SELECT {prior_staging_bytes} FROM frame_payloads WHERE frame_id=NEW.id);
- SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata)
- THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;
+ staging_bytes=staging_bytes+({new_staging_bytes})-(SELECT {prior_staging_bytes} FROM frame_payloads WHERE frame_id=NEW.id);{staging_admission}
UPDATE frame_payloads SET generation=generation+1,bytes=({BYTES}),
policy=(SELECT policy FROM storage_metadata),completed_surfaces={EMPTY_SURFACES},attempts=0,retry_at=NULL,last_error=NULL
WHERE frame_id=NEW.id;
@@ -327,7 +346,7 @@ END;
pub(crate) async fn upgrade_resident_frames(
conn: &mut SqliteConnection,
) -> Result<(), sqlx::Error> {
- let sql = frame_triggers(true);
+ let sql = frame_triggers(true, false);
let checksum = format!("{:x}", Sha256::digest(format!("{sql}:resident-frames-v1")));
let installed: Option<String> =
sqlx::query_scalar("SELECT checksum FROM _hybrid_migrations WHERE version=4")
@@ -365,6 +384,47 @@ pub(crate) async fn upgrade_resident_frames(
Ok(())
}
+/// Archival budgets never reject recording. Replace the v1/v4/v5 write guards
+/// atomically without changing their recorded identities or resident bytes.
+pub(crate) async fn upgrade_recording(
+ conn: &mut SqliteConnection,
+ has_bulk: bool,
+) -> Result<bool, sqlx::Error> {
+ let sql = frame_triggers(true, true);
+ let checksum = format!("{:x}", Sha256::digest(format!("{sql}:recording-v6")));
+ let installed: Option<String> =
+ sqlx::query_scalar("SELECT checksum FROM _hybrid_migrations WHERE version=6")
+ .fetch_optional(&mut *conn)
+ .await?;
+ if let Some(installed) = installed {
+ return if installed == checksum {
+ Ok(false)
+ } else {
+ Err(storage_error("recording schema checksum mismatch"))
+ };
+ }
+ let mut tx = conn.begin().await?;
+ for name in ["insert", "update", "delete", "sealed_guard", "metadata"] {
+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
+ "DROP TRIGGER hybrid_frame_{name}"
+ )))
+ .execute(&mut *tx)
+ .await?;
+ }
+ sqlx::raw_sql(sqlx::AssertSqlSafe(sql))
+ .execute(&mut *tx)
+ .await?;
+ if has_bulk {
+ super::bulk::upgrade_recording(&mut tx).await?;
+ }
+ sqlx::query("INSERT INTO _hybrid_migrations VALUES(6,?)")
+ .bind(checksum)
+ .execute(&mut *tx)
+ .await?;
+ tx.commit().await?;
+ Ok(true)
+}
+
pub(crate) async fn bootstrap(
conn: &mut SqliteConnection,
descriptor: &StorageDescriptor,
@@ -460,6 +520,203 @@ mod checkpoint_tests {
use sqlx::Connection;
use std::time::Duration;
+ #[tokio::test]
+ async fn recording_upgrade_replaces_existing_guards_once() {
+ use crate::storage::{MigrationOptions, PrivacyPolicy};
+ use crate::DatabaseManager;
+
+ for had_v5 in [false, true] {
+ let root = tempfile::tempdir().unwrap();
+ let options = MigrationOptions {
+ privacy: PrivacyPolicy {
+ identity: "private".into(),
+ required_surfaces: 1,
+ },
+ ..Default::default()
+ };
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)
+ .await
+ .unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-17','original'); INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(1,'2026-09-17','text','original'); UPDATE storage_metadata SET staging_limit=1;").await.unwrap();
+ let identities: Vec<(i64, String)> = sqlx::query_as(
+ "SELECT version,checksum FROM _hybrid_migrations WHERE version<5 ORDER BY version",
+ )
+ .fetch_all(&db.pool)
+ .await
+ .unwrap();
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ let mut legacy = frame_triggers(true, false);
+ if had_v5 {
+ let charge = |bytes: &str| {
+ format!("CASE WHEN ({bytes}) <= (SELECT record_limit FROM storage_metadata) THEN ({bytes}) ELSE 0 END")
+ };
+ let guard =
+ "SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata)";
+ legacy = legacy.replace(guard, &format!("SELECT CASE WHEN ({})>({}) AND (SELECT staging_bytes>staging_limit FROM storage_metadata)", charge(BYTES), charge(&BYTES.replace("NEW.", "OLD."))));
+ sqlx::query("INSERT INTO _hybrid_migrations VALUES(5,?)")
+ .bind(format!("{:x}", Sha256::digest("staging-drain-v1")))
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ }
+ for name in ["insert", "update", "delete", "sealed_guard", "metadata"] {
+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
+ "DROP TRIGGER hybrid_frame_{name}"
+ )))
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ }
+ sqlx::raw_sql(sqlx::AssertSqlSafe(legacy))
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ // Restore the earlier bulk admission checks on existing tables.
+ for (name, growth) in [
+ ("hybrid_bulk_ui_events_text_content", "COALESCE(length(CAST(NEW.text_content AS BLOB)),0)>CASE WHEN (OLD._archive_mask & 1)!=0 THEN COALESCE(length(CAST(OLD.text_content AS BLOB)),0) ELSE 0 END AND "),
+ ("hybrid_bulk_pipe_executions_insert", "COALESCE(length(CAST(NEW.stdout AS BLOB)),0)>0 AND "),
+ ] {
+ let sql: String = sqlx::query_scalar("SELECT sql FROM sqlite_master WHERE name=?").bind(name).fetch_one(&mut **tx.conn()).await.unwrap();
+ let guard = format!("BEGIN SELECT CASE WHEN {}(SELECT staging_bytes>staging_limit FROM storage_metadata) THEN RAISE(ABORT,'storage staging budget reached') END;", if had_v5 { growth } else { "" });
+ let sql = sql.replacen("BEGIN", &guard, 1);
+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!("DROP TRIGGER {name}; {sql}"))).execute(&mut **tx.conn()).await.unwrap();
+ }
+ sqlx::query("DELETE FROM _hybrid_migrations WHERE version=6")
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ assert!(db
+ .execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-17','capture')"
+ )
+ .await
+ .is_err());
+ assert!(db
+ .execute_raw_sql_write(
+ "UPDATE ui_events SET text_content='growing capture payload' WHERE id=1"
+ )
+ .await
+ .is_err());
+ let mut expected_bytes: i64 =
+ sqlx::query_scalar("SELECT staging_bytes FROM storage_metadata")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ db.close().await;
+ for iteration in 0..2 {
+ let db = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ // No warm-up SQL on a writer: the first application write must work.
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query(
+ "INSERT INTO frames(timestamp,full_text) VALUES('2026-09-17','capture')",
+ )
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ expected_bytes += "capture".len() as i64;
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ expected_bytes
+ );
+ db.execute_raw_sql_write("UPDATE ui_events SET text_content='growing capture payload' WHERE id=1; INSERT INTO pipe_executions(pipe_name,status,stdout) VALUES('test','running','new payload')").await.unwrap();
+ assert_eq!(sqlx::query_as::<_,(i64,String)>("SELECT version,checksum FROM _hybrid_migrations WHERE version<5 ORDER BY version").fetch_all(&db.pool).await.unwrap(), identities);
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(
+ "SELECT count(*) FROM _hybrid_migrations WHERE version=6"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ 1
+ );
+ if had_v5 {
+ assert_eq!(
+ sqlx::query_scalar::<_, String>(
+ "SELECT checksum FROM _hybrid_migrations WHERE version=5"
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap(),
+ format!("{:x}", Sha256::digest("staging-drain-v1"))
+ );
+ }
+ expected_bytes = sqlx::query_scalar("SELECT staging_bytes FROM storage_metadata")
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ assert!(expected_bytes > 1, "iteration {iteration}");
+ db.verify_storage().await.unwrap();
+ db.close().await;
+ }
+ }
+ }
+
+ #[tokio::test]
+ #[ignore = "manual old/new capture write throughput comparison"]
+ async fn recording_write_cost() {
+ let root = tempfile::tempdir().unwrap();
+ let db =
+ crate::DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ let payload = "x".repeat(4096);
+ for round in 0..6 {
+ let recording = round % 2 != 0;
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ for name in ["insert", "update", "delete", "sealed_guard", "metadata"] {
+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
+ "DROP TRIGGER hybrid_frame_{name}"
+ )))
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ }
+ sqlx::raw_sql(sqlx::AssertSqlSafe(frame_triggers(true, recording)))
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ let start = std::time::Instant::now();
+ for batch in 0..20 {
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query("SELECT name FROM main.sqlite_schema LIMIT 1")
+ .fetch_optional(&mut **tx.conn())
+ .await
+ .unwrap();
+ for row in 0..50 {
+ sqlx::query(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(?,'2026-09-17',?)",
+ )
+ .bind(batch * 50 + row + 1)
+ .bind(&payload)
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ }
+ tx.commit().await.unwrap();
+ }
+ eprintln!(
+ "{}: 1000 4-KiB captures / 20 commits in {:?}",
+ if recording { "new" } else { "old" },
+ start.elapsed()
+ );
+ db.execute_raw_sql_write("DELETE FROM frames")
+ .await
+ .unwrap();
+ }
+ db.close().await;
+ }
+
#[tokio::test]
async fn upgrades_existing_oversized_frame_accounting_once_without_changing_v1_identity() {
use crate::storage::{MigrationOptions, Projection, StorageBudget};
@@ -495,7 +752,7 @@ mod checkpoint_tests {
.execute(&mut **tx.conn())
.await
.unwrap();
- sqlx::raw_sql("DELETE FROM _hybrid_migrations WHERE version=4; UPDATE storage_metadata SET maintenance=1; INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','legacy history',printf('%.*c',2097152,'x')),(2,'2026-09-14T12:00:01Z','pending capture',NULL);")
+ sqlx::raw_sql("DELETE FROM _hybrid_migrations WHERE version>=4; UPDATE storage_metadata SET maintenance=1; INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','legacy history',printf('%.*c',2097152,'x')),(2,'2026-09-14T12:00:01Z','pending capture',NULL);")
.execute(&mut **tx.conn()).await.unwrap();
stage_frames(tx.conn(), 1, 2).await.unwrap();
sqlx::query(
diff --git a/crates/screenpipe-db/src/storage/sealing.rs b/crates/screenpipe-db/src/storage/sealing.rs
index f30b54295..45e85965e 100644
--- a/crates/screenpipe-db/src/storage/sealing.rs
+++ b/crates/screenpipe-db/src/storage/sealing.rs
@@ -9,7 +9,11 @@ use screenpipe_sqlite_coordinator::SqliteWritePool;
use sqlx::{Row, SqlitePool};
use std::path::Path;
use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
+
+const IDLE_POLL: Duration = Duration::from_secs(5);
+const MIN_COOLDOWN: Duration = Duration::from_millis(50);
+const MAX_COOLDOWN: Duration = Duration::from_secs(60);
impl DatabaseManager {
pub(crate) async fn strip_archived_frame_details(
@@ -121,7 +125,7 @@ impl DatabaseManager {
return Err(storage_error("replacement requires hybrid storage"));
};
let mut tx = self.begin_immediate_with_retry().await?;
- let row = sqlx::query("SELECT p.generation,p.state,p.file_id,p.bytes,m.policy,m.staging_bytes,m.staging_limit FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.frame_id=?")
+ let row = sqlx::query("SELECT p.generation,p.state,p.file_id,p.bytes,m.policy,m.staging_bytes FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.frame_id=?")
.bind(payload.id).fetch_optional(&mut **tx.conn()).await?;
let Some(row) = row else {
tx.rollback().await?;
@@ -133,13 +137,6 @@ impl DatabaseManager {
tx.rollback().await?;
return Ok(false);
}
- // Retained legacy frames can still be redacted or have detail removed.
- // New oversized payloads remain subject to normal capture admission.
- if payload.bytes() > storage.descriptor.budget.record_bytes
- && payload.bytes() > row.get::<i64, _>("bytes") as usize
- {
- return Err(storage_error("record budget exceeded"));
- }
let prior_bytes = if row.get::<&str, _>("state") == "staged" {
storage
.descriptor
@@ -153,9 +150,6 @@ impl DatabaseManager {
.descriptor
.budget
.staged_frame_bytes(payload.bytes()) as i64;
- if next_bytes > row.get::<i64, _>("staging_limit") {
- return Err(storage_error("staging budget reached"));
- }
let old_file: Option<String> = row.try_get("file_id")?;
sqlx::query(
"UPDATE storage_metadata SET maintenance=1,revision=revision+1,staging_bytes=?",
@@ -199,26 +193,64 @@ impl HybridStorage {
) {
let storage = Arc::clone(self);
tokio::spawn(async move {
+ let governor = screenpipe_resource::ResourceGovernor::global();
+ tokio::select! {
+ _ = shutdown.cancelled() => return,
+ _ = tokio::time::sleep(IDLE_POLL) => {}
+ }
loop {
- tokio::select! {
+ // Capture never takes this background lane. Keep archival
+ // bounded and share the CPU budget with redaction/indexing.
+ let waiting = Instant::now();
+ let cpu_permit = tokio::select! {
_ = shutdown.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(5)) => {}
- }
+ permit = governor.acquire_background_cpu() => permit,
+ };
+ let cpu_wait = waiting.elapsed();
+ let started = Instant::now();
+ let mut archived = 0;
// An in-flight file job retains ownership until it either
// publishes or leaves its durable reservation for restart.
if storage
.privacy_ready
.load(std::sync::atomic::Ordering::Acquire)
{
- if let Err(error) = storage.seal_once(&pool, &writer).await {
- tracing::warn!(%error,"frame storage sealing deferred");
+ match storage.seal_once(&pool, &writer).await {
+ Ok(rows) => archived = rows,
+ Err(error) => tracing::warn!(%error,"frame storage sealing deferred"),
}
}
if shutdown.is_cancelled() {
break;
}
- if let Err(error) = storage.reclaim_once(&pool, &writer).await {
- tracing::warn!(%error,"frame storage reclamation deferred");
+ let reclaimed = match storage.reclaim_once(&pool, &writer).await {
+ Ok(files) => files,
+ Err(error) => {
+ tracing::warn!(%error,"frame storage reclamation deferred");
+ 0
+ }
+ };
+ let worked = started.elapsed();
+ let sample = cpu_permit.finish(worked, MIN_COOLDOWN, MAX_COOLDOWN);
+ let busy = archived != 0 || reclaimed != 0;
+ let delay = if busy { sample.cooldown } else { IDLE_POLL };
+ tracing::debug!(
+ archived_rows = archived,
+ reclaimed_files = reclaimed,
+ cpu_wait_ms = cpu_wait.as_millis(),
+ worked_ms = worked.as_millis(),
+ cooldown_ms = delay.as_millis(),
+ active_cpu_percent = sample.active_cpu_percent,
+ "storage maintenance batch"
+ );
+ // Successful work continues after its measured CPU cooldown;
+ // only an idle/failed pass gets the full polling backoff.
+ // Hold the shared CPU permit through a busy cooldown, just as
+ // the redactor does, but release it before idle polling.
+ let _cooling = busy.then_some(cpu_permit);
+ tokio::select! {
+ _ = shutdown.cancelled() => break,
+ _ = tokio::time::sleep(delay) => {}
}
}
});
@@ -230,7 +262,43 @@ impl HybridStorage {
writer: &SqliteWritePool,
) -> Result<usize, sqlx::Error> {
super::diagnostics::stage("waiting_for_frame_sealer");
- let _job = self.file_job.lock().await;
+ let mut next = self.file_job.lock().await;
+ let tables = 1 + if self.has_bulk() {
+ super::bulk::TABLES.len()
+ } else {
+ 0
+ };
+ // One bounded job per call, rotating even after an error. Empty or
+ // privacy-blocked tables cannot hide ready work later in the cycle.
+ for _ in 0..tables {
+ let turn = *next;
+ *next = (turn + 1) % tables;
+ let started = Instant::now();
+ let (table, result) = if turn == 0 {
+ ("frames", self.seal_frames(pool, writer).await)
+ } else {
+ let table = &super::bulk::TABLES[turn - 1];
+ (table.name, self.seal_bulk_table(pool, writer, table).await)
+ };
+ tracing::debug!(
+ table,
+ rows = result.as_ref().ok().copied(),
+ failed = result.is_err(),
+ elapsed_ms = started.elapsed().as_millis(),
+ "storage archival turn"
+ );
+ if !matches!(result, Ok(0)) {
+ return result;
+ }
+ }
+ Ok(0)
+ }
+
+ async fn seal_frames(
+ self: &Arc<Self>,
+ pool: &SqlitePool,
+ writer: &SqliteWritePool,
+ ) -> Result<usize, sqlx::Error> {
// Existing oversized frames remain readable through the staged SQLite
// path. Filter before LIMIT so they cannot starve later sealable frames.
super::diagnostics::batch("frames", None, None, None, None);
@@ -249,7 +317,7 @@ impl HybridStorage {
bytes += size;
}
if ids.is_empty() {
- return self.seal_bulk(pool, writer).await;
+ return Ok(0);
}
super::diagnostics::batch(
"frames",
@@ -259,11 +327,13 @@ impl HybridStorage {
Some(bytes as u64),
);
super::diagnostics::stage("reading_frame_payloads");
- let payloads: Vec<_> = self
- .read(pool, &ids, Projection::All)
- .await?
- .into_values()
- .collect();
+ // Ordinary captures must not invalidate a read of older frames.
+ // Reuse request snapshots, retaining deletion/privacy revocation.
+ let payloads: Vec<_> =
+ super::snapshot::run(self, pool, self.read(pool, &ids, Projection::All))
+ .await??
+ .into_values()
+ .collect();
self.publish(pool, writer, payloads, None).await
}
@@ -433,9 +503,8 @@ impl HybridStorage {
.fetch_all(pool)
.await?;
if !ids.is_empty() {
- let rows = self
- .read(pool, &ids, Projection::All)
- .await?
+ let rows = super::snapshot::run(self, pool, self.read(pool, &ids, Projection::All))
+ .await??
.into_values()
.collect();
self.publish(pool, writer, rows, Some(&id)).await?;
diff --git a/crates/screenpipe-db/src/storage/snapshot.rs b/crates/screenpipe-db/src/storage/snapshot.rs
index 6d7ec5f96..260b68415 100644
--- a/crates/screenpipe-db/src/storage/snapshot.rs
+++ b/crates/screenpipe-db/src/storage/snapshot.rs
@@ -142,11 +142,10 @@ impl Drop for RequestSnapshot {
}
pub(super) async fn run<T>(
- db: &crate::DatabaseManager,
+ storage: &Arc<super::HybridStorage>,
+ pool: &SqlitePool,
read: impl std::future::Future<Output = T>,
) -> Result<T, sqlx::Error> {
- let storage = db.storage.as_ref().expect("hybrid read");
- let pool = &db.pool;
let lanes = storage.read_lanes.get_or_init(|| {
Arc::new(tokio::sync::Semaphore::new(
pool.options()
@@ -185,7 +184,7 @@ pub(super) async fn run<T>(
};
let (token, result) = SNAPSHOT
.scope(Arc::clone(&owner.context), async {
- let token = db.storage_read_token().await?;
+ let token = storage.read_token(pool).await?;
Ok::<_, sqlx::Error>((token, read.await))
})
.await?;
@@ -221,7 +220,7 @@ impl crate::DatabaseManager {
}
tokio::time::timeout(
Duration::from_secs(storage.descriptor.budget.operation_timeout_secs),
- run(self, read),
+ run(storage, &self.pool, read),
)
.await
.map_err(|_| super::storage_error("payload read deadline exceeded"))?
diff --git a/crates/screenpipe-db/tests/bulk_storage.rs b/crates/screenpipe-db/tests/bulk_storage.rs
index f9639d65e..923fc9003 100644
--- a/crates/screenpipe-db/tests/bulk_storage.rs
+++ b/crates/screenpipe-db/tests/bulk_storage.rs
@@ -451,18 +451,17 @@ async fn deferred_elements_complete_under_staging_pressure() {
}
tx.commit().await.unwrap();
assert_eq!(db.get_frame_elements(1, None).await.unwrap().len(), 3);
- assert!(db
- .execute_raw_sql_write(
- "INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-11T12:01:00Z','next');"
- )
- .await
- .is_err());
- while db.seal_payloads().await.unwrap() != 0 {}
db.execute_raw_sql_write(
"INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-11T12:01:00Z','next');",
)
.await
.unwrap();
+ while db.seal_payloads().await.unwrap() != 0 {}
+ db.execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-09-11T12:02:00Z','after archival');",
+ )
+ .await
+ .unwrap();
assert_eq!(db.get_frame_elements(1, None).await.unwrap().len(), 3);
db.close().await;
}
diff --git a/crates/screenpipe-db/tests/hybrid_storage.rs b/crates/screenpipe-db/tests/hybrid_storage.rs
index 5ded61f70..5ed8c581b 100644
--- a/crates/screenpipe-db/tests/hybrid_storage.rs
+++ b/crates/screenpipe-db/tests/hybrid_storage.rs
@@ -609,9 +609,10 @@ async fn legacy_exports_can_be_migrated_again() {
}
#[tokio::test]
-async fn storage_budget_rejects_before_acknowledging_and_accepts_after_reclamation() {
+async fn archive_record_budget_keeps_large_captures_resident_and_archives_later_work() {
let root = tempfile::tempdir().unwrap();
let mut options = MigrationOptions::default();
+ options.budget.record_bytes = 1024;
options.budget.staging_bytes = options.budget.record_bytes as u64;
let limit = options.budget.record_bytes;
let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)
@@ -619,22 +620,27 @@ async fn storage_budget_rejects_before_acknowledging_and_accepts_after_reclamati
.unwrap();
let large = "x".repeat(limit + 1);
let mut tx = db.begin_immediate_with_retry().await.unwrap();
- let error = sqlx::query("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-11',?)")
+ sqlx::query("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-11',?)")
.bind(&large)
.execute(&mut **tx.conn())
.await
- .unwrap_err();
- assert!(error.to_string().contains("record budget"));
- tx.rollback().await.unwrap();
+ .unwrap();
+ tx.commit().await.unwrap();
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames")
.fetch_one(&db.pool)
.await
.unwrap(),
- 0
+ 1
);
seed(&db, 2, Some("complete payload"), None).await;
- db.seal_frame_payloads().await.unwrap();
+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 1);
+ assert_eq!(
+ db.frame_payloads(&[1], Projection::All).await.unwrap()[&1]
+ .full_text
+ .as_deref(),
+ Some(large.as_str())
+ );
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")
.fetch_one(&db.pool)
diff --git a/crates/screenpipe-db/tests/storage_staging_drain.rs b/crates/screenpipe-db/tests/storage_staging_drain.rs
new file mode 100644
index 000000000..111332401
--- /dev/null
+++ b/crates/screenpipe-db/tests/storage_staging_drain.rs
@@ -0,0 +1,603 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+use screenpipe_db::storage::{MigrationOptions, PrivacyPolicy, Projection};
+use screenpipe_db::DatabaseManager;
+
+async fn number(db: &DatabaseManager, sql: &str) -> i64 {
+ sqlx::query_scalar(sqlx::AssertSqlSafe(sql.to_owned()))
+ .fetch_one(&db.pool)
+ .await
+ .unwrap()
+}
+
+fn private_options() -> MigrationOptions {
+ let mut options = MigrationOptions::default();
+ options.budget.record_bytes = 1024;
+ options.budget.staging_bytes = 1024;
+ options.privacy = PrivacyPolicy {
+ identity: "private".into(),
+ required_surfaces: 1,
+ };
+ options
+}
+
+async fn record(db: &DatabaseManager, text: &str) -> i64 {
+ // Exercise the same queue/commit acknowledgement used by paired_capture.
+ db.insert_snapshot_frame(
+ "display",
+ chrono::Utc::now(),
+ "",
+ Some("Editor"),
+ None,
+ None,
+ true,
+ Some("test"),
+ Some(text),
+ Some("accessibility"),
+ None,
+ None,
+ None,
+ )
+ .await
+ .unwrap()
+}
+
+async fn reopen(root: &std::path::Path) -> DatabaseManager {
+ DatabaseManager::new(root.join("db.sqlite").to_str().unwrap(), Default::default())
+ .await
+ .unwrap()
+}
+
+async fn record_all_tables(db: &DatabaseManager, id: i64) {
+ let frame = record(db, &format!("capture {id} {}", "x".repeat(256))).await;
+ db.execute_raw_sql_write(&format!("INSERT OR IGNORE INTO audio_chunks(id,file_path) VALUES(1,'test.wav');
+ INSERT OR IGNORE INTO meetings(id,meeting_start,meeting_app) VALUES(1,'2026-09-17','test');
+ INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES({id},{frame},'accessibility','AXText','element {id}',1);
+ INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,redacted_at) VALUES({id},1,{id},'2026-09-17','transcript {id}',1);
+ INSERT INTO ui_events(id,timestamp,event_type,text_content,redacted_at) VALUES({id},'2026-09-17','text','typed {id}',1);
+ INSERT INTO semantic_items(id,entity_fingerprint,version_fingerprint,kind,item_key,identity_quality,title,body,metadata_json) VALUES({id},CAST(printf('%032d',{id}) AS BLOB),CAST(printf('%032d',{id}) AS BLOB),'document','document-{id}','stable','title','body {id}','{{}}');
+ INSERT INTO pipe_executions(id,pipe_name,status,finished_at,stdout) VALUES({id},'test','completed','2026-09-17','output {id}');
+ INSERT INTO meeting_transcript_segments(id,meeting_id,provider,item_id,transcript,captured_at) VALUES({id},1,'test','{id}','meeting {id}','2026-09-17');
+ INSERT INTO outputs(id,source,title,output_path,preview) VALUES({id},'test','output','test-{id}.md','preview {id}');")).await.unwrap();
+}
+
+async fn pending_by_table(db: &DatabaseManager) -> Vec<i64> {
+ let mut counts = vec![
+ number(
+ db,
+ "SELECT count(*) FROM frame_payloads WHERE state='staged'",
+ )
+ .await,
+ number(
+ db,
+ "SELECT count(*) FROM _bulk_element_rows WHERE _archive_deleted=0",
+ )
+ .await,
+ ];
+ for table in [
+ "audio_transcriptions",
+ "ui_events",
+ "semantic_items",
+ "pipe_executions",
+ "meeting_transcript_segments",
+ "outputs",
+ ] {
+ counts.push(
+ number(
+ db,
+ &format!("SELECT count(*) FROM main.{table} WHERE _archive_mask!=0"),
+ )
+ .await,
+ );
+ }
+ counts
+}
+
+#[tokio::test]
+async fn every_table_archives_while_all_tables_keep_receiving_records() {
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ // Keep every queue nonempty before every turn, including frames. Each
+ // table must still receive service within a complete rotation.
+ let tables = pending_by_table(&db).await.len();
+ for id in 1..=tables * 3 {
+ record_all_tables(&db, id as i64).await;
+ assert!(db.seal_payloads().await.unwrap() > 0);
+ if id % tables == 0 {
+ let pending = pending_by_table(&db).await;
+ assert!(
+ pending.iter().all(|&count| count < tables as i64),
+ "turn={id}, pending={pending:?}"
+ );
+ }
+ }
+ let pending = pending_by_table(&db).await;
+ db.verify_storage().await.unwrap();
+ db.close().await;
+ let db = reopen(root.path()).await;
+ assert_eq!(pending_by_table(&db).await, pending);
+ while db.seal_payloads().await.unwrap() != 0 {}
+ assert!(pending_by_table(&db).await.iter().all(|&count| count == 0));
+ assert_eq!(
+ number(&db, "SELECT count(*) FROM frames").await,
+ (tables * 3) as i64
+ );
+ assert_eq!(
+ number(&db, "SELECT count(*) FROM elements").await,
+ (tables * 3) as i64
+ );
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
+#[tokio::test]
+async fn failed_frame_archival_does_not_starve_other_tables() {
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ record_all_tables(&db, 1).await;
+ // Break only this disposable database's frame archive directory.
+ let blocked = root
+ .path()
+ .join(&db.storage_descriptor().unwrap().payloads)
+ .join("1");
+ std::fs::write(&blocked, "owned frame archive failure fixture").unwrap();
+ assert!(db.seal_payloads().await.is_err());
+ let tables = pending_by_table(&db).await.len();
+ for _ in 1..tables {
+ record(&db, "recording continues during a frame archive failure").await;
+ assert_eq!(db.seal_payloads().await.unwrap(), 1);
+ }
+ let pending = pending_by_table(&db).await;
+ assert_eq!(pending[0], tables as i64);
+ assert!(pending[1..].iter().all(|&count| count == 0));
+ std::fs::remove_file(blocked).unwrap();
+ assert_eq!(db.seal_payloads().await.unwrap(), tables);
+ db.reclaim_frame_payloads().await.unwrap();
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
+#[tokio::test]
+async fn blocked_old_element_range_does_not_starve_later_private_work() {
+ let root = tempfile::tempdir().unwrap();
+ let options = MigrationOptions {
+ privacy: PrivacyPolicy {
+ identity: "private".into(),
+ required_surfaces: 1,
+ },
+ ..Default::default()
+ };
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)
+ .await
+ .unwrap();
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-17','private'); INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(1,1,'accessibility','AXText','first',1),(2,1,'accessibility','AXText','second',1);").await.unwrap();
+ assert_eq!(db.seal_payloads().await.unwrap(), 2);
+ db.execute_raw_sql_write("UPDATE elements SET text='private again',redacted_at=NULL WHERE id=2; UPDATE elements SET text='safe' WHERE id=1; INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(3,1,'accessibility','AXText','later safe',1);").await.unwrap();
+ assert_eq!(db.seal_payloads().await.unwrap(), 1);
+ assert_eq!(
+ number(
+ &db,
+ "SELECT count(*) FROM _bulk_element_rows WHERE id IN (1,2)"
+ )
+ .await,
+ 2
+ );
+ assert_eq!(
+ number(&db, "SELECT count(*) FROM _bulk_element_rows WHERE id=3").await,
+ 0
+ );
+ assert_eq!(db.seal_payloads().await.unwrap(), 0);
+ // The blocked range becomes eligible normally after redaction, without
+ // losing either the edits or the independently archived newer record.
+ db.execute_raw_sql_write("UPDATE elements SET text='now safe',redacted_at=1 WHERE id=2;")
+ .await
+ .unwrap();
+ assert_eq!(db.seal_payloads().await.unwrap(), 2);
+ assert_eq!(number(&db, "SELECT count(*) FROM elements").await, 3);
+ assert_eq!(
+ number(
+ &db,
+ "SELECT count(*) FROM elements WHERE text IN ('safe','now safe','later safe')"
+ )
+ .await,
+ 3
+ );
+ assert_eq!(
+ number(
+ &db,
+ "SELECT completed_surfaces FROM frame_payloads WHERE frame_id=1"
+ )
+ .await
+ & 1,
+ 0
+ );
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
+#[tokio::test]
+async fn recording_and_growing_redaction_survive_a_full_backlog_and_reopen() {
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), private_options())
+ .await
+ .unwrap();
+ let mut ids = Vec::new();
+ for i in 0..32 {
+ ids.push(record(&db, &format!("capture {i} {}", "x".repeat(128))).await);
+ }
+ assert!(
+ number(
+ &db,
+ "SELECT staging_bytes>staging_limit FROM storage_metadata"
+ )
+ .await
+ != 0
+ );
+ // Privacy is deliberately pending: no archiver can drain these records.
+ assert_eq!(db.seal_payloads().await.unwrap(), 0);
+ db.execute_raw_sql_write("INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(1,'2026-09-17','text','pending'); INSERT INTO pipe_executions(id,pipe_name,status,stdout) VALUES(1,'test','running','output'); INSERT INTO pipe_executions(id,pipe_name,status) VALUES(2,'test','running');").await.unwrap();
+ let mut payload = db
+ .frame_payloads(&[ids[0]], Projection::All)
+ .await
+ .unwrap()
+ .remove(&ids[0])
+ .unwrap();
+ // Replacement tokens can be longer than the original PII. They must save.
+ payload.full_text = Some("redacted ".repeat(60));
+ assert!(db
+ .replace_frame_payload(&payload, "private", 1, None, None)
+ .await
+ .unwrap());
+ // Stale replacements must still fail the generation check.
+ assert!(!db
+ .replace_frame_payload(&payload, "private", 1, None, None)
+ .await
+ .unwrap());
+ db.execute_raw_sql_write("UPDATE ui_events SET text_content='a longer redacted replacement',redacted_at=1 WHERE id=1; UPDATE pipe_executions SET stdout='longer completed output',status='completed',finished_at='2026-09-17' WHERE id=1;").await.unwrap();
+ let before = db.frame_payloads(&ids, Projection::All).await.unwrap();
+ let bytes = number(&db, "SELECT staging_bytes FROM storage_metadata").await;
+ db.close().await;
+
+ let db = reopen(root.path()).await;
+ assert_eq!(
+ db.frame_payloads(&ids, Projection::All).await.unwrap(),
+ before
+ );
+ assert_eq!(
+ number(&db, "SELECT staging_bytes FROM storage_metadata").await,
+ bytes
+ );
+ assert_eq!(
+ number(
+ &db,
+ "SELECT count(*) FROM frames_fts WHERE frames_fts MATCH 'capture'"
+ )
+ .await,
+ 31
+ );
+ record(&db, "recording after restart").await;
+ while db.seal_payloads().await.unwrap() != 0 {}
+ // Redaction eligibility is preserved; the remaining captures stay resident.
+ assert_eq!(
+ number(
+ &db,
+ "SELECT count(*) FROM frame_payloads WHERE state='sealed'"
+ )
+ .await,
+ 1
+ );
+ // Editing an archived field restages bytes and must work above the old cap.
+ db.execute_raw_sql_write(
+ "UPDATE ui_events SET text_content='updated archived content' WHERE id=1",
+ )
+ .await
+ .unwrap();
+ let archived = db
+ .frame_payloads(&[ids[0]], Projection::All)
+ .await
+ .unwrap()
+ .remove(&ids[0])
+ .unwrap();
+ assert!(db
+ .replace_frame_payload(&archived, "private", 1, None, None)
+ .await
+ .unwrap());
+ db.set_frame_privacy_policy(&PrivacyPolicy::default())
+ .await
+ .unwrap();
+ while db.seal_payloads().await.unwrap() != 0 {}
+ assert_eq!(
+ number(&db, "SELECT staging_bytes FROM storage_metadata").await,
+ 0
+ );
+ assert_eq!(number(&db, "SELECT count(*) FROM frames").await, 33);
+ assert_eq!(
+ number(&db, "SELECT staging_limit FROM storage_metadata").await,
+ 1024
+ );
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
+#[tokio::test]
+async fn oversized_capture_and_bulk_payloads_remain_durable_and_searchable() {
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), private_options())
+ .await
+ .unwrap();
+ let oversized = "oversized 東京 ".repeat(200);
+ let frame = record(&db, &oversized).await;
+ let audio = db.insert_audio_chunk("test.wav", None).await.unwrap();
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query("INSERT INTO audio_transcriptions(audio_chunk_id,transcription,offset_index,timestamp,device,is_input_device) VALUES(?,?,0,'2026-09-17','mic',1)")
+ .bind(audio).bind(&oversized).execute(&mut **tx.conn()).await.unwrap();
+ sqlx::query(
+ "INSERT INTO elements(id,frame_id,source,role,text) VALUES(1,?,'accessibility','AXText',?)",
+ )
+ .bind(frame)
+ .bind(&oversized)
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ for query in [
+ "INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(1,'2026-09-17','text',?)",
+ "INSERT INTO pipe_executions(id,pipe_name,status,stdout) VALUES(1,'test','running',?)",
+ ] {
+ sqlx::query(sqlx::AssertSqlSafe(query)).bind(&oversized).execute(&mut **tx.conn()).await.unwrap();
+ }
+ tx.commit().await.unwrap();
+ let mut payload = db
+ .frame_payloads(&[frame], Projection::All)
+ .await
+ .unwrap()
+ .remove(&frame)
+ .unwrap();
+ payload.full_text = Some(format!("{} extra", oversized));
+ assert!(db
+ .replace_frame_payload(&payload, "private", 1, None, None)
+ .await
+ .unwrap());
+ db.execute_raw_sql_write("UPDATE elements SET text=text||' extra' WHERE id=1; UPDATE ui_events SET text_content=text_content||' extra' WHERE id=1;").await.unwrap();
+ db.set_frame_privacy_policy(&PrivacyPolicy::default())
+ .await
+ .unwrap();
+ let later = record(&db, "later eligible capture").await;
+ while db.seal_payloads().await.unwrap() != 0 {}
+ assert_eq!(
+ number(
+ &db,
+ "SELECT count(*) FROM frame_payloads WHERE state='sealed'"
+ )
+ .await,
+ 1
+ );
+ assert_eq!(
+ number(&db, "SELECT count(*) FROM _bulk_element_rows WHERE id=1").await,
+ 1
+ );
+ let queries = [
+ "SELECT transcription FROM audio_transcriptions",
+ "SELECT text FROM elements",
+ "SELECT text_content FROM ui_events",
+ "SELECT stdout FROM pipe_executions",
+ ];
+ let mut expected = Vec::new();
+ for query in queries {
+ expected.push(db.query_raw_sql(query).await.unwrap());
+ }
+ let bytes = number(&db, "SELECT staging_bytes FROM storage_metadata").await;
+ db.close().await;
+ let db = reopen(root.path()).await;
+ assert_eq!(
+ number(&db, "SELECT staging_bytes FROM storage_metadata").await,
+ bytes
+ );
+ for (query, expected) in queries.into_iter().zip(expected) {
+ assert_eq!(db.query_raw_sql(query).await.unwrap(), expected);
+ }
+ let saved = db
+ .frame_payloads(&[frame, later], Projection::All)
+ .await
+ .unwrap();
+ assert_eq!(saved[&frame].full_text, payload.full_text);
+ assert_eq!(
+ saved[&later].full_text.as_deref(),
+ Some("later eligible capture")
+ );
+ for fts in [
+ "frames_fts",
+ "elements_fts",
+ "ui_events_fts",
+ "audio_transcriptions_fts",
+ ] {
+ assert_eq!(
+ number(
+ &db,
+ &format!("SELECT count(*) FROM {fts} WHERE {fts} MATCH 'oversized'")
+ )
+ .await,
+ 1
+ );
+ }
+ record(&db, "new recording beside oversized history").await;
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
+#[tokio::test]
+async fn failed_archive_write_does_not_block_recording_or_recovery() {
+ let root = tempfile::tempdir().unwrap();
+ let mut options = private_options();
+ options.privacy = PrivacyPolicy::default();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)
+ .await
+ .unwrap();
+ // A file where the archive directory belongs causes a real filesystem
+ // failure, confined to this disposable test's payload storage.
+ let payloads = root.path().join(&db.storage_descriptor().unwrap().payloads);
+ if payloads.exists() {
+ std::fs::remove_dir(&payloads).unwrap();
+ }
+ std::fs::write(&payloads, b"blocked archive path").unwrap();
+ let mut ids = Vec::new();
+ for i in 0..12 {
+ ids.push(record(&db, &format!("durable capture {i} {}", "x".repeat(128))).await);
+ assert!(db.seal_payloads().await.is_err());
+ }
+ let expected = db.frame_payloads(&ids, Projection::All).await.unwrap();
+ db.close().await;
+ let db = reopen(root.path()).await;
+ assert_eq!(
+ db.frame_payloads(&ids, Projection::All).await.unwrap(),
+ expected
+ );
+ record(&db, "recording while archive is still broken").await;
+ std::fs::remove_file(&payloads).unwrap();
+ std::fs::create_dir_all(&payloads).unwrap();
+ db.reclaim_frame_payloads().await.unwrap();
+ while db.seal_payloads().await.unwrap() != 0 {}
+ assert_eq!(number(&db, "SELECT count(*) FROM frames").await, 13);
+ assert_eq!(
+ number(&db, "SELECT staging_bytes FROM storage_metadata").await,
+ 0
+ );
+ assert_eq!(
+ db.frame_payloads(&ids, Projection::All).await.unwrap(),
+ expected
+ );
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
+#[tokio::test]
+async fn acknowledged_backlog_survives_process_exit() {
+ const CHILD_ROOT: &str = "SCREENPIPE_RECORDING_DURABILITY_TEST_ROOT";
+ if let Some(root) = std::env::var_os(CHILD_ROOT) {
+ let db = DatabaseManager::new_hybrid(
+ std::path::Path::new(&root),
+ Default::default(),
+ private_options(),
+ )
+ .await
+ .unwrap();
+ for i in 0..16 {
+ assert_eq!(
+ record(&db, &format!("capture {i} {}", "x".repeat(128))).await,
+ i + 1
+ );
+ }
+ // No graceful queue drain, pool close, checkpoint, or destructor.
+ std::process::exit(86);
+ }
+ let root = tempfile::tempdir().unwrap();
+ let status = std::process::Command::new(std::env::current_exe().unwrap())
+ .args([
+ "--exact",
+ "acknowledged_backlog_survives_process_exit",
+ "--nocapture",
+ ])
+ .env(CHILD_ROOT, root.path())
+ .status()
+ .unwrap();
+ assert_eq!(status.code(), Some(86));
+ let db = reopen(root.path()).await;
+ let rows: Vec<(i64, String)> = sqlx::query_as("SELECT id,full_text FROM frames ORDER BY id")
+ .fetch_all(&db.pool)
+ .await
+ .unwrap();
+ let expected: Vec<_> = (0..16)
+ .map(|i| (i + 1, format!("capture {i} {}", "x".repeat(128))))
+ .collect();
+ assert_eq!(rows, expected);
+ assert_eq!(record(&db, "recording after process recovery").await, 17);
+ db.verify_storage().await.unwrap();
+ db.close().await;
+}
+
+// Exercise the production maintenance task while frames arrive once per second.
+// Keep the workload fixed so before/after backlog timings remain comparable.
+mod background_benchmark {
+ use screenpipe_db::DatabaseManager;
+ use std::time::{Duration, Instant};
+
+ async fn capture(db: &DatabaseManager) {
+ db.insert_snapshot_frame(
+ "display",
+ chrono::Utc::now(),
+ "",
+ Some("Probe"),
+ None,
+ None,
+ true,
+ Some("probe"),
+ Some(&"x".repeat(16 * 1024)),
+ Some("accessibility"),
+ None,
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ }
+
+ async fn report(db: &DatabaseManager, label: &str, start: Instant) {
+ let counts: (i64, i64, i64, i64, i64) = sqlx::query_as(
+ "SELECT (SELECT count(*) FROM frame_payloads WHERE state='staged'),
+ (SELECT count(*) FROM frame_payloads WHERE state='sealed'),
+ (SELECT count(*) FROM _bulk_element_rows),
+ (SELECT count(*) FROM main.audio_transcriptions WHERE _archive_mask!=0),
+ (SELECT staging_bytes FROM storage_metadata)",
+ )
+ .fetch_one(&db.pool)
+ .await
+ .unwrap();
+ eprintln!("PROFILE_COUNTS {label} elapsed={:.3} staged_frames={} sealed_frames={} staged_elements={} staged_audio={} staging_bytes={}", start.elapsed().as_secs_f64(), counts.0, counts.1, counts.2, counts.3, counts.4);
+ }
+
+ #[tokio::test]
+ #[ignore = "manual background archival benchmark with real polling and CPU cooldown"]
+ async fn measure_actual_background_scheduler() {
+ let _ = tracing_subscriber::fmt()
+ .with_env_filter("screenpipe_db::storage::sealing=debug")
+ .with_ansi(false)
+ .try_init();
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ db.set_frame_privacy_policy(&Default::default())
+ .await
+ .unwrap();
+ capture(&db).await;
+ db.execute_raw_sql_write("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1000) INSERT INTO elements(id,frame_id,source,role,text,redacted_at) SELECT x,1,'accessibility','AXText',printf('%0512d',x),1 FROM n;
+ INSERT INTO audio_chunks(id,file_path) VALUES(1,'probe.wav');
+ WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8) INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,redacted_at) SELECT x,1,x,'2026-09-17',printf('%01024d',x),1 FROM n;").await.unwrap();
+ let start = Instant::now();
+ report(&db, "initial", start).await;
+ for second in 1..=31 {
+ tokio::time::sleep(Duration::from_secs(1)).await;
+ capture(&db).await;
+ if second % 5 == 1 {
+ report(&db, "recording", start).await;
+ }
+ }
+ report(&db, "producer_stopped", start).await;
+ let remaining: (i64, i64) = sqlx::query_as("SELECT (SELECT count(*) FROM _bulk_element_rows),(SELECT count(*) FROM main.audio_transcriptions WHERE _archive_mask!=0)").fetch_one(&db.pool).await.unwrap();
+ assert_eq!(
+ remaining,
+ (0, 0),
+ "ready bulk records must drain while frame capture continues"
+ );
+ for _ in 0..3 {
+ tokio::time::sleep(Duration::from_secs(5)).await;
+ report(&db, "quiet", start).await;
+ }
+ db.verify_storage().await.unwrap();
+ db.close().await;
+ }
+}
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 54d3eddf8..6f20b2a27 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 33
-- Mapped Rust files: 357
-- Active test blocks: 3504
-- Ignored/manual test blocks: 144
-- Declared test blocks: 3648
-- Weighted coverage points: 2890.4
+- Mapped Rust files: 358
+- Active test blocks: 3513
+- Ignored/manual test blocks: 146
+- Declared test blocks: 3659
+- Weighted coverage points: 2899.4
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3361 | 138 | 2825.3 | 21 | 11 | 100% |
-| macos | 30 | 3423 | 119 | 2839.3 | 22 | 11 | 100% |
-| linux | 26 | 2991 | 111 | 2484.6 | 20 | 11 | 100% |
+| windows | 30 | 3370 | 140 | 2834.3 | 21 | 11 | 100% |
+| macos | 30 | 3432 | 121 | 2848.3 | 22 | 11 | 100% |
+| linux | 26 | 3000 | 113 | 2493.6 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |
-| screenpipe-db | 5 | 57 | 23 | 535 | 21 | 508.3 | 9 |
+| screenpipe-db | 5 | 58 | 23 | 544 | 23 | 517.3 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts |
| audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 6 suites / 483 active / 17 ignored / 456.3 pts | 6 suites / 483 active / 17 ignored / 456.3 pts | 6 suites / 483 active / 17 ignored / 456.3 pts |
+| database | 6 suites / 492 active / 19 ignored / 465.3 pts | 6 suites / 492 active / 19 ignored / 465.3 pts | 6 suites / 492 active / 19 ignored / 465.3 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts |
| local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts |
| meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1537 active / 72 ignored / 1357.5 pts | 14 suites / 1644 active / 76 ignored / 1400.3 pts | 13 suites / 1537 active / 72 ignored / 1357.5 pts |
+| performance | 13 suites / 1546 active / 74 ignored / 1366.5 pts | 14 suites / 1653 active / 78 ignored / 1409.3 pts | 13 suites / 1546 active / 74 ignored / 1366.5 pts |
| pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
| privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts |
| real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - |
| speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |
-| storage | 4 suites / 600 active / 34 ignored / 494.1 pts | 4 suites / 600 active / 34 ignored / 494.1 pts | 4 suites / 600 active / 34 ignored / 494.1 pts |
+| storage | 4 suites / 609 active / 36 ignored / 503.1 pts | 4 suites / 609 active / 36 ignored / 503.1 pts | 4 suites / 609 active / 36 ignored / 503.1 pts |
| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
-| timeline | 4 suites / 1147 active / 38 ignored / 934.3 pts | 4 suites / 1147 active / 38 ignored / 934.3 pts | 4 suites / 1147 active / 38 ignored / 934.3 pts |
+| timeline | 4 suites / 1156 active / 40 ignored / 943.3 pts | 4 suites / 1156 active / 40 ignored / 943.3 pts | 4 suites / 1156 active / 40 ignored / 943.3 pts |
| transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts |
| ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |
| vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |
@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 30 | 247 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 31 | 256 | 10 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -276,12 +276,12 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | src/failpoint_vfs.rs | source | 4 | 0 | 4 |
| db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 |
| db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |
-| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 6 | 0 | 6 |
+| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 7 | 0 | 7 |
| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 |
| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 |
| db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |
| db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 2 | 1 | 3 |
-| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 2 | 0 | 2 |
+| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 3 | 1 | 4 |
| db-timeline-frames | screenpipe-db | src/storage/sealing.rs | source | 1 | 0 | 1 |
| db-search-indexing | screenpipe-db | src/text_normalizer.rs | source | 19 | 0 | 19 |
| db-search-indexing | screenpipe-db | src/text_similarity.rs | source | 20 | 0 | 20 |
@@ -333,6 +333,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | tests/sqlite_architecture_invariants_test.rs | integration | 2 | 0 | 2 |
| db-runtime-reliability | screenpipe-db | tests/sqlite_runtime_version.rs | integration | 1 | 0 | 1 |
| db-runtime-reliability | screenpipe-db | tests/storage_snapshots.rs | integration | 3 | 0 | 3 |
+| db-timeline-frames | screenpipe-db | tests/storage_staging_drain.rs | integration | 7 | 1 | 8 |
| db-timeline-frames | screenpipe-db | tests/suggestions_frames_table_test.rs | integration | 3 | 0 | 3 |
| db-search-indexing | screenpipe-db | tests/tag_autocomplete_query_test.rs | integration | 5 | 1 | 6 |
| db-search-indexing | screenpipe-db | tests/tag_filter_bench.rs | integration | 0 | 2 | 2 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index adc852fa0..527fafa25 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -686,6 +686,7 @@
"tests/outputs_saf_test.rs",
"tests/range_export_frame_sources_test.rs",
"tests/semantic_storage_test.rs",
+ "tests/storage_staging_drain.rs",
"tests/suggestions_frames_table_test.rs",
"tests/timeline_frameless_audio_test.rs",
"tests/timeline_parquet_test.rs",
diff --git a/packages/sdk/Cargo.lock b/packages/sdk/Cargo.lock
index 85cb57e60..7041d389f 100644
--- a/packages/sdk/Cargo.lock
+++ b/packages/sdk/Cargo.lock
@@ -6999,6 +6999,7 @@ dependencies = [
"regex",
"rusqlite",
"screenpipe-config",
+ "screenpipe-resource",
"screenpipe-semantic",
"screenpipe-sqlite-coordinator",
"serde",
@@ -7054,6 +7055,15 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "screenpipe-resource"
+version = "0.4.50"
+dependencies = [
+ "serde",
+ "sysinfo",
+ "tokio",
+]
+
[[package]]
name = "screenpipe-screen"
version = "0.4.50"
diff --git a/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock b/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock
index 7a215d99e..a453cd767 100644
--- a/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock
+++ b/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock
@@ -7125,6 +7125,7 @@ dependencies = [
"regex",
"rusqlite",
"screenpipe-config",
+ "screenpipe-resource",
"screenpipe-semantic",
"screenpipe-sqlite-coordinator",
"serde",
@@ -7178,6 +7179,15 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "screenpipe-resource"
+version = "0.4.50"
+dependencies = [
+ "serde",
+ "sysinfo",
+ "tokio",
+]
+
[[package]]
name = "screenpipe-screen"
version = "0.4.50"