YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): keep recording and archival progressing together (#7063)
Louis Beaumont0ffa1916092d
diff --git a/AGENTS.md b/AGENTS.mdindex a228b5ab4..3fd237054 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -66,6 +66,15 @@ signed `.app` path used only when persistent macOS TCC identity is required.## Hot paths+Recording is the critical path. Never sacrifice an authorized recording to+archival, compression, indexing, redaction backlog, sync, or other background+work. Persist captures durably through the existing writer; processing budgets+bound background work, never capture admission. When processing cannot keep up,+retain the recording and defer that work. Preserve privacy, user pause controls,+and genuine disk/data-integrity protections. Tests must prove continued durable+recording and recovery under processing failure, backlog, and restart; replace+expectations that require recordings to be rejected or dropped for those reasons.+Capture and encode per frame (`screenpipe-screen`, `-capture`, `-a11y`), audiodevice callbacks (`screenpipe-audio`), and SQLite writes (`screenpipe-db` via`-sqlite-coordinator`) run continuously on every user's machine. No per-framediff --git a/COVERAGE.md b/COVERAGE.mdindex 810612878..28c4f1588 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -35,16 +35,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine- Mapped suites: 33-- Mapped Rust files: 357-- Active test blocks: 3504-- Ignored/manual test blocks: 144-- Weighted coverage points: 2890.4+- Mapped Rust files: 358+- Active test blocks: 3513+- Ignored/manual test blocks: 146+- Weighted coverage points: 2899.4| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3361 | 138 | 2825.3 | 21 | 11 | 100% |-| macos | 30 | 3423 | 119 | 2839.3 | 22 | 11 | 100% |-| linux | 26 | 2991 | 111 | 2484.6 | 20 | 11 | 100% |+| windows | 30 | 3370 | 140 | 2834.3 | 21 | 11 | 100% |+| macos | 30 | 3432 | 121 | 2848.3 | 22 | 11 | 100% |+| linux | 26 | 3000 | 113 | 2493.6 | 20 | 11 | 100% |## Refreshdiff --git a/Cargo.lock b/Cargo.lockindex 3ea6dc455..c76886590 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8860,6 +8860,7 @@ dependencies = ["rusqlite","screenpipe-config","screenpipe-core",+ "screenpipe-resource","screenpipe-secrets","screenpipe-semantic","screenpipe-sqlite-coordinator",diff --git a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock b/apps/screenpipe-app-tauri/src-tauri/Cargo.lockindex eef160dfe..9c9c467d8 100644--- a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock+++ b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock@@ -11535,6 +11535,7 @@ dependencies = ["regex","rusqlite","screenpipe-config",+ "screenpipe-resource","screenpipe-semantic","screenpipe-sqlite-coordinator","serde",diff --git a/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock b/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lockindex 2c8887d41..f86fedb03 100644--- a/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock+++ b/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock@@ -11534,6 +11534,7 @@ dependencies = ["regex","rusqlite","screenpipe-config",+ "screenpipe-resource","screenpipe-semantic","screenpipe-sqlite-coordinator","serde",diff --git a/crates/screenpipe-db/Cargo.toml b/crates/screenpipe-db/Cargo.tomlindex ee82e2ce4..3f91e3429 100644--- a/crates/screenpipe-db/Cargo.toml+++ b/crates/screenpipe-db/Cargo.toml@@ -31,6 +31,7 @@ futures = { workspace = true, features = ["std"] }screenpipe-config = { path = "../screenpipe-config" }screenpipe-semantic = { path = "../screenpipe-semantic" }+screenpipe-resource = { path = "../screenpipe-resource" }screenpipe-sqlite-coordinator = { path = "../screenpipe-sqlite-coordinator" }zerocopy = { version = "0.7.32" }diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex be885ce6d..a953a48ed 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -521,8 +521,25 @@ impl DatabaseManager {}crate::storage::schema::verify(&mut conn, &storage.descriptor).await?;crate::storage::schema::upgrade_resident_frames(&mut conn).await?;+ let upgraded =+ crate::storage::schema::upgrade_recording(&mut conn, storage.has_bulk())+ .await?;crate::storage::read_schema::upgrade(&mut conn, storage).await?;storage.verify_catalog(&db_manager.pool).await?;+ drop(conn);+ if upgraded {+ // Connections opened before the trigger migration need a+ // schema read before their first DML preparation. Refresh+ // every existing writer while startup still owns admission.+ let mut writers = Vec::new();+ for _ in 0..db_manager.write_pool.size() {+ let mut conn = db_manager.write_pool.acquire().await?;+ sqlx::query("SELECT name FROM main.sqlite_schema LIMIT 1")+ .fetch_optional(&mut *conn)+ .await?;+ writers.push(conn);+ }+ }}Ok::<(), sqlx::Error>(())}diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rsindex 4fbfa31e3..2d76307ea 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs@@ -19,6 +19,7 @@ struct Encoded {first: i64,last: i64,rows: usize,+ generations: Vec<(i64, i64)>,}fn record(row: &sqlx::sqlite::SqliteRow) -> Result<Record, sqlx::Error> {@@ -51,17 +52,41 @@ pub(crate) async fn seal() -> Result<usize, sqlx::Error> {crate::storage::diagnostics::batch("elements", None, None, None, None);crate::storage::diagnostics::stage("selecting_staged_elements");- let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(- "SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND ({}) ORDER BY id LIMIT 1",+ // A dirty archive must be rewritten as a whole. If privacy processing+ // blocks that range, continue after it instead of starving newer work.+ let mut after = None;+ loop {+ let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));+ let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(+ "SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND {lower} AND ({}) ORDER BY id LIMIT 1",+ TABLE.sealable()+ )))+ .fetch_optional(pool)+ .await?;+ let Some(id) = id else { return Ok(0) };+ let range: Option<(i64, i64, i64)> = sqlx::query_as("SELECT first_id,last_id,file_id FROM _bulk_element_ranges WHERE first_id=(SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?1) AND last_id>=?1")+ .bind(id).fetch_optional(pool).await?;+ if let Some((first, last, _)) = range {+ if range_blocked(pool, first, last).await? {+ after = Some(last);+ continue;+ }+ }+ // A concurrent edit invalidating publication is retried next tick,+ // rather than spinning over a stream of newly arriving records.+ return rewrite(storage, pool, writer, id, range).await;+ }+}++async fn range_blocked(pool: &SqlitePool, first: i64, last: i64) -> Result<bool, sqlx::Error> {+ sqlx::query_scalar(sqlx::AssertSqlSafe(format!(+ "SELECT EXISTS(SELECT 1 FROM elements WHERE id BETWEEN ? AND ? AND NOT ({}))",TABLE.sealable())))- .fetch_optional(pool)- .await?;- let Some(id) = id else {- return Ok(0);- };- let range:Option<(i64,i64,i64)>=sqlx::query_as("SELECT first_id,last_id,file_id FROM _bulk_element_ranges WHERE first_id=(SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?1) AND last_id>=?1").bind(id).fetch_optional(pool).await?;- rewrite(storage, pool, writer, id, range).await+ .bind(first)+ .bind(last)+ .fetch_one(pool)+ .await}async fn encode(@@ -114,6 +139,7 @@ async fn encode(first: rows.first().unwrap().id,last: rows.last().unwrap().id,rows: rows.len(),+ generations: rows.iter().map(|row| (row.id, row.generation)).collect(),};let budget = storage.descriptor.budget.clone();let file = path.clone();@@ -179,15 +205,7 @@ async fn rewrite(.fetch_one(pool).await?;let (first, last) = if let Some((first, last, _)) = range {- let blocked: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(- "SELECT EXISTS(SELECT 1 FROM elements WHERE id BETWEEN ? AND ? AND NOT ({}))",- TABLE.sealable()- )))- .bind(first)- .bind(last)- .fetch_one(pool)- .await?;- if blocked {+ if range_blocked(pool, first, last).await? {return Ok(0);}(first, last)@@ -256,6 +274,13 @@ async fn rewrite(return Ok(0);};crate::storage::diagnostics::stage("publishing_element_archive");+ #[cfg(test)]+ {+ let hook = storage.bulk.element_publish_hook.lock().unwrap().clone();+ if let Some(hook) = hook {+ tokio::task::spawn_blocking(move || hook()).await.unwrap();+ }+ }let permit = writer.lock().await?;let mut tx = permit.pool().begin().await?;let current: i64 = sqlx::query_scalar("SELECT version FROM _bulk_element_state")@@ -265,7 +290,14 @@ async fn rewrite(sqlx::query_as("SELECT policy,required_surfaces FROM storage_metadata").fetch_one(&mut *tx).await?;- if current != version || current_policy != policy {+ // New captures change the table-wide version while this batch encodes.+ // Only mutations inside the range being published can invalidate it.+ // Retain the O(1) check when nothing changed; otherwise compare resident+ // generations, including tombstones and newly eligible gap inserts.+ let valid = current_policy == policy+ && (current == version+ || resident_range_unchanged(&mut tx, first, last, &files, range.is_some()).await?);+ if !valid {for file in files {sqlx::query("UPDATE _bulk_files SET state='retired' WHERE id=?").bind(file.id)@@ -329,6 +361,42 @@ async fn rewrite(Ok(count)}+async fn resident_range_unchanged(+ conn: &mut sqlx::SqliteConnection,+ first: i64,+ last: i64,+ files: &[Encoded],+ whole_range: bool,+) -> Result<bool, sqlx::Error> {+ let mut rows = sqlx::query(sqlx::AssertSqlSafe(format!(+ "SELECT id,_archive_generation,_archive_deleted,({}) AS sealable FROM _bulk_element_rows WHERE id BETWEEN ? AND ? ORDER BY id",+ TABLE.sealable()+ )))+ .bind(first).bind(last).fetch(&mut *conn);+ while let Some(row) = rows.try_next().await? {+ let id: i64 = row.try_get("id")?;+ let generation: i64 = row.try_get("_archive_generation")?;+ let deleted: bool = row.try_get("_archive_deleted")?;+ let sealable: bool = row.try_get("sealable")?;+ let file = files.partition_point(|file| file.last < id);+ let encoded = files.get(file).and_then(|file| {+ file.generations+ .binary_search_by_key(&id, |(id, _)| *id)+ .ok()+ .map(|index| file.generations[index].1)+ });+ match encoded {+ Some(expected) if deleted || !sealable || generation != expected => return Ok(false),+ // A newly eligible record in a gap must not be deleted without+ // being encoded. Unchanged private gaps and old tombstones stay+ // governed by the existing publication selection.+ None if !deleted && (whole_range || sealable) => return Ok(false),+ _ => {}+ }+ }+ Ok(true)+}+pub(crate) async fn reclaim(storage: &Arc<HybridStorage>,pool: &SqlitePool,diff --git a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rsindex fd940947f..2b341db28 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/vtab.rs@@ -111,9 +111,16 @@ struct RowSql {static ROW_SQL: LazyLock<RowSql> = LazyLock::new(|| {let columns = names();let projection = format!("id,_archive_generation,{columns},_archive_deleted");+ // Preserve the tombstone's generation when an ID is deleted and reused.+ // An encoder holding the old generation must never accept the new row.+ let updates = format!("_archive_deleted,_archive_file,{columns}")+ .split(',')+ .map(|column| format!("{column}=excluded.{column}"))+ .collect::<Vec<_>>()+ .join(",");RowSql {lookup: format!("SELECT {projection} FROM _bulk_element_rows WHERE id=?"),- upsert: format!("INSERT OR REPLACE INTO _bulk_element_rows(id,_archive_generation,_archive_deleted,_archive_file,{columns}) VALUES({})", vec!["?";TABLE.columns.len()+4].join(",")),+ upsert: format!("INSERT INTO _bulk_element_rows(id,_archive_generation,_archive_deleted,_archive_file,{columns}) VALUES({}) ON CONFLICT(id) DO UPDATE SET _archive_generation=max(_bulk_element_rows._archive_generation+1,excluded._archive_generation),{updates}", vec!["?";TABLE.columns.len()+4].join(",")),resident: format!("SELECT COALESCE((SELECT {} FROM _bulk_element_rows WHERE id=?),0)", TABLE.all_bytes("")),projection,}@@ -323,9 +330,6 @@ impl Elements {return Err(constraint("required element column is NULL"));}}- if new.bytes() > self.storage.descriptor.budget.record_bytes {- return Err(constraint("element record budget exceeded"));- }let Value::Integer(frame) = new.values[0] else {return Err(constraint("invalid element frame"));};diff --git a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/lifecycle.rsindex fdd3293cb..5f93f35c7 100644--- a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs@@ -11,23 +11,18 @@ use sqlx::{Row, SqlitePool, ValueRef};use std::{path::Path, sync::Arc};impl HybridStorage {- pub(crate) async fn seal_bulk(+ pub(in crate::storage) async fn seal_bulk_table(self: &Arc<Self>,pool: &SqlitePool,writer: &SqliteWritePool,+ table: &'static Table,) -> Result<usize, sqlx::Error> {- if !self.has_bulk() {- return Ok(0);- }- let elements = super::elements::seal(self, pool, writer).await?;- if elements != 0 {- return Ok(elements);+ if table.name == "elements" {+ return super::elements::seal(self, pool, writer).await;}- for table in TABLES.iter().filter(|t| t.name != "elements") {- let rows = self.select_bulk(pool, table, None).await?;- if !rows.is_empty() {- return self.publish_bulk(pool, writer, table, rows, None).await;- }+ let rows = self.select_bulk(pool, table, None).await?;+ if !rows.is_empty() {+ return self.publish_bulk(pool, writer, table, rows, None).await;}Ok(0)}diff --git a/crates/screenpipe-db/src/storage/bulk/mod.rs b/crates/screenpipe-db/src/storage/bulk/mod.rsindex 3078b7a38..9c03f2ee8 100644--- a/crates/screenpipe-db/src/storage/bulk/mod.rs+++ b/crates/screenpipe-db/src/storage/bulk/mod.rs@@ -18,7 +18,7 @@ use super::{storage_error, HybridStorage};pub(crate) use connection::pool_options;pub(crate) use connection::register_hash;pub(super) use lifecycle::export;-pub(super) use schema::{bootstrap, bootstrap_in_place, finish_indexes};+pub(super) use schema::{bootstrap, bootstrap_in_place, finish_indexes, upgrade_recording};use std::sync::{atomic::AtomicUsize, Arc, Mutex};pub const CAPABILITY: &str = "parquet-bulk-v1";@@ -209,6 +209,8 @@ pub(crate) struct Runtime {element_frames: Mutex<Option<(String, Arc<std::collections::HashMap<i64, Vec<usize>>>)>>,#[cfg(test)]decode_hook: Mutex<Option<Arc<dyn Fn(&str) + Send + Sync>>>,+ #[cfg(test)]+ element_publish_hook: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,}#[derive(Clone)]diff --git a/crates/screenpipe-db/src/storage/bulk/schema.rs b/crates/screenpipe-db/src/storage/bulk/schema.rsindex 6f6d8839f..73e4925d2 100644--- a/crates/screenpipe-db/src/storage/bulk/schema.rs+++ b/crates/screenpipe-db/src/storage/bulk/schema.rs@@ -127,20 +127,18 @@ fn triggers(table: &Table, original_columns: &[String]) -> String {let t = table.name;let new_bytes = table.all_bytes("NEW.");let old_bytes = table.local_bytes("OLD.");- let check=format!("SELECT CASE WHEN ({new_bytes})>(SELECT record_limit FROM storage_metadata) THEN RAISE(ABORT,'bulk record budget exceeded') END;");- let staging_check = "SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata) THEN RAISE(ABORT,'storage staging budget reached') END;";let fts = reindex(table);let fts_delete = if table.fts.is_empty() {String::new()} else {format!("DELETE FROM {t}_fts WHERE rowid=OLD.id;")};- let mut sql=format!("CREATE TRIGGER hybrid_bulk_{t}_id BEFORE UPDATE OF id ON {t} WHEN OLD._archive_file IS NOT NULL AND NEW.id!=OLD.id BEGIN SELECT RAISE(ABORT,'archived record IDs are immutable'); END; CREATE TRIGGER hybrid_bulk_{t}_insert AFTER INSERT ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN {check} UPDATE storage_metadata SET staging_bytes=staging_bytes+({new_bytes}),revision=revision+1; {staging_check} {fts} END; CREATE TRIGGER hybrid_bulk_{t}_delete BEFORE DELETE ON {t} BEGIN {fts_delete} UPDATE _bulk_files SET state='dirty' WHERE id=OLD._archive_file; UPDATE storage_metadata SET staging_bytes=staging_bytes-({old_bytes}),revision=revision+1; END;");+ let mut sql=format!("CREATE TRIGGER hybrid_bulk_{t}_id BEFORE UPDATE OF id ON {t} WHEN OLD._archive_file IS NOT NULL AND NEW.id!=OLD.id BEGIN SELECT RAISE(ABORT,'archived record IDs are immutable'); END; CREATE TRIGGER hybrid_bulk_{t}_insert AFTER INSERT ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN UPDATE storage_metadata SET staging_bytes=staging_bytes+({new_bytes}),revision=revision+1; {fts} END; CREATE TRIGGER hybrid_bulk_{t}_delete BEFORE DELETE ON {t} BEGIN {fts_delete} UPDATE _bulk_files SET state='dirty' WHERE id=OLD._archive_file; UPDATE storage_metadata SET staging_bytes=staging_bytes-({old_bytes}),revision=revision+1; END;");sql.push_str(&format!("CREATE TRIGGER hybrid_bulk_{t}_revision AFTER UPDATE OF {} ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN UPDATE storage_metadata SET revision=revision+1; END;", original_columns.join(",")));for (index, column) in table.columns.iter().enumerate() {let c = column.name;let bit = 1 << index;- sql.push_str(&format!("CREATE TRIGGER hybrid_bulk_{t}_{c} AFTER UPDATE OF {c} ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN {check} UPDATE {t} SET _archive_mask=_archive_mask|{bit},_archive_generation=_archive_generation+1 WHERE id=NEW.id; UPDATE _bulk_files SET state='dirty' WHERE id=OLD._archive_file; UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE(length(CAST(NEW.{c} AS BLOB)),0)-CASE WHEN (OLD._archive_mask & {bit})!=0 THEN COALESCE(length(CAST(OLD.{c} AS BLOB)),0) ELSE 0 END; {staging_check} {fts} END;"));+ sql.push_str(&format!("CREATE TRIGGER hybrid_bulk_{t}_{c} AFTER UPDATE OF {c} ON {t} WHEN (SELECT maintenance=0 FROM storage_metadata) BEGIN UPDATE {t} SET _archive_mask=_archive_mask|{bit},_archive_generation=_archive_generation+1 WHERE id=NEW.id; UPDATE _bulk_files SET state='dirty' WHERE id=OLD._archive_file; UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE(length(CAST(NEW.{c} AS BLOB)),0)-CASE WHEN (OLD._archive_mask & {bit})!=0 THEN COALESCE(length(CAST(OLD.{c} AS BLOB)),0) ELSE 0 END; {fts} END;"));}let metadata = table.fts@@ -153,3 +151,30 @@ fn triggers(table: &Table, original_columns: &[String]) -> String {}sql}++// Called inside the startup schema transaction; preserve v2's identity while+// upgrading the installed triggers for existing as well as new generations.+pub(crate) async fn upgrade_recording(conn: &mut SqliteConnection) -> Result<(), sqlx::Error> {+ for table in TABLES.iter().filter(|t| t.name != "elements") {+ let columns: Vec<String> = sqlx::query_scalar(+ "SELECT name FROM pragma_table_info(?) WHERE name NOT LIKE '_archive_%' ORDER BY cid",+ )+ .bind(table.name)+ .fetch_all(&mut *conn)+ .await?;+ let names: Vec<String> = sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=? AND name GLOB 'hybrid_bulk_*'")+ .bind(table.name).fetch_all(&mut *conn).await?;+ for name in names {+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(+ "DROP TRIGGER \"{}\"",+ name.replace('"', "\"\"")+ )))+ .execute(&mut *conn)+ .await?;+ }+ sqlx::raw_sql(sqlx::AssertSqlSafe(triggers(table, &columns)))+ .execute(&mut *conn)+ .await?;+ }+ Ok(())+}diff --git a/crates/screenpipe-db/src/storage/bulk/tests.rs b/crates/screenpipe-db/src/storage/bulk/tests.rsindex ca2ae038d..addf82e54 100644--- a/crates/screenpipe-db/src/storage/bulk/tests.rs+++ b/crates/screenpipe-db/src/storage/bulk/tests.rs@@ -32,6 +32,57 @@ impl Drop for Pause {}}+#[tokio::test]+async fn element_publication_ignores_new_captures_but_rejects_changed_range_members() {+ for archived in [false, true] {+ for mutation in [+ "INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(30,1,'accessibility','AXText','new capture',1)",+ "UPDATE elements SET text='edited while encoding' WHERE id=10",+ "DELETE FROM elements WHERE id=10",+ "INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(15,1,'accessibility','AXText','inserted gap',1)",+ "INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(15,1,'accessibility','AXText','private gap',NULL)",+ "UPDATE elements SET redacted_at=NULL WHERE id=10",+ "DELETE FROM elements WHERE id=10; INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(10,1,'accessibility','AXText','reused ID',1)",+ ] {+ let root = tempfile::tempdir().unwrap();+ let db = Arc::new(DatabaseManager::new_hybrid(root.path(), Default::default(),+ crate::storage::MigrationOptions {+ privacy: crate::storage::PrivacyPolicy { identity: "private".into(), required_surfaces: 1 },+ ..Default::default()+ }).await.unwrap());+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-17','pending frame'); INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(10,1,'accessibility','AXText','ten',1),(20,1,'accessibility','AXText','twenty',1)").await.unwrap();+ if archived {+ assert_eq!(db.seal_payloads().await.unwrap(), 2);+ db.execute_raw_sql_write("UPDATE elements SET text='restaged ten' WHERE id=10").await.unwrap();+ }+ let storage = db.storage.as_ref().unwrap();+ let encoded = Arc::new(tokio::sync::Notify::new());+ let announced = encoded.clone();+ let pause = Pause::new();+ let held = Pause(pause.0.clone());+ *storage.bulk.element_publish_hook.lock().unwrap() = Some(Arc::new(move || {+ announced.notify_one();+ held.wait();+ }));+ let sealing = db.clone();+ let task = tokio::spawn(async move { sealing.seal_payloads().await });+ tokio::time::timeout(Duration::from_secs(10), encoded.notified()).await.unwrap();+ // The archive is fully encoded while the writer remains available.+ db.execute_raw_sql_write(mutation).await.unwrap();+ let expected = db.query_raw_sql("SELECT id,text,redacted_at FROM elements ORDER BY id").await.unwrap();+ pause.release();+ let count = task.await.unwrap().unwrap();+ assert_eq!(count, if mutation.contains("VALUES(30,") || (!archived && mutation.contains("private gap")) { 2 } else { 0 }, "archived={archived}; {mutation}");+ *storage.bulk.element_publish_hook.lock().unwrap() = None;+ assert_eq!(db.query_raw_sql("SELECT id,text,redacted_at FROM elements ORDER BY id").await.unwrap(), expected);+ while db.seal_payloads().await.unwrap() != 0 {}+ db.verify_storage().await.unwrap();+ assert_eq!(db.query_raw_sql("SELECT id,text,redacted_at FROM elements ORDER BY id").await.unwrap(), expected);+ db.close().await;+ }+ }+}+async fn fixture_mode(hybrid: bool) -> (tempfile::TempDir, Arc<DatabaseManager>) {let root = tempfile::tempdir().unwrap();let db = Arc::new(diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rsindex 106840530..8335886f2 100644--- a/crates/screenpipe-db/src/storage/in_place.rs+++ b/crates/screenpipe-db/src/storage/in_place.rs@@ -141,9 +141,8 @@ async fn run(tx.commit().await?;}if schema::converted_step(&mut conn,"conversion-complete").await? { return Ok(()); }- // A paused migration records resident payloads like the original- // SQLite store. Its disabled sealer must not impose a staging cap- // that eventually stops recording. Explicit completion restores it.+ // Compatibility with pre-v6 triggers during interrupted conversion.+ // The v6 startup upgrade removes capture admission limits entirely.sqlx::query("UPDATE storage_metadata SET staging_limit=?").bind(if archive { storage.descriptor.budget.staging_bytes as i64 } else { i64::MAX }).execute(&mut *conn).await?;@@ -258,8 +257,8 @@ async fn run(let original: Vec<String> = sqlx::query_scalar("SELECT sql FROM _storage_conversion_triggers WHERE restore=1").fetch_all(&mut *tx).await?;for sql in original { sqlx::raw_sql(sqlx::AssertSqlSafe(sql)).execute(&mut *tx).await?; }if archive {- // Existing resident history occupies disk, not the capture work- // queue. Preserve a full configured staging allowance above it.+ // Preserve the legacy allowance for older schema readers. The+ // current recording triggers do not enforce this as a cap.sqlx::query("UPDATE storage_metadata SET staging_limit=staging_bytes+?").bind(storage.descriptor.budget.staging_bytes as i64).execute(&mut *tx).await?;diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rsindex bfd2f92cd..b3648256e 100644--- a/crates/screenpipe-db/src/storage/mod.rs+++ b/crates/screenpipe-db/src/storage/mod.rs@@ -75,10 +75,13 @@ pub struct StorageBudget {pub row_group_rows: usize,pub file_rows: usize,pub file_bytes: usize,+ /// Maximum record the archiver processes; larger captures remain in SQLite.pub record_bytes: usize,pub decode_bytes: usize,pub response_bytes: usize,pub concurrent_decodes: usize,+ /// Background reclamation headroom and legacy staging allowance. It must+ /// never limit durable recording; memory is bounded by processing batches.pub staging_bytes: u64,pub disk_reserve_bytes: u64,pub operation_timeout_secs: u64,@@ -106,7 +109,7 @@ impl Default for StorageBudget {}impl StorageBudget {- /// Oversized legacy frames are resident history, not work for the sealer.+ /// Oversized frames are resident history, not work for the sealer.pub(super) fn staged_frame_bytes(&self, bytes: usize) -> usize {if bytes <= self.record_bytes {bytes@@ -195,7 +198,8 @@ pub(crate) struct HybridStorage {pub gate: Arc<Mutex<()>>,pub leases: Arc<RwLock<()>>,pub decoder: Arc<Semaphore>,- pub file_job: Mutex<()>,+ /// Serializes file jobs; the value is the next table's archival turn.+ pub file_job: Mutex<usize>,pub read_lanes: std::sync::OnceLock<Arc<Semaphore>>,pub closing: tokio_util::sync::CancellationToken,pub privacy_ready: std::sync::atomic::AtomicBool,@@ -471,7 +475,7 @@ impl HybridStorage {descriptor,gate: Arc::new(Mutex::new(())),leases: Arc::new(RwLock::new(())),- file_job: Mutex::new(()),+ file_job: Mutex::new(0),read_lanes: std::sync::OnceLock::new(),closing: tokio_util::sync::CancellationToken::new(),privacy_ready: std::sync::atomic::AtomicBool::new(false),diff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rsindex c18fdb5bc..44b2e617c 100644--- a/crates/screenpipe-db/src/storage/schema.rs+++ b/crates/screenpipe-db/src/storage/schema.rs@@ -239,10 +239,11 @@ const EMPTY_SURFACES: &str = "(CASE WHEN COALESCE(NEW.full_text,'')='' THEN 1 ELfn triggers() -> String {// Version 1's checksum is part of existing generation identities.- frame_triggers(false)+ frame_triggers(false, false)}-fn frame_triggers(retain_oversized: bool) -> String {+// Keep the v1/v4 SQL byte-for-byte stable for installed schema checksums.+fn frame_triggers(retain_oversized: bool, recording: bool) -> String {let charge = |bytes: &str| {if retain_oversized {format!("CASE WHEN ({bytes}) <= (SELECT record_limit FROM storage_metadata) THEN ({bytes}) ELSE 0 END")@@ -257,18 +258,40 @@ fn frame_triggers(retain_oversized: bool) -> String {} else {String::new()};+ let insert_admission = if recording {+ String::new()+ } else {+ format!(+ r#"+ SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata)+ THEN RAISE(ABORT,'frame storage: record budget exceeded') END;+ SELECT CASE WHEN (SELECT staging_bytes+({BYTES})>staging_limit FROM storage_metadata)+ THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;"#+ )+ };+ let update_admission = if recording {+ String::new()+ } else {+ format!(+ r#"+ SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata){growth_guard}+ THEN RAISE(ABORT,'frame storage: record budget exceeded') END;"#+ )+ };+ let staging_admission = if recording {+ ""+ } else {+ "\n SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata)\n THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;"+ };+ let inserted_bytes = if recording { &new_staging_bytes } else { BYTES };format!(r#"CREATE TRIGGER hybrid_frame_insert AFTER INSERT ON framesWHEN (SELECT maintenance=0 FROM storage_metadata WHERE singleton=1)-BEGIN- SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata)- THEN RAISE(ABORT,'frame storage: record budget exceeded') END;- SELECT CASE WHEN (SELECT staging_bytes+({BYTES})>staging_limit FROM storage_metadata)- THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;+BEGIN{insert_admission}INSERT INTO frame_payloads(frame_id,generation,state,bytes,policy,completed_surfaces,capture_version)SELECT NEW.id,1,'staged',({BYTES}),policy,{EMPTY_SURFACES},writer_version FROM storage_metadata;- UPDATE storage_metadata SET staging_bytes=staging_bytes+({BYTES}),revision=revision+1;+ UPDATE storage_metadata SET staging_bytes=staging_bytes+({inserted_bytes}),revision=revision+1;UPDATE frames SET payload_full_text_length=length(COALESCE(NEW.full_text,'')),payload_accessibility_length=length(COALESCE(NEW.accessibility_text,'')),payload_full_text_present=NEW.full_text IS NOT NULL,@@ -285,13 +308,9 @@ WHEN (SELECT maintenance=0 FROM storage_metadata) ANDBEGIN SELECT RAISE(ABORT,'frame storage: sealed frame requires coordinated payload replacement'); END;CREATE TRIGGER hybrid_frame_update AFTER UPDATE OF full_text,accessibility_text,accessibility_tree_json,text_json,app_name,window_name,browser_url ON framesWHEN (SELECT maintenance=0 FROM storage_metadata)-BEGIN- SELECT CASE WHEN ({BYTES}) > (SELECT record_limit FROM storage_metadata){growth_guard}- THEN RAISE(ABORT,'frame storage: record budget exceeded') END;+BEGIN{update_admission}UPDATE storage_metadata SET revision=revision+1,- staging_bytes=staging_bytes+({new_staging_bytes})-(SELECT {prior_staging_bytes} FROM frame_payloads WHERE frame_id=NEW.id);- SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata)- THEN RAISE(ABORT,'frame storage: staging budget reached; capture admission paused') END;+ staging_bytes=staging_bytes+({new_staging_bytes})-(SELECT {prior_staging_bytes} FROM frame_payloads WHERE frame_id=NEW.id);{staging_admission}UPDATE frame_payloads SET generation=generation+1,bytes=({BYTES}),policy=(SELECT policy FROM storage_metadata),completed_surfaces={EMPTY_SURFACES},attempts=0,retry_at=NULL,last_error=NULLWHERE frame_id=NEW.id;@@ -327,7 +346,7 @@ END;pub(crate) async fn upgrade_resident_frames(conn: &mut SqliteConnection,) -> Result<(), sqlx::Error> {- let sql = frame_triggers(true);+ let sql = frame_triggers(true, false);let checksum = format!("{:x}", Sha256::digest(format!("{sql}:resident-frames-v1")));let installed: Option<String> =sqlx::query_scalar("SELECT checksum FROM _hybrid_migrations WHERE version=4")@@ -365,6 +384,47 @@ pub(crate) async fn upgrade_resident_frames(Ok(())}+/// Archival budgets never reject recording. Replace the v1/v4/v5 write guards+/// atomically without changing their recorded identities or resident bytes.+pub(crate) async fn upgrade_recording(+ conn: &mut SqliteConnection,+ has_bulk: bool,+) -> Result<bool, sqlx::Error> {+ let sql = frame_triggers(true, true);+ let checksum = format!("{:x}", Sha256::digest(format!("{sql}:recording-v6")));+ let installed: Option<String> =+ sqlx::query_scalar("SELECT checksum FROM _hybrid_migrations WHERE version=6")+ .fetch_optional(&mut *conn)+ .await?;+ if let Some(installed) = installed {+ return if installed == checksum {+ Ok(false)+ } else {+ Err(storage_error("recording schema checksum mismatch"))+ };+ }+ let mut tx = conn.begin().await?;+ for name in ["insert", "update", "delete", "sealed_guard", "metadata"] {+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(+ "DROP TRIGGER hybrid_frame_{name}"+ )))+ .execute(&mut *tx)+ .await?;+ }+ sqlx::raw_sql(sqlx::AssertSqlSafe(sql))+ .execute(&mut *tx)+ .await?;+ if has_bulk {+ super::bulk::upgrade_recording(&mut tx).await?;+ }+ sqlx::query("INSERT INTO _hybrid_migrations VALUES(6,?)")+ .bind(checksum)+ .execute(&mut *tx)+ .await?;+ tx.commit().await?;+ Ok(true)+}+pub(crate) async fn bootstrap(conn: &mut SqliteConnection,descriptor: &StorageDescriptor,@@ -460,6 +520,203 @@ mod checkpoint_tests {use sqlx::Connection;use std::time::Duration;+ #[tokio::test]+ async fn recording_upgrade_replaces_existing_guards_once() {+ use crate::storage::{MigrationOptions, PrivacyPolicy};+ use crate::DatabaseManager;++ for had_v5 in [false, true] {+ let root = tempfile::tempdir().unwrap();+ let options = MigrationOptions {+ privacy: PrivacyPolicy {+ identity: "private".into(),+ required_surfaces: 1,+ },+ ..Default::default()+ };+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)+ .await+ .unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-17','original'); INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(1,'2026-09-17','text','original'); UPDATE storage_metadata SET staging_limit=1;").await.unwrap();+ let identities: Vec<(i64, String)> = sqlx::query_as(+ "SELECT version,checksum FROM _hybrid_migrations WHERE version<5 ORDER BY version",+ )+ .fetch_all(&db.pool)+ .await+ .unwrap();+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ let mut legacy = frame_triggers(true, false);+ if had_v5 {+ let charge = |bytes: &str| {+ format!("CASE WHEN ({bytes}) <= (SELECT record_limit FROM storage_metadata) THEN ({bytes}) ELSE 0 END")+ };+ let guard =+ "SELECT CASE WHEN (SELECT staging_bytes>staging_limit FROM storage_metadata)";+ legacy = legacy.replace(guard, &format!("SELECT CASE WHEN ({})>({}) AND (SELECT staging_bytes>staging_limit FROM storage_metadata)", charge(BYTES), charge(&BYTES.replace("NEW.", "OLD."))));+ sqlx::query("INSERT INTO _hybrid_migrations VALUES(5,?)")+ .bind(format!("{:x}", Sha256::digest("staging-drain-v1")))+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ }+ for name in ["insert", "update", "delete", "sealed_guard", "metadata"] {+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(+ "DROP TRIGGER hybrid_frame_{name}"+ )))+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ }+ sqlx::raw_sql(sqlx::AssertSqlSafe(legacy))+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ // Restore the earlier bulk admission checks on existing tables.+ for (name, growth) in [+ ("hybrid_bulk_ui_events_text_content", "COALESCE(length(CAST(NEW.text_content AS BLOB)),0)>CASE WHEN (OLD._archive_mask & 1)!=0 THEN COALESCE(length(CAST(OLD.text_content AS BLOB)),0) ELSE 0 END AND "),+ ("hybrid_bulk_pipe_executions_insert", "COALESCE(length(CAST(NEW.stdout AS BLOB)),0)>0 AND "),+ ] {+ let sql: String = sqlx::query_scalar("SELECT sql FROM sqlite_master WHERE name=?").bind(name).fetch_one(&mut **tx.conn()).await.unwrap();+ let guard = format!("BEGIN SELECT CASE WHEN {}(SELECT staging_bytes>staging_limit FROM storage_metadata) THEN RAISE(ABORT,'storage staging budget reached') END;", if had_v5 { growth } else { "" });+ let sql = sql.replacen("BEGIN", &guard, 1);+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!("DROP TRIGGER {name}; {sql}"))).execute(&mut **tx.conn()).await.unwrap();+ }+ sqlx::query("DELETE FROM _hybrid_migrations WHERE version=6")+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ tx.commit().await.unwrap();+ assert!(db+ .execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-17','capture')"+ )+ .await+ .is_err());+ assert!(db+ .execute_raw_sql_write(+ "UPDATE ui_events SET text_content='growing capture payload' WHERE id=1"+ )+ .await+ .is_err());+ let mut expected_bytes: i64 =+ sqlx::query_scalar("SELECT staging_bytes FROM storage_metadata")+ .fetch_one(&db.pool)+ .await+ .unwrap();+ db.close().await;+ for iteration in 0..2 {+ let db = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ // No warm-up SQL on a writer: the first application write must work.+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ sqlx::query(+ "INSERT INTO frames(timestamp,full_text) VALUES('2026-09-17','capture')",+ )+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ tx.commit().await.unwrap();+ expected_bytes += "capture".len() as i64;+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ expected_bytes+ );+ db.execute_raw_sql_write("UPDATE ui_events SET text_content='growing capture payload' WHERE id=1; INSERT INTO pipe_executions(pipe_name,status,stdout) VALUES('test','running','new payload')").await.unwrap();+ assert_eq!(sqlx::query_as::<_,(i64,String)>("SELECT version,checksum FROM _hybrid_migrations WHERE version<5 ORDER BY version").fetch_all(&db.pool).await.unwrap(), identities);+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM _hybrid_migrations WHERE version=6"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 1+ );+ if had_v5 {+ assert_eq!(+ sqlx::query_scalar::<_, String>(+ "SELECT checksum FROM _hybrid_migrations WHERE version=5"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ format!("{:x}", Sha256::digest("staging-drain-v1"))+ );+ }+ expected_bytes = sqlx::query_scalar("SELECT staging_bytes FROM storage_metadata")+ .fetch_one(&db.pool)+ .await+ .unwrap();+ assert!(expected_bytes > 1, "iteration {iteration}");+ db.verify_storage().await.unwrap();+ db.close().await;+ }+ }+ }++ #[tokio::test]+ #[ignore = "manual old/new capture write throughput comparison"]+ async fn recording_write_cost() {+ let root = tempfile::tempdir().unwrap();+ let db =+ crate::DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ let payload = "x".repeat(4096);+ for round in 0..6 {+ let recording = round % 2 != 0;+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ for name in ["insert", "update", "delete", "sealed_guard", "metadata"] {+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(+ "DROP TRIGGER hybrid_frame_{name}"+ )))+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ }+ sqlx::raw_sql(sqlx::AssertSqlSafe(frame_triggers(true, recording)))+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ tx.commit().await.unwrap();+ let start = std::time::Instant::now();+ for batch in 0..20 {+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ sqlx::query("SELECT name FROM main.sqlite_schema LIMIT 1")+ .fetch_optional(&mut **tx.conn())+ .await+ .unwrap();+ for row in 0..50 {+ sqlx::query(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(?,'2026-09-17',?)",+ )+ .bind(batch * 50 + row + 1)+ .bind(&payload)+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ }+ tx.commit().await.unwrap();+ }+ eprintln!(+ "{}: 1000 4-KiB captures / 20 commits in {:?}",+ if recording { "new" } else { "old" },+ start.elapsed()+ );+ db.execute_raw_sql_write("DELETE FROM frames")+ .await+ .unwrap();+ }+ db.close().await;+ }+#[tokio::test]async fn upgrades_existing_oversized_frame_accounting_once_without_changing_v1_identity() {use crate::storage::{MigrationOptions, Projection, StorageBudget};@@ -495,7 +752,7 @@ mod checkpoint_tests {.execute(&mut **tx.conn()).await.unwrap();- sqlx::raw_sql("DELETE FROM _hybrid_migrations WHERE version=4; UPDATE storage_metadata SET maintenance=1; INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','legacy history',printf('%.*c',2097152,'x')),(2,'2026-09-14T12:00:01Z','pending capture',NULL);")+ sqlx::raw_sql("DELETE FROM _hybrid_migrations WHERE version>=4; UPDATE storage_metadata SET maintenance=1; INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(1,'2026-09-14T12:00:00Z','legacy history',printf('%.*c',2097152,'x')),(2,'2026-09-14T12:00:01Z','pending capture',NULL);").execute(&mut **tx.conn()).await.unwrap();stage_frames(tx.conn(), 1, 2).await.unwrap();sqlx::query(diff --git a/crates/screenpipe-db/src/storage/sealing.rs b/crates/screenpipe-db/src/storage/sealing.rsindex f30b54295..45e85965e 100644--- a/crates/screenpipe-db/src/storage/sealing.rs+++ b/crates/screenpipe-db/src/storage/sealing.rs@@ -9,7 +9,11 @@ use screenpipe_sqlite_coordinator::SqliteWritePool;use sqlx::{Row, SqlitePool};use std::path::Path;use std::sync::Arc;-use std::time::Duration;+use std::time::{Duration, Instant};++const IDLE_POLL: Duration = Duration::from_secs(5);+const MIN_COOLDOWN: Duration = Duration::from_millis(50);+const MAX_COOLDOWN: Duration = Duration::from_secs(60);impl DatabaseManager {pub(crate) async fn strip_archived_frame_details(@@ -121,7 +125,7 @@ impl DatabaseManager {return Err(storage_error("replacement requires hybrid storage"));};let mut tx = self.begin_immediate_with_retry().await?;- let row = sqlx::query("SELECT p.generation,p.state,p.file_id,p.bytes,m.policy,m.staging_bytes,m.staging_limit FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.frame_id=?")+ let row = sqlx::query("SELECT p.generation,p.state,p.file_id,p.bytes,m.policy,m.staging_bytes FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.frame_id=?").bind(payload.id).fetch_optional(&mut **tx.conn()).await?;let Some(row) = row else {tx.rollback().await?;@@ -133,13 +137,6 @@ impl DatabaseManager {tx.rollback().await?;return Ok(false);}- // Retained legacy frames can still be redacted or have detail removed.- // New oversized payloads remain subject to normal capture admission.- if payload.bytes() > storage.descriptor.budget.record_bytes- && payload.bytes() > row.get::<i64, _>("bytes") as usize- {- return Err(storage_error("record budget exceeded"));- }let prior_bytes = if row.get::<&str, _>("state") == "staged" {storage.descriptor@@ -153,9 +150,6 @@ impl DatabaseManager {.descriptor.budget.staged_frame_bytes(payload.bytes()) as i64;- if next_bytes > row.get::<i64, _>("staging_limit") {- return Err(storage_error("staging budget reached"));- }let old_file: Option<String> = row.try_get("file_id")?;sqlx::query("UPDATE storage_metadata SET maintenance=1,revision=revision+1,staging_bytes=?",@@ -199,26 +193,64 @@ impl HybridStorage {) {let storage = Arc::clone(self);tokio::spawn(async move {+ let governor = screenpipe_resource::ResourceGovernor::global();+ tokio::select! {+ _ = shutdown.cancelled() => return,+ _ = tokio::time::sleep(IDLE_POLL) => {}+ }loop {- tokio::select! {+ // Capture never takes this background lane. Keep archival+ // bounded and share the CPU budget with redaction/indexing.+ let waiting = Instant::now();+ let cpu_permit = tokio::select! {_ = shutdown.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(5)) => {}- }+ permit = governor.acquire_background_cpu() => permit,+ };+ let cpu_wait = waiting.elapsed();+ let started = Instant::now();+ let mut archived = 0;// An in-flight file job retains ownership until it either// publishes or leaves its durable reservation for restart.if storage.privacy_ready.load(std::sync::atomic::Ordering::Acquire){- if let Err(error) = storage.seal_once(&pool, &writer).await {- tracing::warn!(%error,"frame storage sealing deferred");+ match storage.seal_once(&pool, &writer).await {+ Ok(rows) => archived = rows,+ Err(error) => tracing::warn!(%error,"frame storage sealing deferred"),}}if shutdown.is_cancelled() {break;}- if let Err(error) = storage.reclaim_once(&pool, &writer).await {- tracing::warn!(%error,"frame storage reclamation deferred");+ let reclaimed = match storage.reclaim_once(&pool, &writer).await {+ Ok(files) => files,+ Err(error) => {+ tracing::warn!(%error,"frame storage reclamation deferred");+ 0+ }+ };+ let worked = started.elapsed();+ let sample = cpu_permit.finish(worked, MIN_COOLDOWN, MAX_COOLDOWN);+ let busy = archived != 0 || reclaimed != 0;+ let delay = if busy { sample.cooldown } else { IDLE_POLL };+ tracing::debug!(+ archived_rows = archived,+ reclaimed_files = reclaimed,+ cpu_wait_ms = cpu_wait.as_millis(),+ worked_ms = worked.as_millis(),+ cooldown_ms = delay.as_millis(),+ active_cpu_percent = sample.active_cpu_percent,+ "storage maintenance batch"+ );+ // Successful work continues after its measured CPU cooldown;+ // only an idle/failed pass gets the full polling backoff.+ // Hold the shared CPU permit through a busy cooldown, just as+ // the redactor does, but release it before idle polling.+ let _cooling = busy.then_some(cpu_permit);+ tokio::select! {+ _ = shutdown.cancelled() => break,+ _ = tokio::time::sleep(delay) => {}}}});@@ -230,7 +262,43 @@ impl HybridStorage {writer: &SqliteWritePool,) -> Result<usize, sqlx::Error> {super::diagnostics::stage("waiting_for_frame_sealer");- let _job = self.file_job.lock().await;+ let mut next = self.file_job.lock().await;+ let tables = 1 + if self.has_bulk() {+ super::bulk::TABLES.len()+ } else {+ 0+ };+ // One bounded job per call, rotating even after an error. Empty or+ // privacy-blocked tables cannot hide ready work later in the cycle.+ for _ in 0..tables {+ let turn = *next;+ *next = (turn + 1) % tables;+ let started = Instant::now();+ let (table, result) = if turn == 0 {+ ("frames", self.seal_frames(pool, writer).await)+ } else {+ let table = &super::bulk::TABLES[turn - 1];+ (table.name, self.seal_bulk_table(pool, writer, table).await)+ };+ tracing::debug!(+ table,+ rows = result.as_ref().ok().copied(),+ failed = result.is_err(),+ elapsed_ms = started.elapsed().as_millis(),+ "storage archival turn"+ );+ if !matches!(result, Ok(0)) {+ return result;+ }+ }+ Ok(0)+ }++ async fn seal_frames(+ self: &Arc<Self>,+ pool: &SqlitePool,+ writer: &SqliteWritePool,+ ) -> Result<usize, sqlx::Error> {// Existing oversized frames remain readable through the staged SQLite// path. Filter before LIMIT so they cannot starve later sealable frames.super::diagnostics::batch("frames", None, None, None, None);@@ -249,7 +317,7 @@ impl HybridStorage {bytes += size;}if ids.is_empty() {- return self.seal_bulk(pool, writer).await;+ return Ok(0);}super::diagnostics::batch("frames",@@ -259,11 +327,13 @@ impl HybridStorage {Some(bytes as u64),);super::diagnostics::stage("reading_frame_payloads");- let payloads: Vec<_> = self- .read(pool, &ids, Projection::All)- .await?- .into_values()- .collect();+ // Ordinary captures must not invalidate a read of older frames.+ // Reuse request snapshots, retaining deletion/privacy revocation.+ let payloads: Vec<_> =+ super::snapshot::run(self, pool, self.read(pool, &ids, Projection::All))+ .await??+ .into_values()+ .collect();self.publish(pool, writer, payloads, None).await}@@ -433,9 +503,8 @@ impl HybridStorage {.fetch_all(pool).await?;if !ids.is_empty() {- let rows = self- .read(pool, &ids, Projection::All)- .await?+ let rows = super::snapshot::run(self, pool, self.read(pool, &ids, Projection::All))+ .await??.into_values().collect();self.publish(pool, writer, rows, Some(&id)).await?;diff --git a/crates/screenpipe-db/src/storage/snapshot.rs b/crates/screenpipe-db/src/storage/snapshot.rsindex 6d7ec5f96..260b68415 100644--- a/crates/screenpipe-db/src/storage/snapshot.rs+++ b/crates/screenpipe-db/src/storage/snapshot.rs@@ -142,11 +142,10 @@ impl Drop for RequestSnapshot {}pub(super) async fn run<T>(- db: &crate::DatabaseManager,+ storage: &Arc<super::HybridStorage>,+ pool: &SqlitePool,read: impl std::future::Future<Output = T>,) -> Result<T, sqlx::Error> {- let storage = db.storage.as_ref().expect("hybrid read");- let pool = &db.pool;let lanes = storage.read_lanes.get_or_init(|| {Arc::new(tokio::sync::Semaphore::new(pool.options()@@ -185,7 +184,7 @@ pub(super) async fn run<T>(};let (token, result) = SNAPSHOT.scope(Arc::clone(&owner.context), async {- let token = db.storage_read_token().await?;+ let token = storage.read_token(pool).await?;Ok::<_, sqlx::Error>((token, read.await))}).await?;@@ -221,7 +220,7 @@ impl crate::DatabaseManager {}tokio::time::timeout(Duration::from_secs(storage.descriptor.budget.operation_timeout_secs),- run(self, read),+ run(storage, &self.pool, read),).await.map_err(|_| super::storage_error("payload read deadline exceeded"))?diff --git a/crates/screenpipe-db/tests/bulk_storage.rs b/crates/screenpipe-db/tests/bulk_storage.rsindex f9639d65e..923fc9003 100644--- a/crates/screenpipe-db/tests/bulk_storage.rs+++ b/crates/screenpipe-db/tests/bulk_storage.rs@@ -451,18 +451,17 @@ async fn deferred_elements_complete_under_staging_pressure() {}tx.commit().await.unwrap();assert_eq!(db.get_frame_elements(1, None).await.unwrap().len(), 3);- assert!(db- .execute_raw_sql_write(- "INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-11T12:01:00Z','next');"- )- .await- .is_err());- while db.seal_payloads().await.unwrap() != 0 {}db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-11T12:01:00Z','next');",).await.unwrap();+ while db.seal_payloads().await.unwrap() != 0 {}+ db.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-09-11T12:02:00Z','after archival');",+ )+ .await+ .unwrap();assert_eq!(db.get_frame_elements(1, None).await.unwrap().len(), 3);db.close().await;}diff --git a/crates/screenpipe-db/tests/hybrid_storage.rs b/crates/screenpipe-db/tests/hybrid_storage.rsindex 5ded61f70..5ed8c581b 100644--- a/crates/screenpipe-db/tests/hybrid_storage.rs+++ b/crates/screenpipe-db/tests/hybrid_storage.rs@@ -609,9 +609,10 @@ async fn legacy_exports_can_be_migrated_again() {}#[tokio::test]-async fn storage_budget_rejects_before_acknowledging_and_accepts_after_reclamation() {+async fn archive_record_budget_keeps_large_captures_resident_and_archives_later_work() {let root = tempfile::tempdir().unwrap();let mut options = MigrationOptions::default();+ options.budget.record_bytes = 1024;options.budget.staging_bytes = options.budget.record_bytes as u64;let limit = options.budget.record_bytes;let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)@@ -619,22 +620,27 @@ async fn storage_budget_rejects_before_acknowledging_and_accepts_after_reclamati.unwrap();let large = "x".repeat(limit + 1);let mut tx = db.begin_immediate_with_retry().await.unwrap();- let error = sqlx::query("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-11',?)")+ sqlx::query("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-11',?)").bind(&large).execute(&mut **tx.conn()).await- .unwrap_err();- assert!(error.to_string().contains("record budget"));- tx.rollback().await.unwrap();+ .unwrap();+ tx.commit().await.unwrap();assert_eq!(sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames").fetch_one(&db.pool).await.unwrap(),- 0+ 1);seed(&db, 2, Some("complete payload"), None).await;- db.seal_frame_payloads().await.unwrap();+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 1);+ assert_eq!(+ db.frame_payloads(&[1], Projection::All).await.unwrap()[&1]+ .full_text+ .as_deref(),+ Some(large.as_str())+ );assert_eq!(sqlx::query_scalar::<_, i64>("SELECT staging_bytes FROM storage_metadata").fetch_one(&db.pool)diff --git a/crates/screenpipe-db/tests/storage_staging_drain.rs b/crates/screenpipe-db/tests/storage_staging_drain.rsnew file mode 100644index 000000000..111332401--- /dev/null+++ b/crates/screenpipe-db/tests/storage_staging_drain.rs@@ -0,0 +1,603 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_db::storage::{MigrationOptions, PrivacyPolicy, Projection};+use screenpipe_db::DatabaseManager;++async fn number(db: &DatabaseManager, sql: &str) -> i64 {+ sqlx::query_scalar(sqlx::AssertSqlSafe(sql.to_owned()))+ .fetch_one(&db.pool)+ .await+ .unwrap()+}++fn private_options() -> MigrationOptions {+ let mut options = MigrationOptions::default();+ options.budget.record_bytes = 1024;+ options.budget.staging_bytes = 1024;+ options.privacy = PrivacyPolicy {+ identity: "private".into(),+ required_surfaces: 1,+ };+ options+}++async fn record(db: &DatabaseManager, text: &str) -> i64 {+ // Exercise the same queue/commit acknowledgement used by paired_capture.+ db.insert_snapshot_frame(+ "display",+ chrono::Utc::now(),+ "",+ Some("Editor"),+ None,+ None,+ true,+ Some("test"),+ Some(text),+ Some("accessibility"),+ None,+ None,+ None,+ )+ .await+ .unwrap()+}++async fn reopen(root: &std::path::Path) -> DatabaseManager {+ DatabaseManager::new(root.join("db.sqlite").to_str().unwrap(), Default::default())+ .await+ .unwrap()+}++async fn record_all_tables(db: &DatabaseManager, id: i64) {+ let frame = record(db, &format!("capture {id} {}", "x".repeat(256))).await;+ db.execute_raw_sql_write(&format!("INSERT OR IGNORE INTO audio_chunks(id,file_path) VALUES(1,'test.wav');+ INSERT OR IGNORE INTO meetings(id,meeting_start,meeting_app) VALUES(1,'2026-09-17','test');+ INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES({id},{frame},'accessibility','AXText','element {id}',1);+ INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,redacted_at) VALUES({id},1,{id},'2026-09-17','transcript {id}',1);+ INSERT INTO ui_events(id,timestamp,event_type,text_content,redacted_at) VALUES({id},'2026-09-17','text','typed {id}',1);+ INSERT INTO semantic_items(id,entity_fingerprint,version_fingerprint,kind,item_key,identity_quality,title,body,metadata_json) VALUES({id},CAST(printf('%032d',{id}) AS BLOB),CAST(printf('%032d',{id}) AS BLOB),'document','document-{id}','stable','title','body {id}','{{}}');+ INSERT INTO pipe_executions(id,pipe_name,status,finished_at,stdout) VALUES({id},'test','completed','2026-09-17','output {id}');+ INSERT INTO meeting_transcript_segments(id,meeting_id,provider,item_id,transcript,captured_at) VALUES({id},1,'test','{id}','meeting {id}','2026-09-17');+ INSERT INTO outputs(id,source,title,output_path,preview) VALUES({id},'test','output','test-{id}.md','preview {id}');")).await.unwrap();+}++async fn pending_by_table(db: &DatabaseManager) -> Vec<i64> {+ let mut counts = vec![+ number(+ db,+ "SELECT count(*) FROM frame_payloads WHERE state='staged'",+ )+ .await,+ number(+ db,+ "SELECT count(*) FROM _bulk_element_rows WHERE _archive_deleted=0",+ )+ .await,+ ];+ for table in [+ "audio_transcriptions",+ "ui_events",+ "semantic_items",+ "pipe_executions",+ "meeting_transcript_segments",+ "outputs",+ ] {+ counts.push(+ number(+ db,+ &format!("SELECT count(*) FROM main.{table} WHERE _archive_mask!=0"),+ )+ .await,+ );+ }+ counts+}++#[tokio::test]+async fn every_table_archives_while_all_tables_keep_receiving_records() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ // Keep every queue nonempty before every turn, including frames. Each+ // table must still receive service within a complete rotation.+ let tables = pending_by_table(&db).await.len();+ for id in 1..=tables * 3 {+ record_all_tables(&db, id as i64).await;+ assert!(db.seal_payloads().await.unwrap() > 0);+ if id % tables == 0 {+ let pending = pending_by_table(&db).await;+ assert!(+ pending.iter().all(|&count| count < tables as i64),+ "turn={id}, pending={pending:?}"+ );+ }+ }+ let pending = pending_by_table(&db).await;+ db.verify_storage().await.unwrap();+ db.close().await;+ let db = reopen(root.path()).await;+ assert_eq!(pending_by_table(&db).await, pending);+ while db.seal_payloads().await.unwrap() != 0 {}+ assert!(pending_by_table(&db).await.iter().all(|&count| count == 0));+ assert_eq!(+ number(&db, "SELECT count(*) FROM frames").await,+ (tables * 3) as i64+ );+ assert_eq!(+ number(&db, "SELECT count(*) FROM elements").await,+ (tables * 3) as i64+ );+ db.verify_storage().await.unwrap();+ db.close().await;+}++#[tokio::test]+async fn failed_frame_archival_does_not_starve_other_tables() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ record_all_tables(&db, 1).await;+ // Break only this disposable database's frame archive directory.+ let blocked = root+ .path()+ .join(&db.storage_descriptor().unwrap().payloads)+ .join("1");+ std::fs::write(&blocked, "owned frame archive failure fixture").unwrap();+ assert!(db.seal_payloads().await.is_err());+ let tables = pending_by_table(&db).await.len();+ for _ in 1..tables {+ record(&db, "recording continues during a frame archive failure").await;+ assert_eq!(db.seal_payloads().await.unwrap(), 1);+ }+ let pending = pending_by_table(&db).await;+ assert_eq!(pending[0], tables as i64);+ assert!(pending[1..].iter().all(|&count| count == 0));+ std::fs::remove_file(blocked).unwrap();+ assert_eq!(db.seal_payloads().await.unwrap(), tables);+ db.reclaim_frame_payloads().await.unwrap();+ db.verify_storage().await.unwrap();+ db.close().await;+}++#[tokio::test]+async fn blocked_old_element_range_does_not_starve_later_private_work() {+ let root = tempfile::tempdir().unwrap();+ let options = MigrationOptions {+ privacy: PrivacyPolicy {+ identity: "private".into(),+ required_surfaces: 1,+ },+ ..Default::default()+ };+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)+ .await+ .unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-17','private'); INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(1,1,'accessibility','AXText','first',1),(2,1,'accessibility','AXText','second',1);").await.unwrap();+ assert_eq!(db.seal_payloads().await.unwrap(), 2);+ db.execute_raw_sql_write("UPDATE elements SET text='private again',redacted_at=NULL WHERE id=2; UPDATE elements SET text='safe' WHERE id=1; INSERT INTO elements(id,frame_id,source,role,text,redacted_at) VALUES(3,1,'accessibility','AXText','later safe',1);").await.unwrap();+ assert_eq!(db.seal_payloads().await.unwrap(), 1);+ assert_eq!(+ number(+ &db,+ "SELECT count(*) FROM _bulk_element_rows WHERE id IN (1,2)"+ )+ .await,+ 2+ );+ assert_eq!(+ number(&db, "SELECT count(*) FROM _bulk_element_rows WHERE id=3").await,+ 0+ );+ assert_eq!(db.seal_payloads().await.unwrap(), 0);+ // The blocked range becomes eligible normally after redaction, without+ // losing either the edits or the independently archived newer record.+ db.execute_raw_sql_write("UPDATE elements SET text='now safe',redacted_at=1 WHERE id=2;")+ .await+ .unwrap();+ assert_eq!(db.seal_payloads().await.unwrap(), 2);+ assert_eq!(number(&db, "SELECT count(*) FROM elements").await, 3);+ assert_eq!(+ number(+ &db,+ "SELECT count(*) FROM elements WHERE text IN ('safe','now safe','later safe')"+ )+ .await,+ 3+ );+ assert_eq!(+ number(+ &db,+ "SELECT completed_surfaces FROM frame_payloads WHERE frame_id=1"+ )+ .await+ & 1,+ 0+ );+ db.verify_storage().await.unwrap();+ db.close().await;+}++#[tokio::test]+async fn recording_and_growing_redaction_survive_a_full_backlog_and_reopen() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), private_options())+ .await+ .unwrap();+ let mut ids = Vec::new();+ for i in 0..32 {+ ids.push(record(&db, &format!("capture {i} {}", "x".repeat(128))).await);+ }+ assert!(+ number(+ &db,+ "SELECT staging_bytes>staging_limit FROM storage_metadata"+ )+ .await+ != 0+ );+ // Privacy is deliberately pending: no archiver can drain these records.+ assert_eq!(db.seal_payloads().await.unwrap(), 0);+ db.execute_raw_sql_write("INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(1,'2026-09-17','text','pending'); INSERT INTO pipe_executions(id,pipe_name,status,stdout) VALUES(1,'test','running','output'); INSERT INTO pipe_executions(id,pipe_name,status) VALUES(2,'test','running');").await.unwrap();+ let mut payload = db+ .frame_payloads(&[ids[0]], Projection::All)+ .await+ .unwrap()+ .remove(&ids[0])+ .unwrap();+ // Replacement tokens can be longer than the original PII. They must save.+ payload.full_text = Some("redacted ".repeat(60));+ assert!(db+ .replace_frame_payload(&payload, "private", 1, None, None)+ .await+ .unwrap());+ // Stale replacements must still fail the generation check.+ assert!(!db+ .replace_frame_payload(&payload, "private", 1, None, None)+ .await+ .unwrap());+ db.execute_raw_sql_write("UPDATE ui_events SET text_content='a longer redacted replacement',redacted_at=1 WHERE id=1; UPDATE pipe_executions SET stdout='longer completed output',status='completed',finished_at='2026-09-17' WHERE id=1;").await.unwrap();+ let before = db.frame_payloads(&ids, Projection::All).await.unwrap();+ let bytes = number(&db, "SELECT staging_bytes FROM storage_metadata").await;+ db.close().await;++ let db = reopen(root.path()).await;+ assert_eq!(+ db.frame_payloads(&ids, Projection::All).await.unwrap(),+ before+ );+ assert_eq!(+ number(&db, "SELECT staging_bytes FROM storage_metadata").await,+ bytes+ );+ assert_eq!(+ number(+ &db,+ "SELECT count(*) FROM frames_fts WHERE frames_fts MATCH 'capture'"+ )+ .await,+ 31+ );+ record(&db, "recording after restart").await;+ while db.seal_payloads().await.unwrap() != 0 {}+ // Redaction eligibility is preserved; the remaining captures stay resident.+ assert_eq!(+ number(+ &db,+ "SELECT count(*) FROM frame_payloads WHERE state='sealed'"+ )+ .await,+ 1+ );+ // Editing an archived field restages bytes and must work above the old cap.+ db.execute_raw_sql_write(+ "UPDATE ui_events SET text_content='updated archived content' WHERE id=1",+ )+ .await+ .unwrap();+ let archived = db+ .frame_payloads(&[ids[0]], Projection::All)+ .await+ .unwrap()+ .remove(&ids[0])+ .unwrap();+ assert!(db+ .replace_frame_payload(&archived, "private", 1, None, None)+ .await+ .unwrap());+ db.set_frame_privacy_policy(&PrivacyPolicy::default())+ .await+ .unwrap();+ while db.seal_payloads().await.unwrap() != 0 {}+ assert_eq!(+ number(&db, "SELECT staging_bytes FROM storage_metadata").await,+ 0+ );+ assert_eq!(number(&db, "SELECT count(*) FROM frames").await, 33);+ assert_eq!(+ number(&db, "SELECT staging_limit FROM storage_metadata").await,+ 1024+ );+ db.verify_storage().await.unwrap();+ db.close().await;+}++#[tokio::test]+async fn oversized_capture_and_bulk_payloads_remain_durable_and_searchable() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), private_options())+ .await+ .unwrap();+ let oversized = "oversized 東京 ".repeat(200);+ let frame = record(&db, &oversized).await;+ let audio = db.insert_audio_chunk("test.wav", None).await.unwrap();+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ sqlx::query("INSERT INTO audio_transcriptions(audio_chunk_id,transcription,offset_index,timestamp,device,is_input_device) VALUES(?,?,0,'2026-09-17','mic',1)")+ .bind(audio).bind(&oversized).execute(&mut **tx.conn()).await.unwrap();+ sqlx::query(+ "INSERT INTO elements(id,frame_id,source,role,text) VALUES(1,?,'accessibility','AXText',?)",+ )+ .bind(frame)+ .bind(&oversized)+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ for query in [+ "INSERT INTO ui_events(id,timestamp,event_type,text_content) VALUES(1,'2026-09-17','text',?)",+ "INSERT INTO pipe_executions(id,pipe_name,status,stdout) VALUES(1,'test','running',?)",+ ] {+ sqlx::query(sqlx::AssertSqlSafe(query)).bind(&oversized).execute(&mut **tx.conn()).await.unwrap();+ }+ tx.commit().await.unwrap();+ let mut payload = db+ .frame_payloads(&[frame], Projection::All)+ .await+ .unwrap()+ .remove(&frame)+ .unwrap();+ payload.full_text = Some(format!("{} extra", oversized));+ assert!(db+ .replace_frame_payload(&payload, "private", 1, None, None)+ .await+ .unwrap());+ db.execute_raw_sql_write("UPDATE elements SET text=text||' extra' WHERE id=1; UPDATE ui_events SET text_content=text_content||' extra' WHERE id=1;").await.unwrap();+ db.set_frame_privacy_policy(&PrivacyPolicy::default())+ .await+ .unwrap();+ let later = record(&db, "later eligible capture").await;+ while db.seal_payloads().await.unwrap() != 0 {}+ assert_eq!(+ number(+ &db,+ "SELECT count(*) FROM frame_payloads WHERE state='sealed'"+ )+ .await,+ 1+ );+ assert_eq!(+ number(&db, "SELECT count(*) FROM _bulk_element_rows WHERE id=1").await,+ 1+ );+ let queries = [+ "SELECT transcription FROM audio_transcriptions",+ "SELECT text FROM elements",+ "SELECT text_content FROM ui_events",+ "SELECT stdout FROM pipe_executions",+ ];+ let mut expected = Vec::new();+ for query in queries {+ expected.push(db.query_raw_sql(query).await.unwrap());+ }+ let bytes = number(&db, "SELECT staging_bytes FROM storage_metadata").await;+ db.close().await;+ let db = reopen(root.path()).await;+ assert_eq!(+ number(&db, "SELECT staging_bytes FROM storage_metadata").await,+ bytes+ );+ for (query, expected) in queries.into_iter().zip(expected) {+ assert_eq!(db.query_raw_sql(query).await.unwrap(), expected);+ }+ let saved = db+ .frame_payloads(&[frame, later], Projection::All)+ .await+ .unwrap();+ assert_eq!(saved[&frame].full_text, payload.full_text);+ assert_eq!(+ saved[&later].full_text.as_deref(),+ Some("later eligible capture")+ );+ for fts in [+ "frames_fts",+ "elements_fts",+ "ui_events_fts",+ "audio_transcriptions_fts",+ ] {+ assert_eq!(+ number(+ &db,+ &format!("SELECT count(*) FROM {fts} WHERE {fts} MATCH 'oversized'")+ )+ .await,+ 1+ );+ }+ record(&db, "new recording beside oversized history").await;+ db.verify_storage().await.unwrap();+ db.close().await;+}++#[tokio::test]+async fn failed_archive_write_does_not_block_recording_or_recovery() {+ let root = tempfile::tempdir().unwrap();+ let mut options = private_options();+ options.privacy = PrivacyPolicy::default();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), options)+ .await+ .unwrap();+ // A file where the archive directory belongs causes a real filesystem+ // failure, confined to this disposable test's payload storage.+ let payloads = root.path().join(&db.storage_descriptor().unwrap().payloads);+ if payloads.exists() {+ std::fs::remove_dir(&payloads).unwrap();+ }+ std::fs::write(&payloads, b"blocked archive path").unwrap();+ let mut ids = Vec::new();+ for i in 0..12 {+ ids.push(record(&db, &format!("durable capture {i} {}", "x".repeat(128))).await);+ assert!(db.seal_payloads().await.is_err());+ }+ let expected = db.frame_payloads(&ids, Projection::All).await.unwrap();+ db.close().await;+ let db = reopen(root.path()).await;+ assert_eq!(+ db.frame_payloads(&ids, Projection::All).await.unwrap(),+ expected+ );+ record(&db, "recording while archive is still broken").await;+ std::fs::remove_file(&payloads).unwrap();+ std::fs::create_dir_all(&payloads).unwrap();+ db.reclaim_frame_payloads().await.unwrap();+ while db.seal_payloads().await.unwrap() != 0 {}+ assert_eq!(number(&db, "SELECT count(*) FROM frames").await, 13);+ assert_eq!(+ number(&db, "SELECT staging_bytes FROM storage_metadata").await,+ 0+ );+ assert_eq!(+ db.frame_payloads(&ids, Projection::All).await.unwrap(),+ expected+ );+ db.verify_storage().await.unwrap();+ db.close().await;+}++#[tokio::test]+async fn acknowledged_backlog_survives_process_exit() {+ const CHILD_ROOT: &str = "SCREENPIPE_RECORDING_DURABILITY_TEST_ROOT";+ if let Some(root) = std::env::var_os(CHILD_ROOT) {+ let db = DatabaseManager::new_hybrid(+ std::path::Path::new(&root),+ Default::default(),+ private_options(),+ )+ .await+ .unwrap();+ for i in 0..16 {+ assert_eq!(+ record(&db, &format!("capture {i} {}", "x".repeat(128))).await,+ i + 1+ );+ }+ // No graceful queue drain, pool close, checkpoint, or destructor.+ std::process::exit(86);+ }+ let root = tempfile::tempdir().unwrap();+ let status = std::process::Command::new(std::env::current_exe().unwrap())+ .args([+ "--exact",+ "acknowledged_backlog_survives_process_exit",+ "--nocapture",+ ])+ .env(CHILD_ROOT, root.path())+ .status()+ .unwrap();+ assert_eq!(status.code(), Some(86));+ let db = reopen(root.path()).await;+ let rows: Vec<(i64, String)> = sqlx::query_as("SELECT id,full_text FROM frames ORDER BY id")+ .fetch_all(&db.pool)+ .await+ .unwrap();+ let expected: Vec<_> = (0..16)+ .map(|i| (i + 1, format!("capture {i} {}", "x".repeat(128))))+ .collect();+ assert_eq!(rows, expected);+ assert_eq!(record(&db, "recording after process recovery").await, 17);+ db.verify_storage().await.unwrap();+ db.close().await;+}++// Exercise the production maintenance task while frames arrive once per second.+// Keep the workload fixed so before/after backlog timings remain comparable.+mod background_benchmark {+ use screenpipe_db::DatabaseManager;+ use std::time::{Duration, Instant};++ async fn capture(db: &DatabaseManager) {+ db.insert_snapshot_frame(+ "display",+ chrono::Utc::now(),+ "",+ Some("Probe"),+ None,+ None,+ true,+ Some("probe"),+ Some(&"x".repeat(16 * 1024)),+ Some("accessibility"),+ None,+ None,+ None,+ )+ .await+ .unwrap();+ }++ async fn report(db: &DatabaseManager, label: &str, start: Instant) {+ let counts: (i64, i64, i64, i64, i64) = sqlx::query_as(+ "SELECT (SELECT count(*) FROM frame_payloads WHERE state='staged'),+ (SELECT count(*) FROM frame_payloads WHERE state='sealed'),+ (SELECT count(*) FROM _bulk_element_rows),+ (SELECT count(*) FROM main.audio_transcriptions WHERE _archive_mask!=0),+ (SELECT staging_bytes FROM storage_metadata)",+ )+ .fetch_one(&db.pool)+ .await+ .unwrap();+ eprintln!("PROFILE_COUNTS {label} elapsed={:.3} staged_frames={} sealed_frames={} staged_elements={} staged_audio={} staging_bytes={}", start.elapsed().as_secs_f64(), counts.0, counts.1, counts.2, counts.3, counts.4);+ }++ #[tokio::test]+ #[ignore = "manual background archival benchmark with real polling and CPU cooldown"]+ async fn measure_actual_background_scheduler() {+ let _ = tracing_subscriber::fmt()+ .with_env_filter("screenpipe_db::storage::sealing=debug")+ .with_ansi(false)+ .try_init();+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ db.set_frame_privacy_policy(&Default::default())+ .await+ .unwrap();+ capture(&db).await;+ db.execute_raw_sql_write("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1000) INSERT INTO elements(id,frame_id,source,role,text,redacted_at) SELECT x,1,'accessibility','AXText',printf('%0512d',x),1 FROM n;+ INSERT INTO audio_chunks(id,file_path) VALUES(1,'probe.wav');+ WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8) INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,redacted_at) SELECT x,1,x,'2026-09-17',printf('%01024d',x),1 FROM n;").await.unwrap();+ let start = Instant::now();+ report(&db, "initial", start).await;+ for second in 1..=31 {+ tokio::time::sleep(Duration::from_secs(1)).await;+ capture(&db).await;+ if second % 5 == 1 {+ report(&db, "recording", start).await;+ }+ }+ report(&db, "producer_stopped", start).await;+ let remaining: (i64, i64) = sqlx::query_as("SELECT (SELECT count(*) FROM _bulk_element_rows),(SELECT count(*) FROM main.audio_transcriptions WHERE _archive_mask!=0)").fetch_one(&db.pool).await.unwrap();+ assert_eq!(+ remaining,+ (0, 0),+ "ready bulk records must drain while frame capture continues"+ );+ for _ in 0..3 {+ tokio::time::sleep(Duration::from_secs(5)).await;+ report(&db, "quiet", start).await;+ }+ db.verify_storage().await.unwrap();+ db.close().await;+ }+}diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 54d3eddf8..6f20b2a27 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -8,11 +8,11 @@ confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y- Mapped suites: 33-- Mapped Rust files: 357-- Active test blocks: 3504-- Ignored/manual test blocks: 144-- Declared test blocks: 3648-- Weighted coverage points: 2890.4+- Mapped Rust files: 358+- Active test blocks: 3513+- Ignored/manual test blocks: 146+- Declared test blocks: 3659+- Weighted coverage points: 2899.4Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3361 | 138 | 2825.3 | 21 | 11 | 100% |-| macos | 30 | 3423 | 119 | 2839.3 | 22 | 11 | 100% |-| linux | 26 | 2991 | 111 | 2484.6 | 20 | 11 | 100% |+| windows | 30 | 3370 | 140 | 2834.3 | 21 | 11 | 100% |+| macos | 30 | 3432 | 121 | 2848.3 | 22 | 11 | 100% |+| linux | 26 | 3000 | 113 | 2493.6 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |-| screenpipe-db | 5 | 57 | 23 | 535 | 21 | 508.3 | 9 |+| screenpipe-db | 5 | 58 | 23 | 544 | 23 | 517.3 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts || audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |-| database | 6 suites / 483 active / 17 ignored / 456.3 pts | 6 suites / 483 active / 17 ignored / 456.3 pts | 6 suites / 483 active / 17 ignored / 456.3 pts |+| database | 6 suites / 492 active / 19 ignored / 465.3 pts | 6 suites / 492 active / 19 ignored / 465.3 pts | 6 suites / 492 active / 19 ignored / 465.3 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts || engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts || local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts || meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1537 active / 72 ignored / 1357.5 pts | 14 suites / 1644 active / 76 ignored / 1400.3 pts | 13 suites / 1537 active / 72 ignored / 1357.5 pts |+| performance | 13 suites / 1546 active / 74 ignored / 1366.5 pts | 14 suites / 1653 active / 78 ignored / 1409.3 pts | 13 suites / 1546 active / 74 ignored / 1366.5 pts || pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts || privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts || real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - || speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |-| storage | 4 suites / 600 active / 34 ignored / 494.1 pts | 4 suites / 600 active / 34 ignored / 494.1 pts | 4 suites / 600 active / 34 ignored / 494.1 pts |+| storage | 4 suites / 609 active / 36 ignored / 503.1 pts | 4 suites / 609 active / 36 ignored / 503.1 pts | 4 suites / 609 active / 36 ignored / 503.1 pts || sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |-| timeline | 4 suites / 1147 active / 38 ignored / 934.3 pts | 4 suites / 1147 active / 38 ignored / 934.3 pts | 4 suites / 1147 active / 38 ignored / 934.3 pts |+| timeline | 4 suites / 1156 active / 40 ignored / 943.3 pts | 4 suites / 1156 active / 40 ignored / 943.3 pts | 4 suites / 1156 active / 40 ignored / 943.3 pts || transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts || ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts || vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 30 | 247 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 31 | 256 | 10 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |@@ -276,12 +276,12 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-runtime-reliability | screenpipe-db | src/failpoint_vfs.rs | source | 4 | 0 | 4 || db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 || db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |-| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 6 | 0 | 6 |+| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 7 | 0 | 7 || db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 || db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 || db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 || db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 2 | 1 | 3 |-| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 2 | 0 | 2 |+| db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 3 | 1 | 4 || db-timeline-frames | screenpipe-db | src/storage/sealing.rs | source | 1 | 0 | 1 || db-search-indexing | screenpipe-db | src/text_normalizer.rs | source | 19 | 0 | 19 || db-search-indexing | screenpipe-db | src/text_similarity.rs | source | 20 | 0 | 20 |@@ -333,6 +333,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-runtime-reliability | screenpipe-db | tests/sqlite_architecture_invariants_test.rs | integration | 2 | 0 | 2 || db-runtime-reliability | screenpipe-db | tests/sqlite_runtime_version.rs | integration | 1 | 0 | 1 || db-runtime-reliability | screenpipe-db | tests/storage_snapshots.rs | integration | 3 | 0 | 3 |+| db-timeline-frames | screenpipe-db | tests/storage_staging_drain.rs | integration | 7 | 1 | 8 || db-timeline-frames | screenpipe-db | tests/suggestions_frames_table_test.rs | integration | 3 | 0 | 3 || db-search-indexing | screenpipe-db | tests/tag_autocomplete_query_test.rs | integration | 5 | 1 | 6 || db-search-indexing | screenpipe-db | tests/tag_filter_bench.rs | integration | 0 | 2 | 2 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex adc852fa0..527fafa25 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -686,6 +686,7 @@"tests/outputs_saf_test.rs","tests/range_export_frame_sources_test.rs","tests/semantic_storage_test.rs",+ "tests/storage_staging_drain.rs","tests/suggestions_frames_table_test.rs","tests/timeline_frameless_audio_test.rs","tests/timeline_parquet_test.rs",diff --git a/packages/sdk/Cargo.lock b/packages/sdk/Cargo.lockindex 85cb57e60..7041d389f 100644--- a/packages/sdk/Cargo.lock+++ b/packages/sdk/Cargo.lock@@ -6999,6 +6999,7 @@ dependencies = ["regex","rusqlite","screenpipe-config",+ "screenpipe-resource","screenpipe-semantic","screenpipe-sqlite-coordinator","serde",@@ -7054,6 +7055,15 @@ dependencies = ["tracing",]+[[package]]+name = "screenpipe-resource"+version = "0.4.50"+dependencies = [+ "serde",+ "sysinfo",+ "tokio",+]+[[package]]name = "screenpipe-screen"version = "0.4.50"diff --git a/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock b/packages/sdk/examples/tauri-app/src-tauri/Cargo.lockindex 7a215d99e..a453cd767 100644--- a/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock+++ b/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock@@ -7125,6 +7125,7 @@ dependencies = ["regex","rusqlite","screenpipe-config",+ "screenpipe-resource","screenpipe-semantic","screenpipe-sqlite-coordinator","serde",@@ -7178,6 +7179,15 @@ dependencies = ["tracing",]+[[package]]+name = "screenpipe-resource"+version = "0.4.50"+dependencies = [+ "serde",+ "sysinfo",+ "tokio",+]+[[package]]name = "screenpipe-screen"version = "0.4.50"