YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): coordinate redaction writes
Louis Beaumont7e7520597d3d
diff --git a/Cargo.lock b/Cargo.lockindex 66e03870b..f05b795d0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8749,6 +8749,7 @@ dependencies = ["reqwest 0.13.3","screenpipe-resource","screenpipe-rfdetr-mlx",+ "screenpipe-sqlite-coordinator","serde","serde_json","sha2 0.10.9",@@ -8871,6 +8872,7 @@ name = "screenpipe-sqlite-coordinator"version = "0.4.34"dependencies = ["libsqlite3-sys",+ "sqlx","tempfile","tokio","tracing",diff --git a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock b/apps/screenpipe-app-tauri/src-tauri/Cargo.lockindex 4c1f37ad4..bcdc4854a 100644--- a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock+++ b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock@@ -11509,6 +11509,7 @@ dependencies = ["reqwest 0.13.3","screenpipe-resource","screenpipe-rfdetr-mlx",+ "screenpipe-sqlite-coordinator","serde","serde_json","sha2 0.10.9",@@ -11615,6 +11616,7 @@ name = "screenpipe-sqlite-coordinator"version = "0.4.34"dependencies = ["libsqlite3-sys",+ "sqlx","tokio","tracing",]diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rsindex 58125e6b7..e9eeac641 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs@@ -1070,7 +1070,12 @@ impl ServerCore {poll_interval: std::time::Duration::from_secs(5 * 60),..Default::default()};- let _ = Worker::new(db.pool.clone(), placeholder, cfg)+ let _ = Worker::new_with_writer(+ db.pool.clone(),+ db.coordinated_writer(),+ placeholder,+ cfg,+ ).spawn_with_shutdown(redact_shutdown.clone());}}@@ -1146,7 +1151,12 @@ impl ServerCore {tables: ALL_TARGET_TABLES.to_vec(),..Default::default()};- let _ = Worker::new(db.pool.clone(), pipeline_arc, cfg)+ let _ = Worker::new_with_writer(+ db.pool.clone(),+ db.coordinated_writer(),+ pipeline_arc,+ cfg,+ ).with_database_error_hook(redact_database_error_hook.clone()).spawn_with_shutdown(redact_shutdown.clone());} else {@@ -1155,6 +1165,7 @@ impl ServerCore {// launch. The worker is created inside the spawned// task once the model is ready.let pool = db.pool.clone();+ let writer = db.coordinated_writer();let shutdown = redact_shutdown.clone();let labels = pii_labels.clone();let pseudonymizer = pseudonymizer.clone();@@ -1236,7 +1247,7 @@ impl ServerCore {tables: ALL_TARGET_TABLES.to_vec(),..Default::default()};- let _ = Worker::new(pool, pipeline_arc, cfg)+ let _ = Worker::new_with_writer(pool, writer, pipeline_arc, cfg).with_database_error_hook(database_error_hook).spawn_with_shutdown(shutdown);});@@ -1253,6 +1264,7 @@ impl ServerCore {use screenpipe_redact::ImageRedactor;let pool = db.pool.clone();+ let writer = db.coordinated_writer();if use_tinfoil {let detector = Arc::new(TinfoilImageRedactor::new(TinfoilImageConfig {api_key: tinfoil_api_key.clone(),@@ -1265,8 +1277,9 @@ impl ServerCore {has_api_key = tinfoil_api_key.is_some(),"starting async image-PII worker (backend=tinfoil)");- let _ = ImageWorker::new(+ let _ = ImageWorker::new_with_writer(pool,+ writer,detector,ImageWorkerConfig {policy: ImageRedactionPolicy::from_labels(&pii_labels),@@ -1293,8 +1306,9 @@ impl ServerCore {"starting async image-PII worker (backend=local)");let detector_arc = Arc::new(detector) as Arc<dyn ImageRedactor>;- let _ = ImageWorker::new(+ let _ = ImageWorker::new_with_writer(pool,+ writer,detector_arc,ImageWorkerConfig {policy: ImageRedactionPolicy::from_labels(&labels),diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex e31308c4c..ffae5e9d9 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -82,6 +82,15 @@ fn preflight_existing_database_header(path: &Path) -> Result<(), SqlxError> {}impl DatabaseManager {+ /// Give an independently owned worker access to the dedicated write pool+ /// only while it participates in this database's single-writer protocol.+ pub fn coordinated_writer(&self) -> screenpipe_sqlite_coordinator::SqliteWritePool {+ screenpipe_sqlite_coordinator::SqliteWritePool::new(+ self.write_pool.clone(),+ Arc::clone(&self.write_semaphore),+ )+ }+pub async fn new(database_path: &str, config: DbConfig) -> Result<Self, sqlx::Error> {screenpipe_sqlite_coordinator::verify_sqlite_runtime().map_err(SqlxError::Protocol)?;debug!(diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rsindex 220789125..d8a20194e 100644--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs@@ -1942,7 +1942,9 @@ async fn main() -> anyhow::Result<()> {..Default::default()};info!("starting pi session secret-scrub worker (--redact-agent-session-secrets)");- let _ = Worker::new(db.pool.clone(), placeholder, cfg).spawn();+ let _ =+ Worker::new_with_writer(db.pool.clone(), db.coordinated_writer(), placeholder, cfg)+ .spawn();}}@@ -1989,6 +1991,7 @@ async fn main() -> anyhow::Result<()> {// 4. Regex-only otherwise (still destructive — overwrites// regex-redacted text into the source columns).let pool = db.pool.clone();+ let writer = db.coordinated_writer();let labels = config.pii_redaction_labels.clone();let database_error_hook = redact_database_error_hook.clone();// Consistent-pseudonym tokens (issue #4206), opt-in. Loads (or@@ -2112,7 +2115,7 @@ async fn main() -> anyhow::Result<()> {columns,..Default::default()};- let _worker_handle = Worker::new(pool, pipeline_arc, worker_cfg)+ let _worker_handle = Worker::new_with_writer(pool, writer, pipeline_arc, worker_cfg).with_database_error_hook(database_error_hook).spawn();// The worker runs for the lifetime of the engine. We don't@@ -2211,9 +2214,14 @@ async fn main() -> anyhow::Result<()> {policy: ImageRedactionPolicy::from_labels(&config.pii_redaction_labels),..Default::default()};- let _img_handle = ImageWorker::new(db.pool.clone(), detector, cfg)- .with_database_error_hook(redact_database_error_hook.clone())- .spawn();+ let _img_handle = ImageWorker::new_with_writer(+ db.pool.clone(),+ db.coordinated_writer(),+ detector,+ cfg,+ )+ .with_database_error_hook(redact_database_error_hook.clone())+ .spawn();}}diff --git a/crates/screenpipe-redact/Cargo.toml b/crates/screenpipe-redact/Cargo.tomlindex c7004681e..589dc50e8 100644--- a/crates/screenpipe-redact/Cargo.toml+++ b/crates/screenpipe-redact/Cargo.toml@@ -31,6 +31,7 @@ rand = { workspace = true }regex = { workspace = true }reqwest = { workspace = true }screenpipe-resource = { path = "../screenpipe-resource" }+screenpipe-sqlite-coordinator = { path = "../screenpipe-sqlite-coordinator" }serde = { workspace = true }sha2 = { workspace = true }sha3 = "0.10"diff --git a/crates/screenpipe-redact/src/image/worker.rs b/crates/screenpipe-redact/src/image/worker.rsindex 68146521f..862efe6f0 100644--- a/crates/screenpipe-redact/src/image/worker.rs+++ b/crates/screenpipe-redact/src/image/worker.rs@@ -25,6 +25,7 @@ use std::sync::Arc;use std::time::Duration;use screenpipe_resource::ResourceGovernor;+use screenpipe_sqlite_coordinator::SqliteWritePool;use sqlx::{Row, SqlitePool};use tokio::sync::{Mutex, Notify};use tokio::task::JoinHandle;@@ -93,6 +94,7 @@ pub struct ImageWorkerStatus {#[derive(Clone)]pub struct ImageWorker {pool: SqlitePool,+ writer: SqliteWritePool,redactor: Arc<dyn ImageRedactor>,cfg: ImageWorkerConfig,status: Arc<Mutex<ImageWorkerStatus>>,@@ -102,8 +104,19 @@ pub struct ImageWorker {impl ImageWorker {pub fn new(pool: SqlitePool, redactor: Arc<dyn ImageRedactor>, cfg: ImageWorkerConfig) -> Self {+ let writer = SqliteWritePool::standalone(pool.clone());+ Self::new_with_writer(pool, writer, redactor, cfg)+ }++ pub fn new_with_writer(+ pool: SqlitePool,+ writer: SqliteWritePool,+ redactor: Arc<dyn ImageRedactor>,+ cfg: ImageWorkerConfig,+ ) -> Self {Self {pool,+ writer,redactor,cfg,status: Arc::new(Mutex::new(ImageWorkerStatus::default())),@@ -365,6 +378,7 @@ impl ImageWorker {}async fn mark_redacted(&self, frame_id: i64) -> Result<(), sqlx::Error> {+ let writer = self.writer.lock().await?;sqlx::query(r#"UPDATE frames@@ -373,7 +387,7 @@ impl ImageWorker {"#,).bind(frame_id)- .execute(&self.pool)+ .execute(writer.pool()).await.map(|_| ())}diff --git a/crates/screenpipe-redact/src/worker/mod.rs b/crates/screenpipe-redact/src/worker/mod.rsindex e7cf73e39..33a15880b 100644--- a/crates/screenpipe-redact/src/worker/mod.rs+++ b/crates/screenpipe-redact/src/worker/mod.rs@@ -25,6 +25,7 @@ use std::sync::Arc;use std::time::{Duration, SystemTime};use screenpipe_resource::ResourceGovernor;+use screenpipe_sqlite_coordinator::SqliteWritePool;use sqlx::SqlitePool;use tokio::sync::{Mutex, Notify};use tokio::task::JoinHandle;@@ -117,6 +118,7 @@ pub struct WorkerStatus {#[derive(Clone)]pub struct Worker {pool: SqlitePool,+ writer: SqliteWritePool,redactor: Arc<dyn Redactor>,cfg: WorkerConfig,status: Arc<Mutex<WorkerStatus>>,@@ -126,8 +128,19 @@ pub struct Worker {impl Worker {pub fn new(pool: SqlitePool, redactor: Arc<dyn Redactor>, cfg: WorkerConfig) -> Self {+ let writer = SqliteWritePool::standalone(pool.clone());+ Self::new_with_writer(pool, writer, redactor, cfg)+ }++ pub fn new_with_writer(+ pool: SqlitePool,+ writer: SqliteWritePool,+ redactor: Arc<dyn Redactor>,+ cfg: WorkerConfig,+ ) -> Self {Self {pool,+ writer,redactor,cfg,status: Arc::new(Mutex::new(WorkerStatus::default())),@@ -593,8 +606,9 @@ impl Worker {None};+ let writer = self.writer.lock().await?;tables::write_redacted_element(- &self.pool,+ writer.pool(),row.id,text_out.as_deref(),props_out.as_deref(),@@ -709,7 +723,8 @@ impl Worker {}for (row, redacted) in rows.iter().zip(outputs_by_row.iter()) {- tables::write_redacted_ui_events(&self.pool, &active, row.id, redacted).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_ui_events(writer.pool(), &active, row.id, redacted).await?;}let n = rows.len() as u32;@@ -776,8 +791,9 @@ impl Worker {// they're warned + skipped inside the helper, unfixable by// retry, so they must not wedge full_text).writes += self.propagate_frame_derived(row, &map).await?;+ let writer = self.writer.lock().await?;tables::write_redacted(- &self.pool,+ writer.pool(),TargetTable::FullText,row.id,&map.apply(&row.full_text),@@ -802,8 +818,9 @@ impl Worker {// accessibility_text is left to the Accessibility pass// (it has its own model-pass fallback target).let out = self.redactor.redact(&row.full_text).await?;+ let writer = self.writer.lock().await?;tables::write_redacted(- &self.pool,+ writer.pool(),TargetTable::FullText,row.id,&out.redacted,@@ -842,8 +859,9 @@ impl Worker {if cols.accessibility_text {if let Some(acc) = row.accessibility_text.as_deref() {if !acc.is_empty() && row.accessibility_redacted_at.is_none() {+ let writer = self.writer.lock().await?;tables::write_redacted(- &self.pool,+ writer.pool(),TargetTable::Accessibility,row.id,&map.apply(acc),@@ -864,7 +882,8 @@ impl Worker {&cols.a11y_json_fields(),) {Ok(Some(json)) => {- tables::write_redacted_tree(&self.pool, row.id, &json).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_tree(writer.pool(), row.id, &json).await?;writes += 1;}// Ok(None) means the map was empty — impossible here@@ -884,7 +903,9 @@ impl Worker {if cols.window_name {if let Some(wn) = row.window_name.as_deref() {if !wn.is_empty() && row.window_name_redacted_at.is_none() {- tables::write_redacted_window_name(&self.pool, row.id, &map.apply(wn)).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_window_name(writer.pool(), row.id, &map.apply(wn))+ .await?;writes += 1;}}@@ -896,7 +917,9 @@ impl Worker {if cols.browser_url {if let Some(url) = row.browser_url.as_deref() {if !url.is_empty() && row.browser_url_redacted_at.is_none() {- tables::write_redacted_browser_url(&self.pool, row.id, &map.apply(url)).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_browser_url(writer.pool(), row.id, &map.apply(url))+ .await?;writes += 1;}}@@ -916,7 +939,8 @@ impl Worker {if !tj.is_empty() && row.text_json_redacted_at.is_none() {match crate::ocr_json::redact_ocr_text_json(tj, map) {Ok(Some(json)) => {- tables::write_redacted_text_json(&self.pool, row.id, &json).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_text_json(writer.pool(), row.id, &json).await?;writes += 1;}// Ok(None) means the map was empty — impossible here@@ -964,13 +988,15 @@ impl Worker {.await{Ok(Some(json)) => {- tables::write_redacted_tree(&self.pool, row.id, &json).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_tree(writer.pool(), row.id, &json).await?;writes += 1;}// No redactable text → stamp the verbatim blob so the// row isn't re-scanned for the tree forever.Ok(None) => {- tables::write_redacted_tree(&self.pool, row.id, tree).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_tree(writer.pool(), row.id, tree).await?;writes += 1;}Err(crate::tree_json::TreeRedactError::Json(e)) => warn!(@@ -991,7 +1017,9 @@ impl Worker {if let Some(wn) = row.window_name.as_deref() {if !wn.is_empty() && row.window_name_redacted_at.is_none() {let out = self.redactor.redact(wn).await?;- tables::write_redacted_window_name(&self.pool, row.id, &out.redacted).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_window_name(writer.pool(), row.id, &out.redacted)+ .await?;writes += 1;}}@@ -1001,7 +1029,9 @@ impl Worker {if let Some(url) = row.browser_url.as_deref() {if !url.is_empty() && row.browser_url_redacted_at.is_none() {let out = self.redactor.redact(url).await?;- tables::write_redacted_browser_url(&self.pool, row.id, &out.redacted).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_browser_url(writer.pool(), row.id, &out.redacted)+ .await?;writes += 1;}}@@ -1024,13 +1054,15 @@ impl Worker {.await{Ok(Some(json)) => {- tables::write_redacted_text_json(&self.pool, row.id, &json).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_text_json(writer.pool(), row.id, &json).await?;writes += 1;}// No redactable text → stamp the verbatim blob so the// row isn't re-scanned for text_json forever.Ok(None) => {- tables::write_redacted_text_json(&self.pool, row.id, tj).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted_text_json(writer.pool(), row.id, tj).await?;writes += 1;}Err(crate::tree_json::TreeRedactError::Json(e)) => warn!(@@ -1069,7 +1101,8 @@ impl Worker {}for (row, out) in rows.iter().zip(outputs.iter()) {- tables::write_redacted(&self.pool, table, row.id, &out.redacted).await?;+ let writer = self.writer.lock().await?;+ tables::write_redacted(writer.pool(), table, row.id, &out.redacted).await?;}let n = rows.len() as u32;diff --git a/crates/screenpipe-redact/tests/worker_integration.rs b/crates/screenpipe-redact/tests/worker_integration.rsindex 6b7a8df99..67ff2fbae 100644--- a/crates/screenpipe-redact/tests/worker_integration.rs+++ b/crates/screenpipe-redact/tests/worker_integration.rs@@ -20,6 +20,7 @@ use screenpipe_redact::{worker::{column_keys, RedactColumns, TargetTable, Worker, WorkerConfig, ALL_TARGET_TABLES},Pseudonymizer, RedactError, RedactionMap, RedactionOutput, Redactor,};+use screenpipe_sqlite_coordinator::SqliteWritePool;use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};use sqlx::Row;use tempfile::TempDir;@@ -223,8 +224,24 @@ async fn worker_redacts_all_targets() {let worker = Worker::new(pool.clone(), redactor, cfg);let handle = worker.clone().spawn();- // Give the worker a moment to drain the queue.- tokio::time::sleep(Duration::from_millis(200)).await;+ wait_until("all target watermarks", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT \+ (SELECT COUNT(*) FROM frames \+ WHERE full_text IS NOT NULL AND full_text_redacted_at IS NULL) + \+ (SELECT COUNT(*) FROM frames \+ WHERE accessibility_text IS NOT NULL AND accessibility_redacted_at IS NULL) + \+ (SELECT COUNT(*) FROM audio_transcriptions WHERE redacted_at IS NULL) + \+ (SELECT COUNT(*) FROM ui_events WHERE redacted_at IS NULL) + \+ (SELECT COUNT(*) FROM elements \+ WHERE (text IS NOT NULL OR properties IS NOT NULL) AND redacted_at IS NULL)",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 0+ })+ .await;handle.abort();// Every single-column seeded row should now have its source column@@ -347,6 +364,70 @@ async fn worker_redacts_all_targets() {assert!(status.last_redacted_at.is_some());}+#[tokio::test]+async fn worker_writes_through_capability_with_physically_read_only_reader() {+ let (write_pool, temp_dir) = setup_db().await;+ sqlx::query("INSERT INTO frames (full_text) VALUES ('email [email protected]')")+ .execute(&write_pool)+ .await+ .unwrap();++ let read_options = SqliteConnectOptions::new()+ .filename(temp_dir.path().join("redact-worker-test.sqlite"))+ .read_only(true);+ let read_pool = SqlitePoolOptions::new()+ .max_connections(1)+ .connect_with(read_options)+ .await+ .unwrap();++ let direct_write = sqlx::query("UPDATE frames SET full_text = 'must fail'")+ .execute(&read_pool)+ .await;+ assert!(+ direct_write.is_err(),+ "reader unexpectedly accepted a write"+ );++ let redactor = Arc::new(RegexRedactor::new()) as Arc<dyn Redactor>;+ let cfg = WorkerConfig {+ batch_size: 1,+ idle_between_batches: Duration::from_millis(1),+ poll_interval: Duration::from_millis(20),+ tables: vec![TargetTable::FullText],+ ..test_worker_config()+ };+ let worker = Worker::new_with_writer(+ read_pool,+ SqliteWritePool::standalone(write_pool.clone()),+ redactor,+ cfg,+ );+ let handle = worker.spawn();++ wait_until("coordinated redaction write", || {+ let pool = write_pool.clone();+ async move {+ sqlx::query_scalar::<_, Option<i64>>(+ "SELECT full_text_redacted_at FROM frames WHERE id = 1",+ )+ .fetch_one(&pool)+ .await+ .ok()+ .flatten()+ .is_some()+ }+ })+ .await;+ handle.abort();++ let text: String = sqlx::query_scalar("SELECT full_text FROM frames WHERE id = 1")+ .fetch_one(&write_pool)+ .await+ .unwrap();+ assert_eq!(text, "email [EMAIL]");+}+#[tokio::test]async fn worker_skips_already_redacted_rows() {let (pool, _temp_dir) = setup_db().await;@@ -406,7 +487,16 @@ async fn worker_overwrites_source_columns_destructively() {let worker = Worker::new(pool.clone(), redactor, cfg);let handle = worker.clone().spawn();- tokio::time::sleep(Duration::from_millis(120)).await;+ wait_until("destructive full-text overwrite", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM frames WHERE id = 1 AND full_text_redacted_at IS NOT NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();let row = sqlx::query("SELECT full_text, full_text_redacted_at FROM frames WHERE id = 1")@@ -451,7 +541,16 @@ async fn worker_redacts_frames_full_text_search_surface() {let worker = Worker::new(pool.clone(), redactor, cfg);let handle = worker.clone().spawn();- tokio::time::sleep(Duration::from_millis(120)).await;+ wait_until("full-text search-surface watermark", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM frames WHERE id = 1 AND full_text_redacted_at IS NOT NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();// The issue's repro query: zero verbatim PII left on the search surface.@@ -519,7 +618,16 @@ async fn worker_writes_consistent_pseudonym_tokens() {let worker = Worker::new(pool.clone(), redactor, cfg);let handle = worker.clone().spawn();- tokio::time::sleep(Duration::from_millis(200)).await;+ wait_until("audio pseudonym watermarks", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM audio_transcriptions WHERE redacted_at IS NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 0+ })+ .await;handle.abort();let texts: Vec<String> =@@ -621,7 +729,18 @@ async fn frame_fulltext_redaction_propagates_to_accessibility_once() {};let worker = Worker::new(pool.clone(), redactor.clone(), cfg);let handle = worker.spawn();- tokio::time::sleep(Duration::from_millis(200)).await;+ wait_until("full-text and accessibility watermarks", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM frames WHERE id = 1 \+ AND full_text_redacted_at IS NOT NULL \+ AND accessibility_redacted_at IS NOT NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();let row = sqlx::query(@@ -715,7 +834,21 @@ async fn frame_fulltext_propagates_to_all_derived_copies_once() {..test_worker_config()};let handle = Worker::new(pool.clone(), redactor.clone(), cfg).spawn();- tokio::time::sleep(Duration::from_millis(200)).await;+ wait_until("all derived-copy watermarks", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM frames WHERE id = 1 \+ AND full_text_redacted_at IS NOT NULL \+ AND accessibility_tree_redacted_at IS NOT NULL \+ AND window_name_redacted_at IS NOT NULL \+ AND browser_url_redacted_at IS NOT NULL \+ AND text_json_redacted_at IS NOT NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();let row = sqlx::query(@@ -903,7 +1036,16 @@ async fn frame_fulltext_does_not_clobber_already_redacted_accessibility() {..test_worker_config()};let handle = Worker::new(pool.clone(), redactor.clone(), cfg).spawn();- tokio::time::sleep(Duration::from_millis(150)).await;+ wait_until("full-text watermark", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM frames WHERE id = 1 AND full_text_redacted_at IS NOT NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();let row = sqlx::query(@@ -1020,7 +1162,18 @@ async fn frame_fulltext_pseudonym_token_is_identical_across_columns() {..test_worker_config()};let handle = Worker::new(pool.clone(), redactor, cfg).spawn();- tokio::time::sleep(Duration::from_millis(150)).await;+ wait_until("pseudonymized frame watermarks", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM frames WHERE id = 1 \+ AND full_text_redacted_at IS NOT NULL \+ AND accessibility_redacted_at IS NOT NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();let row = sqlx::query("SELECT full_text, accessibility_text FROM frames WHERE id = 1")@@ -1138,7 +1291,18 @@ async fn frame_fulltext_falls_back_when_no_map() {..test_worker_config()};let handle = Worker::new(pool.clone(), redactor, cfg).spawn();- tokio::time::sleep(Duration::from_millis(200)).await;+ wait_until("fallback frame watermarks", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM frames WHERE id = 1 \+ AND full_text_redacted_at IS NOT NULL \+ AND accessibility_redacted_at IS NOT NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();let row = sqlx::query(@@ -1287,7 +1451,20 @@ async fn frame_fulltext_no_map_path_scrubs_all_derived_copies() {..test_worker_config()};let handle = Worker::new(pool.clone(), redactor, cfg).spawn();- tokio::time::sleep(Duration::from_millis(200)).await;+ wait_until("span-less derived-copy watermarks", || async {+ sqlx::query_scalar::<_, i64>(+ "SELECT COUNT(*) FROM frames WHERE id = 1 \+ AND accessibility_tree_redacted_at IS NOT NULL \+ AND window_name_redacted_at IS NOT NULL \+ AND full_text_redacted_at IS NOT NULL \+ AND browser_url_redacted_at IS NOT NULL",+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();let row = sqlx::query(@@ -1472,7 +1649,14 @@ async fn ui_events_ancestors_json_scrubbed_structure_preserved() {};let worker = Worker::new(pool.clone(), redactor, cfg);let handle = worker.clone().spawn();- tokio::time::sleep(Duration::from_millis(150)).await;+ wait_until("ui-events watermark", || async {+ sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM ui_events WHERE redacted_at IS NOT NULL")+ .fetch_one(&pool)+ .await+ .unwrap()+ == 1+ })+ .await;handle.abort();let (title, ancestors, redacted_at): (String, String, Option<i64>) = sqlx::query_as(diff --git a/crates/screenpipe-sqlite-coordinator/Cargo.toml b/crates/screenpipe-sqlite-coordinator/Cargo.tomlindex 08ef1bb10..5264b3745 100644--- a/crates/screenpipe-sqlite-coordinator/Cargo.toml+++ b/crates/screenpipe-sqlite-coordinator/Cargo.toml@@ -9,6 +9,7 @@ edition.workspace = true[dependencies]libsqlite3-sys = { workspace = true }+sqlx = { workspace = true }tokio = { workspace = true }tracing = { workspace = true }diff --git a/crates/screenpipe-sqlite-coordinator/src/lib.rs b/crates/screenpipe-sqlite-coordinator/src/lib.rsindex 1b26bfafc..bc0862c4a 100644--- a/crates/screenpipe-sqlite-coordinator/src/lib.rs+++ b/crates/screenpipe-sqlite-coordinator/src/lib.rs@@ -6,8 +6,10 @@ use std::collections::HashMap;use std::ffi::CStr;use std::path::{Path, PathBuf};use std::sync::{Arc, Mutex, OnceLock, Weak};+use std::time::Duration;-use tokio::sync::Semaphore;+use sqlx::SqlitePool;+use tokio::sync::{OwnedSemaphorePermit, Semaphore};pub const FIRST_WAL_RESET_SAFE_SQLITE: i32 = 3_051_003;@@ -18,6 +20,59 @@ static SQLITE_WRITE_LOCKS: OnceLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> =static SQLITE_HARD_FAULTS: OnceLock<Mutex<HashMap<PathBuf, i32>>> = OnceLock::new();static SQLITE_RUNTIME_CHECK: OnceLock<Result<SqliteRuntimeIdentity, String>> = OnceLock::new();+/// A cloneable capability for a SQLite pool whose callers must participate in+/// the process-wide single-writer protocol.+///+/// The underlying pool is available only through [`SqliteWritePermit`], so a+/// downstream worker must first join the same coordinator used by+/// DatabaseManager, checkpoints, and SecretStore.+#[derive(Clone)]+pub struct SqliteWritePool {+ pool: SqlitePool,+ coordinator: Arc<Semaphore>,+}++impl SqliteWritePool {+ pub fn new(pool: SqlitePool, coordinator: Arc<Semaphore>) -> Self {+ Self { pool, coordinator }+ }++ /// Build a standalone capability for tests or independently owned+ /// databases that do not share a DatabaseManager coordinator.+ pub fn standalone(pool: SqlitePool) -> Self {+ Self::new(pool, Arc::new(Semaphore::new(1)))+ }++ pub async fn lock(&self) -> Result<SqliteWritePermit, sqlx::Error> {+ let permit = match tokio::time::timeout(+ Duration::from_secs(10),+ Arc::clone(&self.coordinator).acquire_owned(),+ )+ .await+ {+ Ok(Ok(permit)) => permit,+ Ok(Err(_)) => return Err(sqlx::Error::PoolClosed),+ Err(_) => return Err(sqlx::Error::PoolTimedOut),+ };+ Ok(SqliteWritePermit {+ pool: self.pool.clone(),+ _permit: permit,+ })+ }+}++/// Proof that a caller currently owns the coordinated writer lane.+pub struct SqliteWritePermit {+ pool: SqlitePool,+ _permit: OwnedSemaphorePermit,+}++impl SqliteWritePermit {+ pub fn pool(&self) -> &SqlitePool {+ &self.pool+ }+}+#[derive(Debug)]pub struct SqliteRuntimeIdentity {pub version_number: i32,@@ -181,6 +236,25 @@ pub fn sqlite_write_lock(db_path: impl AsRef<Path>) -> Arc<Semaphore> {mod tests {use super::*;+ #[tokio::test]+ async fn write_capability_holds_coordinator_until_permit_drops() {+ let pool = SqlitePool::connect("sqlite::memory:")+ .await+ .expect("open sqlite");+ let coordinator = Arc::new(Semaphore::new(1));+ let writer = SqliteWritePool::new(pool, Arc::clone(&coordinator));++ let permit = writer.lock().await.expect("acquire writer");+ assert_eq!(coordinator.available_permits(), 0);+ sqlx::query("CREATE TABLE capability_test (id INTEGER PRIMARY KEY)")+ .execute(permit.pool())+ .await+ .expect("write through capability");++ drop(permit);+ assert_eq!(coordinator.available_permits(), 1);+ }+#[test]fn canonical_and_relative_paths_share_one_lock() {let dir = tempfile::tempdir().expect("tempdir");