Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): coordinate redaction writes

Louis Beaumont7e7520597d3d
diff --git a/Cargo.lock b/Cargo.lock
index 66e03870b..f05b795d0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8749,6 +8749,7 @@ dependencies = [
"reqwest 0.13.3",
"screenpipe-resource",
"screenpipe-rfdetr-mlx",
+ "screenpipe-sqlite-coordinator",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -8871,6 +8872,7 @@ name = "screenpipe-sqlite-coordinator"
version = "0.4.34"
dependencies = [
"libsqlite3-sys",
+ "sqlx",
"tempfile",
"tokio",
"tracing",
diff --git a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
index 4c1f37ad4..bcdc4854a 100644
--- a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
+++ b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock
@@ -11509,6 +11509,7 @@ dependencies = [
"reqwest 0.13.3",
"screenpipe-resource",
"screenpipe-rfdetr-mlx",
+ "screenpipe-sqlite-coordinator",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -11615,6 +11616,7 @@ name = "screenpipe-sqlite-coordinator"
version = "0.4.34"
dependencies = [
"libsqlite3-sys",
+ "sqlx",
"tokio",
"tracing",
]
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index 58125e6b7..e9eeac641 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -1070,7 +1070,12 @@ impl ServerCore {
poll_interval: std::time::Duration::from_secs(5 * 60),
..Default::default()
};
- let _ = Worker::new(db.pool.clone(), placeholder, cfg)
+ let _ = Worker::new_with_writer(
+ db.pool.clone(),
+ db.coordinated_writer(),
+ placeholder,
+ cfg,
+ )
.spawn_with_shutdown(redact_shutdown.clone());
}
}
@@ -1146,7 +1151,12 @@ impl ServerCore {
tables: ALL_TARGET_TABLES.to_vec(),
..Default::default()
};
- let _ = Worker::new(db.pool.clone(), pipeline_arc, cfg)
+ let _ = Worker::new_with_writer(
+ db.pool.clone(),
+ db.coordinated_writer(),
+ pipeline_arc,
+ cfg,
+ )
.with_database_error_hook(redact_database_error_hook.clone())
.spawn_with_shutdown(redact_shutdown.clone());
} else {
@@ -1155,6 +1165,7 @@ impl ServerCore {
// launch. The worker is created inside the spawned
// task once the model is ready.
let pool = db.pool.clone();
+ let writer = db.coordinated_writer();
let shutdown = redact_shutdown.clone();
let labels = pii_labels.clone();
let pseudonymizer = pseudonymizer.clone();
@@ -1236,7 +1247,7 @@ impl ServerCore {
tables: ALL_TARGET_TABLES.to_vec(),
..Default::default()
};
- let _ = Worker::new(pool, pipeline_arc, cfg)
+ let _ = Worker::new_with_writer(pool, writer, pipeline_arc, cfg)
.with_database_error_hook(database_error_hook)
.spawn_with_shutdown(shutdown);
});
@@ -1253,6 +1264,7 @@ impl ServerCore {
use screenpipe_redact::ImageRedactor;
let pool = db.pool.clone();
+ let writer = db.coordinated_writer();
if use_tinfoil {
let detector = Arc::new(TinfoilImageRedactor::new(TinfoilImageConfig {
api_key: tinfoil_api_key.clone(),
@@ -1265,8 +1277,9 @@ impl ServerCore {
has_api_key = tinfoil_api_key.is_some(),
"starting async image-PII worker (backend=tinfoil)"
);
- let _ = ImageWorker::new(
+ let _ = ImageWorker::new_with_writer(
pool,
+ writer,
detector,
ImageWorkerConfig {
policy: ImageRedactionPolicy::from_labels(&pii_labels),
@@ -1293,8 +1306,9 @@ impl ServerCore {
"starting async image-PII worker (backend=local)"
);
let detector_arc = Arc::new(detector) as Arc<dyn ImageRedactor>;
- let _ = ImageWorker::new(
+ let _ = ImageWorker::new_with_writer(
pool,
+ writer,
detector_arc,
ImageWorkerConfig {
policy: ImageRedactionPolicy::from_labels(&labels),
diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rs
index e31308c4c..ffae5e9d9 100644
--- a/crates/screenpipe-db/src/db/setup.rs
+++ b/crates/screenpipe-db/src/db/setup.rs
@@ -82,6 +82,15 @@ fn preflight_existing_database_header(path: &Path) -> Result<(), SqlxError> {
}
impl DatabaseManager {
+ /// Give an independently owned worker access to the dedicated write pool
+ /// only while it participates in this database's single-writer protocol.
+ pub fn coordinated_writer(&self) -> screenpipe_sqlite_coordinator::SqliteWritePool {
+ screenpipe_sqlite_coordinator::SqliteWritePool::new(
+ self.write_pool.clone(),
+ Arc::clone(&self.write_semaphore),
+ )
+ }
+
pub async fn new(database_path: &str, config: DbConfig) -> Result<Self, sqlx::Error> {
screenpipe_sqlite_coordinator::verify_sqlite_runtime().map_err(SqlxError::Protocol)?;
debug!(
diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
index 220789125..d8a20194e 100644
--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
@@ -1942,7 +1942,9 @@ async fn main() -> anyhow::Result<()> {
..Default::default()
};
info!("starting pi session secret-scrub worker (--redact-agent-session-secrets)");
- let _ = Worker::new(db.pool.clone(), placeholder, cfg).spawn();
+ let _ =
+ Worker::new_with_writer(db.pool.clone(), db.coordinated_writer(), placeholder, cfg)
+ .spawn();
}
}
@@ -1989,6 +1991,7 @@ async fn main() -> anyhow::Result<()> {
// 4. Regex-only otherwise (still destructive — overwrites
// regex-redacted text into the source columns).
let pool = db.pool.clone();
+ let writer = db.coordinated_writer();
let labels = config.pii_redaction_labels.clone();
let database_error_hook = redact_database_error_hook.clone();
// Consistent-pseudonym tokens (issue #4206), opt-in. Loads (or
@@ -2112,7 +2115,7 @@ async fn main() -> anyhow::Result<()> {
columns,
..Default::default()
};
- let _worker_handle = Worker::new(pool, pipeline_arc, worker_cfg)
+ let _worker_handle = Worker::new_with_writer(pool, writer, pipeline_arc, worker_cfg)
.with_database_error_hook(database_error_hook)
.spawn();
// The worker runs for the lifetime of the engine. We don't
@@ -2211,9 +2214,14 @@ async fn main() -> anyhow::Result<()> {
policy: ImageRedactionPolicy::from_labels(&config.pii_redaction_labels),
..Default::default()
};
- let _img_handle = ImageWorker::new(db.pool.clone(), detector, cfg)
- .with_database_error_hook(redact_database_error_hook.clone())
- .spawn();
+ let _img_handle = ImageWorker::new_with_writer(
+ db.pool.clone(),
+ db.coordinated_writer(),
+ detector,
+ cfg,
+ )
+ .with_database_error_hook(redact_database_error_hook.clone())
+ .spawn();
}
}
diff --git a/crates/screenpipe-redact/Cargo.toml b/crates/screenpipe-redact/Cargo.toml
index c7004681e..589dc50e8 100644
--- a/crates/screenpipe-redact/Cargo.toml
+++ b/crates/screenpipe-redact/Cargo.toml
@@ -31,6 +31,7 @@ rand = { workspace = true }
regex = { workspace = true }
reqwest = { workspace = true }
screenpipe-resource = { path = "../screenpipe-resource" }
+screenpipe-sqlite-coordinator = { path = "../screenpipe-sqlite-coordinator" }
serde = { workspace = true }
sha2 = { workspace = true }
sha3 = "0.10"
diff --git a/crates/screenpipe-redact/src/image/worker.rs b/crates/screenpipe-redact/src/image/worker.rs
index 68146521f..862efe6f0 100644
--- a/crates/screenpipe-redact/src/image/worker.rs
+++ b/crates/screenpipe-redact/src/image/worker.rs
@@ -25,6 +25,7 @@ use std::sync::Arc;
use std::time::Duration;
use screenpipe_resource::ResourceGovernor;
+use screenpipe_sqlite_coordinator::SqliteWritePool;
use sqlx::{Row, SqlitePool};
use tokio::sync::{Mutex, Notify};
use tokio::task::JoinHandle;
@@ -93,6 +94,7 @@ pub struct ImageWorkerStatus {
#[derive(Clone)]
pub struct ImageWorker {
pool: SqlitePool,
+ writer: SqliteWritePool,
redactor: Arc<dyn ImageRedactor>,
cfg: ImageWorkerConfig,
status: Arc<Mutex<ImageWorkerStatus>>,
@@ -102,8 +104,19 @@ pub struct ImageWorker {
impl ImageWorker {
pub fn new(pool: SqlitePool, redactor: Arc<dyn ImageRedactor>, cfg: ImageWorkerConfig) -> Self {
+ let writer = SqliteWritePool::standalone(pool.clone());
+ Self::new_with_writer(pool, writer, redactor, cfg)
+ }
+
+ pub fn new_with_writer(
+ pool: SqlitePool,
+ writer: SqliteWritePool,
+ redactor: Arc<dyn ImageRedactor>,
+ cfg: ImageWorkerConfig,
+ ) -> Self {
Self {
pool,
+ writer,
redactor,
cfg,
status: Arc::new(Mutex::new(ImageWorkerStatus::default())),
@@ -365,6 +378,7 @@ impl ImageWorker {
}
async fn mark_redacted(&self, frame_id: i64) -> Result<(), sqlx::Error> {
+ let writer = self.writer.lock().await?;
sqlx::query(
r#"
UPDATE frames
@@ -373,7 +387,7 @@ impl ImageWorker {
"#,
)
.bind(frame_id)
- .execute(&self.pool)
+ .execute(writer.pool())
.await
.map(|_| ())
}
diff --git a/crates/screenpipe-redact/src/worker/mod.rs b/crates/screenpipe-redact/src/worker/mod.rs
index e7cf73e39..33a15880b 100644
--- a/crates/screenpipe-redact/src/worker/mod.rs
+++ b/crates/screenpipe-redact/src/worker/mod.rs
@@ -25,6 +25,7 @@ use std::sync::Arc;
use std::time::{Duration, SystemTime};
use screenpipe_resource::ResourceGovernor;
+use screenpipe_sqlite_coordinator::SqliteWritePool;
use sqlx::SqlitePool;
use tokio::sync::{Mutex, Notify};
use tokio::task::JoinHandle;
@@ -117,6 +118,7 @@ pub struct WorkerStatus {
#[derive(Clone)]
pub struct Worker {
pool: SqlitePool,
+ writer: SqliteWritePool,
redactor: Arc<dyn Redactor>,
cfg: WorkerConfig,
status: Arc<Mutex<WorkerStatus>>,
@@ -126,8 +128,19 @@ pub struct Worker {
impl Worker {
pub fn new(pool: SqlitePool, redactor: Arc<dyn Redactor>, cfg: WorkerConfig) -> Self {
+ let writer = SqliteWritePool::standalone(pool.clone());
+ Self::new_with_writer(pool, writer, redactor, cfg)
+ }
+
+ pub fn new_with_writer(
+ pool: SqlitePool,
+ writer: SqliteWritePool,
+ redactor: Arc<dyn Redactor>,
+ cfg: WorkerConfig,
+ ) -> Self {
Self {
pool,
+ writer,
redactor,
cfg,
status: Arc::new(Mutex::new(WorkerStatus::default())),
@@ -593,8 +606,9 @@ impl Worker {
None
};
+ let writer = self.writer.lock().await?;
tables::write_redacted_element(
- &self.pool,
+ writer.pool(),
row.id,
text_out.as_deref(),
props_out.as_deref(),
@@ -709,7 +723,8 @@ impl Worker {
}
for (row, redacted) in rows.iter().zip(outputs_by_row.iter()) {
- tables::write_redacted_ui_events(&self.pool, &active, row.id, redacted).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_ui_events(writer.pool(), &active, row.id, redacted).await?;
}
let n = rows.len() as u32;
@@ -776,8 +791,9 @@ impl Worker {
// they're warned + skipped inside the helper, unfixable by
// retry, so they must not wedge full_text).
writes += self.propagate_frame_derived(row, &map).await?;
+ let writer = self.writer.lock().await?;
tables::write_redacted(
- &self.pool,
+ writer.pool(),
TargetTable::FullText,
row.id,
&map.apply(&row.full_text),
@@ -802,8 +818,9 @@ impl Worker {
// accessibility_text is left to the Accessibility pass
// (it has its own model-pass fallback target).
let out = self.redactor.redact(&row.full_text).await?;
+ let writer = self.writer.lock().await?;
tables::write_redacted(
- &self.pool,
+ writer.pool(),
TargetTable::FullText,
row.id,
&out.redacted,
@@ -842,8 +859,9 @@ impl Worker {
if cols.accessibility_text {
if let Some(acc) = row.accessibility_text.as_deref() {
if !acc.is_empty() && row.accessibility_redacted_at.is_none() {
+ let writer = self.writer.lock().await?;
tables::write_redacted(
- &self.pool,
+ writer.pool(),
TargetTable::Accessibility,
row.id,
&map.apply(acc),
@@ -864,7 +882,8 @@ impl Worker {
&cols.a11y_json_fields(),
) {
Ok(Some(json)) => {
- tables::write_redacted_tree(&self.pool, row.id, &json).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_tree(writer.pool(), row.id, &json).await?;
writes += 1;
}
// Ok(None) means the map was empty — impossible here
@@ -884,7 +903,9 @@ impl Worker {
if cols.window_name {
if let Some(wn) = row.window_name.as_deref() {
if !wn.is_empty() && row.window_name_redacted_at.is_none() {
- tables::write_redacted_window_name(&self.pool, row.id, &map.apply(wn)).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_window_name(writer.pool(), row.id, &map.apply(wn))
+ .await?;
writes += 1;
}
}
@@ -896,7 +917,9 @@ impl Worker {
if cols.browser_url {
if let Some(url) = row.browser_url.as_deref() {
if !url.is_empty() && row.browser_url_redacted_at.is_none() {
- tables::write_redacted_browser_url(&self.pool, row.id, &map.apply(url)).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_browser_url(writer.pool(), row.id, &map.apply(url))
+ .await?;
writes += 1;
}
}
@@ -916,7 +939,8 @@ impl Worker {
if !tj.is_empty() && row.text_json_redacted_at.is_none() {
match crate::ocr_json::redact_ocr_text_json(tj, map) {
Ok(Some(json)) => {
- tables::write_redacted_text_json(&self.pool, row.id, &json).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_text_json(writer.pool(), row.id, &json).await?;
writes += 1;
}
// Ok(None) means the map was empty — impossible here
@@ -964,13 +988,15 @@ impl Worker {
.await
{
Ok(Some(json)) => {
- tables::write_redacted_tree(&self.pool, row.id, &json).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_tree(writer.pool(), row.id, &json).await?;
writes += 1;
}
// No redactable text → stamp the verbatim blob so the
// row isn't re-scanned for the tree forever.
Ok(None) => {
- tables::write_redacted_tree(&self.pool, row.id, tree).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_tree(writer.pool(), row.id, tree).await?;
writes += 1;
}
Err(crate::tree_json::TreeRedactError::Json(e)) => warn!(
@@ -991,7 +1017,9 @@ impl Worker {
if let Some(wn) = row.window_name.as_deref() {
if !wn.is_empty() && row.window_name_redacted_at.is_none() {
let out = self.redactor.redact(wn).await?;
- tables::write_redacted_window_name(&self.pool, row.id, &out.redacted).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_window_name(writer.pool(), row.id, &out.redacted)
+ .await?;
writes += 1;
}
}
@@ -1001,7 +1029,9 @@ impl Worker {
if let Some(url) = row.browser_url.as_deref() {
if !url.is_empty() && row.browser_url_redacted_at.is_none() {
let out = self.redactor.redact(url).await?;
- tables::write_redacted_browser_url(&self.pool, row.id, &out.redacted).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_browser_url(writer.pool(), row.id, &out.redacted)
+ .await?;
writes += 1;
}
}
@@ -1024,13 +1054,15 @@ impl Worker {
.await
{
Ok(Some(json)) => {
- tables::write_redacted_text_json(&self.pool, row.id, &json).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_text_json(writer.pool(), row.id, &json).await?;
writes += 1;
}
// No redactable text → stamp the verbatim blob so the
// row isn't re-scanned for text_json forever.
Ok(None) => {
- tables::write_redacted_text_json(&self.pool, row.id, tj).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted_text_json(writer.pool(), row.id, tj).await?;
writes += 1;
}
Err(crate::tree_json::TreeRedactError::Json(e)) => warn!(
@@ -1069,7 +1101,8 @@ impl Worker {
}
for (row, out) in rows.iter().zip(outputs.iter()) {
- tables::write_redacted(&self.pool, table, row.id, &out.redacted).await?;
+ let writer = self.writer.lock().await?;
+ tables::write_redacted(writer.pool(), table, row.id, &out.redacted).await?;
}
let n = rows.len() as u32;
diff --git a/crates/screenpipe-redact/tests/worker_integration.rs b/crates/screenpipe-redact/tests/worker_integration.rs
index 6b7a8df99..67ff2fbae 100644
--- a/crates/screenpipe-redact/tests/worker_integration.rs
+++ b/crates/screenpipe-redact/tests/worker_integration.rs
@@ -20,6 +20,7 @@ use screenpipe_redact::{
worker::{column_keys, RedactColumns, TargetTable, Worker, WorkerConfig, ALL_TARGET_TABLES},
Pseudonymizer, RedactError, RedactionMap, RedactionOutput, Redactor,
};
+use screenpipe_sqlite_coordinator::SqliteWritePool;
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use sqlx::Row;
use tempfile::TempDir;
@@ -223,8 +224,24 @@ async fn worker_redacts_all_targets() {
let worker = Worker::new(pool.clone(), redactor, cfg);
let handle = worker.clone().spawn();
- // Give the worker a moment to drain the queue.
- tokio::time::sleep(Duration::from_millis(200)).await;
+ wait_until("all target watermarks", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT \
+ (SELECT COUNT(*) FROM frames \
+ WHERE full_text IS NOT NULL AND full_text_redacted_at IS NULL) + \
+ (SELECT COUNT(*) FROM frames \
+ WHERE accessibility_text IS NOT NULL AND accessibility_redacted_at IS NULL) + \
+ (SELECT COUNT(*) FROM audio_transcriptions WHERE redacted_at IS NULL) + \
+ (SELECT COUNT(*) FROM ui_events WHERE redacted_at IS NULL) + \
+ (SELECT COUNT(*) FROM elements \
+ WHERE (text IS NOT NULL OR properties IS NOT NULL) AND redacted_at IS NULL)",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 0
+ })
+ .await;
handle.abort();
// Every single-column seeded row should now have its source column
@@ -347,6 +364,70 @@ async fn worker_redacts_all_targets() {
assert!(status.last_redacted_at.is_some());
}
+#[tokio::test]
+async fn worker_writes_through_capability_with_physically_read_only_reader() {
+ let (write_pool, temp_dir) = setup_db().await;
+ sqlx::query("INSERT INTO frames (full_text) VALUES ('email [email protected]')")
+ .execute(&write_pool)
+ .await
+ .unwrap();
+
+ let read_options = SqliteConnectOptions::new()
+ .filename(temp_dir.path().join("redact-worker-test.sqlite"))
+ .read_only(true);
+ let read_pool = SqlitePoolOptions::new()
+ .max_connections(1)
+ .connect_with(read_options)
+ .await
+ .unwrap();
+
+ let direct_write = sqlx::query("UPDATE frames SET full_text = 'must fail'")
+ .execute(&read_pool)
+ .await;
+ assert!(
+ direct_write.is_err(),
+ "reader unexpectedly accepted a write"
+ );
+
+ let redactor = Arc::new(RegexRedactor::new()) as Arc<dyn Redactor>;
+ let cfg = WorkerConfig {
+ batch_size: 1,
+ idle_between_batches: Duration::from_millis(1),
+ poll_interval: Duration::from_millis(20),
+ tables: vec![TargetTable::FullText],
+ ..test_worker_config()
+ };
+ let worker = Worker::new_with_writer(
+ read_pool,
+ SqliteWritePool::standalone(write_pool.clone()),
+ redactor,
+ cfg,
+ );
+ let handle = worker.spawn();
+
+ wait_until("coordinated redaction write", || {
+ let pool = write_pool.clone();
+ async move {
+ sqlx::query_scalar::<_, Option<i64>>(
+ "SELECT full_text_redacted_at FROM frames WHERE id = 1",
+ )
+ .fetch_one(&pool)
+ .await
+ .ok()
+ .flatten()
+ .is_some()
+ }
+ })
+ .await;
+ handle.abort();
+
+ let text: String = sqlx::query_scalar("SELECT full_text FROM frames WHERE id = 1")
+ .fetch_one(&write_pool)
+ .await
+ .unwrap();
+ assert_eq!(text, "email [EMAIL]");
+}
+
#[tokio::test]
async fn worker_skips_already_redacted_rows() {
let (pool, _temp_dir) = setup_db().await;
@@ -406,7 +487,16 @@ async fn worker_overwrites_source_columns_destructively() {
let worker = Worker::new(pool.clone(), redactor, cfg);
let handle = worker.clone().spawn();
- tokio::time::sleep(Duration::from_millis(120)).await;
+ wait_until("destructive full-text overwrite", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM frames WHERE id = 1 AND full_text_redacted_at IS NOT NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
let row = sqlx::query("SELECT full_text, full_text_redacted_at FROM frames WHERE id = 1")
@@ -451,7 +541,16 @@ async fn worker_redacts_frames_full_text_search_surface() {
let worker = Worker::new(pool.clone(), redactor, cfg);
let handle = worker.clone().spawn();
- tokio::time::sleep(Duration::from_millis(120)).await;
+ wait_until("full-text search-surface watermark", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM frames WHERE id = 1 AND full_text_redacted_at IS NOT NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
// The issue's repro query: zero verbatim PII left on the search surface.
@@ -519,7 +618,16 @@ async fn worker_writes_consistent_pseudonym_tokens() {
let worker = Worker::new(pool.clone(), redactor, cfg);
let handle = worker.clone().spawn();
- tokio::time::sleep(Duration::from_millis(200)).await;
+ wait_until("audio pseudonym watermarks", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM audio_transcriptions WHERE redacted_at IS NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 0
+ })
+ .await;
handle.abort();
let texts: Vec<String> =
@@ -621,7 +729,18 @@ async fn frame_fulltext_redaction_propagates_to_accessibility_once() {
};
let worker = Worker::new(pool.clone(), redactor.clone(), cfg);
let handle = worker.spawn();
- tokio::time::sleep(Duration::from_millis(200)).await;
+ wait_until("full-text and accessibility watermarks", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM frames WHERE id = 1 \
+ AND full_text_redacted_at IS NOT NULL \
+ AND accessibility_redacted_at IS NOT NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
let row = sqlx::query(
@@ -715,7 +834,21 @@ async fn frame_fulltext_propagates_to_all_derived_copies_once() {
..test_worker_config()
};
let handle = Worker::new(pool.clone(), redactor.clone(), cfg).spawn();
- tokio::time::sleep(Duration::from_millis(200)).await;
+ wait_until("all derived-copy watermarks", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM frames WHERE id = 1 \
+ AND full_text_redacted_at IS NOT NULL \
+ AND accessibility_tree_redacted_at IS NOT NULL \
+ AND window_name_redacted_at IS NOT NULL \
+ AND browser_url_redacted_at IS NOT NULL \
+ AND text_json_redacted_at IS NOT NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
let row = sqlx::query(
@@ -903,7 +1036,16 @@ async fn frame_fulltext_does_not_clobber_already_redacted_accessibility() {
..test_worker_config()
};
let handle = Worker::new(pool.clone(), redactor.clone(), cfg).spawn();
- tokio::time::sleep(Duration::from_millis(150)).await;
+ wait_until("full-text watermark", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM frames WHERE id = 1 AND full_text_redacted_at IS NOT NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
let row = sqlx::query(
@@ -1020,7 +1162,18 @@ async fn frame_fulltext_pseudonym_token_is_identical_across_columns() {
..test_worker_config()
};
let handle = Worker::new(pool.clone(), redactor, cfg).spawn();
- tokio::time::sleep(Duration::from_millis(150)).await;
+ wait_until("pseudonymized frame watermarks", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM frames WHERE id = 1 \
+ AND full_text_redacted_at IS NOT NULL \
+ AND accessibility_redacted_at IS NOT NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
let row = sqlx::query("SELECT full_text, accessibility_text FROM frames WHERE id = 1")
@@ -1138,7 +1291,18 @@ async fn frame_fulltext_falls_back_when_no_map() {
..test_worker_config()
};
let handle = Worker::new(pool.clone(), redactor, cfg).spawn();
- tokio::time::sleep(Duration::from_millis(200)).await;
+ wait_until("fallback frame watermarks", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM frames WHERE id = 1 \
+ AND full_text_redacted_at IS NOT NULL \
+ AND accessibility_redacted_at IS NOT NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
let row = sqlx::query(
@@ -1287,7 +1451,20 @@ async fn frame_fulltext_no_map_path_scrubs_all_derived_copies() {
..test_worker_config()
};
let handle = Worker::new(pool.clone(), redactor, cfg).spawn();
- tokio::time::sleep(Duration::from_millis(200)).await;
+ wait_until("span-less derived-copy watermarks", || async {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT COUNT(*) FROM frames WHERE id = 1 \
+ AND accessibility_tree_redacted_at IS NOT NULL \
+ AND window_name_redacted_at IS NOT NULL \
+ AND full_text_redacted_at IS NOT NULL \
+ AND browser_url_redacted_at IS NOT NULL",
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
let row = sqlx::query(
@@ -1472,7 +1649,14 @@ async fn ui_events_ancestors_json_scrubbed_structure_preserved() {
};
let worker = Worker::new(pool.clone(), redactor, cfg);
let handle = worker.clone().spawn();
- tokio::time::sleep(Duration::from_millis(150)).await;
+ wait_until("ui-events watermark", || async {
+ sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM ui_events WHERE redacted_at IS NOT NULL")
+ .fetch_one(&pool)
+ .await
+ .unwrap()
+ == 1
+ })
+ .await;
handle.abort();
let (title, ancestors, redacted_at): (String, String, Option<i64>) = sqlx::query_as(
diff --git a/crates/screenpipe-sqlite-coordinator/Cargo.toml b/crates/screenpipe-sqlite-coordinator/Cargo.toml
index 08ef1bb10..5264b3745 100644
--- a/crates/screenpipe-sqlite-coordinator/Cargo.toml
+++ b/crates/screenpipe-sqlite-coordinator/Cargo.toml
@@ -9,6 +9,7 @@ edition.workspace = true
[dependencies]
libsqlite3-sys = { workspace = true }
+sqlx = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
diff --git a/crates/screenpipe-sqlite-coordinator/src/lib.rs b/crates/screenpipe-sqlite-coordinator/src/lib.rs
index 1b26bfafc..bc0862c4a 100644
--- a/crates/screenpipe-sqlite-coordinator/src/lib.rs
+++ b/crates/screenpipe-sqlite-coordinator/src/lib.rs
@@ -6,8 +6,10 @@ use std::collections::HashMap;
use std::ffi::CStr;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock, Weak};
+use std::time::Duration;
-use tokio::sync::Semaphore;
+use sqlx::SqlitePool;
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
pub const FIRST_WAL_RESET_SAFE_SQLITE: i32 = 3_051_003;
@@ -18,6 +20,59 @@ static SQLITE_WRITE_LOCKS: OnceLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> =
static SQLITE_HARD_FAULTS: OnceLock<Mutex<HashMap<PathBuf, i32>>> = OnceLock::new();
static SQLITE_RUNTIME_CHECK: OnceLock<Result<SqliteRuntimeIdentity, String>> = OnceLock::new();
+/// A cloneable capability for a SQLite pool whose callers must participate in
+/// the process-wide single-writer protocol.
+///
+/// The underlying pool is available only through [`SqliteWritePermit`], so a
+/// downstream worker must first join the same coordinator used by
+/// DatabaseManager, checkpoints, and SecretStore.
+#[derive(Clone)]
+pub struct SqliteWritePool {
+ pool: SqlitePool,
+ coordinator: Arc<Semaphore>,
+}
+
+impl SqliteWritePool {
+ pub fn new(pool: SqlitePool, coordinator: Arc<Semaphore>) -> Self {
+ Self { pool, coordinator }
+ }
+
+ /// Build a standalone capability for tests or independently owned
+ /// databases that do not share a DatabaseManager coordinator.
+ pub fn standalone(pool: SqlitePool) -> Self {
+ Self::new(pool, Arc::new(Semaphore::new(1)))
+ }
+
+ pub async fn lock(&self) -> Result<SqliteWritePermit, sqlx::Error> {
+ let permit = match tokio::time::timeout(
+ Duration::from_secs(10),
+ Arc::clone(&self.coordinator).acquire_owned(),
+ )
+ .await
+ {
+ Ok(Ok(permit)) => permit,
+ Ok(Err(_)) => return Err(sqlx::Error::PoolClosed),
+ Err(_) => return Err(sqlx::Error::PoolTimedOut),
+ };
+ Ok(SqliteWritePermit {
+ pool: self.pool.clone(),
+ _permit: permit,
+ })
+ }
+}
+
+/// Proof that a caller currently owns the coordinated writer lane.
+pub struct SqliteWritePermit {
+ pool: SqlitePool,
+ _permit: OwnedSemaphorePermit,
+}
+
+impl SqliteWritePermit {
+ pub fn pool(&self) -> &SqlitePool {
+ &self.pool
+ }
+}
+
#[derive(Debug)]
pub struct SqliteRuntimeIdentity {
pub version_number: i32,
@@ -181,6 +236,25 @@ pub fn sqlite_write_lock(db_path: impl AsRef<Path>) -> Arc<Semaphore> {
mod tests {
use super::*;
+ #[tokio::test]
+ async fn write_capability_holds_coordinator_until_permit_drops() {
+ let pool = SqlitePool::connect("sqlite::memory:")
+ .await
+ .expect("open sqlite");
+ let coordinator = Arc::new(Semaphore::new(1));
+ let writer = SqliteWritePool::new(pool, Arc::clone(&coordinator));
+
+ let permit = writer.lock().await.expect("acquire writer");
+ assert_eq!(coordinator.available_permits(), 0);
+ sqlx::query("CREATE TABLE capability_test (id INTEGER PRIMARY KEY)")
+ .execute(permit.pool())
+ .await
+ .expect("write through capability");
+
+ drop(permit);
+ assert_eq!(coordinator.available_permits(), 1);
+ }
+
#[test]
fn canonical_and_relative_paths_share_one_lock() {
let dir = tempfile::tempdir().expect("tempdir");