YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(redact): disable a missing target table instead of retrying it forever (#4315)
Louis Beaumontac94fa0e2607
diff --git a/crates/screenpipe-redact/src/worker/mod.rs b/crates/screenpipe-redact/src/worker/mod.rsindex f2b69b26b..ae7381b52 100644--- a/crates/screenpipe-redact/src/worker/mod.rs+++ b/crates/screenpipe-redact/src/worker/mod.rs@@ -203,6 +203,20 @@ impl Worker {}let mut corruption_logged = false;+ // A "no such table/column" error is non-transient and table-specific:+ // the schema this binary expects doesn't match the DB it opened, so+ // retrying that target can never succeed. This happens during version+ // skew — e.g. an older engine sharing a `db.sqlite` that a newer one+ // already migrated. The `ocr_text` table (retired 2026-06) hit this+ // exactly: a pre-retirement binary kept its `Ocr` target and spammed+ // `no such table: ocr_text` every 2s forever. Detect it, disable that+ // one target for the process lifetime, and keep reconciling the rest.+ fn is_missing_object<E: std::fmt::Display + ?Sized>(e: &E) -> bool {+ let msg = e.to_string().to_lowercase();+ msg.contains("no such table") || msg.contains("no such column")+ }+ let mut disabled: Vec<TargetTable> = Vec::new();+loop {if self.paused.load(std::sync::atomic::Ordering::SeqCst) {self.set_paused(true).await;@@ -219,6 +233,11 @@ impl Worker {let mut any_work = false;for table in &self.cfg.tables {+ // Permanently skip a target whose table/column isn't in this+ // schema — see `is_missing_object` and the Err arm below.+ if disabled.contains(table) {+ continue;+ }// Race the table work against shutdown so a long redact batch// doesn't hold us through tokio teardown.let batch_start = std::time::Instant::now();@@ -263,6 +282,23 @@ impl Worker {let mut s = self.status.lock().await;s.last_error = Some(e.to_string());}+ if is_missing_object(&e) {+ // Non-transient and scoped to this one target: the+ // table or a column it reads isn't in this schema+ // (binary/DB version skew). Retrying spins a core+ // and floods the log, so disable just this target+ // for the run and move on — the others reconcile+ // normally. Logged once per target via the disable.+ warn!(+ table = ?table,+ error = %e,+ "target table/column missing from this schema — disabling its \+ reconciliation for this run (binary predates a migration that \+ retired it?); upgrade screenpipe to clear this"+ );+ disabled.push(*table);+ continue;+ }if is_db_corruption(&e) {// Non-transient: the DB is corrupt and every table// shares it, so retrying now just spins a core.diff --git a/crates/screenpipe-redact/tests/worker_integration.rs b/crates/screenpipe-redact/tests/worker_integration.rsindex 59c3003d3..c5a0eccea 100644--- a/crates/screenpipe-redact/tests/worker_integration.rs+++ b/crates/screenpipe-redact/tests/worker_integration.rs@@ -766,3 +766,78 @@ async fn frame_fulltext_falls_back_when_no_map() {);assert!(!a.contains("[email protected]"), "raw email survived: {a:?}");}++/// A target table missing from the schema (binary/DB version skew — the+/// `ocr_text` retirement is the real-world case) must be disabled, not+/// retried forever. The worker should log once and keep reconciling the+/// other targets at full speed.+///+/// Repro shape: reconcile a missing target (`Elements` — no `elements`+/// table here) *before* a present one (`FullText`). The missing target's+/// error is non-transient; if the worker treated it as transient it would+/// sleep 2s before reaching `FullText` on every sweep, so `full_text`+/// wouldn't be redacted within the short window below. With the fix the+/// missing target is disabled immediately and `full_text` is redacted in+/// the same sweep.+#[tokio::test]+async fn worker_disables_missing_table_and_keeps_reconciling_others() {+ // Schema WITHOUT an `elements` table — mirrors an engine whose code+ // still targets a table this DB's schema no longer has.+ let pool = SqlitePoolOptions::new()+ .max_connections(2)+ .connect("sqlite::memory:")+ .await+ .unwrap();+ sqlx::query(+ r#"+ CREATE TABLE frames (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ full_text TEXT,+ full_text_redacted_at INTEGER,+ accessibility_text TEXT,+ accessibility_redacted_at INTEGER+ );+ "#,+ )+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query("INSERT INTO frames (id, full_text) VALUES (1, 'email [email protected] here')")+ .execute(&pool)+ .await+ .unwrap();++ let redactor = Arc::new(RegexRedactor::new()) as Arc<dyn Redactor>;+ let cfg = WorkerConfig {+ batch_size: 16,+ idle_between_batches: Duration::from_millis(1),+ poll_interval: Duration::from_millis(20),+ // Elements first (missing table) then FullText (present).+ tables: vec![TargetTable::Elements, TargetTable::FullText],+ ..Default::default()+ };+ let worker = Worker::new(pool.clone(), redactor, cfg);+ let handle = worker.clone().spawn();++ // 400ms ≪ the 2s transient backoff: only reachable if the missing+ // `elements` target was disabled rather than slept-on every sweep.+ tokio::time::sleep(Duration::from_millis(400)).await;+ handle.abort();++ let row = sqlx::query("SELECT full_text, full_text_redacted_at FROM frames WHERE id = 1")+ .fetch_one(&pool)+ .await+ .unwrap();+ let red: String = row.get(0);+ let when: Option<i64> = row.get(1);+ assert!(+ red.contains("[EMAIL]") && when.is_some(),+ "full_text must be redacted despite the missing `elements` table — \+ the missing target should be disabled, not block/spam the rotation (got {red:?})"+ );+ // NB: `last_error` is intentionally not asserted here — the successful+ // `FullText` pass that follows the missing-table error clears it back to+ // None, so it's racy by design. The timing-bounded redaction above is the+ // behavioural proof that the missing target was disabled rather than+ // retried on a 2s backoff ahead of `FullText`.+}