Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ci): preserve redaction test database across worker aborts

Louis Beaumontd5b9fe38e69d
diff --git a/crates/screenpipe-redact/tests/worker_integration.rs b/crates/screenpipe-redact/tests/worker_integration.rs
index 910e0142b..54938636c 100644
--- a/crates/screenpipe-redact/tests/worker_integration.rs
+++ b/crates/screenpipe-redact/tests/worker_integration.rs
@@ -8,6 +8,7 @@
//! overwritten with the redacted text and the corresponding
//! `*_redacted_at` timestamp is stamped.
+use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
@@ -19,8 +20,9 @@ use screenpipe_redact::{
worker::{column_keys, RedactColumns, TargetTable, Worker, WorkerConfig, ALL_TARGET_TABLES},
Pseudonymizer, RedactError, RedactionMap, RedactionOutput, Redactor,
};
-use sqlx::sqlite::SqlitePoolOptions;
+use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use sqlx::Row;
+use tempfile::TempDir;
/// Every column enabled — tests that want to verify full coverage opt in to
/// the optional columns (browser_url / element_name+description / url-field)
@@ -38,13 +40,18 @@ fn test_worker_config() -> WorkerConfig {
}
}
-async fn setup_db() -> sqlx::SqlitePool {
- // A plain `sqlite::memory:` database is private to each connection.
- // Keep this pool single-connection so the worker and assertions always
- // observe the schema and rows created by this test.
+async fn setup_db() -> (sqlx::SqlitePool, TempDir) {
+ // Workers are aborted after each assertion window. If an abort cancels a
+ // query, SQLx may discard that connection. A one-connection in-memory DB
+ // would disappear with it, so keep each test isolated in a temporary file.
+ let temp_dir = tempfile::tempdir().unwrap();
+ let db_path: PathBuf = temp_dir.path().join("redact-worker-test.sqlite");
+ let options = SqliteConnectOptions::new()
+ .filename(db_path)
+ .create_if_missing(true);
let pool = SqlitePoolOptions::new()
.max_connections(1)
- .connect("sqlite::memory:")
+ .connect_with(options)
.await
.unwrap();
@@ -109,7 +116,7 @@ async fn setup_db() -> sqlx::SqlitePool {
.await
.unwrap();
- pool
+ (pool, temp_dir)
}
/// Seed each target with a row containing PII the regex catches.
@@ -184,7 +191,7 @@ async fn seed(pool: &sqlx::SqlitePool) {
#[tokio::test]
async fn worker_redacts_all_targets() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
seed(&pool).await;
let redactor = Arc::new(RegexRedactor::new()) as Arc<dyn Redactor>;
@@ -325,7 +332,7 @@ async fn worker_redacts_all_targets() {
#[tokio::test]
async fn worker_skips_already_redacted_rows() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
// Frame 1 is already processed — source already redacted, redacted_at set.
sqlx::query(
"INSERT INTO frames (id, full_text, full_text_redacted_at) VALUES (1, '[EMAIL]', 1)",
@@ -357,7 +364,7 @@ async fn worker_skips_already_redacted_rows() {
#[tokio::test]
async fn worker_overwrites_source_columns_destructively() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
sqlx::query("INSERT INTO frames (id, full_text) VALUES (1, '[email protected] is the email')")
.execute(&pool)
.await
@@ -400,7 +407,7 @@ async fn worker_overwrites_source_columns_destructively() {
/// (`select count(*) from frames where full_text like '%canary%'` → 0).
#[tokio::test]
async fn worker_redacts_frames_full_text_search_surface() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
sqlx::query(
"INSERT INTO frames (full_text) VALUES ('contact canary [email protected] for access')",
)
@@ -451,7 +458,7 @@ async fn worker_redacts_frames_full_text_search_surface() {
/// raw value must be gone — no `token -> value` mapping is stored.
#[tokio::test]
async fn worker_writes_consistent_pseudonym_tokens() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
// Rows 1 & 2 share a secret; row 3 has a different one.
sqlx::query(
"INSERT INTO audio_transcriptions (transcription) VALUES ('key is sk-proj-AbCdEf123456GhIjKlMnOp today')",
@@ -560,7 +567,7 @@ impl Redactor for CountingPipeline {
/// once and `accessibility_text` was never redacted independently.
#[tokio::test]
async fn frame_fulltext_redaction_propagates_to_accessibility_once() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
// accessibility_text ⊆ full_text (full_text = accessibility || ocr),
// both carrying the same secret — mirrors how capture assembles them.
let acc = "AXStaticText[login sk-proj-AbCdEf123456GhIjKlMnOp]";
@@ -641,7 +648,7 @@ async fn frame_fulltext_redaction_propagates_to_accessibility_once() {
/// preserved), watermarks stamped, and detection ran exactly once.
#[tokio::test]
async fn frame_fulltext_propagates_to_all_derived_copies_once() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
let secret = "sk-proj-AbCdEf123456GhIjKlMnOp";
let tree = format!(
r#"[{{"role":"AXStaticText","text":"login {secret}","depth":0,"on_screen":true}},
@@ -775,7 +782,7 @@ async fn frame_fulltext_propagates_to_all_derived_copies_once() {
/// structured tree without persisting the synthetic context.
#[tokio::test]
async fn frame_fulltext_uses_accessibility_input_context_without_extra_model_pass() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
let tree = r#"[{"role":"AXTextField","text":"hunter2","value":"hunter2","placeholder":"API key","depth":1,"bounds":{"left":0.1,"top":0.2,"width":0.3,"height":0.04},"on_screen":true}]"#;
sqlx::query(
"INSERT INTO frames (id, full_text, accessibility_text, accessibility_tree_json) \
@@ -839,7 +846,7 @@ async fn frame_fulltext_uses_accessibility_input_context_without_extra_model_pas
/// prior run (watermark set) — and don't re-stamp it.
#[tokio::test]
async fn frame_fulltext_does_not_clobber_already_redacted_accessibility() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
sqlx::query(
"INSERT INTO frames (id, full_text, accessibility_text, accessibility_redacted_at) \
VALUES (1, 'key sk-proj-AbCdEf123456GhIjKlMnOp here', '[ALREADY]', 999)",
@@ -893,7 +900,7 @@ async fn frame_fulltext_does_not_clobber_already_redacted_accessibility() {
/// from a single detection, without mangling the text.
#[tokio::test]
async fn frame_fulltext_clean_frame_marks_both_done() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
sqlx::query(
"INSERT INTO frames (id, full_text, accessibility_text) \
VALUES (1, 'ordinary text\nmore ordinary text', 'ordinary text')",
@@ -950,7 +957,7 @@ async fn frame_fulltext_clean_frame_marks_both_done() {
/// columns stay correlatable (and propagation didn't re-detect).
#[tokio::test]
async fn frame_fulltext_pseudonym_token_is_identical_across_columns() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
let acc = "login sk-proj-AbCdEf123456GhIjKlMnOp now";
let full = format!("{acc}\nocr sk-proj-AbCdEf123456GhIjKlMnOp");
sqlx::query("INSERT INTO frames (id, full_text, accessibility_text) VALUES (1, ?, ?)")
@@ -995,7 +1002,7 @@ async fn frame_fulltext_pseudonym_token_is_identical_across_columns() {
/// `accessibility_text` is propagated, none re-detected.
#[tokio::test]
async fn frame_fulltext_each_frame_detected_once() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
for id in [1_i64, 2, 3] {
let acc = format!("frame {id} key sk-proj-AbCdEf123456GhIjKlMnOp");
let full = format!("{acc}\nocr line {id}");
@@ -1056,7 +1063,7 @@ async fn frame_fulltext_each_frame_detected_once() {
/// redacted — no silent data loss.
#[tokio::test]
async fn frame_fulltext_falls_back_when_no_map() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
let acc = "send to [email protected]";
let full = format!("{acc}\nocr [email protected]");
sqlx::query("INSERT INTO frames (id, full_text, accessibility_text) VALUES (1, ?, ?)")
@@ -1195,7 +1202,7 @@ async fn worker_disables_missing_table_and_keeps_reconciling_others() {
/// `process_frames_fulltext` / `redact_frame_derived_with_redactor`.
#[tokio::test]
async fn frame_fulltext_no_map_path_scrubs_all_derived_copies() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
let email = "[email protected]";
let tree = format!(
r#"[{{"role":"AXStaticText","text":"mail {email}","depth":0,"automation_id":"keepme"}}]"#
@@ -1283,7 +1290,7 @@ async fn frame_fulltext_no_map_path_scrubs_all_derived_copies() {
/// config is honored (user's "by default url is not processed" requirement).
#[tokio::test]
async fn default_columns_leave_optin_columns_untouched() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
let email = "[email protected]";
// Frame: full_text (core, on) + window_name (core, on) + browser_url (opt-in, off).
sqlx::query("INSERT INTO frames (id, full_text, window_name, browser_url) VALUES (1, ?, ?, ?)")
@@ -1370,7 +1377,7 @@ async fn default_columns_leave_optin_columns_untouched() {
/// defeat the redaction contract for clicks.
#[tokio::test]
async fn ui_events_ancestors_json_scrubbed_structure_preserved() {
- let pool = setup_db().await;
+ let (pool, _temp_dir) = setup_db().await;
sqlx::query(
"INSERT INTO ui_events (event_type, window_title, element_ancestors) VALUES ( \
'click', \