YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: distinguish SQLite operational faults from corruption
Louis Beaumontc42a0c678092
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex f87e4c52d..21f657cac 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 39 git-mined regressions. See+The current app corpus contains 40 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest contributes 20 more.diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex fd5e434a3..fd9ce167f 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-09.1",+ "dataset_version": "2026-09-10.1","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1191,6 +1191,54 @@],"command": "cd packages/ai-gateway && bun test src/test/eval-input-limits.test.ts --timeout=10000"}+ },+ {+ "id": "sqlite-operational-fault-not-corruption",+ "name": "Keep SQLite operational faults distinct from corruption verdicts",+ "source": {+ "kind": "git_regression",+ "fix_commit": "968141d50e944a625230e3df60f243eb4be5f9d5",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/6967"+ },+ "base_ref": "db8bd7ee23c02cbfc0eed2f4b9d99c70fd4cfc2d",+ "oracle_ref": "968141d50e944a625230e3df60f243eb4be5f9d5",+ "oracle_paths": [+ "crates/screenpipe-sqlite-coordinator/src/lib.rs",+ "crates/screenpipe-sqlite-coordinator/src/quarantine.rs",+ "crates/screenpipe-sqlite-coordinator/src/verification.rs"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "crates/screenpipe-sqlite-coordinator/src/**",+ "crates/screenpipe-db/src/db/**"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "sqlite",+ "data-integrity",+ "availability",+ "quarantine",+ "false-positive"+ ],+ "prompt": "A single SQLite I/O failure or full-disk result currently creates corruption quarantine even when database damage has not been verified, unnecessarily routing a healthy database toward repair. Fix this boundary in the shared SQLite coordinator: retire the failed writer generation and retain a durable admission barrier, but do not declare corruption from an operational result alone. Preserve the original database bytes and physical identity. Pending faults must still block writes after process restart; repeated faults or recreating a writer gate must not reopen the failed generation. Preserve the first recorded fault, existing quarantine and malformed-metadata safety, ordinary contention and pool-pressure handling, and independent admission for unrelated databases. Keep the existing public fault-entry and coordination interfaces usable. This task covers incident classification and safe admission, not automatic repair or the complete recording-recovery lifecycle.",+ "agent_timeout_seconds": 1800,+ "grader_dependency_links": [+ {+ "source_path": "target",+ "destination_path": "target"+ }+ ],+ "grader": {+ "timeout_seconds": 600,+ "fixtures": [+ {+ "local_path": "graders/sqlite-operational-fault.rs",+ "destination_path": "crates/screenpipe-sqlite-coordinator/tests/eval_operational_fault.rs"+ }+ ],+ "command": "cargo test -p screenpipe-sqlite-coordinator --locked --test eval_operational_fault -- --test-threads=1"+ }}]}diff --git a/evals/coding-agent/graders/sqlite-operational-fault.rs b/evals/coding-agent/graders/sqlite-operational-fault.rsnew file mode 100644index 000000000..d67f57d05--- /dev/null+++ b/evals/coding-agent/graders/sqlite-operational-fault.rs@@ -0,0 +1,193 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_sqlite_coordinator::{+ latch_sqlite_error, latch_sqlite_hard_fault, persist_sqlite_quarantine,+ prepare_sqlite_quarantine_reserve, registered_sqlite_hard_fault, sqlite_file_identity,+ sqlite_hard_fault_code, sqlite_quarantine_exists, sqlite_quarantine_marker_path,+ sqlite_write_lock,+};+use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection, SqliteJournalMode};+use sqlx::Connection;+use std::path::Path;+use std::sync::Arc;++async fn seed_database(path: &Path) {+ let options = SqliteConnectOptions::new()+ .filename(path)+ .create_if_missing(true)+ .journal_mode(SqliteJournalMode::Delete);+ let mut connection = SqliteConnection::connect_with(&options).await.unwrap();+ sqlx::query("CREATE TABLE samples(id INTEGER PRIMARY KEY, value TEXT NOT NULL)")+ .execute(&mut connection)+ .await+ .unwrap();+ sqlx::query("INSERT INTO samples VALUES (1, 'synthetic alpha'), (2, 'synthetic beta')")+ .execute(&mut connection)+ .await+ .unwrap();+ connection.close().await.unwrap();+}++async fn fault_without_corruption_verdict(code: i32, reserve: bool) {+ let dir = tempfile::tempdir().unwrap();+ let db = dir.path().join("samples.sqlite");+ seed_database(&db).await;+ let bytes = std::fs::read(&db).unwrap();+ let identity = sqlite_file_identity(&db).unwrap();+ if reserve {+ prepare_sqlite_quarantine_reserve(&db).unwrap();+ }+ let old_gate = sqlite_write_lock(&db);+ assert!(!old_gate.is_closed());+ assert!(latch_sqlite_hard_fault(&db, code));+ assert!(old_gate.is_closed(), "faulted writers must remain stopped");+ assert_eq!(registered_sqlite_hard_fault(&db), Some(code));+ assert_eq!(std::fs::read(&db).unwrap(), bytes);+ assert_eq!(sqlite_file_identity(&db).unwrap(), identity);+ assert!(+ !sqlite_quarantine_exists(&db),+ "an unverified result code must not create corruption quarantine"+ );+}++#[tokio::test]+async fn generic_io_error_does_not_declare_corruption() {+ fault_without_corruption_verdict(10, false).await;+}++#[tokio::test]+async fn extended_io_error_does_not_declare_corruption() {+ fault_without_corruption_verdict(522, true).await;+}++#[tokio::test]+async fn full_disk_does_not_declare_corruption() {+ fault_without_corruption_verdict(13, true).await;+}++#[tokio::test]+async fn fault_admission_barrier_survives_process_restart() {+ const PHASE: &str = "SCREENPIPE_SYNTHETIC_FAULT_PHASE";+ const DATABASE: &str = "SCREENPIPE_SYNTHETIC_FAULT_DATABASE";+ if let Ok(phase) = std::env::var(PHASE) {+ let db = std::path::PathBuf::from(std::env::var(DATABASE).unwrap());+ if phase == "fault" {+ prepare_sqlite_quarantine_reserve(&db).unwrap();+ assert!(latch_sqlite_hard_fault(&db, 13));+ } else {+ assert_eq!(registered_sqlite_hard_fault(&db), Some(13));+ assert!(sqlite_write_lock(&db).is_closed());+ }+ return;+ }+ let dir = tempfile::tempdir().unwrap();+ let db = dir.path().join("samples.sqlite");+ seed_database(&db).await;+ let bytes = std::fs::read(&db).unwrap();+ let identity = sqlite_file_identity(&db).unwrap();+ for phase in ["fault", "restart"] {+ let output = std::process::Command::new(std::env::current_exe().unwrap())+ .args([+ "--exact",+ "fault_admission_barrier_survives_process_restart",+ ])+ .env(PHASE, phase)+ .env(DATABASE, &db)+ .output()+ .unwrap();+ assert!(output.status.success(), "durable fault gate phase failed");+ }+ assert_eq!(std::fs::read(&db).unwrap(), bytes);+ assert_eq!(sqlite_file_identity(&db).unwrap(), identity);+}++#[tokio::test]+async fn repeated_faults_never_reopen_old_or_recreated_gates() {+ let dir = tempfile::tempdir().unwrap();+ let db = dir.path().join("samples.sqlite");+ seed_database(&db).await;+ let gate = sqlite_write_lock(&db);+ let held = Arc::clone(&gate).acquire_owned().await.unwrap();+ assert!(latch_sqlite_hard_fault(&db, 522));+ assert!(!latch_sqlite_hard_fault(&db, 13));+ assert_eq!(registered_sqlite_hard_fault(&db), Some(522));+ drop(held);+ assert!(gate.is_closed());+ assert!(gate.try_acquire().is_err());+ drop(gate);+ assert!(sqlite_write_lock(&db).is_closed());+}++#[tokio::test]+async fn ordinary_contention_does_not_poison_writer_admission() {+ let dir = tempfile::tempdir().unwrap();+ let db = dir.path().join("samples.sqlite");+ seed_database(&db).await;+ let gate = sqlite_write_lock(&db);+ for code in [0, 5, 6, 19, 261] {+ assert!(!latch_sqlite_hard_fault(&db, code));+ }+ assert_eq!(registered_sqlite_hard_fault(&db), None);+ assert!(!gate.is_closed());+ assert!(gate.try_acquire().is_ok());+ assert!(!sqlite_quarantine_exists(&db));+}++#[tokio::test]+async fn pool_pressure_is_not_a_database_fault() {+ let dir = tempfile::tempdir().unwrap();+ let db = dir.path().join("samples.sqlite");+ seed_database(&db).await;+ for error in [sqlx::Error::PoolTimedOut, sqlx::Error::PoolClosed] {+ assert!(!latch_sqlite_error(&db, &error));+ }+ assert_eq!(registered_sqlite_hard_fault(&db), None);+ assert!(!sqlite_write_lock(&db).is_closed());+ assert_eq!(+ sqlite_hard_fault_code(&sqlx::Error::Protocol(+ "error returned from database: (code: 522) disk I/O error".into()+ )),+ Some(522)+ );+}++#[tokio::test]+async fn an_unrelated_database_keeps_its_writer_lane() {+ let dir = tempfile::tempdir().unwrap();+ let first = dir.path().join("first.sqlite");+ let second = dir.path().join("second.sqlite");+ seed_database(&first).await;+ seed_database(&second).await;+ let second_gate = sqlite_write_lock(&second);+ assert!(latch_sqlite_hard_fault(&first, 10));+ assert_eq!(registered_sqlite_hard_fault(&second), None);+ assert!(!second_gate.is_closed());+ assert!(second_gate.try_acquire().is_ok());+}++#[tokio::test]+async fn existing_quarantine_is_not_erased_or_ignored() {+ let dir = tempfile::tempdir().unwrap();+ let db = dir.path().join("samples.sqlite");+ seed_database(&db).await;+ persist_sqlite_quarantine(&db, Some(11), "synthetic existing incident").unwrap();+ let marker = sqlite_quarantine_marker_path(&db).unwrap();+ let before = std::fs::read(&marker).unwrap();+ assert_eq!(registered_sqlite_hard_fault(&db), Some(11));+ assert!(sqlite_write_lock(&db).is_closed());+ assert!(sqlite_quarantine_exists(&db));+ assert_eq!(std::fs::read(marker).unwrap(), before);+}++#[tokio::test]+async fn malformed_incident_metadata_remains_fail_closed() {+ let dir = tempfile::tempdir().unwrap();+ let db = dir.path().join("samples.sqlite");+ seed_database(&db).await;+ let marker = sqlite_quarantine_marker_path(&db).unwrap();+ std::fs::write(&marker, b"{incomplete").unwrap();+ assert_eq!(registered_sqlite_hard_fault(&db), Some(10));+ assert!(sqlite_write_lock(&db).is_closed());+ assert_eq!(std::fs::read(marker).unwrap(), b"{incomplete");+}