YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: verify abandoned SQLite read snapshot cleanup
louis98fff2f5b2fe
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 2e960b205..72a8db5cf 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 70 git-mined regressions. See+The current app corpus contains 71 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -729,3 +729,24 @@ is not proof of agent isolation, live authentication/payment/provider operation,transcription policy, playback or model improvement. Separately selected currentproduct route tests preserve transcription behavior; they are not hidden outcomesof this case.++## Abandoned SQLite read snapshots++`app-abandoned-read-wal-cleanup` runs the real database manager, SQLx pool and+SQLite WAL on synthetic temporary files. Six outcomes exercise dropped guards,+explicit release, aborted requests, active-reader consistency, ordinary release+and deadline interruption. Writes during and after cleanup must survive reopen+and an integrity check. The oracle is limited to the historical cancellable-read+implementation; it does not apply scheduler or native recovery changes.++Run `bun test evals/coding-agent/calibrate-wal-read-cleanup.test.js` with the locked+Rust dependencies available offline and a task-owned `target` cache. Calibration+checks the parent/reference, an unused corrected implementation, equivalent+cleanup logic and connection retirement, unconditional rollback, a stale cancellation handler and missing+source classification. Set `SCREENPIPE_EVAL_CALIBRATION_RECEIPTS` to retain+per-control logs. Hidden fixtures and build-cache links are added only for grading.++This case does not establish native recording resumption, scheduler policy,+migration authority, actual device histories, execution isolation or model gains.+Build failures remain infrastructure failures; inspect the executed assertion+logs before treating any failure as the intended historical defect.diff --git a/evals/coding-agent/calibrate-wal-read-cleanup.test.js b/evals/coding-agent/calibrate-wal-read-cleanup.test.jsnew file mode 100644index 000000000..eaa69f241--- /dev/null+++ b/evals/coding-agent/calibrate-wal-read-cleanup.test.js@@ -0,0 +1,77 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { classifyGraderError } from './grader-outcome.mjs';++const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'app-abandoned-read-wal-cleanup');+const root = mkdtempSync(join(tmpdir(), 'wal-read-calibration-'));+const source = 'crates/screenpipe-db/src/cancellable_query.rs';+const archive = execFileSync('git', ['archive', item.base_ref, 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'crates', 'LICENSE.md'], { cwd: repo, maxBuffer: 128 * 1024 * 1024 });+const parent = execFileSync('git', ['show', `${item.base_ref}:${source}`], { cwd: repo, encoding: 'utf8' });+const fixed = execFileSync('git', ['show', `${item.oracle_ref}:${source}`], { cwd: repo, encoding: 'utf8' });+afterAll(() => rmSync(root, { recursive: true, force: true }));++function replace(text, from, to) {+ expect(text.split(from)).toHaveLength(2);+ return text.replace(from, to);+}+function grade(name, text = fixed, extra = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ execFileSync('tar', ['-x', '-C', cwd], { input: archive });+ writeFileSync(join(cwd, source), text);+ mkdirSync(join(cwd, 'crates/screenpipe-db/tests'), { recursive: true });+ writeFileSync(join(cwd, 'crates/screenpipe-db/tests/eval_wal_read_cleanup.rs'), readFileSync(join(import.meta.dir, 'graders/wal-read-cleanup.rs')));+ symlinkSync(join(repo, 'target'), join(cwd, 'target'), 'dir');+ extra(cwd);+ const result = spawnSync('/bin/bash', ['-c', item.grader.command], { cwd, encoding: 'utf8', timeout: 600_000, maxBuffer: 8 * 1024 * 1024 });+ // Leave immutable per-control receipts when explicitly requested by the curator.+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {+ const directory = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(directory, { recursive: true });+ writeFileSync(join(directory, `${name}.json`), JSON.stringify({ command: item.grader.command, status: result.status, signal: result.signal, error: result.error?.message ?? null, stdout: result.stdout, stderr: result.stderr }, null, 2));+ }+ return result;+}+function pass(result) {+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull();+ expect(result.status).toBe(0); expect(result.stdout).toContain('6 passed; 0 failed');+}+function fail(result) {+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull();+ expect(result.status).toBe(101); expect(classifyGraderError(result)).toBeNull();+ expect(result.stdout).toContain('test result: FAILED.');+}+test('parent leaks three snapshots and preserves three neighbors', () => {+ const result = grade('parent', parent); fail(result);+ expect(result.stdout).toContain('3 passed; 3 failed');+ expect(result.stdout).toContain('abandoned reader must release its WAL snapshot');+}, 600_000);+test('reference preserves all six outcomes', () => pass(grade('reference')), 600_000);+test('unused correct implementation cannot mask the actual broken guard', () => {+ fail(grade('unused', parent, cwd => writeFileSync(join(cwd, 'unused-correct.rs'), fixed)));+}, 600_000);+test('equivalent transaction cleanup condition is accepted', () => {+ pass(grade('equivalent', replace(fixed, 'if in_transaction {', 'if !(!in_transaction) {')));+}, 600_000);+test('retiring the connection instead of rolling back is accepted', () => {+ const start = fixed.indexOf('async fn clean_and_return_connection(');+ const end = fixed.indexOf('\n#[cfg(test)]', start);+ expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start);+ pass(grade('retire-connection', fixed.slice(0, start) ++ 'async fn clean_and_return_connection(connection: PoolConnection<Sqlite>) -> Result<(), sqlx::Error> { connection.close().await }\n' + fixed.slice(end)));+}, 600_000);+test('unconditional rollback breaks ordinary release and is rejected', () => {+ fail(grade('unconditional', replace(fixed, 'if in_transaction {', 'if true {')));+}, 600_000);+test('lost progress-handler cleanup is rejected by preserved behavior', () => {+ fail(grade('stale-handler', replace(fixed, 'handle.remove_progress_handler();', '/* mutation: retain the cancelled handler */')));+}, 600_000);+test('missing implementation is a compile error, never regression evidence', () => {+ const result = grade('missing', fixed, cwd => rmSync(join(cwd, source)));+ expect(result.status).toBe(101); expect(classifyGraderError(result)).toBe('rust_compile_error');+ expect(result.stdout).not.toContain('test result: FAILED.');+}, 600_000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 37cc6261a..47d6df1d6 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-22.7",+ "dataset_version": "2026-09-22.8","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -2864,6 +2864,50 @@],"command": "node .eval-onboarding-sdk.mjs && cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts app/onboarding/page.test.tsx -t 'eval SDK assignment outcomes' --maxWorkers=1 --minWorkers=1"}+ },+ {+ "id": "app-abandoned-read-wal-cleanup",+ "name": "Release abandoned SQLite read snapshots without losing writes",+ "source": {+ "kind": "git_regression",+ "fix_commit": "2978ef82da358544450998f81dd5a687651cffee",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/7194"+ },+ "base_ref": "b15733e7b1a252f71a037a968d4f63af578aac47",+ "oracle_ref": "2978ef82da358544450998f81dd5a687651cffee",+ "oracle_paths": [+ "crates/screenpipe-db/src/cancellable_query.rs"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "crates/screenpipe-db/src/cancellable_query.rs",+ "crates/screenpipe-db/src/storage/snapshot.rs"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "sqlite",+ "cancellation",+ "durable-recording"+ ],+ "prompt": "A cancelled read request can leave a raw SQLite read transaction on a pooled connection. Later checkpoints remain pinned even after that request is gone. Fix cancellable read cleanup so dropping the guard, explicitly releasing it, or aborting its owning task releases the abandoned snapshot before returning pool capacity. A single-slot read pool must become usable within three seconds; a passive checkpoint must drain once the abandoned reader is gone. Writes made during the read and after cleanup must remain visible, survive database reopen and retain database integrity. Preserve an active reader's original snapshot while concurrent writes continue, ordinary non-transaction release, interruption of long SQLite work within three seconds when given a 20 ms deadline, and a clean progress handler for the next borrower. Keep the public cancellable-read interfaces usable. This task is limited to read-connection cleanup; it does not change scheduler restart policy, native recovery, user pause or migration authority.",+ "agent_timeout_seconds": 1800,+ "grader_dependency_links": [+ {+ "source_path": "target",+ "destination_path": "target"+ }+ ],+ "grader": {+ "timeout_seconds": 600,+ "fixtures": [+ {+ "local_path": "graders/wal-read-cleanup.rs",+ "destination_path": "crates/screenpipe-db/tests/eval_wal_read_cleanup.rs"+ }+ ],+ "command": "cargo test --offline --locked -p screenpipe-db --test eval_wal_read_cleanup -- --test-threads=1"+ }}]}diff --git a/evals/coding-agent/graders/wal-read-cleanup.rs b/evals/coding-agent/graders/wal-read-cleanup.rsnew file mode 100644index 000000000..cec8aafda--- /dev/null+++ b/evals/coding-agent/graders/wal-read-cleanup.rs@@ -0,0 +1,181 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_db::{is_sqlite_interrupt, CancellableReadConnection, DatabaseManager};+use std::time::{Duration, Instant};+use tokio_util::sync::CancellationToken;++async fn database() -> (tempfile::TempDir, DatabaseManager) {+ let dir = tempfile::tempdir().unwrap();+ let mut config = screenpipe_config::DbConfig::default();+ config.read_pool_max = 1;+ config.read_pool_min = 1;+ let db = DatabaseManager::new(dir.path().join("samples.sqlite").to_str().unwrap(), config)+ .await+ .unwrap();+ db.execute_raw_sql_write("CREATE TABLE samples(id INTEGER PRIMARY KEY, value TEXT)")+ .await+ .unwrap();+ db.wal_checkpoint().await.unwrap();+ (dir, db)+}++async fn reader(db: &DatabaseManager) -> CancellableReadConnection {+ CancellableReadConnection::acquire(+ &db.pool,+ Instant::now() + Duration::from_secs(5),+ CancellationToken::new(),+ )+ .await+ .unwrap()+}++async fn snapshot(db: &DatabaseManager) -> CancellableReadConnection {+ let mut connection = reader(db).await;+ sqlx::query("BEGIN")+ .execute(&mut *connection)+ .await+ .unwrap();+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM samples")+ .fetch_one(&mut *connection)+ .await+ .unwrap();+ assert_eq!(count, 0);+ connection+}++async fn abandoned(mode: &str) {+ let (dir, db) = database().await;+ let connection = snapshot(&db).await;+ db.execute_raw_sql_write("INSERT INTO samples(value) VALUES ('while reading')")+ .await+ .unwrap();+ let (_, pages, checkpointed) = db.wal_checkpoint().await.unwrap();+ assert!(+ pages > checkpointed,+ "control: an active snapshot must pin real WAL frames"+ );+ match mode {+ "drop" => drop(connection),+ "release" => connection.release().await.unwrap(),+ "abort" => {+ let (ready, began) = tokio::sync::oneshot::channel();+ let task = tokio::spawn(async move {+ let _connection = connection;+ ready.send(()).unwrap();+ std::future::pending::<()>().await;+ });+ began.await.unwrap();+ task.abort();+ assert!(task.await.unwrap_err().is_cancelled());+ }+ _ => unreachable!(),+ }+ // The single pool slot cannot be borrowed until guard cleanup finishes.+ let idle = tokio::time::timeout(Duration::from_secs(3), db.pool.acquire())+ .await+ .expect("cleanup must return pool capacity")+ .unwrap();+ db.execute_raw_sql_write("INSERT INTO samples(value) VALUES ('after cleanup')")+ .await+ .unwrap();+ let (busy, pages, checkpointed) = db.wal_checkpoint().await.unwrap();+ assert_eq!(busy, 0);+ assert_eq!(+ pages, checkpointed,+ "abandoned reader must release its WAL snapshot"+ );+ drop(idle);+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM samples")+ .fetch_one(&db.pool)+ .await+ .unwrap();+ assert_eq!(count, 2, "writes must remain visible after cleanup");+ db.close().await;+ let reopened = DatabaseManager::new(+ dir.path().join("samples.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM samples")+ .fetch_one(&reopened.pool)+ .await+ .unwrap();+ assert_eq!(count, 2, "writes must survive database reopen");+ let integrity: String = sqlx::query_scalar("PRAGMA integrity_check")+ .fetch_one(&reopened.pool)+ .await+ .unwrap();+ assert_eq!(integrity, "ok");+ reopened.close().await;+}++#[tokio::test]+async fn dropped_raw_transaction_releases_snapshot() {+ abandoned("drop").await;+}+#[tokio::test]+async fn explicit_release_releases_snapshot() {+ abandoned("release").await;+}+#[tokio::test]+async fn aborted_request_releases_snapshot() {+ abandoned("abort").await;+}++#[tokio::test]+async fn active_reader_retains_snapshot_while_writes_continue() {+ let (_dir, db) = database().await;+ let mut connection = snapshot(&db).await;+ db.execute_raw_sql_write("INSERT INTO samples(value) VALUES ('new capture')")+ .await+ .unwrap();+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM samples")+ .fetch_one(&mut *connection)+ .await+ .unwrap();+ assert_eq!(count, 0, "active reader must retain its original snapshot");+ let (_, pages, checkpointed) = db.wal_checkpoint().await.unwrap();+ assert!(pages > checkpointed);+ sqlx::query("ROLLBACK")+ .execute(&mut *connection)+ .await+ .unwrap();+ connection.release().await.unwrap();+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM samples")+ .fetch_one(&db.pool)+ .await+ .unwrap();+ assert_eq!(count, 1);+ db.close().await;+}++#[tokio::test]+async fn ordinary_release_keeps_pool_usable() {+ let (_dir, db) = database().await;+ reader(&db).await.release().await.unwrap();+ let sum: i64 = sqlx::query_scalar("WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<10000) SELECT sum(x) FROM n")+ .fetch_one(&db.pool).await.unwrap();+ assert_eq!(sum, 50_005_000);+ db.close().await;+}++#[tokio::test]+async fn deadline_interrupts_work_without_poisoning_next_reader() {+ let (_dir, db) = database().await;+ let mut connection = CancellableReadConnection::acquire(+ &db.pool,+ Instant::now() + Duration::from_millis(20),+ CancellationToken::new(),+ )+ .await+ .unwrap();+ let error = tokio::time::timeout(Duration::from_secs(3), sqlx::query_scalar::<_, i64>("WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<100000000) SELECT sum(x) FROM n").fetch_one(&mut *connection))+ .await.expect("SQLite execution must be interruptible").unwrap_err();+ assert!(is_sqlite_interrupt(&error));+ connection.release().await.unwrap();+ let sum: i64 = sqlx::query_scalar("WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<10000) SELECT sum(x) FROM n").fetch_one(&db.pool).await.unwrap();+ assert_eq!(sum, 50_005_000);+ db.close().await;+}