YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover compressed database feature upgrades
Louis Beaumont798ac0866fc6
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex d03e1fc16..b948e98ae 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -33,7 +33,20 @@ images, blanket paste refusal and missing-source classification. Fixtures anddependency links appear only when grading begins. This does not execute nativeclipboard delivery, image encoding, disk autosave, agent isolation or model trials.-The current app corpus contains 123 git-mined regressions. See+`app-compressed-pending-sqlx-upgrades` runs real temporary SQLite and compressed+stores through normal database startup. Six outcomes cover pending feature+migrations, rollback and retry, retained recordings and starred sessions, resident+revision/privacy hooks, and ordinary SQLite behavior. The historical parent fails+four upgrade outcomes and preserves two; the reference and current source pass+six. Run `bun test evals/coding-agent/calibrate-compressed-sqlx-upgrade.test.js`+for eight controls, including equivalent source, unused correct code, skipped+migrations, lost sessions, omitted privacy hooks and missing-source classification.+Fixtures appear only at grading; cold historical arms use the unchanged+180-second limit without build-cache links. This does not verify production-size+histories, process-kill recovery, managed migration authority, live recording,+deployed storage, execution isolation or model performance.++The current app corpus contains 124 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.diff --git a/evals/coding-agent/calibrate-compressed-sqlx-upgrade.test.js b/evals/coding-agent/calibrate-compressed-sqlx-upgrade.test.jsnew file mode 100644index 000000000..8685bc5b6--- /dev/null+++ b/evals/coding-agent/calibrate-compressed-sqlx-upgrade.test.js@@ -0,0 +1,62 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { createHash } from 'node:crypto';+import { classifyGraderError } from './grader-outcome.mjs';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'app-compressed-pending-sqlx-upgrades');+const git = (...args) => execFileSync('git', args, { cwd: repo, encoding: 'utf8', maxBuffer: 128 * 1024 * 1024 });+const setup = item.oracle_paths[0], schema = item.oracle_paths[1], readSchema = item.oracle_paths[2];+const files = ref => Object.fromEntries(item.oracle_paths.map(path => [path, git('show', `${ref}:${path}`)]));+const parent = files(item.base_ref), fixed = files(item.oracle_ref);+const fixture = readFileSync(join(import.meta.dir, 'graders/compressed-sqlx-upgrade.rs'));+const hash = bytes => createHash('sha256').update(bytes).digest('hex');+function change(source, path, from, to) {+ expect(source[path].split(from)).toHaveLength(2);+ return { ...source, [path]: source[path].replace(from, to) };+}+test('calibrate compressed upgrade outcomes with preservation and bypass controls', () => {+ const root = mkdtempSync(join(tmpdir(), 'compressed-upgrade-calibration-'));+ try {+ const archive = execFileSync('git', ['archive', item.base_ref, 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'crates', 'LICENSE.md'], { cwd: repo, maxBuffer: 128 * 1024 * 1024 });+ execFileSync('tar', ['-x', '-C', root], { input: archive });+ writeFileSync(join(root, 'crates/screenpipe-db/tests/eval_compressed_sqlx_upgrade.rs'), fixture);+ const controls = [+ ['parent', parent, 'fail'],+ ['reference', fixed, 'pass'],+ ['equivalent', change(fixed, setup, 'if !bootstrap_storage {', 'if bootstrap_storage == false {'), 'pass'],+ ['unused', parent, 'fail'],+ ['skip-migrations', change(fixed, setup, 'Self::sqlx_migrator().run(&db_manager.write_pool).await?;', '// Synthetic bypass: skip pending migrations.'), 'fail'],+ ['lose-existing-stars', change(fixed, setup, 'if !bootstrap_storage {', 'if !bootstrap_storage {\n sqlx::query("DELETE FROM starred_sessions").execute(&mut *conn).await?;'), 'fail'],+ ['skip-privacy-hooks', change(change(fixed, schema, 'super::read_schema::install_resident_hooks(&mut tx, &table).await?;', '// Synthetic bypass: no resident privacy repair.'), readSchema, 'install_resident_hooks(&mut tx, &table).await?;', '// Synthetic bypass: no initial resident privacy hooks.'), 'fail'],+ ['missing', { ...fixed, [readSchema]: null }, 'error'],+ ];+ for (const [name, source, expected] of controls) {+ for (const [path, body] of Object.entries(source)) {+ if (body === null) rmSync(join(root, path)); else writeFileSync(join(root, path), body);+ }+ const unused = join(root, 'unused-correct.rs');+ if (name === 'unused') writeFileSync(unused, fixed[setup]); else rmSync(unused, { force: true });+ // Calibration alone reuses its local target. Each test owns independent+ // temporary databases. This is not an evaluated agent workspace.+ const result = spawnSync('/bin/bash', ['-c', item.grader.command], { cwd: root, encoding: 'utf8', timeout: item.grader.timeout_seconds * 1000, maxBuffer: 8 * 1024 * 1024 });+ const errorKind = classifyGraderError(result);+ const observed = result.error || result.signal || errorKind ? 'error' : result.status === 0 ? 'pass' : 'fail';+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, status: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: errorKind, source_hashes: Object.fromEntries(Object.entries(source).map(([path, body]) => [path, body === null ? null : hash(body)])), fixture_sha256: hash(fixture), stdout: result.stdout, stderr: result.stderr }, null, 2));+ }+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);+ if (expected === 'pass') expect(result.stdout).toContain('6 passed; 0 failed');+ if (expected === 'fail') { expect(result.status).toBe(101); expect(result.stdout).toContain('test result: FAILED.'); }+ if (name === 'parent') expect(result.stdout).toContain('2 passed; 4 failed');+ if (name === 'lose-existing-stars') expect(result.stdout).toContain('starred_upgrade_preserves_existing_sessions_and_tracks_mutations ... FAILED');+ if (name === 'skip-privacy-hooks') expect(result.stdout).toContain('later_resident_tables_receive_storage_and_privacy_hooks ... FAILED');+ if (name === 'missing') expect(errorKind).toBe('rust_compile_error');+ }+ } finally { rmSync(root, { recursive: true, force: true }); }+}, 1_500_000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 07af85263..7e666290e 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5081,6 +5081,46 @@],"command": "bun --no-env-file test apps/screenpipe-app-tauri/eval-pipe-run-terminal.test.ts"}+ },+ {+ "id": "app-compressed-pending-sqlx-upgrades",+ "name": "Upgrade existing compressed databases without losing history",+ "source": {+ "kind": "git_regression",+ "fix_commit": "9d7d2cd3bd5cbb8b29d40a1b74af4cd606244d49"+ },+ "base_ref": "9123d79bd8fa117174c3a256f92873bc4b785e1b",+ "oracle_ref": "9d7d2cd3bd5cbb8b29d40a1b74af4cd606244d49",+ "oracle_paths": [+ "crates/screenpipe-db/src/db/setup.rs",+ "crates/screenpipe-db/src/storage/schema.rs",+ "crates/screenpipe-db/src/storage/read_schema.rs"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "crates/screenpipe-db/src/db/setup.rs",+ "crates/screenpipe-db/src/storage/",+ "crates/screenpipe-db/src/migrations/"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "migration",+ "data-preservation",+ "privacy"+ ],+ "prompt": "Existing compressed databases created before starred sessions were introduced reopen without applying the pending application schema migration. Make normal startup apply pending feature migrations and record successful completion once, preserving older recordings, exact sealed frame payloads, search results, existing starred sessions and their revisions. Star listing, timestamp membership (including empty input), saving and search ranges must work after restart; new captures must remain writable and searchable. Already-applied migrations must preserve existing data. Resident feature tables introduced after conversion must receive storage revision tracking for insert/update/delete and invalidate previously acquired read-snapshot tokens on deletion or redaction while preserving their admission across ordinary edits. Repair missing tracking hooks on existing resident tables without duplicating their effects across repeated startup. Preserve bulk storage behavior. A failed multi-statement feature migration must roll back its partial schema and completion record; after the conflict is removed, normal startup must retry successfully while preserving recorded history. Ordinary SQLite starred sessions and already-current compressed histories must retain their behavior. Exercise actual disposable databases and public storage methods; no particular Rust helper or implementation is required. This concerns upgrades of already-converted synthetic data. It does not authorize migration rollout, broadening recording or migration permissions, altering pause/cohort/stop controls, deleting retained recovery copies, contacting live services or publishing releases.",+ "agent_timeout_seconds": 1800,+ "grader": {+ "timeout_seconds": 180,+ "fixtures": [+ {+ "local_path": "graders/compressed-sqlx-upgrade.rs",+ "destination_path": "crates/screenpipe-db/tests/eval_compressed_sqlx_upgrade.rs"+ }+ ],+ "command": "cargo test --offline --locked -p screenpipe-db --features storage-fault-injection --test eval_compressed_sqlx_upgrade -- --test-threads=1"+ }}]}diff --git a/evals/coding-agent/graders/compressed-sqlx-upgrade.rs b/evals/coding-agent/graders/compressed-sqlx-upgrade.rsnew file mode 100644index 000000000..18fbb0460--- /dev/null+++ b/evals/coding-agent/graders/compressed-sqlx-upgrade.rs@@ -0,0 +1,349 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_db::storage::{Projection, StorageMode};+use screenpipe_db::{ContentType, DatabaseManager};+use sqlx::Connection;++async fn seed(db: &DatabaseManager, id: i64, text: Option<&str>, accessibility: Option<&str>) {+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ sqlx::query("INSERT INTO frames(id,timestamp,full_text,accessibility_text,text_json,accessibility_tree_json,app_name,window_name,device_name) VALUES(?,'2026-09-11T12:00:00Z',?,?,'[{\"text\":\"hello\",\"left\":0.5}]','{\"text\":\"hello\"}','Editor','notes','display')")+ .bind(id).bind(text).bind(accessibility).execute(&mut **tx.conn()).await.unwrap();+ tx.commit().await.unwrap();+}++async fn search(db: &DatabaseManager, query: &str) -> serde_json::Value {+ let results = db+ .search(+ query,+ ContentType::OCR,+ 100,+ 0,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ )+ .await+ .unwrap();+ serde_json::to_value(results).unwrap()+}++#[tokio::test]+async fn starred_upgrade_restores_older_hybrid_search_and_preserves_recordings() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ seed(&db, 1, Some("starred upgrade history"), None).await;+ db.seal_frame_payloads().await.unwrap();+ let before = db.frame_payloads(&[1], Projection::All).await.unwrap();+ // Reproduce a generation converted before starred sessions shipped. Only+ // mutate this disposable fixture, through its existing serialized writer.+ db.execute_raw_sql_write(+ "DROP TABLE starred_sessions;+ DELETE FROM _sqlx_migrations WHERE version=20261002190000;",+ )+ .await+ .unwrap();+ db.close().await;++ let start = "2026-09-11T12:00:00.000Z";+ let end = "2026-09-11T12:15:00.000Z";+ for reopen in 0..2 {+ let db = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ assert_eq!(db.storage_mode(), StorageMode::HybridParquetV1);+ db.verify_storage().await.unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM _sqlx_migrations WHERE version=20261002190000 AND success=1"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 1,+ );+ assert_eq!(+ before,+ db.frame_payloads(&[1], Projection::All).await.unwrap()+ );+ assert_eq!(search(&db, "history").await.as_array().unwrap().len(), 1);+ assert_eq!(+ db.list_starred_sessions(start, end, 10, 0)+ .await+ .unwrap()+ .len(),+ reopen,+ );+ // Search response assembly calls this even without a starred filter.+ assert_eq!(+ db.starred_timestamps(&[start.into(), end.into()])+ .await+ .unwrap(),+ vec![reopen == 1, false],+ );+ assert!(db.starred_timestamps(&[]).await.unwrap().is_empty());+ if reopen == 0 {+ assert!(db+ .save_starred_session("upgrade", start, end, false, 0, start)+ .await+ .unwrap());+ seed(&db, 2, Some("continued capture"), None).await;+ db.seal_frame_payloads().await.unwrap();+ } else {+ assert_eq!(+ db.get_starred_session("upgrade")+ .await+ .unwrap()+ .unwrap()+ .revision,+ 1+ );+ assert_eq!(+ db.starred_search_ranges(start, end).await.unwrap(),+ vec![(start.into(), end.into())]+ );+ assert_eq!(search(&db, "continued").await.as_array().unwrap().len(), 1);+ }+ db.close().await;+ }+}++#[tokio::test]+async fn starred_upgrade_preserves_existing_sessions_and_tracks_mutations() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ let start = "2026-09-11T12:00:00.000Z";+ let end = "2026-09-11T12:15:00.000Z";+ assert!(db+ .save_starred_session("existing", start, end, true, 0, start)+ .await+ .unwrap());+ // Already-applied SQLx migrations must preserve their data; missing+ // generic storage hooks can still be installed without replaying the DDL.+ db.execute_raw_sql_write(+ "DROP TRIGGER IF EXISTS hybrid_revision_starred_sessions_INSERT;+ DROP TRIGGER IF EXISTS hybrid_revision_starred_sessions_UPDATE;+ DROP TRIGGER IF EXISTS hybrid_revision_starred_sessions_DELETE;+ DROP TRIGGER IF EXISTS hybrid_read_revoke_delete_starred_sessions;",+ )+ .await+ .unwrap();+ db.close().await;++ for _ in 0..2 {+ let db = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ let session = db.get_starred_session("existing").await.unwrap().unwrap();+ assert_eq!(session.revision, 1);+ assert!(session.hd_requested);+ let before = db.storage_read_token().await.unwrap().revision;+ assert!(db+ .save_starred_session("temporary", end, "2026-09-11T12:30:00.000Z", false, 0, end)+ .await+ .unwrap());+ assert_eq!(db.storage_read_token().await.unwrap().revision, before + 1);+ assert!(db+ .save_starred_session("temporary", end, "2026-09-11T12:45:00.000Z", false, 1, end)+ .await+ .unwrap());+ assert_eq!(db.storage_read_token().await.unwrap().revision, before + 2);+ let revocation: i64 =+ sqlx::query_scalar("SELECT revision FROM _storage_revocation WHERE id=1")+ .fetch_one(&db.pool)+ .await+ .unwrap();+ db.execute_raw_sql_write("DELETE FROM starred_sessions WHERE id='temporary'")+ .await+ .unwrap();+ assert_eq!(db.storage_read_token().await.unwrap().revision, before + 3);+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT revision FROM _storage_revocation WHERE id=1")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ revocation + 1+ );+ db.close().await;+ }+}++#[tokio::test]+async fn later_resident_tables_receive_storage_and_privacy_hooks() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ // A future resident feature needs no feature-specific Rust upgrade.+ db.execute_raw_sql_write(+ "CREATE TABLE feature_notes(id INTEGER PRIMARY KEY, text TEXT, redacted_at TEXT);",+ )+ .await+ .unwrap();+ db.close().await;+ for _ in 0..2 {+ let db = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ let before = db.storage_read_token().await.unwrap().revision;+ db.execute_raw_sql_write("INSERT INTO feature_notes VALUES(1,'private',NULL)")+ .await+ .unwrap();+ assert_eq!(db.storage_read_token().await.unwrap().revision, before + 1);+ let token = db.read_snapshot(db.storage_read_token()).await.unwrap().unwrap();+ db.execute_raw_sql_write("UPDATE feature_notes SET text='ordinary edit' WHERE id=1")+ .await+ .unwrap();+ drop(token.admit(&db.pool).await.expect("ordinary edits preserve the read snapshot"));+ db.execute_raw_sql_write(+ "UPDATE feature_notes SET text='redacted',redacted_at='2026-10-03' WHERE id=1",+ )+ .await+ .unwrap();+ assert!(token.admit(&db.pool).await.is_err());+ drop(token);+ assert_eq!(db.storage_read_token().await.unwrap().revision, before + 3);+ let token = db.read_snapshot(db.storage_read_token()).await.unwrap().unwrap();+ db.execute_raw_sql_write("DELETE FROM feature_notes WHERE id=1")+ .await+ .unwrap();+ assert!(token.admit(&db.pool).await.is_err());+ drop(token);+ assert_eq!(db.storage_read_token().await.unwrap().revision, before + 4);+ db.close().await;+ }+}++#[tokio::test]+async fn pending_sqlx_migration_rolls_back_and_retries_on_compressed_storage() {+ use screenpipe_db::storage::StorageDescriptor;+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ seed(&db, 1, Some("migration retry history"), None).await;+ db.seal_frame_payloads().await.unwrap();+ let before = db.frame_payloads(&[1], Projection::All).await.unwrap();+ // Fail the second statement of the pending migration, after CREATE TABLE.+ db.execute_raw_sql_write(+ "DROP TABLE starred_sessions;+ DELETE FROM _sqlx_migrations WHERE version=20261002190000;+ CREATE INDEX idx_starred_sessions_start_end ON tags(name);",+ )+ .await+ .unwrap();+ db.close().await;+ let path = root.path().join("db.sqlite");+ let error = match DatabaseManager::new(path.to_str().unwrap(), Default::default()).await {+ Ok(db) => {+ db.close().await;+ panic!("conflicting index must fail startup");+ }+ Err(error) => error.to_string(),+ };+ assert!(error.contains("idx_starred_sessions_start_end"), "{error}");+ // Inspect only this closed, disposable index. SQLx must roll back both DDL+ // and its completion record before a corrected retry can proceed.+ let descriptor = StorageDescriptor::read(root.path()).unwrap().unwrap();+ let mut conn = sqlx::SqliteConnection::connect_with(+ &sqlx::sqlite::SqliteConnectOptions::new().filename(root.path().join(descriptor.index)),+ )+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM sqlite_master WHERE name='starred_sessions'"+ )+ .fetch_one(&mut conn)+ .await+ .unwrap(),+ 0+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM _sqlx_migrations WHERE version=20261002190000"+ )+ .fetch_one(&mut conn)+ .await+ .unwrap(),+ 0+ );+ sqlx::query("DROP INDEX idx_starred_sessions_start_end")+ .execute(&mut conn)+ .await+ .unwrap();+ conn.close().await.unwrap();+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.verify_storage().await.unwrap();+ assert_eq!(+ before,+ db.frame_payloads(&[1], Projection::All).await.unwrap()+ );+ assert_eq!(search(&db, "history").await.as_array().unwrap().len(), 1);+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM _sqlx_migrations WHERE version=20261002190000 AND success=1"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 1+ );+ db.close().await;+}++#[tokio::test]+async fn ordinary_sqlite_starred_sessions_remain_available() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new(root.path().join("db.sqlite").to_str().unwrap(), Default::default()).await.unwrap();+ let start="2026-09-11T12:00:00.000Z";+ let end="2026-09-11T12:15:00.000Z";+ assert!(db.save_starred_session("ordinary", start, end, false, 0, start).await.unwrap());+ assert_eq!(db.get_starred_session("ordinary").await.unwrap().unwrap().revision, 1);+ assert_eq!(db.starred_timestamps(&[start.into(),end.into()]).await.unwrap(),vec![true,false]);+ db.close().await;+}++#[tokio::test]+async fn existing_compressed_payloads_and_new_capture_survive_restart() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default()).await.unwrap();+ seed(&db, 1, Some("preserved café 東京"), None).await;+ db.seal_frame_payloads().await.unwrap();+ let before = db.frame_payloads(&[1], Projection::All).await.unwrap();+ db.close().await;+ let db = DatabaseManager::new(root.path().join("db.sqlite").to_str().unwrap(), Default::default()).await.unwrap();+ db.verify_storage().await.unwrap();+ assert_eq!(before,db.frame_payloads(&[1],Projection::All).await.unwrap());+ assert_eq!(search(&db,"preserved").await.as_array().unwrap().len(),1);+ seed(&db,2,Some("continued capture"),None).await;+ db.seal_frame_payloads().await.unwrap();+ assert_eq!(search(&db,"continued").await.as_array().unwrap().len(),1);+ db.close().await;+}