YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover Unicode storage migration outcomes
louisac7e1965f624
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 47674a3fa..b6e99a47d 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1184,3 +1184,27 @@ which can capture fetch at initialization. Graders and dependency links areinstalled only after the trajectory. These checks do not establish JWTcryptography, live inference, concurrent spend safety, complete current billingintegration, agent isolation or model performance.++## Unicode migration search probes++`app-migration-unicode-search-probe` converts temporary synthetic SQLite histories+through the actual storage API. Seven outcomes cover control characters, diagram+glyphs, non-Latin text, symbol-only and ordinary histories, missing-index refusal,+low-disk refusal and external-reader refusal followed by explicit retry. The+historical parent fails three conversion outcomes and preserves four; the fix+passes all seven. Reopened payloads, search results, frame counts, completion state+and storage verification are checked without requiring a tokenization helper.++Run `bun test evals/coding-agent/calibrate-migration-unicode.test.js` for the+parent/reference, equivalent implementation, unused fix, skipped search checks,+lost payloads, fabricated success and missing-source controls. Calibration uses+an ordinary task-local target directory and independent temporary databases.+The corpus case has no dependency or build-cache links, and its fixture is+installed only when grading starts. Its offline Cargo command has a 180-second+limit. Compilation and setup errors cannot establish the intended regression.++This case tests explicitly requested offline conversion correctness. It does not+establish native rollout authority, cohort/stop controls, recovery-copy deletion+policy, all interruption paths, live recording continuity, execution isolation+or model performance. See [the rollout review](MIGRATION-ROLLOUT-REVIEW.md) for the+separate authority requirements.diff --git a/evals/coding-agent/calibrate-migration-unicode.test.js b/evals/coding-agent/calibrate-migration-unicode.test.jsnew file mode 100644index 000000000..3b352675a--- /dev/null+++ b/evals/coding-agent/calibrate-migration-unicode.test.js@@ -0,0 +1,62 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { createHash } from 'node:crypto';+import { classifyGraderError } from './grader-outcome.mjs';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'app-migration-unicode-search-probe');+const source = 'crates/screenpipe-db/src/storage/lifecycle.rs';+const git = (...args) => execFileSync('git', args, { cwd: repo, encoding: 'utf8', maxBuffer: 128 * 1024 * 1024 });+const parent = git('show', `${item.base_ref}:${source}`);+const fixed = git('show', `${item.oracle_ref}:${source}`);+const fixture = readFileSync(join(import.meta.dir, 'graders/migration-unicode.rs'));+const hash = bytes => createHash('sha256').update(bytes).digest('hex');+function replace(text, from, to) { expect(text.split(from)).toHaveLength(2); return text.replace(from, to); }+function term(body) {+ const start = fixed.indexOf('fn migration_search_term(text: &str) -> Option<String> {');+ const end = fixed.indexOf('\nasync fn resume_legacy_migration(', start);+ expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start);+ return fixed.slice(0, start) + `fn migration_search_term(text: &str) -> Option<String> { ${body} }\n` + fixed.slice(end);+}+test('calibrate actual migration outcomes with equivalent and broken implementations', () => {+ const root = mkdtempSync(join(tmpdir(), 'migration-unicode-calibration-'));+ try {+ const archive = execFileSync('git', ['archive', item.base_ref, 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'crates', 'LICENSE.md'], { cwd: repo, maxBuffer: 128 * 1024 * 1024 });+ execFileSync('tar', ['-x', '-C', root], { input: archive });+ writeFileSync(join(root, 'crates/screenpipe-db/tests/eval_migration_unicode.rs'), fixture);+ const controls = [+ ['parent', parent, 'fail'],+ ['reference', fixed, 'pass'],+ ['equivalent', term('text.split(|c: char| !c.is_alphanumeric()).find(|word| word.len() > 2).map(|word| format!("\\\"{word}\\\""))'), 'pass'],+ ['unused', parent, 'fail'],+ ['skip-search', term('let _ = text; None'), 'fail'],+ ['lost-payloads', replace(fixed, 'let frozen = source.begin_immediate_with_retry().await?;\n verify_integrity', 'source.execute_raw_sql_write("UPDATE frames SET full_text = \'\'").await?;\n let frozen = source.begin_immediate_with_retry().await?;\n verify_integrity'), 'fail'],+ ['blanket-success', replace(fixed, 'migrate_with_progress(root, config, options, |_| {}).await', 'let _ = (root, config, options); Ok(serde_json::from_value(serde_json::json!({"database_id":"synthetic","generation":"synthetic","frames":2,"tables":[],"source_bytes":0,"index_bytes":0,"payload_bytes":0})).unwrap())'), 'fail'],+ ['missing', null, 'error'],+ ];+ for (const [name, body, expected] of controls) {+ if (body === null) rmSync(join(root, source)); else writeFileSync(join(root, source), body);+ const unused = join(root, 'unused-correct.rs');+ if (name === 'unused') writeFileSync(unused, fixed); else rmSync(unused, { force: true });+ // Only grader calibration reuses this ordinary local target directory.+ // Each test creates independent temporary databases. No cache links or agent run.+ const result = spawnSync('/bin/bash', ['-c', item.grader.command], { cwd: root, encoding: 'utf8', timeout: item.grader.timeout_seconds * 1000, maxBuffer: 8 * 1024 * 1024 });+ const errorKind = classifyGraderError(result);+ const observed = result.error || result.signal || errorKind ? 'error' : result.status === 0 ? 'pass' : 'fail';+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, status: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: errorKind, source_sha256: body === null ? null : hash(body), fixture_sha256: hash(fixture), stdout: result.stdout, stderr: result.stderr }, null, 2));+ }+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);+ if (expected === 'pass') expect(result.stdout).toContain('7 passed; 0 failed');+ if (expected === 'fail') { expect(result.status).toBe(101); expect(result.stdout).toContain('test result: FAILED.'); }+ if (name === 'parent') { expect(result.stdout).toContain('4 passed; 3 failed'); expect(result.stdout).toContain('unterminated string'); }+ if (name === 'skip-search') expect(result.stdout).toContain('genuine_missing_index_is_refused_without_discarding_history ... FAILED');+ if (name === 'missing') expect(errorKind).toBe('rust_compile_error');+ }+ } finally { rmSync(root, { recursive: true, force: true }); }+}, 1_500_000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 1545ffe45..c95a9f0e2 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -4112,6 +4112,41 @@],"command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" test src/test/eval-trusted-runner.test.ts"}+ },+ {+ "id": "app-migration-unicode-search-probe",+ "name": "Preserve Unicode history during explicitly requested storage conversion",+ "source": {+ "kind": "git_regression",+ "fix_commit": "72e09b7dd7425fcb0387715d42782704f9de677a"+ },+ "base_ref": "1a8e311dc8b82725449cd59ae80f814d03bf5c62",+ "oracle_ref": "72e09b7dd7425fcb0387715d42782704f9de677a",+ "oracle_paths": [+ "crates/screenpipe-db/src/storage/lifecycle.rs"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "crates/screenpipe-db/src/storage/lifecycle.rs"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "migration",+ "data-preservation"+ ],+ "prompt": "An explicitly requested offline storage conversion fails on otherwise valid histories containing control characters, diagram glyphs or non-Latin text. Make these histories convert successfully, including symbol-only histories, while retaining exact frame payloads and search results after reopening the converted store. Preserve ordinary-text conversion, frame counts, a verified completion receipt, absence of an unfinished journal, and storage-integrity verification. Do not silently bypass genuine missing source search-index errors: refuse those conversions while retaining the original frame contents and leaving no success receipt or unfinished journal. Low disk headroom and external readers must likewise preserve the original history on refusal; once the external reader releases its transaction, explicit retry must succeed with payloads intact. Exercise actual temporary SQLite histories and reopened storage. No particular helper, token-extraction algorithm or internal call path is required. This task concerns conversion correctness only; it does not authorize automatic migration, broaden recording or migration permissions, alter pause/cohort/stop controls, delete retained recovery copies, contact live services, or publish releases. Preserve public storage interfaces.",+ "agent_timeout_seconds": 1800,+ "grader": {+ "timeout_seconds": 180,+ "fixtures": [+ {+ "local_path": "graders/migration-unicode.rs",+ "destination_path": "crates/screenpipe-db/tests/eval_migration_unicode.rs"+ }+ ],+ "command": "cargo test --offline --locked -p screenpipe-db --test eval_migration_unicode -- --test-threads=1"+ }}]}diff --git a/evals/coding-agent/graders/migration-unicode.rs b/evals/coding-agent/graders/migration-unicode.rsnew file mode 100644index 000000000..b92dfffe2--- /dev/null+++ b/evals/coding-agent/graders/migration-unicode.rs@@ -0,0 +1,102 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+use screenpipe_db::{storage::{migrate, MigrationOptions, Projection, StorageMode}, DatabaseManager};++async fn seed(root: &std::path::Path, texts: [&str; 2]) -> DatabaseManager {+ let db = DatabaseManager::new(root.join("db.sqlite").to_str().unwrap(), Default::default()).await.unwrap();+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ for (index, text) in texts.iter().enumerate() {+ sqlx::query("INSERT INTO frames(id,timestamp,full_text) VALUES(?,'2026-01-01T12:00:00Z',?)")+ .bind(index as i64 + 1).bind(text).execute(&mut **tx.conn()).await.unwrap();+ }+ tx.commit().await.unwrap();+ db+}++async fn converted(texts: [&str; 2]) {+ let root = tempfile::tempdir().unwrap();+ let db = seed(root.path(), texts).await;+ let before = db.frame_payloads(&[1, 2], Projection::All).await.unwrap();+ let mut searches = Vec::new();+ for term in ["foo", "שלום", "café", "東京", "ordinary", "absent"] {+ let ids: Vec<i64> = sqlx::query_scalar("SELECT rowid FROM frames_fts WHERE frames_fts MATCH ? ORDER BY rowid")+ .bind(term).fetch_all(&db.pool).await.unwrap();+ searches.push((term, ids));+ }+ db.close().await;+ let result = migrate(root.path(), Default::default(), Default::default()).await;+ assert!(result.is_ok(), "eligible synthetic history must convert: {result:?}");+ assert_eq!(result.unwrap().frames, 2, "conversion must retain both frames");+ assert!(root.path().join("storage-migration-complete.json").is_file());+ assert!(!root.path().join("storage-migration.json").exists());+ let db = DatabaseManager::new(root.path().join("db.sqlite").to_str().unwrap(), Default::default()).await.unwrap();+ assert_eq!(db.storage_mode(), StorageMode::HybridParquetV1, "success must perform conversion");+ let after = db.frame_payloads(&[1, 2], Projection::All).await.unwrap();+ for id in [1, 2] { assert_eq!(after[&id].full_text, before[&id].full_text, "payload changed for {id}"); }+ for (term, ids) in searches {+ let actual: Vec<i64> = sqlx::query_scalar("SELECT rowid FROM frames_fts WHERE frames_fts MATCH ? ORDER BY rowid")+ .bind(term).fetch_all(&db.pool).await.unwrap();+ assert_eq!(actual, ids, "search changed for {term}");+ }+ db.verify_storage().await.unwrap();+ db.close().await;+}++#[tokio::test]+async fn controls_and_diagrams_preserve_payload_and_search() { converted(["\0\0├───┼───── foo", "┃"]).await; }+#[tokio::test]+async fn unicode_history_preserves_payload_and_search() { converted(["\0שלום עולם", "café\0東京"]).await; }+#[tokio::test]+async fn symbol_only_history_still_converts() { converted(["\0\0├───┼─────", "┃"]).await; }+#[tokio::test]+async fn ordinary_history_still_converts() { converted(["ordinary foo", "café 東京 שלום"]).await; }++fn retained(root: &std::path::Path) {+ assert!(root.join("db.sqlite").is_file(), "refusal must retain original source");+ assert!(!root.join("storage-migration.json").exists());+ assert!(!root.join("storage-migration-complete.json").exists());+ let conn = rusqlite::Connection::open_with_flags(root.join("db.sqlite"), rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap();+ let rows: Vec<String> = conn.prepare("SELECT full_text FROM frames ORDER BY id").unwrap()+ .query_map([], |row| row.get(0)).unwrap().map(Result::unwrap).collect();+ assert_eq!(rows, vec!["ordinary original", "retained original"]);+}+#[tokio::test]+async fn genuine_missing_index_is_refused_without_discarding_history() {+ let root = tempfile::tempdir().unwrap();+ let db = seed(root.path(), ["ordinary original", "retained original"]).await;+ db.execute_raw_sql_write("DROP TABLE frames_fts").await.unwrap();+ db.close().await;+ assert!(migrate(root.path(), Default::default(), Default::default()).await.is_err(), "missing source search index must not be silently accepted");+ retained(root.path());+}+#[tokio::test]+async fn insufficient_headroom_retains_original_history() {+ let root = tempfile::tempdir().unwrap();+ let db = seed(root.path(), ["ordinary original", "retained original"]).await;+ db.close().await;+ let mut options = MigrationOptions::default();+ options.budget.disk_reserve_bytes = fs2::available_space(root.path()).unwrap();+ assert!(migrate(root.path(), Default::default(), options).await.is_err());+ retained(root.path());+}+#[tokio::test]+async fn external_reader_refusal_preserves_history_and_later_retry() {+ use sqlx::Connection;+ let root = tempfile::tempdir().unwrap();+ let db = seed(root.path(), ["ordinary original", "retained original"]).await;+ db.close().await;+ let mut reader = sqlx::SqliteConnection::connect_with(&sqlx::sqlite::SqliteConnectOptions::new().filename(root.path().join("db.sqlite")).read_only(true)).await.unwrap();+ let mut tx = reader.begin().await.unwrap();+ sqlx::query("SELECT count(*) FROM frames").fetch_one(&mut *tx).await.unwrap();+ assert!(migrate(root.path(), Default::default(), Default::default()).await.is_err());+ tx.rollback().await.unwrap();+ reader.close().await.unwrap();+ retained(root.path());+ assert_eq!(migrate(root.path(), Default::default(), Default::default()).await.unwrap().frames, 2);+ let db = DatabaseManager::new(root.path().join("db.sqlite").to_str().unwrap(), Default::default()).await.unwrap();+ let rows = db.frame_payloads(&[1, 2], Projection::All).await.unwrap();+ assert_eq!(rows[&1].full_text.as_deref(), Some("ordinary original"));+ assert_eq!(rows[&2].full_text.as_deref(), Some("retained original"));+ db.verify_storage().await.unwrap();+ db.close().await;+}