YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover vault locking across CLI processes
Louis Beaumont0c107fcae5c9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex a61679a67..7fa9dc4cf 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1855,3 +1855,25 @@ The hidden fixture and runtime link appear only at grading. Synthetic loopbackresponses establish logical invalidation and active-session accounting, not heapreclamation, disconnect cleanup, LAN authentication, concurrency, agent isolationor model performance.++### Fresh-process offline vault locking++`app-vault-lock-fresh-process-key` executes the unchanged CLI handler and full+vault crate in a minimal Cargo workspace. Synthetic ports supply a temporary+data directory and password; setup, lock, status and unlock use fresh processes.+Four outcome groups check encrypted database/WAL/SHM/nested media roundtrip,+wrong-password preservation, unconfigured refusal and repeated-setup refusal.+The roundtrip also preserves locked bytes after wrong unlock and repeated lock.++Run `bun test evals/coding-agent/calibrate-vault-cli.test.js` for historical+parent/reference/current, equivalent, unused-helper, response-only, blanket+refusal, skipped-media, ignored-password and missing-source controls. Offline+Cargo compilation has a 180-second bound; setup errors cannot count as behavior+failures. The hidden fixture appears only at grading and uses no dependency or+build-cache links. The copied diagnostic lock may resolve the smaller workspace;+repository package and lock files are unchanged.++This does not execute the packaged engine, TTY password handling, online HTTP+routes, migration/daemon integration, crash recovery or production recordings.+Current daemon and hybrid-storage refusal policies are separate contracts, not+historical online-lock-success expectations. No model trial or isolation claim.diff --git a/evals/coding-agent/calibrate-vault-cli.test.js b/evals/coding-agent/calibrate-vault-cli.test.jsnew file mode 100644index 000000000..25658b7ce--- /dev/null+++ b/evals/coding-agent/calibrate-vault-cli.test.js@@ -0,0 +1,39 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { test, expect } from 'bun:test';+import fs from 'node:fs';+import path from 'node:path';+import os from 'node:os';+import { spawnSync } from 'node:child_process';+const repo = path.resolve(import.meta.dir,'../..');+const output = process.env.EVAL_CALIBRATION_DIR ? path.resolve(process.env.EVAL_CALIBRATION_DIR) : fs.mkdtempSync(path.join(os.tmpdir(),'vault-calibration-'));+const parent='5dd6c7f0db9a38aa1ffbf71ae9bf5ef46f420a6d', fixed='7d171cc2de988c92b0d38a940fecd8dfb27816b0';+const cli='crates/screenpipe-engine/src/cli/vault.rs', manager='crates/screenpipe-vault/src/manager.rs';+function git(...args){const z=spawnSync('git',['-C',repo,...args],{maxBuffer:20*1024*1024});if(z.status!==0)throw new Error(z.stderr.toString());return z.stdout;}+function mutate(dir,file,from,to){const p=path.join(dir,file),s=fs.readFileSync(p,'utf8');if(!s.includes(from))throw new Error('Missing calibration mutation target');fs.writeFileSync(p,s.replace(from,to));}+const controls=[+ ['parent',parent,'fail'],['reference',fixed,'pass'],['current','HEAD','pass'],+ ['equivalent',fixed,'pass',d=>mutate(d,manager,'let key_bytes: [u8; KEY_SIZE] = *key;','let key_bytes = <[u8; KEY_SIZE]>::clone(&*key);')],+ ['unused-helper',parent,'fail',d=>fs.appendFileSync(path.join(d,cli),'\nfn unused_recovery_example(password: &str) -> bool { !password.is_empty() }\n')],+ ['response-only',fixed,'fail',d=>mutate(d,cli,'let _progress_rx = vault.lock(&password).await?;','println!("vault locked"); return Ok(());')],+ ['refuse-all',fixed,'fail',d=>mutate(d,cli,'let _progress_rx = vault.lock(&password).await?;','anyhow::bail!("refused");')],+ ['skip-media',fixed,'fail',d=>mutate(d,manager,'if data_dir.exists() {','if false {')],+ ['ignore-lock-password',fixed,'fail',d=>mutate(d,manager,'crypto::derive_key(password, &meta.salt)?','crypto::derive_key("synthetic-vault-password", &meta.salt)?')],+ ['missing-source',fixed,'error',d=>fs.unlinkSync(path.join(d,cli))],+];+test('vault actual CLI outcomes accept equivalent fixes and reject incomplete repairs',()=>{+ fs.mkdirSync(output,{recursive:true});const results=[];+ for(const [name,ref,expected,edit] of controls){+ const d=path.join(output,name);fs.mkdirSync(d,{recursive:true});+ const files=[...git('ls-tree','-r','--name-only',ref,'crates/screenpipe-vault').toString().trim().split('\n'),cli,'Cargo.toml','Cargo.lock','crates/screenpipe-engine/Cargo.toml'];+ for(const file of files){fs.mkdirSync(path.dirname(path.join(d,file)),{recursive:true});fs.writeFileSync(path.join(d,file),git('show',ref+':'+file));}+ if(edit)edit(d);+ const fixture=path.join(d,'vault-cli.fixture.js');fs.copyFileSync(path.join(import.meta.dir,'graders/vault-cli.fixture.js'),fixture);+ const z=spawnSync(process.execPath,['--no-env-file',fixture],{cwd:d,encoding:'utf8',timeout:250000,maxBuffer:20*1024*1024});+ fs.writeFileSync(path.join(output,name+'.stdout'),z.stdout ?? '');fs.writeFileSync(path.join(output,name+'.stderr'),z.stderr ?? '');+ const result=JSON.parse(z.stdout);const observed=z.status===127?'error':z.status===0?'pass':z.status===1?'fail':'error';+ results.push({name,expected,observed,exit:z.status,result});fs.writeFileSync(path.join(output,'calibration.json'),JSON.stringify(results,null,2)+'\n');+ expect(observed, name+' '+JSON.stringify(result)).toBe(expected);+ if(expected!=='error'){expect(result.infrastructure_error).toBeNull();expect(result.outcomes).toHaveLength(4);}+ }+},2400000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex c42b93ebd..2411b9a59 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6132,6 +6132,46 @@],"command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 SCREENPIPE_MCP_TELEMETRY_DISABLED=1 DO_NOT_TRACK=1 node node_modules/vitest/vitest.mjs run src/eval-session.test.ts"}+ },+ {+ "id": "app-vault-lock-fresh-process-key",+ "name": "Lock and unlock an offline vault from fresh CLI processes",+ "source": {+ "kind": "git_regression",+ "fix_commit": "7d171cc2de988c92b0d38a940fecd8dfb27816b0"+ },+ "base_ref": "5dd6c7f0db9a38aa1ffbf71ae9bf5ef46f420a6d",+ "oracle_ref": "7d171cc2de988c92b0d38a940fecd8dfb27816b0",+ "oracle_paths": [+ "crates/screenpipe-engine/src/cli/vault.rs",+ "crates/screenpipe-engine/src/routes/vault.rs",+ "crates/screenpipe-vault/src/lib.rs",+ "crates/screenpipe-vault/src/manager.rs"+ ],+ "trigger_paths": [+ "crates/screenpipe-engine/src/cli/vault.rs",+ "crates/screenpipe-vault"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "vault",+ "data-integrity",+ "preserved-behavior"+ ],+ "prompt": "Offline vault setup succeeds, but a later CLI invocation cannot lock the existing vault. Repair the actual CLI flow so setup, lock and unlock can run in independent processes. Setup must leave database, WAL, SHM and nested media bytes unchanged and report unlocked. Lock with the correct password must succeed, report locked, encrypt all those files using the existing SPVAULT1-compatible format without leaving their plaintext content, and allow a fresh unlock process to restore identical original bytes and report unlocked. A wrong lock password must fail while preserving plaintext and unlocked state. A wrong unlock password and a second lock of an already-locked vault must fail without changing encrypted bytes or locked state. Lock before setup must fail without creating vault metadata or altering data. Repeated setup must fail without changing metadata or plaintext. Preserve command status behavior and compatible existing callers; no particular helper or key-storage implementation is required. Grading executes the actual CLI handler and complete vault crate in a minimal offline Cargo workspace, with synthetic filesystem paths and password input, separate processes, and an offline daemon port. It does not run the packaged engine, terminal input, HTTP routes, live recordings or customer files. Do not broaden online locking or migration policy; current daemon and hybrid-storage restrictions remain separate preserved contracts.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 240,+ "fixtures": [+ {+ "local_path": "graders/vault-cli.fixture.js",+ "destination_path": "vault-cli.eval.js"+ }+ ],+ "command": "bun --no-env-file vault-cli.eval.js"+ }}]}diff --git a/evals/coding-agent/graders/vault-cli.fixture.js b/evals/coding-agent/graders/vault-cli.fixture.jsnew file mode 100644index 000000000..de73d61b8--- /dev/null+++ b/evals/coding-agent/graders/vault-cli.fixture.js@@ -0,0 +1,90 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import fs from 'node:fs';+import path from 'node:path';+import { spawn } from 'node:child_process';+import assert from 'node:assert/strict';+import { createHash } from 'node:crypto';++// Only path selection and terminal password input are synthetic. The actual CLI+// handler and complete vault crate execute unchanged in fresh processes.+const source = process.cwd();+const report = { outcomes: [], infrastructure_error: null };+let root;+function command(args, cwd, env = process.env, milliseconds = 10000) {+ return new Promise(resolve => {+ const child = spawn(args[0], args.slice(1), { cwd, env, detached: true, stdio: ['ignore','pipe','pipe'] });+ let stdout = '', stderr = '', timedOut = false;+ child.stdout.on('data', b => stdout += b);+ child.stderr.on('data', b => stderr += b);+ const timer = setTimeout(() => { timedOut = true; try { process.kill(-child.pid, 'SIGKILL'); } catch {} }, milliseconds);+ child.on('error', error => { clearTimeout(timer); resolve({ args, exit: null, stdout, stderr, error: String(error), timedOut }); });+ child.on('close', (exit, signal) => { clearTimeout(timer); resolve({ args, exit, signal, stdout, stderr, timedOut }); });+ });+}+function infrastructure(message) { throw new Error(message); }+function toml(value) {+ if (Array.isArray(value)) return '[' + value.map(toml).join(', ') + ']';+ if (value && typeof value === 'object') return '{' + Object.entries(value).map(([k,v]) => JSON.stringify(k)+' = '+toml(v)).join(', ') + '}';+ return JSON.stringify(value);+}+function table(values) { return Object.entries(values).map(([k,v]) => JSON.stringify(k)+' = '+toml(v)).join('\n'); }+try {+ const read = file => fs.readFileSync(path.join(source,file),'utf8');+ const workspace = Bun.TOML.parse(read('Cargo.toml')).workspace;+ const vault = Bun.TOML.parse(read('crates/screenpipe-vault/Cargo.toml'));+ const engine = Bun.TOML.parse(read('crates/screenpipe-engine/Cargo.toml'));+ const cli = read('crates/screenpipe-engine/src/cli/vault.rs');+ root = fs.mkdtempSync(path.join(source,'.vault-outcomes-'));+ fs.cpSync(path.join(source,'crates/screenpipe-vault'),path.join(root,'vault'),{recursive:true});+ fs.mkdirSync(path.join(root,'probe/src'),{recursive:true});+ fs.writeFileSync(path.join(root,'probe/actual_cli.rs'),cli);+ const names = new Set([...Object.keys(vault.dependencies ?? {}),...Object.keys(vault['dev-dependencies'] ?? {}),'anyhow','tokio','serde_json']);+ const dependencies = Object.fromEntries(Object.entries(workspace.dependencies).filter(([k])=>names.has(k)));+ fs.writeFileSync(path.join(root,'Cargo.toml'),'[workspace]\nresolver = "2"\nmembers = ["vault", "probe"]\n[workspace.package]\n'+table(workspace.package)+'\n[workspace.dependencies]\n'+table(dependencies)+'\n');+ fs.copyFileSync(path.join(source,'Cargo.lock'),path.join(root,'Cargo.lock'));+ fs.writeFileSync(path.join(root,'probe/Cargo.toml'),'[package]\nname = "vault-outcome-probe"\nversion = "0.0.0"\nedition = "2021"\n[dependencies]\nscreenpipe-vault = { path = "../vault" }\n'+table(Object.fromEntries(['anyhow','tokio','serde_json'].map(k=>[k,engine.dependencies[k]])))+'\n');+ fs.writeFileSync(path.join(root,'probe/src/main.rs'),`// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+extern crate self as screenpipe_core;+extern crate self as rpassword;+pub mod paths { pub fn default_screenpipe_data_dir()->std::path::PathBuf { std::env::var("VAULT_EVAL_DIRECTORY").expect("synthetic directory required").into() } }+pub fn read_password()->std::io::Result<String> { Ok(std::env::var("VAULT_EVAL_PASSWORD").expect("synthetic password required")) }+pub enum VaultCommand { Status {port:u16}, Setup {port:u16}, Lock {port:u16}, Unlock {port:u16} }+#[path="../actual_cli.rs"] mod vault_cli;+#[tokio::main]+async fn main()->anyhow::Result<()> { let args:Vec<_>=std::env::args().collect();let command=match args.get(1).map(|x|x.as_str()) {Some("setup")=>VaultCommand::Setup{port:9},Some("lock")=>VaultCommand::Lock{port:9},Some("unlock")=>VaultCommand::Unlock{port:9},_=>VaultCommand::Status{port:9}};vault_cli::handle_vault_command(&command).await }+`);+ report.build = await command(['cargo','build','--offline','-p','vault-outcome-probe'],root,process.env,180000);+ if (report.build.exit !== 0 || report.build.timedOut || report.build.error) infrastructure('Cargo setup/compilation failed before behavioral grading');+ const binary = path.join(root,'target/debug/vault-outcome-probe');+ const hash = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');+ report.resolved_lock_sha256 = hash(path.join(root,'Cargo.lock'));+ report.binary_sha256 = hash(binary);+ report.runtime = { bun: Bun.version, cargo: await command(['cargo','--version'],root), rustc: await command(['rustc','--version'],root) };+ const invoke = async (op,data,password='synthetic-vault-password') => {+ const result = await command([binary,op],root,{PATH:process.env.PATH,VAULT_EVAL_DIRECTORY:data,VAULT_EVAL_PASSWORD:password});+ if (result.timedOut || result.error || result.signal) infrastructure('CLI process unavailable or timed out: '+JSON.stringify(result));+ return result;+ };+ const state = async data => { const z=await invoke('status',data);assert.equal(z.exit,0,z.stderr);return JSON.parse(z.stdout).state; };+ const contents = {'db.sqlite':Buffer.from('synthetic database\0bytes'),'db.sqlite-wal':Buffer.from('synthetic WAL'),'db.sqlite-shm':Buffer.from('synthetic SHM'),'data/nested/media.txt':Buffer.from('synthetic media bytes')};+ function seed(name) { const d=path.join(root,'data-'+name);for(const [f,b] of Object.entries(contents)){fs.mkdirSync(path.dirname(path.join(d,f)),{recursive:true});fs.writeFileSync(path.join(d,f),b);}return d; }+ const fileBytes = data => Object.fromEntries(Object.keys(contents).map(f=>[f,fs.readFileSync(path.join(data,f)).toString('hex')]));+ const plain = Object.fromEntries(Object.entries(contents).map(([f,b])=>[f,b.toString('hex')]));+ async function outcome(name,fn) { try { await fn();report.outcomes.push({name,passed:true}); } catch(error) { if (!(error instanceof assert.AssertionError)) throw error;report.outcomes.push({name,passed:false,error:String(error)}); } }+ const setup = async d => { const z=await invoke('setup',d);assert.equal(z.exit,0,z.stderr);assert.equal(await state(d),'unlocked');assert.deepEqual(fileBytes(d),plain); };+ await outcome('fresh CLI lock encrypts every file and fresh unlock restores identical bytes',async()=>{+ const d=seed('roundtrip');await setup(d);const z=await invoke('lock',d);assert.equal(z.exit,0,z.stderr);assert.equal(await state(d),'locked');+ for(const [f,b] of Object.entries(contents)){const encrypted=fs.readFileSync(path.join(d,f));assert.equal(encrypted.subarray(0,8).toString(),'SPVAULT1');assert.notDeepEqual(encrypted,b);assert.equal(encrypted.includes(b),false);}+ const before=fileBytes(d);const wrong=await invoke('unlock',d,'wrong-synthetic-password');assert.notEqual(wrong.exit,0);assert.equal(await state(d),'locked');assert.deepEqual(fileBytes(d),before);+ const again=await invoke('lock',d);assert.notEqual(again.exit,0);assert.equal(await state(d),'locked');assert.deepEqual(fileBytes(d),before);+ const unlocked=await invoke('unlock',d);assert.equal(unlocked.exit,0,unlocked.stderr);assert.equal(await state(d),'unlocked');assert.deepEqual(fileBytes(d),plain);+ });+ await outcome('wrong lock password preserves plaintext and unlocked state',async()=>{const d=seed('wrong');await setup(d);const z=await invoke('lock',d,'wrong-synthetic-password');assert.notEqual(z.exit,0);assert.equal(await state(d),'unlocked');assert.deepEqual(fileBytes(d),plain);});+ await outcome('unconfigured lock refuses without mutating data',async()=>{const d=seed('none');const z=await invoke('lock',d);assert.notEqual(z.exit,0);assert.equal(await state(d),'none');assert.deepEqual(fileBytes(d),plain);assert.equal(fs.existsSync(path.join(d,'vault.meta')),false);});+ await outcome('setup preserves data and repeated setup refuses',async()=>{const d=seed('setup');await setup(d);const metadata=fs.readFileSync(path.join(d,'vault.meta'));const z=await invoke('setup',d);assert.notEqual(z.exit,0);assert.equal(await state(d),'unlocked');assert.deepEqual(fileBytes(d),plain);assert.deepEqual(fs.readFileSync(path.join(d,'vault.meta')),metadata);});+} catch(error) { report.infrastructure_error=String(error); }+console.log(JSON.stringify(report));+if (root) fs.rmSync(root,{recursive:true,force:true});+process.exitCode=report.infrastructure_error ? 127 : report.outcomes.every(x=>x.passed) ? 0 : 1;