YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: preserve complete pipe runs across intermediate turns
Louis Beaumontb952fb8cba5d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 9a64f74b5..d03e1fc16 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -33,7 +33,7 @@ images, blanket paste refusal and missing-source classification. Fixtures anddependency links appear only when grading begins. This does not execute nativeclipboard delivery, image encoding, disk autosave, agent isolation or model trials.-The current app corpus contains 122 git-mined regressions. See+The current app corpus contains 123 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1514,3 +1514,13 @@ wrong credentials and lost caching. Fixtures appear only at grading and requireno dependency links. Missing provider source is a setup failure. This does notestablish real provider availability, gateway HTTP admission, concurrent orcross-account cache isolation, execution isolation or model performance.++`app-pipe-run-terminal-preservation` mounts the actual recorder through a synthetic+event bus and uses its real NDJSON parser. Eight outcomes cover intermediate turns,+complete terminal transcripts, ordinary completion, disabled history, foreign events,+foreground ownership and duplicate terminals. The parent fails two outcomes and+preserves six; the historical recorder fix and current source pass eight. Run+`bun test evals/coding-agent/calibrate-pipe-run-terminal.test.js` for calibration.+Storage and settings are synthetic; this does not establish native delivery, disk+durability, crash recovery, continued-chat behavior, agent isolation or model quality.+The hidden fixture is installed only when grading begins.diff --git a/evals/coding-agent/calibrate-pipe-run-terminal.test.js b/evals/coding-agent/calibrate-pipe-run-terminal.test.jsnew file mode 100644index 000000000..b735c4ffc--- /dev/null+++ b/evals/coding-agent/calibrate-pipe-run-terminal.test.js@@ -0,0 +1,38 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync,mkdtempSync,readFileSync,writeFileSync,rmSync } from 'node:fs';+import { dirname,join,resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(x=>x.id==='app-pipe-run-terminal-preservation');+const base='apps/screenpipe-app-tauri/';const path=base+'lib/events/pipe-run-recorder.ts';+const show=(ref,p)=>execFileSync('git',['show',`${ref}:${p}`],{cwd:repo,encoding:'utf8'});+const broken=show(item.base_ref,path),fixed=show(item.oracle_ref,path);+const root=mkdtempSync(join(tmpdir(),'pipe-terminal-calibration-'));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,source,ref=item.base_ref,unused=false){+ const cwd=join(root,name);const write=(p,t)=>{mkdirSync(dirname(join(cwd,p)),{recursive:true});writeFileSync(join(cwd,p),t);};+ if(source!==null)write(path,source);+ for(const p of ['lib/events/types.ts','lib/pipe-ndjson-to-chat.ts',...(ref==='HEAD'?['lib/pipe-conversation.ts','lib/pipe-execution-status.ts']:[])])write(base+p,show(ref,base+p));+ if(unused)write(base+'lib/events/unused-repair.ts',fixed);+ write(base+'tsconfig.json',JSON.stringify({compilerOptions:{baseUrl:'.',paths:{'@/*':['./*']}}}));+ const f=item.grader.fixtures[0];write(f.destination_path,readFileSync(join(import.meta.dir,f.local_path),'utf8'));+ const r=spawnSync(process.execPath,['--no-env-file','test',f.destination_path],{cwd,encoding:'utf8',timeout:30000,env:{PATH:dirname(process.execPath)}});+ if(process.env.PIPE_TERMINAL_RESULTS){const out=resolve(process.env.PIPE_TERMINAL_RESULTS);mkdirSync(out,{recursive:true});for(const ext of ['stdout','stderr'])writeFileSync(join(out,name+'.'+ext),r[ext]??'');writeFileSync(join(out,name+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message??null}));}+ return r;+}+function pass(r){expect(r.error).toBeUndefined();expect(r.status).toBe(0);expect(r.stderr).toContain('8 pass');}+function fail(r){expect(r.error).toBeUndefined();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}+function patch(a,b){expect(fixed.split(a)).toHaveLength(2);return fixed.replace(a,b);}+test('parent fails two behaviors and preserves six',()=>{const r=grade('parent',broken);fail(r);expect(r.stderr).toContain('2 fail');expect(r.stderr).toContain('6 pass');});+test('historical recorder fix passes with base parser',()=>pass(grade('reference',fixed)));+test('current source passes',()=>pass(grade('current',show('HEAD',path),'HEAD')));+test('equivalent guard passes',()=>pass(grade('equivalent',patch('typeof inner.type === "string" && TERMINAL_EVENT_TYPES.has(inner.type)','inner.type === "agent_end" || inner.type === "pipe_done"'))));+test('unused correct recorder cannot bypass mounted behavior',()=>fail(grade('unused',broken,item.base_ref,true)));+test('intermediate save mutation is rejected',()=>fail(grade('intermediate',patch('"agent_end",\n "pipe_done",','"agent_end",\n "turn_end",\n "pipe_done",'))));+test('blanket no-save workaround rejected',()=>fail(grade('nosave',patch('async function finalizeBuffer(sid: string, buf: PipeRunBuffer): Promise<void> {','async function finalizeBuffer(sid: string, buf: PipeRunBuffer): Promise<void> { return;'))));+test('history preference bypass rejected',()=>fail(grade('history',patch('if (settings?.chatHistory?.historyEnabled === false) return;','if (false) return;'))));+test('foreign source bypass rejected',()=>fail(grade('foreign',patch('if (envelope.source !== "pipe") return;','if (false) return;'))));+test('missing recorder remains setup error',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex eaf59d365..07af85263 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5045,6 +5045,42 @@],"command": "bun --no-env-file test packages/ai-gateway/eval-wif-token-retry.test.ts"}+ },+ {+ "id": "app-pipe-run-terminal-preservation",+ "name": "Keep complete pipe transcripts across intermediate turns",+ "source": {+ "kind": "git_regression",+ "fix_commit": "4f58c512232267cc66eb5bf76846107afff11931"+ },+ "base_ref": "523615534af1bc9174d777ba917a3603354a164f",+ "oracle_ref": "4f58c512232267cc66eb5bf76846107afff11931",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/events/pipe-run-recorder.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/events/pipe-run-recorder.ts",+ "apps/screenpipe-app-tauri/lib/pipe-ndjson-to-chat.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "data-preservation",+ "pipe-history"+ ],+ "prompt": "The pipe-run recorder can save an incomplete conversation between model turns, losing later output. Repair the actual mounted recorder so intermediate turn_end events do not persist an incomplete run. When agent_end or pipe_done finishes a run, save one complete conversation containing its original user prompt and all streamed assistant turns. Preserve ordinary single-turn completion, the execution identity in saved pipeContext, rejection of non-pipe events, disabled chat-history settings, and the foreground pipe-watch writer's ownership. Repeated terminal events must not overwrite the saved conversation. Keep the public mount/unmount lifecycle and normal event-envelope contract; no particular terminal guard or internal helper is required. Use synthetic events and intercepted storage/settings/bus ports. Native delivery, actual disk durability, crash recovery, continued-chat merging, and presentation labels are outside this historical task.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/pipe-run-terminal.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-pipe-run-terminal.test.ts"+ }+ ],+ "command": "bun --no-env-file test apps/screenpipe-app-tauri/eval-pipe-run-terminal.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/pipe-run-terminal.fixture.ts.txt b/evals/coding-agent/graders/pipe-run-terminal.fixture.ts.txtnew file mode 100644index 000000000..89ba851a4--- /dev/null+++ b/evals/coding-agent/graders/pipe-run-terminal.fixture.ts.txt@@ -0,0 +1,25 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {test,expect,mock,beforeEach,afterEach} from 'bun:test';+let callback:any; let saved:any[]=[]; let enabled=true; let sessions:any={};let unmount:any;+globalThis.fetch=async()=>{throw new Error('Unexpected network');};+const register=(fn:any)=>{callback=fn;return()=>{callback=undefined;};};+mock.module('./lib/events/bus',()=>({mountAgentEventBus:async()=>{},registerDefault:register,registerObserver:register}));+mock.module('@/lib/chat-storage',()=>({loadConversationFile:async()=>null,saveConversationFile:async(c:any)=>{saved.push(structuredClone(c));},conversationMetaFromJson:(c:any)=>c}));+mock.module('@/lib/stores/chat-store',()=>({useChatStore:{getState:()=>({sessions,actions:{upsert:(c:any)=>{sessions[c.id]=c;}}})},sessionRecordFromMeta:(c:any)=>c}));+mock.module('@/lib/hooks/use-settings',()=>({getStore:async()=>({get:async()=>({chatHistory:{historyEnabled:enabled}})})}));+mock.module('@/components/settings/pipes-section',()=>({cleanPipeStdout:()=>{throw new Error('Unexpected plain-output fallback');}}));+const recorder=await import('./lib/events/pipe-run-recorder');+const sid='pipe:synthetic-review:42';+async function emit(event:any,source='pipe',sessionId=sid){await callback({source,sessionId,event});}+const start=async()=>{await emit({type:'message_start',message:{role:'user',content:'Summarize synthetic notes.'}});await emit({type:'message_start',message:{role:'assistant'}});await emit({type:'message_update',assistantMessageEvent:{type:'text_delta',delta:'First result.'}});};+beforeEach(async()=>{saved=[];enabled=true;sessions={};unmount=await recorder.mountPipeRunRecorder();expect(typeof callback).toBe('function');});+afterEach(()=>unmount());+test('intermediate turn does not save incomplete run',async()=>{await start();await emit({type:'turn_end'});expect(saved).toHaveLength(0);});+test('terminal save retains prompt and both turns',async()=>{await start();await emit({type:'turn_end'});await emit({type:'message_start',message:{role:'assistant'}});await emit({type:'message_update',assistantMessageEvent:{type:'text_delta',delta:'Second result.'}});await emit({type:'agent_end'});expect(saved).toHaveLength(1);const text=saved[0].messages.map((m:any)=>m.content).join('\n');expect(text).toContain('Summarize synthetic notes.');expect(text).toContain('First result.');expect(text).toContain('Second result.');});+test('ordinary agent_end saves',async()=>{await start();await emit({type:'agent_end'});expect(saved).toHaveLength(1);expect(saved[0].pipeContext.executionId).toBe(42);});+test('pipe_done saves',async()=>{await start();await emit({type:'pipe_done'});expect(saved).toHaveLength(1);});+test('disabled history suppresses persistence',async()=>{enabled=false;await start();await emit({type:'agent_end'});expect(saved).toHaveLength(0);});+test('foreign source never saves',async()=>{await emit({type:'agent_end',messages:[{role:'assistant',content:'other'}]},'pi');expect(saved).toHaveLength(0);});+test('foreground pipe writer is preserved',async()=>{sessions[sid]={kind:'pipe-watch',messages:[{role:'assistant',content:'owned'}]};await start();await emit({type:'agent_end'});expect(saved).toHaveLength(0);expect(sessions[sid].messages[0].content).toBe('owned');});+test('duplicate terminal does not overwrite',async()=>{await start();await emit({type:'agent_end'});await emit({type:'pipe_done'});expect(saved).toHaveLength(1);});