Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: preserve complete pipe runs across intermediate turns

Louis Beaumontb952fb8cba5d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 9a64f74b5..d03e1fc16 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -33,7 +33,7 @@ images, blanket paste refusal and missing-source classification. Fixtures and
dependency links appear only when grading begins. This does not execute native
clipboard delivery, image encoding, disk autosave, agent isolation or model trials.
-The current app corpus contains 122 git-mined regressions. See
+The current app corpus contains 123 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -1514,3 +1514,13 @@ wrong credentials and lost caching. Fixtures appear only at grading and require
no dependency links. Missing provider source is a setup failure. This does not
establish real provider availability, gateway HTTP admission, concurrent or
cross-account cache isolation, execution isolation or model performance.
+
+`app-pipe-run-terminal-preservation` mounts the actual recorder through a synthetic
+event bus and uses its real NDJSON parser. Eight outcomes cover intermediate turns,
+complete terminal transcripts, ordinary completion, disabled history, foreign events,
+foreground ownership and duplicate terminals. The parent fails two outcomes and
+preserves six; the historical recorder fix and current source pass eight. Run
+`bun test evals/coding-agent/calibrate-pipe-run-terminal.test.js` for calibration.
+Storage and settings are synthetic; this does not establish native delivery, disk
+durability, crash recovery, continued-chat behavior, agent isolation or model quality.
+The hidden fixture is installed only when grading begins.
diff --git a/evals/coding-agent/calibrate-pipe-run-terminal.test.js b/evals/coding-agent/calibrate-pipe-run-terminal.test.js
new file mode 100644
index 000000000..b735c4ffc
--- /dev/null
+++ b/evals/coding-agent/calibrate-pipe-run-terminal.test.js
@@ -0,0 +1,38 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync,mkdtempSync,readFileSync,writeFileSync,rmSync } from 'node:fs';
+import { dirname,join,resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(x=>x.id==='app-pipe-run-terminal-preservation');
+const base='apps/screenpipe-app-tauri/';const path=base+'lib/events/pipe-run-recorder.ts';
+const show=(ref,p)=>execFileSync('git',['show',`${ref}:${p}`],{cwd:repo,encoding:'utf8'});
+const broken=show(item.base_ref,path),fixed=show(item.oracle_ref,path);
+const root=mkdtempSync(join(tmpdir(),'pipe-terminal-calibration-'));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,source,ref=item.base_ref,unused=false){
+ const cwd=join(root,name);const write=(p,t)=>{mkdirSync(dirname(join(cwd,p)),{recursive:true});writeFileSync(join(cwd,p),t);};
+ if(source!==null)write(path,source);
+ for(const p of ['lib/events/types.ts','lib/pipe-ndjson-to-chat.ts',...(ref==='HEAD'?['lib/pipe-conversation.ts','lib/pipe-execution-status.ts']:[])])write(base+p,show(ref,base+p));
+ if(unused)write(base+'lib/events/unused-repair.ts',fixed);
+ write(base+'tsconfig.json',JSON.stringify({compilerOptions:{baseUrl:'.',paths:{'@/*':['./*']}}}));
+ const f=item.grader.fixtures[0];write(f.destination_path,readFileSync(join(import.meta.dir,f.local_path),'utf8'));
+ const r=spawnSync(process.execPath,['--no-env-file','test',f.destination_path],{cwd,encoding:'utf8',timeout:30000,env:{PATH:dirname(process.execPath)}});
+ if(process.env.PIPE_TERMINAL_RESULTS){const out=resolve(process.env.PIPE_TERMINAL_RESULTS);mkdirSync(out,{recursive:true});for(const ext of ['stdout','stderr'])writeFileSync(join(out,name+'.'+ext),r[ext]??'');writeFileSync(join(out,name+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message??null}));}
+ return r;
+}
+function pass(r){expect(r.error).toBeUndefined();expect(r.status).toBe(0);expect(r.stderr).toContain('8 pass');}
+function fail(r){expect(r.error).toBeUndefined();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}
+function patch(a,b){expect(fixed.split(a)).toHaveLength(2);return fixed.replace(a,b);}
+test('parent fails two behaviors and preserves six',()=>{const r=grade('parent',broken);fail(r);expect(r.stderr).toContain('2 fail');expect(r.stderr).toContain('6 pass');});
+test('historical recorder fix passes with base parser',()=>pass(grade('reference',fixed)));
+test('current source passes',()=>pass(grade('current',show('HEAD',path),'HEAD')));
+test('equivalent guard passes',()=>pass(grade('equivalent',patch('typeof inner.type === "string" && TERMINAL_EVENT_TYPES.has(inner.type)','inner.type === "agent_end" || inner.type === "pipe_done"'))));
+test('unused correct recorder cannot bypass mounted behavior',()=>fail(grade('unused',broken,item.base_ref,true)));
+test('intermediate save mutation is rejected',()=>fail(grade('intermediate',patch('"agent_end",\n "pipe_done",','"agent_end",\n "turn_end",\n "pipe_done",'))));
+test('blanket no-save workaround rejected',()=>fail(grade('nosave',patch('async function finalizeBuffer(sid: string, buf: PipeRunBuffer): Promise<void> {','async function finalizeBuffer(sid: string, buf: PipeRunBuffer): Promise<void> { return;'))));
+test('history preference bypass rejected',()=>fail(grade('history',patch('if (settings?.chatHistory?.historyEnabled === false) return;','if (false) return;'))));
+test('foreign source bypass rejected',()=>fail(grade('foreign',patch('if (envelope.source !== "pipe") return;','if (false) return;'))));
+test('missing recorder remains setup error',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index eaf59d365..07af85263 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -5045,6 +5045,42 @@
],
"command": "bun --no-env-file test packages/ai-gateway/eval-wif-token-retry.test.ts"
}
+ },
+ {
+ "id": "app-pipe-run-terminal-preservation",
+ "name": "Keep complete pipe transcripts across intermediate turns",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "4f58c512232267cc66eb5bf76846107afff11931"
+ },
+ "base_ref": "523615534af1bc9174d777ba917a3603354a164f",
+ "oracle_ref": "4f58c512232267cc66eb5bf76846107afff11931",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/events/pipe-run-recorder.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/events/pipe-run-recorder.ts",
+ "apps/screenpipe-app-tauri/lib/pipe-ndjson-to-chat.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "data-preservation",
+ "pipe-history"
+ ],
+ "prompt": "The pipe-run recorder can save an incomplete conversation between model turns, losing later output. Repair the actual mounted recorder so intermediate turn_end events do not persist an incomplete run. When agent_end or pipe_done finishes a run, save one complete conversation containing its original user prompt and all streamed assistant turns. Preserve ordinary single-turn completion, the execution identity in saved pipeContext, rejection of non-pipe events, disabled chat-history settings, and the foreground pipe-watch writer's ownership. Repeated terminal events must not overwrite the saved conversation. Keep the public mount/unmount lifecycle and normal event-envelope contract; no particular terminal guard or internal helper is required. Use synthetic events and intercepted storage/settings/bus ports. Native delivery, actual disk durability, crash recovery, continued-chat merging, and presentation labels are outside this historical task.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/pipe-run-terminal.fixture.ts.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-pipe-run-terminal.test.ts"
+ }
+ ],
+ "command": "bun --no-env-file test apps/screenpipe-app-tauri/eval-pipe-run-terminal.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/pipe-run-terminal.fixture.ts.txt b/evals/coding-agent/graders/pipe-run-terminal.fixture.ts.txt
new file mode 100644
index 000000000..89ba851a4
--- /dev/null
+++ b/evals/coding-agent/graders/pipe-run-terminal.fixture.ts.txt
@@ -0,0 +1,25 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {test,expect,mock,beforeEach,afterEach} from 'bun:test';
+let callback:any; let saved:any[]=[]; let enabled=true; let sessions:any={};let unmount:any;
+globalThis.fetch=async()=>{throw new Error('Unexpected network');};
+const register=(fn:any)=>{callback=fn;return()=>{callback=undefined;};};
+mock.module('./lib/events/bus',()=>({mountAgentEventBus:async()=>{},registerDefault:register,registerObserver:register}));
+mock.module('@/lib/chat-storage',()=>({loadConversationFile:async()=>null,saveConversationFile:async(c:any)=>{saved.push(structuredClone(c));},conversationMetaFromJson:(c:any)=>c}));
+mock.module('@/lib/stores/chat-store',()=>({useChatStore:{getState:()=>({sessions,actions:{upsert:(c:any)=>{sessions[c.id]=c;}}})},sessionRecordFromMeta:(c:any)=>c}));
+mock.module('@/lib/hooks/use-settings',()=>({getStore:async()=>({get:async()=>({chatHistory:{historyEnabled:enabled}})})}));
+mock.module('@/components/settings/pipes-section',()=>({cleanPipeStdout:()=>{throw new Error('Unexpected plain-output fallback');}}));
+const recorder=await import('./lib/events/pipe-run-recorder');
+const sid='pipe:synthetic-review:42';
+async function emit(event:any,source='pipe',sessionId=sid){await callback({source,sessionId,event});}
+const start=async()=>{await emit({type:'message_start',message:{role:'user',content:'Summarize synthetic notes.'}});await emit({type:'message_start',message:{role:'assistant'}});await emit({type:'message_update',assistantMessageEvent:{type:'text_delta',delta:'First result.'}});};
+beforeEach(async()=>{saved=[];enabled=true;sessions={};unmount=await recorder.mountPipeRunRecorder();expect(typeof callback).toBe('function');});
+afterEach(()=>unmount());
+test('intermediate turn does not save incomplete run',async()=>{await start();await emit({type:'turn_end'});expect(saved).toHaveLength(0);});
+test('terminal save retains prompt and both turns',async()=>{await start();await emit({type:'turn_end'});await emit({type:'message_start',message:{role:'assistant'}});await emit({type:'message_update',assistantMessageEvent:{type:'text_delta',delta:'Second result.'}});await emit({type:'agent_end'});expect(saved).toHaveLength(1);const text=saved[0].messages.map((m:any)=>m.content).join('\n');expect(text).toContain('Summarize synthetic notes.');expect(text).toContain('First result.');expect(text).toContain('Second result.');});
+test('ordinary agent_end saves',async()=>{await start();await emit({type:'agent_end'});expect(saved).toHaveLength(1);expect(saved[0].pipeContext.executionId).toBe(42);});
+test('pipe_done saves',async()=>{await start();await emit({type:'pipe_done'});expect(saved).toHaveLength(1);});
+test('disabled history suppresses persistence',async()=>{enabled=false;await start();await emit({type:'agent_end'});expect(saved).toHaveLength(0);});
+test('foreign source never saves',async()=>{await emit({type:'agent_end',messages:[{role:'assistant',content:'other'}]},'pi');expect(saved).toHaveLength(0);});
+test('foreground pipe writer is preserved',async()=>{sessions[sid]={kind:'pipe-watch',messages:[{role:'assistant',content:'owned'}]};await start();await emit({type:'agent_end'});expect(saved).toHaveLength(0);expect(sessions[sid].messages[0].content).toBe('owned');});
+test('duplicate terminal does not overwrite',async()=>{await start();await emit({type:'agent_end'});await emit({type:'pipe_done'});expect(saved).toHaveLength(1);});