Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: keep continued Pipe sessions beside active recordings

louisa9e7e60a2ca1
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 08fcb440b..950a3fc15 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 60 git-mined regressions. See
+The current app corpus contains 61 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -514,3 +514,19 @@ accepting equivalent internal naming. Missing imports remain setup failures.
Only the extension is the historical oracle; the existing public transport
normalizer runs unchanged. These outcomes do not prove native persistence,
real agent repair choices, model performance or enforced process isolation.
+
+## Continued Pipe recording directories
+
+`app-continued-pipe-recording-directory` exercises the actual directory resolver
+and session parser against synthetic native path/command ports. Eighteen outcomes
+cover custom roots, later root changes, Windows-style drive paths, spaces, lookup
+fallbacks, and generic routing for ordinary, per-execution and unsafe sessions.
+The parent fails seven outcomes and preserves eleven; reference/current pass all
+eighteen. Nine calibration controls reject unused fixes, stale caches, unsafe
+routing and lost fallback, while accepting equivalent variable naming.
+
+Run `bun test evals/coding-agent/calibrate-pipe-directory.test.js`. Missing modules
+remain setup errors. Only the directory resolver is the historical oracle.
+The synthetic join uses forward slashes; it does not prove Windows-native path
+semantics, native command correctness, complete chat continuation, account-state
+relocation or durable session persistence. No model, scheduler or native build runs.
diff --git a/evals/coding-agent/calibrate-pipe-directory.test.js b/evals/coding-agent/calibrate-pipe-directory.test.js
new file mode 100644
index 000000000..efbf0c5b9
--- /dev/null
+++ b/evals/coding-agent/calibrate-pipe-directory.test.js
@@ -0,0 +1,29 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdirSync,mkdtempSync,readFileSync,rmSync,writeFileSync} from 'node:fs';
+import {dirname,join,resolve} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),root=mkdtempSync(join(tmpdir(),'eval-calibration-'));
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==="app-continued-pipe-recording-directory");
+const source="apps/screenpipe-app-tauri/lib/chat/pi-project-dir.ts",archive=["apps/screenpipe-app-tauri/lib/chat/pi-project-dir.ts", "apps/screenpipe-app-tauri/lib/events/types.ts"];
+const show=(ref,path=source)=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'}),fixed=show(item.oracle_ref),broken=show(item.base_ref);
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,body=fixed,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);execFileSync('tar',['-xf','-','-C',cwd],{input:execFileSync('git',['archive',item.base_ref,...archive],{cwd:repo,maxBuffer:32*1024*1024})});
+ if(body===null)rmSync(join(cwd,source));else writeFileSync(join(cwd,source),body);
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ mutate(cwd);return spawnSync(process.execPath,['test',item.grader.fixtures[0].destination_path],{cwd,encoding:'utf8',timeout:15000,env:{PATH:process.env.PATH,HOME:cwd,NO_COLOR:'1'}});
+}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toMatch(/Cannot find|ModuleNotFound|SyntaxError|Unhandled error between tests/);expect(r.stderr).toContain('Ran 18 tests across 1 file');}
+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}
+test('parent fails intended outcomes and preserves neighboring behavior',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('7 fail');expect(r.stderr).toContain('11 pass');});
+test('reference passes all outcomes',()=>{const r=grade('reference');expect(r.status).toBe(0);expect(r.stderr).toContain('18 pass');});
+test('an unused correct implementation cannot rescue the caller',()=>fails(grade('unused',broken,cwd=>writeFileSync(join(cwd,source+'.unused.ts'),fixed))));
+test('equivalent internal variable naming passes',()=>expect(grade('equivalent',fixed.replaceAll(/\bbaseDir\b/g,'workingRoot')).status).toBe(0));
+test('unsafe names cannot reach a pipe directory',()=>fails(grade('unsafe',replace(fixed,'if (\n pipeName.trim()', 'if (false && (\n pipeName.trim()').replace('pipeName.includes("\\0")\n )','pipeName.includes("\\0")\n ))'))));
+test('ordinary sessions cannot use the pipe path',()=>fails(grade('ordinary',replace(fixed,'if (!pipeName) return','if (false) return'))));
+test('a stale cached recording root is rejected',()=>fails(grade('cached','let cached: any;\n'+replace(fixed,'await commands.getActiveDataDir()','await (cached ??= commands.getActiveDataDir())'))));
+test('failed native lookup retains the default root',()=>fails(grade('fallback',replace(fixed,'} catch {','} catch { baseDir = "/wrong-root";'))));
+test('missing source is a setup error',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toMatch(/Cannot find|ModuleNotFound/);expect(r.stderr).not.toContain('expect(received)');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 94c363ca4..48f9f8ae2 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-20.11",
+ "dataset_version": "2026-09-20.12",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1980,6 +1980,37 @@
],
"command": "\"${BUN_BIN:-bun}\" test crates/screenpipe-core/assets/extensions/workflow-workspace.eval.test.ts"
}
+ },
+ {
+ "id": "app-continued-pipe-recording-directory",
+ "name": "Keep continued Pipe chats with their active recording directory",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "a6a611a159f6d8704acc6ad00d21a6b87db42a95"
+ },
+ "base_ref": "d300cfd9c355ab3568bffe802a5132bc40211615",
+ "oracle_ref": "a6a611a159f6d8704acc6ad00d21a6b87db42a95",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/chat/pi-project-dir.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "workflow",
+ "compatibility"
+ ],
+ "prompt": "Continued Pipe chat follow-ups can select the account settings directory instead of the recording directory, separating their conversation from scheduled runs. Resolve continued Pipe session working directories beneath the currently active recording root, retaining drive identity, spaces and the exact valid pipe name. Read a changed root for later calls. If the native root lookup throws, returns an error or an empty value, fall back to the default home/.screenpipe/pipes directory. Preserve generic home/.screenpipe/pi-chat routing for ordinary chats, per-execution sessions, malformed IDs and unsafe pipe names (dot segments, slash/backslash, NUL or surrounding whitespace); those generic paths must not query native account or recording directories. Use the actual exported directory resolver and existing session parser with synthetic native path/command ports. No particular helper extraction is required. No real chats, scheduler/agent dispatch, native build or filesystem operations; this grades directory selection, not native persistence or complete account-state relocation.",
+ "agent_timeout_seconds": 1200,
+ "grader": {
+ "timeout_seconds": 90,
+ "fixtures": [
+ {
+ "local_path": "graders/pipe-directory.test.ts",
+ "destination_path": "evals/coding-agent/graders/pipe-directory.test.ts"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/pipe-directory.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/pipe-directory.test.ts b/evals/coding-agent/graders/pipe-directory.test.ts
new file mode 100644
index 000000000..73f752fa3
--- /dev/null
+++ b/evals/coding-agent/graders/pipe-directory.test.ts
@@ -0,0 +1,34 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {beforeEach,expect,mock,test} from 'bun:test';
+import * as events from '../../../apps/screenpipe-app-tauri/lib/events/types';
+mock.module('@/lib/events/types',()=>events);
+let active:any, activeReads:number, accountReads:number;
+mock.module('@tauri-apps/api/path',()=>({homeDir:async()=>'/synthetic/home',join:async(...parts:string[])=>parts.join('/')}));
+mock.module('@/lib/utils/tauri',()=>({commands:{
+ getScreenpipeBaseDir:async()=>{accountReads++;return {status:'ok',data:'/synthetic/account'};},
+ getActiveDataDir:async()=>{activeReads++;if(active instanceof Error)throw active;return active;},
+}}));
+const {piProjectDirForSession}=await import('../../../apps/screenpipe-app-tauri/lib/chat/pi-project-dir');
+beforeEach(()=>{active={status:'ok',data:'/synthetic/recordings'};activeReads=0;accountReads=0;});
+test('continued chat uses recordings rather than the distinct account directory',async()=>{
+ expect(await piProjectDirForSession('pipe:daily-brief:continuous')).toBe('/synthetic/recordings/pipes/daily-brief');
+});
+test('changed recording location is read again for later continuations',async()=>{
+ expect(await piProjectDirForSession('pipe:daily-brief:continuous')).toBe('/synthetic/recordings/pipes/daily-brief');
+ active={status:'ok',data:'/another drive/recordings'};
+ expect(await piProjectDirForSession('pipe:daily-brief:continuous')).toBe('/another drive/recordings/pipes/daily-brief');
+});
+test('a Windows-style recording root preserves spaces and drive identity',async()=>{
+ active={status:'ok',data:'E:/Screenpipe recording'};
+ expect(await piProjectDirForSession('pipe:weekly-note:continuous')).toBe('E:/Screenpipe recording/pipes/weekly-note');
+});
+for(const [name,value] of [['error status',{status:'error',error:'synthetic'}],['empty root',{status:'ok',data:''}],['thrown error',new Error('synthetic native error')]] as const)
+ test(`${name} falls back to the default pipe root`,async()=>{active=value;expect(await piProjectDirForSession('pipe:daily-brief:continuous')).toBe('/synthetic/home/.screenpipe/pipes/daily-brief');});
+for(const session of ['ordinary-session','pipe:daily-brief:42','pipe:daily-brief:NaN','pipe::continuous','pipe:../escape:continuous','pipe:bad\\name:continuous','pipe:..:continuous','pipe:.:continuous','pipe: leading:continuous','pipe:trailing :continuous','pipe:nul\0name:continuous'])
+ test(`ordinary, per-run or unsafe session ${JSON.stringify(session)} keeps generic chat isolated`,async()=>{
+ expect(await piProjectDirForSession(session)).toBe('/synthetic/home/.screenpipe/pi-chat');expect(activeReads).toBe(0);expect(accountReads).toBe(0);
+ });
+test('valid pipe names retain literal spaces without touching the filesystem',async()=>{
+ expect(await piProjectDirForSession('pipe:daily briefing:continuous')).toBe('/synthetic/recordings/pipes/daily briefing');
+});