Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: verify cloud-session host boundaries through fetch interceptor

Louis Beaumont32e59fef0c63
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index b948e98ae..866851387 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -46,7 +46,18 @@ Fixtures appear only at grading; cold historical arms use the unchanged
histories, process-kill recovery, managed migration authority, live recording,
deployed storage, execution isolation or model performance.
-The current app corpus contains 124 git-mined regressions. See
+`app-cloud-auth-host-boundary` installs the actual fetch interceptor with
+synthetic HTTP, session and UI ports. Eleven outcomes cover local and unrelated
+401 responses, misleading hosts, genuine website expiry, successful requests,
+signed-out calls and network-error propagation. Request and response identities
+are preserved. The parent fails six outcomes and preserves five; the historical
+fix and current source pass all eleven. Run
+`bun test evals/coding-agent/calibrate-auth-host.test.js` for calibration.
+The hidden fixture appears only at grading and uses no dependency links.
+This does not establish native recording continuity, real authentication,
+account-switch race safety, browser integration, isolation or model performance.
+
+The current app corpus contains 125 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
diff --git a/evals/coding-agent/calibrate-auth-host.test.js b/evals/coding-agent/calibrate-auth-host.test.js
new file mode 100644
index 000000000..4d2d0376b
--- /dev/null
+++ b/evals/coding-agent/calibrate-auth-host.test.js
@@ -0,0 +1,31 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll, expect, test} from 'bun:test';
+import {execFileSync, spawnSync} from 'node:child_process';
+import {mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
+import {dirname, join, resolve} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),path='apps/screenpipe-app-tauri/lib/auth-guard.tsx';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-cloud-auth-host-boundary');
+const source=ref=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});
+const parent=source(item.base_ref),reference=source(item.oracle_ref),fixture=readFileSync(join(import.meta.dir,'graders/auth-host.fixture.tsx.txt'));
+const root=mkdtempSync(join(tmpdir(),'auth-host-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,body){
+ const cwd=join(root,name);mkdirSync(join(cwd,dirname(path)),{recursive:true});if(body!==null)writeFileSync(join(cwd,path),body);
+ if(name==='unused')writeFileSync(join(cwd,'unused-correct-module.tsx'),reference);
+ writeFileSync(join(cwd,'auth-host.eval.test.tsx'),fixture);
+ const args=['--no-env-file','test','auth-host.eval.test.tsx'];const r=spawnSync(process.execPath,args,{cwd,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});
+ if(process.env.EVAL_CALIBRATION_RESULTS_DIR){const d=process.env.EVAL_CALIBRATION_RESULTS_DIR;mkdirSync(d,{recursive:true});for(const k of ['stdout','stderr'])writeFileSync(join(d,name+'.'+k),r[k]??'');writeFileSync(join(d,name+'.json'),JSON.stringify({command:[process.execPath,...args],exit:r.status,error:r.error?.message??null,signal:r.signal},null,2)+'\n');}return r;
+}
+function pass(r){expect(r.error).toBeUndefined();expect(r.status).toBe(0);expect(r.stderr).toContain('11 pass');expect(r.stderr).toContain('0 fail');}
+function fail(r){expect(r.error).toBeUndefined();expect(r.status).toBe(1);expect(r.stderr).toContain('error: expect(received)');expect(r.stderr).not.toContain('GRADER_SETUP_ERROR');}
+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}
+test('parent fails six unwanted clears with five preserved outcomes',()=>{const r=grade('parent',parent);fail(r);expect(r.stderr).toContain('5 pass');expect(r.stderr).toContain('6 fail');});
+test('historical fix passes',()=>pass(grade('reference',reference)));
+test('current interceptor passes',()=>pass(grade('current',source('HEAD'))));
+test('equivalent host matching passes',()=>pass(grade('equivalent',replace(reference,'host.endsWith(".screenpi.pe")','/\\.screenpi\\.pe$/.test(host)').replace('host.endsWith(".screenpipe.com")','/\\.screenpipe\\.com$/.test(host)'))));
+test('unused correct module cannot hide broken interceptor',()=>fail(grade('unused',parent)));
+test('blanket sign-out suppression loses genuine expiry',()=>fail(grade('blanket',replace(reference,'if (isScreenpipeApi(url) && (res.status === 401 || res.status === 403))','if (false)'))));
+test('reconstructed response loses response identity',()=>fail(grade('response',replace(reference,'return res;','return new Response(null, { status: res.status });'))));
+test('lost request options are rejected',()=>fail(grade('arguments',replace(reference,'originalFetch.call(this, input, init)','originalFetch.call(this, input)'))));
+test('missing source is infrastructure, not intended regression',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('0 pass');expect(r.stderr).not.toContain('error: expect(received)');expect(r.stderr).toMatch(/ModuleNotFound|File not found|GRADER_SETUP_ERROR/);});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 7e666290e..bd327a27b 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -5121,6 +5121,41 @@
],
"command": "cargo test --offline --locked -p screenpipe-db --features storage-fault-injection --test eval_compressed_sqlx_upgrade -- --test-threads=1"
}
+ },
+ {
+ "id": "app-cloud-auth-host-boundary",
+ "name": "Preserve sessions across local and unrelated HTTP authorization failures",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "aad18c86ab9ca120840b8048f025b7ed7c7839ab"
+ },
+ "base_ref": "84e6faba1affb73707239f217a7c379e4a41c014",
+ "oracle_ref": "aad18c86ab9ca120840b8048f025b7ed7c7839ab",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/auth-guard.tsx"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/auth-guard.tsx"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "auth",
+ "session-preservation"
+ ],
+ "prompt": "The desktop fetch interceptor can clear a valid cloud session when a local connection returns 401 and its URL happens to contain a Screenpipe domain. Preserve the cloud session for localhost, IPv4 and IPv6 loopback responses, and for unrelated hosts mentioning screenpi.pe or screenpipe.com in their query or a misleading hostname. Keep session expiry for a signed-in request returning 401 from the genuine screenpipe.com or screenpi.pe website. Successful responses and requests without a signed-in token must not clear a session. Preserve fetch arguments and the original response object, and propagate network errors without clearing the session. Support string URLs, URL objects and Request objects.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 45,
+ "fixtures": [
+ {
+ "local_path": "graders/auth-host.fixture.tsx.txt",
+ "destination_path": "auth-host.eval.test.tsx"
+ }
+ ],
+ "command": "bun --no-env-file test auth-host.eval.test.tsx"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/auth-host.fixture.tsx.txt b/evals/coding-agent/graders/auth-host.fixture.tsx.txt
new file mode 100644
index 000000000..7f91abc3d
--- /dev/null
+++ b/evals/coding-agent/graders/auth-host.fixture.tsx.txt
@@ -0,0 +1,124 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll, expect, test} from 'bun:test';
+import {mkdtempSync, rmSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join, resolve} from 'node:path';
+const root=process.cwd(),entry=resolve(root,'apps/screenpipe-app-tauri/lib/auth-guard.tsx');
+const scratch=mkdtempSync(join(tmpdir(),'auth-host-grader-'));
+afterAll(()=>rmSync(scratch,{recursive:true,force:true}));
+const globals:any=globalThis;
+globals.fetch=()=>{throw Error('LIVE_NETWORK_FORBIDDEN')};
+const mocks:Record<string,string>={
+'react':`export default {createElement:()=>null};export const useEffect=()=>{},useRef=()=>{},useCallback=()=>{};`,
+'react/jsx-dev-runtime':`export const jsxDEV=()=>null,Fragment='fragment';`,
+'react/jsx-runtime':`export const jsx=()=>null,jsxs=()=>null,Fragment='fragment';`,
+'gt-react':`export const T=()=>null,useGT=()=>((s)=>s);`,
+'@/lib/hooks/use-settings':`export const useSettings=()=>{throw Error('UNEXPECTED_HOOK')};`,
+'@/components/ui/use-toast':`export const toast=()=>{};`,
+'@/components/ui/toast':`export const ToastAction=()=>null;`,
+'posthog-js':`export default {capture:()=>{globalThis.__captures++}};`,
+'@/lib/utils/tauri':`export const commands={setCloudToken:()=>{throw Error('UNEXPECTED_NATIVE')}};`,
+'@/lib/web-url':`export const PROD_WEB_BASE='https://screenpipe.com',screenpipeWebBase=(x)=>x,screenpipeWebUrl=()=>{throw Error('UNEXPECTED_LOGIN')};`,
+'@tauri-apps/plugin-shell':`export const open=()=>{throw Error('UNEXPECTED_SHELL')};`
+};
+const scenarios=[
+ {
+ "id": "local-query",
+ "url": "http://localhost:3030/connections/events?instance=fixture%40screenpi.pe",
+ "status": 401,
+ "clear": 0
+ },
+ {
+ "id": "loopback-query",
+ "url": "http://127.0.0.1:3030/[email protected]",
+ "status": 401,
+ "clear": 0
+ },
+ {
+ "id": "ipv6-query",
+ "url": "http://[::1]:3030/connections?ref=screenpipe.com",
+ "status": 401,
+ "clear": 0
+ },
+ {
+ "id": "external-query",
+ "url": "https://fixture.example/?ref=screenpi.pe",
+ "status": 401,
+ "clear": 0
+ },
+ {
+ "id": "suffix-host",
+ "url": "https://notscreenpipe.com/api",
+ "status": 401,
+ "clear": 0
+ },
+ {
+ "id": "prefix-host",
+ "url": "https://screenpipe.com.fixture.example/api",
+ "status": 401,
+ "clear": 0
+ },
+ {
+ "id": "cloud-401",
+ "url": "https://screenpipe.com/api/user",
+ "status": 401,
+ "clear": 1
+ },
+ {
+ "id": "legacy-cloud-401",
+ "url": "https://screenpi.pe/api/user",
+ "status": 401,
+ "clear": 1
+ },
+ {
+ "id": "cloud-200",
+ "url": "https://screenpipe.com/api/user",
+ "status": 200,
+ "clear": 0
+ },
+ {
+ "id": "signed-out-cloud",
+ "url": "https://screenpipe.com/api/user",
+ "status": 401,
+ "clear": 0,
+ "noToken": true
+ },
+ {
+ "id": "network-error",
+ "url": "https://screenpipe.com/api/user",
+ "status": 401,
+ "clear": 0,
+ "networkError": true
+ }
+];
+// Build and exercise every scenario before registering assertions. Missing
+// source, unsupported adapters and setup exceptions are infrastructure errors.
+const built=await Bun.build({entrypoints:[entry],target:'bun',plugins:[{name:'synthetic-ports',setup(b){
+ b.onResolve({filter:/.*/},(a)=>{
+ if(mocks[a.path])return {path:a.path,namespace:'mock'};
+ if(a.path.startsWith('@/'))return {path:resolve(root,'apps/screenpipe-app-tauri',a.path.slice(2))};
+ if(a.path===entry||a.path.startsWith('.')||a.path.startsWith('/'))return;
+ throw Error('UNSUPPORTED_PORT '+a.path);
+ });
+ b.onLoad({filter:/.*/,namespace:'mock'},a=>({contents:mocks[a.path],loader:'js'}));
+}}]});
+if(!built.success)throw Error('GRADER_SETUP_ERROR '+built.logs.join('\n'));
+const bundle=join(scratch,'candidate.js');await Bun.write(bundle,built.outputs[0]);
+const rows=[];
+for(const [index,s] of scenarios.entries()){
+ let cleared=0,seenInput:any,seenInit:any;globals.__captures=0;
+ const response=new Response('synthetic',{status:s.status}),networkError=new Error('synthetic network failure');
+ globals.window={location:{href:'http://localhost:1420/'},fetch:async(input:any,init:any)=>{seenInput=input;seenInit=init;if(s.networkError)throw networkError;return response}};
+ const mod=await import(bundle+`?scenario=${index}`);
+ mod.installAuthInterceptor(()=>s.noToken?undefined:'synthetic-token',async()=>{cleared++});
+ const input=index%3===0?new URL(s.url):index%3===1?new Request(s.url):s.url;
+ const init={headers:{'x-synthetic':'1'}};let actual,err;
+ try{actual=await globals.window.fetch(input,init)}catch(e){err=e}
+ rows.push({spec:s,cleared,requestPreserved:seenInput===input&&seenInit===init,responsePreserved:s.networkError?err===networkError:actual===response});
+}
+for(const r of rows)test(r.spec.id,()=>{
+ expect(r.cleared).toBe(r.spec.clear);
+ expect(r.requestPreserved).toBe(true);
+ expect(r.responsePreserved).toBe(true);
+});