YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: verify cloud-session host boundaries through fetch interceptor
Louis Beaumont32e59fef0c63
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b948e98ae..866851387 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -46,7 +46,18 @@ Fixtures appear only at grading; cold historical arms use the unchangedhistories, process-kill recovery, managed migration authority, live recording,deployed storage, execution isolation or model performance.-The current app corpus contains 124 git-mined regressions. See+`app-cloud-auth-host-boundary` installs the actual fetch interceptor with+synthetic HTTP, session and UI ports. Eleven outcomes cover local and unrelated+401 responses, misleading hosts, genuine website expiry, successful requests,+signed-out calls and network-error propagation. Request and response identities+are preserved. The parent fails six outcomes and preserves five; the historical+fix and current source pass all eleven. Run+`bun test evals/coding-agent/calibrate-auth-host.test.js` for calibration.+The hidden fixture appears only at grading and uses no dependency links.+This does not establish native recording continuity, real authentication,+account-switch race safety, browser integration, isolation or model performance.++The current app corpus contains 125 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.diff --git a/evals/coding-agent/calibrate-auth-host.test.js b/evals/coding-agent/calibrate-auth-host.test.jsnew file mode 100644index 000000000..4d2d0376b--- /dev/null+++ b/evals/coding-agent/calibrate-auth-host.test.js@@ -0,0 +1,31 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll, expect, test} from 'bun:test';+import {execFileSync, spawnSync} from 'node:child_process';+import {mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';+import {dirname, join, resolve} from 'node:path';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..'),path='apps/screenpipe-app-tauri/lib/auth-guard.tsx';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-cloud-auth-host-boundary');+const source=ref=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});+const parent=source(item.base_ref),reference=source(item.oracle_ref),fixture=readFileSync(join(import.meta.dir,'graders/auth-host.fixture.tsx.txt'));+const root=mkdtempSync(join(tmpdir(),'auth-host-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,body){+ const cwd=join(root,name);mkdirSync(join(cwd,dirname(path)),{recursive:true});if(body!==null)writeFileSync(join(cwd,path),body);+ if(name==='unused')writeFileSync(join(cwd,'unused-correct-module.tsx'),reference);+ writeFileSync(join(cwd,'auth-host.eval.test.tsx'),fixture);+ const args=['--no-env-file','test','auth-host.eval.test.tsx'];const r=spawnSync(process.execPath,args,{cwd,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});+ if(process.env.EVAL_CALIBRATION_RESULTS_DIR){const d=process.env.EVAL_CALIBRATION_RESULTS_DIR;mkdirSync(d,{recursive:true});for(const k of ['stdout','stderr'])writeFileSync(join(d,name+'.'+k),r[k]??'');writeFileSync(join(d,name+'.json'),JSON.stringify({command:[process.execPath,...args],exit:r.status,error:r.error?.message??null,signal:r.signal},null,2)+'\n');}return r;+}+function pass(r){expect(r.error).toBeUndefined();expect(r.status).toBe(0);expect(r.stderr).toContain('11 pass');expect(r.stderr).toContain('0 fail');}+function fail(r){expect(r.error).toBeUndefined();expect(r.status).toBe(1);expect(r.stderr).toContain('error: expect(received)');expect(r.stderr).not.toContain('GRADER_SETUP_ERROR');}+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}+test('parent fails six unwanted clears with five preserved outcomes',()=>{const r=grade('parent',parent);fail(r);expect(r.stderr).toContain('5 pass');expect(r.stderr).toContain('6 fail');});+test('historical fix passes',()=>pass(grade('reference',reference)));+test('current interceptor passes',()=>pass(grade('current',source('HEAD'))));+test('equivalent host matching passes',()=>pass(grade('equivalent',replace(reference,'host.endsWith(".screenpi.pe")','/\\.screenpi\\.pe$/.test(host)').replace('host.endsWith(".screenpipe.com")','/\\.screenpipe\\.com$/.test(host)'))));+test('unused correct module cannot hide broken interceptor',()=>fail(grade('unused',parent)));+test('blanket sign-out suppression loses genuine expiry',()=>fail(grade('blanket',replace(reference,'if (isScreenpipeApi(url) && (res.status === 401 || res.status === 403))','if (false)'))));+test('reconstructed response loses response identity',()=>fail(grade('response',replace(reference,'return res;','return new Response(null, { status: res.status });'))));+test('lost request options are rejected',()=>fail(grade('arguments',replace(reference,'originalFetch.call(this, input, init)','originalFetch.call(this, input)'))));+test('missing source is infrastructure, not intended regression',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('0 pass');expect(r.stderr).not.toContain('error: expect(received)');expect(r.stderr).toMatch(/ModuleNotFound|File not found|GRADER_SETUP_ERROR/);});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 7e666290e..bd327a27b 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5121,6 +5121,41 @@],"command": "cargo test --offline --locked -p screenpipe-db --features storage-fault-injection --test eval_compressed_sqlx_upgrade -- --test-threads=1"}+ },+ {+ "id": "app-cloud-auth-host-boundary",+ "name": "Preserve sessions across local and unrelated HTTP authorization failures",+ "source": {+ "kind": "git_regression",+ "fix_commit": "aad18c86ab9ca120840b8048f025b7ed7c7839ab"+ },+ "base_ref": "84e6faba1affb73707239f217a7c379e4a41c014",+ "oracle_ref": "aad18c86ab9ca120840b8048f025b7ed7c7839ab",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/auth-guard.tsx"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/auth-guard.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "auth",+ "session-preservation"+ ],+ "prompt": "The desktop fetch interceptor can clear a valid cloud session when a local connection returns 401 and its URL happens to contain a Screenpipe domain. Preserve the cloud session for localhost, IPv4 and IPv6 loopback responses, and for unrelated hosts mentioning screenpi.pe or screenpipe.com in their query or a misleading hostname. Keep session expiry for a signed-in request returning 401 from the genuine screenpipe.com or screenpi.pe website. Successful responses and requests without a signed-in token must not clear a session. Preserve fetch arguments and the original response object, and propagate network errors without clearing the session. Support string URLs, URL objects and Request objects.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 45,+ "fixtures": [+ {+ "local_path": "graders/auth-host.fixture.tsx.txt",+ "destination_path": "auth-host.eval.test.tsx"+ }+ ],+ "command": "bun --no-env-file test auth-host.eval.test.tsx"+ }}]}diff --git a/evals/coding-agent/graders/auth-host.fixture.tsx.txt b/evals/coding-agent/graders/auth-host.fixture.tsx.txtnew file mode 100644index 000000000..7f91abc3d--- /dev/null+++ b/evals/coding-agent/graders/auth-host.fixture.tsx.txt@@ -0,0 +1,124 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll, expect, test} from 'bun:test';+import {mkdtempSync, rmSync} from 'node:fs';+import {tmpdir} from 'node:os';+import {join, resolve} from 'node:path';+const root=process.cwd(),entry=resolve(root,'apps/screenpipe-app-tauri/lib/auth-guard.tsx');+const scratch=mkdtempSync(join(tmpdir(),'auth-host-grader-'));+afterAll(()=>rmSync(scratch,{recursive:true,force:true}));+const globals:any=globalThis;+globals.fetch=()=>{throw Error('LIVE_NETWORK_FORBIDDEN')};+const mocks:Record<string,string>={+'react':`export default {createElement:()=>null};export const useEffect=()=>{},useRef=()=>{},useCallback=()=>{};`,+'react/jsx-dev-runtime':`export const jsxDEV=()=>null,Fragment='fragment';`,+'react/jsx-runtime':`export const jsx=()=>null,jsxs=()=>null,Fragment='fragment';`,+'gt-react':`export const T=()=>null,useGT=()=>((s)=>s);`,+'@/lib/hooks/use-settings':`export const useSettings=()=>{throw Error('UNEXPECTED_HOOK')};`,+'@/components/ui/use-toast':`export const toast=()=>{};`,+'@/components/ui/toast':`export const ToastAction=()=>null;`,+'posthog-js':`export default {capture:()=>{globalThis.__captures++}};`,+'@/lib/utils/tauri':`export const commands={setCloudToken:()=>{throw Error('UNEXPECTED_NATIVE')}};`,+'@/lib/web-url':`export const PROD_WEB_BASE='https://screenpipe.com',screenpipeWebBase=(x)=>x,screenpipeWebUrl=()=>{throw Error('UNEXPECTED_LOGIN')};`,+'@tauri-apps/plugin-shell':`export const open=()=>{throw Error('UNEXPECTED_SHELL')};`+};+const scenarios=[+ {+ "id": "local-query",+ "url": "http://localhost:3030/connections/events?instance=fixture%40screenpi.pe",+ "status": 401,+ "clear": 0+ },+ {+ "id": "loopback-query",+ "url": "http://127.0.0.1:3030/[email protected]",+ "status": 401,+ "clear": 0+ },+ {+ "id": "ipv6-query",+ "url": "http://[::1]:3030/connections?ref=screenpipe.com",+ "status": 401,+ "clear": 0+ },+ {+ "id": "external-query",+ "url": "https://fixture.example/?ref=screenpi.pe",+ "status": 401,+ "clear": 0+ },+ {+ "id": "suffix-host",+ "url": "https://notscreenpipe.com/api",+ "status": 401,+ "clear": 0+ },+ {+ "id": "prefix-host",+ "url": "https://screenpipe.com.fixture.example/api",+ "status": 401,+ "clear": 0+ },+ {+ "id": "cloud-401",+ "url": "https://screenpipe.com/api/user",+ "status": 401,+ "clear": 1+ },+ {+ "id": "legacy-cloud-401",+ "url": "https://screenpi.pe/api/user",+ "status": 401,+ "clear": 1+ },+ {+ "id": "cloud-200",+ "url": "https://screenpipe.com/api/user",+ "status": 200,+ "clear": 0+ },+ {+ "id": "signed-out-cloud",+ "url": "https://screenpipe.com/api/user",+ "status": 401,+ "clear": 0,+ "noToken": true+ },+ {+ "id": "network-error",+ "url": "https://screenpipe.com/api/user",+ "status": 401,+ "clear": 0,+ "networkError": true+ }+];+// Build and exercise every scenario before registering assertions. Missing+// source, unsupported adapters and setup exceptions are infrastructure errors.+const built=await Bun.build({entrypoints:[entry],target:'bun',plugins:[{name:'synthetic-ports',setup(b){+ b.onResolve({filter:/.*/},(a)=>{+ if(mocks[a.path])return {path:a.path,namespace:'mock'};+ if(a.path.startsWith('@/'))return {path:resolve(root,'apps/screenpipe-app-tauri',a.path.slice(2))};+ if(a.path===entry||a.path.startsWith('.')||a.path.startsWith('/'))return;+ throw Error('UNSUPPORTED_PORT '+a.path);+ });+ b.onLoad({filter:/.*/,namespace:'mock'},a=>({contents:mocks[a.path],loader:'js'}));+}}]});+if(!built.success)throw Error('GRADER_SETUP_ERROR '+built.logs.join('\n'));+const bundle=join(scratch,'candidate.js');await Bun.write(bundle,built.outputs[0]);+const rows=[];+for(const [index,s] of scenarios.entries()){+ let cleared=0,seenInput:any,seenInit:any;globals.__captures=0;+ const response=new Response('synthetic',{status:s.status}),networkError=new Error('synthetic network failure');+ globals.window={location:{href:'http://localhost:1420/'},fetch:async(input:any,init:any)=>{seenInput=input;seenInit=init;if(s.networkError)throw networkError;return response}};+ const mod=await import(bundle+`?scenario=${index}`);+ mod.installAuthInterceptor(()=>s.noToken?undefined:'synthetic-token',async()=>{cleared++});+ const input=index%3===0?new URL(s.url):index%3===1?new Request(s.url):s.url;+ const init={headers:{'x-synthetic':'1'}};let actual,err;+ try{actual=await globals.window.fetch(input,init)}catch(e){err=e}+ rows.push({spec:s,cleared,requestPreserved:seenInput===input&&seenInit===init,responsePreserved:s.networkError?err===networkError:actual===response});+}+for(const r of rows)test(r.spec.id,()=>{+ expect(r.cleared).toBe(r.spec.clear);+ expect(r.requestPreserved).toBe(true);+ expect(r.responsePreserved).toBe(true);+});