Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover model catalog session freshness

louis65e9c78d7ff8
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 33f635ef1..a1f7c71f5 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,18 @@ The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-model-catalog-session-freshness` runs the actual model-catalog hook with
+synthetic settings and HTTP ports. Ten outcomes cover hydration, token changes,
+late responses and bodies, loading ownership, failure recovery and preserved
+anonymous access and metadata. The parent fails five outcomes and preserves five;
+the reference and current hook pass ten. Run
+`bun test evals/coding-agent/calibrate-model-catalog.test.js` for eleven controls,
+including equivalent guards, unused correct code, stale state, empty catalogs,
+lost metadata and missing-source classification. The current check also runs the
+actual gateway URL adapter with synthetic native ports. Fixtures and dependency
+links appear only at grading. This does not establish native credential security,
+live entitlement delivery, execution isolation or model improvement.
+
`app-shared-running-pipes-lifecycle` mounts the actual React hook and Zustand
store with synthetic HTTP/event ports and timers. Ten outcomes cover overlapping
consumers, final unmount, remount, delayed fetch and event-bus completion, pipe
diff --git a/evals/coding-agent/calibrate-model-catalog.test.js b/evals/coding-agent/calibrate-model-catalog.test.js
new file mode 100644
index 000000000..5ba103dba
--- /dev/null
+++ b/evals/coding-agent/calibrate-model-catalog.test.js
@@ -0,0 +1,38 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-model-catalog-session-freshness');
+const hook=`${app}/lib/hooks/use-pi-models.ts`, root=mkdtempSync(join(tmpdir(),'model-catalog-calibration-'));
+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function replace(cwd,from,to){const p=join(cwd,hook),text=readFileSync(p,'utf8');expect(text.split(from)).toHaveLength(2);writeFileSync(p,text.replace(from,to));}
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);
+ // Exact product modules only; no agent, network, native process or user config.
+ for(const file of [hook,...(ref==='HEAD'?[`${app}/lib/ai-gateway-url.ts`]:[])]){const dest=join(cwd,file);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,execFileSync('git',['show',`${ref}:${file}`],{cwd:repo}));}
+ for(const file of ['lib/hooks/use-settings.ts','lib/utils/tauri.ts','lib/http/tauri-fetch.ts']){const dest=join(cwd,app,file);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,'throw new Error("unmocked external port");');}
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);
+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});
+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message,classification:classifyGraderError(r)},null,2));}
+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;
+}
+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('10 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/);}
+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(classifyGraderError(r)).toBeNull();expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/);}
+test('parent fails session outcomes and preserves nearby behavior',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('5 failed | 5 passed')},60000);
+test('reference passes',()=>passes(grade('reference')),60000);
+test('current passes',()=>passes(grade('current','HEAD')),60000);
+test('declared oracle path repairs parent',()=>passes(grade('oracle-only',item.base_ref,cwd=>{const patch=execFileSync('git',['diff',item.base_ref,item.oracle_ref,'--',...item.oracle_paths],{cwd:repo});execFileSync('git',['apply','--whitespace=nowarn','-'],{cwd,input:patch})})),60000);
+test('unused correct hook cannot hide active broken hook',()=>fails(grade('unused',item.oracle_ref,cwd=>{writeFileSync(join(cwd,app,'unused-correct-hook.ts'),readFileSync(join(cwd,hook)));writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}))})),60000);
+test('equivalent stale-request guard preserves outcomes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('generation !== requestGeneration.current','!(generation === requestGeneration.current)'))})),60000);
+test('lost body and header guards are rejected',()=>fails(grade('stale',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('if (generation !== requestGeneration.current) return;',''))})),60000);
+test('stale completion clearing loading is rejected',()=>fails(grade('loading',item.oracle_ref,cwd=>replace(cwd,'if (generation === requestGeneration.current) setIsLoading(false);','setIsLoading(false);'))),60000);
+test('blanket catalog suppression is rejected',()=>fails(grade('empty',item.oracle_ref,cwd=>replace(cwd,'setPiModels(models);','setPiModels([]);'))),60000);
+test('lost model names are rejected',()=>fails(grade('metadata',item.oracle_ref,cwd=>replace(cwd,'name: model.name || model.id,','name: model.id,'))),60000);
+test('missing hook is a setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(classifyGraderError(r)).toBe('vitest_collection_error')},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 86ea3e864..69a1487c4 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -4502,6 +4502,53 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-running-pipes.config.mjs"
}
+ },
+ {
+ "id": "app-model-catalog-session-freshness",
+ "name": "Keep model catalogs tied to the current hydrated session",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "757808f1043865752a05013e1a77973ce880f6f7",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5139"
+ },
+ "base_ref": "074e5512469da7a74fa08711484a5b7f859706fb",
+ "oracle_ref": "757808f1043865752a05013e1a77973ce880f6f7",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/hooks/use-pi-models.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/hooks/use-pi-models.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "session",
+ "stale-response",
+ "model-catalog"
+ ],
+ "prompt": "The model picker can load before settings hydrate and later replace a new session\u2019s catalog with an older request. Fix the existing usePiModels hook. Do not request models until settings are loaded. Once loaded, support anonymous requests and include the current bearer token when present; refresh after token changes. Keep the last successful catalog visible during refresh and after HTTP or network failures. Responses from an abandoned session, including delayed response bodies, must never replace the current catalog or clear a newer request\u2019s loading state. Unloading settings must invalidate pending work and report no loading. Preserve the existing hook interface, model metadata, name fallback to ID, and first-occurrence deduplication by model ID. This task concerns React behavior with controlled settings and HTTP ports; native credential persistence is outside scope.",
+ "agent_timeout_seconds": 1200,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/model-catalog.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-model-catalog.test.tsx"
+ },
+ {
+ "local_path": "graders/model-catalog.config.mjs",
+ "destination_path": "apps/screenpipe-app-tauri/eval-model-catalog.config.mjs"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-model-catalog.config.mjs"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/model-catalog.config.mjs b/evals/coding-agent/graders/model-catalog.config.mjs
new file mode 100644
index 000000000..44ca9801e
--- /dev/null
+++ b/evals/coding-agent/graders/model-catalog.config.mjs
@@ -0,0 +1,5 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { defineConfig } from 'vitest/config';
+import path from 'node:path';
+export default defineConfig({esbuild:{jsx:'automatic'},resolve:{alias:{'@':path.resolve('.')}},test:{include:['eval-model-catalog.test.tsx'],environment:'jsdom',pool:'forks',poolOptions:{forks:{singleFork:true}}},css:{postcss:{plugins:[]}}});
diff --git a/evals/coding-agent/graders/model-catalog.fixture.tsx.txt b/evals/coding-agent/graders/model-catalog.fixture.tsx.txt
new file mode 100644
index 000000000..931c97ab4
--- /dev/null
+++ b/evals/coding-agent/graders/model-catalog.fixture.tsx.txt
@@ -0,0 +1,28 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { act, cleanup, renderHook } from '@testing-library/react';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
+const state = vi.hoisted(() => ({ loaded: true, token: 'session-a' }));
+vi.mock('@/lib/hooks/use-settings', () => ({ useSettings: () => ({ isSettingsLoaded: state.loaded, settings: { user: { token: state.token } } }) }));
+vi.mock('@/lib/utils/tauri', () => ({ commands: { getScreenpipeAiGatewayUrl: async () => ({ status: 'ok', data: 'https://api.screenpipe.com/v1' }) } }));
+vi.mock('@/lib/http/tauri-fetch', () => ({ tauriFetchWithDeadline: () => { throw new Error('unexpected native transport'); } }));
+import { usePiModels } from './lib/hooks/use-pi-models';
+function deferred() { let resolve!: (v:any)=>void, reject!: (e:any)=>void; const promise = new Promise<any>((a,b)=>{resolve=a;reject=b}); return {promise,resolve,reject}; }
+let requests: { reply: ReturnType<typeof deferred>, init: RequestInit }[];
+beforeEach(() => { state.loaded=true; state.token='session-a'; requests=[]; vi.stubGlobal('fetch',vi.fn((_url,init)=>{const reply=deferred();requests.push({reply,init});return reply.promise;})); });
+afterEach(()=>{cleanup();vi.unstubAllGlobals();});
+const rows=(id:string)=>[{id}];
+const settle=async (fn:()=>void)=>{await act(async()=>{});await act(async()=>{fn();await new Promise(r=>setTimeout(r,0));});};
+async function answer(index:number,data:any[],status=200){await settle(()=>requests[index].reply.resolve({ok:status===200,status,json:async()=>({data})}));}
+async function switchToken(r:any,token='session-b'){state.token=token;r.rerender();await act(async()=>{});}
+const ids=(r:any)=>r.result.current.piModels.map((m:any)=>m.id);
+test('waits for hydration then loads the current bearer',async()=>{state.loaded=false;const r=renderHook(usePiModels);await act(async()=>{});expect(requests).toHaveLength(0);expect(r.result.current.isLoading).toBe(false);state.loaded=true;r.rerender();await act(async()=>{});expect(requests).toHaveLength(1);expect(new Headers(requests[0].init.headers).get('Authorization')).toBe('Bearer session-a');await answer(0,rows('ready'));expect(ids(r)).toEqual(['ready']);});
+test('loaded anonymous sessions still load without bearer',async()=>{state.token='';const r=renderHook(usePiModels);await act(async()=>{});expect(requests).toHaveLength(1);expect(new Headers(requests[0].init.headers).has('Authorization')).toBe(false);await answer(0,rows('free'));expect(ids(r)).toEqual(['free']);expect(r.result.current.isLoading).toBe(false);});
+test('refresh keeps the catalog and uses the replacement token',async()=>{const r=renderHook(usePiModels);await answer(0,rows('old'));await switchToken(r);expect(ids(r)).toEqual(['old']);expect(r.result.current.isLoading).toBe(true);expect(new Headers(requests[1].init.headers).get('Authorization')).toBe('Bearer session-b');await answer(1,rows('new'));expect(ids(r)).toEqual(['new']);expect(r.result.current.isLoading).toBe(false);});
+test('late response cannot overwrite a newer catalog',async()=>{const r=renderHook(usePiModels);await switchToken(r);await answer(1,rows('new'));await answer(0,rows('stale'));expect(ids(r)).toEqual(['new']);});
+test('late body cannot overwrite a newer catalog',async()=>{const r=renderHook(usePiModels);const body=deferred();await settle(()=>requests[0].reply.resolve({ok:true,json:()=>body.promise}));await switchToken(r);await answer(1,rows('new'));await settle(()=>body.resolve({data:rows('stale')}));expect(ids(r)).toEqual(['new']);});
+test('stale completion cannot clear current loading',async()=>{const r=renderHook(usePiModels);await switchToken(r);await answer(0,rows('stale'));expect(r.result.current.isLoading).toBe(true);await answer(1,rows('new'));expect(r.result.current.isLoading).toBe(false);});
+test('settings unload invalidates the pending body and stops loading',async()=>{const r=renderHook(usePiModels);await answer(0,rows('saved'));await switchToken(r);const body=deferred();await settle(()=>requests[1].reply.resolve({ok:true,json:()=>body.promise}));state.loaded=false;r.rerender();expect(r.result.current.isLoading).toBe(false);await settle(()=>body.resolve({data:rows('stale')}));expect(ids(r)).toEqual(['saved']);expect(requests).toHaveLength(2);});
+test('HTTP refusal preserves the last good catalog',async()=>{const r=renderHook(usePiModels);await answer(0,rows('saved'));await switchToken(r);await answer(1,[],401);expect(ids(r)).toEqual(['saved']);expect(r.result.current.isLoading).toBe(false);});
+test('network failure preserves the catalog and allows later recovery',async()=>{const r=renderHook(usePiModels);await answer(0,rows('saved'));await switchToken(r);await settle(()=>requests[1].reply.reject(new Error('synthetic offline')));expect(ids(r)).toEqual(['saved']);expect(r.result.current.isLoading).toBe(false);await switchToken(r,'session-c');await answer(2,rows('recovered'));expect(ids(r)).toEqual(['recovered']);});
+test('deduplicates by first ID and preserves metadata and name fallback',async()=>{const r=renderHook(usePiModels);const first={id:'model-a',name:'First',free:true,cost_tier:'low',recommended_for:['chat'],warning:'synthetic warning',locked:false,health:{status:'ok',error_rate_5m:0}};await answer(0,[first,{id:'model-a',name:'Duplicate'},{id:'model-b'}]);expect(r.result.current.piModels).toHaveLength(2);expect(r.result.current.piModels[0]).toMatchObject(first);expect(r.result.current.piModels[1]).toMatchObject({id:'model-b',name:'model-b'});});