Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): verify trusted runner chat admission

louisbd35658f56c9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index d24b2ec3c..47674a3fa 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1169,3 +1169,18 @@ parent/reference/current, an equivalent anchor scheme, unused fix, bypass, empty
output, one-role-only repair, dropped safe HTML, styles, broken fragments and
missing-source setup failure. These are corpus/grader checks, not agent trials,
a full browser exploit defense or proof of execution isolation.
+
+`ai-gateway-trusted-runner-admission` runs the actual HTTP entrypoint, authentication,
+Free-plan gates, chat handler and provider adapter with synthetic external ports.
+Fourteen outcomes cover successful machine and paid-human replies, forged or
+unconfigured credentials, Free background refusal, unknown plans, rate limits
+and invalid input. The parent fails five outcomes and preserves nine; the
+reference and current source pass fourteen. Twelve calibration controls reject
+partial fixes, header/prefix bypasses, unused correct code, fabricated success
+and blanket refusal, and accept an equivalent guard expression. Run
+`bun test evals/coding-agent/calibrate-trusted-runner.test.js` with gateway
+dependencies installed. The fetch boundary is installed before SDK imports,
+which can capture fetch at initialization. Graders and dependency links are
+installed only after the trajectory. These checks do not establish JWT
+cryptography, live inference, concurrent spend safety, complete current billing
+integration, agent isolation or model performance.
diff --git a/evals/coding-agent/calibrate-trusted-runner.test.js b/evals/coding-agent/calibrate-trusted-runner.test.js
new file mode 100644
index 000000000..d2db3ce26
--- /dev/null
+++ b/evals/coding-agent/calibrate-trusted-runner.test.js
@@ -0,0 +1,43 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='ai-gateway-trusted-runner-admission');
+const root=mkdtempSync(join(tmpdir(),'trusted-runner-calibration-'));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const template=join(root,'reference');mkdirSync(template);
+execFileSync('tar',['-x','-C',template],{input:execFileSync('git',['archive',item.oracle_ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:16*1024*1024})});
+const index='packages/ai-gateway/src/index.ts',free='packages/ai-gateway/src/services/free-chat-limit.ts',auth='packages/ai-gateway/src/utils/auth.ts';
+const sources=Object.fromEntries([index,free,auth].map(p=>[p,readFileSync(join(template,p),'utf8')]));
+const broken=Object.fromEntries([index,free].map(p=>[p,execFileSync('git',['show',`${item.base_ref}:${p}`],{cwd:repo,encoding:'utf8'})]));
+const fixture='packages/ai-gateway/src/test/eval-trusted-runner.test.ts';
+const bytes=readFileSync(join(import.meta.dir,'graders/trusted-runner-admission.fixture.ts.txt'));
+function replace(path,from,to){expect(sources[path].split(from)).toHaveLength(2);return sources[path].replace(from,to);}
+function grade(name,changes={},unused=false){
+ const cwd=join(root,name);cpSync(template,cwd,{recursive:true});
+ for(const [p,value] of Object.entries(changes)){if(value===null)rmSync(join(cwd,p));else writeFileSync(join(cwd,p),value);}
+ if(unused)writeFileSync(join(cwd,'unused-correct-entrypoint.ts'),sources[index]);
+ symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');
+ writeFileSync(join(cwd,fixture),bytes);
+ const result=spawnSync(process.execPath,['test',fixture],{cwd,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});
+ const receipts=process.env.EVAL_CALIBRATION_RESULTS_DIR;
+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),result.stdout??'');writeFileSync(join(receipts,name+'.stderr'),result.stderr??'');}
+ return result;
+}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}
+test('parent fails five intended outcomes and preserves nine',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('5 fail');expect(r.stderr).toContain('9 pass');});
+test('reference passes fourteen outcomes',()=>{const r=grade('fixed');expect(r.status).toBe(0);expect(r.stderr).toContain('14 pass');});
+test('unused correct entrypoint cannot repair the route',()=>fails(grade('unused',broken,true)));
+test('only the outer guard repaired is insufficient',()=>fails(grade('outer-only',{[free]:broken[free]})));
+test('only the Free gate repaired is insufficient',()=>fails(grade('inner-only',{[index]:broken[index]})));
+test('equivalent explicit service predicate passes',()=>{const r=grade('equivalent',{[index]:replace(index,'(!authResult.userId && authResult.service !== true)','!(authResult.userId || authResult.service === true)'),[free]:replace(free,'auth.service === true && hasPaidHostedAiPlan(auth)','hasPaidHostedAiPlan(auth) && auth.service === true')});expect(r.status).toBe(0);});
+test('client headers cannot manufacture service authority',()=>fails(grade('header-bypass',{[auth]:replace(auth,'env.AI_GATEWAY_SERVICE_TOKEN && token === env.AI_GATEWAY_SERVICE_TOKEN',"env.AI_GATEWAY_SERVICE_TOKEN && (token === env.AI_GATEWAY_SERVICE_TOKEN || request.headers.get('x-device-id') === 'cloud-runner')")})));
+test('credential prefix acceptance is rejected',()=>fails(grade('prefix',{[auth]:replace(auth,'token === env.AI_GATEWAY_SERVICE_TOKEN','token.startsWith(env.AI_GATEWAY_SERVICE_TOKEN)')})));
+test('blanket service refusal is rejected',()=>fails(grade('deny',{[index]:replace(index,"if (path === '/v1/chat/completions' && request.method === 'POST') {","if (path === '/v1/chat/completions' && request.method === 'POST') {\n if (authResult.service) return new Response('denied',{status:401});")})));
+test('fabricated successful response without provider effect is rejected',()=>fails(grade('fake',{[index]:replace(index,"if (path === '/v1/chat/completions' && request.method === 'POST') {","if (path === '/v1/chat/completions' && request.method === 'POST') {\n if (authResult.service) return Response.json({choices:[{message:{content:'synthetic accepted response'}}]});")})));
+test('ignoring rate refusal is rejected',()=>fails(grade('rate',{[index]:sources[index].replaceAll('if (!rateLimit.allowed && rateLimit.response) {','if (false && !rateLimit.allowed && rateLimit.response) {')})));
+test('missing route is setup error, not intended baseline failure',()=>{const r=grade('missing',{[index]:null});expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 4e7f588c0..1545ffe45 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -4072,6 +4072,46 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-html.config.mjs components/chat/eval-html-outcome.test.tsx"
}
+ },
+ {
+ "id": "ai-gateway-trusted-runner-admission",
+ "name": "Admit verified machine chat while preserving access refusals",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "25716cb17c4a11a75377df02b830891cc83229db"
+ },
+ "base_ref": "2168f5a3e118c28b79f6ba2ab15fff437c9cec33",
+ "oracle_ref": "25716cb17c4a11a75377df02b830891cc83229db",
+ "oracle_paths": [
+ "packages/ai-gateway/src/index.ts",
+ "packages/ai-gateway/src/services/free-chat-limit.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "access",
+ "machine-identity",
+ "background"
+ ],
+ "prompt": "A trusted backend runner supplies the configured service bearer but hosted chat rejects it because it has no human user ID. Repair the existing POST /v1/chat/completions path so that exact configured machine credential can complete both interactive and background requests for its requested paid model. The actual chat/provider path must make one successful request carrying the original user message and requested model, then return the provider answer. Preserve successful paid human chat. Missing, incorrect, public-user-ID, unsupported-scheme and unconfigured service credentials must return 401 before provider work, even with runner-like device, workload and background headers. A verified Free human background request must remain 403; missing human plan truth must remain 503. Machine and paid-human rate-limit refusals must remain 429 without provider work. A machine request with an empty model or empty message list must return 400 without provider work. Use synthetic credentials, entitlement, rate-limit, storage and provider ports only. Do not assume any particular private helper or guard structure. This case exercises authentication through HTTP dispatch and provider-response propagation, not JWT cryptography, live inference, SQL/concurrent spend accounting, full current entitlement integration, agent isolation or model quality.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/ai-gateway/node_modules",
+ "destination_path": "packages/ai-gateway/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/trusted-runner-admission.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/src/test/eval-trusted-runner.test.ts"
+ }
+ ],
+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" test src/test/eval-trusted-runner.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/trusted-runner-admission.fixture.ts.txt b/evals/coding-agent/graders/trusted-runner-admission.fixture.ts.txt
new file mode 100644
index 000000000..58b79baf5
--- /dev/null
+++ b/evals/coding-agent/graders/trusted-runner-admission.fixture.ts.txt
@@ -0,0 +1,35 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeEach, afterAll, expect, mock, test } from 'bun:test';
+let state: any;
+const originalFetch = globalThis.fetch;
+// Install before importing SDKs, which capture fetch at module initialization.
+globalThis.fetch = (...args: any[]) => { if (!state?.fetch) throw Error("network denied before fixture setup"); return state.fetch(...args); };
+mock.module('@clerk/backend',()=>({verifyToken:async(token:string)=>{
+ if(token==='eyJ.synthetic.user')return {sub:state.subject};throw Error('synthetic invalid token');
+}}));
+const {handleRequest}=await import('../index');
+const auth=await import('../utils/auth');
+const env:any={NODE_ENV:'production',AI_GATEWAY_SERVICE_TOKEN:'synthetic-machine-token',CLERK_SECRET_KEY:'synthetic-clerk',OPENAI_API_KEY:'synthetic-provider',CLOUDFLARE_AI_GATEWAY_ID:'synthetic',AI:{gateway:()=>({getUrl:async()=> 'https://provider.example.invalid/v1'})},
+ RATE_LIMITER:{idFromName:(x:any)=>x,get:()=>({fetch:async()=>Response.json({allowed:!state.rateDenied,standing:'good',remaining:10,reset_in:60,tier:'subscribed',rpm_limit:25})})},
+ DB:{prepare:(sql:string)=>({bind:(...values:any[])=>({first:async()=>null,run:async()=>({success:true,meta:{changes:1}}),all:async()=>({results:[]})}),run:async()=>({success:true})})}};
+const ctx={waitUntil:(p:Promise<any>)=>state.pending.push(p),passThroughOnException:()=>{}};
+let serial=0;
+beforeEach(()=>{env.AI_GATEWAY_SERVICE_TOKEN='synthetic-machine-token';state={subject:'user_synthetic_'+(++serial),plan:'pro',dispatch:[],pending:[],unexpected:[],rateDenied:false};auth.__resetAuthEntitlementCacheForTests?.();
+ state.fetch=async(input:any,init:any)=>{const url=input instanceof Request?input.url:String(input);if(['https://api.openai.com/v1/chat/completions','https://provider.example.invalid/v1/chat/completions'].includes(url)){state.dispatch.push({body:JSON.parse(init?.body ?? await input.text())});return Response.json({id:'synthetic-response',model:'gpt-5.6-sol',created:1,object:'chat.completion',choices:[{index:0,message:{role:'assistant',content:'synthetic accepted response'},finish_reason:'stop'}],usage:{prompt_tokens:1,completion_tokens:1,total_tokens:2}});}if(url==='https://screenpipe.com/api/user')return Response.json({success:true,user:state.unknown?{clerk_id:state.subject}:{clerk_id:state.subject,cloud_subscribed:state.plan==='pro',app_entitled:state.plan!=='none',subscription_plan:state.plan,entitlement:{active:state.plan!=='none',plan:state.plan,features:{app:state.plan!=='none',cloud:state.plan==='pro'}}}});
+ state.unexpected.push(url);throw Error('unexpected network forbidden');};});
+afterAll(()=>{globalThis.fetch=originalFetch;});
+async function request(token?:string,headers:any={},override:any={}){const r=await handleRequest(new Request('https://gateway.example.invalid/v1/chat/completions',{method:'POST',headers:{'content-type':'application/json',...(token?{authorization:'Bearer '+token}:{}),...headers},body:JSON.stringify({model:'gpt-5.6-sol',messages:[{role:'user',content:'synthetic task'}],...override})}),env,ctx);await Promise.all(state.pending);expect(state.unexpected).toEqual([]);return r;}
+async function accepted(token:string,headers:any={}){const r=await request(token,headers);expect(r.status).toBe(200);expect((await r.json()).choices[0].message.content).toBe('synthetic accepted response');expect(state.dispatch).toHaveLength(1);expect(state.dispatch[0].body.messages).toContainEqual({role:'user',content:'synthetic task'});}
+for(const background of [false,true])test('trusted machine successful dispatch background='+background,async()=>{await accepted(env.AI_GATEWAY_SERVICE_TOKEN,background?{'x-screenpipe-latency':'background'}:{});expect(state.dispatch[0].body.model).toBe('gpt-5.6-sol');});
+for(const token of [undefined,'synthetic-machine-token-wrong','user_public_fixture'])test('untrusted caller denied '+token,async()=>{const r=await request(token,{'x-screenpipe-latency':'background','x-screenpipe-workload':'pipe','x-device-id':'cloud-runner'});expect(r.status).toBe(401);expect(state.dispatch).toEqual([]);});
+test('Free user cannot spoof background authority',async()=>{state.plan='none';const r=await request('eyJ.synthetic.user',{'x-screenpipe-latency':'background'});expect(r.status).toBe(403);expect(state.dispatch).toEqual([]);});
+test('paid human still dispatches',()=>accepted('eyJ.synthetic.user'));
+test('machine obeys rate refusal',async()=>{state.rateDenied=true;const r=await request(env.AI_GATEWAY_SERVICE_TOKEN);expect(r.status).toBe(429);expect(state.dispatch).toEqual([]);});
+test('machine malformed model never dispatches',async()=>{const r=await request(env.AI_GATEWAY_SERVICE_TOKEN,{}, {model:''});expect(r.status).toBe(400);expect(state.dispatch).toEqual([]);});
+
+test('unconfigured service token cannot authorize',async()=>{env.AI_GATEWAY_SERVICE_TOKEN='';const r=await request('synthetic-machine-token');expect(r.status).toBe(401);expect(state.dispatch).toEqual([]);});
+test('unsupported credential scheme cannot authorize',async()=>{const r=await request(undefined,{authorization:'Basic synthetic-machine-token'});expect(r.status).toBe(401);expect(state.dispatch).toEqual([]);});
+test('paid human rate refusal remains enforced',async()=>{state.rateDenied=true;const r=await request('eyJ.synthetic.user');expect(r.status).toBe(429);expect(state.dispatch).toEqual([]);});
+test('unknown human plan cannot dispatch',async()=>{state.unknown=true;const r=await request('eyJ.synthetic.user');expect(r.status).toBe(503);expect(state.dispatch).toEqual([]);});
+test('empty machine messages remain invalid',async()=>{const r=await request(env.AI_GATEWAY_SERVICE_TOKEN,{}, {messages:[]});expect(r.status).toBe(400);expect(state.dispatch).toEqual([]);});