YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): verify trusted runner chat admission
louisbd35658f56c9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex d24b2ec3c..47674a3fa 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1169,3 +1169,18 @@ parent/reference/current, an equivalent anchor scheme, unused fix, bypass, emptyoutput, one-role-only repair, dropped safe HTML, styles, broken fragments andmissing-source setup failure. These are corpus/grader checks, not agent trials,a full browser exploit defense or proof of execution isolation.++`ai-gateway-trusted-runner-admission` runs the actual HTTP entrypoint, authentication,+Free-plan gates, chat handler and provider adapter with synthetic external ports.+Fourteen outcomes cover successful machine and paid-human replies, forged or+unconfigured credentials, Free background refusal, unknown plans, rate limits+and invalid input. The parent fails five outcomes and preserves nine; the+reference and current source pass fourteen. Twelve calibration controls reject+partial fixes, header/prefix bypasses, unused correct code, fabricated success+and blanket refusal, and accept an equivalent guard expression. Run+`bun test evals/coding-agent/calibrate-trusted-runner.test.js` with gateway+dependencies installed. The fetch boundary is installed before SDK imports,+which can capture fetch at initialization. Graders and dependency links are+installed only after the trajectory. These checks do not establish JWT+cryptography, live inference, concurrent spend safety, complete current billing+integration, agent isolation or model performance.diff --git a/evals/coding-agent/calibrate-trusted-runner.test.js b/evals/coding-agent/calibrate-trusted-runner.test.jsnew file mode 100644index 000000000..d2db3ce26--- /dev/null+++ b/evals/coding-agent/calibrate-trusted-runner.test.js@@ -0,0 +1,43 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='ai-gateway-trusted-runner-admission');+const root=mkdtempSync(join(tmpdir(),'trusted-runner-calibration-'));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+const template=join(root,'reference');mkdirSync(template);+execFileSync('tar',['-x','-C',template],{input:execFileSync('git',['archive',item.oracle_ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:16*1024*1024})});+const index='packages/ai-gateway/src/index.ts',free='packages/ai-gateway/src/services/free-chat-limit.ts',auth='packages/ai-gateway/src/utils/auth.ts';+const sources=Object.fromEntries([index,free,auth].map(p=>[p,readFileSync(join(template,p),'utf8')]));+const broken=Object.fromEntries([index,free].map(p=>[p,execFileSync('git',['show',`${item.base_ref}:${p}`],{cwd:repo,encoding:'utf8'})]));+const fixture='packages/ai-gateway/src/test/eval-trusted-runner.test.ts';+const bytes=readFileSync(join(import.meta.dir,'graders/trusted-runner-admission.fixture.ts.txt'));+function replace(path,from,to){expect(sources[path].split(from)).toHaveLength(2);return sources[path].replace(from,to);}+function grade(name,changes={},unused=false){+ const cwd=join(root,name);cpSync(template,cwd,{recursive:true});+ for(const [p,value] of Object.entries(changes)){if(value===null)rmSync(join(cwd,p));else writeFileSync(join(cwd,p),value);}+ if(unused)writeFileSync(join(cwd,'unused-correct-entrypoint.ts'),sources[index]);+ symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');+ writeFileSync(join(cwd,fixture),bytes);+ const result=spawnSync(process.execPath,['test',fixture],{cwd,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});+ const receipts=process.env.EVAL_CALIBRATION_RESULTS_DIR;+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),result.stdout??'');writeFileSync(join(receipts,name+'.stderr'),result.stderr??'');}+ return result;+}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}+test('parent fails five intended outcomes and preserves nine',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('5 fail');expect(r.stderr).toContain('9 pass');});+test('reference passes fourteen outcomes',()=>{const r=grade('fixed');expect(r.status).toBe(0);expect(r.stderr).toContain('14 pass');});+test('unused correct entrypoint cannot repair the route',()=>fails(grade('unused',broken,true)));+test('only the outer guard repaired is insufficient',()=>fails(grade('outer-only',{[free]:broken[free]})));+test('only the Free gate repaired is insufficient',()=>fails(grade('inner-only',{[index]:broken[index]})));+test('equivalent explicit service predicate passes',()=>{const r=grade('equivalent',{[index]:replace(index,'(!authResult.userId && authResult.service !== true)','!(authResult.userId || authResult.service === true)'),[free]:replace(free,'auth.service === true && hasPaidHostedAiPlan(auth)','hasPaidHostedAiPlan(auth) && auth.service === true')});expect(r.status).toBe(0);});+test('client headers cannot manufacture service authority',()=>fails(grade('header-bypass',{[auth]:replace(auth,'env.AI_GATEWAY_SERVICE_TOKEN && token === env.AI_GATEWAY_SERVICE_TOKEN',"env.AI_GATEWAY_SERVICE_TOKEN && (token === env.AI_GATEWAY_SERVICE_TOKEN || request.headers.get('x-device-id') === 'cloud-runner')")})));+test('credential prefix acceptance is rejected',()=>fails(grade('prefix',{[auth]:replace(auth,'token === env.AI_GATEWAY_SERVICE_TOKEN','token.startsWith(env.AI_GATEWAY_SERVICE_TOKEN)')})));+test('blanket service refusal is rejected',()=>fails(grade('deny',{[index]:replace(index,"if (path === '/v1/chat/completions' && request.method === 'POST') {","if (path === '/v1/chat/completions' && request.method === 'POST') {\n if (authResult.service) return new Response('denied',{status:401});")})));+test('fabricated successful response without provider effect is rejected',()=>fails(grade('fake',{[index]:replace(index,"if (path === '/v1/chat/completions' && request.method === 'POST') {","if (path === '/v1/chat/completions' && request.method === 'POST') {\n if (authResult.service) return Response.json({choices:[{message:{content:'synthetic accepted response'}}]});")})));+test('ignoring rate refusal is rejected',()=>fails(grade('rate',{[index]:sources[index].replaceAll('if (!rateLimit.allowed && rateLimit.response) {','if (false && !rateLimit.allowed && rateLimit.response) {')})));+test('missing route is setup error, not intended baseline failure',()=>{const r=grade('missing',{[index]:null});expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 4e7f588c0..1545ffe45 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -4072,6 +4072,46 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-html.config.mjs components/chat/eval-html-outcome.test.tsx"}+ },+ {+ "id": "ai-gateway-trusted-runner-admission",+ "name": "Admit verified machine chat while preserving access refusals",+ "source": {+ "kind": "git_regression",+ "fix_commit": "25716cb17c4a11a75377df02b830891cc83229db"+ },+ "base_ref": "2168f5a3e118c28b79f6ba2ab15fff437c9cec33",+ "oracle_ref": "25716cb17c4a11a75377df02b830891cc83229db",+ "oracle_paths": [+ "packages/ai-gateway/src/index.ts",+ "packages/ai-gateway/src/services/free-chat-limit.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "access",+ "machine-identity",+ "background"+ ],+ "prompt": "A trusted backend runner supplies the configured service bearer but hosted chat rejects it because it has no human user ID. Repair the existing POST /v1/chat/completions path so that exact configured machine credential can complete both interactive and background requests for its requested paid model. The actual chat/provider path must make one successful request carrying the original user message and requested model, then return the provider answer. Preserve successful paid human chat. Missing, incorrect, public-user-ID, unsupported-scheme and unconfigured service credentials must return 401 before provider work, even with runner-like device, workload and background headers. A verified Free human background request must remain 403; missing human plan truth must remain 503. Machine and paid-human rate-limit refusals must remain 429 without provider work. A machine request with an empty model or empty message list must return 400 without provider work. Use synthetic credentials, entitlement, rate-limit, storage and provider ports only. Do not assume any particular private helper or guard structure. This case exercises authentication through HTTP dispatch and provider-response propagation, not JWT cryptography, live inference, SQL/concurrent spend accounting, full current entitlement integration, agent isolation or model quality.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "packages/ai-gateway/node_modules",+ "destination_path": "packages/ai-gateway/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/trusted-runner-admission.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/src/test/eval-trusted-runner.test.ts"+ }+ ],+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" test src/test/eval-trusted-runner.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/trusted-runner-admission.fixture.ts.txt b/evals/coding-agent/graders/trusted-runner-admission.fixture.ts.txtnew file mode 100644index 000000000..58b79baf5--- /dev/null+++ b/evals/coding-agent/graders/trusted-runner-admission.fixture.ts.txt@@ -0,0 +1,35 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterAll, expect, mock, test } from 'bun:test';+let state: any;+const originalFetch = globalThis.fetch;+// Install before importing SDKs, which capture fetch at module initialization.+globalThis.fetch = (...args: any[]) => { if (!state?.fetch) throw Error("network denied before fixture setup"); return state.fetch(...args); };+mock.module('@clerk/backend',()=>({verifyToken:async(token:string)=>{+ if(token==='eyJ.synthetic.user')return {sub:state.subject};throw Error('synthetic invalid token');+}}));+const {handleRequest}=await import('../index');+const auth=await import('../utils/auth');+const env:any={NODE_ENV:'production',AI_GATEWAY_SERVICE_TOKEN:'synthetic-machine-token',CLERK_SECRET_KEY:'synthetic-clerk',OPENAI_API_KEY:'synthetic-provider',CLOUDFLARE_AI_GATEWAY_ID:'synthetic',AI:{gateway:()=>({getUrl:async()=> 'https://provider.example.invalid/v1'})},+ RATE_LIMITER:{idFromName:(x:any)=>x,get:()=>({fetch:async()=>Response.json({allowed:!state.rateDenied,standing:'good',remaining:10,reset_in:60,tier:'subscribed',rpm_limit:25})})},+ DB:{prepare:(sql:string)=>({bind:(...values:any[])=>({first:async()=>null,run:async()=>({success:true,meta:{changes:1}}),all:async()=>({results:[]})}),run:async()=>({success:true})})}};+const ctx={waitUntil:(p:Promise<any>)=>state.pending.push(p),passThroughOnException:()=>{}};+let serial=0;+beforeEach(()=>{env.AI_GATEWAY_SERVICE_TOKEN='synthetic-machine-token';state={subject:'user_synthetic_'+(++serial),plan:'pro',dispatch:[],pending:[],unexpected:[],rateDenied:false};auth.__resetAuthEntitlementCacheForTests?.();+ state.fetch=async(input:any,init:any)=>{const url=input instanceof Request?input.url:String(input);if(['https://api.openai.com/v1/chat/completions','https://provider.example.invalid/v1/chat/completions'].includes(url)){state.dispatch.push({body:JSON.parse(init?.body ?? await input.text())});return Response.json({id:'synthetic-response',model:'gpt-5.6-sol',created:1,object:'chat.completion',choices:[{index:0,message:{role:'assistant',content:'synthetic accepted response'},finish_reason:'stop'}],usage:{prompt_tokens:1,completion_tokens:1,total_tokens:2}});}if(url==='https://screenpipe.com/api/user')return Response.json({success:true,user:state.unknown?{clerk_id:state.subject}:{clerk_id:state.subject,cloud_subscribed:state.plan==='pro',app_entitled:state.plan!=='none',subscription_plan:state.plan,entitlement:{active:state.plan!=='none',plan:state.plan,features:{app:state.plan!=='none',cloud:state.plan==='pro'}}}});+ state.unexpected.push(url);throw Error('unexpected network forbidden');};});+afterAll(()=>{globalThis.fetch=originalFetch;});+async function request(token?:string,headers:any={},override:any={}){const r=await handleRequest(new Request('https://gateway.example.invalid/v1/chat/completions',{method:'POST',headers:{'content-type':'application/json',...(token?{authorization:'Bearer '+token}:{}),...headers},body:JSON.stringify({model:'gpt-5.6-sol',messages:[{role:'user',content:'synthetic task'}],...override})}),env,ctx);await Promise.all(state.pending);expect(state.unexpected).toEqual([]);return r;}+async function accepted(token:string,headers:any={}){const r=await request(token,headers);expect(r.status).toBe(200);expect((await r.json()).choices[0].message.content).toBe('synthetic accepted response');expect(state.dispatch).toHaveLength(1);expect(state.dispatch[0].body.messages).toContainEqual({role:'user',content:'synthetic task'});}+for(const background of [false,true])test('trusted machine successful dispatch background='+background,async()=>{await accepted(env.AI_GATEWAY_SERVICE_TOKEN,background?{'x-screenpipe-latency':'background'}:{});expect(state.dispatch[0].body.model).toBe('gpt-5.6-sol');});+for(const token of [undefined,'synthetic-machine-token-wrong','user_public_fixture'])test('untrusted caller denied '+token,async()=>{const r=await request(token,{'x-screenpipe-latency':'background','x-screenpipe-workload':'pipe','x-device-id':'cloud-runner'});expect(r.status).toBe(401);expect(state.dispatch).toEqual([]);});+test('Free user cannot spoof background authority',async()=>{state.plan='none';const r=await request('eyJ.synthetic.user',{'x-screenpipe-latency':'background'});expect(r.status).toBe(403);expect(state.dispatch).toEqual([]);});+test('paid human still dispatches',()=>accepted('eyJ.synthetic.user'));+test('machine obeys rate refusal',async()=>{state.rateDenied=true;const r=await request(env.AI_GATEWAY_SERVICE_TOKEN);expect(r.status).toBe(429);expect(state.dispatch).toEqual([]);});+test('machine malformed model never dispatches',async()=>{const r=await request(env.AI_GATEWAY_SERVICE_TOKEN,{}, {model:''});expect(r.status).toBe(400);expect(state.dispatch).toEqual([]);});++test('unconfigured service token cannot authorize',async()=>{env.AI_GATEWAY_SERVICE_TOKEN='';const r=await request('synthetic-machine-token');expect(r.status).toBe(401);expect(state.dispatch).toEqual([]);});+test('unsupported credential scheme cannot authorize',async()=>{const r=await request(undefined,{authorization:'Basic synthetic-machine-token'});expect(r.status).toBe(401);expect(state.dispatch).toEqual([]);});+test('paid human rate refusal remains enforced',async()=>{state.rateDenied=true;const r=await request('eyJ.synthetic.user');expect(r.status).toBe(429);expect(state.dispatch).toEqual([]);});+test('unknown human plan cannot dispatch',async()=>{state.unknown=true;const r=await request('eyJ.synthetic.user');expect(r.status).toBe(503);expect(state.dispatch).toEqual([]);});+test('empty machine messages remain invalid',async()=>{const r=await request(env.AI_GATEWAY_SERVICE_TOKEN,{}, {messages:[]});expect(r.status).toBe(400);expect(state.dispatch).toEqual([]);});