YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): verify provider spend-cap recovery guidance
louisad25d201876d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b7526fa65..6d4da6fab 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1041,3 +1041,20 @@ pass seven. Run `bun test evals/coding-agent/calibrate-admission-proof.test.js`for ten controls, including an equivalent row-reading method. This is not HTTPauthentication, provider execution, settlement, production fault prevalence,agent isolation or a model trial.++## Provider spend-cap guidance++`ai-gateway-provider-cap-guidance` invokes the existing chat handler with+synthetic provider and telemetry ports. Nine outcomes cover JSON and streaming+cap errors, exhausted chains, successful fallback, intact input, healthy replies,+malformed images, empty input, authentication failures and ordinary throttling.+The parent fails three guidance outcomes and preserves six; reference/current+pass nine. Fallback already works in the parent.++Run `bun test evals/coding-agent/calibrate-provider-cap-guidance.test.js` with+gateway dependencies available. Eleven controls include unused correct code,+raw error leakage, blanket classification, lost fallback/tools, duplicate alerts,+equivalent private names/prose and missing-source setup classification. Fixtures+and dependency links are added at grading time. This does not establish HTTP+authentication, accounting, actual provider availability, complete log redaction,+execution isolation or model performance.diff --git a/evals/coding-agent/calibrate-provider-cap-guidance.test.js b/evals/coding-agent/calibrate-provider-cap-guidance.test.jsnew file mode 100644index 000000000..0564ae03a--- /dev/null+++ b/evals/coding-agent/calibrate-provider-cap-guidance.test.js@@ -0,0 +1,43 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll,expect,test} from 'bun:test';+import {execFileSync,spawnSync} from 'node:child_process';+import {mkdirSync,mkdtempSync,readFileSync,rmSync,symlinkSync,writeFileSync} from 'node:fs';+import {dirname,join,resolve} from 'node:path';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='ai-gateway-provider-cap-guidance');+const fixture=readFileSync(join(import.meta.dir,'graders/provider-cap-guidance.fixture.ts.txt'));+const root=mkdtempSync(join(tmpdir(),'provider-cap-calibration-'));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name){+ const cwd=join(root,name);mkdirSync(cwd);+ const ref=['parent','unused'].includes(name)?item.base_ref:name==='current'?'HEAD':item.oracle_ref;+ const paths=['packages/ai-gateway/src'];+ const extension='crates/screenpipe-core/assets/extensions/lib';+ if(execFileSync('git',['ls-tree',ref,'--',extension],{cwd:repo,encoding:'utf8'}).trim())paths.push(extension);+ execFileSync('tar',['-x','-C',cwd],{input:execFileSync('git',['archive',ref,...paths],{cwd:repo,maxBuffer:32*1024*1024})});+ const service=join(cwd,'packages/ai-gateway/src/handlers/chat.ts');let source=readFileSync(service,'utf8');+ function change(from,to){expect(source.split(from)).toHaveLength(2);source=source.replace(from,to)}+ if(name==='unused')writeFileSync(service+'.unused.ts',execFileSync('git',['show',`${item.oracle_ref}:packages/ai-gateway/src/handlers/chat.ts`],{cwd:repo}));+ if(name==='leak')change('error.userMessage = `${model} is temporarily at capacity (the provider\'s usage limit was reached). Pick Auto or a model from a different provider, or try again later.`;','error.userMessage = msg;');+ if(name==='blanket')change('if (isProviderUsageCapped(status, msg)) {','if (status === 400) {');+ if(name==='no-fallback')change('const fallbacks = MODEL_FALLBACKS[body.model];','const fallbacks: string[] | undefined = undefined;');+ if(name==='lost-tools')change('const reqBody = { ...body, model };','const reqBody = { ...body, model }; delete reqBody.tools;');+ if(name==='alerts')change('if (isProviderUsageCapped(status, msg)) {','if (isProviderUsageCapped(status, msg)) { captureException(error);');+ if(name==='equivalent'){+ expect(source).toContain('isProviderUsageCapped');source=source.replaceAll('isProviderUsageCapped','providerBudgetUnavailable');+ change('error.userMessage = `${model} is temporarily at capacity (the provider\'s usage limit was reached). Pick Auto or a model from a different provider, or try again later.`;',"error.userMessage = 'The provider reached its usage limit. Try again later or select a different provider.';");+ }+ writeFileSync(service,source);if(name==='missing')rmSync(service);+ writeFileSync(join(cwd,'packages/ai-gateway/src/test/eval-provider-cap.test.ts'),fixture);+ symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');+ const result=spawnSync(process.execPath,['--no-env-file','test','src/test/eval-provider-cap.test.ts'],{cwd:join(cwd,'packages/ai-gateway'),encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath),HOME:cwd,CI:'true'}});+ if(process.env.EVAL_CALIBRATION_RESULTS){const out=resolve(process.env.EVAL_CALIBRATION_RESULTS);mkdirSync(out,{recursive:true});for(const ext of ['stdout','stderr'])writeFileSync(join(out,name+'.'+ext),result[ext]||'');writeFileSync(join(out,name+'.json'),JSON.stringify({status:result.status,signal:result.signal,error:result.error?.message||null})+'\n')}+ return result;+}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module')}+test('parent fails three guidance outcomes and preserves six',()=>{const r=grade('parent');fails(r);expect(r.stderr).toContain('3 fail');expect(r.stderr).toContain('6 pass')});+for(const name of ['reference','current','equivalent'])test(name+' passes nine outcomes',()=>{const r=grade(name);expect(r.error).toBeUndefined();expect(r.status).toBe(0);expect(r.stderr).toContain('9 pass')});+for(const name of ['unused','leak','blanket','no-fallback','lost-tools','alerts'])test(name+' is rejected',()=>fails(grade(name)));+test('missing handler is a setup error',()=>{const r=grade('missing');expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)')});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 0d4515064..e67878e0c 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-27.1",+ "dataset_version": "2026-09-27.2","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3730,6 +3730,48 @@],"command": "cd packages/ai-gateway && bun test src/services/eval-admission-proof.test.ts"}+ },+ {+ "id": "ai-gateway-provider-cap-guidance",+ "name": "Explain provider spend-cap outages without blaming client input",+ "source": {+ "kind": "git_regression",+ "fix_commit": "4e6c53848e337eacb8e5eff18843604800be05d8",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5653"+ },+ "base_ref": "6b8f11c0c1ed933f4db6342d9b38c6160572cdff",+ "oracle_ref": "4e6c53848e337eacb8e5eff18843604800be05d8",+ "oracle_paths": [+ "packages/ai-gateway/src/handlers/chat.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/ai-gateway/src/handlers/chat.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "provider-error",+ "preserved-behavior"+ ],+ "prompt": "Repair the existing chat-completions handler when a provider rejects work with HTTP 400 because its organization API usage limit has been reached. For explicit single-model JSON and streaming requests, and when every model in an explicit fallback chain has this cap, retain the 400 response but explain the provider capacity/usage limit and recovery through retrying later, a reset time or a different provider. Do not return nested provider error JSON, provider request identifiers, or blame malformed tools or unsupported parameters. Suppress redundant exception reports for this known provider cap while retaining failed-model health records. Preserve a successful fallback response, its original messages/tools and the caller request; fallback already works in the broken revision. Preserve healthy explicit replies, malformed-image client errors, rejection of empty messages before provider work, provider authentication failures with 401 and their API-key detail, and ordinary 429 throttling with retry guidance rather than a provider-cap explanation. Keep the public handler entrypoint. Grade observable response and provider/telemetry effects, not private helper names or exact advisory wording. Tests use synthetic provider and telemetry ports, no live requests. HTTP authentication, budget accounting, actual provider availability, log redaction beyond returned error payloads and execution isolation are outside this case.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/provider-cap-guidance.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/src/test/eval-provider-cap.test.ts"+ }+ ],+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" --no-env-file test src/test/eval-provider-cap.test.ts"+ },+ "grader_dependency_links": [+ {+ "source_path": "packages/ai-gateway/node_modules",+ "destination_path": "packages/ai-gateway/node_modules"+ }+ ]}]}diff --git a/evals/coding-agent/graders/provider-cap-guidance.fixture.ts.txt b/evals/coding-agent/graders/provider-cap-guidance.fixture.ts.txtnew file mode 100644index 000000000..dfa5ce3b9--- /dev/null+++ b/evals/coding-agent/graders/provider-cap-guidance.fixture.ts.txt@@ -0,0 +1,23 @@+import {beforeEach,afterEach,expect,mock,test} from 'bun:test';+let state:any;+mock.module('@sentry/cloudflare',()=>({captureException:(error:any)=>state.alerts.push(error.message)}));+mock.module('../services/model-health',()=>({logModelOutcome:async(_env:any,entry:any)=>state.health.push(entry)}));+mock.module('../providers',()=>({resolveModelAlias:(model:string)=>model,createProvider:(model:string)=>({supportsTools:true,createCompletion:async(body:any)=>deliver(model,body,false),createStreamingCompletion:async(body:any)=>(await deliver(model,body,true)).body})}));+const {handleChatCompletions}=await import('../handlers/chat');+const originalFetch=globalThis.fetch;+const cap='400 {"error":{"type":"invalid_request_error","message":"You have reached your specified API usage limits. You will regain access on 2030-01-01 at 00:00 UTC."},"request_id":"synthetic-provider-id"}';+const payload={id:'synthetic-reply',choices:[{message:{role:'assistant',content:'synthetic answer'}}]};+beforeEach(()=>{state={calls:[],health:[],alerts:[],network:[],mode:'cap',message:cap,status:400};globalThis.fetch=(async(...args:any[])=>{state.network.push(String(args[0]));throw Error('unexpected external request')}) as any});+afterEach(()=>{globalThis.fetch=originalFetch;expect(state.network).toEqual([])});+async function deliver(model:string,body:any,stream:boolean){state.calls.push({model,body:structuredClone(body)});if(state.mode!=='success'&&!(state.mode==='recover'&&state.calls.length>1))throw Object.assign(Error(state.message),{status:state.status});return stream?new Response('data: '+JSON.stringify(payload)+'\n\ndata: [DONE]\n\n'):Response.json(payload)}+function request(extra:any={}){return {model:'synthetic-single-model',messages:[{role:'system',content:'screenpipe synthetic instruction'},{role:'user',content:'synthetic request'}],max_tokens:120,tools:[{type:'function',function:{name:'lookup',parameters:{type:'object',properties:{}}}}],stream:false,...extra}}+function invoke(body:any){return handleChatCompletions(body,{} as any,'interactive','',false,{backgroundFallback:false,safetyRefusalFallback:false} as any)}+function advisory(text:string){expect(text).toMatch(/provider/i);expect(text).toMatch(/capacity|usage limit/i);expect(text).toMatch(/try.*later|try again|different provider|reset/i);expect(text).not.toMatch(/invalid_request_error|synthetic-provider-id|malformed tool|unsupported parameter/i)}+for(const stream of [false,true])test('single model cap returns safe recovery guidance '+stream,async()=>{const response=await invoke(request({stream}));expect(response.status).toBe(400);advisory(await response.text());expect(state.calls.length).toBe(1);expect(state.alerts).toEqual([]);expect(state.health).toContainEqual({model:'synthetic-single-model',outcome:'error'})});+test('exhausted explicit fallback chain returns provider recovery guidance',async()=>{const response=await invoke(request({model:'claude-sonnet-5'}));expect(response.status).toBe(400);expect(state.calls.length).toBeGreaterThan(1);advisory(await response.text());expect(state.alerts).toEqual([])});+test('provider cap preserves successful fallback and original request',async()=>{state.mode='recover';const body=request({model:'claude-sonnet-5'}),saved=structuredClone(body);const response=await invoke(body);expect(response.status).toBe(200);expect(await response.json()).toEqual(payload);expect(state.calls.length).toBeGreaterThan(1);expect(state.calls[1].body.messages).toEqual(saved.messages);expect(state.calls[1].body.tools).toEqual(saved.tools);expect(body).toEqual(saved)});+test('healthy explicit provider remains successful',async()=>{state.mode='success';const response=await invoke(request());expect(response.status).toBe(200);expect(await response.json()).toEqual(payload);expect(state.calls.length).toBe(1)});+test('malformed image remains a client payload error',async()=>{state.message='Failed to decode image data';const response=await invoke(request());expect(response.status).toBe(400);const text=await response.text();expect(text).toMatch(/image/i);expect(text).not.toMatch(/provider.*usage limit|temporarily at capacity/i);expect(state.calls.length).toBe(1)});+test('empty request fails before provider work',async()=>{expect((await invoke(request({messages:[]}))).status).toBe(400);expect(state.calls).toEqual([])});+test('provider authentication failure retains unauthorized status',async()=>{state.status=401;state.message='invalid API key';const response=await invoke(request());expect(response.status).toBe(401);expect(await response.text()).toContain('invalid API key');expect(state.calls.length).toBe(1)});+test('ordinary throttle retains retry guidance without cap advisory',async()=>{state.status=429;state.message='Too many requests';const response=await invoke(request());expect(response.status).toBe(429);const text=await response.text();expect(text).toMatch(/rate limit|try again/i);expect(text).not.toMatch(/provider.*usage limit|temporarily at capacity/i);expect(state.calls.length).toBe(1)});