Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: preserve background requests after allowance exhaustion

louis81856c7c5972
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 1658c0aa1..de1fdf164 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -935,3 +935,7 @@ from intended behavior failures. Use the shared runner's `--verify` for the case
### Attachment owner handoff
`app-attachment-owner-handoff` exercises the actual attachment hook and in-memory chat store across delayed file reads, pane unmount, foreground navigation, pointer-scoped drops, cancellation and read failure. Hidden/deleted owners stay absent and ordinary multiple-file attachment survives. Native file/dialog/extraction and locale ports are synthetic; this does not establish native delivery, disk persistence or full composer integration. Run `bun test evals/coding-agent/calibrate-attachment-owner.test.js` with the desktop test dependencies available. Historical verification and mutation calibration are not model trials or an isolation claim.
+
+### Background allowance request preservation
+
+`ai-gateway-background-request-preservation` drives the existing chat handler through synthetic provider ports. Thirteen outcomes preserve message roles, context, tool schemas, output budgets, response formats and streamed tool results during enabled allowance recovery, plus successful-primary and refusal behavior. Parent verification fails four intended outcomes and preserves nine; the historical fix passes thirteen. Run `bun test evals/coding-agent/calibrate-background-request.test.js`. Paid-plan HTTP admission, safety-refusal policy, actual provider compatibility, concurrent accounting, execution isolation and model performance remain separate. Fixtures and dependency links are installed only when grading starts.
diff --git a/evals/coding-agent/calibrate-background-request.test.js b/evals/coding-agent/calibrate-background-request.test.js
new file mode 100644
index 000000000..39396fcc7
--- /dev/null
+++ b/evals/coding-agent/calibrate-background-request.test.js
@@ -0,0 +1,42 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='ai-gateway-background-request-preservation');
+const root=mkdtempSync(join(tmpdir(),'background-request-calibration-'));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const fixture=readFileSync(join(import.meta.dir,'graders/background-request.fixture.ts.txt'));
+function grade(name) {
+ const cwd=join(root,name);mkdirSync(cwd);
+ const ref=['parent','unused'].includes(name)?item.base_ref:item.oracle_ref;
+ execFileSync('tar',['-x','-C',cwd],{input:execFileSync('git',['archive',ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:16*1024*1024})});
+ const src=join(cwd,'packages/ai-gateway/src'),service=join(src,'services/background-limit-fallback.ts');
+ if(name==='unused') writeFileSync(service+'.unused.ts',execFileSync('git',['show',`${item.oracle_ref}:packages/ai-gateway/src/services/background-limit-fallback.ts`],{cwd:repo}));
+ if(['budget','tools','blanket'].includes(name)) {
+ const original=readFileSync(service,'utf8'),marker='return { ...body, model: BACKGROUND_FALLBACK_MODEL };';expect(original.split(marker)).toHaveLength(2);
+ const replacement=name==='budget'?'return { ...body, model: BACKGROUND_FALLBACK_MODEL, max_completion_tokens: 512 };':name==='tools'?'return { ...body, model: BACKGROUND_FALLBACK_MODEL, tools: [] };':'return { ...body, model: BACKGROUND_FALLBACK_MODEL, messages: [] };';
+ writeFileSync(service,original.replace(marker,replacement));
+ }
+ if(name==='disabled') {const original=readFileSync(service,'utf8');expect(original.split('return input.enabled &&')).toHaveLength(2);writeFileSync(service,original.replace('return input.enabled &&','return true &&'));}
+ if(name==='equivalent') for(const file of ['services/background-limit-fallback.ts','handlers/chat.ts']) {const path=join(src,file),text=readFileSync(path,'utf8');expect(text).toContain('BACKGROUND_FALLBACK_MODEL');writeFileSync(path,text.replaceAll('BACKGROUND_FALLBACK_MODEL','RECOVERY_MODEL'));}
+ if(name==='missing') rmSync(join(src,'handlers/chat.ts'));
+ writeFileSync(join(src,'test/eval-background-request.test.ts'),fixture);
+ symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');
+ const result=spawnSync(process.execPath,['test','src/test/eval-background-request.test.ts'],{cwd:join(cwd,'packages/ai-gateway'),encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath),HOME:cwd,CI:'true'}});
+ if(process.env.EVAL_CALIBRATION_RESULTS) {const out=resolve(process.env.EVAL_CALIBRATION_RESULTS);mkdirSync(out,{recursive:true});for(const ext of ['stdout','stderr'])writeFileSync(join(out,name+'.'+ext),result[ext]||'');writeFileSync(join(out,name+'.json'),JSON.stringify({status:result.status,signal:result.signal,error:result.error?.message||null})+'\n');}
+ return result;
+}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}
+test('parent fails four outcomes and preserves nine',()=>{const r=grade('parent');fails(r);expect(r.stderr).toContain('4 fail');expect(r.stderr).toContain('9 pass');});
+test('reference passes thirteen outcomes',()=>{const r=grade('reference');expect(r.status).toBe(0);expect(r.stderr).toContain('13 pass');});
+test('unused correct service does not repair dispatch',()=>fails(grade('unused')));
+test('clamped output budget is rejected',()=>fails(grade('budget')));
+test('lost tool schema is rejected',()=>fails(grade('tools')));
+test('blanket context loss is rejected',()=>fails(grade('blanket')));
+test('disabled lane remains protected',()=>fails(grade('disabled')));
+test('equivalent private model binding is accepted',()=>{expect(grade('equivalent').status).toBe(0);});
+test('missing handler is setup failure',()=>{const r=grade('missing');expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index c1a2e2d61..cdd068ff3 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-25.4",
+ "dataset_version": "2026-09-25.5",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3388,6 +3388,45 @@
"destination_path": "apps/screenpipe-app-tauri/node_modules"
}
]
+ },
+ {
+ "id": "ai-gateway-background-request-preservation",
+ "name": "Preserve the complete request when recovering an exhausted background allowance",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "1c0a271403a4128a2238c355e5820b7f18c98eac"
+ },
+ "base_ref": "5eb2bd80952a7302f20f46f561abdbb136950f86",
+ "oracle_ref": "1c0a271403a4128a2238c355e5820b7f18c98eac",
+ "oracle_paths": [
+ "packages/ai-gateway/src/handlers/chat.ts",
+ "packages/ai-gateway/src/services/background-limit-fallback.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "background",
+ "allowance",
+ "request-integrity"
+ ],
+ "prompt": "An already-authorized background request exhausts its hosted allowance. Its recovery attempt can silently lose developer instructions, older context, tool descriptions and schema metadata, or most of its output budget. Repair the existing chat completion handler so an enabled, configured text-only recovery makes one alternate provider attempt with the complete message sequence, roles, tool-call associations, tool schemas, requested output budget, response format and other request options preserved. Do not mutate the caller request. Preserve the real provider response and streamed tool-call result. A model substitution is allowed; no particular private helper name or provider model is required. A successful primary needs no recovery. Disabled or unconfigured recovery and requests with images or files must retain the original allowance refusal without dropping input. If recovery is unavailable, preserve the original hosted_ai_allowance_exceeded 429 response. Ordinary requests-per-minute throttling must not activate allowance recovery; an empty message list must return 400 before provider work. Tests use the existing handler entrypoint with both legacy and current option spellings, synthetic provider and telemetry ports, and denied external network. Authorization and paid-plan HTTP dispatch, safety-refusal recovery policy, actual provider compatibility, concurrent spend accounting and model quality are outside this case. Add focused tests for the outcomes.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/ai-gateway/node_modules",
+ "destination_path": "packages/ai-gateway/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/background-request.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/src/test/eval-background-request.test.ts"
+ }
+ ],
+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" test src/test/eval-background-request.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/background-request.fixture.ts.txt b/evals/coding-agent/graders/background-request.fixture.ts.txt
new file mode 100644
index 000000000..28e21b4a7
--- /dev/null
+++ b/evals/coding-agent/graders/background-request.fixture.ts.txt
@@ -0,0 +1,25 @@
+import {beforeEach,afterEach,expect,mock,test} from 'bun:test';
+let state: any;
+mock.module('@sentry/cloudflare',()=>({captureException:()=>{}}));
+mock.module('../services/model-health',()=>({logModelOutcome:async()=>{}}));
+mock.module('../providers',()=>({resolveModelAlias:(model:string)=>model,createProvider:(model:string,_env:unknown,connection:unknown)=>({supportsTools:true,createCompletion:async(body:any)=>deliver(model,body,connection,false),createStreamingCompletion:async(body:any)=>(await deliver(model,body,connection,true)).body})}));
+const {HostedChatAllowanceExceededError}=await import('../services/cloudflare-ai-gateway');
+const {handleChatCompletions}=await import('../handlers/chat');
+const originalFetch=globalThis.fetch;
+beforeEach(()=>{state={calls:[],success:false,rescueFails:false,error:null,network:[]};globalThis.fetch=(async(url:any)=>{state.network.push(String(url));throw Error('unexpected network')}) as any});
+afterEach(()=>{globalThis.fetch=originalFetch;expect(state.network).toEqual([])});
+const env={OPENAI_API_KEY:'synthetic-key',SCREENPIPE_QWEN35_API_KEY:'synthetic-key',SCREENPIPE_QWEN35_URL:'https://synthetic.invalid'} as any;
+const context={user_id:'synthetic',plan:'business_max',lane:'explicit',workload:'background',trial:false} as any;
+const payload={id:'reply_synthetic',choices:[{message:{role:'assistant',tool_calls:[{id:'call_synthetic',type:'function',function:{name:'lookup',arguments:'{"query":"synthetic"}'}}]},finish_reason:'tool_calls'}]};
+async function deliver(model:string,body:any,connection:unknown,stream:boolean){state.calls.push({model,body:structuredClone(body),connection});if(model==='fixture-primary'&&!state.success)throw state.error??new HostedChatAllowanceExceededError(context);if(model!=='fixture-primary'&&state.rescueFails)throw Object.assign(new Error('synthetic rescue unavailable'),{status:503});return stream?new Response('data: '+JSON.stringify(payload)+'\n\ndata: [DONE]\n\n'):Response.json(payload)}
+function body(extra:any={}){return {model:'fixture-primary',stream:false,messages:[{role:'system',content:'screenpipe synthetic context'},{role:'developer',content:'Retain these synthetic task requirements.'},{role:'user',content:'old-context-start:'+ 'sample '.repeat(4000)+':old-context-end'},{role:'assistant',content:null,tool_calls:[{id:'call_prior',type:'function',function:{name:'lookup',arguments:'{"query":"past"}'}}]},{role:'tool',tool_call_id:'call_prior',content:'Synthetic prior tool result'},{role:'user',content:'Finish the synthetic summary.'}],tools:[{type:'function',function:{name:'lookup',description:'Required synthetic tool semantics. '.repeat(20),parameters:{type:'object',title:'SyntheticQuery',description:'Preserve schema guidance',properties:{query:{type:'string',description:'Synthetic search terms',default:'synthetic',examples:['example']}},required:['query']}}}],tool_choice:'auto',response_format:{type:'json_object'},max_completion_tokens:32000,temperature:0.2,...extra}}
+async function invoke(request:any,enabled=true,settings:any=env){return handleChatCompletions(request,settings,'background','',false,{backgroundFallback:enabled,argusBackgroundFallback:enabled,safetyRefusalFallback:false,argusSafetyRefusalFallback:false} as any)}
+for(const field of ['messages','tools','max_completion_tokens','response_format'])test('allowance rescue preserves '+field,async()=>{const request=body();const saved=structuredClone(request);const response=await invoke(request);expect(response.status).toBe(200);expect(state.calls.length).toBe(2);expect(state.calls[1].model).not.toBe('fixture-primary');expect(state.calls[1].body[field]).toEqual(saved[field]);expect(request).toEqual(saved);expect(await response.json()).toEqual(payload)});
+test('streaming rescue preserves context and emits provider tool result',async()=>{const request=body({stream:true});const response=await invoke(request);expect(response.status).toBe(200);expect(state.calls.length).toBe(2);const {model,...actual}=state.calls[1].body;const {model:_requested,...expected}=request;expect(actual).toEqual(expected);expect(await response.text()).toContain('call_synthetic')});
+test('successful primary needs no rescue',async()=>{state.success=true;expect((await invoke(body())).status).toBe(200);expect(state.calls.length).toBe(1)});
+test('disabled rescue preserves allowance response',async()=>{const response=await invoke(body(),false);expect(response.status).toBe(429);expect(await response.text()).toContain('hosted_ai_allowance_exceeded');expect(state.calls.length).toBe(1)});
+for(const type of ['image_url','file'])test('unsupported '+type+' remains refused without input loss',async()=>{const response=await invoke(body({messages:[{role:'user',content:[{type,[type]:{url:'https://synthetic.invalid/input'}}]}]}));expect(response.status).toBe(429);expect(state.calls.length).toBe(1)});
+test('unconfigured rescue preserves allowance refusal',async()=>{expect((await invoke(body(),true,{})).status).toBe(429);expect(state.calls.length).toBe(1)});
+test('unavailable rescue preserves original allowance error',async()=>{state.rescueFails=true;const response=await invoke(body());expect(response.status).toBe(429);expect(await response.text()).toContain('hosted_ai_allowance_exceeded');expect(state.calls.length).toBe(2)});
+test('ordinary throttle cannot activate allowance rescue',async()=>{state.error=Object.assign(new Error('500 requests per minute'),{status:429});expect((await invoke(body())).status).toBe(429);expect(state.calls.length).toBe(1)});
+test('empty messages are refused before provider work',async()=>{expect((await invoke(body({messages:[]}))).status).toBe(400);expect(state.calls).toEqual([])});