Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): enforce public credits at request admission

louis7037af803902
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 45d269373..61842f2f9 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1023,3 +1023,11 @@ grading time; dependency links do not establish agent isolation. Current product
tests separately cover additional ledger windows. This case does not establish
HTTP admission, hold release, every budget window, deployed migrations, actual
provider billing or model performance.
+
+`app-public-hosted-ai-allowance` exercises actual request reservations against
+local workerd D1. Fifteen outcomes check public monthly/trial ceilings, healthy
+releasable holds, concurrent admission, stricter private limits and malformed
+configuration. Run `bun test evals/coding-agent/calibrate-public-allowance.test.js`
+with gateway dependencies available. This does not execute HTTP entitlement,
+provider delivery, final settlement, production activation, native behavior,
+agent isolation or model trials. Fixtures and dependencies are added at grading.
diff --git a/evals/coding-agent/calibrate-public-allowance.test.js b/evals/coding-agent/calibrate-public-allowance.test.js
new file mode 100644
index 000000000..f8ccb6d04
--- /dev/null
+++ b/evals/coding-agent/calibrate-public-allowance.test.js
@@ -0,0 +1,26 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdtempSync,mkdirSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';
+import {resolve,join,dirname,basename} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),root=mkdtempSync(join(tmpdir(),'public-allowance-'));
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-public-hosted-ai-allowance');
+const cap='packages/ai-gateway/src/services/cost-cap.ts',controls='packages/ai-gateway/src/services/hosted-ai-cost-controls.ts';
+const source=(path,ref=item.oracle_ref)=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function prepare(name,ref=item.base_ref,oracle=true){const cwd=join(root,name);mkdirSync(cwd);const archive=execFileSync('git',['archive',ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:64*1024*1024});execFileSync('tar',['-xf','-','-C',cwd],{input:archive});if(oracle)for(const f of item.oracle_paths)writeFileSync(join(cwd,f),source(f));for(const f of item.grader.fixtures){const out=join(cwd,f.destination_path);mkdirSync(dirname(out),{recursive:true});writeFileSync(out,readFileSync(join(import.meta.dir,f.local_path)));}symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');return cwd;}
+function run(cwd){const r=spawnSync(process.execPath,['test','src/services/eval-public-allowance.test.ts'],{cwd:join(cwd,'packages/ai-gateway'),encoding:'utf8',timeout:20000,env:{PATH:process.env.PATH,HOME:cwd,TZ:'UTC',CI:'true'}});if(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS){mkdirSync(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS,{recursive:true});writeFileSync(join(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS,basename(cwd)+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message,stdout:r.stdout,stderr:r.stderr},null,2));}return r;}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('error: expect(');expect(r.stderr).not.toMatch(/Cannot find module|error: (?:Unexpected|Could not resolve)/);}
+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}
+function mutant(name,path,text){const cwd=prepare(name);writeFileSync(join(cwd,path),text);return run(cwd);}
+test('parent fails seven ceilings and preserves eight outcomes',()=>{const r=run(prepare('parent',item.base_ref,false));fails(r);expect(r.stderr).toContain('8 pass\n 7 fail');});
+test('reference passes all fifteen outcomes',()=>{const r=run(prepare('reference'));expect(r.status).toBe(0);expect(r.stderr).toContain('15 pass\n 0 fail');});
+test('current contract passes all fifteen outcomes',()=>{const r=run(prepare('current','HEAD',false));expect(r.status).toBe(0);expect(r.stderr).toContain('15 pass\n 0 fail');});
+test('unused correct controls cannot repair admission',()=>{const cwd=prepare('unused');writeFileSync(join(cwd,controls+'.unused.ts'),source(controls));writeFileSync(join(cwd,controls),source(controls,item.base_ref));fails(run(cwd));});
+test('equivalent private helper naming passes',()=>expect(mutant('equivalent',controls,source(controls).replace(/\bclampWindowsToIncludedAllowance\b/g,'boundedIncludedWindows')).status).toBe(0));
+test('claimed admission without a persisted hold fails',()=>fails(mutant('no-hold',cap,replace(source(cap),'if (isZeroCostModel(model)) return { allowed: true, reservation: null };','return { allowed: true, reservation: null };'))));
+test('blanket refusal loses healthy admission',()=>fails(mutant('blanket',cap,replace(source(cap),'if (isZeroCostModel(model)) return { allowed: true, reservation: null };','return { allowed: false, response: new Response("monthly_cost_limit_exceeded", {status:429}) };'))));
+test('ignoring stricter private windows fails',()=>fails(mutant('private-bypass',controls,replace(source(controls),'Math.min(total, includedAllowance)','includedAllowance'))));
+test('missing admission module is setup failure',()=>{const cwd=prepare('missing');rmSync(join(cwd,cap));const r=run(cwd);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('error: expect(');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index ab12022e3..36174b08d 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-26.5",
+ "dataset_version": "2026-09-26.8",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3644,6 +3644,50 @@
],
"command": "cd apps/screenpipe-app-tauri && \"${BUN_BIN:-bun}\" ../../evals/coding-agent/graders/stale-policy.mjs"
}
+ },
+ {
+ "id": "app-public-hosted-ai-allowance",
+ "name": "Keep actual hosted text admission inside public included credits",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "a448803c30e4220d60e8d55003a796b425d9e349"
+ },
+ "base_ref": "f073b89db6894546cf72fddfb2ce39cbe86dfec6",
+ "oracle_ref": "a448803c30e4220d60e8d55003a796b425d9e349",
+ "oracle_paths": [
+ "packages/ai-gateway/src/services/hosted-ai-cost-controls.ts",
+ "packages/ai-gateway/src/services/hosted-ai-policy.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "billing",
+ "admission",
+ "concurrency"
+ ],
+ "trigger_paths": [
+ "packages/ai-gateway/src/services/cost-cap.ts",
+ "packages/ai-gateway/src/services/hosted-ai-cost-controls.ts"
+ ],
+ "prompt": "The existing hosted text reserveDailyCostCap entrypoint can admit spending beyond the public included-credit ceiling when private controls are permissive. Repair actual local-D1 request reservation. With synthetic private request/daily/monthly and trial windows set to10 USD, a request whose estimated hold would cross the public monthly ceiling must return allowed:false and429 with monthly_cost_limit_exceeded, without changing existing spend or leaving a reservation. Public monthly provider-cost ceilings are Free0.10USD, Basic1.50USD and Business/BusinessMax/BusinessUltra4USD; an active Business trial has a4USD total ceiling and must similarly refuse with trial_cost_limit_exceeded. A request with room for two estimated holds must still be admitted for every listed plan/period, persist its positive hold in the existing usage ledger, preserve prior spend, and release exactly that hold through releaseDailyCostReservation. Two simultaneous Basic requests with room for only one-and-a-half holds must admit exactly one; the other must receive429, and releasing the accepted hold must restore prior ledger state. Do not prescribe a specific monthly-versus-temporary-capacity error code for that concurrent refusal. Preserve a stricter private0.80USD monthly ceiling, and fail closed with503/cost_control_unavailable and no hold for malformed private configuration. Use the existing entrypoint, ledger and release interface with a fixed clock and synthetic workerd D1. No particular private helper name, SQL formatting or algorithm is required. This case does not execute HTTP authentication/entitlement, provider dispatch, final cost settlement, global or background windows, incident resets, deployment activation, real financial operations, native behavior, execution isolation or model trials.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/ai-gateway/node_modules",
+ "destination_path": "packages/ai-gateway/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/public-allowance.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/src/services/eval-public-allowance.test.ts"
+ }
+ ],
+ "command": "cd packages/ai-gateway && bun test src/services/eval-public-allowance.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/public-allowance.fixture.ts.txt b/evals/coding-agent/graders/public-allowance.fixture.ts.txt
new file mode 100644
index 000000000..fa9073c4d
--- /dev/null
+++ b/evals/coding-agent/graders/public-allowance.fixture.ts.txt
@@ -0,0 +1,50 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterEach,beforeEach,expect,test} from 'bun:test';
+import {Miniflare} from 'miniflare';
+import {reserveDailyCostCap,releaseDailyCostReservation} from './cost-cap';
+import {getCostReservationMicroUsd,monthlyCostKey,trialCostKey,utcMonth} from './cost-tracker';
+import {TEST_PRIVATE_COST_CONTROLS} from '../test/fixtures/private-cost-controls';
+let mf:Miniflare,env:any;
+const now=new Date('2026-07-14T12:00:00Z');
+const model='gpt-5.6-luna';
+const amount=getCostReservationMicroUsd(model)/1_000_000;
+beforeEach(async()=>{
+ mf=new Miniflare({compatibilityDate:'2026-01-01',d1Databases:{DB:'synthetic-allowance'},modules:true,script:'export default {fetch(){return new Response("ok")}}'});
+ const DB=await mf.getD1Database('DB');
+ await DB.prepare(`CREATE TABLE usage (device_id TEXT PRIMARY KEY,user_id TEXT,daily_count INTEGER DEFAULT 0,last_reset TEXT NOT NULL,tier TEXT DEFAULT 'anonymous',created_at TEXT DEFAULT (datetime('now')),updated_at TEXT DEFAULT (datetime('now')),cost_day TEXT,daily_cost_usd REAL NOT NULL DEFAULT 0)`).run();
+ env={...TEST_PRIVATE_COST_CONTROLS,DB};
+ for(const plan of ['FREE','BASIC','BUSINESS'])for(const window of ['REQUEST','DAILY','MONTHLY'])env[`MAX_${window}_${plan}_TEXT_COST`]='10';
+ env.MAX_REQUEST_TRIAL_TEXT_COST='10';env.MAX_DAILY_TRIAL_TEXT_COST='10';env.MAX_TRIAL_TEXT_COST='10';
+});
+afterEach(async()=>{await mf?.dispose()});
+async function seed(id:string,cost:number,trial=false){
+ const key=trial?trialCostKey(id):monthlyCostKey(id),period=trial?'trial':utcMonth(now);
+ await env.DB.prepare(`INSERT INTO usage(device_id,daily_count,last_reset,tier,cost_day,daily_cost_usd) VALUES (?,0,?,'synthetic-spend',?,?)`).bind(key,period,period,cost).run();
+ return key;
+}
+const reserve=(id:string,plan='basic',trial=false)=>reserveDailyCostCap(env,id,plan==='basic'?'logged_in':'subscribed',model,now,'interactive',{},plan as any,trial);
+async function snapshot(){return (await env.DB.prepare('SELECT device_id,user_id,daily_count,last_reset,tier,cost_day,daily_cost_usd FROM usage ORDER BY device_id').all()).results}
+async function refused(r:any,code:string,status=429){expect(r.allowed).toBe(false);expect(r.response.status).toBe(status);expect(await r.response.text()).toContain(code)}
+for(const [plan,cap,trial] of [['free',0.1,false],['basic',1.5,false],['business',4,false],['business_max',4,false],['business_ultra',4,false],['business',4,true]] as const){
+ test(`${plan} ${trial?'trial':'monthly'} ceiling refuses crossing hold without changing spend or reservations`,async()=>{
+ await seed('capped',cap-amount/2,trial);const before=await snapshot();await refused(await reserve('capped',plan,trial),trial?'trial_cost_limit_exceeded':'monthly_cost_limit_exceeded');expect(await snapshot()).toEqual(before);
+ });
+ test(`${plan} ${trial?'trial':'monthly'} healthy allowance persists an exact releasable hold`,async()=>{
+ const key=await seed('healthy',cap-amount*2,trial);const before=await snapshot();const r=await reserve('healthy',plan,trial);expect(r.allowed).toBe(true);if(!r.allowed||!r.reservation)throw new Error('missing reservation');expect(r.reservation.reservedMicroUsd).toBeGreaterThan(0);
+ const rows=await snapshot();expect(rows.length).toBe(before.length+1);expect(rows.find((v:any)=>v.device_id===key)).toEqual(before[0]);
+ await releaseDailyCostReservation(env,r.reservation);expect(await snapshot()).toEqual(before);
+ });
+}
+test('concurrent requests cannot jointly cross the public monthly ceiling',async()=>{
+ await seed('concurrent',1.5-amount*1.5);const before=await snapshot();const results=await Promise.all([reserve('concurrent'),reserve('concurrent')]);expect(results.filter(r=>r.allowed)).toHaveLength(1);
+ for(const r of results){if(r.allowed&&r.reservation)await releaseDailyCostReservation(env,r.reservation);else {expect(r.allowed).toBe(false);expect(r.response.status).toBe(429)}}
+ expect(await snapshot()).toEqual(before);
+});
+test('stricter private monthly controls remain enforced',async()=>{
+ for(const plan of ['FREE','BASIC','BUSINESS'])for(const window of ['REQUEST','DAILY','MONTHLY'])env[`MAX_${window}_${plan}_TEXT_COST`]='0.8';
+ await seed('private',0.8-amount/2);const before=await snapshot();await refused(await reserve('private'),'monthly_cost_limit_exceeded');expect(await snapshot()).toEqual(before);
+});
+test('invalid control configuration refuses without a hold',async()=>{
+ env.MAX_MONTHLY_BASIC_TEXT_COST='1oops';const before=await snapshot();await refused(await reserve('invalid'),'cost_control_unavailable',503);expect(await snapshot()).toEqual(before);
+});