Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): verify persisted admission under ambiguous receipts

louis02a62fd6b33e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 61842f2f9..b7526fa65 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1031,3 +1031,13 @@ configuration. Run `bun test evals/coding-agent/calibrate-public-allowance.test.
with gateway dependencies available. This does not execute HTTP entitlement,
provider delivery, final settlement, production activation, native behavior,
agent isolation or model trials. Fixtures and dependencies are added at grading.
+
+`app-reservation-admission-proof` executes reservation and exact release against
+local workerd D1 with synthetic write-result faults. Zero or missing change
+metadata must not deny stored reservations; positive metadata must not fabricate
+admission. Seven outcomes include concurrency, full capacity, sibling retention
+and write failure. The parent fails five and preserves two; reference/current
+pass seven. Run `bun test evals/coding-agent/calibrate-admission-proof.test.js`
+for ten controls, including an equivalent row-reading method. This is not HTTP
+authentication, provider execution, settlement, production fault prevalence,
+agent isolation or a model trial.
diff --git a/evals/coding-agent/calibrate-admission-proof.test.js b/evals/coding-agent/calibrate-admission-proof.test.js
new file mode 100644
index 000000000..d328a90c0
--- /dev/null
+++ b/evals/coding-agent/calibrate-admission-proof.test.js
@@ -0,0 +1,27 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdtempSync,mkdirSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';
+import {resolve,join,dirname,basename} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),root=mkdtempSync(join(tmpdir(),'admission-proof-'));
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-reservation-admission-proof');
+const cap='packages/ai-gateway/src/services/cost-cap.ts',controls='packages/ai-gateway/src/services/hosted-ai-cost-controls.ts';
+const source=(path,ref=item.oracle_ref)=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function prepare(name,ref=item.base_ref,oracle=true){const cwd=join(root,name);mkdirSync(cwd);const archive=execFileSync('git',['archive',ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:64*1024*1024});execFileSync('tar',['-xf','-','-C',cwd],{input:archive});if(oracle)for(const f of item.oracle_paths)writeFileSync(join(cwd,f),source(f));for(const f of item.grader.fixtures){const out=join(cwd,f.destination_path);mkdirSync(dirname(out),{recursive:true});writeFileSync(out,readFileSync(join(import.meta.dir,f.local_path)));}symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');return cwd;}
+function run(cwd){const r=spawnSync(process.execPath,['test','src/services/eval-admission-proof.test.ts'],{cwd:join(cwd,'packages/ai-gateway'),encoding:'utf8',timeout:20000,env:{PATH:process.env.PATH,HOME:cwd,TZ:'UTC',CI:'true'}});if(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS){mkdirSync(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS,{recursive:true});writeFileSync(join(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS,basename(cwd)+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message,stdout:r.stdout,stderr:r.stderr},null,2));}return r;}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('error: expect(');expect(r.stderr).not.toMatch(/Cannot find module|error: (?:Unexpected|Could not resolve)/);}
+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}
+function mutant(name,path,text){const cwd=prepare(name);writeFileSync(join(cwd,path),text);return run(cwd);}
+test('parent fails five admission outcomes and preserves two',()=>{const r=run(prepare('parent',item.base_ref,false));fails(r);expect(r.stderr).toContain('2 pass\n 5 fail');});
+test('reference passes all seven outcomes',()=>{const r=run(prepare('reference'));expect(r.status).toBe(0);expect(r.stderr).toContain('7 pass\n 0 fail');});
+test('current contract passes all seven outcomes',()=>{const r=run(prepare('current','HEAD',false));expect(r.status).toBe(0);expect(r.stderr).toContain('7 pass\n 0 fail');});
+test('unused fixed module cannot repair admission',()=>{const cwd=prepare('unused');writeFileSync(join(cwd,cap+'.unused.ts'),source(cap));writeFileSync(join(cwd,cap),source(cap,item.base_ref));fails(run(cwd));});
+test('equivalent row-reading method passes',()=>expect(mutant('equivalent',cap,replace(source(cap),'.first<{ reservation_key: string }>()','.all<{ reservation_key: string }>().then(result => result.results[0])')).status).toBe(0));
+test('claimed admission without a persisted hold fails',()=>fails(mutant('no-hold',cap,replace(source(cap),'if (isZeroCostModel(model)) return { allowed: true, reservation: null };','return { allowed: true, reservation: null };'))));
+test('blanket refusal loses healthy admission',()=>fails(mutant('blanket',cap,replace(source(cap),'if (isZeroCostModel(model)) return { allowed: true, reservation: null };','return { allowed: false, response: new Response("refused", {status:429}) };'))));
+test('ignoring a missing claimed row fails',()=>fails(mutant('claim-bypass',cap,replace(source(cap),'if (claimed?.reservation_key === key)','if (true)'))));
+test('release must preserve siblings',()=>fails(mutant('release-bypass',cap,replace(source(cap),'DELETE FROM usage\n\t\t\tWHERE device_id = ? AND user_id = ? AND tier = ?', 'DELETE FROM usage\n\t\t\tWHERE device_id = ? OR user_id = ? AND tier = ?'))));
+test('missing admission module is setup failure',()=>{const cwd=prepare('missing');rmSync(join(cwd,cap));const r=run(cwd);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('error: expect(');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 36174b08d..0d4515064 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-26.8",
+ "dataset_version": "2026-09-27.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3688,6 +3688,48 @@
],
"command": "cd packages/ai-gateway && bun test src/services/eval-public-allowance.test.ts"
}
+ },
+ {
+ "id": "app-reservation-admission-proof",
+ "name": "Use persisted reservation evidence despite ambiguous write metadata",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "a5749d6d922fec0f642720dbb9a0cbbc3719ea82"
+ },
+ "base_ref": "7a1462b5d0684f5ad4a7c5059f8070ee8a8f2eb6",
+ "oracle_ref": "a5749d6d922fec0f642720dbb9a0cbbc3719ea82",
+ "oracle_paths": [
+ "packages/ai-gateway/src/services/cost-cap.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "billing",
+ "admission",
+ "concurrency"
+ ],
+ "trigger_paths": [
+ "packages/ai-gateway/src/services/cost-cap.ts"
+ ],
+ "prompt": "Repair the existing reserveDailyCostCap request-admission boundary. A successful local-D1 write may store a reservation while its change-count metadata is zero or absent. Healthy requests in those conditions must be admitted with a positive stored hold matching the returned key, account and reserved amount, and releaseDailyCostReservation must remove exactly that hold. Conversely, positive change-count metadata is not sufficient when no row was inserted: a full account with three active interactive reservations must reject the next request with 429 and preserve every existing hold; a reported successful write that stores nothing must refuse with 429 or 503. With a configured interactive limit of three, six concurrent requests under ambiguous write metadata must yield exactly three allowed responses and three 429 responses, matching exactly three stored holds. Preserve ordinary healthy admission and exact release without deleting sibling reservations; a storage write exception must refuse with 503 and leave no hold. The fixture uses a fixed clock, synthetic Business requests for gpt-5.6-luna, the existing usage ledger, and real local workerd D1 with faults restricted to write execution or returned metadata. Keep the public reservation/release entrypoints; no specific private name, SQL formatting or row-reading method is required. This case does not test HTTP authentication, provider dispatch, actual settlement, production D1 fault prevalence, global or background limits, native behavior, execution isolation or model capability.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/ai-gateway/node_modules",
+ "destination_path": "packages/ai-gateway/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/admission-proof.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/src/services/eval-admission-proof.test.ts"
+ }
+ ],
+ "command": "cd packages/ai-gateway && bun test src/services/eval-admission-proof.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/admission-proof.fixture.ts.txt b/evals/coding-agent/graders/admission-proof.fixture.ts.txt
new file mode 100644
index 000000000..3529244bf
--- /dev/null
+++ b/evals/coding-agent/graders/admission-proof.fixture.ts.txt
@@ -0,0 +1,35 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterEach,beforeEach,expect,test} from 'bun:test';
+import {Miniflare} from 'miniflare';
+import {reserveDailyCostCap,releaseDailyCostReservation} from './cost-cap';
+import {TEST_PRIVATE_COST_CONTROLS} from '../test/fixtures/private-cost-controls';
+let mf:Miniflare,env:any,realDb:any;
+const now=new Date('2026-07-14T12:00:00Z');
+beforeEach(async()=>{
+ mf=new Miniflare({compatibilityDate:'2026-01-01',d1Databases:{DB:'synthetic-admission-proof'},modules:true,script:'export default {fetch(){return new Response("ok")}}'});
+ realDb=await mf.getD1Database('DB');
+ await realDb.prepare(`CREATE TABLE usage (device_id TEXT PRIMARY KEY,user_id TEXT,daily_count INTEGER DEFAULT 0,last_reset TEXT NOT NULL,tier TEXT DEFAULT 'anonymous',created_at TEXT DEFAULT (datetime('now')),updated_at TEXT DEFAULT (datetime('now')),cost_day TEXT,daily_cost_usd REAL NOT NULL DEFAULT 0)`).run();
+ env={...TEST_PRIVATE_COST_CONTROLS,DB:realDb,PRIVATE_MAX_ACTIVE_INTERACTIVE_RESERVATIONS:'3'};
+});
+afterEach(async()=>{await mf?.dispose()});
+// All reads and normal writes use real workerd D1. Only the fault under test is synthesized.
+function fault(mode:string){
+ function statement(sql:string,bound:any):any {const write=/^\s*(INSERT|UPDATE|DELETE)\b/i.test(sql);return new Proxy(bound,{get(target,prop){if(prop==='bind')return(...args:any[])=>statement(sql,target.bind(...args));if(['run','first','all'].includes(String(prop)))return async(...args:any[])=>{
+ if(write&&mode==='throw')throw new Error('synthetic storage unavailable');
+ if(write&&mode==='drop')return prop==='first'?null:{success:true,meta:{changes:1},results:[]};
+ const result=await target[prop](...args);
+ if(write&&prop!=='first')return {...result,meta:mode==='missing'?{}:{...result.meta,changes:mode==='positive'?1:0}};
+ return result;
+ };const v=target[prop];return typeof v==='function'?v.bind(target):v;}});}
+ env.DB={prepare:(sql:string)=>statement(sql,realDb.prepare(sql))};
+}
+const reserve=()=>reserveDailyCostCap(env,'synthetic-user','subscribed','gpt-5.6-luna',now,'interactive',{},'business');
+async function rows(){return (await realDb.prepare('SELECT device_id,user_id,daily_count,last_reset,tier FROM usage ORDER BY device_id').all()).results;}
+async function hold(r:any){expect(r.allowed).toBe(true);expect(r.reservation).toBeTruthy();expect(r.reservation.reservedMicroUsd).toBeGreaterThan(0);const stored=(await rows()).find((x:any)=>x.device_id===r.reservation.key);expect(stored).toBeTruthy();expect(stored.user_id).toBe('synthetic-user');expect(stored.daily_count).toBe(r.reservation.reservedMicroUsd);return r.reservation;}
+for(const mode of ['zero','missing'])test(`committed write with ${mode} change metadata still admits its persisted hold`,async()=>{fault(mode);const r=await reserve();expect(await rows()).toHaveLength(1);const h=await hold(r);await releaseDailyCostReservation(env,h);expect(await rows()).toHaveLength(0)});
+test('concurrent ambiguous receipts agree with the three stored admissions',async()=>{fault('zero');const rs=await Promise.all(Array.from({length:6},reserve));expect(await rows()).toHaveLength(3);expect(rs.filter(r=>r.allowed)).toHaveLength(3);for(const r of rs){if(r.allowed)await hold(r);else expect(r.response.status).toBe(429)}});
+test('positive write metadata cannot admit beyond full capacity',async()=>{for(let i=0;i<3;i++)await hold(await reserve());const before=await rows();fault('positive');const r=await reserve();expect(r.allowed).toBe(false);expect(r.response.status).toBe(429);expect(await rows()).toEqual(before)});
+test('reported write success without a stored reservation must refuse',async()=>{fault('drop');const r=await reserve();expect(r.allowed).toBe(false);expect([429,503]).toContain(r.response.status);expect(await rows()).toHaveLength(0)});
+test('healthy admission and exact release preserve sibling holds',async()=>{const a=await hold(await reserve()),b=await hold(await reserve());await releaseDailyCostReservation(env,a);expect((await rows()).map((x:any)=>x.device_id)).toEqual([b.key]);await hold(await reserve());expect(await rows()).toHaveLength(2)});
+test('storage write error refuses without a reservation',async()=>{fault('throw');const r=await reserve();expect(r.allowed).toBe(false);expect(r.response.status).toBe(503);expect(await rows()).toHaveLength(0)});