Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover canonical gateway billing capacity and access boundaries

Louis Beaumont7170697d7b7b
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index b0a9d6c07..4690450a0 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 44 git-mined regressions. See
+The current app corpus contains 45 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -158,3 +158,11 @@ and successful exits retain their previous results. This does not classify all
PostCSS/compiler errors or prove whether the candidate caused a setup failure.
The 27 synthetic CLI controls include both baseline/reference startup failure,
quoted diagnostics with and without a Vitest test summary, and successful output.
+
+The ai-gateway-billing-capacity-compatibility case exercises actual gateway auth
+with synthetic Clerk and website ports and the real entitlement cache. It covers
+canonical Max/Ultra capacity, older access labels, malformed or contradictory
+plan truth, identity binding and cache admission. It does not test live billing,
+cryptographic verification, provider settlement or cross-service deployment.
+Run `bun test evals/coding-agent/calibrate-billing-capacity.test.js` to check
+correct, broken, equivalent and bypass implementations of the grader contract.
diff --git a/evals/coding-agent/calibrate-billing-capacity.test.js b/evals/coding-agent/calibrate-billing-capacity.test.js
new file mode 100644
index 000000000..f1658eaa2
--- /dev/null
+++ b/evals/coding-agent/calibrate-billing-capacity.test.js
@@ -0,0 +1,44 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c => c.id === 'ai-gateway-billing-capacity-compatibility');
+const authPath = 'packages/ai-gateway/src/utils/auth.ts';
+const paths = [authPath, 'packages/ai-gateway/src/utils/ttl-single-flight-cache.ts','packages/ai-gateway/src/types.ts'];
+const show = (ref,path) => execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});
+const fixed = Object.fromEntries(paths.map(p => [p,show(item.oracle_ref,p)]));
+const broken = { ...fixed, [authPath]: show(item.base_ref,authPath) };
+const graderPath = 'evals/coding-agent/graders/gateway-billing-capacity.test.js';
+const grader = readFileSync(join(repo,graderPath),'utf8');
+const root = mkdtempSync(join(tmpdir(),'capacity-calibration-'));
+afterAll(() => rmSync(root,{recursive:true,force:true}));
+function replace(source,from,to) { expect(source.split(from)).toHaveLength(2); return source.replace(from,to); }
+function grade(name,sources) {
+ const cwd = join(root,name);
+ for (const [p,s] of Object.entries({...sources,[graderPath]:grader})) {
+ const file = join(cwd,p); mkdirSync(dirname(file),{recursive:true}); writeFileSync(file,s);
+ }
+ return spawnSync(process.execPath,['test',graderPath],{cwd,encoding:'utf8',timeout:30000,env:{PATH:dirname(process.execPath)}});
+}
+function fail(result) {
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(result.status).toBe(1);
+ expect(result.stderr).toContain('expect(received)'); expect(result.stderr).not.toContain('Cannot find module'); expect(result.stderr).not.toContain('Unhandled error');
+}
+const mutation = (name,from,to) => fail(grade(name,{...fixed,[authPath]:replace(fixed[authPath],from,to)}));
+test('known broken parent fails actual capacity outcomes',()=>fail(grade('parent',broken)));
+test('known historical fix passes all outcomes',()=>expect(grade('reference',fixed).status).toBe(0));
+test('equivalent private helper names are accepted',()=>expect(grade('renamed',{...fixed,[authPath]:fixed[authPath].replaceAll('billingPlanMatchesAccessPlan','compatibleCapacity').replaceAll('resolveAccountPlan','accountCapacity')}).status).toBe(0));
+test('ignoring contradictory billing cannot pass',()=>mutation('ignore-invalid',"return 'unknown';\n }\n accountPlan = billingPlan;","return accessPlan;\n }\n accountPlan = billingPlan;"));
+test('billing cannot override an incompatible access grant',()=>mutation('wrong-grant','!billingPlanMatchesAccessPlan(billingPlan, accessPlan)','false'));
+test('billing labels cannot substitute for active entitlement',()=>mutation('inactive',"user.entitlement?.active === true &&","true &&"));
+test('ignoring verified subject cannot transfer another account capacity',()=>mutation('identity','screenpipeUser.clerkUserId === resolvedUserId','true'));
+test('caching unknown plan truth is rejected',()=>mutation('unknown-cache',"if (!result.accountPlan || result.accountPlan === 'unknown') return null;",''));
+test('removing explicit denial loses preserved free behavior',()=>mutation('denial','if (user.app_entitled === false && user.cloud_subscribed === false)','if (false)'));
+test('missing source is setup failure, not a behavioral contrast',()=>{
+ const sources = {...fixed};delete sources[authPath];const r=grade('missing',sources);
+ expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');
+});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index b66972218..cee74bc97 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-19.1",
+ "dataset_version": "2026-09-19.2",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1410,6 +1410,41 @@
],
"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-verified-identity.test.js"
}
+ },
+ {
+ "id": "ai-gateway-billing-capacity-compatibility",
+ "name": "Resolve canonical billing capacity without widening access",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "d9ff95802e6566a7b3ea3242800aef59cacb9aa0",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5779"
+ },
+ "base_ref": "8b455607ffbaef204a6f6a0a62e973bac6211e48",
+ "oracle_ref": "d9ff95802e6566a7b3ea3242800aef59cacb9aa0",
+ "oracle_paths": [
+ "packages/ai-gateway/src/utils/auth.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "ai-gateway",
+ "billing",
+ "entitlement",
+ "compatibility",
+ "identity"
+ ],
+ "prompt": "Customers with Max or Ultra billing plans receive ordinary Business capacity when the website returns the older compatible pro/Business access labels. Repair validateAuth so canonical billing_plan refines a matching, active app entitlement to the purchased capacity. Preserve normalized aliases and legacy responses with no billing_plan, ordinary Basic/Business access and explicit app/cloud denial. Billing data must not independently establish access: reject null, empty, unknown, free, malformed or incompatible billing values as unknown plan truth, and require matching subscription/entitlement plans plus app_entitled, entitlement.active and features.app. Keep broad cloud subscription access separate from capacity, and do not issue a power usage tier without subscribed access. Bind plan truth to the verified Clerk subject; mismatched identity and unsuccessful or unavailable website responses must not transfer capacity. Retry unknown or mismatched plan responses on the next request; retain caching of valid paid plan truth. Preserve anonymous access without credentials and the dedicated service credential. Exercise synthetic credentials only; no live billing operations and no prescribed helper structure.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/gateway-billing-capacity.test.js",
+ "destination_path": "evals/coding-agent/graders/gateway-billing-capacity.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-billing-capacity.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/gateway-billing-capacity.test.js b/evals/coding-agent/graders/gateway-billing-capacity.test.js
new file mode 100644
index 000000000..eaad2b4df
--- /dev/null
+++ b/evals/coding-agent/graders/gateway-billing-capacity.test.js
@@ -0,0 +1,90 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeEach, afterAll, expect, mock, test } from 'bun:test';
+const subject = 'user_capacity_fixture';
+const token = 'eyJ.synthetic.capacity';
+const originalFetch = globalThis.fetch;
+let state;
+mock.module('@clerk/backend', () => ({ verifyToken: async () => ({ sub: subject }) }));
+globalThis.fetch = async (url, options) => {
+ state.calls.push({ url: String(url), body: JSON.parse(options.body) });
+ if (String(url) !== 'https://screenpipe.com/api/user') throw new Error('unexpected external port');
+ if (state.unavailable) throw new Error('synthetic outage');
+ return Response.json(state.body, { status: state.status });
+};
+const auth = await import('../../../packages/ai-gateway/src/utils/auth');
+afterAll(() => { globalThis.fetch = originalFetch; });
+const env = { CLERK_SECRET_KEY: 'fixture', AI_GATEWAY_SERVICE_TOKEN: 'fixture-service' };
+function user(plan = 'pro', extra = {}) {
+ return { id: 'fixture-db-id', clerk_id: subject, cloud_subscribed: true,
+ app_entitled: true, subscription_plan: plan,
+ entitlement: { plan, active: true, features: { app: true } }, ...extra };
+}
+beforeEach(() => {
+ auth.__resetAuthEntitlementCacheForTests?.();
+ state = { calls: [], status: 200, body: { success: true, user: user() } };
+});
+async function run(credential = token) {
+ return auth.validateAuth(new Request('https://gateway.fixture.invalid/usage', { headers: {
+ 'X-Device-Id': 'fixture-device', ...(credential ? { Authorization: `Bearer ${credential}` } : {}),
+ } }), env);
+}
+async function check(plan, tier = 'subscribed') {
+ const value = await run();
+ expect(value).toMatchObject({ isValid: true, accountPlan: plan, tier, userId: subject, deviceId: subject });
+ expect(value.usageTier ?? value.tier).toBe(tier === 'subscribed' && ['business_max', 'business_ultra'].includes(plan) ? plan : tier);
+ for (const call of state.calls) expect(call).toEqual({ url: 'https://screenpipe.com/api/user', body: { token } });
+ return value;
+}
+for (const [billing, plan] of [['pro_max','business_max'],['business_max','business_max'],['pro_ultra','business_ultra'],['business_ultra','business_ultra'],[' PRO_MAX ','business_max']]) {
+ test(`canonical ${billing} refines compatible older Business labels`, async () => {
+ state.body.user.billing_plan = billing; await check(plan);
+ });
+}
+for (const [label, plan, cloud, tier] of [['standard','basic',false,'logged_in'],['lifetime','basic',false,'logged_in'],['pro','business',true,'subscribed'],['pro_max','business_max',true,'subscribed'],['pro_ultra','business_ultra',true,'subscribed']]) {
+ test(`legacy ${label} without billing remains supported`, async () => {
+ state.body.user = user(label, { cloud_subscribed: cloud }); await check(plan, tier);
+ });
+}
+for (const [name,billing] of [['null',null],['empty',''],['unknown','premium'],['free','free'],['numeric',9],['object',{}],['basic','standard'],['team','team']]) {
+ test(`${name} billing fails capacity closed and retries unknown truth`, async () => {
+ state.body.user.billing_plan = billing; await check('unknown'); await check('unknown'); expect(state.calls).toHaveLength(2);
+ });
+}
+test('power billing cannot refine a Basic grant', async () => {
+ state.body.user = user('standard', { billing_plan: 'pro_max', cloud_subscribed: false }); await check('unknown','logged_in');
+});
+for (const name of ['mismatch','missing entitlement','inactive','missing app feature','app denied']) {
+ test(`${name} prevents billing data from granting capacity`, async () => {
+ const u = state.body.user; u.billing_plan = 'pro_ultra';
+ if (name === 'mismatch') u.entitlement.plan = 'standard';
+ if (name === 'missing entitlement') delete u.entitlement;
+ if (name === 'inactive') u.entitlement.active = false;
+ if (name === 'missing app feature') u.entitlement.features = {};
+ if (name === 'app denied') u.app_entitled = false;
+ await check('unknown');
+ });
+}
+test('explicit fresh denial overrides stale paid labels', async () => {
+ state.body.user = user('pro', { billing_plan: 'pro_ultra', app_entitled: false, cloud_subscribed: false }); await check('free','logged_in');
+});
+for (const name of ['wrong subject','missing subject','unsuccessful','outage','rejected HTTP']) {
+ test(`${name} cannot transfer capacity and retries`, async () => {
+ state.body.user.billing_plan = 'pro_ultra';
+ if (name === 'wrong subject') state.body.user.clerk_id = 'user_other_fixture';
+ if (name === 'missing subject') delete state.body.user.clerk_id;
+ if (name === 'unsuccessful') state.body.success = false;
+ if (name === 'outage') state.unavailable = true;
+ if (name === 'rejected HTTP') state.status = 401;
+ await check('unknown','logged_in'); await check('unknown','logged_in'); expect(state.calls).toHaveLength(2);
+ });
+}
+test('complete paid truth is cached and bound to the verified caller', async () => {
+ state.body.user.billing_plan = 'pro_max'; await check('business_max'); await check('business_max'); expect(state.calls).toHaveLength(1);
+});
+test('absent credentials remain anonymous without website lookup', async () => {
+ expect(await run(null)).toMatchObject({ tier: 'anonymous', accountPlan: 'unknown', deviceId: 'fixture-device' }); expect(state.calls).toHaveLength(0);
+});
+test('dedicated service credential retains Business access without website lookup', async () => {
+ expect(await run(env.AI_GATEWAY_SERVICE_TOKEN)).toMatchObject({ tier: 'subscribed', accountPlan: 'business', service: true }); expect(state.calls).toHaveLength(0);
+});