YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover canonical gateway billing capacity and access boundaries
Louis Beaumont7170697d7b7b
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b0a9d6c07..4690450a0 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 44 git-mined regressions. See+The current app corpus contains 45 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -158,3 +158,11 @@ and successful exits retain their previous results. This does not classify allPostCSS/compiler errors or prove whether the candidate caused a setup failure.The 27 synthetic CLI controls include both baseline/reference startup failure,quoted diagnostics with and without a Vitest test summary, and successful output.++The ai-gateway-billing-capacity-compatibility case exercises actual gateway auth+with synthetic Clerk and website ports and the real entitlement cache. It covers+canonical Max/Ultra capacity, older access labels, malformed or contradictory+plan truth, identity binding and cache admission. It does not test live billing,+cryptographic verification, provider settlement or cross-service deployment.+Run `bun test evals/coding-agent/calibrate-billing-capacity.test.js` to check+correct, broken, equivalent and bypass implementations of the grader contract.diff --git a/evals/coding-agent/calibrate-billing-capacity.test.js b/evals/coding-agent/calibrate-billing-capacity.test.jsnew file mode 100644index 000000000..f1658eaa2--- /dev/null+++ b/evals/coding-agent/calibrate-billing-capacity.test.js@@ -0,0 +1,44 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c => c.id === 'ai-gateway-billing-capacity-compatibility');+const authPath = 'packages/ai-gateway/src/utils/auth.ts';+const paths = [authPath, 'packages/ai-gateway/src/utils/ttl-single-flight-cache.ts','packages/ai-gateway/src/types.ts'];+const show = (ref,path) => execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});+const fixed = Object.fromEntries(paths.map(p => [p,show(item.oracle_ref,p)]));+const broken = { ...fixed, [authPath]: show(item.base_ref,authPath) };+const graderPath = 'evals/coding-agent/graders/gateway-billing-capacity.test.js';+const grader = readFileSync(join(repo,graderPath),'utf8');+const root = mkdtempSync(join(tmpdir(),'capacity-calibration-'));+afterAll(() => rmSync(root,{recursive:true,force:true}));+function replace(source,from,to) { expect(source.split(from)).toHaveLength(2); return source.replace(from,to); }+function grade(name,sources) {+ const cwd = join(root,name);+ for (const [p,s] of Object.entries({...sources,[graderPath]:grader})) {+ const file = join(cwd,p); mkdirSync(dirname(file),{recursive:true}); writeFileSync(file,s);+ }+ return spawnSync(process.execPath,['test',graderPath],{cwd,encoding:'utf8',timeout:30000,env:{PATH:dirname(process.execPath)}});+}+function fail(result) {+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(result.status).toBe(1);+ expect(result.stderr).toContain('expect(received)'); expect(result.stderr).not.toContain('Cannot find module'); expect(result.stderr).not.toContain('Unhandled error');+}+const mutation = (name,from,to) => fail(grade(name,{...fixed,[authPath]:replace(fixed[authPath],from,to)}));+test('known broken parent fails actual capacity outcomes',()=>fail(grade('parent',broken)));+test('known historical fix passes all outcomes',()=>expect(grade('reference',fixed).status).toBe(0));+test('equivalent private helper names are accepted',()=>expect(grade('renamed',{...fixed,[authPath]:fixed[authPath].replaceAll('billingPlanMatchesAccessPlan','compatibleCapacity').replaceAll('resolveAccountPlan','accountCapacity')}).status).toBe(0));+test('ignoring contradictory billing cannot pass',()=>mutation('ignore-invalid',"return 'unknown';\n }\n accountPlan = billingPlan;","return accessPlan;\n }\n accountPlan = billingPlan;"));+test('billing cannot override an incompatible access grant',()=>mutation('wrong-grant','!billingPlanMatchesAccessPlan(billingPlan, accessPlan)','false'));+test('billing labels cannot substitute for active entitlement',()=>mutation('inactive',"user.entitlement?.active === true &&","true &&"));+test('ignoring verified subject cannot transfer another account capacity',()=>mutation('identity','screenpipeUser.clerkUserId === resolvedUserId','true'));+test('caching unknown plan truth is rejected',()=>mutation('unknown-cache',"if (!result.accountPlan || result.accountPlan === 'unknown') return null;",''));+test('removing explicit denial loses preserved free behavior',()=>mutation('denial','if (user.app_entitled === false && user.cloud_subscribed === false)','if (false)'));+test('missing source is setup failure, not a behavioral contrast',()=>{+ const sources = {...fixed};delete sources[authPath];const r=grade('missing',sources);+ expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');+});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex b66972218..cee74bc97 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-19.1",+ "dataset_version": "2026-09-19.2","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1410,6 +1410,41 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-verified-identity.test.js"}+ },+ {+ "id": "ai-gateway-billing-capacity-compatibility",+ "name": "Resolve canonical billing capacity without widening access",+ "source": {+ "kind": "git_regression",+ "fix_commit": "d9ff95802e6566a7b3ea3242800aef59cacb9aa0",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5779"+ },+ "base_ref": "8b455607ffbaef204a6f6a0a62e973bac6211e48",+ "oracle_ref": "d9ff95802e6566a7b3ea3242800aef59cacb9aa0",+ "oracle_paths": [+ "packages/ai-gateway/src/utils/auth.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "ai-gateway",+ "billing",+ "entitlement",+ "compatibility",+ "identity"+ ],+ "prompt": "Customers with Max or Ultra billing plans receive ordinary Business capacity when the website returns the older compatible pro/Business access labels. Repair validateAuth so canonical billing_plan refines a matching, active app entitlement to the purchased capacity. Preserve normalized aliases and legacy responses with no billing_plan, ordinary Basic/Business access and explicit app/cloud denial. Billing data must not independently establish access: reject null, empty, unknown, free, malformed or incompatible billing values as unknown plan truth, and require matching subscription/entitlement plans plus app_entitled, entitlement.active and features.app. Keep broad cloud subscription access separate from capacity, and do not issue a power usage tier without subscribed access. Bind plan truth to the verified Clerk subject; mismatched identity and unsuccessful or unavailable website responses must not transfer capacity. Retry unknown or mismatched plan responses on the next request; retain caching of valid paid plan truth. Preserve anonymous access without credentials and the dedicated service credential. Exercise synthetic credentials only; no live billing operations and no prescribed helper structure.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/gateway-billing-capacity.test.js",+ "destination_path": "evals/coding-agent/graders/gateway-billing-capacity.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-billing-capacity.test.js"+ }}]}diff --git a/evals/coding-agent/graders/gateway-billing-capacity.test.js b/evals/coding-agent/graders/gateway-billing-capacity.test.jsnew file mode 100644index 000000000..eaad2b4df--- /dev/null+++ b/evals/coding-agent/graders/gateway-billing-capacity.test.js@@ -0,0 +1,90 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterAll, expect, mock, test } from 'bun:test';+const subject = 'user_capacity_fixture';+const token = 'eyJ.synthetic.capacity';+const originalFetch = globalThis.fetch;+let state;+mock.module('@clerk/backend', () => ({ verifyToken: async () => ({ sub: subject }) }));+globalThis.fetch = async (url, options) => {+ state.calls.push({ url: String(url), body: JSON.parse(options.body) });+ if (String(url) !== 'https://screenpipe.com/api/user') throw new Error('unexpected external port');+ if (state.unavailable) throw new Error('synthetic outage');+ return Response.json(state.body, { status: state.status });+};+const auth = await import('../../../packages/ai-gateway/src/utils/auth');+afterAll(() => { globalThis.fetch = originalFetch; });+const env = { CLERK_SECRET_KEY: 'fixture', AI_GATEWAY_SERVICE_TOKEN: 'fixture-service' };+function user(plan = 'pro', extra = {}) {+ return { id: 'fixture-db-id', clerk_id: subject, cloud_subscribed: true,+ app_entitled: true, subscription_plan: plan,+ entitlement: { plan, active: true, features: { app: true } }, ...extra };+}+beforeEach(() => {+ auth.__resetAuthEntitlementCacheForTests?.();+ state = { calls: [], status: 200, body: { success: true, user: user() } };+});+async function run(credential = token) {+ return auth.validateAuth(new Request('https://gateway.fixture.invalid/usage', { headers: {+ 'X-Device-Id': 'fixture-device', ...(credential ? { Authorization: `Bearer ${credential}` } : {}),+ } }), env);+}+async function check(plan, tier = 'subscribed') {+ const value = await run();+ expect(value).toMatchObject({ isValid: true, accountPlan: plan, tier, userId: subject, deviceId: subject });+ expect(value.usageTier ?? value.tier).toBe(tier === 'subscribed' && ['business_max', 'business_ultra'].includes(plan) ? plan : tier);+ for (const call of state.calls) expect(call).toEqual({ url: 'https://screenpipe.com/api/user', body: { token } });+ return value;+}+for (const [billing, plan] of [['pro_max','business_max'],['business_max','business_max'],['pro_ultra','business_ultra'],['business_ultra','business_ultra'],[' PRO_MAX ','business_max']]) {+ test(`canonical ${billing} refines compatible older Business labels`, async () => {+ state.body.user.billing_plan = billing; await check(plan);+ });+}+for (const [label, plan, cloud, tier] of [['standard','basic',false,'logged_in'],['lifetime','basic',false,'logged_in'],['pro','business',true,'subscribed'],['pro_max','business_max',true,'subscribed'],['pro_ultra','business_ultra',true,'subscribed']]) {+ test(`legacy ${label} without billing remains supported`, async () => {+ state.body.user = user(label, { cloud_subscribed: cloud }); await check(plan, tier);+ });+}+for (const [name,billing] of [['null',null],['empty',''],['unknown','premium'],['free','free'],['numeric',9],['object',{}],['basic','standard'],['team','team']]) {+ test(`${name} billing fails capacity closed and retries unknown truth`, async () => {+ state.body.user.billing_plan = billing; await check('unknown'); await check('unknown'); expect(state.calls).toHaveLength(2);+ });+}+test('power billing cannot refine a Basic grant', async () => {+ state.body.user = user('standard', { billing_plan: 'pro_max', cloud_subscribed: false }); await check('unknown','logged_in');+});+for (const name of ['mismatch','missing entitlement','inactive','missing app feature','app denied']) {+ test(`${name} prevents billing data from granting capacity`, async () => {+ const u = state.body.user; u.billing_plan = 'pro_ultra';+ if (name === 'mismatch') u.entitlement.plan = 'standard';+ if (name === 'missing entitlement') delete u.entitlement;+ if (name === 'inactive') u.entitlement.active = false;+ if (name === 'missing app feature') u.entitlement.features = {};+ if (name === 'app denied') u.app_entitled = false;+ await check('unknown');+ });+}+test('explicit fresh denial overrides stale paid labels', async () => {+ state.body.user = user('pro', { billing_plan: 'pro_ultra', app_entitled: false, cloud_subscribed: false }); await check('free','logged_in');+});+for (const name of ['wrong subject','missing subject','unsuccessful','outage','rejected HTTP']) {+ test(`${name} cannot transfer capacity and retries`, async () => {+ state.body.user.billing_plan = 'pro_ultra';+ if (name === 'wrong subject') state.body.user.clerk_id = 'user_other_fixture';+ if (name === 'missing subject') delete state.body.user.clerk_id;+ if (name === 'unsuccessful') state.body.success = false;+ if (name === 'outage') state.unavailable = true;+ if (name === 'rejected HTTP') state.status = 401;+ await check('unknown','logged_in'); await check('unknown','logged_in'); expect(state.calls).toHaveLength(2);+ });+}+test('complete paid truth is cached and bound to the verified caller', async () => {+ state.body.user.billing_plan = 'pro_max'; await check('business_max'); await check('business_max'); expect(state.calls).toHaveLength(1);+});+test('absent credentials remain anonymous without website lookup', async () => {+ expect(await run(null)).toMatchObject({ tier: 'anonymous', accountPlan: 'unknown', deviceId: 'fixture-device' }); expect(state.calls).toHaveLength(0);+});+test('dedicated service credential retains Business access without website lookup', async () => {+ expect(await run(env.AI_GATEWAY_SERVICE_TOKEN)).toMatchObject({ tier: 'subscribed', accountPlan: 'business', service: true }); expect(state.calls).toHaveLength(0);+});