Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: verify enterprise allowance through gateway metadata

louis402411d24a8c
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index d2726b435..f23f2e8f2 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -357,3 +357,26 @@ The host applies real onSaved callbacks to the meeting prop and permits the
existing autosave of that same note; this is not proof of backend persistence,
crash recovery, real generation, native capture or filesystem isolation.
Dependencies are exposed only during grading. No model trial is implied.
+
+
+## Verified enterprise allowance through gateway metadata
+
+`ai-gateway-enterprise-allowance` joins real authentication, gateway context and
+connection construction with synthetic Clerk and website ports. Twenty outcomes
+cover enterprise precedence only after complete entitlement proof, ordinary
+consumer capacity, rejection boundaries, hashed identity, serialized metadata,
+OpenAI/Anthropic credential removal, cache identity and service/anonymous paths.
+The parent fails five intended assertions and preserves fifteen; reference and
+selected current source pass twenty. Only auth and gateway service are oracle
+patches. This standalone Bun suite needs no installed dependencies.
+
+Run `bun test evals/coding-agent/calibrate-enterprise-allowance.test.js` for
+parent/reference, equivalent naming, auth-only/gateway-only bypass, weak flag,
+early entitlement bypass, blanket Ultra, wrong wire metadata and missing-source
+controls. No website product code is included in this public case.
+
+The actual internal plan policy, model classification and account cache execute;
+an unrelated newer GLM provider import is replaced by an unused constant.
+This verifies connection configuration, not full route dispatch, cryptographic
+verification, website membership computation, real gateway spend enforcement,
+settlement, SDK requests, model outcomes or filesystem isolation.
diff --git a/evals/coding-agent/calibrate-enterprise-allowance.test.js b/evals/coding-agent/calibrate-enterprise-allowance.test.js
new file mode 100644
index 000000000..46ad95b36
--- /dev/null
+++ b/evals/coding-agent/calibrate-enterprise-allowance.test.js
@@ -0,0 +1,28 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='ai-gateway-enterprise-allowance');
+const auth='packages/ai-gateway/src/utils/auth.ts',gateway='packages/ai-gateway/src/services/cloudflare-ai-gateway.ts';
+const paths=[auth,gateway,'packages/ai-gateway/src/utils/ttl-single-flight-cache.ts','packages/ai-gateway/src/services/cost-tracker.ts','packages/ai-gateway/src/services/hosted-ai-policy.ts','packages/ai-gateway/src/services/hosted-ai-settlement-ledger.ts'];
+const show=(ref,p)=>execFileSync('git',['show',`${ref}:${p}`],{cwd:repo,encoding:'utf8'});
+const fixed=Object.fromEntries(paths.map(p=>[p,show(item.oracle_ref,p)])),broken=Object.fromEntries(paths.map(p=>[p,show(item.base_ref,p)]));
+const fixture='evals/coding-agent/graders/enterprise-allowance.test.js',grader=readFileSync(join(repo,fixture),'utf8');
+const root=mkdtempSync(join(tmpdir(),'enterprise-allowance-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}
+function grade(name,sources){const cwd=join(root,name);for(const [p,s] of Object.entries({...sources,[fixture]:grader})){const f=join(cwd,p);mkdirSync(dirname(f),{recursive:true});writeFileSync(f,s);}return spawnSync(process.execPath,['test',fixture],{cwd,encoding:'utf8',timeout:30000,env:{PATH:dirname(process.execPath)}});}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toMatch(/Cannot find module|Unhandled error/);}
+test('parent has five intended failures and fifteen preserved outcomes',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('5 fail');expect(r.stderr).toContain('15 pass');});
+test('reference passes twenty joined auth/context/connection outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('20 pass');});
+test('equivalent auth helper naming is accepted',()=>{expect(grade('renamed',{...fixed,[auth]:fixed[auth].replaceAll('resolveAccountPlan','deriveVerifiedAccountPlan')}).status).toBe(0);});
+test('gateway fix cannot hide the unchanged auth caller',()=>fails(grade('old-auth',{...fixed,[auth]:broken[auth]})));
+test('auth fix cannot hide the unchanged gateway mapping',()=>fails(grade('old-gateway',{...fixed,[gateway]:broken[gateway]})));
+test('truthy enterprise strings cannot grant enterprise allowance',()=>fails(grade('truthy',{...fixed,[auth]:replace(fixed[auth],'user.is_enterprise_user === true ?','Boolean(user.is_enterprise_user) ?')})));
+test('enterprise flag cannot bypass entitlement checks',()=>fails(grade('early',{...fixed,[auth]:replace(fixed[auth],'function resolveAccountPlan(user: ScreenpipeUserData): AccountPlan {','function resolveAccountPlan(user: ScreenpipeUserData): AccountPlan {\n if(user.is_enterprise_user === true) return "enterprise";')})));
+test('blanket Ultra mapping fails consumer preservation',()=>fails(grade('all-ultra',{...fixed,[gateway]:replace(fixed[gateway],"if (auth.accountPlan === 'enterprise') return 'business_ultra';","if (true) return 'business_ultra';")})));
+test('correct context without correct serialized wire metadata is rejected',()=>fails(grade('bad-header',{...fixed,[gateway]:replace(fixed[gateway],"'cf-aig-metadata': JSON.stringify(context),","'cf-aig-metadata': JSON.stringify({ ...context, plan: 'business' }),")})));
+test('missing auth source remains setup failure',()=>{const sources={...fixed};delete sources[auth];const r=grade('missing',sources);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 2a13eadba..0ce4843bc 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-20.3",
+ "dataset_version": "2026-09-20.4",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1734,6 +1734,42 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config meeting-handoff.config.ts"
}
+ },
+ {
+ "id": "ai-gateway-enterprise-allowance",
+ "name": "Carry verified enterprise allowance from authenticated account truth to gateway metadata",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "8663ecbef34d7cd786b064a541c0e3740f4a3c14",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/6837"
+ },
+ "base_ref": "d52da0a6484f1b54a349224ed67c6ea52637dcb3",
+ "oracle_ref": "8663ecbef34d7cd786b064a541c0e3740f4a3c14",
+ "oracle_paths": [
+ "packages/ai-gateway/src/utils/auth.ts",
+ "packages/ai-gateway/src/services/cloudflare-ai-gateway.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "auth",
+ "enterprise",
+ "gateway",
+ "entitlement",
+ "capacity"
+ ],
+ "prompt": "Verified enterprise users inherit a lower consumer allowance when authenticated website account truth includes both enterprise membership and a compatible consumer billing plan. Repair the authentication-to-gateway handoff so an explicit boolean enterprise flag produces enterprise account identity and business_ultra gateway metadata after the existing complete, active, matching entitlement checks pass. Preserve compatible Pro, Max and Ultra billing fields, and ensure the serialized metadata reaches both OpenAI and Anthropic gateway connection headers. A false, missing or string enterprise flag must not upgrade an ordinary consumer. Inactive or denied app entitlement, missing app features, conflicting entitlement/billing plans, wrong verified subject and unsuccessful website responses must not mint paid gateway capacity. Explicit app/cloud denial stays free despite stale enterprise labels. Preserve consumer Max/Ultra capacity, cached verified account identity across device changes, service internal allowance and anonymous refusal without account lookup. Metadata must use a stable hashed account identifier and retain auto lane, caller workload and trial status without exposing the account ID or bearer token. Keep payload logging disabled, remove direct provider authorization headers and keep provider retry amplification disabled. Exercise actual authentication, gateway context and connection functions using synthetic verification/website ports; no live identity, billing, provider or native operations. No particular helper name is required.",
+ "agent_timeout_seconds": 1200,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/enterprise-allowance.test.js",
+ "destination_path": "evals/coding-agent/graders/enterprise-allowance.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/enterprise-allowance.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/enterprise-allowance.test.js b/evals/coding-agent/graders/enterprise-allowance.test.js
new file mode 100644
index 000000000..3abe9e0c5
--- /dev/null
+++ b/evals/coding-agent/graders/enterprise-allowance.test.js
@@ -0,0 +1,66 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeEach, afterEach, afterAll, expect, mock, test } from 'bun:test';
+const originalFetch=globalThis.fetch,subject='user_enterprise_fixture',token='eyJ.synthetic.enterprise';
+let state;
+mock.module('@clerk/backend',()=>({verifyToken:async()=>({sub:subject})}));
+// Unrelated provider import added on newer main; no provider request is executed.
+mock.module('../../../packages/ai-gateway/src/providers/screenpipe-glm',()=>({SCREENPIPE_GLM_MODEL:'synthetic-glm-unused'}));
+globalThis.fetch=async (url,options)=>{
+ state.calls.push({url:String(url),body:JSON.parse(options.body)});
+ if(String(url)!=='https://screenpipe.com/api/user'){state.unexpected.push(String(url));throw Error('network forbidden');}
+ return Response.json(state.body,{status:state.status});
+};
+const auth=await import('../../../packages/ai-gateway/src/utils/auth');
+const gateway=await import('../../../packages/ai-gateway/src/services/cloudflare-ai-gateway');
+const env={CLERK_SECRET_KEY:'fixture',AI_GATEWAY_SERVICE_TOKEN:'fixture-service',CLOUDFLARE_AI_GATEWAY_ID:'fixture-gateway',CLOUDFLARE_AI_GATEWAY_BASE_URL:'https://gateway.ai.cloudflare.com/v1/fixture-account/fixture-gateway'};
+function user(extra={}){return {clerk_id:subject,cloud_subscribed:true,app_entitled:true,subscription_plan:'pro',billing_plan:'pro',is_enterprise_user:true,entitlement:{plan:'pro',active:true,features:{app:true}},...extra};}
+beforeEach(()=>{auth.__resetAuthEntitlementCacheForTests?.();state={calls:[],unexpected:[],status:200,body:{success:true,user:user()}};});
+afterEach(()=>expect(state.unexpected).toEqual([]));afterAll(()=>{globalThis.fetch=originalFetch;});
+function run(credential=token,device='fixture-device'){return auth.validateAuth(new Request('https://gateway.fixture.invalid/chat',{headers:{'X-Device-Id':device,...(credential?{Authorization:`Bearer ${credential}`}:{})}}),env);}
+async function joined(plan='business_ultra',provider='openai'){
+ const verified=await run();const context=await gateway.buildHostedChatGatewayContext(verified,'auto','background');
+ const connection=await gateway.getHostedChatGatewayConnection(env,provider,context);
+ expect(context.plan).toBe(plan);expect(context).toMatchObject({lane:'auto',workload:'background',trial:false});
+ expect(context.user_id).toMatch(/^[a-f0-9]{64}$/);expect(context.user_id).not.toContain(subject);
+ expect(JSON.parse(connection.defaultHeaders['cf-aig-metadata'])).toEqual(context);
+ expect(connection.baseURL).toBe(`https://gateway.ai.cloudflare.com/v1/fixture-account/fixture-gateway/${provider}`);
+ expect(connection.maxRetries).toBe(0);expect(connection.defaultHeaders['cf-aig-collect-log-payload']).toBe('false');
+ expect(connection.defaultHeaders[provider==='openai'?'Authorization':'x-api-key']).toBeNull();
+ expect(JSON.stringify(connection)).not.toContain(token);expect(JSON.stringify(connection)).not.toContain(subject);
+ for(const call of state.calls)expect(call).toEqual({url:'https://screenpipe.com/api/user',body:{token}});
+ return {verified,context};
+}
+for(const billing of ['pro','pro_max','pro_ultra'])test(`verified enterprise overrides compatible consumer ${billing} through gateway metadata`,async()=>{
+ state.body.user.billing_plan=billing;const {verified}=await joined();expect(verified).toMatchObject({accountPlan:'enterprise',userId:subject,isValid:true});
+});
+test('enterprise context reaches the Anthropic gateway without a provider credential',async()=>{await joined('business_ultra','anthropic');});
+for(const flag of [false,undefined,'true'])test(`unverified enterprise flag ${String(flag)} preserves consumer allowance`,async()=>{state.body.user.is_enterprise_user=flag;await joined('business');});
+for(const [billing,plan] of [['pro_max','business_max'],['pro_ultra','business_ultra']])test(`consumer ${billing} retains purchased capacity`,async()=>{state.body.user.is_enterprise_user=false;state.body.user.billing_plan=billing;await joined(plan);});
+for(const kind of ['inactive','app denied','missing app feature','mismatched entitlement','incompatible billing','wrong subject','failed response'])test(`${kind} cannot promote enterprise metadata into paid capacity`,async()=>{
+ const u=state.body.user;
+ if(kind==='inactive')u.entitlement.active=false;
+ if(kind==='app denied')u.app_entitled=false;
+ if(kind==='missing app feature')u.entitlement.features={};
+ if(kind==='mismatched entitlement')u.entitlement.plan='standard';
+ if(kind==='incompatible billing')u.billing_plan='standard';
+ if(kind==='wrong subject')u.clerk_id='user_other_fixture';
+ if(kind==='failed response')state.body.success=false;
+ const result=await run();expect(result.accountPlan).toBe('unknown');
+ await expect(gateway.buildHostedChatGatewayContext(result,'auto','interactive')).rejects.toThrow();
+});
+test('explicit app/cloud denial remains free despite stale enterprise flag',async()=>{state.body.user.app_entitled=false;state.body.user.cloud_subscribed=false;await joined('free');});
+test('verified account identity is stable across device changes',async()=>{
+ const first=await run();const second=await run(token,'second-device');
+ const a=await gateway.buildHostedChatGatewayContext(first,'auto','background');
+ const b=await gateway.buildHostedChatGatewayContext(second,'auto','interactive');
+ expect(a.user_id).toBe(b.user_id);expect(a.plan).toBe('business_ultra');expect(b.workload).toBe('interactive');expect(state.calls).toHaveLength(1);
+});
+test('service credential retains internal allowance without website lookup',async()=>{
+ const value=await run(env.AI_GATEWAY_SERVICE_TOKEN);expect(value.service).toBe(true);
+ expect((await gateway.buildHostedChatGatewayContext(value,'auto','background')).plan).toBe('internal');expect(state.calls).toEqual([]);
+});
+test('anonymous request never gains an account or enterprise allowance',async()=>{
+ const value=await run(null);expect(value).toMatchObject({tier:'anonymous',accountPlan:'unknown'});expect(state.calls).toEqual([]);
+ await expect(gateway.buildHostedChatGatewayContext(value,'auto','background')).rejects.toThrow();
+});