YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover account-standing provenance through HTTP admission
louis0e559af750a2
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 63a5029ea..98695f711 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -915,3 +915,19 @@ a setup error. Only URL configuration is mocked. The companion paid-plan-labelcase covers a separate omission invariant. Fixtures are installed at grading;no UI/native recording, live provider verification, agent isolation or modelimprovement is established.++## Account-standing provenance++`ai-gateway-account-standing-provenance` exercises the real gateway HTTP router,+authentication and durable-object admission logic. Twelve synthetic scenarios+cover temporary sign-in locks, proven deletion versus a stale website account ID,+banned accounts, preserved model discovery and refusal before provider work.+With explicit denial reason checks, the historical parent fails ten complete+outcomes and preserves two; the reference+passes all twelve. This does not establish live identity verification, cache+migration, native retry behavior, concurrency safety or model improvement.++Run `bun test evals/coding-agent/calibrate-account-standing.test.js` for the+known-broken/correct, unused fix, permissive admission, blanket refusal, equivalent+private field and missing-source controls. Dependency setup errors are separate+from intended behavior failures. Use the shared runner's `--verify` for the case.diff --git a/evals/coding-agent/calibrate-account-standing.test.js b/evals/coding-agent/calibrate-account-standing.test.jsnew file mode 100644index 000000000..6b8ff87dd--- /dev/null+++ b/evals/coding-agent/calibrate-account-standing.test.js@@ -0,0 +1,45 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='ai-gateway-account-standing-provenance');+const root=mkdtempSync(join(tmpdir(),'standing-calibration-'));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+const fixture=readFileSync(join(import.meta.dir,'graders/account-standing.fixture.ts.txt'));+function grade(name) {+ const cwd=join(root,name);mkdirSync(cwd);+ const ref=['parent','unused'].includes(name)?item.base_ref:item.oracle_ref;+ execFileSync('tar',['-x','-C',cwd],{input:execFileSync('git',['archive',ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:16*1024*1024})});+ const src=join(cwd,'packages/ai-gateway/src');+ if(name==='unused') for(const path of ['utils/auth.ts','utils/rate-limiter.ts']) writeFileSync(join(src,path+'.unused.ts'),execFileSync('git',['show',`${item.oracle_ref}:packages/ai-gateway/src/${path}`],{cwd:repo}));+ if(['permissive','blanket'].includes(name)) {+ const path=join(src,'utils/rate-limiter.ts'),text=readFileSync(path,'utf8'),marker=' const clerkUserId = authResult?.clerkUserId;';+ expect(text.split(marker)).toHaveLength(2);+ writeFileSync(path,text.replace(marker,(name==='permissive'?' return {allowed:true};':" return {allowed:false,response:new Response('',{status:403})};")+'\n'+marker));+ }+ if(name==='equivalent') for(const file of ['utils/auth.ts','utils/rate-limiter.ts','types.ts']) {+ const path=join(src,file),text=readFileSync(path,'utf8');expect(text).toContain('clerkUserIdVerified');writeFileSync(path,text.replaceAll('clerkUserIdVerified','requestSubjectVerified'));+ }+ if(name==='missing') rmSync(join(src,'index.ts'));+ writeFileSync(join(src,'test/eval-standing-http.test.ts'),fixture);+ symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');+ const result=spawnSync(process.execPath,['test','src/test/eval-standing-http.test.ts'],{cwd:join(cwd,'packages/ai-gateway'),encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath),HOME:cwd,CI:'true'}});+ if(process.env.EVAL_CALIBRATION_RESULTS) {+ const out=resolve(process.env.EVAL_CALIBRATION_RESULTS);mkdirSync(out,{recursive:true});+ for(const ext of ['stdout','stderr'])writeFileSync(join(out,name+'.'+ext),result[ext]||'');+ writeFileSync(join(out,name+'.json'),JSON.stringify({status:result.status,signal:result.signal,error:result.error?.message||null})+'\n');+ }+ return result;+}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}+test('parent fails ten complete outcomes and preserves two',()=>{const r=grade('parent');fails(r);expect(r.stderr).toContain('10 fail');expect(r.stderr).toContain('2 pass');});+test('reference passes twelve HTTP outcomes',()=>{const r=grade('reference');expect(r.status).toBe(0);expect(r.stderr).toContain('12 pass');});+test('unused correct admission cannot repair callers',()=>fails(grade('unused')));+test('permissive admission loses denial and provenance outcomes',()=>fails(grade('permissive')));+test('blanket refusal loses preserved model listing',()=>fails(grade('blanket')));+test('equivalent private provenance field remains accepted',()=>{expect(grade('equivalent').status).toBe(0);});+test('missing router is classified as setup failure',()=>{const r=grade('missing');expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 0f93e14cd..bbf0f1c16 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -3300,6 +3300,47 @@],"command": "bun test ./evals/coding-agent/graders/free-evidence.test.ts"}+ },+ {+ "id": "ai-gateway-account-standing-provenance",+ "name": "Keep temporary sign-in locks and stale account IDs distinct from revocation",+ "source": {+ "kind": "git_regression",+ "fix_commit": "e8732d64bb543cd17b7cd798e74027fcf168573c"+ },+ "base_ref": "a9ddc64913d5a0cee60de9a582cb5293298f6ec7",+ "oracle_ref": "e8732d64bb543cd17b7cd798e74027fcf168573c",+ "oracle_paths": [+ "packages/ai-gateway/src/utils/auth.ts",+ "packages/ai-gateway/src/utils/clerk-standing.ts",+ "packages/ai-gateway/src/utils/rate-limiter.ts",+ "packages/ai-gateway/src/types.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "authentication",+ "account-standing",+ "gateway"+ ],+ "prompt": "The gateway can refuse an otherwise entitled account because its identity provider temporarily locks new sign-ins, and can treat a stale database account ID as proof of permanent deletion. Repair actual HTTP admission for paid accounts. On GET /anthropic/v1/models, both good and temporarily locked accounts must return 200 with the provider's nonempty model list and exactly one model-list provider call. On that route and POST /v1/chat/completions, a banned account must receive 403 with account_not_in_good_standing and reason banned. A deleted subject verified from this request's signed credential must receive the same 403 error with reason deleted. A provider 404 for a subject obtained only from the website account row must instead return retryable 503 with account_status_unavailable. Preserve the established JSON error envelope. Refused requests must not reach the model provider. Exercise both request-verified and website-row identity paths through the real router, authentication and durable-object admission logic. Identity verification, entitlement/standing responses, provider model listing and durable storage are synthetic ports; other network traffic is refused. Any implementation achieving these outcomes is acceptable, without requiring particular private field names. This case does not cover cryptographic verification, cache migration, live provider behavior, native transcription retries, concurrent admission, model quality or agent isolation.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "packages/ai-gateway/node_modules",+ "destination_path": "packages/ai-gateway/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/account-standing.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/src/test/eval-standing-http.test.ts"+ }+ ],+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" test src/test/eval-standing-http.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/account-standing.fixture.ts.txt b/evals/coding-agent/graders/account-standing.fixture.ts.txtnew file mode 100644index 000000000..fa7b09060--- /dev/null+++ b/evals/coding-agent/graders/account-standing.fixture.ts.txt@@ -0,0 +1,106 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test, mock, afterAll } from 'bun:test';+import { RateLimiter, checkRateLimit } from '../utils/rate-limiter';+import type { Env } from '../types';+mock.module('@clerk/backend', () => ({ verifyToken: async (token: string) => {+ if (token.includes('.verified.')) return { sub: token.split('.').at(-1) };+ throw new Error('synthetic alternate credential');+}}));+const { handleRequest } = await import('../index');+const originalFetch = globalThis.fetch;+afterAll(() => { globalThis.fetch = originalFetch; });+function makeHarness(over: Record<string, unknown> = {}) {+ const instances = new Map<string, RateLimiter>();+ const storageByName = new Map<string, Map<string, unknown>>();+ const requestedNames: string[] = [];+ const valuesFor = (name: string) => {+ let values = storageByName.get(name);+ if (!values) {+ values = new Map();+ storageByName.set(name, values);+ }+ return values;+ };+ const storageFor = (name: string) => {+ const values = valuesFor(name);+ return {+ get: async <T>(key: string): Promise<T | undefined> => values.get(key) as T | undefined,+ put: async (key: string, value: unknown): Promise<void> => {+ values.set(key, value);+ },+ };+ };+ const harnessEnv = {+ CLERK_SECRET_KEY: 'synthetic-standing-key',+ ...over,+ } as unknown as Env;+ const RATE_LIMITER = {+ idFromName: (name: string) => ({ name, toString: () => name }),+ get: (id: { name: string }) => {+ requestedNames.push(id.name);+ let inst = instances.get(id.name);+ if (!inst) {+ inst = new RateLimiter(+ { storage: storageFor(id.name) } as any,+ harnessEnv,+ );+ instances.set(id.name, inst);+ }+ return {+ fetch: (input: string | Request) =>+ inst!.fetch(typeof input === 'string' ? new Request(input) : input),+ };+ },+ };+ (harnessEnv as any).RATE_LIMITER = RATE_LIMITER;+ return {+ env: harnessEnv,+ evict: (name: string) => instances.delete(name),+ fetchDo: (name: string, request: Request) => RATE_LIMITER.get({ name }).fetch(request),+ readStorage: <T>(name: string, key: string): T | undefined =>+ valuesFor(name).get(key) as T | undefined,+ seedStorage: (name: string, key: string, value: unknown) => {+ valuesFor(name).set(key, value);+ },+ requestedNames,+ };+}++++for (const route of ['models', 'chat']) {+ for (const provenance of ['verified', 'database']) {+ for (const standing of (route==='models' ? ['good','locked','deleted','banned'] : ['deleted','banned'])) {+ test(`${route} ${provenance} ${standing} through HTTP dispatch`, async () => {+ const subject=`user_${route}_${provenance}_${standing}`;+ const calls: string[]=[];+ globalThis.fetch=mock(async (input: any) => {+ const url=String(input);calls.push(url);+ if(url==='https://api.anthropic.com/v1/models') return Response.json({data:[{id:'claude-sonnet-5',type:'model',created_at:'2026-07-01T00:00:00Z'}]});+ if(url==='https://screenpipe.com/api/user') return Response.json({success:true,user:{id:'synthetic_db_user',clerk_id:subject,cloud_subscribed:true,app_entitled:true,subscription_plan:'pro',entitlement:{active:true,plan:'pro',features:{app:true,cloud:true}}}});+ if(url===`https://api.clerk.com/v1/users/${subject}`) return standing==='deleted'?new Response('',{status:404}):Response.json({id:subject,banned:standing==='banned',locked:standing==='locked'});+ throw new Error('unexpected network port '+url);+ }) as typeof fetch;+ const env=makeHarness({NODE_ENV:'production',MODEL_GATING_ENABLED:'false',ANTHROPIC_API_KEY:'synthetic-provider-key'}).env;+ const request=new Request(route==='models'?'https://gateway.test/anthropic/v1/models':'https://gateway.test/v1/chat/completions',{+ method:route==='models'?'GET':'POST',headers:{Authorization:`Bearer eyJ.${provenance}.${subject}`,'Content-Type':'application/json'},+ ...(route==='chat'?{body:JSON.stringify({model:'auto',messages:[{role:'user',content:'synthetic'}]})}:{})+ });+ const result=await handleRequest(request,env,{waitUntil:()=>{},passThroughOnException:()=>{}} as any);+ expect(calls.every(url=>url==='https://screenpipe.com/api/user'||url===`https://api.clerk.com/v1/users/${subject}`||url==='https://api.anthropic.com/v1/models')).toBe(true);+ expect(calls.filter(url=>url==='https://api.anthropic.com/v1/models').length).toBe(standing==='good'||standing==='locked'?1:0);+ if (standing==='good'||standing==='locked') {+ expect(result.status).toBe(200);+ const body=await result.json() as any;expect(Array.isArray(body.data)).toBe(true);expect(body.data.length).toBeGreaterThan(0);+ } else {+ const unavailable=standing==='deleted'&&provenance==='database';+ expect(result.status).toBe(unavailable?503:403);+ const outer=await result.json() as any;const error=JSON.parse(outer.error);+ expect(error.error).toBe(unavailable?'account_status_unavailable':'account_not_in_good_standing');+ if(!unavailable) expect(error.reason).toBe(standing);+ }+ });+ }+ }+}