YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): verify paid TTS dispatch before provider work
louisdd661ebaed83
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 76d446f06..152912aee 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 68 git-mined regressions. See+The current app corpus contains 69 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -697,3 +697,17 @@ SDK and completion factory are fail-closed ports. Gateway authentication/ratelimits, actual playback, live provider compatibility and full WAV containerintegrity are outside this signature-level contract. No real audio, native build,model call or dependency link is used; execution isolation remains unproven.++`ai-gateway-paid-tts-dispatch` invokes the actual gateway dispatcher, authentication,+paid-plan policy and voice handler with synthetic verifier, entitlement API,+rate-limiter and provider ports. Seven outcomes cover anonymous, Free, unknown+plan, Basic, Business, denied-standing and invalid-input behavior. Paid responses+retain exact provider bytes and one original-text request. The oracle applies+only the historical dispatcher fix.++Run `bun test evals/coding-agent/calibrate-paid-tts.test.js` with gateway JavaScript+dependencies installed. Grading links them only after the trajectory ends. This+is not proof of agent isolation, live authentication/payment/provider operation,+transcription policy, playback or model improvement. Separately selected current+product route tests preserve transcription behavior; they are not hidden outcomes+of this case.diff --git a/evals/coding-agent/calibrate-paid-tts.test.js b/evals/coding-agent/calibrate-paid-tts.test.jsnew file mode 100644index 000000000..ff15155c1--- /dev/null+++ b/evals/coding-agent/calibrate-paid-tts.test.js@@ -0,0 +1,39 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='ai-gateway-paid-tts-dispatch');+const root=mkdtempSync(join(tmpdir(),'paid-tts-calibration-'));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+const template=join(root,'template');mkdirSync(template);+const archive=execFileSync('git',['archive',item.oracle_ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:16*1024*1024});+execFileSync('tar',['-x','-C',template],{input:archive});+const indexPath='packages/ai-gateway/src/index.ts';+const fixed=readFileSync(join(template,indexPath),'utf8');+const broken=execFileSync('git',['show',`${item.base_ref}:${indexPath}`],{cwd:repo,encoding:'utf8'});+const graderPath='evals/coding-agent/graders/paid-tts-dispatch.test.js';+const grader=readFileSync(join(repo,graderPath));+const gate="if (path === '/v1/text-to-speech' && request.method === 'POST') {\n\t\t\tconst gate = paidHostedAiRouteError(authResult);\n\t\t\tif (gate) return gate;";+function change(from,to){expect(fixed.split(from)).toHaveLength(2);return fixed.replace(from,to);}+function grade(name,source=fixed,unused=false){+ const cwd=join(root,name);cpSync(template,cwd,{recursive:true});+ if(source===null)rmSync(join(cwd,indexPath));else writeFileSync(join(cwd,indexPath),source);+ if(unused)writeFileSync(join(cwd,'unused-correct-dispatcher.ts'),fixed);+ symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');+ mkdirSync(dirname(join(cwd,graderPath)),{recursive:true});writeFileSync(join(cwd,graderPath),grader);+ return spawnSync(process.execPath,['test',graderPath],{cwd,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});+}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}+test('broken parent fails three access outcomes and preserves four',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('3 fail');expect(r.stderr).toContain('4 pass');});+test('historical reference passes seven outcomes',()=>{const r=grade('reference');expect(r.status).toBe(0);expect(r.stderr).toContain('7 pass');});+test('unused correct dispatcher does not repair actual caller',()=>fails(grade('unused',broken,true)));+test('equivalent private binding name passes',()=>{const s=fixed.replace('paidHostedAiRouteError,','paidHostedAiRouteError as paidSpeechGate,').replaceAll('paidHostedAiRouteError(authResult)','paidSpeechGate(authResult)');expect(grade('rename',s).status).toBe(0);});+test('ignoring the real gate is rejected',()=>fails(grade('ignored',change(gate,gate.replace('if (gate) return gate;','void gate;')))));+test('provider work before a refusal is rejected',()=>fails(grade('late',change(gate,gate.replace('if (gate) return gate;','if (gate) { await handleTextToSpeech(request, env); return gate; }')))));+test('blanket refusal loses paid preserved behavior',()=>fails(grade('deny',change(gate,gate+'\n\t\t\treturn new Response("blocked",{status:401});'))));+test('response-only audio without provider effect is rejected',()=>fails(grade('fake',change('return await handleTextToSpeech(request, env);','return new Response(new Uint8Array([82,73,70,70,4,0,0,0,87,65,86,69]), {headers:{"content-type":"audio/wav"}});'))));+test('missing dispatcher is setup failure',()=>{const r=grade('missing',null);expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex e6859b305..7e99054c6 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-22.4",+ "dataset_version": "2026-09-22.5","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -2263,6 +2263,45 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/tts-wav.test.js"}+ },+ {+ "id": "ai-gateway-paid-tts-dispatch",+ "name": "Enforce paid access before text-to-speech provider work",+ "source": {+ "kind": "git_regression",+ "fix_commit": "6301ad06342d4bad420598db86db9e44423b05b6"+ },+ "base_ref": "8d2fd0135dbfe71e64aa9db2c4300601648c2077",+ "oracle_ref": "6301ad06342d4bad420598db86db9e44423b05b6",+ "oracle_paths": [+ "packages/ai-gateway/src/index.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "billing",+ "access",+ "voice",+ "provider-effects"+ ],+ "prompt": "The text-to-speech gateway route can send speech to the provider without a paid account. Repair the actual POST /v1/text-to-speech boundary. For a valid text body, an anonymous caller must receive 401, a verified Free caller 403, and a verified caller with missing plan truth 503, all as JSON errors before any speech-provider request. Preserve paid Basic and Business callers: one provider request carrying the original text must return the exact provider audio bytes with audio/wav. Paid accounts denied good standing must still receive 403 before provider work; paid requests with invalid empty text must still receive 400 with no provider work. All refusals must be JSON errors. Use synthetic identity-verifier, entitlement API, rate-limit and audio-provider ports only. Do not assume a helper name or particular implementation. This case does not test authentication cryptography, live payment or provider delivery, transcription, native playback, database concurrency, agent isolation or model capability.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "packages/ai-gateway/node_modules",+ "destination_path": "packages/ai-gateway/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/paid-tts-dispatch.test.js",+ "destination_path": "evals/coding-agent/graders/paid-tts-dispatch.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/paid-tts-dispatch.test.js"+ }}]}diff --git a/evals/coding-agent/graders/paid-tts-dispatch.test.js b/evals/coding-agent/graders/paid-tts-dispatch.test.jsnew file mode 100644index 000000000..2e774d0b7--- /dev/null+++ b/evals/coding-agent/graders/paid-tts-dispatch.test.js@@ -0,0 +1,49 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterAll, expect, mock, test } from 'bun:test';+let s, serial = 0;+const oldFetch = globalThis.fetch;+const audio = new Uint8Array([82,73,70,70,4,0,0,0,87,65,86,69]);+mock.module('@clerk/backend', () => ({verifyToken: async () => ({sub:s.subject})}));+const {handleRequest} = await import('../../../packages/ai-gateway/src/index');+const {__resetAuthEntitlementCacheForTests: reset} = await import('../../../packages/ai-gateway/src/utils/auth');+const env = {NODE_ENV:'production',CLERK_SECRET_KEY:'synthetic',DEEPGRAM_API_KEY:'synthetic',+ RATE_LIMITER:{idFromName:x=>x,get:()=>({fetch:async()=>Response.json(s.denied+ ? {allowed:false,standing:'denied',standing_reason:'banned'}+ : {allowed:true,standing:'good',remaining:10,reset_in:60,tier:'logged_in',rpm_limit:25})})},+ DB:{prepare:()=>{throw Error('Unexpected database operation');}}};+const ctx={waitUntil:()=>{},passThroughOnException:()=>{}};+beforeEach(()=>{+ reset?.(); s={subject:`user_probe_${++serial}`,plan:'standard',provider:[],lookups:0,denied:false,unknown:false};+ globalThis.fetch=async(input,init)=>{+ const url=String(input);+ if(url==='https://screenpipe.com/api/user'){+ s.lookups++; const paid=s.plan!=='none';+ return Response.json({success:true,user:s.unknown?{clerk_id:s.subject}:{clerk_id:s.subject,+ cloud_subscribed:s.plan==='pro',app_entitled:paid,subscription_plan:s.plan,+ entitlement:{active:paid,plan:s.plan,features:{app:paid,cloud:s.plan==='pro'}}}});+ }+ if(url.startsWith('https://api.deepgram.com/v1/speak')){s.provider.push({url,body:JSON.parse(init.body)});return new Response(audio);}+ throw Error('Unexpected network port');+ };+});+afterAll(()=>{globalThis.fetch=oldFetch;});+async function request(auth=true,text='synthetic speech'){+ return handleRequest(new Request('https://gateway.example.invalid/v1/text-to-speech',{+ method:'POST',headers:{'content-type':'application/json',...(auth?{authorization:'Bearer eyJ.synthetic.probe'}:{})},+ body:JSON.stringify({text})}),env,ctx);+}+async function refused(status,auth=true,text='synthetic speech'){+ const r=await request(auth,text);expect(r.status).toBe(status);expect(s.provider).toEqual([]);+ expect(r.headers.get('content-type')).toContain('application/json');+}+test('anonymous valid speech is refused before provider effect',()=>refused(401,false));+test('verified free valid speech is refused before provider effect',async()=>{s.plan='none';await refused(403);});+test('unknown plan cannot grant provider access',async()=>{s.unknown=true;await refused(503);});+for(const plan of ['standard','pro'])test(`${plan} retains exact provider audio and one effect`,async()=>{+ s.plan=plan;const r=await request();expect(r.status).toBe(200);expect(new Uint8Array(await r.arrayBuffer())).toEqual(audio);+ expect(r.headers.get('content-type')).toBe('audio/wav');expect(s.provider).toHaveLength(1);+ expect(s.provider[0].body).toEqual({text:'synthetic speech'});+});+test('denied good standing blocks paid provider access',async()=>{s.denied=true;await refused(403);});+test('paid invalid input remains refused without provider access',()=>refused(400,true,''));