Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): verify paid TTS dispatch before provider work

louisdd661ebaed83
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 76d446f06..152912aee 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 68 git-mined regressions. See
+The current app corpus contains 69 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -697,3 +697,17 @@ SDK and completion factory are fail-closed ports. Gateway authentication/rate
limits, actual playback, live provider compatibility and full WAV container
integrity are outside this signature-level contract. No real audio, native build,
model call or dependency link is used; execution isolation remains unproven.
+
+`ai-gateway-paid-tts-dispatch` invokes the actual gateway dispatcher, authentication,
+paid-plan policy and voice handler with synthetic verifier, entitlement API,
+rate-limiter and provider ports. Seven outcomes cover anonymous, Free, unknown
+plan, Basic, Business, denied-standing and invalid-input behavior. Paid responses
+retain exact provider bytes and one original-text request. The oracle applies
+only the historical dispatcher fix.
+
+Run `bun test evals/coding-agent/calibrate-paid-tts.test.js` with gateway JavaScript
+dependencies installed. Grading links them only after the trajectory ends. This
+is not proof of agent isolation, live authentication/payment/provider operation,
+transcription policy, playback or model improvement. Separately selected current
+product route tests preserve transcription behavior; they are not hidden outcomes
+of this case.
diff --git a/evals/coding-agent/calibrate-paid-tts.test.js b/evals/coding-agent/calibrate-paid-tts.test.js
new file mode 100644
index 000000000..ff15155c1
--- /dev/null
+++ b/evals/coding-agent/calibrate-paid-tts.test.js
@@ -0,0 +1,39 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='ai-gateway-paid-tts-dispatch');
+const root=mkdtempSync(join(tmpdir(),'paid-tts-calibration-'));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const template=join(root,'template');mkdirSync(template);
+const archive=execFileSync('git',['archive',item.oracle_ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:16*1024*1024});
+execFileSync('tar',['-x','-C',template],{input:archive});
+const indexPath='packages/ai-gateway/src/index.ts';
+const fixed=readFileSync(join(template,indexPath),'utf8');
+const broken=execFileSync('git',['show',`${item.base_ref}:${indexPath}`],{cwd:repo,encoding:'utf8'});
+const graderPath='evals/coding-agent/graders/paid-tts-dispatch.test.js';
+const grader=readFileSync(join(repo,graderPath));
+const gate="if (path === '/v1/text-to-speech' && request.method === 'POST') {\n\t\t\tconst gate = paidHostedAiRouteError(authResult);\n\t\t\tif (gate) return gate;";
+function change(from,to){expect(fixed.split(from)).toHaveLength(2);return fixed.replace(from,to);}
+function grade(name,source=fixed,unused=false){
+ const cwd=join(root,name);cpSync(template,cwd,{recursive:true});
+ if(source===null)rmSync(join(cwd,indexPath));else writeFileSync(join(cwd,indexPath),source);
+ if(unused)writeFileSync(join(cwd,'unused-correct-dispatcher.ts'),fixed);
+ symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');
+ mkdirSync(dirname(join(cwd,graderPath)),{recursive:true});writeFileSync(join(cwd,graderPath),grader);
+ return spawnSync(process.execPath,['test',graderPath],{cwd,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});
+}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}
+test('broken parent fails three access outcomes and preserves four',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('3 fail');expect(r.stderr).toContain('4 pass');});
+test('historical reference passes seven outcomes',()=>{const r=grade('reference');expect(r.status).toBe(0);expect(r.stderr).toContain('7 pass');});
+test('unused correct dispatcher does not repair actual caller',()=>fails(grade('unused',broken,true)));
+test('equivalent private binding name passes',()=>{const s=fixed.replace('paidHostedAiRouteError,','paidHostedAiRouteError as paidSpeechGate,').replaceAll('paidHostedAiRouteError(authResult)','paidSpeechGate(authResult)');expect(grade('rename',s).status).toBe(0);});
+test('ignoring the real gate is rejected',()=>fails(grade('ignored',change(gate,gate.replace('if (gate) return gate;','void gate;')))));
+test('provider work before a refusal is rejected',()=>fails(grade('late',change(gate,gate.replace('if (gate) return gate;','if (gate) { await handleTextToSpeech(request, env); return gate; }')))));
+test('blanket refusal loses paid preserved behavior',()=>fails(grade('deny',change(gate,gate+'\n\t\t\treturn new Response("blocked",{status:401});'))));
+test('response-only audio without provider effect is rejected',()=>fails(grade('fake',change('return await handleTextToSpeech(request, env);','return new Response(new Uint8Array([82,73,70,70,4,0,0,0,87,65,86,69]), {headers:{"content-type":"audio/wav"}});'))));
+test('missing dispatcher is setup failure',()=>{const r=grade('missing',null);expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index e6859b305..7e99054c6 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-22.4",
+ "dataset_version": "2026-09-22.5",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -2263,6 +2263,45 @@
],
"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/tts-wav.test.js"
}
+ },
+ {
+ "id": "ai-gateway-paid-tts-dispatch",
+ "name": "Enforce paid access before text-to-speech provider work",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "6301ad06342d4bad420598db86db9e44423b05b6"
+ },
+ "base_ref": "8d2fd0135dbfe71e64aa9db2c4300601648c2077",
+ "oracle_ref": "6301ad06342d4bad420598db86db9e44423b05b6",
+ "oracle_paths": [
+ "packages/ai-gateway/src/index.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "billing",
+ "access",
+ "voice",
+ "provider-effects"
+ ],
+ "prompt": "The text-to-speech gateway route can send speech to the provider without a paid account. Repair the actual POST /v1/text-to-speech boundary. For a valid text body, an anonymous caller must receive 401, a verified Free caller 403, and a verified caller with missing plan truth 503, all as JSON errors before any speech-provider request. Preserve paid Basic and Business callers: one provider request carrying the original text must return the exact provider audio bytes with audio/wav. Paid accounts denied good standing must still receive 403 before provider work; paid requests with invalid empty text must still receive 400 with no provider work. All refusals must be JSON errors. Use synthetic identity-verifier, entitlement API, rate-limit and audio-provider ports only. Do not assume a helper name or particular implementation. This case does not test authentication cryptography, live payment or provider delivery, transcription, native playback, database concurrency, agent isolation or model capability.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/ai-gateway/node_modules",
+ "destination_path": "packages/ai-gateway/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/paid-tts-dispatch.test.js",
+ "destination_path": "evals/coding-agent/graders/paid-tts-dispatch.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/paid-tts-dispatch.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/paid-tts-dispatch.test.js b/evals/coding-agent/graders/paid-tts-dispatch.test.js
new file mode 100644
index 000000000..2e774d0b7
--- /dev/null
+++ b/evals/coding-agent/graders/paid-tts-dispatch.test.js
@@ -0,0 +1,49 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeEach, afterAll, expect, mock, test } from 'bun:test';
+let s, serial = 0;
+const oldFetch = globalThis.fetch;
+const audio = new Uint8Array([82,73,70,70,4,0,0,0,87,65,86,69]);
+mock.module('@clerk/backend', () => ({verifyToken: async () => ({sub:s.subject})}));
+const {handleRequest} = await import('../../../packages/ai-gateway/src/index');
+const {__resetAuthEntitlementCacheForTests: reset} = await import('../../../packages/ai-gateway/src/utils/auth');
+const env = {NODE_ENV:'production',CLERK_SECRET_KEY:'synthetic',DEEPGRAM_API_KEY:'synthetic',
+ RATE_LIMITER:{idFromName:x=>x,get:()=>({fetch:async()=>Response.json(s.denied
+ ? {allowed:false,standing:'denied',standing_reason:'banned'}
+ : {allowed:true,standing:'good',remaining:10,reset_in:60,tier:'logged_in',rpm_limit:25})})},
+ DB:{prepare:()=>{throw Error('Unexpected database operation');}}};
+const ctx={waitUntil:()=>{},passThroughOnException:()=>{}};
+beforeEach(()=>{
+ reset?.(); s={subject:`user_probe_${++serial}`,plan:'standard',provider:[],lookups:0,denied:false,unknown:false};
+ globalThis.fetch=async(input,init)=>{
+ const url=String(input);
+ if(url==='https://screenpipe.com/api/user'){
+ s.lookups++; const paid=s.plan!=='none';
+ return Response.json({success:true,user:s.unknown?{clerk_id:s.subject}:{clerk_id:s.subject,
+ cloud_subscribed:s.plan==='pro',app_entitled:paid,subscription_plan:s.plan,
+ entitlement:{active:paid,plan:s.plan,features:{app:paid,cloud:s.plan==='pro'}}}});
+ }
+ if(url.startsWith('https://api.deepgram.com/v1/speak')){s.provider.push({url,body:JSON.parse(init.body)});return new Response(audio);}
+ throw Error('Unexpected network port');
+ };
+});
+afterAll(()=>{globalThis.fetch=oldFetch;});
+async function request(auth=true,text='synthetic speech'){
+ return handleRequest(new Request('https://gateway.example.invalid/v1/text-to-speech',{
+ method:'POST',headers:{'content-type':'application/json',...(auth?{authorization:'Bearer eyJ.synthetic.probe'}:{})},
+ body:JSON.stringify({text})}),env,ctx);
+}
+async function refused(status,auth=true,text='synthetic speech'){
+ const r=await request(auth,text);expect(r.status).toBe(status);expect(s.provider).toEqual([]);
+ expect(r.headers.get('content-type')).toContain('application/json');
+}
+test('anonymous valid speech is refused before provider effect',()=>refused(401,false));
+test('verified free valid speech is refused before provider effect',async()=>{s.plan='none';await refused(403);});
+test('unknown plan cannot grant provider access',async()=>{s.unknown=true;await refused(503);});
+for(const plan of ['standard','pro'])test(`${plan} retains exact provider audio and one effect`,async()=>{
+ s.plan=plan;const r=await request();expect(r.status).toBe(200);expect(new Uint8Array(await r.arrayBuffer())).toEqual(audio);
+ expect(r.headers.get('content-type')).toBe('audio/wav');expect(s.provider).toHaveLength(1);
+ expect(s.provider[0].body).toEqual({text:'synthetic speech'});
+});
+test('denied good standing blocks paid provider access',async()=>{s.denied=true;await refused(403);});
+test('paid invalid input remains refused without provider access',()=>refused(400,true,''));