YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: verify TTS WAV response boundaries
louisb775a9bd5bab
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex d8f1a7964..76d446f06 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 67 git-mined regressions. See+The current app corpus contains 68 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -678,3 +678,22 @@ changes only the scanner. UI, native filesystem permissions, actual importpersistence and newer maintenance/sync behavior are outside this case. Theseparate current product dialog check uses a synthetic scan/import port. No realchat histories are read, and no execution-isolation claim follows.++## TTS WAV response boundary++`ai-gateway-tts-wav-response` executes the actual speech handler, voice utility+and response helpers against a synthetic provider. Fifteen outcomes cover WAV+request parameters and byte preservation, malformed signatures and short bodies,+provider HTTP/transport/body failures, explicit recovery, invalid input and+preserved MP3 utility calls. Only the voice utility is the historical oracle.++Run `bun test evals/coding-agent/calibrate-tts-wav.test.js`. Ten controls cover+parent/reference, an unused correct utility, equivalent helper naming, missing+container selection, arbitrary-byte acceptance, discarded audio, forced WAV+validation, blanket refusal and missing-source setup failure.++These are corpus and grader checks, not model trials. The unrelated transcription+SDK and completion factory are fail-closed ports. Gateway authentication/rate+limits, actual playback, live provider compatibility and full WAV container+integrity are outside this signature-level contract. No real audio, native build,+model call or dependency link is used; execution isolation remains unproven.diff --git a/evals/coding-agent/calibrate-tts-wav.test.js b/evals/coding-agent/calibrate-tts-wav.test.jsnew file mode 100644index 000000000..1ae9c85c0--- /dev/null+++ b/evals/coding-agent/calibrate-tts-wav.test.js@@ -0,0 +1,25 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..'),path='packages/ai-gateway/src/utils/voice-utils.ts';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='ai-gateway-tts-wav-response');+const source=(ref,p=path)=>execFileSync('git',['show',`${ref}:${p}`],{cwd:repo,encoding:'utf8'});+const broken=source(item.base_ref),fixed=source(item.oracle_ref),fixture='evals/coding-agent/graders/tts-wav.test.js';+const grader=readFileSync(join(repo,fixture)),root=mkdtempSync(join(tmpdir(),'tts-wav-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,code,unused){const cwd=join(root,name);const put=(p,b)=>{mkdirSync(dirname(join(cwd,p)),{recursive:true});writeFileSync(join(cwd,p),b);};for(const p of ['packages/ai-gateway/src/handlers/voice.ts','packages/ai-gateway/src/utils/cors.ts','packages/ai-gateway/src/types.ts'])put(p,source(item.base_ref,p));if(code!==null)put(path,code);if(unused)put('packages/ai-gateway/src/utils/unused.ts',unused);put(fixture,grader);return spawnSync(process.execPath,['test','./'+fixture],{cwd,encoding:'utf8',timeout:15_000,maxBuffer:8*1024*1024,env:{PATH:dirname(process.execPath)}});}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toMatch(/[1-9][0-9]* fail/);expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');}+function change(old,next){expect(fixed.split(old)).toHaveLength(2);return fixed.replace(old,next);}+test('parent exposes intended format failures with preserved refusals',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toMatch(/[1-9][0-9]* pass/);});+test('reference preserves all15outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('15 pass');});+test('equivalent signature helper naming passes',()=>expect(grade('equivalent',fixed.replaceAll('hasWavHeader','isRequestedWaveContainer')).status).toBe(0));+test('unused correct utility cannot repair caller',()=>fails(grade('unused',broken,fixed)));+test('omitting explicit container fails provider contract',()=>fails(grade('container',change("url.searchParams.set('container', 'wav');",''))));+test('accepting arbitrary bytes fails malformed results',()=>fails(grade('accept-bytes',change("if (encoding === 'linear16' && !hasWavHeader(audioBuffer))",'if (false)'))));+test('dropping returned provider bytes fails successful responses',()=>fails(grade('drop-bytes',change('return audioBuffer;','return new ArrayBuffer(0);'))));+test('forcing WAV validation for mp3 loses preserved behavior',()=>fails(grade('force-wav',change("encoding === 'linear16' && !hasWavHeader(audioBuffer)",'!hasWavHeader(audioBuffer)'))));+test('blanket refusal cannot satisfy the grader',()=>fails(grade('deny',change('return audioBuffer;','return null;'))));+test('missing utility is setup failure',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Could not resolve');expect(r.stderr).toContain('Unhandled error between tests');expect(r.stderr).toContain('0 pass');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex cd396f886..e6859b305 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-22.1",+ "dataset_version": "2026-09-22.4","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -2230,6 +2230,39 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/app-external-chat-scan.test.js"}+ },+ {+ "id": "ai-gateway-tts-wav-response",+ "name": "Return requested WAV audio without mislabelling provider bytes",+ "source": {+ "kind": "git_regression",+ "fix_commit": "ffff7de62e09bcd7058e219dbc13620e38c7189a"+ },+ "base_ref": "b68c120070351b90afb9e1df1e5f2c866f7fe396",+ "oracle_ref": "ffff7de62e09bcd7058e219dbc13620e38c7189a",+ "oracle_paths": [+ "packages/ai-gateway/src/utils/voice-utils.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "gateway",+ "tts",+ "binary-response",+ "provider-failure"+ ],+ "prompt": "The text-to-speech handler labels provider bytes audio/wav even when the provider sends another format. Repair the existing handleTextToSpeech/textToSpeech path. The HTTP handler must request linear16 audio in a WAV container from the existing speech endpoint, using the default voice or the explicitly supported voice, the configured provider authorization and the literal text. A successful provider WAV response must reach the caller byte-for-byte with status200 and audio/wav. Empty or shorter-than12-byte results, MP3 bytes, and buffers without both RIFF and WAVE signatures must produce a JSON error with status500, never successful audio. This is a format/signature boundary, not a full WAV decoder requirement. Provider HTTP rejection, transport rejection or body-read failure must similarly fail once without automatic retries, and a later explicit request must recover. Preserve rejection of missing/nonstring/empty text and unsupported voices with400 before provider access; whitespace-only text remains a500 error without provider access. Preserve the utility public API for explicitly requested non-WAV encoding such as mp3: forward that encoding, do not force a WAV container or WAV validation, and preserve successful bytes. Use only synthetic provider responses and text. No specific private helper name or implementation structure is required. This evaluates the actual TTS handler and utility, not gateway authentication/rate limits, transcription, real playback, provider connectivity or full container integrity.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/tts-wav.test.js",+ "destination_path": "evals/coding-agent/graders/tts-wav.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/tts-wav.test.js"+ }}]}diff --git a/evals/coding-agent/graders/tts-wav.test.js b/evals/coding-agent/graders/tts-wav.test.jsnew file mode 100644index 000000000..1da42c7a8--- /dev/null+++ b/evals/coding-agent/graders/tts-wav.test.js@@ -0,0 +1,33 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterEach, expect, test } from 'bun:test';+import { resolve } from 'node:path';+const root=process.cwd();+async function load(path){+ const built=await Bun.build({entrypoints:[resolve(root,path)],target:'bun',write:false,plugins:[{name:'synthetic-unrelated-ports',setup(build){+ build.onResolve({filter:/^(@deepgram\/sdk|\.\.\/providers)$/},args=>({path:args.path,namespace:'unrelated'}));+ build.onLoad({filter:/.*/,namespace:'unrelated'},args=>({contents:args.path==='@deepgram/sdk'?'export function createClient(){throw Error("Unrelated transcription forbidden")}':'export function createProvider(){throw Error("Unrelated completion forbidden")}',loader:'js'}));+ }}]});+ if(!built.success)throw Error('Cannot find module or compile source: '+built.logs.map(String).join('\n'));+ return import('data:text/javascript;base64,'+Buffer.from(await built.outputs[0].text()).toString('base64'));+}+const {handleTextToSpeech}=await load('packages/ai-gateway/src/handlers/voice.ts');+const {textToSpeech}=await load('packages/ai-gateway/src/utils/voice-utils.ts');+const wav=Uint8Array.from([82,73,70,70,38,0,0,0,87,65,86,69,102,109,116,32,16,0,0,0,1,0,1,0,128,62,0,0,0,125,0,0,2,0,16,0,100,97,116,97,2,0,0,0,0,0]);+const env={DEEPGRAM_API_KEY:'synthetic-provider-key'};let calls,reply;const originalFetch=globalThis.fetch;+beforeEach(()=>{calls=[];reply=()=>new Response(wav);globalThis.fetch=async(input,init)=>{const url=new URL(String(input));expect(url.origin+url.pathname).toBe('https://api.deepgram.com/v1/speak');calls.push({url,init});return reply();};});+afterEach(()=>{globalThis.fetch=originalFetch;});+function request(body){return new Request('https://gateway.example.invalid/v1/text-to-speech',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify(body)});}+async function success(body={text:'Synthetic speech'},voice='aura-asteria-en'){+ const r=await handleTextToSpeech(request(body),env);expect(r.status).toBe(200);expect(r.headers.get('content-type')).toBe('audio/wav');expect(new Uint8Array(await r.arrayBuffer())).toEqual(wav);expect(calls).toHaveLength(1);+ const {url,init}=calls[0];expect(url.searchParams.get('model')).toBe(voice);expect(url.searchParams.get('encoding')).toBe('linear16');expect(url.searchParams.get('container')).toBe('wav');expect(init.method).toBe('POST');expect(JSON.parse(init.body)).toEqual({text:body.text});expect(new Headers(init.headers).get('authorization')).toBe('Token synthetic-provider-key');+}+test('default handler requests WAV PCM and preserves actual provider bytes',()=>success());+test('explicit supported voice and literal text survive the request',()=>success({text:' Synthetic "speech"\nsecond line 🧪',voice:'aura-luna-en'},'aura-luna-en'));+const bad=[['empty',new Uint8Array()],['short RIFF',Uint8Array.from([82,73,70,70])],['eleven bytes',wav.slice(0,11)],['MP3',Uint8Array.from([255,243,96,196])],['wrong RIFF',Uint8Array.from(wav.map((v,i)=>i===0?0:v))],['wrong WAVE',Uint8Array.from(wav.map((v,i)=>i===8?0:v))]];+for(const [name,bytes] of bad)test(`rejects ${name} instead of labelling it audio/wav`,async()=>{reply=()=>new Response(bytes);const r=await handleTextToSpeech(request({text:'Synthetic speech'}),env);expect(r.status).toBe(500);expect(r.headers.get('content-type')).toContain('application/json');expect((await r.json()).error).toBeTruthy();expect(calls).toHaveLength(1);});+for(const kind of ['HTTP','transport','body'])test(`provider ${kind} failure refuses audio; explicit retry can recover`,async()=>{reply=()=>{if(kind==='transport')throw Error('synthetic offline');if(kind==='body')return {ok:true,arrayBuffer:async()=>{throw Error('synthetic body failure')}};return new Response('synthetic unavailable',{status:503});};const r=await handleTextToSpeech(request({text:'Synthetic speech'}),env);expect(r.status).toBe(500);expect(r.headers.get('content-type')).toContain('application/json');expect((await r.json()).error).toBeTruthy();expect(calls).toHaveLength(1);calls=[];reply=()=>new Response(wav);await success();});+test('missing, nonstring and invalid voice requests refuse before provider access',async()=>{for(const body of [{},{text:5},{text:''},{text:'hello',voice:'unsupported-model'}]){const r=await handleTextToSpeech(request(body),env);expect(r.status).toBe(400);expect((await r.json()).error).toBeTruthy();}expect(calls).toEqual([]);});+test('whitespace-only text fails without a provider request',async()=>{const r=await handleTextToSpeech(request({text:' \n\t'}),env);expect(r.status).toBe(500);expect((await r.json()).error).toBeTruthy();expect(calls).toEqual([]);});+test('non-WAV utility encoding remains supported without a WAV container',async()=>{const mp3=Uint8Array.from([255,243,96,196,7,8]);reply=()=>new Response(mp3);expect(new Uint8Array(await textToSpeech('Synthetic mp3',env,{encoding:'mp3',voice:'aura-luna-en'}))).toEqual(mp3);expect(calls).toHaveLength(1);expect(calls[0].url.searchParams.get('encoding')).toBe('mp3');expect(calls[0].url.searchParams.has('container')).toBe(false);});+test('rejection of wrong bytes does not poison a later explicit WAV request',async()=>{reply=()=>new Response(Uint8Array.from([1,2,3]));expect((await handleTextToSpeech(request({text:'Synthetic speech'}),env)).status).toBe(500);calls=[];reply=()=>new Response(wav);await success();});