YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover transcription language at the provider boundary
Louis Beaumont669bced489b7
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 6d9a728fd..89916863d 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.This does not establish native recording continuity, real authentication,account-switch race safety, browser integration, isolation or model performance.-The current app corpus contains 125 git-mined regressions. See+The current app corpus contains 127 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1558,3 +1558,17 @@ preserves six; the historical recorder fix and current source pass eight. RunStorage and settings are synthetic; this does not establish native delivery, diskdurability, crash recovery, continued-chat behavior, agent isolation or model quality.The hidden fixture is installed only when grading begins.+++`app-transcription-nova-language` runs the actual batch transcription handler and+A/B service with synthetic audio and intercepted provider transport. Twelve+outcomes cover automatic, single, multiple, normalized and legacy language+selection, query precedence, audio preservation and successful/error payloads.+The parent fails nine language outcomes and preserves three; the service-only+historical fix and current source pass all twelve. Run+`bun test evals/coding-agent/calibrate-transcription-language.test.js` for eleven+controls, including an equivalent implementation, unused correct code, wrong+language shortcuts, lost audio and missing-source/syntax infrastructure errors.+Fixtures appear only at grading and need no dependency links. This does not+establish speech recognition quality, native settings persistence, real provider+delivery, alternate-provider behavior, execution isolation or model improvement.diff --git a/evals/coding-agent/calibrate-transcription-language.test.js b/evals/coding-agent/calibrate-transcription-language.test.jsnew file mode 100644index 000000000..141f3062a--- /dev/null+++ b/evals/coding-agent/calibrate-transcription-language.test.js@@ -0,0 +1,44 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+import { classifyGraderError } from './grader-outcome.mjs';++const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(x=>x.id==='app-transcription-nova-language');+const handler='packages/ai-gateway/src/handlers/transcription.ts';+const service='packages/ai-gateway/src/services/transcription-ab.ts';+const cors='packages/ai-gateway/src/utils/cors.ts';+const show=(ref,path)=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});+const broken=show(item.base_ref,service), fixed=show(item.oracle_ref,service);+const root=mkdtempSync(join(tmpdir(),'transcription-language-calibration-'));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name, source, ref=item.base_ref, missing=false) {+ const workspace=join(root,name);+ const write=(path,data)=>{mkdirSync(dirname(join(workspace,path)),{recursive:true});writeFileSync(join(workspace,path),data);};+ if(!missing)write(handler,show(ref,handler));+ write(service,source); write(cors,show(ref,cors));+ write('packages/ai-gateway/src/providers/vertex.ts','export class VertexAIProvider { constructor(){throw Error("unexpected alternate provider")} }');+ const fixture=item.grader.fixtures[0];write(fixture.destination_path,readFileSync(join(import.meta.dir,fixture.local_path),'utf8'));+ const result=spawnSync(process.execPath,['--no-env-file','test',fixture.destination_path],{cwd:workspace,encoding:'utf8',timeout:30000,env:{PATH:dirname(process.execPath)}});+ if(process.env.TRANSCRIPTION_CALIBRATION_RESULTS){const out=resolve(process.env.TRANSCRIPTION_CALIBRATION_RESULTS);mkdirSync(out,{recursive:true});writeFileSync(join(out,name+'.stdout'),result.stdout??'');writeFileSync(join(out,name+'.stderr'),result.stderr??'');writeFileSync(join(out,name+'.json'),JSON.stringify({status:result.status,signal:result.signal,error:result.error?.message??null,classification:classifyGraderError(result)}));}+ return result;+}+function pass(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(0);expect(r.stderr).toContain('12 pass');}+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(classifyGraderError(r)).toBeNull();}+function patch(anchor,replacement){expect(fixed.split(anchor)).toHaveLength(2);return fixed.replace(anchor,replacement);}+test('broken caller fails nine language outcomes and preserves three',()=>{const r=grade('parent',broken);fail(r);expect(r.stderr).toContain('9 fail');expect(r.stderr).toContain('3 pass');},40000);+test('historical service-only repair passes actual handler outcomes',()=>pass(grade('reference',fixed)),40000);+test('current source preserves the contract',()=>pass(grade('current',show('HEAD',service),'HEAD')),40000);+test('unused complete correct helper cannot hide the broken call',()=>{const helper=fixed.slice(fixed.indexOf('export function deepgramLanguageQuery'),fixed.indexOf('export async function callDeepgram'));fail(grade('unused',broken+'\n'+helper));},40000);+test('always English cannot replace language selection',()=>fail(grade('english',patch('deepgramLanguageQuery(req.languages)',"'&language=en'"))),40000);+test('always multilingual cannot replace a single selected language',()=>fail(grade('multi',patch('deepgramLanguageQuery(req.languages)',"'&language=multi'"))),40000);+test('audio bytes must survive the repaired request',()=>fail(grade('audio',fixed.replace('body: req.audioBuffer,','body: new ArrayBuffer(0),'))),40000);+test('equivalent inline language construction passes',()=>pass(grade('equivalent',patch('deepgramLanguageQuery(req.languages)',"((codes)=>'&language='+encodeURIComponent(codes.length===1?codes[0]:'multi'))(req.languages.map(x=>x.trim().toLowerCase()).filter(x=>x&&x!=='true'&&x!=='false'))"))),40000);+test('missing active handler is an infrastructure error',()=>{const r=grade('missing',fixed,item.base_ref,true);expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(classifyGraderError(r)).toBe('bun_unhandled_error');expect(r.stderr).not.toContain('expect(received)');},40000);+test('service syntax failure is an infrastructure error',()=>{const r=grade('syntax','export const = ;');expect(r.status).not.toBe(0);expect(classifyGraderError(r)).toBe('bun_unhandled_error');expect(r.stderr).not.toContain('expect(received)');},40000);++test('equivalent Request transport preserves the same observable request',()=>{const source=patch('const resp = await fetch(url, {','const resp = await fetch(new Request(url, {').replace('signal: AbortSignal.timeout(180_000),\n });','signal: AbortSignal.timeout(180_000),\n }));');pass(grade('request',source));},40000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 179a50906..d1a87f36a 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5200,6 +5200,43 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/html-viewer.test.ts"}+ },+ {+ "id": "app-transcription-nova-language",+ "name": "Preserve language selection through cloud transcription",+ "source": {+ "kind": "git_regression",+ "fix_commit": "a52a6c640457308fec92daa270f423b79ce7a34a"+ },+ "base_ref": "066291bc4ccdd986e3a29e339c48b610d8fd2a2f",+ "oracle_ref": "a52a6c640457308fec92daa270f423b79ce7a34a",+ "oracle_paths": [+ "packages/ai-gateway/src/services/transcription-ab.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/ai-gateway/src/handlers/transcription.ts",+ "packages/ai-gateway/src/services/transcription-ab.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "transcription",+ "language",+ "gateway"+ ],+ "prompt": "Cloud batch transcription loses the selected language and returns English even when another language was requested. Repair the actual handleFileTranscription request path so the outgoing Nova-3 request uses a language parameter: a single selected code, trimmed and lowercased, or multi when no language or multiple languages are selected. Do not send the obsolete detect_language parameter to the provider. Preserve parsing of language/languages query values, repeated or comma-separated detect_language query codes, and legacy language/detect_language headers. Query codes take precedence over headers; boolean detection flags are not codes and must allow a usable legacy header fallback. Empty or auto-detection-only requests use multi. Preserve the default Deepgram POST endpoint, Nova-3 model, explicit sample rate, original audio bytes, audio content type and provider credential. With alternate-provider traffic disabled, make one provider call for a successful request or a permanent 400 response. Pass through the successful transcript payload unchanged, including an empty transcript; preserve HTTP 500 with the provider's error detail for permanent rejection. Use the existing public handler contract. No particular helper, export, or implementation structure is required. Use synthetic audio and intercepted provider transport only. Actual speech recognition quality, native language settings persistence, alternate providers, real credentials, deployed gateway admission, concurrent requests and infrastructure isolation are outside this task.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/transcription-language.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/eval-transcription-language.test.ts"+ }+ ],+ "command": "bun --no-env-file test packages/ai-gateway/eval-transcription-language.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/transcription-language.fixture.ts.txt b/evals/coding-agent/graders/transcription-language.fixture.ts.txtnew file mode 100644index 000000000..14360a2cc--- /dev/null+++ b/evals/coding-agent/graders/transcription-language.fixture.ts.txt@@ -0,0 +1,30 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll, expect, mock, test} from 'bun:test';+mock.module('./src/providers/vertex',()=>({VertexAIProvider:class {constructor(){throw Error('unexpected alternate provider')}}}));+const route=await import('./src/handlers/transcription.ts');+const originalFetch=globalThis.fetch;+afterAll(()=>{globalThis.fetch=originalFetch});+const cases=[+ {name:'auto',query:'detect_language=true',expected:'multi'},+ {name:'none',query:'',expected:'multi'},+ {name:'single',query:'language=es',expected:'es'},+ {name:'multiple',query:'detect_language=es&detect_language=pt',expected:'multi'},+ {name:'normalized',query:'language=%20ES%20',expected:'es'},+ {name:'legacy',query:'',headers:{detect_language:'fr'},expected:'fr'},+ {name:'precedence',query:'language=ja',headers:{detect_language:'fr'},expected:'ja'},+ {name:'comma',query:'languages=es,pt',expected:'multi'},+ {name:'boolean-fallback',query:'detect_language=false',headers:{language:'de'},expected:'de'},+ {name:'transport',query:'language=en',preserved:true},+ {name:'provider-refusal',query:'language=en',preserved:true,refusal:true},+ {name:'empty-transcript',query:'language=en',preserved:true,empty:true}+];+for(const c of cases) test(c.name,async()=>{+ const calls:any[]=[];const payload={results:{channels:[{alternatives:[{transcript:c.empty?'':'synthetic transcript',confidence:0.9}]}]}};+ globalThis.fetch=async(input,init)=>{const outgoing=new Request(input,init);calls.push({url:outgoing.url,method:outgoing.method,headers:Object.fromEntries(outgoing.headers.entries()),body:Array.from(new Uint8Array(await outgoing.arrayBuffer()))});return c.refusal?new Response('synthetic refusal',{status:400}):Response.json(payload)};+ const response=await route.handleFileTranscription(new Request('https://synthetic.invalid/v1/listen?'+c.query+'&sample_rate=22050',{method:'POST',headers:{'Content-Type':'audio/mpeg',...c.headers},body:new Uint8Array([1,2,3,4])}),{DEEPGRAM_API_KEY:'synthetic-local-key',WHISPER_TRAFFIC_PCT:'0',PARAKEET_TRAFFIC_PCT:'0',DUAL_SEND_PCT:'0'});+ const body=await response.json();expect(calls.length).toBe(1);expect(response.status).toBe(c.refusal?500:200);+ if(c.refusal){expect(body.error).toMatch(/synthetic refusal/)}else{expect(body).toEqual(payload)}+ const q=new URL(calls[0].url);expect(q.origin).toBe('https://api.deepgram.com');expect(q.pathname).toBe('/v1/listen');expect(q.searchParams.get('model')).toBe('nova-3');expect(q.searchParams.get('sample_rate')).toBe('22050');expect(calls[0].method).toBe('POST');expect(calls[0].body).toEqual([1,2,3,4]);expect(calls[0].headers['content-type']).toBe('audio/mpeg');expect(calls[0].headers.authorization).toBe('Token synthetic-local-key');+ if(!c.preserved){expect(q.searchParams.get('language')).toBe(c.expected);expect(q.searchParams.has('detect_language')).toBe(false)}+});