Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover partial meeting transcript recovery

Louis Beaumontffeac69dd6b5
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index a548447d7..b98bea9af 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1894,3 +1894,7 @@ refusal. Grading uses synthetic loopback requests, an empty home and disabled
telemetry; it does not install from npm or call upstream tools. Compilation and
unresponsive startup stop collection as setup errors. Calibrate with
`bun test evals/coding-agent/calibrate-mcp-package-dispatch.test.js`.
+
+### Meeting background recovery
+
+`app-meeting-background-recovery` executes the actual meeting audio reader with synthetic local API responses. Eight outcomes cover partial live plus recovered speech, ordering/capping, duplicate and distinct-device behavior, source failures, malformed rows and metadata preservation. The known fix changes only the reader for this case; native transcription and database recovery are outside its evidence boundary. Run `bun test evals/coding-agent/calibrate-meeting-background.test.js` for calibration controls.
diff --git a/evals/coding-agent/calibrate-meeting-background.test.js b/evals/coding-agent/calibrate-meeting-background.test.js
new file mode 100644
index 000000000..88408feaf
--- /dev/null
+++ b/evals/coding-agent/calibrate-meeting-background.test.js
@@ -0,0 +1,44 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='app-meeting-background-recovery');
+const sourcePath=item.oracle_paths[0];
+const source=ref=>execFileSync('git',['show',`${ref}:${sourcePath}`],{cwd:repo,encoding:'utf8'});
+const parent=source(item.base_ref),fixed=source(item.oracle_ref);
+const fixture=readFileSync(join(import.meta.dir,'graders/meeting-background.fixture.ts.txt'));
+const hash=x=>createHash('sha256').update(x).digest('hex');
+function change(from,to){expect(fixed.split(from)).toHaveLength(2);return fixed.replace(from,to);}
+const controls=[
+ ['parent',parent,'fail'],['reference',fixed,'pass'],['current',source('HEAD'),'pass'],
+ ['equivalent',change('const merged = sortAudioChunks([...liveRows, ...backgroundRows]);','const merged = sortAudioChunks(liveRows.concat(backgroundRows));'),'pass'],
+ ['unused-correct',parent,'fail'],
+ ['drop-live',change('return mergeMeetingAudioChunks(routedRows, out, cap);','return mergeMeetingAudioChunks([], out, cap);'),'fail'],
+ ['drop-background',change('return mergeMeetingAudioChunks(routedRows, out, cap);','return routedRows;'),'fail'],
+ ['reverse-order',change('return mergeMeetingAudioChunks(routedRows, out, cap);','return mergeMeetingAudioChunks(routedRows, out, cap).reverse();'),'fail'],
+ ['retain-duplicates',change('if (seen.has(key)) continue;\n seen.add(key);\n out.push(chunk);','seen.add(key);\n out.push(chunk);'),'fail'],
+ ['blanket-empty',change('return mergeMeetingAudioChunks(routedRows, out, cap);','return [];'),'fail'],
+ ['missing-source',null,'error'],
+];
+for(const [name,body,expected]of controls)test(`meeting recovery calibration: ${name}`,()=>{
+ const root=mkdtempSync(join(tmpdir(),'meeting-recovery-'));
+ try{
+ const target=join(root,sourcePath);mkdirSync(dirname(target),{recursive:true});
+ if(body!==null)writeFileSync(target,body);
+ if(name==='unused-correct')writeFileSync(join(root,'unused-correct.ts'),fixed);
+ writeFileSync(join(root,item.grader.fixtures[0].destination_path),fixture);
+ const result=spawnSync(process.execPath,['--no-env-file','test',item.grader.fixtures[0].destination_path],{cwd:root,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024});
+ const kind=classifyGraderError(result);const observed=result.error||result.signal||kind?'error':result.status===0?'pass':'fail';
+ if(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS){const dir=resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS);mkdirSync(dir,{recursive:true});writeFileSync(join(dir,`${name}.json`),JSON.stringify({expected,observed,exit:result.status,error_kind:kind,source_sha256:body===null?null:hash(body),fixture_sha256:hash(fixture),stdout:result.stdout,stderr:result.stderr},null,2));}
+ expect(result.error).toBeUndefined();expect(result.signal).toBeNull();expect(observed).toBe(expected);
+ if(expected==='pass')expect(result.stderr).toContain('8 pass');
+ if(name==='parent'||name==='unused-correct'){expect(result.stderr).toContain('4 fail');expect(result.stderr).toContain('4 pass');}
+ if(name==='missing-source')expect(kind).toBe('bun_unhandled_error');
+ }finally{rmSync(root,{recursive:true,force:true});}
+},65000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index f337c3c88..2f48d12eb 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6253,6 +6253,41 @@
],
"command": "bun --no-env-file test packages/ai-gateway/eval-outbound-chat.test.ts"
}
+ },
+ {
+ "id": "app-meeting-background-recovery",
+ "name": "Keep recovered meeting speech beside partial live transcripts",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "2b83241eee061b953edcbac3e66c70199bf5d8ca"
+ },
+ "base_ref": "ed2496416c20fc0b1d68c1355481036ddf79975f",
+ "oracle_ref": "2b83241eee061b953edcbac3e66c70199bf5d8ca",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/utils/meeting-context.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "meeting",
+ "transcript",
+ "recovery"
+ ],
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/utils/meeting-context.ts"
+ ],
+ "prompt": "A partial routed meeting transcript hides speech recovered by background transcription. Repair the existing fetchMeetingAudio function so it returns both routed and recovered speech in chronological order, up to the requested positive cap. Remove identical text/time/device/speaker duplicates across sources while preserving distinct speech and separate input/output devices. Preserve background-only recovery when the routed request fails, retain available routed words when search fails, and return an empty list when both sources are unavailable. Filter malformed background rows and preserve valid chunk identity, audio path, timestamp, speaker and device metadata. Keep the public interface compatible; no particular helper, state layout or merge algorithm is required. Tests run the actual complete utility module with synthetic local API responses. This task does not establish native capture, live transcription delivery, database reconciliation, current cross-source speaker-label normalization, recording authority, browser rendering, agent isolation or model performance. Do not contact live services.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/meeting-background.fixture.ts.txt",
+ "destination_path": "apps/screenpipe-app-tauri/lib/utils/meeting-background.eval.test.ts"
+ }
+ ],
+ "command": "bun --no-env-file test apps/screenpipe-app-tauri/lib/utils/meeting-background.eval.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/meeting-background.fixture.ts.txt b/evals/coding-agent/graders/meeting-background.fixture.ts.txt
new file mode 100644
index 000000000..094a597f1
--- /dev/null
+++ b/evals/coding-agent/graders/meeting-background.fixture.ts.txt
@@ -0,0 +1,25 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+// Synthetic local API ports; the actual complete module is imported below.
+import { beforeEach, expect, mock, test } from 'bun:test';
+let routed: unknown[] = []; let background: unknown[] = []; let routedFails=false; let backgroundFails=false;
+mock.module('@/lib/api', () => ({ localFetch: async (url: string) => {
+ if(url.startsWith('/meetings/')) { if(routedFails)throw Error('synthetic unavailable');return {ok:true,json:async()=>routed}; }
+ if(url.startsWith('/search?')) {if(backgroundFails)throw Error('synthetic unavailable');return {ok:true,json:async()=>({data:background})};}
+ throw Error('unexpected endpoint');
+}}));
+const { fetchMeetingAudio } = await import('./meeting-context.ts');
+const ts=(second:number)=>`2026-01-01T00:00:${String(second).padStart(2,'0')}.000Z`;
+const live=(text:string,second:number)=>({id:second,meetingId:42,provider:'synthetic',itemId:String(second),deviceName:'speaker',deviceType:'output',speakerName:'speaker',transcript:text,capturedAt:ts(second)});
+const back=(text:string,second:number)=>({content:{chunk_id:second,file_path:'/synthetic/audio.mp4',timestamp:ts(second),transcription:text,device:'speaker',device_type:'output',speaker:{id:1,name:'speaker'}}});
+const fetch=()=>fetchMeetingAudio(ts(0),ts(59),100,42);
+beforeEach(()=>{routed=[];background=[];routedFails=false;backgroundFails=false;});
+test('partial routed transcript retains later recovered words',async()=>{routed=[live('early live',1)];background=[back('later recovered',5)];expect((await fetch()).map(x=>x.transcription)).toEqual(['early live','later recovered']);});
+test('background-only retrieval works when routed service fails',async()=>{routedFails=true;background=[back('recovered only',5)];expect((await fetch()).map(x=>x.transcription)).toEqual(['recovered only']);});
+test('retains available live words when background fails',async()=>{routed=[live('available live',1)];backgroundFails=true;expect((await fetch()).map(x=>x.transcription)).toEqual(['available live']);});
+test('combines sources chronologically, removes exact duplicate and respects cap',async()=>{routed=[live('middle',5)];background=[back('last',9),back('middle',5),back('first',1)];expect((await fetchMeetingAudio(ts(0),ts(59),2,42)).map(x=>x.transcription)).toEqual(['first','middle']);});
+
+test('deduplicates identical routed and recovered utterances without losing distinct speech',async()=>{routed=[live('shared words',5)];background=[back('shared words',5),back('other words',7)];expect((await fetch()).map(x=>x.transcription)).toEqual(['shared words','other words']);});
+test('preserves distinct devices even when text and time match',async()=>{routed=[live('same words',5)];const mic=back('same words',5);mic.content.device_type='input';background=[mic];const rows=await fetch();expect(rows).toHaveLength(2);expect(rows.map(x=>x.isInput)).toEqual([false,true]);});
+test('filters malformed background records and retains valid record metadata',async()=>{background=[{content:{}},back('',2),back('valid words',3)];const rows=await fetch();expect(rows).toHaveLength(1);expect(rows[0]).toMatchObject({audioChunkId:3,audioFilePath:'/synthetic/audio.mp4',transcription:'valid words',timestamp:ts(3),speakerId:1,deviceType:'output',isInput:false});});
+test('unavailable sources return an empty result',async()=>{routedFails=true;backgroundFails=true;expect(await fetch()).toEqual([]);});