YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: preserve source identity in meeting transcript text
Louis Beaumonta3a337402017
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex cc754c134..fac597207 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1,5 +1,18 @@# Coding-agent regression evals+`app-meeting-speaker-source` executes the actual transcript renderer and full+meeting Markdown builder with synthetic chunks. Twelve outcomes distinguish+per-stream generic labels while preserving explicit names, grouping, the gap+boundary, sorting, whitespace, empty speech, notes and stored identities.+The parent fails five outcomes and preserves seven; the historical fix and+current source pass twelve. Run+`bun test evals/coding-agent/calibrate-meeting-speaker-source.test.js` for twelve+controls, including equivalent code, unused correct code, no-op package scripts,+wrong source labels, lost grouping and missing-source classification.+The fixture appears only at grading and needs no dependency links. This does+not test the live announcement, panel UI, native clipboard, diarization,+execution isolation or model performance.+`app-acp-declined-warmup` mounts the actual automatic ACP warmup hook withsynthetic native command replies. Fifteen outcomes cover cancelled, failed andpending starts across new chats/remounts, successful handshakes, independentdiff --git a/evals/coding-agent/calibrate-meeting-speaker-source.test.js b/evals/coding-agent/calibrate-meeting-speaker-source.test.jsnew file mode 100644index 000000000..281d9d80c--- /dev/null+++ b/evals/coding-agent/calibrate-meeting-speaker-source.test.js@@ -0,0 +1,46 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, resolve } from 'node:path';+import { createHash } from 'node:crypto';+import { classifyGraderError } from './grader-outcome.mjs';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='app-meeting-speaker-source');+const sourcePath=item.oracle_paths[0];+const source=ref=>execFileSync('git',['show',`${ref}:${sourcePath}`],{cwd:repo,encoding:'utf8'});+const parent=source(item.base_ref),fixed=source(item.oracle_ref);+const fixture=readFileSync(join(import.meta.dir,'graders/meeting-speaker-source.fixture.ts.txt'));+const hash=x=>createHash('sha256').update(x).digest('hex');+function change(from,to){expect(fixed.split(from)).toHaveLength(2);return fixed.replace(from,to);}+const controls=[+ ['parent',parent,'fail'],['reference',fixed,'pass'],['current',source('HEAD'),'pass'],+ ['equivalent',change('const speakerName = sourceQualifiedSpeakerName(c.speakerName ?? "", c.deviceType);','const speakerName = sourceQualifiedSpeakerName(c.speakerName == null ? "" : c.speakerName, c.deviceType);'),'pass'],+ ['unused-correct',parent,'fail'],+ ['no-op-script',parent,'fail'],+ ['drop-qualification',change('const speakerName = sourceQualifiedSpeakerName(c.speakerName ?? "", c.deviceType);','const speakerName = c.speakerName ?? "";'),'fail'],+ ['wrong-source',change('direction === "input" ? "Microphone"','direction === "input" ? "Meeting audio"'),'fail'],+ ['lose-grouping',change('if (last && last.speakerName === speakerName && gap < MAX_GAP_MS)', 'if (false && last && last.speakerName === speakerName && gap < MAX_GAP_MS)'),'fail'],+ ['merge-gap-boundary',change('gap < MAX_GAP_MS','gap <= MAX_GAP_MS'),'fail'],+ ['mutate-input',change('const text = c.transcription.replace(/\\s+/g, " ").trim();','c.speakerName = "changed"; const text = c.transcription.replace(/\\s+/g, " ").trim();'),'fail'],+ ['missing-source',null,'error'],+];+for(const [name,body,expected]of controls)test(`speaker source calibration: ${name}`,()=>{+ const root=mkdtempSync(join(tmpdir(),'speaker-source-'));+ try{+ const target=join(root,sourcePath);mkdirSync(dirname(target),{recursive:true});+ if(body!==null)writeFileSync(target,body);+ if(name==='no-op-script')writeFileSync(join(root,'package.json'),JSON.stringify({scripts:{test:'true'}}));+ if(name==='unused-correct')writeFileSync(join(root,'unused-correct.ts'),fixed);+ writeFileSync(join(root,item.grader.fixtures[0].destination_path),fixture);+ const result=spawnSync(process.execPath,['--no-env-file','test',item.grader.fixtures[0].destination_path],{cwd:root,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024});+ const kind=classifyGraderError(result);const observed=result.error||result.signal||kind?'error':result.status===0?'pass':'fail';+ if(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS){const dir=resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS);mkdirSync(dir,{recursive:true});writeFileSync(join(dir,`${name}.json`),JSON.stringify({expected,observed,exit:result.status,error_kind:kind,source_sha256:body===null?null:hash(body),fixture_sha256:hash(fixture),stdout:result.stdout,stderr:result.stderr},null,2));}+ expect(result.error).toBeUndefined();expect(result.signal).toBeNull();expect(observed).toBe(expected);+ if(expected==='pass')expect(result.stderr).toContain('12 pass');+ if(name==='parent'||name==='unused-correct'){expect(result.stderr).toContain('5 fail');expect(result.stderr).toContain('7 pass');}+ if(name==='missing-source')expect(kind).toBe('bun_unhandled_error');+ }finally{rmSync(root,{recursive:true,force:true});}+},65000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex f4981cb95..fd3f84d63 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6365,6 +6365,39 @@"destination_path": "apps/screenpipe-app-tauri/node_modules"}]+ },+ {+ "id": "app-meeting-speaker-source",+ "name": "Keep per-stream speaker labels distinct in meeting transcript text",+ "source": {+ "kind": "git_regression",+ "fix_commit": "13d960d27f7c4b95b1e35c23c76a45ea241d912b"+ },+ "base_ref": "b9cff1112aff37a2e461c620db4e03209c164b54",+ "oracle_ref": "13d960d27f7c4b95b1e35c23c76a45ea241d912b",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/utils/meeting-context.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "meetings",+ "transcript",+ "speaker-identity"+ ],+ "prompt": "Meeting transcript text merges microphone and meeting-audio turns when independent streams reuse generic speaker labels. Repair the actual transcript renderer and the transcript section of full meeting Markdown. Generic labels speaker, speaker followed by a space and digits, or speaker followed by an underscore and digits are case-insensitive: qualify them as Microphone for input, Meeting audio for output, and Unidentified for an unknown direction. Preserve the original label alongside that source using the existing bracketed speaker format, for example [Microphone · speaker 1]. Adjacent generic turns from different sources must remain separate and retain all words. Preserve explicit names, including names that only resemble generic labels; an explicit matching name may still coalesce across sources. Keep same-source consecutive turns grouped below a ten-second gap, but separate turns at exactly ten seconds. Preserve chronological sorting, whitespace normalization, empty/null inputs, omission of blank speech and omission of the unknown speaker label. Never mutate input chunks or their stored identities. Full meeting Markdown must retain notes. The grader invokes the complete existing module with synthetic chunks and refuses local API work. Any implementation achieving these outputs is acceptable. This task covers rendered transcript text only, not live announcements, the transcript panel, native clipboard delivery, audio diarization, storage identity, execution isolation or model capability.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/meeting-speaker-source.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/lib/utils/meeting-speaker-source.eval.test.ts"+ }+ ],+ "command": "bun --no-env-file test apps/screenpipe-app-tauri/lib/utils/meeting-speaker-source.eval.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/meeting-speaker-source.fixture.ts.txt b/evals/coding-agent/graders/meeting-speaker-source.fixture.ts.txtnew file mode 100644index 000000000..a52b1d213--- /dev/null+++ b/evals/coding-agent/graders/meeting-speaker-source.fixture.ts.txt@@ -0,0 +1,23 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, mock, test } from 'bun:test';+mock.module('@/lib/api',()=>({localFetch:async()=>{throw Error('Unexpected local API call');}}));+const {renderMeetingTranscript:render,buildMeetingMarkdown}=await import('./meeting-context.ts');+const chunk=(overrides:Record<string,unknown>={})=>({audioChunkId:1,audioFilePath:'/synthetic/audio',speakerId:null,speakerName:'speaker 1',deviceType:'output',isInput:false,transcription:'remote words',timestamp:'2026-01-01T00:00:01Z',source:'live',...overrides});+const pair=(name='speaker 1')=>[chunk({speakerName:name}),chunk({audioChunkId:2,speakerName:name,deviceType:'input',isInput:true,transcription:'local words',timestamp:'2026-01-01T00:00:03Z'})];+for(const name of ['speaker 1','SPEAKER_2','speaker'])test(`source labels stay separate: ${name}`,()=>{+ const lines=render(pair(name)).split('\n');expect(lines).toHaveLength(2);+ expect(lines[0]).toContain(`[Meeting audio · ${name}] remote words`);expect(lines[1]).toContain(`[Microphone · ${name}] local words`);+});+test('unknown direction stays distinct from known microphone',()=>{const rows=pair();rows[0].deviceType='';const lines=render(rows).split('\n');expect(lines).toHaveLength(2);expect(lines[0]).toContain('[Unidentified · speaker 1] remote words');expect(lines[1]).toContain('[Microphone · speaker 1] local words');});+test('full meeting markdown uses the same qualified transcript and preserves notes',()=>{+ const markdown=buildMeetingMarkdown({meeting:{id:42,title:'Synthetic meeting',meeting_start:'2026-01-01T00:00:00Z',meeting_end:'2026-01-01T00:01:00Z',note:'Preserved note'},context:null,transcript:pair()});+ expect(markdown).toContain('[Meeting audio · speaker 1] remote words');expect(markdown).toContain('[Microphone · speaker 1] local words');expect(markdown).toContain('## Notes\n\nPreserved note');+});+test('named identity remains unchanged and groups across sources',()=>{const lines=render(pair('Jordan')).split('\n');expect(lines).toHaveLength(1);expect(lines[0]).toContain('[Jordan] remote words local words');});+test('names resembling generic labels are not relabeled',()=>{for(const name of ['speaker coach','speaker-1','SpeakerOne'])expect(render([chunk({speakerName:name})])).toContain(`[${name}] remote words`);});+test('same source neighboring chunks still coalesce',()=>{const rows=pair();rows[1].deviceType='output';const lines=render(rows).split('\n');expect(lines).toHaveLength(1);expect(lines[0]).toEndWith('remote words local words');});+test('ten second gap separates otherwise matching turns',()=>{const rows=pair('Jordan');rows[1].timestamp='2026-01-01T00:00:11Z';expect(render(rows).split('\n')).toHaveLength(2);});+test('sorts by timestamp without changing chunks or stored speaker identities',()=>{const rows=pair('Jordan').reverse();const before=JSON.stringify(rows);expect(render(rows)).toContain('remote words local words');expect(JSON.stringify(rows)).toBe(before);});+test('empty inputs and blank text produce no transcript',()=>{expect(render(null)).toBe('');expect(render([])).toBe('');expect(render([chunk({transcription:' \n '})])).toBe('');});+test('normalizes text whitespace and preserves unknown unlabelled speaker',()=>{const out=render([chunk({speakerName:'unknown',transcription:' preserved \n words '})]);expect(out).toEndWith('preserved words');expect(out).not.toContain('[');});