YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: verify meeting summary completion and saved-note handoff
louis6fd3cbd1583b
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex c2f4d967a..d2726b435 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -334,3 +334,26 @@ current shared protocol are real. This does not execute tools or establish livemodel behavior, full schema validation, every malformed XML form, prompt-injectionresistance, native permission checks, encrypted transport or agent isolation.These are corpus and grader checks, not model trials.+++## Meeting summary completion and saved-note handoff++`app-meeting-summary-handoff` mounts the real NoteView, summary surface and+Markdown renderer with the real lifecycle, stream reducer and save queue.+Nine DOM/IO outcomes cover retained streamed text, empty/stale saved reads,+read failure and recovery, already-saved initial state, execution identity,+active-run preservation and unmount cleanup. The parent fails five intended+assertions and preserves four outcomes; reference and selected current source+pass all nine. Only the two production component files are applied as oracle.++Run `bun test evals/coding-agent/calibrate-meeting-handoff.test.js` with the+app JavaScript dependencies installed. Calibration distinguishes behavioral+failures from missing-source setup and rejects an unused helper, stale reads,+lost completion rendering, early refresh bookkeeping and blanket refusal.+Equivalent helper naming remains accepted.++API, native, analytics, context, chat and unrelated child UI ports are synthetic.+The host applies real onSaved callbacks to the meeting prop and permits the+existing autosave of that same note; this is not proof of backend persistence,+crash recovery, real generation, native capture or filesystem isolation.+Dependencies are exposed only during grading. No model trial is implied.diff --git a/evals/coding-agent/calibrate-meeting-handoff.test.js b/evals/coding-agent/calibrate-meeting-handoff.test.jsnew file mode 100644index 000000000..2b3ac28eb--- /dev/null+++ b/evals/coding-agent/calibrate-meeting-handoff.test.js@@ -0,0 +1,33 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-meeting-summary-handoff');+const note=`${app}/components/meeting-notes/note-view.tsx`, surface=`${app}/components/meeting-notes/meeting-workspace.tsx`;+const root=mkdtempSync(join(tmpdir(),'meeting-handoff-calibration-')), archives=new Map();+const paths=['components','lib','package.json','tsconfig.json'].map(p=>`${app}/${p}`).concat('crates/screenpipe-core/assets/acp/agents.json');+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function change(cwd,path,from,to){const p=join(cwd,path),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}) {+ const cwd=join(root,name);mkdirSync(cwd);+ if(!archives.has(ref)) archives.set(ref,execFileSync('git',['archive',ref,...paths],{cwd:repo,maxBuffer:128*1024*1024}));+ execFileSync('tar',['-x','-C',cwd],{input:archives.get(ref)});+ for(const f of item.grader.fixtures)writeFileSync(join(cwd,f.destination_path),readFileSync(join(import.meta.dir,f.local_path)));+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);+ return spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:120000,maxBuffer:4*1024*1024});+}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/AssertionError|Unable to find an element/);expect(r.stdout+r.stderr).not.toMatch(/Failed to resolve import|Failed to load url|Cannot find module|No .* export is defined/);}+test('parent has five intended failures and four preserved outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('5 failed | 4 passed');},120000);+test('reference passes nine real component outcomes',()=>{const r=grade('reference');expect(r.status).toBe(0);expect(r.stdout).toContain('9 passed');},120000);+test('equivalent helper naming is accepted',()=>{const r=grade('renamed',item.oracle_ref,cwd=>{for(const p of [note,surface])writeFileSync(join(cwd,p),readFileSync(join(cwd,p),'utf8').replaceAll('meetingSummarySaveIsVisible','summaryReadHasCaughtUp'));});expect(r.status).toBe(0);},120000);+test('a correct unused helper cannot conceal the broken note caller',()=>fails(grade('old-caller',item.oracle_ref,cwd=>writeFileSync(join(cwd,note),execFileSync('git',['show',`${item.base_ref}:${note}`],{cwd:repo})))),120000);+test('discarding the completed stream in the surface is rejected',()=>fails(grade('old-surface',item.oracle_ref,cwd=>change(cwd,surface,'(state === "working" || state === "ready")','state === "working"'))),120000);+test('accepting stale reads is rejected',()=>fails(grade('stale',item.oracle_ref,cwd=>change(cwd,surface,'return !awaitedNewSummary || refreshedNote !== currentNote;','return true;'))),120000);+test('marking completion before the note read prevents retries and is rejected',()=>fails(grade('early-mark',item.oracle_ref,cwd=>change(cwd,note,'const meetingResponse = await localFetch(`/meetings/${meeting.id}`);','refreshedSummaryExecutionRef.current = next.execution.id;\n const meetingResponse = await localFetch(`/meetings/${meeting.id}`);'))),120000);+test('blanket refusal fails legitimate saved-note outcomes',()=>fails(grade('deny-all',item.oracle_ref,cwd=>change(cwd,surface,'return !awaitedNewSummary || refreshedNote !== currentNote;','return false;'))),120000);+test('removing pipe identity filtering is rejected',()=>fails(grade('wrong-pipe',item.oracle_ref,cwd=>change(cwd,note,'if (!pipe || pipe.pipeName !== summaryPipeSlug) return;','if (!pipe) return;'))),120000);+test('missing source remains a setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,note)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stdout).not.toContain('9 passed');},120000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex b75e755ea..2a13eadba 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-20.2",+ "dataset_version": "2026-09-20.3","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1689,6 +1689,51 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/glm-tool-call-boundary.test.js"}+ },+ {+ "id": "app-meeting-summary-handoff",+ "name": "Keep completed meeting summaries visible until authoritative save catches up",+ "source": {+ "kind": "git_regression",+ "fix_commit": "fc5a65ffbfc785d3b2e2240096ef33fa71d2f5c2",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/6846"+ },+ "base_ref": "d86ad74d99fe6df68232ae04fcb6b7b5b3a99c40",+ "oracle_ref": "fc5a65ffbfc785d3b2e2240096ef33fa71d2f5c2",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/meeting-notes/note-view.tsx",+ "apps/screenpipe-app-tauri/components/meeting-notes/meeting-workspace.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "meetings",+ "streaming",+ "persistence",+ "recovery"+ ],+ "prompt": "A completed meeting summary disappears while its saved note is still catching up. Repair the real note view and summary surface so the matching streamed summary remains visible when execution changes from running to ready. With either no prior summary or an older saved summary, do not report a stale completed-note read as a successful save. Continue reading until the new summary is visible, including after a temporary failed read, and deliver the refreshed note without remounting. Once that execution has been refreshed, do not repeatedly reload its saved note. Preserve an already-saved summary when opening an initially completed meeting, ignore unrelated pipe/execution streams, do not refresh the saved note while execution is still running, and stop polling and event subscriptions on unmount. Preserve legitimate note autosave callbacks and existing title/content. Verify observable UI and save-boundary outcomes using synthetic local API, native and analytics ports. No live meeting data, provider calls or native builds are needed. No particular helper name or state layout is required.",+ "agent_timeout_seconds": 1200,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 120,+ "fixtures": [+ {+ "local_path": "graders/meeting-handoff.eval.test.tsx",+ "destination_path": "apps/screenpipe-app-tauri/meeting-handoff.eval.test.tsx"+ },+ {+ "local_path": "graders/meeting-handoff.config.ts",+ "destination_path": "apps/screenpipe-app-tauri/meeting-handoff.config.ts"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config meeting-handoff.config.ts"+ }}]}diff --git a/evals/coding-agent/graders/meeting-handoff.config.ts b/evals/coding-agent/graders/meeting-handoff.config.tsnew file mode 100644index 000000000..123785412--- /dev/null+++ b/evals/coding-agent/graders/meeting-handoff.config.ts@@ -0,0 +1,6 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { defineConfig } from 'vitest/config';+import { resolve } from 'node:path';+export default defineConfig({esbuild:{jsx:'automatic'},resolve:{alias:{'@':resolve(process.cwd())}},css:{postcss:{plugins:[]}},+ test:{environment:'jsdom',include:['meeting-handoff.eval.test.tsx'],pool:'forks',poolOptions:{forks:{singleFork:true}}}});diff --git a/evals/coding-agent/graders/meeting-handoff.eval.test.tsx b/evals/coding-agent/graders/meeting-handoff.eval.test.tsxnew file mode 100644index 000000000..954ff7c52--- /dev/null+++ b/evals/coding-agent/graders/meeting-handoff.eval.test.tsx@@ -0,0 +1,148 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import React from 'react';+import { beforeEach, afterEach, describe, test, expect, vi } from 'vitest';+import { render, screen, act, cleanup } from '@testing-library/react';+import { NoteView } from '@/components/meeting-notes/note-view';++const ports = vi.hoisted(() => ({+ observers: new Set<any>(), status: 'running', note: null as string | null,+ readStatus: 200, calls: [] as string[], unexpected: [] as string[],+ saved: vi.fn(), settings: { aiPresets: [], languages: ['english'], user: null },+ toast: vi.fn(), updateSettings: vi.fn(),+}));+vi.mock('@/lib/api', () => ({ localFetch: async (path: string, options?: any) => {+ ports.calls.push(`${options?.method || 'GET'} ${path}`);+ if (path === '/meetings/42' && options?.method === 'PUT') {+ const body = JSON.parse(options.body);+ if (body.note !== ports.note || body.title !== 'Synthetic meeting') {+ ports.unexpected.push('unexpected saved note');+ }+ return {ok: true};+ }+ if (options?.method && options.method !== 'GET') {+ ports.unexpected.push(`mutation ${path}`); throw new Error(`unexpected mutation ${path}`);+ }+ if (path.includes('/summary-status?')) return {ok: true, json: async () => ({+ state: ports.status, auto_summary_enabled: true, execution_id: 91,+ execution_status: ports.status === 'running' ? 'running' : 'completed',+ })};+ if (path === '/meetings/42') return {ok: ports.readStatus === 200, status: ports.readStatus,+ json: async () => meeting(ports.note)};+ if (path.endsWith('/transcript')) return {ok: true, json: async () => []};+ if (path === '/health') return {ok: true, json: async () => ({})};+ if (path === '/pipes/meeting-summary') return {ok: true, json: async () => ({data:{config:{preset:[]}}})};+ ports.unexpected.push(path); throw new Error(`unexpected path ${path}`);+}}));+vi.mock('@/lib/hooks/use-settings', () => ({ useSettings: () => ({settings: ports.settings, updateSettings: ports.updateSettings}) }));+vi.mock('@/lib/hooks/use-managed-policy', () => ({useManagedPolicy: () => ({isManagedDeployment: false, policy:{aiPresetPolicy:{}}})}));+vi.mock('@/lib/acp-rollout', async (original) => ({...await original<any>(), useAcpRolloutEnabled: () => false}));+vi.mock('@/lib/hooks/use-usage-status', () => ({useUsageStatus: () => null, hostedAiAllowanceForModel: () => null}));+vi.mock('@/components/ui/use-toast', () => ({useToast: () => ({toast: ports.toast})}));+vi.mock('@/lib/utils/tauri', () => ({commands: new Proxy({}, {get: () => () => {throw new Error('native command forbidden')}})}));+vi.mock('@tauri-apps/api/event', () => ({listen: async () => () => {}}));+vi.mock('@tauri-apps/api/webview', () => ({getCurrentWebview: () => ({onDragDropEvent: async () => () => {}})}));+vi.mock('@tauri-apps/plugin-shell', () => ({open: () => {throw new Error('external navigation forbidden')}}));+vi.mock('@tauri-apps/plugin-dialog', () => ({save: () => {throw new Error('native dialog forbidden')}}));+vi.mock('@tauri-apps/plugin-fs', () => ({readFile: () => {throw new Error('native filesystem forbidden')}}));+vi.mock('posthog-js', () => ({default: {capture: vi.fn()}}));+vi.mock('@/lib/analytics/qualified-value', () => ({qualifiedValue: {meetingNoteOpened: vi.fn()}}));+vi.mock('@/lib/logging/browser-log', () => ({writeBrowserLogNow: vi.fn()}));+vi.mock('@/lib/events/tauri-events', () => ({listenTyped: async () => () => {}, TAURI_EVENTS:{}}));+vi.mock('@/lib/events/bus', () => ({mountAgentEventBus: vi.fn(), registerObserver: (fn: any) => {+ ports.observers.add(fn); return () => ports.observers.delete(fn);+}}));+vi.mock('@/lib/utils/meeting-context', () => ({fetchMeetingContext: async () => null,+ fetchMeetingAudio: async () => [], renderMeetingTranscript: () => ''}));+vi.mock('@/components/meeting-notes/use-meeting-chat', () => ({useMeetingChat: () => ({inFlight:false, turns:[]})}));+vi.mock('@/components/meeting-notes/use-meeting-one-tap-send', () => ({useMeetingOneTapSend: () => ({label:null,suggestions:[]})}));+vi.mock('@/components/meeting-notes/attendees-pill', () => ({AttendeesPill: () => null}));+vi.mock('@/components/meeting-notes/receipts', () => ({Receipts: () => null}));+vi.mock('@/components/meeting-notes/replay-strip', () => ({ReplayStrip: () => null}));+vi.mock('@/components/meeting-notes/listening-sticks', () => ({ListeningSticks: () => null}));+vi.mock('@/components/meeting-notes/transcript-panel', () => ({TranscriptPanel: () => null}));+vi.mock('@/components/meeting-notes/meeting-share-menu', () => ({MeetingShareMenu: () => null}));+vi.mock('@/components/meeting-notes/meeting-chat-panel', () => ({MeetingChatPanel: () => null}));+vi.mock('@/components/meeting-notes/meeting-summary-transition', () => ({MeetingSummaryTransition: () => null}));+vi.mock('@/components/meeting-notes/note-editor', () => ({NoteEditor: React.forwardRef(() => null)}));+vi.mock('@/components/connected-share-dialog', () => ({ConnectedShareDialog: () => null}));++const fresh = '## Summary\nFresh synthetic summary.';+const old = '## Summary\nEarlier synthetic summary.';+function meeting(note: string | null) { return {id:42, meeting_start:'2026-09-02T10:00:00Z',+ meeting_end:'2026-09-02T10:30:00Z', meeting_app:'zoom', title:'Synthetic meeting',+ attendees:null,note,detection_source:'auto',created_at:'2026-09-02T10:00:00Z'}; }+async function mount(note: string | null = null) {+ function Host() {+ const [value,setValue] = React.useState(meeting(note));+ return <NoteView meeting={value as any} isLive={false} stopping={false} resuming={false}+ initialWorkspaceTab="summary" onBack={() => {}} onStop={async () => {}} onResume={async () => {}}+ onDeleted={() => {}} onSaved={(updated: any) => {ports.saved(updated);setValue(updated)}} />;+ }+ let view: any;+ await act(async () => {view = render(<Host/>)}); return view;+}+async function tick(ms = 2000) { await act(async () => {await vi.advanceTimersByTimeAsync(ms)}); }+async function emit(text = fresh, sessionId = 'pipe:meeting-summary:91', executionId = 91) {+ await act(async () => { for (const fn of ports.observers) fn({source:'pipe',sessionId,executionId,+ event:{type:'message_update',assistantMessageEvent:{type:'text_delta',delta:text}}}); });+}+beforeEach(() => {+ vi.useFakeTimers(); vi.setSystemTime(new Date('2026-09-02T11:00:00Z'));+ ports.observers.clear();ports.status='running';ports.note=null;ports.readStatus=200;+ ports.calls=[];ports.unexpected=[];ports.saved.mockClear();localStorage.clear();+ Object.defineProperty(window,'matchMedia',{writable:true,value:() => ({matches:true,addListener(){},removeListener(){},addEventListener(){},removeEventListener(){}})});+ globalThis.ResizeObserver = class {observe(){} unobserve(){} disconnect(){}} as any;+ HTMLElement.prototype.scrollIntoView = () => {};+});+afterEach(() => { cleanup();vi.useRealTimers();expect(ports.unexpected).toEqual([]); });++describe('completed meeting summary handoff', () => {+ test.each([null,old])('retains stream while waiting for a fresh save from %s', async (initial) => {+ ports.note=initial;await mount(initial);await emit();+ expect(screen.getByText('Fresh synthetic summary.')).toBeTruthy();+ ports.status='ready';await tick();+ expect(screen.getByText('Fresh synthetic summary.')).toBeTruthy();+ expect(ports.saved).not.toHaveBeenCalled();+ ports.note=fresh;await tick(10000);+ expect(ports.saved).toHaveBeenCalled();+ expect(ports.saved.mock.calls.every(([value]) => value.note === fresh)).toBe(true);+ expect(screen.getByText('Fresh synthetic summary.')).toBeTruthy();+ const reads=ports.calls.filter(c => c === 'GET /meetings/42').length;+ await tick(10000);expect(ports.calls.filter(c => c === 'GET /meetings/42')).toHaveLength(reads);+ });+ test.each([null,old])('does not report a stale completed note as saved from %s', async (initial) => {+ ports.note=initial;await mount(initial);ports.status='ready';await tick();+ expect(ports.saved).not.toHaveBeenCalled();ports.note=fresh;await tick(10000);+ expect(ports.saved).toHaveBeenCalled();+ expect(ports.saved.mock.calls.every(([value]) => value.note === fresh)).toBe(true);+ });+ test('does not save while status remains running', async () => {+ ports.note=fresh;await mount();await emit();await tick(10000);+ expect(ports.saved).not.toHaveBeenCalled();+ expect(ports.calls.filter(c => c === 'GET /meetings/42')).toHaveLength(0);+ });+ test('retries a failed completed-note read and then delivers the saved summary', async () => {+ await mount();await emit();ports.status='ready';ports.readStatus=503;await tick();+ expect(ports.saved).not.toHaveBeenCalled();ports.readStatus=200;ports.note=fresh;+ await tick(10000);expect(ports.saved).toHaveBeenCalled();+ expect(ports.saved.mock.calls.every(([value]) => value.note === fresh)).toBe(true);+ });+ test('accepts an already saved summary on initial completed status once', async () => {+ ports.status='ready';ports.note=old;await mount(old);+ expect(ports.saved).toHaveBeenCalledTimes(1);+ expect(screen.getByText('Earlier synthetic summary.')).toBeTruthy();+ await tick(20000);expect(ports.saved).toHaveBeenCalledTimes(1);+ });+ test('ignores another pipe and another execution while retaining the matching stream', async () => {+ await mount();await emit('Wrong pipe','pipe:other:91',91);+ await emit('Wrong execution','pipe:meeting-summary:92',92);+ expect(screen.queryByText(/Wrong pipe|Wrong execution/)).toBeNull();await emit();+ expect(screen.getByText('Fresh synthetic summary.')).toBeTruthy();+ });+ test('unmount removes stream subscription and stops polling', async () => {+ const view = await mount();view.unmount();const count=ports.calls.length;+ await tick(30000);expect(ports.calls).toHaveLength(count);+ expect(ports.observers.size).toBe(0);expect(ports.saved).not.toHaveBeenCalled();+ });+});