Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover meeting Stop after note-save refusal

Louis Beaumont6f66d1874e1c
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index d37ec8e48..bb14694c4 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1639,3 +1639,19 @@ suppression and missing-module setup errors. Only the frontend calendar module
is applied from the historical fix. Native endpoint mappings, OS permissions,
OAuth refresh, publisher backoff, enforced agent isolation and model performance
remain outside this evidence. Fixtures and dependencies are installed at grading.
+
+
+`app-meeting-stop-save-refusal` mounts the real historical NoteView with synthetic
+editor-input, HTTP and native ports. Five outcomes preserve draft-save ordering,
+overlapping edits, ordinary autosave and unchanged notes, while ensuring an
+explicit Stop still runs once after a failed save, with a warning and no false
+Saved indication. The parent fails that Stop outcome and preserves four; the
+historical reference passes five. Separate current-component checks preserve
+these outcomes with current display and inactive-service adapters.
+Run `bun test evals/coding-agent/calibrate-meeting-stop.test.js` with desktop test
+dependencies available for eleven historical grader controls, including inline warnings, empty feedback, changed
+warning copy, unused correct source, early/duplicate Stop, silent refusal, false
+Saved status and missing-source classification. Hidden fixtures and dependency
+links appear only when grading begins. This does not test editor internals,
+native stop/deferral release, database durability, global recording pause, full
+current dependency parity, execution isolation or model improvement.
diff --git a/evals/coding-agent/calibrate-meeting-stop.test.js b/evals/coding-agent/calibrate-meeting-stop.test.js
new file mode 100644
index 000000000..8ee626aff
--- /dev/null
+++ b/evals/coding-agent/calibrate-meeting-stop.test.js
@@ -0,0 +1,73 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-meeting-stop-save-refusal');
+const hook = `${app}/components/meeting-notes/note-view.tsx`;
+const root = mkdtempSync(join(tmpdir(), 'meeting-stop-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('5 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+const warning = ` toast({
+ title: "couldn't save notes",
+ description: "stopping anyway — your latest edits may not be saved.",
+ variant: "destructive",
+ });`;
+test('parent blocks Stop on rejected save and preserves four outcomes', () => {
+ const r = grade('parent', item.base_ref); fails(r);
+ expect(r.stdout).toContain('1 failed | 4 passed');
+ expect(r.stderr).toContain('to be called 1 times');
+}, 60000);
+test('reference passes all five outcomes', () => passes(grade('reference')), 60000);
+test('different warning copy remains acceptable', () => passes(grade('equivalent', item.oracle_ref, cwd => {
+ replace(cwd, hook, warning, warning.replace("couldn't save notes", 'Notes could not be saved').replace('stopping anyway — your latest edits may not be saved.', 'Your Stop request will continue; recent edits could not be stored.'));
+})), 60000);
+test('unused corrected source cannot hide the broken active action', () => fails(grade('unused', item.oracle_ref, cwd => {
+ writeFileSync(join(cwd, 'unused-correct-note-view.tsx'), readFileSync(join(cwd, hook)));
+ writeFileSync(join(cwd, hook), execFileSync('git', ['show', `${item.base_ref}:${hook}`], {cwd: repo}));
+})), 60000);
+test('early Stop fails successful-save ordering', () => fails(grade('early-stop', item.oracle_ref, cwd => {
+ replace(cwd, hook, ' if (savingBeforeStop) return;', ' if (savingBeforeStop) return;\n await onStop(); return;');
+})), 60000);
+test('silent save refusal fails the warning outcome', () => fails(grade('silent', item.oracle_ref, cwd => replace(cwd, hook, warning, ''))), 60000);
+test('duplicate Stop cannot pass', () => fails(grade('duplicate', item.oracle_ref, cwd => replace(cwd, hook, ' await onStop();', ' await onStop(); await onStop();'))), 60000);
+test('a false Saved status cannot pass', () => fails(grade('false-saved', item.oracle_ref, cwd => replace(cwd, hook, '} catch (saveErr) {', '} catch (saveErr) {\n setSaveState({kind: "saved", at: Date.now()});'))), 60000);
+test('missing component is setup failure, not a regression', () => {
+ const r = grade('missing', item.oracle_ref, cwd => rmSync(join(cwd, hook)));
+ expect(r.status).toBe(1); expect(r.stdout).toContain('no tests');
+ expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);
+}, 60000);
+
+test('an inline accessible warning is also accepted', () => passes(grade('inline-warning', item.oracle_ref, cwd => {
+ replace(cwd, hook, warning, '');
+ replace(cwd, hook, '<span className="text-destructive">offline — will retry</span>', '<span role="alert">The note could not be saved; Stop will continue.</span>');
+})), 60000);
+test('empty feedback is not a user warning', () => fails(grade('empty-warning', item.oracle_ref, cwd => replace(cwd, hook, warning, ' toast({});'))), 60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 66ed2f77c..09e0bbd46 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-10-02-remote-images-visible-fallbacks",
+ "dataset_version": "2026-10-07-meeting-stop-save-refusal",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -5396,6 +5396,53 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-calendar-unavailable.config.mjs"
}
+ },
+ {
+ "id": "app-meeting-stop-save-refusal",
+ "name": "Preserve explicit meeting Stop when saving the current note fails",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "f3cd3f8b8d462a8df686b236e75a702db8cad033"
+ },
+ "base_ref": "c89551d84a2f2c8065aa0e08dc706ef467643f74",
+ "oracle_ref": "f3cd3f8b8d462a8df686b236e75a702db8cad033",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/meeting-notes/note-view.tsx"
+ ],
+ "tags": [
+ "regression",
+ "meetings",
+ "user-control",
+ "persistence",
+ "escaped-failure"
+ ],
+ "gate": "advisory",
+ "prompt": "The live meeting note view prevents Stop when saving the latest draft fails. Repair the actual NoteView Stop action so a rejected pre-stop save warns the user and still invokes the supplied Stop callback exactly once, without reporting that the note was saved. Preserve successful-save ordering: wait for an in-flight save of the current draft before stopping. An unchanged note must stop without an unnecessary save. Preserve ordinary autosave and ensure an older in-flight save cannot overwrite a newer draft after overlapping edits, including the visible note and persisted request data. Keep the public component interfaces compatible. This task checks the real React component with synthetic editor-input, HTTP and native ports; actual editor internals, native meeting-stop/deferral release, database durability and global recording pause are outside scope. No particular queue implementation, callback name or warning copy is required.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/meeting-stop.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/note-save-probe.test.tsx"
+ },
+ {
+ "local_path": "graders/meeting-stop.config.ts",
+ "destination_path": "apps/screenpipe-app-tauri/note-save.config.ts"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config note-save.config.ts"
+ },
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/meeting-notes/note-view.tsx",
+ "apps/screenpipe-app-tauri/components/meeting-notes/note-save-queue.ts"
+ ]
}
]
}
diff --git a/evals/coding-agent/graders/meeting-stop.config.ts b/evals/coding-agent/graders/meeting-stop.config.ts
new file mode 100644
index 000000000..e2e14d766
--- /dev/null
+++ b/evals/coding-agent/graders/meeting-stop.config.ts
@@ -0,0 +1,6 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { defineConfig } from 'vitest/config';
+import { resolve } from 'node:path';
+export default defineConfig({esbuild:{jsx:'automatic'},resolve:{alias:{'@':resolve(process.cwd())}},css:{postcss:{plugins:[]}},
+ test:{environment:'jsdom',include:['note-save-probe.test.tsx'],pool:'forks',poolOptions:{forks:{singleFork:true}}}});
diff --git a/evals/coding-agent/graders/meeting-stop.fixture.tsx.txt b/evals/coding-agent/graders/meeting-stop.fixture.tsx.txt
new file mode 100644
index 000000000..eead7930a
--- /dev/null
+++ b/evals/coding-agent/graders/meeting-stop.fixture.tsx.txt
@@ -0,0 +1,40 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import React from 'react';
+import {test,expect,vi,beforeEach,afterEach} from 'vitest';
+import {render,screen,act,fireEvent,cleanup} from '@testing-library/react';
+import {NoteView} from '@/components/meeting-notes/note-view';
+const p=vi.hoisted(()=>({requests:[] as any[],persisted:'initial',saved:vi.fn(),stop:vi.fn(),toast:vi.fn(),unexpected:[] as string[]}));
+vi.mock('@/lib/api',()=>({localFetch:async(path:string,opts:any={})=>{
+ if(path==='/meetings/42'&&opts.method==='PUT'){const body=JSON.parse(opts.body);return new Promise(resolve=>p.requests.push({body,finish:(ok=true)=>{if(ok)p.persisted=body.note;resolve({ok,status:ok?200:503})}}))}
+ if(path==='/health')return {ok:true,json:async()=>({})};
+ p.unexpected.push(path);throw Error('Unexpected API '+path);
+}}));
+vi.mock('@/lib/hooks/use-settings',()=>({useSettings:()=>({settings:{aiPresets:[],languages:['english'],user:null},updateSettings:vi.fn()})}));
+vi.mock('@/components/ui/use-toast',()=>({useToast:()=>({toast:p.toast})}));
+vi.mock('@/lib/chat-utils',()=>({showChatWithPrefill:()=>{throw Error('Chat forbidden')}}));
+vi.mock('@/lib/utils/tauri',()=>({commands:new Proxy({},{get:()=>()=>{throw Error('Native forbidden')}})}));
+vi.mock('@tauri-apps/api/event',()=>({listen:async()=>()=>{}}));
+vi.mock('@tauri-apps/api/webview',()=>({getCurrentWebview:()=>({onDragDropEvent:async()=>()=>{}})}));
+vi.mock('@tauri-apps/plugin-shell',()=>({open:()=>{throw Error('Navigation forbidden')}}));
+vi.mock('@tauri-apps/plugin-dialog',()=>({save:()=>{throw Error('Dialog forbidden')}}));
+vi.mock('@tauri-apps/plugin-fs',()=>({readFile:()=>{throw Error('Filesystem forbidden')}}));
+vi.mock('posthog-js',()=>({default:{capture:vi.fn()}}));
+vi.mock('@/lib/utils/meeting-context',()=>({fetchMeetingContext:async()=>null,fetchMeetingAudio:async()=>[]}));
+vi.mock('@/components/meeting-notes/receipts',()=>({Receipts:()=>null}));
+vi.mock('@/components/meeting-notes/replay-strip',()=>({ReplayStrip:()=>null}));
+vi.mock('@/components/meeting-notes/listening-sticks',()=>({ListeningSticks:()=>null}));
+vi.mock('@/components/meeting-notes/transcript-panel',()=>({TranscriptPanel:()=>null}));
+vi.mock('@/components/meeting-notes/note-editor',()=>({NoteEditor:React.forwardRef((props:any,_ref)=><textarea aria-label="Synthetic note editor" value={props.value} onChange={e=>props.onChange(e.target.value)}/>)}));
+async function mount(){function Host(){const [meeting,setMeeting]=React.useState({id:42,meeting_start:'2026-06-11T10:00:00Z',meeting_end:null,meeting_app:'zoom',title:'Synthetic meeting',attendees:null,note:'initial',detection_source:'manual',created_at:'2026-06-11T10:00:00Z'});return <NoteView meeting={meeting as any} isLive={true} stopping={false} resuming={false} onBack={()=>{}} onStop={()=>{p.stop(p.persisted)}} onResume={()=>{}} onDeleted={()=>{}} onSaved={(value:any)=>{p.saved(value);setMeeting(value)}}/>}await act(async()=>{render(<Host/>)});}
+async function edit(text:string){await act(async()=>{fireEvent.change(screen.getByRole('textbox',{name:'Synthetic note editor'}),{target:{value:text}})});}
+async function tick(ms=810){await act(async()=>{await vi.advanceTimersByTimeAsync(ms)})}
+async function finish(r:any,ok=true){await act(async()=>{r.finish(ok)});}
+async function stop(){await act(async()=>{fireEvent.click(screen.getByRole('button',{name:/^stop$/i}))})}
+beforeEach(()=>{vi.useFakeTimers();vi.setSystemTime(new Date('2026-06-11T10:10:00Z'));p.requests=[];p.persisted='initial';p.saved.mockClear();p.stop.mockClear();p.toast.mockClear();p.unexpected=[];localStorage.clear();globalThis.fetch=vi.fn(async()=>{throw Error('External network forbidden')}) as any;Object.defineProperty(window,'matchMedia',{writable:true,value:()=>({matches:true,addListener(){},removeListener(){},addEventListener(){},removeEventListener(){}})});globalThis.ResizeObserver=class{observe(){}unobserve(){}disconnect(){}} as any;HTMLElement.prototype.scrollIntoView=()=>{};});
+afterEach(()=>{cleanup();vi.useRealTimers();expect(p.unexpected).toEqual([])});
+test('latest draft wins even if an older in-flight save completes later',async()=>{await mount();await edit('older');await tick();const first=p.requests[0];expect(first).toBeTruthy();await edit('latest');await tick();const completed=new Set<any>();if(p.requests[1]){await finish(p.requests[1]);completed.add(p.requests[1])}await finish(first);completed.add(first);for(const r of p.requests)if(!completed.has(r))await finish(r);expect(p.persisted).toBe('latest');expect((screen.getByRole('textbox',{name:'Synthetic note editor'}) as HTMLTextAreaElement).value).toBe('latest');});
+test('Stop waits until the current draft is persisted',async()=>{await mount();await edit('latest');await stop();expect(p.stop).not.toHaveBeenCalled();expect(p.requests.length).toBeGreaterThan(0);await finish(p.requests[0]);expect(p.stop).toHaveBeenCalledWith('latest');});
+test('failed save warns without blocking explicit Stop or pretending the note was saved',async()=>{await mount();await edit('latest');await stop();expect(p.requests.length).toBeGreaterThan(0);await finish(p.requests[0],false);expect(p.stop).toHaveBeenCalledTimes(1);expect(p.toast.mock.calls.some(([v])=>Boolean(v?.title||v?.description)) || screen.queryAllByRole('alert').some(el=>Boolean(el.textContent?.trim()))).toBe(true);expect(p.saved).not.toHaveBeenCalled();expect(screen.queryAllByText(/^saved\s*·/i)).toHaveLength(0);expect(p.persisted).toBe('initial');});
+test('unchanged note stops without a write',async()=>{await mount();await stop();expect(p.requests).toHaveLength(0);expect(p.stop).toHaveBeenCalledWith('initial');});
+test('ordinary autosave retains note and reports the saved value',async()=>{await mount();await edit('ordinary');await tick();await finish(p.requests[0]);expect(p.persisted).toBe('ordinary');expect(p.saved.mock.calls.at(-1)[0].note).toBe('ordinary');});