YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover recording status recovery and pause authority
Louis Beaumont5ccfeb24c4c3
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 116aee4ca..b42e005ae 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1,5 +1,15 @@# Coding-agent regression evals+`app-recording-status-authority` mounts the actual recording popover with+synthetic device, settings and callback inputs. Fourteen outcomes distinguish+empty or stale status from capture authority, preserve global/device actions,+and require status retry without capture mutation. The parent fails four and+preserves ten; the historical fix and current component pass fourteen.+Run `bun test evals/coding-agent/calibrate-recording-status-authority.test.js`+for equivalent, unused-fix, retry, routing, rollback and setup-error controls.+Fixtures and dependency links appear only at grading. This does not execute+Home wiring, native recording, enforced isolation or model trials.+`ai-gateway-vertex-stream-fallback` executes the public streaming provider withsynthetic authentication and transport. Twelve outcomes cover reasoning-onlyfinish, terminal marker, EOF, split Unicode, CRLF and empty deltas, whilediff --git a/evals/coding-agent/calibrate-recording-status-authority.test.js b/evals/coding-agent/calibrate-recording-status-authority.test.jsnew file mode 100644index 000000000..053948e21--- /dev/null+++ b/evals/coding-agent/calibrate-recording-status-authority.test.js@@ -0,0 +1,61 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-recording-status-authority');+const component = `${app}/components/recording-status.tsx`;+const root = mkdtempSync(join(tmpdir(), 'recording-status-authority-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);++ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('14 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent loses four status outcomes and preserves ten outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 10 passed');},60000);+test('reference passes fourteen outcomes',()=>passes(grade('reference')),60000);+test('component-only oracle repairs the parent',()=>passes(grade('oracle-only',item.base_ref,cwd=>writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.oracle_ref}:${component}`],{cwd:repo})))),60000);+test('current component passes the historical contract',()=>passes(grade('current','HEAD')),60000);+test('equivalent private names and removed test ids pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{+ const p=join(cwd,component);writeFileSync(p,readFileSync(p,'utf8').replaceAll('allPaused','everyDevicePaused').replaceAll('data-testid=','data-eval-neutral='));+})),60000);+test('unused correct component does not hide broken active component',()=>fails(grade('unused',item.oracle_ref,cwd=>{+ writeFileSync(join(cwd,`${app}/components/unused-recording-status.tsx`),readFileSync(join(cwd,component)));+ writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.base_ref}:${component}`],{cwd:repo}));+})),60000);+test('reversed global mode routing fails',()=>fails(grade('reversed-mode',item.oracle_ref,cwd=>replace(cwd,component,'if (!isGloballyPaused)','if (isGloballyPaused)'))),60000);+test('broken global pause fails',()=>fails(grade('broken-pause',item.oracle_ref,cwd=>replace(cwd,component,'await onPauseRecording();','await onResumeRecording?.();'))),60000);+test('lost device rollback fails',()=>fails(grade('broken-rollback',item.oracle_ref,cwd=>replace(cwd,component,'active: device.active','active: !device.active'))),60000);+test('missing component is setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,component)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);++test('status retry cannot mutate global capture',()=>fails(grade('retry-mutates-capture',item.oracle_ref,cwd=>replace(cwd,component,'await onRefreshDevices();','await onResumeRecording?.();'))),60000);+test('pending status retry must remain disabled',()=>fails(grade('retry-enabled',item.oracle_ref,cwd=>replace(cwd,component,'disabled={refreshLoading}','disabled={false}'))),60000);+test('lost empty global-pause precedence fails',()=>fails(grade('empty-global',item.oracle_ref,cwd=>replace(cwd,component,'(isGloballyPaused || (visibleDevices.length > 0 && !canPauseRecording))','(visibleDevices.length > 0 && !canPauseRecording)'))),60000);+test('blanket disabled capture fails preserved behavior',()=>fails(grade('blanket-disabled',item.oracle_ref,cwd=>replace(cwd,component,'const ui = useGT();','allCaptureDisabled = true; const ui = useGT();'))),60000);+test('no-op package script does not hide broken component',()=>fails(grade('parent-noop-script',item.base_ref,cwd=>{+ const file=join(cwd,app,'package.json');const pkg=JSON.parse(readFileSync(file,'utf8'));pkg.scripts={...pkg.scripts,test:'true','test:vitest':'true'};writeFileSync(file,JSON.stringify(pkg));+})),60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex f708d514f..e07079a21 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6518,6 +6518,56 @@],"command": "bun --no-env-file test packages/ai-gateway/eval-vertex-stream-fallback.test.ts"}+ },+ {+ "id": "app-recording-status-authority",+ "name": "Recover unavailable recording status without changing capture authority",+ "source": {+ "kind": "git_regression",+ "fix_commit": "dd332af338ba8302b3df12ef424d8a4c7e7bf291"+ },+ "base_ref": "bbc17e34d453abb9449be47e0cfc9d7a9ecd8565",+ "oracle_ref": "dd332af338ba8302b3df12ef424d8a4c7e7bf291",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/recording-status.tsx"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/recording-status.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "recording",+ "user-pause",+ "status-recovery"+ ],+ "prompt": "The recording status popover offers the wrong controls when device information is empty or stale. Preserve the RecordingStatus public interface. For a running session with no reported devices, provide accessible Check again and Open settings actions, without offering a global pause/resume action or changing capture. Check again calls onRefreshDevices once, keeps an accessible Check again or Checking button disabled while pending, prevents duplicate requests, and becomes usable again after either success or rejection. Open settings invokes onOpenRecordingSettings. Global pause must take precedence over an empty device list or stale active devices: show an accessible Resume all recording action that invokes onResumeRecording once, with no individual Pause/Resume controls or device requests. When settings disable all capture, show no capture or status-retry controls, but keep Open settings available, regardless of reported device or global-pause state. Preserve existing behavior: a live session whose devices are individually paused resumes those devices through POST /vision/device/start with monitor_id and POST /audio/device/start with device_name, without global callbacks; active or mixed devices pause through onPauseRecording; a globally paused session resumes through the global callback. A monitor without an id must issue no request while eligible audio devices still resume. Failed individual requests must restore their previous paused state. Hide the global control when both callbacks are absent; an unavailable global resume callback must leave its action disabled or absent. Preserve accessible action names pause all recording, resume all recording and resume (case-insensitive). No private function names, test attributes or implementation structure are required. This task covers frontend controls with synthetic callback/API boundaries, not Home wiring or native recording.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/recording-status-authority.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/.eval-hidden/recording-resume.test.tsx"+ },+ {+ "local_path": "graders/recording-resume-config.mjs.txt",+ "destination_path": "apps/screenpipe-app-tauri/.eval-hidden/recording-resume.config.mjs"+ },+ {+ "local_path": "graders/recording-resume-translation.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/.eval-hidden/recording-resume-translation.ts"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config .eval-hidden/recording-resume.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/recording-status-authority.fixture.tsx.txt b/evals/coding-agent/graders/recording-status-authority.fixture.tsx.txtnew file mode 100644index 000000000..09ad335d1--- /dev/null+++ b/evals/coding-agent/graders/recording-status-authority.fixture.tsx.txt@@ -0,0 +1,118 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import React from 'react';+import { afterEach, beforeEach, expect, test, vi } from 'vitest';+import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';+import { RecordingStatus } from '../components/recording-status';+import { TooltipProvider } from '../components/ui/tooltip';+const ports = vi.hoisted(() => ({ fetch: vi.fn() }));+vi.mock('@/lib/api', () => ({ localFetch: ports.fetch }));+vi.mock('@/lib/utils', () => ({ cn: (...xs: unknown[]) => xs.filter(Boolean).join(' ') }));+vi.mock('posthog-js', () => ({ default: { capture: vi.fn() } }));+afterEach(cleanup);+beforeEach(() => { ports.fetch.mockReset(); ports.fetch.mockResolvedValue({ok: true}); });+const paused = [+ {name:'Display', fullName:'Display synthetic', kind:'monitor', id:41, active:false},+ {name:'Mic', fullName:'Synthetic microphone (input)', kind:'input', active:false},+ {name:'Speaker', fullName:'Synthetic speakers (output)', kind:'output', active:false},+] as any;+function mount(devices: any[], global: boolean, extra: any = {}) {+ const pause=vi.fn(), resume=vi.fn();+ function Host() {+ const [state,setState]=React.useState(devices);+ return <TooltipProvider><RecordingStatus devices={state} onDevicesChange={setState}+ meetingActive={false} meetingLoading={false} onToggleMeeting={()=>{}}+ onPauseRecording={pause} onResumeRecording={resume} isGloballyPaused={global} {...extra}/>+ <output data-testid="device-state">{JSON.stringify(state)}</output></TooltipProvider>;+ }+ render(<Host/>);+ fireEvent.click(screen.getByRole('button', {name:/recording|paused/i}));+ return {pause,resume};+}+test('global stopped session resumes through global callback only', async()=>{+ const p=mount(paused,true);+ fireEvent.click(screen.getByRole('button',{name:/resume all recording/i}));+ await waitFor(()=>expect(p.resume).toHaveBeenCalledTimes(1));+ expect(p.pause).not.toHaveBeenCalled(); expect(ports.fetch).not.toHaveBeenCalled();+});+test('live session with every device paused resumes individual display and audio ports',async()=>{+ const p=mount(paused,false);+ fireEvent.click(screen.getByRole('button',{name:/resume all recording/i}));+ await waitFor(()=>expect(ports.fetch).toHaveBeenCalledTimes(3));+ expect(ports.fetch.mock.calls.map(([url,init])=>[url,init.method,JSON.parse(init.body)]).sort()).toEqual([+ ['/vision/device/start','POST',{monitor_id:41}],+ ['/audio/device/start','POST',{device_name:'Synthetic microphone (input)'}],+ ['/audio/device/start','POST',{device_name:'Synthetic speakers (output)'}],+ ].sort());+ expect(p.resume).not.toHaveBeenCalled();expect(p.pause).not.toHaveBeenCalled();+});+test('active capture retains global pause action',async()=>{+ const p=mount(paused.map((d:any)=>({...d,active:true})),false);+ fireEvent.click(screen.getByRole('button',{name:/pause all recording/i}));+ await waitFor(()=>expect(p.pause).toHaveBeenCalledTimes(1));+ expect(p.resume).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();+});+test('failed individual request restores the device pause state',async()=>{+ ports.fetch.mockResolvedValue({ok:false,status:503});+ mount([paused[1]],false);+ fireEvent.click(screen.getByRole('button',{name:/^resume$/i}));+ await waitFor(()=>expect(ports.fetch).toHaveBeenCalledTimes(1));+ await waitFor(()=>expect(JSON.parse(screen.getByTestId('device-state').textContent!)[0].active).toBe(false));+});++test('legacy monitor without an id makes no request while audio resumes',async()=>{+ const p=mount([{...paused[0],id:undefined},paused[1]],false);+ fireEvent.click(screen.getByRole('button',{name:/resume all recording/i}));+ await waitFor(()=>expect(ports.fetch).toHaveBeenCalledTimes(1));+ expect(ports.fetch.mock.calls[0][0]).toBe('/audio/device/start');+ expect(JSON.parse(ports.fetch.mock.calls[0][1].body)).toEqual({device_name:paused[1].fullName});+ expect(p.resume).not.toHaveBeenCalled();expect(p.pause).not.toHaveBeenCalled();+});+test('no global callbacks keeps the global control absent',()=>{+ mount(paused,false,{onPauseRecording:undefined,onResumeRecording:undefined});+ expect(screen.queryByRole('button',{name:/all recording/i})).toBeNull();+ expect(ports.fetch).not.toHaveBeenCalled();+});+test('stopped session without a resume callback cannot trigger a global action',()=>{+ const p=mount(paused,true,{onResumeRecording:undefined});+ const button=screen.queryByRole('button',{name:/resume all recording/i});+ // An unavailable action may be disabled or omitted.+ if(button) {expect((button as HTMLButtonElement).disabled).toBe(true);fireEvent.click(button);}+ expect(p.pause).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();+});+test('mixed active and paused devices keep the global pause action',async()=>{+ const p=mount([{...paused[0],active:true},paused[1]],false);+ fireEvent.click(screen.getByRole('button',{name:/pause all recording/i}));+ await waitFor(()=>expect(p.pause).toHaveBeenCalledTimes(1));+ expect(p.resume).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();+});++test('empty live status offers recovery without capture mutations',async()=>{+ const refresh=vi.fn(),settings=vi.fn();const p=mount([],false,{onRefreshDevices:refresh,onOpenRecordingSettings:settings});+ expect(screen.queryByRole('button',{name:/all recording/i})).toBeNull();+ fireEvent.click(screen.getByRole('button',{name:/check again/i}));+ await waitFor(()=>expect(refresh).toHaveBeenCalledTimes(1));+ expect(p.pause).not.toHaveBeenCalled();expect(p.resume).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();+ fireEvent.click(screen.getByRole('button',{name:/open settings/i}));expect(settings).toHaveBeenCalledTimes(1);+});+test('status retry prevents duplicate work and remains usable after rejection',async()=>{+ let reject!: (error:Error)=>void;const refresh=vi.fn().mockImplementationOnce(()=>new Promise((_resolve,rejectFn)=>{reject=rejectFn})).mockResolvedValue(undefined);+ const p=mount([],false,{onRefreshDevices:refresh});fireEvent.click(screen.getByRole('button',{name:/check again/i}));+ const pending=screen.getByRole('button',{name:/check again|checking/i});expect((pending as HTMLButtonElement).disabled).toBe(true);fireEvent.click(pending);expect(refresh).toHaveBeenCalledTimes(1);+ reject(new Error('synthetic status unavailable'));+ await waitFor(()=>expect((screen.getByRole('button',{name:/check again/i}) as HTMLButtonElement).disabled).toBe(false));+ fireEvent.click(screen.getByRole('button',{name:/check again/i}));await waitFor(()=>expect(refresh).toHaveBeenCalledTimes(2));+ await waitFor(()=>expect((screen.getByRole('button',{name:/check again/i}) as HTMLButtonElement).disabled).toBe(false));+ expect(p.pause).not.toHaveBeenCalled();expect(p.resume).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();+});+test.each([[],paused.map((d:any)=>({...d,active:true}))].map(snapshot=>[snapshot]))('global pause overrides empty or stale active snapshot %#',async devices=>{+ const p=mount(devices,true);expect(screen.queryByRole('button',{name:/^pause$/i})).toBeNull();expect(screen.queryByRole('button',{name:/^resume$/i})).toBeNull();+ fireEvent.click(screen.getByRole('button',{name:/resume all recording/i}));await waitFor(()=>expect(p.resume).toHaveBeenCalledTimes(1));+ expect(p.pause).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();+});+test.each([false,true])('disabled settings preserve capture authority %#',global=>{+ const settings=vi.fn(),refresh=vi.fn();const p=mount(paused.map((d:any)=>({...d,active:true})),global,{allCaptureDisabled:true,onOpenRecordingSettings:settings,onRefreshDevices:refresh});+ expect(screen.queryByRole('button',{name:/all recording|check again|^pause$|^resume$/i})).toBeNull();+ fireEvent.click(screen.getByRole('button',{name:/open settings/i}));expect(settings).toHaveBeenCalledTimes(1);+ expect(refresh).not.toHaveBeenCalled();expect(p.pause).not.toHaveBeenCalled();expect(p.resume).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();+});