Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): verify recording resume control effects

louis4340a80723e5
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index a1f7c71f5..d058ba036 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1344,3 +1344,27 @@ against an in-memory SQLite ledger; assertions inspect the resulting row. This i
not native Workers termination, live D1 durability, retry or concurrent billing
coverage. Hidden fixtures and dependency links appear only at grading time;
execution isolation and model performance remain unproven.
+
+## Recording resume controls
+
+`app-recording-global-device-resume` renders the actual `RecordingStatus`
+component with real Radix popover and tooltip controls in jsdom. Synthetic API
+and callback ports observe global session resume separately from per-device
+resume. Eight outcomes cover both resume modes, global pause, mixed device
+states, legacy monitors without IDs, missing callbacks and request-failure
+rollback. The parent fails three resume outcomes and preserves five; applying
+only the historical component fix passes all eight.
+
+Run `bun test evals/coding-agent/calibrate-recording-resume.test.js` with the
+frontend dependencies installed. Eleven controls include the current component,
+an equivalent private-name change, unused correct code, reversed mode routing,
+broken pause, lost rollback and missing-source classification. A separate
+current-only check preserves disabled-capture settings recovery; that newer
+prop is not required by the historical task. The grading config uses a plain
+translation port, so translation delivery is outside this evidence.
+
+This case covers frontend decisions and synthetic effect boundaries. It does
+not establish Home callback wiring, native recording resumption, operating-system
+capture, agent isolation or model improvement. The hidden fixture and dependency
+link are installed only after the trajectory ends. No evaluated agent runs in
+calibration or baseline/reference verification.
diff --git a/evals/coding-agent/calibrate-recording-resume.test.js b/evals/coding-agent/calibrate-recording-resume.test.js
new file mode 100644
index 000000000..0b51c1043
--- /dev/null
+++ b/evals/coding-agent/calibrate-recording-resume.test.js
@@ -0,0 +1,70 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-recording-global-device-resume');
+const component = `${app}/components/recording-status.tsx`;
+const root = mkdtempSync(join(tmpdir(), 'recording-resume-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('8 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent loses three resume outcomes and preserves five outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('3 failed | 5 passed');},60000);
+test('reference passes eight outcomes',()=>passes(grade('reference')),60000);
+test('component-only oracle repairs the parent',()=>passes(grade('oracle-only',item.base_ref,cwd=>writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.oracle_ref}:${component}`],{cwd:repo})))),60000);
+test('current component passes the historical contract',()=>passes(grade('current','HEAD')),60000);
+test('equivalent private names and removed test ids pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{
+ const p=join(cwd,component);writeFileSync(p,readFileSync(p,'utf8').replaceAll('allPaused','everyDevicePaused').replaceAll('data-testid=','data-eval-neutral='));
+})),60000);
+test('unused correct component does not hide broken active component',()=>fails(grade('unused',item.oracle_ref,cwd=>{
+ writeFileSync(join(cwd,`${app}/components/unused-recording-status.tsx`),readFileSync(join(cwd,component)));
+ writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.base_ref}:${component}`],{cwd:repo}));
+})),60000);
+test('reversed global mode routing fails',()=>fails(grade('reversed-mode',item.oracle_ref,cwd=>replace(cwd,component,'if (!isGloballyPaused)','if (isGloballyPaused)'))),60000);
+test('broken global pause fails',()=>fails(grade('broken-pause',item.oracle_ref,cwd=>replace(cwd,component,'await onPauseRecording();','await onResumeRecording?.();'))),60000);
+test('lost device rollback fails',()=>fails(grade('broken-rollback',item.oracle_ref,cwd=>replace(cwd,component,'active: device.active','active: !device.active'))),60000);
+test('missing component is setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,component)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);
+// Current-only preservation checks are deliberately outside the historical grader.
+test('current disabled capture hides controls and opens settings',()=>{
+ const r=grade('current-disabled','HEAD',cwd=>{
+ const f=join(cwd,app,'.eval-hidden/recording-resume.test.tsx');
+ writeFileSync(f,readFileSync(f,'utf8')+`
+test('disabled capture preserves settings recovery without recording actions',()=>{
+ const open=vi.fn();
+ const p=mount(paused,true,{allCaptureDisabled:true,onOpenRecordingSettings:open});
+ expect(screen.queryByRole('button',{name:/all recording/i})).toBeNull();
+ expect(screen.queryByRole('button',{name:/^resume$/i})).toBeNull();
+ fireEvent.click(screen.getByRole('button',{name:/open settings/i}));
+ expect(open).toHaveBeenCalledTimes(1);
+ expect(p.pause).not.toHaveBeenCalled();expect(p.resume).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();
+});
+`);
+ });expect(r.status).toBe(0);expect(r.stdout).toContain('9 passed');
+},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 69a1487c4..a4be9229a 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -4549,6 +4549,55 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-model-catalog.config.mjs"
}
+ },
+ {
+ "id": "app-recording-global-device-resume",
+ "name": "Resume globally stopped recording and individually paused devices",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "cdc3afb06cf62a6570cd196d1e9d0df87742498e"
+ },
+ "base_ref": "66936aa58874d10020132c80b679d42f47a63a3e",
+ "oracle_ref": "cdc3afb06cf62a6570cd196d1e9d0df87742498e",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/recording-status.tsx"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/recording-status.tsx"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "recording",
+ "user-pause"
+ ],
+ "prompt": "The recording status popover has no working resume-all action when every device is paused. Preserve the RecordingStatus public interface and provide an accessible resume-all control. When the entire session is globally paused, resume through onResumeRecording once without issuing individual device requests. When the session is running but every device is individually paused, resume those devices through their existing POST /vision/device/start (monitor_id) and /audio/device/start (device_name) routes without invoking global callbacks. Keep pause-all routed through onPauseRecording for active or mixed device states. A legacy monitor without an id must issue no device request, while eligible audio devices still resume. Hide the global control when neither callback exists; when globally paused without a resume callback, the unavailable resume action must be disabled or absent. Preserve individual device request failure rollback. Keep the observable action names pause all recording, resume all recording, and resume accessible to assistive technology. No particular private function names, test attributes, or implementation structure are required. This task covers the historical frontend control and its synthetic callback/API boundaries, not native recording, the Home wiring, current rollout activation, or operating-system capture.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/recording-resume.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/.eval-hidden/recording-resume.test.tsx"
+ },
+ {
+ "local_path": "graders/recording-resume-config.mjs.txt",
+ "destination_path": "apps/screenpipe-app-tauri/.eval-hidden/recording-resume.config.mjs"
+ },
+ {
+ "local_path": "graders/recording-resume-translation.ts.txt",
+ "destination_path": "apps/screenpipe-app-tauri/.eval-hidden/recording-resume-translation.ts"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config .eval-hidden/recording-resume.config.mjs"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/recording-resume-config.mjs.txt b/evals/coding-agent/graders/recording-resume-config.mjs.txt
new file mode 100644
index 000000000..91da25e3e
--- /dev/null
+++ b/evals/coding-agent/graders/recording-resume-config.mjs.txt
@@ -0,0 +1,6 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {defineConfig} from 'vitest/config';
+import {fileURLToPath} from 'node:url';
+const root=fileURLToPath(new URL('../',import.meta.url));
+export default defineConfig({root,resolve:{alias:{'@':root,'gt-react':fileURLToPath(new URL('./recording-resume-translation.ts',import.meta.url))}},test:{environment:'jsdom',include:['.eval-hidden/recording-resume.test.tsx'],maxWorkers:1,minWorkers:1}});
diff --git a/evals/coding-agent/graders/recording-resume-translation.ts.txt b/evals/coding-agent/graders/recording-resume-translation.ts.txt
new file mode 100644
index 000000000..892da6222
--- /dev/null
+++ b/evals/coding-agent/graders/recording-resume-translation.ts.txt
@@ -0,0 +1,3 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+export const useGT = () => (s: string) => s;
diff --git a/evals/coding-agent/graders/recording-resume.fixture.tsx.txt b/evals/coding-agent/graders/recording-resume.fixture.tsx.txt
new file mode 100644
index 000000000..675bbbf58
--- /dev/null
+++ b/evals/coding-agent/graders/recording-resume.fixture.tsx.txt
@@ -0,0 +1,88 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import React from 'react';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
+import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { RecordingStatus } from '../components/recording-status';
+import { TooltipProvider } from '../components/ui/tooltip';
+const ports = vi.hoisted(() => ({ fetch: vi.fn() }));
+vi.mock('@/lib/api', () => ({ localFetch: ports.fetch }));
+vi.mock('@/lib/utils', () => ({ cn: (...xs: unknown[]) => xs.filter(Boolean).join(' ') }));
+vi.mock('posthog-js', () => ({ default: { capture: vi.fn() } }));
+afterEach(cleanup);
+beforeEach(() => { ports.fetch.mockReset(); ports.fetch.mockResolvedValue({ok: true}); });
+const paused = [
+ {name:'Display', fullName:'Display synthetic', kind:'monitor', id:41, active:false},
+ {name:'Mic', fullName:'Synthetic microphone (input)', kind:'input', active:false},
+ {name:'Speaker', fullName:'Synthetic speakers (output)', kind:'output', active:false},
+] as any;
+function mount(devices: any[], global: boolean, extra: any = {}) {
+ const pause=vi.fn(), resume=vi.fn();
+ function Host() {
+ const [state,setState]=React.useState(devices);
+ return <TooltipProvider><RecordingStatus devices={state} onDevicesChange={setState}
+ meetingActive={false} meetingLoading={false} onToggleMeeting={()=>{}}
+ onPauseRecording={pause} onResumeRecording={resume} isGloballyPaused={global} {...extra}/>
+ <output data-testid="device-state">{JSON.stringify(state)}</output></TooltipProvider>;
+ }
+ render(<Host/>);
+ fireEvent.click(screen.getByRole('button', {name:/recording|paused/i}));
+ return {pause,resume};
+}
+test('global stopped session resumes through global callback only', async()=>{
+ const p=mount(paused,true);
+ fireEvent.click(screen.getByRole('button',{name:/resume all recording/i}));
+ await waitFor(()=>expect(p.resume).toHaveBeenCalledTimes(1));
+ expect(p.pause).not.toHaveBeenCalled(); expect(ports.fetch).not.toHaveBeenCalled();
+});
+test('live session with every device paused resumes individual display and audio ports',async()=>{
+ const p=mount(paused,false);
+ fireEvent.click(screen.getByRole('button',{name:/resume all recording/i}));
+ await waitFor(()=>expect(ports.fetch).toHaveBeenCalledTimes(3));
+ expect(ports.fetch.mock.calls.map(([url,init])=>[url,init.method,JSON.parse(init.body)]).sort()).toEqual([
+ ['/vision/device/start','POST',{monitor_id:41}],
+ ['/audio/device/start','POST',{device_name:'Synthetic microphone (input)'}],
+ ['/audio/device/start','POST',{device_name:'Synthetic speakers (output)'}],
+ ].sort());
+ expect(p.resume).not.toHaveBeenCalled();expect(p.pause).not.toHaveBeenCalled();
+});
+test('active capture retains global pause action',async()=>{
+ const p=mount(paused.map((d:any)=>({...d,active:true})),false);
+ fireEvent.click(screen.getByRole('button',{name:/pause all recording/i}));
+ await waitFor(()=>expect(p.pause).toHaveBeenCalledTimes(1));
+ expect(p.resume).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();
+});
+test('failed individual request restores the device pause state',async()=>{
+ ports.fetch.mockResolvedValue({ok:false,status:503});
+ mount([paused[1]],false);
+ fireEvent.click(screen.getByRole('button',{name:/^resume$/i}));
+ await waitFor(()=>expect(ports.fetch).toHaveBeenCalledTimes(1));
+ await waitFor(()=>expect(JSON.parse(screen.getByTestId('device-state').textContent!)[0].active).toBe(false));
+});
+
+test('legacy monitor without an id makes no request while audio resumes',async()=>{
+ const p=mount([{...paused[0],id:undefined},paused[1]],false);
+ fireEvent.click(screen.getByRole('button',{name:/resume all recording/i}));
+ await waitFor(()=>expect(ports.fetch).toHaveBeenCalledTimes(1));
+ expect(ports.fetch.mock.calls[0][0]).toBe('/audio/device/start');
+ expect(JSON.parse(ports.fetch.mock.calls[0][1].body)).toEqual({device_name:paused[1].fullName});
+ expect(p.resume).not.toHaveBeenCalled();expect(p.pause).not.toHaveBeenCalled();
+});
+test('no global callbacks keeps the global control absent',()=>{
+ mount(paused,false,{onPauseRecording:undefined,onResumeRecording:undefined});
+ expect(screen.queryByRole('button',{name:/all recording/i})).toBeNull();
+ expect(ports.fetch).not.toHaveBeenCalled();
+});
+test('stopped session without a resume callback cannot trigger a global action',()=>{
+ const p=mount(paused,true,{onResumeRecording:undefined});
+ const button=screen.queryByRole('button',{name:/resume all recording/i});
+ // An unavailable action may be disabled or omitted.
+ if(button) {expect((button as HTMLButtonElement).disabled).toBe(true);fireEvent.click(button);}
+ expect(p.pause).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();
+});
+test('mixed active and paused devices keep the global pause action',async()=>{
+ const p=mount([{...paused[0],active:true},paused[1]],false);
+ fireEvent.click(screen.getByRole('button',{name:/pause all recording/i}));
+ await waitFor(()=>expect(p.pause).toHaveBeenCalledTimes(1));
+ expect(p.resume).not.toHaveBeenCalled();expect(ports.fetch).not.toHaveBeenCalled();
+});