Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover replay recovery from unavailable frame images

Louis Beaumont6c0f26bfaf53
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 738fc0a6e..5b1347ba8 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.
This does not establish native recording continuity, real authentication,
account-switch race safety, browser integration, isolation or model performance.
-The current app corpus contains 134 git-mined regressions. See
+The current app corpus contains 135 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -1701,3 +1701,14 @@ mutations and missing-source controls. Hidden fixtures appear only at grading;
no build-cache links are declared. This does not prove recording continuity,
HTTP or hybrid-storage integration, concurrent writes, host isolation or model
performance.
+
+`app-replay-unavailable-images` renders the actual replay strip with synthetic
+image load/error events and frame/transcript, native and navigation ports. Six
+outcomes cover active and thumbnail failures, all-unavailable fallback, accurate
+frame counts, range reset, successful loads and retained caption/navigation. The
+parent fails four recovery outcomes and preserves two; the source-only reference
+and current source pass six. Run
+`bun test evals/coding-agent/calibrate-replay-images.test.js` for calibration.
+Fixtures and dependency links appear only at grading. This does not execute
+native image decoding, audio playback, real network delivery, browser layout,
+enforced agent isolation or model trials.
diff --git a/evals/coding-agent/calibrate-replay-images.test.js b/evals/coding-agent/calibrate-replay-images.test.js
new file mode 100644
index 000000000..ae2f94f4f
--- /dev/null
+++ b/evals/coding-agent/calibrate-replay-images.test.js
@@ -0,0 +1,50 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-replay-unavailable-images');
+const hook = `${app}/components/meeting-notes/replay-strip.tsx`;
+const root = mkdtempSync(join(tmpdir(), 'replay-images-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('6 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toMatch(/AssertionError|TestingLibraryElementError: Unable to find an element/); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent fails four image recovery outcomes and preserves two',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 2 passed');},60000);
+test('reference passes six outcomes',()=>passes(grade('reference')),60000);
+test('current source passes six outcomes',()=>passes(grade('current','HEAD')),60000);
+test('equivalent private state names are accepted',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('unavailableFrameIds','failedImages').replaceAll('setUnavailableFrameIds','setFailedImages').replaceAll('markFrameUnavailable','evictImage'));})),60000);
+test('unused repair cannot hide active broken code',()=>fails(grade('bypass',item.oracle_ref,cwd=>{writeFileSync(join(cwd,'unused-correct.tsx'),readFileSync(join(cwd,hook)));writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}));})),60000);
+test('active image handler cannot be omitted',()=>fails(grade('active-handler',item.oracle_ref,cwd=>replace(cwd,hook,'onError={() => markFrameUnavailable(activeFrame.frameId)}',''))),60000);
+test('thumbnail handler cannot be omitted',()=>fails(grade('thumbnail-handler',item.oracle_ref,cwd=>replace(cwd,hook,'onError={() => markFrameUnavailable(f.frameId)}',''))),60000);
+test('unavailable state must reset for a new range',()=>fails(grade('no-reset',item.oracle_ref,cwd=>replace(cwd,hook,' setUnavailableFrameIds(new Set());',''))),60000);
+test('all images cannot be suppressed',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,hook,'() => frames.filter((f) => !unavailableFrameIds.has(f.frameId)),','() => [],'))),60000);
+test('frame count must reflect available images',()=>fails(grade('stale-count',item.oracle_ref,cwd=>replace(cwd,hook,'${renderableFrames.length} frames','${frames.length} frames'))),60000);
+test('timeline handoff remains available without frames',()=>fails(grade('no-navigation',item.oracle_ref,cwd=>replace(cwd,hook,'router.push("/home?section=timeline");',''))),60000);
+test('missing component remains a setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout).toContain('no tests');expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 81c6947d5..a62387b9e 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -5550,6 +5550,56 @@
],
"command": "cargo test --offline --locked -p screenpipe-db --test eval_audio_segments -- --test-threads=1"
}
+ },
+ {
+ "id": "app-replay-unavailable-images",
+ "name": "Recover meeting replay from unavailable frame images",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "5c1632dda6f5129f22f1b43b2ee250d12ec55f2f"
+ },
+ "base_ref": "335651f8047977e6f4c4819070abd0f991c6280e",
+ "oracle_ref": "5c1632dda6f5129f22f1b43b2ee250d12ec55f2f",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsx"
+ ],
+ "tags": [
+ "regression",
+ "meetings",
+ "images",
+ "recovery",
+ "escaped-failure"
+ ],
+ "gate": "advisory",
+ "prompt": "Meeting replay keeps broken images visible when a sampled frame cannot render. Repair the actual ReplayStrip so a load error in either the active image or a thumbnail removes every displayed occurrence of that frame and excludes it from subsequent active-image selection, while keeping healthy images usable. Keep the active image accessible through nonempty alternative text and thumbnails decorative. When all samples are unavailable, show a visible no-screen-images-available message and no broken image elements. Keep the displayed number of available frames truthful. A new time range must retry frames that failed in the previous range. Preserve successful image loads, transcript captions, and the Open in timeline action, including the selected timestamp and /home?section=timeline destination, even without screen images. Keep existing public component interfaces. The checks render the actual React component with synthetic frame/transcript responses, native/navigation ports and image load/error events. Native image decoding, network delivery, audio playback and full browser layout are outside this task. No particular state structure or private helper name is required.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/replay-images.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/replay-probe.test.tsx"
+ },
+ {
+ "local_path": "graders/replay-images.config.ts",
+ "destination_path": "apps/screenpipe-app-tauri/replay-probe.config.ts"
+ },
+ {
+ "local_path": "graders/replay-images-i18n.fixture.ts.txt",
+ "destination_path": "apps/screenpipe-app-tauri/replay-i18n.ts"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config replay-probe.config.ts"
+ },
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsx"
+ ]
}
]
}
diff --git a/evals/coding-agent/graders/replay-images-i18n.fixture.ts.txt b/evals/coding-agent/graders/replay-images-i18n.fixture.ts.txt
new file mode 100644
index 000000000..89695f872
--- /dev/null
+++ b/evals/coding-agent/graders/replay-images-i18n.fixture.ts.txt
@@ -0,0 +1,3 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+export const useGT=()=> (s:string,v:Record<string,unknown>={})=>s.replace(/\{(\w+)\}/g,(_,k)=>String(v[k]??''));
diff --git a/evals/coding-agent/graders/replay-images.config.ts b/evals/coding-agent/graders/replay-images.config.ts
new file mode 100644
index 000000000..4881beee5
--- /dev/null
+++ b/evals/coding-agent/graders/replay-images.config.ts
@@ -0,0 +1,5 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {defineConfig} from 'vitest/config';
+import path from 'node:path';
+export default defineConfig({esbuild:{jsx:'automatic'},test:{environment:'jsdom',include:['replay-probe.test.tsx'],maxWorkers:1,minWorkers:1},resolve:{alias:{'@':path.resolve(__dirname),'gt-react':path.resolve(__dirname,'replay-i18n.ts')}}});
diff --git a/evals/coding-agent/graders/replay-images.fixture.tsx.txt b/evals/coding-agent/graders/replay-images.fixture.tsx.txt
new file mode 100644
index 000000000..f285a84fd
--- /dev/null
+++ b/evals/coding-agent/graders/replay-images.fixture.tsx.txt
@@ -0,0 +1,27 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import React from 'react';
+import {cleanup,fireEvent,render,screen,waitFor} from '@testing-library/react';
+import {afterEach,beforeEach,expect,test,vi} from 'vitest';
+const p=vi.hoisted(()=>({frames:vi.fn(),audio:vi.fn(),push:vi.fn(),navigate:vi.fn(),emit:vi.fn()}));
+vi.mock('@tauri-apps/api/event',()=>({emit:p.emit}));
+vi.mock('next/navigation',()=>({useRouter:()=>({push:p.push})}));
+vi.mock('@/lib/api',()=>({appendAuthToken:(s:string)=>s,getApiBaseUrl:()=> 'http://localhost:3030'}));
+vi.mock('@/lib/hooks/use-timeline-store',()=>({useTimelineStore:(s:any)=>s({setPendingNavigation:p.navigate})}));
+vi.mock('@/components/speaker-assign-popover',()=>({SpeakerAssignPopover:({children}:any)=>children}));
+vi.mock('@/lib/utils/meeting-context',()=>({fetchFrameSamples:p.frames,fetchMeetingAudio:p.audio}));
+import {ReplayStrip} from './components/meeting-notes/replay-strip';
+const start='2026-06-12T10:00:00.000Z',middle='2026-06-12T10:02:00.000Z',end='2026-06-12T10:10:00.000Z';
+const segments=[{transcription:'Synthetic preserved caption',speaker:'speaker',device:'synthetic',timestamp:middle}];
+const frames=[{frameId:11,timestamp:middle},{frameId:22,timestamp:'2026-06-12T10:03:00.000Z'}];
+function view(a=start,b=end){return <ReplayStrip meetingId={1} segments={segments} timeRange={{start:a,end:b}}/>}
+function images(c:HTMLElement,id?:number){return [...c.querySelectorAll('img')].filter(x=>id===undefined||x.src.endsWith('/frames/'+id))}
+async function ready(c:HTMLElement){await waitFor(()=>expect(images(c,11).length).toBeGreaterThan(0));await screen.findByText('Synthetic preserved caption')}
+beforeEach(()=>{vi.clearAllMocks();p.frames.mockResolvedValue(frames);p.audio.mockResolvedValue([{audioChunkId:7,audioFilePath:'',speakerId:null,speakerName:'speaker',deviceType:'input',isInput:true,transcription:'Synthetic preserved caption',timestamp:middle}]);});
+afterEach(()=>cleanup());
+test('active image failure evicts every occurrence and preserves healthy frames',async()=>{const {container:c}=render(view());await ready(c);const a=images(c,11).find(x=>x.alt)!;expect(a).toBeTruthy();fireEvent.error(a);await waitFor(()=>expect(images(c,11)).toHaveLength(0));expect(images(c,22).length).toBeGreaterThan(0);expect(images(c).find(x=>x.alt)?.src).toMatch(/\/frames\/22$/);expect(screen.getByText(/1 frames/)).toBeTruthy();});
+test('thumbnail failure also prevents the unavailable frame becoming active',async()=>{const {container:c}=render(view());await ready(c);const t=images(c,22).find(x=>!x.alt)!;fireEvent.error(t);await waitFor(()=>expect(images(c,22)).toHaveLength(0));expect(images(c,11).length).toBeGreaterThan(0);});
+test('all unavailable images leave truthful empty state and preserve caption and navigation',async()=>{p.frames.mockResolvedValue([frames[0]]);const {container:c}=render(view());await ready(c);fireEvent.error(images(c)[0]);await waitFor(()=>expect(images(c)).toHaveLength(0));expect(screen.getByText(/no screen images available/i)).toBeTruthy();expect(screen.getByText(/0 frames/)).toBeTruthy();expect(screen.getByText('Synthetic preserved caption')).toBeTruthy();fireEvent.click(screen.getByRole('button',{name:/open in timeline/i}));expect(p.push).toHaveBeenCalledWith('/home?section=timeline');expect(p.navigate).toHaveBeenCalledWith({timestamp:middle});});
+test('a new time range can load an image that was unavailable before',async()=>{p.frames.mockResolvedValue([frames[0]]);const {container:c,rerender}=render(view());await ready(c);fireEvent.error(images(c)[0]);await waitFor(()=>expect(images(c)).toHaveLength(0));p.frames.mockResolvedValue([{frameId:11,timestamp:'2026-06-12T11:02:00.000Z'}]);rerender(view('2026-06-12T11:00:00.000Z','2026-06-12T11:10:00.000Z'));await waitFor(()=>expect(images(c,11).length).toBeGreaterThan(0));expect(screen.queryByText(/no screen images available/i)).toBeNull();});
+test('successful image loads preserve both frames and caption',async()=>{const {container:c}=render(view());await ready(c);for(const i of images(c))fireEvent.load(i);expect(images(c,11).length).toBeGreaterThan(0);expect(images(c,22).length).toBeGreaterThan(0);expect(screen.getByText(/2 frames/)).toBeTruthy();expect(screen.getByText('Synthetic preserved caption')).toBeTruthy();});
+test('no sampled images still preserves transcript and timeline entry',async()=>{p.frames.mockResolvedValue([]);const {container:c}=render(view());await screen.findByText('Synthetic preserved caption');expect(images(c)).toHaveLength(0);expect(screen.getByRole('button',{name:/open in timeline/i})).toBeTruthy();expect(screen.getByText(/0 frames/)).toBeTruthy();});