YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover replay recovery from unavailable frame images
Louis Beaumont6c0f26bfaf53
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 738fc0a6e..5b1347ba8 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.This does not establish native recording continuity, real authentication,account-switch race safety, browser integration, isolation or model performance.-The current app corpus contains 134 git-mined regressions. See+The current app corpus contains 135 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1701,3 +1701,14 @@ mutations and missing-source controls. Hidden fixtures appear only at grading;no build-cache links are declared. This does not prove recording continuity,HTTP or hybrid-storage integration, concurrent writes, host isolation or modelperformance.++`app-replay-unavailable-images` renders the actual replay strip with synthetic+image load/error events and frame/transcript, native and navigation ports. Six+outcomes cover active and thumbnail failures, all-unavailable fallback, accurate+frame counts, range reset, successful loads and retained caption/navigation. The+parent fails four recovery outcomes and preserves two; the source-only reference+and current source pass six. Run+`bun test evals/coding-agent/calibrate-replay-images.test.js` for calibration.+Fixtures and dependency links appear only at grading. This does not execute+native image decoding, audio playback, real network delivery, browser layout,+enforced agent isolation or model trials.diff --git a/evals/coding-agent/calibrate-replay-images.test.js b/evals/coding-agent/calibrate-replay-images.test.jsnew file mode 100644index 000000000..ae2f94f4f--- /dev/null+++ b/evals/coding-agent/calibrate-replay-images.test.js@@ -0,0 +1,50 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-replay-unavailable-images');+const hook = `${app}/components/meeting-notes/replay-strip.tsx`;+const root = mkdtempSync(join(tmpdir(), 'replay-images-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('6 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toMatch(/AssertionError|TestingLibraryElementError: Unable to find an element/); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails four image recovery outcomes and preserves two',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 2 passed');},60000);+test('reference passes six outcomes',()=>passes(grade('reference')),60000);+test('current source passes six outcomes',()=>passes(grade('current','HEAD')),60000);+test('equivalent private state names are accepted',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('unavailableFrameIds','failedImages').replaceAll('setUnavailableFrameIds','setFailedImages').replaceAll('markFrameUnavailable','evictImage'));})),60000);+test('unused repair cannot hide active broken code',()=>fails(grade('bypass',item.oracle_ref,cwd=>{writeFileSync(join(cwd,'unused-correct.tsx'),readFileSync(join(cwd,hook)));writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}));})),60000);+test('active image handler cannot be omitted',()=>fails(grade('active-handler',item.oracle_ref,cwd=>replace(cwd,hook,'onError={() => markFrameUnavailable(activeFrame.frameId)}',''))),60000);+test('thumbnail handler cannot be omitted',()=>fails(grade('thumbnail-handler',item.oracle_ref,cwd=>replace(cwd,hook,'onError={() => markFrameUnavailable(f.frameId)}',''))),60000);+test('unavailable state must reset for a new range',()=>fails(grade('no-reset',item.oracle_ref,cwd=>replace(cwd,hook,' setUnavailableFrameIds(new Set());',''))),60000);+test('all images cannot be suppressed',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,hook,'() => frames.filter((f) => !unavailableFrameIds.has(f.frameId)),','() => [],'))),60000);+test('frame count must reflect available images',()=>fails(grade('stale-count',item.oracle_ref,cwd=>replace(cwd,hook,'${renderableFrames.length} frames','${frames.length} frames'))),60000);+test('timeline handoff remains available without frames',()=>fails(grade('no-navigation',item.oracle_ref,cwd=>replace(cwd,hook,'router.push("/home?section=timeline");',''))),60000);+test('missing component remains a setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout).toContain('no tests');expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 81c6947d5..a62387b9e 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5550,6 +5550,56 @@],"command": "cargo test --offline --locked -p screenpipe-db --test eval_audio_segments -- --test-threads=1"}+ },+ {+ "id": "app-replay-unavailable-images",+ "name": "Recover meeting replay from unavailable frame images",+ "source": {+ "kind": "git_regression",+ "fix_commit": "5c1632dda6f5129f22f1b43b2ee250d12ec55f2f"+ },+ "base_ref": "335651f8047977e6f4c4819070abd0f991c6280e",+ "oracle_ref": "5c1632dda6f5129f22f1b43b2ee250d12ec55f2f",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsx"+ ],+ "tags": [+ "regression",+ "meetings",+ "images",+ "recovery",+ "escaped-failure"+ ],+ "gate": "advisory",+ "prompt": "Meeting replay keeps broken images visible when a sampled frame cannot render. Repair the actual ReplayStrip so a load error in either the active image or a thumbnail removes every displayed occurrence of that frame and excludes it from subsequent active-image selection, while keeping healthy images usable. Keep the active image accessible through nonempty alternative text and thumbnails decorative. When all samples are unavailable, show a visible no-screen-images-available message and no broken image elements. Keep the displayed number of available frames truthful. A new time range must retry frames that failed in the previous range. Preserve successful image loads, transcript captions, and the Open in timeline action, including the selected timestamp and /home?section=timeline destination, even without screen images. Keep existing public component interfaces. The checks render the actual React component with synthetic frame/transcript responses, native/navigation ports and image load/error events. Native image decoding, network delivery, audio playback and full browser layout are outside this task. No particular state structure or private helper name is required.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/replay-images.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/replay-probe.test.tsx"+ },+ {+ "local_path": "graders/replay-images.config.ts",+ "destination_path": "apps/screenpipe-app-tauri/replay-probe.config.ts"+ },+ {+ "local_path": "graders/replay-images-i18n.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/replay-i18n.ts"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config replay-probe.config.ts"+ },+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsx"+ ]}]}diff --git a/evals/coding-agent/graders/replay-images-i18n.fixture.ts.txt b/evals/coding-agent/graders/replay-images-i18n.fixture.ts.txtnew file mode 100644index 000000000..89695f872--- /dev/null+++ b/evals/coding-agent/graders/replay-images-i18n.fixture.ts.txt@@ -0,0 +1,3 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+export const useGT=()=> (s:string,v:Record<string,unknown>={})=>s.replace(/\{(\w+)\}/g,(_,k)=>String(v[k]??''));diff --git a/evals/coding-agent/graders/replay-images.config.ts b/evals/coding-agent/graders/replay-images.config.tsnew file mode 100644index 000000000..4881beee5--- /dev/null+++ b/evals/coding-agent/graders/replay-images.config.ts@@ -0,0 +1,5 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {defineConfig} from 'vitest/config';+import path from 'node:path';+export default defineConfig({esbuild:{jsx:'automatic'},test:{environment:'jsdom',include:['replay-probe.test.tsx'],maxWorkers:1,minWorkers:1},resolve:{alias:{'@':path.resolve(__dirname),'gt-react':path.resolve(__dirname,'replay-i18n.ts')}}});diff --git a/evals/coding-agent/graders/replay-images.fixture.tsx.txt b/evals/coding-agent/graders/replay-images.fixture.tsx.txtnew file mode 100644index 000000000..f285a84fd--- /dev/null+++ b/evals/coding-agent/graders/replay-images.fixture.tsx.txt@@ -0,0 +1,27 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import React from 'react';+import {cleanup,fireEvent,render,screen,waitFor} from '@testing-library/react';+import {afterEach,beforeEach,expect,test,vi} from 'vitest';+const p=vi.hoisted(()=>({frames:vi.fn(),audio:vi.fn(),push:vi.fn(),navigate:vi.fn(),emit:vi.fn()}));+vi.mock('@tauri-apps/api/event',()=>({emit:p.emit}));+vi.mock('next/navigation',()=>({useRouter:()=>({push:p.push})}));+vi.mock('@/lib/api',()=>({appendAuthToken:(s:string)=>s,getApiBaseUrl:()=> 'http://localhost:3030'}));+vi.mock('@/lib/hooks/use-timeline-store',()=>({useTimelineStore:(s:any)=>s({setPendingNavigation:p.navigate})}));+vi.mock('@/components/speaker-assign-popover',()=>({SpeakerAssignPopover:({children}:any)=>children}));+vi.mock('@/lib/utils/meeting-context',()=>({fetchFrameSamples:p.frames,fetchMeetingAudio:p.audio}));+import {ReplayStrip} from './components/meeting-notes/replay-strip';+const start='2026-06-12T10:00:00.000Z',middle='2026-06-12T10:02:00.000Z',end='2026-06-12T10:10:00.000Z';+const segments=[{transcription:'Synthetic preserved caption',speaker:'speaker',device:'synthetic',timestamp:middle}];+const frames=[{frameId:11,timestamp:middle},{frameId:22,timestamp:'2026-06-12T10:03:00.000Z'}];+function view(a=start,b=end){return <ReplayStrip meetingId={1} segments={segments} timeRange={{start:a,end:b}}/>}+function images(c:HTMLElement,id?:number){return [...c.querySelectorAll('img')].filter(x=>id===undefined||x.src.endsWith('/frames/'+id))}+async function ready(c:HTMLElement){await waitFor(()=>expect(images(c,11).length).toBeGreaterThan(0));await screen.findByText('Synthetic preserved caption')}+beforeEach(()=>{vi.clearAllMocks();p.frames.mockResolvedValue(frames);p.audio.mockResolvedValue([{audioChunkId:7,audioFilePath:'',speakerId:null,speakerName:'speaker',deviceType:'input',isInput:true,transcription:'Synthetic preserved caption',timestamp:middle}]);});+afterEach(()=>cleanup());+test('active image failure evicts every occurrence and preserves healthy frames',async()=>{const {container:c}=render(view());await ready(c);const a=images(c,11).find(x=>x.alt)!;expect(a).toBeTruthy();fireEvent.error(a);await waitFor(()=>expect(images(c,11)).toHaveLength(0));expect(images(c,22).length).toBeGreaterThan(0);expect(images(c).find(x=>x.alt)?.src).toMatch(/\/frames\/22$/);expect(screen.getByText(/1 frames/)).toBeTruthy();});+test('thumbnail failure also prevents the unavailable frame becoming active',async()=>{const {container:c}=render(view());await ready(c);const t=images(c,22).find(x=>!x.alt)!;fireEvent.error(t);await waitFor(()=>expect(images(c,22)).toHaveLength(0));expect(images(c,11).length).toBeGreaterThan(0);});+test('all unavailable images leave truthful empty state and preserve caption and navigation',async()=>{p.frames.mockResolvedValue([frames[0]]);const {container:c}=render(view());await ready(c);fireEvent.error(images(c)[0]);await waitFor(()=>expect(images(c)).toHaveLength(0));expect(screen.getByText(/no screen images available/i)).toBeTruthy();expect(screen.getByText(/0 frames/)).toBeTruthy();expect(screen.getByText('Synthetic preserved caption')).toBeTruthy();fireEvent.click(screen.getByRole('button',{name:/open in timeline/i}));expect(p.push).toHaveBeenCalledWith('/home?section=timeline');expect(p.navigate).toHaveBeenCalledWith({timestamp:middle});});+test('a new time range can load an image that was unavailable before',async()=>{p.frames.mockResolvedValue([frames[0]]);const {container:c,rerender}=render(view());await ready(c);fireEvent.error(images(c)[0]);await waitFor(()=>expect(images(c)).toHaveLength(0));p.frames.mockResolvedValue([{frameId:11,timestamp:'2026-06-12T11:02:00.000Z'}]);rerender(view('2026-06-12T11:00:00.000Z','2026-06-12T11:10:00.000Z'));await waitFor(()=>expect(images(c,11).length).toBeGreaterThan(0));expect(screen.queryByText(/no screen images available/i)).toBeNull();});+test('successful image loads preserve both frames and caption',async()=>{const {container:c}=render(view());await ready(c);for(const i of images(c))fireEvent.load(i);expect(images(c,11).length).toBeGreaterThan(0);expect(images(c,22).length).toBeGreaterThan(0);expect(screen.getByText(/2 frames/)).toBeTruthy();expect(screen.getByText('Synthetic preserved caption')).toBeTruthy();});+test('no sampled images still preserves transcript and timeline entry',async()=>{p.frames.mockResolvedValue([]);const {container:c}=render(view());await screen.findByText('Synthetic preserved caption');expect(images(c)).toHaveLength(0);expect(screen.getByRole('button',{name:/open in timeline/i})).toBeTruthy();expect(screen.getByText(/0 frames/)).toBeTruthy();});