YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover timeline keyboard filter freshness
Louis Beaumont8ebdc63cbb69
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 5a4cdd75d..744fdf654 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.This does not establish native recording continuity, real authentication,account-switch race safety, browser integration, isolation or model performance.-The current app corpus contains 136 git-mined regressions. See+The current app corpus contains 137 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1724,3 +1724,15 @@ nine. Run `bun test evals/coding-agent/calibrate-recency.test.js` for calibratioHidden fixtures and dependency links appear only when grading starts. This doesnot execute native persistence, foreground/background save callbacks, fullsidebar rendering, enforced agent isolation or model trials.++## Timeline keyboard filter freshness++`app-timeline-keyboard-filter-freshness` mounts the actual keyboard hook with+synthetic navigation/native/translation ports. Three parent failures cover stale+left/right filters and a replaced state receiver; nine nearby outcomes pass.+The source-only historical fix and current hook pass all twelve. Text input,+search review, playback seeking, no-successor behavior and unmount cleanup remain+required. Run `bun test evals/coding-agent/calibrate-keyboard.test.js`. Hidden+fixtures and runtime links appear only at grading. This does not verify the+separate live-capture prepend index repair, full timeline rendering, native+event delivery, recording persistence, enforced isolation or model performance.diff --git a/evals/coding-agent/calibrate-keyboard.test.js b/evals/coding-agent/calibrate-keyboard.test.jsnew file mode 100644index 000000000..44f8c7ad8--- /dev/null+++ b/evals/coding-agent/calibrate-keyboard.test.js@@ -0,0 +1,49 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-timeline-keyboard-filter-freshness');+const hook = `${app}/components/rewind/hooks/use-timeline-keyboard.ts`;+const root = mkdtempSync(join(tmpdir(), 'timeline-keyboard-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('12 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toMatch(/AssertionError|TestingLibraryElementError: Unable to find an element/); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails three freshness outcomes and preserves nine',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('3 failed | 9 passed');},60000);+test('reference passes twelve outcomes',()=>passes(grade('reference')),60000);+test('current source passes twelve outcomes',()=>passes(grade('current','HEAD')),60000);+test('equivalent callback names are accepted',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const p=join(cwd,hook);let s=readFileSync(p,'utf8');s=s.replace('findNextDevice,','findNextDevice: chooseFilteredFrame,');s=s.replaceAll('findNextDevice(prev','chooseFilteredFrame(prev').replace('frames, findNextDevice, setCurrentIndex','frames, chooseFilteredFrame, setCurrentIndex');writeFileSync(p,s);})),60000);+test('unused correct hook cannot hide stale active listener',()=>fails(grade('bypass',item.oracle_ref,cwd=>{writeFileSync(join(cwd,'unused-correct.ts'),readFileSync(join(cwd,hook)));writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}));})),60000);+test('blanket arrow refusal is rejected',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,hook,'const handleArrowKeys = (e: KeyboardEvent) => {','const handleArrowKeys = (e: KeyboardEvent) => { return;'))),60000);+test('text editing must retain arrows',()=>fails(grade('input',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement || target.isContentEditable','false'));})),60000);+test('open modal must retain arrows',()=>fails(grade('modal',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('if (showSearchModal) return;','if (false) return;'));})),60000);+test('playback seek must reach selected timestamp',()=>fails(grade('seek',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('if (isPlaying) seekPlayback(new Date(frames[next].timestamp).getTime());','if (isPlaying) void 0;'));})),60000);+test('unmount must remove the active listener',()=>fails(grade('unmount',item.oracle_ref,cwd=>replace(cwd,hook,'return () => window.removeEventListener("keydown", handleArrowKeys);','return () => {};'))),60000);+test('missing hook is setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stdout+r.stderr).not.toContain('AssertionError');},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 594ff7e2d..383cc4103 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5643,6 +5643,56 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config recency-probe.config.ts"}+ },+ {+ "id": "app-timeline-keyboard-filter-freshness",+ "name": "Keep timeline arrow navigation on the current filter and state receiver",+ "source": {+ "kind": "git_regression",+ "fix_commit": "a3f49b7c66bcc80d97d99abed54fca406db4fb91"+ },+ "base_ref": "f5d0c1550f8c37ed0cb034ba93248b2bb4116639",+ "oracle_ref": "a3f49b7c66bcc80d97d99abed54fca406db4fb91",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/rewind/hooks/use-timeline-keyboard.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "timeline",+ "keyboard",+ "preserved-behavior"+ ],+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/rewind/hooks/use-timeline-keyboard.ts"+ ],+ "prompt": "Timeline arrow navigation can keep using the previous filter until new frames arrive. Fix the actual mounted keyboard hook so ArrowLeft and ArrowRight immediately use the current filter/navigation callback after a rerender, even when the frame array stays unchanged. Do not invoke the stale selection callback. If the index state receiver changes, the next key must reach the new receiver exactly once, not the old receiver. Keep the selected frame and index aligned with the current callback result. Preserve ordinary older/newer navigation, no-successor behavior, and keyboard-listener cleanup on unmount. Text inputs, textareas, editable content and an open search modal retain their arrow keys without moving the timeline. In search review, arrows navigate results and pause playback without ordinary frame navigation. While ordinary playback is running, arrow navigation seeks to the selected frame timestamp without pausing it. Public hook interfaces remain compatible; no dependency-list shape or private implementation is required. Checks mount the actual React hook and dispatch DOM keyboard events with synthetic navigation/native/translation ports. This task does not cover the separate live-capture prepend index repair, full timeline rendering, native event delivery, capture persistence or execution isolation.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/keyboard.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/keyboard-probe.test.tsx"+ },+ {+ "local_path": "graders/keyboard.config.ts",+ "destination_path": "apps/screenpipe-app-tauri/keyboard-probe.config.ts"+ },+ {+ "local_path": "graders/keyboard-translation.ts",+ "destination_path": "apps/screenpipe-app-tauri/keyboard-translation.ts"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config keyboard-probe.config.ts"+ }}]}diff --git a/evals/coding-agent/graders/keyboard-translation.ts b/evals/coding-agent/graders/keyboard-translation.tsnew file mode 100644index 000000000..a8123cf60--- /dev/null+++ b/evals/coding-agent/graders/keyboard-translation.ts@@ -0,0 +1,3 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+export const useGT=()=> (text:string)=>text;diff --git a/evals/coding-agent/graders/keyboard.config.ts b/evals/coding-agent/graders/keyboard.config.tsnew file mode 100644index 000000000..0a7c85832--- /dev/null+++ b/evals/coding-agent/graders/keyboard.config.ts@@ -0,0 +1,5 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {defineConfig} from 'vitest/config';+import path from 'node:path';+export default defineConfig({test:{environment:'jsdom',include:['keyboard-probe.test.tsx'],maxWorkers:1,minWorkers:1},resolve:{alias:{'@':path.resolve(__dirname),'gt-react':path.resolve(__dirname,'keyboard-translation.ts')}}});diff --git a/evals/coding-agent/graders/keyboard.fixture.tsx.txt b/evals/coding-agent/graders/keyboard.fixture.tsx.txtnew file mode 100644index 000000000..f76a31565--- /dev/null+++ b/evals/coding-agent/graders/keyboard.fixture.tsx.txt@@ -0,0 +1,31 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {renderHook,fireEvent,cleanup} from '@testing-library/react';+import {afterEach,beforeEach,expect,it,vi} from 'vitest';+vi.mock('@tauri-apps/api/event',()=>({listen:vi.fn(async()=>()=>{})}));+vi.mock('@/lib/utils/tauri',()=>({commands:{closeWindow:vi.fn(),copyFrameToClipboard:vi.fn(async()=>{})}}));+vi.mock('@/components/rewind/timeline/timeline',()=>({getFrameAppName:(f:any)=>f?.app}));+vi.mock('@/components/ui/use-toast',()=>({toast:vi.fn()}));+import {useTimelineKeyboard} from './components/rewind/hooks/use-timeline-keyboard';+beforeEach(()=>vi.useFakeTimers());+afterEach(()=>{cleanup();document.body.replaceChildren();vi.clearAllTimers();vi.useRealTimers();});+function setup(extra:any={}){+ let index=2;+ const frames=Array.from({length:5},(_,i)=>({timestamp:new Date(1700000000000-i*1000).toISOString(),devices:[{frame_id:i+1}],app:'synthetic'}));+ const setCurrentIndex=vi.fn((next:any)=>{index=typeof next==='function'?next(index):next});+ const opts:any={frames,currentIndex:index,currentFrame:frames[index],setCurrentIndex,setCurrentFrame:vi.fn(),showSearchModal:false,setShowSearchModal:vi.fn(),inSearchReviewMode:false,matchingIndices:null,isPlaying:false,pausePlayback:vi.fn(),seekPlayback:vi.fn(),navigateToSearchResultRef:{current:vi.fn()},findNextDevice:vi.fn((n:number,d:number)=>Math.max(0,Math.min(4,n+d))),embedded:true,isMac:false,searchResultIndex:1,searchResults:[{}, {}, {}],dismissSearchHighlight:vi.fn(),clearSearchHighlight:vi.fn(),hasSearchHighlight:false,setIsArrowNav:vi.fn(),arrowNavTimerRef:{current:null},resetFilters:vi.fn(),selectionRange:null,sendSelectionToChat:vi.fn(async()=>{}),selectedDeviceIdRef:{current:'all'},selectedAppNameRef:{current:'all'},...extra};+ const view=renderHook((props:any)=>useTimelineKeyboard(props),{initialProps:opts});+ return {opts,view,index:()=>index,key:(key:string,target:Element|Window=window)=>fireEvent.keyDown(target,{key})};+}+for(const [key,selected] of [['ArrowLeft',4],['ArrowRight',0]] as const)it(key+' immediately uses the updated filter on unchanged frames',()=>{+ const f=setup(),fresh=vi.fn(()=>selected);f.view.rerender({...f.opts,findNextDevice:fresh});f.key(key);expect(f.index()).toBe(selected);expect(f.opts.setCurrentFrame).toHaveBeenLastCalledWith(f.opts.frames[selected]);expect(f.opts.findNextDevice).not.toHaveBeenCalled();+});+it('replaced navigation state receiver gets the next key',()=>{const f=setup(),next=vi.fn();f.view.rerender({...f.opts,setCurrentIndex:next});f.key('ArrowLeft');expect(next).toHaveBeenCalledTimes(1);expect(f.opts.setCurrentIndex).not.toHaveBeenCalled();});+for(const [key,index] of [['ArrowLeft',3],['ArrowRight',1]] as const)it(key+' still moves one frame in ordinary browsing',()=>{const f=setup();f.key(key);expect(f.index()).toBe(index);expect(f.opts.setCurrentFrame).toHaveBeenLastCalledWith(f.opts.frames[index]);});+it('text inputs and textareas retain arrow keys',()=>{const f=setup();for(const tag of ['input','textarea']){const el=document.createElement(tag);document.body.append(el);f.key('ArrowLeft',el)}expect(f.opts.setCurrentIndex).not.toHaveBeenCalled();expect(f.opts.setCurrentFrame).not.toHaveBeenCalled();});+it('editable content retains arrow keys',()=>{const f=setup(),el=document.createElement('div');Object.defineProperty(el,'isContentEditable',{value:true});document.body.append(el);f.key('ArrowRight',el);expect(f.opts.setCurrentIndex).not.toHaveBeenCalled();});+it('search modal retains arrow keys',()=>{const f=setup({showSearchModal:true});f.key('ArrowLeft');expect(f.opts.setCurrentIndex).not.toHaveBeenCalled();});+it('search review navigates results without moving ordinary frames',()=>{const f=setup({inSearchReviewMode:true,isPlaying:true});f.key('ArrowLeft');expect(f.opts.navigateToSearchResultRef.current).toHaveBeenLastCalledWith(2);expect(f.opts.pausePlayback).toHaveBeenCalledTimes(1);expect(f.opts.setCurrentIndex).not.toHaveBeenCalled();});+it('playing navigation seeks the selected timestamp',()=>{const f=setup({isPlaying:true});f.key('ArrowRight');expect(f.opts.seekPlayback).toHaveBeenLastCalledWith(Date.parse(f.opts.frames[1].timestamp));expect(f.opts.pausePlayback).not.toHaveBeenCalled();});+it('no matching successor preserves the current frame index',()=>{const f=setup({findNextDevice:(n:number)=>n});f.key('ArrowRight');expect(f.index()).toBe(2);expect(f.opts.setCurrentFrame).toHaveBeenLastCalledWith(f.opts.frames[2]);});+it('unmount removes keyboard navigation',()=>{const f=setup();f.view.unmount();f.key('ArrowRight');expect(f.opts.setCurrentIndex).not.toHaveBeenCalled();expect(f.opts.setCurrentFrame).not.toHaveBeenCalled();});