YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: preserve watched pipe text across chat switches
Louis Beaumontc7431c5e9cfa
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b42e005ae..f7c04505c 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1974,3 +1974,5 @@ unresponsive startup stop collection as setup errors. Calibrate with### Vertex upstream JSON failuresThe Vertex upstream JSON case executes public JSON and streaming completion methods with synthetic OAuth and completion transport. Empty or malformed successful bodies must produce status 502 while valid content, reasoning fallback, client refusal and streaming remain intact. It does not prove gateway retry behavior, live provider reliability or agent isolation.++`app-pipe-watch-switch-preservation` mounts the real conversation-switch hook and session store. It distinguishes equal-count stale pipe-watch text from normal panel ownership, preserves longer stored transcripts, and checks incoming chat identity. This is separate from saved terminal pipe transcripts and overlapping-load races. Run `bun test evals/coding-agent/calibrate-pipe-watch-switch.test.js` for calibration. Native delivery and disk durability are outside its scope.diff --git a/evals/coding-agent/calibrate-pipe-watch-switch.test.js b/evals/coding-agent/calibrate-pipe-watch-switch.test.jsnew file mode 100644index 000000000..cc4ecb010--- /dev/null+++ b/evals/coding-agent/calibrate-pipe-watch-switch.test.js@@ -0,0 +1,48 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-pipe-watch-switch-preservation');+const component = `${app}/components/hooks/use-chat-conversations.ts`;+const root = mkdtempSync(join(tmpdir(), 'pipe-watch-switch-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);++ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('4 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails stale watched-pipe text and preserves three controls',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('1 failed | 3 passed');expect(r.stdout).toContain('switch from watched pipe');},60000);+test('reference preserves all four outcomes',()=>passes(grade('reference')),60000);+test('hook-only oracle repairs the parent',()=>passes(grade('oracle-only',item.base_ref,cwd=>writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.oracle_ref}:${component}`],{cwd:repo})))),60000);+test('current hook preserves the historical contract',()=>passes(grade('current','HEAD')),60000);+test('equivalent guard passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,component,'if (outgoingKind !== "pipe-watch") {','if (!["pipe-watch"].includes(outgoingKind ?? "")) {'))),60000);+test('unconditional snapshots reject stale pipe text',()=>fails(grade('blind-snapshot',item.oracle_ref,cwd=>replace(cwd,component,'if (outgoingKind !== "pipe-watch") {','if (true) {'))),60000);+test('skipping every snapshot rejects normal chat ownership',()=>fails(grade('skip-all',item.oracle_ref,cwd=>replace(cwd,component,'if (outgoingKind !== "pipe-watch") {','if (false) {'))),60000);+test('no-op navigation rejects incoming chat restoration',()=>fails(grade('noop-load',item.oracle_ref,cwd=>replace(cwd,component,'const loadConversation = async (conv: ChatConversation) => {','const loadConversation = async (conv: ChatConversation) => { return;'))),60000);+test('unused fixed hook cannot hide the broken live hook',()=>fails(grade('unused',item.oracle_ref,cwd=>{writeFileSync(join(cwd,component+'.unused.ts'),readFileSync(join(cwd,component)));writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.base_ref}:${component}`],{cwd:repo}));})),60000);+test('missing hook is a setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,component)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex e07079a21..af9b5ac10 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6568,6 +6568,52 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config .eval-hidden/recording-resume.config.mjs"}+ },+ {+ "id": "app-pipe-watch-switch-preservation",+ "name": "Keep authoritative watched-pipe text when switching conversations",+ "source": {+ "kind": "git_regression",+ "fix_commit": "3a427b22626db983406d917b43abe4888c647dfc"+ },+ "base_ref": "6b4edca9b022ddfdc5c489e6175507f626298f71",+ "oracle_ref": "3a427b22626db983406d917b43abe4888c647dfc",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/hooks/use-chat-conversations.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/hooks/use-chat-conversations.ts",+ "apps/screenpipe-app-tauri/lib/stores/chat-store.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "data-preservation",+ "pipe-history"+ ],+ "prompt": "Switching conversations can replace a watched pipe's newer streamed text with the panel's stale copy when both copies have the same message count. Repair the actual useChatConversations loadConversation path so leaving a pipe-watch session retains its authoritative store messages. Preserve normal chat ownership: an equally sized panel transcript must still replace the normal chat's older store transcript, while a shorter panel snapshot must not discard additional stored messages. Loading a warm incoming chat must restore its stored messages and update both the current store identity and the panel session reference to that incoming chat. Keep the public hook and session-store interfaces compatible. Checks mount the real React hook with its real store and synthetic native, storage and translation ports. No particular guard, helper, or reducer architecture is required. New-chat creation, stream-event delivery, native rendering, disk durability and crash recovery are outside this historical task.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/pipe-watch-switch.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/.eval-hidden/pipe-watch-switch.test.tsx"+ },+ {+ "local_path": "graders/pipe-watch-switch-config.mjs.txt",+ "destination_path": "apps/screenpipe-app-tauri/.eval-hidden/pipe-watch-switch.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config .eval-hidden/pipe-watch-switch.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/pipe-watch-switch-config.mjs.txt b/evals/coding-agent/graders/pipe-watch-switch-config.mjs.txtnew file mode 100644index 000000000..56d49a764--- /dev/null+++ b/evals/coding-agent/graders/pipe-watch-switch-config.mjs.txt@@ -0,0 +1,3 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {defineConfig} from 'vitest/config';import {fileURLToPath} from 'node:url';const root=fileURLToPath(new URL('../',import.meta.url));export default defineConfig({root,resolve:{alias:{'@':root}},test:{environment:'jsdom',include:['.eval-hidden/pipe-watch-switch.test.tsx'],maxWorkers:1,minWorkers:1}});diff --git a/evals/coding-agent/graders/pipe-watch-switch.fixture.tsx.txt b/evals/coding-agent/graders/pipe-watch-switch.fixture.tsx.txtnew file mode 100644index 000000000..067b57e6c--- /dev/null+++ b/evals/coding-agent/graders/pipe-watch-switch.fixture.tsx.txt@@ -0,0 +1,30 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterEach,beforeEach,expect,test,vi} from 'vitest';+import {act,cleanup,renderHook} from '@testing-library/react';+import {useChatConversations} from '../components/hooks/use-chat-conversations';+import {useChatStore} from '../lib/stores/chat-store';+vi.mock('@/lib/utils/tauri',()=>({commands:{}}));+vi.mock('@/lib/hooks/use-settings',()=>({getStore:async()=>({get:async()=>undefined,set:async()=>{},save:async()=>{}})}));+vi.mock('@/lib/chat-storage',()=>({saveConversationFile:vi.fn(),deleteConversationFile:vi.fn(),loadAllConversations:async()=>[],loadConversationFile:async()=>null,migrateFromStoreBin:async()=>{},listConversations:async()=>[],searchConversations:async()=>[],CHAT_HISTORY_INITIAL_LIMIT:50}));+afterEach(cleanup);+beforeEach(()=>{useChatStore.setState({sessions:{},currentId:null});globalThis.fetch=vi.fn(()=>{throw Error('network disabled')}) as any;});+const msg=(id:string,content:string)=>({id,role:'assistant',content,timestamp:1});+function session(id:string,kind='chat',messages:any[]=[]){useChatStore.getState().actions.upsert({id,kind,title:id,preview:'',status:'streaming',messageCount:messages.length,createdAt:1,updatedAt:1,pinned:false,unread:false,messages,hydratedAt:1} as any);}+function mount(kind:string){+ const fresh=[msg('one','first turn with newly streamed text')],stale=[msg('one','first')];session('outgoing',kind,fresh);session('incoming','chat',[msg('target','incoming complete')]);useChatStore.getState().actions.setCurrent('outgoing');+ const opts:any={messages:stale,conversationId:'outgoing',setMessages:vi.fn(),setConversationId:vi.fn(),setInput:vi.fn(),inputRef:{current:null},isLoading:true,isStreaming:true,piInfo:null,piStreamingTextRef:{current:'stale'},piMessageIdRef:{current:'one'},piContentBlocksRef:{current:[]},piSessionSyncedRef:{current:true},piSessionIdRef:{current:'outgoing'},setIsLoading:vi.fn(),setIsStreaming:vi.fn(),setPastedImages:vi.fn(),settings:{chatHistory:{historyEnabled:false}}};+ const hook=renderHook(()=>useChatConversations(opts));return {hook,opts,fresh,stale};+}+test('switch from watched pipe preserves newer authoritative messages',async()=>{const p=mount('pipe-watch');await act(async()=>{await p.hook.result.current.loadConversation({id:'incoming',title:'incoming',messages:[]} as any)});expect(useChatStore.getState().sessions.outgoing.messages).toEqual(p.fresh);});+test('switch from normal chat preserves the panel-owned messages',async()=>{const p=mount('chat');await act(async()=>{await p.hook.result.current.loadConversation({id:'incoming',title:'incoming',messages:[]} as any)});expect(useChatStore.getState().sessions.outgoing.messages).toEqual(p.stale);});+test('incoming warm chat restores messages and foreground identity',async()=>{const p=mount('pipe-watch');await act(async()=>{await p.hook.result.current.loadConversation({id:'incoming',title:'incoming',messages:[]} as any)});expect(p.opts.setMessages).toHaveBeenLastCalledWith([msg('target','incoming complete')]);expect(useChatStore.getState().currentId).toBe('incoming');expect(p.opts.piSessionIdRef.current).toBe('incoming');});++test('shorter panel snapshot cannot discard additional normal-chat messages',async()=>{const p=mount('chat');const full=[...p.fresh,msg('two','second turn')];useChatStore.getState().actions.setMessages('outgoing',full as any);await act(async()=>{await p.hook.result.current.loadConversation({id:'incoming',title:'incoming',messages:[]} as any)});expect(useChatStore.getState().sessions.outgoing.messages).toEqual(full);});++vi.mock('@tauri-apps/api/event',()=>({emit:async()=>{},listen:async()=>()=>{}}));+vi.mock('@/lib/utils/generate-title-with-preset',()=>({titleCreatedByAI:async()=>null}));+vi.mock('@/lib/browser-state-cache',()=>({getCachedBrowserStateEntry:()=>null,resolveNewestBrowserState:()=>null}));+vi.mock('@/components/chat/archive-undo-toast',()=>({showChatArchiveUndoToast:()=>{}}));+vi.mock('@/lib/i18n/provider',()=>({useUiLocale:()=> 'en'}));+vi.mock('gt-react',()=>({useGT:()=> (s:string)=>s,msg:(s:string)=>s}));