Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): protect foreground conversation save ownership

louis8752574c5a5a
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 2f9fdfd55..af3044ea1 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,19 @@ The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-chat-foreground-save-ownership` runs the real foreground event hook and
+React lifecycle with synthetic event, persistence and native ports. Four outcomes
+cover switches to new and existing empty sessions, stable-session saves and empty
+events. The parent fails both switched saves and preserves two outcomes; the
+reference and current source pass four. Run
+`bun test evals/coding-agent/calibrate-foreground-save.test.js` for eight controls,
+including an equivalent callback-refresh solution, unused correct code, disabled
+saves, wrong destinations and missing-source setup classification. The persistence
+port follows the existing save callback contract and serializes synthetic records.
+This does not test actual filesystem durability, full session-switch integration,
+concurrent/background saves, native event delivery, isolation or model quality.
+Fixtures and dependency links are installed only when grading starts.
+
`app-settings-restart-persistence` runs the actual settings provider, write queue
and update banner with synthetic React, storage and native restart ports. Four
outcomes cover a delayed save, another edit during the drain, failed-save refusal
diff --git a/evals/coding-agent/calibrate-foreground-save.test.js b/evals/coding-agent/calibrate-foreground-save.test.js
new file mode 100644
index 000000000..77721b286
--- /dev/null
+++ b/evals/coding-agent/calibrate-foreground-save.test.js
@@ -0,0 +1,52 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-chat-foreground-save-ownership');
+const hook = `${app}/components/chat/standalone/hooks/use-pi-foreground-events.ts`;
+const root = mkdtempSync(join(tmpdir(), 'foreground-save-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('4 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent fails two switched saves and preserves two outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('2 failed | 2 passed');},60000);
+test('reference passes all four outcomes',()=>passes(grade('reference')),60000);
+test('current source preserves all four outcomes',()=>passes(grade('current','HEAD')),60000);
+test('refreshing the captured callback is an equivalent solution',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{
+ replace(cwd,hook,'idOverride: piSessionIdRef.current,','');
+ replace(cwd,hook,' }, []);',' }, [saveConversation]);');
+})),60000);
+test('unused correct source cannot hide the broken active hook',()=>fails(grade('unused',item.oracle_ref,cwd=>{
+ writeFileSync(join(cwd,`${app}/unused-correct-foreground.ts`),readFileSync(join(cwd,hook)));
+ writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}));
+})),60000);
+test('blanket save suppression fails preserved stable saves',()=>fails(grade('no-save',item.oracle_ref,cwd=>replace(cwd,hook,'if (nextRows) {','if (false && nextRows) {'))),60000);
+test('an incorrect destination cannot pass',()=>fails(grade('wrong-target',item.oracle_ref,cwd=>replace(cwd,hook,'idOverride: piSessionIdRef.current,','idOverride: "wrong-conversation",'))),60000);
+test('missing active hook is a setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout).toContain('no tests');expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 6b526f312..0454c37ec 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-27.5",
+ "dataset_version": "2026-09-28.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3845,6 +3845,49 @@
],
"command": "cd apps/screenpipe-app-tauri && \"${BUN_BIN:-bun}\" test ../../evals/coding-agent/graders/settings-restart-persistence.test.js"
}
+ },
+ {
+ "id": "app-chat-foreground-save-ownership",
+ "name": "Keep foreground event saves within the active conversation",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "a5230a0d7a993a1a9a20781ca5abd6ace69e77b4"
+ },
+ "base_ref": "4c2f43f5c7b519dab72564a66541e4078a38963f",
+ "oracle_ref": "a5230a0d7a993a1a9a20781ca5abd6ace69e77b4",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-foreground-events.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-foreground-events.ts",
+ "apps/screenpipe-app-tauri/components/hooks/use-chat-conversations.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "chat",
+ "persistence",
+ "session-ownership"
+ ],
+ "prompt": "After switching the foreground chat session, an echoed user message can replace the previous conversation's saved messages. Fix the foreground event path so a nonempty user message-start saves its user row and one assistant placeholder into the active session only, whether that target is new or already exists as an empty conversation. Preserve the previous conversation's contents and avoid creating extra conversation records. Preserve ordinary stable-session saves. An empty user event after switching must neither write a conversation nor add visible messages. Keep the public hooks and save callback interfaces compatible. This task covers the real React foreground event lifecycle with synthetic event, storage and native ports; it does not require native filesystem execution. No particular private helper name, callback-refresh strategy or implementation is prescribed.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/foreground-save-ownership.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-stale-session.test.tsx"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts eval-stale-session.test.tsx"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/foreground-save-ownership.fixture.tsx.txt b/evals/coding-agent/graders/foreground-save-ownership.fixture.tsx.txt
new file mode 100644
index 000000000..a281a4198
--- /dev/null
+++ b/evals/coding-agent/graders/foreground-save-ownership.fixture.tsx.txt
@@ -0,0 +1,47 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { useRef, useState } from 'react';
+import { act, renderHook, cleanup } from '@testing-library/react';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { usePiChatState } from '@/components/chat/standalone/hooks/use-pi-chat-state';
+import { usePiForegroundEvents } from '@/components/chat/standalone/hooks/use-pi-foreground-events';
+vi.mock('@tauri-apps/api/event', () => ({emit:vi.fn(async()=>{})}));
+vi.mock('@/lib/stores/chat-store',()=>({useChatStore:{getState:()=>({sessions:{},actions:{patch:vi.fn(),setStreaming:vi.fn(),setMessages:vi.fn(),appendMessage:vi.fn()}})}}));
+vi.mock('@/lib/events/bus',()=>({mountAgentEventBus:vi.fn(async()=>{}),onTerminated:vi.fn(()=>()=>{})}));
+vi.mock('@/components/chat/standalone/hooks/pi-log-listener',()=>({registerPiLogListener:vi.fn(async()=>()=>{})}));
+vi.mock('@/components/chat/standalone/hooks/pi-reauth-listener',()=>({registerPiReauthListener:vi.fn(async()=>()=>{})}));
+vi.mock('posthog-js',()=>({default:{capture:vi.fn(),getFeatureFlag:vi.fn()}}));
+const oldRows=[{id:'old-row',role:'user',content:'Synthetic old conversation',timestamp:1}];
+function useHarness({session,files}:{session:string,files:Map<string,any[]>}) {
+ const pi=usePiChatState(); pi.piSessionIdRef.current=session;
+ const [messages,setMessages]=useState<any[]>([]); const messagesRef=useRef(messages);messagesRef.current=messages;
+ const handler=useRef<any>(null);
+ // This port preserves the real save API's captured-id preference and explicit
+ // override, with serialized synthetic records instead of filesystem/native IO.
+ const saveConversation=async(rows:any[],options:any={})=>{files.set(options.idOverride||session,JSON.parse(JSON.stringify(rows)));};
+ usePiForegroundEvents({...pi,activePreset:undefined,buildProviderConfig:()=>null,cancelStreamingMessageRender:vi.fn(),clearPipeExecution:vi.fn(),consumeQueuedDisplayForStartedMessage:()=>undefined,findTurnIntentForUserStart:()=>undefined,flushPendingSteerBatch:async()=>{},flushStreamingMessageRender:vi.fn(),handleAgentEventDataRef:handler,handleAgentActionEvent:()=>false,clearAgentActionsForSession:vi.fn(),handleInvalidatedAuthToken:vi.fn(),lastUserMessageRef:useRef(''),markTurnIntentConsumed:vi.fn(),messages,messagesRef,setMessages,mountedRef:useRef(true),optimisticSteerRef:useRef(null),pendingNextPiUserDisplayRef:useRef(null),pendingNextPiUserIntentRef:useRef(null),pendingSteerBatchRef:useRef([]),saveConversation,scheduleStreamingMessageRender:vi.fn(),setIsLoading:vi.fn(),setIsStreaming:vi.fn(),settings:{} as never,syncThinkingLevelAfterStart:vi.fn(),turnIntentLedgerRef:useRef([]),turnIntentTextValuesMatch:()=>false,noteTurnLivenessEvent:vi.fn()} as any);
+ return {pi,messages,handler};
+}
+beforeEach(()=>{vi.stubGlobal('fetch',vi.fn(()=>{throw new Error('live fetch forbidden')}));});
+afterEach(()=>{cleanup();vi.restoreAllMocks();vi.unstubAllGlobals();});
+it.each(['new-chat','other-existing-chat'])('saves a switched foreground turn only into %s',async(target)=>{
+ const files=new Map<string,any[]>([['old-chat',structuredClone(oldRows)]]);
+ if(target==='other-existing-chat')files.set(target,[]);
+ const {result,rerender}=renderHook(useHarness,{initialProps:{session:'old-chat',files}});
+ rerender({session:target,files});
+ await act(async()=>{result.current.handler.current({type:'message_start',message:{role:'user',content:[{type:'text',text:'Synthetic new prompt'}]}});});
+ expect(files.get('old-chat')).toEqual(oldRows);
+ expect(files.get(target)?.filter(x=>x.role==='user').map(x=>x.content)).toEqual(['Synthetic new prompt']);
+ expect(files.get(target)?.filter(x=>x.role==='assistant')).toHaveLength(1);
+ expect([...files.keys()].sort()).toEqual(['old-chat',target].sort());
+});
+it('preserves saving an ordinary stable-session turn',async()=>{
+ const files=new Map<string,any[]>();const {result}=renderHook(useHarness,{initialProps:{session:'stable-chat',files}});
+ await act(async()=>{result.current.handler.current({type:'message_start',message:{role:'user',content:[{type:'text',text:'Stable prompt'}]}});});
+ expect(files.get('stable-chat')?.map(x=>x.role)).toEqual(['user','assistant']);expect(files.get('stable-chat')?.[0].content).toBe('Stable prompt');
+});
+it('does not overwrite either conversation for an empty user event',async()=>{
+ const files=new Map<string,any[]>([['old-chat',structuredClone(oldRows)]]);const {result,rerender}=renderHook(useHarness,{initialProps:{session:'old-chat',files}});rerender({session:'new-chat',files});
+ await act(async()=>{result.current.handler.current({type:'message_start',message:{role:'user',content:[]}});});
+ expect([...files]).toEqual([['old-chat',oldRows]]);expect(result.current.messages).toEqual([]);
+});