Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover saved chat user recency

Louis Beaumont440575492809
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 5b1347ba8..5a4cdd75d 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.
This does not establish native recording continuity, real authentication,
account-switch race safety, browser integration, isolation or model performance.
-The current app corpus contains 135 git-mined regressions. See
+The current app corpus contains 136 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -1712,3 +1712,15 @@ and current source pass six. Run
Fixtures and dependency links appear only at grading. This does not execute
native image decoding, audio playback, real network delivery, browser layout,
enforced agent isolation or model trials.
+
+## Saved chat user recency
+
+`app-chat-saved-user-recency` calls the actual saved-chat listing and load APIs
+with synthetic filesystem/native ports. Nine outcomes cover stale, unordered,
+malformed and missing timestamps plus newer persisted values, assistant-only
+activity and retained conversation metadata/content. The parent fails three
+outcomes and preserves six; the source-only reference and current source pass
+nine. Run `bun test evals/coding-agent/calibrate-recency.test.js` for calibration.
+Hidden fixtures and dependency links appear only when grading starts. This does
+not execute native persistence, foreground/background save callbacks, full
+sidebar rendering, enforced agent isolation or model trials.
diff --git a/evals/coding-agent/calibrate-recency.test.js b/evals/coding-agent/calibrate-recency.test.js
new file mode 100644
index 000000000..9f5af77c2
--- /dev/null
+++ b/evals/coding-agent/calibrate-recency.test.js
@@ -0,0 +1,49 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-chat-saved-user-recency');
+const hook = `${app}/lib/chat-storage.ts`;
+const root = mkdtempSync(join(tmpdir(), 'chat-recency-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('9 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toMatch(/AssertionError|TestingLibraryElementError: Unable to find an element/); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent fails three recency outcomes and preserves six',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('3 failed | 6 passed');},60000);
+test('reference passes nine outcomes',()=>passes(grade('reference')),60000);
+test('current source passes nine outcomes',()=>passes(grade('current','HEAD')),60000);
+test('equivalent private timestamp names are accepted',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('newestUserMessageAt','latestNumericUserTime').replaceAll('persistedLastUserMessageAt','storedUserTime'));})),60000);
+test('unused repair cannot hide broken listing',()=>fails(grade('bypass',item.oracle_ref,cwd=>{writeFileSync(join(cwd,'unused-correct.ts'),readFileSync(join(cwd,hook)));writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}));})),60000);
+test('a newer saved marker must survive',()=>fails(grade('newer-marker',item.oracle_ref,cwd=>replace(cwd,hook,'Math.max(persistedLastUserMessageAt ?? 0, newestUserMessageAt)','newestUserMessageAt'))),60000);
+test('assistant activity must not advance user recency',()=>fails(grade('assistant-time',item.oracle_ref,cwd=>replace(cwd,hook,'m?.role === "user" && typeof m.timestamp === "number"','typeof m?.timestamp === "number"'))),60000);
+test('transcript array order cannot choose the wrong time',()=>fails(grade('last-array-item',item.oracle_ref,cwd=>replace(cwd,hook,'newestUserMessageAt == null || m.timestamp > newestUserMessageAt','true'))),60000);
+test('listing must not hide the affected conversation',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,hook,'if (!conv || typeof conv.id !== "string") return null;','return null;'))),60000);
+test('conversation title must survive metadata repair',()=>fails(grade('title-loss',item.oracle_ref,cwd=>replace(cwd,hook,'title: typeof conv.title === "string" ? conv.title : "untitled",','title: "lost",'))),60000);
+test('missing storage source is a setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stdout+r.stderr).not.toContain('AssertionError');},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index a62387b9e..594ff7e2d 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -5600,6 +5600,49 @@
"trigger_paths": [
"apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsx"
]
+ },
+ {
+ "id": "app-chat-saved-user-recency",
+ "name": "Recover saved chat recency from the latest user message",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "fa7c1db8d6d9ca9e945aca1d522a37d2b5158f16"
+ },
+ "base_ref": "5d3a539aab4134b9dbaa7467a44c31d096644e00",
+ "oracle_ref": "fa7c1db8d6d9ca9e945aca1d522a37d2b5158f16",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/chat-storage.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "chat",
+ "recents",
+ "persistence"
+ ],
+ "prompt": "Reloading saved chats can leave a conversation with an old Recents timestamp even though its saved transcript contains newer user messages. Fix the saved-chat listing API so lastUserMessageAt reflects the greatest numeric user-message timestamp or an even newer numeric persisted lastUserMessageAt. Transcript order must not matter. Ignore non-numeric persisted timestamps and non-numeric message timestamps; assistant messages must not advance user recency. When there are no numeric user-message timestamps, preserve a numeric persisted value, otherwise leave user recency absent. Legacy files without recency metadata must still derive it from user messages. Preserve conversation identity, title, createdAt, updatedAt, pinned state, message count and reloadable message contents. Keep the public listing and load APIs compatible. Checks call the actual APIs with synthetic filesystem/native ports. Native persistence, foreground/background save callbacks and sidebar rendering are outside this task; no private helper or specific implementation is required.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/recency.fixture.ts.txt",
+ "destination_path": "apps/screenpipe-app-tauri/recency-probe.test.ts"
+ },
+ {
+ "local_path": "graders/recency.config.ts",
+ "destination_path": "apps/screenpipe-app-tauri/recency-probe.config.ts"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config recency-probe.config.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/recency.config.ts b/evals/coding-agent/graders/recency.config.ts
new file mode 100644
index 000000000..98d59c7e5
--- /dev/null
+++ b/evals/coding-agent/graders/recency.config.ts
@@ -0,0 +1,5 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {defineConfig} from 'vitest/config';
+import path from 'node:path';
+export default defineConfig({test:{environment:'node',include:['recency-probe.test.ts'],maxWorkers:1,minWorkers:1},resolve:{alias:{'@':path.resolve(__dirname)}}});
diff --git a/evals/coding-agent/graders/recency.fixture.ts.txt b/evals/coding-agent/graders/recency.fixture.ts.txt
new file mode 100644
index 000000000..ac5c8a4d7
--- /dev/null
+++ b/evals/coding-agent/graders/recency.fixture.ts.txt
@@ -0,0 +1,35 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {beforeEach,expect,test,vi} from 'vitest';
+const port=vi.hoisted(()=>({files:new Map<string,{text:string,mtime:number}>()}));
+vi.mock('@tauri-apps/api/path',()=>({homeDir:async()=>'/synthetic',join:async(...x:string[])=>x.join('/')}));
+vi.mock('@/lib/utils/tauri',()=>({commands:{getChatsDir:async()=>({status:'ok',data:'/synthetic/.screenpipe/chats'}),listChatEntriesByMtime:async()=>{throw Error('synthetic unavailable native bulk listing')}}}));
+vi.mock('@/lib/browser-state-cache',()=>({deleteCachedBrowserState:async()=>{}}));
+vi.mock('@tauri-apps/plugin-fs',()=>({
+ exists:async(p:string)=>p==='/synthetic/.screenpipe/chats'||port.files.has(p),mkdir:async()=>{},
+ readDir:async(dir:string)=>[...port.files.keys()].filter(p=>p.startsWith(dir+'/')).map(p=>({name:p.slice(dir.length+1)})),
+ readTextFile:async(p:string)=>{const f=port.files.get(p);if(!f)throw Error('missing synthetic file');return f.text},
+ writeTextFile:async(p:string,text:string)=>{port.files.set(p,{text,mtime:90000})},remove:async(p:string)=>{port.files.delete(p)},
+ rename:async(a:string,b:string)=>{const f=port.files.get(a);if(!f)throw Error('missing synthetic file');port.files.set(b,f);port.files.delete(a)},
+ stat:async(p:string)=>({mtime:new Date(port.files.get(p)?.mtime??0)})
+}));
+let api:any;
+beforeEach(async()=>{port.files.clear();vi.resetModules();api=await import('./lib/chat-storage');globalThis.fetch=async()=>{throw Error('network forbidden')}});
+const user=(id:string,timestamp:any)=>({id,role:'user',content:'synthetic question '+id,timestamp});
+const assistant=(timestamp:number)=>({id:'reply',role:'assistant',content:'synthetic reply',timestamp});
+async function listed(messages:any[],lastUserMessageAt?:any){
+ const c={id:'chat_fixture',title:'Synthetic saved chat',createdAt:100,updatedAt:50000,messages,lastUserMessageAt,pinned:true};
+ port.files.set('/synthetic/.screenpipe/chats/chat_fixture.json',{text:JSON.stringify(c),mtime:50000});
+ const rows=await api.listConversations();expect(rows).toHaveLength(1);const row=rows[0];
+ expect(row.id).toBe(c.id);expect(row.title).toBe(c.title);expect(row.createdAt).toBe(100);expect(row.updatedAt).toBe(50000);expect(row.messageCount).toBe(messages.length);expect(row.pinned).toBe(true);
+ const loaded=await api.loadConversationFile(c.id);expect(loaded.messages).toEqual(messages);return row;
+}
+test('listing repairs an old persisted user timestamp from the newer saved transcript',async()=>{expect((await listed([user('a',1000),assistant(2000),user('b',9000)],1000)).lastUserMessageAt).toBe(9000)});
+test('listing finds the newest user timestamp even when transcript order differs',async()=>{expect((await listed([user('a',9000),assistant(40000),user('b',3000)],1000)).lastUserMessageAt).toBe(9000)});
+test('listing ignores malformed persisted timestamp when valid user time exists',async()=>{expect((await listed([user('a',9000)],'stale')).lastUserMessageAt).toBe(9000)});
+test('newer persisted user activity is not rolled backward by an older transcript',async()=>{expect((await listed([user('a',1000),assistant(40000)],12000)).lastUserMessageAt).toBe(12000)});
+test('assistant activity does not advance user recency',async()=>{expect((await listed([user('a',1000),assistant(40000)],1000)).lastUserMessageAt).toBe(1000)});
+test('legacy file without metadata derives recency from user messages',async()=>{expect((await listed([user('a',1000),user('b',9000),assistant(40000)])).lastUserMessageAt).toBe(9000)});
+test('assistant-only file preserves an existing user recency marker',async()=>{expect((await listed([assistant(40000)],5000)).lastUserMessageAt).toBe(5000)});
+test('file without a user message or metadata has no invented user timestamp',async()=>{expect((await listed([assistant(40000)])).lastUserMessageAt).toBeUndefined()});
+test('non-numeric message timestamps do not override valid user recency',async()=>{expect((await listed([user('a','9000'),assistant(40000)],5000)).lastUserMessageAt).toBe(5000)});