Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: preserve independent chats with repeated prompts

Louis Beaumont5fe11930074e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 47175a6d6..e19e6c6eb 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1436,3 +1436,17 @@ bun test evals/coding-agent/calibrate-openai-tool-choice.test.js
```
This covers the provider boundary with synthetic transport. It does not establish model adherence, actual tool execution, gateway authentication, native Pi policy, agent isolation or model gains. The existing GLM case covers response conversion and remains separate.
+
+## Repeated chat prompts
+
+`app-chat-repeat-prompt-identity` runs the actual sidebar selector with synthetic
+in-memory records. Independent sends and legacy records stay separate; true
+message copies still collapse. Twelve outcomes also preserve wrappers, branches,
+pipe runs, the existing creation window, visible survivors and input records.
+
+Run `bun test evals/coding-agent/calibrate-chat-repeat-identity.test.js` with
+frontend dependencies installed. Controls include an equivalent identity encoding,
+unused correct code, disabled deduplication, identity-field omissions, a hidden
+survivor and missing-source classification. Fixtures and dependency links appear
+only at grading. This does not execute disk history/search, metadata transport,
+a mounted sidebar, native delivery, agent isolation or model trials.
diff --git a/evals/coding-agent/calibrate-chat-repeat-identity.test.js b/evals/coding-agent/calibrate-chat-repeat-identity.test.js
new file mode 100644
index 000000000..89fd88306
--- /dev/null
+++ b/evals/coding-agent/calibrate-chat-repeat-identity.test.js
@@ -0,0 +1,100 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+
+const repo = resolve(import.meta.dir, '../..');
+const prefix = 'apps/screenpipe-app-tauri/';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8'))
+ .cases.find(c => c.id === 'app-chat-repeat-prompt-identity');
+const paths = ['tsconfig.json', 'lib/stores/chat-store.ts', 'lib/chat-dedup.ts',
+ 'lib/utils/chat-title.ts', 'lib/chat/ephemeral-side-conversation.ts'];
+function sources(ref) {
+ const available = new Set(execFileSync('git', ['ls-tree', '-r', '--name-only', ref, prefix],
+ { cwd: repo, encoding: 'utf8' }).trim().split('\n'));
+ return Object.fromEntries(paths.filter(p => available.has(prefix + p)).map(p =>
+ [p, execFileSync('git', ['show', `${ref}:${prefix}${p}`], { cwd: repo, encoding: 'utf8' })]));
+}
+const broken = sources(item.base_ref), fixed = sources(item.oracle_ref);
+const current = sources('HEAD');
+const fixture = readFileSync(join(import.meta.dir, 'graders/chat-repeat-identity.fixture.ts.txt'));
+const root = mkdtempSync(join(tmpdir(), 'chat-repeat-identity-calibration-'));
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+const logRoot = process.env.EVAL_CALIBRATION_RESULTS_DIR;
+function grade(name, files) {
+ const cwd = join(root, name);
+ for (const [path, source] of Object.entries(files)) {
+ const file = join(cwd, path);
+ mkdirSync(dirname(file), { recursive: true });
+ writeFileSync(file, source);
+ }
+ symlinkSync(join(repo, prefix, 'node_modules'), join(cwd, 'node_modules'), 'dir');
+ writeFileSync(join(cwd, 'eval-chat-repeat-identity.test.ts'), fixture);
+ const args = ['--no-env-file', 'test', 'eval-chat-repeat-identity.test.ts'];
+ const result = spawnSync(process.execPath, args, { cwd, encoding: 'utf8', timeout: 30_000,
+ env: { PATH: dirname(process.execPath) } });
+ if (logRoot) {
+ mkdirSync(logRoot, { recursive: true });
+ writeFileSync(join(logRoot, `${name}.stdout`), result.stdout ?? '');
+ writeFileSync(join(logRoot, `${name}.stderr`), result.stderr ?? '');
+ writeFileSync(join(logRoot, `${name}.json`), JSON.stringify({ command: [process.execPath, ...args],
+ cwd, exit_code: result.status, signal: result.signal, error: result.error?.message ?? null,
+ grader_error_kind: classifyGraderError(result) }, null, 2) + '\n');
+ }
+ return result;
+}
+function passes(r) {
+ expect(r.error).toBeUndefined();
+ expect(r.signal).toBeNull();
+ expect(r.status).toBe(0);
+ expect(r.stderr).toContain('12 pass');
+}
+function fails(r, count) {
+ expect(r.error).toBeUndefined();
+ expect(r.signal).toBeNull();
+ expect(r.status).toBe(1);
+ expect(r.stderr).toContain('expect(received).toEqual(expected)');
+ expect(r.stderr).toContain(`${count} fail`);
+ expect(r.stderr).toContain(`${12 - count} pass`);
+ expect(classifyGraderError(r)).toBeNull();
+}
+function replace(files, path, old, next) {
+ expect(files[path].split(old)).toHaveLength(2);
+ return { ...files, [path]: files[path].replace(old, next) };
+}
+const store = 'lib/stores/chat-store.ts', dedup = 'lib/chat-dedup.ts';
+test('parent fails five identity outcomes and preserves seven neighbors', () => fails(grade('parent', broken), 5));
+test('reference passes all outcomes', () => passes(grade('reference', fixed)));
+test('current source preserves the scoped contract', () => passes(grade('current', current)));
+test('equivalent identity encoding is accepted', () => passes(grade('equivalent', replace(fixed, dedup,
+ 'return JSON.stringify([semanticKey, messageId, messageTimestamp]);',
+ 'return JSON.stringify({ text: semanticKey, id: messageId, time: messageTimestamp });'))));
+test('unused corrected helper cannot hide the broken selector', () => fails(grade('unused-helper',
+ { ...broken, [dedup]: fixed[dedup] }), 5));
+test('blanket duplicate suppression removal is rejected', () => fails(grade('no-dedup', replace(fixed, store,
+ 'const all = dedupeSessionRecords(Object.values(state.sessions));',
+ 'const all = Object.values(state.sessions);')), 3));
+test('a hidden copy cannot replace the visible survivor', () => fails(grade('hidden-wins', replace(fixed, store,
+ 'if (!!a.hidden !== !!b.hidden) return !a.hidden;',
+ 'if (!!a.hidden !== !!b.hidden) return !!a.hidden;')), 1));
+test('timestamp alone is insufficient identity', () => fails(grade('timestamp-only', replace(fixed, dedup,
+ 'return JSON.stringify([semanticKey, messageId, messageTimestamp]);',
+ 'return JSON.stringify([semanticKey, messageTimestamp]);')), 1));
+test('message id alone is insufficient identity', () => fails(grade('id-only', replace(fixed, dedup,
+ 'return JSON.stringify([semanticKey, messageId, messageTimestamp]);',
+ 'return JSON.stringify([semanticKey, messageId]);')), 1));
+test('empty output cannot hide the regression', () => fails(grade('empty-output', replace(fixed, store,
+ 'const all = dedupeSessionRecords(Object.values(state.sessions));', 'const all = [];')), 12));
+test('missing actual selector is an infrastructure error', () => {
+ const files = { ...fixed };
+ delete files[store];
+ const r = grade('missing-source', files);
+ expect(r.status).toBe(1);
+ expect(r.stderr).toContain('Cannot find module');
+ expect(r.stderr).toContain('0 pass');
+ expect(classifyGraderError(r)).toBe('bun_unhandled_error');
+});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 73cd99feb..5b1b56bcd 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -4801,6 +4801,50 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts eval-mixed-note-paste.test.tsx"
}
+ },
+ {
+ "id": "app-chat-repeat-prompt-identity",
+ "name": "Keep independent same-prompt chats visible in the sidebar",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "c2051b6d02bed075e3acc97a9dc6d89c55b58db3"
+ },
+ "base_ref": "4c2f43f5c7b519dab72564a66541e4078a38963f",
+ "oracle_ref": "c2051b6d02bed075e3acc97a9dc6d89c55b58db3",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/chat-dedup.ts",
+ "apps/screenpipe-app-tauri/lib/stores/chat-store.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/chat-dedup.ts",
+ "apps/screenpipe-app-tauri/lib/stores/chat-store.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "chat",
+ "sidebar",
+ "identity"
+ ],
+ "prompt": "Starting two chats with the same opening prompt can make one disappear from the sidebar. Repair the existing in-memory sidebar selection behavior. Independent sends must remain separate, including messages with different IDs at the same timestamp or the same ID at different timestamps. Preserve legacy conversations when a first user message lacks its ID or timestamp. Actual cross-window copies of the same first user message should still collapse within the existing thirty-minute creation window, including copies whose text carries connections context. Keep the copy with a completed reply instead of a Processing... placeholder; a visible copy must survive a hidden twin. Preserve distinct text, pipe runs, branches and conversations outside the creation window. Return the surviving records intact without mutating the input records or deleting stored conversations. Preserve the existing public selector interface. This task concerns in-memory sidebar selection; disk history/search and native UI transport are outside scope.",
+ "agent_timeout_seconds": 1200,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/chat-repeat-identity.fixture.ts.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-chat-repeat-identity.test.ts"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && \"${BUN_BIN:-bun}\" --no-env-file test eval-chat-repeat-identity.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/chat-repeat-identity.fixture.ts.txt b/evals/coding-agent/graders/chat-repeat-identity.fixture.ts.txt
new file mode 100644
index 000000000..5a027fc89
--- /dev/null
+++ b/evals/coding-agent/graders/chat-repeat-identity.fixture.ts.txt
@@ -0,0 +1,34 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { test, expect } from 'bun:test';
+import { selectOrderedSessions } from './lib/stores/chat-store';
+
+const epoch = 1700000000000;
+function row(id: string, message: Record<string, unknown> = {}, fields: Record<string, unknown> = {}) {
+ return {id, title: `Title ${id}`, kind: 'chat', status: 'idle', pinned: false,
+ hidden: false, draft: false, createdAt: epoch, updatedAt: epoch + 100,
+ lastUserMessageAt: epoch, messageCount: 2,
+ messages: [{id: 'shared-message', role: 'user', content: 'summarize this', timestamp: epoch, ...message},
+ {id: id + '-reply', role: 'assistant', content: 'answer for ' + id, timestamp: epoch + 10}], ...fields};
+}
+function assertRows(rows: ReturnType<typeof row>[], ids: string[]) {
+ const state = {sessions: Object.fromEntries(rows.map(r => [r.id, r]))};
+ const snapshot = JSON.stringify(state);
+ const actual = selectOrderedSessions(state as any);
+ expect(actual.map(r => r.id).sort()).toEqual([...ids].sort());
+ for (const value of actual) expect(value).toEqual(state.sessions[value.id]);
+ expect(JSON.stringify(state)).toBe(snapshot);
+}
+
+test('separate sends with the same text stay visible', () => assertRows([row('one', {id:'send-a'}), row('two', {id:'send-b', timestamp: epoch+1})], ['one','two']));
+test('different message ids at the same time stay visible', () => assertRows([row('one', {id:'send-a'}), row('two', {id:'send-b'})], ['one','two']));
+test('same message id at different times stays visible', () => assertRows([row('one'), row('two', {timestamp:epoch+1})], ['one','two']));
+test('legacy missing message ids stay visible', () => assertRows([row('one', {id:undefined}), row('two', {id:undefined})], ['one','two']));
+test('legacy missing message timestamps stay visible', () => assertRows([row('one', {timestamp:undefined}), row('two', {timestamp:undefined})], ['one','two']));
+test('copies of the same message still collapse to the completed reply', () => assertRows([row('ghost', {}, {messages:[{id:'shared-message',role:'user',content:'summarize this',timestamp:epoch},{role:'assistant',content:'Processing...'}]}), row('real')], ['real']));
+test('equivalent wrappers still collapse', () => assertRows([row('real'), row('ghost', {content:'<connections_context>\nsynthetic data\n</connections_context>\n\nsummarize this'}, {messageCount:1,messages:[{id:'shared-message',role:'user',content:'<connections_context>\nsynthetic data\n</connections_context>\n\nsummarize this',timestamp:epoch}]})], ['real']));
+test('different text with matching metadata stays visible', () => assertRows([row('one'), row('two', {content:'a different question'})], ['one','two']));
+test('separate pipe runs stay visible', () => assertRows([row('one', {}, {kind:'pipe-run'}), row('two', {}, {kind:'pipe-run'})], ['one','two']));
+test('branch sharing its parent opener stays visible', () => assertRows([row('one'), row('two', {}, {branchedFrom:'one'})], ['one','two']));
+test('copies outside the existing thirty-minute window stay visible', () => assertRows([row('one'), row('two', {}, {createdAt:epoch+31*60*1000})], ['one','two']));
+test('visible copy survives a more complete hidden twin', () => assertRows([row('hidden', {}, {hidden:true}), row('visible', {}, {messageCount:1,messages:[{id:'shared-message',role:'user',content:'summarize this',timestamp:epoch}]})], ['visible']));