Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: cover declined ACP warmup across chat remounts

Louis Beaumont7b54db6f2b34
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 05916d3df..cc754c134 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,18 @@
# Coding-agent regression evals
+`app-acp-declined-warmup` mounts the actual automatic ACP warmup hook with
+synthetic native command replies. Fifteen outcomes cover cancelled, failed and
+pending starts across new chats/remounts, successful handshakes, independent
+presets/agents, feature/provider/start guards, request inputs and late cleanup.
+The parent fails four intended outcomes and preserves eleven; the historical
+fix and current hook pass all fifteen. Run
+`bun test evals/coding-agent/calibrate-acp-warmup.test.js` for twelve controls,
+including an equivalent Map implementation, unused correct code, no-op package
+scripts, blanket suppression, lost guards and missing-source setup errors.
+Fixtures and dependency links appear only at grading. This does not execute
+native sign-in UI, manual retry/send paths, cross-webview persistence, enforced
+agent isolation or model trials.
+
`ai-gateway-outbound-chat-compatibility` executes the actual provider through its
completion and streaming methods with synthetic SDK and telemetry ports. Nine
outcomes cover response-only field removal, orphan and valid tool history,
diff --git a/evals/coding-agent/calibrate-acp-warmup.test.js b/evals/coding-agent/calibrate-acp-warmup.test.js
new file mode 100644
index 000000000..d01cc8e3f
--- /dev/null
+++ b/evals/coding-agent/calibrate-acp-warmup.test.js
@@ -0,0 +1,62 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(x => x.id === 'app-acp-declined-warmup');
+const sourcePath = item.oracle_paths[0];
+const source = ref => execFileSync('git', ['show', `${ref}:${sourcePath}`], { cwd: repo, encoding: 'utf8' });
+const parent = source(item.base_ref), fixed = source(item.oracle_ref);
+const hash = bytes => createHash('sha256').update(bytes).digest('hex');
+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }
+const controls = [
+ ['parent', parent, 'fail'], ['reference', fixed, 'pass'], ['current', source('HEAD'), 'pass'],
+ ['equivalent', fixed.replace('new Set<string>()', 'new Map<string, boolean>()').replace('pendingOrFailedWarmups.add(agentKey)', 'pendingOrFailedWarmups.set(agentKey, true)'), 'pass'],
+ ['unused-correct', parent, 'fail'], ['no-op-package-script', parent, 'fail'],
+ ['blanket-no-warmup', 'export function useAcpWarmup() {}', 'fail'],
+ ['all-agents-blocked', change('JSON.stringify([presetId, agentId])', '"shared-key"'), 'fail'],
+ ['success-stays-blocked', change('pendingOrFailedWarmups.delete(agentKey);', '// Synthetic failure to release successful key.'), 'fail'],
+ ['disabled-start', change('if (!enabled) return;', '// Synthetic removed feature guard.'), 'fail'],
+ ['liveness-releases-guard', change('[enabled, provider, presetId, agentId, sessionId]', '[enabled, provider, presetId, agentId, sessionId, piInfo?.running]'), 'fail'],
+ ['missing-source', null, 'error'],
+];
+for (const [name, body, expected] of controls) test(`ACP warmup calibration: ${name}`, () => {
+ const root = mkdtempSync(join(tmpdir(), 'acp-warmup-calibration-'));
+ try {
+ const app = join(root, 'apps/screenpipe-app-tauri');
+ mkdirSync(dirname(join(root, sourcePath)), { recursive: true });
+ if (body !== null) writeFileSync(join(root, sourcePath), body);
+ mkdirSync(join(app, 'lib/utils'), { recursive: true });
+ writeFileSync(join(app, 'lib/utils/tauri.ts'), 'export const commands = {};');
+ symlinkSync(join(repo, 'apps/screenpipe-app-tauri/node_modules'), join(app, 'node_modules'), 'dir');
+ const fixtureHashes = {};
+ for (const f of item.grader.fixtures) {
+ const bytes = readFileSync(join(import.meta.dir, f.local_path));
+ writeFileSync(join(root, f.destination_path), bytes); fixtureHashes[f.local_path] = hash(bytes);
+ }
+ if (name === 'unused-correct') writeFileSync(join(app, 'unused-correct.ts'), fixed);
+ if (name === 'no-op-package-script') writeFileSync(join(app, 'package.json'), JSON.stringify({ scripts: { test: 'exit 0', 'test:eval': 'exit 0' } }));
+ const command = ['node', 'node_modules/vitest/vitest.mjs', 'run', '--config', 'eval-acp-warmup.config.mjs'];
+ const result = spawnSync(command[0], command.slice(1), { cwd: app, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024 });
+ const kind = classifyGraderError(result);
+ const observed = result.error || result.signal || kind ? 'error' : result.status === 0 ? 'pass' : 'fail';
+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {
+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command, expected, observed, exit: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: kind, source_sha256: body === null ? null : hash(body), fixture_sha256: fixtureHashes, stdout: result.stdout, stderr: result.stderr }, null, 2));
+ }
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);
+ if (expected === 'pass') expect(result.stdout).toContain('15 passed');
+ if (['parent', 'unused-correct', 'no-op-package-script'].includes(name)) {
+ expect(result.stdout).toContain('4 failed | 11 passed');
+ expect(result.stderr).toContain('unsuccessful cancelled');
+ expect(result.stderr).toContain('pending attempt is not duplicated');
+ }
+ if (name === 'disabled-start') expect(result.stderr).toContain('preserves disabled start guard');
+ if (name === 'missing-source') expect(kind).toBe('vitest_collection_error');
+ } finally { rmSync(root, { recursive: true, force: true }); }
+}, 65000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index ce93560c2..f4981cb95 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6323,6 +6323,48 @@
],
"command": "bun --no-env-file test packages/ai-gateway/eval-vertex-upstream-json.test.ts"
}
+ },
+ {
+ "id": "app-acp-declined-warmup",
+ "name": "Keep declined automatic agent warmups quiet across chat remounts",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "6a443fb1ff9e0a11848f7dbddd923ec6748be87a"
+ },
+ "base_ref": "67f8fdf8222305796a9cafe31cdb74c289a8121d",
+ "oracle_ref": "6a443fb1ff9e0a11848f7dbddd923ec6748be87a",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "authentication",
+ "chat",
+ "react"
+ ],
+ "prompt": "The chat view can repeatedly reopen an agent sign-in after the user declined an automatic warmup. Fix the existing useAcpWarmup hook so a pending or unsuccessful automatic start for the same preset and ACP agent stays quiet across new conversations and chat remounts within the same webview lifetime. Non-running replies, error replies and rejected starts all count as unsuccessful. A successful running handshake must still permit automatic warming for a later conversation. Failure for one preset/agent must not suppress another preset or agent. Preserve disabled-feature, hosted-provider, missing preset/agent, already-running and in-flight start guards. A process-liveness update must not release a handshake guard before its call completes. Release the guard on settlement or unmount; ignore late UI/config/thinking updates after unmount, and only update them after successful running handshakes. Preserve session, project directory, token and provider config passed to the start command, and successful state/config/thinking synchronization. The hidden checks mount the actual hook with synthetic native command responses; native sign-in UI, cross-webview persistence and manual send/retry paths are outside this case. Keep changes scoped to automatic warmup and add or update tests as needed.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 120,
+ "fixtures": [
+ {
+ "local_path": "graders/acp-declined-warmup.fixture.ts.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-acp-warmup.test.ts"
+ },
+ {
+ "local_path": "graders/acp-warmup.config.mjs.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-acp-warmup.config.mjs"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-acp-warmup.config.mjs"
+ },
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-acp-warmup.ts"
+ ],
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ]
}
]
}
diff --git a/evals/coding-agent/graders/acp-declined-warmup.fixture.ts.txt b/evals/coding-agent/graders/acp-declined-warmup.fixture.ts.txt
new file mode 100644
index 000000000..eaa7db9fd
--- /dev/null
+++ b/evals/coding-agent/graders/acp-declined-warmup.fixture.ts.txt
@@ -0,0 +1,101 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { act, cleanup, renderHook } from '@testing-library/react';
+import { afterEach, expect, it, vi } from 'vitest';
+const port = vi.hoisted(() => ({ start: vi.fn() }));
+vi.mock('@/lib/utils/tauri', () => ({ commands: { piStart: port.start } }));
+import { useAcpWarmup } from './components/chat/standalone/hooks/use-acp-warmup';
+
+let sequence = 0;
+function options() {
+ const id = `synthetic-${++sequence}`;
+ return {
+ enabled: true, activePreset: { id, provider: 'acp', acpAgent: { id: 'test-agent' } } as any,
+ piInfo: null as any, piStartInFlightRef: { current: false }, piSessionIdRef: { current: `${id}-chat` },
+ piProjectDirForSession: vi.fn(async (session: string) => `/synthetic/${session}`),
+ buildProviderConfig: vi.fn(() => ({ provider: 'acp', model: 'test-agent' }) as any),
+ userToken: 'synthetic-token', setPiInfo: vi.fn(), setPiStarting: vi.fn(),
+ setRunningConfigFromProviderConfig: vi.fn(), syncThinkingLevelAfterStart: vi.fn(),
+ };
+}
+const settle = () => act(async () => { await Promise.resolve(); await Promise.resolve(); });
+afterEach(() => { cleanup(); port.start.mockReset(); });
+
+it.each(['cancelled', 'error', 'rejected'])('unsuccessful %s attempt stays quiet across new chat and remount', async kind => {
+ if (kind === 'rejected') port.start.mockRejectedValue(new Error('synthetic refusal'));
+ else port.start.mockResolvedValue(kind === 'cancelled' ? { status: 'ok', data: { running: false } } : { status: 'error', error: 'synthetic refusal' });
+ const o = options(); const first = renderHook(() => useAcpWarmup(o)); await settle();
+ expect(port.start).toHaveBeenCalledTimes(1);
+ expect(o.piStartInFlightRef.current).toBe(false);
+ expect(o.setPiStarting).toHaveBeenLastCalledWith(false);
+ expect(o.setPiInfo).not.toHaveBeenCalled();
+ expect(o.setRunningConfigFromProviderConfig).not.toHaveBeenCalled();
+ expect(o.syncThinkingLevelAfterStart).not.toHaveBeenCalled();
+ o.piSessionIdRef.current += '-next'; first.rerender(); await settle(); first.unmount();
+ const second = renderHook(() => useAcpWarmup(o)); await settle();
+ expect(port.start).toHaveBeenCalledTimes(1); second.unmount();
+});
+
+it('pending attempt is not duplicated by remount with a new conversation', async () => {
+ let resolve!: (value: any) => void;
+ port.start.mockImplementation(() => new Promise(r => { resolve = r; }));
+ const o = options(); const first = renderHook(() => useAcpWarmup(o)); await settle();
+ expect(port.start).toHaveBeenCalledTimes(1); const finishFirst = resolve;
+ first.unmount(); expect(o.piStartInFlightRef.current).toBe(false);
+ o.piSessionIdRef.current += '-next'; const second = renderHook(() => useAcpWarmup(o)); await settle();
+ expect(port.start).toHaveBeenCalledTimes(1);
+ await act(async () => finishFirst({ status: 'ok', data: { running: false } })); second.unmount();
+});
+
+it('successful handshakes continue on new conversations with state and request inputs preserved', async () => {
+ const info = { running: true, busy: false }; port.start.mockResolvedValue({ status: 'ok', data: info });
+ const o = options(); const firstSession = o.piSessionIdRef.current;
+ const hook = renderHook(() => useAcpWarmup(o)); await settle();
+ expect(port.start).toHaveBeenLastCalledWith(firstSession, `/synthetic/${firstSession}`, 'synthetic-token', { provider: 'acp', model: 'test-agent' });
+ expect(o.setPiInfo).toHaveBeenLastCalledWith(info);
+ expect(o.setRunningConfigFromProviderConfig).toHaveBeenLastCalledWith({ provider: 'acp', model: 'test-agent' });
+ expect(o.syncThinkingLevelAfterStart).toHaveBeenLastCalledWith(firstSession);
+ o.piSessionIdRef.current += '-next'; hook.rerender(); await settle();
+ expect(port.start).toHaveBeenCalledTimes(2);
+ expect(o.syncThinkingLevelAfterStart).toHaveBeenLastCalledWith(o.piSessionIdRef.current);
+ expect(o.piStartInFlightRef.current).toBe(false);
+});
+
+it.each(['preset', 'agent'])('failure does not suppress a different %s', async field => {
+ port.start.mockResolvedValue({ status: 'error', error: 'synthetic refusal' });
+ const o = options(); const hook = renderHook(() => useAcpWarmup(o)); await settle();
+ expect(port.start).toHaveBeenCalledTimes(1);
+ o.activePreset = field === 'preset' ? { ...o.activePreset, id: o.activePreset.id + '-other' } : { ...o.activePreset, acpAgent: { id: 'other-agent' } };
+ hook.rerender(); await settle(); expect(port.start).toHaveBeenCalledTimes(2);
+});
+
+it.each(['disabled', 'hosted', 'missing-preset', 'missing-agent', 'running', 'in-flight'])('preserves %s start guard', async guard => {
+ port.start.mockResolvedValue({ status: 'ok', data: { running: true } }); const o = options();
+ if (guard === 'disabled') o.enabled = false;
+ if (guard === 'hosted') o.activePreset.provider = 'openai';
+ if (guard === 'missing-preset') o.activePreset.id = '';
+ if (guard === 'missing-agent') o.activePreset.acpAgent.id = '';
+ if (guard === 'running') o.piInfo = { running: true, sessionId: o.piSessionIdRef.current };
+ if (guard === 'in-flight') o.piStartInFlightRef.current = true;
+ renderHook(() => useAcpWarmup(o)); await settle();
+ expect(port.start).not.toHaveBeenCalled(); expect(o.setPiStarting).not.toHaveBeenCalled();
+});
+
+it('process liveness does not release a still-pending handshake guard', async () => {
+ let finish!: (value: any) => void; port.start.mockReturnValue(new Promise(r => { finish = r; }));
+ const o = options(); const hook = renderHook(() => useAcpWarmup(o)); await settle();
+ expect(port.start).toHaveBeenCalledTimes(1); expect(o.piStartInFlightRef.current).toBe(true);
+ o.piInfo = { running: true, sessionId: o.piSessionIdRef.current }; hook.rerender();
+ expect(o.piStartInFlightRef.current).toBe(true); expect(o.setPiStarting).not.toHaveBeenCalledWith(false);
+ await act(async () => finish({ status: 'ok', data: o.piInfo }));
+ expect(o.piStartInFlightRef.current).toBe(false); expect(o.setPiStarting).toHaveBeenLastCalledWith(false);
+});
+
+it('unmount releases its guard and ignores late UI state updates', async () => {
+ let finish!: (value: any) => void; port.start.mockReturnValue(new Promise(r => { finish = r; }));
+ const o = options(); const hook = renderHook(() => useAcpWarmup(o)); await settle();
+ expect(o.piStartInFlightRef.current).toBe(true); hook.unmount();
+ expect(o.piStartInFlightRef.current).toBe(false); expect(o.setPiStarting).toHaveBeenLastCalledWith(false);
+ await act(async () => finish({ status: 'ok', data: { running: true } }));
+ expect(o.setPiInfo).not.toHaveBeenCalled(); expect(o.setRunningConfigFromProviderConfig).not.toHaveBeenCalled(); expect(o.syncThinkingLevelAfterStart).not.toHaveBeenCalled();
+});
diff --git a/evals/coding-agent/graders/acp-warmup.config.mjs.txt b/evals/coding-agent/graders/acp-warmup.config.mjs.txt
new file mode 100644
index 000000000..ca8536dbf
--- /dev/null
+++ b/evals/coding-agent/graders/acp-warmup.config.mjs.txt
@@ -0,0 +1,9 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { defineConfig } from 'vitest/config';
+import { fileURLToPath } from 'node:url';
+export default defineConfig({
+ resolve: { alias: { '@': fileURLToPath(new URL('.', import.meta.url)) } },
+ css: { postcss: { plugins: [] } },
+ test: { environment: 'jsdom', include: ['eval-acp-warmup.test.ts'], pool: 'forks', poolOptions: { forks: { singleFork: true } } },
+});