YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover declined ACP warmup across chat remounts
Louis Beaumont7b54db6f2b34
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 05916d3df..cc754c134 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1,5 +1,18 @@# Coding-agent regression evals+`app-acp-declined-warmup` mounts the actual automatic ACP warmup hook with+synthetic native command replies. Fifteen outcomes cover cancelled, failed and+pending starts across new chats/remounts, successful handshakes, independent+presets/agents, feature/provider/start guards, request inputs and late cleanup.+The parent fails four intended outcomes and preserves eleven; the historical+fix and current hook pass all fifteen. Run+`bun test evals/coding-agent/calibrate-acp-warmup.test.js` for twelve controls,+including an equivalent Map implementation, unused correct code, no-op package+scripts, blanket suppression, lost guards and missing-source setup errors.+Fixtures and dependency links appear only at grading. This does not execute+native sign-in UI, manual retry/send paths, cross-webview persistence, enforced+agent isolation or model trials.+`ai-gateway-outbound-chat-compatibility` executes the actual provider through itscompletion and streaming methods with synthetic SDK and telemetry ports. Nineoutcomes cover response-only field removal, orphan and valid tool history,diff --git a/evals/coding-agent/calibrate-acp-warmup.test.js b/evals/coding-agent/calibrate-acp-warmup.test.jsnew file mode 100644index 000000000..d01cc8e3f--- /dev/null+++ b/evals/coding-agent/calibrate-acp-warmup.test.js@@ -0,0 +1,62 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, resolve } from 'node:path';+import { createHash } from 'node:crypto';+import { classifyGraderError } from './grader-outcome.mjs';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(x => x.id === 'app-acp-declined-warmup');+const sourcePath = item.oracle_paths[0];+const source = ref => execFileSync('git', ['show', `${ref}:${sourcePath}`], { cwd: repo, encoding: 'utf8' });+const parent = source(item.base_ref), fixed = source(item.oracle_ref);+const hash = bytes => createHash('sha256').update(bytes).digest('hex');+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }+const controls = [+ ['parent', parent, 'fail'], ['reference', fixed, 'pass'], ['current', source('HEAD'), 'pass'],+ ['equivalent', fixed.replace('new Set<string>()', 'new Map<string, boolean>()').replace('pendingOrFailedWarmups.add(agentKey)', 'pendingOrFailedWarmups.set(agentKey, true)'), 'pass'],+ ['unused-correct', parent, 'fail'], ['no-op-package-script', parent, 'fail'],+ ['blanket-no-warmup', 'export function useAcpWarmup() {}', 'fail'],+ ['all-agents-blocked', change('JSON.stringify([presetId, agentId])', '"shared-key"'), 'fail'],+ ['success-stays-blocked', change('pendingOrFailedWarmups.delete(agentKey);', '// Synthetic failure to release successful key.'), 'fail'],+ ['disabled-start', change('if (!enabled) return;', '// Synthetic removed feature guard.'), 'fail'],+ ['liveness-releases-guard', change('[enabled, provider, presetId, agentId, sessionId]', '[enabled, provider, presetId, agentId, sessionId, piInfo?.running]'), 'fail'],+ ['missing-source', null, 'error'],+];+for (const [name, body, expected] of controls) test(`ACP warmup calibration: ${name}`, () => {+ const root = mkdtempSync(join(tmpdir(), 'acp-warmup-calibration-'));+ try {+ const app = join(root, 'apps/screenpipe-app-tauri');+ mkdirSync(dirname(join(root, sourcePath)), { recursive: true });+ if (body !== null) writeFileSync(join(root, sourcePath), body);+ mkdirSync(join(app, 'lib/utils'), { recursive: true });+ writeFileSync(join(app, 'lib/utils/tauri.ts'), 'export const commands = {};');+ symlinkSync(join(repo, 'apps/screenpipe-app-tauri/node_modules'), join(app, 'node_modules'), 'dir');+ const fixtureHashes = {};+ for (const f of item.grader.fixtures) {+ const bytes = readFileSync(join(import.meta.dir, f.local_path));+ writeFileSync(join(root, f.destination_path), bytes); fixtureHashes[f.local_path] = hash(bytes);+ }+ if (name === 'unused-correct') writeFileSync(join(app, 'unused-correct.ts'), fixed);+ if (name === 'no-op-package-script') writeFileSync(join(app, 'package.json'), JSON.stringify({ scripts: { test: 'exit 0', 'test:eval': 'exit 0' } }));+ const command = ['node', 'node_modules/vitest/vitest.mjs', 'run', '--config', 'eval-acp-warmup.config.mjs'];+ const result = spawnSync(command[0], command.slice(1), { cwd: app, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024 });+ const kind = classifyGraderError(result);+ const observed = result.error || result.signal || kind ? 'error' : result.status === 0 ? 'pass' : 'fail';+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command, expected, observed, exit: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: kind, source_sha256: body === null ? null : hash(body), fixture_sha256: fixtureHashes, stdout: result.stdout, stderr: result.stderr }, null, 2));+ }+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);+ if (expected === 'pass') expect(result.stdout).toContain('15 passed');+ if (['parent', 'unused-correct', 'no-op-package-script'].includes(name)) {+ expect(result.stdout).toContain('4 failed | 11 passed');+ expect(result.stderr).toContain('unsuccessful cancelled');+ expect(result.stderr).toContain('pending attempt is not duplicated');+ }+ if (name === 'disabled-start') expect(result.stderr).toContain('preserves disabled start guard');+ if (name === 'missing-source') expect(kind).toBe('vitest_collection_error');+ } finally { rmSync(root, { recursive: true, force: true }); }+}, 65000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex ce93560c2..f4981cb95 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6323,6 +6323,48 @@],"command": "bun --no-env-file test packages/ai-gateway/eval-vertex-upstream-json.test.ts"}+ },+ {+ "id": "app-acp-declined-warmup",+ "name": "Keep declined automatic agent warmups quiet across chat remounts",+ "source": {+ "kind": "git_regression",+ "fix_commit": "6a443fb1ff9e0a11848f7dbddd923ec6748be87a"+ },+ "base_ref": "67f8fdf8222305796a9cafe31cdb74c289a8121d",+ "oracle_ref": "6a443fb1ff9e0a11848f7dbddd923ec6748be87a",+ "tags": [+ "regression",+ "escaped-failure",+ "authentication",+ "chat",+ "react"+ ],+ "prompt": "The chat view can repeatedly reopen an agent sign-in after the user declined an automatic warmup. Fix the existing useAcpWarmup hook so a pending or unsuccessful automatic start for the same preset and ACP agent stays quiet across new conversations and chat remounts within the same webview lifetime. Non-running replies, error replies and rejected starts all count as unsuccessful. A successful running handshake must still permit automatic warming for a later conversation. Failure for one preset/agent must not suppress another preset or agent. Preserve disabled-feature, hosted-provider, missing preset/agent, already-running and in-flight start guards. A process-liveness update must not release a handshake guard before its call completes. Release the guard on settlement or unmount; ignore late UI/config/thinking updates after unmount, and only update them after successful running handshakes. Preserve session, project directory, token and provider config passed to the start command, and successful state/config/thinking synchronization. The hidden checks mount the actual hook with synthetic native command responses; native sign-in UI, cross-webview persistence and manual send/retry paths are outside this case. Keep changes scoped to automatic warmup and add or update tests as needed.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 120,+ "fixtures": [+ {+ "local_path": "graders/acp-declined-warmup.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-acp-warmup.test.ts"+ },+ {+ "local_path": "graders/acp-warmup.config.mjs.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-acp-warmup.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-acp-warmup.config.mjs"+ },+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-acp-warmup.ts"+ ],+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ]}]}diff --git a/evals/coding-agent/graders/acp-declined-warmup.fixture.ts.txt b/evals/coding-agent/graders/acp-declined-warmup.fixture.ts.txtnew file mode 100644index 000000000..eaa7db9fd--- /dev/null+++ b/evals/coding-agent/graders/acp-declined-warmup.fixture.ts.txt@@ -0,0 +1,101 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { act, cleanup, renderHook } from '@testing-library/react';+import { afterEach, expect, it, vi } from 'vitest';+const port = vi.hoisted(() => ({ start: vi.fn() }));+vi.mock('@/lib/utils/tauri', () => ({ commands: { piStart: port.start } }));+import { useAcpWarmup } from './components/chat/standalone/hooks/use-acp-warmup';++let sequence = 0;+function options() {+ const id = `synthetic-${++sequence}`;+ return {+ enabled: true, activePreset: { id, provider: 'acp', acpAgent: { id: 'test-agent' } } as any,+ piInfo: null as any, piStartInFlightRef: { current: false }, piSessionIdRef: { current: `${id}-chat` },+ piProjectDirForSession: vi.fn(async (session: string) => `/synthetic/${session}`),+ buildProviderConfig: vi.fn(() => ({ provider: 'acp', model: 'test-agent' }) as any),+ userToken: 'synthetic-token', setPiInfo: vi.fn(), setPiStarting: vi.fn(),+ setRunningConfigFromProviderConfig: vi.fn(), syncThinkingLevelAfterStart: vi.fn(),+ };+}+const settle = () => act(async () => { await Promise.resolve(); await Promise.resolve(); });+afterEach(() => { cleanup(); port.start.mockReset(); });++it.each(['cancelled', 'error', 'rejected'])('unsuccessful %s attempt stays quiet across new chat and remount', async kind => {+ if (kind === 'rejected') port.start.mockRejectedValue(new Error('synthetic refusal'));+ else port.start.mockResolvedValue(kind === 'cancelled' ? { status: 'ok', data: { running: false } } : { status: 'error', error: 'synthetic refusal' });+ const o = options(); const first = renderHook(() => useAcpWarmup(o)); await settle();+ expect(port.start).toHaveBeenCalledTimes(1);+ expect(o.piStartInFlightRef.current).toBe(false);+ expect(o.setPiStarting).toHaveBeenLastCalledWith(false);+ expect(o.setPiInfo).not.toHaveBeenCalled();+ expect(o.setRunningConfigFromProviderConfig).not.toHaveBeenCalled();+ expect(o.syncThinkingLevelAfterStart).not.toHaveBeenCalled();+ o.piSessionIdRef.current += '-next'; first.rerender(); await settle(); first.unmount();+ const second = renderHook(() => useAcpWarmup(o)); await settle();+ expect(port.start).toHaveBeenCalledTimes(1); second.unmount();+});++it('pending attempt is not duplicated by remount with a new conversation', async () => {+ let resolve!: (value: any) => void;+ port.start.mockImplementation(() => new Promise(r => { resolve = r; }));+ const o = options(); const first = renderHook(() => useAcpWarmup(o)); await settle();+ expect(port.start).toHaveBeenCalledTimes(1); const finishFirst = resolve;+ first.unmount(); expect(o.piStartInFlightRef.current).toBe(false);+ o.piSessionIdRef.current += '-next'; const second = renderHook(() => useAcpWarmup(o)); await settle();+ expect(port.start).toHaveBeenCalledTimes(1);+ await act(async () => finishFirst({ status: 'ok', data: { running: false } })); second.unmount();+});++it('successful handshakes continue on new conversations with state and request inputs preserved', async () => {+ const info = { running: true, busy: false }; port.start.mockResolvedValue({ status: 'ok', data: info });+ const o = options(); const firstSession = o.piSessionIdRef.current;+ const hook = renderHook(() => useAcpWarmup(o)); await settle();+ expect(port.start).toHaveBeenLastCalledWith(firstSession, `/synthetic/${firstSession}`, 'synthetic-token', { provider: 'acp', model: 'test-agent' });+ expect(o.setPiInfo).toHaveBeenLastCalledWith(info);+ expect(o.setRunningConfigFromProviderConfig).toHaveBeenLastCalledWith({ provider: 'acp', model: 'test-agent' });+ expect(o.syncThinkingLevelAfterStart).toHaveBeenLastCalledWith(firstSession);+ o.piSessionIdRef.current += '-next'; hook.rerender(); await settle();+ expect(port.start).toHaveBeenCalledTimes(2);+ expect(o.syncThinkingLevelAfterStart).toHaveBeenLastCalledWith(o.piSessionIdRef.current);+ expect(o.piStartInFlightRef.current).toBe(false);+});++it.each(['preset', 'agent'])('failure does not suppress a different %s', async field => {+ port.start.mockResolvedValue({ status: 'error', error: 'synthetic refusal' });+ const o = options(); const hook = renderHook(() => useAcpWarmup(o)); await settle();+ expect(port.start).toHaveBeenCalledTimes(1);+ o.activePreset = field === 'preset' ? { ...o.activePreset, id: o.activePreset.id + '-other' } : { ...o.activePreset, acpAgent: { id: 'other-agent' } };+ hook.rerender(); await settle(); expect(port.start).toHaveBeenCalledTimes(2);+});++it.each(['disabled', 'hosted', 'missing-preset', 'missing-agent', 'running', 'in-flight'])('preserves %s start guard', async guard => {+ port.start.mockResolvedValue({ status: 'ok', data: { running: true } }); const o = options();+ if (guard === 'disabled') o.enabled = false;+ if (guard === 'hosted') o.activePreset.provider = 'openai';+ if (guard === 'missing-preset') o.activePreset.id = '';+ if (guard === 'missing-agent') o.activePreset.acpAgent.id = '';+ if (guard === 'running') o.piInfo = { running: true, sessionId: o.piSessionIdRef.current };+ if (guard === 'in-flight') o.piStartInFlightRef.current = true;+ renderHook(() => useAcpWarmup(o)); await settle();+ expect(port.start).not.toHaveBeenCalled(); expect(o.setPiStarting).not.toHaveBeenCalled();+});++it('process liveness does not release a still-pending handshake guard', async () => {+ let finish!: (value: any) => void; port.start.mockReturnValue(new Promise(r => { finish = r; }));+ const o = options(); const hook = renderHook(() => useAcpWarmup(o)); await settle();+ expect(port.start).toHaveBeenCalledTimes(1); expect(o.piStartInFlightRef.current).toBe(true);+ o.piInfo = { running: true, sessionId: o.piSessionIdRef.current }; hook.rerender();+ expect(o.piStartInFlightRef.current).toBe(true); expect(o.setPiStarting).not.toHaveBeenCalledWith(false);+ await act(async () => finish({ status: 'ok', data: o.piInfo }));+ expect(o.piStartInFlightRef.current).toBe(false); expect(o.setPiStarting).toHaveBeenLastCalledWith(false);+});++it('unmount releases its guard and ignores late UI state updates', async () => {+ let finish!: (value: any) => void; port.start.mockReturnValue(new Promise(r => { finish = r; }));+ const o = options(); const hook = renderHook(() => useAcpWarmup(o)); await settle();+ expect(o.piStartInFlightRef.current).toBe(true); hook.unmount();+ expect(o.piStartInFlightRef.current).toBe(false); expect(o.setPiStarting).toHaveBeenLastCalledWith(false);+ await act(async () => finish({ status: 'ok', data: { running: true } }));+ expect(o.setPiInfo).not.toHaveBeenCalled(); expect(o.setRunningConfigFromProviderConfig).not.toHaveBeenCalled(); expect(o.syncThinkingLevelAfterStart).not.toHaveBeenCalled();+});diff --git a/evals/coding-agent/graders/acp-warmup.config.mjs.txt b/evals/coding-agent/graders/acp-warmup.config.mjs.txtnew file mode 100644index 000000000..ca8536dbf--- /dev/null+++ b/evals/coding-agent/graders/acp-warmup.config.mjs.txt@@ -0,0 +1,9 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { defineConfig } from 'vitest/config';+import { fileURLToPath } from 'node:url';+export default defineConfig({+ resolve: { alias: { '@': fileURLToPath(new URL('.', import.meta.url)) } },+ css: { postcss: { plugins: [] } },+ test: { environment: 'jsdom', include: ['eval-acp-warmup.test.ts'], pool: 'forks', poolOptions: { forks: { singleFork: true } } },+});