YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover saved ACP billing recovery and retry boundaries
louisbfff9ce15239
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex de4aad429..f1a647fc4 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,10 +9,22 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 73 git-mined regressions. See+The current app corpus contains 74 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.+`app-acp-billing-saved-recovery` exercises the real foreground event hook,+provider presentation and saved-message rendering with synthetic native, event,+storage and network ports. Twelve outcomes cover provider-specific account+recovery with and without HTTP 429, persistence, automatic versus manual retries,+stale-session retry suppression, context overflow and successful recovery.+The parent fails seven intended outcomes and preserves five; the reference+passes twelve. Guidance is checked semantically, with an equivalent-prose control.+Run `bun test evals/coding-agent/calibrate-acp-billing-recovery.test.js` with the+desktop test dependencies installed. Graders and dependency links are installed+only after grading starts; those links do not establish execution isolation.+No native ACP, live billing, entitlement safety or model performance is measured.+`app-daily-summary-terminal-error` runs the actual summary orchestration andits internal helpers against synthetic native commands and event delivery.Thirteen outcomes cover terminal message/agent errors, original provider detail,diff --git a/evals/coding-agent/calibrate-acp-billing-recovery.test.js b/evals/coding-agent/calibrate-acp-billing-recovery.test.jsnew file mode 100644index 000000000..0563b7fb8--- /dev/null+++ b/evals/coding-agent/calibrate-acp-billing-recovery.test.js@@ -0,0 +1,55 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-acp-billing-saved-recovery');+const provider = `${app}/lib/chat/provider-errors.ts`, hook = `${app}/components/chat/standalone/hooks/use-pi-foreground-events.ts`;+const root = mkdtempSync(join(tmpdir(), 'acp-billing-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);+ paths.push('packages/workflows-ui', 'crates/screenpipe-core/assets');+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('12 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails seven intended outcomes and preserves five',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('7 failed | 5 passed');},60000);+test('reference preserves all twelve outcomes',()=>passes(grade('reference')),60000);+test('equivalent recovery prose and helper naming pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{+ replace(cwd,provider,'Add credits in ${billing}, then send your message again.','Purchase credits through ${billing}, then submit the request again.');+ replace(cwd,provider,'To continue now, choose another AI preset.','Switch to a different AI preset to continue.');+ const f=join(cwd,provider);writeFileSync(f,readFileSync(f,'utf8').replaceAll('buildAgentBillingMessage','accountRecoveryGuidance'));+})),60000);+test('unused correct helper cannot hide a broken active caller',()=>fails(grade('unused',item.oracle_ref,cwd=>{+ writeFileSync(join(cwd,`${app}/lib/chat/unused-correct-provider.ts`),readFileSync(join(cwd,provider)));+ for(const f of [provider,hook])writeFileSync(join(cwd,f),execFileSync('git',['show',`${item.base_ref}:${f}`],{cwd:repo}));+})),60000);+test('billing 429 cannot fall through to automatic throttling retry',()=>fails(grade('billing-retry',item.oracle_ref,cwd=>replace(cwd,hook,'if (buildProviderErrorPresentation(error, getActivePreset())?.kind === "agent_billing") {','if (false) {'))),60000);+test('blanket retry suppression fails preserved transient behavior',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,hook,'return classifyHostedQuotaError(error);','return "none";'))),60000);+test('discarding durable saved messages fails',()=>fails(grade('lost-save',item.oracle_ref,cwd=>replace(cwd,hook,'.actions.setMessages(finalizedSessionId, nextMessages as any);','.actions.setMessages(finalizedSessionId, []);'))),60000);+test('missing provider source is a setup error, not a regression',()=>{+ const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,provider)));expect(r.status).toBe(1);expect(r.stdout).toContain('no tests');expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);+},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex ab87f262c..a43f5b98a 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-23.2",+ "dataset_version": "2026-09-23.5","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3001,6 +3001,52 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-pipe-run-outputs.config.mjs"}+ },+ {+ "id": "app-acp-billing-saved-recovery",+ "name": "Preserve own-account ACP billing recovery without automatic retries",+ "source": {+ "kind": "git_regression",+ "fix_commit": "02e8048c8138e8a2bb4fccb0a92ab945672d5b31"+ },+ "base_ref": "6661cc543625435bd8992efba6aa3277cb519883",+ "oracle_ref": "02e8048c8138e8a2bb4fccb0a92ab945672d5b31",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/chat/provider-errors.ts",+ "apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-foreground-events.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/chat/provider-errors.ts",+ "apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-foreground-events.ts",+ "apps/screenpipe-app-tauri/components/chat/standalone/message-content.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "billing-recovery",+ "acp",+ "retry",+ "persistence"+ ],+ "prompt": "Own-account ACP billing failures currently leave an opaque or misleading failed turn. When the selected Claude Code account has insufficient credits, Cursor reaches a spending limit, or Codex reaches a usage limit, retain actionable guidance naming that selected provider and the relevant account recovery (add credits, review or adjust spending limits, or review usage reset/plan options), plus the option to switch AI presets. This must survive message error events, message end, terminal agent end and saving/reloading the message as an accessible error alert. A billing failure prefixed with HTTP 429 is still a billing failure: do not offer an immediate retry button or automatically resend the prompt. Preserve an ordinary transient HTTP 429 response retry with the original prompt, and prevent a delayed retry from sending into a changed session. Preserve context-overflow recovery after earlier tool progress, retaining the progress blocks and a manual retry prompt, whether or not earlier error events were delivered. A subsequent successful answer must replace the error and clear its retry action. Keep public hook/message interfaces compatible. Exact prose, private helper names and logging format are not prescribed. Use synthetic local native, event, storage and network ports only. This task covers the foreground event hook, internal provider presentation and saved-message rendering, not native ACP execution, live provider billing or financial entitlement policy.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/acp-billing-recovery.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-acp-billing-semantic.test.tsx"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts eval-acp-billing-semantic.test.tsx"+ }}]}diff --git a/evals/coding-agent/graders/acp-billing-recovery.fixture.tsx.txt b/evals/coding-agent/graders/acp-billing-recovery.fixture.tsx.txtnew file mode 100644index 000000000..fcf9ad7e5--- /dev/null+++ b/evals/coding-agent/graders/acp-billing-recovery.fixture.tsx.txt@@ -0,0 +1,212 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { useRef, useState } from "react";+import { act, render, renderHook, screen } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { Message } from "@/lib/chat/types";+import { commands } from "@/lib/utils/tauri";+import type { AIPreset } from "@/lib/utils/tauri";+import { buildContextOverflowMessage } from "@/lib/chat/provider-errors";+import { MessageContent } from "@/components/chat/standalone/message-content";+import { usePiChatState } from "@/components/chat/standalone/hooks/use-pi-chat-state";+import { usePiForegroundEvents } from "@/components/chat/standalone/hooks/use-pi-foreground-events";++const storeMessages = vi.hoisted(() => vi.fn());+vi.mock("@tauri-apps/api/event", () => ({ emit: vi.fn(async () => {}) }));+vi.mock("@/lib/stores/chat-store", () => ({+ useChatStore: { getState: () => ({+ sessions: {},+ actions: { patch: vi.fn(), setStreaming: vi.fn(), setMessages: storeMessages },+ }) },+}));+vi.mock("@/lib/events/bus", () => ({+ mountAgentEventBus: vi.fn(async () => {}),+ onTerminated: vi.fn(() => () => {}),+}));+vi.mock("@/components/chat/standalone/hooks/pi-log-listener", () => ({ registerPiLogListener: vi.fn(async () => () => {}) }));+vi.mock("@/components/chat/standalone/hooks/pi-reauth-listener", () => ({ registerPiReauthListener: vi.fn(async () => () => {}) }));++vi.mock("posthog-js", () => ({ default: {capture: vi.fn(), getFeatureFlag: vi.fn()} }));+const progress = "I'll look at the repo to see what it actually contains.";+const overflow = "400 request (33252 tokens) exceeds the available context size (32768 tokens), try increasing it";+const initial: Message = {+ id: "assistant-1", role: "assistant", content: progress, timestamp: 0,+ contentBlocks: [+ { type: "text", text: progress, phase: "commentary" },+ { type: "tool", toolCall: { id: "search-1", toolName: "sp_web_search", args: {}, isRunning: false } },+ ],+};++function useErrorHarness(activePreset?: AIPreset) {+ const pi = usePiChatState();+ const [messages, setMessages] = useState<Message[]>([initial]);+ const messagesRef = useRef(messages);+ messagesRef.current = messages;+ const handler = useRef<((event: unknown) => void) | null>(null);+ usePiForegroundEvents({+ ...pi,+ activePreset,+ buildProviderConfig: () => null,+ cancelStreamingMessageRender: vi.fn(),+ clearPipeExecution: vi.fn(),+ consumeQueuedDisplayForStartedMessage: () => undefined,+ findTurnIntentForUserStart: () => undefined,+ flushPendingSteerBatch: async () => {},+ flushStreamingMessageRender: vi.fn(),+ handleAgentEventDataRef: handler,+ handleAgentActionEvent: () => false,+ clearAgentActionsForSession: vi.fn(),+ handleInvalidatedAuthToken: vi.fn(),+ lastUserMessageRef: useRef("Explain this repo"),+ markTurnIntentConsumed: vi.fn(),+ messages, messagesRef, setMessages,+ mountedRef: useRef(true),+ optimisticSteerRef: useRef(null),+ pendingNextPiUserDisplayRef: useRef(null),+ pendingNextPiUserIntentRef: useRef(null),+ pendingSteerBatchRef: useRef([]),+ saveConversation: async () => {},+ scheduleStreamingMessageRender: vi.fn(),+ setIsLoading: vi.fn(), setIsStreaming: vi.fn(),+ settings: {} as never,+ syncThinkingLevelAfterStart: vi.fn(),+ turnIntentLedgerRef: useRef([]),+ turnIntentTextValuesMatch: () => false,+ noteTurnLivenessEvent: vi.fn(),+ });+ return { pi, messages, setMessages, handler };+}++describe("Pi terminal provider errors", () => {+ beforeEach(() => { vi.useFakeTimers(); vi.clearAllMocks(); vi.spyOn(commands, "piPrompt").mockResolvedValue({status: "ok", data: null} as never); vi.stubGlobal("fetch", vi.fn(() => { throw new Error("live network forbidden in probe"); })); });+ afterEach(() => {+ vi.clearAllTimers();+ vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllGlobals();+ });+ it.each([false, true])("retains an error after tool progress (earlier error events: %s)", (emitEarlierError) => {+ const { result } = renderHook(useErrorHarness);+ const errorMessage = { role: "assistant", stopReason: "error", errorMessage: overflow, content: [] };+ const sessionId = result.current.pi.piSessionIdRef.current;++ act(() => {+ const { pi, handler } = result.current;+ pi.piMessageIdRef.current = initial.id;+ pi.piStreamingTextRef.current = progress;+ pi.piContentBlocksRef.current = [...initial.contentBlocks!];+ if (emitEarlierError) {+ handler.current!({ type: "message_start", message: errorMessage });+ handler.current!({ type: "message_end", message: errorMessage });+ }+ handler.current!({ type: "agent_end", messages: [+ { role: "assistant", stopReason: "toolUse", content: [{ type: "text", text: progress }] },+ errorMessage,+ ] });+ });++ expect(result.current.messages[0]).toMatchObject({+ content: `Error: ${buildContextOverflowMessage(overflow)}`,+ retryPrompt: "Explain this repo",+ contentBlocks: initial.contentBlocks,+ });+ expect(storeMessages).toHaveBeenLastCalledWith(sessionId, result.current.messages);+ expect(result.current.pi.piMessageIdRef.current).toBeNull();+ });++ it("clears the retry action when the provider recovers and answers", () => {+ const { result } = renderHook(useErrorHarness);+ const answer = "Here is how to run the repository.";+ act(() => {+ result.current.setMessages([{ ...initial, content: buildContextOverflowMessage(overflow), retryPrompt: "Explain this repo" }]);+ result.current.pi.piMessageIdRef.current = initial.id;+ result.current.pi.piStreamingTextRef.current = answer;+ result.current.pi.piContentBlocksRef.current = [+ ...initial.contentBlocks!, { type: "text", text: answer },+ ];+ result.current.handler.current!({ type: "agent_end", messages: [+ { role: "assistant", stopReason: "stop", content: [{ type: "text", text: answer }] },+ ] });+ });+ expect(result.current.messages[0].content).toBe(answer);+ expect(result.current.messages[0].retryPrompt).toBeUndefined();+ });++ it.each([+ { agentId: "claude-acp", agentName: "Claude Code", error: 'Internal error: Credit balance is too low: { "errorKind": "billing_error" }', recovery: /(?:add|buy|top.up|replenish|purchase).*credits|credits.*(?:add|buy|top.up|replenish|purchase)/i },+ { agentId: "cursor", agentName: "Cursor", error: "spending_limit_exceeded", recovery: /(?:adjust|change|raise|increase|reset|review|check).*limit|limit.*(?:adjust|change|raise|increase|reset|review|check)/i },+ { agentId: "codex-acp", agentName: "Codex", error: 'Internal error: {"codexErrorInfo":"usageLimitExceeded"}', recovery: /(?:usage|plan).*(?:reset|change|upgrade|check|review)|(?:reset|change|upgrade|check|review).*(?:usage|plan)/i },+ ].flatMap((agent) => [false, true].map((with429) => ({ ...agent, with429 }))))(+ "retains $agentName billing recovery through the saved turn (HTTP 429: $with429)",+ ({ agentId, agentName, error, recovery, with429 }) => {+ const preset: AIPreset = {+ id: agentId, provider: "acp", model: agentId, prompt: "",+ acpAgent: { id: agentId, useScreenpipeCloud: false },+ defaultPreset: true, apiKey: null, maxContextChars: 100000,+ };+ const { result } = renderHook(() => useErrorHarness(preset));+ const raw = `${with429 ? "429 " : ""}${error}`;+ const errorMessage = { role: "assistant", stopReason: "error", errorMessage: raw, content: [] };+ const rawLog = vi.spyOn(console, "error").mockImplementation(() => {});+ try {+ act(() => {+ const { pi, handler } = result.current;+ pi.piMessageIdRef.current = initial.id;+ pi.piStreamingTextRef.current = progress;+ pi.piContentBlocksRef.current = [...initial.contentBlocks!];+ handler.current!({ type: "message_update", assistantMessageEvent: {+ type: "error", reason: "ACP request failed", error: raw,+ } });+ handler.current!({ type: "message_end", message: errorMessage });+ handler.current!({ type: "agent_end", messages: [errorMessage] });+ });+ const message = result.current.messages[0];+ expect(message.content).toContain(agentName);+ expect(message.content).toMatch(recovery);+ expect(message.content).toMatch(/(?:another|different|switch|change).*(?:AI|preset|provider)/i);+ expect(message.retryPrompt).toBeUndefined();+ // Keep the original technical cause in the console logs collected by+ // support, while persisted chat history carries the recovery message.+ expect(message.content).not.toContain("Error: undefined");+ const saved = storeMessages.mock.calls.at(-1)?.[1]?.[0];+ expect(saved).toEqual(message);+ render(<MessageContent message={JSON.parse(JSON.stringify(saved))} />);+ expect(screen.getByRole("alert")).toHaveTextContent(agentName);+ expect(screen.getByRole("alert")).toHaveTextContent(recovery);+ expect(screen.queryByRole("button", { name: "Try again" })).not.toBeInTheDocument();+ } finally {+ rawLog.mockRestore();+ }+ },+ );+ it.each([+ {raw: '429 Internal error: Credit balance is too low: {"errorKind":"billing_error"}', retries: 0},+ {raw: '429 Too many requests reset_in=1', retries: 1},+ ])('response failure preserves correct retry policy: $raw', async ({raw, retries}) => {+ const preset = {id: 'claude-acp', provider: 'acp', model: 'claude-acp', prompt: '', acpAgent: {id: 'claude-acp', useScreenpipeCloud: false}, defaultPreset: true, apiKey: null, maxContextChars: 100000} as AIPreset;+ const {result} = renderHook(() => useErrorHarness(preset));+ act(() => {+ result.current.pi.piMessageIdRef.current = initial.id;+ result.current.handler.current!({type: 'response', success: false, error: raw});+ });+ await act(async () => { await vi.advanceTimersByTimeAsync(61000); });+ expect(commands.piPrompt).toHaveBeenCalledTimes(retries);+ if (!retries) {+ expect(result.current.messages[0].content).toMatch(/Claude Code/);+ expect(result.current.messages[0].content).toMatch(/credits/i);+ expect(result.current.messages[0].retryPrompt).toBeUndefined();+ } else {+ expect(commands.piPrompt).toHaveBeenCalledWith(expect.any(String), 'Explain this repo', null, null);+ }+ });+ it('does not inject a delayed transient retry after the session changes', async () => {+ const {result} = renderHook(useErrorHarness);+ act(() => {+ result.current.pi.piMessageIdRef.current = initial.id;+ result.current.handler.current!({type: 'response', success: false, error: '429 Too many requests reset_in=1'});+ result.current.pi.piSessionIdRef.current = 'other-synthetic-session';+ });+ await act(async () => { await vi.advanceTimersByTimeAsync(61000); });+ expect(commands.piPrompt).not.toHaveBeenCalled();+ });++});