Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): exercise terminal quota retry outcomes

louisae02634b0c11
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index b39edf76b..b83b6ffe3 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,18 @@ The current app corpus contains 76 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-terminal-quota-retry` exercises the actual foreground event hook with
+synthetic native commands and event delivery. Seven event outcomes cover terminal
+quota/model retry cancellation, finalization, retained transient retries, delayed
+throttling recovery and successful answers; 76 historical helper assertions remain.
+The parent fails three terminal outcomes and preserves 80 assertions; the reference
+passes all 83. A disabled stop condition previously passed the source-name grader.
+Run `bun test evals/coding-agent/calibrate-terminal-quota.test.js` for eight controls
+including disabled/blanket stopping, lost stop effects, equivalent detector naming,
+blanket retry suppression and missing-source setup classification. Runtime links
+are installed only for grading. This does not establish native process termination,
+live billing, isolation or model performance.
+
`app-acp-billing-saved-recovery` exercises the real foreground event hook,
provider presentation and saved-message rendering with synthetic native, event,
storage and network ports. Twelve outcomes cover provider-specific account
diff --git a/evals/coding-agent/calibrate-terminal-quota.test.js b/evals/coding-agent/calibrate-terminal-quota.test.js
new file mode 100644
index 000000000..6eeb51bcd
--- /dev/null
+++ b/evals/coding-agent/calibrate-terminal-quota.test.js
@@ -0,0 +1,51 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-terminal-quota-retry');
+const provider = `${app}/lib/chat/provider-errors.ts`, hook = `${app}/components/chat/standalone/hooks/use-pi-foreground-events.ts`;
+const root = mkdtempSync(join(tmpdir(), 'terminal-quota-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+ paths.push('crates/screenpipe-core/assets');
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path || f.source_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, f.local_path ? readFileSync(join(import.meta.dir, f.local_path)) : execFileSync("git", ["show", `${f.source_ref}:${f.source_path}`], {cwd: repo})); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('83 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent fails terminal event outcomes, preserving other behavior',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('3 failed | 80 passed');},60000);
+test('reference passes actual terminal, transient and success events',()=>passes(grade('reference')),60000);
+
+test('disabled terminal guard cannot pass through a source-name match',()=>fails(grade('disabled-guard',item.oracle_ref,cwd=>replace(cwd,hook,'if (isTerminalQuotaError(piLastErrorRef.current ?? "")) {','if (false && isTerminalQuotaError(piLastErrorRef.current ?? "")) {'))),60000);
+test('blanket stop fails preserved provider retries',()=>fails(grade('blanket-stop',item.oracle_ref,cwd=>replace(cwd,hook,'if (isTerminalQuotaError(piLastErrorRef.current ?? "")) {','if (true) {'))),60000);
+test('missing actual stop cannot hide behind finalized state',()=>fails(grade('no-stop',item.oracle_ref,cwd=>replace(cwd,hook,'void commands.piStop(sid);','void Promise.resolve(sid);'))),60000);
+test('equivalent private detector naming passes',()=>passes(grade('renamed',item.oracle_ref,cwd=>{
+ for(const file of [hook,`${app}/components/chat/standalone/hooks/pi-event-handlers.ts`]) {const f=join(cwd,file);writeFileSync(f,readFileSync(f,'utf8').replaceAll('isTerminalQuotaError','terminalUsageFailure'));}
+})),60000);
+test('blanket response retry suppression fails',()=>fails(grade('no-response-retry',item.oracle_ref,cwd=>replace(cwd,hook,'classifyQuotaError(errorStr) === "rate" &&','false && classifyQuotaError(errorStr) === "rate" &&'))),60000);
+test('missing hook source is setup failure',()=>{
+ const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);
+},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index d098e636f..966ac6e11 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -64,14 +64,8 @@
"quota",
"retry"
],
- "prompt": "The Pi chat transport receives provider 429 responses that mean the account has exhausted a daily allowance or the selected model is not permitted. The app currently treats these as transient and keeps retrying, leaving the user in a loop. Make terminal quota/plan errors stop the retry path and surface the assistant error, while ordinary transient failures remain retryable. Preserve existing silent prompt-timeout and connection-card behavior.",
+ "prompt": "The Pi chat transport receives provider 429 responses that mean the account has exhausted a daily allowance or the selected model is not permitted. The app currently treats these as transient and keeps retrying, leaving the user in a loop. Make terminal quota/plan errors stop the retry path and surface the assistant error, while ordinary transient failures remain retryable. Preserve existing silent prompt-timeout and connection-card behavior. Exercise the actual foreground event handling: terminal message errors followed by an agent-end event promising another retry must stop that session, finalize the assistant error, and clear loading/streaming state without submitting another prompt. Ordinary provider retry events stay active; a transient throttled response can retry after its wait, and successful answers still finalize. No live provider or native process is needed.",
"agent_timeout_seconds": 720,
- "dependency_links": [
- {
- "source_path": "apps/screenpipe-app-tauri/node_modules",
- "destination_path": "apps/screenpipe-app-tauri/node_modules"
- }
- ],
"grader": {
"timeout_seconds": 300,
"fixtures": [
@@ -89,7 +83,13 @@
}
],
"command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts lib/chat/__tests__/eval-terminal-quota.test.ts lib/chat/__tests__/quota-errors.test.ts lib/chat/__tests__/provider-errors.test.ts"
- }
+ },
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ]
},
{
"id": "app-timeline-stream-reliability",
diff --git a/evals/coding-agent/graders/app-terminal-quota.test.ts b/evals/coding-agent/graders/app-terminal-quota.test.ts
index d3811d0d7..c8506a466 100644
--- a/evals/coding-agent/graders/app-terminal-quota.test.ts
+++ b/evals/coding-agent/graders/app-terminal-quota.test.ts
@@ -1,48 +1,139 @@
// screenpipe — AI that knows everything you've seen, said, or heard
-// https://screenpi.pe
+// https://screenpipe.com
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-import { describe, expect, it } from "vitest";
-import * as handlers from "../../../components/chat/standalone/hooks/pi-event-handlers";
-import * as quota from "../quota-errors";
+import { useRef, useState } from "react";
+import { act, renderHook } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { Message } from "@/lib/chat/types";
+import { commands } from "@/lib/utils/tauri";
+import type { AIPreset } from "@/lib/utils/tauri";
+import { buildContextOverflowMessage } from "@/lib/chat/provider-errors";
+import { usePiChatState } from "@/components/chat/standalone/hooks/use-pi-chat-state";
+import { usePiForegroundEvents } from "@/components/chat/standalone/hooks/use-pi-foreground-events";
-type Detector = (error: string) => boolean;
+const storeMessages = vi.hoisted(() => vi.fn());
+const loading = vi.hoisted(() => vi.fn());
+const streaming = vi.hoisted(() => vi.fn());
+vi.mock("@tauri-apps/api/event", () => ({ emit: vi.fn(async () => {}) }));
+vi.mock("@/lib/stores/chat-store", () => ({
+ useChatStore: { getState: () => ({
+ sessions: {},
+ actions: { patch: vi.fn(), setStreaming: vi.fn(), setMessages: storeMessages },
+ }) },
+}));
+vi.mock("@/lib/events/bus", () => ({
+ mountAgentEventBus: vi.fn(async () => {}),
+ onTerminated: vi.fn(() => () => {}),
+}));
+vi.mock("@/components/chat/standalone/hooks/pi-log-listener", () => ({ registerPiLogListener: vi.fn(async () => () => {}) }));
+vi.mock("@/components/chat/standalone/hooks/pi-reauth-listener", () => ({ registerPiReauthListener: vi.fn(async () => () => {}) }));
-const terminalDetector =
- (handlers as Record<string, unknown>).isTerminalQuotaError ??
- (quota as Record<string, unknown>).isTerminalQuotaOrPlanError;
+vi.mock("posthog-js", () => ({ default: {capture: vi.fn(), getFeatureFlag: vi.fn()} }));
+const progress = "I'll look at the repo to see what it actually contains.";
+const overflow = "400 request (33252 tokens) exceeds the available context size (32768 tokens), try increasing it";
+const initial: Message = {
+ id: "assistant-1", role: "assistant", content: progress, timestamp: 0,
+ contentBlocks: [
+ { type: "text", text: progress, phase: "commentary" },
+ { type: "tool", toolCall: { id: "search-1", toolName: "sp_web_search", args: {}, isRunning: false } },
+ ],
+};
-describe("terminal Pi retry outcome", () => {
- it("provides one pure terminal detector with balanced positive and negative behavior", () => {
- expect(typeof terminalDetector).toBe("function");
- const detect = terminalDetector as Detector;
- for (const error of [
- 'HTTP 429 {"error":"daily_cost_limit_exceeded","required_plan":"business"}',
- "HTTP 429 credits_exhausted",
- '{"error":"model_not_allowed","required_plan":"business"}',
- ]) {
- expect(detect(error), error).toBe(true);
- }
- for (const error of [
- "HTTP 429 rate limit exceeded; wait 9 seconds",
- "500 upstream unavailable",
- "socket hang up",
- ]) {
- expect(detect(error), error).toBe(false);
- }
+function useErrorHarness(activePreset?: AIPreset) {
+ const pi = usePiChatState();
+ const [messages, setMessages] = useState<Message[]>([initial]);
+ const messagesRef = useRef(messages);
+ messagesRef.current = messages;
+ const handler = useRef<((event: unknown) => void) | null>(null);
+ usePiForegroundEvents({
+ ...pi,
+ activePreset,
+ buildProviderConfig: () => null,
+ cancelStreamingMessageRender: vi.fn(),
+ clearPipeExecution: vi.fn(),
+ consumeQueuedDisplayForStartedMessage: () => undefined,
+ findTurnIntentForUserStart: () => undefined,
+ flushPendingSteerBatch: async () => {},
+ flushStreamingMessageRender: vi.fn(),
+ handleAgentEventDataRef: handler,
+ handleAgentActionEvent: () => false,
+ clearAgentActionsForSession: vi.fn(),
+ handleInvalidatedAuthToken: vi.fn(),
+ lastUserMessageRef: useRef("Explain this repo"),
+ markTurnIntentConsumed: vi.fn(),
+ messages, messagesRef, setMessages,
+ mountedRef: useRef(true),
+ optimisticSteerRef: useRef(null),
+ pendingNextPiUserDisplayRef: useRef(null),
+ pendingNextPiUserIntentRef: useRef(null),
+ pendingSteerBatchRef: useRef([]),
+ saveConversation: async () => {},
+ scheduleStreamingMessageRender: vi.fn(),
+ setIsLoading: loading, setIsStreaming: streaming,
+ settings: {} as never,
+ syncThinkingLevelAfterStart: vi.fn(),
+ turnIntentLedgerRef: useRef([]),
+ turnIntentTextValuesMatch: () => false,
+ noteTurnLivenessEvent: vi.fn(),
});
+ return { pi, messages, setMessages, handler };
+}
- it("uses the terminal detector in the foreground event path that owns retries", () => {
- const source = readFileSync(
- join(
- process.cwd(),
- "components/chat/standalone/hooks/use-pi-foreground-events.ts",
- ),
- "utf8",
- );
- expect(source).toMatch(/\bisTerminalQuota(?:OrPlan)?Error\s*\(/);
- expect(source).toContain("classifyQuotaError");
- expect(source).toContain("PI_MAX_RATE_LIMIT_RETRIES");
+
+describe("terminal quota event outcomes", () => {
+ beforeEach(() => {
+ vi.useFakeTimers(); vi.clearAllMocks();
+ vi.spyOn(commands, "piStop").mockResolvedValue({status: "ok", data: null} as never);
+ vi.spyOn(commands, "piPrompt").mockResolvedValue({status: "ok", data: null} as never);
+ vi.stubGlobal("fetch", vi.fn(() => {throw new Error("live network forbidden");}));
+ });
+ afterEach(() => {vi.clearAllTimers();vi.useRealTimers();vi.restoreAllMocks();vi.unstubAllGlobals();});
+ const terminal = [
+ {raw: '429 {"error":"daily_cost_limit_exceeded"}', visible: /limit|allowance|budget/i},
+ {raw: '429 credits_exhausted', visible: /credit|limit|allowance|used.*included.*queries/i},
+ {raw: '429 {"error":"model_not_allowed"}', visible: /model|upgrade/i},
+ ];
+ it.each(terminal)("stops a futile provider retry and finalizes $raw", async ({raw,visible}) => {
+ const {result}=renderHook(useErrorHarness);
+ const error={role:"assistant",stopReason:"error",errorMessage:raw,content:[]};
+ const sid=result.current.pi.piSessionIdRef.current;
+ act(() => {
+ result.current.pi.piMessageIdRef.current=initial.id;
+ result.current.handler.current!({type:"message_end",message:error});
+ result.current.handler.current!({type:"agent_end",willRetry:true,messages:[error]});
+ });
+ await act(async () => {await vi.advanceTimersByTimeAsync(61000);});
+ expect(commands.piStop).toHaveBeenCalledWith(sid);
+ expect(commands.piPrompt).not.toHaveBeenCalled();
+ expect(result.current.messages[0].content).toMatch(visible);
+ expect(loading).toHaveBeenLastCalledWith(false);
+ expect(streaming).toHaveBeenLastCalledWith(false);
+ expect(result.current.pi.piMessageIdRef.current).toBeNull();
+ });
+ it.each(['429 Too many requests reset_in=1','500 upstream unavailable'])("leaves an ordinary provider retry active: %s", raw => {
+ const {result}=renderHook(useErrorHarness);
+ const error={role:"assistant",stopReason:"error",errorMessage:raw,content:[]};
+ act(() => {
+ result.current.pi.piMessageIdRef.current=initial.id;
+ result.current.handler.current!({type:"message_end",message:error});
+ result.current.handler.current!({type:"agent_end",willRetry:true,messages:[error]});
+ });
+ expect(commands.piStop).not.toHaveBeenCalled();
+ expect(result.current.pi.piMessageIdRef.current).toBe(initial.id);
+ expect(loading).toHaveBeenLastCalledWith(true);
+ expect(streaming).toHaveBeenLastCalledWith(true);
+ });
+ it("retries an ordinary throttled response after its wait",async()=>{
+ const {result}=renderHook(useErrorHarness);
+ act(()=>{result.current.pi.piMessageIdRef.current=initial.id;result.current.handler.current!({type:"response",success:false,error:"429 Too many requests reset_in=1"});});
+ await act(async()=>{await vi.advanceTimersByTimeAsync(61000);});
+ expect(commands.piPrompt).toHaveBeenCalledTimes(1);
+ expect(commands.piStop).not.toHaveBeenCalled();
+ });
+ it("finalizes a successful answer",()=>{
+ const {result}=renderHook(useErrorHarness);
+ act(()=>{result.current.pi.piMessageIdRef.current=initial.id;result.current.handler.current!({type:"agent_end",messages:[{role:"assistant",content:[{type:"text",text:"Synthetic successful answer"}]}]});});
+ expect(result.current.messages[0].content).toBe("Synthetic successful answer");
+ expect(commands.piStop).not.toHaveBeenCalled();expect(loading).toHaveBeenLastCalledWith(false);
});
});