YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): exercise terminal quota retry outcomes
louisae02634b0c11
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b39edf76b..b83b6ffe3 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -13,6 +13,18 @@ The current app corpus contains 76 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.+`app-terminal-quota-retry` exercises the actual foreground event hook with+synthetic native commands and event delivery. Seven event outcomes cover terminal+quota/model retry cancellation, finalization, retained transient retries, delayed+throttling recovery and successful answers; 76 historical helper assertions remain.+The parent fails three terminal outcomes and preserves 80 assertions; the reference+passes all 83. A disabled stop condition previously passed the source-name grader.+Run `bun test evals/coding-agent/calibrate-terminal-quota.test.js` for eight controls+including disabled/blanket stopping, lost stop effects, equivalent detector naming,+blanket retry suppression and missing-source setup classification. Runtime links+are installed only for grading. This does not establish native process termination,+live billing, isolation or model performance.+`app-acp-billing-saved-recovery` exercises the real foreground event hook,provider presentation and saved-message rendering with synthetic native, event,storage and network ports. Twelve outcomes cover provider-specific accountdiff --git a/evals/coding-agent/calibrate-terminal-quota.test.js b/evals/coding-agent/calibrate-terminal-quota.test.jsnew file mode 100644index 000000000..6eeb51bcd--- /dev/null+++ b/evals/coding-agent/calibrate-terminal-quota.test.js@@ -0,0 +1,51 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-terminal-quota-retry');+const provider = `${app}/lib/chat/provider-errors.ts`, hook = `${app}/components/chat/standalone/hooks/use-pi-foreground-events.ts`;+const root = mkdtempSync(join(tmpdir(), 'terminal-quota-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);+ paths.push('crates/screenpipe-core/assets');+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path || f.source_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, f.local_path ? readFileSync(join(import.meta.dir, f.local_path)) : execFileSync("git", ["show", `${f.source_ref}:${f.source_path}`], {cwd: repo})); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('83 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails terminal event outcomes, preserving other behavior',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('3 failed | 80 passed');},60000);+test('reference passes actual terminal, transient and success events',()=>passes(grade('reference')),60000);++test('disabled terminal guard cannot pass through a source-name match',()=>fails(grade('disabled-guard',item.oracle_ref,cwd=>replace(cwd,hook,'if (isTerminalQuotaError(piLastErrorRef.current ?? "")) {','if (false && isTerminalQuotaError(piLastErrorRef.current ?? "")) {'))),60000);+test('blanket stop fails preserved provider retries',()=>fails(grade('blanket-stop',item.oracle_ref,cwd=>replace(cwd,hook,'if (isTerminalQuotaError(piLastErrorRef.current ?? "")) {','if (true) {'))),60000);+test('missing actual stop cannot hide behind finalized state',()=>fails(grade('no-stop',item.oracle_ref,cwd=>replace(cwd,hook,'void commands.piStop(sid);','void Promise.resolve(sid);'))),60000);+test('equivalent private detector naming passes',()=>passes(grade('renamed',item.oracle_ref,cwd=>{+ for(const file of [hook,`${app}/components/chat/standalone/hooks/pi-event-handlers.ts`]) {const f=join(cwd,file);writeFileSync(f,readFileSync(f,'utf8').replaceAll('isTerminalQuotaError','terminalUsageFailure'));}+})),60000);+test('blanket response retry suppression fails',()=>fails(grade('no-response-retry',item.oracle_ref,cwd=>replace(cwd,hook,'classifyQuotaError(errorStr) === "rate" &&','false && classifyQuotaError(errorStr) === "rate" &&'))),60000);+test('missing hook source is setup failure',()=>{+ const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);+},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex d098e636f..966ac6e11 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -64,14 +64,8 @@"quota","retry"],- "prompt": "The Pi chat transport receives provider 429 responses that mean the account has exhausted a daily allowance or the selected model is not permitted. The app currently treats these as transient and keeps retrying, leaving the user in a loop. Make terminal quota/plan errors stop the retry path and surface the assistant error, while ordinary transient failures remain retryable. Preserve existing silent prompt-timeout and connection-card behavior.",+ "prompt": "The Pi chat transport receives provider 429 responses that mean the account has exhausted a daily allowance or the selected model is not permitted. The app currently treats these as transient and keeps retrying, leaving the user in a loop. Make terminal quota/plan errors stop the retry path and surface the assistant error, while ordinary transient failures remain retryable. Preserve existing silent prompt-timeout and connection-card behavior. Exercise the actual foreground event handling: terminal message errors followed by an agent-end event promising another retry must stop that session, finalize the assistant error, and clear loading/streaming state without submitting another prompt. Ordinary provider retry events stay active; a transient throttled response can retry after its wait, and successful answers still finalize. No live provider or native process is needed.","agent_timeout_seconds": 720,- "dependency_links": [- {- "source_path": "apps/screenpipe-app-tauri/node_modules",- "destination_path": "apps/screenpipe-app-tauri/node_modules"- }- ],"grader": {"timeout_seconds": 300,"fixtures": [@@ -89,7 +83,13 @@}],"command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts lib/chat/__tests__/eval-terminal-quota.test.ts lib/chat/__tests__/quota-errors.test.ts lib/chat/__tests__/provider-errors.test.ts"- }+ },+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ]},{"id": "app-timeline-stream-reliability",diff --git a/evals/coding-agent/graders/app-terminal-quota.test.ts b/evals/coding-agent/graders/app-terminal-quota.test.tsindex d3811d0d7..c8506a466 100644--- a/evals/coding-agent/graders/app-terminal-quota.test.ts+++ b/evals/coding-agent/graders/app-terminal-quota.test.ts@@ -1,48 +1,139 @@// screenpipe — AI that knows everything you've seen, said, or heard-// https://screenpi.pe+// https://screenpipe.com-import { readFileSync } from "node:fs";-import { join } from "node:path";-import { describe, expect, it } from "vitest";-import * as handlers from "../../../components/chat/standalone/hooks/pi-event-handlers";-import * as quota from "../quota-errors";+import { useRef, useState } from "react";+import { act, renderHook } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { Message } from "@/lib/chat/types";+import { commands } from "@/lib/utils/tauri";+import type { AIPreset } from "@/lib/utils/tauri";+import { buildContextOverflowMessage } from "@/lib/chat/provider-errors";+import { usePiChatState } from "@/components/chat/standalone/hooks/use-pi-chat-state";+import { usePiForegroundEvents } from "@/components/chat/standalone/hooks/use-pi-foreground-events";-type Detector = (error: string) => boolean;+const storeMessages = vi.hoisted(() => vi.fn());+const loading = vi.hoisted(() => vi.fn());+const streaming = vi.hoisted(() => vi.fn());+vi.mock("@tauri-apps/api/event", () => ({ emit: vi.fn(async () => {}) }));+vi.mock("@/lib/stores/chat-store", () => ({+ useChatStore: { getState: () => ({+ sessions: {},+ actions: { patch: vi.fn(), setStreaming: vi.fn(), setMessages: storeMessages },+ }) },+}));+vi.mock("@/lib/events/bus", () => ({+ mountAgentEventBus: vi.fn(async () => {}),+ onTerminated: vi.fn(() => () => {}),+}));+vi.mock("@/components/chat/standalone/hooks/pi-log-listener", () => ({ registerPiLogListener: vi.fn(async () => () => {}) }));+vi.mock("@/components/chat/standalone/hooks/pi-reauth-listener", () => ({ registerPiReauthListener: vi.fn(async () => () => {}) }));-const terminalDetector =- (handlers as Record<string, unknown>).isTerminalQuotaError ??- (quota as Record<string, unknown>).isTerminalQuotaOrPlanError;+vi.mock("posthog-js", () => ({ default: {capture: vi.fn(), getFeatureFlag: vi.fn()} }));+const progress = "I'll look at the repo to see what it actually contains.";+const overflow = "400 request (33252 tokens) exceeds the available context size (32768 tokens), try increasing it";+const initial: Message = {+ id: "assistant-1", role: "assistant", content: progress, timestamp: 0,+ contentBlocks: [+ { type: "text", text: progress, phase: "commentary" },+ { type: "tool", toolCall: { id: "search-1", toolName: "sp_web_search", args: {}, isRunning: false } },+ ],+};-describe("terminal Pi retry outcome", () => {- it("provides one pure terminal detector with balanced positive and negative behavior", () => {- expect(typeof terminalDetector).toBe("function");- const detect = terminalDetector as Detector;- for (const error of [- 'HTTP 429 {"error":"daily_cost_limit_exceeded","required_plan":"business"}',- "HTTP 429 credits_exhausted",- '{"error":"model_not_allowed","required_plan":"business"}',- ]) {- expect(detect(error), error).toBe(true);- }- for (const error of [- "HTTP 429 rate limit exceeded; wait 9 seconds",- "500 upstream unavailable",- "socket hang up",- ]) {- expect(detect(error), error).toBe(false);- }+function useErrorHarness(activePreset?: AIPreset) {+ const pi = usePiChatState();+ const [messages, setMessages] = useState<Message[]>([initial]);+ const messagesRef = useRef(messages);+ messagesRef.current = messages;+ const handler = useRef<((event: unknown) => void) | null>(null);+ usePiForegroundEvents({+ ...pi,+ activePreset,+ buildProviderConfig: () => null,+ cancelStreamingMessageRender: vi.fn(),+ clearPipeExecution: vi.fn(),+ consumeQueuedDisplayForStartedMessage: () => undefined,+ findTurnIntentForUserStart: () => undefined,+ flushPendingSteerBatch: async () => {},+ flushStreamingMessageRender: vi.fn(),+ handleAgentEventDataRef: handler,+ handleAgentActionEvent: () => false,+ clearAgentActionsForSession: vi.fn(),+ handleInvalidatedAuthToken: vi.fn(),+ lastUserMessageRef: useRef("Explain this repo"),+ markTurnIntentConsumed: vi.fn(),+ messages, messagesRef, setMessages,+ mountedRef: useRef(true),+ optimisticSteerRef: useRef(null),+ pendingNextPiUserDisplayRef: useRef(null),+ pendingNextPiUserIntentRef: useRef(null),+ pendingSteerBatchRef: useRef([]),+ saveConversation: async () => {},+ scheduleStreamingMessageRender: vi.fn(),+ setIsLoading: loading, setIsStreaming: streaming,+ settings: {} as never,+ syncThinkingLevelAfterStart: vi.fn(),+ turnIntentLedgerRef: useRef([]),+ turnIntentTextValuesMatch: () => false,+ noteTurnLivenessEvent: vi.fn(),});+ return { pi, messages, setMessages, handler };+}- it("uses the terminal detector in the foreground event path that owns retries", () => {- const source = readFileSync(- join(- process.cwd(),- "components/chat/standalone/hooks/use-pi-foreground-events.ts",- ),- "utf8",- );- expect(source).toMatch(/\bisTerminalQuota(?:OrPlan)?Error\s*\(/);- expect(source).toContain("classifyQuotaError");- expect(source).toContain("PI_MAX_RATE_LIMIT_RETRIES");++describe("terminal quota event outcomes", () => {+ beforeEach(() => {+ vi.useFakeTimers(); vi.clearAllMocks();+ vi.spyOn(commands, "piStop").mockResolvedValue({status: "ok", data: null} as never);+ vi.spyOn(commands, "piPrompt").mockResolvedValue({status: "ok", data: null} as never);+ vi.stubGlobal("fetch", vi.fn(() => {throw new Error("live network forbidden");}));+ });+ afterEach(() => {vi.clearAllTimers();vi.useRealTimers();vi.restoreAllMocks();vi.unstubAllGlobals();});+ const terminal = [+ {raw: '429 {"error":"daily_cost_limit_exceeded"}', visible: /limit|allowance|budget/i},+ {raw: '429 credits_exhausted', visible: /credit|limit|allowance|used.*included.*queries/i},+ {raw: '429 {"error":"model_not_allowed"}', visible: /model|upgrade/i},+ ];+ it.each(terminal)("stops a futile provider retry and finalizes $raw", async ({raw,visible}) => {+ const {result}=renderHook(useErrorHarness);+ const error={role:"assistant",stopReason:"error",errorMessage:raw,content:[]};+ const sid=result.current.pi.piSessionIdRef.current;+ act(() => {+ result.current.pi.piMessageIdRef.current=initial.id;+ result.current.handler.current!({type:"message_end",message:error});+ result.current.handler.current!({type:"agent_end",willRetry:true,messages:[error]});+ });+ await act(async () => {await vi.advanceTimersByTimeAsync(61000);});+ expect(commands.piStop).toHaveBeenCalledWith(sid);+ expect(commands.piPrompt).not.toHaveBeenCalled();+ expect(result.current.messages[0].content).toMatch(visible);+ expect(loading).toHaveBeenLastCalledWith(false);+ expect(streaming).toHaveBeenLastCalledWith(false);+ expect(result.current.pi.piMessageIdRef.current).toBeNull();+ });+ it.each(['429 Too many requests reset_in=1','500 upstream unavailable'])("leaves an ordinary provider retry active: %s", raw => {+ const {result}=renderHook(useErrorHarness);+ const error={role:"assistant",stopReason:"error",errorMessage:raw,content:[]};+ act(() => {+ result.current.pi.piMessageIdRef.current=initial.id;+ result.current.handler.current!({type:"message_end",message:error});+ result.current.handler.current!({type:"agent_end",willRetry:true,messages:[error]});+ });+ expect(commands.piStop).not.toHaveBeenCalled();+ expect(result.current.pi.piMessageIdRef.current).toBe(initial.id);+ expect(loading).toHaveBeenLastCalledWith(true);+ expect(streaming).toHaveBeenLastCalledWith(true);+ });+ it("retries an ordinary throttled response after its wait",async()=>{+ const {result}=renderHook(useErrorHarness);+ act(()=>{result.current.pi.piMessageIdRef.current=initial.id;result.current.handler.current!({type:"response",success:false,error:"429 Too many requests reset_in=1"});});+ await act(async()=>{await vi.advanceTimersByTimeAsync(61000);});+ expect(commands.piPrompt).toHaveBeenCalledTimes(1);+ expect(commands.piStop).not.toHaveBeenCalled();+ });+ it("finalizes a successful answer",()=>{+ const {result}=renderHook(useErrorHarness);+ act(()=>{result.current.pi.piMessageIdRef.current=initial.id;result.current.handler.current!({type:"agent_end",messages:[{role:"assistant",content:[{type:"text",text:"Synthetic successful answer"}]}]});});+ expect(result.current.messages[0].content).toBe("Synthetic successful answer");+ expect(commands.piStop).not.toHaveBeenCalled();expect(loading).toHaveBeenLastCalledWith(false);});});