YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: distinguish hosted outages from usage limits
louis3e70fc0a53f9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex a2d15304f..7c0e98c51 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,9 +9,9 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 41 git-mined regressions. See+The current app corpus contains 42 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and-history-mining workflow. The companion website manifest contributes 20 more.+history-mining workflow. The companion website manifest uses this same harness.The MCP config symlink case grades Settings-side IO with synthetic files andreal links on a symlink-capable host. Its platform path-resolution port is@@ -62,3 +62,5 @@ one harness implementation.- `--verify`: require baseline failure plus oracle success.Do not turn capability scores into a release gate after one run. Establish matched-environment repeated baselines first. Regression cases intended to block should target reliable `pass^k`, not a lucky `pass@k`.++The app-provider-outage-vs-limit case uses standalone Bun tests without installed package dependencies. `BUN_BIN` may select an explicit Bun binary. It checks provider presentation semantics, not live model availability.diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex a37a99e8e..325d01a0b 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-11.1",+ "dataset_version": "2026-09-14.1","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1285,6 +1285,42 @@],"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts lib/__tests__/ai-tools-mcp.test.ts lib/__tests__/eval-mcp-config-symlinks.test.ts"}+ },+ {+ "id": "app-provider-outage-vs-limit",+ "name": "Separate hosted outages from usage limits",+ "source": {+ "kind": "git_regression",+ "fix_commit": "42593c9b330a66ff39bc632bf87b2f34ad801b55",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/6897"+ },+ "base_ref": "f04a0c8071ee38397cfe138e4ec3e1f433229ddd",+ "oracle_ref": "42593c9b330a66ff39bc632bf87b2f34ad801b55",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/chat/provider-errors.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/chat/provider-errors.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "classification",+ "preserved-behavior"+ ],+ "prompt": "Screenpipe hosted chat labels service-unavailable errors as usage throttling and suggests upgrading. Treat hosted unavailability, including differently cased and status-prefixed messages, as a retryable cloud outage with retry guidance and no quota or upgrade claim. Preserve explicit rate-limit guidance (wait/retry and upgrade options), real daily allowance limits and tomorrow guidance, ordinary cloud connectivity recovery, local Ollama recovery instructions with the configured model, and non-retryable model refusals. Keep the existing public provider error message/presentation interfaces. Grade user-visible semantics rather than exact sentences; no live provider calls are needed.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/app-provider-outage-vs-limit.test.ts",+ "destination_path": "evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test ./evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts b/evals/coding-agent/graders/app-provider-outage-vs-limit.test.tsnew file mode 100644index 000000000..e098c3f64--- /dev/null+++ b/evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts@@ -0,0 +1,56 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { describe, expect, test } from "bun:test";+import { buildProviderErrorMessage, buildProviderErrorPresentation } from "../../../apps/screenpipe-app-tauri/lib/chat/provider-errors";++const hosted = { provider: "screenpipe-cloud", model: "auto" };++describe("hosted outage and usage-limit boundary", () => {+ for (const raw of ["service temporarily unavailable", "Service UNAVAILABLE", "503: upstream unavailable"]) {+ test(`unavailability is a retryable outage: ${raw}`, () => {+ const result = buildProviderErrorPresentation(raw, hosted);+ expect(result).toMatchObject({ kind: "provider", retryable: true });+ expect(result?.message.toLowerCase()).toContain("screenpipe");+ expect(result?.message.toLowerCase()).toMatch(/outage|connect|unavailable/);+ expect(result?.message.toLowerCase()).toMatch(/retry|try again/);+ expect(result?.message.toLowerCase()).not.toMatch(/upgrade|rate.limit|quota/);+ });+ }++ for (const raw of ["rate-limited", "rate limit exceeded", "too many requests"]) {+ test(`explicit throttling keeps limit guidance: ${raw}`, () => {+ const text = buildProviderErrorMessage(raw, hosted)?.toLowerCase();+ expect(text).toMatch(/rate.limit|too many requests/);+ expect(text).toMatch(/wait|retry|try again/);+ expect(text).toContain("upgrade");+ expect(text).not.toContain("outage on our end");+ });+ }++ test("a real daily allowance remains a daily allowance", () => {+ const text = buildProviderErrorMessage('{"error":"free_chat_limit_exceeded","limit":2}', hosted);+ expect(text).toContain("2 free AI messages");+ expect(text).toContain("tomorrow");+ expect(text).toContain("upgrade");+ });++ test("ordinary cloud transport errors keep retryable connectivity guidance", () => {+ const result = buildProviderErrorPresentation("Connection error.", hosted);+ expect(result).toMatchObject({ kind: "provider", retryable: true });+ expect(result?.message.toLowerCase()).toContain("screenpipe");+ expect(result?.message.toLowerCase()).not.toContain("upgrade");+ });++ test("local Ollama errors retain local recovery instructions", () => {+ const text = buildProviderErrorMessage("Connection error.", { provider: "native-ollama", model: "fixture-model" });+ expect(text).toContain("ollama serve");+ expect(text).toContain("fixture-model");+ expect(text?.toLowerCase()).not.toContain("upgrade");+ });++ test("an explicit model refusal remains non-retryable", () => {+ const result = buildProviderErrorPresentation("finish_reason: content_filter", hosted);+ expect(result).toMatchObject({ kind: "safety_refusal", retryable: false });+ });+});