Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: distinguish hosted outages from usage limits

louis3e70fc0a53f9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index a2d15304f..7c0e98c51 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,9 +9,9 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 41 git-mined regressions. See
+The current app corpus contains 42 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
-history-mining workflow. The companion website manifest contributes 20 more.
+history-mining workflow. The companion website manifest uses this same harness.
The MCP config symlink case grades Settings-side IO with synthetic files and
real links on a symlink-capable host. Its platform path-resolution port is
@@ -62,3 +62,5 @@ one harness implementation.
- `--verify`: require baseline failure plus oracle success.
Do not turn capability scores into a release gate after one run. Establish matched-environment repeated baselines first. Regression cases intended to block should target reliable `pass^k`, not a lucky `pass@k`.
+
+The app-provider-outage-vs-limit case uses standalone Bun tests without installed package dependencies. `BUN_BIN` may select an explicit Bun binary. It checks provider presentation semantics, not live model availability.
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index a37a99e8e..325d01a0b 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-11.1",
+ "dataset_version": "2026-09-14.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1285,6 +1285,42 @@
],
"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts lib/__tests__/ai-tools-mcp.test.ts lib/__tests__/eval-mcp-config-symlinks.test.ts"
}
+ },
+ {
+ "id": "app-provider-outage-vs-limit",
+ "name": "Separate hosted outages from usage limits",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "42593c9b330a66ff39bc632bf87b2f34ad801b55",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/6897"
+ },
+ "base_ref": "f04a0c8071ee38397cfe138e4ec3e1f433229ddd",
+ "oracle_ref": "42593c9b330a66ff39bc632bf87b2f34ad801b55",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/chat/provider-errors.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/chat/provider-errors.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "classification",
+ "preserved-behavior"
+ ],
+ "prompt": "Screenpipe hosted chat labels service-unavailable errors as usage throttling and suggests upgrading. Treat hosted unavailability, including differently cased and status-prefixed messages, as a retryable cloud outage with retry guidance and no quota or upgrade claim. Preserve explicit rate-limit guidance (wait/retry and upgrade options), real daily allowance limits and tomorrow guidance, ordinary cloud connectivity recovery, local Ollama recovery instructions with the configured model, and non-retryable model refusals. Keep the existing public provider error message/presentation interfaces. Grade user-visible semantics rather than exact sentences; no live provider calls are needed.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/app-provider-outage-vs-limit.test.ts",
+ "destination_path": "evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test ./evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts b/evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts
new file mode 100644
index 000000000..e098c3f64
--- /dev/null
+++ b/evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts
@@ -0,0 +1,56 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { describe, expect, test } from "bun:test";
+import { buildProviderErrorMessage, buildProviderErrorPresentation } from "../../../apps/screenpipe-app-tauri/lib/chat/provider-errors";
+
+const hosted = { provider: "screenpipe-cloud", model: "auto" };
+
+describe("hosted outage and usage-limit boundary", () => {
+ for (const raw of ["service temporarily unavailable", "Service UNAVAILABLE", "503: upstream unavailable"]) {
+ test(`unavailability is a retryable outage: ${raw}`, () => {
+ const result = buildProviderErrorPresentation(raw, hosted);
+ expect(result).toMatchObject({ kind: "provider", retryable: true });
+ expect(result?.message.toLowerCase()).toContain("screenpipe");
+ expect(result?.message.toLowerCase()).toMatch(/outage|connect|unavailable/);
+ expect(result?.message.toLowerCase()).toMatch(/retry|try again/);
+ expect(result?.message.toLowerCase()).not.toMatch(/upgrade|rate.limit|quota/);
+ });
+ }
+
+ for (const raw of ["rate-limited", "rate limit exceeded", "too many requests"]) {
+ test(`explicit throttling keeps limit guidance: ${raw}`, () => {
+ const text = buildProviderErrorMessage(raw, hosted)?.toLowerCase();
+ expect(text).toMatch(/rate.limit|too many requests/);
+ expect(text).toMatch(/wait|retry|try again/);
+ expect(text).toContain("upgrade");
+ expect(text).not.toContain("outage on our end");
+ });
+ }
+
+ test("a real daily allowance remains a daily allowance", () => {
+ const text = buildProviderErrorMessage('{"error":"free_chat_limit_exceeded","limit":2}', hosted);
+ expect(text).toContain("2 free AI messages");
+ expect(text).toContain("tomorrow");
+ expect(text).toContain("upgrade");
+ });
+
+ test("ordinary cloud transport errors keep retryable connectivity guidance", () => {
+ const result = buildProviderErrorPresentation("Connection error.", hosted);
+ expect(result).toMatchObject({ kind: "provider", retryable: true });
+ expect(result?.message.toLowerCase()).toContain("screenpipe");
+ expect(result?.message.toLowerCase()).not.toContain("upgrade");
+ });
+
+ test("local Ollama errors retain local recovery instructions", () => {
+ const text = buildProviderErrorMessage("Connection error.", { provider: "native-ollama", model: "fixture-model" });
+ expect(text).toContain("ollama serve");
+ expect(text).toContain("fixture-model");
+ expect(text?.toLowerCase()).not.toContain("upgrade");
+ });
+
+ test("an explicit model refusal remains non-retryable", () => {
+ const result = buildProviderErrorPresentation("finish_reason: content_filter", hosted);
+ expect(result).toMatchObject({ kind: "safety_refusal", retryable: false });
+ });
+});