Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover gateway input-limit classification

Louis Beaumont53049736dd78
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index a9452d7d3..f87e4c52d 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 38 git-mined regressions. See
+The current app corpus contains 39 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest contributes 20 more.
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index bd5ee7115..fd5e434a3 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-08.1",
+ "dataset_version": "2026-09-09.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1145,6 +1145,52 @@
],
"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts components/settings/referral-card.test.tsx lib/auth-guard.test.tsx"
}
+ },
+ {
+ "id": "ai-gateway-input-limit-classification",
+ "name": "Recognize current provider input and tool-call limits",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "739090783bb1116dca26621522e8034134e65028",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/6945"
+ },
+ "base_ref": "655dfc3ad50fedfa9ba9561ebdf3a1467bc55517",
+ "oracle_ref": "739090783bb1116dca26621522e8034134e65028",
+ "oracle_paths": [
+ "packages/ai-gateway/src/handlers/chat.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/ai-gateway/src/handlers/chat.ts",
+ "packages/ai-gateway/src/test/chat-fallback.unit.test.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "ai-gateway",
+ "input-limit",
+ "tool-call",
+ "error-classification",
+ "false-positive"
+ ],
+ "prompt": "The gateway fails to recognize provider errors saying input tokens exceed the configured limit or a request exceeds the available context size. It also misses an excess-length tool-call array, leaving the caller without useful payload guidance. Recognize these input-size and tool-call-limit failures on their supported client-error statuses, and provide actionable tool-call guidance without leaking the raw nested error. Preserve older input-limit phrasings, image and empty-message guidance, transient status handling, terminal safety outcomes, and bounded ordered fallback attempts. Do not classify unrelated schemas, non-tool arrays, thin messages, authentication errors, or server failures as these client payload problems.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/ai-gateway/node_modules",
+ "destination_path": "packages/ai-gateway/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 300,
+ "fixtures": [
+ {
+ "local_path": "graders/ai-gateway-input-limits.test.ts",
+ "destination_path": "packages/ai-gateway/src/test/eval-input-limits.test.ts"
+ }
+ ],
+ "command": "cd packages/ai-gateway && bun test src/test/eval-input-limits.test.ts --timeout=10000"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/ai-gateway-input-limits.test.ts b/evals/coding-agent/graders/ai-gateway-input-limits.test.ts
new file mode 100644
index 000000000..dfc7d8f1c
--- /dev/null
+++ b/evals/coding-agent/graders/ai-gateway-input-limits.test.ts
@@ -0,0 +1,93 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { describe, expect, it } from "bun:test";
+import {
+ boundedModelChain,
+ clientPayloadMessage,
+ isTransient,
+ isUserInputTooLarge,
+ selectCascadeError,
+} from "../handlers/chat";
+
+const configuredLimit =
+ "Input tokens exceed the configured limit of 100 tokens. Your messages resulted in 101 tokens.";
+const availableContext =
+ "request (201 tokens) exceeds the available context size (200 tokens), try increasing it";
+const toolCallLimit =
+ "Invalid 'messages[0].tool_calls': array too long. Expected an array with maximum length 2, but got an array with length 3 instead.";
+
+describe("gateway input error outcomes", () => {
+ it("recognizes configured input limits on supported client statuses", () => {
+ for (const status of [400, 413]) {
+ expect(isUserInputTooLarge(status, configuredLimit)).toBe(true);
+ expect(isUserInputTooLarge(status, configuredLimit.toUpperCase())).toBe(true);
+ }
+ });
+
+ it("recognizes an exceeded available context size", () => {
+ expect(isUserInputTooLarge(400, availableContext)).toBe(true);
+ });
+
+ it("gives an actionable client message for excess tool calls", () => {
+ for (const status of [400, 422]) {
+ const message = clientPayloadMessage(status, toolCallLimit);
+ expect(message).toMatch(/too many tool calls/i);
+ expect(message).toMatch(/compact|new chat/i);
+ expect(message).not.toContain("messages[0]");
+ }
+ });
+
+ it("preserves previously supported input-size errors", () => {
+ for (const message of [
+ "prompt is too long: 101 tokens > 100 maximum",
+ "maximum context length exceeded",
+ "The input (101 tokens) is longer than the model's context length (100 tokens).",
+ "The input token count (101) exceeds the maximum number of tokens allowed (100).",
+ ]) expect(isUserInputTooLarge(400, message)).toBe(true);
+ });
+
+ it("does not relabel unrelated errors or unsupported status boundaries", () => {
+ expect(isUserInputTooLarge(400, "invalid tool schema")).toBe(false);
+ expect(isUserInputTooLarge(400, "")).toBe(false);
+ for (const status of [0, 200, 401, 403, 429, 500]) {
+ expect(isUserInputTooLarge(status, configuredLimit)).toBe(false);
+ expect(isUserInputTooLarge(status, availableContext)).toBe(false);
+ expect(clientPayloadMessage(status, toolCallLimit)).toBeNull();
+ }
+ });
+
+ it("keeps near-neighbor non-tool arrays and thin errors unclassified", () => {
+ for (const message of [
+ toolCallLimit.replace("tool_calls", "attachments"),
+ "tool_calls: invalid schema",
+ "array too long",
+ "",
+ ]) expect(clientPayloadMessage(400, message)).toBeNull();
+ });
+
+ it("preserves image and empty-message guidance", () => {
+ expect(clientPayloadMessage(400, "Failed to decode image data.")).toMatch(/image/i);
+ expect(clientPayloadMessage(400, "at least one message is required")).toMatch(/message/i);
+ expect(clientPayloadMessage(400, "invalid tool schema")).toBeNull();
+ });
+
+ it("preserves ordinary transient failures and non-transient client errors", () => {
+ for (const status of [403, 404, 408, 429, 500, 503, 599]) {
+ expect(isTransient(status, "upstream unavailable")).toBe(true);
+ }
+ expect(isTransient(400, "bad request")).toBe(false);
+ expect(isTransient(401, "unauthorized")).toBe(false);
+ });
+
+ it("preserves terminal safety outcomes over earlier capacity advisories", () => {
+ const capacity = Object.assign(new Error("provider capacity"), { userMessage: "try later" });
+ const terminal = Object.assign(new Error("request declined"), { code: "safety_refusal" });
+ expect(selectCascadeError(capacity, terminal)).toBe(terminal);
+ });
+
+ it("keeps model attempts bounded and ordered", () => {
+ expect(boundedModelChain(["first-model", "second-model", "third-model"], 2))
+ .toEqual(["first-model", "second-model"]);
+ });
+});