YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover gateway input-limit classification
Louis Beaumont53049736dd78
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex a9452d7d3..f87e4c52d 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 38 git-mined regressions. See+The current app corpus contains 39 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest contributes 20 more.diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex bd5ee7115..fd5e434a3 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-08.1",+ "dataset_version": "2026-09-09.1","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1145,6 +1145,52 @@],"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts components/settings/referral-card.test.tsx lib/auth-guard.test.tsx"}+ },+ {+ "id": "ai-gateway-input-limit-classification",+ "name": "Recognize current provider input and tool-call limits",+ "source": {+ "kind": "git_regression",+ "fix_commit": "739090783bb1116dca26621522e8034134e65028",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/6945"+ },+ "base_ref": "655dfc3ad50fedfa9ba9561ebdf3a1467bc55517",+ "oracle_ref": "739090783bb1116dca26621522e8034134e65028",+ "oracle_paths": [+ "packages/ai-gateway/src/handlers/chat.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/ai-gateway/src/handlers/chat.ts",+ "packages/ai-gateway/src/test/chat-fallback.unit.test.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "ai-gateway",+ "input-limit",+ "tool-call",+ "error-classification",+ "false-positive"+ ],+ "prompt": "The gateway fails to recognize provider errors saying input tokens exceed the configured limit or a request exceeds the available context size. It also misses an excess-length tool-call array, leaving the caller without useful payload guidance. Recognize these input-size and tool-call-limit failures on their supported client-error statuses, and provide actionable tool-call guidance without leaking the raw nested error. Preserve older input-limit phrasings, image and empty-message guidance, transient status handling, terminal safety outcomes, and bounded ordered fallback attempts. Do not classify unrelated schemas, non-tool arrays, thin messages, authentication errors, or server failures as these client payload problems.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "packages/ai-gateway/node_modules",+ "destination_path": "packages/ai-gateway/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 300,+ "fixtures": [+ {+ "local_path": "graders/ai-gateway-input-limits.test.ts",+ "destination_path": "packages/ai-gateway/src/test/eval-input-limits.test.ts"+ }+ ],+ "command": "cd packages/ai-gateway && bun test src/test/eval-input-limits.test.ts --timeout=10000"+ }}]}diff --git a/evals/coding-agent/graders/ai-gateway-input-limits.test.ts b/evals/coding-agent/graders/ai-gateway-input-limits.test.tsnew file mode 100644index 000000000..dfc7d8f1c--- /dev/null+++ b/evals/coding-agent/graders/ai-gateway-input-limits.test.ts@@ -0,0 +1,93 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { describe, expect, it } from "bun:test";+import {+ boundedModelChain,+ clientPayloadMessage,+ isTransient,+ isUserInputTooLarge,+ selectCascadeError,+} from "../handlers/chat";++const configuredLimit =+ "Input tokens exceed the configured limit of 100 tokens. Your messages resulted in 101 tokens.";+const availableContext =+ "request (201 tokens) exceeds the available context size (200 tokens), try increasing it";+const toolCallLimit =+ "Invalid 'messages[0].tool_calls': array too long. Expected an array with maximum length 2, but got an array with length 3 instead.";++describe("gateway input error outcomes", () => {+ it("recognizes configured input limits on supported client statuses", () => {+ for (const status of [400, 413]) {+ expect(isUserInputTooLarge(status, configuredLimit)).toBe(true);+ expect(isUserInputTooLarge(status, configuredLimit.toUpperCase())).toBe(true);+ }+ });++ it("recognizes an exceeded available context size", () => {+ expect(isUserInputTooLarge(400, availableContext)).toBe(true);+ });++ it("gives an actionable client message for excess tool calls", () => {+ for (const status of [400, 422]) {+ const message = clientPayloadMessage(status, toolCallLimit);+ expect(message).toMatch(/too many tool calls/i);+ expect(message).toMatch(/compact|new chat/i);+ expect(message).not.toContain("messages[0]");+ }+ });++ it("preserves previously supported input-size errors", () => {+ for (const message of [+ "prompt is too long: 101 tokens > 100 maximum",+ "maximum context length exceeded",+ "The input (101 tokens) is longer than the model's context length (100 tokens).",+ "The input token count (101) exceeds the maximum number of tokens allowed (100).",+ ]) expect(isUserInputTooLarge(400, message)).toBe(true);+ });++ it("does not relabel unrelated errors or unsupported status boundaries", () => {+ expect(isUserInputTooLarge(400, "invalid tool schema")).toBe(false);+ expect(isUserInputTooLarge(400, "")).toBe(false);+ for (const status of [0, 200, 401, 403, 429, 500]) {+ expect(isUserInputTooLarge(status, configuredLimit)).toBe(false);+ expect(isUserInputTooLarge(status, availableContext)).toBe(false);+ expect(clientPayloadMessage(status, toolCallLimit)).toBeNull();+ }+ });++ it("keeps near-neighbor non-tool arrays and thin errors unclassified", () => {+ for (const message of [+ toolCallLimit.replace("tool_calls", "attachments"),+ "tool_calls: invalid schema",+ "array too long",+ "",+ ]) expect(clientPayloadMessage(400, message)).toBeNull();+ });++ it("preserves image and empty-message guidance", () => {+ expect(clientPayloadMessage(400, "Failed to decode image data.")).toMatch(/image/i);+ expect(clientPayloadMessage(400, "at least one message is required")).toMatch(/message/i);+ expect(clientPayloadMessage(400, "invalid tool schema")).toBeNull();+ });++ it("preserves ordinary transient failures and non-transient client errors", () => {+ for (const status of [403, 404, 408, 429, 500, 503, 599]) {+ expect(isTransient(status, "upstream unavailable")).toBe(true);+ }+ expect(isTransient(400, "bad request")).toBe(false);+ expect(isTransient(401, "unauthorized")).toBe(false);+ });++ it("preserves terminal safety outcomes over earlier capacity advisories", () => {+ const capacity = Object.assign(new Error("provider capacity"), { userMessage: "try later" });+ const terminal = Object.assign(new Error("request declined"), { code: "safety_refusal" });+ expect(selectCascadeError(capacity, terminal)).toBe(terminal);+ });++ it("keeps model attempts bounded and ordered", () => {+ expect(boundedModelChain(["first-model", "second-model", "third-model"], 2))+ .toEqual(["first-model", "second-model"]);+ });+});