A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
recognize litellm maximum-context rejections (#2129)
kt55ade48b73f6
diff --git a/packages/ai/.changes/eng-6011-litellm-context-overflow.md b/packages/ai/.changes/eng-6011-litellm-context-overflow.mdnew file mode 100644index 0000000000..9dd157ec0f--- /dev/null+++ b/packages/ai/.changes/eng-6011-litellm-context-overflow.md@@ -0,0 +1 @@+- Fixed detection of LiteLLM context-limit errors when input plus requested output exceed the model's context window.diff --git a/packages/ai/src/utils/overflow.ts b/packages/ai/src/utils/overflow.tsindex bf59405d82..8913dfb187 100644--- a/packages/ai/src/utils/overflow.ts+++ b/packages/ai/src/utils/overflow.ts@@ -15,6 +15,7 @@ import type { AssistantMessage } from "../types.js";* - xAI: "This model's maximum prompt length is 131072 but the request contains 537812 tokens"* - Groq: "Please reduce the length of the messages or completion"* - OpenRouter: "This endpoint's maximum context length is X tokens. However, you requested about Y tokens"+ * - LiteLLM: "Requested token count exceeds the model's maximum context length of X tokens"* - llama.cpp: "the request exceeds the available context size, try increasing it"* - LM Studio: "tokens to keep from the initial prompt is greater than the context length"* - GitHub Copilot: "prompt token count of X exceeds the limit of Y"@@ -37,6 +38,7 @@ const OVERFLOW_PATTERNS = [/maximum prompt length is \d+/i, // xAI (Grok)/reduce the length of the messages/i, // Groq/maximum context length is \d+ tokens/i, // OpenRouter (all backends)+ /exceeds the model's maximum context length/i, // LiteLLM (input + requested output)/exceeds the limit of \d+/i, // GitHub Copilot/exceeds the available context size/i, // llama.cpp server/greater than the context length/i, // LM Studio@@ -86,6 +88,7 @@ const NON_OVERFLOW_PATTERNS = [* - Cerebras: 400/413 status code (no body)* - Mistral: "Prompt contains X tokens ... too large for model with Y maximum context length"* - OpenRouter (all backends): "maximum context length is X tokens"+ * - LiteLLM: "exceeds the model's maximum context length of X tokens"* - llama.cpp: "exceeds the available context size"* - LM Studio: "greater than the context length"* - Kimi For Coding: "exceeded model token limit: X (requested: Y)"diff --git a/packages/ai/test/overflow.test.ts b/packages/ai/test/overflow.test.tsindex 3343e730b3..30d7d494d4 100644--- a/packages/ai/test/overflow.test.ts+++ b/packages/ai/test/overflow.test.ts@@ -30,6 +30,33 @@ function createErrorMessage(errorMessage: string): AssistantMessage {}describe("isContextOverflow", () => {+ const litellmError =+ "400 litellm.BadRequestError: OpenAIException - Requested token count exceeds the model's maximum context length of 262144 tokens. You requested a total of 270128 tokens: 261936 tokens from the input messages and 8192 tokens for the completion. Please reduce the number of tokens in the input messages or the completion to fit within the limit.. Received Model Group=qwen3.8-27b-nvfp4";++ it.each([262144, undefined])("detects LiteLLM context rejection with contextWindow=%s", (contextWindow) => {+ expect(isContextOverflow(createErrorMessage(litellmError), contextWindow)).toBe(true);+ });++ it("detects context rejection without a proxy prefix, regardless of case", () => {+ const message = createErrorMessage(+ "REQUESTED TOKEN COUNT EXCEEDS THE MODEL'S MAXIMUM CONTEXT LENGTH OF 262144 TOKENS.",+ );+ expect(isContextOverflow(message)).toBe(true);+ });++ it.each([+ "429 rate limit: too many tokens",+ "Requested token count exceeds the per-minute token quota.",+ "400 litellm.BadRequestError: unsupported parameter: temperature",+ `429 rate limit: upstream previously returned ${litellmError}`,+ ])("does not classify an unrelated or rate-limited rejection as overflow: %s", (errorMessage) => {+ expect(isContextOverflow(createErrorMessage(errorMessage), 262144)).toBe(false);+ });++ it.each(["stop", "aborted"] as const)("ignores context error text with stopReason=%s", (stopReason) => {+ expect(isContextOverflow({ ...createErrorMessage(litellmError), stopReason }, 262144)).toBe(false);+ });+it("detects explicit Ollama prompt-too-long errors", () => {const message = createErrorMessage("400 `prompt too long; exceeded max context length by 100918 tokens`");expect(isContextOverflow(message, 32768)).toBe(true);diff --git a/packages/coding-agent/.changes/eng-6011-litellm-context-overflow.md b/packages/coding-agent/.changes/eng-6011-litellm-context-overflow.mdnew file mode 100644index 0000000000..45e4eb1252--- /dev/null+++ b/packages/coding-agent/.changes/eng-6011-litellm-context-overflow.md@@ -0,0 +1 @@+- Fixed automatic compaction and recovery for LiteLLM maximum-context rejections.diff --git a/packages/coding-agent/test/suite/regressions/6011-litellm-context-overflow.test.ts b/packages/coding-agent/test/suite/regressions/6011-litellm-context-overflow.test.tsnew file mode 100644index 0000000000..51dd11ac67--- /dev/null+++ b/packages/coding-agent/test/suite/regressions/6011-litellm-context-overflow.test.ts@@ -0,0 +1,85 @@+import { type Context, fauxAssistantMessage } from "@earendil-works/pi-ai";+import { afterEach, describe, expect, it, vi } from "vitest";+import { createHarness, getMessageText, type Harness } from "../harness.js";++const litellmError =+ "400 litellm.BadRequestError: OpenAIException - Requested token count exceeds the model's maximum context length of 262144 tokens. You requested a total of 270128 tokens: 261936 tokens from the input messages and 8192 tokens for the completion. Please reduce the number of tokens in the input messages or the completion to fit within the limit.. Received Model Group=qwen3.8-27b-nvfp4";++describe("LiteLLM context overflow recovery", () => {+ const harnesses: Harness[] = [];++ afterEach(() => {+ while (harnesses.length > 0) harnesses.pop()?.cleanup();+ });++ async function setup() {+ const harness = await createHarness({+ models: [{ id: "litellm-fixture", contextWindow: 262144, maxTokens: 8192 }],+ settings: {+ autoRefine: { enabled: false },+ compaction: { enabled: true, reserveTokens: 8192, keepRecentTokens: 50 },+ retry: { enabled: true, maxRetries: 1, baseDelayMs: 1 },+ },+ });+ harnesses.push(harness);+ harness.setResponses([fauxAssistantMessage("Earlier findings recorded.")]);+ await harness.session.prompt("Earlier context. ".repeat(100));+ return harness;+ }++ it.each([+ ["LiteLLM rejection", litellmError],+ ["known overflow control", "prompt is too long: 270128 tokens > 262144 maximum"],+ ])("compacts and continues after %s without retrying the unchanged request", async (_name, errorMessage) => {+ const harness = await setup();+ const requests: Context[] = [];+ harness.setResponses([+ fauxAssistantMessage("", { stopReason: "error", errorMessage }),+ (context) => {+ requests.push({ systemPrompt: context.systemPrompt, messages: structuredClone(context.messages) });+ return fauxAssistantMessage("Summary of earlier findings.");+ },+ (context) => {+ requests.push({ systemPrompt: context.systemPrompt, messages: structuredClone(context.messages) });+ return fauxAssistantMessage("Recovered.");+ },+ ]);++ await harness.session.prompt("Continue the task using the earlier findings. ".repeat(8));++ expect(harness.eventsOfType("compaction_start").map((event) => event.reason)).toEqual(["overflow"]);+ expect(harness.eventsOfType("compaction_end")).toEqual([+ expect.objectContaining({ reason: "overflow", aborted: false, willRetry: true, result: expect.any(Object) }),+ ]);+ await vi.waitFor(() => {+ expect(harness.session.messages.at(-1)).toEqual(+ expect.objectContaining({ content: [{ type: "text", text: "Recovered." }], stopReason: "stop" }),+ );+ });+ expect(harness.eventsOfType("auto_retry_start")).toEqual([]);+ expect(requests).toHaveLength(2);+ expect(getMessageText(requests[0].messages[0])).toContain("<conversation>");+ expect(requests[1].messages.map(getMessageText).join("\n")).toContain("Summary of earlier findings.");+ expect(+ requests[1].messages.some((message) => message.role === "assistant" && message.stopReason === "error"),+ ).toBe(false);+ expect(harness.sessionManager.getBranch().filter((entry) => entry.type === "compaction")).toHaveLength(1);+ expect(harness.faux.state.callCount).toBe(4);+ expect(harness.getPendingResponseCount()).toBe(0);+ });++ it("retries a token-rate-limit rejection without compacting", async () => {+ const harness = await setup();+ harness.setResponses([+ fauxAssistantMessage("", { stopReason: "error", errorMessage: "429 rate limit: too many tokens" }),+ fauxAssistantMessage("Recovered after rate limit."),+ ]);++ await harness.session.prompt("Continue the task.");++ expect(harness.eventsOfType("compaction_start")).toEqual([]);+ expect(harness.eventsOfType("auto_retry_start")).toHaveLength(1);+ expect(getMessageText(harness.session.messages.at(-1))).toBe("Recovered after rate limit.");+ expect(harness.faux.state.callCount).toBe(3);+ });+});