Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: report context overflow the way Anthropic does, so Claude Code can compact (#10090)

Nilay6b06e31aebbd
diff --git a/studio/backend/core/inference/context_refusal.py b/studio/backend/core/inference/context_refusal.py
index 77ac3a8dbb9..ea46fea591c 100644
--- a/studio/backend/core/inference/context_refusal.py
+++ b/studio/backend/core/inference/context_refusal.py
@@ -221,18 +221,14 @@ _ROLE_ADVICE["function"] = _ROLE_ADVICE["tool"]
_ROLE_ADVICE["developer"] = _ROLE_ADVICE["system"]
-def describe_oversize(request_tokens: int, context_tokens: int) -> str:
- """The user-facing message for a prompt that exceeds the loaded context window.
+def oversize_advice(context_tokens: int) -> str:
+ """The remedy half of an oversize refusal: what the user can actually do.
- The advice splits on the only two things that change what the user can do: whose
- turn is the bulk of the prompt, and whether that turn is merely most of the prompt
- or actually too big to send at all. An unrecognised role falls back to the generic
- wording rather than blaming a turn it cannot describe.
+ Split out from :func:`describe_oversize` so a surface that must keep its own head
+ wording -- the Anthropic passthrough sends Anthropic's "Prompt is too long: N
+ tokens > M maximum", which is what its clients key on -- can still pair it with
+ this diagnosis instead of prescribing compaction for a prompt no compaction fits.
"""
- head = (
- f"Message too long: {request_tokens} tokens exceeds the "
- f"{context_tokens}-token context window. "
- )
blamed = _blame_latest_turn(context_tokens)
advice = _ROLE_ADVICE.get(blamed[0]) if blamed else None
if advice is None:
@@ -243,25 +239,36 @@ def describe_oversize(request_tokens: int, context_tokens: int) -> str:
# say which of them it is -- `shared_prompt_tokens` bundles the template wrapper with the catalogue, so a
# large one does not prove there are tools. Both levers are offered, and neither is claimed to be the cause.
return (
- head + "Even with every earlier turn dropped, this prompt would still be "
+ "Even with every earlier turn dropped, this prompt would still be "
"too long, so shortening the conversation will not help. Increase the "
"Context Length in Model settings, or reduce what every request carries: "
"the system prompt and any tools that are enabled."
)
- return (
- head + "Try increasing the Context Length in Model settings, or shorten the "
- "conversation."
- )
+ return "Try increasing the Context Length in Model settings, or shorten the conversation."
dominant_cause, oversize_cause, lever = advice
fits_alone = blamed[1]
cause = dominant_cause if fits_alone else oversize_cause
hedge = "will not help much" if fits_alone else "will not help"
return (
- f"{head}{cause}, so shortening the conversation {hedge}. Increase the Context "
+ f"{cause}, so shortening the conversation {hedge}. Increase the Context "
f"Length in Model settings, or {lever}."
)
+def describe_oversize(request_tokens: int, context_tokens: int) -> str:
+ """The user-facing message for a prompt that exceeds the loaded context window.
+
+ The advice splits on the only two things that change what the user can do: whose
+ turn is the bulk of the prompt, and whether that turn is merely most of the prompt
+ or actually too big to send at all. An unrecognised role falls back to the generic
+ wording rather than blaming a turn it cannot describe.
+ """
+ return (
+ f"Message too long: {request_tokens} tokens exceeds the "
+ f"{context_tokens}-token context window. "
+ ) + oversize_advice(context_tokens)
+
+
# anything absent gets the neutral line: an MCP tool's payload is not a file and not a program
# What the user can actually shorten, per tool. Anything absent gets the neutral line: an MCP tool's payload is not a
# file and not a program, and guessing at it is worse than saying the one thing that is true of every tool.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index f32eced8e68..0b6924e61c8 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -72,7 +72,11 @@ from core.inference.memory_contract import (
build_memory_estimate,
project_estimate_memory_response,
)
-from core.inference.stream_errors import LlamaStreamError
+from core.inference.stream_errors import (
+ KV_STARVATION_MESSAGE,
+ LlamaStreamError,
+ is_kv_starvation,
+)
from core.inference.orchestrator import (
AUDIO_GENERATION_MAX_TOKENS,
GenStreamError,
@@ -246,6 +250,11 @@ _LOST_CONNECTION_MSG = (
)
+_OVERSIZE_TOKENS_RE = _re.compile(
+ r"request \((\d+) tokens?\) exceeds the available context size \((\d+) tokens?\)"
+)
+
+
def _friendly_error(exc: Exception) -> str:
"""Extract a user-friendly message from known llama-server errors."""
if isinstance(exc, httpx.ReadTimeout):
@@ -268,10 +277,7 @@ def _friendly_error(exc: Exception) -> str:
if isinstance(exc, httpx.RequestError):
return _LOST_CONNECTION_MSG
msg = str(exc)
- m = _re.search(
- r"request \((\d+) tokens?\) exceeds the available context size \((\d+) tokens?\)",
- msg,
- )
+ m = _OVERSIZE_TOKENS_RE.search(msg)
if m:
# llama-server knows only the prompt total, so its "shorten the conversation" is
# wrong for a single oversized turn: let the fit's diagnosis pick the wording.
@@ -317,6 +323,46 @@ def _friendly_upstream_error(text: str) -> str:
return f"llama-server error: {text}"
+def _oversize_counts(text: str):
+ """(prompt, context) for an oversize refusal, or None.
+
+ The prose counts first, then the structured `n_prompt_tokens`/`n_ctx` that an
+ `exceed_context_size_error` body carries: these paths are handed the whole body, so
+ when the message itself has no numbers the fields in it are still the real totals.
+ """
+ m = _OVERSIZE_TOKENS_RE.search(text)
+ if m:
+ return int(m.group(1)), int(m.group(2))
+ return _parse_overflow_counts(text)
+
+
+def _anthropic_upstream_error(text: str, counts_source: Optional[str] = None) -> str:
+ """`text` is what the client is told; `counts_source` is the fuller body it came
+ from, used only to recover token counts the message itself does not spell out."""
+ # Starvation is a shared-cache capacity failure, not an oversized prompt: the right
+ # response is to retry, so it must not be reworded into "shorten the conversation".
+ if is_kv_starvation(text):
+ return KV_STARVATION_MESSAGE
+ if _classify_llama_generation_error(Exception(text)):
+ counts = _oversize_counts(text)
+ if counts is None and counts_source:
+ counts = _oversize_counts(counts_source)
+ if counts:
+ # Anthropic's own head wording, which its clients key on to compact, paired
+ # with the fit's remedy rather than a flat "shorten the conversation": when
+ # the latest turn or the system prompt is what does not fit, compacting the
+ # history cannot help and the client would just retry it.
+ return (
+ f"Prompt is too long: {counts[0]} tokens > {counts[1]} maximum. "
+ + context_refusal.oversize_advice(counts[1])
+ )
+ return (
+ "Prompt is too long. Try increasing the Context Length in Model settings, "
+ "or shorten the conversation."
+ )
+ return _friendly_upstream_error(text)
+
+
def _clamp_finish_reason(value) -> str:
"""Coerce an upstream finish_reason into OpenAI's known chat values.
@@ -1029,6 +1075,35 @@ def _anthropic_stream_error_event(exc, *, force: bool = False):
)
+def _json_dumps_safe(value) -> Optional[str]:
+ """`json.dumps` for count recovery only; never raises on an odd payload."""
+ try:
+ return json.dumps(value)
+ except (TypeError, ValueError):
+ return None
+
+
+def _anthropic_upstream_stream_error_event(text: str, counts_source: Optional[str] = None):
+ """In-band Anthropic error event for a 200 stream that later reports an upstream failure.
+
+ Same wording and status rule as the non-200 branch. Routing the raw body through
+ `_friendly_error` instead flattens the count-less oversize refusal ("the request
+ exceeds the available context size") to "An internal error occurred", so a streaming
+ client is told 400/invalid_request_error with nothing it can read or compact on.
+ """
+ from core.inference.llama_keepwarm import mark_current_response_failed
+
+ mark_current_response_failed()
+ over = bool(_classify_llama_generation_error(Exception(text)))
+ return build_anthropic_sse_event(
+ "error",
+ anthropic_error_body(
+ _anthropic_upstream_error(text, counts_source = counts_source),
+ status = 400 if over else 500,
+ ),
+ )
+
+
def _drop_parallel_tool_call_deltas(chunk) -> bool:
"""In-place: drop tool_call deltas whose index >= 1 from a parsed OpenAI
streaming chunk so only the first tool call survives (parallel_tool_calls=false
@@ -1454,6 +1529,11 @@ def _classify_llama_generation_error(exc: Exception) -> Optional[bool]:
return True if exc.context_oversize else None
msg = str(exc)
msg_l = msg.lower()
+ # Same reasoning as the typed branch, which only reaches errors carrying the parsed
+ # flag. Raw upstream bodies arrive here as a plain Exception, and "Context size has
+ # been exceeded" trips the substring test below while meaning the opposite.
+ if is_kv_starvation(msg):
+ return None
if "n_ctx" in msg_l or (
"context" in msg_l and any(t in msg_l for t in ("exceed", "length", "window", "too long"))
):
@@ -29966,11 +30046,12 @@ async def _anthropic_passthrough_stream(
from core.inference.llama_keepwarm import mark_response_failed
mark_response_failed(getattr(request, "scope", None))
+ _over = bool(_classify_llama_generation_error(Exception(_err_text)))
yield build_anthropic_sse_event(
"error",
anthropic_error_body(
- _friendly_upstream_error(_err_text),
- status = resp.status_code,
+ _anthropic_upstream_error(_err_text),
+ status = 400 if _over else resp.status_code,
),
)
return
@@ -30009,12 +30090,13 @@ async def _anthropic_passthrough_stream(
from core.inference.llama_keepwarm import mark_response_failed
mark_response_failed(getattr(request, "scope", None))
- event = _anthropic_stream_error_event(
- RuntimeError(error_message),
- force = True,
+ # The message alone can be count-less while the chunk still
+ # carries n_prompt_tokens/n_ctx; keep the clean message and let
+ # the formatter recover the totals from the payload.
+ yield _anthropic_upstream_stream_error_event(
+ error_message,
+ counts_source = _json_dumps_safe(chunk.get("error", chunk)),
)
- if event is not None:
- yield event
return
if disable_parallel_tool_use:
_drop_parallel_tool_call_deltas(chunk)
@@ -30176,9 +30258,11 @@ async def _anthropic_passthrough_non_streaming(
resp = await _post(body)
if resp.status_code != 200:
+ _err_text = resp.text[:500]
+ _over = bool(_classify_llama_generation_error(Exception(_err_text)))
raise HTTPException(
- status_code = resp.status_code,
- detail = _friendly_upstream_error(resp.text[:500]),
+ status_code = 400 if _over else resp.status_code,
+ detail = _anthropic_upstream_error(_err_text),
)
data = resp.json()
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index e626270eb13..fe7d0a0292c 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -4,6 +4,7 @@
"""Tests for the OpenAI /v1/chat/completions client-side tool pass-through."""
import os
+import re
import sys
import asyncio
import base64
@@ -161,6 +162,201 @@ class TestFriendlyUpstreamError:
# An unrelated upstream error still passes through verbatim.
assert "llama-server error:" in _openai_passthrough_error(500, "disk full").detail
+ def test_anthropic_upstream_error_rewords_context_overflow(self):
+ from routes.inference import _anthropic_upstream_error
+
+ raw = (
+ '{"error":{"code":500,"message":"the request exceeds the available context '
+ 'size. try increasing the context size or enable context shift",'
+ '"type":"server_error"}}'
+ )
+ msg = _anthropic_upstream_error(raw)
+ assert "prompt is too long" in msg.lower()
+ assert "llama-server error:" not in msg
+
+ def test_anthropic_upstream_error_includes_token_counts_when_known(self):
+ from routes.inference import _anthropic_upstream_error
+
+ msg = _anthropic_upstream_error(
+ "the request (214331 tokens) exceeds the available context size (131072 tokens)"
+ )
+ m = re.search(r"prompt is too long[^0-9]*(\d+)\s*tokens?\s*>\s*(\d+)", msg, re.I)
+ assert m and m.groups() == ("214331", "131072")
+
+ def test_anthropic_upstream_error_leaves_other_failures_alone(self):
+ from routes.inference import _anthropic_upstream_error
+ assert _anthropic_upstream_error("disk full") == "llama-server error: disk full"
+ assert "compile a grammar" in _anthropic_upstream_error("failed to parse grammar")
+
+ def test_anthropic_upstream_error_keeps_kv_starvation_out_of_the_overflow_rewrite(self):
+ """llama-server says "Context size has been exceeded" when concurrent
+ generations drain the shared KV cache. Nothing about the request was too
+ long, so rewording it as an oversized prompt sends the client compacting a
+ valid conversation instead of retrying."""
+ from routes.inference import _anthropic_upstream_error, _classify_llama_generation_error
+ from core.inference.stream_errors import KV_STARVATION_MESSAGE
+
+ for raw in (
+ "Context size has been exceeded.",
+ '{"error":{"message":"Context size has been exceeded.","code":500}}',
+ ):
+ assert _anthropic_upstream_error(raw) == KV_STARVATION_MESSAGE
+ assert "too long" not in _anthropic_upstream_error(raw).lower()
+ # None keeps the upstream status: a 400 would blame the caller's request.
+ assert _classify_llama_generation_error(Exception(raw)) is None
+
+ def test_in_band_sse_recovers_counts_from_the_chunk_the_message_came_from(self):
+ """`_monitor_openai_error_message` returns only the message string. When that
+ string carries no numbers but the chunk around it does, the totals -- and the
+ window `oversize_advice` needs -- have to come from the payload."""
+ import json as _json
+
+ from core.inference import context_refusal
+ from routes.inference import (
+ _anthropic_upstream_stream_error_event,
+ _json_dumps_safe,
+ _monitor_openai_error_message,
+ )
+
+ chunk = {
+ "error": {
+ "code": 400,
+ "message": "the request exceeds the available context size",
+ "type": "exceed_context_size_error",
+ "n_prompt_tokens": 70494,
+ "n_ctx": 67584,
+ }
+ }
+ message = _monitor_openai_error_message(chunk)
+ source = _json_dumps_safe(chunk.get("error", chunk))
+
+ def body(text, counts_source):
+ event = _anthropic_upstream_stream_error_event(text, counts_source = counts_source)
+ return _json.loads(event.split("data: ", 1)[1].strip().splitlines()[0])["error"]
+
+ assert "70494" not in body(message, None)["message"]
+ assert (
+ "Prompt is too long: 70494 tokens > 67584 maximum" in body(message, source)["message"]
+ )
+
+ try:
+ context_refusal.record_fit(
+ {
+ "fits": False,
+ "context_length": 67584,
+ "irreducible_tokens": 70000,
+ "latest_turn_tokens": 500,
+ }
+ )
+ # The recovered window is what lets the fit be consulted at all.
+ assert "shortening the conversation will not help" in body(message, source)["message"]
+ finally:
+ context_refusal.clear()
+
+ # An unrelated error must not have the payload spliced into its text.
+ assert body("disk full", source)["message"] == "llama-server error: disk full"
+
+ def test_counts_come_from_the_structured_fields_when_the_message_has_none(self):
+ """An exceed_context_size_error body carries n_prompt_tokens/n_ctx even when its
+ message spells out no numbers. These paths are handed the whole body, so the
+ totals are right there; dropping them sends the client a count-less refusal."""
+ from routes.inference import _anthropic_upstream_error, _oversize_counts
+
+ body = (
+ '{"error":{"code":400,"message":"the request exceeds the available context '
+ 'size","type":"exceed_context_size_error","n_prompt_tokens":70494,'
+ '"n_ctx":67584}}'
+ )
+ assert _oversize_counts(body) == (70494, 67584)
+ assert "Prompt is too long: 70494 tokens > 67584 maximum" in _anthropic_upstream_error(body)
+
+ def test_the_remedy_comes_from_the_fit_not_a_flat_shorten_the_conversation(self):
+ """When the latest turn or the irreducible floor is what does not fit, compacting
+ the history cannot make it fit, and a client told to compact just retries."""
+ from core.inference import context_refusal
+ from routes.inference import _anthropic_upstream_error
+
+ body = "the request (214331 tokens) exceeds the available context size (131072 tokens)"
+ try:
+ context_refusal.record_fit(
+ {
+ "fits": False,
+ "context_length": 131072,
+ "irreducible_tokens": 140000,
+ "latest_turn_tokens": 500,
+ }
+ )
+ msg = _anthropic_upstream_error(body)
+ # The head Anthropic clients key on survives.
+ assert msg.startswith("Prompt is too long: 214331 tokens > 131072 maximum.")
+ assert "shortening the conversation will not help" in msg
+
+ context_refusal.record_fit(
+ {
+ "fits": False,
+ "context_length": 131072,
+ "irreducible_tokens": 140000,
+ "latest_turn_tokens": 139000,
+ "latest_turn_role": "tool",
+ "latest_turn_counted": True,
+ }
+ )
+ msg = _anthropic_upstream_error(body)
+ assert "A tool returned more than this context window can hold" in msg
+ assert "shortening the conversation will not help" in msg
+ finally:
+ context_refusal.clear()
+
+ # With no recorded fit it stays the generic wording.
+ assert "Try increasing the Context Length" in _anthropic_upstream_error(body)
+
+ def test_in_band_sse_error_gets_the_same_wording_as_the_non_200_branch(self):
+ """A 200 stream that later emits data: {"error": ...} used to be wrapped in a
+ plain RuntimeError, and _friendly_error flattens the count-less oversize
+ refusal to "An internal error occurred" -- a 400/invalid_request_error with
+ nothing the client can read or compact on."""
+ import json as _json
+
+ from routes.inference import _anthropic_upstream_stream_error_event
+
+ def body(text):
+ event = _anthropic_upstream_stream_error_event(text)
+ return _json.loads(event.split("data: ", 1)[1].strip().splitlines()[0])["error"]
+
+ no_counts = body(
+ "the request exceeds the available context size. try increasing the context size"
+ )
+ assert "internal error" not in no_counts["message"].lower()
+ assert no_counts["message"].startswith("Prompt is too long")
+ assert no_counts["type"] == "invalid_request_error"
+
+ counted = body(
+ "the request (214331 tokens) exceeds the available context size (131072 tokens)"
+ )
+ assert "Prompt is too long: 214331 tokens > 131072 maximum" in counted["message"]
+ assert counted["type"] == "invalid_request_error"
+
+ # Starvation keeps the retry advice and must not become a 400.
+ starved = body("Context size has been exceeded.")
+ assert "shared pool of context" in starved["message"]
+ assert "too long" not in starved["message"].lower()
+ assert starved["type"] == "api_error"
+
+ unrelated = body("disk full")
+ assert unrelated["message"] == "llama-server error: disk full"
+ assert unrelated["type"] == "api_error"
+
+ def test_a_genuine_oversize_body_is_still_classified_as_an_overflow(self):
+ from routes.inference import _classify_llama_generation_error
+ assert (
+ _classify_llama_generation_error(
+ Exception(
+ "the request (214331 tokens) exceeds the available context size (131072 tokens)"
+ )
+ )
+ is True
+ )
+
# =====================================================================
# ChatMessage — tool role, tool_calls, optional content