Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: report context overflow the way Anthropic does, so Claude Code can compact (#10090)
Nilay6b06e31aebbd
diff --git a/studio/backend/core/inference/context_refusal.py b/studio/backend/core/inference/context_refusal.pyindex 77ac3a8dbb9..ea46fea591c 100644--- a/studio/backend/core/inference/context_refusal.py+++ b/studio/backend/core/inference/context_refusal.py@@ -221,18 +221,14 @@ _ROLE_ADVICE["function"] = _ROLE_ADVICE["tool"]_ROLE_ADVICE["developer"] = _ROLE_ADVICE["system"]-def describe_oversize(request_tokens: int, context_tokens: int) -> str:- """The user-facing message for a prompt that exceeds the loaded context window.+def oversize_advice(context_tokens: int) -> str:+ """The remedy half of an oversize refusal: what the user can actually do.- The advice splits on the only two things that change what the user can do: whose- turn is the bulk of the prompt, and whether that turn is merely most of the prompt- or actually too big to send at all. An unrecognised role falls back to the generic- wording rather than blaming a turn it cannot describe.+ Split out from :func:`describe_oversize` so a surface that must keep its own head+ wording -- the Anthropic passthrough sends Anthropic's "Prompt is too long: N+ tokens > M maximum", which is what its clients key on -- can still pair it with+ this diagnosis instead of prescribing compaction for a prompt no compaction fits."""- head = (- f"Message too long: {request_tokens} tokens exceeds the "- f"{context_tokens}-token context window. "- )blamed = _blame_latest_turn(context_tokens)advice = _ROLE_ADVICE.get(blamed[0]) if blamed else Noneif advice is None:@@ -243,25 +239,36 @@ def describe_oversize(request_tokens: int, context_tokens: int) -> str:# say which of them it is -- `shared_prompt_tokens` bundles the template wrapper with the catalogue, so a# large one does not prove there are tools. Both levers are offered, and neither is claimed to be the cause.return (- head + "Even with every earlier turn dropped, this prompt would still be "+ "Even with every earlier turn dropped, this prompt would still be ""too long, so shortening the conversation will not help. Increase the ""Context Length in Model settings, or reduce what every request carries: ""the system prompt and any tools that are enabled.")- return (- head + "Try increasing the Context Length in Model settings, or shorten the "- "conversation."- )+ return "Try increasing the Context Length in Model settings, or shorten the conversation."dominant_cause, oversize_cause, lever = advicefits_alone = blamed[1]cause = dominant_cause if fits_alone else oversize_causehedge = "will not help much" if fits_alone else "will not help"return (- f"{head}{cause}, so shortening the conversation {hedge}. Increase the Context "+ f"{cause}, so shortening the conversation {hedge}. Increase the Context "f"Length in Model settings, or {lever}.")+def describe_oversize(request_tokens: int, context_tokens: int) -> str:+ """The user-facing message for a prompt that exceeds the loaded context window.++ The advice splits on the only two things that change what the user can do: whose+ turn is the bulk of the prompt, and whether that turn is merely most of the prompt+ or actually too big to send at all. An unrecognised role falls back to the generic+ wording rather than blaming a turn it cannot describe.+ """+ return (+ f"Message too long: {request_tokens} tokens exceeds the "+ f"{context_tokens}-token context window. "+ ) + oversize_advice(context_tokens)++# anything absent gets the neutral line: an MCP tool's payload is not a file and not a program# What the user can actually shorten, per tool. Anything absent gets the neutral line: an MCP tool's payload is not a# file and not a program, and guessing at it is worse than saying the one thing that is true of every tool.diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex f32eced8e68..0b6924e61c8 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -72,7 +72,11 @@ from core.inference.memory_contract import (build_memory_estimate,project_estimate_memory_response,)-from core.inference.stream_errors import LlamaStreamError+from core.inference.stream_errors import (+ KV_STARVATION_MESSAGE,+ LlamaStreamError,+ is_kv_starvation,+)from core.inference.orchestrator import (AUDIO_GENERATION_MAX_TOKENS,GenStreamError,@@ -246,6 +250,11 @@ _LOST_CONNECTION_MSG = ()+_OVERSIZE_TOKENS_RE = _re.compile(+ r"request \((\d+) tokens?\) exceeds the available context size \((\d+) tokens?\)"+)++def _friendly_error(exc: Exception) -> str:"""Extract a user-friendly message from known llama-server errors."""if isinstance(exc, httpx.ReadTimeout):@@ -268,10 +277,7 @@ def _friendly_error(exc: Exception) -> str:if isinstance(exc, httpx.RequestError):return _LOST_CONNECTION_MSGmsg = str(exc)- m = _re.search(- r"request \((\d+) tokens?\) exceeds the available context size \((\d+) tokens?\)",- msg,- )+ m = _OVERSIZE_TOKENS_RE.search(msg)if m:# llama-server knows only the prompt total, so its "shorten the conversation" is# wrong for a single oversized turn: let the fit's diagnosis pick the wording.@@ -317,6 +323,46 @@ def _friendly_upstream_error(text: str) -> str:return f"llama-server error: {text}"+def _oversize_counts(text: str):+ """(prompt, context) for an oversize refusal, or None.++ The prose counts first, then the structured `n_prompt_tokens`/`n_ctx` that an+ `exceed_context_size_error` body carries: these paths are handed the whole body, so+ when the message itself has no numbers the fields in it are still the real totals.+ """+ m = _OVERSIZE_TOKENS_RE.search(text)+ if m:+ return int(m.group(1)), int(m.group(2))+ return _parse_overflow_counts(text)+++def _anthropic_upstream_error(text: str, counts_source: Optional[str] = None) -> str:+ """`text` is what the client is told; `counts_source` is the fuller body it came+ from, used only to recover token counts the message itself does not spell out."""+ # Starvation is a shared-cache capacity failure, not an oversized prompt: the right+ # response is to retry, so it must not be reworded into "shorten the conversation".+ if is_kv_starvation(text):+ return KV_STARVATION_MESSAGE+ if _classify_llama_generation_error(Exception(text)):+ counts = _oversize_counts(text)+ if counts is None and counts_source:+ counts = _oversize_counts(counts_source)+ if counts:+ # Anthropic's own head wording, which its clients key on to compact, paired+ # with the fit's remedy rather than a flat "shorten the conversation": when+ # the latest turn or the system prompt is what does not fit, compacting the+ # history cannot help and the client would just retry it.+ return (+ f"Prompt is too long: {counts[0]} tokens > {counts[1]} maximum. "+ + context_refusal.oversize_advice(counts[1])+ )+ return (+ "Prompt is too long. Try increasing the Context Length in Model settings, "+ "or shorten the conversation."+ )+ return _friendly_upstream_error(text)++def _clamp_finish_reason(value) -> str:"""Coerce an upstream finish_reason into OpenAI's known chat values.@@ -1029,6 +1075,35 @@ def _anthropic_stream_error_event(exc, *, force: bool = False):)+def _json_dumps_safe(value) -> Optional[str]:+ """`json.dumps` for count recovery only; never raises on an odd payload."""+ try:+ return json.dumps(value)+ except (TypeError, ValueError):+ return None+++def _anthropic_upstream_stream_error_event(text: str, counts_source: Optional[str] = None):+ """In-band Anthropic error event for a 200 stream that later reports an upstream failure.++ Same wording and status rule as the non-200 branch. Routing the raw body through+ `_friendly_error` instead flattens the count-less oversize refusal ("the request+ exceeds the available context size") to "An internal error occurred", so a streaming+ client is told 400/invalid_request_error with nothing it can read or compact on.+ """+ from core.inference.llama_keepwarm import mark_current_response_failed++ mark_current_response_failed()+ over = bool(_classify_llama_generation_error(Exception(text)))+ return build_anthropic_sse_event(+ "error",+ anthropic_error_body(+ _anthropic_upstream_error(text, counts_source = counts_source),+ status = 400 if over else 500,+ ),+ )++def _drop_parallel_tool_call_deltas(chunk) -> bool:"""In-place: drop tool_call deltas whose index >= 1 from a parsed OpenAIstreaming chunk so only the first tool call survives (parallel_tool_calls=false@@ -1454,6 +1529,11 @@ def _classify_llama_generation_error(exc: Exception) -> Optional[bool]:return True if exc.context_oversize else Nonemsg = str(exc)msg_l = msg.lower()+ # Same reasoning as the typed branch, which only reaches errors carrying the parsed+ # flag. Raw upstream bodies arrive here as a plain Exception, and "Context size has+ # been exceeded" trips the substring test below while meaning the opposite.+ if is_kv_starvation(msg):+ return Noneif "n_ctx" in msg_l or ("context" in msg_l and any(t in msg_l for t in ("exceed", "length", "window", "too long"))):@@ -29966,11 +30046,12 @@ async def _anthropic_passthrough_stream(from core.inference.llama_keepwarm import mark_response_failedmark_response_failed(getattr(request, "scope", None))+ _over = bool(_classify_llama_generation_error(Exception(_err_text)))yield build_anthropic_sse_event("error",anthropic_error_body(- _friendly_upstream_error(_err_text),- status = resp.status_code,+ _anthropic_upstream_error(_err_text),+ status = 400 if _over else resp.status_code,),)return@@ -30009,12 +30090,13 @@ async def _anthropic_passthrough_stream(from core.inference.llama_keepwarm import mark_response_failedmark_response_failed(getattr(request, "scope", None))- event = _anthropic_stream_error_event(- RuntimeError(error_message),- force = True,+ # The message alone can be count-less while the chunk still+ # carries n_prompt_tokens/n_ctx; keep the clean message and let+ # the formatter recover the totals from the payload.+ yield _anthropic_upstream_stream_error_event(+ error_message,+ counts_source = _json_dumps_safe(chunk.get("error", chunk)),)- if event is not None:- yield eventreturnif disable_parallel_tool_use:_drop_parallel_tool_call_deltas(chunk)@@ -30176,9 +30258,11 @@ async def _anthropic_passthrough_non_streaming(resp = await _post(body)if resp.status_code != 200:+ _err_text = resp.text[:500]+ _over = bool(_classify_llama_generation_error(Exception(_err_text)))raise HTTPException(- status_code = resp.status_code,- detail = _friendly_upstream_error(resp.text[:500]),+ status_code = 400 if _over else resp.status_code,+ detail = _anthropic_upstream_error(_err_text),)data = resp.json()diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex e626270eb13..fe7d0a0292c 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -4,6 +4,7 @@"""Tests for the OpenAI /v1/chat/completions client-side tool pass-through."""import os+import reimport sysimport asyncioimport base64@@ -161,6 +162,201 @@ class TestFriendlyUpstreamError:# An unrelated upstream error still passes through verbatim.assert "llama-server error:" in _openai_passthrough_error(500, "disk full").detail+ def test_anthropic_upstream_error_rewords_context_overflow(self):+ from routes.inference import _anthropic_upstream_error++ raw = (+ '{"error":{"code":500,"message":"the request exceeds the available context '+ 'size. try increasing the context size or enable context shift",'+ '"type":"server_error"}}'+ )+ msg = _anthropic_upstream_error(raw)+ assert "prompt is too long" in msg.lower()+ assert "llama-server error:" not in msg++ def test_anthropic_upstream_error_includes_token_counts_when_known(self):+ from routes.inference import _anthropic_upstream_error++ msg = _anthropic_upstream_error(+ "the request (214331 tokens) exceeds the available context size (131072 tokens)"+ )+ m = re.search(r"prompt is too long[^0-9]*(\d+)\s*tokens?\s*>\s*(\d+)", msg, re.I)+ assert m and m.groups() == ("214331", "131072")++ def test_anthropic_upstream_error_leaves_other_failures_alone(self):+ from routes.inference import _anthropic_upstream_error+ assert _anthropic_upstream_error("disk full") == "llama-server error: disk full"+ assert "compile a grammar" in _anthropic_upstream_error("failed to parse grammar")++ def test_anthropic_upstream_error_keeps_kv_starvation_out_of_the_overflow_rewrite(self):+ """llama-server says "Context size has been exceeded" when concurrent+ generations drain the shared KV cache. Nothing about the request was too+ long, so rewording it as an oversized prompt sends the client compacting a+ valid conversation instead of retrying."""+ from routes.inference import _anthropic_upstream_error, _classify_llama_generation_error+ from core.inference.stream_errors import KV_STARVATION_MESSAGE++ for raw in (+ "Context size has been exceeded.",+ '{"error":{"message":"Context size has been exceeded.","code":500}}',+ ):+ assert _anthropic_upstream_error(raw) == KV_STARVATION_MESSAGE+ assert "too long" not in _anthropic_upstream_error(raw).lower()+ # None keeps the upstream status: a 400 would blame the caller's request.+ assert _classify_llama_generation_error(Exception(raw)) is None++ def test_in_band_sse_recovers_counts_from_the_chunk_the_message_came_from(self):+ """`_monitor_openai_error_message` returns only the message string. When that+ string carries no numbers but the chunk around it does, the totals -- and the+ window `oversize_advice` needs -- have to come from the payload."""+ import json as _json++ from core.inference import context_refusal+ from routes.inference import (+ _anthropic_upstream_stream_error_event,+ _json_dumps_safe,+ _monitor_openai_error_message,+ )++ chunk = {+ "error": {+ "code": 400,+ "message": "the request exceeds the available context size",+ "type": "exceed_context_size_error",+ "n_prompt_tokens": 70494,+ "n_ctx": 67584,+ }+ }+ message = _monitor_openai_error_message(chunk)+ source = _json_dumps_safe(chunk.get("error", chunk))++ def body(text, counts_source):+ event = _anthropic_upstream_stream_error_event(text, counts_source = counts_source)+ return _json.loads(event.split("data: ", 1)[1].strip().splitlines()[0])["error"]++ assert "70494" not in body(message, None)["message"]+ assert (+ "Prompt is too long: 70494 tokens > 67584 maximum" in body(message, source)["message"]+ )++ try:+ context_refusal.record_fit(+ {+ "fits": False,+ "context_length": 67584,+ "irreducible_tokens": 70000,+ "latest_turn_tokens": 500,+ }+ )+ # The recovered window is what lets the fit be consulted at all.+ assert "shortening the conversation will not help" in body(message, source)["message"]+ finally:+ context_refusal.clear()++ # An unrelated error must not have the payload spliced into its text.+ assert body("disk full", source)["message"] == "llama-server error: disk full"++ def test_counts_come_from_the_structured_fields_when_the_message_has_none(self):+ """An exceed_context_size_error body carries n_prompt_tokens/n_ctx even when its+ message spells out no numbers. These paths are handed the whole body, so the+ totals are right there; dropping them sends the client a count-less refusal."""+ from routes.inference import _anthropic_upstream_error, _oversize_counts++ body = (+ '{"error":{"code":400,"message":"the request exceeds the available context '+ 'size","type":"exceed_context_size_error","n_prompt_tokens":70494,'+ '"n_ctx":67584}}'+ )+ assert _oversize_counts(body) == (70494, 67584)+ assert "Prompt is too long: 70494 tokens > 67584 maximum" in _anthropic_upstream_error(body)++ def test_the_remedy_comes_from_the_fit_not_a_flat_shorten_the_conversation(self):+ """When the latest turn or the irreducible floor is what does not fit, compacting+ the history cannot make it fit, and a client told to compact just retries."""+ from core.inference import context_refusal+ from routes.inference import _anthropic_upstream_error++ body = "the request (214331 tokens) exceeds the available context size (131072 tokens)"+ try:+ context_refusal.record_fit(+ {+ "fits": False,+ "context_length": 131072,+ "irreducible_tokens": 140000,+ "latest_turn_tokens": 500,+ }+ )+ msg = _anthropic_upstream_error(body)+ # The head Anthropic clients key on survives.+ assert msg.startswith("Prompt is too long: 214331 tokens > 131072 maximum.")+ assert "shortening the conversation will not help" in msg++ context_refusal.record_fit(+ {+ "fits": False,+ "context_length": 131072,+ "irreducible_tokens": 140000,+ "latest_turn_tokens": 139000,+ "latest_turn_role": "tool",+ "latest_turn_counted": True,+ }+ )+ msg = _anthropic_upstream_error(body)+ assert "A tool returned more than this context window can hold" in msg+ assert "shortening the conversation will not help" in msg+ finally:+ context_refusal.clear()++ # With no recorded fit it stays the generic wording.+ assert "Try increasing the Context Length" in _anthropic_upstream_error(body)++ def test_in_band_sse_error_gets_the_same_wording_as_the_non_200_branch(self):+ """A 200 stream that later emits data: {"error": ...} used to be wrapped in a+ plain RuntimeError, and _friendly_error flattens the count-less oversize+ refusal to "An internal error occurred" -- a 400/invalid_request_error with+ nothing the client can read or compact on."""+ import json as _json++ from routes.inference import _anthropic_upstream_stream_error_event++ def body(text):+ event = _anthropic_upstream_stream_error_event(text)+ return _json.loads(event.split("data: ", 1)[1].strip().splitlines()[0])["error"]++ no_counts = body(+ "the request exceeds the available context size. try increasing the context size"+ )+ assert "internal error" not in no_counts["message"].lower()+ assert no_counts["message"].startswith("Prompt is too long")+ assert no_counts["type"] == "invalid_request_error"++ counted = body(+ "the request (214331 tokens) exceeds the available context size (131072 tokens)"+ )+ assert "Prompt is too long: 214331 tokens > 131072 maximum" in counted["message"]+ assert counted["type"] == "invalid_request_error"++ # Starvation keeps the retry advice and must not become a 400.+ starved = body("Context size has been exceeded.")+ assert "shared pool of context" in starved["message"]+ assert "too long" not in starved["message"].lower()+ assert starved["type"] == "api_error"++ unrelated = body("disk full")+ assert unrelated["message"] == "llama-server error: disk full"+ assert unrelated["type"] == "api_error"++ def test_a_genuine_oversize_body_is_still_classified_as_an_overflow(self):+ from routes.inference import _classify_llama_generation_error+ assert (+ _classify_llama_generation_error(+ Exception(+ "the request (214331 tokens) exceeds the available context size (131072 tokens)"+ )+ )+ is True+ )+# =====================================================================# ChatMessage — tool role, tool_calls, optional content