Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: honour context_overflow on the /v1/messages passthrough (#15418)

Sreesanth Rddc75e9bcef8
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 3ef38f75a5b..77f8f5d2020 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1003,10 +1003,15 @@ def _truncate_middle_messages(messages: list, keep_ratio: float):
return messages, 0
new_messages = head + anchor
- for grp in middle[first_kept:]:
- new_messages.extend(grp)
- for grp in tail:
- new_messages.extend(grp)
+ kept = [msg for grp in middle[first_kept:] + tail for msg in grp]
+ # Merge the seam: strict templates (e.g. Mistral) reject two user turns in a row.
+ if kept and new_messages[-1].get("role") == kept[0].get("role") == "user":
+ new_messages[-1] = {
+ **new_messages[-1],
+ "content": _merge_user_content(new_messages[-1].get("content"), kept[0].get("content")),
+ }
+ kept = kept[1:]
+ new_messages.extend(kept)
return new_messages, dropped
@@ -1163,6 +1168,20 @@ def _remember_passthrough_cut(
prefix_boundary.remember(original, body.get("messages") or [], dropped, scope)
+async def _prepare_passthrough_cut(llama_backend, body: dict, policy: Optional[str]):
+ """(uncut messages, cut scope, messages a remembered cut dropped). Only truncate_oldest
+ remembers cuts, so every other policy skips the copy, the scope and the thread hop."""
+ if policy != "truncate_oldest":
+ return None, None, 0
+ uncut = list(body.get("messages") or [])
+ scope = _passthrough_cut_scope(llama_backend, body)
+ # Off the event loop: the history can hold megabytes of images.
+ dropped = await asyncio.to_thread(
+ _replay_passthrough_cut, body, policy, getattr(llama_backend, "context_length", None), scope
+ )
+ return uncut, scope, dropped
+
+
def _anthropic_stream_error_event(exc, *, force: bool = False):
"""Return an Anthropic in-band stream error event when one is useful."""
# Flag failed even when returning None: middleware must not claim a preview-owned model.
@@ -43238,6 +43257,7 @@ async def anthropic_messages(
auto_heal_tool_calls = payload.auto_heal_tool_calls,
response_format = response_format,
parse_think = _think_parsing_expected(llama_backend, payload),
+ context_overflow = _overflow_truncation_policy(payload),
**_reasoning_args,
)
)
@@ -43265,6 +43285,7 @@ async def anthropic_messages(
cancel_event = cancel_event,
response_format = response_format,
parse_think = _think_parsing_expected(llama_backend, payload),
+ context_overflow = _overflow_truncation_policy(payload),
**_reasoning_args,
)
)
@@ -44384,6 +44405,7 @@ async def _anthropic_passthrough_stream(
preserve_thinking = None,
parse_think = True,
thinking_budget_tokens = None,
+ context_overflow = None,
):
"""Streaming client-side pass-through: forward tools to llama-server and
translate its stream to Anthropic SSE without executing anything."""
@@ -44511,29 +44533,64 @@ async def _anthropic_passthrough_stream(
items_iter = None
cancel_watcher = None
disconnect_watcher = None
+ # Resolved by the route from the request field or UNSLOTH_CONTEXT_OVERFLOW; None keeps the 400.
+ _truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if context_overflow else 0
+
+ def _apply_passthrough_truncation(err_text: str) -> bool:
+ nonlocal _truncated_messages
+ dropped = _apply_measured_overflow_truncation(body, err_text, context_overflow)
+ if dropped is None:
+ return False
+ _truncated_messages += dropped
+ _remember_passthrough_cut(
+ _uncut_messages, body, context_overflow, _truncated_messages, _cut_scope
+ )
+ return True
+
try:
+ # Inside the try so a disconnect during the replay still releases the tracker.
+ _uncut_messages, _cut_scope, _truncated_messages = await _prepare_passthrough_cut(
+ llama_backend, body, context_overflow
+ )
upstream = _LlamaUpstream(llama_backend, "/v1/chat/completions")
- async with aclosing(
- upstream.stream_response(client, body, cancel_event, request)
- ) as opening:
- async for opened in opening:
- if opened is _LLAMA_STREAM_KEEPALIVE:
- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
- else:
- resp = opened
- first_token_deadline = upstream.first_token_deadline
- if resp is None:
- return
+ while True:
+ async with aclosing(
+ upstream.stream_response(client, body, cancel_event, request)
+ ) as opening:
+ async for opened in opening:
+ if opened is _LLAMA_STREAM_KEEPALIVE:
+ yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
+ else:
+ resp = opened
+ first_token_deadline = upstream.first_token_deadline
+ if resp is None:
+ return
- # Headers flushed, so upstream 4xx must surface as in-band Anthropic error, not silent end_turn.
- if resp.status_code != 200:
+ if resp.status_code == 200:
+ break
+
+ # Headers flushed, so upstream 4xx must surface as in-band Anthropic error, not silent end_turn.
_err_bytes = await resp.aread()
- _err_text = _err_bytes.decode("utf-8", "replace")[:500]
+ _err_text = _err_bytes.decode("utf-8", "replace")
logger.error(
"anthropic passthrough upstream error: status=%s body=%s",
resp.status_code,
- _err_text,
+ _err_text[:500],
)
+ # Opt-in overflow policy, as on /v1/chat/completions: shrink and retry instead of a 400.
+ if (
+ _truncate_budget > 0
+ and _classify_llama_generation_error(Exception(_err_text))
+ and _apply_passthrough_truncation(_err_text)
+ ):
+ _truncate_budget -= 1
+ try:
+ await resp.aclose()
+ except Exception:
+ pass
+ resp = None
+ continue
+ _err_text = _err_text[:500]
# The outer 200 stream headers are already flushed, so flag the failure: the
# middleware must not claim a preview-owned model on a passthrough whose
# upstream errored before any SSE.
@@ -44766,6 +44823,7 @@ async def _anthropic_passthrough_non_streaming(
preserve_thinking = None,
parse_think = True,
thinking_budget_tokens = None,
+ context_overflow = None,
):
body = _build_passthrough_payload(
openai_messages,
@@ -44794,7 +44852,26 @@ async def _anthropic_passthrough_non_streaming(
async with _LlamaUpstream(
llama_backend, "/v1/chat/completions", cancel_event = cancel_event, request = request
) as upstream:
+ _truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if context_overflow else 0
+ _uncut_messages, _cut_scope, _truncated_messages = await _prepare_passthrough_cut(
+ llama_backend, body, context_overflow
+ )
resp = await upstream.post(body, timeout = _timeout)
+ # Opt-in overflow policy, as on /v1/chat/completions: shrink and retry instead of a 400.
+ while (
+ resp.status_code != 200
+ and _truncate_budget > 0
+ and _classify_llama_generation_error(Exception(resp.text))
+ ):
+ dropped = _apply_measured_overflow_truncation(body, resp.text, context_overflow)
+ if dropped is None:
+ break
+ _truncate_budget -= 1
+ _truncated_messages += dropped
+ _remember_passthrough_cut(
+ _uncut_messages, body, context_overflow, _truncated_messages, _cut_scope
+ )
+ resp = await upstream.post(body, timeout = _timeout)
if resp.status_code != 200:
_err_text = resp.text[:500]
@@ -45603,15 +45680,8 @@ async def _openai_passthrough_stream_admitted(
client = _llama_stream_client()
_truncate_policy = _overflow_truncation_policy(payload)
_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0
- _uncut_messages = list(body.get("messages") or [])
- # Off the event loop: the history can hold megabytes of images.
- _cut_scope = _passthrough_cut_scope(llama_backend, body)
- _truncated_messages = await asyncio.to_thread(
- _replay_passthrough_cut,
- body,
- _truncate_policy,
- getattr(llama_backend, "context_length", None),
- _cut_scope,
+ _uncut_messages, _cut_scope, _truncated_messages = await _prepare_passthrough_cut(
+ llama_backend, body, _truncate_policy
)
_context_was_truncated = _truncated_messages > 0
@@ -46372,15 +46442,8 @@ async def _openai_passthrough_non_streaming_upstream(
_truncate_policy = _overflow_truncation_policy(payload)
_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0
- _uncut_messages = list(body.get("messages") or [])
- # Off the event loop: the history can hold megabytes of images.
- _cut_scope = _passthrough_cut_scope(llama_backend, body)
- _truncated_messages = await asyncio.to_thread(
- _replay_passthrough_cut,
- body,
- _truncate_policy,
- getattr(llama_backend, "context_length", None),
- _cut_scope,
+ _uncut_messages, _cut_scope, _truncated_messages = await _prepare_passthrough_cut(
+ llama_backend, body, _truncate_policy
)
_context_was_truncated = _truncated_messages > 0
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index 6575bdd568f..d04d91e26cf 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -3373,6 +3373,47 @@ class TestAnthropicMessagesToolRouting:
assert captured.get("thinking_budget_tokens") == expected
+ @pytest.mark.parametrize(
+ ("env", "field", "expected"),
+ [
+ (None, None, None),
+ ("truncate_oldest", None, "truncate_oldest"),
+ (None, "truncate_middle", "truncate_middle"),
+ ("truncate_oldest", "error", None),
+ ],
+ )
+ @pytest.mark.parametrize("stream", [False, True])
+ def test_context_overflow_reaches_anthropic_client_tool_passthrough(
+ self, monkeypatch, stream, env, field, expected
+ ):
+ import routes.inference as inf_mod
+ from fastapi.responses import JSONResponse
+
+ if env is None:
+ monkeypatch.delenv("UNSLOTH_CONTEXT_OVERFLOW", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_CONTEXT_OVERFLOW", env)
+ _mock_backend(monkeypatch)
+ captured = {}
+
+ async def _passthrough(*args, **kwargs):
+ captured.update(kwargs)
+ return JSONResponse({"type": "message", "content": []})
+
+ helper = (
+ "_anthropic_passthrough_stream" if stream else "_anthropic_passthrough_non_streaming"
+ )
+ monkeypatch.setattr(inf_mod, helper, _passthrough)
+ payload = _basic_payload(
+ stream = stream,
+ tools = [{"name": "lookup", "input_schema": {"type": "object"}}],
+ **({} if field is None else {"context_overflow": field}),
+ )
+
+ _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t"))
+
+ assert captured.get("context_overflow") == expected
+
@pytest.mark.parametrize("budget", [None, 128])
@pytest.mark.parametrize("stream", [False, True])
def test_passthrough_puts_thinking_budget_on_the_llama_server_body(
diff --git a/studio/backend/tests/test_anthropic_passthrough_overflow.py b/studio/backend/tests/test_anthropic_passthrough_overflow.py
new file mode 100644
index 00000000000..a5a0f4a86bc
--- /dev/null
+++ b/studio/backend/tests/test_anthropic_passthrough_overflow.py
@@ -0,0 +1,249 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""/v1/messages client-tool passthrough honours context_overflow like /v1/chat/completions (#15417)."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import sys
+import threading
+
+import httpx
+import pytest
+
+_backend = os.path.join(os.path.dirname(__file__), "..")
+sys.path.insert(0, _backend)
+
+import routes.inference as inf_mod
+from routes.inference import (
+ _anthropic_passthrough_non_streaming,
+ _anthropic_passthrough_stream,
+)
+
+_OVERFLOW = json.dumps(
+ {
+ "error": {
+ "code": 400,
+ "message": "request (10000 tokens) exceeds the available context size (2048 tokens), try increasing it",
+ "type": "exceed_context_size_error",
+ "n_prompt_tokens": 10000,
+ "n_ctx": 2048,
+ }
+ }
+).encode()
+
+
+class _Backend:
+ base_url = "http://127.0.0.1:59999"
+ context_length = 2048
+
+ def count_chat_tokens(self, *_args, **_kwargs):
+ return 2
+
+
+class _Request:
+ async def is_disconnected(self):
+ return False
+
+
+def _history():
+ messages = [{"role": "system", "content": "system"}]
+ for idx in range(8):
+ messages.append({"role": "user", "content": f"question {idx} " + "x" * 1000})
+ messages.append({"role": "assistant", "content": f"answer {idx} " + "y" * 1000})
+ messages.append({"role": "user", "content": "latest question"})
+ return messages
+
+
[email protected](autouse = True)
+def _fresh_cut_memory():
+ from core.inference import prefix_boundary
+
+ prefix_boundary.clear()
+ yield
+ prefix_boundary.clear()
+
+
+def _install_transport(monkeypatch, bodies, *, stream):
+ def handler(request: httpx.Request) -> httpx.Response:
+ bodies.append(json.loads(request.content))
+ if len(bodies) == 1:
+ return httpx.Response(400, content = _OVERFLOW)
+ if stream:
+ content = (
+ f"data: {json.dumps({'choices': [{'delta': {'content': 'hi'}}]})}\n\n"
+ "data: [DONE]\n\n"
+ )
+ return httpx.Response(
+ 200, content = content.encode(), headers = {"content-type": "text/event-stream"}
+ )
+ return httpx.Response(
+ 200,
+ json = {
+ "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 2, "completion_tokens": 1},
+ },
+ )
+
+ transport = httpx.MockTransport(handler)
+ real_client = httpx.AsyncClient
+
+ def _client(*_args, **kwargs):
+ return real_client(transport = transport, timeout = kwargs.get("timeout", 600))
+
+ monkeypatch.setattr(inf_mod.httpx, "AsyncClient", _client)
+
+
+async def _stream_text(**kwargs):
+ response = await _anthropic_passthrough_stream(
+ _Request(),
+ threading.Event(),
+ _Backend(),
+ _history(),
+ [],
+ 0.7,
+ 0.95,
+ 20,
+ 16,
+ "msg_1",
+ "test-model",
+ **kwargs,
+ )
+ chunks = []
+ async for chunk in response.body_iterator:
+ chunks.append(chunk.decode() if isinstance(chunk, (bytes, bytearray)) else chunk)
+ return "".join(chunks)
+
+
+async def _non_streaming(**kwargs):
+ return await _anthropic_passthrough_non_streaming(
+ _Backend(),
+ _history(),
+ [],
+ 0.7,
+ 0.95,
+ 20,
+ 16,
+ "msg_1",
+ "test-model",
+ **kwargs,
+ )
+
+
[email protected]("policy", ["truncate_oldest", "truncate_middle"])
+def test_stream_overflow_is_truncated_and_retried(monkeypatch, policy):
+ bodies = []
+ _install_transport(monkeypatch, bodies, stream = True)
+
+ text = asyncio.run(_stream_text(context_overflow = policy))
+
+ assert len(bodies) == 2
+ assert len(bodies[1]["messages"]) < len(bodies[0]["messages"])
+ assert bodies[1]["messages"][0] == {"role": "system", "content": "system"}
+ assert bodies[1]["messages"][-1]["content"] == "latest question"
+ assert "event: error" not in text
+ assert '"text": "hi"' in text or '"text":"hi"' in text
+ assert "message_stop" in text
+
+
+def test_stream_overflow_without_opt_in_keeps_the_error(monkeypatch):
+ bodies = []
+ _install_transport(monkeypatch, bodies, stream = True)
+
+ text = asyncio.run(_stream_text())
+
+ assert len(bodies) == 1
+ assert "event: error" in text
+
+
[email protected]("policy", ["truncate_oldest", "truncate_middle"])
+def test_non_streaming_overflow_is_truncated_and_retried(monkeypatch, policy):
+ bodies = []
+ _install_transport(monkeypatch, bodies, stream = False)
+
+ result = asyncio.run(_non_streaming(context_overflow = policy))
+
+ assert len(bodies) == 2
+ assert len(bodies[1]["messages"]) < len(bodies[0]["messages"])
+ assert bodies[1]["messages"][-1]["content"] == "latest question"
+ assert json.loads(result.body)["content"][0]["text"] == "hi"
+
+
+def test_non_streaming_overflow_without_opt_in_keeps_the_error(monkeypatch):
+ from fastapi import HTTPException
+
+ bodies = []
+ _install_transport(monkeypatch, bodies, stream = False)
+
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(_non_streaming())
+
+ assert excinfo.value.status_code == 400
+ assert len(bodies) == 1
+
+
[email protected]("stream", [True, False])
+def test_messages_route_honours_the_server_overflow_default(monkeypatch, stream):
+ from types import SimpleNamespace
+
+ from fastapi.responses import StreamingResponse
+ from models.inference import AnthropicMessagesRequest
+ from routes.inference import anthropic_messages
+
+ monkeypatch.setenv("UNSLOTH_CONTEXT_OVERFLOW", "truncate_oldest")
+ monkeypatch.setattr(inf_mod, "current_date_prompt_line", lambda **_kwargs: "")
+ backend = SimpleNamespace(
+ is_loaded = True,
+ is_vision = False,
+ supports_tools = True,
+ model_identifier = "test-model",
+ base_url = _Backend.base_url,
+ context_length = 2048,
+ count_chat_tokens = lambda *args, **kwargs: 2,
+ )
+ monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)
+ bodies = []
+ _install_transport(monkeypatch, bodies, stream = stream)
+ history = _history()
+ payload = AnthropicMessagesRequest(
+ max_tokens = 16,
+ stream = stream,
+ system = history[0]["content"],
+ messages = history[1:],
+ tools = [{"name": "lookup", "input_schema": {"type": "object"}}],
+ )
+
+ async def _run():
+ response = await anthropic_messages(payload, request = _Request(), current_subject = "t")
+ if not isinstance(response, StreamingResponse):
+ return response.body.decode()
+ chunks = []
+ async for chunk in response.body_iterator:
+ chunks.append(chunk.decode() if isinstance(chunk, (bytes, bytearray)) else chunk)
+ return "".join(chunks)
+
+ text = asyncio.run(_run())
+
+ assert len(bodies) == 2
+ assert len(bodies[1]["messages"]) < len(bodies[0]["messages"])
+ assert "exceeds the available context size" not in text
+ assert "hi" in text
+
+
[email protected]("policy", [None, "truncate_middle"])
+def test_only_truncate_oldest_pays_for_the_cut_replay(monkeypatch, policy):
+ def _no_hop(*_args, **_kwargs):
+ raise AssertionError("no thread hop without a policy that remembers cuts")
+
+ monkeypatch.setattr(inf_mod.asyncio, "to_thread", _no_hop)
+ body = {"messages": _history()}
+
+ assert asyncio.run(inf_mod._prepare_passthrough_cut(_Backend(), body, policy)) == (
+ None,
+ None,
+ 0,
+ )
+ assert body["messages"] == _history()
diff --git a/studio/backend/tests/test_context_overflow_truncation.py b/studio/backend/tests/test_context_overflow_truncation.py
index 7e809d0fb99..24af110b06a 100644
--- a/studio/backend/tests/test_context_overflow_truncation.py
+++ b/studio/backend/tests/test_context_overflow_truncation.py
@@ -101,6 +101,22 @@ def test_truncation_drops_middle_keeps_anchors():
assert new[-2] == msgs[-2]
+def test_truncation_keeps_roles_alternating_at_the_cut():
+ messages = [
+ {"role": "system", "content": "s"},
+ {"role": "user", "content": "u0"},
+ {"role": "assistant", "content": "A" * 20000},
+ {"role": "user", "content": "u1"},
+ {"role": "assistant", "content": "a1"},
+ {"role": "user", "content": "u2"},
+ ]
+ new, dropped = _truncate_middle_messages(messages, 0.5)
+ assert dropped == 1
+ assert [m["role"] for m in new] == ["system", "user", "assistant", "user"]
+ assert new[1]["content"] == "u0\n\nu1"
+ assert messages[1]["content"] == "u0"
+
+
def test_truncation_never_orphans_tool_results():
msgs = _conversation()
new, dropped = _truncate_middle_messages(msgs, keep_ratio = 0.4)