Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: honour context_overflow on the /v1/messages passthrough (#15418)
Sreesanth Rddc75e9bcef8
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 3ef38f75a5b..77f8f5d2020 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1003,10 +1003,15 @@ def _truncate_middle_messages(messages: list, keep_ratio: float):return messages, 0new_messages = head + anchor- for grp in middle[first_kept:]:- new_messages.extend(grp)- for grp in tail:- new_messages.extend(grp)+ kept = [msg for grp in middle[first_kept:] + tail for msg in grp]+ # Merge the seam: strict templates (e.g. Mistral) reject two user turns in a row.+ if kept and new_messages[-1].get("role") == kept[0].get("role") == "user":+ new_messages[-1] = {+ **new_messages[-1],+ "content": _merge_user_content(new_messages[-1].get("content"), kept[0].get("content")),+ }+ kept = kept[1:]+ new_messages.extend(kept)return new_messages, dropped@@ -1163,6 +1168,20 @@ def _remember_passthrough_cut(prefix_boundary.remember(original, body.get("messages") or [], dropped, scope)+async def _prepare_passthrough_cut(llama_backend, body: dict, policy: Optional[str]):+ """(uncut messages, cut scope, messages a remembered cut dropped). Only truncate_oldest+ remembers cuts, so every other policy skips the copy, the scope and the thread hop."""+ if policy != "truncate_oldest":+ return None, None, 0+ uncut = list(body.get("messages") or [])+ scope = _passthrough_cut_scope(llama_backend, body)+ # Off the event loop: the history can hold megabytes of images.+ dropped = await asyncio.to_thread(+ _replay_passthrough_cut, body, policy, getattr(llama_backend, "context_length", None), scope+ )+ return uncut, scope, dropped++def _anthropic_stream_error_event(exc, *, force: bool = False):"""Return an Anthropic in-band stream error event when one is useful."""# Flag failed even when returning None: middleware must not claim a preview-owned model.@@ -43238,6 +43257,7 @@ async def anthropic_messages(auto_heal_tool_calls = payload.auto_heal_tool_calls,response_format = response_format,parse_think = _think_parsing_expected(llama_backend, payload),+ context_overflow = _overflow_truncation_policy(payload),**_reasoning_args,))@@ -43265,6 +43285,7 @@ async def anthropic_messages(cancel_event = cancel_event,response_format = response_format,parse_think = _think_parsing_expected(llama_backend, payload),+ context_overflow = _overflow_truncation_policy(payload),**_reasoning_args,))@@ -44384,6 +44405,7 @@ async def _anthropic_passthrough_stream(preserve_thinking = None,parse_think = True,thinking_budget_tokens = None,+ context_overflow = None,):"""Streaming client-side pass-through: forward tools to llama-server andtranslate its stream to Anthropic SSE without executing anything."""@@ -44511,29 +44533,64 @@ async def _anthropic_passthrough_stream(items_iter = Nonecancel_watcher = Nonedisconnect_watcher = None+ # Resolved by the route from the request field or UNSLOTH_CONTEXT_OVERFLOW; None keeps the 400.+ _truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if context_overflow else 0++ def _apply_passthrough_truncation(err_text: str) -> bool:+ nonlocal _truncated_messages+ dropped = _apply_measured_overflow_truncation(body, err_text, context_overflow)+ if dropped is None:+ return False+ _truncated_messages += dropped+ _remember_passthrough_cut(+ _uncut_messages, body, context_overflow, _truncated_messages, _cut_scope+ )+ return True+try:+ # Inside the try so a disconnect during the replay still releases the tracker.+ _uncut_messages, _cut_scope, _truncated_messages = await _prepare_passthrough_cut(+ llama_backend, body, context_overflow+ )upstream = _LlamaUpstream(llama_backend, "/v1/chat/completions")- async with aclosing(- upstream.stream_response(client, body, cancel_event, request)- ) as opening:- async for opened in opening:- if opened is _LLAMA_STREAM_KEEPALIVE:- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE- else:- resp = opened- first_token_deadline = upstream.first_token_deadline- if resp is None:- return+ while True:+ async with aclosing(+ upstream.stream_response(client, body, cancel_event, request)+ ) as opening:+ async for opened in opening:+ if opened is _LLAMA_STREAM_KEEPALIVE:+ yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE+ else:+ resp = opened+ first_token_deadline = upstream.first_token_deadline+ if resp is None:+ return- # Headers flushed, so upstream 4xx must surface as in-band Anthropic error, not silent end_turn.- if resp.status_code != 200:+ if resp.status_code == 200:+ break++ # Headers flushed, so upstream 4xx must surface as in-band Anthropic error, not silent end_turn._err_bytes = await resp.aread()- _err_text = _err_bytes.decode("utf-8", "replace")[:500]+ _err_text = _err_bytes.decode("utf-8", "replace")logger.error("anthropic passthrough upstream error: status=%s body=%s",resp.status_code,- _err_text,+ _err_text[:500],)+ # Opt-in overflow policy, as on /v1/chat/completions: shrink and retry instead of a 400.+ if (+ _truncate_budget > 0+ and _classify_llama_generation_error(Exception(_err_text))+ and _apply_passthrough_truncation(_err_text)+ ):+ _truncate_budget -= 1+ try:+ await resp.aclose()+ except Exception:+ pass+ resp = None+ continue+ _err_text = _err_text[:500]# The outer 200 stream headers are already flushed, so flag the failure: the# middleware must not claim a preview-owned model on a passthrough whose# upstream errored before any SSE.@@ -44766,6 +44823,7 @@ async def _anthropic_passthrough_non_streaming(preserve_thinking = None,parse_think = True,thinking_budget_tokens = None,+ context_overflow = None,):body = _build_passthrough_payload(openai_messages,@@ -44794,7 +44852,26 @@ async def _anthropic_passthrough_non_streaming(async with _LlamaUpstream(llama_backend, "/v1/chat/completions", cancel_event = cancel_event, request = request) as upstream:+ _truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if context_overflow else 0+ _uncut_messages, _cut_scope, _truncated_messages = await _prepare_passthrough_cut(+ llama_backend, body, context_overflow+ )resp = await upstream.post(body, timeout = _timeout)+ # Opt-in overflow policy, as on /v1/chat/completions: shrink and retry instead of a 400.+ while (+ resp.status_code != 200+ and _truncate_budget > 0+ and _classify_llama_generation_error(Exception(resp.text))+ ):+ dropped = _apply_measured_overflow_truncation(body, resp.text, context_overflow)+ if dropped is None:+ break+ _truncate_budget -= 1+ _truncated_messages += dropped+ _remember_passthrough_cut(+ _uncut_messages, body, context_overflow, _truncated_messages, _cut_scope+ )+ resp = await upstream.post(body, timeout = _timeout)if resp.status_code != 200:_err_text = resp.text[:500]@@ -45603,15 +45680,8 @@ async def _openai_passthrough_stream_admitted(client = _llama_stream_client()_truncate_policy = _overflow_truncation_policy(payload)_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0- _uncut_messages = list(body.get("messages") or [])- # Off the event loop: the history can hold megabytes of images.- _cut_scope = _passthrough_cut_scope(llama_backend, body)- _truncated_messages = await asyncio.to_thread(- _replay_passthrough_cut,- body,- _truncate_policy,- getattr(llama_backend, "context_length", None),- _cut_scope,+ _uncut_messages, _cut_scope, _truncated_messages = await _prepare_passthrough_cut(+ llama_backend, body, _truncate_policy)_context_was_truncated = _truncated_messages > 0@@ -46372,15 +46442,8 @@ async def _openai_passthrough_non_streaming_upstream(_truncate_policy = _overflow_truncation_policy(payload)_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0- _uncut_messages = list(body.get("messages") or [])- # Off the event loop: the history can hold megabytes of images.- _cut_scope = _passthrough_cut_scope(llama_backend, body)- _truncated_messages = await asyncio.to_thread(- _replay_passthrough_cut,- body,- _truncate_policy,- getattr(llama_backend, "context_length", None),- _cut_scope,+ _uncut_messages, _cut_scope, _truncated_messages = await _prepare_passthrough_cut(+ llama_backend, body, _truncate_policy)_context_was_truncated = _truncated_messages > 0diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex 6575bdd568f..d04d91e26cf 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -3373,6 +3373,47 @@ class TestAnthropicMessagesToolRouting:assert captured.get("thinking_budget_tokens") == expected+ @pytest.mark.parametrize(+ ("env", "field", "expected"),+ [+ (None, None, None),+ ("truncate_oldest", None, "truncate_oldest"),+ (None, "truncate_middle", "truncate_middle"),+ ("truncate_oldest", "error", None),+ ],+ )+ @pytest.mark.parametrize("stream", [False, True])+ def test_context_overflow_reaches_anthropic_client_tool_passthrough(+ self, monkeypatch, stream, env, field, expected+ ):+ import routes.inference as inf_mod+ from fastapi.responses import JSONResponse++ if env is None:+ monkeypatch.delenv("UNSLOTH_CONTEXT_OVERFLOW", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_CONTEXT_OVERFLOW", env)+ _mock_backend(monkeypatch)+ captured = {}++ async def _passthrough(*args, **kwargs):+ captured.update(kwargs)+ return JSONResponse({"type": "message", "content": []})++ helper = (+ "_anthropic_passthrough_stream" if stream else "_anthropic_passthrough_non_streaming"+ )+ monkeypatch.setattr(inf_mod, helper, _passthrough)+ payload = _basic_payload(+ stream = stream,+ tools = [{"name": "lookup", "input_schema": {"type": "object"}}],+ **({} if field is None else {"context_overflow": field}),+ )++ _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t"))++ assert captured.get("context_overflow") == expected+@pytest.mark.parametrize("budget", [None, 128])@pytest.mark.parametrize("stream", [False, True])def test_passthrough_puts_thinking_budget_on_the_llama_server_body(diff --git a/studio/backend/tests/test_anthropic_passthrough_overflow.py b/studio/backend/tests/test_anthropic_passthrough_overflow.pynew file mode 100644index 00000000000..a5a0f4a86bc--- /dev/null+++ b/studio/backend/tests/test_anthropic_passthrough_overflow.py@@ -0,0 +1,249 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""/v1/messages client-tool passthrough honours context_overflow like /v1/chat/completions (#15417)."""++from __future__ import annotations++import asyncio+import json+import os+import sys+import threading++import httpx+import pytest++_backend = os.path.join(os.path.dirname(__file__), "..")+sys.path.insert(0, _backend)++import routes.inference as inf_mod+from routes.inference import (+ _anthropic_passthrough_non_streaming,+ _anthropic_passthrough_stream,+)++_OVERFLOW = json.dumps(+ {+ "error": {+ "code": 400,+ "message": "request (10000 tokens) exceeds the available context size (2048 tokens), try increasing it",+ "type": "exceed_context_size_error",+ "n_prompt_tokens": 10000,+ "n_ctx": 2048,+ }+ }+).encode()+++class _Backend:+ base_url = "http://127.0.0.1:59999"+ context_length = 2048++ def count_chat_tokens(self, *_args, **_kwargs):+ return 2+++class _Request:+ async def is_disconnected(self):+ return False+++def _history():+ messages = [{"role": "system", "content": "system"}]+ for idx in range(8):+ messages.append({"role": "user", "content": f"question {idx} " + "x" * 1000})+ messages.append({"role": "assistant", "content": f"answer {idx} " + "y" * 1000})+ messages.append({"role": "user", "content": "latest question"})+ return messages++[email protected](autouse = True)+def _fresh_cut_memory():+ from core.inference import prefix_boundary++ prefix_boundary.clear()+ yield+ prefix_boundary.clear()+++def _install_transport(monkeypatch, bodies, *, stream):+ def handler(request: httpx.Request) -> httpx.Response:+ bodies.append(json.loads(request.content))+ if len(bodies) == 1:+ return httpx.Response(400, content = _OVERFLOW)+ if stream:+ content = (+ f"data: {json.dumps({'choices': [{'delta': {'content': 'hi'}}]})}\n\n"+ "data: [DONE]\n\n"+ )+ return httpx.Response(+ 200, content = content.encode(), headers = {"content-type": "text/event-stream"}+ )+ return httpx.Response(+ 200,+ json = {+ "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 2, "completion_tokens": 1},+ },+ )++ transport = httpx.MockTransport(handler)+ real_client = httpx.AsyncClient++ def _client(*_args, **kwargs):+ return real_client(transport = transport, timeout = kwargs.get("timeout", 600))++ monkeypatch.setattr(inf_mod.httpx, "AsyncClient", _client)+++async def _stream_text(**kwargs):+ response = await _anthropic_passthrough_stream(+ _Request(),+ threading.Event(),+ _Backend(),+ _history(),+ [],+ 0.7,+ 0.95,+ 20,+ 16,+ "msg_1",+ "test-model",+ **kwargs,+ )+ chunks = []+ async for chunk in response.body_iterator:+ chunks.append(chunk.decode() if isinstance(chunk, (bytes, bytearray)) else chunk)+ return "".join(chunks)+++async def _non_streaming(**kwargs):+ return await _anthropic_passthrough_non_streaming(+ _Backend(),+ _history(),+ [],+ 0.7,+ 0.95,+ 20,+ 16,+ "msg_1",+ "test-model",+ **kwargs,+ )++[email protected]("policy", ["truncate_oldest", "truncate_middle"])+def test_stream_overflow_is_truncated_and_retried(monkeypatch, policy):+ bodies = []+ _install_transport(monkeypatch, bodies, stream = True)++ text = asyncio.run(_stream_text(context_overflow = policy))++ assert len(bodies) == 2+ assert len(bodies[1]["messages"]) < len(bodies[0]["messages"])+ assert bodies[1]["messages"][0] == {"role": "system", "content": "system"}+ assert bodies[1]["messages"][-1]["content"] == "latest question"+ assert "event: error" not in text+ assert '"text": "hi"' in text or '"text":"hi"' in text+ assert "message_stop" in text+++def test_stream_overflow_without_opt_in_keeps_the_error(monkeypatch):+ bodies = []+ _install_transport(monkeypatch, bodies, stream = True)++ text = asyncio.run(_stream_text())++ assert len(bodies) == 1+ assert "event: error" in text++[email protected]("policy", ["truncate_oldest", "truncate_middle"])+def test_non_streaming_overflow_is_truncated_and_retried(monkeypatch, policy):+ bodies = []+ _install_transport(monkeypatch, bodies, stream = False)++ result = asyncio.run(_non_streaming(context_overflow = policy))++ assert len(bodies) == 2+ assert len(bodies[1]["messages"]) < len(bodies[0]["messages"])+ assert bodies[1]["messages"][-1]["content"] == "latest question"+ assert json.loads(result.body)["content"][0]["text"] == "hi"+++def test_non_streaming_overflow_without_opt_in_keeps_the_error(monkeypatch):+ from fastapi import HTTPException++ bodies = []+ _install_transport(monkeypatch, bodies, stream = False)++ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(_non_streaming())++ assert excinfo.value.status_code == 400+ assert len(bodies) == 1++[email protected]("stream", [True, False])+def test_messages_route_honours_the_server_overflow_default(monkeypatch, stream):+ from types import SimpleNamespace++ from fastapi.responses import StreamingResponse+ from models.inference import AnthropicMessagesRequest+ from routes.inference import anthropic_messages++ monkeypatch.setenv("UNSLOTH_CONTEXT_OVERFLOW", "truncate_oldest")+ monkeypatch.setattr(inf_mod, "current_date_prompt_line", lambda **_kwargs: "")+ backend = SimpleNamespace(+ is_loaded = True,+ is_vision = False,+ supports_tools = True,+ model_identifier = "test-model",+ base_url = _Backend.base_url,+ context_length = 2048,+ count_chat_tokens = lambda *args, **kwargs: 2,+ )+ monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)+ bodies = []+ _install_transport(monkeypatch, bodies, stream = stream)+ history = _history()+ payload = AnthropicMessagesRequest(+ max_tokens = 16,+ stream = stream,+ system = history[0]["content"],+ messages = history[1:],+ tools = [{"name": "lookup", "input_schema": {"type": "object"}}],+ )++ async def _run():+ response = await anthropic_messages(payload, request = _Request(), current_subject = "t")+ if not isinstance(response, StreamingResponse):+ return response.body.decode()+ chunks = []+ async for chunk in response.body_iterator:+ chunks.append(chunk.decode() if isinstance(chunk, (bytes, bytearray)) else chunk)+ return "".join(chunks)++ text = asyncio.run(_run())++ assert len(bodies) == 2+ assert len(bodies[1]["messages"]) < len(bodies[0]["messages"])+ assert "exceeds the available context size" not in text+ assert "hi" in text++[email protected]("policy", [None, "truncate_middle"])+def test_only_truncate_oldest_pays_for_the_cut_replay(monkeypatch, policy):+ def _no_hop(*_args, **_kwargs):+ raise AssertionError("no thread hop without a policy that remembers cuts")++ monkeypatch.setattr(inf_mod.asyncio, "to_thread", _no_hop)+ body = {"messages": _history()}++ assert asyncio.run(inf_mod._prepare_passthrough_cut(_Backend(), body, policy)) == (+ None,+ None,+ 0,+ )+ assert body["messages"] == _history()diff --git a/studio/backend/tests/test_context_overflow_truncation.py b/studio/backend/tests/test_context_overflow_truncation.pyindex 7e809d0fb99..24af110b06a 100644--- a/studio/backend/tests/test_context_overflow_truncation.py+++ b/studio/backend/tests/test_context_overflow_truncation.py@@ -101,6 +101,22 @@ def test_truncation_drops_middle_keeps_anchors():assert new[-2] == msgs[-2]+def test_truncation_keeps_roles_alternating_at_the_cut():+ messages = [+ {"role": "system", "content": "s"},+ {"role": "user", "content": "u0"},+ {"role": "assistant", "content": "A" * 20000},+ {"role": "user", "content": "u1"},+ {"role": "assistant", "content": "a1"},+ {"role": "user", "content": "u2"},+ ]+ new, dropped = _truncate_middle_messages(messages, 0.5)+ assert dropped == 1+ assert [m["role"] for m in new] == ["system", "user", "assistant", "user"]+ assert new[1]["content"] == "u0\n\nu1"+ assert messages[1]["content"] == "u0"++def test_truncation_never_orphans_tool_results():msgs = _conversation()new, dropped = _truncate_middle_messages(msgs, keep_ratio = 0.4)