Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Respect the thinking budget on /v1/messages (#11724)
Nilay7ff3d23af6ba
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex c2c021edadc..c80060abb9a 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -35220,6 +35220,7 @@ class LlamaCppBackend:compaction_headroom_ratio: Optional[float] = None,thread_id: Optional[str] = None,tools_withheld: bool = False,+ thinking_budget_tokens: Optional[int] = None,_allow_respawn_retry: bool = True,) -> Generator[Union[str, dict], None, None]:"""@@ -35274,6 +35275,8 @@ class LlamaCppBackend:)if _reasoning_kw is not None:payload["chat_template_kwargs"] = _reasoning_kw+ if thinking_budget_tokens is not None:+ payload["thinking_budget_tokens"] = thinking_budget_tokensif continue_final_message:# llama-server applies the template; it rejects both flags set true.payload["continue_final_message"] = True@@ -35582,6 +35585,7 @@ class LlamaCppBackend:# The retry refits, so it must be told the same about this request's# tools as the first attempt was.tools_withheld = tools_withheld,+ thinking_budget_tokens = thinking_budget_tokens,_allow_respawn_retry = False,)return@@ -35643,6 +35647,7 @@ class LlamaCppBackend:# where the previous round's request has completed.on_conversation_grew: Optional[Callable[[list], None]] = None,on_decode_slot: Optional[Callable[[str, int], None]] = None,+ thinking_budget_tokens: Optional[int] = None,) -> Generator[dict, None, None]:"""Agentic loop: let the model call tools, execute them, and continue.@@ -36385,6 +36390,8 @@ class LlamaCppBackend:payload["tool_choice"] = requested_choiceif _reasoning_kw is not None:payload["chat_template_kwargs"] = _reasoning_kw+ if thinking_budget_tokens is not None:+ payload["thinking_budget_tokens"] = thinking_budget_tokens# Re-checked per iteration: once a tool result is appended the partial is# no longer trailing, so later turns are normal.if continue_final_message and trailing_assistant_text(conversation):@@ -39064,6 +39071,8 @@ class LlamaCppBackend:stream_payload["logit_bias"] = logit_biasif _reasoning_kw is not None:stream_payload["chat_template_kwargs"] = _reasoning_kw+ if thinking_budget_tokens is not None:+ stream_payload["thinking_budget_tokens"] = thinking_budget_tokensstream_payload["max_tokens"] = _final_max_tokensif stop:stream_payload["stop"] = stopdiff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex fa2748757ab..e04eab53edb 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -3720,7 +3720,7 @@ class AnthropicThinkingConfig(BaseModel):# (adaptive tiers) and Claude Code sends them, and a strict Literal turns an unrecognized value# into a hard 400. Only "disabled" means off; treat anything else as a request to think.type: str = "enabled"- # Accepted for wire compatibility; llama-server has no thinking budget.+ # Forwarded to llama-server as thinking_budget_tokens when thinking is on.budget_tokens: Optional[int] = Nonemodel_config = {"extra": "allow"}@@ -3759,8 +3759,6 @@ class AnthropicMessagesRequest(BaseModel):)enable_tools: Optional[bool] = Noneenabled_tools: Optional[list[str]] = None- # Anthropic's native extended-thinking control. Only `type` is honored: llama-server has no- # thinking-token budget, so `budget_tokens` is accepted and ignored rather than 400'd.thinking: Optional[AnthropicThinkingConfig] = None# [x-unsloth] reasoning controls mirroring the OpenAI endpoint. These win# over `thinking` when both are present, matching enable_tools precedence.diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 09dd5e28646..cfbd9519cd3 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -4976,11 +4976,15 @@ def _anthropic_reasoning_args(payload) -> dict:resolver = getattr(payload, "resolved_enable_thinking", None)if resolver is not None:enable_thinking = resolver()- return {+ args = {"enable_thinking": enable_thinking,"reasoning_effort": reasoning_effort,"preserve_thinking": payload.preserve_thinking,}+ budget = getattr(getattr(payload, "thinking", None), "budget_tokens", None)+ if enable_thinking and isinstance(budget, int) and budget > 0:+ args["thinking_budget_tokens"] = budget+ return argsdef _anthropic_preserve_thinking(llama_backend, payload) -> bool:@@ -38321,6 +38325,7 @@ def _build_passthrough_payload(seed = None,stream_options = None,markup = None,+ thinking_budget_tokens = None,):from core.inference.chat_template_helpers import (forced_tool_catalog,@@ -38391,6 +38396,8 @@ def _build_passthrough_payload(# llama-server renders the Jinja template in the caller's mode instead# of the model's load-time default.body["chat_template_kwargs"] = chat_template_kwargs+ if thinking_budget_tokens is not None:+ body["thinking_budget_tokens"] = thinking_budget_tokensreturn body@@ -38489,6 +38496,7 @@ async def _anthropic_passthrough_stream(reasoning_effort = None,preserve_thinking = None,parse_think = True,+ thinking_budget_tokens = None,):"""Streaming client-side pass-through: forward tools to llama-server andtranslate its stream to Anthropic SSE without executing anything."""@@ -38511,6 +38519,7 @@ async def _anthropic_passthrough_stream(chat_template_kwargs = _reasoning_template_kwargs(llama_backend, enable_thinking, reasoning_effort, preserve_thinking),+ thinking_budget_tokens = thinking_budget_tokens,backend_ctx = llama_backend.context_length,stream_options = {"include_usage": True},markup = getattr(llama_backend, "markup_profile", None),@@ -38825,6 +38834,7 @@ async def _anthropic_passthrough_non_streaming(reasoning_effort = None,preserve_thinking = None,parse_think = True,+ thinking_budget_tokens = None,):"""Non-streaming client-side pass-through.@@ -38852,6 +38862,7 @@ async def _anthropic_passthrough_non_streaming(chat_template_kwargs = _reasoning_template_kwargs(llama_backend, enable_thinking, reasoning_effort, preserve_thinking),+ thinking_budget_tokens = thinking_budget_tokens,backend_ctx = llama_backend.context_length,markup = getattr(llama_backend, "markup_profile", None),)diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex 012c5229a82..133eb7f9f49 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -2563,8 +2563,7 @@ class TestAnthropicReasoningArgs:assert payload.resolved_enable_thinking() is Truedef test_budget_tokens_accepted_not_rejected(self):- """Claude Code always sends budget_tokens; llama-server has no budget,- so it must be ignored rather than 400'd."""+ """Claude Code always sends budget_tokens; it must parse rather than 400."""payload = self._payload(thinking = {"type": "enabled", "budget_tokens": 4096})assert payload.thinking.budget_tokens == 4096assert payload.resolved_enable_thinking() is True@@ -2953,6 +2952,105 @@ class TestAnthropicMessagesToolRouting:assert captured["seed"] == 3407+ _BUDGET_CASES = [+ ({"thinking": {"type": "enabled", "budget_tokens": 128}}, 128),+ ({"thinking": {"type": "adaptive", "budget_tokens": 128}}, 128),+ ({"thinking": {"type": "enabled"}}, None),+ ({"thinking": {"type": "enabled", "budget_tokens": 0}}, None),+ ({"thinking": {"type": "disabled", "budget_tokens": 128}}, None),+ ({"thinking": {"type": "enabled", "budget_tokens": 128}, "enable_thinking": False}, None),+ ({"thinking": {"type": "enabled", "budget_tokens": 128}, "reasoning_effort": "none"}, None),+ ({}, None),+ ]++ @pytest.mark.parametrize(("fields", "expected"), _BUDGET_CASES)+ @pytest.mark.parametrize(+ ("extra", "expected_path"),+ [+ ({}, "plain"),+ ({"enable_tools": True, "permission_mode": "off"}, "tools"),+ ],+ ids = ["plain", "server-tools"],+ )+ def test_thinking_budget_reaches_internal_anthropic_generation(+ self, monkeypatch, extra, expected_path, fields, expected+ ):+ backend = _mock_backend(monkeypatch)++ _drive(+ anthropic_messages(+ _basic_payload(**fields, **extra),+ request = self._Request(),+ current_subject = "t",+ )+ )++ [(path, kwargs)] = backend.calls+ assert path == expected_path+ assert kwargs.get("thinking_budget_tokens") == expected++ @pytest.mark.parametrize(("fields", "expected"), _BUDGET_CASES)+ @pytest.mark.parametrize("stream", [False, True])+ def test_thinking_budget_reaches_anthropic_client_tool_passthrough(+ self, monkeypatch, stream, fields, expected+ ):+ import routes.inference as inf_mod+ from fastapi.responses import JSONResponse++ _mock_backend(monkeypatch)+ captured = {}++ async def _passthrough(*args, **kwargs):+ captured.update(kwargs)+ return JSONResponse({"type": "message", "content": []})++ helper = (+ "_anthropic_passthrough_stream" if stream else "_anthropic_passthrough_non_streaming"+ )+ monkeypatch.setattr(inf_mod, helper, _passthrough)+ payload = _basic_payload(+ stream = stream,+ tools = [{"name": "lookup", "input_schema": {"type": "object"}}],+ **fields,+ )++ _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t"))++ assert captured.get("thinking_budget_tokens") == expected++ @pytest.mark.parametrize("budget", [None, 128])+ @pytest.mark.parametrize("stream", [False, True])+ def test_passthrough_puts_thinking_budget_on_the_llama_server_body(+ self, monkeypatch, stream, budget+ ):+ import routes.inference as inf_mod++ real_builder = inf_mod._build_passthrough_payload+ bodies = []++ def _builder(*args, **kwargs):+ bodies.append(real_builder(*args, **kwargs))+ raise RuntimeError("body built")++ monkeypatch.setattr(inf_mod, "_build_passthrough_payload", _builder)+ backend = SimpleNamespace(base_url = "http://llama.test", context_length = 4096)+ messages = [{"role": "user", "content": "hi"}]+ common = (messages, [], 0.7, 0.95, 20, 16, "msg_1", "test-model")+ if stream:+ coro = inf_mod._anthropic_passthrough_stream(+ self._Request(), threading.Event(), backend, *common, thinking_budget_tokens = budget+ )+ else:+ coro = inf_mod._anthropic_passthrough_non_streaming(+ backend, *common, thinking_budget_tokens = budget+ )++ with pytest.raises(RuntimeError, match = "body built"):+ _drive(coro)++ [body] = bodies+ assert body.get("thinking_budget_tokens") == budget+def test_client_tool_catalog_without_passthrough_is_rejected(self, monkeypatch):# /v1/chat/completions 400s this; /v1/messages answered in prose instead.backend = _mock_backend(monkeypatch, supports_tools = False, supports_tool_passthrough = False)diff --git a/studio/backend/tests/test_llama_admission_compat_matrix.py b/studio/backend/tests/test_llama_admission_compat_matrix.pyindex b4b4688523d..c376c9b6d50 100644--- a/studio/backend/tests/test_llama_admission_compat_matrix.py+++ b/studio/backend/tests/test_llama_admission_compat_matrix.py@@ -185,7 +185,7 @@ class TestNothingChangesWhenTheBudgetIsUnknown:# In the order they were added, so older positional callers keep their meaning.-_TOOL_LOOP_HOOKS = ("on_conversation_grew", "on_decode_slot")+_TOOL_LOOP_HOOKS = ("on_conversation_grew", "on_decode_slot", "thinking_budget_tokens")class TestOldCallers:diff --git a/studio/backend/tests/test_llama_thinking_budget.py b/studio/backend/tests/test_llama_thinking_budget.pynew file mode 100644index 00000000000..0f282e0be03--- /dev/null+++ b/studio/backend/tests/test_llama_thinking_budget.py@@ -0,0 +1,153 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A per-request thinking budget must reach every llama-server request a generation makes."""++from __future__ import annotations++import contextlib+import copy+import json+import sys+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from core.inference.llama_cpp import LlamaCppBackend++_LOOKUP_TOOL = {+ "type": "function",+ "function": {+ "name": "web_search",+ "description": "Search the web.",+ "parameters": {+ "type": "object",+ "properties": {"query": {"type": "string"}},+ "required": ["query"],+ },+ },+}+++def _sse(delta: dict) -> str:+ return "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n"+++def _done() -> str:+ return "data: [DONE]\n"+++def _tool_call() -> str:+ return _sse(+ {+ "tool_calls": [+ {+ "index": 0,+ "id": "call_0",+ "type": "function",+ "function": {+ "name": "web_search",+ "arguments": json.dumps({"query": "q"}),+ },+ }+ ]+ }+ )+++def _backend(monkeypatch, streams: list, payloads: list) -> LlamaCppBackend:+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._process = object()+ backend._healthy = True+ backend._port = 48852+ backend._api_key = None+ backend._effective_context_length = 4096+ backend._supports_reasoning = True+ backend._reasoning_always_on = False+ backend._reasoning_style = "enable_thinking"+ backend._supports_preserve_thinking = False++ @contextlib.contextmanager+ def fake_stream_with_retry(_client, _url, payload, _cancel_event, **_kwargs):+ payloads.append(copy.deepcopy(payload))+ yield type("FakeResponse", (), {"status_code": 200, "chunks": streams.pop(0)})()++ def fake_iter_text_cancellable(+ response,+ _cancel_event,+ first_token_deadline = None,+ ):+ yield from response.chunks++ monkeypatch.setattr(backend, "_stream_with_retry", fake_stream_with_retry)+ monkeypatch.setattr(backend, "_iter_text_cancellable", fake_iter_text_cancellable)+ monkeypatch.setattr(backend, "_maybe_recover_from_mtp_crash", lambda *_a, **_k: False)+ return backend++[email protected]("budget", [None, 128])+def test_plain_generation_sends_the_budget(monkeypatch, budget):+ payloads: list[dict] = []+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)++ list(+ backend.generate_chat_completion(+ messages = [{"role": "user", "content": "hi"}],+ enable_thinking = True,+ thinking_budget_tokens = budget,+ )+ )++ [payload] = payloads+ assert payload.get("thinking_budget_tokens") == budget+ assert payload["chat_template_kwargs"] == {"enable_thinking": True}++[email protected]("budget", [None, 128])+def test_every_tool_loop_round_sends_the_budget(monkeypatch, budget):+ payloads: list[dict] = []+ backend = _backend(+ monkeypatch,+ [[_tool_call(), _done()], [_sse({"content": "done"}), _done()]],+ payloads,+ )+ monkeypatch.setattr(+ "core.inference.tools.execute_tool",+ lambda name, arguments, **_kwargs: "a result",+ )++ list(+ backend.generate_chat_completion_with_tools(+ messages = [{"role": "user", "content": "search"}],+ tools = [_LOOKUP_TOOL],+ enable_thinking = True,+ max_tool_iterations = 3,+ thinking_budget_tokens = budget,+ )+ )++ assert len(payloads) == 2+ assert [p.get("thinking_budget_tokens") for p in payloads] == [budget, budget]++[email protected]("budget", [None, 128])+def test_tool_loop_final_pass_sends_the_budget(monkeypatch, budget):+ payloads: list[dict] = []+ backend = _backend(monkeypatch, [[_sse({"content": "done"}), _done()]], payloads)++ list(+ backend.generate_chat_completion_with_tools(+ messages = [{"role": "user", "content": "hi"}],+ tools = [],+ enable_thinking = True,+ max_tool_iterations = 0,+ thinking_budget_tokens = budget,+ )+ )++ [payload] = payloads+ assert payload.get("thinking_budget_tokens") == budget