Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Respect the thinking budget on /v1/messages (#11724)

Nilay7ff3d23af6ba
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index c2c021edadc..c80060abb9a 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -35220,6 +35220,7 @@ class LlamaCppBackend:
compaction_headroom_ratio: Optional[float] = None,
thread_id: Optional[str] = None,
tools_withheld: bool = False,
+ thinking_budget_tokens: Optional[int] = None,
_allow_respawn_retry: bool = True,
) -> Generator[Union[str, dict], None, None]:
"""
@@ -35274,6 +35275,8 @@ class LlamaCppBackend:
)
if _reasoning_kw is not None:
payload["chat_template_kwargs"] = _reasoning_kw
+ if thinking_budget_tokens is not None:
+ payload["thinking_budget_tokens"] = thinking_budget_tokens
if continue_final_message:
# llama-server applies the template; it rejects both flags set true.
payload["continue_final_message"] = True
@@ -35582,6 +35585,7 @@ class LlamaCppBackend:
# The retry refits, so it must be told the same about this request's
# tools as the first attempt was.
tools_withheld = tools_withheld,
+ thinking_budget_tokens = thinking_budget_tokens,
_allow_respawn_retry = False,
)
return
@@ -35643,6 +35647,7 @@ class LlamaCppBackend:
# where the previous round's request has completed.
on_conversation_grew: Optional[Callable[[list], None]] = None,
on_decode_slot: Optional[Callable[[str, int], None]] = None,
+ thinking_budget_tokens: Optional[int] = None,
) -> Generator[dict, None, None]:
"""
Agentic loop: let the model call tools, execute them, and continue.
@@ -36385,6 +36390,8 @@ class LlamaCppBackend:
payload["tool_choice"] = requested_choice
if _reasoning_kw is not None:
payload["chat_template_kwargs"] = _reasoning_kw
+ if thinking_budget_tokens is not None:
+ payload["thinking_budget_tokens"] = thinking_budget_tokens
# Re-checked per iteration: once a tool result is appended the partial is
# no longer trailing, so later turns are normal.
if continue_final_message and trailing_assistant_text(conversation):
@@ -39064,6 +39071,8 @@ class LlamaCppBackend:
stream_payload["logit_bias"] = logit_bias
if _reasoning_kw is not None:
stream_payload["chat_template_kwargs"] = _reasoning_kw
+ if thinking_budget_tokens is not None:
+ stream_payload["thinking_budget_tokens"] = thinking_budget_tokens
stream_payload["max_tokens"] = _final_max_tokens
if stop:
stream_payload["stop"] = stop
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index fa2748757ab..e04eab53edb 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -3720,7 +3720,7 @@ class AnthropicThinkingConfig(BaseModel):
# (adaptive tiers) and Claude Code sends them, and a strict Literal turns an unrecognized value
# into a hard 400. Only "disabled" means off; treat anything else as a request to think.
type: str = "enabled"
- # Accepted for wire compatibility; llama-server has no thinking budget.
+ # Forwarded to llama-server as thinking_budget_tokens when thinking is on.
budget_tokens: Optional[int] = None
model_config = {"extra": "allow"}
@@ -3759,8 +3759,6 @@ class AnthropicMessagesRequest(BaseModel):
)
enable_tools: Optional[bool] = None
enabled_tools: Optional[list[str]] = None
- # Anthropic's native extended-thinking control. Only `type` is honored: llama-server has no
- # thinking-token budget, so `budget_tokens` is accepted and ignored rather than 400'd.
thinking: Optional[AnthropicThinkingConfig] = None
# [x-unsloth] reasoning controls mirroring the OpenAI endpoint. These win
# over `thinking` when both are present, matching enable_tools precedence.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 09dd5e28646..cfbd9519cd3 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -4976,11 +4976,15 @@ def _anthropic_reasoning_args(payload) -> dict:
resolver = getattr(payload, "resolved_enable_thinking", None)
if resolver is not None:
enable_thinking = resolver()
- return {
+ args = {
"enable_thinking": enable_thinking,
"reasoning_effort": reasoning_effort,
"preserve_thinking": payload.preserve_thinking,
}
+ budget = getattr(getattr(payload, "thinking", None), "budget_tokens", None)
+ if enable_thinking and isinstance(budget, int) and budget > 0:
+ args["thinking_budget_tokens"] = budget
+ return args
def _anthropic_preserve_thinking(llama_backend, payload) -> bool:
@@ -38321,6 +38325,7 @@ def _build_passthrough_payload(
seed = None,
stream_options = None,
markup = None,
+ thinking_budget_tokens = None,
):
from core.inference.chat_template_helpers import (
forced_tool_catalog,
@@ -38391,6 +38396,8 @@ def _build_passthrough_payload(
# llama-server renders the Jinja template in the caller's mode instead
# of the model's load-time default.
body["chat_template_kwargs"] = chat_template_kwargs
+ if thinking_budget_tokens is not None:
+ body["thinking_budget_tokens"] = thinking_budget_tokens
return body
@@ -38489,6 +38496,7 @@ async def _anthropic_passthrough_stream(
reasoning_effort = None,
preserve_thinking = None,
parse_think = True,
+ thinking_budget_tokens = None,
):
"""Streaming client-side pass-through: forward tools to llama-server and
translate its stream to Anthropic SSE without executing anything."""
@@ -38511,6 +38519,7 @@ async def _anthropic_passthrough_stream(
chat_template_kwargs = _reasoning_template_kwargs(
llama_backend, enable_thinking, reasoning_effort, preserve_thinking
),
+ thinking_budget_tokens = thinking_budget_tokens,
backend_ctx = llama_backend.context_length,
stream_options = {"include_usage": True},
markup = getattr(llama_backend, "markup_profile", None),
@@ -38825,6 +38834,7 @@ async def _anthropic_passthrough_non_streaming(
reasoning_effort = None,
preserve_thinking = None,
parse_think = True,
+ thinking_budget_tokens = None,
):
"""Non-streaming client-side pass-through.
@@ -38852,6 +38862,7 @@ async def _anthropic_passthrough_non_streaming(
chat_template_kwargs = _reasoning_template_kwargs(
llama_backend, enable_thinking, reasoning_effort, preserve_thinking
),
+ thinking_budget_tokens = thinking_budget_tokens,
backend_ctx = llama_backend.context_length,
markup = getattr(llama_backend, "markup_profile", None),
)
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index 012c5229a82..133eb7f9f49 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -2563,8 +2563,7 @@ class TestAnthropicReasoningArgs:
assert payload.resolved_enable_thinking() is True
def test_budget_tokens_accepted_not_rejected(self):
- """Claude Code always sends budget_tokens; llama-server has no budget,
- so it must be ignored rather than 400'd."""
+ """Claude Code always sends budget_tokens; it must parse rather than 400."""
payload = self._payload(thinking = {"type": "enabled", "budget_tokens": 4096})
assert payload.thinking.budget_tokens == 4096
assert payload.resolved_enable_thinking() is True
@@ -2953,6 +2952,105 @@ class TestAnthropicMessagesToolRouting:
assert captured["seed"] == 3407
+ _BUDGET_CASES = [
+ ({"thinking": {"type": "enabled", "budget_tokens": 128}}, 128),
+ ({"thinking": {"type": "adaptive", "budget_tokens": 128}}, 128),
+ ({"thinking": {"type": "enabled"}}, None),
+ ({"thinking": {"type": "enabled", "budget_tokens": 0}}, None),
+ ({"thinking": {"type": "disabled", "budget_tokens": 128}}, None),
+ ({"thinking": {"type": "enabled", "budget_tokens": 128}, "enable_thinking": False}, None),
+ ({"thinking": {"type": "enabled", "budget_tokens": 128}, "reasoning_effort": "none"}, None),
+ ({}, None),
+ ]
+
+ @pytest.mark.parametrize(("fields", "expected"), _BUDGET_CASES)
+ @pytest.mark.parametrize(
+ ("extra", "expected_path"),
+ [
+ ({}, "plain"),
+ ({"enable_tools": True, "permission_mode": "off"}, "tools"),
+ ],
+ ids = ["plain", "server-tools"],
+ )
+ def test_thinking_budget_reaches_internal_anthropic_generation(
+ self, monkeypatch, extra, expected_path, fields, expected
+ ):
+ backend = _mock_backend(monkeypatch)
+
+ _drive(
+ anthropic_messages(
+ _basic_payload(**fields, **extra),
+ request = self._Request(),
+ current_subject = "t",
+ )
+ )
+
+ [(path, kwargs)] = backend.calls
+ assert path == expected_path
+ assert kwargs.get("thinking_budget_tokens") == expected
+
+ @pytest.mark.parametrize(("fields", "expected"), _BUDGET_CASES)
+ @pytest.mark.parametrize("stream", [False, True])
+ def test_thinking_budget_reaches_anthropic_client_tool_passthrough(
+ self, monkeypatch, stream, fields, expected
+ ):
+ import routes.inference as inf_mod
+ from fastapi.responses import JSONResponse
+
+ _mock_backend(monkeypatch)
+ captured = {}
+
+ async def _passthrough(*args, **kwargs):
+ captured.update(kwargs)
+ return JSONResponse({"type": "message", "content": []})
+
+ helper = (
+ "_anthropic_passthrough_stream" if stream else "_anthropic_passthrough_non_streaming"
+ )
+ monkeypatch.setattr(inf_mod, helper, _passthrough)
+ payload = _basic_payload(
+ stream = stream,
+ tools = [{"name": "lookup", "input_schema": {"type": "object"}}],
+ **fields,
+ )
+
+ _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t"))
+
+ assert captured.get("thinking_budget_tokens") == expected
+
+ @pytest.mark.parametrize("budget", [None, 128])
+ @pytest.mark.parametrize("stream", [False, True])
+ def test_passthrough_puts_thinking_budget_on_the_llama_server_body(
+ self, monkeypatch, stream, budget
+ ):
+ import routes.inference as inf_mod
+
+ real_builder = inf_mod._build_passthrough_payload
+ bodies = []
+
+ def _builder(*args, **kwargs):
+ bodies.append(real_builder(*args, **kwargs))
+ raise RuntimeError("body built")
+
+ monkeypatch.setattr(inf_mod, "_build_passthrough_payload", _builder)
+ backend = SimpleNamespace(base_url = "http://llama.test", context_length = 4096)
+ messages = [{"role": "user", "content": "hi"}]
+ common = (messages, [], 0.7, 0.95, 20, 16, "msg_1", "test-model")
+ if stream:
+ coro = inf_mod._anthropic_passthrough_stream(
+ self._Request(), threading.Event(), backend, *common, thinking_budget_tokens = budget
+ )
+ else:
+ coro = inf_mod._anthropic_passthrough_non_streaming(
+ backend, *common, thinking_budget_tokens = budget
+ )
+
+ with pytest.raises(RuntimeError, match = "body built"):
+ _drive(coro)
+
+ [body] = bodies
+ assert body.get("thinking_budget_tokens") == budget
+
def test_client_tool_catalog_without_passthrough_is_rejected(self, monkeypatch):
# /v1/chat/completions 400s this; /v1/messages answered in prose instead.
backend = _mock_backend(monkeypatch, supports_tools = False, supports_tool_passthrough = False)
diff --git a/studio/backend/tests/test_llama_admission_compat_matrix.py b/studio/backend/tests/test_llama_admission_compat_matrix.py
index b4b4688523d..c376c9b6d50 100644
--- a/studio/backend/tests/test_llama_admission_compat_matrix.py
+++ b/studio/backend/tests/test_llama_admission_compat_matrix.py
@@ -185,7 +185,7 @@ class TestNothingChangesWhenTheBudgetIsUnknown:
# In the order they were added, so older positional callers keep their meaning.
-_TOOL_LOOP_HOOKS = ("on_conversation_grew", "on_decode_slot")
+_TOOL_LOOP_HOOKS = ("on_conversation_grew", "on_decode_slot", "thinking_budget_tokens")
class TestOldCallers:
diff --git a/studio/backend/tests/test_llama_thinking_budget.py b/studio/backend/tests/test_llama_thinking_budget.py
new file mode 100644
index 00000000000..0f282e0be03
--- /dev/null
+++ b/studio/backend/tests/test_llama_thinking_budget.py
@@ -0,0 +1,153 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A per-request thinking budget must reach every llama-server request a generation makes."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.llama_cpp import LlamaCppBackend
+
+_LOOKUP_TOOL = {
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "description": "Search the web.",
+ "parameters": {
+ "type": "object",
+ "properties": {"query": {"type": "string"}},
+ "required": ["query"],
+ },
+ },
+}
+
+
+def _sse(delta: dict) -> str:
+ return "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n"
+
+
+def _done() -> str:
+ return "data: [DONE]\n"
+
+
+def _tool_call() -> str:
+ return _sse(
+ {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_0",
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "arguments": json.dumps({"query": "q"}),
+ },
+ }
+ ]
+ }
+ )
+
+
+def _backend(monkeypatch, streams: list, payloads: list) -> LlamaCppBackend:
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._process = object()
+ backend._healthy = True
+ backend._port = 48852
+ backend._api_key = None
+ backend._effective_context_length = 4096
+ backend._supports_reasoning = True
+ backend._reasoning_always_on = False
+ backend._reasoning_style = "enable_thinking"
+ backend._supports_preserve_thinking = False
+
+ @contextlib.contextmanager
+ def fake_stream_with_retry(_client, _url, payload, _cancel_event, **_kwargs):
+ payloads.append(copy.deepcopy(payload))
+ yield type("FakeResponse", (), {"status_code": 200, "chunks": streams.pop(0)})()
+
+ def fake_iter_text_cancellable(
+ response,
+ _cancel_event,
+ first_token_deadline = None,
+ ):
+ yield from response.chunks
+
+ monkeypatch.setattr(backend, "_stream_with_retry", fake_stream_with_retry)
+ monkeypatch.setattr(backend, "_iter_text_cancellable", fake_iter_text_cancellable)
+ monkeypatch.setattr(backend, "_maybe_recover_from_mtp_crash", lambda *_a, **_k: False)
+ return backend
+
+
[email protected]("budget", [None, 128])
+def test_plain_generation_sends_the_budget(monkeypatch, budget):
+ payloads: list[dict] = []
+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)
+
+ list(
+ backend.generate_chat_completion(
+ messages = [{"role": "user", "content": "hi"}],
+ enable_thinking = True,
+ thinking_budget_tokens = budget,
+ )
+ )
+
+ [payload] = payloads
+ assert payload.get("thinking_budget_tokens") == budget
+ assert payload["chat_template_kwargs"] == {"enable_thinking": True}
+
+
[email protected]("budget", [None, 128])
+def test_every_tool_loop_round_sends_the_budget(monkeypatch, budget):
+ payloads: list[dict] = []
+ backend = _backend(
+ monkeypatch,
+ [[_tool_call(), _done()], [_sse({"content": "done"}), _done()]],
+ payloads,
+ )
+ monkeypatch.setattr(
+ "core.inference.tools.execute_tool",
+ lambda name, arguments, **_kwargs: "a result",
+ )
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "search"}],
+ tools = [_LOOKUP_TOOL],
+ enable_thinking = True,
+ max_tool_iterations = 3,
+ thinking_budget_tokens = budget,
+ )
+ )
+
+ assert len(payloads) == 2
+ assert [p.get("thinking_budget_tokens") for p in payloads] == [budget, budget]
+
+
[email protected]("budget", [None, 128])
+def test_tool_loop_final_pass_sends_the_budget(monkeypatch, budget):
+ payloads: list[dict] = []
+ backend = _backend(monkeypatch, [[_sse({"content": "done"}), _done()]], payloads)
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "hi"}],
+ tools = [],
+ enable_thinking = True,
+ max_tool_iterations = 0,
+ thinking_budget_tokens = budget,
+ )
+ )
+
+ [payload] = payloads
+ assert payload.get("thinking_budget_tokens") == budget