Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: forward a request's other chat_template_kwargs to llama-server (#16567)
Debojyoti Banerjeeaccbbcc599fe
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex f879b187070..9a8bf41d04b 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -2472,6 +2472,13 @@ _MAX_IDENTICAL_TOOL_RESULTS = 3_UNSET = object()+def _with_template_extras(reasoning_kw: Optional[dict], extras: Optional[dict]) -> Optional[dict]:+ """The caller's other template tags (e.g. verification) under Studio's resolved reasoning keys."""+ if not extras:+ return reasoning_kw+ return {**extras, **(reasoning_kw or {})}++# A result with this tail is the window's answer, not the tool's._WINDOW_NOTICE_MARKER = "chars for the model;"# Zero-budget results use _zero_room_stub, which lacks the marker above; missing it lets a model re-read slices forever, so catch it too.@@ -36972,6 +36979,7 @@ class LlamaCppBackend:thinking_budget_tokens: Optional[int] = None,_allow_respawn_retry: bool = True,compact_context: bool = False,+ chat_template_kwargs: Optional[dict] = None,) -> Generator[Union[str, dict], None, None]:"""Streams a chat completion via /v1/chat/completions (llama-server applies template and vision); yields cumulative text like InferenceBackend."""if not self.is_loaded:@@ -37020,9 +37028,10 @@ class LlamaCppBackend:payload["timings_per_token"] = Trueif logit_bias:payload["logit_bias"] = logit_bias- # Per-request enable_thinking / reasoning_effort / preserve_thinking- _reasoning_kw = self._request_reasoning_kwargs(- enable_thinking, reasoning_effort, preserve_thinking+ # Per-request enable_thinking / reasoning_effort / preserve_thinking, plus the caller's other tags+ _reasoning_kw = _with_template_extras(+ self._request_reasoning_kwargs(enable_thinking, reasoning_effort, preserve_thinking),+ chat_template_kwargs,)if _reasoning_kw is not None:payload["chat_template_kwargs"] = _reasoning_kw@@ -37196,6 +37205,7 @@ class LlamaCppBackend:tools_withheld = tools_withheld,thinking_budget_tokens = thinking_budget_tokens,compact_context = compact_context,+ chat_template_kwargs = chat_template_kwargs,_allow_respawn_retry = False,)@@ -37416,6 +37426,7 @@ class LlamaCppBackend:compact_context: bool = False,website_policy: Optional[dict] = None,live_permissions = None,+ chat_template_kwargs: Optional[dict] = None,) -> Generator[dict, None, None]:"""Agentic loop: let the model call tools, execute them, and continue.@@ -37990,8 +38001,11 @@ class LlamaCppBackend:neutralize_control_markup_in_messages,)- _reasoning_kw = self._request_reasoning_kwargs(- _effective_enable_thinking, _effective_reasoning_effort, preserve_thinking+ _reasoning_kw = _with_template_extras(+ self._request_reasoning_kwargs(+ _effective_enable_thinking, _effective_reasoning_effort, preserve_thinking+ ),+ chat_template_kwargs,)# On a continuation only the remainder of the caller's cap; read before the payload._iteration_max_tokens = (@@ -39155,8 +39169,9 @@ class LlamaCppBackend:)_cap_left_l = _loop_budget_left(_spent_l)# Recompute kwargs with thinking off so admission prices the real request.- _off_kw_l = self._request_reasoning_kwargs(- False, None, preserve_thinking+ _off_kw_l = _with_template_extras(+ self._request_reasoning_kwargs(False, None, preserve_thinking),+ chat_template_kwargs,)# Counted ONCE normally: eviction below runs only on a candidate already found too large._fits_l = _cap_left_l != 0 and _loop_continuation_fits(@@ -40612,8 +40627,9 @@ class LlamaCppBackend:_final_tools = _last_sent_toolsfrom core.inference.chat_template_helpers import neutralize_control_markup_in_messages- _reasoning_kw = self._request_reasoning_kwargs(- enable_thinking, reasoning_effort, preserve_thinking+ _reasoning_kw = _with_template_extras(+ self._request_reasoning_kwargs(enable_thinking, reasoning_effort, preserve_thinking),+ chat_template_kwargs,)_final_max_tokens = (max_tokens@@ -41252,7 +41268,10 @@ class LlamaCppBackend:_candidate_r, None, self.markup_profile)# Admit with thinking-off kwargs, matching the request actually sent.- _off_kw = self._request_reasoning_kwargs(False, None, preserve_thinking)+ _off_kw = _with_template_extras(+ self._request_reasoning_kwargs(False, None, preserve_thinking),+ chat_template_kwargs,+ )_next_cap_r = _remaining_output_budget()# Protect the current user turn and recovery tail._recovery_tail = _candidate_r[-2:]diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex cf1b9dfaebd..86d867c306f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -2337,6 +2337,7 @@ def _count_gguf_admission_prompt(payload.reasoning_effort,_replay_preserve_thinking(llama_backend, payload, messages),)+ template_kwargs = _with_extra_chat_template_kwargs(payload, template_kwargs)continue_final = (_continue_final_message(payload, thought = True)and trailing_assistant_resume_kind(messages) is not None@@ -3734,6 +3735,7 @@ import zlibfrom utils.current_date_prompt_settings import (CURRENT_DATE_PROMPT_PREFIX,+ PROBE_RENDER_KEYS,PROBE_TOOLS,contains_current_date_prompt_line,current_date_prompt_line,@@ -6419,13 +6421,21 @@ def _local_template_system_turn(def _template_controls(payload: Any) -> tuple:- """The reasoning controls this request's chat template is rendered with."""+ """The template kwargs this request's chat template is rendered with."""names = ("enable_thinking", "reasoning_effort", "preserve_thinking")values = [getattr(payload, name, None) for name in names]try:llama = get_llama_cpp_backend()if llama.is_loaded:- return tuple(sorted((llama._request_reasoning_kwargs(*values) or {}).items()))+ # Only llama-server receives the extras; scalars only, the probe is lru-cached on them.+ extras = {+ key: value+ for key, value in (_extra_chat_template_kwargs(payload) or {}).items()+ if key not in PROBE_RENDER_KEYS+ and (value is None or isinstance(value, (str, int, float, bool)))+ }+ kwargs = llama._request_reasoning_kwargs(*values) or {}+ return tuple(sorted({**extras, **kwargs}.items()))except Exception:passreturn tuple((name, value) for name, value in zip(names, values) if value is not None)@@ -30230,6 +30240,26 @@ def _consume_chat_template_reasoning_kwargs(payload) -> None:}+def _extra_chat_template_kwargs(payload) -> Optional[dict]:+ """Non-reasoning ``chat_template_kwargs`` (e.g. ``verification``), forwarded to llama-server as sent (#16545)."""+ extra = getattr(payload, "model_extra", None)+ template_kwargs = extra.get("chat_template_kwargs") if isinstance(extra, dict) else None+ if not isinstance(template_kwargs, dict):+ return None+ extras = {+ key: value+ for key, value in template_kwargs.items()+ if isinstance(key, str) and key not in _CHAT_TEMPLATE_REASONING_KEYS+ }+ return extras or None+++def _with_extra_chat_template_kwargs(payload, template_kwargs: Optional[dict]) -> Optional[dict]:+ """Studio's resolved reasoning keys win over same-named caller extras."""+ extras = _extra_chat_template_kwargs(payload)+ return {**extras, **(template_kwargs or {})} if extras else template_kwargs++def _resolve_reasoning_controls(enable_thinking: Optional[bool], reasoning_effort: Optional[str]) -> tuple[Optional[bool], Optional[str]]:@@ -32195,6 +32225,7 @@ async def produce_openai_chat_completions(sandbox_level = payload.sandbox_level,perf_callback = _gguf_perf_callback,on_conversation_grew = _gguf_recost,+ chat_template_kwargs = _extra_chat_template_kwargs(payload),# Only streams park and reclaim; an MCP image forces the approval wait in any mode.on_decode_slot = _gguf_record_decode_slotif payload.stream@@ -33050,6 +33081,7 @@ async def produce_openai_chat_completions(thread_id = payload.thread_id,# Tell the epoch gate tools are withheld; it cannot see that from process policy.tools_withheld = _tool_loop_unusable,+ chat_template_kwargs = _extra_chat_template_kwargs(payload),)_gguf_sentinel = object()@@ -39274,6 +39306,10 @@ def _build_chat_request(if response_format is not None:chat_kwargs["response_format"] = response_format+ _template_extras = _extra_chat_template_kwargs(payload)+ if _template_extras:+ chat_kwargs["chat_template_kwargs"] = _template_extras+chat_request = ChatCompletionRequest(**chat_kwargs)_normalize_chat_reasoning_controls(chat_request)return chat_request@@ -42383,6 +42419,7 @@ async def chat_count_tokens(# Off the loop: first use probes the template.await asyncio.to_thread(_replay_preserve_thinking, llama_backend, payload, openai_messages),)+ _template_kwargs = _with_extra_chat_template_kwargs(payload, _template_kwargs)# Another tab's load can swap the tokenizer unseen by the caller's guard, so report the id._tokenizer_model = _llama_status_checkpoint_id(llama_backend)@@ -46413,6 +46450,7 @@ def _build_openai_passthrough_body(if llama_backend is not Noneelse None)+ tpl_kwargs = _with_extra_chat_template_kwargs(payload, tpl_kwargs)response_format = _response_format_for_llama_server(_response_format_beside_tools(payload, tools))diff --git a/studio/backend/tests/test_extra_chat_template_kwargs.py b/studio/backend/tests/test_extra_chat_template_kwargs.pynew file mode 100644index 00000000000..459123669f5--- /dev/null+++ b/studio/backend/tests/test_extra_chat_template_kwargs.py@@ -0,0 +1,317 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A caller's own chat_template_kwargs (e.g. a model's ``verification`` tag) must reach llama-server+beside Studio's reasoning keys instead of being dropped (#16545)."""++from __future__ import annotations++import contextlib+import copy+import json+import sys+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from core.inference.llama_cpp import LlamaCppBackend+from models.inference import ChatCompletionRequest, ChatMessage, ResponsesRequest+from routes.inference import (+ _build_chat_request,+ _build_openai_passthrough_body,+ _extra_chat_template_kwargs,+ _normalize_chat_reasoning_controls,+)++_LOOKUP_TOOL = {+ "type": "function",+ "function": {+ "name": "web_search",+ "description": "Search the web.",+ "parameters": {+ "type": "object",+ "properties": {"query": {"type": "string"}},+ "required": ["query"],+ },+ },+}+++def _sse(delta: dict) -> str:+ return "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n"+++def _done() -> str:+ return "data: [DONE]\n"+++def _tool_call() -> str:+ return _sse(+ {+ "tool_calls": [+ {+ "index": 0,+ "id": "call_0",+ "type": "function",+ "function": {+ "name": "web_search",+ "arguments": json.dumps({"query": "q"}),+ },+ }+ ]+ }+ )+++def _reasoning_backend() -> LlamaCppBackend:+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._supports_reasoning = True+ backend._reasoning_always_on = False+ backend._reasoning_style = "enable_thinking"+ backend._supports_preserve_thinking = False+ return backend+++def _backend(monkeypatch, streams: list, payloads: list) -> LlamaCppBackend:+ backend = _reasoning_backend()+ backend._process = object()+ backend._healthy = True+ backend._port = 48853+ backend._api_key = None+ backend._effective_context_length = 4096++ @contextlib.contextmanager+ def fake_stream_with_retry(_client, _url, payload, _cancel_event, **_kwargs):+ payloads.append(copy.deepcopy(payload))+ yield type("FakeResponse", (), {"status_code": 200, "chunks": streams.pop(0)})()++ def fake_iter_text_cancellable(+ response,+ _cancel_event,+ first_token_deadline = None,+ ):+ yield from response.chunks++ monkeypatch.setattr(backend, "_stream_with_retry", fake_stream_with_retry)+ monkeypatch.setattr(backend, "_iter_text_cancellable", fake_iter_text_cancellable)+ monkeypatch.setattr(backend, "_maybe_recover_from_mtp_crash", lambda *_a, **_k: False)+ return backend+++def _chat_payload(**fields) -> ChatCompletionRequest:+ return ChatCompletionRequest(+ model = "default", messages = [{"role": "user", "content": "hi"}], **fields+ )+++def test_plain_generation_sends_the_extra_keys(monkeypatch):+ payloads: list[dict] = []+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)++ list(+ backend.generate_chat_completion(+ messages = [{"role": "user", "content": "hi"}],+ enable_thinking = True,+ chat_template_kwargs = {"verification": True},+ )+ )++ [payload] = payloads+ assert payload["chat_template_kwargs"] == {"verification": True, "enable_thinking": True}+++def test_extra_keys_go_out_even_without_a_reasoning_control(monkeypatch):+ payloads: list[dict] = []+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)++ list(+ backend.generate_chat_completion(+ messages = [{"role": "user", "content": "hi"}],+ chat_template_kwargs = {"verification": True},+ )+ )++ [payload] = payloads+ assert payload["chat_template_kwargs"] == {"verification": True}+++def test_studio_reasoning_keys_win_over_the_callers(monkeypatch):+ payloads: list[dict] = []+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)++ list(+ backend.generate_chat_completion(+ messages = [{"role": "user", "content": "hi"}],+ enable_thinking = True,+ chat_template_kwargs = {"enable_thinking": False, "verification": True},+ )+ )++ [payload] = payloads+ assert payload["chat_template_kwargs"] == {"enable_thinking": True, "verification": True}+++def test_no_extra_keys_keeps_the_payload_unchanged(monkeypatch):+ payloads: list[dict] = []+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)++ list(backend.generate_chat_completion(messages = [{"role": "user", "content": "hi"}]))++ [payload] = payloads+ assert "chat_template_kwargs" not in payload+++def test_every_tool_loop_round_sends_the_extra_keys(monkeypatch):+ payloads: list[dict] = []+ backend = _backend(+ monkeypatch,+ [[_tool_call(), _done()], [_sse({"content": "done"}), _done()]],+ payloads,+ )+ monkeypatch.setattr(+ "core.inference.tools.execute_tool",+ lambda name, arguments, **_kwargs: "a result",+ )++ list(+ backend.generate_chat_completion_with_tools(+ messages = [{"role": "user", "content": "search"}],+ tools = [_LOOKUP_TOOL],+ enable_thinking = True,+ max_tool_iterations = 3,+ chat_template_kwargs = {"verification": True},+ )+ )++ assert len(payloads) == 2+ for payload in payloads:+ assert payload["chat_template_kwargs"]["verification"] is True+ assert payload["chat_template_kwargs"]["enable_thinking"] is True+++def test_tool_loop_final_pass_sends_the_extra_keys(monkeypatch):+ payloads: list[dict] = []+ backend = _backend(monkeypatch, [[_sse({"content": "done"}), _done()]], payloads)++ list(+ backend.generate_chat_completion_with_tools(+ messages = [{"role": "user", "content": "hi"}],+ tools = [],+ enable_thinking = True,+ max_tool_iterations = 0,+ chat_template_kwargs = {"verification": True},+ )+ )++ [payload] = payloads+ assert payload["chat_template_kwargs"] == {"verification": True, "enable_thinking": True}+++def test_the_route_lifts_reasoning_keys_and_keeps_the_rest():+ payload = _chat_payload(+ chat_template_kwargs = {"enable_thinking": False, "verification": True, "mode": "strict"}+ )++ _normalize_chat_reasoning_controls(payload)++ assert payload.enable_thinking is False+ assert _extra_chat_template_kwargs(payload) == {"verification": True, "mode": "strict"}+++ "template_kwargs",+ [None, {}, {"enable_thinking": True}, "verification"],+)+def test_nothing_extra_to_forward(template_kwargs):+ fields = {} if template_kwargs is None else {"chat_template_kwargs": template_kwargs}+ payload = _chat_payload(**fields)++ _normalize_chat_reasoning_controls(payload)++ assert _extra_chat_template_kwargs(payload) is None+++def test_client_tools_passthrough_sends_the_extra_keys():+ payload = _chat_payload(+ chat_template_kwargs = {"enable_thinking": True, "verification": True},+ tools = [_LOOKUP_TOOL],+ )+ _normalize_chat_reasoning_controls(payload)++ body = _build_openai_passthrough_body(+ payload, backend_ctx = 4096, llama_backend = _reasoning_backend()+ )++ assert body["chat_template_kwargs"] == {"verification": True, "enable_thinking": True}+++def test_responses_requests_carry_the_extra_keys():+ payload = ResponsesRequest(+ input = "hi",+ chat_template_kwargs = {"enable_thinking": True, "verification": True},+ )+ messages = [ChatMessage(role = "user", content = "hi")]++ chat_req = _build_chat_request(payload, messages, stream = False)++ assert chat_req.enable_thinking is True+ assert _extra_chat_template_kwargs(chat_req) == {"verification": True}+++def test_the_public_token_count_prices_the_extra_keys(monkeypatch):+ from .test_openai_auto_switch import _count_request, _count_tokens_backend, _counted_body++ _switched, counted = _count_tokens_backend(monkeypatch)+ _counted_body(+ _count_request(+ [{"role": "user", "content": "hi"}],+ chat_template_kwargs = {"enable_thinking": True, "verification": True},+ )+ )++ assert counted["chat_template_kwargs"] == {"verification": True, "enable_thinking": True}+++def test_the_default_system_probe_renders_with_the_extra_keys(monkeypatch):+ import routes.inference as inference_route+ from utils.current_date_prompt_settings import template_system_turn+ from datetime import date++ backend = _reasoning_backend()+ backend._process = object()+ backend._healthy = True+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend)+ payload = _chat_payload(+ chat_template_kwargs = {+ "enable_thinking": True,+ "verification": True,+ "nested": {"a": 1},+ "documents": None,+ }+ )+ _normalize_chat_reasoning_controls(payload)++ controls = inference_route._template_controls(payload)++ assert dict(controls) == {"verification": True, "enable_thinking": True}+ template = (+ "{% if messages[0].role != 'system' %}<system>{{ 'VERIFY' if verification else 'PLAIN' }}</system>"+ "{% endif %}{% for m in messages %}<{{ m.role }}>{{ m.content }}</{{ m.role }}>{% endfor %}"+ )+ assert template_system_turn(template, date(2026, 1, 1), False, controls)[1] == "VERIFY"+++def test_in_process_backends_probe_without_the_extras(monkeypatch):+ import routes.inference as inference_route++ backend = _reasoning_backend()+ backend._process = None+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend)+ payload = _chat_payload(chat_template_kwargs = {"enable_thinking": True, "verification": True})+ _normalize_chat_reasoning_controls(payload)++ assert dict(inference_route._template_controls(payload)) == {"enable_thinking": True}diff --git a/studio/backend/tests/test_llama_admission_compat_matrix.py b/studio/backend/tests/test_llama_admission_compat_matrix.pyindex f8f3314b089..2dc78a533f4 100644--- a/studio/backend/tests/test_llama_admission_compat_matrix.py+++ b/studio/backend/tests/test_llama_admission_compat_matrix.py@@ -237,7 +237,12 @@ class TestOldCallers:inspect.signature(LlamaCppBackend.generate_chat_completion_with_tools).parameters)assert names.index("sandbox_level") == names.index("instruction_anchor_ids") + 1- assert names[-3:] == ["compact_context", "website_policy", "live_permissions"]+ assert names[-4:] == [+ "compact_context",+ "website_policy",+ "live_permissions",+ "chat_template_kwargs",+ ]assert names.index("promote_reasoning_only") == names.index("permission_mode") + 1def test_the_wait_timeout_has_a_sane_default(self):diff --git a/studio/backend/utils/current_date_prompt_settings.py b/studio/backend/utils/current_date_prompt_settings.pyindex 7652b078785..a9924cbe258 100644--- a/studio/backend/utils/current_date_prompt_settings.py+++ b/studio/backend/utils/current_date_prompt_settings.py@@ -122,6 +122,10 @@ _PROBE_SPECIAL_TOKENS = {f"{name}_token": f"UNSLOTH_DATE_PROBE_{name.upper()}"for name in ("bos", "eos", "pad", "unk", "sep", "cls", "mask")}+# Names the probe passes itself; a caller's template kwarg of the same name would collide.+PROBE_RENDER_KEYS = frozenset(+ {"messages", "tools", "documents", "add_generation_prompt", *_PROBE_SPECIAL_TOKENS}+)# a catalog a tool request's branch can render, for probing the template it selects.PROBE_TOOLS = [{