Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: forward a request's other chat_template_kwargs to llama-server (#16567)

Debojyoti Banerjeeaccbbcc599fe
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index f879b187070..9a8bf41d04b 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2472,6 +2472,13 @@ _MAX_IDENTICAL_TOOL_RESULTS = 3
_UNSET = object()
+def _with_template_extras(reasoning_kw: Optional[dict], extras: Optional[dict]) -> Optional[dict]:
+ """The caller's other template tags (e.g. verification) under Studio's resolved reasoning keys."""
+ if not extras:
+ return reasoning_kw
+ return {**extras, **(reasoning_kw or {})}
+
+
# A result with this tail is the window's answer, not the tool's.
_WINDOW_NOTICE_MARKER = "chars for the model;"
# Zero-budget results use _zero_room_stub, which lacks the marker above; missing it lets a model re-read slices forever, so catch it too.
@@ -36972,6 +36979,7 @@ class LlamaCppBackend:
thinking_budget_tokens: Optional[int] = None,
_allow_respawn_retry: bool = True,
compact_context: bool = False,
+ chat_template_kwargs: Optional[dict] = None,
) -> Generator[Union[str, dict], None, None]:
"""Streams a chat completion via /v1/chat/completions (llama-server applies template and vision); yields cumulative text like InferenceBackend."""
if not self.is_loaded:
@@ -37020,9 +37028,10 @@ class LlamaCppBackend:
payload["timings_per_token"] = True
if logit_bias:
payload["logit_bias"] = logit_bias
- # Per-request enable_thinking / reasoning_effort / preserve_thinking
- _reasoning_kw = self._request_reasoning_kwargs(
- enable_thinking, reasoning_effort, preserve_thinking
+ # Per-request enable_thinking / reasoning_effort / preserve_thinking, plus the caller's other tags
+ _reasoning_kw = _with_template_extras(
+ self._request_reasoning_kwargs(enable_thinking, reasoning_effort, preserve_thinking),
+ chat_template_kwargs,
)
if _reasoning_kw is not None:
payload["chat_template_kwargs"] = _reasoning_kw
@@ -37196,6 +37205,7 @@ class LlamaCppBackend:
tools_withheld = tools_withheld,
thinking_budget_tokens = thinking_budget_tokens,
compact_context = compact_context,
+ chat_template_kwargs = chat_template_kwargs,
_allow_respawn_retry = False,
)
@@ -37416,6 +37426,7 @@ class LlamaCppBackend:
compact_context: bool = False,
website_policy: Optional[dict] = None,
live_permissions = None,
+ chat_template_kwargs: Optional[dict] = None,
) -> Generator[dict, None, None]:
"""
Agentic loop: let the model call tools, execute them, and continue.
@@ -37990,8 +38001,11 @@ class LlamaCppBackend:
neutralize_control_markup_in_messages,
)
- _reasoning_kw = self._request_reasoning_kwargs(
- _effective_enable_thinking, _effective_reasoning_effort, preserve_thinking
+ _reasoning_kw = _with_template_extras(
+ self._request_reasoning_kwargs(
+ _effective_enable_thinking, _effective_reasoning_effort, preserve_thinking
+ ),
+ chat_template_kwargs,
)
# On a continuation only the remainder of the caller's cap; read before the payload.
_iteration_max_tokens = (
@@ -39155,8 +39169,9 @@ class LlamaCppBackend:
)
_cap_left_l = _loop_budget_left(_spent_l)
# Recompute kwargs with thinking off so admission prices the real request.
- _off_kw_l = self._request_reasoning_kwargs(
- False, None, preserve_thinking
+ _off_kw_l = _with_template_extras(
+ self._request_reasoning_kwargs(False, None, preserve_thinking),
+ chat_template_kwargs,
)
# Counted ONCE normally: eviction below runs only on a candidate already found too large.
_fits_l = _cap_left_l != 0 and _loop_continuation_fits(
@@ -40612,8 +40627,9 @@ class LlamaCppBackend:
_final_tools = _last_sent_tools
from core.inference.chat_template_helpers import neutralize_control_markup_in_messages
- _reasoning_kw = self._request_reasoning_kwargs(
- enable_thinking, reasoning_effort, preserve_thinking
+ _reasoning_kw = _with_template_extras(
+ self._request_reasoning_kwargs(enable_thinking, reasoning_effort, preserve_thinking),
+ chat_template_kwargs,
)
_final_max_tokens = (
max_tokens
@@ -41252,7 +41268,10 @@ class LlamaCppBackend:
_candidate_r, None, self.markup_profile
)
# Admit with thinking-off kwargs, matching the request actually sent.
- _off_kw = self._request_reasoning_kwargs(False, None, preserve_thinking)
+ _off_kw = _with_template_extras(
+ self._request_reasoning_kwargs(False, None, preserve_thinking),
+ chat_template_kwargs,
+ )
_next_cap_r = _remaining_output_budget()
# Protect the current user turn and recovery tail.
_recovery_tail = _candidate_r[-2:]
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index cf1b9dfaebd..86d867c306f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2337,6 +2337,7 @@ def _count_gguf_admission_prompt(
payload.reasoning_effort,
_replay_preserve_thinking(llama_backend, payload, messages),
)
+ template_kwargs = _with_extra_chat_template_kwargs(payload, template_kwargs)
continue_final = (
_continue_final_message(payload, thought = True)
and trailing_assistant_resume_kind(messages) is not None
@@ -3734,6 +3735,7 @@ import zlib
from utils.current_date_prompt_settings import (
CURRENT_DATE_PROMPT_PREFIX,
+ PROBE_RENDER_KEYS,
PROBE_TOOLS,
contains_current_date_prompt_line,
current_date_prompt_line,
@@ -6419,13 +6421,21 @@ def _local_template_system_turn(
def _template_controls(payload: Any) -> tuple:
- """The reasoning controls this request's chat template is rendered with."""
+ """The template kwargs this request's chat template is rendered with."""
names = ("enable_thinking", "reasoning_effort", "preserve_thinking")
values = [getattr(payload, name, None) for name in names]
try:
llama = get_llama_cpp_backend()
if llama.is_loaded:
- return tuple(sorted((llama._request_reasoning_kwargs(*values) or {}).items()))
+ # Only llama-server receives the extras; scalars only, the probe is lru-cached on them.
+ extras = {
+ key: value
+ for key, value in (_extra_chat_template_kwargs(payload) or {}).items()
+ if key not in PROBE_RENDER_KEYS
+ and (value is None or isinstance(value, (str, int, float, bool)))
+ }
+ kwargs = llama._request_reasoning_kwargs(*values) or {}
+ return tuple(sorted({**extras, **kwargs}.items()))
except Exception:
pass
return tuple((name, value) for name, value in zip(names, values) if value is not None)
@@ -30230,6 +30240,26 @@ def _consume_chat_template_reasoning_kwargs(payload) -> None:
}
+def _extra_chat_template_kwargs(payload) -> Optional[dict]:
+ """Non-reasoning ``chat_template_kwargs`` (e.g. ``verification``), forwarded to llama-server as sent (#16545)."""
+ extra = getattr(payload, "model_extra", None)
+ template_kwargs = extra.get("chat_template_kwargs") if isinstance(extra, dict) else None
+ if not isinstance(template_kwargs, dict):
+ return None
+ extras = {
+ key: value
+ for key, value in template_kwargs.items()
+ if isinstance(key, str) and key not in _CHAT_TEMPLATE_REASONING_KEYS
+ }
+ return extras or None
+
+
+def _with_extra_chat_template_kwargs(payload, template_kwargs: Optional[dict]) -> Optional[dict]:
+ """Studio's resolved reasoning keys win over same-named caller extras."""
+ extras = _extra_chat_template_kwargs(payload)
+ return {**extras, **(template_kwargs or {})} if extras else template_kwargs
+
+
def _resolve_reasoning_controls(
enable_thinking: Optional[bool], reasoning_effort: Optional[str]
) -> tuple[Optional[bool], Optional[str]]:
@@ -32195,6 +32225,7 @@ async def produce_openai_chat_completions(
sandbox_level = payload.sandbox_level,
perf_callback = _gguf_perf_callback,
on_conversation_grew = _gguf_recost,
+ chat_template_kwargs = _extra_chat_template_kwargs(payload),
# Only streams park and reclaim; an MCP image forces the approval wait in any mode.
on_decode_slot = _gguf_record_decode_slot
if payload.stream
@@ -33050,6 +33081,7 @@ async def produce_openai_chat_completions(
thread_id = payload.thread_id,
# Tell the epoch gate tools are withheld; it cannot see that from process policy.
tools_withheld = _tool_loop_unusable,
+ chat_template_kwargs = _extra_chat_template_kwargs(payload),
)
_gguf_sentinel = object()
@@ -39274,6 +39306,10 @@ def _build_chat_request(
if response_format is not None:
chat_kwargs["response_format"] = response_format
+ _template_extras = _extra_chat_template_kwargs(payload)
+ if _template_extras:
+ chat_kwargs["chat_template_kwargs"] = _template_extras
+
chat_request = ChatCompletionRequest(**chat_kwargs)
_normalize_chat_reasoning_controls(chat_request)
return chat_request
@@ -42383,6 +42419,7 @@ async def chat_count_tokens(
# Off the loop: first use probes the template.
await asyncio.to_thread(_replay_preserve_thinking, llama_backend, payload, openai_messages),
)
+ _template_kwargs = _with_extra_chat_template_kwargs(payload, _template_kwargs)
# Another tab's load can swap the tokenizer unseen by the caller's guard, so report the id.
_tokenizer_model = _llama_status_checkpoint_id(llama_backend)
@@ -46413,6 +46450,7 @@ def _build_openai_passthrough_body(
if llama_backend is not None
else None
)
+ tpl_kwargs = _with_extra_chat_template_kwargs(payload, tpl_kwargs)
response_format = _response_format_for_llama_server(
_response_format_beside_tools(payload, tools)
)
diff --git a/studio/backend/tests/test_extra_chat_template_kwargs.py b/studio/backend/tests/test_extra_chat_template_kwargs.py
new file mode 100644
index 00000000000..459123669f5
--- /dev/null
+++ b/studio/backend/tests/test_extra_chat_template_kwargs.py
@@ -0,0 +1,317 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A caller's own chat_template_kwargs (e.g. a model's ``verification`` tag) must reach llama-server
+beside Studio's reasoning keys instead of being dropped (#16545)."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.llama_cpp import LlamaCppBackend
+from models.inference import ChatCompletionRequest, ChatMessage, ResponsesRequest
+from routes.inference import (
+ _build_chat_request,
+ _build_openai_passthrough_body,
+ _extra_chat_template_kwargs,
+ _normalize_chat_reasoning_controls,
+)
+
+_LOOKUP_TOOL = {
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "description": "Search the web.",
+ "parameters": {
+ "type": "object",
+ "properties": {"query": {"type": "string"}},
+ "required": ["query"],
+ },
+ },
+}
+
+
+def _sse(delta: dict) -> str:
+ return "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n"
+
+
+def _done() -> str:
+ return "data: [DONE]\n"
+
+
+def _tool_call() -> str:
+ return _sse(
+ {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_0",
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "arguments": json.dumps({"query": "q"}),
+ },
+ }
+ ]
+ }
+ )
+
+
+def _reasoning_backend() -> LlamaCppBackend:
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._supports_reasoning = True
+ backend._reasoning_always_on = False
+ backend._reasoning_style = "enable_thinking"
+ backend._supports_preserve_thinking = False
+ return backend
+
+
+def _backend(monkeypatch, streams: list, payloads: list) -> LlamaCppBackend:
+ backend = _reasoning_backend()
+ backend._process = object()
+ backend._healthy = True
+ backend._port = 48853
+ backend._api_key = None
+ backend._effective_context_length = 4096
+
+ @contextlib.contextmanager
+ def fake_stream_with_retry(_client, _url, payload, _cancel_event, **_kwargs):
+ payloads.append(copy.deepcopy(payload))
+ yield type("FakeResponse", (), {"status_code": 200, "chunks": streams.pop(0)})()
+
+ def fake_iter_text_cancellable(
+ response,
+ _cancel_event,
+ first_token_deadline = None,
+ ):
+ yield from response.chunks
+
+ monkeypatch.setattr(backend, "_stream_with_retry", fake_stream_with_retry)
+ monkeypatch.setattr(backend, "_iter_text_cancellable", fake_iter_text_cancellable)
+ monkeypatch.setattr(backend, "_maybe_recover_from_mtp_crash", lambda *_a, **_k: False)
+ return backend
+
+
+def _chat_payload(**fields) -> ChatCompletionRequest:
+ return ChatCompletionRequest(
+ model = "default", messages = [{"role": "user", "content": "hi"}], **fields
+ )
+
+
+def test_plain_generation_sends_the_extra_keys(monkeypatch):
+ payloads: list[dict] = []
+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)
+
+ list(
+ backend.generate_chat_completion(
+ messages = [{"role": "user", "content": "hi"}],
+ enable_thinking = True,
+ chat_template_kwargs = {"verification": True},
+ )
+ )
+
+ [payload] = payloads
+ assert payload["chat_template_kwargs"] == {"verification": True, "enable_thinking": True}
+
+
+def test_extra_keys_go_out_even_without_a_reasoning_control(monkeypatch):
+ payloads: list[dict] = []
+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)
+
+ list(
+ backend.generate_chat_completion(
+ messages = [{"role": "user", "content": "hi"}],
+ chat_template_kwargs = {"verification": True},
+ )
+ )
+
+ [payload] = payloads
+ assert payload["chat_template_kwargs"] == {"verification": True}
+
+
+def test_studio_reasoning_keys_win_over_the_callers(monkeypatch):
+ payloads: list[dict] = []
+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)
+
+ list(
+ backend.generate_chat_completion(
+ messages = [{"role": "user", "content": "hi"}],
+ enable_thinking = True,
+ chat_template_kwargs = {"enable_thinking": False, "verification": True},
+ )
+ )
+
+ [payload] = payloads
+ assert payload["chat_template_kwargs"] == {"enable_thinking": True, "verification": True}
+
+
+def test_no_extra_keys_keeps_the_payload_unchanged(monkeypatch):
+ payloads: list[dict] = []
+ backend = _backend(monkeypatch, [[_sse({"content": "hi"}), _done()]], payloads)
+
+ list(backend.generate_chat_completion(messages = [{"role": "user", "content": "hi"}]))
+
+ [payload] = payloads
+ assert "chat_template_kwargs" not in payload
+
+
+def test_every_tool_loop_round_sends_the_extra_keys(monkeypatch):
+ payloads: list[dict] = []
+ backend = _backend(
+ monkeypatch,
+ [[_tool_call(), _done()], [_sse({"content": "done"}), _done()]],
+ payloads,
+ )
+ monkeypatch.setattr(
+ "core.inference.tools.execute_tool",
+ lambda name, arguments, **_kwargs: "a result",
+ )
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "search"}],
+ tools = [_LOOKUP_TOOL],
+ enable_thinking = True,
+ max_tool_iterations = 3,
+ chat_template_kwargs = {"verification": True},
+ )
+ )
+
+ assert len(payloads) == 2
+ for payload in payloads:
+ assert payload["chat_template_kwargs"]["verification"] is True
+ assert payload["chat_template_kwargs"]["enable_thinking"] is True
+
+
+def test_tool_loop_final_pass_sends_the_extra_keys(monkeypatch):
+ payloads: list[dict] = []
+ backend = _backend(monkeypatch, [[_sse({"content": "done"}), _done()]], payloads)
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "hi"}],
+ tools = [],
+ enable_thinking = True,
+ max_tool_iterations = 0,
+ chat_template_kwargs = {"verification": True},
+ )
+ )
+
+ [payload] = payloads
+ assert payload["chat_template_kwargs"] == {"verification": True, "enable_thinking": True}
+
+
+def test_the_route_lifts_reasoning_keys_and_keeps_the_rest():
+ payload = _chat_payload(
+ chat_template_kwargs = {"enable_thinking": False, "verification": True, "mode": "strict"}
+ )
+
+ _normalize_chat_reasoning_controls(payload)
+
+ assert payload.enable_thinking is False
+ assert _extra_chat_template_kwargs(payload) == {"verification": True, "mode": "strict"}
+
+
+ "template_kwargs",
+ [None, {}, {"enable_thinking": True}, "verification"],
+)
+def test_nothing_extra_to_forward(template_kwargs):
+ fields = {} if template_kwargs is None else {"chat_template_kwargs": template_kwargs}
+ payload = _chat_payload(**fields)
+
+ _normalize_chat_reasoning_controls(payload)
+
+ assert _extra_chat_template_kwargs(payload) is None
+
+
+def test_client_tools_passthrough_sends_the_extra_keys():
+ payload = _chat_payload(
+ chat_template_kwargs = {"enable_thinking": True, "verification": True},
+ tools = [_LOOKUP_TOOL],
+ )
+ _normalize_chat_reasoning_controls(payload)
+
+ body = _build_openai_passthrough_body(
+ payload, backend_ctx = 4096, llama_backend = _reasoning_backend()
+ )
+
+ assert body["chat_template_kwargs"] == {"verification": True, "enable_thinking": True}
+
+
+def test_responses_requests_carry_the_extra_keys():
+ payload = ResponsesRequest(
+ input = "hi",
+ chat_template_kwargs = {"enable_thinking": True, "verification": True},
+ )
+ messages = [ChatMessage(role = "user", content = "hi")]
+
+ chat_req = _build_chat_request(payload, messages, stream = False)
+
+ assert chat_req.enable_thinking is True
+ assert _extra_chat_template_kwargs(chat_req) == {"verification": True}
+
+
+def test_the_public_token_count_prices_the_extra_keys(monkeypatch):
+ from .test_openai_auto_switch import _count_request, _count_tokens_backend, _counted_body
+
+ _switched, counted = _count_tokens_backend(monkeypatch)
+ _counted_body(
+ _count_request(
+ [{"role": "user", "content": "hi"}],
+ chat_template_kwargs = {"enable_thinking": True, "verification": True},
+ )
+ )
+
+ assert counted["chat_template_kwargs"] == {"verification": True, "enable_thinking": True}
+
+
+def test_the_default_system_probe_renders_with_the_extra_keys(monkeypatch):
+ import routes.inference as inference_route
+ from utils.current_date_prompt_settings import template_system_turn
+ from datetime import date
+
+ backend = _reasoning_backend()
+ backend._process = object()
+ backend._healthy = True
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend)
+ payload = _chat_payload(
+ chat_template_kwargs = {
+ "enable_thinking": True,
+ "verification": True,
+ "nested": {"a": 1},
+ "documents": None,
+ }
+ )
+ _normalize_chat_reasoning_controls(payload)
+
+ controls = inference_route._template_controls(payload)
+
+ assert dict(controls) == {"verification": True, "enable_thinking": True}
+ template = (
+ "{% if messages[0].role != 'system' %}<system>{{ 'VERIFY' if verification else 'PLAIN' }}</system>"
+ "{% endif %}{% for m in messages %}<{{ m.role }}>{{ m.content }}</{{ m.role }}>{% endfor %}"
+ )
+ assert template_system_turn(template, date(2026, 1, 1), False, controls)[1] == "VERIFY"
+
+
+def test_in_process_backends_probe_without_the_extras(monkeypatch):
+ import routes.inference as inference_route
+
+ backend = _reasoning_backend()
+ backend._process = None
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend)
+ payload = _chat_payload(chat_template_kwargs = {"enable_thinking": True, "verification": True})
+ _normalize_chat_reasoning_controls(payload)
+
+ assert dict(inference_route._template_controls(payload)) == {"enable_thinking": True}
diff --git a/studio/backend/tests/test_llama_admission_compat_matrix.py b/studio/backend/tests/test_llama_admission_compat_matrix.py
index f8f3314b089..2dc78a533f4 100644
--- a/studio/backend/tests/test_llama_admission_compat_matrix.py
+++ b/studio/backend/tests/test_llama_admission_compat_matrix.py
@@ -237,7 +237,12 @@ class TestOldCallers:
inspect.signature(LlamaCppBackend.generate_chat_completion_with_tools).parameters
)
assert names.index("sandbox_level") == names.index("instruction_anchor_ids") + 1
- assert names[-3:] == ["compact_context", "website_policy", "live_permissions"]
+ assert names[-4:] == [
+ "compact_context",
+ "website_policy",
+ "live_permissions",
+ "chat_template_kwargs",
+ ]
assert names.index("promote_reasoning_only") == names.index("permission_mode") + 1
def test_the_wait_timeout_has_a_sane_default(self):
diff --git a/studio/backend/utils/current_date_prompt_settings.py b/studio/backend/utils/current_date_prompt_settings.py
index 7652b078785..a9924cbe258 100644
--- a/studio/backend/utils/current_date_prompt_settings.py
+++ b/studio/backend/utils/current_date_prompt_settings.py
@@ -122,6 +122,10 @@ _PROBE_SPECIAL_TOKENS = {
f"{name}_token": f"UNSLOTH_DATE_PROBE_{name.upper()}"
for name in ("bos", "eos", "pad", "unk", "sep", "cls", "mask")
}
+# Names the probe passes itself; a caller's template kwarg of the same name would collide.
+PROBE_RENDER_KEYS = frozenset(
+ {"messages", "tools", "documents", "add_generation_prompt", *_PROBE_SPECIAL_TOKENS}
+)
# a catalog a tool request's branch can render, for probing the template it selects.
PROBE_TOOLS = [
{