Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): classify MLX requests from the applied chat template override (#11903)

Long Yixing4cfeec94d923
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 5f7856f8a0e..18aa872c967 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3938,6 +3938,43 @@ async def artifact_preview_frame(allow_network: bool = False):
_BARE_JSON_NAME_MARKER_RE = _re.compile(r'\{\s*\\?"(?:name|function)\\?"\s*:')
+def _sf_rendered_features(
+ backend,
+ model_info: dict,
+ tools = None,
+ **kwargs,
+):
+ """Classify the template generation renders, returning ``(features, template)``.
+
+ An applied MLX override renders every turn but one on a text model: a tool turn whose schemas
+ the override drops renders through the shipped template (``render_with_native_template_fallback``),
+ so it is classified from it, and a plain turn keeps that route to tools. A vision model renders
+ through the processor, which has no such fallback. ``chat_template_info`` keeps the shipped
+ template as the editor's default; the MLX backend records a reason for every override it did
+ not install.
+ """
+ info = model_info.get("chat_template_info")
+ shipped = info.get("template") if isinstance(info, dict) else None
+ if tools is not None:
+ kwargs["tools"] = tools
+ override = model_info.get("chat_template_override_requested")
+ if (
+ not isinstance(override, str)
+ or not override.strip()
+ or model_info.get("chat_template_override_reason")
+ ):
+ return _detect_safetensors_features(backend, shipped, **kwargs), shipped
+ features = _detect_safetensors_features(backend, override, **kwargs)
+ if features.get("supports_tools") or model_info.get("is_vision"):
+ return features, override
+ fallback = _detect_safetensors_features(backend, shipped, **kwargs)
+ if not fallback.get("supports_tools"):
+ return features, override
+ if tools:
+ return fallback, shipped
+ return dict(features, supports_tools = True), override
+
+
def _detect_safetensors_features(
backend,
chat_template: Optional[str],
@@ -16375,7 +16412,7 @@ async def _load_model_impl(
f"Could not retrieve chat template for {backend.active_model_name}: {e}"
)
# Classify via the same path as GGUF.
- _sf_flags = _detect_safetensors_features(backend, _chat_template)
+ _sf_flags, _ = _sf_rendered_features(backend, _model_info)
_sf_supports_reasoning = _sf_flags["supports_reasoning"]
_sf_reasoning_style = _sf_flags["reasoning_style"]
# Requested chat model already resident: assert CHAT ownership (no-op when held) to correct a drifted owner.
@@ -17173,7 +17210,7 @@ async def _load_model_impl(
pass
# Classify reasoning/tool flags via the GGUF sniffer.
- _sf_flags = _detect_safetensors_features(backend, _chat_template)
+ _sf_flags, _ = _sf_rendered_features(backend, _model_info)
# Report validate_model's requirement (raw auto_map OR YAML) plus the value the
# load used, and persist it, so a later retry/rollback doesn't send
@@ -19620,7 +19657,7 @@ async def get_status(current_subject: str):
)
# Non-GGUF: classify from the loaded template.
- _sf_flags = _detect_safetensors_features(backend, chat_template)
+ _sf_flags, _ = _sf_rendered_features(backend, model_info)
inference_config = (
load_inference_config(backend.active_model_name) if backend.active_model_name else None
)
@@ -28200,7 +28237,6 @@ async def produce_openai_chat_completions(
decode_cache = _sf_mcp_decode_cache,
caller_images = images or served_images,
)
- _sf_tpl = (_sf_model_info.get("chat_template_info") or {}).get("template")
# Resolve the tool policy BEFORE the protocol is classified: the template
# branch chosen here must be the one generation renders. Reading the raw
# policy and withdrawing it later would classify with the ``tool_use``
@@ -28284,13 +28320,15 @@ async def produce_openai_chat_completions(
template = None,
prefer_tool_use = True,
):
- body = _sf_tpl if template is None else template
# Forward only the non-default: unconditional breaks stubs predating the parameter.
_pref = {} if prefer_tool_use else {"prefer_tool_use": False}
- if template is not None:
+ if template is None:
+ features, body = _sf_rendered_features(backend, _sf_model_info, tools, **_pref)
+ else:
+ body = template
# One specific body, so no reasoning rescue from an unselected branch.
_pref["reasoning_fallback"] = False
- features = _detect_safetensors_features(backend, body, tools = tools, **_pref)
+ features = _detect_safetensors_features(backend, body, tools = tools, **_pref)
# The prefill probe needs the ONE body that renders, not the collection (#10092).
try:
from core.inference.chat_template_helpers import (
@@ -34682,7 +34720,6 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon
# exposes tool markup only in its tool_use branch, and which branch is read depends on
# the tools handed in -- so hand it a placeholder for the schemas selected below.
# Without them it reads the plain branch and prices away the whole catalog.
- _tpl = (entry.get("chat_template_info") or {}).get("template")
# Detection only, exactly as the completion draws it: which branch is read, never what is
# rendered, so it must not follow tool_choice. A count reading the plain branch for a tool
# conversation loses the assistant's calls and the result correlation fields to
@@ -34695,9 +34732,7 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon
):
_template_tools = ({},)
_takes_tools = bool(
- _detect_safetensors_features(backend, _tpl, tools = _template_tools).get(
- "supports_tools", False
- )
+ _sf_rendered_features(backend, entry, tools = _template_tools)[0].get("supports_tools", False)
)
# A budget of zero never enters the loop, so the relay is what renders.
_budget = getattr(payload, "max_tool_calls_per_message", None)
diff --git a/studio/backend/tests/test_inference_status_route.py b/studio/backend/tests/test_inference_status_route.py
index f83cfc63992..71696423ccb 100644
--- a/studio/backend/tests/test_inference_status_route.py
+++ b/studio/backend/tests/test_inference_status_route.py
@@ -341,3 +341,43 @@ def test_status_does_not_list_one_transformers_load_twice(monkeypatch):
response = asyncio.run(inference_route.get_status(current_subject = "test"))
assert response.loading == ["/home/alice/models/local-llama"]
+
+
+ "override, reason, applied",
+ [
+ ("{{ override }}", None, True),
+ ("{{ override }}", "it could not render a conversation", False),
+ (" ", None, False),
+ ],
+)
+def test_status_classifies_the_template_generation_renders(monkeypatch, override, reason, applied):
+ """Capabilities follow an applied MLX override; the editor's default stays the shipped one."""
+ backend = _FakeInferenceBackend()
+ backend.active_model_name = "org/mlx-model"
+ backend.models = {
+ "org/mlx-model": {
+ "is_mlx": True,
+ "chat_template_info": {"template": "{{ shipped }}"},
+ "chat_template_override_requested": override,
+ "chat_template_override_reason": reason,
+ }
+ }
+ backend.loading_models = set()
+ _patch_fast_status(monkeypatch, backend)
+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)
+ detect = inference_route._detect_safetensors_features
+ monkeypatch.setattr(
+ inference_route,
+ "_detect_safetensors_features",
+ lambda _backend, template, *a, **k: dict(
+ detect(_backend, template),
+ supports_tools = True,
+ supports_reasoning = template == "{{ override }}",
+ ),
+ )
+
+ response = asyncio.run(inference_route.get_status(current_subject = "test"))
+
+ assert response.supports_reasoning is applied
+ assert response.chat_template == "{{ shipped }}"
diff --git a/studio/backend/tests/test_safetensors_capability_advertise.py b/studio/backend/tests/test_safetensors_capability_advertise.py
index 8e404ce1095..f4ee2590387 100644
--- a/studio/backend/tests/test_safetensors_capability_advertise.py
+++ b/studio/backend/tests/test_safetensors_capability_advertise.py
@@ -631,6 +631,50 @@ def test_detect_safetensors_features_qwen35_keeps_tools_on():
assert flags["reasoning_style"] == "enable_thinking"
+# No tools and no reasoning, so features read from the shipped template cannot pass for it.
+_PLAIN_OVERRIDE = (
+ "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"
+ "{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+)
+_TOOLS_OVERRIDE = QWEN35_TOOL_INSTRUCTION + "{# override #}"
+_REFUSED = "it could not render a conversation"
+_TOOL = [{"type": "function"}]
+
+
+ "shipped, override, reason, is_vision, tools, rendered, supports_tools",
+ [
+ (QWEN35_TOOL_INSTRUCTION, None, None, False, None, "shipped", True),
+ (QWEN35_TOOL_INSTRUCTION, " ", None, False, None, "shipped", True),
+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, _REFUSED, False, None, "shipped", True),
+ (QWEN35_TOOL_INSTRUCTION, _TOOLS_OVERRIDE, None, False, _TOOL, "override", True),
+ # A text model renders a tool turn the override drops through the shipped template.
+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, None, False, _TOOL, "shipped", True),
+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, None, False, None, "override", True),
+ (_PLAIN_OVERRIDE + "{# shipped #}", _PLAIN_OVERRIDE, None, False, _TOOL, "override", False),
+ # A vision model renders through the processor, which has no such fallback.
+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, None, True, _TOOL, "override", False),
+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, None, True, None, "override", False),
+ ],
+)
+def test_rendered_features_classify_the_template_generation_renders(
+ shipped, override, reason, is_vision, tools, rendered, supports_tools
+):
+ from routes.inference import _detect_safetensors_features, _sf_rendered_features
+
+ backend = SimpleNamespace(active_model_name = "mlx/model", models = {})
+ model_info = {
+ "is_vision": is_vision,
+ "chat_template_info": {"template": shipped},
+ "chat_template_override_requested": override,
+ "chat_template_override_reason": reason,
+ }
+ features, template = _sf_rendered_features(backend, model_info, tools = tools)
+ assert template == {"shipped": shipped, "override": override}[rendered]
+ expected = _detect_safetensors_features(backend, template, tools = tools)
+ assert features == dict(expected, supports_tools = supports_tools)
+
+
# ── Tests: IPC bridge contract ───────────────────────────────────────
diff --git a/studio/backend/tests/test_safetensors_reasoning_stream.py b/studio/backend/tests/test_safetensors_reasoning_stream.py
index a819dd7858b..58b63241697 100644
--- a/studio/backend/tests/test_safetensors_reasoning_stream.py
+++ b/studio/backend/tests/test_safetensors_reasoning_stream.py
@@ -40,6 +40,8 @@ from routes.inference import (
import importlib # noqa: E402
import types # noqa: E402
+
+import pytest # noqa: E402
from unittest.mock import MagicMock # noqa: E402
@@ -145,6 +147,14 @@ _MESSAGE_SHAPE_TPL = (
"{% if add_generation_prompt %}<|im_start|>assistant\n"
"{% if ns.think %}<think>\n{% else %}<think></think>{% endif %}{% endif %}"
)
+# Qwen override shape: an inline ``<|think_off|>`` in any message closes the block.
+_THINK_OFF_TAG_TPL = (
+ "{% set ns = namespace(off = false) %}"
+ "{% for m in messages %}{% if '<|think_off|>' in m['content'] %}{% set ns.off = true %}"
+ "{% endif %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"
+ "{% if add_generation_prompt %}<|im_start|>assistant\n"
+ "{% if ns.off %}<think>\n\n</think>\n\n{% else %}<think>\n{% endif %}{% endif %}"
+)
# Kimi shape: renders history into the prompt but opens no block of its own.
_HISTORY_ONLY_TPL = (
"{% for m in messages %}<|im_user|>{{ m['role'] }}<|im_middle|>{{ m['content'] }}<|im_end|>"
@@ -622,6 +632,7 @@ def _sf_route_message(
is_vision = False,
is_mlx = False,
features = None,
+ model_info = None,
**body,
):
"""POST a non-streaming safetensors chat completion and return the assistant message."""
@@ -643,6 +654,7 @@ def _sf_route_message(
"chat_template_info": {"template": template},
"is_vision": is_vision,
"is_mlx": is_mlx,
+ **(model_info or {}),
}
}
@@ -742,6 +754,39 @@ def test_route_keeps_literal_think_text_on_an_mlx_image_turn(monkeypatch):
assert not message["reasoning_content"]
+_THINK_OFF_MESSAGES = [
+ {"role": "system", "content": "You are helpful. <|think_off|>"},
+ {"role": "user", "content": "What is the capital of France?"},
+]
+
+
+ "reason, content, reasoning",
+ [
+ (None, "Paris.", ""),
+ # A refused override is not what renders, so the shipped template's open block decides.
+ ("it could not render a conversation", "", "Paris."),
+ ],
+)
+def test_route_probes_the_applied_mlx_override_not_the_shipped_template(
+ monkeypatch, reason, content, reasoning
+):
+ """An override closing the block on an inline tag must not be read through the shipped one."""
+ message = _sf_route_message(
+ monkeypatch,
+ _THINK_TPL,
+ ["Paris."],
+ is_mlx = True,
+ model_info = {
+ "chat_template_override_requested": _THINK_OFF_TAG_TPL,
+ "chat_template_override_reason": reason,
+ },
+ messages = _THINK_OFF_MESSAGES,
+ )
+ assert (message["content"] or "") == content
+ assert (message["reasoning_content"] or "") == reasoning
+
+
def test_route_carries_the_effort_field_into_the_gate(monkeypatch):
answer = "Use <think>hi</think> in your prompt."
message = _sf_route_message(