Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): classify MLX requests from the applied chat template override (#11903)
Long Yixing4cfeec94d923
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 5f7856f8a0e..18aa872c967 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -3938,6 +3938,43 @@ async def artifact_preview_frame(allow_network: bool = False):_BARE_JSON_NAME_MARKER_RE = _re.compile(r'\{\s*\\?"(?:name|function)\\?"\s*:')+def _sf_rendered_features(+ backend,+ model_info: dict,+ tools = None,+ **kwargs,+):+ """Classify the template generation renders, returning ``(features, template)``.++ An applied MLX override renders every turn but one on a text model: a tool turn whose schemas+ the override drops renders through the shipped template (``render_with_native_template_fallback``),+ so it is classified from it, and a plain turn keeps that route to tools. A vision model renders+ through the processor, which has no such fallback. ``chat_template_info`` keeps the shipped+ template as the editor's default; the MLX backend records a reason for every override it did+ not install.+ """+ info = model_info.get("chat_template_info")+ shipped = info.get("template") if isinstance(info, dict) else None+ if tools is not None:+ kwargs["tools"] = tools+ override = model_info.get("chat_template_override_requested")+ if (+ not isinstance(override, str)+ or not override.strip()+ or model_info.get("chat_template_override_reason")+ ):+ return _detect_safetensors_features(backend, shipped, **kwargs), shipped+ features = _detect_safetensors_features(backend, override, **kwargs)+ if features.get("supports_tools") or model_info.get("is_vision"):+ return features, override+ fallback = _detect_safetensors_features(backend, shipped, **kwargs)+ if not fallback.get("supports_tools"):+ return features, override+ if tools:+ return fallback, shipped+ return dict(features, supports_tools = True), override++def _detect_safetensors_features(backend,chat_template: Optional[str],@@ -16375,7 +16412,7 @@ async def _load_model_impl(f"Could not retrieve chat template for {backend.active_model_name}: {e}")# Classify via the same path as GGUF.- _sf_flags = _detect_safetensors_features(backend, _chat_template)+ _sf_flags, _ = _sf_rendered_features(backend, _model_info)_sf_supports_reasoning = _sf_flags["supports_reasoning"]_sf_reasoning_style = _sf_flags["reasoning_style"]# Requested chat model already resident: assert CHAT ownership (no-op when held) to correct a drifted owner.@@ -17173,7 +17210,7 @@ async def _load_model_impl(pass# Classify reasoning/tool flags via the GGUF sniffer.- _sf_flags = _detect_safetensors_features(backend, _chat_template)+ _sf_flags, _ = _sf_rendered_features(backend, _model_info)# Report validate_model's requirement (raw auto_map OR YAML) plus the value the# load used, and persist it, so a later retry/rollback doesn't send@@ -19620,7 +19657,7 @@ async def get_status(current_subject: str):)# Non-GGUF: classify from the loaded template.- _sf_flags = _detect_safetensors_features(backend, chat_template)+ _sf_flags, _ = _sf_rendered_features(backend, model_info)inference_config = (load_inference_config(backend.active_model_name) if backend.active_model_name else None)@@ -28200,7 +28237,6 @@ async def produce_openai_chat_completions(decode_cache = _sf_mcp_decode_cache,caller_images = images or served_images,)- _sf_tpl = (_sf_model_info.get("chat_template_info") or {}).get("template")# Resolve the tool policy BEFORE the protocol is classified: the template# branch chosen here must be the one generation renders. Reading the raw# policy and withdrawing it later would classify with the ``tool_use``@@ -28284,13 +28320,15 @@ async def produce_openai_chat_completions(template = None,prefer_tool_use = True,):- body = _sf_tpl if template is None else template# Forward only the non-default: unconditional breaks stubs predating the parameter._pref = {} if prefer_tool_use else {"prefer_tool_use": False}- if template is not None:+ if template is None:+ features, body = _sf_rendered_features(backend, _sf_model_info, tools, **_pref)+ else:+ body = template# One specific body, so no reasoning rescue from an unselected branch._pref["reasoning_fallback"] = False- features = _detect_safetensors_features(backend, body, tools = tools, **_pref)+ features = _detect_safetensors_features(backend, body, tools = tools, **_pref)# The prefill probe needs the ONE body that renders, not the collection (#10092).try:from core.inference.chat_template_helpers import (@@ -34682,7 +34720,6 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon# exposes tool markup only in its tool_use branch, and which branch is read depends on# the tools handed in -- so hand it a placeholder for the schemas selected below.# Without them it reads the plain branch and prices away the whole catalog.- _tpl = (entry.get("chat_template_info") or {}).get("template")# Detection only, exactly as the completion draws it: which branch is read, never what is# rendered, so it must not follow tool_choice. A count reading the plain branch for a tool# conversation loses the assistant's calls and the result correlation fields to@@ -34695,9 +34732,7 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon):_template_tools = ({},)_takes_tools = bool(- _detect_safetensors_features(backend, _tpl, tools = _template_tools).get(- "supports_tools", False- )+ _sf_rendered_features(backend, entry, tools = _template_tools)[0].get("supports_tools", False))# A budget of zero never enters the loop, so the relay is what renders._budget = getattr(payload, "max_tool_calls_per_message", None)diff --git a/studio/backend/tests/test_inference_status_route.py b/studio/backend/tests/test_inference_status_route.pyindex f83cfc63992..71696423ccb 100644--- a/studio/backend/tests/test_inference_status_route.py+++ b/studio/backend/tests/test_inference_status_route.py@@ -341,3 +341,43 @@ def test_status_does_not_list_one_transformers_load_twice(monkeypatch):response = asyncio.run(inference_route.get_status(current_subject = "test"))assert response.loading == ["/home/alice/models/local-llama"]+++ "override, reason, applied",+ [+ ("{{ override }}", None, True),+ ("{{ override }}", "it could not render a conversation", False),+ (" ", None, False),+ ],+)+def test_status_classifies_the_template_generation_renders(monkeypatch, override, reason, applied):+ """Capabilities follow an applied MLX override; the editor's default stays the shipped one."""+ backend = _FakeInferenceBackend()+ backend.active_model_name = "org/mlx-model"+ backend.models = {+ "org/mlx-model": {+ "is_mlx": True,+ "chat_template_info": {"template": "{{ shipped }}"},+ "chat_template_override_requested": override,+ "chat_template_override_reason": reason,+ }+ }+ backend.loading_models = set()+ _patch_fast_status(monkeypatch, backend)+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)+ detect = inference_route._detect_safetensors_features+ monkeypatch.setattr(+ inference_route,+ "_detect_safetensors_features",+ lambda _backend, template, *a, **k: dict(+ detect(_backend, template),+ supports_tools = True,+ supports_reasoning = template == "{{ override }}",+ ),+ )++ response = asyncio.run(inference_route.get_status(current_subject = "test"))++ assert response.supports_reasoning is applied+ assert response.chat_template == "{{ shipped }}"diff --git a/studio/backend/tests/test_safetensors_capability_advertise.py b/studio/backend/tests/test_safetensors_capability_advertise.pyindex 8e404ce1095..f4ee2590387 100644--- a/studio/backend/tests/test_safetensors_capability_advertise.py+++ b/studio/backend/tests/test_safetensors_capability_advertise.py@@ -631,6 +631,50 @@ def test_detect_safetensors_features_qwen35_keeps_tools_on():assert flags["reasoning_style"] == "enable_thinking"+# No tools and no reasoning, so features read from the shipped template cannot pass for it.+_PLAIN_OVERRIDE = (+ "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"+ "{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"+)+_TOOLS_OVERRIDE = QWEN35_TOOL_INSTRUCTION + "{# override #}"+_REFUSED = "it could not render a conversation"+_TOOL = [{"type": "function"}]+++ "shipped, override, reason, is_vision, tools, rendered, supports_tools",+ [+ (QWEN35_TOOL_INSTRUCTION, None, None, False, None, "shipped", True),+ (QWEN35_TOOL_INSTRUCTION, " ", None, False, None, "shipped", True),+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, _REFUSED, False, None, "shipped", True),+ (QWEN35_TOOL_INSTRUCTION, _TOOLS_OVERRIDE, None, False, _TOOL, "override", True),+ # A text model renders a tool turn the override drops through the shipped template.+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, None, False, _TOOL, "shipped", True),+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, None, False, None, "override", True),+ (_PLAIN_OVERRIDE + "{# shipped #}", _PLAIN_OVERRIDE, None, False, _TOOL, "override", False),+ # A vision model renders through the processor, which has no such fallback.+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, None, True, _TOOL, "override", False),+ (QWEN35_TOOL_INSTRUCTION, _PLAIN_OVERRIDE, None, True, None, "override", False),+ ],+)+def test_rendered_features_classify_the_template_generation_renders(+ shipped, override, reason, is_vision, tools, rendered, supports_tools+):+ from routes.inference import _detect_safetensors_features, _sf_rendered_features++ backend = SimpleNamespace(active_model_name = "mlx/model", models = {})+ model_info = {+ "is_vision": is_vision,+ "chat_template_info": {"template": shipped},+ "chat_template_override_requested": override,+ "chat_template_override_reason": reason,+ }+ features, template = _sf_rendered_features(backend, model_info, tools = tools)+ assert template == {"shipped": shipped, "override": override}[rendered]+ expected = _detect_safetensors_features(backend, template, tools = tools)+ assert features == dict(expected, supports_tools = supports_tools)++# ── Tests: IPC bridge contract ───────────────────────────────────────diff --git a/studio/backend/tests/test_safetensors_reasoning_stream.py b/studio/backend/tests/test_safetensors_reasoning_stream.pyindex a819dd7858b..58b63241697 100644--- a/studio/backend/tests/test_safetensors_reasoning_stream.py+++ b/studio/backend/tests/test_safetensors_reasoning_stream.py@@ -40,6 +40,8 @@ from routes.inference import (import importlib # noqa: E402import types # noqa: E402++import pytest # noqa: E402from unittest.mock import MagicMock # noqa: E402@@ -145,6 +147,14 @@ _MESSAGE_SHAPE_TPL = ("{% if add_generation_prompt %}<|im_start|>assistant\n""{% if ns.think %}<think>\n{% else %}<think></think>{% endif %}{% endif %}")+# Qwen override shape: an inline ``<|think_off|>`` in any message closes the block.+_THINK_OFF_TAG_TPL = (+ "{% set ns = namespace(off = false) %}"+ "{% for m in messages %}{% if '<|think_off|>' in m['content'] %}{% set ns.off = true %}"+ "{% endif %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"+ "{% if add_generation_prompt %}<|im_start|>assistant\n"+ "{% if ns.off %}<think>\n\n</think>\n\n{% else %}<think>\n{% endif %}{% endif %}"+)# Kimi shape: renders history into the prompt but opens no block of its own._HISTORY_ONLY_TPL = ("{% for m in messages %}<|im_user|>{{ m['role'] }}<|im_middle|>{{ m['content'] }}<|im_end|>"@@ -622,6 +632,7 @@ def _sf_route_message(is_vision = False,is_mlx = False,features = None,+ model_info = None,**body,):"""POST a non-streaming safetensors chat completion and return the assistant message."""@@ -643,6 +654,7 @@ def _sf_route_message("chat_template_info": {"template": template},"is_vision": is_vision,"is_mlx": is_mlx,+ **(model_info or {}),}}@@ -742,6 +754,39 @@ def test_route_keeps_literal_think_text_on_an_mlx_image_turn(monkeypatch):assert not message["reasoning_content"]+_THINK_OFF_MESSAGES = [+ {"role": "system", "content": "You are helpful. <|think_off|>"},+ {"role": "user", "content": "What is the capital of France?"},+]+++ "reason, content, reasoning",+ [+ (None, "Paris.", ""),+ # A refused override is not what renders, so the shipped template's open block decides.+ ("it could not render a conversation", "", "Paris."),+ ],+)+def test_route_probes_the_applied_mlx_override_not_the_shipped_template(+ monkeypatch, reason, content, reasoning+):+ """An override closing the block on an inline tag must not be read through the shipped one."""+ message = _sf_route_message(+ monkeypatch,+ _THINK_TPL,+ ["Paris."],+ is_mlx = True,+ model_info = {+ "chat_template_override_requested": _THINK_OFF_TAG_TPL,+ "chat_template_override_reason": reason,+ },+ messages = _THINK_OFF_MESSAGES,+ )+ assert (message["content"] or "") == content+ assert (message["reasoning_content"] or "") == reasoning++def test_route_carries_the_effort_field_into_the_gate(monkeypatch):answer = "Use <think>hi</think> in your prompt."message = _sf_route_message(