Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix reasoning classification for generation-tag chat templates (#16121)

Leo Borcherding8f5b576bbf26
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 7e84b2e53e1..411076cdebb 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -4365,23 +4365,8 @@ _PROBE_MESSAGES = ({"role": "user", "content": "hi"},)
def _compile_generation_prompt_probe(template: str):
"""Cache the compiled template only: rendering is cheap, and messages must vary per request."""
try:
- from datetime import datetime
-
- from jinja2.sandbox import ImmutableSandboxedEnvironment
-
- def _raise_exception(message: str):
- raise RuntimeError(message)
-
- env = ImmutableSandboxedEnvironment(
- trim_blocks = True,
- lstrip_blocks = True,
- extensions = ["jinja2.ext.loopcontrols"],
- )
- env.filters["tojson"] = lambda value, **kwargs: json.dumps(value, ensure_ascii = False)
- env.globals["raise_exception"] = _raise_exception
- # transformers exposes strftime_now to every template; without it date templates raise.
- env.globals["strftime_now"] = lambda fmt: datetime.now().strftime(fmt)
- return env.from_string(template)
+ from transformers.utils.chat_template_utils import _compile_jinja_template
+ return _compile_jinja_template(template)
except Exception:
return None
@@ -4409,6 +4394,8 @@ def _render_generation_prompt_probe(
add_generation_prompt = generation_prompt,
bos_token = "",
eos_token = "",
+ tools = None,
+ documents = None,
**reasoning_kwargs,
)
except Exception:
@@ -4421,18 +4408,16 @@ def _generation_prompt_opens_think(
reasoning_effort: Optional[str] = None,
messages: Optional[Collection] = None,
) -> bool:
- """Diffs the no-prompt render so typed think tags do not count; unrenderable returns True."""
- if not template:
+ """Diffs the no-prompt render so typed think tags do not count; unrenderable returns False."""
+ if not template or not isinstance(template, str):
return False
- if not isinstance(template, str):
- return True
args = (template, enable_thinking, reasoning_effort)
rendered = _render_generation_prompt_probe(*args, messages)
if rendered is None and messages is not None:
messages = None
rendered = _render_generation_prompt_probe(*args)
if rendered is None:
- return True
+ return False
# Ignoring the flag gives an empty prefix, so nothing is prefilled; correct when no block opens.
body = _render_generation_prompt_probe(*args, messages, generation_prompt = False)
prefix = (
diff --git a/studio/backend/tests/test_safetensors_reasoning_stream.py b/studio/backend/tests/test_safetensors_reasoning_stream.py
index e6ab71e1a11..5a743904c4f 100644
--- a/studio/backend/tests/test_safetensors_reasoning_stream.py
+++ b/studio/backend/tests/test_safetensors_reasoning_stream.py
@@ -120,6 +120,14 @@ _STRICT_HISTORY_TPL = (
"{% endif %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"
"{% if add_generation_prompt %}<|im_start|>assistant\n<think>\n\n</think>\n\n{% endif %}"
)
+_GENERATION_TAG_TPL = (
+ "{% set enable_thinking = enable_thinking | default(false) %}"
+ "{% for m in messages %}<|im_start|>{{ m['role'] }}\n"
+ "{% if m['role'] == 'assistant' %}{% generation %}{{ m['content'] }}{% endgeneration %}"
+ "{% else %}{{ m['content'] }}{% endif %}<|im_end|>\n{% endfor %}"
+ "{% if add_generation_prompt %}<|im_start|>assistant\n"
+ "{% if enable_thinking %}<think>{% else %}<think></think>{% endif %}{% endif %}"
+)
_TINY_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAE0lEQVR4nGM8ISfHAANMcBZeDgA0dgEMydTl/QAAAABJRU5ErkJggg=="
_ETHINK = {"reasoning_style": "enable_thinking", "supports_reasoning": True}
_ETHINK_EFFORT = {"reasoning_style": "enable_thinking_effort", "supports_reasoning": True}
@@ -158,6 +166,62 @@ def test_a_template_that_stamps_a_date_still_renders():
assert _sf_reasoning_prefill_mode(_ETHINK, None, _STRFTIME_TPL) is False
+def test_a_generation_tag_template_is_not_read_as_an_open_think():
+ assert _sf_reasoning_prefill_mode(_ETHINK, None, _GENERATION_TAG_TPL) is False
+ assert _sf_reasoning_prefill_mode(_ETHINK, False, _GENERATION_TAG_TPL) is False
+ assert _sf_reasoning_prefill_mode(_ETHINK, True, _GENERATION_TAG_TPL) is True
+
+
[email protected]("thinking", (False, True))
+def test_generation_block_assignments_stay_local(thinking):
+ from transformers.utils.chat_template_utils import _compile_jinja_template
+
+ template = (
+ "{% set thinking = " + str(thinking).lower() + " %}"
+ "{% generation %}{% set thinking = not thinking %}previous answer{% endgeneration %}"
+ "{% if add_generation_prompt %}<|assistant|>"
+ "{% if thinking %}<think>{% else %}<think></think>{% endif %}{% endif %}"
+ )
+ rendered = _compile_jinja_template(template).render(
+ messages = [{"role": "user", "content": "hi"}], add_generation_prompt = True
+ )
+ assert rendered.endswith("<think>" if thinking else "<think></think>")
+ assert _sf_reasoning_prefill_mode(_ETHINK, None, template) is thinking
+
+
+ "template",
+ (
+ "{% unknown %}{% for m in messages %}{{ m['content'] }}{% endfor %}<think></think>",
+ "{{ tools | tojson }}{% for m in messages %}{{ m['content'] }}{% endfor %}<think></think>",
+ ),
+)
+def test_closed_or_unrenderable_templates_keep_the_answer_visible(template):
+ extractor = _ResponsesReasoningExtractor(
+ parse_think_markers = _sf_parse_think_markers(_ETHINK),
+ reasoning_prefilled = _sf_reasoning_prefill_mode(_ETHINK, None, template),
+ )
+ reasoning, visible = extractor.feed("Four")
+ tail_reasoning, tail_visible = extractor.finish()
+ assert (reasoning + tail_reasoning, visible + tail_visible) == ("", "Four")
+
+
[email protected]("context_name", ("tools", "documents"))
+def test_default_render_context_preserves_prefilled_reasoning(context_name):
+ template = (
+ "{{ " + context_name + " | tojson }}"
+ "{% for m in messages %}{{ m['content'] }}{% endfor %}"
+ "{% if add_generation_prompt %}<|assistant|><think>{% endif %}"
+ )
+ extractor = _ResponsesReasoningExtractor(
+ parse_think_markers = True,
+ reasoning_prefilled = _sf_reasoning_prefill_mode(_ETHINK, None, template),
+ )
+ reasoning, visible = extractor.feed("Let me calculate.</think>Four")
+ tail_reasoning, tail_visible = extractor.finish()
+ assert (reasoning + tail_reasoning, visible + tail_visible) == ("Let me calculate.", "Four")
+
+
def test_prefill_mode_off_without_think_markers():
assert _sf_reasoning_prefill_mode(_ETHINK, None, "no markers here") is False
@@ -656,6 +720,36 @@ def test_route_returns_the_answer_as_content_when_the_template_closes_its_block(
assert not message["reasoning_content"]
[email protected]("stream", (False, True))
[email protected]("is_mlx", (False, True))
+def test_route_generation_tag_keeps_the_answer_visible(monkeypatch, stream, is_mlx):
+ message = _sf_route_message(
+ monkeypatch, _GENERATION_TAG_TPL, ["Four"], stream = stream, is_mlx = is_mlx
+ )
+ assert message["content"] == "Four"
+ assert not message["reasoning_content"]
+
+
[email protected]("stream", (False, True))
[email protected]("is_mlx", (False, True))
[email protected]("context_name", ("tools", "documents"))
+def test_route_default_context_preserves_reasoning(monkeypatch, stream, is_mlx, context_name):
+ template = (
+ "{{ " + context_name + " | tojson }}"
+ "{% for m in messages %}{{ m['content'] }}{% endfor %}"
+ "{% if add_generation_prompt %}<|assistant|><think>{% endif %}"
+ )
+ message = _sf_route_message(
+ monkeypatch,
+ template,
+ ["Let me calculate.</think>Four"],
+ stream = stream,
+ is_mlx = is_mlx,
+ )
+ assert message["content"] == "Four"
+ assert message["reasoning_content"] == "Let me calculate."
+
+
def test_route_keeps_literal_think_text_when_the_request_turns_thinking_off(monkeypatch):
answer = "Use <think>hi</think> in your prompt."
message = _sf_route_message(