Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: detect tool support from Jinja syntax instead of substrings (#10690)

Daniel Han7a4b0416191d
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index f218d8d55fb..4eaa8c6c449 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2591,29 +2591,6 @@ def _extract_model_size_b(model_id: str):
return extract_model_size_b(model_id)
-_TOOL_TEMPLATE_MARKERS = (
- "{%- if tools %}",
- "{%- if tools -%}",
- "{% if tools %}",
- "{% if tools -%}",
- # Defensive templates guard with `tools is defined` before truth-testing
- # (e.g. Inkling: `{%- if tools is defined and tools -%}`).
- "{%- if tools is defined",
- "{% if tools is defined",
- '"role" == "tool"',
- "'role' == 'tool'",
- 'message.role == "tool"',
- "message.role == 'tool'",
- # DeepSeek: no top-level ``{% if tools %}`` block; it gates emission on
- # ``message['role'] == 'tool'`` plus ``message['tool_calls'] is defined``.
- "message['role'] == 'tool'",
- 'message["role"] == "tool"',
- "message['tool_calls']",
- 'message["tool_calls"]',
- "tool_calls is defined",
-)
-
-
# Canonical reasoning_effort levels, weakest -> strongest. Used to read the
# discrete set a template branches on (e.g. GLM-5.2 uses 'high' | 'max', Inkling
# uses the full 'none'..'max' ladder) so we only ever offer levels the template
@@ -2785,7 +2762,9 @@ def detect_reasoning_flags(
flags["preserve_thinking_default"] = bool(_QWEN38_MODEL_RE.search(model_identifier or ""))
_log(f"{prefix}model supports preserve_thinking")
- if any(marker in tpl for marker in _TOOL_TEMPLATE_MARKERS):
+ from core.inference.template_capabilities import template_supports_tools
+
+ if isinstance(tpl, str) and template_supports_tools(tpl):
flags["supports_tools"] = True
_log(f"{prefix}model supports tool calling")
diff --git a/studio/backend/core/inference/template_capabilities.py b/studio/backend/core/inference/template_capabilities.py
new file mode 100644
index 00000000000..ed647f274fe
--- /dev/null
+++ b/studio/backend/core/inference/template_capabilities.py
@@ -0,0 +1,270 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Does this chat template ever put the tool catalog into the prompt?
+
+Answers Studio's one yes/no per model, deciding whether the tool controls are live.
+
+Do not go back to matching substrings: that reads spelling, not meaning, and called
+Granite 3.3 tool-less for writing `{%- if tools and not available_tools -%}`.
+
+Over-approximates on purpose. A spurious yes shows a control the backend re-checks;
+a spurious no silently disables a working feature, which is the bug this replaces.
+"""
+
+from functools import lru_cache
+
+from jinja2 import nodes
+from jinja2.ext import Extension
+from jinja2.sandbox import ImmutableSandboxedEnvironment
+
+
+class _Generation(Extension):
+ """Not real Jinja, so without it HuggingFaceTB/SmolLM3-3B fails to parse."""
+
+ tags = {"generation"}
+
+ def parse(self, parser):
+ next(parser.stream)
+ return parser.parse_statements(("name:endgeneration",), drop_needle = True)
+
+
+_ENVIRONMENT = ImmutableSandboxedEnvironment(
+ extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do", _Generation],
+)
+
+# Ask about the catalog rather than serialise it: a number reaches the prompt.
+_REDUCING = frozenset({"length", "count"})
+
+
+def _names(node):
+ """Every name this expression reads. find_all skips the node, so add it by hand."""
+ found = {item.name for item in node.find_all(nodes.Name)}
+ return found | ({node.name} if isinstance(node, nodes.Name) else set())
+
+
+def _field(node):
+ """The field a member access reads, for `a.b`, `a['b']` and `a.get('b')` alike."""
+ if isinstance(node, nodes.Getattr):
+ return node.attr
+ if isinstance(node, nodes.Getitem) and isinstance(node.arg, nodes.Const):
+ return node.arg.value
+ # `.get('tool_calls')` is the same read spelled as a call.
+ if (
+ isinstance(node, nodes.Call)
+ and isinstance(node.node, nodes.Getattr)
+ and node.node.attr == "get"
+ and node.args
+ and isinstance(node.args[0], nodes.Const)
+ ):
+ return node.args[0].value
+ return None
+
+
+def _reads_catalog(node, aliases):
+ """Whether this expression reads the tool catalog or a message's tool calls."""
+ if _names(node) & aliases:
+ return True
+ members = list(node.find_all((nodes.Getattr, nodes.Getitem, nodes.Call)))
+ if isinstance(node, (nodes.Getattr, nodes.Getitem, nodes.Call)):
+ members.append(node)
+ return any(_field(member) == "tool_calls" for member in members)
+
+
+def _checks_tool_role(node):
+ """`message.role == 'tool'` - the branch that handles a tool result."""
+ compares = list(node.find_all(nodes.Compare))
+ if isinstance(node, nodes.Compare):
+ # `{% if m.role == 'tool' %}` IS the comparison, which find_all skips.
+ compares.append(node)
+ for compare in compares:
+ if len(compare.ops) != 1 or compare.ops[0].op != "eq":
+ continue
+ pairs = (
+ (compare.expr, compare.ops[0].expr),
+ (compare.ops[0].expr, compare.expr),
+ )
+ for field, value in pairs:
+ if _field(field) == "role" and isinstance(value, nodes.Const) and value.value == "tool":
+ return True
+ return False
+
+
+def _is_payload(node):
+ """Whether rendering this would put something meaningful in the prompt."""
+ if isinstance(node, (nodes.Not, nodes.Compare, nodes.Test)):
+ return False
+ if isinstance(node, nodes.Filter) and node.name in _REDUCING:
+ return False
+ if isinstance(node, nodes.TemplateData):
+ return bool(node.data.strip())
+ return True
+
+
+def _bound_names(node):
+ """The names a `{% set %}` or `{% for %}` target binds, tuple targets included."""
+ if isinstance(node, nodes.Tuple):
+ return {name for item in node.items for name in _bound_names(item)}
+ # `{% set ns.catalog = ... %}` is an NSRef, already naming the container.
+ while isinstance(node, (nodes.Getattr, nodes.Getitem)):
+ node = node.node
+ return {node.name} if isinstance(node, (nodes.Name, nodes.NSRef)) else set()
+
+
+def _rebound_names(node):
+ """Names a target REBINDS: `{% set ns.catalog = x %}` cannot un-hold `ns`."""
+ if isinstance(node, nodes.Name):
+ return {node.name}
+ if isinstance(node, nodes.Tuple):
+ return {name for item in node.items for name in _rebound_names(item)}
+ return set()
+
+
+def _receiver_gaining_catalog(
+ node,
+ aliases,
+ guarded = False,
+):
+ """Names `catalog.append(tools)` fills, or any receiver written under a guard."""
+ if not (isinstance(node, nodes.Call) and isinstance(node.node, nodes.Getattr)):
+ return set()
+ arguments = list(node.args) + [keyword.value for keyword in node.kwargs]
+ if not guarded and not any(_reads_catalog(argument, aliases) for argument in arguments):
+ return set()
+ return _bound_names(node.node.node)
+
+
+def _scan_maybe(
+ body,
+ aliases,
+ guarded,
+ bound = frozenset(),
+ killed = frozenset(),
+):
+ """Walk a body that may not run. Keep what it learns, drop what it unbinds; the
+ body's own `bound`/`killed` names escape in neither direction."""
+ local = (set(aliases) | set(bound)) - set(killed)
+ found = _scan(body, local, guarded)
+ aliases |= local - (set(bound) - aliases)
+ return found
+
+
+def _join(aliases, arms, exhaustive):
+ """Union the arms: a name holds the catalog if it does on any. Falling past every
+ arm is a path too, so the incoming set counts unless an `{% else %}` is present -
+ which is what keeps a rebinding on every arm a rebinding."""
+ merged = set() if exhaustive else set(aliases)
+ for arm in arms:
+ merged |= arm
+ aliases.clear()
+ aliases.update(merged)
+
+
+def _scan(body, aliases, guarded):
+ """Walk statements, tracking which names hold the catalog; True on first emission.
+ `guarded` means the branch runs only when tools exist, so its prose advertises them."""
+ for node in body:
+ if isinstance(node, nodes.Output):
+ for value in node.nodes:
+ # `{{ m.content if m.role == 'tool' else '' }}` holds its check in
+ # the expression, where an `{% if %}` usually would.
+ if _is_payload(value) and (
+ guarded or _reads_catalog(value, aliases) or _checks_tool_role(value)
+ ):
+ return True
+ elif isinstance(node, nodes.ExprStmt):
+ # `{% do catalog.append(tools) %}`
+ aliases |= _receiver_gaining_catalog(node.node, aliases, guarded)
+ continue
+ elif isinstance(node, nodes.Assign):
+ # `{% set _ = catalog.append(tools) %}`: the same mutation without `do`.
+ aliases |= _receiver_gaining_catalog(node.node, aliases, guarded)
+ if _reads_catalog(node.node, aliases) or guarded or _checks_tool_role(node.node):
+ # Three ways a name comes to hold tool-conditional content: LFM2 fills
+ # `ns.system_prompt` inside the guard and renders it outside; a guard
+ # runs the statement only when tools exist, so even a constant counts;
+ # and a stored `m.role == 'tool'` renders what the inline test does.
+ aliases |= _bound_names(node.target)
+ else:
+ # Plain names only: one field says nothing about the rest of the
+ # container. glm-4-9b-chat rebinds `tools` off a message.
+ aliases -= _rebound_names(node.target)
+ continue
+
+ if isinstance(node, nodes.If):
+ arms = []
+ for branch in [node, *node.elif_]:
+ inner = (
+ guarded
+ or _reads_catalog(branch.test, aliases)
+ or _checks_tool_role(branch.test)
+ )
+ arm = set(aliases)
+ if _scan(branch.body, arm, inner):
+ return True
+ arms.append(arm)
+ if node.else_:
+ arm = set(aliases)
+ if _scan(node.else_, arm, guarded):
+ return True
+ arms.append(arm)
+ _join(aliases, arms, exhaustive = bool(node.else_))
+ elif isinstance(node, nodes.For):
+ over_catalog = _reads_catalog(node.iter, aliases)
+ # `{% for m in messages if m.role == 'tool' %}`: the guard is the filter.
+ filtered = node.test is not None and (
+ _reads_catalog(node.test, aliases) or _checks_tool_role(node.test)
+ )
+ # Each item is catalog data, so the loop variable carries it too.
+ bound = _bound_names(node.target) if over_catalog else frozenset()
+ if _scan_maybe(node.body, aliases, guarded or over_catalog or filtered, bound):
+ return True
+ if _scan_maybe(node.else_, aliases, guarded):
+ return True
+ elif isinstance(node, nodes.With):
+ # `{% with catalog = tools %}` binds like a set, for the block only.
+ bound = set()
+ killed = set()
+ for target, value in zip(node.targets, node.values):
+ if _reads_catalog(value, aliases):
+ bound |= _bound_names(target)
+ else:
+ killed |= _rebound_names(target)
+ if _scan_maybe(node.body, aliases, guarded, bound, killed):
+ return True
+ elif hasattr(node, "body"):
+ # Macros, blocks, filters, autoescape: the body can still render.
+ if _scan_maybe(node.body, aliases, guarded):
+ return True
+ return False
+
+
+def template_supports_tools(template) -> bool:
+ """Inspect syntax only; rendering and parser support remain backend checks."""
+ # Outside the cache: lru_cache hashes first, so a dict-valued template would
+ # raise past every fail-closed branch below.
+ if not isinstance(template, str):
+ return False
+ # `str.__str__`, not `str(...)`: an override would raise out here, unhandled.
+ return _analyse_template(str.__str__(template))
+
+
+@lru_cache(maxsize = 128)
+def _analyse_template(template: str) -> bool:
+ if "tool" not in template:
+ return False
+ try:
+ tree = _ENVIRONMENT.parse(template)
+ aliases = {"tools"}
+ # Twice: a template may render an alias before the statement that binds it.
+ for _ in range(2):
+ if _scan(tree.body, aliases, False):
+ return True
+ return False
+ except Exception:
+ # Fail closed: this runs on the model-load path and must not stop it.
+ return False
+
+
+template_supports_tools.cache_clear = _analyse_template.cache_clear
+template_supports_tools.cache_info = _analyse_template.cache_info
diff --git a/studio/backend/tests/data/chat_templates/granite-3.3.jinja b/studio/backend/tests/data/chat_templates/granite-3.3.jinja
new file mode 100644
index 00000000000..976e657fcd7
--- /dev/null
+++ b/studio/backend/tests/data/chat_templates/granite-3.3.jinja
@@ -0,0 +1,63 @@
+{# Source: https://huggingface.co/ibm-granite/granite-3.3-8b-instruct/blob/main/tokenizer_config.json #}
+{# Alias tools -> available_tools #}
+{%- if tools and not available_tools -%}
+ {%- set available_tools = tools -%}
+{%- endif -%}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content'] %}
+ {%- set loop_messages = messages[1:] %}
+ {%- else %}
+ {%- set system_message = "Knowledge Cutoff Date: April 2024.
+Today's Date: " + strftime_now('%B %d, %Y') + ".
+You are Granite, developed by IBM." %}
+ {%- if available_tools and documents %}
+ {%- set system_message = system_message + " You are a helpful assistant with access to the following tools. When a tool is required to answer the user's query, respond only with <|tool_call|> followed by a JSON list of tools used. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request.
+Write the response to the user's input by strictly aligning with the facts in the provided documents. If the information needed to answer the question is not available in the documents, inform the user that the question cannot be answered based on the available data." %}
+ {%- elif available_tools %}
+ {%- set system_message = system_message + " You are a helpful assistant with access to the following tools. When a tool is required to answer the user's query, respond only with <|tool_call|> followed by a JSON list of tools used. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request." %}
+ {%- elif documents %}
+ {%- set system_message = system_message + " Write the response to the user's input by strictly aligning with the facts in the provided documents. If the information needed to answer the question is not available in the documents, inform the user that the question cannot be answered based on the available data." %}
+ {%- elif thinking %}
+ {%- set system_message = system_message + " You are a helpful AI assistant.
+Respond to every user query in a comprehensive and detailed way. You can write down your thoughts and reasoning process before responding. In the thought process, engage in a comprehensive cycle of analysis, summarization, exploration, reassessment, reflection, backtracing, and iteration to develop well-considered thinking process. In the response section, based on various attempts, explorations, and reflections from the thoughts section, systematically present the final solution that you deem correct. The response should summarize the thought process. Write your thoughts between <think></think> and write your response between <response></response> for each user query." %}
+ {%- else %}
+ {%- set system_message = system_message + " You are a helpful AI assistant." %}
+ {%- endif %}
+ {%- if 'citations' in controls and documents %}
+ {%- set system_message = system_message + '
+Use the symbols <|start_of_cite|> and <|end_of_cite|> to indicate when a fact comes from a document in the search result, e.g <|start_of_cite|> {document_id: 1}my fact <|end_of_cite|> for a fact from document 1. Afterwards, list all the citations with their corresponding documents in an ordered list.' %}
+ {%- endif %}
+ {%- if 'hallucinations' in controls and documents %}
+ {%- set system_message = system_message + '
+Finally, after the response is written, include a numbered list of sentences from the response with a corresponding risk value that are hallucinated and not based in the documents.' %}
+ {%- endif %}
+ {%- set loop_messages = messages %}
+ {%- endif %}
+ {{- '<|start_of_role|>system<|end_of_role|>' + system_message + '<|end_of_text|>
+' }}
+ {%- if available_tools %}
+ {{- '<|start_of_role|>available_tools<|end_of_role|>' }}
+ {{- available_tools | tojson(indent=4) }}
+ {{- '<|end_of_text|>
+' }}
+ {%- endif %}
+ {%- if documents %}
+ {%- for document in documents %}
+ {{- '<|start_of_role|>document {"document_id": "' + document['doc_id'] | string + '"}<|end_of_role|>
+' }}
+ {{- document['text'] }}
+ {{- '<|end_of_text|>
+' }}
+ {%- endfor %}
+ {%- endif %}
+ {%- for message in loop_messages %}
+ {{- '<|start_of_role|>' + message['role'] + '<|end_of_role|>' + message['content'] + '<|end_of_text|>
+' }}
+ {%- if loop.last and add_generation_prompt %}
+ {{- '<|start_of_role|>assistant' }}
+ {%- if controls %}
+ {{- ' ' + controls | tojson()}}
+ {%- endif %}
+ {{- '<|end_of_role|>' }}
+ {%- endif %}
+ {%- endfor %}
diff --git a/studio/backend/tests/data/chat_templates/lfm2-tool.jinja b/studio/backend/tests/data/chat_templates/lfm2-tool.jinja
new file mode 100644
index 00000000000..99f3593d02f
--- /dev/null
+++ b/studio/backend/tests/data/chat_templates/lfm2-tool.jinja
@@ -0,0 +1,7 @@
+{{- bos_token -}}{%- set system_prompt = "" -%}{%- set ns = namespace(system_prompt="") -%}{%- if messages[0]["role"] == "system" -%} {%- set ns.system_prompt = messages[0]["content"] -%} {%- set messages = messages[1:] -%}{%- endif -%}{%- if tools -%} {%- set ns.system_prompt = ns.system_prompt + ("
+" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%} {%- for tool in tools -%} {%- if tool is not string -%} {%- set tool = tool | tojson -%} {%- endif -%} {%- set ns.system_prompt = ns.system_prompt + tool -%} {%- if not loop.last -%} {%- set ns.system_prompt = ns.system_prompt + ", " -%} {%- endif -%} {%- endfor -%} {%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}{%- endif -%}{%- if ns.system_prompt -%} {{- "<|im_start|>system
+" + ns.system_prompt + "<|im_end|>
+" -}}{%- endif -%}{%- for message in messages -%} {{- "<|im_start|>" + message["role"] + "
+" -}} {%- set content = message["content"] -%} {%- if content is not string -%} {%- set content = content | tojson -%} {%- endif -%} {%- if message["role"] == "tool" -%} {%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%} {%- endif -%} {{- content + "<|im_end|>
+" -}}{%- endfor -%}{%- if add_generation_prompt -%} {{- "<|im_start|>assistant
+" -}}{%- endif -%}
\ No newline at end of file
diff --git a/studio/backend/tests/data/chat_templates/phi-4-mini.jinja b/studio/backend/tests/data/chat_templates/phi-4-mini.jinja
new file mode 100644
index 00000000000..b637de6b04d
--- /dev/null
+++ b/studio/backend/tests/data/chat_templates/phi-4-mini.jinja
@@ -0,0 +1,2 @@
+{# Source: https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/main/tokenizer_config.json #}
+{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}
diff --git a/studio/backend/tests/data/refactor_guard/ast_inventory.json b/studio/backend/tests/data/refactor_guard/ast_inventory.json
index 9b1c0dbb1c9..6e646f60a65 100644
--- a/studio/backend/tests/data/refactor_guard/ast_inventory.json
+++ b/studio/backend/tests/data/refactor_guard/ast_inventory.json
@@ -374,9 +374,6 @@
"_THREAD_OVERRIDE_FLAGS": {
"kind": "assign"
},
- "_TOOL_TEMPLATE_MARKERS": {
- "kind": "assign"
- },
"_UNRECOVERABLE_LOCAL_ERRNOS": {
"kind": "assign"
},
diff --git a/studio/backend/tests/test_safetensors_capability_advertise.py b/studio/backend/tests/test_safetensors_capability_advertise.py
index 6b42a873895..8e404ce1095 100644
--- a/studio/backend/tests/test_safetensors_capability_advertise.py
+++ b/studio/backend/tests/test_safetensors_capability_advertise.py
@@ -163,6 +163,72 @@ def test_detect_reasoning_flags_none_template_returns_all_false():
assert flags["reasoning_style"] == "enable_thinking"
+# Tool guards shipped templates actually write, each read as "no tools" by the older
+# exact-substring scan. A false greys out the Search and Code pills, so the user cannot correct it.
+ "label, guard",
+ [
+ # Granite 3.3 aliases the list before branching on it.
+ (
+ "granite_alias",
+ "{%- if tools and not available_tools -%}{{- tools | tojson }}{%- endif -%}",
+ ),
+ # No spaces inside the tag.
+ ("tight_whitespace", "{%-if tools%}{{- tools | tojson }}{%- endif -%}"),
+ # `is not none` rather than a truth test.
+ ("is_not_none", "{% if tools is not none %}{{ tools | tojson }}{% endif %}"),
+ # No guard at all, straight into the loop.
+ ("unguarded_loop", "{%- for tool in tools %}{{- tool | tojson }}{%- endfor %}"),
+ # An elif arm.
+ (
+ "elif_arm",
+ "{%- if documents %}{{- documents }}{%- elif tools %}{{- tools }}{%- endif %}",
+ ),
+ ],
+)
+def test_detect_reasoning_flags_reads_tool_guards_however_they_are_written(label, guard):
+ from core.inference.llama_cpp import detect_reasoning_flags
+ flags = detect_reasoning_flags(guard, f"vendor/{label}")
+ assert flags["supports_tools"] is True
+
+
+ "label, template",
+ [
+ # Prose, not a tool block.
+ ("prose_only", "{{- 'You are a helpful assistant with access to tools.' }}"),
+ # Studio passes tools as a kwarg, not a message field.
+ (
+ "phi4_message_scoped",
+ "{% if message['role'] == 'system' and 'tools' in message"
+ " and message['tools'] is not none %}{{ message['tools'] }}{% endif %}",
+ ),
+ # Excluding tool turns is not handling them.
+ (
+ "negated_role_check",
+ "{% for m in messages %}{% if m.role != 'tool' %}{{ m.content }}"
+ "{% endif %}{% endfor %}",
+ ),
+ # The word in a Jinja comment is not a capability.
+ (
+ "tool_calls_in_comment",
+ "{# tool_calls are deliberately unsupported #}"
+ "{% for m in messages %}{{ m.content }}{% endfor %}",
+ ),
+ # Llama 3.1's other switches: neither renders a schema.
+ (
+ "adjacent_switch_names",
+ "{%- if builtin_tools %}{{- 'x' }}{%- endif %}"
+ "{%- if tools_in_user_message %}{{- 'y' }}{%- endif %}",
+ ),
+ ],
+)
+def test_detect_reasoning_flags_does_not_invent_tool_support(label, template):
+ from core.inference.llama_cpp import detect_reasoning_flags
+ flags = detect_reasoning_flags(template, f"vendor/{label}")
+ assert flags["supports_tools"] is False
+
+
def test_detect_reasoning_flags_deepseek_v4_exposes_none_high_max():
"""DeepSeek-V4-Flash: enable_thinking gate + reasoning_effort 'max' preamble.
Classified as the hybrid style with the full none/high/max ladder even
diff --git a/studio/backend/tests/test_tool_template_detection.py b/studio/backend/tests/test_tool_template_detection.py
new file mode 100644
index 00000000000..ba1ecc5c4d6
--- /dev/null
+++ b/studio/backend/tests/test_tool_template_detection.py
@@ -0,0 +1,394 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tool detection reads Jinja syntax, not the spelling of it.
+
+The detector is an over-approximation on purpose, so the tests are split three ways:
+what it must get right, what it deliberately answers loosely, and what it must never
+do (raise, or disagree with itself over line endings).
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+from jinja2 import Environment
+
+_backend_root = Path(__file__).resolve().parent.parent
+if str(_backend_root) not in sys.path:
+ sys.path.insert(0, str(_backend_root))
+
+from core.inference.template_capabilities import ( # noqa: E402
+ template_supports_tools,
+)
+
+TOOLS = [
+ {"type": "function", "function": {"name": "get_weather", "parameters": {}}},
+ {"type": "function", "function": {"name": "get_time", "parameters": {}}},
+]
+
+
+def _published(name):
+ return (Path(__file__).parent / "data" / "chat_templates" / f"{name}.jinja").read_text(
+ encoding = "utf-8"
+ )
+
+
+def _renders_catalog(template, **context):
+ """Ground truth: does rendering this actually put a tool name in the output?"""
+ environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
+ output = environment.from_string(template).render(tools = list(TOOLS), **context)
+ return "get_weather" in output
+
+
+# ── the published templates this exists for ─────────────────────────────────
+ ("name", "detected"),
+ [
+ # Aliases the catalog and renders the alias; no marker matched. The bug.
+ ("granite-3.3", True),
+ # Fills a namespace field inside the guard, renders it outside, so nothing
+ # is emitted where the walk sees the guard. The marker scan said yes.
+ ("lfm2-tool", True),
+ # Names a `tools` variable but never renders anything derived from it.
+ ("phi-4-mini", False),
+ ],
+)
+def test_published_templates(name, detected):
+ assert template_supports_tools(_published(name)) is detected
+
+
+def test_lfm2_really_does_render_its_catalog():
+ """Pins the ground truth rather than the detector. LiquidAI/LFM2-1.2B-Tool is a
+ tool-calling model, and reporting it tool-less is the bug this file is about."""
+ environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
+ rendered = environment.from_string(_published("lfm2-tool")).render(
+ tools = list(TOOLS),
+ messages = [{"role": "user", "content": "what is the weather"}],
+ bos_token = "<s>",
+ add_generation_prompt = True,
+ )
+ assert "get_weather" in rendered
+
+
+def test_granite_really_does_render_its_catalog():
+ """Pins the ground truth rather than the detector, so this test still means
+ something if the detector changes."""
+ environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
+ rendered = environment.from_string(_published("granite-3.3")).render(
+ tools = list(TOOLS),
+ messages = [{"role": "user", "content": "what is the weather"}],
+ documents = [],
+ controls = {},
+ thinking = False,
+ add_generation_prompt = True,
+ strftime_now = lambda fmt: "January 01, 2026",
+ )
+ assert "get_weather" in rendered
+
+
+# ── the spellings a guard comes in ──────────────────────────────────────────
+ ("template", "detected"),
+ [
+ ("{% if tools %}{{ tools|tojson }}{% endif %}", True),
+ ("{%- if tools -%}{{ tools|tojson }}{%- endif -%}", True),
+ ("{% if tools is defined and tools %}{{ tools|tojson }}{% endif %}", True),
+ ("{% if tools and not available_tools %}{{ tools|tojson }}{% endif %}", True),
+ # A guarded branch advertises tools even when its body is only prose.
+ ("{% if tools %}You may call the tools listed above.{% endif %}", True),
+ # Tool results coming back from the model.
+ ("{% for m in messages %}{{ m.tool_calls|tojson }}{% endfor %}", True),
+ ("{% if message.role == 'tool' %}{{ message.content }}{% endif %}", True),
+ ("{% if message['role'] == 'tool' %}{{ message.content }}{% endif %}", True),
+ # Nothing to do with tools.
+ ("{% for m in messages %}{{ m.content }}{% endfor %}", False),
+ ("{% if enable_thinking %}<think>{% endif %}", False),
+ ("", False),
+ ("plain text, no jinja at all", False),
+ ],
+)
+def test_guard_spellings(template, detected):
+ assert template_supports_tools(template) is detected
+
+
+# ── carrying the catalog through names ──────────────────────────────────────
+ ("template", "detected"),
+ [
+ ("{% set catalog = tools %}{{ catalog|tojson }}", True),
+ ("{% set a = tools %}{% set b = a %}{{ b|tojson }}", True),
+ ("{% if tools %}{% set c = tools %}{% endif %}{{ c|tojson }}", True),
+ ("{% set catalog = tools %}{{ catalog|length }}", False),
+ # glm-4-9b-chat reads `tools` off a message inside the branch that renders
+ # it, so it never consumes what Studio passes in.
+ ("{% set tools = item['tools'] %}{{ tools|tojson }}", False),
+ ("{% set tools = none %}{{ tools }}", False),
+ ("{% set catalog = tools %}{% set catalog = [] %}{{ catalog|tojson }}", False),
+ # LFM2 builds its whole tool block this way: loop variable to namespace
+ # field, rendered outside any guard the walk can see.
+ (
+ "{% set ns = namespace(p='') %}{% for t in tools %}"
+ "{% set ns.p = ns.p + (t|tojson) %}{% endfor %}{{ ns.p }}",
+ True,
+ ),
+ ("{% set ns = namespace(c=none) %}{% set ns.c = tools %}{{ ns.c|tojson }}", True),
+ ("{% set ns = namespace(c=none) %}{% set ns.c = messages %}{{ ns.c|tojson }}", False),
+ ("{% set a, b = tools, none %}{{ a|tojson }}", True),
+ # Skip the branch and the catalog is still bound, so a rebinding inside one
+ # must not follow the walk out. The unconditional cases above still kill.
+ ("{% if legacy %}{% set tools = none %}{% endif %}{{ tools|tojson }}", True),
+ ("{% if legacy %}{% set tools = none %}{% else %}{{ tools|tojson }}{% endif %}", True),
+ (
+ "{% if a %}{% if b %}{% set tools = none %}{% endif %}{% endif %}{{ tools|tojson }}",
+ True,
+ ),
+ ("{% if a %}x{% elif b %}{% set tools = none %}{% endif %}{{ tools|tojson }}", True),
+ ("{% for m in messages %}{% set tools = m.tools %}{% endfor %}{{ tools|tojson }}", True),
+ ("{% macro unused() %}{% set tools = none %}{% endmacro %}{{ tools|tojson }}", True),
+ # On EVERY arm there is no surviving path, so an exhaustive chain kills.
+ (
+ "{% if x %}{% set tools = none %}{% else %}{% set tools = none %}{% endif %}"
+ "{{ tools|tojson }}",
+ False,
+ ),
+ (
+ "{% set c = tools %}{% if x %}{% set c = [] %}{% else %}{% set c = [] %}"
+ "{% endif %}{{ c|tojson }}",
+ False,
+ ),
+ (
+ "{% if x %}{% set tools = none %}{% elif y %}{% set tools = none %}"
+ "{% else %}{% set tools = none %}{% endif %}{{ tools|tojson }}",
+ False,
+ ),
+ # One arm leaving it alone is a path where it survives.
+ ("{% if x %}{% set tools = none %}{% else %}prose{% endif %}{{ tools|tojson }}", True),
+ # Undefined after `{% endfor %}`, so it carries nothing out.
+ ("{% for t in tools %}{% endfor %}{{ t|tojson }}", False),
+ (
+ "{% for t in tools %}{% endfor %}{% for t in messages %}{{ t.content }}{% endfor %}",
+ False,
+ ),
+ # A tuple target binds and rebinds like a plain name, both ways round.
+ ("{% set tools, flag = none, false %}{{ tools|tojson }}", False),
+ ("{% set a, b = tools, none %}{% set a, b = none, none %}{{ a|tojson }}", False),
+ # Stored under a tools guard, so tool-conditional whatever the value is.
+ (
+ "{% set intro = '' %}{% if tools %}{% set intro = 'You may call functions.' %}"
+ "{% endif %}{{ intro }}",
+ True,
+ ),
+ (
+ "{% set intro = '' %}{% if enable_thinking %}{% set intro = 'Think.' %}"
+ "{% endif %}{{ intro }}",
+ False,
+ ),
+ # The check can sit in the output expression, and a guarded branch can
+ # store its instructions through a side effect. Markers matched both.
+ ("{{ message.content if message.role == 'tool' else '' }}", True),
+ ("{{ message.content if message.role != 'tool' else '' }}", False),
+ ("{{ message.content if message.role == 'user' else '' }}", False),
+ (
+ "{% set ns = namespace(lines=[]) %}{% if tools %}"
+ "{% do ns.lines.append('You may call functions') %}{% endif %}"
+ "{{ ns.lines|join(',') }}",
+ True,
+ ),
+ (
+ "{% set ns = namespace(lines=[]) %}{% if enable_thinking %}"
+ "{% do ns.lines.append('Think') %}{% endif %}{{ ns.lines|join(',') }}",
+ False,
+ ),
+ # `.get('tool_calls')` is the same member read spelled as a call.
+ (
+ "{% set tool_calls = message.get('tool_calls') %}"
+ "{% if tool_calls is defined %}{{ tool_calls }}{% endif %}",
+ True,
+ ),
+ ("{% set tc = message.get('tool_calls') %}{% if tc %}{{ tc }}{% endif %}", True),
+ ("{% set x = message.get('content') %}{{ x }}", False),
+ # A stored predicate renders what the inline one does.
+ (
+ "{% set handles_tool = message.role == 'tool' %}"
+ "{% if handles_tool %}{{ message.content }}{% endif %}",
+ True,
+ ),
+ (
+ "{% set is_user = message.role == 'user' %}"
+ "{% if is_user %}{{ message.content }}{% endif %}",
+ False,
+ ),
+ # A loop's inline filter is a guard like any other.
+ ("{% for m in messages if m.role == 'tool' %}{{ m.content }}{% endfor %}", True),
+ ('{% for m in messages if m["role"] == "tool" %}{{ m.content }}{% endfor %}', True),
+ ("{% for m in messages if m.role != 'tool' %}{{ m.content }}{% endfor %}", False),
+ ('{% for m in messages if m.role == "user" %}{{ m.content }}{% endfor %}', False),
+ # `{% with %}` binds for its block and for nothing else.
+ ("{% with catalog = tools %}{{ catalog|tojson }}{% endwith %}", True),
+ ("{% with tools = none %}{{ tools|tojson }}{% endwith %}", False),
+ ("{% with tools = none %}x{% endwith %}{{ tools|tojson }}", True),
+ # Handing the catalog to a container puts it in that container.
+ ("{% set c = [] %}{% do c.append(tools) %}{{ c|tojson }}", True),
+ ("{% set c = [] %}{% set _ = c.append(tools) %}{{ c|tojson }}", True),
+ ("{% set ns = namespace(c=[]) %}{% do ns.c.extend(tools) %}{{ ns.c|tojson }}", True),
+ ("{% set c = [] %}{% do c.append(messages) %}{{ c|tojson }}", False),
+ ],
+)
+def test_names_carrying_the_catalog(template, detected):
+ assert template_supports_tools(template) is detected
+
+
+ "template",
+ [
+ "{% set catalog = tools %}{{ catalog|tojson }}",
+ "{% set a = tools %}{% set b = a %}{{ b|tojson }}",
+ "{% if tools %}{{ tools|tojson }}{% endif %}",
+ "{% for t in tools %}{{ t|tojson }}{% endfor %}",
+ "{% set c = [] %}{% do c.append(tools) %}{{ c|tojson }}",
+ "{% set ns = namespace(p='') %}{% for t in tools %}"
+ "{% set ns.p = ns.p + (t|tojson) %}{% endfor %}{{ ns.p }}",
+ "{% set ns = namespace(c=none) %}{% set ns.c = tools %}{{ ns.c|tojson }}",
+ "{% set a, b = tools, none %}{{ a|tojson }}",
+ "{% if legacy %}{% set tools = none %}{% endif %}{{ tools|tojson }}",
+ "{% macro unused() %}{% set tools = none %}{% endmacro %}{{ tools|tojson }}",
+ "{% with catalog = tools %}{{ catalog|tojson }}{% endwith %}",
+ "{% with tools = none %}x{% endwith %}{{ tools|tojson }}",
+ ],
+)
+def test_positives_match_a_real_render(template):
+ """Every case claimed positive here really does emit a tool name."""
+ assert _renders_catalog(template, item = {}, message = {}, messages = [], legacy = False)
+ assert template_supports_tools(template) is True
+
+
+# ── deliberate over-approximation ───────────────────────────────────────────
+ "template",
+ [
+ # A branch that can never run.
+ "{% if false %}{{ tools|tojson }}{% endif %}",
+ # A catalog put somewhere and then taken back out again.
+ "{% set c = [] %}{% do c.append(tools) %}{% do c.clear() %}{{ c|tojson }}",
+ # Iterating a mapping yields its keys, not the catalog under them.
+ "{% for key in {'weather': tools} %}{{ key }}{% endfor %}",
+ ],
+)
+def test_known_over_approximations_answer_yes(template):
+ """These render no schema, and the detector says yes anyway.
+
+ A value handed to a container is assumed to stay there, a branch is walked whether
+ or not it can run, and a mapping's values are not told apart from its keys.
+ Tracking any of that properly needs a much larger analysis, and none of it is
+ reachable from the 120 published templates checked. The error runs towards showing
+ a tool control that the backend then re-checks, rather than hiding one that works,
+ so it is left here on purpose rather than papered over.
+ """
+ assert not _renders_catalog(template, item = {}, message = {}, messages = [])
+ assert template_supports_tools(template) is True
+
+
+def test_a_guarded_branch_counts_even_without_naming_the_catalog():
+ """`{% if tools %}<prose>{% endif %}` renders no schema but is still telling the
+ model it has tools, so it counts. Asserted because it looks like a false positive
+ and is not one."""
+ template = "{% if tools %}You may call tools.{% endif %}"
+ assert not _renders_catalog(template)
+ assert template_supports_tools(template) is True
+
+
+# ── robustness ──────────────────────────────────────────────────────────────
+# Named, not generated: pytest exports the id in PYTEST_CURRENT_TEST and Windows
+# refuses an env var over 32767 characters, which the 5000-repeat id exceeds.
+ "template",
+ [
+ pytest.param("{% if tools %}", id = "unterminated_tag"),
+ pytest.param("{{ tools", id = "unterminated_expression"),
+ pytest.param("{%", id = "bare_tag_opener"),
+ pytest.param("{% if tools %}" * 400 + "x" + "{% endif %}" * 400, id = "deeply_nested_tags"),
+ pytest.param(
+ "{{ " + "(" * 300 + "tools" + ")" * 300 + " }}", id = "deeply_nested_expression"
+ ),
+ pytest.param("{{ tools|tojson }}" * 5000, id = "very_long"),
+ pytest.param("{% unknown_tag %}{{ tools|tojson }}", id = "unknown_tag"),
+ ],
+)
+def test_hostile_templates_never_raise(template):
+ """This runs on the GGUF metadata read and the llama-server launch, so it must
+ return a verdict rather than stop a model from loading."""
+ assert template_supports_tools(template) in (True, False)
+
+
[email protected]("template", [None, 123, b"bytes", {"default": "x"}, [{"name": "x"}]])
+def test_non_string_templates_are_turned_away(template):
+ """Hugging Face ships named-template maps and Hermes-style lists. These reach the
+ detector unhashable, so the type check has to sit outside the cache."""
+ assert template_supports_tools(template) is False
+
+
+def test_a_string_subclass_cannot_escape_the_fail_closed_branch():
+ """`isinstance` lets a subclass through, and then its own `__hash__` and
+ `__contains__` run before the try: the cache hashes the argument and the early-out
+ does `"tool" not in template`. Narrowing to str is what keeps those from reaching
+ a caller that has no except around the model load."""
+
+ class NoHash(str):
+ __hash__ = None
+
+ class BadContains(str):
+ def __contains__(self, item):
+ raise ValueError("injected")
+
+ template = "{% if tools %}{{ tools|tojson }}{% endif %}"
+ assert template_supports_tools(NoHash(template)) is True
+ assert template_supports_tools(BadContains(template)) is True
+
+
+ "template",
+ [
+ "{% if tools %}{{ tools|tojson }}{% endif %}",
+ "{% set catalog = tools %}{{ catalog|tojson }}",
+ "{% for m in messages %}{{ m.content }}{% endfor %}",
+ ],
+)
+def test_line_endings_and_byte_order_marks_do_not_change_the_verdict(template):
+ """A Windows checkout and a file saved with a BOM have to agree with everyone
+ else, which is the whole point of reading syntax rather than bytes."""
+ base = template_supports_tools(template)
+ assert template_supports_tools(template.replace("\n", "\r\n")) is base
+ assert template_supports_tools("" + template) is base
+
+
+def test_the_generation_tag_still_parses():
+ """`{% generation %}` marks the assistant span for training masks and is not a
+ real Jinja tag. Without the extension the template fails to parse and the
+ fail-closed branch would turn tools off. HuggingFaceTB/SmolLM3-3B needs this."""
+ template = (
+ "{% if tools %}{{ tools|tojson }}{% endif %}"
+ "{% generation %}{{ message.content }}{% endgeneration %}"
+ )
+ assert template_supports_tools(template) is True
+
+
+def test_unrelated_conditions_do_not_degrade():
+ """Nothing here forks a state per condition, so a template with many unrelated
+ guards costs no more than a linear walk. Qwen3-Coder-30B is the reason to keep
+ checking: it is long enough that any per-branch blow-up would reach it."""
+ template = (
+ "".join("{%% if flag%d %%}plain{%% endif %%}" % index for index in range(300))
+ + "{% if tools %}{{ tools|tojson }}{% endif %}"
+ )
+ assert template_supports_tools(template) is True
+
+
+def test_the_cache_is_keyed_on_the_template_text():
+ template = "{% if tools %}{{ tools|tojson }}{% endif %}"
+ template_supports_tools.cache_clear()
+ assert template_supports_tools(template) is True
+ assert template_supports_tools("".join([template])) is True
+ assert template_supports_tools.cache_info().hits >= 1