Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Tool detection reads Jinja syntax and deliberately over-approximates; it must never raise."""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
from jinja2 import Environment
_backend_root = Path(__file__).resolve().parent.parent
if str(_backend_root) not in sys.path:
sys.path.insert(0, str(_backend_root))
from core.inference.template_capabilities import ( # noqa: E402
template_supports_tools,
)
TOOLS = [
{"type": "function", "function": {"name": "get_weather", "parameters": {}}},
{"type": "function", "function": {"name": "get_time", "parameters": {}}},
]
def _published(name):
return (Path(__file__).parent / "data" / "chat_templates" / f"{name}.jinja").read_text(
encoding = "utf-8"
)
def _renders_catalog(template, **context):
"""Ground truth: does rendering this actually put a tool name in the output?"""
environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
output = environment.from_string(template).render(tools = list(TOOLS), **context)
return "get_weather" in output
@pytest.mark.parametrize(
("name", "detected"),
[
# Aliases the catalog and renders the alias; no marker matched. The bug.
("granite-3.3", True),
# Namespace field filled inside the guard, rendered outside; the marker scan said yes.
("lfm2-tool", True),
# Names a `tools` variable but never renders anything derived from it.
("phi-4-mini", False),
],
)
def test_published_templates(name, detected):
assert template_supports_tools(_published(name)) is detected
def test_lfm2_really_does_render_its_catalog():
"""Pins the ground truth rather than the detector. LiquidAI/LFM2-1.2B-Tool is a
tool-calling model, and reporting it tool-less is the bug this file is about."""
environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
rendered = environment.from_string(_published("lfm2-tool")).render(
tools = list(TOOLS),
messages = [{"role": "user", "content": "what is the weather"}],
bos_token = "<s>",
add_generation_prompt = True,
)
assert "get_weather" in rendered
def test_granite_really_does_render_its_catalog():
"""Pins Jinja's real rendering, not the detector, so it outlives detector changes."""
environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
rendered = environment.from_string(_published("granite-3.3")).render(
tools = list(TOOLS),
messages = [{"role": "user", "content": "what is the weather"}],
documents = [],
controls = {},
thinking = False,
add_generation_prompt = True,
strftime_now = lambda fmt: "January 01, 2026",
)
assert "get_weather" in rendered
@pytest.mark.parametrize(
("template", "detected"),
[
("{% if tools %}{{ tools|tojson }}{% endif %}", True),
("{%- if tools -%}{{ tools|tojson }}{%- endif -%}", True),
("{% if tools is defined and tools %}{{ tools|tojson }}{% endif %}", True),
("{% if tools and not available_tools %}{{ tools|tojson }}{% endif %}", True),
("{% if tools %}You may call the tools listed above.{% endif %}", True),
("{% for m in messages %}{{ m.tool_calls|tojson }}{% endfor %}", True),
("{% if message.role == 'tool' %}{{ message.content }}{% endif %}", True),
("{% if message['role'] == 'tool' %}{{ message.content }}{% endif %}", True),
("{% for m in messages %}{{ m.content }}{% endfor %}", False),
("{% if enable_thinking %}<think>{% endif %}", False),
("", False),
("plain text, no jinja at all", False),
],
)
def test_guard_spellings(template, detected):
assert template_supports_tools(template) is detected
@pytest.mark.parametrize(
("template", "detected"),
[
("{% set catalog = tools %}{{ catalog|tojson }}", True),
("{% set a = tools %}{% set b = a %}{{ b|tojson }}", True),
("{% if tools %}{% set c = tools %}{% endif %}{{ c|tojson }}", True),
("{% set catalog = tools %}{{ catalog|length }}", False),
# glm-4-9b-chat reads tools off a message, never the catalog Studio passes in.
("{% set tools = item['tools'] %}{{ tools|tojson }}", False),
("{% set tools = none %}{{ tools }}", False),
("{% set catalog = tools %}{% set catalog = [] %}{{ catalog|tojson }}", False),
# LFM2 copies a loop variable into a namespace field, rendered outside any guard.
(
"{% set ns = namespace(p='') %}{% for t in tools %}"
"{% set ns.p = ns.p + (t|tojson) %}{% endfor %}{{ ns.p }}",
True,
),
("{% set ns = namespace(c=none) %}{% set ns.c = tools %}{{ ns.c|tojson }}", True),
("{% set ns = namespace(c=none) %}{% set ns.c = messages %}{{ ns.c|tojson }}", False),
("{% set a, b = tools, none %}{{ a|tojson }}", True),
# Skipping the branch leaves the catalog bound, so a rebinding inside must not leak.
("{% if legacy %}{% set tools = none %}{% endif %}{{ tools|tojson }}", True),
("{% if legacy %}{% set tools = none %}{% else %}{{ tools|tojson }}{% endif %}", True),
(
"{% if a %}{% if b %}{% set tools = none %}{% endif %}{% endif %}{{ tools|tojson }}",
True,
),
("{% if a %}x{% elif b %}{% set tools = none %}{% endif %}{{ tools|tojson }}", True),
("{% for m in messages %}{% set tools = m.tools %}{% endfor %}{{ tools|tojson }}", True),
("{% macro unused() %}{% set tools = none %}{% endmacro %}{{ tools|tojson }}", True),
(
"{% if x %}{% set tools = none %}{% else %}{% set tools = none %}{% endif %}"
"{{ tools|tojson }}",
False,
),
(
"{% set c = tools %}{% if x %}{% set c = [] %}{% else %}{% set c = [] %}"
"{% endif %}{{ c|tojson }}",
False,
),
(
"{% if x %}{% set tools = none %}{% elif y %}{% set tools = none %}"
"{% else %}{% set tools = none %}{% endif %}{{ tools|tojson }}",
False,
),
("{% if x %}{% set tools = none %}{% else %}prose{% endif %}{{ tools|tojson }}", True),
# Undefined after `{% endfor %}`, so it carries nothing out.
("{% for t in tools %}{% endfor %}{{ t|tojson }}", False),
(
"{% for t in tools %}{% endfor %}{% for t in messages %}{{ t.content }}{% endfor %}",
False,
),
("{% set tools, flag = none, false %}{{ tools|tojson }}", False),
("{% set a, b = tools, none %}{% set a, b = none, none %}{{ a|tojson }}", False),
(
"{% set intro = '' %}{% if tools %}{% set intro = 'You may call functions.' %}"
"{% endif %}{{ intro }}",
True,
),
(
"{% set intro = '' %}{% if enable_thinking %}{% set intro = 'Think.' %}"
"{% endif %}{{ intro }}",
False,
),
("{{ message.content if message.role == 'tool' else '' }}", True),
("{{ message.content if message.role != 'tool' else '' }}", False),
("{{ message.content if message.role == 'user' else '' }}", False),
(
"{% set ns = namespace(lines=[]) %}{% if tools %}"
"{% do ns.lines.append('You may call functions') %}{% endif %}"
"{{ ns.lines|join(',') }}",
True,
),
(
"{% set ns = namespace(lines=[]) %}{% if enable_thinking %}"
"{% do ns.lines.append('Think') %}{% endif %}{{ ns.lines|join(',') }}",
False,
),
(
"{% set tool_calls = message.get('tool_calls') %}"
"{% if tool_calls is defined %}{{ tool_calls }}{% endif %}",
True,
),
("{% set tc = message.get('tool_calls') %}{% if tc %}{{ tc }}{% endif %}", True),
("{% set x = message.get('content') %}{{ x }}", False),
(
"{% set handles_tool = message.role == 'tool' %}"
"{% if handles_tool %}{{ message.content }}{% endif %}",
True,
),
(
"{% set is_user = message.role == 'user' %}"
"{% if is_user %}{{ message.content }}{% endif %}",
False,
),
("{% for m in messages if m.role == 'tool' %}{{ m.content }}{% endfor %}", True),
('{% for m in messages if m["role"] == "tool" %}{{ m.content }}{% endfor %}', True),
("{% for m in messages if m.role != 'tool' %}{{ m.content }}{% endfor %}", False),
('{% for m in messages if m.role == "user" %}{{ m.content }}{% endfor %}', False),
("{% with catalog = tools %}{{ catalog|tojson }}{% endwith %}", True),
("{% with tools = none %}{{ tools|tojson }}{% endwith %}", False),
("{% with tools = none %}x{% endwith %}{{ tools|tojson }}", True),
("{% set c = [] %}{% do c.append(tools) %}{{ c|tojson }}", True),
("{% set c = [] %}{% set _ = c.append(tools) %}{{ c|tojson }}", True),
("{% set ns = namespace(c=[]) %}{% do ns.c.extend(tools) %}{{ ns.c|tojson }}", True),
("{% set c = [] %}{% do c.append(messages) %}{{ c|tojson }}", False),
],
)
def test_names_carrying_the_catalog(template, detected):
assert template_supports_tools(template) is detected
@pytest.mark.parametrize(
"template",
[
"{% set catalog = tools %}{{ catalog|tojson }}",
"{% set a = tools %}{% set b = a %}{{ b|tojson }}",
"{% if tools %}{{ tools|tojson }}{% endif %}",
"{% for t in tools %}{{ t|tojson }}{% endfor %}",
"{% set c = [] %}{% do c.append(tools) %}{{ c|tojson }}",
"{% set ns = namespace(p='') %}{% for t in tools %}"
"{% set ns.p = ns.p + (t|tojson) %}{% endfor %}{{ ns.p }}",
"{% set ns = namespace(c=none) %}{% set ns.c = tools %}{{ ns.c|tojson }}",
"{% set a, b = tools, none %}{{ a|tojson }}",
"{% if legacy %}{% set tools = none %}{% endif %}{{ tools|tojson }}",
"{% macro unused() %}{% set tools = none %}{% endmacro %}{{ tools|tojson }}",
"{% with catalog = tools %}{{ catalog|tojson }}{% endwith %}",
"{% with tools = none %}x{% endwith %}{{ tools|tojson }}",
],
)
def test_positives_match_a_real_render(template):
"""Every case claimed positive here really does emit a tool name."""
assert _renders_catalog(template, item = {}, message = {}, messages = [], legacy = False)
assert template_supports_tools(template) is True
@pytest.mark.parametrize(
"template",
[
"{% if false %}{{ tools|tojson }}{% endif %}",
"{% set c = [] %}{% do c.append(tools) %}{% do c.clear() %}{{ c|tojson }}",
"{% for key in {'weather': tools} %}{{ key }}{% endfor %}",
],
)
def test_known_over_approximations_answer_yes(template):
"""Over-approximations answer yes: a wrongly shown tool control is re-checked, not wrongly hidden."""
assert not _renders_catalog(template, item = {}, message = {}, messages = [])
assert template_supports_tools(template) is True
def test_a_guarded_branch_counts_even_without_naming_the_catalog():
"""`{% if tools %}<prose>{% endif %}` renders no schema but is still telling the
model it has tools, so it counts. Asserted because it looks like a false positive
and is not one."""
template = "{% if tools %}You may call tools.{% endif %}"
assert not _renders_catalog(template)
assert template_supports_tools(template) is True
# Named, not generated: Windows refuses PYTEST_CURRENT_TEST over 32767 chars.
@pytest.mark.parametrize(
"template",
[
pytest.param("{% if tools %}", id = "unterminated_tag"),
pytest.param("{{ tools", id = "unterminated_expression"),
pytest.param("{%", id = "bare_tag_opener"),
pytest.param("{% if tools %}" * 400 + "x" + "{% endif %}" * 400, id = "deeply_nested_tags"),
pytest.param(
"{{ " + "(" * 300 + "tools" + ")" * 300 + " }}", id = "deeply_nested_expression"
),
pytest.param("{{ tools|tojson }}" * 5000, id = "very_long"),
pytest.param("{% unknown_tag %}{{ tools|tojson }}", id = "unknown_tag"),
],
)
def test_hostile_templates_never_raise(template):
"""Runs on the GGUF metadata read and llama-server launch, so it must never raise."""
assert template_supports_tools(template) in (True, False)
@pytest.mark.parametrize("template", [None, 123, b"bytes", {"default": "x"}, [{"name": "x"}]])
def test_non_string_templates_are_turned_away(template):
"""Unhashable template maps and lists: the type check must sit outside the cache."""
assert template_supports_tools(template) is False
def test_a_string_subclass_cannot_escape_the_fail_closed_branch():
"""Only exact str passes: a str subclass's own __hash__ or __contains__ could run unguarded."""
class NoHash(str):
__hash__ = None
class BadContains(str):
def __contains__(self, item):
raise ValueError("injected")
template = "{% if tools %}{{ tools|tojson }}{% endif %}"
assert template_supports_tools(NoHash(template)) is True
assert template_supports_tools(BadContains(template)) is True
@pytest.mark.parametrize(
"template",
[
"{% if tools %}{{ tools|tojson }}{% endif %}",
"{% set catalog = tools %}{{ catalog|tojson }}",
"{% for m in messages %}{{ m.content }}{% endfor %}",
],
)
def test_line_endings_and_byte_order_marks_do_not_change_the_verdict(template):
"""Line endings and a BOM must not change the verdict; detection reads syntax, not bytes."""
base = template_supports_tools(template)
assert template_supports_tools(template.replace("\n", "\r\n")) is base
assert template_supports_tools("" + template) is base
def test_the_generation_tag_still_parses():
"""`{% generation %}` marks the assistant span for training masks and is not a
real Jinja tag. Without the extension the template fails to parse and the
fail-closed branch would turn tools off. HuggingFaceTB/SmolLM3-3B needs this."""
template = (
"{% if tools %}{{ tools|tojson }}{% endif %}"
"{% generation %}{{ message.content }}{% endgeneration %}"
)
assert template_supports_tools(template) is True
def test_unrelated_conditions_do_not_degrade():
"""Unrelated guards must not fork state per condition; Qwen3-Coder-30B catches any blow-up."""
template = (
"".join("{%% if flag%d %%}plain{%% endif %%}" % index for index in range(300))
+ "{% if tools %}{{ tools|tojson }}{% endif %}"
)
assert template_supports_tools(template) is True
def test_the_cache_is_keyed_on_the_template_text():
template = "{% if tools %}{{ tools|tojson }}{% endif %}"
template_supports_tools.cache_clear()
assert template_supports_tools(template) is True
assert template_supports_tools("".join([template])) is True
assert template_supports_tools.cache_info().hits >= 1