Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

14.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Tool detection reads Jinja syntax and deliberately over-approximates; it must never raise."""

from __future__ import annotations

import sys
from pathlib import Path

import pytest
from jinja2 import Environment

_backend_root = Path(__file__).resolve().parent.parent
if str(_backend_root) not in sys.path:
    sys.path.insert(0, str(_backend_root))

from core.inference.template_capabilities import (  # noqa: E402
    template_supports_tools,
)

TOOLS = [
    {"type": "function", "function": {"name": "get_weather", "parameters": {}}},
    {"type": "function", "function": {"name": "get_time", "parameters": {}}},
]


def _published(name):
    return (Path(__file__).parent / "data" / "chat_templates" / f"{name}.jinja").read_text(
        encoding = "utf-8"
    )


def _renders_catalog(template, **context):
    """Ground truth: does rendering this actually put a tool name in the output?"""
    environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
    output = environment.from_string(template).render(tools = list(TOOLS), **context)
    return "get_weather" in output


@pytest.mark.parametrize(
    ("name", "detected"),
    [
        # Aliases the catalog and renders the alias; no marker matched. The bug.
        ("granite-3.3", True),
        # Namespace field filled inside the guard, rendered outside; the marker scan said yes.
        ("lfm2-tool", True),
        # Names a `tools` variable but never renders anything derived from it.
        ("phi-4-mini", False),
    ],
)
def test_published_templates(name, detected):
    assert template_supports_tools(_published(name)) is detected


def test_lfm2_really_does_render_its_catalog():
    """Pins the ground truth rather than the detector. LiquidAI/LFM2-1.2B-Tool is a
    tool-calling model, and reporting it tool-less is the bug this file is about."""
    environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
    rendered = environment.from_string(_published("lfm2-tool")).render(
        tools = list(TOOLS),
        messages = [{"role": "user", "content": "what is the weather"}],
        bos_token = "<s>",
        add_generation_prompt = True,
    )
    assert "get_weather" in rendered


def test_granite_really_does_render_its_catalog():
    """Pins Jinja's real rendering, not the detector, so it outlives detector changes."""
    environment = Environment(extensions = ["jinja2.ext.loopcontrols", "jinja2.ext.do"])
    rendered = environment.from_string(_published("granite-3.3")).render(
        tools = list(TOOLS),
        messages = [{"role": "user", "content": "what is the weather"}],
        documents = [],
        controls = {},
        thinking = False,
        add_generation_prompt = True,
        strftime_now = lambda fmt: "January 01, 2026",
    )
    assert "get_weather" in rendered


@pytest.mark.parametrize(
    ("template", "detected"),
    [
        ("{% if tools %}{{ tools|tojson }}{% endif %}", True),
        ("{%- if tools -%}{{ tools|tojson }}{%- endif -%}", True),
        ("{% if tools is defined and tools %}{{ tools|tojson }}{% endif %}", True),
        ("{% if tools and not available_tools %}{{ tools|tojson }}{% endif %}", True),
        ("{% if tools %}You may call the tools listed above.{% endif %}", True),
        ("{% for m in messages %}{{ m.tool_calls|tojson }}{% endfor %}", True),
        ("{% if message.role == 'tool' %}{{ message.content }}{% endif %}", True),
        ("{% if message['role'] == 'tool' %}{{ message.content }}{% endif %}", True),
        ("{% for m in messages %}{{ m.content }}{% endfor %}", False),
        ("{% if enable_thinking %}<think>{% endif %}", False),
        ("", False),
        ("plain text, no jinja at all", False),
    ],
)
def test_guard_spellings(template, detected):
    assert template_supports_tools(template) is detected


@pytest.mark.parametrize(
    ("template", "detected"),
    [
        ("{% set catalog = tools %}{{ catalog|tojson }}", True),
        ("{% set a = tools %}{% set b = a %}{{ b|tojson }}", True),
        ("{% if tools %}{% set c = tools %}{% endif %}{{ c|tojson }}", True),
        ("{% set catalog = tools %}{{ catalog|length }}", False),
        # glm-4-9b-chat reads tools off a message, never the catalog Studio passes in.
        ("{% set tools = item['tools'] %}{{ tools|tojson }}", False),
        ("{% set tools = none %}{{ tools }}", False),
        ("{% set catalog = tools %}{% set catalog = [] %}{{ catalog|tojson }}", False),
        # LFM2 copies a loop variable into a namespace field, rendered outside any guard.
        (
            "{% set ns = namespace(p='') %}{% for t in tools %}"
            "{% set ns.p = ns.p + (t|tojson) %}{% endfor %}{{ ns.p }}",
            True,
        ),
        ("{% set ns = namespace(c=none) %}{% set ns.c = tools %}{{ ns.c|tojson }}", True),
        ("{% set ns = namespace(c=none) %}{% set ns.c = messages %}{{ ns.c|tojson }}", False),
        ("{% set a, b = tools, none %}{{ a|tojson }}", True),
        # Skipping the branch leaves the catalog bound, so a rebinding inside must not leak.
        ("{% if legacy %}{% set tools = none %}{% endif %}{{ tools|tojson }}", True),
        ("{% if legacy %}{% set tools = none %}{% else %}{{ tools|tojson }}{% endif %}", True),
        (
            "{% if a %}{% if b %}{% set tools = none %}{% endif %}{% endif %}{{ tools|tojson }}",
            True,
        ),
        ("{% if a %}x{% elif b %}{% set tools = none %}{% endif %}{{ tools|tojson }}", True),
        ("{% for m in messages %}{% set tools = m.tools %}{% endfor %}{{ tools|tojson }}", True),
        ("{% macro unused() %}{% set tools = none %}{% endmacro %}{{ tools|tojson }}", True),
        (
            "{% if x %}{% set tools = none %}{% else %}{% set tools = none %}{% endif %}"
            "{{ tools|tojson }}",
            False,
        ),
        (
            "{% set c = tools %}{% if x %}{% set c = [] %}{% else %}{% set c = [] %}"
            "{% endif %}{{ c|tojson }}",
            False,
        ),
        (
            "{% if x %}{% set tools = none %}{% elif y %}{% set tools = none %}"
            "{% else %}{% set tools = none %}{% endif %}{{ tools|tojson }}",
            False,
        ),
        ("{% if x %}{% set tools = none %}{% else %}prose{% endif %}{{ tools|tojson }}", True),
        # Undefined after `{% endfor %}`, so it carries nothing out.
        ("{% for t in tools %}{% endfor %}{{ t|tojson }}", False),
        (
            "{% for t in tools %}{% endfor %}{% for t in messages %}{{ t.content }}{% endfor %}",
            False,
        ),
        ("{% set tools, flag = none, false %}{{ tools|tojson }}", False),
        ("{% set a, b = tools, none %}{% set a, b = none, none %}{{ a|tojson }}", False),
        (
            "{% set intro = '' %}{% if tools %}{% set intro = 'You may call functions.' %}"
            "{% endif %}{{ intro }}",
            True,
        ),
        (
            "{% set intro = '' %}{% if enable_thinking %}{% set intro = 'Think.' %}"
            "{% endif %}{{ intro }}",
            False,
        ),
        ("{{ message.content if message.role == 'tool' else '' }}", True),
        ("{{ message.content if message.role != 'tool' else '' }}", False),
        ("{{ message.content if message.role == 'user' else '' }}", False),
        (
            "{% set ns = namespace(lines=[]) %}{% if tools %}"
            "{% do ns.lines.append('You may call functions') %}{% endif %}"
            "{{ ns.lines|join(',') }}",
            True,
        ),
        (
            "{% set ns = namespace(lines=[]) %}{% if enable_thinking %}"
            "{% do ns.lines.append('Think') %}{% endif %}{{ ns.lines|join(',') }}",
            False,
        ),
        (
            "{% set tool_calls = message.get('tool_calls') %}"
            "{% if tool_calls is defined %}{{ tool_calls }}{% endif %}",
            True,
        ),
        ("{% set tc = message.get('tool_calls') %}{% if tc %}{{ tc }}{% endif %}", True),
        ("{% set x = message.get('content') %}{{ x }}", False),
        (
            "{% set handles_tool = message.role == 'tool' %}"
            "{% if handles_tool %}{{ message.content }}{% endif %}",
            True,
        ),
        (
            "{% set is_user = message.role == 'user' %}"
            "{% if is_user %}{{ message.content }}{% endif %}",
            False,
        ),
        ("{% for m in messages if m.role == 'tool' %}{{ m.content }}{% endfor %}", True),
        ('{% for m in messages if m["role"] == "tool" %}{{ m.content }}{% endfor %}', True),
        ("{% for m in messages if m.role != 'tool' %}{{ m.content }}{% endfor %}", False),
        ('{% for m in messages if m.role == "user" %}{{ m.content }}{% endfor %}', False),
        ("{% with catalog = tools %}{{ catalog|tojson }}{% endwith %}", True),
        ("{% with tools = none %}{{ tools|tojson }}{% endwith %}", False),
        ("{% with tools = none %}x{% endwith %}{{ tools|tojson }}", True),
        ("{% set c = [] %}{% do c.append(tools) %}{{ c|tojson }}", True),
        ("{% set c = [] %}{% set _ = c.append(tools) %}{{ c|tojson }}", True),
        ("{% set ns = namespace(c=[]) %}{% do ns.c.extend(tools) %}{{ ns.c|tojson }}", True),
        ("{% set c = [] %}{% do c.append(messages) %}{{ c|tojson }}", False),
    ],
)
def test_names_carrying_the_catalog(template, detected):
    assert template_supports_tools(template) is detected


@pytest.mark.parametrize(
    "template",
    [
        "{% set catalog = tools %}{{ catalog|tojson }}",
        "{% set a = tools %}{% set b = a %}{{ b|tojson }}",
        "{% if tools %}{{ tools|tojson }}{% endif %}",
        "{% for t in tools %}{{ t|tojson }}{% endfor %}",
        "{% set c = [] %}{% do c.append(tools) %}{{ c|tojson }}",
        "{% set ns = namespace(p='') %}{% for t in tools %}"
        "{% set ns.p = ns.p + (t|tojson) %}{% endfor %}{{ ns.p }}",
        "{% set ns = namespace(c=none) %}{% set ns.c = tools %}{{ ns.c|tojson }}",
        "{% set a, b = tools, none %}{{ a|tojson }}",
        "{% if legacy %}{% set tools = none %}{% endif %}{{ tools|tojson }}",
        "{% macro unused() %}{% set tools = none %}{% endmacro %}{{ tools|tojson }}",
        "{% with catalog = tools %}{{ catalog|tojson }}{% endwith %}",
        "{% with tools = none %}x{% endwith %}{{ tools|tojson }}",
    ],
)
def test_positives_match_a_real_render(template):
    """Every case claimed positive here really does emit a tool name."""
    assert _renders_catalog(template, item = {}, message = {}, messages = [], legacy = False)
    assert template_supports_tools(template) is True


@pytest.mark.parametrize(
    "template",
    [
        "{% if false %}{{ tools|tojson }}{% endif %}",
        "{% set c = [] %}{% do c.append(tools) %}{% do c.clear() %}{{ c|tojson }}",
        "{% for key in {'weather': tools} %}{{ key }}{% endfor %}",
    ],
)
def test_known_over_approximations_answer_yes(template):
    """Over-approximations answer yes: a wrongly shown tool control is re-checked, not wrongly hidden."""
    assert not _renders_catalog(template, item = {}, message = {}, messages = [])
    assert template_supports_tools(template) is True


def test_a_guarded_branch_counts_even_without_naming_the_catalog():
    """`{% if tools %}<prose>{% endif %}` renders no schema but is still telling the
    model it has tools, so it counts. Asserted because it looks like a false positive
    and is not one."""
    template = "{% if tools %}You may call tools.{% endif %}"
    assert not _renders_catalog(template)
    assert template_supports_tools(template) is True


# Named, not generated: Windows refuses PYTEST_CURRENT_TEST over 32767 chars.
@pytest.mark.parametrize(
    "template",
    [
        pytest.param("{% if tools %}", id = "unterminated_tag"),
        pytest.param("{{ tools", id = "unterminated_expression"),
        pytest.param("{%", id = "bare_tag_opener"),
        pytest.param("{% if tools %}" * 400 + "x" + "{% endif %}" * 400, id = "deeply_nested_tags"),
        pytest.param(
            "{{ " + "(" * 300 + "tools" + ")" * 300 + " }}", id = "deeply_nested_expression"
        ),
        pytest.param("{{ tools|tojson }}" * 5000, id = "very_long"),
        pytest.param("{% unknown_tag %}{{ tools|tojson }}", id = "unknown_tag"),
    ],
)
def test_hostile_templates_never_raise(template):
    """Runs on the GGUF metadata read and llama-server launch, so it must never raise."""
    assert template_supports_tools(template) in (True, False)


@pytest.mark.parametrize("template", [None, 123, b"bytes", {"default": "x"}, [{"name": "x"}]])
def test_non_string_templates_are_turned_away(template):
    """Unhashable template maps and lists: the type check must sit outside the cache."""
    assert template_supports_tools(template) is False


def test_a_string_subclass_cannot_escape_the_fail_closed_branch():
    """Only exact str passes: a str subclass's own __hash__ or __contains__ could run unguarded."""

    class NoHash(str):
        __hash__ = None

    class BadContains(str):
        def __contains__(self, item):
            raise ValueError("injected")

    template = "{% if tools %}{{ tools|tojson }}{% endif %}"
    assert template_supports_tools(NoHash(template)) is True
    assert template_supports_tools(BadContains(template)) is True


@pytest.mark.parametrize(
    "template",
    [
        "{% if tools %}{{ tools|tojson }}{% endif %}",
        "{% set catalog = tools %}{{ catalog|tojson }}",
        "{% for m in messages %}{{ m.content }}{% endfor %}",
    ],
)
def test_line_endings_and_byte_order_marks_do_not_change_the_verdict(template):
    """Line endings and a BOM must not change the verdict; detection reads syntax, not bytes."""
    base = template_supports_tools(template)
    assert template_supports_tools(template.replace("\n", "\r\n")) is base
    assert template_supports_tools("" + template) is base


def test_the_generation_tag_still_parses():
    """`{% generation %}` marks the assistant span for training masks and is not a
    real Jinja tag. Without the extension the template fails to parse and the
    fail-closed branch would turn tools off. HuggingFaceTB/SmolLM3-3B needs this."""
    template = (
        "{% if tools %}{{ tools|tojson }}{% endif %}"
        "{% generation %}{{ message.content }}{% endgeneration %}"
    )
    assert template_supports_tools(template) is True


def test_unrelated_conditions_do_not_degrade():
    """Unrelated guards must not fork state per condition; Qwen3-Coder-30B catches any blow-up."""
    template = (
        "".join("{%% if flag%d %%}plain{%% endif %%}" % index for index in range(300))
        + "{% if tools %}{{ tools|tojson }}{% endif %}"
    )
    assert template_supports_tools(template) is True


def test_the_cache_is_keyed_on_the_template_text():
    template = "{% if tools %}{{ tools|tojson }}{% endif %}"
    template_supports_tools.cache_clear()
    assert template_supports_tools(template) is True
    assert template_supports_tools("".join([template])) is True
    assert template_supports_tools.cache_info().hits >= 1