Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Regressions for native tool-token provenance and opaque markerless spans."""
import json
import pytest
import torch
import importlib
import sys
import types
from unittest.mock import MagicMock
# inference.py imports unsloth; stub explicitly, the source scan misses this path.
_STUBBED: list[str] = []
def _stub_if_missing(name, attrs):
"""Stub a dep the backend test matrix does not install, or collection fails and takes the job down."""
if name in sys.modules:
return
try:
importlib.import_module(name)
return
except Exception: # noqa: BLE001 - unusable here either way, so stub it
pass
_STUBBED.append(name)
mod = types.ModuleType(name)
mod.__spec__ = None
for attr in attrs:
setattr(mod, attr, MagicMock())
sys.modules[name] = mod
parent, _, child = name.rpartition(".")
if parent and parent in sys.modules:
setattr(sys.modules[parent], child, mod)
_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
from core.inference.native_tool_tokens import (
NATIVE_TOOL_CONTROL_TOKENS,
NativeToolTokenDecoder,
decode_with_native_tool_tokens,
decoder_preserves_token,
stop_token_text,
)
from core.inference.safetensors_agentic import run_safetensors_tool_loop
from core.inference.tool_call_parser import parse_tool_calls_from_text
# Bind under the stubs, then drop them: leftover stubs make _shared_policy return None.
import core.inference.inference # noqa: F401,E402 - imported to bind it under the stubs
for _name in reversed(_STUBBED):
sys.modules.pop(_name, None)
class _GemmaTokenDecoder:
_pieces = {
1: "<|tool_call>",
2: "call:terminal{command:",
3: '<|"|>',
4: "id",
5: "}",
6: "<tool_call|>",
7: "<eos>",
8: 'The page quoted call:terminal{command:"id"}, but I did not run it.',
}
all_special_ids = [1, 3, 6, 7]
all_special_tokens = ["<|tool_call>", '<|"|>', "<tool_call|>", "<eos>"]
def convert_ids_to_tokens(self, token_id):
return self._pieces[token_id]
def decode(
self,
token_ids,
*,
skip_special_tokens = False,
**_kwargs,
):
special = set(self.all_special_ids) if skip_special_tokens else set()
return "".join(
self._pieces[int(token_id)] for token_id in token_ids if token_id not in special
)
class _PieceTokenizer:
def __init__(self, pieces, special_ids):
self.pieces = dict(enumerate(pieces, start = 1))
self.all_special_ids = list(special_ids)
self.all_special_tokens = [self.pieces[token_id] for token_id in special_ids]
def convert_ids_to_tokens(self, token_id):
return self.pieces[token_id]
def decode(
self,
token_ids,
*,
skip_special_tokens = False,
**_kwargs,
):
special = set(self.all_special_ids) if skip_special_tokens else set()
return "".join(
self.pieces[int(token_id)] for token_id in token_ids if token_id not in special
)
class _SlowSpacingTokenizer(_PieceTokenizer):
def decode(
self,
token_ids,
*,
skip_special_tokens = False,
spaces_between_special_tokens = True,
**_kwargs,
):
special = set(self.all_special_ids) if skip_special_tokens else set()
pieces = [self.pieces[int(token_id)] for token_id in token_ids if token_id not in special]
return (" " if spaces_between_special_tokens else "").join(pieces)
def _decode_transformers_tokens(token_ids):
from core.inference.inference import InferenceBackend
backend = InferenceBackend.__new__(InferenceBackend)
streamer = backend._make_text_streamer(
_GemmaTokenDecoder(),
reasoning_channel_markers = None,
reasoning_channel_markers_resolved = True,
skip_prompt = False,
preserve_tool_tokens = True,
)
streamer.put(torch.tensor(token_ids))
streamer.end()
return "".join(streamer)
def _run_tool_loop(first_turn):
turns = iter((first_turn, "done"))
executed = []
def _single_turn(_messages, **_kwargs):
try:
yield next(turns)
except StopIteration:
return
def _execute(name, arguments, **_kwargs):
executed.append((name, arguments))
return "ok"
events = list(
run_safetensors_tool_loop(
single_turn = _single_turn,
messages = [{"role": "user", "content": "run it"}],
tools = [{"type": "function", "function": {"name": "terminal"}}],
execute_tool = _execute,
nudge_tool_calls = False,
max_tool_iterations = 2,
permission_mode = "off",
)
)
return executed, events
def test_transformers_native_gemma_token_ids_reach_execution_with_wrapper_provenance():
decoded = _decode_transformers_tokens([1, 2, 3, 4, 3, 5, 6, 7])
assert decoded == '<|tool_call>call:terminal{command:<|"|>id<|"|>}<tool_call|>'
executed, events = _run_tool_loop(decoded)
assert executed == [("terminal", {"command": "id"})]
assert any(event.get("type") == "tool_start" for event in events)
def test_transformers_bare_injected_prose_remains_untrusted():
decoded = _decode_transformers_tokens([8, 7])
executed, events = _run_tool_loop(decoded)
assert executed == []
assert any("call:terminal" in event.get("text", "") for event in events)
@pytest.mark.parametrize("control", sorted(NATIVE_TOOL_CONTROL_TOKENS))
def test_complete_native_tool_control_vocabulary_survives_special_token_decode(control):
tokenizer = _PieceTokenizer([control, "<eos>"], {1, 2})
assert decode_with_native_tool_tokens(tokenizer, [1, 2]) == control
def test_slow_tokenizer_does_not_space_preserved_gemma_segments():
parts = [
"<|tool_call>",
"call:edit_file{path:",
'<|"|>',
"/tmp/x",
'<|"|>',
",edits:[]}",
"<tool_call|>",
]
tokenizer = _SlowSpacingTokenizer(parts, {1, 3, 5, 7})
decoded = decode_with_native_tool_tokens(tokenizer, range(1, len(parts) + 1))
assert decoded == '<|tool_call>call:edit_file{path:<|"|>/tmp/x<|"|>,edits:[]}<tool_call|>'
calls = parse_tool_calls_from_text(decoded, enabled_tool_names = {"edit_file"})
assert json.loads(calls[0]["function"]["arguments"])["path"] == "/tmp/x"
@pytest.mark.parametrize("opener", ["<|channel>", "<|channel>thought\n"])
def test_request_reasoning_marker_preserves_component_or_full_special_token(opener):
tokenizer = _PieceTokenizer([opener, "<eos>"], {1, 2})
assert (
decode_with_native_tool_tokens(tokenizer, [1, 2], preserved_tokens = {"<|channel>thought"})
== opener
)
@pytest.mark.parametrize(
"parts",
[
[
"<tool_call>",
'{"name":"terminal","arguments":{"command":"id"}}',
"</tool_call>",
],
["<|python_tag|>", 'terminal.call(command="id")'],
["[TOOL_CALLS]", "terminal", "[ARGS]", '{"command":"id"}'],
[
"<|tool▁calls▁begin|>",
"<|tool▁call▁begin|>",
"terminal",
"<|tool▁sep|>",
'{"command":"id"}',
"<|tool▁call▁end|>",
"<|tool▁calls▁end|>",
],
[
"<|tool_calls_section_begin|>",
"<|tool_call_begin|>",
"functions.terminal:0",
"<|tool_call_argument_begin|>",
'{"command":"id"}',
"<|tool_call_end|>",
"<|tool_calls_section_end|>",
],
[
"<|message_model|>",
"terminal",
"<|content_invoke_tool_json|>",
'{"name":"terminal","args":{"command":"id"}}',
"<|end_message|>",
],
[
"<tool_call>",
"terminal\n",
"<arg_key>",
"command",
"</arg_key>",
"<arg_value>",
"id",
"</arg_value>",
"</tool_call>",
],
[
"<|tool_call>",
"call:terminal{command:",
'<|"|>',
"id",
'<|"|>',
"}",
"<tool_call|>",
],
],
ids = ["qwen", "llama", "mistral", "deepseek", "kimi", "tml", "glm", "gemma"],
)
def test_native_protocol_families_parse_after_special_token_decode(parts):
controls = {
index for index, part in enumerate(parts, start = 1) if part in NATIVE_TOOL_CONTROL_TOKENS
}
tokenizer = _PieceTokenizer(parts + ["<eos>"], controls | {len(parts) + 1})
decoded = decode_with_native_tool_tokens(tokenizer, range(1, len(parts) + 2))
assert "<eos>" not in decoded
calls = parse_tool_calls_from_text(decoded, enabled_tool_names = {"terminal"})
assert [call["function"]["name"] for call in calls] == ["terminal"]
def test_transformers_reasoning_stream_preserves_reasoning_and_tool_ids_but_not_eos():
from core.inference.inference import InferenceBackend
parts = [
"<|channel>",
"thought\n",
"reasoned",
"<channel|>",
"<|tool_call>",
"call:terminal{command:id}",
"<tool_call|>",
"<eos>",
]
tokenizer = _PieceTokenizer(parts, {1, 4, 5, 7, 8})
backend = InferenceBackend.__new__(InferenceBackend)
streamer = backend._make_text_streamer(
tokenizer,
reasoning_channel_markers = ("<|channel>thought", "<channel|>"),
reasoning_channel_markers_resolved = True,
skip_prompt = False,
preserve_tool_tokens = True,
)
streamer.put(torch.tensor(range(1, len(parts) + 1)))
streamer.end()
decoded = "".join(streamer)
assert decoded == "<think>reasoned</think><|tool_call>call:terminal{command:id}<tool_call|>"
executed, _events = _run_tool_loop(decoded)
assert executed == [("terminal", {"command": "id"})]
def test_disabled_execution_json_stops_before_later_enabled_call():
text = ";".join(
(
json.dumps({"name": "terminal", "arguments": {"command": "id"}}),
json.dumps({"name": "web_search", "arguments": {"query": "cats"}}),
)
)
assert parse_tool_calls_from_text(text, enabled_tool_names = {"web_search"}) == []
def test_blocked_rehearsal_is_opaque_but_outside_sibling_remains_eligible():
blocked = 'terminal[ARGS]{"command":"call:web_search{query:secret}"}'
assert parse_tool_calls_from_text(blocked, enabled_tool_names = {"terminal", "web_search"}) == []
calls = parse_tool_calls_from_text(
blocked + "\ncall:web_search{query:outside}",
enabled_tool_names = {"terminal", "web_search"},
)
assert [call["function"]["name"] for call in calls] == ["web_search"]
assert json.loads(calls[0]["function"]["arguments"]) == {"query": "outside"}
def test_a_tokenizer_whose_special_ids_raise_falls_back_instead_of_killing_the_turn():
"""Adapters may raise any type from all_special_ids: every failure must reach the fail-closed path."""
class RaisesOnSpecialIds:
def __init__(self, exc):
self._exc = exc
@property
def all_special_ids(self):
raise self._exc
def convert_ids_to_tokens(self, token_id):
raise self._exc
def decode(self, token_ids, **kwargs):
return "".join(f"<{int(i)}>" for i in token_ids)
for exc in (
RuntimeError("adapter has no special ids"),
KeyError("all_special_ids"),
NotImplementedError(),
OSError("backing file went away"),
):
tokenizer = RaisesOnSpecialIds(exc)
decoder = NativeToolTokenDecoder(tokenizer)
assert decoder.decode([1, 2]) == "<1><2>"
assert decoder.preserves("<tool_call>") is False
assert decoder_preserves_token(tokenizer, "<tool_call>") is False
def test_a_stop_token_is_named_by_decoding_when_conversion_cannot_name_it():
"""Controls named only by decoding must stay special, or an allowlisted EOS control leaks as markup."""
class ConvertUnavailable:
all_special_ids = [7]
def __init__(self, raises = False):
self._raises = raises
def convert_ids_to_tokens(self, token_id):
if self._raises:
raise RuntimeError("adapter cannot name ids")
return None
def decode(
self,
token_ids,
skip_special_tokens = False,
**kwargs,
):
out = []
for i in token_ids:
if int(i) == 7:
if not skip_special_tokens:
out.append("<|end_message|>")
else:
out.append("hi")
return "".join(out)
for raises in (False, True):
tokenizer = ConvertUnavailable(raises = raises)
assert "<|end_message|>" in NativeToolTokenDecoder(tokenizer).decode([1, 7])
assert stop_token_text(tokenizer, 7) == "<|end_message|>"
class Unusable:
all_special_ids = [7]
def convert_ids_to_tokens(self, token_id):
return None
def decode(self, token_ids, **kwargs):
raise RuntimeError("nothing works")
assert stop_token_text(Unusable(), 7) is None