Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import importlib
import sys
import threading
import types
from pathlib import Path
from unittest.mock import MagicMock
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_STUBBED: list[str] = []
def _stub_if_missing(name, attrs):
if name in sys.modules:
return
try:
importlib.import_module(name)
return
except Exception: # noqa: BLE001
pass
_STUBBED.append(name)
mod = types.ModuleType(name)
mod.__spec__ = None
for attr in attrs:
setattr(mod, attr, MagicMock())
sys.modules[name] = mod
parent, _, child = name.rpartition(".")
if parent and parent in sys.modules:
setattr(sys.modules[parent], child, mod)
_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
try:
import core.inference.inference # noqa: E402,F401
except ImportError:
pass
for _name in reversed(_STUBBED):
sys.modules.pop(_name, None)
class _Batch(dict):
def to(self, *_args, **_kwargs):
return self
class _Tokenizer:
all_special_tokens: list = []
eos_token_id = 1
pad_token_id = None
chat_template = "{{ messages }}"
def __call__(self, *_args, **_kwargs):
torch = pytest.importorskip("torch")
return _Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)})
def decode(self, ids, **kwargs):
return "".join(self.pieces.get(int(i), "") for i in ids)
class _Processor:
chat_template = None
tokenizer = _Tokenizer()
class _Model:
device = "cpu"
generation_config = type("Cfg", (), {"eos_token_id": 1})()
config = generation_config
def __init__(self, pieces):
self.pieces = pieces
self.sent = []
def generate(self, streamer, stopping_criteria, max_new_tokens, **_kwargs):
torch = pytest.importorskip("torch")
ids = torch.zeros((1, 1), dtype = torch.long)
streamer.put(ids)
for index, piece in enumerate(self.pieces[:max_new_tokens], 2):
ids = torch.cat([ids, torch.tensor([[index]])], dim = 1)
streamer.put(torch.tensor([index]))
self.sent.append(piece)
if stopping_criteria is not None and bool(stopping_criteria(ids, None).all()):
break
streamer.end()
return ids
def _backend(pieces, tokenizer):
inf = pytest.importorskip("core.inference.inference")
model = _Model(pieces)
raw_tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
raw_tokenizer.pieces = dict(enumerate(pieces, 2))
streamer = inf.TextIteratorStreamer(raw_tokenizer, skip_prompt = True, timeout = 0.2)
backend = inf.InferenceBackend.__new__(inf.InferenceBackend)
backend.active_model_name = "stop-test"
backend._generation_lock = threading.Lock()
backend.models = {"stop-test": {"model": model, "tokenizer": tokenizer, "processor": tokenizer}}
backend._make_text_streamer = lambda *_args, **_kwargs: streamer
return backend, model
class _Responses(list):
def put(self, response):
self.append(response)
def test_worker_forwards_stop_and_the_reply_ends_before_it(monkeypatch):
from core.inference import worker
backend, model = _backend(["Hello ", "ST", "OP", " world"], _Tokenizer())
monkeypatch.setattr(
backend, "_apply_chat_template_for_generation", lambda *a, **k: "PROMPT", raising = False
)
responses = _Responses()
cmd = {
"request_id": "r",
"messages": [{"role": "user", "content": "hi"}],
"max_new_tokens": 3,
"stop": ["STOP"],
}
worker._handle_generate(backend, cmd, responses, threading.Event())
assert [r["text"] for r in responses if r["type"] == "token"] == ["Hello"]
assert model.sent == ["Hello ", "ST", "OP"]
done = next(r for r in responses if r["type"] == "gen_done")
assert done["stats"]["truncated"] is False
def test_vision_reply_ends_before_a_stop_sequence():
backend, model = _backend(["Hello ", "ST", "OP", " world"], _Processor())
backend.format_chat_prompt = lambda *_args, **_kwargs: "PROMPT"
snapshots = list(
backend._generate_vision_response(
messages = [{"role": "user", "content": "hi"}],
system_prompt = "",
image = None,
temperature = 0.0,
top_p = 1.0,
top_k = 0,
min_p = 0.0,
max_new_tokens = 3,
repetition_penalty = 1.0,
stop = ["STOP"],
)
)
assert snapshots == ["Hello"]
assert model.sent == ["Hello ", "ST", "OP"]
assert backend.last_generation_stats["truncated"] is False
def test_a_partial_stop_sequence_is_released_when_the_reply_ends():
backend, _ = _backend(["Hello ", "ST"], _Tokenizer())
snapshots = list(backend.generate_stream("PROMPT", max_new_tokens = 8, stop = ["STOP"]))
assert snapshots == ["Hello", "Hello ST"]
@pytest.mark.parametrize("vision", [False, True])
def test_compact_json_stops_in_producer_before_word_flush(vision):
pieces = ['{"id":1}', ',{"id":2}'] * 100
backend, model = _backend(pieces, _Processor() if vision else _Tokenizer())
if vision:
backend.format_chat_prompt = lambda *a, **k: "PROMPT"
output = list(
backend._generate_vision_response(
messages = [{"role": "user", "content": "hi"}],
system_prompt = "",
image = None,
temperature = 0.0,
top_p = 1.0,
top_k = 0,
min_p = 0.0,
max_new_tokens = 200,
repetition_penalty = 1.0,
stop = ["}"],
)
)
else:
output = list(backend.generate_stream("PROMPT", max_new_tokens = 200, stop = ["}"]))
assert model.sent == [pieces[0]]
assert output == ['{"id":1']
assert backend.last_generation_stats["truncated"] is False
@pytest.mark.parametrize(
"kind,raw,stop",
[
(
"harmony",
"<|channel|>analysis<|message|>Reasoning<|channel|>final<|message|>Answer",
"<think>",
),
("reasoning", "<|channel>thoughtReasoning <channel|>Answer ", "</think>"),
],
)
def test_synthetic_reasoning_tags_do_not_match_stops(kind, raw, stop):
inf = pytest.importorskip("core.inference.inference")
torch = pytest.importorskip("torch")
tokenizer = _Tokenizer()
tokenizer.pieces = {2: raw}
if kind == "harmony":
streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = False)
else:
streamer = inf.ReasoningTextIteratorStreamer(
tokenizer,
markers = ("<|channel>thought", "<channel|>"),
skip_prompt = False,
)
wrapped = inf._StopSequenceStreamer(streamer, [stop])
wrapped.put(torch.tensor([2]))
wrapped.end()
output = []
while True:
try:
output.append(next(wrapped))
except StopIteration:
break
assert not wrapped.matched.is_set()
assert "Answer" in "".join(output)
@pytest.mark.parametrize("stop", [None, [], ["missing"]])
def test_requests_without_a_matching_stop_keep_all_text(stop):
backend, model = _backend(["Hello ", "world"], _Tokenizer())
output = list(backend.generate_stream("PROMPT", max_new_tokens = 8, stop = stop))
assert output[-1] == "Hello world"
assert model.sent == ["Hello ", "world"]
@pytest.mark.parametrize("abort", [False, True])
def test_stop_wrapper_finishes_reasoning_once(abort):
inf = pytest.importorskip("core.inference.inference")
torch = pytest.importorskip("torch")
tokenizer = _Tokenizer()
tokenizer.pieces = {2: "<|channel>thoughtReasoning ST"}
streamer = inf.ReasoningTextIteratorStreamer(
tokenizer,
markers = ("<|channel>thought", "<channel|>"),
skip_prompt = False,
)
wrapped = inf._StopSequenceStreamer(streamer, ["STOP"])
wrapped.put(torch.tensor([2]))
if abort:
wrapped.abort()
wrapped.end()
wrapped.end()
output = "".join(wrapped)
assert "Reasoning ST" in output
assert output.count("</think>") == (0 if abort else 1)
def test_stop_matching_decodes_split_unicode_without_leaking_partial_bytes():
inf = pytest.importorskip("core.inference.inference")
torch = pytest.importorskip("torch")
class Tokenizer(_Tokenizer):
pieces = {2: b"Hello \xc3", 3: b"\xa9END"}
def decode(self, ids, **kwargs):
return b"".join(self.pieces[int(i)] for i in ids).decode("utf-8", errors = "replace")
streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)
wrapped = inf._StopSequenceStreamer(streamer, ["éEND"])
wrapped.put(torch.tensor([2]))
assert not wrapped.matched.is_set()
wrapped.put(torch.tensor([3]))
assert wrapped.matched.is_set()
wrapped.end()
assert "".join(wrapped) == "Hello "
def test_a_stop_created_by_decode_cleanup_of_settled_text_still_matches():
inf = pytest.importorskip("core.inference.inference")
torch = pytest.importorskip("torch")
class Tokenizer(_Tokenizer):
pieces = {2: "Hello world", 3: " ", 4: ".", 5: " More"}
def decode(self, ids, **kwargs):
# Mirrors clean_up_tokenization_spaces, which rewrites " ." as ".".
return super().decode(ids, **kwargs).replace(" .", ".")
streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)
wrapped = inf._StopSequenceStreamer(streamer, ["."])
for token in (2, 3, 4):
wrapped.put(torch.tensor([token]))
assert wrapped.matched.is_set()
wrapped.end()
assert "".join(wrapped) == "Hello world "
@pytest.mark.parametrize(
"pieces, rule, stop",
[
# " n't" completes one char after the apostrophe, " 've" three after the space.
({2: "I", 3: " ca", 4: " ", 5: "n", 6: "'t"}, (" n't", "n't"), "can't"),
({2: "we", 3: " ", 4: "'", 5: "v", 6: "e"}, (" 've", "'ve"), "we've"),
({2: "it", 3: " ", 4: "'", 5: "s"}, (" 's", "'s"), "it's"),
],
)
def test_a_cleanup_rewrite_split_across_tokens_still_matches_a_stop(pieces, rule, stop):
inf = pytest.importorskip("core.inference.inference")
torch = pytest.importorskip("torch")
class Tokenizer(_Tokenizer):
# Reported and applied: the re-decode is gated on this, pure cost when cleanup is off.
clean_up_tokenization_spaces = True
def decode(self, ids, **kwargs):
return super().decode(ids, **kwargs).replace(*rule)
# Split across tokens, each half decodes unchanged, so a short window misses the rewrite.
tokenizer = Tokenizer()
tokenizer.pieces = pieces
streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = False)
wrapped = inf._StopSequenceStreamer(streamer, [stop])
for token in sorted(pieces):
wrapped.put(torch.tensor([token]))
assert wrapped.matched.is_set()
wrapped.end()
assert stop not in "".join(wrapped)
def test_an_unfinished_byte_run_is_not_settled_before_it_completes():
inf = pytest.importorskip("core.inference.inference")
torch = pytest.importorskip("torch")
class Tokenizer(_Tokenizer):
pieces = {2: "Hi ", 3: b"\xf0", 4: b"\x9f", 5: b"\x99", 6: b"\x82"}
pieces.update({7: b"\xf0", 8: b"\x9f", 9: b"\x99", 10: b"\x83", 11: " done"})
def decode(self, ids, **kwargs):
# Like SentencePiece byte fallback: an unfinished byte run is all replacements.
out, run = [], b""
for token in ids:
piece = self.pieces[int(token)]
if isinstance(piece, bytes):
run += piece
continue
out += [self._flush(run), piece]
run = b""
return "".join(out + [self._flush(run)])
@staticmethod
def _flush(run):
try:
return run.decode("utf-8")
except UnicodeDecodeError:
return "\ufffd" * len(run)
streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)
wrapped = inf._StopSequenceStreamer(streamer, ["never"])
for token in range(2, 12):
wrapped.put(torch.tensor([token]))
wrapped.end()
assert "".join(wrapped) == "Hi \U0001f642\U0001f643 done"
def test_stop_matching_decodes_a_bounded_window_per_token():
inf = pytest.importorskip("core.inference.inference")
torch = pytest.importorskip("torch")
class Tokenizer(_Tokenizer):
decoded = 0
def decode(self, ids, **kwargs):
self.decoded += len(ids)
return super().decode(ids, **kwargs)
tokenizer = Tokenizer()
words = 2000
tokenizer.pieces = {i: "word " for i in range(2, words + 2)}
tokenizer.pieces.update({words + 2 + i: ch for i, ch in enumerate("STOP")})
streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = False)
wrapped = inf._StopSequenceStreamer(streamer, ["STOP"])
for token in range(2, words + 2):
wrapped.put(torch.tensor([token]))
assert not wrapped.matched.is_set()
assert tokenizer.decoded < 10 * words
for token in range(words + 2, words + 6):
wrapped.put(torch.tensor([token]))
assert wrapped.matched.is_set()
wrapped.end()
assert "".join(wrapped) == "word " * words
@pytest.mark.parametrize("stop", [None, ["never"]])
def test_harmony_stream_waits_for_split_multibyte_characters(stop):
inf = pytest.importorskip("core.inference.inference")
torch = pytest.importorskip("torch")
body = "饺子 🦥"
split = [bytes([b]) for b in body.encode()]
parts = [b"<|channel|>analysis<|message|>", *split, b"\xf0"]
parts += [b"<|end|><|start|>assistant<|channel|>final<|message|>", *split, b"\xf0", b"\x9f"]
class Tokenizer(_Tokenizer):
pieces = dict(enumerate(parts, start = 2))
def decode(self, ids, **kwargs):
return b"".join(self.pieces[int(i)] for i in ids).decode("utf-8", errors = "replace")
streamer = inf.HarmonyTextStreamer(Tokenizer(), skip_prompt = False)
if stop:
streamer = inf._StopSequenceStreamer(streamer, stop)
for token in Tokenizer.pieces:
streamer.put(torch.tensor([token]))
streamer.end()
assert "".join(streamer) == f"<think>{body}\ufffd</think>{body}\ufffd"