Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

6.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import sys
import threading
from pathlib import Path

import pytest

_BACKEND = Path(__file__).resolve().parent.parent
if str(_BACKEND) not in sys.path:
    sys.path.insert(0, str(_BACKEND))

try:
    from core.inference import inference as inf_mod  # noqa: E402
    from core.inference.inference import InferenceBackend  # noqa: E402
except (ImportError, RuntimeError) as exc:  # pragma: no cover - env-dependent
    pytest.skip(
        f"full inference backend unavailable ({type(exc).__name__}: {exc})",
        allow_module_level = True,
    )

_CHATML = "{% for m in messages %}<|im_start|>{{m.role}}\n{{m.content}}<|im_end|>{% endfor %}"
_NAME = "unsloth/gemma-2-2b-it"


class _FakeTokenizer:
    def __init__(
        self,
        eos_id,
        chat_template = "",
        token_ids = None,
    ):
        self.eos_token_id = eos_id
        self.chat_template = chat_template
        self.pad_token_id = eos_id
        self.unk_token_id = None
        self._ids = dict(token_ids or {})

    def convert_tokens_to_ids(self, tok):
        return self._ids.get(tok)


def _backend(monkeypatch, model_info, get_chat_template):
    import utils.datasets as ds

    backend = InferenceBackend.__new__(InferenceBackend)
    backend.active_model_name = _NAME
    backend.models = {_NAME: model_info}
    monkeypatch.setattr(inf_mod, "get_chat_template", get_chat_template)
    monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {_NAME.lower(): "gemma2"}, raising = False)
    monkeypatch.setattr(backend, "_normalize_top_k", lambda k: k, raising = False)
    rendered_with = []

    def _render(tok, *a, **k):
        rendered_with.append(tok)
        return "PROMPT"

    monkeypatch.setattr(backend, "_apply_chat_template_for_generation", _render, raising = False)
    monkeypatch.setattr(backend, "generate_stream", lambda *a, **k: iter(()), raising = False)
    return backend, rendered_with


def _send(backend, text = "hi"):
    list(backend._generate_chat_response_inner(messages = [{"role": "user", "content": text}]))


class _Counter:
    def __init__(self, make):
        self.calls = []
        self.make = make

    def __call__(
        self,
        tok,
        chat_template = None,
    ):
        self.calls.append((tok, chat_template))
        return self.make(tok)


def _remapped(tok):
    return _FakeTokenizer(1, chat_template = _CHATML, token_ids = {"<|im_end|>": 1})


def test_template_applied_once_and_reused_across_messages(monkeypatch):
    orig = _FakeTokenizer(1, token_ids = {"<|im_end|>": 107})
    model_info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": [1]}
    gct = _Counter(_remapped)
    backend, rendered_with = _backend(monkeypatch, model_info, gct)

    for i in range(5):
        _send(backend, f"msg {i}")

    assert len(gct.calls) == 1
    assert gct.calls[0] == (orig, "gemma2")
    assert len(rendered_with) == 5
    assert all(t is rendered_with[0] for t in rendered_with)
    assert rendered_with[0] is not orig
    assert model_info["chat_turn_end_eos_ids"] == [1, 107]


def test_eos_union_matches_uncached_repeat(monkeypatch):
    orig = _FakeTokenizer(1, token_ids = {"<|im_end|>": 107})
    cached_info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": [5]}
    backend, _ = _backend(monkeypatch, cached_info, _Counter(_remapped))
    for _ in range(3):
        _send(backend)

    fresh_ids = [5]
    for _ in range(3):
        info = {"tokenizer": orig, "is_vision": False, "chat_turn_end_eos_ids": fresh_ids}
        backend.models = {_NAME: info}
        _send(backend)
        fresh_ids = info["chat_turn_end_eos_ids"]
    assert cached_info["chat_turn_end_eos_ids"] == fresh_ids == [1, 5, 107]


def test_reload_and_tokenizer_swap_reapply(monkeypatch):
    orig = _FakeTokenizer(1)
    model_info = {"tokenizer": orig, "is_vision": False}
    gct = _Counter(_remapped)
    backend, rendered_with = _backend(monkeypatch, model_info, gct)
    _send(backend)
    _send(backend)
    assert len(gct.calls) == 1

    new_tok = _FakeTokenizer(1)
    model_info["tokenizer"] = new_tok
    _send(backend)
    assert len(gct.calls) == 2 and gct.calls[-1][0] is new_tok

    backend.models = {_NAME: {"tokenizer": orig, "is_vision": False}}
    _send(backend)
    assert len(gct.calls) == 3
    assert rendered_with[-1] is not rendered_with[0]


def test_template_name_change_reapplies(monkeypatch):
    import utils.datasets as ds

    orig = _FakeTokenizer(1)
    model_info = {"tokenizer": orig, "is_vision": False}
    gct = _Counter(_remapped)
    backend, _ = _backend(monkeypatch, model_info, gct)
    _send(backend)
    monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {_NAME.lower(): "chatml"}, raising = False)
    _send(backend)
    _send(backend)
    assert [c[1] for c in gct.calls] == ["gemma2", "chatml"]


def test_failure_is_not_cached_and_falls_back(monkeypatch):
    orig = _FakeTokenizer(1, chat_template = _CHATML)
    model_info = {"tokenizer": orig, "is_vision": False}
    state = {"fail": True}

    def make(tok):
        if state["fail"]:
            raise KeyError("qwen3.5")
        return _remapped(tok)

    gct = _Counter(make)
    backend, rendered_with = _backend(monkeypatch, model_info, gct)
    _send(backend)
    _send(backend)
    assert rendered_with == [orig, orig]
    assert len(gct.calls) == 2
    assert "_mapped_template_tokenizer" not in model_info
    state["fail"] = False
    _send(backend)
    _send(backend)
    assert len(gct.calls) == 3
    assert rendered_with[2] is rendered_with[3] is not orig


def test_concurrent_first_messages_apply_once(monkeypatch):
    orig = _FakeTokenizer(1)
    model_info = {"tokenizer": orig, "is_vision": False}
    gate = threading.Event()

    def slow(tok):
        gate.wait(5)
        return _remapped(tok)

    gct = _Counter(slow)
    backend, rendered_with = _backend(monkeypatch, model_info, gct)
    threads = [threading.Thread(target = _send, args = (backend,)) for _ in range(4)]
    for t in threads:
        t.start()
    gate.set()
    for t in threads:
        t.join(10)
    assert len(gct.calls) == 1
    assert len(rendered_with) == 4 and all(t is rendered_with[0] for t in rendered_with)


def test_unmapped_model_never_calls_get_chat_template(monkeypatch):
    import utils.datasets as ds

    orig = _FakeTokenizer(1, chat_template = _CHATML)
    model_info = {"tokenizer": orig, "is_vision": False}
    gct = _Counter(_remapped)
    backend, rendered_with = _backend(monkeypatch, model_info, gct)
    monkeypatch.setattr(ds, "MODEL_TO_TEMPLATE_MAPPER", {}, raising = False)
    _send(backend)
    assert gct.calls == [] and rendered_with == [orig]
    assert "_mapped_template_tokenizer" not in model_info