Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

6.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Studio chat generate() graphs decode by default; unload frees graphs. _decode_graphs.py is
loaded by file path since importing unsloth needs an accelerator."""

import importlib
import importlib.util
import sys
import threading
import types
from pathlib import Path
from unittest.mock import MagicMock

import pytest

torch = pytest.importorskip("torch")

_BACKEND_DIR = Path(__file__).resolve().parent.parent
_DECODE_GRAPHS = _BACKEND_DIR.parent.parent / "unsloth" / "models" / "_decode_graphs.py"
if str(_BACKEND_DIR) not in sys.path:
    sys.path.insert(0, str(_BACKEND_DIR))

_STUBBED: list[str] = []


def _stub_if_missing(name, attrs):
    if name in sys.modules:
        return
    try:
        importlib.import_module(name)
        return
    except Exception:  # noqa: BLE001
        pass
    _STUBBED.append(name)
    mod = types.ModuleType(name)
    mod.__spec__ = None
    for attr in attrs:
        setattr(mod, attr, MagicMock())
    sys.modules[name] = mod
    parent, _, child = name.rpartition(".")
    if parent and parent in sys.modules:
        setattr(sys.modules[parent], child, mod)


_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))

try:
    import core.inference.inference  # noqa: E402,F401
except ImportError:
    pass

for _name in reversed(_STUBBED):
    sys.modules.pop(_name, None)

ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"


@pytest.fixture
def graphs(monkeypatch):
    """Real _decode_graphs module with the default decision on a spoofed NVIDIA CUDA host."""
    spec = importlib.util.spec_from_file_location("_studio_decode_graphs", _DECODE_GRAPHS)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    module.ENABLED = None
    module._DEFAULT_OK = True
    module._STUDIO_DEFAULT_OK = True  # Linux default (native Windows stays eager)
    unsloth = types.ModuleType("unsloth")
    models = types.ModuleType("unsloth.models")
    unsloth.models = models
    models._decode_graphs = module
    monkeypatch.setitem(sys.modules, "unsloth", unsloth)
    monkeypatch.setitem(sys.modules, "unsloth.models", models)
    monkeypatch.setitem(sys.modules, "unsloth.models._decode_graphs", module)
    monkeypatch.delenv(ENV, raising = False)
    return module


class _Batch(dict):
    def to(self, *_args, **_kwargs):
        return self


class _Tokenizer:
    all_special_tokens: list = []
    eos_token_id = 1
    pad_token_id = None
    pieces = {2: "Hello", 3: " world"}

    def __call__(self, *_args, **_kwargs):
        return _Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)})

    def decode(self, ids, **kwargs):
        return "".join(self.pieces.get(int(i), "") for i in ids)


class _Model:
    device = "cpu"
    generation_config = type("Cfg", (), {"eos_token_id": 1})()
    config = generation_config

    def __init__(self, graphs):
        self.graphs = graphs
        self.seen = []

    def generate(self, streamer, max_new_tokens, **_kwargs):
        self.seen.append(self.graphs.enabled())
        ids = torch.tensor([[0, 2, 3]])
        streamer.put(ids[:, :1])
        streamer.put(torch.tensor([2]))
        streamer.put(torch.tensor([3]))
        streamer.end()
        return ids


def _backend(graphs):
    inf = pytest.importorskip("core.inference.inference")
    model = _Model(graphs)
    tokenizer = _Tokenizer()
    streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = True, timeout = 0.2)
    backend = inf.InferenceBackend.__new__(inf.InferenceBackend)
    backend.active_model_name = "m"
    backend._generation_lock = threading.Lock()
    backend.models = {"m": {"model": model, "tokenizer": tokenizer}}
    backend._make_text_streamer = lambda *_args, **_kwargs: streamer
    return backend, model


def _chat(backend):
    return list(backend.generate_stream("PROMPT", temperature = 0.0, max_new_tokens = 4))


def test_chat_generate_runs_with_graphs_on_by_default(graphs):
    backend, model = _backend(graphs)
    assert _chat(backend)[-1] == "Hello world"
    assert model.seen == [True]
    # Scoped to the generate thread: this thread, and Core's plain generate(), stay eager.
    assert graphs.enabled() is False


def test_opt_out_variable_still_disables_graphs(graphs, monkeypatch):
    monkeypatch.setenv(ENV, "0")
    backend, model = _backend(graphs)
    _chat(backend)
    assert model.seen == [False]


def test_non_nvidia_hosts_stay_eager(graphs, monkeypatch):
    # ROCm / XPU / MPS / CPU: _default_ok() is False there.
    monkeypatch.setattr(graphs, "_DEFAULT_OK", False)
    backend, model = _backend(graphs)
    _chat(backend)
    assert model.seen == [False]


def test_an_unsloth_without_the_scope_keeps_generate_unchanged(graphs, monkeypatch):
    monkeypatch.delattr(graphs, "studio_generation", raising = False)
    backend, model = _backend(graphs)
    assert _chat(backend)[-1] == "Hello world"
    assert model.seen == [False]


def _graphed_model(graphs):
    model = torch.nn.Module()
    attn = torch.nn.Module()
    model.attn = attn
    graphs._state(model).graphs[(1, "cuda")] = object()
    attn._unsloth_graph_buffers = {(1, torch.float16, "cuda"): ()}
    attn.paged_attention = torch.zeros(1)
    attn.attention = torch.zeros(1)
    return model, attn


def _unload(backend, monkeypatch):
    inf = sys.modules["core.inference.inference"]
    monkeypatch.setattr(inf, "clear_gpu_cache", lambda: None)
    cleanup = pytest.importorskip("utils.cache_cleanup")
    monkeypatch.setattr(cleanup, "clear_unsloth_compiled_cache", lambda **_kw: None)
    return backend.unload_model("m")


def test_unload_frees_decode_graphs_and_buffers(graphs, monkeypatch):
    backend, _ = _backend(graphs)
    model, attn = _graphed_model(graphs)
    backend.models["m"]["model"] = model
    assert _unload(backend, monkeypatch) is True
    assert graphs.stats(model)["graphs"] == 0
    assert "_unsloth_graph_buffers" not in attn.__dict__
    assert "paged_attention" not in attn.__dict__ and "attention" not in attn.__dict__
    assert backend.models == {}


def test_unload_leaves_a_running_generation_alone(graphs, monkeypatch):
    backend, _ = _backend(graphs)
    model, attn = _graphed_model(graphs)
    backend.models["m"]["model"] = model
    with backend._generation_lock:
        assert _unload(backend, monkeypatch) is True
    # The generate() thread still owns the buffers; they go with its model reference.
    assert "_unsloth_graph_buffers" in attn.__dict__
    assert graphs.stats(model)["graphs"] == 1
    assert backend.models == {}