Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""studio_generation() turns graphed decode on per thread on NVIDIA CUDA; =0 wins, others stay eager.
Loads _decode_graphs.py by file path (importing unsloth needs an accelerator)."""
import importlib.util
import threading
from pathlib import Path
import pytest
import torch
from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"
_SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_decode_graphs.py"
@pytest.fixture
def G(monkeypatch):
spec = importlib.util.spec_from_file_location("_unsloth_decode_graphs_studio", _SOURCE)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.ENABLED = None
module._STUDIO_DEFAULT_OK = True # the Linux default; the Windows test overrides it
monkeypatch.delenv(ENV, raising = False)
return module
def test_studio_scope_turns_graphs_on_for_nvidia_cuda(G, monkeypatch):
monkeypatch.setattr(G, "_DEFAULT_OK", True)
assert G.enabled() is False
with G.studio_generation():
assert G.enabled() is True
with G.rl_generation():
assert G.enabled() is True
assert G.enabled() is True
assert G.enabled() is False
def test_studio_scope_honours_the_variable(G, monkeypatch):
monkeypatch.setattr(G, "_DEFAULT_OK", True)
monkeypatch.setenv(ENV, "0")
with G.studio_generation():
assert G.enabled() is False
monkeypatch.setattr(G, "ENABLED", False)
monkeypatch.setenv(ENV, "1")
with G.studio_generation():
assert G.enabled() is False
def test_studio_scope_stays_eager_off_nvidia(G, monkeypatch):
# ROCm (HIP), XPU, MPS and CPU hosts: _default_ok() is False.
monkeypatch.setattr(G, "_DEFAULT_OK", False)
with G.studio_generation():
assert G.enabled() is False
monkeypatch.setattr(torch.version, "hip", "6.4.0", raising = False)
assert G._default_ok() is False
def test_studio_scope_is_per_thread(G, monkeypatch):
monkeypatch.setattr(G, "_DEFAULT_OK", True)
seen = []
with G.studio_generation():
t = threading.Thread(target = lambda: seen.append(G.enabled()))
t.start()
t.join()
assert G.enabled() is True
assert seen == [False]
def test_studio_scope_unwinds_on_error(G, monkeypatch):
monkeypatch.setattr(G, "_DEFAULT_OK", True)
with pytest.raises(RuntimeError):
with G.studio_generation():
raise RuntimeError("generate failed")
assert G.enabled() is False
def test_studio_scope_is_off_on_native_windows(G, monkeypatch):
# Native Windows CUDA capture is unverified: the Studio default stays eager there, =1 still forces.
monkeypatch.setattr(G, "_DEFAULT_OK", True)
monkeypatch.setattr(G, "_STUDIO_DEFAULT_OK", False)
with G.studio_generation():
assert G.enabled() is False
with G.rl_generation():
assert G.enabled() is True
assert G.enabled() is False
assert G.enabled() is False
monkeypatch.setenv(ENV, "1")
with G.studio_generation():
assert G.enabled() is True
@pytest.mark.skipif(not has_real_cuda(), reason = "CUDA only")
def test_studio_generate_graphed_plain_generate_eager(monkeypatch):
import unsloth # noqa: F401
from unsloth import FastLanguageModel
import unsloth.models._decode_graphs as G
if not G._DEFAULT_OK:
pytest.skip("the default is NVIDIA CUDA only")
if not G._STUDIO_DEFAULT_OK:
pytest.skip("the Studio default stays eager on native Windows")
model, tok = FastLanguageModel.from_pretrained(
"trl-internal-testing/tiny-Qwen3ForCausalLM", max_seq_length = 2048, load_in_4bit = False
)
FastLanguageModel.for_inference(model)
enc = tok(["Once upon a time there was"], return_tensors = "pt").to("cuda")
def gen():
out = model.generate(
**enc, max_new_tokens = 24, min_new_tokens = 24, do_sample = False,
output_logits = True, return_dict_in_generate = True,
) # fmt: skip
return out.sequences.cpu(), torch.stack(out.logits).cpu()
def replays():
return G.stats(model).get("replays", 0)
# Plain generate() graphs by default too (generate_scope); =0 is the eager reference.
monkeypatch.setenv("UNSLOTH_DECODE_CUDA_GRAPHS", "0")
eager = gen()
assert replays() == 0
monkeypatch.delenv("UNSLOTH_DECODE_CUDA_GRAPHS")
with G.studio_generation():
graphed = gen()
assert replays() == 24 - 2 and G.stats(model)["disabled"] is None
assert torch.equal(eager[0], graphed[0]) and torch.equal(eager[1], graphed[1])
assert torch.equal(gen()[1], eager[1]) and replays() == 2 * (24 - 2)
G.release_for_training(model)
assert G.stats(model)["graphs"] == 0
assert not any("paged_attention" in m.__dict__ for m in model.modules())