Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""A block-streamed image denoiser takes the fused int8 GEMM: the speed layer installs it against the onload device
when the plan moves the denoiser, and the weights that arrive through the event-fenced prefetch run it bit-identically
to stock torchao. The CPU tests need no GPU; the last ones need NVIDIA sm80+ CUDA, Triton and torchao."""
from __future__ import annotations
import ast
import pathlib
import types
import pytest
torch = pytest.importorskip("torch")
from core.inference import diffusion_int8_gemm as g8 # noqa: E402
from core.inference import diffusion_speed as ds # noqa: E402
_INFERENCE = pathlib.Path(ds.__file__).resolve().parent
class _DiT:
def compile_repeated_blocks(self, **kwargs):
return None
def _record_install(monkeypatch):
seen = []
def _install(
transformer,
logger = None,
offload_active = False,
device = None,
):
seen.append({"offload_active": offload_active, "device": device})
return 7
monkeypatch.setattr(g8, "install", _install)
monkeypatch.setattr(ds, "_denoiser_dits", lambda pipe: [_DiT()])
return seen
@pytest.mark.parametrize(
"offload_active, denoiser_offloaded, onload, expected",
[
(True, True, "cuda:1", {"offload_active": False, "device": "cuda:1"}),
(True, None, "cuda", {"offload_active": False, "device": "cuda"}),
# no onload device (video backend, ROCm, CPU): still refused
(True, True, None, {"offload_active": True, "device": None}),
(True, False, "cuda", {"offload_active": False, "device": None}),
(False, None, "cuda", {"offload_active": False, "device": None}),
],
)
def test_moved_denoiser_installs_against_the_onload_device(
monkeypatch, offload_active, denoiser_offloaded, onload, expected
):
seen = _record_install(monkeypatch)
ds._compile_repeated_blocks(
types.SimpleNamespace(),
None,
offload_active = offload_active,
denoiser_offloaded = denoiser_offloaded,
onload_device = onload,
)
assert seen == [expected]
@pytest.mark.parametrize(
"target, expected",
[
(types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:1"), "cuda:1"),
(types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda"), "cuda"),
(types.SimpleNamespace(device = "cuda", backend = "cuda"), "cuda"),
(types.SimpleNamespace(device = "cuda", backend = "rocm", torch_device = "cuda"), None),
(types.SimpleNamespace(device = "cpu", backend = "cpu", torch_device = "cpu"), None),
(types.SimpleNamespace(device = "mps", backend = "mps", torch_device = "mps"), None),
],
)
def test_onload_device_is_nvidia_cuda_only(target, expected):
assert ds._onload_device(target) == expected
def _speed_calls(path: pathlib.Path) -> list:
tree = ast.parse(path.read_text(encoding = "utf-8"))
return [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and getattr(node.func, "id", getattr(node.func, "attr", None)) == "apply_speed_optims"
]
def test_image_backend_asks_for_the_streamed_install():
"""Image speed passes request the streamed install; video keeps its own H3-only one."""
calls = _speed_calls(_INFERENCE / "diffusion.py")
assert len(calls) == 2
for call in calls:
kw = {k.arg: k.value for k in call.keywords}
assert (
isinstance(kw.get("stream_int8_gemm"), ast.Constant)
and kw["stream_int8_gemm"].value is True
)
for call in _speed_calls(_INFERENCE / "video.py"):
assert "stream_int8_gemm" not in {k.arg for k in call.keywords}
def test_apply_speed_optims_passes_the_onload_device_only_when_asked(monkeypatch):
seen = []
def _compile(pipe, logger, **kwargs):
seen.append(kwargs.get("onload_device"))
return False
monkeypatch.setattr(ds, "_compile_repeated_blocks", _compile)
monkeypatch.setattr(ds, "compile_eligible", lambda *a, **k: True)
monkeypatch.setattr(ds, "fp16_unet_offloaded", lambda *a, **k: False)
monkeypatch.setattr(ds, "_denoiser_unet", lambda pipe: None)
target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:2", dtype = None)
pipe = types.SimpleNamespace()
for mode in (ds.SPEED_DEFAULT, ds.SPEED_MAX):
for ask in (True, False):
try:
ds.apply_speed_optims(
pipe,
target,
is_gguf = False,
family = types.SimpleNamespace(supports_cuda_graph = False),
speed_mode = mode,
offload_active = True,
denoiser_offloaded = True,
stream_int8_gemm = ask,
)
except Exception: # noqa: BLE001 - only the compile call is under test
pass
assert seen == ["cuda:2", None, "cuda:2", None]
def _cuda_ready() -> bool:
if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
return False
if torch.cuda.get_device_capability() < (8, 0):
return False
try:
import torchao # noqa: F401
import triton # noqa: F401
except Exception: # noqa: BLE001
return False
return True
needs_cuda = pytest.mark.skipif(
not _cuda_ready(), reason = "needs NVIDIA sm80+ CUDA, Triton and torchao"
)
@pytest.fixture
def forced(monkeypatch):
monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")
monkeypatch.delenv(g8.INT8_GEMM_STREAMED_ENV, raising = False)
g8._DEVICE_CFG.clear()
cfg = g8.device_config(torch.cuda.current_device())
if cfg is None:
pytest.skip("int8 GEMM probe refused this device")
yield cfg
g8._DEVICE_CFG.clear()
def _blocks(n, k, version):
from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
class Holder(torch.nn.Module):
def __init__(self):
super().__init__()
torch.manual_seed(0)
self.blocks = torch.nn.ModuleList(
[torch.nn.Sequential(torch.nn.Linear(k, k, bias = i % 2 == 0)) for i in range(n)]
)
def forward(self, x):
for block in self.blocks:
x = block(x) * 0.5
return x
holder = Holder().cuda().to(torch.bfloat16)
cfg = Int8DynamicActivationInt8WeightConfig(set_inductor_config = False)
if version is not None:
if not hasattr(cfg, "version"):
pytest.skip("torchao without config versions")
cfg.version = version
quantize_(holder, cfg)
return holder.requires_grad_(False)
@needs_cuda
@pytest.mark.parametrize("version", [None, 2])
def test_prefetched_weights_run_the_fused_gemm_bit_identically(forced, version):
"""Block streaming via event-fenced prefetch two groups ahead: every int8 Linear still takes the
fused GEMM and matches stock torchao, forward after forward."""
pytest.importorskip("diffusers.hooks")
from core.inference import diffusion_memory as dm
from core.inference.diffusion_offload_prefetch import module_prefetcher
n, k = 6, 1024
stock = _blocks(n, k, version)
streamed = _blocks(n, k, version).cpu()
pipe = types.SimpleNamespace(transformer = streamed, components = {"transformer": streamed})
assert dm._apply_group_offload(pipe, "cuda", None)
pf = module_prefetcher(streamed)
if pf is None:
pytest.skip("this diffusers keeps its own stream prefetch")
assert all(lin.weight.device.type == "cpu" for block in streamed.blocks for lin in block)
assert g8.install(streamed, device = "cuda") == n
xs = [torch.randn(300, k, device = "cuda", dtype = torch.bfloat16) * 3 for _ in range(4)]
with torch.no_grad(): # group offload's swap_tensors onload cannot run on inference tensors
for x in xs:
before = g8.call_count()
out = streamed(x)
assert g8.call_count() == before + n
assert torch.equal(out, stock(x))
assert pf.stats["prefetched"] > 0 and pf.stats["missed"] == 0