Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fused int8 GEMM for block-streamed image denoisers (#12677)
Daniel Han29bcc2306895
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 9a9653d15f7..8cf0a47bd2c 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -7104,6 +7104,7 @@ class DiffusionBackend:cache_engaged = cache_graph_break,offload_active = plan.offload_policy != OFFLOAD_NONE,denoiser_offloaded = not plan_keeps_transformer_resident(plan),+ stream_int8_gemm = True,logger = logger,)if vae_fp16:@@ -9191,6 +9192,7 @@ class DiffusionBackend:offload_active = state.offload_policy != OFFLOAD_NONE,denoiser_offloaded = state.offload_policy != OFFLOAD_NONEand _denoiser_hooked(state.pipe),+ stream_int8_gemm = True,logger = logger,)if denoisers_pinned_resident(state.pipe):diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 0bc5dccc672..5a1de2936a2 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -427,6 +427,7 @@ def apply_speed_optims(cuda_graph_default: bool = True,cache_engaged: Optional[bool] = None,denoiser_offloaded: Optional[bool] = None,+ stream_int8_gemm: bool = False,logger: Any = None,) -> dict[str, bool]:"""Apply the opt-in speed optims for ``speed_mode`` to a built pipeline, BEFORE placement /@@ -435,6 +436,7 @@ def apply_speed_optims(``offload_active`` (offload policy != none) installs ``@torch.compiler.disable``d onload hooks,so the compile must drop ``fullgraph`` (like an active step cache) or it crashes at step 1.``denoiser_offloaded`` (None = ``offload_active``) limits the CUDA-graph refusal to a moved denoiser.+ ``stream_int8_gemm``: a moved denoiser still takes the fused int8 GEMM, installed against its onload device.``cuda_graph_default`` is what the CUDA-graph arm assumes for a family that declares nothing:True on the image backend, False on video, where ``supports_cuda_graph`` opts in.@@ -517,6 +519,7 @@ def apply_speed_optims(cache_active = cache_active,offload_active = offload_active,denoiser_offloaded = denoiser_offloaded,+ onload_device = _onload_device(target) if stream_int8_gemm else None,filter_reductions = filter_reductions,)elif (@@ -531,6 +534,7 @@ def apply_speed_optims(cache_active = cache_active,offload_active = offload_active,denoiser_offloaded = denoiser_offloaded,+ onload_device = _onload_device(target) if stream_int8_gemm else None,filter_reductions = filter_reductions,)@@ -606,6 +610,14 @@ def apply_speed_optims(return applied+def _onload_device(target: Any) -> Optional[str]:+ """Onload device of an offloaded denoiser, else None. NVIDIA only: ROCm runs int8 weight-only, never this GEMM."""+ if getattr(target, "device", None) != "cuda" or getattr(target, "backend", "cuda") != "cuda":+ return None+ device = getattr(target, "torch_device", None)+ return str(device) if isinstance(device, str) and device.startswith("cuda") else "cuda"++def engage_pinned_denoisers(pipe: Any,applied: dict,@@ -931,6 +943,7 @@ def _compile_repeated_blocks(cache_active: bool = False,offload_active: bool = False,denoiser_offloaded: Optional[bool] = None,+ onload_device: Any = None,filter_reductions: bool = False,) -> bool:dits = [@@ -1049,14 +1062,17 @@ def _compile_repeated_blocks(try:from .diffusion_int8_gemm import install as install_int8_gemm- # Keyed on the DENOISER's placement: a group plan that streams only the encoders keeps it resident.- transformer._unsloth_int8_gemm = install_int8_gemm(- transformer,- logger,- offload_active = offload_active- if denoiser_offloaded is None- else bool(denoiser_offloaded),- )+ # Keyed on the DENOISER's placement (an encoder-only stream keeps it resident). A moved denoiser installs+ # against its onload device; a weight found elsewhere at call time keeps the stock path.+ moved = offload_active if denoiser_offloaded is None else bool(denoiser_offloaded)+ if moved and onload_device is not None:+ transformer._unsloth_int8_gemm = install_int8_gemm(+ transformer, logger, device = onload_device+ )+ else:+ transformer._unsloth_int8_gemm = install_int8_gemm(+ transformer, logger, offload_active = moved+ )except Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "int8 fused-dequant gemm", exc)if type(transformer).__name__ == "QwenImageTransformer2DModel":diff --git a/studio/backend/tests/test_diffusion_stream_int8_gemm.py b/studio/backend/tests/test_diffusion_stream_int8_gemm.pynew file mode 100644index 00000000000..d424ea410b1--- /dev/null+++ b/studio/backend/tests/test_diffusion_stream_int8_gemm.py@@ -0,0 +1,227 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A block-streamed image denoiser takes the fused int8 GEMM: the speed layer installs it against the onload device+when the plan moves the denoiser, and the weights that arrive through the event-fenced prefetch run it bit-identically+to stock torchao. The CPU tests need no GPU; the last ones need NVIDIA sm80+ CUDA, Triton and torchao."""++from __future__ import annotations++import ast+import pathlib+import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_int8_gemm as g8 # noqa: E402+from core.inference import diffusion_speed as ds # noqa: E402++_INFERENCE = pathlib.Path(ds.__file__).resolve().parent+++class _DiT:+ def compile_repeated_blocks(self, **kwargs):+ return None+++def _record_install(monkeypatch):+ seen = []++ def _install(+ transformer,+ logger = None,+ offload_active = False,+ device = None,+ ):+ seen.append({"offload_active": offload_active, "device": device})+ return 7++ monkeypatch.setattr(g8, "install", _install)+ monkeypatch.setattr(ds, "_denoiser_dits", lambda pipe: [_DiT()])+ return seen+++ "offload_active, denoiser_offloaded, onload, expected",+ [+ # moved denoiser with a known onload device: installed against it+ (True, True, "cuda:1", {"offload_active": False, "device": "cuda:1"}),+ (True, None, "cuda", {"offload_active": False, "device": "cuda"}),+ # no onload device (video backend, ROCm, CPU): the old refusal+ (True, True, None, {"offload_active": True, "device": None}),+ # a resident denoiser never needs the device+ (True, False, "cuda", {"offload_active": False, "device": None}),+ (False, None, "cuda", {"offload_active": False, "device": None}),+ ],+)+def test_moved_denoiser_installs_against_the_onload_device(+ monkeypatch, offload_active, denoiser_offloaded, onload, expected+):+ seen = _record_install(monkeypatch)+ ds._compile_repeated_blocks(+ types.SimpleNamespace(),+ None,+ offload_active = offload_active,+ denoiser_offloaded = denoiser_offloaded,+ onload_device = onload,+ )+ assert seen == [expected]+++ "target, expected",+ [+ (types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:1"), "cuda:1"),+ (types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda"), "cuda"),+ (types.SimpleNamespace(device = "cuda", backend = "cuda"), "cuda"), # video's namespace target+ (types.SimpleNamespace(device = "cuda", backend = "rocm", torch_device = "cuda"), None),+ (types.SimpleNamespace(device = "cpu", backend = "cpu", torch_device = "cpu"), None),+ (types.SimpleNamespace(device = "mps", backend = "mps", torch_device = "mps"), None),+ ],+)+def test_onload_device_is_nvidia_cuda_only(target, expected):+ assert ds._onload_device(target) == expected+++def _speed_calls(path: pathlib.Path) -> list:+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ return [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", getattr(node.func, "attr", None)) == "apply_speed_optims"+ ]+++def test_image_backend_asks_for_the_streamed_install():+ """Both image-backend speed passes (load, and the deferred compile on the 3rd image) ask for it; the video+ backend keeps its own H3-only streamed install."""+ calls = _speed_calls(_INFERENCE / "diffusion.py")+ assert len(calls) == 2+ for call in calls:+ kw = {k.arg: k.value for k in call.keywords}+ assert (+ isinstance(kw.get("stream_int8_gemm"), ast.Constant)+ and kw["stream_int8_gemm"].value is True+ )+ for call in _speed_calls(_INFERENCE / "video.py"):+ assert "stream_int8_gemm" not in {k.arg for k in call.keywords}+++def test_apply_speed_optims_passes_the_onload_device_only_when_asked(monkeypatch):+ seen = []++ def _compile(pipe, logger, **kwargs):+ seen.append(kwargs.get("onload_device"))+ return False++ monkeypatch.setattr(ds, "_compile_repeated_blocks", _compile)+ monkeypatch.setattr(ds, "compile_eligible", lambda *a, **k: True)+ monkeypatch.setattr(ds, "fp16_unet_offloaded", lambda *a, **k: False)+ monkeypatch.setattr(ds, "_denoiser_unet", lambda pipe: None)+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:2", dtype = None)+ pipe = types.SimpleNamespace()+ for mode in (ds.SPEED_DEFAULT, ds.SPEED_MAX):+ for ask in (True, False):+ try:+ ds.apply_speed_optims(+ pipe,+ target,+ is_gguf = False,+ family = types.SimpleNamespace(supports_cuda_graph = False),+ speed_mode = mode,+ offload_active = True,+ denoiser_offloaded = True,+ stream_int8_gemm = ask,+ )+ except Exception: # noqa: BLE001 - only the compile call is under test+ pass+ assert seen == ["cuda:2", None, "cuda:2", None]+++def _cuda_ready() -> bool:+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return False+ if torch.cuda.get_device_capability() < (8, 0):+ return False+ try:+ import torchao # noqa: F401+ import triton # noqa: F401+ except Exception: # noqa: BLE001+ return False+ return True+++needs_cuda = pytest.mark.skipif(+ not _cuda_ready(), reason = "needs NVIDIA sm80+ CUDA, Triton and torchao"+)+++def forced(monkeypatch):+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")+ monkeypatch.delenv(g8.INT8_GEMM_STREAMED_ENV, raising = False)+ g8._DEVICE_CFG.clear()+ cfg = g8.device_config(torch.cuda.current_device())+ if cfg is None:+ pytest.skip("int8 GEMM probe refused this device")+ yield cfg+ g8._DEVICE_CFG.clear()+++def _blocks(n, k, version):+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ class Holder(torch.nn.Module):+ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.blocks = torch.nn.ModuleList(+ [torch.nn.Sequential(torch.nn.Linear(k, k, bias = i % 2 == 0)) for i in range(n)]+ )++ def forward(self, x):+ for block in self.blocks:+ x = block(x) * 0.5+ return x++ holder = Holder().cuda().to(torch.bfloat16)+ cfg = Int8DynamicActivationInt8WeightConfig(set_inductor_config = False)+ if version is not None:+ if not hasattr(cfg, "version"):+ pytest.skip("torchao without config versions")+ cfg.version = version+ quantize_(holder, cfg)+ return holder.requires_grad_(False)+++@needs_cuda[email protected]("version", [None, 2])+def test_prefetched_weights_run_the_fused_gemm_bit_identically(forced, version):+ """Studio's block streaming (diffusers group offload driven by the event-fenced prefetch, two groups ahead): the+ weights reach each block through the copy stream, every int8 Linear still takes the fused GEMM, and the output+ equals stock torchao on resident weights, forward after forward."""+ pytest.importorskip("diffusers.hooks")+ from core.inference import diffusion_memory as dm+ from core.inference.diffusion_offload_prefetch import module_prefetcher++ n, k = 6, 1024+ stock = _blocks(n, k, version)+ streamed = _blocks(n, k, version).cpu()+ pipe = types.SimpleNamespace(transformer = streamed, components = {"transformer": streamed})+ assert dm._apply_group_offload(pipe, "cuda", None)+ pf = module_prefetcher(streamed)+ if pf is None:+ pytest.skip("this diffusers keeps its own stream prefetch")+ assert all(lin.weight.device.type == "cpu" for block in streamed.blocks for lin in block)+ assert g8.install(streamed, device = "cuda") == n+ xs = [torch.randn(300, k, device = "cuda", dtype = torch.bfloat16) * 3 for _ in range(4)]+ with torch.no_grad(): # group offload's swap_tensors onload cannot run on inference tensors+ for x in xs:+ before = g8.call_count()+ out = streamed(x)+ assert g8.call_count() == before + n+ assert torch.equal(out, stock(x))+ assert pf.stats["prefetched"] > 0 and pf.stats["missed"] == 0