Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: fuse the int8 SwiGLU MLP on block-streamed Qwen-Image-2.1 denoisers (#15370)
Daniel Hanaa9b15211e7b
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex e783b7b85bb..7992150062f 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -42,6 +42,9 @@ A fully swapped Qwen-Image-2.1 SwiGLU MLP takes its down projection's codes straA block-streamed denoiser installs against its onload device (``install(..., device = ...)``): group offload's``swap_tensors`` keeps each Parameter's identity and the forward reads the payload off the live Parameter, so everycall sees the onloaded copy. Kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED=0``.++It also patches host-resident SwiGLU MLPs (the group hook onloads the block first; off-device weights fall back to+stock). Kill switch ``UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED=0``."""from __future__ import annotations@@ -57,6 +60,7 @@ INT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"INT8_GEMM_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_TILES"+INT8_SWIGLU_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED"_MIN_TRITON = (3, 2)# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge._MIN_ROWS = 17@@ -124,6 +128,11 @@ def streamed_enabled() -> bool:return raw not in ("0", "off", "false", "no")+def swiglu_streamed_enabled() -> bool:+ raw = (os.environ.get(INT8_SWIGLU_STREAMED_ENV) or "").strip().lower()+ return raw not in ("0", "off", "false", "no")++def tiles_enabled() -> bool:raw = (os.environ.get(INT8_GEMM_TILES_ENV) or "").strip().lower()return raw not in ("0", "off", "false", "no")@@ -774,14 +783,21 @@ def _swiglu_ff_eligible(module: Any) -> bool:return bool(parts[2].__dict__[_REC][2])-def _install_swiglu_ff(transformer: Any, index: int) -> int:- """Patch the eligible SwiGLU MLPs on card ``index``."""+def _install_swiglu_ff(+ transformer: Any,+ index: int,+ streamed: bool = False,+) -> int:+ """Patch the eligible SwiGLU MLPs on card ``index`` (``streamed``: also host-resident ones)."""count = 0for module in transformer.modules():if _FF_MARK in module.__dict__:count += 1continue- if not _swiglu_ff_eligible(module) or _weight_card(module.out) != index:+ if not _swiglu_ff_eligible(module):+ continue+ card = _weight_card(module.out)+ if card != index and not (streamed and card is None):continuemodule.__dict__[_FF_MARK] = Truemodule.forward = types.MethodType(_swiglu_ff_forward, module)@@ -951,7 +967,7 @@ def _swap(return 0index = dev.index if dev.index is not None else torch.cuda.current_device()- return _swap_on(transformer, logger, index)+ return _swap_on(transformer, logger, index, streamed = device is not None)def _split_cards(transformer: Any) -> tuple:@@ -977,7 +993,12 @@ def _weight_card(module: Any) -> Optional[int]:return dev.index if dev is not None and dev.type == "cuda" else None-def _swap_on(transformer: Any, logger: Any, index: int) -> int:+def _swap_on(+ transformer: Any,+ logger: Any,+ index: int,+ streamed: bool = False,+) -> int:"""Swap the eligible int8 Linears of ``transformer`` against card ``index``'s probed kernels. On a layer split(several cards) only the Linears whose weight sits on ``index``."""global _OP_HANDLE@@ -1028,7 +1049,15 @@ def _swap_on(transformer: Any, logger: Any, index: int) -> int:module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)module.forward = types.MethodType(_linear_forward, module)count += 1- ff = _install_swiglu_ff(transformer, index) if rotq else 0+ ff = (+ _install_swiglu_ff(+ transformer,+ index,+ streamed = streamed and not multi and swiglu_streamed_enabled(),+ )+ if rotq+ else 0+ )if logger is not None and count:from .diffusion_convrot_quant import _H16_DEVICElogger.info(diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.pyindex b8f8a7f6173..b025db78743 100644--- a/studio/backend/tests/test_diffusion_int8_gemm.py+++ b/studio/backend/tests/test_diffusion_int8_gemm.py@@ -1279,6 +1279,95 @@ def test_swiglu_mlp_without_the_prologue_still_shares_the_gate_value_quant(h16_rg8.uninstall(torch.nn.Sequential(ff))+def _host_swiglu_ff(monkeypatch):+ ff = torch.nn.Module()+ ff.gate_layer, ff.proj, ff.out = (+ torch.nn.Linear(8, 8),+ torch.nn.Linear(8, 8),+ torch.nn.Linear(8, 8),+ )+ ff.activation_fn = torch.nn.SiLU()+ monkeypatch.setattr(g8, "_swiglu_ff_eligible", lambda m: m is ff)+ return ff++[email protected]("streamed, patched", [(False, False), (True, True)])+def test_streamed_install_patches_the_host_swiglu_mlp(monkeypatch, streamed, patched):+ monkeypatch.delenv(g8.INT8_SWIGLU_STREAMED_ENV, raising = False)+ ff = _host_swiglu_ff(monkeypatch)+ assert g8._install_swiglu_ff(torch.nn.Sequential(ff), 0, streamed = streamed) == int(patched)+ assert bool(ff.__dict__.get(g8._FF_MARK)) is patched+++def test_streamed_install_keeps_a_swiglu_mlp_on_another_card(monkeypatch):+ ff = _host_swiglu_ff(monkeypatch)+ monkeypatch.setattr(g8, "_weight_card", lambda m: 1)+ assert g8._install_swiglu_ff(torch.nn.Sequential(ff), 0, streamed = True) == 0+ assert g8._FF_MARK not in ff.__dict__+++def test_swap_passes_streamed_only_for_the_onload_device_install(monkeypatch):+ seen = []+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(+ g8, "_swap_on", lambda t, lg, i, streamed = False: seen.append((i, streamed)) or 0+ )+ import core.inference.diffusion_int8_fused as fused++ monkeypatch.setattr(fused, "resident_cuda_device", lambda t: torch.device("cuda", 0))+ g8._swap(torch.nn.Linear(2, 2), device = "cuda:0")+ g8._swap(torch.nn.Linear(2, 2))+ assert seen == [(0, True), (0, False)]+++@needs_cuda+def test_block_streamed_swiglu_mlp_runs_fused_after_the_onload(h16_rotq, monkeypatch):+ """Streamed host MLP runs fused, bit-identical to resident; the kill switch keeps it stock."""+ hooks = pytest.importorskip("diffusers.hooks")+ if not cq._SWIGLU_DEVICE.get(torch.cuda.current_device()):+ pytest.skip("SwiGLU prologue probe refused this device")+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ monkeypatch.delenv(g8.INT8_GEMM_STREAMED_ENV, raising = False)++ class Holder(torch.nn.Module):+ def __init__(self, ff):+ super().__init__()+ self.blocks = torch.nn.ModuleList([torch.nn.Sequential(ff)])++ def forward(self, x):+ return self.blocks[0](x)++ resident = _swiglu_ff(1024, 3072, 2)+ assert g8.install(torch.nn.Sequential(resident)) == 3 and resident.__dict__.get(g8._FF_MARK)+ x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ref = resident(x)+ for kill, fused in ((None, True), ("0", False)):+ if kill is None:+ monkeypatch.delenv("UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED", kill)+ ff = _swiglu_ff(1024, 3072, 2)+ holder = Holder(ff).requires_grad_(False).cpu()+ hooks.apply_group_offloading(+ holder,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ )+ assert ff.out.weight.device.type == "cpu"+ assert g8.install(holder, device = "cuda") == 3+ assert bool(ff.__dict__.get(g8._FF_MARK)) is fused+ with torch.no_grad():+ sw = cq.swiglu_call_count()+ out = holder(x)+ assert cq.swiglu_call_count() == sw + int(fused)+ if fused:+ assert torch.equal(out, ref)+ assert ff.out.weight.device.type == "cpu"++def test_hooked_linear_is_not_shared():"""A forward hook or pre-hook (module-level or global) on a swapped Linear must see its call, so the shared QKV /SwiGLU paths, which run the GEMM without calling the module, decline it and the per-module call runs instead."""