Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: fuse the int8 SwiGLU MLP on block-streamed Qwen-Image-2.1 denoisers (#15370)

Daniel Hanaa9b15211e7b
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index e783b7b85bb..7992150062f 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -42,6 +42,9 @@ A fully swapped Qwen-Image-2.1 SwiGLU MLP takes its down projection's codes stra
A block-streamed denoiser installs against its onload device (``install(..., device = ...)``): group offload's
``swap_tensors`` keeps each Parameter's identity and the forward reads the payload off the live Parameter, so every
call sees the onloaded copy. Kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED=0``.
+
+It also patches host-resident SwiGLU MLPs (the group hook onloads the block first; off-device weights fall back to
+stock). Kill switch ``UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED=0``.
"""
from __future__ import annotations
@@ -57,6 +60,7 @@ INT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"
INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"
INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"
INT8_GEMM_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_TILES"
+INT8_SWIGLU_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED"
_MIN_TRITON = (3, 2)
# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge.
_MIN_ROWS = 17
@@ -124,6 +128,11 @@ def streamed_enabled() -> bool:
return raw not in ("0", "off", "false", "no")
+def swiglu_streamed_enabled() -> bool:
+ raw = (os.environ.get(INT8_SWIGLU_STREAMED_ENV) or "").strip().lower()
+ return raw not in ("0", "off", "false", "no")
+
+
def tiles_enabled() -> bool:
raw = (os.environ.get(INT8_GEMM_TILES_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
@@ -774,14 +783,21 @@ def _swiglu_ff_eligible(module: Any) -> bool:
return bool(parts[2].__dict__[_REC][2])
-def _install_swiglu_ff(transformer: Any, index: int) -> int:
- """Patch the eligible SwiGLU MLPs on card ``index``."""
+def _install_swiglu_ff(
+ transformer: Any,
+ index: int,
+ streamed: bool = False,
+) -> int:
+ """Patch the eligible SwiGLU MLPs on card ``index`` (``streamed``: also host-resident ones)."""
count = 0
for module in transformer.modules():
if _FF_MARK in module.__dict__:
count += 1
continue
- if not _swiglu_ff_eligible(module) or _weight_card(module.out) != index:
+ if not _swiglu_ff_eligible(module):
+ continue
+ card = _weight_card(module.out)
+ if card != index and not (streamed and card is None):
continue
module.__dict__[_FF_MARK] = True
module.forward = types.MethodType(_swiglu_ff_forward, module)
@@ -951,7 +967,7 @@ def _swap(
return 0
index = dev.index if dev.index is not None else torch.cuda.current_device()
- return _swap_on(transformer, logger, index)
+ return _swap_on(transformer, logger, index, streamed = device is not None)
def _split_cards(transformer: Any) -> tuple:
@@ -977,7 +993,12 @@ def _weight_card(module: Any) -> Optional[int]:
return dev.index if dev is not None and dev.type == "cuda" else None
-def _swap_on(transformer: Any, logger: Any, index: int) -> int:
+def _swap_on(
+ transformer: Any,
+ logger: Any,
+ index: int,
+ streamed: bool = False,
+) -> int:
"""Swap the eligible int8 Linears of ``transformer`` against card ``index``'s probed kernels. On a layer split
(several cards) only the Linears whose weight sits on ``index``."""
global _OP_HANDLE
@@ -1028,7 +1049,15 @@ def _swap_on(transformer: Any, logger: Any, index: int) -> int:
module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
module.forward = types.MethodType(_linear_forward, module)
count += 1
- ff = _install_swiglu_ff(transformer, index) if rotq else 0
+ ff = (
+ _install_swiglu_ff(
+ transformer,
+ index,
+ streamed = streamed and not multi and swiglu_streamed_enabled(),
+ )
+ if rotq
+ else 0
+ )
if logger is not None and count:
from .diffusion_convrot_quant import _H16_DEVICE
logger.info(
diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.py
index b8f8a7f6173..b025db78743 100644
--- a/studio/backend/tests/test_diffusion_int8_gemm.py
+++ b/studio/backend/tests/test_diffusion_int8_gemm.py
@@ -1279,6 +1279,95 @@ def test_swiglu_mlp_without_the_prologue_still_shares_the_gate_value_quant(h16_r
g8.uninstall(torch.nn.Sequential(ff))
+def _host_swiglu_ff(monkeypatch):
+ ff = torch.nn.Module()
+ ff.gate_layer, ff.proj, ff.out = (
+ torch.nn.Linear(8, 8),
+ torch.nn.Linear(8, 8),
+ torch.nn.Linear(8, 8),
+ )
+ ff.activation_fn = torch.nn.SiLU()
+ monkeypatch.setattr(g8, "_swiglu_ff_eligible", lambda m: m is ff)
+ return ff
+
+
[email protected]("streamed, patched", [(False, False), (True, True)])
+def test_streamed_install_patches_the_host_swiglu_mlp(monkeypatch, streamed, patched):
+ monkeypatch.delenv(g8.INT8_SWIGLU_STREAMED_ENV, raising = False)
+ ff = _host_swiglu_ff(monkeypatch)
+ assert g8._install_swiglu_ff(torch.nn.Sequential(ff), 0, streamed = streamed) == int(patched)
+ assert bool(ff.__dict__.get(g8._FF_MARK)) is patched
+
+
+def test_streamed_install_keeps_a_swiglu_mlp_on_another_card(monkeypatch):
+ ff = _host_swiglu_ff(monkeypatch)
+ monkeypatch.setattr(g8, "_weight_card", lambda m: 1)
+ assert g8._install_swiglu_ff(torch.nn.Sequential(ff), 0, streamed = True) == 0
+ assert g8._FF_MARK not in ff.__dict__
+
+
+def test_swap_passes_streamed_only_for_the_onload_device_install(monkeypatch):
+ seen = []
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(
+ g8, "_swap_on", lambda t, lg, i, streamed = False: seen.append((i, streamed)) or 0
+ )
+ import core.inference.diffusion_int8_fused as fused
+
+ monkeypatch.setattr(fused, "resident_cuda_device", lambda t: torch.device("cuda", 0))
+ g8._swap(torch.nn.Linear(2, 2), device = "cuda:0")
+ g8._swap(torch.nn.Linear(2, 2))
+ assert seen == [(0, True), (0, False)]
+
+
+@needs_cuda
+def test_block_streamed_swiglu_mlp_runs_fused_after_the_onload(h16_rotq, monkeypatch):
+ """Streamed host MLP runs fused, bit-identical to resident; the kill switch keeps it stock."""
+ hooks = pytest.importorskip("diffusers.hooks")
+ if not cq._SWIGLU_DEVICE.get(torch.cuda.current_device()):
+ pytest.skip("SwiGLU prologue probe refused this device")
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ monkeypatch.delenv(g8.INT8_GEMM_STREAMED_ENV, raising = False)
+
+ class Holder(torch.nn.Module):
+ def __init__(self, ff):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList([torch.nn.Sequential(ff)])
+
+ def forward(self, x):
+ return self.blocks[0](x)
+
+ resident = _swiglu_ff(1024, 3072, 2)
+ assert g8.install(torch.nn.Sequential(resident)) == 3 and resident.__dict__.get(g8._FF_MARK)
+ x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = resident(x)
+ for kill, fused in ((None, True), ("0", False)):
+ if kill is None:
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_INT8_SWIGLU_STREAMED", kill)
+ ff = _swiglu_ff(1024, 3072, 2)
+ holder = Holder(ff).requires_grad_(False).cpu()
+ hooks.apply_group_offloading(
+ holder,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ )
+ assert ff.out.weight.device.type == "cpu"
+ assert g8.install(holder, device = "cuda") == 3
+ assert bool(ff.__dict__.get(g8._FF_MARK)) is fused
+ with torch.no_grad():
+ sw = cq.swiglu_call_count()
+ out = holder(x)
+ assert cq.swiglu_call_count() == sw + int(fused)
+ if fused:
+ assert torch.equal(out, ref)
+ assert ff.out.weight.device.type == "cpu"
+
+
def test_hooked_linear_is_not_shared():
"""A forward hook or pre-hook (module-level or global) on a swapped Linear must see its call, so the shared QKV /
SwiGLU paths, which run the GEMM without calling the module, decline it and the per-module call runs instead."""