Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fused int8 GEMM for block-streamed image denoisers (#12677)

Daniel Han29bcc2306895
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 9a9653d15f7..8cf0a47bd2c 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -7104,6 +7104,7 @@ class DiffusionBackend:
cache_engaged = cache_graph_break,
offload_active = plan.offload_policy != OFFLOAD_NONE,
denoiser_offloaded = not plan_keeps_transformer_resident(plan),
+ stream_int8_gemm = True,
logger = logger,
)
if vae_fp16:
@@ -9191,6 +9192,7 @@ class DiffusionBackend:
offload_active = state.offload_policy != OFFLOAD_NONE,
denoiser_offloaded = state.offload_policy != OFFLOAD_NONE
and _denoiser_hooked(state.pipe),
+ stream_int8_gemm = True,
logger = logger,
)
if denoisers_pinned_resident(state.pipe):
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 0bc5dccc672..5a1de2936a2 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -427,6 +427,7 @@ def apply_speed_optims(
cuda_graph_default: bool = True,
cache_engaged: Optional[bool] = None,
denoiser_offloaded: Optional[bool] = None,
+ stream_int8_gemm: bool = False,
logger: Any = None,
) -> dict[str, bool]:
"""Apply the opt-in speed optims for ``speed_mode`` to a built pipeline, BEFORE placement /
@@ -435,6 +436,7 @@ def apply_speed_optims(
``offload_active`` (offload policy != none) installs ``@torch.compiler.disable``d onload hooks,
so the compile must drop ``fullgraph`` (like an active step cache) or it crashes at step 1.
``denoiser_offloaded`` (None = ``offload_active``) limits the CUDA-graph refusal to a moved denoiser.
+ ``stream_int8_gemm``: a moved denoiser still takes the fused int8 GEMM, installed against its onload device.
``cuda_graph_default`` is what the CUDA-graph arm assumes for a family that declares nothing:
True on the image backend, False on video, where ``supports_cuda_graph`` opts in.
@@ -517,6 +519,7 @@ def apply_speed_optims(
cache_active = cache_active,
offload_active = offload_active,
denoiser_offloaded = denoiser_offloaded,
+ onload_device = _onload_device(target) if stream_int8_gemm else None,
filter_reductions = filter_reductions,
)
elif (
@@ -531,6 +534,7 @@ def apply_speed_optims(
cache_active = cache_active,
offload_active = offload_active,
denoiser_offloaded = denoiser_offloaded,
+ onload_device = _onload_device(target) if stream_int8_gemm else None,
filter_reductions = filter_reductions,
)
@@ -606,6 +610,14 @@ def apply_speed_optims(
return applied
+def _onload_device(target: Any) -> Optional[str]:
+ """Onload device of an offloaded denoiser, else None. NVIDIA only: ROCm runs int8 weight-only, never this GEMM."""
+ if getattr(target, "device", None) != "cuda" or getattr(target, "backend", "cuda") != "cuda":
+ return None
+ device = getattr(target, "torch_device", None)
+ return str(device) if isinstance(device, str) and device.startswith("cuda") else "cuda"
+
+
def engage_pinned_denoisers(
pipe: Any,
applied: dict,
@@ -931,6 +943,7 @@ def _compile_repeated_blocks(
cache_active: bool = False,
offload_active: bool = False,
denoiser_offloaded: Optional[bool] = None,
+ onload_device: Any = None,
filter_reductions: bool = False,
) -> bool:
dits = [
@@ -1049,14 +1062,17 @@ def _compile_repeated_blocks(
try:
from .diffusion_int8_gemm import install as install_int8_gemm
- # Keyed on the DENOISER's placement: a group plan that streams only the encoders keeps it resident.
- transformer._unsloth_int8_gemm = install_int8_gemm(
- transformer,
- logger,
- offload_active = offload_active
- if denoiser_offloaded is None
- else bool(denoiser_offloaded),
- )
+ # Keyed on the DENOISER's placement (an encoder-only stream keeps it resident). A moved denoiser installs
+ # against its onload device; a weight found elsewhere at call time keeps the stock path.
+ moved = offload_active if denoiser_offloaded is None else bool(denoiser_offloaded)
+ if moved and onload_device is not None:
+ transformer._unsloth_int8_gemm = install_int8_gemm(
+ transformer, logger, device = onload_device
+ )
+ else:
+ transformer._unsloth_int8_gemm = install_int8_gemm(
+ transformer, logger, offload_active = moved
+ )
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "int8 fused-dequant gemm", exc)
if type(transformer).__name__ == "QwenImageTransformer2DModel":
diff --git a/studio/backend/tests/test_diffusion_stream_int8_gemm.py b/studio/backend/tests/test_diffusion_stream_int8_gemm.py
new file mode 100644
index 00000000000..d424ea410b1
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_stream_int8_gemm.py
@@ -0,0 +1,227 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A block-streamed image denoiser takes the fused int8 GEMM: the speed layer installs it against the onload device
+when the plan moves the denoiser, and the weights that arrive through the event-fenced prefetch run it bit-identically
+to stock torchao. The CPU tests need no GPU; the last ones need NVIDIA sm80+ CUDA, Triton and torchao."""
+
+from __future__ import annotations
+
+import ast
+import pathlib
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_int8_gemm as g8 # noqa: E402
+from core.inference import diffusion_speed as ds # noqa: E402
+
+_INFERENCE = pathlib.Path(ds.__file__).resolve().parent
+
+
+class _DiT:
+ def compile_repeated_blocks(self, **kwargs):
+ return None
+
+
+def _record_install(monkeypatch):
+ seen = []
+
+ def _install(
+ transformer,
+ logger = None,
+ offload_active = False,
+ device = None,
+ ):
+ seen.append({"offload_active": offload_active, "device": device})
+ return 7
+
+ monkeypatch.setattr(g8, "install", _install)
+ monkeypatch.setattr(ds, "_denoiser_dits", lambda pipe: [_DiT()])
+ return seen
+
+
+ "offload_active, denoiser_offloaded, onload, expected",
+ [
+ # moved denoiser with a known onload device: installed against it
+ (True, True, "cuda:1", {"offload_active": False, "device": "cuda:1"}),
+ (True, None, "cuda", {"offload_active": False, "device": "cuda"}),
+ # no onload device (video backend, ROCm, CPU): the old refusal
+ (True, True, None, {"offload_active": True, "device": None}),
+ # a resident denoiser never needs the device
+ (True, False, "cuda", {"offload_active": False, "device": None}),
+ (False, None, "cuda", {"offload_active": False, "device": None}),
+ ],
+)
+def test_moved_denoiser_installs_against_the_onload_device(
+ monkeypatch, offload_active, denoiser_offloaded, onload, expected
+):
+ seen = _record_install(monkeypatch)
+ ds._compile_repeated_blocks(
+ types.SimpleNamespace(),
+ None,
+ offload_active = offload_active,
+ denoiser_offloaded = denoiser_offloaded,
+ onload_device = onload,
+ )
+ assert seen == [expected]
+
+
+ "target, expected",
+ [
+ (types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:1"), "cuda:1"),
+ (types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda"), "cuda"),
+ (types.SimpleNamespace(device = "cuda", backend = "cuda"), "cuda"), # video's namespace target
+ (types.SimpleNamespace(device = "cuda", backend = "rocm", torch_device = "cuda"), None),
+ (types.SimpleNamespace(device = "cpu", backend = "cpu", torch_device = "cpu"), None),
+ (types.SimpleNamespace(device = "mps", backend = "mps", torch_device = "mps"), None),
+ ],
+)
+def test_onload_device_is_nvidia_cuda_only(target, expected):
+ assert ds._onload_device(target) == expected
+
+
+def _speed_calls(path: pathlib.Path) -> list:
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ return [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", getattr(node.func, "attr", None)) == "apply_speed_optims"
+ ]
+
+
+def test_image_backend_asks_for_the_streamed_install():
+ """Both image-backend speed passes (load, and the deferred compile on the 3rd image) ask for it; the video
+ backend keeps its own H3-only streamed install."""
+ calls = _speed_calls(_INFERENCE / "diffusion.py")
+ assert len(calls) == 2
+ for call in calls:
+ kw = {k.arg: k.value for k in call.keywords}
+ assert (
+ isinstance(kw.get("stream_int8_gemm"), ast.Constant)
+ and kw["stream_int8_gemm"].value is True
+ )
+ for call in _speed_calls(_INFERENCE / "video.py"):
+ assert "stream_int8_gemm" not in {k.arg for k in call.keywords}
+
+
+def test_apply_speed_optims_passes_the_onload_device_only_when_asked(monkeypatch):
+ seen = []
+
+ def _compile(pipe, logger, **kwargs):
+ seen.append(kwargs.get("onload_device"))
+ return False
+
+ monkeypatch.setattr(ds, "_compile_repeated_blocks", _compile)
+ monkeypatch.setattr(ds, "compile_eligible", lambda *a, **k: True)
+ monkeypatch.setattr(ds, "fp16_unet_offloaded", lambda *a, **k: False)
+ monkeypatch.setattr(ds, "_denoiser_unet", lambda pipe: None)
+ target = types.SimpleNamespace(device = "cuda", backend = "cuda", torch_device = "cuda:2", dtype = None)
+ pipe = types.SimpleNamespace()
+ for mode in (ds.SPEED_DEFAULT, ds.SPEED_MAX):
+ for ask in (True, False):
+ try:
+ ds.apply_speed_optims(
+ pipe,
+ target,
+ is_gguf = False,
+ family = types.SimpleNamespace(supports_cuda_graph = False),
+ speed_mode = mode,
+ offload_active = True,
+ denoiser_offloaded = True,
+ stream_int8_gemm = ask,
+ )
+ except Exception: # noqa: BLE001 - only the compile call is under test
+ pass
+ assert seen == ["cuda:2", None, "cuda:2", None]
+
+
+def _cuda_ready() -> bool:
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ if torch.cuda.get_device_capability() < (8, 0):
+ return False
+ try:
+ import torchao # noqa: F401
+ import triton # noqa: F401
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
+needs_cuda = pytest.mark.skipif(
+ not _cuda_ready(), reason = "needs NVIDIA sm80+ CUDA, Triton and torchao"
+)
+
+
+def forced(monkeypatch):
+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")
+ monkeypatch.delenv(g8.INT8_GEMM_STREAMED_ENV, raising = False)
+ g8._DEVICE_CFG.clear()
+ cfg = g8.device_config(torch.cuda.current_device())
+ if cfg is None:
+ pytest.skip("int8 GEMM probe refused this device")
+ yield cfg
+ g8._DEVICE_CFG.clear()
+
+
+def _blocks(n, k, version):
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ class Holder(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.blocks = torch.nn.ModuleList(
+ [torch.nn.Sequential(torch.nn.Linear(k, k, bias = i % 2 == 0)) for i in range(n)]
+ )
+
+ def forward(self, x):
+ for block in self.blocks:
+ x = block(x) * 0.5
+ return x
+
+ holder = Holder().cuda().to(torch.bfloat16)
+ cfg = Int8DynamicActivationInt8WeightConfig(set_inductor_config = False)
+ if version is not None:
+ if not hasattr(cfg, "version"):
+ pytest.skip("torchao without config versions")
+ cfg.version = version
+ quantize_(holder, cfg)
+ return holder.requires_grad_(False)
+
+
+@needs_cuda
[email protected]("version", [None, 2])
+def test_prefetched_weights_run_the_fused_gemm_bit_identically(forced, version):
+ """Studio's block streaming (diffusers group offload driven by the event-fenced prefetch, two groups ahead): the
+ weights reach each block through the copy stream, every int8 Linear still takes the fused GEMM, and the output
+ equals stock torchao on resident weights, forward after forward."""
+ pytest.importorskip("diffusers.hooks")
+ from core.inference import diffusion_memory as dm
+ from core.inference.diffusion_offload_prefetch import module_prefetcher
+
+ n, k = 6, 1024
+ stock = _blocks(n, k, version)
+ streamed = _blocks(n, k, version).cpu()
+ pipe = types.SimpleNamespace(transformer = streamed, components = {"transformer": streamed})
+ assert dm._apply_group_offload(pipe, "cuda", None)
+ pf = module_prefetcher(streamed)
+ if pf is None:
+ pytest.skip("this diffusers keeps its own stream prefetch")
+ assert all(lin.weight.device.type == "cpu" for block in streamed.blocks for lin in block)
+ assert g8.install(streamed, device = "cuda") == n
+ xs = [torch.randn(300, k, device = "cuda", dtype = torch.bfloat16) * 3 for _ in range(4)]
+ with torch.no_grad(): # group offload's swap_tensors onload cannot run on inference tensors
+ for x in xs:
+ before = g8.call_count()
+ out = streamed(x)
+ assert g8.call_count() == before + n
+ assert torch.equal(out, stock(x))
+ assert pf.stats["prefetched"] > 0 and pf.stats["missed"] == 0