Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: MiniMax-H3 int8 Linears take the fused-dequant int8 GEMM (A100 9% faster per step, ahead of ComfyUI) (#12507)

Daniel Hanbbe2467f9f7b
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index 32c8276be64..865da207e09 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -19,6 +19,13 @@ Per-arch gate: sm80 / sm89 / sm120 on (measured), everything else stock. sm75: T
sm100: Triton int8 is slower than cuBLAS. ROCm (weight-only int8 there) and CPU never reach it.
Kill switch: ``UNSLOTH_DIFFUSION_INT8_GEMM=0``; ``=1`` also enables it on an unmeasured arch (still probe-gated).
+
+ConvRot Linears (MiniMax-H3) run ``ConvRotLinear.forward``'s own rotation first, so eager output stays bit-identical;
+kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT=0``.
+
+A block-streamed denoiser installs against its onload device (``install(..., device = ...)``): group offload's
+``swap_tensors`` keeps each Parameter's identity and the forward reads the payload off the live Parameter, so every
+call sees the onloaded copy. Kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED=0``.
"""
from __future__ import annotations
@@ -30,6 +37,8 @@ from functools import lru_cache
from typing import Any, Optional
INT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"
+INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"
+INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"
_MIN_TRITON = (3, 2)
# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge.
_MIN_ROWS = 17
@@ -72,6 +81,16 @@ def int8_gemm_mode() -> str:
return "auto"
+def convrot_enabled() -> bool:
+ raw = (os.environ.get(INT8_GEMM_CONVROT_ENV) or "").strip().lower()
+ return raw not in ("0", "off", "false", "no")
+
+
+def streamed_enabled() -> bool:
+ raw = (os.environ.get(INT8_GEMM_STREAMED_ENV) or "").strip().lower()
+ return raw not in ("0", "off", "false", "no")
+
+
def _triton_version_ok(version: Optional[str] = None) -> bool:
if version is None:
try:
@@ -263,7 +282,7 @@ def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
_CALLS[0] += 1
a = a if a.stride(-1) == 1 else a.contiguous()
cfg = _DEVICE_CFG.get(a.device.index)
- if cfg is not None and w.stride(-1) == 1 and _aligned(a, w):
+ if cfg is not None and w.stride(-1) == 1 and w.device == a.device and _aligned(a, w):
try:
return _launch(a, w, xs, ws, bias, cfg)
except Exception: # noqa: BLE001 - a failed launch keeps the stock math
@@ -507,7 +526,7 @@ def _linear_forward(self: Any, x: Any) -> Any:
rec = self.__dict__.get(_REC)
if rec is None or x.dtype != torch.bfloat16 or not x.is_cuda:
return type(self).forward(self, x)
- kind, _wq, _ws, weight = rec
+ kind, group, _ws, weight = rec
if self.weight is not weight: # weight replaced since install (reload / LoRA bake): stock
return type(self).forward(self, x)
# Payload off the live parameter, never a cached alias: a moved weight must not leave a stale device or pin a copy.
@@ -519,9 +538,14 @@ def _linear_forward(self: Any, x: Any) -> Any:
if wq.device != x.device:
return type(self).forward(self, x)
lead = x.shape[:-1]
- x2d = x.reshape(-1, x.shape[-1])
- if x2d.shape[0] < _MIN_ROWS:
+ if x.numel() < _MIN_ROWS * x.shape[-1]:
return type(self).forward(self, x)
+ if group is not None:
+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
+ x = rotate_convrot_activation(
+ x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group
+ )
+ x2d = x.reshape(-1, x.shape[-1])
if kind == "v1":
xq, xs = _act_quant_v1(x2d)
else:
@@ -553,11 +577,31 @@ def linear_from_q(q: Any, xs: Any, weight: Any, bias: Any) -> Optional[Any]:
return _OP_HANDLE(q, parts[0], xs.reshape(-1), parts[1], bias)
+def _rotation_group(module: Any) -> Optional[int]:
+ """The ConvRot group the forward above can reproduce, else None."""
+ try:
+ from .diffusion_convrot import convrot_linear_class, is_power_of_four
+
+ if type(module) is not convrot_linear_class():
+ return None
+ group = module.convrot_groupsize
+ if not is_power_of_four(group) or module.in_features % group:
+ return None
+ return int(group)
+ except Exception: # noqa: BLE001
+ return None
+
+
def _eligible(module: Any) -> Optional[tuple]:
+ """(kind, rotation group or None, scale, weight) for a Linear the fused GEMM can run, else None."""
from torch import nn
- if type(module) is not nn.Linear: # a subclass (ConvRotLinear) transforms the input first
- return None
+ group = None
+ if type(module) is not nn.Linear:
+ # any other subclass transforms its input in a way this forward would skip
+ group = _rotation_group(module) if convrot_enabled() else None
+ if group is None:
+ return None
w = module.weight
bias = module.bias
import torch
@@ -570,10 +614,10 @@ def _eligible(module: Any) -> Optional[tuple]:
return None
parts = _v1_parts(w)
if parts is not None and parts[1].dtype == torch.bfloat16:
- return ("v1",) + parts + (w,)
+ return ("v1", group, parts[1], w)
parts = _v2_parts(w)
if parts is not None and parts[1].dtype in (torch.bfloat16, torch.float32):
- return ("v2",) + parts + (w,)
+ return ("v2", group, parts[1], w)
return None
@@ -588,9 +632,19 @@ def install(
transformer: Any,
logger: Any = None,
offload_active: bool = False,
+ device: Any = None,
) -> int:
- """Idempotent; returns the (candidate) count. Must run before the first compiled forward."""
- if int8_gemm_mode() == "off" or transformer is None or offload_active:
+ """Idempotent; returns the (candidate) count. Must run before the first compiled forward.
+
+ ``device``: the onload device of a block-streamed denoiser, whose weights sit on the host between blocks; the
+ probe runs there and the swap happens now. Without it an offloaded denoiser keeps the stock path."""
+ if int8_gemm_mode() == "off" or transformer is None:
+ return 0
+ if device is not None:
+ if not streamed_enabled():
+ return 0
+ return _finalize(transformer, logger, device = device)
+ if offload_active:
return 0
from .diffusion_int8_fused import resident_cuda_device, run_on_first_call
@@ -614,8 +668,12 @@ def install(
return n
-def _finalize(transformer: Any, logger: Any = None) -> int:
- count = _swap(transformer, logger)
+def _finalize(
+ transformer: Any,
+ logger: Any = None,
+ device: Any = None,
+) -> int:
+ count = _swap(transformer, logger, device = device)
try:
transformer._unsloth_int8_gemm = count # the deferred install recorded the candidate count
except Exception: # noqa: BLE001
@@ -623,14 +681,31 @@ def _finalize(transformer: Any, logger: Any = None) -> int:
return count
-def _swap(transformer: Any, logger: Any = None) -> int:
+def _swap(
+ transformer: Any,
+ logger: Any = None,
+ device: Any = None,
+) -> int:
global _OP_HANDLE
from .diffusion_int8_fused import resident_cuda_device
- dev = resident_cuda_device(transformer)
+ import torch
+
+ if device is not None:
+ try:
+ dev = torch.device(device)
+ except Exception: # noqa: BLE001
+ return 0
+ if (
+ dev.type != "cuda"
+ or getattr(torch.version, "hip", None)
+ or not torch.cuda.is_available()
+ ):
+ return 0
+ else:
+ dev = resident_cuda_device(transformer)
if dev is None:
return 0
- import torch
index = dev.index if dev.index is not None else torch.cuda.current_device()
recs = [(m, _eligible(m)) for m in transformer.modules()]
@@ -650,10 +725,10 @@ def _swap(transformer: Any, logger: Any = None) -> int:
continue
module.__dict__[_REC] = (
rec[0],
- None,
+ rec[1],
None,
rec[3],
- ) # kind + the Parameter, no payload alias
+ ) # kind, rotation, the Parameter; no payload alias
module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
module.forward = types.MethodType(_linear_forward, module)
count += 1
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index d4acd2aec1e..95a6716264f 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -7471,11 +7471,22 @@ class VideoBackend:
# The conditioner and the VAEs stay in the rotation even when the denoiser is pinned, so the onload
# hooks are live and fullgraph has to drop.
offload_active = offload_policy != "none",
+ # a denoiser pinned under the others' offload rotation never moves, so the fused int8 GEMM can engage
+ denoiser_offloaded = bool(denoiser_streamed) or not denoiser_pinned,
cuda_graph_default = False,
logger = logger,
)
speed_optims = tuple(k for k, v in applied.items() if v) + h3_attn_levers
if denoiser_streamed and "compiled" in speed_optims:
+ # before the first forward traces the blocks below their hooks
+ try:
+ from .diffusion_int8_gemm import install as install_int8_gemm
+ if install_int8_gemm(
+ getattr(pipe, denoiser_component, None), logger, device = device
+ ):
+ speed_optims += ("int8_gemm",)
+ except Exception as exc: # noqa: BLE001 -- optimisation only, the stock GEMM stays
+ logger.warning("video.h3_int8_gemm: streamed install skipped: %s", exc)
from .video_minimax_h3_residency import compile_blocks_below_offload_hooks
compile_blocks_below_offload_hooks(
getattr(pipe, denoiser_component, None), logger = logger
diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.py
index 93b03ec3cff..82e7c45a12a 100644
--- a/studio/backend/tests/test_diffusion_int8_gemm.py
+++ b/studio/backend/tests/test_diffusion_int8_gemm.py
@@ -393,3 +393,127 @@ def test_status_drops_int8_gemm_once_the_deferred_probe_swapped_nothing():
dit._unsloth_int8_gemm = 0
assert int8_gemm_live(pipe, ("compiled", "int8_gemm")) == ["compiled"]
assert int8_gemm_live(pipe, ("compiled",)) == ["compiled"]
+
+
+def _convrot(lin, group = 256):
+ from core.inference.diffusion_convrot import _install_rotation
+ _install_rotation(lin, group)
+ return lin
+
+
+def test_convrot_linear_is_eligible_with_its_group(monkeypatch):
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ scale = torch.ones(256, dtype = torch.bfloat16)
+ monkeypatch.setattr(g8, "_v1_parts", lambda w: (torch.zeros(256, 512, dtype = torch.int8), scale))
+ lin = _convrot(torch.nn.Linear(512, 256, bias = False))
+ rec = g8._eligible(lin)
+ assert rec is not None and rec[:2] == ("v1", 256) and rec[3] is lin.weight
+ monkeypatch.setenv(g8.INT8_GEMM_CONVROT_ENV, "0")
+ assert g8._eligible(lin) is None
+ assert g8._eligible(torch.nn.Linear(512, 256, bias = False))[:2] == ("v1", None)
+
+
+def test_other_linear_subclasses_and_bad_groups_stay_stock(monkeypatch):
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ monkeypatch.setattr(
+ g8,
+ "_v1_parts",
+ lambda w: (torch.zeros(256, 512, dtype = torch.int8), torch.ones(256, dtype = torch.bfloat16)),
+ )
+
+ class Other(torch.nn.Linear):
+ pass
+
+ assert g8._eligible(Other(512, 256, bias = False)) is None
+ lin = _convrot(torch.nn.Linear(512, 256, bias = False))
+ lin.convrot_groupsize = 1024 # does not divide in_features: the forward could not rotate it
+ assert g8._eligible(lin) is None
+
+
+@needs_cuda
[email protected]("version", [None, 2])
+def test_convrot_linear_swap_is_bit_identical_eager_and_compiled(forced, monkeypatch, version):
+ """MiniMax-H3's rotated int8 Linear (and its fused QKV): rotation, act quant, fused GEMM == ConvRotLinear.forward."""
+ from torch._dynamo.utils import counters
+
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ stock = _convrot(_int8_linear(1024, 768, False, version))
+ fused = _convrot(_int8_linear(1024, 768, False, version))
+ holder = torch.nn.Sequential(fused)
+ assert g8.install(holder) == 1 and g8.is_installed(fused)
+ x = torch.randn(2, 300, 1024, device = "cuda", dtype = torch.bfloat16) * 3
+ x[0, 7, :5] *= 200 # an outlier row, what the rotation exists for
+ with torch.inference_mode():
+ before = g8.call_count()
+ assert torch.equal(fused(x), stock(x))
+ assert g8.call_count() == before + 1
+ counters.clear()
+ torch._dynamo.reset()
+ with torch._inductor.config.patch(emulate_precision_casts = True):
+ out = torch.compile(fused, fullgraph = True)(x)
+ assert not counters["graph_break"]
+ assert torch.equal(out, torch.compile(stock, fullgraph = True)(x))
+ g8.uninstall(holder)
+ assert not g8.is_installed(fused)
+
+
+@needs_cuda
+def test_convrot_kill_switch_keeps_rotated_linears_stock(forced, monkeypatch):
+ monkeypatch.setenv(g8.INT8_GEMM_CONVROT_ENV, "0")
+ rotated = _convrot(_int8_linear(1024, 768, False, None))
+ plain = _int8_linear(1024, 768, False, None)
+ holder = torch.nn.Sequential(rotated, plain)
+ assert g8.install(holder) == 1
+ assert g8.is_installed(plain) and not g8.is_installed(rotated)
+
+
+@needs_cuda
[email protected]("use_stream, version", [(False, None), (False, 2), (True, 2)])
+def test_block_streamed_denoiser_installs_against_its_onload_device(
+ forced, monkeypatch, use_stream, version
+):
+ """Group-offloaded int8 blocks (H3 on 40 GB): installed while on the host, the fused GEMM follows each onload."""
+ hooks = pytest.importorskip("diffusers.hooks")
+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)
+ monkeypatch.delenv(g8.INT8_GEMM_STREAMED_ENV, raising = False)
+
+ class Holder(torch.nn.Module):
+ def __init__(self, lin):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList([torch.nn.Sequential(lin)])
+
+ def forward(self, x):
+ return self.blocks[0](x)
+
+ stock = _convrot(_int8_linear(1024, 768, False, version))
+ fused = _convrot(_int8_linear(1024, 768, False, version))
+ holder = Holder(fused).requires_grad_(False).cpu()
+ hooks.apply_group_offloading(
+ holder,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = use_stream,
+ )
+ assert fused.weight.device.type == "cpu"
+ assert (
+ g8.install(holder, offload_active = True) == 0
+ ) # no onload device: an offloaded denoiser stays stock
+ assert g8.install(holder, device = "cuda") == 1 and g8.is_installed(fused)
+ x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16) * 3
+ with torch.no_grad(): # group offload's swap_tensors onload cannot run on inference tensors
+ before = g8.call_count()
+ out = holder(x)
+ assert g8.call_count() == before + 1
+ assert torch.equal(out, stock(x))
+ assert torch.equal(holder(x), out)
+ if not use_stream: # the stream path keeps the last group resident until the next onload
+ assert fused.weight.device.type == "cpu"
+
+
+@needs_cuda
+def test_streamed_kill_switch(forced, monkeypatch):
+ monkeypatch.setenv(g8.INT8_GEMM_STREAMED_ENV, "0")
+ holder = torch.nn.Sequential(_int8_linear(1024, 768, False, None))
+ assert g8.install(holder, device = "cuda") == 0 and not g8.is_installed(holder[0])
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 71a5c000ef9..a3707ee95b9 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -2355,3 +2355,34 @@ def test_family_compiles_regionally_closes_the_dynamo_import_window_first(monkey
fam = types.SimpleNamespace(transformer_class = "Lumina2Transformer2DModel")
assert family_compiles_regionally(fam) is True
assert events[:2] == ["guard", "probe"]
+
+
+ "offload_active, denoiser_offloaded, expected",
+ [(True, False, False), (True, True, True), (True, None, True), (False, None, False)],
+)
+def test_int8_gemm_install_follows_the_denoiser_placement(
+ monkeypatch, offload_active, denoiser_offloaded, expected
+):
+ """Only a moving denoiser keeps the stock GEMM; one pinned under the others' offload rotation takes the fused one."""
+ from core.inference import diffusion_int8_gemm, diffusion_speed as ds_mod
+
+ seen = []
+ monkeypatch.setattr(
+ diffusion_int8_gemm,
+ "install",
+ lambda t, logger = None, offload_active = False: seen.append(offload_active) or 0,
+ )
+
+ class _DiT:
+ def compile_repeated_blocks(self, **kwargs):
+ return None
+
+ monkeypatch.setattr(ds_mod, "_denoiser_dits", lambda pipe: [_DiT()])
+ ds_mod._compile_repeated_blocks(
+ types.SimpleNamespace(),
+ None,
+ offload_active = offload_active,
+ denoiser_offloaded = denoiser_offloaded,
+ )
+ assert seen == [expected]
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 9b5635c2ccb..3bdede35194 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -9621,6 +9621,53 @@ def test_every_rebuilt_speed_target_carries_the_backend():
assert "backend" in fields, f"video.py:{call.lineno} target lacks backend: {sorted(fields)}"
+def test_h3_speed_optims_get_the_denoisers_own_placement():
+ """A denoiser pinned under H3's offload policy must not read as offloaded."""
+ import ast
+ from pathlib import Path
+
+ source = Path(__file__).resolve().parents[1] / "core" / "inference" / "video.py"
+ text = source.read_text(encoding = "utf-8")
+ tree = ast.parse(text)
+ h3 = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "_load_h3_modular_pipeline"
+ )
+ calls = [
+ n
+ for n in ast.walk(h3)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "apply_speed_optims"
+ ]
+ assert len(calls) == 1
+ kw = {k.arg: k.value for k in calls[0].keywords}
+ assert "denoiser_offloaded" in kw
+ expr = ast.unparse(kw["denoiser_offloaded"])
+ names = {n.id for n in ast.walk(kw["denoiser_offloaded"]) if isinstance(n, ast.Name)}
+ assert (
+ {"denoiser_pinned", "denoiser_streamed"}
+ <= names
+ <= {"denoiser_pinned", "denoiser_streamed", "bool"}
+ ), expr
+ for pinned, streamed, offloaded in (
+ (True, None, False),
+ (True, "group", True),
+ (False, None, True),
+ ):
+ got = eval(
+ compile(ast.Expression(kw["denoiser_offloaded"]), "<h3>", "eval"),
+ {"bool": bool},
+ {"denoiser_pinned": pinned, "denoiser_streamed": streamed},
+ )
+ assert bool(got) is offloaded, (expr, pinned, streamed)
+ installs = [
+ n
+ for n in ast.walk(h3)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "install_int8_gemm"
+ ]
+ assert len(installs) == 1 and "device" in {k.arg for k in installs[0].keywords}
+
+
class _GraphHandle:
"""Stands in for a captured denoiser graph: only reset() matters to generate()."""