Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: MiniMax-H3 int8 Linears take the fused-dequant int8 GEMM (A100 9% faster per step, ahead of ComfyUI) (#12507)
Daniel Hanbbe2467f9f7b
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex 32c8276be64..865da207e09 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -19,6 +19,13 @@ Per-arch gate: sm80 / sm89 / sm120 on (measured), everything else stock. sm75: Tsm100: Triton int8 is slower than cuBLAS. ROCm (weight-only int8 there) and CPU never reach it.Kill switch: ``UNSLOTH_DIFFUSION_INT8_GEMM=0``; ``=1`` also enables it on an unmeasured arch (still probe-gated).++ConvRot Linears (MiniMax-H3) run ``ConvRotLinear.forward``'s own rotation first, so eager output stays bit-identical;+kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT=0``.++A block-streamed denoiser installs against its onload device (``install(..., device = ...)``): group offload's+``swap_tensors`` keeps each Parameter's identity and the forward reads the payload off the live Parameter, so every+call sees the onloaded copy. Kill switch ``UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED=0``."""from __future__ import annotations@@ -30,6 +37,8 @@ from functools import lru_cachefrom typing import Any, OptionalINT8_GEMM_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM"+INT8_GEMM_CONVROT_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT"+INT8_GEMM_STREAMED_ENV = "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED"_MIN_TRITON = (3, 2)# torch._int_mm needs M > 16; below that the stock path (safe_int_mm padding) stays in charge._MIN_ROWS = 17@@ -72,6 +81,16 @@ def int8_gemm_mode() -> str:return "auto"+def convrot_enabled() -> bool:+ raw = (os.environ.get(INT8_GEMM_CONVROT_ENV) or "").strip().lower()+ return raw not in ("0", "off", "false", "no")+++def streamed_enabled() -> bool:+ raw = (os.environ.get(INT8_GEMM_STREAMED_ENV) or "").strip().lower()+ return raw not in ("0", "off", "false", "no")++def _triton_version_ok(version: Optional[str] = None) -> bool:if version is None:try:@@ -263,7 +282,7 @@ def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:_CALLS[0] += 1a = a if a.stride(-1) == 1 else a.contiguous()cfg = _DEVICE_CFG.get(a.device.index)- if cfg is not None and w.stride(-1) == 1 and _aligned(a, w):+ if cfg is not None and w.stride(-1) == 1 and w.device == a.device and _aligned(a, w):try:return _launch(a, w, xs, ws, bias, cfg)except Exception: # noqa: BLE001 - a failed launch keeps the stock math@@ -507,7 +526,7 @@ def _linear_forward(self: Any, x: Any) -> Any:rec = self.__dict__.get(_REC)if rec is None or x.dtype != torch.bfloat16 or not x.is_cuda:return type(self).forward(self, x)- kind, _wq, _ws, weight = rec+ kind, group, _ws, weight = recif self.weight is not weight: # weight replaced since install (reload / LoRA bake): stockreturn type(self).forward(self, x)# Payload off the live parameter, never a cached alias: a moved weight must not leave a stale device or pin a copy.@@ -519,9 +538,14 @@ def _linear_forward(self: Any, x: Any) -> Any:if wq.device != x.device:return type(self).forward(self, x)lead = x.shape[:-1]- x2d = x.reshape(-1, x.shape[-1])- if x2d.shape[0] < _MIN_ROWS:+ if x.numel() < _MIN_ROWS * x.shape[-1]:return type(self).forward(self, x)+ if group is not None:+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation+ x = rotate_convrot_activation(+ x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group+ )+ x2d = x.reshape(-1, x.shape[-1])if kind == "v1":xq, xs = _act_quant_v1(x2d)else:@@ -553,11 +577,31 @@ def linear_from_q(q: Any, xs: Any, weight: Any, bias: Any) -> Optional[Any]:return _OP_HANDLE(q, parts[0], xs.reshape(-1), parts[1], bias)+def _rotation_group(module: Any) -> Optional[int]:+ """The ConvRot group the forward above can reproduce, else None."""+ try:+ from .diffusion_convrot import convrot_linear_class, is_power_of_four++ if type(module) is not convrot_linear_class():+ return None+ group = module.convrot_groupsize+ if not is_power_of_four(group) or module.in_features % group:+ return None+ return int(group)+ except Exception: # noqa: BLE001+ return None++def _eligible(module: Any) -> Optional[tuple]:+ """(kind, rotation group or None, scale, weight) for a Linear the fused GEMM can run, else None."""from torch import nn- if type(module) is not nn.Linear: # a subclass (ConvRotLinear) transforms the input first- return None+ group = None+ if type(module) is not nn.Linear:+ # any other subclass transforms its input in a way this forward would skip+ group = _rotation_group(module) if convrot_enabled() else None+ if group is None:+ return Nonew = module.weightbias = module.biasimport torch@@ -570,10 +614,10 @@ def _eligible(module: Any) -> Optional[tuple]:return Noneparts = _v1_parts(w)if parts is not None and parts[1].dtype == torch.bfloat16:- return ("v1",) + parts + (w,)+ return ("v1", group, parts[1], w)parts = _v2_parts(w)if parts is not None and parts[1].dtype in (torch.bfloat16, torch.float32):- return ("v2",) + parts + (w,)+ return ("v2", group, parts[1], w)return None@@ -588,9 +632,19 @@ def install(transformer: Any,logger: Any = None,offload_active: bool = False,+ device: Any = None,) -> int:- """Idempotent; returns the (candidate) count. Must run before the first compiled forward."""- if int8_gemm_mode() == "off" or transformer is None or offload_active:+ """Idempotent; returns the (candidate) count. Must run before the first compiled forward.++ ``device``: the onload device of a block-streamed denoiser, whose weights sit on the host between blocks; the+ probe runs there and the swap happens now. Without it an offloaded denoiser keeps the stock path."""+ if int8_gemm_mode() == "off" or transformer is None:+ return 0+ if device is not None:+ if not streamed_enabled():+ return 0+ return _finalize(transformer, logger, device = device)+ if offload_active:return 0from .diffusion_int8_fused import resident_cuda_device, run_on_first_call@@ -614,8 +668,12 @@ def install(return n-def _finalize(transformer: Any, logger: Any = None) -> int:- count = _swap(transformer, logger)+def _finalize(+ transformer: Any,+ logger: Any = None,+ device: Any = None,+) -> int:+ count = _swap(transformer, logger, device = device)try:transformer._unsloth_int8_gemm = count # the deferred install recorded the candidate countexcept Exception: # noqa: BLE001@@ -623,14 +681,31 @@ def _finalize(transformer: Any, logger: Any = None) -> int:return count-def _swap(transformer: Any, logger: Any = None) -> int:+def _swap(+ transformer: Any,+ logger: Any = None,+ device: Any = None,+) -> int:global _OP_HANDLEfrom .diffusion_int8_fused import resident_cuda_device- dev = resident_cuda_device(transformer)+ import torch++ if device is not None:+ try:+ dev = torch.device(device)+ except Exception: # noqa: BLE001+ return 0+ if (+ dev.type != "cuda"+ or getattr(torch.version, "hip", None)+ or not torch.cuda.is_available()+ ):+ return 0+ else:+ dev = resident_cuda_device(transformer)if dev is None:return 0- import torchindex = dev.index if dev.index is not None else torch.cuda.current_device()recs = [(m, _eligible(m)) for m in transformer.modules()]@@ -650,10 +725,10 @@ def _swap(transformer: Any, logger: Any = None) -> int:continuemodule.__dict__[_REC] = (rec[0],- None,+ rec[1],None,rec[3],- ) # kind + the Parameter, no payload alias+ ) # kind, rotation, the Parameter; no payload aliasmodule.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)module.forward = types.MethodType(_linear_forward, module)count += 1diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex d4acd2aec1e..95a6716264f 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -7471,11 +7471,22 @@ class VideoBackend:# The conditioner and the VAEs stay in the rotation even when the denoiser is pinned, so the onload# hooks are live and fullgraph has to drop.offload_active = offload_policy != "none",+ # a denoiser pinned under the others' offload rotation never moves, so the fused int8 GEMM can engage+ denoiser_offloaded = bool(denoiser_streamed) or not denoiser_pinned,cuda_graph_default = False,logger = logger,)speed_optims = tuple(k for k, v in applied.items() if v) + h3_attn_leversif denoiser_streamed and "compiled" in speed_optims:+ # before the first forward traces the blocks below their hooks+ try:+ from .diffusion_int8_gemm import install as install_int8_gemm+ if install_int8_gemm(+ getattr(pipe, denoiser_component, None), logger, device = device+ ):+ speed_optims += ("int8_gemm",)+ except Exception as exc: # noqa: BLE001 -- optimisation only, the stock GEMM stays+ logger.warning("video.h3_int8_gemm: streamed install skipped: %s", exc)from .video_minimax_h3_residency import compile_blocks_below_offload_hookscompile_blocks_below_offload_hooks(getattr(pipe, denoiser_component, None), logger = loggerdiff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.pyindex 93b03ec3cff..82e7c45a12a 100644--- a/studio/backend/tests/test_diffusion_int8_gemm.py+++ b/studio/backend/tests/test_diffusion_int8_gemm.py@@ -393,3 +393,127 @@ def test_status_drops_int8_gemm_once_the_deferred_probe_swapped_nothing():dit._unsloth_int8_gemm = 0assert int8_gemm_live(pipe, ("compiled", "int8_gemm")) == ["compiled"]assert int8_gemm_live(pipe, ("compiled",)) == ["compiled"]+++def _convrot(lin, group = 256):+ from core.inference.diffusion_convrot import _install_rotation+ _install_rotation(lin, group)+ return lin+++def test_convrot_linear_is_eligible_with_its_group(monkeypatch):+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ scale = torch.ones(256, dtype = torch.bfloat16)+ monkeypatch.setattr(g8, "_v1_parts", lambda w: (torch.zeros(256, 512, dtype = torch.int8), scale))+ lin = _convrot(torch.nn.Linear(512, 256, bias = False))+ rec = g8._eligible(lin)+ assert rec is not None and rec[:2] == ("v1", 256) and rec[3] is lin.weight+ monkeypatch.setenv(g8.INT8_GEMM_CONVROT_ENV, "0")+ assert g8._eligible(lin) is None+ assert g8._eligible(torch.nn.Linear(512, 256, bias = False))[:2] == ("v1", None)+++def test_other_linear_subclasses_and_bad_groups_stay_stock(monkeypatch):+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ monkeypatch.setattr(+ g8,+ "_v1_parts",+ lambda w: (torch.zeros(256, 512, dtype = torch.int8), torch.ones(256, dtype = torch.bfloat16)),+ )++ class Other(torch.nn.Linear):+ pass++ assert g8._eligible(Other(512, 256, bias = False)) is None+ lin = _convrot(torch.nn.Linear(512, 256, bias = False))+ lin.convrot_groupsize = 1024 # does not divide in_features: the forward could not rotate it+ assert g8._eligible(lin) is None+++@needs_cuda[email protected]("version", [None, 2])+def test_convrot_linear_swap_is_bit_identical_eager_and_compiled(forced, monkeypatch, version):+ """MiniMax-H3's rotated int8 Linear (and its fused QKV): rotation, act quant, fused GEMM == ConvRotLinear.forward."""+ from torch._dynamo.utils import counters++ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ stock = _convrot(_int8_linear(1024, 768, False, version))+ fused = _convrot(_int8_linear(1024, 768, False, version))+ holder = torch.nn.Sequential(fused)+ assert g8.install(holder) == 1 and g8.is_installed(fused)+ x = torch.randn(2, 300, 1024, device = "cuda", dtype = torch.bfloat16) * 3+ x[0, 7, :5] *= 200 # an outlier row, what the rotation exists for+ with torch.inference_mode():+ before = g8.call_count()+ assert torch.equal(fused(x), stock(x))+ assert g8.call_count() == before + 1+ counters.clear()+ torch._dynamo.reset()+ with torch._inductor.config.patch(emulate_precision_casts = True):+ out = torch.compile(fused, fullgraph = True)(x)+ assert not counters["graph_break"]+ assert torch.equal(out, torch.compile(stock, fullgraph = True)(x))+ g8.uninstall(holder)+ assert not g8.is_installed(fused)+++@needs_cuda+def test_convrot_kill_switch_keeps_rotated_linears_stock(forced, monkeypatch):+ monkeypatch.setenv(g8.INT8_GEMM_CONVROT_ENV, "0")+ rotated = _convrot(_int8_linear(1024, 768, False, None))+ plain = _int8_linear(1024, 768, False, None)+ holder = torch.nn.Sequential(rotated, plain)+ assert g8.install(holder) == 1+ assert g8.is_installed(plain) and not g8.is_installed(rotated)+++@needs_cuda[email protected]("use_stream, version", [(False, None), (False, 2), (True, 2)])+def test_block_streamed_denoiser_installs_against_its_onload_device(+ forced, monkeypatch, use_stream, version+):+ """Group-offloaded int8 blocks (H3 on 40 GB): installed while on the host, the fused GEMM follows each onload."""+ hooks = pytest.importorskip("diffusers.hooks")+ monkeypatch.delenv(g8.INT8_GEMM_CONVROT_ENV, raising = False)+ monkeypatch.delenv(g8.INT8_GEMM_STREAMED_ENV, raising = False)++ class Holder(torch.nn.Module):+ def __init__(self, lin):+ super().__init__()+ self.blocks = torch.nn.ModuleList([torch.nn.Sequential(lin)])++ def forward(self, x):+ return self.blocks[0](x)++ stock = _convrot(_int8_linear(1024, 768, False, version))+ fused = _convrot(_int8_linear(1024, 768, False, version))+ holder = Holder(fused).requires_grad_(False).cpu()+ hooks.apply_group_offloading(+ holder,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = use_stream,+ )+ assert fused.weight.device.type == "cpu"+ assert (+ g8.install(holder, offload_active = True) == 0+ ) # no onload device: an offloaded denoiser stays stock+ assert g8.install(holder, device = "cuda") == 1 and g8.is_installed(fused)+ x = torch.randn(300, 1024, device = "cuda", dtype = torch.bfloat16) * 3+ with torch.no_grad(): # group offload's swap_tensors onload cannot run on inference tensors+ before = g8.call_count()+ out = holder(x)+ assert g8.call_count() == before + 1+ assert torch.equal(out, stock(x))+ assert torch.equal(holder(x), out)+ if not use_stream: # the stream path keeps the last group resident until the next onload+ assert fused.weight.device.type == "cpu"+++@needs_cuda+def test_streamed_kill_switch(forced, monkeypatch):+ monkeypatch.setenv(g8.INT8_GEMM_STREAMED_ENV, "0")+ holder = torch.nn.Sequential(_int8_linear(1024, 768, False, None))+ assert g8.install(holder, device = "cuda") == 0 and not g8.is_installed(holder[0])diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 71a5c000ef9..a3707ee95b9 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -2355,3 +2355,34 @@ def test_family_compiles_regionally_closes_the_dynamo_import_window_first(monkeyfam = types.SimpleNamespace(transformer_class = "Lumina2Transformer2DModel")assert family_compiles_regionally(fam) is Trueassert events[:2] == ["guard", "probe"]+++ "offload_active, denoiser_offloaded, expected",+ [(True, False, False), (True, True, True), (True, None, True), (False, None, False)],+)+def test_int8_gemm_install_follows_the_denoiser_placement(+ monkeypatch, offload_active, denoiser_offloaded, expected+):+ """Only a moving denoiser keeps the stock GEMM; one pinned under the others' offload rotation takes the fused one."""+ from core.inference import diffusion_int8_gemm, diffusion_speed as ds_mod++ seen = []+ monkeypatch.setattr(+ diffusion_int8_gemm,+ "install",+ lambda t, logger = None, offload_active = False: seen.append(offload_active) or 0,+ )++ class _DiT:+ def compile_repeated_blocks(self, **kwargs):+ return None++ monkeypatch.setattr(ds_mod, "_denoiser_dits", lambda pipe: [_DiT()])+ ds_mod._compile_repeated_blocks(+ types.SimpleNamespace(),+ None,+ offload_active = offload_active,+ denoiser_offloaded = denoiser_offloaded,+ )+ assert seen == [expected]diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 9b5635c2ccb..3bdede35194 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -9621,6 +9621,53 @@ def test_every_rebuilt_speed_target_carries_the_backend():assert "backend" in fields, f"video.py:{call.lineno} target lacks backend: {sorted(fields)}"+def test_h3_speed_optims_get_the_denoisers_own_placement():+ """A denoiser pinned under H3's offload policy must not read as offloaded."""+ import ast+ from pathlib import Path++ source = Path(__file__).resolve().parents[1] / "core" / "inference" / "video.py"+ text = source.read_text(encoding = "utf-8")+ tree = ast.parse(text)+ h3 = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "_load_h3_modular_pipeline"+ )+ calls = [+ n+ for n in ast.walk(h3)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "apply_speed_optims"+ ]+ assert len(calls) == 1+ kw = {k.arg: k.value for k in calls[0].keywords}+ assert "denoiser_offloaded" in kw+ expr = ast.unparse(kw["denoiser_offloaded"])+ names = {n.id for n in ast.walk(kw["denoiser_offloaded"]) if isinstance(n, ast.Name)}+ assert (+ {"denoiser_pinned", "denoiser_streamed"}+ <= names+ <= {"denoiser_pinned", "denoiser_streamed", "bool"}+ ), expr+ for pinned, streamed, offloaded in (+ (True, None, False),+ (True, "group", True),+ (False, None, True),+ ):+ got = eval(+ compile(ast.Expression(kw["denoiser_offloaded"]), "<h3>", "eval"),+ {"bool": bool},+ {"denoiser_pinned": pinned, "denoiser_streamed": streamed},+ )+ assert bool(got) is offloaded, (expr, pinned, streamed)+ installs = [+ n+ for n in ast.walk(h3)+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "install_int8_gemm"+ ]+ assert len(installs) == 1 and "device" in {k.arg for k in installs[0].keywords}++class _GraphHandle:"""Stands in for a captured denoiser graph: only reset() matters to generate()."""