Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: feed the SwiGLU halves straight into the down projection's int8 activation quant (L4 -0.85% s/step, bit-identical) (#17422)

Daniel Han150d223e79f7
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 83c601ffc9f..d490cfc965b 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -12753,6 +12753,16 @@ class VideoBackend:
compile_blocks_below_offload_hooks(
getattr(pipe, denoiser_component, None), logger = logger
)
+ if "int8_gemm" in speed_optims:
+ # before the first forward traces the blocks
+ try:
+ from .video_minimax_h3_ffn import install as install_h3_ffn
+ if install_h3_ffn(
+ getattr(pipe, denoiser_component, None), device = device, logger = logger
+ ):
+ speed_optims += ("h3_fused_swiglu",)
+ except Exception as exc: # noqa: BLE001 -- optimisation only, the stock MLP stays
+ logger.warning("video.h3_ffn: skipped: %s", exc)
if "h3_strided_attn" in speed_optims:
from .video_minimax_h3_attn import shared_qkv_count
shared_qkv = shared_qkv_count(getattr(pipe, denoiser_component, None))
diff --git a/studio/backend/core/inference/video_minimax_h3_ffn.py b/studio/backend/core/inference/video_minimax_h3_ffn.py
new file mode 100644
index 00000000000..63110fec00a
--- /dev/null
+++ b/studio/backend/core/inference/video_minimax_h3_ffn.py
@@ -0,0 +1,158 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 SwiGLU MLP: the int8 down projection reads value / gate halves straight from the input projection
+(``rotq_h16x``); anything ineligible per call runs the stock forward. Kill switch ``UNSLOTH_H3_FUSED_SWIGLU=0``."""
+
+from __future__ import annotations
+
+import os
+import types
+from typing import Any
+
+FUSED_SWIGLU_ENV = "UNSLOTH_H3_FUSED_SWIGLU"
+_MARK = "_unsloth_h3_fused_swiglu"
+_FIRST_CALL_KEY = "h3_fused_swiglu"
+# Engagement census, eager calls only (a traced forward counts nothing).
+_CALLS = [0]
+# Traced gates compare classes, never functions.
+_CLS: dict = {}
+
+
+def fused_swiglu_enabled() -> bool:
+ return (os.environ.get(FUSED_SWIGLU_ENV) or "1").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _classes() -> bool:
+ if not _CLS:
+ try:
+ from diffusers.models.activations import SwiGLU
+ from diffusers.models.attention import FeedForward
+ from torch import nn
+ except Exception: # noqa: BLE001 - diffusers without these
+ return False
+ _CLS.update(ff = FeedForward, swiglu = SwiGLU, dropout = nn.Dropout)
+ return True
+
+
+def _eligible(module: Any) -> bool:
+ if not _classes() or type(module) is not _CLS["ff"]:
+ return False
+ net = getattr(module, "net", None)
+ if net is None or len(net) < 3 or type(net[0]) is not _CLS["swiglu"]:
+ return False
+ return all(type(extra) is _CLS["dropout"] for extra in (net[1], *list(net)[3:]))
+
+
+def _ready(ff: Any) -> bool:
+ """Per call: the fused path skips ``SwiGLU.forward``, the Dropouts and the down projection's own forward."""
+ from . import diffusion_int8_gemm as gemm
+
+ net = ff.net
+ act = net[0]
+ if gemm._hooked(act) or not gemm._plain_silu(getattr(act, "activation", None)):
+ return False
+ for drop in (net[1], *list(net)[3:]):
+ if drop.training and drop.p > 0:
+ return False
+ if gemm._hooked(drop):
+ return False
+ rec = gemm._swapped_rec(net[2])
+ return rec is not None and bool(rec[2]) and rec[1] == 256
+
+
+def _forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:
+ stock = type(self).forward
+ if args or kwargs or not _ready(self):
+ return stock(self, hidden_states, *args, **kwargs)
+ from . import diffusion_int8_gemm as gemm
+
+ net = self.net
+ act = net[0]
+ y = act.proj(hidden_states)
+ half = y.shape[-1] // 2
+ # SwiGLU.forward: value, gate = proj(x).chunk(2) -> value * silu(gate)
+ out = gemm.producer_linear(net[2], None, y[..., half:], y[..., :half])
+ if out is None:
+ out = net[2](net[1](y[..., :half] * act.activation(y[..., half:])))
+ else:
+ import torch
+ if not torch.compiler.is_compiling():
+ _CALLS[0] += 1
+ for extra in list(net)[3:]:
+ out = extra(out)
+ return out
+
+
+def _arm(index: int) -> bool:
+ from .diffusion_convrot_quant import _H16_DEVICE, _arm_producer
+ return bool(_H16_DEVICE.get(index)) and _arm_producer(index)
+
+
+def install(
+ transformer: Any,
+ device: Any = None,
+ logger: Any = None,
+) -> int:
+ """Call after the int8 GEMM swap, before the first (compiled) forward. Returns the patched count."""
+ if transformer is None or not fused_swiglu_enabled():
+ return 0
+ modules = getattr(transformer, "modules", None)
+ if not callable(modules):
+ return 0
+ patched = 0
+ for module in modules():
+ if _MARK in module.__dict__:
+ patched += 1
+ continue
+ if not _eligible(module) or "forward" in module.__dict__:
+ continue
+ module.__dict__[_MARK] = True
+ module.forward = types.MethodType(_forward, module)
+ patched += 1
+ if not patched:
+ return 0
+ try:
+ import torch
+
+ if not torch.cuda.is_available():
+ return patched
+ dev = torch.device(device) if device is not None else torch.device("cuda")
+ if dev.type != "cuda":
+ return patched
+ index = dev.index if dev.index is not None else torch.cuda.current_device()
+ except Exception: # noqa: BLE001
+ return patched
+ armed = _arm(index)
+ if not armed:
+ from .diffusion_int8_fused import run_on_first_call
+ run_on_first_call(transformer, _FIRST_CALL_KEY, lambda _t: _arm(index))
+ if logger is not None:
+ logger.info(
+ "video.h3_ffn: %d SwiGLU MLP(s) feed the down projection's act quant from the projection halves "
+ "(card %d %s)",
+ patched,
+ index,
+ "armed" if armed else "arms at the first forward",
+ )
+ return patched
+
+
+def uninstall(transformer: Any) -> int:
+ restored = 0
+ for module in getattr(transformer, "modules", lambda: ())():
+ if module.__dict__.pop(_MARK, None) is not None:
+ fwd = module.__dict__.get("forward")
+ if getattr(fwd, "__func__", None) is _forward:
+ del module.__dict__["forward"]
+ restored += 1
+ return restored
+
+
+def call_count() -> int:
+ return _CALLS[0]
diff --git a/studio/backend/tests/test_h3_fused_swiglu.py b/studio/backend/tests/test_h3_fused_swiglu.py
new file mode 100644
index 00000000000..f8ba9f778e8
--- /dev/null
+++ b/studio/backend/tests/test_h3_fused_swiglu.py
@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 SwiGLU MLP feeding the down projection's act quant from the projection halves (video_minimax_h3_ffn).
+The int8 piece (``producer_linear``) is replaced by the plain modules, so the patched forward has to match the stock
+diffusers ``FeedForward`` bit for bit on CPU, eager and traced; the GPU kernel is checked by its own probe."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parents[1]
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+
+def env(monkeypatch):
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("diffusers")
+ from diffusers.models.attention import FeedForward
+
+ from core.inference import diffusion_int8_gemm as gemm
+ from core.inference import video_minimax_h3_ffn as ffn
+
+ monkeypatch.delenv(ffn.FUSED_SWIGLU_ENV, raising = False)
+ calls = {"producer": 0}
+
+ def producer(down, prefix, gate, value):
+ assert prefix is None
+ calls["producer"] += 1
+ calls["halves"] = (gate, value)
+ return down(value * torch.nn.functional.silu(gate))
+
+ monkeypatch.setattr(gemm, "producer_linear", producer)
+ downs = set()
+ # only the down projections registered here count as swapped rotated int8 Linears
+ monkeypatch.setattr(
+ gemm,
+ "_swapped_rec",
+ lambda m: ("v2", 256, True, m.weight, True) if id(m) in downs else None,
+ )
+
+ class Holder(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.ff = FeedForward(64, inner_dim = 96, activation_fn = "swiglu", bias = False)
+
+ def forward(self, x):
+ return self.ff(x) + x
+
+ def build(swapped = True):
+ torch.manual_seed(0)
+ h = Holder().eval()
+ if swapped:
+ downs.add(id(h.ff.net[2]))
+ return h
+
+ yield torch, ffn, build, calls
+
+
+def _x(torch):
+ return torch.randn(2, 33, 64, generator = torch.Generator().manual_seed(1))
+
+
[email protected]("mode", ["eager", "compiled"])
+def test_patched_mlp_matches_stock(env, mode):
+ torch, ffn, build, calls = env
+ h = build()
+ x = _x(torch)
+ with torch.no_grad():
+ ref = h(x)
+ assert ffn.install(h) == 1
+ assert "forward" in h.ff.__dict__
+ if mode == "compiled":
+ torch._dynamo.reset()
+ out = torch.compile(h, backend = "eager", fullgraph = True)(x)
+ torch._dynamo.reset()
+ else:
+ out = h(x)
+ assert calls["producer"] >= 1
+ assert torch.equal(out, ref)
+ if mode == "eager":
+ gate, value = calls["halves"]
+ # halves of the one projection output, no copy: value first, gate second (diffusers' SwiGLU.chunk order)
+ assert gate.stride() == value.stride() and gate.stride(-2) == 2 * 96
+ assert gate.data_ptr() - value.data_ptr() == 96 * gate.element_size()
+ assert ffn.uninstall(h) == 1 and "forward" not in h.ff.__dict__
+ with torch.no_grad():
+ assert torch.equal(h(x), ref)
+
+
+def test_kill_switch_keeps_stock(env, monkeypatch):
+ torch, ffn, build, calls = env
+ monkeypatch.setenv(ffn.FUSED_SWIGLU_ENV, "0")
+ h = build()
+ assert ffn.install(h) == 0
+ assert "forward" not in h.ff.__dict__
+ with torch.no_grad():
+ h(_x(torch))
+ assert calls["producer"] == 0
+
+
+def test_per_call_gates_run_stock(env):
+ torch, ffn, build, calls = env
+ x = _x(torch)
+ h = build(swapped = False)
+ with torch.no_grad():
+ ref = h(x)
+ ffn.install(h)
+ assert torch.equal(h(x), ref)
+ assert calls["producer"] == 0
+ h = build()
+ h.ff.net[1].p = 0.5
+ h.ff.net[1].train()
+ ffn.install(h)
+ assert not ffn._ready(h.ff)
+ h = build()
+ with torch.no_grad():
+ ref = h(x)
+ h.ff.net[0].register_forward_hook(lambda m, a, o: o)
+ ffn.install(h)
+ with torch.no_grad():
+ assert torch.equal(h(x), ref)
+ assert calls["producer"] == 0
+
+
+def test_h3_blocks_are_eligible(env):
+ torch, ffn, _build, _calls = env
+ h3 = pytest.importorskip("diffusers.models.transformers.transformer_minimax_h3")
+ block_cls = getattr(h3, "MiniMaxH3TransformerBlock", None)
+ if block_cls is None:
+ pytest.skip("this diffusers has no MiniMaxH3TransformerBlock")
+ try:
+ block = block_cls(
+ hidden_size = 64,
+ num_attention_heads = 2,
+ attention_head_dim = 32,
+ ffn_dim = 96,
+ time_embed_dim = 32,
+ norm_eps = 1e-6,
+ qk_norm_eps = 1e-6,
+ )
+ except TypeError:
+ pytest.skip("MiniMaxH3TransformerBlock signature changed")
+ assert ffn._eligible(block.ff)
+ assert ffn.install(block) == 1
+
+
+def test_video_installs_it_before_the_first_forward():
+ src = (_BACKEND / "core" / "inference" / "video.py").read_text()
+ i = src.index("from .video_minimax_h3_ffn import install as install_h3_ffn")
+ assert src.index("compile_blocks_below_offload_hooks(", src.index("install_int8_gemm(")) < i
+ assert '"h3_fused_swiglu"' in src[i : i + 600]