Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MiniMax-H3: feed the SwiGLU halves straight into the down projection's int8 activation quant (L4 -0.85% s/step, bit-identical) (#17422)
Daniel Han150d223e79f7
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 83c601ffc9f..d490cfc965b 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -12753,6 +12753,16 @@ class VideoBackend:compile_blocks_below_offload_hooks(getattr(pipe, denoiser_component, None), logger = logger)+ if "int8_gemm" in speed_optims:+ # before the first forward traces the blocks+ try:+ from .video_minimax_h3_ffn import install as install_h3_ffn+ if install_h3_ffn(+ getattr(pipe, denoiser_component, None), device = device, logger = logger+ ):+ speed_optims += ("h3_fused_swiglu",)+ except Exception as exc: # noqa: BLE001 -- optimisation only, the stock MLP stays+ logger.warning("video.h3_ffn: skipped: %s", exc)if "h3_strided_attn" in speed_optims:from .video_minimax_h3_attn import shared_qkv_countshared_qkv = shared_qkv_count(getattr(pipe, denoiser_component, None))diff --git a/studio/backend/core/inference/video_minimax_h3_ffn.py b/studio/backend/core/inference/video_minimax_h3_ffn.pynew file mode 100644index 00000000000..63110fec00a--- /dev/null+++ b/studio/backend/core/inference/video_minimax_h3_ffn.py@@ -0,0 +1,158 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 SwiGLU MLP: the int8 down projection reads value / gate halves straight from the input projection+(``rotq_h16x``); anything ineligible per call runs the stock forward. Kill switch ``UNSLOTH_H3_FUSED_SWIGLU=0``."""++from __future__ import annotations++import os+import types+from typing import Any++FUSED_SWIGLU_ENV = "UNSLOTH_H3_FUSED_SWIGLU"+_MARK = "_unsloth_h3_fused_swiglu"+_FIRST_CALL_KEY = "h3_fused_swiglu"+# Engagement census, eager calls only (a traced forward counts nothing).+_CALLS = [0]+# Traced gates compare classes, never functions.+_CLS: dict = {}+++def fused_swiglu_enabled() -> bool:+ return (os.environ.get(FUSED_SWIGLU_ENV) or "1").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _classes() -> bool:+ if not _CLS:+ try:+ from diffusers.models.activations import SwiGLU+ from diffusers.models.attention import FeedForward+ from torch import nn+ except Exception: # noqa: BLE001 - diffusers without these+ return False+ _CLS.update(ff = FeedForward, swiglu = SwiGLU, dropout = nn.Dropout)+ return True+++def _eligible(module: Any) -> bool:+ if not _classes() or type(module) is not _CLS["ff"]:+ return False+ net = getattr(module, "net", None)+ if net is None or len(net) < 3 or type(net[0]) is not _CLS["swiglu"]:+ return False+ return all(type(extra) is _CLS["dropout"] for extra in (net[1], *list(net)[3:]))+++def _ready(ff: Any) -> bool:+ """Per call: the fused path skips ``SwiGLU.forward``, the Dropouts and the down projection's own forward."""+ from . import diffusion_int8_gemm as gemm++ net = ff.net+ act = net[0]+ if gemm._hooked(act) or not gemm._plain_silu(getattr(act, "activation", None)):+ return False+ for drop in (net[1], *list(net)[3:]):+ if drop.training and drop.p > 0:+ return False+ if gemm._hooked(drop):+ return False+ rec = gemm._swapped_rec(net[2])+ return rec is not None and bool(rec[2]) and rec[1] == 256+++def _forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any:+ stock = type(self).forward+ if args or kwargs or not _ready(self):+ return stock(self, hidden_states, *args, **kwargs)+ from . import diffusion_int8_gemm as gemm++ net = self.net+ act = net[0]+ y = act.proj(hidden_states)+ half = y.shape[-1] // 2+ # SwiGLU.forward: value, gate = proj(x).chunk(2) -> value * silu(gate)+ out = gemm.producer_linear(net[2], None, y[..., half:], y[..., :half])+ if out is None:+ out = net[2](net[1](y[..., :half] * act.activation(y[..., half:])))+ else:+ import torch+ if not torch.compiler.is_compiling():+ _CALLS[0] += 1+ for extra in list(net)[3:]:+ out = extra(out)+ return out+++def _arm(index: int) -> bool:+ from .diffusion_convrot_quant import _H16_DEVICE, _arm_producer+ return bool(_H16_DEVICE.get(index)) and _arm_producer(index)+++def install(+ transformer: Any,+ device: Any = None,+ logger: Any = None,+) -> int:+ """Call after the int8 GEMM swap, before the first (compiled) forward. Returns the patched count."""+ if transformer is None or not fused_swiglu_enabled():+ return 0+ modules = getattr(transformer, "modules", None)+ if not callable(modules):+ return 0+ patched = 0+ for module in modules():+ if _MARK in module.__dict__:+ patched += 1+ continue+ if not _eligible(module) or "forward" in module.__dict__:+ continue+ module.__dict__[_MARK] = True+ module.forward = types.MethodType(_forward, module)+ patched += 1+ if not patched:+ return 0+ try:+ import torch++ if not torch.cuda.is_available():+ return patched+ dev = torch.device(device) if device is not None else torch.device("cuda")+ if dev.type != "cuda":+ return patched+ index = dev.index if dev.index is not None else torch.cuda.current_device()+ except Exception: # noqa: BLE001+ return patched+ armed = _arm(index)+ if not armed:+ from .diffusion_int8_fused import run_on_first_call+ run_on_first_call(transformer, _FIRST_CALL_KEY, lambda _t: _arm(index))+ if logger is not None:+ logger.info(+ "video.h3_ffn: %d SwiGLU MLP(s) feed the down projection's act quant from the projection halves "+ "(card %d %s)",+ patched,+ index,+ "armed" if armed else "arms at the first forward",+ )+ return patched+++def uninstall(transformer: Any) -> int:+ restored = 0+ for module in getattr(transformer, "modules", lambda: ())():+ if module.__dict__.pop(_MARK, None) is not None:+ fwd = module.__dict__.get("forward")+ if getattr(fwd, "__func__", None) is _forward:+ del module.__dict__["forward"]+ restored += 1+ return restored+++def call_count() -> int:+ return _CALLS[0]diff --git a/studio/backend/tests/test_h3_fused_swiglu.py b/studio/backend/tests/test_h3_fused_swiglu.pynew file mode 100644index 00000000000..f8ba9f778e8--- /dev/null+++ b/studio/backend/tests/test_h3_fused_swiglu.py@@ -0,0 +1,157 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 SwiGLU MLP feeding the down projection's act quant from the projection halves (video_minimax_h3_ffn).+The int8 piece (``producer_linear``) is replaced by the plain modules, so the patched forward has to match the stock+diffusers ``FeedForward`` bit for bit on CPU, eager and traced; the GPU kernel is checked by its own probe."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parents[1]+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))+++def env(monkeypatch):+ torch = pytest.importorskip("torch")+ pytest.importorskip("diffusers")+ from diffusers.models.attention import FeedForward++ from core.inference import diffusion_int8_gemm as gemm+ from core.inference import video_minimax_h3_ffn as ffn++ monkeypatch.delenv(ffn.FUSED_SWIGLU_ENV, raising = False)+ calls = {"producer": 0}++ def producer(down, prefix, gate, value):+ assert prefix is None+ calls["producer"] += 1+ calls["halves"] = (gate, value)+ return down(value * torch.nn.functional.silu(gate))++ monkeypatch.setattr(gemm, "producer_linear", producer)+ downs = set()+ # only the down projections registered here count as swapped rotated int8 Linears+ monkeypatch.setattr(+ gemm,+ "_swapped_rec",+ lambda m: ("v2", 256, True, m.weight, True) if id(m) in downs else None,+ )++ class Holder(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.ff = FeedForward(64, inner_dim = 96, activation_fn = "swiglu", bias = False)++ def forward(self, x):+ return self.ff(x) + x++ def build(swapped = True):+ torch.manual_seed(0)+ h = Holder().eval()+ if swapped:+ downs.add(id(h.ff.net[2]))+ return h++ yield torch, ffn, build, calls+++def _x(torch):+ return torch.randn(2, 33, 64, generator = torch.Generator().manual_seed(1))++[email protected]("mode", ["eager", "compiled"])+def test_patched_mlp_matches_stock(env, mode):+ torch, ffn, build, calls = env+ h = build()+ x = _x(torch)+ with torch.no_grad():+ ref = h(x)+ assert ffn.install(h) == 1+ assert "forward" in h.ff.__dict__+ if mode == "compiled":+ torch._dynamo.reset()+ out = torch.compile(h, backend = "eager", fullgraph = True)(x)+ torch._dynamo.reset()+ else:+ out = h(x)+ assert calls["producer"] >= 1+ assert torch.equal(out, ref)+ if mode == "eager":+ gate, value = calls["halves"]+ # halves of the one projection output, no copy: value first, gate second (diffusers' SwiGLU.chunk order)+ assert gate.stride() == value.stride() and gate.stride(-2) == 2 * 96+ assert gate.data_ptr() - value.data_ptr() == 96 * gate.element_size()+ assert ffn.uninstall(h) == 1 and "forward" not in h.ff.__dict__+ with torch.no_grad():+ assert torch.equal(h(x), ref)+++def test_kill_switch_keeps_stock(env, monkeypatch):+ torch, ffn, build, calls = env+ monkeypatch.setenv(ffn.FUSED_SWIGLU_ENV, "0")+ h = build()+ assert ffn.install(h) == 0+ assert "forward" not in h.ff.__dict__+ with torch.no_grad():+ h(_x(torch))+ assert calls["producer"] == 0+++def test_per_call_gates_run_stock(env):+ torch, ffn, build, calls = env+ x = _x(torch)+ h = build(swapped = False)+ with torch.no_grad():+ ref = h(x)+ ffn.install(h)+ assert torch.equal(h(x), ref)+ assert calls["producer"] == 0+ h = build()+ h.ff.net[1].p = 0.5+ h.ff.net[1].train()+ ffn.install(h)+ assert not ffn._ready(h.ff)+ h = build()+ with torch.no_grad():+ ref = h(x)+ h.ff.net[0].register_forward_hook(lambda m, a, o: o)+ ffn.install(h)+ with torch.no_grad():+ assert torch.equal(h(x), ref)+ assert calls["producer"] == 0+++def test_h3_blocks_are_eligible(env):+ torch, ffn, _build, _calls = env+ h3 = pytest.importorskip("diffusers.models.transformers.transformer_minimax_h3")+ block_cls = getattr(h3, "MiniMaxH3TransformerBlock", None)+ if block_cls is None:+ pytest.skip("this diffusers has no MiniMaxH3TransformerBlock")+ try:+ block = block_cls(+ hidden_size = 64,+ num_attention_heads = 2,+ attention_head_dim = 32,+ ffn_dim = 96,+ time_embed_dim = 32,+ norm_eps = 1e-6,+ qk_norm_eps = 1e-6,+ )+ except TypeError:+ pytest.skip("MiniMaxH3TransformerBlock signature changed")+ assert ffn._eligible(block.ff)+ assert ffn.install(block) == 1+++def test_video_installs_it_before_the_first_forward():+ src = (_BACKEND / "core" / "inference" / "video.py").read_text()+ i = src.index("from .video_minimax_h3_ffn import install as install_h3_ffn")+ assert src.index("compile_blocks_below_offload_hooks(", src.index("install_int8_gemm(")) < i+ assert '"h3_fused_swiglu"' in src[i : i + 600]