Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: run the MiniMax-H3 int8 video decoder on the fused ConvRot act quant and int8 GEMM (#16314)

Daniel Han163267ff6eee
diff --git a/studio/backend/core/inference/video_minimax_h3_vae.py b/studio/backend/core/inference/video_minimax_h3_vae.py
index fbb13e3dac2..c48f3f1a4ad 100644
--- a/studio/backend/core/inference/video_minimax_h3_vae.py
+++ b/studio/backend/core/inference/video_minimax_h3_vae.py
@@ -18,6 +18,7 @@ from .rocm_bf16 import is_rocm_torch
H3_VAE_FAST_ENV = "UNSLOTH_H3_VAE_FAST"
H3_VAE_INT8_ENV = "UNSLOTH_H3_VAE_INT8"
H3_VAE_TILE_BATCH_ENV = "UNSLOTH_H3_VAE_TILE_BATCH"
+H3_VAE_INT8_FUSED_ENV = "UNSLOTH_H3_VAE_INT8_FUSED"
FP16_ACCUM_DISABLE_ENV = "UNSLOTH_DISABLE_FP16_ACCUM"
LEVER_FUSED_ENCODER = "fused_encoder"
@@ -1053,6 +1054,58 @@ def _install_tile_batch(vae: Any) -> bool:
return True
+_FUSED_DEVICES: dict = {}
+
+
+def _fused_int8_ready(index: Optional[int]) -> bool:
+ """Probe once per card, never inside a forward."""
+ if index is None or _env(H3_VAE_INT8_FUSED_ENV) in _FALSE:
+ return False
+ if index not in _FUSED_DEVICES:
+ ok = False
+ try:
+ from . import diffusion_int8_gemm as gemm
+ from .diffusion_convrot_quant import rotquant_device_config
+ ok = (
+ gemm.int8_gemm_mode() != "off"
+ and gemm._op() is not None
+ and gemm.device_config(index) is not None
+ and rotquant_device_config(index) is not None
+ )
+ except Exception: # noqa: BLE001 - the unfused path stays
+ ok = False
+ _FUSED_DEVICES[index] = ok
+ return _FUSED_DEVICES[index]
+
+
+def _fused_int8_linear(
+ linear: Any, x2: Any, wq: Any, ws: Any, out_dtype: Any, group: int
+) -> Optional[Any]:
+ """ConvRot rotation + int8 act quant, then int8 GEMM with fused dequant. None when unsupported."""
+ import torch
+
+ bias = linear.bias
+ if (
+ _env(H3_VAE_INT8_FUSED_ENV) in _FALSE
+ or out_dtype != torch.bfloat16
+ or x2.dtype != torch.bfloat16
+ or not _FUSED_DEVICES.get(x2.device.index)
+ or (bias is not None and bias.dtype != torch.bfloat16)
+ or x2.shape[1] % 64
+ or wq.shape[0] % 16
+ ):
+ return None
+ from . import diffusion_int8_gemm as gemm
+ from .diffusion_convrot_quant import convrot_act_quant
+
+ q = convrot_act_quant(x2, group, True)
+ op = gemm._op()
+ if q is None or op is None:
+ return None
+ xq, xs = q
+ return op(xq, wq, xs.reshape(-1), ws, bias)
+
+
def _int8_linear(linear: Any, x: Any) -> Any:
"""W8A8: per-token int8 activations, per-output-channel int8 weights, int32 accumulate via ``torch._int_mm``."""
import torch
@@ -1064,6 +1117,18 @@ def _int8_linear(linear: Any, x: Any) -> Any:
wq, ws = linear._unsloth_int8
out_dtype = torch.get_autocast_dtype("cuda") if torch.is_autocast_enabled("cuda") else x.dtype
rot = getattr(linear, "_unsloth_rot", None)
+ if (
+ rot is not None
+ and x.is_cuda
+ and x.shape[-1] >= rot.shape[0]
+ and x.numel() >= 17 * x.shape[-1]
+ ):
+ x2 = x.reshape(-1, x.shape[-1])
+ out = _fused_int8_linear(
+ linear, x2 if x2.is_contiguous() else x2.contiguous(), wq, ws, out_dtype, rot.shape[0]
+ )
+ if out is not None:
+ return out.view(*x.shape[:-1], out.shape[-1])
if rot is not None:
# ConvRot: weight stored as W @ H.T, so x @ H keeps the product while spreading outlier channels
g = rot.shape[0]
@@ -1153,6 +1218,15 @@ def _install_int8_decoder(vae: Any, *, keep_blocks: Optional[tuple] = None) -> i
wq, ws = _quantize_int8_weight(w, hadamard if rotate else None)
staged.append((linear, wq, ws, rotate))
cls = _int8_linear_class()
+ if hadamard is not None:
+ try:
+ if torch.cuda.is_available():
+ # weights' card, plus the current one for host-offloaded weights
+ if hadamard.is_cuda:
+ _fused_int8_ready(hadamard.device.index)
+ _fused_int8_ready(torch.cuda.current_device())
+ except Exception: # noqa: BLE001 - the unfused path stays
+ pass
freed = 0
for linear, wq, ws, rotate in staged:
if rotate:
diff --git a/studio/backend/tests/test_video_minimax_h3_vae.py b/studio/backend/tests/test_video_minimax_h3_vae.py
index eb522311827..e01c4b10875 100644
--- a/studio/backend/tests/test_video_minimax_h3_vae.py
+++ b/studio/backend/tests/test_video_minimax_h3_vae.py
@@ -1313,3 +1313,91 @@ def test_the_h3_load_reserves_the_fused_decoder_before_the_speed_optims():
for kw in ("speed_mode", "workflow"):
got = {k.arg: ast.dump(k.value) for k in reserve.keywords}[kw]
assert got == {k.arg: ast.dump(k.value) for k in apply_vae.keywords}[kw]
+
+
+def _rotated_bf16_linear(
+ k = 1024,
+ n = 512,
+ device = "cpu",
+):
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(k, n).to(device = device, dtype = torch.bfloat16)
+ ref = copy.deepcopy(lin)
+ holder = torch.nn.Module()
+ holder.transformer_blocks = torch.nn.ModuleList([torch.nn.ModuleDict({"l": lin})])
+ assert H._install_int8_decoder(types.SimpleNamespace(decoder = holder), keep_blocks = ()) > 0
+ assert lin._unsloth_rot.shape[0] == H.H3_VAE_INT8_ROT_GROUP
+ return lin, ref
+
+
+def test_the_fused_int8_path_hands_the_rotation_group_and_operands_to_the_dit_kernels(monkeypatch):
+ from core.inference import diffusion_convrot_quant as cq
+ from core.inference import diffusion_int8_gemm as gemm
+
+ lin, _ = _rotated_bf16_linear()
+ seen = {}
+
+ def act_quant(x2d, group, v2):
+ seen["quant"] = (tuple(x2d.shape), group, v2)
+ return torch.zeros(x2d.shape, dtype = torch.int8), torch.ones(x2d.shape[0])
+
+ def op(xq, wq, xs, ws, bias):
+ seen["gemm"] = (wq is lin._unsloth_wq, ws.dtype, bias is lin.bias, tuple(xs.shape))
+ return torch.full((xq.shape[0], wq.shape[0]), 2.0, dtype = torch.bfloat16)
+
+ monkeypatch.setattr(cq, "convrot_act_quant", act_quant)
+ monkeypatch.setattr(gemm, "_op", lambda: op)
+ monkeypatch.setitem(H._FUSED_DEVICES, None, True)
+ x = torch.randn(2, 20, 1024, dtype = torch.bfloat16)
+ wq, ws = lin._unsloth_int8
+ out = H._fused_int8_linear(lin, x.reshape(-1, 1024), wq, ws, torch.bfloat16, 256)
+ assert out is not None and out.shape == (40, 512) and bool((out == 2).all())
+ assert seen["quant"] == ((40, 1024), 256, True)
+ assert seen["gemm"] == (True, torch.float32, True, (40,))
+
+
[email protected]("case", ["fp16_out", "fp16_in", "not_probed", "fp32_bias", "ragged_k"])
+def test_the_fused_int8_path_declines_what_the_dit_kernels_do_not_cover(monkeypatch, case):
+ lin, _ = _rotated_bf16_linear()
+ wq, ws = lin._unsloth_int8
+ x2 = torch.randn(40, 1024, dtype = torch.bfloat16)
+ out_dtype = torch.bfloat16
+ monkeypatch.setitem(H._FUSED_DEVICES, None, case != "not_probed")
+ if case == "fp16_out":
+ out_dtype = torch.float16
+ elif case == "fp16_in":
+ x2 = x2.half()
+ elif case == "fp32_bias":
+ lin.bias = torch.nn.Parameter(lin.bias.float())
+ elif case == "ragged_k":
+ x2 = x2[:, :1000]
+ assert H._fused_int8_linear(lin, x2, wq, ws, out_dtype, 256) is None
+
+
+def test_the_fused_int8_kill_switch_keeps_the_unfused_path(monkeypatch):
+ monkeypatch.setenv(H.H3_VAE_INT8_FUSED_ENV, "0")
+ monkeypatch.delitem(H._FUSED_DEVICES, 7, raising = False)
+ assert H._fused_int8_ready(7) is False
+ assert 7 not in H._FUSED_DEVICES # the switch is read per call, never cached
+
+
+@needs_cuda
+def test_the_fused_int8_path_matches_the_unfused_int8_path_on_cuda(monkeypatch):
+ from core.inference import diffusion_int8_gemm as gemm
+
+ lin, ref = _rotated_bf16_linear(k = 2048, n = 1024, device = "cuda")
+ x = torch.randn(4, 1024, 2048, device = "cuda", dtype = torch.bfloat16)
+ x[..., :8] *= 30 # outlier channels: what the rotation is for
+ with torch.no_grad():
+ dense = ref(x).float()
+ monkeypatch.setenv(H.H3_VAE_INT8_FUSED_ENV, "0")
+ unfused = lin(x).float()
+ monkeypatch.delenv(H.H3_VAE_INT8_FUSED_ENV)
+ if not H._fused_int8_ready(torch.cuda.current_device()):
+ pytest.skip("the DiT's ConvRot act quant / fused GEMM is not armed on this card")
+ calls = gemm.call_count()
+ fused = lin(x).float()
+ assert gemm.call_count() > calls, "the fused-dequant GEMM did not run"
+ rel = lambda a, b: float((a - b).norm() / b.norm()) # noqa: E731
+ assert rel(unfused, dense) < 0.02 and rel(fused, dense) < 0.02
+ assert rel(fused, unfused) < 0.01