Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: run the MiniMax-H3 int8 video decoder on the fused ConvRot act quant and int8 GEMM (#16314)
Daniel Han163267ff6eee
diff --git a/studio/backend/core/inference/video_minimax_h3_vae.py b/studio/backend/core/inference/video_minimax_h3_vae.pyindex fbb13e3dac2..c48f3f1a4ad 100644--- a/studio/backend/core/inference/video_minimax_h3_vae.py+++ b/studio/backend/core/inference/video_minimax_h3_vae.py@@ -18,6 +18,7 @@ from .rocm_bf16 import is_rocm_torchH3_VAE_FAST_ENV = "UNSLOTH_H3_VAE_FAST"H3_VAE_INT8_ENV = "UNSLOTH_H3_VAE_INT8"H3_VAE_TILE_BATCH_ENV = "UNSLOTH_H3_VAE_TILE_BATCH"+H3_VAE_INT8_FUSED_ENV = "UNSLOTH_H3_VAE_INT8_FUSED"FP16_ACCUM_DISABLE_ENV = "UNSLOTH_DISABLE_FP16_ACCUM"LEVER_FUSED_ENCODER = "fused_encoder"@@ -1053,6 +1054,58 @@ def _install_tile_batch(vae: Any) -> bool:return True+_FUSED_DEVICES: dict = {}+++def _fused_int8_ready(index: Optional[int]) -> bool:+ """Probe once per card, never inside a forward."""+ if index is None or _env(H3_VAE_INT8_FUSED_ENV) in _FALSE:+ return False+ if index not in _FUSED_DEVICES:+ ok = False+ try:+ from . import diffusion_int8_gemm as gemm+ from .diffusion_convrot_quant import rotquant_device_config+ ok = (+ gemm.int8_gemm_mode() != "off"+ and gemm._op() is not None+ and gemm.device_config(index) is not None+ and rotquant_device_config(index) is not None+ )+ except Exception: # noqa: BLE001 - the unfused path stays+ ok = False+ _FUSED_DEVICES[index] = ok+ return _FUSED_DEVICES[index]+++def _fused_int8_linear(+ linear: Any, x2: Any, wq: Any, ws: Any, out_dtype: Any, group: int+) -> Optional[Any]:+ """ConvRot rotation + int8 act quant, then int8 GEMM with fused dequant. None when unsupported."""+ import torch++ bias = linear.bias+ if (+ _env(H3_VAE_INT8_FUSED_ENV) in _FALSE+ or out_dtype != torch.bfloat16+ or x2.dtype != torch.bfloat16+ or not _FUSED_DEVICES.get(x2.device.index)+ or (bias is not None and bias.dtype != torch.bfloat16)+ or x2.shape[1] % 64+ or wq.shape[0] % 16+ ):+ return None+ from . import diffusion_int8_gemm as gemm+ from .diffusion_convrot_quant import convrot_act_quant++ q = convrot_act_quant(x2, group, True)+ op = gemm._op()+ if q is None or op is None:+ return None+ xq, xs = q+ return op(xq, wq, xs.reshape(-1), ws, bias)++def _int8_linear(linear: Any, x: Any) -> Any:"""W8A8: per-token int8 activations, per-output-channel int8 weights, int32 accumulate via ``torch._int_mm``."""import torch@@ -1064,6 +1117,18 @@ def _int8_linear(linear: Any, x: Any) -> Any:wq, ws = linear._unsloth_int8out_dtype = torch.get_autocast_dtype("cuda") if torch.is_autocast_enabled("cuda") else x.dtyperot = getattr(linear, "_unsloth_rot", None)+ if (+ rot is not None+ and x.is_cuda+ and x.shape[-1] >= rot.shape[0]+ and x.numel() >= 17 * x.shape[-1]+ ):+ x2 = x.reshape(-1, x.shape[-1])+ out = _fused_int8_linear(+ linear, x2 if x2.is_contiguous() else x2.contiguous(), wq, ws, out_dtype, rot.shape[0]+ )+ if out is not None:+ return out.view(*x.shape[:-1], out.shape[-1])if rot is not None:# ConvRot: weight stored as W @ H.T, so x @ H keeps the product while spreading outlier channelsg = rot.shape[0]@@ -1153,6 +1218,15 @@ def _install_int8_decoder(vae: Any, *, keep_blocks: Optional[tuple] = None) -> iwq, ws = _quantize_int8_weight(w, hadamard if rotate else None)staged.append((linear, wq, ws, rotate))cls = _int8_linear_class()+ if hadamard is not None:+ try:+ if torch.cuda.is_available():+ # weights' card, plus the current one for host-offloaded weights+ if hadamard.is_cuda:+ _fused_int8_ready(hadamard.device.index)+ _fused_int8_ready(torch.cuda.current_device())+ except Exception: # noqa: BLE001 - the unfused path stays+ passfreed = 0for linear, wq, ws, rotate in staged:if rotate:diff --git a/studio/backend/tests/test_video_minimax_h3_vae.py b/studio/backend/tests/test_video_minimax_h3_vae.pyindex eb522311827..e01c4b10875 100644--- a/studio/backend/tests/test_video_minimax_h3_vae.py+++ b/studio/backend/tests/test_video_minimax_h3_vae.py@@ -1313,3 +1313,91 @@ def test_the_h3_load_reserves_the_fused_decoder_before_the_speed_optims():for kw in ("speed_mode", "workflow"):got = {k.arg: ast.dump(k.value) for k in reserve.keywords}[kw]assert got == {k.arg: ast.dump(k.value) for k in apply_vae.keywords}[kw]+++def _rotated_bf16_linear(+ k = 1024,+ n = 512,+ device = "cpu",+):+ torch.manual_seed(0)+ lin = torch.nn.Linear(k, n).to(device = device, dtype = torch.bfloat16)+ ref = copy.deepcopy(lin)+ holder = torch.nn.Module()+ holder.transformer_blocks = torch.nn.ModuleList([torch.nn.ModuleDict({"l": lin})])+ assert H._install_int8_decoder(types.SimpleNamespace(decoder = holder), keep_blocks = ()) > 0+ assert lin._unsloth_rot.shape[0] == H.H3_VAE_INT8_ROT_GROUP+ return lin, ref+++def test_the_fused_int8_path_hands_the_rotation_group_and_operands_to_the_dit_kernels(monkeypatch):+ from core.inference import diffusion_convrot_quant as cq+ from core.inference import diffusion_int8_gemm as gemm++ lin, _ = _rotated_bf16_linear()+ seen = {}++ def act_quant(x2d, group, v2):+ seen["quant"] = (tuple(x2d.shape), group, v2)+ return torch.zeros(x2d.shape, dtype = torch.int8), torch.ones(x2d.shape[0])++ def op(xq, wq, xs, ws, bias):+ seen["gemm"] = (wq is lin._unsloth_wq, ws.dtype, bias is lin.bias, tuple(xs.shape))+ return torch.full((xq.shape[0], wq.shape[0]), 2.0, dtype = torch.bfloat16)++ monkeypatch.setattr(cq, "convrot_act_quant", act_quant)+ monkeypatch.setattr(gemm, "_op", lambda: op)+ monkeypatch.setitem(H._FUSED_DEVICES, None, True)+ x = torch.randn(2, 20, 1024, dtype = torch.bfloat16)+ wq, ws = lin._unsloth_int8+ out = H._fused_int8_linear(lin, x.reshape(-1, 1024), wq, ws, torch.bfloat16, 256)+ assert out is not None and out.shape == (40, 512) and bool((out == 2).all())+ assert seen["quant"] == ((40, 1024), 256, True)+ assert seen["gemm"] == (True, torch.float32, True, (40,))++[email protected]("case", ["fp16_out", "fp16_in", "not_probed", "fp32_bias", "ragged_k"])+def test_the_fused_int8_path_declines_what_the_dit_kernels_do_not_cover(monkeypatch, case):+ lin, _ = _rotated_bf16_linear()+ wq, ws = lin._unsloth_int8+ x2 = torch.randn(40, 1024, dtype = torch.bfloat16)+ out_dtype = torch.bfloat16+ monkeypatch.setitem(H._FUSED_DEVICES, None, case != "not_probed")+ if case == "fp16_out":+ out_dtype = torch.float16+ elif case == "fp16_in":+ x2 = x2.half()+ elif case == "fp32_bias":+ lin.bias = torch.nn.Parameter(lin.bias.float())+ elif case == "ragged_k":+ x2 = x2[:, :1000]+ assert H._fused_int8_linear(lin, x2, wq, ws, out_dtype, 256) is None+++def test_the_fused_int8_kill_switch_keeps_the_unfused_path(monkeypatch):+ monkeypatch.setenv(H.H3_VAE_INT8_FUSED_ENV, "0")+ monkeypatch.delitem(H._FUSED_DEVICES, 7, raising = False)+ assert H._fused_int8_ready(7) is False+ assert 7 not in H._FUSED_DEVICES # the switch is read per call, never cached+++@needs_cuda+def test_the_fused_int8_path_matches_the_unfused_int8_path_on_cuda(monkeypatch):+ from core.inference import diffusion_int8_gemm as gemm++ lin, ref = _rotated_bf16_linear(k = 2048, n = 1024, device = "cuda")+ x = torch.randn(4, 1024, 2048, device = "cuda", dtype = torch.bfloat16)+ x[..., :8] *= 30 # outlier channels: what the rotation is for+ with torch.no_grad():+ dense = ref(x).float()+ monkeypatch.setenv(H.H3_VAE_INT8_FUSED_ENV, "0")+ unfused = lin(x).float()+ monkeypatch.delenv(H.H3_VAE_INT8_FUSED_ENV)+ if not H._fused_int8_ready(torch.cuda.current_device()):+ pytest.skip("the DiT's ConvRot act quant / fused GEMM is not armed on this card")+ calls = gemm.call_count()+ fused = lin(x).float()+ assert gemm.call_count() > calls, "the fused-dequant GEMM did not run"+ rel = lambda a, b: float((a - b).norm() / b.norm()) # noqa: E731+ assert rel(unfused, dense) < 0.02 and rel(fused, dense) < 0.02+ assert rel(fused, unfused) < 0.01