Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: MiniMax-H3 unfused Q / K / V share one ConvRot rotation and act quant (#15956)
Daniel Hanf6e47fe78b5d
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 0f9971f67c6..d9c0c036e06 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -11124,6 +11124,16 @@ class VideoBackend:compile_blocks_below_offload_hooks(getattr(pipe, denoiser_component, None), logger = logger)+ if "h3_strided_attn" in speed_optims:+ from .video_minimax_h3_attn import shared_qkv_count+ shared_qkv = shared_qkv_count(getattr(pipe, denoiser_component, None))+ if shared_qkv:+ speed_optims += ("h3_shared_qkv_quant",)+ logger.info(+ "video.h3_attn: Q / K / V share one ConvRot rotation + act quant on %d "+ "attention modules",+ shared_qkv,+ )except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a loadlogger.warning("video.h3_speed_optims failed, continuing unoptimised: %s", exc)# H3 declines graphs; this only acts when a family override forces them.diff --git a/studio/backend/core/inference/video_minimax_h3_attn.py b/studio/backend/core/inference/video_minimax_h3_attn.pyindex 39a416dba73..28b19a78669 100644--- a/studio/backend/core/inference/video_minimax_h3_attn.py+++ b/studio/backend/core/inference/video_minimax_h3_attn.py@@ -3,7 +3,8 @@"""MiniMax-H3 attention fast paths, each off with its env var set to 0: UNSLOTH_H3_STRIDED_ATTN(permuted SDPA views, no copies), UNSLOTH_H3_FUSED_QKV (one row-concatenated int8 to_qkv),-UNSLOTH_H3_ATTN_ARCH (flash only on H3_FLASH_FASTER_ARCHS); the first two are bit-identical."""+UNSLOTH_H3_SHARED_QKV_QUANT (unfused Q / K / V on one rotation + act quant), UNSLOTH_H3_ATTN_ARCH+(flash only on H3_FLASH_FASTER_ARCHS); all but the last are bit-identical."""from __future__ import annotations@@ -14,6 +15,7 @@ from .rocm_bf16 import is_rocm_torchSTRIDED_ATTN_ENV = "UNSLOTH_H3_STRIDED_ATTN"FUSED_QKV_ENV = "UNSLOTH_H3_FUSED_QKV"+SHARED_QKV_ENV = "UNSLOTH_H3_SHARED_QKV_QUANT"ATTN_ARCH_ENV = "UNSLOTH_H3_ATTN_ARCH"# Measured at H3's shape (torch 2.11): flash beats cuDNN on sm80 / sm89, ties or loses on sm120; unmeasured keep cuDNN.@@ -76,6 +78,8 @@ def strided_processor_class() -> Any:)from torch.nn.attention import SDPBackend, sdpa_kernel+ from . import diffusion_int8_gemm as int8_gemm+def _fusable_norm(norm: Any) -> bool:return (isinstance(norm, torch.nn.RMSNorm)@@ -96,6 +100,7 @@ def strided_processor_class() -> Any:class UnslothH3StridedAttnProcessor(MiniMaxH3AttnProcessor):_unsloth_qk_rope = False+ _unsloth_shared_qkv = Falsedef __call__(self,@@ -107,12 +112,20 @@ def strided_processor_class() -> Any:kernel = kernels.get(_backend_value(self._attention_backend))if kernel is None or attention_mask is not None or self._parallel_config is not None:return super().__call__(attn, hidden_states, rotary_emb, attention_mask)+ qkv = Noneif attn.fused_projections:query, key, value = attn.to_qkv(hidden_states).chunk(3, dim = -1)else:- query = attn.to_q(hidden_states)- key = attn.to_k(hidden_states)- value = attn.to_v(hidden_states)+ if self._unsloth_shared_qkv:+ qkv = int8_gemm.shared_rotq_linears(+ hidden_states, (attn.to_q, attn.to_k, attn.to_v)+ )+ if qkv is not None:+ query, key, value = qkv+ else:+ query = attn.to_q(hidden_states)+ key = attn.to_k(hidden_states)+ value = attn.to_v(hidden_states)query = query.unflatten(-1, (attn.heads, -1))key = key.unflatten(-1, (attn.heads, -1))@@ -190,6 +203,7 @@ def install_strided_attention(transformer: Any, logger: Any = None) -> int:continuenew = cls()new._unsloth_qk_rope = qk_rope_on+ new._unsloth_shared_qkv = _enabled(SHARED_QKV_ENV)new._attention_backend = getattr(old, "_attention_backend", None)new._parallel_config = getattr(old, "_parallel_config", None)module.set_processor(new)@@ -212,6 +226,23 @@ def strided_attention_count(transformer: Any) -> int:)+def shared_qkv_count(transformer: Any) -> int:+ """Engagement census: unfused attention modules whose Q / K / V take the shared rotation + act quant."""+ from .diffusion_int8_gemm import _swapped_rec++ count = 0+ for m in _h3_attention_modules(transformer):+ if getattr(m, "fused_projections", False):+ continue+ if not getattr(getattr(m, "processor", None), "_unsloth_shared_qkv", False):+ continue+ parts = [getattr(m, n, None) for n in ("to_q", "to_k", "to_v")]+ recs = [None if p is None else _swapped_rec(p) for p in parts]+ if all(r is not None and r[2] for r in recs) and len({(r[0], r[1]) for r in recs}) == 1:+ count += 1+ return count++def _per_token_int8(weight: Any) -> bool:"""Per-token activation quant (depends on the input alone, so appending weight rows cannot change it)."""name = type(weight).__name__diff --git a/studio/backend/tests/test_video_minimax_h3_attn.py b/studio/backend/tests/test_video_minimax_h3_attn.pyindex ab57b5a3166..90b2cb29af6 100644--- a/studio/backend/tests/test_video_minimax_h3_attn.py+++ b/studio/backend/tests/test_video_minimax_h3_attn.py@@ -106,9 +106,13 @@ def _quantize(return model-def _inputs(seed: int = 1) -> dict:+def _inputs(+ seed: int = 1,+ n_text: int = 5,+ n_video: int = 8,+ n_audio: int = 3,+) -> dict:g = torch.Generator().manual_seed(seed)- n_text, n_video, n_audio = 5, 8, 3seq = n_text + n_video + n_audiotext_idx = torch.arange(0, n_text)video_idx = torch.arange(n_text, n_text + n_video)@@ -243,6 +247,112 @@ def test_strided_kill_switch(quantized, monkeypatch):assert A.strided_attention_count(quantized) == 0+def test_unfused_qkv_takes_the_shared_quant_and_matches_stock(quantized, monkeypatch):+ from core.inference import diffusion_int8_gemm as g8++ quantized.set_attention_backend("_native_math")+ stock = copy.deepcopy(quantized)+ seen = []++ def shared(x, linears):+ seen.append(len(linears))+ return [m(x) for m in linears]++ monkeypatch.setattr(g8, "shared_rotq_linears", shared)+ assert A.install_strided_attention(quantized) == 3+ assert _equal(_run(stock), _run(quantized))+ assert seen and set(seen) == {3}+++def test_shared_qkv_kill_switch(quantized, monkeypatch):+ from core.inference import diffusion_int8_gemm as g8++ monkeypatch.setenv(A.SHARED_QKV_ENV, "0")+ monkeypatch.setattr(+ g8, "shared_rotq_linears", lambda x, linears: pytest.fail("shared path used")+ )+ A.install_strided_attention(quantized)+ quantized.set_attention_backend("_native_math")+ _run(quantized)+ assert A.shared_qkv_count(quantized) == 0+++def test_shared_qkv_census_needs_the_int8_swap(quantized):+ A.install_strided_attention(quantized)+ assert A.shared_qkv_count(quantized) == 0++[email protected](not torch.cuda.is_available(), reason = "int8 GEMM swap needs CUDA")+def test_shared_qkv_on_the_int8_swap_is_bit_identical_on_cuda(monkeypatch):+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ from core.inference import diffusion_convrot_quant as cq+ from core.inference import diffusion_int8_gemm as g8++ monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")+ index = torch.cuda.current_device()+ if g8.device_config(index) is None or cq.rotquant_device_config(index) is None:+ pytest.skip("int8 GEMM / fused rotation probe refused this device")+ torch.manual_seed(0)+ model = (+ h3.MiniMaxH3Transformer3DModel(+ num_attention_heads = 8,+ attention_head_dim = 128,+ hidden_size = 1024,+ num_layers = 2,+ num_refiner_layers = 1,+ ffn_dim = 2048,+ in_channels = 4,+ audio_in_channels = 4,+ patch_size = (1, 2, 2),+ text_dim = 32,+ freq_dim = 32,+ time_embed_hidden_dim = 64,+ time_embed_dim = 32,+ rope_freq_dim = 4,+ )+ .eval()+ .to("cuda", torch.bfloat16)+ )+ fqns = [+ name+ for name, m in model.named_modules()+ if isinstance(m, torch.nn.Linear)+ and name.startswith("transformer_blocks.")+ and ".attn." in name+ ]+ rotate_linears_(model, fqns, 256)+ quantize_(+ model,+ Int8DynamicActivationInt8WeightConfig(set_inductor_config = False),+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear) and fqn in fqns,+ )+ model.set_attention_backend("_native_math")+ assert g8.install(model)+ stock = copy.deepcopy(model)+ A.install_strided_attention(stock)+ for m in stock.modules():+ if isinstance(m, h3.MiniMaxH3Attention):+ m.processor._unsloth_shared_qkv = False+ A.install_strided_attention(model)+ assert A.shared_qkv_count(model) == 2+ inputs = _inputs(n_text = 16, n_video = 48, n_audio = 8)+ inputs = {+ k: (v.to("cuda", torch.bfloat16) if torch.is_floating_point(v) else v.to("cuda"))+ if torch.is_tensor(v)+ else v+ for k, v in inputs.items()+ }+ with torch.no_grad():+ before = cq.rotquant_call_count()+ ref = stock(**inputs)+ mid = cq.rotquant_call_count()+ out = model(**inputs)+ after = cq.rotquant_call_count()+ assert _equal(ref, out)+ assert after - mid == mid - before - 2 * A.shared_qkv_count(model) > 0++def test_install_is_idempotent(quantized):assert A.install_strided_attention(quantized) == 3assert A.install_strided_attention(quantized) == 0