Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: MiniMax-H3 unfused Q / K / V share one ConvRot rotation and act quant (#15956)

Daniel Hanf6e47fe78b5d
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 0f9971f67c6..d9c0c036e06 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -11124,6 +11124,16 @@ class VideoBackend:
compile_blocks_below_offload_hooks(
getattr(pipe, denoiser_component, None), logger = logger
)
+ if "h3_strided_attn" in speed_optims:
+ from .video_minimax_h3_attn import shared_qkv_count
+ shared_qkv = shared_qkv_count(getattr(pipe, denoiser_component, None))
+ if shared_qkv:
+ speed_optims += ("h3_shared_qkv_quant",)
+ logger.info(
+ "video.h3_attn: Q / K / V share one ConvRot rotation + act quant on %d "
+ "attention modules",
+ shared_qkv,
+ )
except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load
logger.warning("video.h3_speed_optims failed, continuing unoptimised: %s", exc)
# H3 declines graphs; this only acts when a family override forces them.
diff --git a/studio/backend/core/inference/video_minimax_h3_attn.py b/studio/backend/core/inference/video_minimax_h3_attn.py
index 39a416dba73..28b19a78669 100644
--- a/studio/backend/core/inference/video_minimax_h3_attn.py
+++ b/studio/backend/core/inference/video_minimax_h3_attn.py
@@ -3,7 +3,8 @@
"""MiniMax-H3 attention fast paths, each off with its env var set to 0: UNSLOTH_H3_STRIDED_ATTN
(permuted SDPA views, no copies), UNSLOTH_H3_FUSED_QKV (one row-concatenated int8 to_qkv),
-UNSLOTH_H3_ATTN_ARCH (flash only on H3_FLASH_FASTER_ARCHS); the first two are bit-identical."""
+UNSLOTH_H3_SHARED_QKV_QUANT (unfused Q / K / V on one rotation + act quant), UNSLOTH_H3_ATTN_ARCH
+(flash only on H3_FLASH_FASTER_ARCHS); all but the last are bit-identical."""
from __future__ import annotations
@@ -14,6 +15,7 @@ from .rocm_bf16 import is_rocm_torch
STRIDED_ATTN_ENV = "UNSLOTH_H3_STRIDED_ATTN"
FUSED_QKV_ENV = "UNSLOTH_H3_FUSED_QKV"
+SHARED_QKV_ENV = "UNSLOTH_H3_SHARED_QKV_QUANT"
ATTN_ARCH_ENV = "UNSLOTH_H3_ATTN_ARCH"
# Measured at H3's shape (torch 2.11): flash beats cuDNN on sm80 / sm89, ties or loses on sm120; unmeasured keep cuDNN.
@@ -76,6 +78,8 @@ def strided_processor_class() -> Any:
)
from torch.nn.attention import SDPBackend, sdpa_kernel
+ from . import diffusion_int8_gemm as int8_gemm
+
def _fusable_norm(norm: Any) -> bool:
return (
isinstance(norm, torch.nn.RMSNorm)
@@ -96,6 +100,7 @@ def strided_processor_class() -> Any:
class UnslothH3StridedAttnProcessor(MiniMaxH3AttnProcessor):
_unsloth_qk_rope = False
+ _unsloth_shared_qkv = False
def __call__(
self,
@@ -107,12 +112,20 @@ def strided_processor_class() -> Any:
kernel = kernels.get(_backend_value(self._attention_backend))
if kernel is None or attention_mask is not None or self._parallel_config is not None:
return super().__call__(attn, hidden_states, rotary_emb, attention_mask)
+ qkv = None
if attn.fused_projections:
query, key, value = attn.to_qkv(hidden_states).chunk(3, dim = -1)
else:
- query = attn.to_q(hidden_states)
- key = attn.to_k(hidden_states)
- value = attn.to_v(hidden_states)
+ if self._unsloth_shared_qkv:
+ qkv = int8_gemm.shared_rotq_linears(
+ hidden_states, (attn.to_q, attn.to_k, attn.to_v)
+ )
+ if qkv is not None:
+ query, key, value = qkv
+ else:
+ query = attn.to_q(hidden_states)
+ key = attn.to_k(hidden_states)
+ value = attn.to_v(hidden_states)
query = query.unflatten(-1, (attn.heads, -1))
key = key.unflatten(-1, (attn.heads, -1))
@@ -190,6 +203,7 @@ def install_strided_attention(transformer: Any, logger: Any = None) -> int:
continue
new = cls()
new._unsloth_qk_rope = qk_rope_on
+ new._unsloth_shared_qkv = _enabled(SHARED_QKV_ENV)
new._attention_backend = getattr(old, "_attention_backend", None)
new._parallel_config = getattr(old, "_parallel_config", None)
module.set_processor(new)
@@ -212,6 +226,23 @@ def strided_attention_count(transformer: Any) -> int:
)
+def shared_qkv_count(transformer: Any) -> int:
+ """Engagement census: unfused attention modules whose Q / K / V take the shared rotation + act quant."""
+ from .diffusion_int8_gemm import _swapped_rec
+
+ count = 0
+ for m in _h3_attention_modules(transformer):
+ if getattr(m, "fused_projections", False):
+ continue
+ if not getattr(getattr(m, "processor", None), "_unsloth_shared_qkv", False):
+ continue
+ parts = [getattr(m, n, None) for n in ("to_q", "to_k", "to_v")]
+ recs = [None if p is None else _swapped_rec(p) for p in parts]
+ if all(r is not None and r[2] for r in recs) and len({(r[0], r[1]) for r in recs}) == 1:
+ count += 1
+ return count
+
+
def _per_token_int8(weight: Any) -> bool:
"""Per-token activation quant (depends on the input alone, so appending weight rows cannot change it)."""
name = type(weight).__name__
diff --git a/studio/backend/tests/test_video_minimax_h3_attn.py b/studio/backend/tests/test_video_minimax_h3_attn.py
index ab57b5a3166..90b2cb29af6 100644
--- a/studio/backend/tests/test_video_minimax_h3_attn.py
+++ b/studio/backend/tests/test_video_minimax_h3_attn.py
@@ -106,9 +106,13 @@ def _quantize(
return model
-def _inputs(seed: int = 1) -> dict:
+def _inputs(
+ seed: int = 1,
+ n_text: int = 5,
+ n_video: int = 8,
+ n_audio: int = 3,
+) -> dict:
g = torch.Generator().manual_seed(seed)
- n_text, n_video, n_audio = 5, 8, 3
seq = n_text + n_video + n_audio
text_idx = torch.arange(0, n_text)
video_idx = torch.arange(n_text, n_text + n_video)
@@ -243,6 +247,112 @@ def test_strided_kill_switch(quantized, monkeypatch):
assert A.strided_attention_count(quantized) == 0
+def test_unfused_qkv_takes_the_shared_quant_and_matches_stock(quantized, monkeypatch):
+ from core.inference import diffusion_int8_gemm as g8
+
+ quantized.set_attention_backend("_native_math")
+ stock = copy.deepcopy(quantized)
+ seen = []
+
+ def shared(x, linears):
+ seen.append(len(linears))
+ return [m(x) for m in linears]
+
+ monkeypatch.setattr(g8, "shared_rotq_linears", shared)
+ assert A.install_strided_attention(quantized) == 3
+ assert _equal(_run(stock), _run(quantized))
+ assert seen and set(seen) == {3}
+
+
+def test_shared_qkv_kill_switch(quantized, monkeypatch):
+ from core.inference import diffusion_int8_gemm as g8
+
+ monkeypatch.setenv(A.SHARED_QKV_ENV, "0")
+ monkeypatch.setattr(
+ g8, "shared_rotq_linears", lambda x, linears: pytest.fail("shared path used")
+ )
+ A.install_strided_attention(quantized)
+ quantized.set_attention_backend("_native_math")
+ _run(quantized)
+ assert A.shared_qkv_count(quantized) == 0
+
+
+def test_shared_qkv_census_needs_the_int8_swap(quantized):
+ A.install_strided_attention(quantized)
+ assert A.shared_qkv_count(quantized) == 0
+
+
[email protected](not torch.cuda.is_available(), reason = "int8 GEMM swap needs CUDA")
+def test_shared_qkv_on_the_int8_swap_is_bit_identical_on_cuda(monkeypatch):
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ from core.inference import diffusion_convrot_quant as cq
+ from core.inference import diffusion_int8_gemm as g8
+
+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "1")
+ index = torch.cuda.current_device()
+ if g8.device_config(index) is None or cq.rotquant_device_config(index) is None:
+ pytest.skip("int8 GEMM / fused rotation probe refused this device")
+ torch.manual_seed(0)
+ model = (
+ h3.MiniMaxH3Transformer3DModel(
+ num_attention_heads = 8,
+ attention_head_dim = 128,
+ hidden_size = 1024,
+ num_layers = 2,
+ num_refiner_layers = 1,
+ ffn_dim = 2048,
+ in_channels = 4,
+ audio_in_channels = 4,
+ patch_size = (1, 2, 2),
+ text_dim = 32,
+ freq_dim = 32,
+ time_embed_hidden_dim = 64,
+ time_embed_dim = 32,
+ rope_freq_dim = 4,
+ )
+ .eval()
+ .to("cuda", torch.bfloat16)
+ )
+ fqns = [
+ name
+ for name, m in model.named_modules()
+ if isinstance(m, torch.nn.Linear)
+ and name.startswith("transformer_blocks.")
+ and ".attn." in name
+ ]
+ rotate_linears_(model, fqns, 256)
+ quantize_(
+ model,
+ Int8DynamicActivationInt8WeightConfig(set_inductor_config = False),
+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear) and fqn in fqns,
+ )
+ model.set_attention_backend("_native_math")
+ assert g8.install(model)
+ stock = copy.deepcopy(model)
+ A.install_strided_attention(stock)
+ for m in stock.modules():
+ if isinstance(m, h3.MiniMaxH3Attention):
+ m.processor._unsloth_shared_qkv = False
+ A.install_strided_attention(model)
+ assert A.shared_qkv_count(model) == 2
+ inputs = _inputs(n_text = 16, n_video = 48, n_audio = 8)
+ inputs = {
+ k: (v.to("cuda", torch.bfloat16) if torch.is_floating_point(v) else v.to("cuda"))
+ if torch.is_tensor(v)
+ else v
+ for k, v in inputs.items()
+ }
+ with torch.no_grad():
+ before = cq.rotquant_call_count()
+ ref = stock(**inputs)
+ mid = cq.rotquant_call_count()
+ out = model(**inputs)
+ after = cq.rotquant_call_count()
+ assert _equal(ref, out)
+ assert after - mid == mid - before - 2 * A.shared_qkv_count(model) > 0
+
+
def test_install_is_idempotent(quantized):
assert A.install_strided_attention(quantized) == 3
assert A.install_strided_attention(quantized) == 0