Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: gate MiniMax-H3 Auto flash on a real flash SDPA kernel (#16294) (#16296)
Indrajit swain5e167e60b32c
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex b03cf918b9a..fd798dd4c24 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -3476,6 +3476,23 @@ def apply_attention_backend(and not _cudnn_serves_pipe(pipe, target, logger)):backend = None+ if backend == "_native_flash" and target is not None:+ # H3 Auto can pick flash after select(); refuse exclusive flash when this build has no kernel (#16294).+ try:+ kernels = available_sdpa_kernels(target)+ except Exception: # noqa: BLE001 -- an unreadable probe must not force flash+ kernels = ()+ if SDPA_FLASH not in kernels:+ # A probe that answered without cuDNN keeps the default dispatch; an empty one keeps Auto's cuDNN.+ cudnn_ok = (not kernels or SDPA_CUDNN in kernels) and _cudnn_serves_pipe(+ pipe, target, logger+ )+ if logger is not None:+ logger.info(+ "diffusion.attention: flash SDPA unavailable on this torch build; using %s",+ "cuDNN" if cudnn_ok else "the default SDPA dispatch",+ )+ backend = AUTO_CUDNN if cudnn_ok else Noneif (isinstance(backend, _AutoCudnn)and target is not Nonediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 1e33e127324..61fbaf623e0 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -11306,7 +11306,11 @@ class VideoBackend:umem_target, attention_backend, speed_active = effective_speed != SPEED_OFF)try:- h3_attn_backend = h3_attention_backend(h3_attn_backend, requested = attention_backend)+ h3_attn_backend = h3_attention_backend(+ h3_attn_backend,+ requested = attention_backend,+ target = types.SimpleNamespace(device = device, dtype = dtype),+ )except Exception: # noqa: BLE001 -- keep the generic pickpassattention_engaged = apply_attention_backend(diff --git a/studio/backend/core/inference/video_minimax_h3_attn.py b/studio/backend/core/inference/video_minimax_h3_attn.pyindex 28b19a78669..342e0ecfc6c 100644--- a/studio/backend/core/inference/video_minimax_h3_attn.py+++ b/studio/backend/core/inference/video_minimax_h3_attn.py@@ -4,7 +4,7 @@"""MiniMax-H3 attention fast paths, each off with its env var set to 0: UNSLOTH_H3_STRIDED_ATTN(permuted SDPA views, no copies), UNSLOTH_H3_FUSED_QKV (one row-concatenated int8 to_qkv),UNSLOTH_H3_SHARED_QKV_QUANT (unfused Q / K / V on one rotation + act quant), UNSLOTH_H3_ATTN_ARCH-(flash only on H3_FLASH_FASTER_ARCHS); all but the last are bit-identical."""+(flash only on H3_FLASH_FASTER_ARCHS when flash SDPA is available); all but the last are bit-identical."""from __future__ import annotations@@ -39,13 +39,35 @@ def _capability(device: Any = None) -> Optional[tuple[int, int]]:return None+def _flash_sdpa_ready(target: Any = None) -> bool:+ """Whether this torch build can actually run flash SDPA. Empty / failed probes refuse the upgrade (#16294)."""+ if target is not None:+ try:+ from .diffusion_attention import SDPA_FLASH, available_sdpa_kernels+ kernels = available_sdpa_kernels(target)+ # Empty means unanswerable, not "flash works".+ return bool(kernels) and SDPA_FLASH in kernels+ except Exception: # noqa: BLE001 -- a probe failure must not force flash+ return False+ try:+ import torch+ check = getattr(torch.backends.cuda, "is_flash_attention_available", None)+ if callable(check):+ return bool(check())+ except Exception: # noqa: BLE001+ return False+ # No target and no torch flag: keep the measured arch upgrade for builds that never expose the check.+ return True++def h3_attention_backend(selected: Optional[str],capability: Optional[tuple[int, int]] = None,requested: Optional[str] = None,+ target: Any = None,) -> Optional[str]:- """Moves only the automatic cuDNN pick (``requested`` auto / unset), and only to flash on a measured-faster arch;- anything else unchanged."""+ """Moves only the automatic cuDNN pick (``requested`` auto / unset), and only to flash on a measured-faster arch+ whose torch build actually has flash SDPA; anything else unchanged (keep ``selected`` so ``_AutoCudnn`` survives)."""if selected != _CUDNN or not _enabled(ATTN_ARCH_ENV):return selectedtry:@@ -57,6 +79,8 @@ def h3_attention_backend(cap = capability if capability is not None else _capability()if cap is None or tuple(cap) not in H3_FLASH_FASTER_ARCHS:return selected+ if not _flash_sdpa_ready(target):+ return selectedreturn _FLASHdiff --git a/studio/backend/tests/test_diffusion_attention_cudnn_head_dim.py b/studio/backend/tests/test_diffusion_attention_cudnn_head_dim.pyindex 43876abab89..a08d48b7b4c 100644--- a/studio/backend/tests/test_diffusion_attention_cudnn_head_dim.py+++ b/studio/backend/tests/test_diffusion_attention_cudnn_head_dim.py@@ -175,3 +175,27 @@ def test_kill_switch_pins_cudnn_without_asking(monkeypatch):types.SimpleNamespace(transformer = t), "_native_cudnn", target = _target()) == ("_native_cudnn")assert seen == []+++ "kernels, expected",+ [+ (("cudnn", "mem_efficient", "math"), "_native_cudnn"),+ (("mem_efficient", "math"), None),+ (("math",), None),+ (("flash", "cudnn", "math"), "_native_flash"),+ ],+)+def test_exclusive_flash_needs_a_flash_kernel_and_falls_back_to_cudnn_only_where_it_runs(+ monkeypatch, kernels, expected+):+ # #16294: a torch build without flash SDPA must not pin FLASH_ATTENTION, nor fall back to a cuDNN it lacks.+ _stub_probe(monkeypatch, served = (64, 128))+ monkeypatch.setattr(att, "available_sdpa_kernels", lambda target: kernels)+ monkeypatch.setattr(att, "_native_dispatches_cudnn", lambda *a, **k: False)+ t = _Transformer(config = {"attention_head_dim": 128})+ engaged = apply_attention_backend(+ types.SimpleNamespace(transformer = t), "_native_flash", target = _target()+ )+ assert engaged == expected+ assert ("_native_flash" in t.calls) == (expected == "_native_flash")diff --git a/studio/backend/tests/test_video_minimax_h3_attn.py b/studio/backend/tests/test_video_minimax_h3_attn.pyindex 90b2cb29af6..adcb906173a 100644--- a/studio/backend/tests/test_video_minimax_h3_attn.py+++ b/studio/backend/tests/test_video_minimax_h3_attn.py@@ -373,16 +373,39 @@ def test_install_is_idempotent(quantized):(None, (8, 0), None),],)-def test_arch_pick_moves_only_the_automatic_cudnn_pick_on_measured_archs(selected, cap, expected):+def test_arch_pick_moves_only_the_automatic_cudnn_pick_on_measured_archs(+ selected, cap, expected, monkeypatch+):+ monkeypatch.setattr(A, "_flash_sdpa_ready", lambda target = None: True)assert A.h3_attention_backend(selected, cap) == expected@pytest.mark.parametrize("requested", ["cudnn", "flash", "native"])-def test_arch_pick_keeps_an_explicit_request(requested):+def test_arch_pick_keeps_an_explicit_request(requested, monkeypatch):+ monkeypatch.setattr(A, "_flash_sdpa_ready", lambda target = None: True)assert A.h3_attention_backend("_native_cudnn", (8, 0), requested = requested) == "_native_cudnn"assert A.h3_attention_backend("_native_cudnn", (8, 0), requested = "auto") == "_native_flash"[email protected]("cap", [(8, 0), (8, 9)])+def test_arch_pick_keeps_cudnn_when_flash_sdpa_is_missing(cap, monkeypatch):+ # Windows CUDA wheels often ship without flash; Auto must not exclusive-pin FLASH (#16294).+ monkeypatch.setattr(A, "_flash_sdpa_ready", lambda target = None: False)+ assert A.h3_attention_backend("_native_cudnn", cap, requested = "auto") == "_native_cudnn"+++def test_arch_pick_keeps_cudnn_when_sdpa_probe_is_empty(monkeypatch):+ monkeypatch.setattr(+ "core.inference.diffusion_attention.available_sdpa_kernels",+ lambda target: (),+ )+ target = type("T", (), {"device": "cuda", "dtype": "torch.bfloat16"})()+ assert (+ A.h3_attention_backend("_native_cudnn", (8, 9), requested = "auto", target = target)+ == "_native_cudnn"+ )++def test_h3_load_wires_the_request_and_the_speed_gate():import astimport inspect@@ -397,6 +420,7 @@ def test_h3_load_wires_the_request_and_the_speed_gate():if isinstance(n, ast.Call) and getattr(n.func, "id", "") == "h3_attention_backend"]assert calls and all(any(k.arg == "requested" for k in c.keywords) for c in calls)+ assert calls and all(any(k.arg == "target" for k in c.keywords) for c in calls)gates = [nfor n in ast.walk(tree)