Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Windows ROCm: use math attention where the fused SDPA kernels fail (#12639)
Daniel Hanad13de2e0159
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex 91339218315..755be7d36e0 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -155,7 +155,9 @@ def _probe_sdpa_kernels(device: str, dtype: Any) -> tuple[str, ...]:continuetry:with sdpa_kernel([backend]):- torch.nn.functional.scaled_dot_product_attention(q, q, q)+ out = torch.nn.functional.scaled_dot_product_attention(q, q, q)+ # Windows ROCm raises a failed fused launch only at the next checked kernel: take it here.+ out.float().sum().item()available.append(name)except Exception: # noqa: BLE001 - "No available kernel" is the answer, not an errorcontinuediff --git a/studio/backend/tests/test_diffusion_attention.py b/studio/backend/tests/test_diffusion_attention.pyindex 74b2c796268..6c5beffeae3 100644--- a/studio/backend/tests/test_diffusion_attention.py+++ b/studio/backend/tests/test_diffusion_attention.py@@ -884,6 +884,26 @@ def test_any_subquadratic_kernel_means_not_math_only(kernels, monkeypatch):assert att.sdpa_math_only(_target()) is False+def test_a_fused_launch_that_fails_late_is_not_reported_available(monkeypatch):+ # Windows ROCm gfx1151: the fused call returns, and its hipErrorInvalidValue only surfaces on+ # the next checked kernel. The probe must take that error itself, not hand it to a later op.+ torch = pytest.importorskip("torch")++ class _Pending:+ def float(self):+ raise RuntimeError("CUDA error: invalid argument")++ real = torch.nn.functional.scaled_dot_product_attention++ def _sdpa(q, k, v, *a, **kw):+ if not torch.backends.cuda.math_sdp_enabled():+ return _Pending()+ return real(q, k, v, *a, **kw)++ monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", _sdpa)+ assert att._probe_sdpa_kernels("cpu", torch.float32) == ("math",)++def test_an_unanswerable_probe_is_not_a_math_only_verdict(monkeypatch):# "Only math" is a claim about the hardware. A probe that could not run (no torch, no device,# an allocator failure) has made no such claim, and must not refuse or warn on a guess.diff --git a/tests/test_rocm_windows_fused_sdpa.py b/tests/test_rocm_windows_fused_sdpa.pynew file mode 100644index 00000000000..034cd78db71--- /dev/null+++ b/tests/test_rocm_windows_fused_sdpa.py@@ -0,0 +1,158 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""On Windows ROCm (torch 2.11.0+rocm7.14.1, gfx1151) flash and memory-efficient SDPA fail with+hipErrorInvalidValue on every call. `import unsloth` must turn off exactly the backends that fail,+so attention runs on the math kernel. The failed launch only surfaces at the next checked kernel+launch, so the probe must not blame a later call. The last two tests need the real host."""++import json+import subprocess+import sys+import textwrap++import pytest++torch = pytest.importorskip("torch")+++def _on_windows_rocm():+ if sys.platform != "win32" or not torch.cuda.is_available():+ return False+ # Same detection as the fix: some AMD wheels tag only the torch version.+ from unsloth.import_fixes import _is_rocm_torch_build+ return bool(getattr(torch.version, "hip", None)) or _is_rocm_torch_build()+++def _run(code, *args):+ p = subprocess.run(+ [sys.executable, "-c", code, *args], capture_output = True, text = True, timeout = 900+ )+ line = [l for l in p.stdout.splitlines() if l.startswith("RESULT ")]+ assert line, p.stdout[-2000:] + p.stderr[-3000:]+ return json.loads(line[-1].split(" ", 1)[1])+++def test_probe_is_windows_rocm_only():+ if _on_windows_rocm():+ pytest.skip(reason = "this host is the Windows ROCm one the probe targets")+ from unsloth.import_fixes import _rocm_windows_broken_sdpa_backends+ assert _rocm_windows_broken_sdpa_backends() == []+++ "broken, expected",+ [+ ([], [True, True, True]),+ (["flash"], [False, True, True]),+ (["mem_efficient"], [True, False, True]),+ (["flash", "mem_efficient"], [False, False, True]),+ ],+)+def test_only_broken_backends_are_turned_off(monkeypatch, broken, expected):+ import unsloth.import_fixes as fixes++ cuda = torch.backends.cuda+ saved = (cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled())+ monkeypatch.delenv("UNSLOTH_ALLOW_ROCM_FUSED_SDPA", raising = False)+ monkeypatch.setattr(fixes, "_rocm_windows_broken_sdpa_backends", lambda: list(broken))+ try:+ cuda.enable_flash_sdp(True)+ cuda.enable_mem_efficient_sdp(True)+ fixes.fix_rocm_windows_fused_sdpa()+ got = [cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled(), cuda.math_sdp_enabled()]+ finally:+ cuda.enable_flash_sdp(saved[0])+ cuda.enable_mem_efficient_sdp(saved[1])+ assert got == expected+++def test_opt_out_skips_the_probe(monkeypatch):+ import unsloth.import_fixes as fixes++ monkeypatch.setenv("UNSLOTH_ALLOW_ROCM_FUSED_SDPA", "1")+ monkeypatch.setattr(+ fixes,+ "_rocm_windows_broken_sdpa_backends",+ lambda: pytest.fail("probed despite the opt-out"),+ )+ fixes.fix_rocm_windows_fused_sdpa()+++_PROBE_TWICE = textwrap.dedent(+ """+ import json+ import torch+ from unsloth.import_fixes import _rocm_windows_broken_sdpa_backends+ first = _rocm_windows_broken_sdpa_backends()+ with torch.inference_mode(): # an import inside inference_mode must probe the same+ second = _rocm_windows_broken_sdpa_backends()+ # Nothing may be left pending: a stale launch error would raise on this checked kernel.+ torch.ones(4, device = "cuda").float().sum().item()+ print("RESULT " + json.dumps([first, second]))+ """+)+++def test_windows_rocm_probe_is_stable_and_leaves_no_pending_error():+ if not _on_windows_rocm():+ pytest.skip(reason = "needs a Windows ROCm GPU to reproduce the launch failure")+ first, second = _run(_PROBE_TWICE)+ assert first == second+++_ATTENTION = textwrap.dedent(+ """+ import json+ import unsloth # noqa: F401 (applies the import-time fixes)+ import torch+ import torch.nn.functional as F+ from torch.nn.attention import SDPBackend, sdpa_kernel++ g = torch.Generator(device = "cuda").manual_seed(0)+ # fp16: gfx10 claims bf16 it lacks, and fp16 exercises the same fused kernels.+ q, k, v = (torch.randn(2, 4, 24, 64, device = "cuda", dtype = torch.float16, generator = g)+ for _ in range(3))+ mask = torch.ones(24, 24, device = "cuda", dtype = torch.bool).tril()+ out = {}+ for name, grad, kw, nkv in (+ ("causal_fwd", False, {"is_causal": True}, 4),+ ("mask_fwd", False, {"attn_mask": mask}, 4),+ ("causal_bwd", True, {"is_causal": True}, 4),+ ("mask_bwd", True, {"attn_mask": mask}, 4),+ ("gqa_bwd", True, {"is_causal": True, "enable_gqa": True}, 2),+ ):+ def run(math):+ qq = q.clone().requires_grad_(grad)+ if math:+ with sdpa_kernel([SDPBackend.MATH]):+ o = F.scaled_dot_product_attention(qq, k[:, :nkv], v[:, :nkv], **kw)+ else:+ o = F.scaled_dot_product_attention(qq, k[:, :nkv], v[:, :nkv], **kw)+ o = o.float() # a checked launch: a failed attention launch raises here+ if grad:+ o.sum().backward()+ return o.detach(), qq.grad.float()+ return o, o+ try:+ o, d = run(False)+ ro, rd = run(True)+ out[name] = bool(torch.allclose(o, ro, atol = 2e-2, rtol = 2e-2)+ and torch.allclose(d, rd, atol = 2e-2, rtol = 2e-2))+ except Exception as e:+ out[name] = f"{type(e).__name__}: {e}"[:200]+ print("RESULT " + json.dumps(out))+ """+)+++def test_windows_rocm_attention_runs_after_import():+ if not _on_windows_rocm():+ pytest.skip(reason = "needs a Windows ROCm GPU to reproduce the launch failure")+ assert _run(_ATTENTION) == {+ "causal_fwd": True,+ "mask_fwd": True,+ "causal_bwd": True,+ "mask_bwd": True,+ "gqa_bwd": True,+ }diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 3f8790078df..127f35bc4cf 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -282,6 +282,7 @@ from .import_fixes import (fix_vllm_guided_decoding_params,fix_vllm_pdl_blackwell,fix_cudnn_sdpa_d256_masked_backward,+ fix_rocm_windows_fused_sdpa,fix_triton_compiled_kernel_missing_attrs,fix_dynamo_config_thread_visibility,patch_trunc_normal_precision_issue,@@ -366,6 +367,8 @@ fix_vllm_guided_decoding_params()fix_trl_vllm_ascend()fix_vllm_pdl_blackwell()fix_cudnn_sdpa_d256_masked_backward()+# Windows ROCm only, probe-gated: fused attention fails on every call there (gfx1151, torch 2.11).+fix_rocm_windows_fused_sdpa()fix_triton_compiled_kernel_missing_attrs()# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss patches raise the dynamo# recompile limits, so those settings reach the autograd worker threads on torch >= 2.12.@@ -425,6 +428,7 @@ del fix_vllm_guided_decoding_paramsdel fix_trl_vllm_ascenddel fix_vllm_pdl_blackwelldel fix_cudnn_sdpa_d256_masked_backward+del fix_rocm_windows_fused_sdpadel fix_triton_compiled_kernel_missing_attrsdel fix_dynamo_config_thread_visibilitydel patch_trunc_normal_precision_issuediff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex ed1e6b04b2e..109274fa51c 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -6639,6 +6639,96 @@ def fix_cudnn_sdpa_d256_masked_backward():)+def _rocm_windows_broken_sdpa_backends():+ """Fused SDPA backends ("flash", "mem_efficient") failing on this process's Windows ROCm GPU.++ torch 2.11.0+rocm7.14.1 on gfx1151 fails every fused call (hipErrorInvalidValue), raised only+ by the next checked launch, so each probe ends in one and is compared against math.+ """+ if sys.platform != "win32":+ return []+ try:+ import torch+ import torch.nn.functional as F+ from torch.nn.attention import SDPBackend, sdpa_kernel++ # Some AMD wheels tag only __version__, so torch.version.hip alone misses them.+ is_rocm = bool(getattr(torch.version, "hip", None)) or _is_rocm_torch_build()+ if not is_rocm or not torch.cuda.is_available():+ return []+ except Exception:+ return []++ def attend(device, backend):+ g = torch.Generator(device = device).manual_seed(0)+ q, k, v = (+ torch.randn(1, 2, 16, 64, device = device, dtype = dtype, generator = g) for _ in range(3)+ )+ q.requires_grad_(True)+ with sdpa_kernel([backend]):+ out = F.scaled_dot_product_attention(q, k, v, is_causal = True)+ out = out.float() # checked launch: a failed attention launch raises here+ out.sum().backward()+ dq = q.grad.float()+ dq.sum().item()+ return torch.cat([out.detach().flatten(), dq.flatten()])++ # Only this process's device: probing every visible GPU would open a HIP context on each.+ device = torch.device("cuda", torch.cuda.current_device())+ try:+ from .device_type import arch_lacks_bf16+ arch = torch.cuda.get_device_properties(device).gcnArchName+ except Exception:+ arch_lacks_bf16, arch = (lambda _: True), None+ # gfx10 claims bf16 it lacks (device_type.arch_lacks_bf16); fp16 is safe everywhere.+ dtype = torch.float16 if arch_lacks_bf16(arch) else torch.bfloat16+ broken = []+ # An import under inference_mode would leave nothing to backpropagate through.+ with torch.inference_mode(False), torch.enable_grad():+ try:+ reference = attend(device, SDPBackend.MATH)+ except Exception:+ return [] # math fails too: nothing to learn about the fused kernels+ for name, backend in (+ ("flash", SDPBackend.FLASH_ATTENTION),+ ("mem_efficient", SDPBackend.EFFICIENT_ATTENTION),+ ):+ try:+ ok = torch.allclose(attend(device, backend), reference, atol = 2e-2, rtol = 2e-2)+ except Exception as e:+ ok = "No available kernel" in str(e)+ if not ok:+ broken.append(name)+ return broken+++def fix_rocm_windows_fused_sdpa():+ """Turn off fused SDPA backends that fail on this Windows ROCm GPU, so attention uses math.++ Probe-gated, so it is a no-op once the kernels work. UNSLOTH_ALLOW_ROCM_FUSED_SDPA=1 opts out.+ """+ if os.environ.get("UNSLOTH_ALLOW_ROCM_FUSED_SDPA", "0") == "1":+ return+ broken = _rocm_windows_broken_sdpa_backends()+ if not broken:+ return+ try:+ import torch+ if "flash" in broken:+ torch.backends.cuda.enable_flash_sdp(False)+ if "mem_efficient" in broken:+ torch.backends.cuda.enable_mem_efficient_sdp(False)+ except Exception as e:+ logger.info(f"Unsloth: Skipping the Windows ROCm SDPA fix ({e})")+ return+ names = " or ".join("flash" if b == "flash" else "memory-efficient" for b in broken)+ fallback = "the math kernel" if len(broken) == 2 else "the remaining kernels"+ logger.warning(+ f"Unsloth: this Windows ROCm torch cannot run {names} attention; using {fallback} "+ "instead (set UNSLOTH_ALLOW_ROCM_FUSED_SDPA=1 to keep them)."+ )++def patch_openspiel_env_async():"""Apply nest_asyncio for OpenEnv EnvClient async compatibility.