Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Windows ROCm: use math attention where the fused SDPA kernels fail (#12639)

Daniel Hanad13de2e0159
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 91339218315..755be7d36e0 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -155,7 +155,9 @@ def _probe_sdpa_kernels(device: str, dtype: Any) -> tuple[str, ...]:
continue
try:
with sdpa_kernel([backend]):
- torch.nn.functional.scaled_dot_product_attention(q, q, q)
+ out = torch.nn.functional.scaled_dot_product_attention(q, q, q)
+ # Windows ROCm raises a failed fused launch only at the next checked kernel: take it here.
+ out.float().sum().item()
available.append(name)
except Exception: # noqa: BLE001 - "No available kernel" is the answer, not an error
continue
diff --git a/studio/backend/tests/test_diffusion_attention.py b/studio/backend/tests/test_diffusion_attention.py
index 74b2c796268..6c5beffeae3 100644
--- a/studio/backend/tests/test_diffusion_attention.py
+++ b/studio/backend/tests/test_diffusion_attention.py
@@ -884,6 +884,26 @@ def test_any_subquadratic_kernel_means_not_math_only(kernels, monkeypatch):
assert att.sdpa_math_only(_target()) is False
+def test_a_fused_launch_that_fails_late_is_not_reported_available(monkeypatch):
+ # Windows ROCm gfx1151: the fused call returns, and its hipErrorInvalidValue only surfaces on
+ # the next checked kernel. The probe must take that error itself, not hand it to a later op.
+ torch = pytest.importorskip("torch")
+
+ class _Pending:
+ def float(self):
+ raise RuntimeError("CUDA error: invalid argument")
+
+ real = torch.nn.functional.scaled_dot_product_attention
+
+ def _sdpa(q, k, v, *a, **kw):
+ if not torch.backends.cuda.math_sdp_enabled():
+ return _Pending()
+ return real(q, k, v, *a, **kw)
+
+ monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", _sdpa)
+ assert att._probe_sdpa_kernels("cpu", torch.float32) == ("math",)
+
+
def test_an_unanswerable_probe_is_not_a_math_only_verdict(monkeypatch):
# "Only math" is a claim about the hardware. A probe that could not run (no torch, no device,
# an allocator failure) has made no such claim, and must not refuse or warn on a guess.
diff --git a/tests/test_rocm_windows_fused_sdpa.py b/tests/test_rocm_windows_fused_sdpa.py
new file mode 100644
index 00000000000..034cd78db71
--- /dev/null
+++ b/tests/test_rocm_windows_fused_sdpa.py
@@ -0,0 +1,158 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""On Windows ROCm (torch 2.11.0+rocm7.14.1, gfx1151) flash and memory-efficient SDPA fail with
+hipErrorInvalidValue on every call. `import unsloth` must turn off exactly the backends that fail,
+so attention runs on the math kernel. The failed launch only surfaces at the next checked kernel
+launch, so the probe must not blame a later call. The last two tests need the real host."""
+
+import json
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+
+def _on_windows_rocm():
+ if sys.platform != "win32" or not torch.cuda.is_available():
+ return False
+ # Same detection as the fix: some AMD wheels tag only the torch version.
+ from unsloth.import_fixes import _is_rocm_torch_build
+ return bool(getattr(torch.version, "hip", None)) or _is_rocm_torch_build()
+
+
+def _run(code, *args):
+ p = subprocess.run(
+ [sys.executable, "-c", code, *args], capture_output = True, text = True, timeout = 900
+ )
+ line = [l for l in p.stdout.splitlines() if l.startswith("RESULT ")]
+ assert line, p.stdout[-2000:] + p.stderr[-3000:]
+ return json.loads(line[-1].split(" ", 1)[1])
+
+
+def test_probe_is_windows_rocm_only():
+ if _on_windows_rocm():
+ pytest.skip(reason = "this host is the Windows ROCm one the probe targets")
+ from unsloth.import_fixes import _rocm_windows_broken_sdpa_backends
+ assert _rocm_windows_broken_sdpa_backends() == []
+
+
+ "broken, expected",
+ [
+ ([], [True, True, True]),
+ (["flash"], [False, True, True]),
+ (["mem_efficient"], [True, False, True]),
+ (["flash", "mem_efficient"], [False, False, True]),
+ ],
+)
+def test_only_broken_backends_are_turned_off(monkeypatch, broken, expected):
+ import unsloth.import_fixes as fixes
+
+ cuda = torch.backends.cuda
+ saved = (cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled())
+ monkeypatch.delenv("UNSLOTH_ALLOW_ROCM_FUSED_SDPA", raising = False)
+ monkeypatch.setattr(fixes, "_rocm_windows_broken_sdpa_backends", lambda: list(broken))
+ try:
+ cuda.enable_flash_sdp(True)
+ cuda.enable_mem_efficient_sdp(True)
+ fixes.fix_rocm_windows_fused_sdpa()
+ got = [cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled(), cuda.math_sdp_enabled()]
+ finally:
+ cuda.enable_flash_sdp(saved[0])
+ cuda.enable_mem_efficient_sdp(saved[1])
+ assert got == expected
+
+
+def test_opt_out_skips_the_probe(monkeypatch):
+ import unsloth.import_fixes as fixes
+
+ monkeypatch.setenv("UNSLOTH_ALLOW_ROCM_FUSED_SDPA", "1")
+ monkeypatch.setattr(
+ fixes,
+ "_rocm_windows_broken_sdpa_backends",
+ lambda: pytest.fail("probed despite the opt-out"),
+ )
+ fixes.fix_rocm_windows_fused_sdpa()
+
+
+_PROBE_TWICE = textwrap.dedent(
+ """
+ import json
+ import torch
+ from unsloth.import_fixes import _rocm_windows_broken_sdpa_backends
+ first = _rocm_windows_broken_sdpa_backends()
+ with torch.inference_mode(): # an import inside inference_mode must probe the same
+ second = _rocm_windows_broken_sdpa_backends()
+ # Nothing may be left pending: a stale launch error would raise on this checked kernel.
+ torch.ones(4, device = "cuda").float().sum().item()
+ print("RESULT " + json.dumps([first, second]))
+ """
+)
+
+
+def test_windows_rocm_probe_is_stable_and_leaves_no_pending_error():
+ if not _on_windows_rocm():
+ pytest.skip(reason = "needs a Windows ROCm GPU to reproduce the launch failure")
+ first, second = _run(_PROBE_TWICE)
+ assert first == second
+
+
+_ATTENTION = textwrap.dedent(
+ """
+ import json
+ import unsloth # noqa: F401 (applies the import-time fixes)
+ import torch
+ import torch.nn.functional as F
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ # fp16: gfx10 claims bf16 it lacks, and fp16 exercises the same fused kernels.
+ q, k, v = (torch.randn(2, 4, 24, 64, device = "cuda", dtype = torch.float16, generator = g)
+ for _ in range(3))
+ mask = torch.ones(24, 24, device = "cuda", dtype = torch.bool).tril()
+ out = {}
+ for name, grad, kw, nkv in (
+ ("causal_fwd", False, {"is_causal": True}, 4),
+ ("mask_fwd", False, {"attn_mask": mask}, 4),
+ ("causal_bwd", True, {"is_causal": True}, 4),
+ ("mask_bwd", True, {"attn_mask": mask}, 4),
+ ("gqa_bwd", True, {"is_causal": True, "enable_gqa": True}, 2),
+ ):
+ def run(math):
+ qq = q.clone().requires_grad_(grad)
+ if math:
+ with sdpa_kernel([SDPBackend.MATH]):
+ o = F.scaled_dot_product_attention(qq, k[:, :nkv], v[:, :nkv], **kw)
+ else:
+ o = F.scaled_dot_product_attention(qq, k[:, :nkv], v[:, :nkv], **kw)
+ o = o.float() # a checked launch: a failed attention launch raises here
+ if grad:
+ o.sum().backward()
+ return o.detach(), qq.grad.float()
+ return o, o
+ try:
+ o, d = run(False)
+ ro, rd = run(True)
+ out[name] = bool(torch.allclose(o, ro, atol = 2e-2, rtol = 2e-2)
+ and torch.allclose(d, rd, atol = 2e-2, rtol = 2e-2))
+ except Exception as e:
+ out[name] = f"{type(e).__name__}: {e}"[:200]
+ print("RESULT " + json.dumps(out))
+ """
+)
+
+
+def test_windows_rocm_attention_runs_after_import():
+ if not _on_windows_rocm():
+ pytest.skip(reason = "needs a Windows ROCm GPU to reproduce the launch failure")
+ assert _run(_ATTENTION) == {
+ "causal_fwd": True,
+ "mask_fwd": True,
+ "causal_bwd": True,
+ "mask_bwd": True,
+ "gqa_bwd": True,
+ }
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 3f8790078df..127f35bc4cf 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -282,6 +282,7 @@ from .import_fixes import (
fix_vllm_guided_decoding_params,
fix_vllm_pdl_blackwell,
fix_cudnn_sdpa_d256_masked_backward,
+ fix_rocm_windows_fused_sdpa,
fix_triton_compiled_kernel_missing_attrs,
fix_dynamo_config_thread_visibility,
patch_trunc_normal_precision_issue,
@@ -366,6 +367,8 @@ fix_vllm_guided_decoding_params()
fix_trl_vllm_ascend()
fix_vllm_pdl_blackwell()
fix_cudnn_sdpa_d256_masked_backward()
+# Windows ROCm only, probe-gated: fused attention fails on every call there (gfx1151, torch 2.11).
+fix_rocm_windows_fused_sdpa()
fix_triton_compiled_kernel_missing_attrs()
# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss patches raise the dynamo
# recompile limits, so those settings reach the autograd worker threads on torch >= 2.12.
@@ -425,6 +428,7 @@ del fix_vllm_guided_decoding_params
del fix_trl_vllm_ascend
del fix_vllm_pdl_blackwell
del fix_cudnn_sdpa_d256_masked_backward
+del fix_rocm_windows_fused_sdpa
del fix_triton_compiled_kernel_missing_attrs
del fix_dynamo_config_thread_visibility
del patch_trunc_normal_precision_issue
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index ed1e6b04b2e..109274fa51c 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -6639,6 +6639,96 @@ def fix_cudnn_sdpa_d256_masked_backward():
)
+def _rocm_windows_broken_sdpa_backends():
+ """Fused SDPA backends ("flash", "mem_efficient") failing on this process's Windows ROCm GPU.
+
+ torch 2.11.0+rocm7.14.1 on gfx1151 fails every fused call (hipErrorInvalidValue), raised only
+ by the next checked launch, so each probe ends in one and is compared against math.
+ """
+ if sys.platform != "win32":
+ return []
+ try:
+ import torch
+ import torch.nn.functional as F
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ # Some AMD wheels tag only __version__, so torch.version.hip alone misses them.
+ is_rocm = bool(getattr(torch.version, "hip", None)) or _is_rocm_torch_build()
+ if not is_rocm or not torch.cuda.is_available():
+ return []
+ except Exception:
+ return []
+
+ def attend(device, backend):
+ g = torch.Generator(device = device).manual_seed(0)
+ q, k, v = (
+ torch.randn(1, 2, 16, 64, device = device, dtype = dtype, generator = g) for _ in range(3)
+ )
+ q.requires_grad_(True)
+ with sdpa_kernel([backend]):
+ out = F.scaled_dot_product_attention(q, k, v, is_causal = True)
+ out = out.float() # checked launch: a failed attention launch raises here
+ out.sum().backward()
+ dq = q.grad.float()
+ dq.sum().item()
+ return torch.cat([out.detach().flatten(), dq.flatten()])
+
+ # Only this process's device: probing every visible GPU would open a HIP context on each.
+ device = torch.device("cuda", torch.cuda.current_device())
+ try:
+ from .device_type import arch_lacks_bf16
+ arch = torch.cuda.get_device_properties(device).gcnArchName
+ except Exception:
+ arch_lacks_bf16, arch = (lambda _: True), None
+ # gfx10 claims bf16 it lacks (device_type.arch_lacks_bf16); fp16 is safe everywhere.
+ dtype = torch.float16 if arch_lacks_bf16(arch) else torch.bfloat16
+ broken = []
+ # An import under inference_mode would leave nothing to backpropagate through.
+ with torch.inference_mode(False), torch.enable_grad():
+ try:
+ reference = attend(device, SDPBackend.MATH)
+ except Exception:
+ return [] # math fails too: nothing to learn about the fused kernels
+ for name, backend in (
+ ("flash", SDPBackend.FLASH_ATTENTION),
+ ("mem_efficient", SDPBackend.EFFICIENT_ATTENTION),
+ ):
+ try:
+ ok = torch.allclose(attend(device, backend), reference, atol = 2e-2, rtol = 2e-2)
+ except Exception as e:
+ ok = "No available kernel" in str(e)
+ if not ok:
+ broken.append(name)
+ return broken
+
+
+def fix_rocm_windows_fused_sdpa():
+ """Turn off fused SDPA backends that fail on this Windows ROCm GPU, so attention uses math.
+
+ Probe-gated, so it is a no-op once the kernels work. UNSLOTH_ALLOW_ROCM_FUSED_SDPA=1 opts out.
+ """
+ if os.environ.get("UNSLOTH_ALLOW_ROCM_FUSED_SDPA", "0") == "1":
+ return
+ broken = _rocm_windows_broken_sdpa_backends()
+ if not broken:
+ return
+ try:
+ import torch
+ if "flash" in broken:
+ torch.backends.cuda.enable_flash_sdp(False)
+ if "mem_efficient" in broken:
+ torch.backends.cuda.enable_mem_efficient_sdp(False)
+ except Exception as e:
+ logger.info(f"Unsloth: Skipping the Windows ROCm SDPA fix ({e})")
+ return
+ names = " or ".join("flash" if b == "flash" else "memory-efficient" for b in broken)
+ fallback = "the math kernel" if len(broken) == 2 else "the remaining kernels"
+ logger.warning(
+ f"Unsloth: this Windows ROCm torch cannot run {names} attention; using {fallback} "
+ "instead (set UNSLOTH_ALLOW_ROCM_FUSED_SDPA=1 to keep them)."
+ )
+
+
def patch_openspiel_env_async():
"""Apply nest_asyncio for OpenEnv EnvClient async compatibility.