Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Attention: packed fp32 rerouted to xformers keeps an fp32 output without autocast (#15410)

Daniel Hanf001b5baa919
diff --git a/tests/utils/test_attention_packed_fp32_routing.py b/tests/utils/test_attention_packed_fp32_routing.py
index 89063da677e..91543a8425a 100644
--- a/tests/utils/test_attention_packed_fp32_routing.py
+++ b/tests/utils/test_attention_packed_fp32_routing.py
@@ -229,3 +229,35 @@ def test_packed_fp32_cuda_parity(monkeypatch):
torch.testing.assert_close(out.double().reshape(ref.shape), ref, atol = tol, rtol = tol)
for a, b in zip(got, want):
torch.testing.assert_close(a.double(), b, atol = tol, rtol = tol)
+
+
+def test_packed_fp32_reroute_keeps_fp32_output_on_sm100(native_packed_host, monkeypatch):
+ # sm_100+ downcasts fp32 for xformers; without autocast the output must stay fp32.
+ monkeypatch.setattr(ad, "_xformers_rejects_fp32", lambda device: True)
+ lengths = [3, 5]
+ Q, K, V = _packed_qkv(torch.float32, lengths)
+ out = _attend(Q, K, V, _seq_info(lengths), ad.select_attention_backend(use_varlen = True))
+ assert native_packed_host == {"xformers": 1, "sdpa": 0}
+ assert out.dtype == torch.float32
+ o_proj = torch.nn.Linear(H * D, H * D)
+ o_proj(out.as_subclass(torch.Tensor).reshape(1, sum(lengths), H * D))
+
+
[email protected](not has_real_cuda(), reason = "needs a real CUDA device")
+def test_packed_fp32_cuda_feeds_an_fp32_o_proj(monkeypatch):
+ if not ad.HAS_XFORMERS:
+ pytest.skip("needs xformers")
+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)
+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)
+ lengths = [40, 32, 24]
+ T = sum(lengths)
+ leaves = [torch.randn(1, h, T, 64, device = "cuda", requires_grad = True) for h in (H, KV, KV)]
+ o_proj = torch.nn.Linear(H * 64, H * 64, device = "cuda")
+ out = _attend(*leaves, _seq_info(lengths, "cuda"), ad.select_attention_backend(True))
+ assert out.dtype == torch.float32
+ o_proj(out.reshape(1, T, H * 64)).sum().backward()
+ assert all(t.grad is not None and t.grad.dtype == torch.float32 for t in leaves)
+ with torch.autocast("cuda", dtype = torch.bfloat16):
+ out = _attend(*leaves, _seq_info(lengths, "cuda"), ad.select_attention_backend(True))
+ if ad._xformers_rejects_fp32(leaves[0].device):
+ assert out.dtype == torch.bfloat16
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index e77d471e950..ce2f09d6698 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -1287,8 +1287,12 @@ def _run_attention(
if backend == FLASH_VARLEN and context.seq_info is None:
backend = FLASH_DENSE if HAS_FLASH_ATTENTION else SDPA
xformers_fp32_unsupported = _xformers_rejects_fp32(Q.device)
+ # sm_100+ downcast below must not hand bf16 to an fp32 o_proj (old SDPA path returned fp32).
+ packed_out_dtype = None
if backend == SDPA and _packed_fp32_prefers_xformers(config, context, Q, K, V):
backend = XFORMERS
+ if not torch.is_autocast_enabled():
+ packed_out_dtype = Q.dtype
if _DEVICE_BACKENDS is not None and backend != SDPA:
device_backend = _backend_for_device(backend, Q.device, Q.dtype)
if device_backend != backend and _configured_softcap(config):
@@ -1513,7 +1517,7 @@ def _run_attention(
out = out.reshape(bsz, q_len, n_heads, head_dim)
else:
out = out.view(bsz, q_len, n_heads, head_dim)
- return out
+ return out if packed_out_dtype is None else out.to(packed_out_dtype)
else:
local_mask = context.attention_mask
is_causal_local = False