Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Attention: packed fp32 rerouted to xformers keeps an fp32 output without autocast (#15410)
Daniel Hanf001b5baa919
diff --git a/tests/utils/test_attention_packed_fp32_routing.py b/tests/utils/test_attention_packed_fp32_routing.pyindex 89063da677e..91543a8425a 100644--- a/tests/utils/test_attention_packed_fp32_routing.py+++ b/tests/utils/test_attention_packed_fp32_routing.py@@ -229,3 +229,35 @@ def test_packed_fp32_cuda_parity(monkeypatch):torch.testing.assert_close(out.double().reshape(ref.shape), ref, atol = tol, rtol = tol)for a, b in zip(got, want):torch.testing.assert_close(a.double(), b, atol = tol, rtol = tol)+++def test_packed_fp32_reroute_keeps_fp32_output_on_sm100(native_packed_host, monkeypatch):+ # sm_100+ downcasts fp32 for xformers; without autocast the output must stay fp32.+ monkeypatch.setattr(ad, "_xformers_rejects_fp32", lambda device: True)+ lengths = [3, 5]+ Q, K, V = _packed_qkv(torch.float32, lengths)+ out = _attend(Q, K, V, _seq_info(lengths), ad.select_attention_backend(use_varlen = True))+ assert native_packed_host == {"xformers": 1, "sdpa": 0}+ assert out.dtype == torch.float32+ o_proj = torch.nn.Linear(H * D, H * D)+ o_proj(out.as_subclass(torch.Tensor).reshape(1, sum(lengths), H * D))++[email protected](not has_real_cuda(), reason = "needs a real CUDA device")+def test_packed_fp32_cuda_feeds_an_fp32_o_proj(monkeypatch):+ if not ad.HAS_XFORMERS:+ pytest.skip("needs xformers")+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)+ lengths = [40, 32, 24]+ T = sum(lengths)+ leaves = [torch.randn(1, h, T, 64, device = "cuda", requires_grad = True) for h in (H, KV, KV)]+ o_proj = torch.nn.Linear(H * 64, H * 64, device = "cuda")+ out = _attend(*leaves, _seq_info(lengths, "cuda"), ad.select_attention_backend(True))+ assert out.dtype == torch.float32+ o_proj(out.reshape(1, T, H * 64)).sum().backward()+ assert all(t.grad is not None and t.grad.dtype == torch.float32 for t in leaves)+ with torch.autocast("cuda", dtype = torch.bfloat16):+ out = _attend(*leaves, _seq_info(lengths, "cuda"), ad.select_attention_backend(True))+ if ad._xformers_rejects_fp32(leaves[0].device):+ assert out.dtype == torch.bfloat16diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex e77d471e950..ce2f09d6698 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -1287,8 +1287,12 @@ def _run_attention(if backend == FLASH_VARLEN and context.seq_info is None:backend = FLASH_DENSE if HAS_FLASH_ATTENTION else SDPAxformers_fp32_unsupported = _xformers_rejects_fp32(Q.device)+ # sm_100+ downcast below must not hand bf16 to an fp32 o_proj (old SDPA path returned fp32).+ packed_out_dtype = Noneif backend == SDPA and _packed_fp32_prefers_xformers(config, context, Q, K, V):backend = XFORMERS+ if not torch.is_autocast_enabled():+ packed_out_dtype = Q.dtypeif _DEVICE_BACKENDS is not None and backend != SDPA:device_backend = _backend_for_device(backend, Q.device, Q.dtype)if device_backend != backend and _configured_softcap(config):@@ -1513,7 +1517,7 @@ def _run_attention(out = out.reshape(bsz, q_len, n_heads, head_dim)else:out = out.view(bsz, q_len, n_heads, head_dim)- return out+ return out if packed_out_dtype is None else out.to(packed_out_dtype)else:local_mask = context.attention_maskis_causal_local = False