Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Packed attention: full finetuning rows reach varlen (fp32 Q / K under autocast) (#15714)

Daniel Hanafa02342df89
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.py
index d0e1c1bd4a5..1eb50bfe3ff 100644
--- a/tests/test_hf_packed_attention.py
+++ b/tests/test_hf_packed_attention.py
@@ -2,6 +2,7 @@
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Packed rows on the transformers modeling path run varlen attention, and nothing else changes."""
+import contextlib
import inspect
import types
@@ -1682,3 +1683,128 @@ def test_reentrant_checkpoint_first_trace_keeps_the_block_varlen_path(monkeypatc
torch.testing.assert_close(out.float(), ref_out.float(), atol = 2e-2, rtol = 2e-2)
for g, r in zip(got, ref):
torch.testing.assert_close(g.float(), r.float(), atol = 3e-2, rtol = 3e-2)
+
+
+def _dtype_seen():
+ seen = []
+
+ def orig(
+ module,
+ q,
+ k,
+ v,
+ mask,
+ dropout = 0.0,
+ scaling = None,
+ is_causal = None,
+ **kwargs,
+ ):
+ seen.append((q.dtype, k.dtype, v.dtype))
+ return torch.nn.functional.scaled_dot_product_attention(
+ q, k, v, attn_mask = mask, scale = scaling
+ ), None
+
+ return seen, orig
+
+
+ "case, autocast, dtype, packed, expected",
+ [
+ # Full finetuning: fp32 q_norm / k_norm leave Q and K in fp32 under bf16 autocast.
+ ("fp32_under_bf16_autocast", torch.bfloat16, torch.float32, True, torch.bfloat16),
+ ("fp32_under_fp16_autocast", torch.float16, torch.float32, True, torch.float16),
+ ("fp32_without_autocast", None, torch.float32, True, torch.float32),
+ ("bf16_under_fp16_autocast", torch.float16, torch.bfloat16, True, torch.bfloat16),
+ ("unpacked_fp32_under_autocast", torch.bfloat16, torch.float32, False, torch.float32),
+ ],
+)
+def test_packed_fp32_rows_under_autocast_run_in_the_autocast_dtype(
+ monkeypatch, case, autocast, dtype, packed, expected
+):
+ """Packed fp32 Q / K / V under autocast reach the varlen checks in the dtype SDPA runs in."""
+ seen, orig = _dtype_seen()
+ hpa._ORIG_SDPA[0] = orig
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ lengths = [3, 5]
+ q, k, v = (torch.randn(1, 2, 8, 16, dtype = dtype) for _ in range(3))
+ kwargs = {"packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32)} if packed else {}
+ module = types.SimpleNamespace(is_causal = True)
+ mask = _block_causal(lengths)
+ context = (
+ torch.autocast("cpu", dtype = autocast) if autocast is not None else contextlib.nullcontext()
+ )
+ with context:
+ out, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 0.25, **kwargs)
+ ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask = mask, scale = 0.25)
+ assert seen == [(dtype, dtype, dtype)]
+ assert out.dtype == ref.dtype
+ torch.testing.assert_close(out, ref, atol = 0, rtol = 0)
+ if packed:
+ with context:
+ assert {t.dtype for t in hpa._autocast_float32(q, k, v)} == {expected}
+
+
[email protected]("autocast", [torch.bfloat16, torch.float16])
[email protected]("n_kv", [2, 8])
+def test_packed_fp32_fallback_rows_keep_their_gradients(monkeypatch, autocast, n_kv):
+ """Fallback rows must get uncast Q / K / V: GQA's dK / dV reduce in fp32 after repeat_kv."""
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ torch.manual_seed(0)
+ lengths = [37, 100, 2, 5, 56]
+ T, H, D = sum(lengths), 8, 64
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)
+ q = torch.randn(1, H, T, D, requires_grad = True)
+ k = torch.randn(1, n_kv, T, D, requires_grad = True)
+ v = torch.randn(1, n_kv, T, D, requires_grad = True)
+ mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
+ with torch.autocast("cpu", dtype = autocast):
+ out, _ = hpa._sdpa_packed_varlen(
+ module, q, k, v, mask, scaling = 0.125, packed_seq_lengths = psl
+ )
+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.125)
+ g = torch.randn_like(ref)
+ got = torch.autograd.grad(out, (q, k, v), g)
+ want = torch.autograd.grad(ref, (q, k, v), g)
+ torch.testing.assert_close(out, ref, atol = 0, rtol = 0)
+ for a, b in zip(got, want):
+ assert a.dtype == b.dtype == torch.float32
+ torch.testing.assert_close(a, b, atol = 0, rtol = 0)
+
+
+@_CUDA
+@_TORCH_VARLEN_HOST
[email protected]("autocast", [torch.bfloat16, torch.float16])
+def test_packed_fp32_rows_under_autocast_take_varlen(monkeypatch, autocast):
+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip("this torch has no window_size varlen_attn")
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ torch.manual_seed(0)
+ lengths = [37, 100, 2, 5, 230]
+ T, H, n_kv, D = sum(lengths), 8, 2, 64
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)
+ q = torch.randn(1, H, T, D, device = "cuda", requires_grad = True)
+ k = torch.randn(1, n_kv, T, D, device = "cuda", requires_grad = True)
+ v = torch.randn(1, n_kv, T, D, device = "cuda", requires_grad = True)
+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ with torch.autocast("cuda", dtype = autocast):
+ fast, _ = hpa._sdpa_packed_varlen(
+ module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl
+ )
+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1
+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"]
+ assert fast.dtype == ref.dtype == autocast and fast.shape == ref.shape
+ torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)
+ g = torch.randn_like(ref)
+ grads_fast = torch.autograd.grad(fast, (q, k, v), g)
+ grads_ref = torch.autograd.grad(ref, (q, k, v), g)
+ for a, b in zip(grads_fast, grads_ref):
+ assert a.dtype == torch.float32
+ torch.testing.assert_close(a, b, atol = 5e-2, rtol = 5e-2)
diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.py
index bf9f47af261..8cd43e11416 100644
--- a/unsloth/utils/hf_packed_attention.py
+++ b/unsloth/utils/hf_packed_attention.py
@@ -93,14 +93,27 @@ def _mask_is_block_causal(
return verdict
-def _autocast_dtype_differs(query) -> bool:
+def _autocast_dtype(device_type):
try:
- enabled = torch.is_autocast_enabled(query.device.type)
- dtype = torch.get_autocast_dtype(query.device.type) if enabled else None
+ enabled = torch.is_autocast_enabled(device_type)
+ return torch.get_autocast_dtype(device_type) if enabled else None
except (AttributeError, TypeError):
- enabled = torch.is_autocast_enabled()
- dtype = torch.get_autocast_gpu_dtype() if enabled else None
- return enabled and dtype != query.dtype
+ return torch.get_autocast_gpu_dtype() if torch.is_autocast_enabled() else None
+
+
+def _autocast_dtype_differs(query) -> bool:
+ dtype = _autocast_dtype(query.device.type)
+ return dtype is not None and dtype != query.dtype
+
+
+def _autocast_float32(query, key, value):
+ """Cast fp32 Q / K / V to the autocast dtype as SDPA would (fp32 q_norm / k_norm in full finetuning)."""
+ if not (query.dtype == key.dtype == value.dtype == torch.float32):
+ return query, key, value
+ dtype = _autocast_dtype(query.device.type)
+ if dtype not in (torch.float16, torch.bfloat16):
+ return query, key, value
+ return query.to(dtype), key.to(dtype), value.to(dtype)
_NO_WINDOW = object()
@@ -424,6 +437,10 @@ def _sdpa_packed_varlen(
HF_PACKED_ATTENTION_STATS["fallback"] += 1
return orig(module, query, key, value, attention_mask, *args, **named, **kwargs)
block_varlen = kwargs.pop(BLOCK_VARLEN_KWARG, None)
+ # Fallback gets the uncast Q / K / V so GQA's dK / dV reduce in fp32 after repeat_kv.
+ given = (query, key, value)
+ if kwargs.get("packed_seq_lengths") is not None and not _disabled():
+ query, key, value = _autocast_float32(query, key, value)
# Before any counter while tracing: Dynamo would guard on it and recompile every forward.
if block_varlen is not None and torch.compiler.is_compiling() and not _disabled():
window = _static_window(module, query, key, value, dropout, is_causal, kwargs)
@@ -458,9 +475,7 @@ def _sdpa_packed_varlen(
):
return _sdpa_packed_varlen_eager(
module,
- query,
- key,
- value,
+ *given,
attention_mask,
dropout = dropout,
scaling = scaling,
@@ -470,9 +485,7 @@ def _sdpa_packed_varlen(
if kwargs.get("packed_seq_lengths") is None or _disabled() or torch.compiler.is_compiling():
return orig(
module,
- query,
- key,
- value,
+ *given,
attention_mask,
dropout = dropout,
scaling = scaling,
@@ -527,9 +540,7 @@ def _sdpa_packed_varlen(
HF_PACKED_ATTENTION_STATS["fallback"] += 1
return orig(
module,
- query,
- key,
- value,
+ *given,
attention_mask,
dropout = dropout,
scaling = scaling,
@@ -558,9 +569,7 @@ def _sdpa_packed_varlen(
HF_PACKED_ATTENTION_STATS["fallback"] += 1
return orig(
module,
- query,
- key,
- value,
+ *given,
attention_mask,
dropout = dropout,
scaling = scaling,
@@ -587,9 +596,7 @@ def _sdpa_packed_varlen(
HF_PACKED_ATTENTION_STATS["fallback"] += 1
return orig(
module,
- query,
- key,
- value,
+ *given,
attention_mask,
dropout = dropout,
scaling = scaling,
@@ -812,7 +819,12 @@ def _block_varlen_prepare(
n_kv_heads = getattr(config, "num_key_value_heads", None) or n_heads
if not all(isinstance(n, int) for n in (head_dim, n_heads, n_kv_heads)):
return None
- policy_args = (hidden_states.device, hidden_states.dtype, head_dim, n_heads, n_kv_heads)
+ # SDPA runs in the autocast dtype when autocast casts fp32 activations (_autocast_float32).
+ dtype = hidden_states.dtype
+ if dtype == torch.float32:
+ dtype = _autocast_dtype(hidden_states.device.type) or dtype
+ dtype = dtype if dtype in (torch.float16, torch.bfloat16, torch.float32) else torch.float32
+ policy_args = (hidden_states.device, dtype, head_dim, n_heads, n_kv_heads)
wants_sdpa = _sdpa_policy_requested(*policy_args)
if policy is not None:
policy.append((policy_args, wants_sdpa))