Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Packed attention: full finetuning rows reach varlen (fp32 Q / K under autocast) (#15714)
Daniel Hanafa02342df89
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.pyindex d0e1c1bd4a5..1eb50bfe3ff 100644--- a/tests/test_hf_packed_attention.py+++ b/tests/test_hf_packed_attention.py@@ -2,6 +2,7 @@# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved."""Packed rows on the transformers modeling path run varlen attention, and nothing else changes."""+import contextlibimport inspectimport types@@ -1682,3 +1683,128 @@ def test_reentrant_checkpoint_first_trace_keeps_the_block_varlen_path(monkeypatctorch.testing.assert_close(out.float(), ref_out.float(), atol = 2e-2, rtol = 2e-2)for g, r in zip(got, ref):torch.testing.assert_close(g.float(), r.float(), atol = 3e-2, rtol = 3e-2)+++def _dtype_seen():+ seen = []++ def orig(+ module,+ q,+ k,+ v,+ mask,+ dropout = 0.0,+ scaling = None,+ is_causal = None,+ **kwargs,+ ):+ seen.append((q.dtype, k.dtype, v.dtype))+ return torch.nn.functional.scaled_dot_product_attention(+ q, k, v, attn_mask = mask, scale = scaling+ ), None++ return seen, orig+++ "case, autocast, dtype, packed, expected",+ [+ # Full finetuning: fp32 q_norm / k_norm leave Q and K in fp32 under bf16 autocast.+ ("fp32_under_bf16_autocast", torch.bfloat16, torch.float32, True, torch.bfloat16),+ ("fp32_under_fp16_autocast", torch.float16, torch.float32, True, torch.float16),+ ("fp32_without_autocast", None, torch.float32, True, torch.float32),+ ("bf16_under_fp16_autocast", torch.float16, torch.bfloat16, True, torch.bfloat16),+ ("unpacked_fp32_under_autocast", torch.bfloat16, torch.float32, False, torch.float32),+ ],+)+def test_packed_fp32_rows_under_autocast_run_in_the_autocast_dtype(+ monkeypatch, case, autocast, dtype, packed, expected+):+ """Packed fp32 Q / K / V under autocast reach the varlen checks in the dtype SDPA runs in."""+ seen, orig = _dtype_seen()+ hpa._ORIG_SDPA[0] = orig+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ lengths = [3, 5]+ q, k, v = (torch.randn(1, 2, 8, 16, dtype = dtype) for _ in range(3))+ kwargs = {"packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32)} if packed else {}+ module = types.SimpleNamespace(is_causal = True)+ mask = _block_causal(lengths)+ context = (+ torch.autocast("cpu", dtype = autocast) if autocast is not None else contextlib.nullcontext()+ )+ with context:+ out, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 0.25, **kwargs)+ ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask = mask, scale = 0.25)+ assert seen == [(dtype, dtype, dtype)]+ assert out.dtype == ref.dtype+ torch.testing.assert_close(out, ref, atol = 0, rtol = 0)+ if packed:+ with context:+ assert {t.dtype for t in hpa._autocast_float32(q, k, v)} == {expected}++[email protected]("autocast", [torch.bfloat16, torch.float16])[email protected]("n_kv", [2, 8])+def test_packed_fp32_fallback_rows_keep_their_gradients(monkeypatch, autocast, n_kv):+ """Fallback rows must get uncast Q / K / V: GQA's dK / dV reduce in fp32 after repeat_kv."""+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ torch.manual_seed(0)+ lengths = [37, 100, 2, 5, 56]+ T, H, D = sum(lengths), 8, 64+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)+ q = torch.randn(1, H, T, D, requires_grad = True)+ k = torch.randn(1, n_kv, T, D, requires_grad = True)+ v = torch.randn(1, n_kv, T, D, requires_grad = True)+ mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)+ with torch.autocast("cpu", dtype = autocast):+ out, _ = hpa._sdpa_packed_varlen(+ module, q, k, v, mask, scaling = 0.125, packed_seq_lengths = psl+ )+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.125)+ g = torch.randn_like(ref)+ got = torch.autograd.grad(out, (q, k, v), g)+ want = torch.autograd.grad(ref, (q, k, v), g)+ torch.testing.assert_close(out, ref, atol = 0, rtol = 0)+ for a, b in zip(got, want):+ assert a.dtype == b.dtype == torch.float32+ torch.testing.assert_close(a, b, atol = 0, rtol = 0)+++@_CUDA+@_TORCH_VARLEN_HOST[email protected]("autocast", [torch.bfloat16, torch.float16])+def test_packed_fp32_rows_under_autocast_take_varlen(monkeypatch, autocast):+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip("this torch has no window_size varlen_attn")+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ torch.manual_seed(0)+ lengths = [37, 100, 2, 5, 230]+ T, H, n_kv, D = sum(lengths), 8, 2, 64+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)+ q = torch.randn(1, H, T, D, device = "cuda", requires_grad = True)+ k = torch.randn(1, n_kv, T, D, device = "cuda", requires_grad = True)+ v = torch.randn(1, n_kv, T, D, device = "cuda", requires_grad = True)+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ with torch.autocast("cuda", dtype = autocast):+ fast, _ = hpa._sdpa_packed_varlen(+ module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl+ )+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"]+ assert fast.dtype == ref.dtype == autocast and fast.shape == ref.shape+ torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)+ g = torch.randn_like(ref)+ grads_fast = torch.autograd.grad(fast, (q, k, v), g)+ grads_ref = torch.autograd.grad(ref, (q, k, v), g)+ for a, b in zip(grads_fast, grads_ref):+ assert a.dtype == torch.float32+ torch.testing.assert_close(a, b, atol = 5e-2, rtol = 5e-2)diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.pyindex bf9f47af261..8cd43e11416 100644--- a/unsloth/utils/hf_packed_attention.py+++ b/unsloth/utils/hf_packed_attention.py@@ -93,14 +93,27 @@ def _mask_is_block_causal(return verdict-def _autocast_dtype_differs(query) -> bool:+def _autocast_dtype(device_type):try:- enabled = torch.is_autocast_enabled(query.device.type)- dtype = torch.get_autocast_dtype(query.device.type) if enabled else None+ enabled = torch.is_autocast_enabled(device_type)+ return torch.get_autocast_dtype(device_type) if enabled else Noneexcept (AttributeError, TypeError):- enabled = torch.is_autocast_enabled()- dtype = torch.get_autocast_gpu_dtype() if enabled else None- return enabled and dtype != query.dtype+ return torch.get_autocast_gpu_dtype() if torch.is_autocast_enabled() else None+++def _autocast_dtype_differs(query) -> bool:+ dtype = _autocast_dtype(query.device.type)+ return dtype is not None and dtype != query.dtype+++def _autocast_float32(query, key, value):+ """Cast fp32 Q / K / V to the autocast dtype as SDPA would (fp32 q_norm / k_norm in full finetuning)."""+ if not (query.dtype == key.dtype == value.dtype == torch.float32):+ return query, key, value+ dtype = _autocast_dtype(query.device.type)+ if dtype not in (torch.float16, torch.bfloat16):+ return query, key, value+ return query.to(dtype), key.to(dtype), value.to(dtype)_NO_WINDOW = object()@@ -424,6 +437,10 @@ def _sdpa_packed_varlen(HF_PACKED_ATTENTION_STATS["fallback"] += 1return orig(module, query, key, value, attention_mask, *args, **named, **kwargs)block_varlen = kwargs.pop(BLOCK_VARLEN_KWARG, None)+ # Fallback gets the uncast Q / K / V so GQA's dK / dV reduce in fp32 after repeat_kv.+ given = (query, key, value)+ if kwargs.get("packed_seq_lengths") is not None and not _disabled():+ query, key, value = _autocast_float32(query, key, value)# Before any counter while tracing: Dynamo would guard on it and recompile every forward.if block_varlen is not None and torch.compiler.is_compiling() and not _disabled():window = _static_window(module, query, key, value, dropout, is_causal, kwargs)@@ -458,9 +475,7 @@ def _sdpa_packed_varlen():return _sdpa_packed_varlen_eager(module,- query,- key,- value,+ *given,attention_mask,dropout = dropout,scaling = scaling,@@ -470,9 +485,7 @@ def _sdpa_packed_varlen(if kwargs.get("packed_seq_lengths") is None or _disabled() or torch.compiler.is_compiling():return orig(module,- query,- key,- value,+ *given,attention_mask,dropout = dropout,scaling = scaling,@@ -527,9 +540,7 @@ def _sdpa_packed_varlen(HF_PACKED_ATTENTION_STATS["fallback"] += 1return orig(module,- query,- key,- value,+ *given,attention_mask,dropout = dropout,scaling = scaling,@@ -558,9 +569,7 @@ def _sdpa_packed_varlen(HF_PACKED_ATTENTION_STATS["fallback"] += 1return orig(module,- query,- key,- value,+ *given,attention_mask,dropout = dropout,scaling = scaling,@@ -587,9 +596,7 @@ def _sdpa_packed_varlen(HF_PACKED_ATTENTION_STATS["fallback"] += 1return orig(module,- query,- key,- value,+ *given,attention_mask,dropout = dropout,scaling = scaling,@@ -812,7 +819,12 @@ def _block_varlen_prepare(n_kv_heads = getattr(config, "num_key_value_heads", None) or n_headsif not all(isinstance(n, int) for n in (head_dim, n_heads, n_kv_heads)):return None- policy_args = (hidden_states.device, hidden_states.dtype, head_dim, n_heads, n_kv_heads)+ # SDPA runs in the autocast dtype when autocast casts fp32 activations (_autocast_float32).+ dtype = hidden_states.dtype+ if dtype == torch.float32:+ dtype = _autocast_dtype(hidden_states.device.type) or dtype+ dtype = dtype if dtype in (torch.float16, torch.bfloat16, torch.float32) else torch.float32+ policy_args = (hidden_states.device, dtype, head_dim, n_heads, n_kv_heads)wants_sdpa = _sdpa_policy_requested(*policy_args)if policy is not None:policy.append((policy_args, wants_sdpa))