Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Packed attention: recheck the SDPA policy before serving cached varlen metadata (#15402)
Daniel Hanf27e9788d24d
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.pyindex b3b4b5510e7..3d817edc1e3 100644--- a/tests/test_hf_packed_attention.py+++ b/tests/test_hf_packed_attention.py@@ -1415,6 +1415,33 @@ def test_block_varlen_kwargs_honours_the_sdpa_policy_each_step(monkeypatch):assert got is None if wants_sdpa else got == (cu, 5)+def test_block_varlen_kwargs_rechecks_the_sdpa_policy_on_reused_tensors(monkeypatch):+ """A cache hit must still refuse cached metadata once the sdpa policy flips."""+ hpa._ORIG_SDPA[0] = _plain_sdpa+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())+ monkeypatch.setattr(hpa, "_BLOCK_PREP", {})+ lengths = [3, 5]+ cu = torch.tensor([0, 3, 8], dtype = torch.int32)+ monkeypatch.setattr(+ "unsloth.utils.packing.get_packed_info_from_kwargs",+ lambda kwargs, device: (torch.tensor(lengths), cu, 5),+ )+ policy = [False]+ monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: policy[0])+ hidden = torch.randn(1, 8, 8)+ kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": _block_causal(lengths)}+ layer = _block_layer()+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)+ policy[0] = True+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) is None+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) is None+ policy[0] = False+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)++@pytest.mark.parametrize("in_graph", [True, False])def test_traced_block_varlen_call_stays_in_the_graph(monkeypatch, in_graph):"""With prepared metadata the traced wrapper calls the varlen op without a graph break; when thediff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.pyindex 7fb9b8aae6c..895c8e22034 100644--- a/unsloth/utils/hf_packed_attention.py+++ b/unsloth/utils/hf_packed_attention.py@@ -736,11 +736,14 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):and entry[2] == (hidden_states.device, seq_lengths._version, mask._version)# cu_seqlens built under inference_mode cannot be saved for a training step's backward.and (not entry[4] or torch.is_inference_mode_enabled())+ # sdpa_kernel / deterministic mode can change between calls on the same tensors.+ and (entry[5] is None or _sdpa_policy_requested(*entry[5]) == entry[6])):if entry[3] is not None:HF_PACKED_ATTENTION_STATS["compiled"] += 1return entry[3]- result = _block_varlen_prepare(layer, attention, hidden_states, kwargs)+ policy = []+ result = _block_varlen_prepare(layer, attention, hidden_states, kwargs, policy)if (isinstance(seq_lengths, torch.Tensor)and isinstance(mask, torch.Tensor)@@ -755,6 +758,7 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):(hidden_states.device, seq_lengths._version, mask._version),result,torch.is_inference_mode_enabled(),+ *(policy[0] if policy else (None, None)),)return result@@ -762,7 +766,13 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):_BLOCK_PREP = {}-def _block_varlen_prepare(layer, attention, hidden_states, kwargs):+def _block_varlen_prepare(+ layer,+ attention,+ hidden_states,+ kwargs,+ policy = None,+):if (_disabled()or _ORIG_SDPA[0] is None@@ -788,9 +798,11 @@ def _block_varlen_prepare(layer, attention, hidden_states, kwargs):n_kv_heads = getattr(config, "num_key_value_heads", None) or n_headsif not all(isinstance(n, int) for n in (head_dim, n_heads, n_kv_heads)):return None- if _sdpa_policy_requested(- hidden_states.device, hidden_states.dtype, head_dim, n_heads, n_kv_heads- ):+ policy_args = (hidden_states.device, hidden_states.dtype, head_dim, n_heads, n_kv_heads)+ wants_sdpa = _sdpa_policy_requested(*policy_args)+ if policy is not None:+ policy.append((policy_args, wants_sdpa))+ if wants_sdpa:return Noneif not _VARLEN_INT32_GUARD_DISABLED and _varlen_backward_overflows_int32(kwargs["packed_seq_lengths"].numel(), total, n_heads, head_dim