Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Packed attention: recheck the SDPA policy before serving cached varlen metadata (#15402)

Daniel Hanf27e9788d24d
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.py
index b3b4b5510e7..3d817edc1e3 100644
--- a/tests/test_hf_packed_attention.py
+++ b/tests/test_hf_packed_attention.py
@@ -1415,6 +1415,33 @@ def test_block_varlen_kwargs_honours_the_sdpa_policy_each_step(monkeypatch):
assert got is None if wants_sdpa else got == (cu, 5)
+def test_block_varlen_kwargs_rechecks_the_sdpa_policy_on_reused_tensors(monkeypatch):
+ """A cache hit must still refuse cached metadata once the sdpa policy flips."""
+ hpa._ORIG_SDPA[0] = _plain_sdpa
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
+ monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
+ lengths = [3, 5]
+ cu = torch.tensor([0, 3, 8], dtype = torch.int32)
+ monkeypatch.setattr(
+ "unsloth.utils.packing.get_packed_info_from_kwargs",
+ lambda kwargs, device: (torch.tensor(lengths), cu, 5),
+ )
+ policy = [False]
+ monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: policy[0])
+ hidden = torch.randn(1, 8, 8)
+ kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": _block_causal(lengths)}
+ layer = _block_layer()
+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)
+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)
+ policy[0] = True
+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) is None
+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) is None
+ policy[0] = False
+ assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)
+
+
@pytest.mark.parametrize("in_graph", [True, False])
def test_traced_block_varlen_call_stays_in_the_graph(monkeypatch, in_graph):
"""With prepared metadata the traced wrapper calls the varlen op without a graph break; when the
diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.py
index 7fb9b8aae6c..895c8e22034 100644
--- a/unsloth/utils/hf_packed_attention.py
+++ b/unsloth/utils/hf_packed_attention.py
@@ -736,11 +736,14 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):
and entry[2] == (hidden_states.device, seq_lengths._version, mask._version)
# cu_seqlens built under inference_mode cannot be saved for a training step's backward.
and (not entry[4] or torch.is_inference_mode_enabled())
+ # sdpa_kernel / deterministic mode can change between calls on the same tensors.
+ and (entry[5] is None or _sdpa_policy_requested(*entry[5]) == entry[6])
):
if entry[3] is not None:
HF_PACKED_ATTENTION_STATS["compiled"] += 1
return entry[3]
- result = _block_varlen_prepare(layer, attention, hidden_states, kwargs)
+ policy = []
+ result = _block_varlen_prepare(layer, attention, hidden_states, kwargs, policy)
if (
isinstance(seq_lengths, torch.Tensor)
and isinstance(mask, torch.Tensor)
@@ -755,6 +758,7 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):
(hidden_states.device, seq_lengths._version, mask._version),
result,
torch.is_inference_mode_enabled(),
+ *(policy[0] if policy else (None, None)),
)
return result
@@ -762,7 +766,13 @@ def block_varlen_kwargs(layer, hidden_states, kwargs):
_BLOCK_PREP = {}
-def _block_varlen_prepare(layer, attention, hidden_states, kwargs):
+def _block_varlen_prepare(
+ layer,
+ attention,
+ hidden_states,
+ kwargs,
+ policy = None,
+):
if (
_disabled()
or _ORIG_SDPA[0] is None
@@ -788,9 +798,11 @@ def _block_varlen_prepare(layer, attention, hidden_states, kwargs):
n_kv_heads = getattr(config, "num_key_value_heads", None) or n_heads
if not all(isinstance(n, int) for n in (head_dim, n_heads, n_kv_heads)):
return None
- if _sdpa_policy_requested(
- hidden_states.device, hidden_states.dtype, head_dim, n_heads, n_kv_heads
- ):
+ policy_args = (hidden_states.device, hidden_states.dtype, head_dim, n_heads, n_kv_heads)
+ wants_sdpa = _sdpa_policy_requested(*policy_args)
+ if policy is not None:
+ policy.append((policy_args, wants_sdpa))
+ if wants_sdpa:
return None
if not _VARLEN_INT32_GUARD_DISABLED and _varlen_backward_overflows_int32(
kwargs["packed_seq_lengths"].numel(), total, n_heads, head_dim