Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Packed attention: run the in-graph varlen probe with grad enabled, so a checkpoint's no_grad first trace keeps the varlen path (#15414)

Daniel Han707f023d4d98
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.py
index 3d817edc1e3..5b67875decd 100644
--- a/tests/test_hf_packed_attention.py
+++ b/tests/test_hf_packed_attention.py
@@ -1549,3 +1549,134 @@ def test_block_varlen_runs_inside_the_graph_and_matches_dense_sdpa(monkeypatch,
assert sum(torch._dynamo.utils.counters["graph_break"].values()) == 0
# max_seqlen specializes once, then is dynamic: at most two graphs for four layouts.
assert torch._dynamo.utils.counters["stats"]["unique_graphs"] <= 2
+
+
[email protected]("mode", ["no_grad", "inference_mode"])
+def test_varlen_in_graph_probe_runs_its_backward_with_grad_enabled(monkeypatch, mode):
+ """Probe under no_grad must still run and check a backward (verdict is cached for all modes)."""
+ seen = []
+
+ def torch_varlen(device):
+ seen.append((torch.is_grad_enabled(), torch.is_inference_mode_enabled()))
+ return None
+
+ monkeypatch.setattr(hpa, "_torch_varlen", torch_varlen)
+ context = torch.no_grad() if mode == "no_grad" else torch.inference_mode()
+ with context:
+ assert hpa._varlen_in_graph_probe(0, torch.bfloat16, 64, 8, 2, None) is False
+ assert torch.is_grad_enabled() is False # the caller's mode is restored
+ assert seen == [(True, False)]
+
+
[email protected]("compiled", [False, True])
+def test_varlen_in_graph_probe_inside_a_nonreentrant_checkpoint(monkeypatch, compiled):
+ """First trace inside a non-reentrant checkpoint (VLMs under DDP): the probe's own backward must
+ not go through the checkpoint's saved-tensor hooks, and the outer backward must still recompute."""
+ from torch.utils.checkpoint import checkpoint
+
+ seen = []
+
+ def torch_varlen(device):
+ # Stands in for the probe's kernel forward and backward (CUDA only).
+ try:
+ q = torch.zeros(4, 4, requires_grad = True)
+ (q @ q).sin().sum().backward()
+ seen.append(q.grad is not None)
+ except Exception as error:
+ seen.append(repr(error))
+ return None
+
+ monkeypatch.setattr(hpa, "_torch_varlen", torch_varlen)
+ monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})
+
+ def block(x):
+ if hpa._varlen_in_graph(0, torch.bfloat16, 64, 8, 2, None):
+ return (x * 3).cos()
+ return (x * 2).sin()
+
+ if compiled:
+ # Dynamo calls the assume_constant_result probe eagerly while tracing, hooks still live.
+ torch._dynamo.reset()
+ block = torch.compile(block, backend = "eager", fullgraph = True)
+ x = torch.randn(4, 4, requires_grad = True)
+ checkpoint(block, x, use_reentrant = False).sum().backward()
+ assert seen == [True]
+ torch.testing.assert_close(x.grad, 2 * torch.cos(2 * x.detach()))
+
+
+@_CUDA
+@_TORCH_VARLEN_HOST
[email protected]("mode", ["no_grad", "inference_mode"])
+def test_varlen_in_graph_verdict_does_not_depend_on_the_first_grad_mode(monkeypatch, mode):
+ """Real probe under no_grad / inference_mode caches the grad-enabled answer."""
+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip(
+ reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"
+ )
+ key = (0, torch.bfloat16, 64, 8, 2, None)
+ expected = hpa._varlen_in_graph_probe(*key)
+ if not expected:
+ pytest.skip(reason = "this build's flash varlen does not take a 64-wide GQA row")
+ monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})
+ context = torch.no_grad() if mode == "no_grad" else torch.inference_mode()
+ with context:
+ assert hpa._varlen_in_graph_eager(*key) is expected
+ assert hpa._varlen_in_graph_eager(*key) is expected
+
+
+@_CUDA
+@_TORCH_VARLEN_HOST
+def test_reentrant_checkpoint_first_trace_keeps_the_block_varlen_path(monkeypatch):
+ """First compiled inside a reentrant checkpoint: forward and recompute both run varlen."""
+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip(
+ reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"
+ )
+ from torch.utils.checkpoint import checkpoint
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ def _dense(*args, **kwargs):
+ raise AssertionError("packed row fell back to SDPA over the dense mask")
+
+ hpa._ORIG_SDPA[0] = _dense
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})
+ H, n_kv, D = 8, 2, 64
+ module = types.SimpleNamespace(
+ is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = None
+ )
+ lengths = [37, 100, 2, 5, 230]
+ T = sum(lengths)
+ mask = _block_causal(lengths, "cuda")
+ psl = torch.tensor(lengths, dtype = torch.int32)
+ cu = _cu(lengths, "cuda")
+ fn = torch.compile(
+ lambda q, k, v: hpa._sdpa_packed_varlen(
+ module,
+ q,
+ k,
+ v,
+ mask,
+ scaling = 0.11,
+ packed_seq_lengths = psl,
+ **{hpa.BLOCK_VARLEN_KWARG: (cu, max(lengths))},
+ )[0],
+ fullgraph = True,
+ )
+ torch._dynamo.reset()
+ torch.manual_seed(0)
+ q, k, v = (
+ torch.randn(1, h, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ for h in (H, n_kv, n_kv)
+ )
+ out = checkpoint(fn, q, k, v, use_reentrant = True)
+ grad = torch.randn_like(out)
+ out.backward(grad) # reentrant checkpoint refuses autograd.grad
+ got = (q.grad, k.grad, v.grad)
+ # Math reference: the dense-mask backend sdpa picks can change with earlier tests in this file.
+ with torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.MATH):
+ ref_out, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
+ ref = torch.autograd.grad(ref_out, (q, k, v), grad)
+ torch.testing.assert_close(out.float(), ref_out.float(), atol = 2e-2, rtol = 2e-2)
+ for g, r in zip(got, ref):
+ torch.testing.assert_close(g.float(), r.float(), atol = 3e-2, rtol = 3e-2)
diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.py
index 895c8e22034..86677d5b6ac 100644
--- a/unsloth/utils/hf_packed_attention.py
+++ b/unsloth/utils/hf_packed_attention.py
@@ -5,6 +5,7 @@ flash-attn) instead of SDPA over a dense block-causal mask; heads past flash's 2
layers, 512) and shapes an FA4 / FA3 impl refuses run SDPA's memory-efficient kernel over the same
cu_seqlens. Anything else reaches the wrapped "sdpa"."""
+import contextlib
import inspect
import logging
import os
@@ -691,26 +692,41 @@ def _varlen_in_graph_eager(device_index, dtype, head_dim, n_heads, n_kv_heads, w
return _VARLEN_IN_GRAPH[key]
+def _no_saved_tensor_hooks():
+ # Only the innermost pair applies. Pushing raises while torch has hooks disabled: keep the ambient state.
+ enabled = getattr(torch._C._autograd, "_saved_tensors_hooks_is_enabled", lambda: True)
+ if enabled():
+ return torch.autograd.graph.saved_tensors_hooks(_identity, _identity)
+ return contextlib.nullcontext()
+
+
+def _identity(tensor):
+ return tensor
+
+
def _varlen_in_graph_probe(device_index, dtype, head_dim, n_heads, n_kv_heads, window):
"""aten's flash varlen takes this shape on this device: one real forward and backward (a build
that refuses at run time would otherwise fail inside the compiled graph)."""
- try:
- from .flash_impl import flash_impl_allows
-
- device = torch.device("cuda", device_index)
- query = torch.zeros(1, n_heads, 16, head_dim, device = device, dtype = dtype)
- key = torch.zeros(1, n_kv_heads, 16, head_dim, device = device, dtype = dtype)
- torch_varlen = _torch_varlen(device)
- # Hardware and build only: the sdpa policy can change, block_varlen_kwargs checks it each step.
- if torch_varlen is None or not flash_impl_allows(head_dim, window, device):
+ # Cached for all grad modes; first trace may be no_grad (reentrant checkpoint), where backward raises.
+ # Identity hooks: a non-reentrant checkpoint's pack hooks would make the probe's backward recompute the block.
+ with torch.inference_mode(False), torch.enable_grad(), _no_saved_tensor_hooks():
+ try:
+ from .flash_impl import flash_impl_allows
+
+ device = torch.device("cuda", device_index)
+ torch_varlen = _torch_varlen(device)
+ # Hardware and build only: the sdpa policy can change, block_varlen_kwargs checks it each step.
+ if torch_varlen is None or not flash_impl_allows(head_dim, window, device):
+ return False
+ query = torch.zeros(1, n_heads, 16, head_dim, device = device, dtype = dtype)
+ key = torch.zeros(1, n_kv_heads, 16, head_dim, device = device, dtype = dtype)
+ query.requires_grad_()
+ cu_seqlens = torch.tensor([0, 7, 16], dtype = torch.int32, device = device)
+ out = _flash_varlen(query, key, key, cu_seqlens, 9, 1.0, window)
+ out.float().sum().backward()
+ return True
+ except Exception:
return False
- query.requires_grad_()
- cu_seqlens = torch.tensor([0, 7, 16], dtype = torch.int32, device = device)
- out = _flash_varlen(query, key, key, cu_seqlens, 9, 1.0, window)
- out.float().sum().backward()
- return True
- except Exception:
- return False
_varlen_in_graph = torch._dynamo.assume_constant_result(_varlen_in_graph_eager)