Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Packed attention: run the in-graph varlen probe with grad enabled, so a checkpoint's no_grad first trace keeps the varlen path (#15414)
Daniel Han707f023d4d98
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.pyindex 3d817edc1e3..5b67875decd 100644--- a/tests/test_hf_packed_attention.py+++ b/tests/test_hf_packed_attention.py@@ -1549,3 +1549,134 @@ def test_block_varlen_runs_inside_the_graph_and_matches_dense_sdpa(monkeypatch,assert sum(torch._dynamo.utils.counters["graph_break"].values()) == 0# max_seqlen specializes once, then is dynamic: at most two graphs for four layouts.assert torch._dynamo.utils.counters["stats"]["unique_graphs"] <= 2++[email protected]("mode", ["no_grad", "inference_mode"])+def test_varlen_in_graph_probe_runs_its_backward_with_grad_enabled(monkeypatch, mode):+ """Probe under no_grad must still run and check a backward (verdict is cached for all modes)."""+ seen = []++ def torch_varlen(device):+ seen.append((torch.is_grad_enabled(), torch.is_inference_mode_enabled()))+ return None++ monkeypatch.setattr(hpa, "_torch_varlen", torch_varlen)+ context = torch.no_grad() if mode == "no_grad" else torch.inference_mode()+ with context:+ assert hpa._varlen_in_graph_probe(0, torch.bfloat16, 64, 8, 2, None) is False+ assert torch.is_grad_enabled() is False # the caller's mode is restored+ assert seen == [(True, False)]++[email protected]("compiled", [False, True])+def test_varlen_in_graph_probe_inside_a_nonreentrant_checkpoint(monkeypatch, compiled):+ """First trace inside a non-reentrant checkpoint (VLMs under DDP): the probe's own backward must+ not go through the checkpoint's saved-tensor hooks, and the outer backward must still recompute."""+ from torch.utils.checkpoint import checkpoint++ seen = []++ def torch_varlen(device):+ # Stands in for the probe's kernel forward and backward (CUDA only).+ try:+ q = torch.zeros(4, 4, requires_grad = True)+ (q @ q).sin().sum().backward()+ seen.append(q.grad is not None)+ except Exception as error:+ seen.append(repr(error))+ return None++ monkeypatch.setattr(hpa, "_torch_varlen", torch_varlen)+ monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})++ def block(x):+ if hpa._varlen_in_graph(0, torch.bfloat16, 64, 8, 2, None):+ return (x * 3).cos()+ return (x * 2).sin()++ if compiled:+ # Dynamo calls the assume_constant_result probe eagerly while tracing, hooks still live.+ torch._dynamo.reset()+ block = torch.compile(block, backend = "eager", fullgraph = True)+ x = torch.randn(4, 4, requires_grad = True)+ checkpoint(block, x, use_reentrant = False).sum().backward()+ assert seen == [True]+ torch.testing.assert_close(x.grad, 2 * torch.cos(2 * x.detach()))+++@_CUDA+@_TORCH_VARLEN_HOST[email protected]("mode", ["no_grad", "inference_mode"])+def test_varlen_in_graph_verdict_does_not_depend_on_the_first_grad_mode(monkeypatch, mode):+ """Real probe under no_grad / inference_mode caches the grad-enabled answer."""+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip(+ reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"+ )+ key = (0, torch.bfloat16, 64, 8, 2, None)+ expected = hpa._varlen_in_graph_probe(*key)+ if not expected:+ pytest.skip(reason = "this build's flash varlen does not take a 64-wide GQA row")+ monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})+ context = torch.no_grad() if mode == "no_grad" else torch.inference_mode()+ with context:+ assert hpa._varlen_in_graph_eager(*key) is expected+ assert hpa._varlen_in_graph_eager(*key) is expected+++@_CUDA+@_TORCH_VARLEN_HOST+def test_reentrant_checkpoint_first_trace_keeps_the_block_varlen_path(monkeypatch):+ """First compiled inside a reentrant checkpoint: forward and recompute both run varlen."""+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip(+ reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"+ )+ from torch.utils.checkpoint import checkpoint+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ def _dense(*args, **kwargs):+ raise AssertionError("packed row fell back to SDPA over the dense mask")++ hpa._ORIG_SDPA[0] = _dense+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})+ H, n_kv, D = 8, 2, 64+ module = types.SimpleNamespace(+ is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = None+ )+ lengths = [37, 100, 2, 5, 230]+ T = sum(lengths)+ mask = _block_causal(lengths, "cuda")+ psl = torch.tensor(lengths, dtype = torch.int32)+ cu = _cu(lengths, "cuda")+ fn = torch.compile(+ lambda q, k, v: hpa._sdpa_packed_varlen(+ module,+ q,+ k,+ v,+ mask,+ scaling = 0.11,+ packed_seq_lengths = psl,+ **{hpa.BLOCK_VARLEN_KWARG: (cu, max(lengths))},+ )[0],+ fullgraph = True,+ )+ torch._dynamo.reset()+ torch.manual_seed(0)+ q, k, v = (+ torch.randn(1, h, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ for h in (H, n_kv, n_kv)+ )+ out = checkpoint(fn, q, k, v, use_reentrant = True)+ grad = torch.randn_like(out)+ out.backward(grad) # reentrant checkpoint refuses autograd.grad+ got = (q.grad, k.grad, v.grad)+ # Math reference: the dense-mask backend sdpa picks can change with earlier tests in this file.+ with torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.MATH):+ ref_out, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)+ ref = torch.autograd.grad(ref_out, (q, k, v), grad)+ torch.testing.assert_close(out.float(), ref_out.float(), atol = 2e-2, rtol = 2e-2)+ for g, r in zip(got, ref):+ torch.testing.assert_close(g.float(), r.float(), atol = 3e-2, rtol = 3e-2)diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.pyindex 895c8e22034..86677d5b6ac 100644--- a/unsloth/utils/hf_packed_attention.py+++ b/unsloth/utils/hf_packed_attention.py@@ -5,6 +5,7 @@ flash-attn) instead of SDPA over a dense block-causal mask; heads past flash's 2layers, 512) and shapes an FA4 / FA3 impl refuses run SDPA's memory-efficient kernel over the samecu_seqlens. Anything else reaches the wrapped "sdpa"."""+import contextlibimport inspectimport loggingimport os@@ -691,26 +692,41 @@ def _varlen_in_graph_eager(device_index, dtype, head_dim, n_heads, n_kv_heads, wreturn _VARLEN_IN_GRAPH[key]+def _no_saved_tensor_hooks():+ # Only the innermost pair applies. Pushing raises while torch has hooks disabled: keep the ambient state.+ enabled = getattr(torch._C._autograd, "_saved_tensors_hooks_is_enabled", lambda: True)+ if enabled():+ return torch.autograd.graph.saved_tensors_hooks(_identity, _identity)+ return contextlib.nullcontext()+++def _identity(tensor):+ return tensor++def _varlen_in_graph_probe(device_index, dtype, head_dim, n_heads, n_kv_heads, window):"""aten's flash varlen takes this shape on this device: one real forward and backward (a buildthat refuses at run time would otherwise fail inside the compiled graph)."""- try:- from .flash_impl import flash_impl_allows-- device = torch.device("cuda", device_index)- query = torch.zeros(1, n_heads, 16, head_dim, device = device, dtype = dtype)- key = torch.zeros(1, n_kv_heads, 16, head_dim, device = device, dtype = dtype)- torch_varlen = _torch_varlen(device)- # Hardware and build only: the sdpa policy can change, block_varlen_kwargs checks it each step.- if torch_varlen is None or not flash_impl_allows(head_dim, window, device):+ # Cached for all grad modes; first trace may be no_grad (reentrant checkpoint), where backward raises.+ # Identity hooks: a non-reentrant checkpoint's pack hooks would make the probe's backward recompute the block.+ with torch.inference_mode(False), torch.enable_grad(), _no_saved_tensor_hooks():+ try:+ from .flash_impl import flash_impl_allows++ device = torch.device("cuda", device_index)+ torch_varlen = _torch_varlen(device)+ # Hardware and build only: the sdpa policy can change, block_varlen_kwargs checks it each step.+ if torch_varlen is None or not flash_impl_allows(head_dim, window, device):+ return False+ query = torch.zeros(1, n_heads, 16, head_dim, device = device, dtype = dtype)+ key = torch.zeros(1, n_kv_heads, 16, head_dim, device = device, dtype = dtype)+ query.requires_grad_()+ cu_seqlens = torch.tensor([0, 7, 16], dtype = torch.int32, device = device)+ out = _flash_varlen(query, key, key, cu_seqlens, 9, 1.0, window)+ out.float().sum().backward()+ return True+ except Exception:return False- query.requires_grad_()- cu_seqlens = torch.tensor([0, 7, 16], dtype = torch.int32, device = device)- out = _flash_varlen(query, key, key, cu_seqlens, 9, 1.0, window)- out.float().sum().backward()- return True- except Exception:- return False_varlen_in_graph = torch._dynamo.assume_constant_result(_varlen_in_graph_eager)