Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GatedDeltaNet fla op engages in training; large-head self-check survives a non-reentrant checkpoint (#14233)
Daniel Han745bdb297903
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex a856e81f639..3eaff4ac76d 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -598,6 +598,62 @@ def test_gdn_adapter_routes_only_the_self_checked_call(monkeypatch):assert seen == ["op"] + ["fla"] * 6+def _hub_wrapper(implementation, applicable_params):+ # Shape of transformers' use_kernel_func_from_hub_with_fallback wrapper.+ def torch_chunk_gated_delta_rule(*args, **kwargs):+ kwargs = {k: v for k, v in kwargs.items() if k in applicable_params}+ return implementation(*args, **kwargs)++ return torch_chunk_gated_delta_rule+++def test_gdn_adapter_ignores_kwargs_the_hub_wrapper_drops(monkeypatch):+ # The decoder's TransformersKwargs (num_items_in_batch, use_cache, ...) reach the hub wrapper,+ # which drops them; the adapter declined on them, so the op never ran in training.+ M = _load()+ seen = []+ monkeypatch.setattr(M, "_GDN_OP", [True])+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ monkeypatch.setattr(+ torch.ops,+ "unsloth",+ types.SimpleNamespace(qwen3_5_gdn_chunk = lambda *a: seen.append("op") or ("out",)),+ raising = False,+ )+ fla = lambda *a, **kw: seen.append("fla") or ("fla", None)+ params = ("q", "k", "v", "g", "beta", "scale", "initial_state", "output_final_state")+ params += ("use_qk_l2norm_in_kernel", "cu_seqlens")+ adapter = M._gdn_chunk_adapter(_hub_wrapper(fla, params))+ t, gb = torch.zeros(1, 4, 2, 8), torch.zeros(1, 4, 2)+ stock = dict(use_qk_l2norm_in_kernel = True, initial_state = None, output_final_state = False)+ adapter(t, t, t, g = gb, beta = gb, num_items_in_batch = torch.tensor(8), use_cache = False, **stock)+ adapter(t, t, t, g = gb, beta = gb, cu_seqlens = torch.tensor([0, 4]), **stock)+ assert seen == ["op", "fla"]+++def test_route_fla_kernel_sees_a_frozen_heap(monkeypatch):+ # Trainer.train() gc.freeze()s the heap (_gc_train_scope) before the first GatedDeltaNet+ # forward routes the kernel, and gc.get_referrers skips frozen objects: nothing was routed.+ import gc++ if gc.get_freeze_count():+ pytest.skip("the heap is already frozen")+ M = _load()+ chunk = types.ModuleType("fake_frozen.fla.ops.gated_delta_rule.chunk")+ chunk.chunk_gated_delta_rule = lambda *a, **kw: ("fla", None)+ monkeypatch.setitem(sys.modules, chunk.__name__, chunk)+ modeling = types.ModuleType("fake_frozen_modeling")+ modeling.torch_chunk_gated_delta_rule = _hub_wrapper(chunk.chunk_gated_delta_rule, ("q",))+ monkeypatch.setitem(sys.modules, modeling.__name__, modeling)+ gc.freeze()+ try:+ assert M._route_fla_kernel() >= 1+ assert gc.get_freeze_count() > 0 # the caller's freeze is kept+ finally:+ gc.unfreeze()+ assert getattr(modeling.torch_chunk_gated_delta_rule, "_unsloth_block_adapter", False)++@pytest.mark.parametrize("linear_env", ["auto", "0"])def test_nf4_block_route_matches_both_checkpoint_halves(monkeypatch, linear_env):# Package context so `from ..kernels import bnb_override` resolves to the fakes below.diff --git a/tests/test_large_head_attention.py b/tests/test_large_head_attention.pyindex 1c5c439d06d..ed624ad6044 100644--- a/tests/test_large_head_attention.py+++ b/tests/test_large_head_attention.py@@ -255,6 +255,49 @@ def test_self_check_under_inference_mode():assert LH._self_check(device, dtype) is True+def _self_check_in_checkpoint(device, dtype):+ # Cold first call from inside a non-reentrant checkpointed layer (DDP / FSDP2, Gemma-4+ # E-series): returns the cached verdict, the layer's run count and whether w got a grad.+ from torch.utils.checkpoint import checkpoint++ LH._SELF_CHECK.pop((str(device), dtype), None)+ calls = []+ w = (torch.randn(8, 8, device = device) * 0.1).requires_grad_()++ def layer(x):+ calls.append(1)+ h = (x @ w).sigmoid()+ LH._self_check(device, dtype)+ return (h @ w).sigmoid()++ checkpoint(layer, torch.randn(4, 8, device = device), use_reentrant = False).sum().backward()+ return LH._SELF_CHECK[(str(device), dtype)], len(calls), w.grad is not None+++def test_self_check_stays_out_of_a_non_reentrant_checkpoint(monkeypatch):+ # The check's tensors were packed into the layer's checkpoint and its autograd.grad recomputed+ # the layer mid-forward: CheckpointError, cached as False, the kernel off for the process.+ def kernel(q, k, v, lo, hi, scale):+ groups = q.shape[1] // k.shape[1]+ out = torch.nn.functional.scaled_dot_product_attention(+ q, k.repeat_interleave(groups, 1), v.repeat_interleave(groups, 1), is_causal = True+ )+ return out.transpose(1, 2)++ monkeypatch.setattr(LH, "large_head_attention", kernel)+ monkeypatch.setattr(LH, "_measured_faster_on", lambda device: True)+ monkeypatch.setattr(LH, "_dense_intervals", lambda *args: (None, None))+ key = (str(torch.device("cpu")), torch.float32)+ monkeypatch.setitem(LH._SELF_CHECK, key, None)+ assert _self_check_in_checkpoint(torch.device("cpu"), torch.float32) == (True, 2, True)++[email protected](not _SM80, reason = "the router engages on sm80+ only")+def test_real_self_check_stays_out_of_a_non_reentrant_checkpoint():+ device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()+ assert _self_check_in_checkpoint(device, dtype) == (True, 2, True)++@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")def test_patch_self_checks_so_compiled_graphs_use_the_kernel(monkeypatch):from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONSdiff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.pyindex 48f6508f32e..e0e9673a0ea 100644--- a/unsloth/kernels/large_head_attention.py+++ b/unsloth/kernels/large_head_attention.py@@ -757,8 +757,11 @@ def _self_check(device, dtype):# A local generator: torch.manual_seed would reseed every CUDA device's RNG.gen = torch.Generator(device = device)gen.manual_seed(0)- # Inference tensors cannot record autograd, even under enable_grad.- with torch.inference_mode(False), torch.enable_grad():+ # Inference tensors cannot record autograd, even under enable_grad. A first call inside a+ # non-reentrant checkpoint (DDP / FSDP2, Gemma-4 E-series) would pack the check's tensors+ # into the layer's checkpoint and recompute the layer mid-forward (CheckpointError).+ hooks = torch.autograd.graph.saved_tensors_hooks(lambda t: t.detach(), lambda t: t)+ with torch.inference_mode(False), torch.enable_grad(), hooks:q, k, v, g = [torch.randn(shape, device = device, dtype = dtype, generator = gen)for shape in ((1, 4, 80, 512), (1, 2, 80, 512), (1, 2, 80, 512), (1, 80, 4, 512))@@ -775,6 +778,8 @@ def _self_check(device, dtype):is_causal = True,).transpose(1, 2)ref_grads = torch.autograd.grad(ref, (qf, kf, vf), g.float())+ # Compared detached: autograd ops on `out` here would also be saved into that checkpoint.+ out, ref = out.detach(), ref.detach()passed = all(torch.isfinite(a).all().item()and ((a.float() - b).abs().max() <= 0.05 * b.abs().max() + 0.02).item()diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 898659cc275..53f2f621ed0 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -130,7 +130,27 @@ def _mark_seq_dynamic(seq_len, obj):_GDN_OP = [None]+def _hub_forwarded_params(fn):+ """The keywords transformers' hub-kernel wrapper passes on to fla (it drops the rest, e.g. the+ decoder's num_items_in_batch), unwrapping Unsloth's disable wrappers; None if not a hub call."""+ for _ in range(8):+ code = getattr(fn, "__code__", None)+ if code is not None and fn.__closure__:+ cells = dict(zip(code.co_freevars, fn.__closure__))+ if "implementation" in cells and "applicable_params" in cells:+ try:+ return frozenset(cells["applicable_params"].cell_contents)+ except (ValueError, TypeError):+ return None+ fn = getattr(fn, "_unsloth_undisabled", None) or getattr(fn, "__wrapped__", None)+ if fn is None:+ break+ return None++def _gdn_chunk_adapter(original):+ forwarded = _hub_forwarded_params(original)+# updated = (): the original's __dict__ carries torch.compiler.disable's marker.@functools.wraps(original, updated = ())def chunk_gated_delta_rule(@@ -149,7 +169,7 @@ def _gdn_chunk_adapter(original):and all(value is None or value is Falsefor key, value in kwargs.items()- if key != "use_qk_l2norm_in_kernel"+ if key != "use_qk_l2norm_in_kernel" and (forwarded is None or key in forwarded))):# Inductor pins a custom op's input strides; a sliced input fails at the next length.@@ -169,6 +189,21 @@ def _route_fla_kernel():referrers), not by name. Closures are left alone: only namespace entries are swapped."""import gc+ # gc.get_referrers skips the permanent generation, and Trainer.train() freezes the setup-time+ # heap (_gc_train_scope) before the first GatedDeltaNet forward gets here: the walk found no+ # caller and the op never engaged. Thaw for the walk; refreezing also freezes what train()+ # allocated so far, which train()'s exit unfreezes with the rest.+ frozen = gc.get_freeze_count() > 0+ if frozen:+ gc.unfreeze()+ try:+ return _route_fla_callers(gc)+ finally:+ if frozen:+ gc.freeze()+++def _route_fla_callers(gc):kernels = []for name, module in list(sys.modules.items()):if module is not None and name.endswith("fla.ops.gated_delta_rule.chunk"):