Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GatedDeltaNet fla op engages in training; large-head self-check survives a non-reentrant checkpoint (#14233)

Daniel Han745bdb297903
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index a856e81f639..3eaff4ac76d 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -598,6 +598,62 @@ def test_gdn_adapter_routes_only_the_self_checked_call(monkeypatch):
assert seen == ["op"] + ["fla"] * 6
+def _hub_wrapper(implementation, applicable_params):
+ # Shape of transformers' use_kernel_func_from_hub_with_fallback wrapper.
+ def torch_chunk_gated_delta_rule(*args, **kwargs):
+ kwargs = {k: v for k, v in kwargs.items() if k in applicable_params}
+ return implementation(*args, **kwargs)
+
+ return torch_chunk_gated_delta_rule
+
+
+def test_gdn_adapter_ignores_kwargs_the_hub_wrapper_drops(monkeypatch):
+ # The decoder's TransformersKwargs (num_items_in_batch, use_cache, ...) reach the hub wrapper,
+ # which drops them; the adapter declined on them, so the op never ran in training.
+ M = _load()
+ seen = []
+ monkeypatch.setattr(M, "_GDN_OP", [True])
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ monkeypatch.setattr(
+ torch.ops,
+ "unsloth",
+ types.SimpleNamespace(qwen3_5_gdn_chunk = lambda *a: seen.append("op") or ("out",)),
+ raising = False,
+ )
+ fla = lambda *a, **kw: seen.append("fla") or ("fla", None)
+ params = ("q", "k", "v", "g", "beta", "scale", "initial_state", "output_final_state")
+ params += ("use_qk_l2norm_in_kernel", "cu_seqlens")
+ adapter = M._gdn_chunk_adapter(_hub_wrapper(fla, params))
+ t, gb = torch.zeros(1, 4, 2, 8), torch.zeros(1, 4, 2)
+ stock = dict(use_qk_l2norm_in_kernel = True, initial_state = None, output_final_state = False)
+ adapter(t, t, t, g = gb, beta = gb, num_items_in_batch = torch.tensor(8), use_cache = False, **stock)
+ adapter(t, t, t, g = gb, beta = gb, cu_seqlens = torch.tensor([0, 4]), **stock)
+ assert seen == ["op", "fla"]
+
+
+def test_route_fla_kernel_sees_a_frozen_heap(monkeypatch):
+ # Trainer.train() gc.freeze()s the heap (_gc_train_scope) before the first GatedDeltaNet
+ # forward routes the kernel, and gc.get_referrers skips frozen objects: nothing was routed.
+ import gc
+
+ if gc.get_freeze_count():
+ pytest.skip("the heap is already frozen")
+ M = _load()
+ chunk = types.ModuleType("fake_frozen.fla.ops.gated_delta_rule.chunk")
+ chunk.chunk_gated_delta_rule = lambda *a, **kw: ("fla", None)
+ monkeypatch.setitem(sys.modules, chunk.__name__, chunk)
+ modeling = types.ModuleType("fake_frozen_modeling")
+ modeling.torch_chunk_gated_delta_rule = _hub_wrapper(chunk.chunk_gated_delta_rule, ("q",))
+ monkeypatch.setitem(sys.modules, modeling.__name__, modeling)
+ gc.freeze()
+ try:
+ assert M._route_fla_kernel() >= 1
+ assert gc.get_freeze_count() > 0 # the caller's freeze is kept
+ finally:
+ gc.unfreeze()
+ assert getattr(modeling.torch_chunk_gated_delta_rule, "_unsloth_block_adapter", False)
+
+
@pytest.mark.parametrize("linear_env", ["auto", "0"])
def test_nf4_block_route_matches_both_checkpoint_halves(monkeypatch, linear_env):
# Package context so `from ..kernels import bnb_override` resolves to the fakes below.
diff --git a/tests/test_large_head_attention.py b/tests/test_large_head_attention.py
index 1c5c439d06d..ed624ad6044 100644
--- a/tests/test_large_head_attention.py
+++ b/tests/test_large_head_attention.py
@@ -255,6 +255,49 @@ def test_self_check_under_inference_mode():
assert LH._self_check(device, dtype) is True
+def _self_check_in_checkpoint(device, dtype):
+ # Cold first call from inside a non-reentrant checkpointed layer (DDP / FSDP2, Gemma-4
+ # E-series): returns the cached verdict, the layer's run count and whether w got a grad.
+ from torch.utils.checkpoint import checkpoint
+
+ LH._SELF_CHECK.pop((str(device), dtype), None)
+ calls = []
+ w = (torch.randn(8, 8, device = device) * 0.1).requires_grad_()
+
+ def layer(x):
+ calls.append(1)
+ h = (x @ w).sigmoid()
+ LH._self_check(device, dtype)
+ return (h @ w).sigmoid()
+
+ checkpoint(layer, torch.randn(4, 8, device = device), use_reentrant = False).sum().backward()
+ return LH._SELF_CHECK[(str(device), dtype)], len(calls), w.grad is not None
+
+
+def test_self_check_stays_out_of_a_non_reentrant_checkpoint(monkeypatch):
+ # The check's tensors were packed into the layer's checkpoint and its autograd.grad recomputed
+ # the layer mid-forward: CheckpointError, cached as False, the kernel off for the process.
+ def kernel(q, k, v, lo, hi, scale):
+ groups = q.shape[1] // k.shape[1]
+ out = torch.nn.functional.scaled_dot_product_attention(
+ q, k.repeat_interleave(groups, 1), v.repeat_interleave(groups, 1), is_causal = True
+ )
+ return out.transpose(1, 2)
+
+ monkeypatch.setattr(LH, "large_head_attention", kernel)
+ monkeypatch.setattr(LH, "_measured_faster_on", lambda device: True)
+ monkeypatch.setattr(LH, "_dense_intervals", lambda *args: (None, None))
+ key = (str(torch.device("cpu")), torch.float32)
+ monkeypatch.setitem(LH._SELF_CHECK, key, None)
+ assert _self_check_in_checkpoint(torch.device("cpu"), torch.float32) == (True, 2, True)
+
+
[email protected](not _SM80, reason = "the router engages on sm80+ only")
+def test_real_self_check_stays_out_of_a_non_reentrant_checkpoint():
+ device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
+ assert _self_check_in_checkpoint(device, dtype) == (True, 2, True)
+
+
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_patch_self_checks_so_compiled_graphs_use_the_kernel(monkeypatch):
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.py
index 48f6508f32e..e0e9673a0ea 100644
--- a/unsloth/kernels/large_head_attention.py
+++ b/unsloth/kernels/large_head_attention.py
@@ -757,8 +757,11 @@ def _self_check(device, dtype):
# A local generator: torch.manual_seed would reseed every CUDA device's RNG.
gen = torch.Generator(device = device)
gen.manual_seed(0)
- # Inference tensors cannot record autograd, even under enable_grad.
- with torch.inference_mode(False), torch.enable_grad():
+ # Inference tensors cannot record autograd, even under enable_grad. A first call inside a
+ # non-reentrant checkpoint (DDP / FSDP2, Gemma-4 E-series) would pack the check's tensors
+ # into the layer's checkpoint and recompute the layer mid-forward (CheckpointError).
+ hooks = torch.autograd.graph.saved_tensors_hooks(lambda t: t.detach(), lambda t: t)
+ with torch.inference_mode(False), torch.enable_grad(), hooks:
q, k, v, g = [
torch.randn(shape, device = device, dtype = dtype, generator = gen)
for shape in ((1, 4, 80, 512), (1, 2, 80, 512), (1, 2, 80, 512), (1, 80, 4, 512))
@@ -775,6 +778,8 @@ def _self_check(device, dtype):
is_causal = True,
).transpose(1, 2)
ref_grads = torch.autograd.grad(ref, (qf, kf, vf), g.float())
+ # Compared detached: autograd ops on `out` here would also be saved into that checkpoint.
+ out, ref = out.detach(), ref.detach()
passed = all(
torch.isfinite(a).all().item()
and ((a.float() - b).abs().max() <= 0.05 * b.abs().max() + 0.02).item()
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index 898659cc275..53f2f621ed0 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -130,7 +130,27 @@ def _mark_seq_dynamic(seq_len, obj):
_GDN_OP = [None]
+def _hub_forwarded_params(fn):
+ """The keywords transformers' hub-kernel wrapper passes on to fla (it drops the rest, e.g. the
+ decoder's num_items_in_batch), unwrapping Unsloth's disable wrappers; None if not a hub call."""
+ for _ in range(8):
+ code = getattr(fn, "__code__", None)
+ if code is not None and fn.__closure__:
+ cells = dict(zip(code.co_freevars, fn.__closure__))
+ if "implementation" in cells and "applicable_params" in cells:
+ try:
+ return frozenset(cells["applicable_params"].cell_contents)
+ except (ValueError, TypeError):
+ return None
+ fn = getattr(fn, "_unsloth_undisabled", None) or getattr(fn, "__wrapped__", None)
+ if fn is None:
+ break
+ return None
+
+
def _gdn_chunk_adapter(original):
+ forwarded = _hub_forwarded_params(original)
+
# updated = (): the original's __dict__ carries torch.compiler.disable's marker.
@functools.wraps(original, updated = ())
def chunk_gated_delta_rule(
@@ -149,7 +169,7 @@ def _gdn_chunk_adapter(original):
and all(
value is None or value is False
for key, value in kwargs.items()
- if key != "use_qk_l2norm_in_kernel"
+ if key != "use_qk_l2norm_in_kernel" and (forwarded is None or key in forwarded)
)
):
# Inductor pins a custom op's input strides; a sliced input fails at the next length.
@@ -169,6 +189,21 @@ def _route_fla_kernel():
referrers), not by name. Closures are left alone: only namespace entries are swapped."""
import gc
+ # gc.get_referrers skips the permanent generation, and Trainer.train() freezes the setup-time
+ # heap (_gc_train_scope) before the first GatedDeltaNet forward gets here: the walk found no
+ # caller and the op never engaged. Thaw for the walk; refreezing also freezes what train()
+ # allocated so far, which train()'s exit unfreezes with the rest.
+ frozen = gc.get_freeze_count() > 0
+ if frozen:
+ gc.unfreeze()
+ try:
+ return _route_fla_callers(gc)
+ finally:
+ if frozen:
+ gc.freeze()
+
+
+def _route_fla_callers(gc):
kernels = []
for name, module in list(sys.modules.items()):
if module is not None and name.endswith("fla.ops.gated_delta_rule.chunk"):