Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Compiled blocks: packed GatedDeltaNet op, flex main kernel in training blocks, PrefixGrouper calls skip the block (0 graph breaks) (#14782)

Daniel Han3c16b36ceb77
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index 00fe6f21ab7..5418a6f9567 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -434,6 +434,41 @@ def test_cache_calls_keep_the_stock_forward(monkeypatch):
assert calls # no cache: compiled
[email protected]("key", ["packed_seq_lengths", "prefix_seg_info"])
+def test_unsloth_host_metadata_calls_keep_the_stock_forward(monkeypatch, key):
+ """Host-read packed lengths / PrefixGrouper tables recompiled a block at every new length."""
+ M = _load()
+ calls = []
+
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ calls.append(1)
+ return fn(*args, **kw)
+
+ return compiled
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+
+ class UnslothDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, hidden_states, **kwargs):
+ return self.proj(hidden_states)
+
+ UnslothDecoderLayer.forward.__module__ = "unsloth.models.llama"
+ M._wrap(UnslothDecoderLayer, {})
+ layer = UnslothDecoderLayer()
+ x = torch.randn(1, 4, 8)
+ with torch.no_grad():
+ layer(x, **{key: object()})
+ assert calls == []
+ with torch.no_grad():
+ layer(x, **{key: None})
+ assert calls # no host metadata: compiled
+
+
def test_gemma4_band_fast_path_in_blocks(monkeypatch):
M = _load()
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
@@ -746,6 +781,119 @@ def test_qwen3_block_matches_stock():
_check_block(layers, run, x, modeling.Qwen3DecoderLayer)
+def test_block_flex_forces_the_main_kernel(monkeypatch):
+ """Inside a block every flex call asks for the main kernel; a caller's choice is kept."""
+ M = _load()
+ seen = []
+ funcs = {"flex_attention": lambda *a, **kw: seen.append(kw.get("kernel_options"))}
+ tmu = types.ModuleType("transformers.modeling_utils")
+ tmu.ALL_ATTENTION_FUNCTIONS = funcs
+ monkeypatch.setitem(sys.modules, "transformers.modeling_utils", tmu)
+ M._flex_split_hook()
+ flex = funcs["flex_attention"]
+ flex(None, None, None, None)
+ for mode in (M._FULL, M._SPLIT):
+ monkeypatch.setattr(M, "_IN_BLOCK", [mode])
+ flex(None, None, None, None)
+ flex(None, None, None, None, kernel_options = {"BLOCK_M": 32})
+ flex(None, None, None, None, kernel_options = {"FORCE_USE_FLEX_ATTENTION": False})
+ forced = {"FORCE_USE_FLEX_ATTENTION": True}
+ assert seen == [None] + 2 * [
+ forced,
+ {"BLOCK_M": 32, **forced},
+ {"FORCE_USE_FLEX_ATTENTION": False},
+ ]
+
+
[email protected](not has_real_cuda(), reason = "flex_attention kernels need CUDA")
+def test_block_flex_options_lower_short_gqa_training_queries():
+ """Qwen3.5 GRPO shapes: with the block options, compiled flex fwd + bwd lower and match eager."""
+ from torch.nn.attention.flex_attention import create_block_mask, flex_attention
+
+ M = _load()
+ gen = torch.Generator(device = "cuda").manual_seed(0)
+ mk = lambda *s: torch.randn(*s, device = "cuda", dtype = torch.bfloat16, generator = gen)
+ q, k, v = mk(4, 8, 84, 256), mk(4, 2, 84, 256), mk(4, 2, 84, 256)
+ bm = create_block_mask(lambda b, h, qi, ki: qi >= ki, 4, None, 84, 84, device = "cuda")
+ ins = [t.clone().requires_grad_() for t in (q, k, v)]
+ torch._dynamo.reset()
+ try:
+ fn = torch.compile(flex_attention, dynamic = False)
+ got = fn(
+ *ins,
+ block_mask = bm,
+ enable_gqa = True,
+ scale = 1 / 16,
+ kernel_options = M._training_flex_options(None),
+ )
+ got.float().sum().backward()
+ finally:
+ torch._dynamo.reset()
+ ref_ins = [t.detach().float().requires_grad_() for t in (q, k, v)]
+ ref = flex_attention(*ref_ins, block_mask = bm, enable_gqa = True, scale = 1 / 16)
+ ref.sum().backward()
+ torch.testing.assert_close(got.float(), ref, atol = 2e-2, rtol = 2e-2)
+ for a, b in zip(ins, ref_ins):
+ torch.testing.assert_close(a.grad.float(), b.grad, atol = 5e-2, rtol = 5e-2)
+
+
[email protected](not has_real_cuda(), reason = "compiled block needs CUDA")
+def test_short_gqa_flex_block_stays_compiled():
+ """Short GQA head_dim 256 batches used to fail flex decoding lowering and demote every block."""
+ pytest.importorskip("unsloth")
+ from transformers.models.qwen3 import modeling_qwen3 as modeling
+ from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
+
+ import unsloth.models._block_compile as M
+
+ cfg = Qwen3Config(
+ vocab_size = 64,
+ hidden_size = 256,
+ intermediate_size = 256,
+ num_hidden_layers = 2,
+ num_attention_heads = 8,
+ num_key_value_heads = 2,
+ head_dim = 256,
+ )
+ cfg._attn_implementation = "flex_attention"
+ torch.manual_seed(0)
+ layers = [
+ modeling.Qwen3DecoderLayer(cfg, layer_idx = i).cuda().to(torch.bfloat16) for i in range(2)
+ ]
+ rotary = modeling.Qwen3RotaryEmbedding(cfg).cuda()
+ M._flex_split_hook()
+ from torch.nn.attention.flex_attention import create_block_mask
+
+ def run(layer, h):
+ B, L = h.shape[0], h.shape[1]
+ pos = torch.arange(L, device = h.device)[None].expand(B, -1)
+ mask = create_block_mask(lambda b, hd, q, k: q >= k, B, None, L, L, device = h.device)
+ out = layer(h, attention_mask = mask, position_embeddings = rotary(h, pos), position_ids = pos)
+ return out[0] if isinstance(out, tuple) else out
+
+ from torch._dynamo.utils import counters
+
+ x = [torch.randn(4, L, 256, device = "cuda", dtype = torch.bfloat16) for L in (84, 100)]
+ cls = modeling.Qwen3DecoderLayer
+ original = cls.forward
+ want = [_grads_after_step(layers, run, xi.clone().requires_grad_()) for xi in x]
+ try:
+ M._wrap(M._rebind_class(cls) or cls, M._COMPILE_OPTIONS)
+ counters.clear()
+ got = [_grads_after_step(layers, run, xi.clone().requires_grad_()) for xi in x]
+ modes = [layer.__dict__.get("_unsloth_block_mode") for layer in layers]
+ assert modes == [M._FULL, M._FULL], modes
+ assert sum(counters["graph_break"].values()) == 0, dict(counters["graph_break"])
+ finally:
+ cls.forward = original
+ torch._dynamo.reset()
+ for (lw, gw), (lg, gg) in zip(want, got):
+ torch.testing.assert_close(lg, lw, rtol = 2e-2, atol = 1e-3)
+ for k in gw:
+ num = (gg[k].float() - gw[k].float()).norm()
+ assert num <= 0.05 * gw[k].float().norm() + 1e-4, (k, float(num))
+
+
@pytest.mark.skipif(not has_real_cuda(), reason = "fla kernels need CUDA")
def test_qwen3_5_gdn_block_has_no_break():
pytest.importorskip("unsloth")
@@ -820,6 +968,7 @@ def test_opaque_fla_op_matches_fla_function():
g = -mk(B, T, H, dt = torch.float32).abs() * 0.1
beta = mk(B, T, H).sigmoid()
assert M._fla_ops(lambda: (q, k, v, g, beta))
+ assert M._VARLEN[0], "the packed (cu_seqlens) layout failed its self-check"
# A wrong fake meta or a numeric mismatch must keep the op out of the block.
M._FLA = None
orig = M._chunk_meta
@@ -860,6 +1009,37 @@ def test_gdn_adapter_routes_only_the_self_checked_call(monkeypatch):
assert seen == ["op"] + ["fla"] * 6
+def test_gdn_adapter_routes_packed_batches_once_self_checked(monkeypatch):
+ """Packed (cu_seqlens) fla calls stay in the block once the op's packed self-check passed."""
+ M = _load()
+ seen = []
+ monkeypatch.setattr(M, "_GDN_OP", [True])
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+
+ def op(*a):
+ seen.append(("op", a[6].tolist() if len(a) > 6 else None))
+ return ("out",)
+
+ monkeypatch.setattr(
+ torch.ops, "unsloth", types.SimpleNamespace(qwen3_5_gdn_chunk = op), raising = False
+ )
+ adapter = M._gdn_chunk_adapter(lambda *a, **kw: seen.append(("fla", None)) or ("fla", None))
+ stock = dict(use_qk_l2norm_in_kernel = True, initial_state = None, output_final_state = False)
+ cu = torch.tensor([0, 3, 4], dtype = torch.int32)
+ one, two = torch.zeros(1, 4, 2, 8), torch.zeros(2, 4, 2, 8)
+ g1, g2 = torch.zeros(1, 4, 2), torch.zeros(2, 4, 2)
+ for varlen in (False, True):
+ monkeypatch.setattr(M, "_GDN_VARLEN", [varlen])
+ adapter(one, one, one, g = g1, beta = g1, cu_seqlens = cu, **stock)
+ adapter(two, two, two, g = g2, beta = g2, cu_seqlens = cu, **stock) # fla wants B = 1
+ adapter(one, one, one, g = g1, beta = g1, cu_seqlens = None, **stock)
+ assert seen == [("fla", None), ("fla", None), ("op", None)] + [
+ ("op", [0, 3, 4]),
+ ("fla", None),
+ ("op", None),
+ ]
+
+
def _hub_wrapper(implementation, applicable_params):
# Shape of transformers' use_kernel_func_from_hub_with_fallback wrapper.
def torch_chunk_gated_delta_rule(*args, **kwargs):
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index be2a2bd4a5a..4fae6116fbe 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -144,6 +144,8 @@ def _mark_seq_dynamic(seq_len, obj):
# fla's opaque op state: None until a GatedDeltaNet layer ran its self-check, then True / False.
_GDN_OP = [None]
+# Packed batches (cu_seqlens) through the op: set eagerly with _GDN_OP from the op's self-check.
+_GDN_VARLEN = [False]
def _hub_forwarded_params(fn):
@@ -177,20 +179,31 @@ def _gdn_chunk_adapter(original):
beta = None,
**kwargs,
):
+ cu_seqlens = (
+ kwargs.get("cu_seqlens") if forwarded is None or "cu_seqlens" in forwarded else None
+ )
if (
_GDN_OP[0]
and torch.compiler.is_compiling()
and kwargs.get("use_qk_l2norm_in_kernel") is True
- # The op is the self-checked default: no state, varlen, scale, layout or gating flags.
+ # Only the self-checked default; packed batches only if their self-check passed.
+ and (cu_seqlens is None or (_GDN_VARLEN[0] and q.shape[0] == 1))
and all(
value is None or value is False
for key, value in kwargs.items()
- if key != "use_qk_l2norm_in_kernel" and (forwarded is None or key in forwarded)
+ if key not in ("use_qk_l2norm_in_kernel", "cu_seqlens")
+ and (forwarded is None or key in forwarded)
)
):
# Inductor pins a custom op's input strides; a sliced input fails at the next length.
q, k, v, g, beta = (t.contiguous() for t in (q, k, v, g, beta))
- out = torch.ops.unsloth.qwen3_5_gdn_chunk(q, k, v, g, beta, q.shape[-1] ** -0.5)
+ scale = q.shape[-1] ** -0.5
+ if cu_seqlens is None:
+ out = torch.ops.unsloth.qwen3_5_gdn_chunk(q, k, v, g, beta, scale)
+ else:
+ out = torch.ops.unsloth.qwen3_5_gdn_chunk(
+ q, k, v, g, beta, scale, cu_seqlens.contiguous()
+ )
return out[0], None
return original(q, k, v, g, beta, **kwargs)
@@ -392,7 +405,7 @@ def _prepare_gdn(layer, hidden_states):
except Exception:
pass
try:
- from ._gdn_op import _fla_ops
+ from ._gdn_op import _fla_ops, varlen_ok
heads, dk, dv = gdn.num_v_heads, gdn.head_k_dim, gdn.head_v_dim
B, T = hidden_states.shape[0], hidden_states.shape[1]
@@ -412,6 +425,7 @@ def _prepare_gdn(layer, hidden_states):
if not _fla_ops(make_inputs):
return
+ _GDN_VARLEN[0] = varlen_ok()
_GDN_OP[0] = _route_fla_kernel() > 0
if _GDN_OP[0]:
_route_causal_conv(gdn)
@@ -553,6 +567,16 @@ _DEMOTION = (_FULL, _SPLIT, _OFF)
_FIRST_MODE = _SPLIT if os.environ.get("UNSLOTH_BLOCK_FLEX", "inside") == "outside" else _FULL
+def _training_flex_options(options):
+ """Force the main flex kernel: Inductor's decoding kernel for queries < 128 fails to lower
+ (GQA, head_dim 256), demoting every block. A caller's explicit choice is kept."""
+ if options is None:
+ return {"FORCE_USE_FLEX_ATTENTION": True}
+ if isinstance(options, dict) and "FORCE_USE_FLEX_ATTENTION" not in options:
+ return {**options, "FORCE_USE_FLEX_ATTENTION": True}
+ return options
+
+
def _flex_split_hook():
"""Lets a block keep flex_attention out of its graph. Some masks do not lower inside a larger
graph (torch 2.13, Gemma-4 12B blockwise mask: `unsupported operand type(s) for &:
@@ -568,6 +592,8 @@ def _flex_split_hook():
@functools.wraps(flex)
def flex_attention_forward(*args, **kwargs):
+ if _IN_BLOCK[0] != _OFF:
+ kwargs["kernel_options"] = _training_flex_options(kwargs.get("kernel_options"))
if _IN_BLOCK[0] == _SPLIT and torch.compiler.is_compiling():
return outside(*args, **kwargs)
return flex(*args, **kwargs)
@@ -660,7 +686,7 @@ def _layer_kind(layer):
def _wrap(cls, compile_options):
original = cls.forward
- # Unsloth layer forwards read packed lengths on the host (packing.py): packed calls stay stock.
+ # Host-read packed lengths / PrefixGrouper segments stay stock.
unsloth_forward = getattr(original, "__module__", "").startswith("unsloth.models.")
compiled = torch.compile(original, fullgraph = False, dynamic = None, options = compile_options)
try:
@@ -709,6 +735,8 @@ def _wrap(cls, compile_options):
or _forced_float32(self)
or _zero3_partitioned(self)
or (unsloth_forward and kwargs.get("packed_seq_lengths") is not None)
+ # PrefixGrouper's int segment table would recompile at every new GRPO length.
+ or (unsloth_forward and kwargs.get("prefix_seg_info") is not None)
):
return original(self, hidden_states, *args, **kwargs)
if tracing:
diff --git a/unsloth/models/_gdn_op.py b/unsloth/models/_gdn_op.py
index ff4038f1a73..bc27cd5c711 100644
--- a/unsloth/models/_gdn_op.py
+++ b/unsloth/models/_gdn_op.py
@@ -5,6 +5,9 @@
checked bit-equal to ChunkGatedDeltaRuleFunction before use, so a Gated DeltaNet layer compiles
as one graph."""
+import inspect
+from typing import Optional
+
import torch
import torch.nn.functional as F
from torch.autograd.graph import saved_tensors_hooks
@@ -49,11 +52,33 @@ def _fla_ops(make_inputs):
with torch.enable_grad(), saved_tensors_hooks(lambda t: t.detach(), lambda t: t):
_FLA = fns
_FLA = fns if _self_check(*inputs) else False
+ # Packed (padding-free) batches: B = 1 plus cu_seqlens, checked the same way.
+ _VARLEN[0] = bool(_FLA) and _self_check_varlen(*inputs)
except Exception:
_FLA = False
+ _VARLEN[0] = False
return _FLA
+# True once the op matched fla's Function on a packed (cu_seqlens) batch too.
+_VARLEN = [False]
+
+
+def varlen_ok() -> bool:
+ return _VARLEN[0]
+
+
+def _varlen_kwargs(fn, cu_seqlens):
+ """cu_seqlens (+ the chunk_indices fla's Function derives from it) for fla's fwd / bwd."""
+ if cu_seqlens is None:
+ return {}
+ kwargs = {"cu_seqlens": cu_seqlens}
+ if "chunk_indices" in inspect.signature(fn).parameters:
+ from fla.ops.utils.index import prepare_chunk_indices
+ kwargs["chunk_indices"] = prepare_chunk_indices(cu_seqlens, 64)
+ return kwargs
+
+
def _chunk_meta(q, k, v, g):
# (qn, q_rstd, kn, k_rstd, g_cumsum, A), as fla 0.5 allocates them
return (
@@ -76,6 +101,7 @@ try:
g: torch.Tensor,
beta: torch.Tensor,
scale: float,
+ cu_seqlens: Optional[torch.Tensor] = None,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -99,12 +125,21 @@ try:
scale = scale,
initial_state = None,
output_final_state = False,
+ **_varlen_kwargs(fwd, cu_seqlens),
)
# Fake metas are contiguous; keep the real outputs the same.
return tuple(t.contiguous() for t in (o.to(q.dtype), qn, q_rstd, kn, k_rstd, g_cum, A))
@_gdn_chunk.register_fake
- def _gdn_chunk_fake(q, k, v, g, beta, scale):
+ def _gdn_chunk_fake(
+ q,
+ k,
+ v,
+ g,
+ beta,
+ scale,
+ cu_seqlens = None,
+ ):
o = v.new_empty(v.shape, dtype = q.dtype)
return (o,) + tuple(
q.new_empty(shape, dtype = dtype) for shape, dtype in _chunk_meta(q, k, v, g)
@@ -122,6 +157,7 @@ try:
A: torch.Tensor,
do: torch.Tensor,
scale: float,
+ cu_seqlens: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
_, bwd, _, l2_bwd = _FLA
v, beta, do = v.contiguous(), beta.contiguous(), do.contiguous()
@@ -137,6 +173,7 @@ try:
initial_state = None,
do = do,
dht = None,
+ **_varlen_kwargs(bwd, cu_seqlens),
)
dq = l2_bwd(qn, q_rstd, dq)
dk = l2_bwd(kn, k_rstd, dk)
@@ -150,21 +187,35 @@ try:
return tuple(t.contiguous() for t in out)
@_gdn_chunk_bwd.register_fake
- def _gdn_chunk_bwd_fake(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A, do, scale):
+ def _gdn_chunk_bwd_fake(
+ qn,
+ q_rstd,
+ kn,
+ k_rstd,
+ v,
+ g_cum,
+ beta,
+ A,
+ do,
+ scale,
+ cu_seqlens = None,
+ ):
return tuple(qn.new_empty(t.shape, dtype = t.dtype) for t in (qn, kn, v, g_cum, beta))
def _gdn_chunk_setup(ctx, inputs, output):
- q, k, v, g, beta, scale = inputs
+ q, k, v, g, beta, scale = inputs[:6]
+ cu_seqlens = inputs[6] if len(inputs) > 6 else None
_, qn, q_rstd, kn, k_rstd, g_cum, A = output
- ctx.save_for_backward(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A)
+ ctx.save_for_backward(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A, cu_seqlens)
ctx.scale = scale
ctx.dtypes = (q.dtype, k.dtype, g.dtype)
def _gdn_chunk_backward(ctx, do, *_):
- saved = [t.contiguous() for t in ctx.saved_tensors]
- dq, dk, dv, dg, db = _gdn_chunk_bwd(*saved, do.contiguous(), ctx.scale)
+ *saved, cu_seqlens = ctx.saved_tensors
+ saved = [t.contiguous() for t in saved]
+ dq, dk, dv, dg, db = _gdn_chunk_bwd(*saved, do.contiguous(), ctx.scale, cu_seqlens)
q_dtype, k_dtype, g_dtype = ctx.dtypes
- return dq.to(q_dtype), dk.to(k_dtype), dv, dg.to(g_dtype), db, None
+ return dq.to(q_dtype), dk.to(k_dtype), dv, dg.to(g_dtype), db, None, None
_gdn_chunk.register_autograd(_gdn_chunk_backward, setup_context = _gdn_chunk_setup)
except Exception:
@@ -193,3 +244,36 @@ def _self_check(q, k, v, g, beta):
and torch.equal(out[0], ref)
and all(torch.equal(a, b) for a, b in zip(got_g, ref_g))
)
+
+
+def _self_check_varlen(q, k, v, g, beta):
+ """Packed batch: bit-equal to fla's Function; False if fla's fwd / bwd lack cu_seqlens."""
+ if _gdn_chunk is None or not q.is_cuda:
+ return False
+ fwd, bwd, _, _ = _FLA
+ for fn in (fwd, bwd):
+ if "cu_seqlens" not in inspect.signature(fn).parameters:
+ return False
+ from fla.ops.gated_delta_rule import chunk_gated_delta_rule
+
+ B, T = q.shape[0], q.shape[1]
+ flat = [t.reshape(1, B * T, *t.shape[2:]) for t in (q, k, v, g, beta)]
+ total = B * T
+ # Uneven documents, one shorter than a chunk and one crossing a chunk edge.
+ cuts = sorted({0, min(17, total), min(total // 2 + 5, total), total})
+ cu_seqlens = torch.tensor(cuts, device = q.device, dtype = torch.int32)
+ gen = torch.Generator(device = v.device).manual_seed(1)
+ do = torch.randn(flat[2].shape, dtype = v.dtype, device = v.device, generator = gen)
+ ins = [t.contiguous().requires_grad_() for t in flat]
+ ref, _ = chunk_gated_delta_rule(*ins, use_qk_l2norm_in_kernel = True, cu_seqlens = cu_seqlens)
+ ref_g = torch.autograd.grad(ref, ins, do)
+ ins2 = [t.detach().clone().requires_grad_() for t in ins]
+ out = _gdn_chunk(*ins2, flat[0].shape[-1] ** -0.5, cu_seqlens)
+ got_g = torch.autograd.grad(out[0], ins2, do)
+ metas = [(o.shape, o.dtype) for o in out[1:]]
+ want = [(torch.Size(s), d) for s, d in _chunk_meta(*flat[:4])]
+ return (
+ metas == want
+ and torch.equal(out[0], ref)
+ and all(torch.equal(a, b) for a, b in zip(got_g, ref_g))
+ )