Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Compiled blocks: packed GatedDeltaNet op, flex main kernel in training blocks, PrefixGrouper calls skip the block (0 graph breaks) (#14782)
Daniel Han3c16b36ceb77
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex 00fe6f21ab7..5418a6f9567 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -434,6 +434,41 @@ def test_cache_calls_keep_the_stock_forward(monkeypatch):assert calls # no cache: compiled[email protected]("key", ["packed_seq_lengths", "prefix_seg_info"])+def test_unsloth_host_metadata_calls_keep_the_stock_forward(monkeypatch, key):+ """Host-read packed lengths / PrefixGrouper tables recompiled a block at every new length."""+ M = _load()+ calls = []++ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ calls.append(1)+ return fn(*args, **kw)++ return compiled++ monkeypatch.setattr(torch, "compile", fake_compile)++ class UnslothDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, hidden_states, **kwargs):+ return self.proj(hidden_states)++ UnslothDecoderLayer.forward.__module__ = "unsloth.models.llama"+ M._wrap(UnslothDecoderLayer, {})+ layer = UnslothDecoderLayer()+ x = torch.randn(1, 4, 8)+ with torch.no_grad():+ layer(x, **{key: object()})+ assert calls == []+ with torch.no_grad():+ layer(x, **{key: None})+ assert calls # no host metadata: compiled++def test_gemma4_band_fast_path_in_blocks(monkeypatch):M = _load()from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS@@ -746,6 +781,119 @@ def test_qwen3_block_matches_stock():_check_block(layers, run, x, modeling.Qwen3DecoderLayer)+def test_block_flex_forces_the_main_kernel(monkeypatch):+ """Inside a block every flex call asks for the main kernel; a caller's choice is kept."""+ M = _load()+ seen = []+ funcs = {"flex_attention": lambda *a, **kw: seen.append(kw.get("kernel_options"))}+ tmu = types.ModuleType("transformers.modeling_utils")+ tmu.ALL_ATTENTION_FUNCTIONS = funcs+ monkeypatch.setitem(sys.modules, "transformers.modeling_utils", tmu)+ M._flex_split_hook()+ flex = funcs["flex_attention"]+ flex(None, None, None, None)+ for mode in (M._FULL, M._SPLIT):+ monkeypatch.setattr(M, "_IN_BLOCK", [mode])+ flex(None, None, None, None)+ flex(None, None, None, None, kernel_options = {"BLOCK_M": 32})+ flex(None, None, None, None, kernel_options = {"FORCE_USE_FLEX_ATTENTION": False})+ forced = {"FORCE_USE_FLEX_ATTENTION": True}+ assert seen == [None] + 2 * [+ forced,+ {"BLOCK_M": 32, **forced},+ {"FORCE_USE_FLEX_ATTENTION": False},+ ]++[email protected](not has_real_cuda(), reason = "flex_attention kernels need CUDA")+def test_block_flex_options_lower_short_gqa_training_queries():+ """Qwen3.5 GRPO shapes: with the block options, compiled flex fwd + bwd lower and match eager."""+ from torch.nn.attention.flex_attention import create_block_mask, flex_attention++ M = _load()+ gen = torch.Generator(device = "cuda").manual_seed(0)+ mk = lambda *s: torch.randn(*s, device = "cuda", dtype = torch.bfloat16, generator = gen)+ q, k, v = mk(4, 8, 84, 256), mk(4, 2, 84, 256), mk(4, 2, 84, 256)+ bm = create_block_mask(lambda b, h, qi, ki: qi >= ki, 4, None, 84, 84, device = "cuda")+ ins = [t.clone().requires_grad_() for t in (q, k, v)]+ torch._dynamo.reset()+ try:+ fn = torch.compile(flex_attention, dynamic = False)+ got = fn(+ *ins,+ block_mask = bm,+ enable_gqa = True,+ scale = 1 / 16,+ kernel_options = M._training_flex_options(None),+ )+ got.float().sum().backward()+ finally:+ torch._dynamo.reset()+ ref_ins = [t.detach().float().requires_grad_() for t in (q, k, v)]+ ref = flex_attention(*ref_ins, block_mask = bm, enable_gqa = True, scale = 1 / 16)+ ref.sum().backward()+ torch.testing.assert_close(got.float(), ref, atol = 2e-2, rtol = 2e-2)+ for a, b in zip(ins, ref_ins):+ torch.testing.assert_close(a.grad.float(), b.grad, atol = 5e-2, rtol = 5e-2)++[email protected](not has_real_cuda(), reason = "compiled block needs CUDA")+def test_short_gqa_flex_block_stays_compiled():+ """Short GQA head_dim 256 batches used to fail flex decoding lowering and demote every block."""+ pytest.importorskip("unsloth")+ from transformers.models.qwen3 import modeling_qwen3 as modeling+ from transformers.models.qwen3.configuration_qwen3 import Qwen3Config++ import unsloth.models._block_compile as M++ cfg = Qwen3Config(+ vocab_size = 64,+ hidden_size = 256,+ intermediate_size = 256,+ num_hidden_layers = 2,+ num_attention_heads = 8,+ num_key_value_heads = 2,+ head_dim = 256,+ )+ cfg._attn_implementation = "flex_attention"+ torch.manual_seed(0)+ layers = [+ modeling.Qwen3DecoderLayer(cfg, layer_idx = i).cuda().to(torch.bfloat16) for i in range(2)+ ]+ rotary = modeling.Qwen3RotaryEmbedding(cfg).cuda()+ M._flex_split_hook()+ from torch.nn.attention.flex_attention import create_block_mask++ def run(layer, h):+ B, L = h.shape[0], h.shape[1]+ pos = torch.arange(L, device = h.device)[None].expand(B, -1)+ mask = create_block_mask(lambda b, hd, q, k: q >= k, B, None, L, L, device = h.device)+ out = layer(h, attention_mask = mask, position_embeddings = rotary(h, pos), position_ids = pos)+ return out[0] if isinstance(out, tuple) else out++ from torch._dynamo.utils import counters++ x = [torch.randn(4, L, 256, device = "cuda", dtype = torch.bfloat16) for L in (84, 100)]+ cls = modeling.Qwen3DecoderLayer+ original = cls.forward+ want = [_grads_after_step(layers, run, xi.clone().requires_grad_()) for xi in x]+ try:+ M._wrap(M._rebind_class(cls) or cls, M._COMPILE_OPTIONS)+ counters.clear()+ got = [_grads_after_step(layers, run, xi.clone().requires_grad_()) for xi in x]+ modes = [layer.__dict__.get("_unsloth_block_mode") for layer in layers]+ assert modes == [M._FULL, M._FULL], modes+ assert sum(counters["graph_break"].values()) == 0, dict(counters["graph_break"])+ finally:+ cls.forward = original+ torch._dynamo.reset()+ for (lw, gw), (lg, gg) in zip(want, got):+ torch.testing.assert_close(lg, lw, rtol = 2e-2, atol = 1e-3)+ for k in gw:+ num = (gg[k].float() - gw[k].float()).norm()+ assert num <= 0.05 * gw[k].float().norm() + 1e-4, (k, float(num))++@pytest.mark.skipif(not has_real_cuda(), reason = "fla kernels need CUDA")def test_qwen3_5_gdn_block_has_no_break():pytest.importorskip("unsloth")@@ -820,6 +968,7 @@ def test_opaque_fla_op_matches_fla_function():g = -mk(B, T, H, dt = torch.float32).abs() * 0.1beta = mk(B, T, H).sigmoid()assert M._fla_ops(lambda: (q, k, v, g, beta))+ assert M._VARLEN[0], "the packed (cu_seqlens) layout failed its self-check"# A wrong fake meta or a numeric mismatch must keep the op out of the block.M._FLA = Noneorig = M._chunk_meta@@ -860,6 +1009,37 @@ def test_gdn_adapter_routes_only_the_self_checked_call(monkeypatch):assert seen == ["op"] + ["fla"] * 6+def test_gdn_adapter_routes_packed_batches_once_self_checked(monkeypatch):+ """Packed (cu_seqlens) fla calls stay in the block once the op's packed self-check passed."""+ M = _load()+ seen = []+ monkeypatch.setattr(M, "_GDN_OP", [True])+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)++ def op(*a):+ seen.append(("op", a[6].tolist() if len(a) > 6 else None))+ return ("out",)++ monkeypatch.setattr(+ torch.ops, "unsloth", types.SimpleNamespace(qwen3_5_gdn_chunk = op), raising = False+ )+ adapter = M._gdn_chunk_adapter(lambda *a, **kw: seen.append(("fla", None)) or ("fla", None))+ stock = dict(use_qk_l2norm_in_kernel = True, initial_state = None, output_final_state = False)+ cu = torch.tensor([0, 3, 4], dtype = torch.int32)+ one, two = torch.zeros(1, 4, 2, 8), torch.zeros(2, 4, 2, 8)+ g1, g2 = torch.zeros(1, 4, 2), torch.zeros(2, 4, 2)+ for varlen in (False, True):+ monkeypatch.setattr(M, "_GDN_VARLEN", [varlen])+ adapter(one, one, one, g = g1, beta = g1, cu_seqlens = cu, **stock)+ adapter(two, two, two, g = g2, beta = g2, cu_seqlens = cu, **stock) # fla wants B = 1+ adapter(one, one, one, g = g1, beta = g1, cu_seqlens = None, **stock)+ assert seen == [("fla", None), ("fla", None), ("op", None)] + [+ ("op", [0, 3, 4]),+ ("fla", None),+ ("op", None),+ ]++def _hub_wrapper(implementation, applicable_params):# Shape of transformers' use_kernel_func_from_hub_with_fallback wrapper.def torch_chunk_gated_delta_rule(*args, **kwargs):diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex be2a2bd4a5a..4fae6116fbe 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -144,6 +144,8 @@ def _mark_seq_dynamic(seq_len, obj):# fla's opaque op state: None until a GatedDeltaNet layer ran its self-check, then True / False._GDN_OP = [None]+# Packed batches (cu_seqlens) through the op: set eagerly with _GDN_OP from the op's self-check.+_GDN_VARLEN = [False]def _hub_forwarded_params(fn):@@ -177,20 +179,31 @@ def _gdn_chunk_adapter(original):beta = None,**kwargs,):+ cu_seqlens = (+ kwargs.get("cu_seqlens") if forwarded is None or "cu_seqlens" in forwarded else None+ )if (_GDN_OP[0]and torch.compiler.is_compiling()and kwargs.get("use_qk_l2norm_in_kernel") is True- # The op is the self-checked default: no state, varlen, scale, layout or gating flags.+ # Only the self-checked default; packed batches only if their self-check passed.+ and (cu_seqlens is None or (_GDN_VARLEN[0] and q.shape[0] == 1))and all(value is None or value is Falsefor key, value in kwargs.items()- if key != "use_qk_l2norm_in_kernel" and (forwarded is None or key in forwarded)+ if key not in ("use_qk_l2norm_in_kernel", "cu_seqlens")+ and (forwarded is None or key in forwarded))):# Inductor pins a custom op's input strides; a sliced input fails at the next length.q, k, v, g, beta = (t.contiguous() for t in (q, k, v, g, beta))- out = torch.ops.unsloth.qwen3_5_gdn_chunk(q, k, v, g, beta, q.shape[-1] ** -0.5)+ scale = q.shape[-1] ** -0.5+ if cu_seqlens is None:+ out = torch.ops.unsloth.qwen3_5_gdn_chunk(q, k, v, g, beta, scale)+ else:+ out = torch.ops.unsloth.qwen3_5_gdn_chunk(+ q, k, v, g, beta, scale, cu_seqlens.contiguous()+ )return out[0], Nonereturn original(q, k, v, g, beta, **kwargs)@@ -392,7 +405,7 @@ def _prepare_gdn(layer, hidden_states):except Exception:passtry:- from ._gdn_op import _fla_ops+ from ._gdn_op import _fla_ops, varlen_okheads, dk, dv = gdn.num_v_heads, gdn.head_k_dim, gdn.head_v_dimB, T = hidden_states.shape[0], hidden_states.shape[1]@@ -412,6 +425,7 @@ def _prepare_gdn(layer, hidden_states):if not _fla_ops(make_inputs):return+ _GDN_VARLEN[0] = varlen_ok()_GDN_OP[0] = _route_fla_kernel() > 0if _GDN_OP[0]:_route_causal_conv(gdn)@@ -553,6 +567,16 @@ _DEMOTION = (_FULL, _SPLIT, _OFF)_FIRST_MODE = _SPLIT if os.environ.get("UNSLOTH_BLOCK_FLEX", "inside") == "outside" else _FULL+def _training_flex_options(options):+ """Force the main flex kernel: Inductor's decoding kernel for queries < 128 fails to lower+ (GQA, head_dim 256), demoting every block. A caller's explicit choice is kept."""+ if options is None:+ return {"FORCE_USE_FLEX_ATTENTION": True}+ if isinstance(options, dict) and "FORCE_USE_FLEX_ATTENTION" not in options:+ return {**options, "FORCE_USE_FLEX_ATTENTION": True}+ return options++def _flex_split_hook():"""Lets a block keep flex_attention out of its graph. Some masks do not lower inside a largergraph (torch 2.13, Gemma-4 12B blockwise mask: `unsupported operand type(s) for &:@@ -568,6 +592,8 @@ def _flex_split_hook():@functools.wraps(flex)def flex_attention_forward(*args, **kwargs):+ if _IN_BLOCK[0] != _OFF:+ kwargs["kernel_options"] = _training_flex_options(kwargs.get("kernel_options"))if _IN_BLOCK[0] == _SPLIT and torch.compiler.is_compiling():return outside(*args, **kwargs)return flex(*args, **kwargs)@@ -660,7 +686,7 @@ def _layer_kind(layer):def _wrap(cls, compile_options):original = cls.forward- # Unsloth layer forwards read packed lengths on the host (packing.py): packed calls stay stock.+ # Host-read packed lengths / PrefixGrouper segments stay stock.unsloth_forward = getattr(original, "__module__", "").startswith("unsloth.models.")compiled = torch.compile(original, fullgraph = False, dynamic = None, options = compile_options)try:@@ -709,6 +735,8 @@ def _wrap(cls, compile_options):or _forced_float32(self)or _zero3_partitioned(self)or (unsloth_forward and kwargs.get("packed_seq_lengths") is not None)+ # PrefixGrouper's int segment table would recompile at every new GRPO length.+ or (unsloth_forward and kwargs.get("prefix_seg_info") is not None)):return original(self, hidden_states, *args, **kwargs)if tracing:diff --git a/unsloth/models/_gdn_op.py b/unsloth/models/_gdn_op.pyindex ff4038f1a73..bc27cd5c711 100644--- a/unsloth/models/_gdn_op.py+++ b/unsloth/models/_gdn_op.py@@ -5,6 +5,9 @@checked bit-equal to ChunkGatedDeltaRuleFunction before use, so a Gated DeltaNet layer compilesas one graph."""+import inspect+from typing import Optional+import torchimport torch.nn.functional as Ffrom torch.autograd.graph import saved_tensors_hooks@@ -49,11 +52,33 @@ def _fla_ops(make_inputs):with torch.enable_grad(), saved_tensors_hooks(lambda t: t.detach(), lambda t: t):_FLA = fns_FLA = fns if _self_check(*inputs) else False+ # Packed (padding-free) batches: B = 1 plus cu_seqlens, checked the same way.+ _VARLEN[0] = bool(_FLA) and _self_check_varlen(*inputs)except Exception:_FLA = False+ _VARLEN[0] = Falsereturn _FLA+# True once the op matched fla's Function on a packed (cu_seqlens) batch too.+_VARLEN = [False]+++def varlen_ok() -> bool:+ return _VARLEN[0]+++def _varlen_kwargs(fn, cu_seqlens):+ """cu_seqlens (+ the chunk_indices fla's Function derives from it) for fla's fwd / bwd."""+ if cu_seqlens is None:+ return {}+ kwargs = {"cu_seqlens": cu_seqlens}+ if "chunk_indices" in inspect.signature(fn).parameters:+ from fla.ops.utils.index import prepare_chunk_indices+ kwargs["chunk_indices"] = prepare_chunk_indices(cu_seqlens, 64)+ return kwargs++def _chunk_meta(q, k, v, g):# (qn, q_rstd, kn, k_rstd, g_cumsum, A), as fla 0.5 allocates themreturn (@@ -76,6 +101,7 @@ try:g: torch.Tensor,beta: torch.Tensor,scale: float,+ cu_seqlens: Optional[torch.Tensor] = None,) -> tuple[torch.Tensor,torch.Tensor,@@ -99,12 +125,21 @@ try:scale = scale,initial_state = None,output_final_state = False,+ **_varlen_kwargs(fwd, cu_seqlens),)# Fake metas are contiguous; keep the real outputs the same.return tuple(t.contiguous() for t in (o.to(q.dtype), qn, q_rstd, kn, k_rstd, g_cum, A))@_gdn_chunk.register_fake- def _gdn_chunk_fake(q, k, v, g, beta, scale):+ def _gdn_chunk_fake(+ q,+ k,+ v,+ g,+ beta,+ scale,+ cu_seqlens = None,+ ):o = v.new_empty(v.shape, dtype = q.dtype)return (o,) + tuple(q.new_empty(shape, dtype = dtype) for shape, dtype in _chunk_meta(q, k, v, g)@@ -122,6 +157,7 @@ try:A: torch.Tensor,do: torch.Tensor,scale: float,+ cu_seqlens: Optional[torch.Tensor] = None,) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:_, bwd, _, l2_bwd = _FLAv, beta, do = v.contiguous(), beta.contiguous(), do.contiguous()@@ -137,6 +173,7 @@ try:initial_state = None,do = do,dht = None,+ **_varlen_kwargs(bwd, cu_seqlens),)dq = l2_bwd(qn, q_rstd, dq)dk = l2_bwd(kn, k_rstd, dk)@@ -150,21 +187,35 @@ try:return tuple(t.contiguous() for t in out)@_gdn_chunk_bwd.register_fake- def _gdn_chunk_bwd_fake(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A, do, scale):+ def _gdn_chunk_bwd_fake(+ qn,+ q_rstd,+ kn,+ k_rstd,+ v,+ g_cum,+ beta,+ A,+ do,+ scale,+ cu_seqlens = None,+ ):return tuple(qn.new_empty(t.shape, dtype = t.dtype) for t in (qn, kn, v, g_cum, beta))def _gdn_chunk_setup(ctx, inputs, output):- q, k, v, g, beta, scale = inputs+ q, k, v, g, beta, scale = inputs[:6]+ cu_seqlens = inputs[6] if len(inputs) > 6 else None_, qn, q_rstd, kn, k_rstd, g_cum, A = output- ctx.save_for_backward(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A)+ ctx.save_for_backward(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A, cu_seqlens)ctx.scale = scalectx.dtypes = (q.dtype, k.dtype, g.dtype)def _gdn_chunk_backward(ctx, do, *_):- saved = [t.contiguous() for t in ctx.saved_tensors]- dq, dk, dv, dg, db = _gdn_chunk_bwd(*saved, do.contiguous(), ctx.scale)+ *saved, cu_seqlens = ctx.saved_tensors+ saved = [t.contiguous() for t in saved]+ dq, dk, dv, dg, db = _gdn_chunk_bwd(*saved, do.contiguous(), ctx.scale, cu_seqlens)q_dtype, k_dtype, g_dtype = ctx.dtypes- return dq.to(q_dtype), dk.to(k_dtype), dv, dg.to(g_dtype), db, None+ return dq.to(q_dtype), dk.to(k_dtype), dv, dg.to(g_dtype), db, None, None_gdn_chunk.register_autograd(_gdn_chunk_backward, setup_context = _gdn_chunk_setup)except Exception:@@ -193,3 +244,36 @@ def _self_check(q, k, v, g, beta):and torch.equal(out[0], ref)and all(torch.equal(a, b) for a, b in zip(got_g, ref_g)))+++def _self_check_varlen(q, k, v, g, beta):+ """Packed batch: bit-equal to fla's Function; False if fla's fwd / bwd lack cu_seqlens."""+ if _gdn_chunk is None or not q.is_cuda:+ return False+ fwd, bwd, _, _ = _FLA+ for fn in (fwd, bwd):+ if "cu_seqlens" not in inspect.signature(fn).parameters:+ return False+ from fla.ops.gated_delta_rule import chunk_gated_delta_rule++ B, T = q.shape[0], q.shape[1]+ flat = [t.reshape(1, B * T, *t.shape[2:]) for t in (q, k, v, g, beta)]+ total = B * T+ # Uneven documents, one shorter than a chunk and one crossing a chunk edge.+ cuts = sorted({0, min(17, total), min(total // 2 + 5, total), total})+ cu_seqlens = torch.tensor(cuts, device = q.device, dtype = torch.int32)+ gen = torch.Generator(device = v.device).manual_seed(1)+ do = torch.randn(flat[2].shape, dtype = v.dtype, device = v.device, generator = gen)+ ins = [t.contiguous().requires_grad_() for t in flat]+ ref, _ = chunk_gated_delta_rule(*ins, use_qk_l2norm_in_kernel = True, cu_seqlens = cu_seqlens)+ ref_g = torch.autograd.grad(ref, ins, do)+ ins2 = [t.detach().clone().requires_grad_() for t in ins]+ out = _gdn_chunk(*ins2, flat[0].shape[-1] ** -0.5, cu_seqlens)+ got_g = torch.autograd.grad(out[0], ins2, do)+ metas = [(o.shape, o.dtype) for o in out[1:]]+ want = [(torch.Size(s), d) for s, d in _chunk_meta(*flat[:4])]+ return (+ metas == want+ and torch.equal(out[0], ref)+ and all(torch.equal(a, b) for a, b in zip(got_g, ref_g))+ )