Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Compile each transformer block as one region for every FastModel model (LoRA and QLoRA) (#10744)
Hexu Zhaoc69969d58d0e
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pynew file mode 100644index 00000000000..63d3b93562e--- /dev/null+++ b/tests/test_block_compile.py@@ -0,0 +1,715 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""One compiled region per decoder layer (unsloth/models/_block_compile.py).++CPU: sequence dims (not widths) are marked dynamic, rebound globals resolve, and a block that+fails to compile retries with flex_attention outside, then keeps the stock forward.+GPU: compiled Qwen3 and Qwen3.5 layers match the stock forward and gradients with no graph break+(Qwen3.5's fla kernel runs as the opaque op inside the block).+"""++import importlib.util+import os+import sys+import types++import pytest+from real_accelerator import has_real_cuda++torch = pytest.importorskip("torch")++_DIR = os.path.join(os.path.dirname(__file__), "..", "unsloth", "models")++[email protected](autouse = True)+def _block_env(monkeypatch):+ # Other test modules set these process-wide (e.g. UNSLOTH_COMPILE_DISABLE=1 at import).+ for key in ("UNSLOTH_BLOCK_COMPILE", "UNSLOTH_COMPILE_DISABLE", "UNSLOTH_FORCE_FLOAT32"):+ monkeypatch.delenv(key, raising = False)+++def _load(name = "_block_compile"):+ # By file path: importing the unsloth package needs an accelerator.+ spec = importlib.util.spec_from_file_location(f"_u{name}", os.path.join(_DIR, f"{name}.py"))+ mod = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = mod+ spec.loader.exec_module(mod)+ return mod+++def test_mark_seq_dynamic_skips_widths(monkeypatch):+ M = _load()+ marked = []+ monkeypatch.setattr(+ torch._dynamo, "maybe_mark_dynamic", lambda t, d: marked.append((t.dim(), d))+ )+ L = 16+ hidden = torch.zeros(2, L, L) # hidden size equal to the sequence length+ mask4d = torch.zeros(2, 1, L, L)+ mask2d = torch.zeros(2, L)+ mrope_ids = torch.zeros(3, 2, L, dtype = torch.long)+ M._mark_seq_dynamic(L, (hidden, {"m": mask4d}, [mask2d, "x", None], mrope_ids))+ assert sorted(marked) == [(2, 1), (3, 1), (3, 2), (4, 2), (4, 3)]+++def test_rebind_globals_resolves():+ M = _load()+ ns = {"__name__": "not_a_registered_module", "OFFSET": 3}+ exec("def f(x):\n return x + OFFSET\n", ns)+ f = M._rebind_globals(ns["f"])+ assert f is not ns["f"] and f(1) == 4+ assert sys.modules[f.__module__].__dict__ is f.__globals__+ g = M._rebind_globals(f)+ assert g is f # already owned by a registered module+ exec("def h():\n global OFFSET\n OFFSET += 1\n", ns)+ assert M._rebind_globals(ns["h"]) is ns["h"] # a copied namespace would miss the write+++class _Boom(torch._dynamo.exc.TorchDynamoException):+ pass+++def _toy_class():+ class ToyDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(+ self,+ hidden_states,+ attention_mask = None,+ ):+ return self.proj(hidden_states) + hidden_states++ return ToyDecoderLayer++[email protected]("fails_in", [("full",), ("full", "split")])+def test_compile_failure_falls_back(monkeypatch, fails_in):+ M = _load()+ calls = []++ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ mode = {M._FULL: "full", M._SPLIT: "split"}[M._IN_BLOCK[0]]+ calls.append(mode)+ if mode in fails_in:+ raise _Boom("lowering failed")+ return fn(*args, **kw)++ return compiled++ monkeypatch.setattr(torch, "compile", fake_compile)+ cls = _toy_class()+ original = cls.forward+ M._wrap(cls, {})+ layer = cls()+ x = torch.randn(2, 4, 8)+ with torch.no_grad():+ want = original(layer, x)+ got = layer(x)+ torch.testing.assert_close(got, want)+ layer(x)+ if fails_in == ("full",):+ assert layer._unsloth_block_mode == M._SPLIT+ # Grad-mode trial (full fails, split), then the no-grad call twice.+ assert calls == ["full", "split", "split", "split"]+ else:+ assert layer._unsloth_block_mode == M._OFF+ assert calls == ["full", "split"] # then the stock forward, never compiled again+ # A sibling of the same kind starts where this layer ended, without retrying what failed.+ calls.clear()+ with torch.no_grad():+ cls()(x)+ assert calls == (["split", "split"] if fails_in == ("full",) else [])+ # Eval keeps the stock forward (logits equal transformers bit for bit).+ calls.clear()+ sibling = cls().eval()+ sibling(x)+ assert calls == []+++def test_cache_calls_keep_the_stock_forward(monkeypatch):+ from transformers.cache_utils import DynamicCache++ M = _load()+ calls = []++ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ calls.append(1)+ return fn(*args, **kw)++ return compiled++ monkeypatch.setattr(torch, "compile", fake_compile)++ class CacheLayerDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(+ self,+ hidden_states,+ past_key_value = None,+ **kwargs,+ ):+ return self.proj(hidden_states)++ M._wrap(CacheLayerDecoderLayer, {})+ layer = CacheLayerDecoderLayer()+ x = torch.randn(2, 4, 8)+ with torch.no_grad():+ layer(x, past_key_value = DynamicCache())+ layer(x, DynamicCache())+ layer(x, past_key_values = DynamicCache())+ assert calls == []+ with torch.no_grad():+ layer(x)+ assert calls # no cache: compiled+++def test_gemma4_band_fast_path_in_blocks(monkeypatch):+ M = _load()+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS++ scans, routed, plain_calls = [], [], []+ band = types.ModuleType("fake_banded_attention")+ band._mask_is_plain_band = lambda mask, S, w: scans.append((S, w)) or True+ zoo = types.ModuleType("fake_flash_sliding")++ def plain(module, q, k, v, mask, **kw):+ plain_calls.append(mask)+ return "plain", None++ def router(module, q, k, v, mask, **kw):+ routed.append(mask)+ return "router", None++ router._unsloth_gemma4_flash = True+ router.__module__ = zoo.__name__+ zoo._ORIG_SDPA = [plain]+ monkeypatch.setitem(sys.modules, band.__name__, band)+ monkeypatch.setitem(sys.modules, zoo.__name__, zoo)+ monkeypatch.setattr(M, "_BAND_MODULE", band.__name__)+ monkeypatch.setitem(ALL_ATTENTION_FUNCTIONS, "sdpa", router)+ M._gemma4_router_hook()+ wrapped = ALL_ATTENTION_FUNCTIONS["sdpa"]+ assert wrapped is not router+ # Carries zoo's marker, so zoo's patch on a later model load does not wrap it again+ # with it as the fallback (that loop recursed forever).+ assert wrapped._unsloth_gemma4_flash+ M._gemma4_router_hook()+ assert ALL_ATTENTION_FUNCTIONS["sdpa"] is wrapped++ class Gemma4TextAttention(torch.nn.Module):+ is_sliding, sliding_window, is_causal = True, 64, True++ layer = torch.nn.Module()+ layer.self_attn = attn = Gemma4TextAttention()+ short, long = torch.zeros(1, 1, 32, 32), torch.zeros(1, 1, 96, 96)+ M._precheck_band(layer, 32, (), {"attention_mask": short})+ M._precheck_band(layer, 96, (long,), {})+ assert scans == [(32, 64)] # longer than the window: never scanned+ assert (short._unsloth_block_band, long._unsloth_block_band) == (True, False)++ q = torch.randn(1, 2, 32, 16)+ assert wrapped(attn, q, q, q, short)[0] == "router" # eager: the stock router+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ M._IN_BLOCK[0] = M._FULL+ try:+ out, _ = wrapped(attn, q, q, q, short, scaling = 0.25)+ want = torch.nn.functional.scaled_dot_product_attention(q, q, q, is_causal = True, scale = 0.25)+ torch.testing.assert_close(out, want.transpose(1, 2))+ ql = torch.randn(1, 2, 96, 16)+ assert wrapped(attn, ql, ql, ql, long)[0] == "plain"+ finally:+ M._IN_BLOCK[0] = M._OFF+ assert routed == [short] and plain_calls == [long]+++def test_kill_switch(monkeypatch):+ M = _load()+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE", "0")+ assert M.patch_block_compile(torch.nn.Module()) == []+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE", "1")+ monkeypatch.setenv("UNSLOTH_COMPILE_DISABLE", "1")+ assert not M._enabled()+ monkeypatch.setenv("UNSLOTH_COMPILE_DISABLE", "0")+ assert M._enabled()+ monkeypatch.setattr(torch, "__version__", "2.6.0+cu124")+ assert not M._torch_supported()+ monkeypatch.setattr(torch, "__version__", "2.7.1+cu126")+ assert M._torch_supported()+++def test_forced_float32_is_decided_per_model(monkeypatch):+ M = _load()+ calls = []++ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ calls.append(1)+ return fn(*args, **kw)++ return compiled++ monkeypatch.setattr(torch, "compile", fake_compile)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(M, "_torch_supported", lambda: True)+ cls = _toy_class()++ def model_of(forced):+ model = torch.nn.Module()+ model.layers = torch.nn.ModuleList([cls(), cls()])+ if forced is not None:+ model._unsloth_forced_float32 = forced+ return model++ # Forced by this load's env (the stamp lands after the first post_patch_model).+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "1")+ forced = model_of(None)+ assert M.patch_block_compile(forced) == []+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "0")+ plain = model_of(False)+ assert M.patch_block_compile(plain) == [cls.__name__]+ x = torch.randn(2, 4, 8)+ with torch.no_grad():+ forced.layers[0](x)+ assert calls == []+ with torch.no_grad():+ plain.layers[0](x)+ assert len(calls) == 2 # grad-mode trial + the call+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "1")+ with torch.no_grad():+ plain.layers[1](x)+ assert len(calls) == 4+ # get_peft_model re-patches a wrapper after another load reset the env: the stamp holds.+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "0")++ class Delegating(torch.nn.Module):+ def __init__(self, inner):+ super().__init__()+ self.inner = inner++ def __getattr__(self, name):+ try:+ return super().__getattr__(name)+ except AttributeError:+ return getattr(self.inner, name)++ forced._unsloth_forced_float32 = True+ wrapper = torch.nn.Module()+ wrapper.base = forced+ M.patch_block_compile(wrapper)+ M.patch_block_compile(Delegating(forced))+ calls.clear()+ with torch.no_grad():+ forced.layers[0](x)+ assert calls == []+++def test_checkpoint_recompute_keeps_the_no_grad_mode(monkeypatch):+ M = _load()+ calls = []++ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ mode = {M._FULL: "full", M._SPLIT: "split"}[M._IN_BLOCK[0]]+ grad = torch.is_grad_enabled()+ calls.append((mode, grad))+ if grad:+ torch.rand(3)+ if mode == "full":+ raise _Boom("grad-mode lowering failed")+ return fn(*args, **kw)++ return compiled++ monkeypatch.setattr(torch, "compile", fake_compile)+ cls = _toy_class()+ M._wrap(cls, {})+ layer = cls()+ x = torch.randn(2, 4, 8)+ rng = torch.get_rng_state()+ with torch.no_grad():+ layer(x)+ # The grad-mode trial demoted first, so the no-grad half already ran in SPLIT.+ assert calls == [("full", True), ("split", True), ("split", False)]+ assert torch.equal(torch.get_rng_state(), rng)+ calls.clear()+ layer(x.requires_grad_())+ assert calls == [("split", True)]+++def test_gdn_self_check_leaves_default_rng_alone():+ # Checkpoint recompute replays the default generator for LoRA dropout masks.+ pytest.importorskip("fla.ops.gated_delta_rule.chunk")+ if not has_real_cuda():+ pytest.skip("needs a real CUDA device")+ G = _load("_gdn_op")+ gen = torch.Generator(device = "cuda").manual_seed(0)+ B, T, H, D = 1, 70, 2, 128+ mk = lambda *s, dt = torch.bfloat16: torch.randn(*s, device = "cuda", dtype = dt, generator = gen)+ q, k, v = mk(B, T, H, D), mk(B, T, H, D), mk(B, T, H, D)+ g = -mk(B, T, H, dt = torch.float32).abs() * 0.1+ beta = mk(B, T, H).sigmoid()+ state = torch.cuda.get_rng_state()+ assert G._fla_ops(lambda: (q, k, v, g, beta))+ assert torch.equal(torch.cuda.get_rng_state(), state)+++def test_sm_count_is_read_outside_the_trace(monkeypatch):+ # Stands in for Triton's hip_utils.get_device_properties, which Dynamo cannot trace.+ M = _load()+ fla = types.ModuleType("fake.fla.modules.fused_norm_gate")+ calls = []++ @torch._dynamo.disable+ def get_multiprocessor_count(tensor_idx = 0):+ calls.append(tensor_idx)+ return 7++ fla.get_multiprocessor_count = get_multiprocessor_count+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)+ M._route_sm_count({fla.__name__: fla})+ out = torch.compile(+ lambda x: x * fla.get_multiprocessor_count(0), fullgraph = True, backend = "eager"+ )(torch.ones(2))+ assert out.tolist() == [7.0, 7.0] and calls == [0]+++def _grads_after_step(layers, run, x):+ for layer in layers:+ layer.zero_grad(set_to_none = True)+ out = x+ for layer in layers:+ out = run(layer, out)+ loss = out.float().pow(2).mean()+ loss.backward()+ return loss.detach(), {+ n: p.grad.clone() for layer in layers for n, p in layer.named_parameters()+ }+++def _check_block(layers, run, x, cls):+ """Stock forward + backward vs the compiled block, two sequence lengths, no graph break."""+ from torch._dynamo.utils import counters++ import unsloth.models._block_compile as M++ want = [_grads_after_step(layers, run, xi) for xi in x]+ original = cls.forward+ try:+ M._wrap(M._rebind_class(cls) or cls, M._COMPILE_OPTIONS)+ counters.clear()+ got = [_grads_after_step(layers, run, xi) for xi in x]+ assert sum(counters["graph_break"].values()) == 0, dict(counters["graph_break"])+ finally:+ cls.forward = original+ torch._dynamo.reset()+ for (lw, gw), (lg, gg) in zip(want, got):+ torch.testing.assert_close(lg, lw, rtol = 1e-4, atol = 1e-5)+ for k in gw:+ torch.testing.assert_close(gg[k], gw[k], rtol = 1e-3, atol = 1e-5)++[email protected](not has_real_cuda(), reason = "compiled block needs CUDA")+def test_qwen3_block_matches_stock():+ pytest.importorskip("unsloth")+ from transformers.models.qwen3 import modeling_qwen3 as modeling+ from transformers.models.qwen3.configuration_qwen3 import Qwen3Config++ cfg = Qwen3Config(+ vocab_size = 64,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 32,+ )+ cfg._attn_implementation = "sdpa"+ torch.manual_seed(0)+ layers = [modeling.Qwen3DecoderLayer(cfg, layer_idx = i).cuda() for i in range(2)]+ rotary = modeling.Qwen3RotaryEmbedding(cfg).cuda()++ def run(layer, h):+ pos = torch.arange(h.shape[1], device = h.device)[None].expand(h.shape[0], -1)+ out = layer(h, position_embeddings = rotary(h, pos), position_ids = pos)+ return out[0] if isinstance(out, tuple) else out++ x = [torch.randn(2, L, 64, device = "cuda", requires_grad = True) for L in (24, 40)]+ _check_block(layers, run, x, modeling.Qwen3DecoderLayer)++[email protected](not has_real_cuda(), reason = "fla kernels need CUDA")+def test_qwen3_5_gdn_block_has_no_break():+ pytest.importorskip("unsloth")+ pytest.importorskip("fla.ops.gated_delta_rule")+ from transformers.models.qwen3_5 import modeling_qwen3_5 as modeling+ from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig++ import unsloth.models._block_compile as M++ cfg = Qwen3_5TextConfig(+ vocab_size = 64,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 32,+ linear_num_key_heads = 4,+ linear_num_value_heads = 16,+ linear_key_head_dim = 32,+ linear_value_head_dim = 32,+ layer_types = ["linear_attention", "full_attention"],+ )+ cfg._attn_implementation = "sdpa"+ torch.manual_seed(0)+ layers = [+ modeling.Qwen3_5DecoderLayer(cfg, layer_idx = i).cuda().to(torch.bfloat16) for i in range(2)+ ]+ rotary = modeling.Qwen3_5TextRotaryEmbedding(cfg).cuda()+ x = [torch.randn(2, L, 64, device = "cuda", dtype = torch.bfloat16) for L in (64, 128)]+ M._prepare_gdn(layers[0], x[0])+ if not M._GDN_OP[0]:+ pytest.skip("fla's opaque op failed its self-check on this GPU")++ def run(layer, h):+ pos = torch.arange(h.shape[1], device = h.device)[None, None].expand(3, h.shape[0], -1)+ out = layer(h, position_embeddings = rotary(h, pos), position_ids = pos)+ return out[0] if isinstance(out, tuple) else out++ from torch._dynamo.utils import counters++ cls = modeling.Qwen3_5DecoderLayer+ original = cls.forward+ want = [_grads_after_step(layers, run, xi.clone().requires_grad_()) for xi in x]+ try:+ M._wrap(cls, M._COMPILE_OPTIONS)+ counters.clear()+ got = [_grads_after_step(layers, run, xi.clone().requires_grad_()) for xi in x]+ assert sum(counters["graph_break"].values()) == 0, dict(counters["graph_break"])+ finally:+ cls.forward = original+ torch._dynamo.reset()+ for (lw, gw), (lg, gg) in zip(want, got):+ torch.testing.assert_close(lg, lw, rtol = 2e-2, atol = 1e-3)+ for k in gw:+ num = (gg[k].float() - gw[k].float()).norm()+ assert num <= 0.05 * gw[k].float().norm() + 1e-4, (+ k,+ float(num),+ float(gw[k].float().norm()),+ )++[email protected](not has_real_cuda(), reason = "fla kernels need a GPU")+def test_opaque_fla_op_matches_fla_function():+ pytest.importorskip("fla.ops.gated_delta_rule.chunk")+ M = _load("_gdn_op")+ gen = torch.Generator(device = "cuda").manual_seed(0)+ B, T, H, D = 2, 150, 4, 128+ mk = lambda *s, dt = torch.bfloat16: torch.randn(*s, device = "cuda", dtype = dt, generator = gen)+ q, k, v = mk(B, T, H, D), mk(B, T, H, D), mk(B, T, H, D)+ g = -mk(B, T, H, dt = torch.float32).abs() * 0.1+ beta = mk(B, T, H).sigmoid()+ assert M._fla_ops(lambda: (q, k, v, g, beta))+ # A wrong fake meta or a numeric mismatch must keep the op out of the block.+ M._FLA = None+ orig = M._chunk_meta+ M._chunk_meta = lambda *a: orig(*a)[:-1] + (((1,), torch.bfloat16),)+ try:+ assert M._fla_ops(lambda: (q, k, v, g, beta)) is False+ finally:+ M._chunk_meta = orig+++def test_gdn_adapter_routes_only_the_self_checked_call(monkeypatch):+ M = _load()+ seen = []+ monkeypatch.setattr(M, "_GDN_OP", [True])+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ monkeypatch.setattr(+ torch.ops,+ "unsloth",+ types.SimpleNamespace(qwen3_5_gdn_chunk = lambda *a: seen.append("op") or ("out",)),+ raising = False,+ )+ adapter = M._gdn_chunk_adapter(lambda *a, **kw: seen.append("fla") or ("fla", None))+ t = torch.zeros(1, 4, 2, 8)+ gb = torch.zeros(1, 4, 2)+ stock = dict(+ use_qk_l2norm_in_kernel = True, initial_state = None, output_final_state = False, cu_seqlens = None+ )+ adapter(t, t, t, g = gb, beta = gb, **stock)+ for extra in (+ {"head_first": True},+ {"use_beta_sigmoid_in_kernel": True},+ {"scale": 0.1},+ {"output_final_state": True},+ {"cu_seqlens": torch.tensor([0, 4])},+ {"use_qk_l2norm_in_kernel": False},+ ):+ adapter(t, t, t, g = gb, beta = gb, **dict(stock, **extra))+ assert seen == ["op"] + ["fla"] * 6++[email protected]("linear_env", ["auto", "0"])+def test_nf4_block_route_matches_both_checkpoint_halves(monkeypatch, linear_env):+ # Package context so `from ..kernels import bnb_override` resolves to the fakes below.+ pkg = types.ModuleType("_ufake")+ pkg.__path__ = []+ models = types.ModuleType("_ufake.models")+ models.__path__ = []+ kernels = types.ModuleType("_ufake.kernels")+ kernels.__path__ = []+ O = types.ModuleType("_ufake.kernels.bnb_override")+ O._TRACE, O._TRACE_BF16, O._LINEAR = True, True, linear_env+ O._eligible = lambda self, x, weight, quant_state: True+ O._linear = lambda x, weight, quant_state, bias: torch.full_like(x, 7.0)+ kernels.bnb_override = O++ class Linear4bit(torch.nn.Module):+ def __init__(self, dtype, bias):+ super().__init__()+ self.weight = types.SimpleNamespace(quant_state = types.SimpleNamespace(dtype = dtype))+ self.bias = torch.nn.Parameter(torch.zeros(8)) if bias else None++ def forward(self, x):+ return torch.zeros_like(x)++ bnb = types.ModuleType("bitsandbytes")+ bnb.nn = types.SimpleNamespace(Linear4bit = Linear4bit)+ for name, mod in (+ ("_ufake", pkg),+ ("_ufake.models", models),+ ("_ufake.kernels", kernels),+ ("_ufake.kernels.bnb_override", O),+ ("bitsandbytes", bnb),+ ):+ monkeypatch.setitem(sys.modules, name, mod)+ spec = importlib.util.spec_from_file_location(+ "_ufake.models._block_compile", os.path.join(_DIR, "_block_compile.py")+ )+ M = importlib.util.module_from_spec(spec)+ monkeypatch.setitem(sys.modules, spec.name, M)+ spec.loader.exec_module(M)+ M._nf4_in_blocks()+ if linear_env == "0":+ assert not getattr(Linear4bit.forward, "_unsloth_block_nf4", False)+ return+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ monkeypatch.setattr(M, "_IN_BLOCK", [M._FULL])++ def routed(+ dtype,+ rows,+ bias = False,+ ):+ return bool(Linear4bit(dtype, bias)(torch.ones(rows, 8, dtype = dtype)).eq(7).all())++ assert routed(torch.bfloat16, 16)+ assert routed(torch.float32, 8)+ assert not routed(torch.bfloat16, 1) # one row: fast_gemv only without grad+ assert not routed(torch.float32, 4) # bitsandbytes' fp32 small-row kernel+ assert not routed(torch.bfloat16, 16, bias = True) # trainable bias+++def test_causal_depthwise_conv_matches_conv1d():+ M = _load("_gdn_op")+ torch.manual_seed(0)+ x, w = torch.randn(2, 6, 37), torch.randn(6, 4)+ want = torch.nn.functional.conv1d(x, w[:, None], groups = 6, padding = 3)[..., :37]+ torch.testing.assert_close(M._causal_depthwise_conv(x, w), want, rtol = 1e-5, atol = 1e-5)+++def test_disable_unless_decode_keeps_original_traceable_on_old_torch(monkeypatch):+ # torch < 2.8: compiler.disable(fn, recursive = False) marks fn itself and returns it.+ import unsloth.import_fixes as F+ import unsloth_zoo.temporary_patches.utils as zoo_utils++ marked = []++ def old_disable(+ fn = None,+ recursive = True,+ **kwargs,+ ):+ fn._old_disable_marked = True+ marked.append(fn)+ return fn++ monkeypatch.setattr(torch.compiler, "disable", old_disable)+ monkeypatch.setattr(+ zoo_utils,+ "torch_compiler_disable_unless_decode",+ zoo_utils.torch_compiler_disable_unless_decode,+ )+ F.fix_torch_compiler_disable_unless_decode()+ fixed = zoo_utils.torch_compiler_disable_unless_decode+ assert fixed._unsloth_disable_copy++ def attention(x):+ return x + 1++ wrapped = fixed(attention)+ assert wrapped._unsloth_undisabled is attention+ assert not getattr(attention, "_old_disable_marked", False)+ disabled_copy = marked[-1]+ assert disabled_copy is not attention and disabled_copy.__code__ is not attention.__code__+ assert disabled_copy(1) == attention(1) == 2+ assert torch.compiler.disable is old_disable+++def test_moe_experts_run_outside_the_block(monkeypatch):+ M = _load()++ class ToyExperts(torch.nn.Module):+ def forward(self, x):+ return x * 2++ class LoraWrapper(torch.nn.Module): # PEFT-style wrapper around the experts+ def __init__(self):+ super().__init__()+ self.base_layer = ToyExperts()++ def forward(self, x):+ return self.base_layer(x) + 1++ layer = torch.nn.Module()+ layer.mlp = torch.nn.Module()+ layer.mlp.experts = LoraWrapper()+ layer.self_attn = torch.nn.Linear(4, 4)+ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: True)+ M._experts_outside(layer)+ outer = layer.mlp.experts+ assert (+ outer.forward.__func__ is not LoraWrapper.forward+ if hasattr(outer.forward, "__func__")+ else True+ )+ assert outer._unsloth_block_outside and outer.base_layer._unsloth_block_outside+ assert not getattr(layer.self_attn, "_unsloth_block_outside", False)+ from torch._dynamo.utils import counters++ counters.clear()+ out = torch.compile(lambda x: layer.mlp.experts(x) * 3, backend = "eager")(torch.ones(2))+ assert out.tolist() == [9.0, 9.0] and sum(counters["graph_break"].values()) >= 1+ # A stash-reading experts forward (no fold) stays inside; so does everything under "inside".+ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: False)+ other = torch.nn.Module()+ other.experts = LoraWrapper()+ M._experts_outside(other)+ assert not getattr(other.experts, "_unsloth_block_outside", False)+ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: True)+ monkeypatch.setenv("UNSLOTH_BLOCK_MOE", "inside")+ third = torch.nn.Module()+ third.experts = LoraWrapper()+ M._experts_outside(third)+ assert not getattr(third.experts, "_unsloth_block_outside", False)diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex f0c7dd8cb8d..5aa2536177f 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -253,6 +253,13 @@ except ModuleNotFoundError:except:raise+try:+ from .import_fixes import fix_torch_compiler_disable_unless_decode+ fix_torch_compiler_disable_unless_decode()+ del fix_torch_compiler_disable_unless_decode+except Exception:+ pass+# Re-assert after unsloth_zoo's patch_torch_compile, which historically popped# TORCHINDUCTOR_COMPILE_THREADS.if os.environ.get("UNSLOTH_FORCE_SINGLE_COMPILE_WORKER", "0") == "1":diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 12da2bc8777..bf0ccc38e75 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -264,6 +264,64 @@ if not UNSLOTH_ENABLE_LOGGING:)+def fix_torch_compiler_disable_unless_decode():+ """torch < 2.8 marks `fn` itself in `compiler.disable(fn, recursive = False)`, so zoo's+ decode-aware disable disabled the function it runs: hand it a copy to disable."""+ import functools+ import types++ try:+ import torch+ import unsloth_zoo.temporary_patches.utils as zoo_utils+ except Exception:+ return+ current = getattr(zoo_utils, "torch_compiler_disable_unless_decode", None)+ if current is None or getattr(current, "_unsloth_disable_copy", False):+ return++ def _probe(x):+ return x++ try:+ if torch.compiler.disable(_probe, recursive = False) is not _probe:+ return+ except Exception:+ return+ real_disable = torch.compiler.disable++ def _copy(fn):+ clone = types.FunctionType(+ fn.__code__.replace(), fn.__globals__, fn.__name__, fn.__defaults__, fn.__closure__+ )+ clone.__kwdefaults__ = fn.__kwdefaults__+ clone.__dict__.update(fn.__dict__)+ clone.__qualname__ = fn.__qualname__+ clone.__module__ = fn.__module__+ return clone++ def disable_copy(+ fn = None,+ *args,+ **kwargs,+ ):+ if isinstance(fn, types.FunctionType):+ fn = _copy(fn)+ return real_disable(fn, *args, **kwargs)++ @functools.wraps(current)+ def torch_compiler_disable_unless_decode(func = None, *, recursive = False):+ if func is None:+ return functools.partial(torch_compiler_disable_unless_decode, recursive = recursive)+ torch.compiler.disable = disable_copy+ try:+ return current(func, recursive = recursive)+ finally:+ torch.compiler.disable = real_disable++ torch_compiler_disable_unless_decode._unsloth_disable_copy = True+ zoo_utils.torch_compiler_disable_unless_decode = torch_compiler_disable_unless_decode++def fix_torch_check_is_size():"""Shim torch._check_is_size if a future torch removes it (bitsandbytes 4-bitdequant calls it). The FutureWarning is silenced in suppress_cuda_printf."""diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pynew file mode 100644index 00000000000..bbbbab30d3c--- /dev/null+++ b/unsloth/models/_block_compile.py@@ -0,0 +1,771 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""One torch.compile region per decoder layer for training (stock forward: attention, MLP or MoE,+norms, LoRA, NF4). UNSLOTH_BLOCK_COMPILE=0 or UNSLOTH_COMPILE_DISABLE turns it off."""++import functools+import os+import sys+import types++import torch++__all__ = ["patch_block_compile"]++_MARK = "_unsloth_block_compiled"+# No max_autotune: it multiplies first-step compile time for every block.+_COMPILE_OPTIONS = {+ "epilogue_fusion": True,+ "shape_padding": True,+ "triton.cudagraphs": False,+}+++# gpt-oss builds its flex mask inside attention: the block broke and recompiled per length.+_SKIP = {"GptOssDecoderLayer"}+++def _enabled():+ return os.environ.get("UNSLOTH_BLOCK_COMPILE", "1") != "0" and os.environ.get(+ "UNSLOTH_COMPILE_DISABLE", "0"+ ) not in ("1", "partial")+++def _forced_float32(module):+ # Zoo's float32-forced patches round to fp16 on purpose; Inductor drops those roundings.+ # Stamped per layer at load: the env var only describes the latest load.+ forced = module.__dict__.get("_unsloth_forced_float32")+ if forced is None:+ forced = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"+ return forced+++def _writes_globals(code):+ import dis+ if any(i.opname in ("STORE_GLOBAL", "DELETE_GLOBAL") for i in dis.get_instructions(code)):+ return True+ return any(_writes_globals(c) for c in code.co_consts if isinstance(c, types.CodeType))+++def _rebind_globals(fn):+ """dynamo resolves a function's globals through sys.modules[__globals__["__name__"]]; Unsloth's+ generated modules exec code into a dict that module does not own (Gemma-4: `module+ 'transformers...modeling_gemma4' has no attribute 'Gemma4RMSNorm_forward'`). Rebuild such a+ function on a registered module holding a copy of that dict. A copy cannot follow later+ writes, so functions that assign globals are left alone."""+ g = getattr(fn, "__globals__", None)+ if g is None or _writes_globals(fn.__code__):+ return fn+ mod = sys.modules.get(g.get("__name__"))+ if mod is not None and mod.__dict__ is g:+ return fn+ name = f"unsloth_block_globals_{id(g)}"+ holder = sys.modules.get(name)+ if holder is None:+ holder = types.ModuleType(name)+ holder.__dict__.update(g)+ holder.__dict__["__name__"] = name+ sys.modules[name] = holder+ new = types.FunctionType(+ fn.__code__, holder.__dict__, fn.__name__, fn.__defaults__, fn.__closure__+ )+ new.__kwdefaults__ = fn.__kwdefaults__+ new.__dict__.update(fn.__dict__)+ new.__qualname__ = fn.__qualname__+ new.__module__ = name+ return new+++def _inline_compiled(fn):+ """A method that is itself torch.compile'd (zoo's patch_function): inside a block, call the+ original so dynamo inlines it. Inlining through the compiled wrapper guards on the wrapper's+ closure as if it were the original's (torch 2.13 / 2.14, Gemma-4 fp16 MLP: `Unexpected type+ CatchErrorsWrapper`). Outside a block the compiled wrapper runs as before."""+ original = getattr(fn, "_torchdynamo_orig_callable", None)+ if (+ not isinstance(original, types.FunctionType)+ or getattr(fn, "_torchdynamo_disable", False)+ or getattr(fn, "_unsloth_block_inline", False)+ ):+ return fn+ compiled = fn+ original = _rebind_globals(original)++ @functools.wraps(original, updated = ())+ def dispatch(*args, **kwargs):+ if torch.compiler.is_compiling():+ return original(*args, **kwargs)+ return compiled(*args, **kwargs)++ dispatch._unsloth_block_inline = True+ return dispatch+++def _rebind_class(cls):+ for key, value in list(cls.__dict__.items()):+ new = _inline_compiled(value) if callable(value) else value+ if isinstance(new, types.FunctionType):+ new = _rebind_globals(new)+ if new is not value:+ setattr(cls, key, new)+++def _mark_seq_dynamic(seq_len, obj):+ if isinstance(obj, torch.Tensor):+ # A float tensor's last dim is a width even when it equals seq_len (mrope ids are int).+ last = obj.dim() - 1 if obj.dim() == 3 and obj.is_floating_point() else None+ for dim, size in enumerate(obj.shape):+ if size == seq_len and dim != last:+ torch._dynamo.maybe_mark_dynamic(obj, dim)+ elif isinstance(obj, (tuple, list)):+ for o in obj:+ _mark_seq_dynamic(seq_len, o)+ elif isinstance(obj, dict):+ for o in obj.values():+ _mark_seq_dynamic(seq_len, o)+++# fla's opaque op state: None until a GatedDeltaNet layer ran its self-check, then True / False.+_GDN_OP = [None]+++def _gdn_chunk_adapter(original):+ # updated = (): the original's __dict__ carries torch.compiler.disable's marker.+ @functools.wraps(original, updated = ())+ def chunk_gated_delta_rule(+ q,+ k,+ v,+ g = None,+ beta = None,+ **kwargs,+ ):+ if (+ _GDN_OP[0]+ and torch.compiler.is_compiling()+ and kwargs.get("use_qk_l2norm_in_kernel") is True+ # The op is the self-checked default: no state, varlen, scale, layout or gating flags.+ and all(+ value is None or value is False+ for key, value in kwargs.items()+ if key != "use_qk_l2norm_in_kernel"+ )+ ):+ # Inductor pins a custom op's input strides; a sliced input fails at the next length.+ q, k, v, g, beta = (t.contiguous() for t in (q, k, v, g, beta))+ out = torch.ops.unsloth.qwen3_5_gdn_chunk(q, k, v, g, beta, q.shape[-1] ** -0.5)+ return out[0], None+ return original(q, k, v, g, beta, **kwargs)++ chunk_gated_delta_rule._unsloth_block_adapter = True+ return chunk_gated_delta_rule+++def _route_fla_kernel():+ """Point modeling code's references to fla's chunk_gated_delta_rule at the adapter. Callers+ reach it through module globals, transformers' hub-kernel wrapper (a closure over it) and+ Unsloth's compiled-module copies, so callers are found from the function object (gc+ referrers), not by name. Closures are left alone: only namespace entries are swapped."""+ import gc++ kernels = []+ for name, module in list(sys.modules.items()):+ if module is not None and name.endswith("fla.ops.gated_delta_rule.chunk"):+ fn = getattr(module, "chunk_gated_delta_rule", None)+ if callable(fn) and fn not in kernels:+ kernels.append(fn)+ callers, frontier = list(kernels), list(kernels)+ for _ in range(4):+ found = []+ for inner in frontier:+ for cell in gc.get_referrers(inner):+ if type(cell).__name__ != "cell":+ continue+ for closure in gc.get_referrers(cell):+ if not isinstance(closure, tuple):+ continue+ for fn in gc.get_referrers(closure):+ if (+ isinstance(fn, types.FunctionType)+ and fn.__closure__ is closure+ and fn not in callers+ and not getattr(fn, "_unsloth_block_adapter", False)+ ):+ callers.append(fn)+ found.append(fn)+ frontier = found+ if not frontier:+ break+ routed = 0+ for caller in callers:+ if getattr(caller, "_unsloth_block_adapter", False):+ continue+ adapter = _gdn_chunk_adapter(caller)+ for space in gc.get_referrers(caller):+ if not isinstance(space, dict) or space is adapter.__dict__:+ continue+ for key, value in list(space.items()):+ if (+ value is caller+ and isinstance(key, str)+ and key.endswith("chunk_gated_delta_rule")+ ):+ space[key] = adapter+ routed += 1+ return routed+++def _causal_conv1d_adapter(original, depthwise_conv):+ # Inductor's conv backward guards on the exact sequence length: recompile per length.+ @functools.wraps(original, updated = ())+ def causal_conv1d_fn(+ hidden_states,+ weight,+ bias = None,+ activation = None,+ **kwargs,+ ):+ if (+ _IN_BLOCK[0]+ and torch.compiler.is_compiling()+ and weight.dim() == 2+ and activation in (None, "silu", "swish")+ and all(kwargs.get(key) is None for key in ("seq_idx", "cu_seqlens", "initial_states"))+ and not kwargs.get("return_final_states")+ ):+ out = depthwise_conv(hidden_states.to(weight.dtype), weight)+ if bias is not None:+ out = out + bias[:, None]+ if activation is not None:+ out = torch.nn.functional.silu(out)+ return out.to(hidden_states.dtype)+ return original(hidden_states, weight, bias, activation = activation, **kwargs)++ causal_conv1d_fn._unsloth_block_adapter = True+ return causal_conv1d_fn+++def _namespaces(fn, depth = 3):+ """Global namespaces a forward reaches through wrappers (closures) and module-level helpers."""+ seen, out, todo = set(), [], [(fn, 0)]+ while todo:+ f, d = todo.pop()+ if not isinstance(f, types.FunctionType) or id(f) in seen:+ continue+ seen.add(id(f))+ if all(f.__globals__ is not g for g in out):+ out.append(f.__globals__)+ if d >= depth:+ continue+ for cell in f.__closure__ or ():+ try:+ todo.append((cell.cell_contents, d + 1))+ except ValueError:+ pass+ for name in f.__code__.co_names:+ todo.append((f.__globals__.get(name), d + 1))+ return out+++def _route_causal_conv(gdn):+ from ._gdn_op import _causal_depthwise_conv+ for space in _namespaces(type(gdn).forward):+ fn = space.get("causal_conv1d_fn")+ if callable(fn) and not getattr(fn, "_unsloth_block_adapter", False):+ space["causal_conv1d_fn"] = _causal_conv1d_adapter(fn, _causal_depthwise_conv)+++def _route_sm_count(modules = None):+ # ROCm: Triton's hip_utils.get_device_properties is untraceable; the SM count is per device.+ for name, module in list((sys.modules if modules is None else modules).items()):+ if module is None or "fla." not in name:+ continue+ fn = getattr(module, "get_multiprocessor_count", None)+ if not callable(fn) or getattr(fn, "_unsloth_block_adapter", False):+ continue+ try:+ counts = {i: int(fn(i)) for i in range(torch.cuda.device_count())}+ except Exception:+ continue++ @functools.wraps(fn, updated = ())+ def get_multiprocessor_count(+ tensor_idx = 0,+ _original = fn,+ _counts = counts,+ ):+ count = _counts.get(tensor_idx)+ return _original(tensor_idx) if count is None else count++ get_multiprocessor_count._unsloth_block_adapter = True+ module.get_multiprocessor_count = get_multiprocessor_count+++def _prepare_gdn(layer, hidden_states):+ """Self-check fla's opaque op on this layer's shapes, then point the GatedDeltaNet forward's+ chunk kernel at the adapter. Runs eagerly, before the first trace."""+ gdn = getattr(layer, "linear_attn", None)+ if gdn is None or _GDN_OP[0] is not None:+ return+ _GDN_OP[0] = False+ try:+ _route_sm_count()+ except Exception:+ pass+ try:+ from ._gdn_op import _fla_ops++ heads, dk, dv = gdn.num_v_heads, gdn.head_k_dim, gdn.head_v_dim+ B, T = hidden_states.shape[0], hidden_states.shape[1]+ dev, dt = hidden_states.device, hidden_states.dtype++ def make_inputs():+ gen = torch.Generator(device = dev).manual_seed(0)+ r = lambda *s, dtype = dt: torch.randn(*s, device = dev, dtype = dtype, generator = gen)+ g = -torch.rand(B, T, heads, device = dev, generator = gen) * 0.1+ return (+ r(B, T, heads, dk),+ r(B, T, heads, dk),+ r(B, T, heads, dv),+ g,+ r(B, T, heads).sigmoid(),+ )++ if not _fla_ops(make_inputs):+ return+ _GDN_OP[0] = _route_fla_kernel() > 0+ if _GDN_OP[0]:+ _route_causal_conv(gdn)+ except Exception:+ _GDN_OP[0] = False+++# zoo's Gemma-4 sliding router scans the mask (data dependent) and declines while compiling, so+# the block runs the scan eagerly and a router wrapper reads the verdict stored on the mask.+_BAND_MODULE = "unsloth_zoo.temporary_patches.gemma4_banded_attention"+_BAND_CHECK = [None]+++def _gemma4_router_hook():+ """Inside a block, a sliding layer whose mask is a plain causal band no longer than the+ window runs is_causal SDPA, as the stock router does; longer bands run masked SDPA. The+ router itself is not traced: its engagement counters drive a log-once branch dynamo guards+ on, and its banded kernel traced in a checkpointed block recomputes different saved tensors+ than its forward (CheckpointError)."""+ try:+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS+ except Exception:+ return+ router = ALL_ATTENTION_FUNCTIONS.get("sdpa")+ if not getattr(router, "_unsloth_gemma4_flash", False) or getattr(+ router, "_unsloth_block_band", False+ ):+ return+ zoo = sys.modules.get(router.__module__)+ plain = getattr(zoo, "_ORIG_SDPA", [None])[0]+ check = getattr(sys.modules.get(_BAND_MODULE), "_mask_is_plain_band", None)+ if plain is None or not callable(check):+ return+ _BAND_CHECK[0] = check++ # wraps() keeps zoo's marker, else a later load wraps this as its own fallback (endless loop).+ @functools.wraps(router)+ def sdpa_attention_forward(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = 0.0,+ scaling = None,+ is_causal = None,+ **kwargs,+ ):+ if (+ _IN_BLOCK[0]+ and torch.compiler.is_compiling()+ and getattr(module, "is_sliding", False)+ and type(module).__name__.startswith("Gemma4")+ and query.dim() == 4+ and query.shape[-1] <= 256+ ):+ window = kwargs.get("sliding_window", None) or getattr(module, "sliding_window", None)+ causal = is_causal if is_causal is not None else getattr(module, "is_causal", True)+ if (+ window+ and query.shape[2] == key.shape[2]+ and query.shape[2] <= window+ and (+ causal+ if attention_mask is None+ else getattr(attention_mask, "_unsloth_block_band", False) is True+ )+ ):+ out = torch.nn.functional.scaled_dot_product_attention(+ query,+ key,+ value,+ dropout_p = dropout if module.training else 0.0,+ is_causal = True,+ scale = scaling,+ enable_gqa = True,+ )+ return out.transpose(1, 2).contiguous(), None+ return plain(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = dropout,+ scaling = scaling,+ is_causal = is_causal,+ **kwargs,+ )+ return router(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = dropout,+ scaling = scaling,+ is_causal = is_causal,+ **kwargs,+ )++ sdpa_attention_forward._unsloth_block_band = True+ ALL_ATTENTION_FUNCTIONS["sdpa"] = sdpa_attention_forward+++def _sliding_window(layer):+ window = layer.__dict__.get("_unsloth_block_window", 0)+ if window == 0:+ window = None+ for module in layer.modules():+ if getattr(module, "is_sliding", False) and getattr(module, "sliding_window", None):+ window = module.sliding_window+ break+ layer.__dict__["_unsloth_block_window"] = window+ return window+++def _precheck_band(layer, seq_len, args, kwargs):+ # Every 4D mask gets a verdict: a missing attribute would recompile.+ check = _BAND_CHECK[0]+ if check is None:+ return+ window = _sliding_window(layer)+ for obj in (*args, *kwargs.values()):+ if isinstance(obj, torch.Tensor) and obj.dim() == 4 and not obj.requires_grad:+ if window is not None and seq_len <= window:+ try:+ obj._unsloth_block_band = bool(check(obj, seq_len, window))+ except Exception:+ obj._unsloth_block_band = False+ elif not hasattr(obj, "_unsloth_block_band"):+ obj._unsloth_block_band = False+++# Mode of the block being traced (_SPLIT: flex_attention outside); dynamo guards on it.+_IN_BLOCK = [0]+_OFF, _FULL, _SPLIT = 0, 1, 2+_DEMOTION = (_FULL, _SPLIT, _OFF)+_FIRST_MODE = _SPLIT if os.environ.get("UNSLOTH_BLOCK_FLEX", "inside") == "outside" else _FULL+++def _flex_split_hook():+ """Lets a block keep flex_attention out of its graph. Some masks do not lower inside a larger+ graph (torch 2.13, Gemma-4 12B blockwise mask: `unsupported operand type(s) for &:+ 'FloorDiv' and 'Add'`); flex then runs as its own compiled region, one break per layer."""+ try:+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS+ except Exception:+ return+ flex = ALL_ATTENTION_FUNCTIONS.get("flex_attention")+ if flex is None or getattr(flex, "_unsloth_block_split", False):+ return+ outside = torch.compiler.disable(flex)++ @functools.wraps(flex)+ def flex_attention_forward(*args, **kwargs):+ if _IN_BLOCK[0] == _SPLIT and torch.compiler.is_compiling():+ return outside(*args, **kwargs)+ return flex(*args, **kwargs)++ flex_attention_forward._unsloth_block_split = True+ ALL_ATTENTION_FUNCTIONS["flex_attention"] = flex_attention_forward+++def _nf4_in_blocks():+ """bitsandbytes NF4 Linear4bit inside a block: Unsloth's Triton dequantize + F.linear (bit-exact+ to bitsandbytes' dequantize; kernels/bnb_override.py), on every GPU. Outside blocks the+ override keeps its own per-GPU default."""+ try:+ import bitsandbytes as bnb++ from ..kernels import bnb_override as O+ except Exception:+ return+ if not O._TRACE or O._LINEAR == "0":+ return # torch < 2.11 miscompiles the traced backward; "0" = the user's opt-out+ Linear4bit = bnb.nn.Linear4bit+ current = Linear4bit.forward+ if getattr(current, "_unsloth_block_nf4", False):+ return++ @functools.wraps(current)+ def forward(self, x, *args, **kwargs):+ # Both checkpoint halves must take one kernel: a trainable bias fails _eligible only under+ # grad, and one row picks fast_gemv only without grad (O._linear).+ bias = self.bias+ rows = x.numel() // max(x.shape[-1], 1)+ if (+ _IN_BLOCK[0]+ and torch.compiler.is_compiling()+ and not args+ and not kwargs+ and (bias is None or not bias.requires_grad)+ and rows > 1+ ):+ weight = self.weight+ quant_state = getattr(weight, "quant_state", None)+ if (+ O._eligible(self, x, weight, quant_state)+ and (quant_state.dtype is not torch.bfloat16 or O._TRACE_BF16)+ # bitsandbytes' fp32 small-row kernel is more accurate (O._forward keeps it too).+ and not (quant_state.dtype is torch.float32 and rows < 8)+ ):+ dtype = quant_state.dtype+ bias = None if self.bias is None else self.bias.to(dtype)+ return O._linear(x.to(dtype), weight, quant_state, bias).to(x.dtype)+ return current(self, x, *args, **kwargs)++ forward._unsloth_block_nf4 = True+ Linear4bit.forward = forward+++_CACHE_KWARGS = ("past_key_values", "past_key_value", "cache_params", "layer_past")+++def _has_cache(args, kwargs):+ # transformers 4.x layers take the singular `past_key_value`, sometimes positionally.+ if any(kwargs.get(key) is not None for key in _CACHE_KWARGS):+ return True+ try:+ from transformers.cache_utils import Cache+ except Exception:+ return False+ return any(isinstance(arg, Cache) for arg in args)+++def _layer_kind(layer):+ attn = getattr(layer, "self_attn", None) or getattr(layer, "linear_attn", None)+ return (+ type(attn).__name__,+ getattr(layer, "attention_type", None),+ getattr(attn, "is_sliding", None),+ getattr(attn, "sliding_window", None),+ getattr(attn, "head_dim", None),+ )+++def _wrap(cls, compile_options):+ original = cls.forward+ compiled = torch.compile(original, fullgraph = False, dynamic = None, options = compile_options)+ try:+ from unsloth_zoo.temporary_patches.utils import UNSLOTH_DECODE_COMPILE as inline_flag+ except Exception:+ inline_flag = [False]+ # First call per grad mode compiles with errors raised: on failure retry with flex outside,+ # then the stock forward. The mode is per layer (a checkpoint recompute must not switch graph);+ # a failure is also the starting mode for later layers of the same kind.+ state = {"reported": set(), "start": {}}++ def run(self, hidden_states, args, kwargs, mode):+ previous = inline_flag[0]+ inline_flag[0] = True+ _IN_BLOCK[0] = mode+ try:+ return compiled(self, hidden_states, *args, **kwargs)+ finally:+ inline_flag[0] = previous+ _IN_BLOCK[0] = _OFF++ @functools.wraps(original)+ def forward(self, hidden_states, *args, **kwargs):+ mode = self.__dict__.get("_unsloth_block_mode")+ if mode is None:+ mode = state["start"].get(_layer_kind(self), _FIRST_MODE)+ self.__dict__["_unsloth_block_mode"] = mode+ # Training only: eval logits stay bit-equal to transformers.+ if (+ mode == _OFF+ or not self.training+ or _has_cache(args, kwargs)+ or not _enabled()+ or _forced_float32(self)+ ):+ return original(self, hidden_states, *args, **kwargs)+ if not self.__dict__.get("_unsloth_block_moe_done", False):+ _experts_outside(self)+ if _GDN_OP[0] is None and hasattr(self, "linear_attn"):+ _prepare_gdn(self, hidden_states)+ if hidden_states.dim() == 3 and hidden_states.shape[1] > 1:+ _mark_seq_dynamic(hidden_states.shape[1], (hidden_states, args, kwargs))+ _precheck_band(self, hidden_states.shape[1], args, kwargs)+ bit = 2 if torch.is_grad_enabled() else 1+ checked = self.__dict__.get("_unsloth_block_checked", 0)+ if bit == 1 and not checked & 2:+ # Grad-mode trial first (RNG forked): a checkpoint recompute must not demote below this call.+ devices = [hidden_states.device.index] if hidden_states.is_cuda else []+ with torch.random.fork_rng(devices = devices), torch.enable_grad():+ forward(self, hidden_states.detach().requires_grad_(), *args, **kwargs)+ mode = self.__dict__["_unsloth_block_mode"]+ if mode == _OFF:+ return original(self, hidden_states, *args, **kwargs)+ checked = self.__dict__.get("_unsloth_block_checked", 0)+ if checked & bit:+ return run(self, hidden_states, args, kwargs, mode)+ while True:+ try:+ with torch._dynamo.config.patch(suppress_errors = False):+ out = run(self, hidden_states, args, kwargs, mode)+ self.__dict__["_unsloth_block_checked"] = checked | bit+ return out+ except torch.OutOfMemoryError:+ raise+ except Exception as e:+ if os.environ.get("UNSLOTH_BLOCK_COMPILE_DEBUG") == "1":+ import traceback+ traceback.print_exc()+ mode = _SPLIT if mode == _FULL else _OFF+ self.__dict__["_unsloth_block_mode"] = mode+ kind = _layer_kind(self)+ state["start"][kind] = max(+ state["start"].get(kind, _FIRST_MODE), mode, key = _DEMOTION.index+ )+ first = str(e).strip().splitlines()[0][:200] if str(e).strip() else type(e).__name__+ if (first, mode) not in state["reported"]:+ state["reported"].add((first, mode))+ print(+ f"Unsloth: {cls.__name__} block compile failed ({first}); "+ + (+ "retrying with flex_attention outside the block."+ if mode == _SPLIT+ else "keeping the stock forward for this layer."+ )+ )+ if mode == _OFF:+ return original(self, hidden_states, *args, **kwargs)++ forward._unsloth_block_original = original+ forward._unsloth_block_state = state+ setattr(forward, _MARK, True)+ cls.forward = forward+++def _layer_lists(model):+ """ModuleLists of decoder layers (one class each): the text stack, not vision towers."""+ return [+ module+ for module in model.modules()+ if isinstance(module, torch.nn.ModuleList)+ and len(module) > 0+ and type(module[0]).__name__.endswith("DecoderLayer")+ and all(type(m) is type(module[0]) for m in module)+ ]+++def _decoder_layer_classes(model):+ classes = []+ for layer_list in _layer_lists(model):+ if type(layer_list[0]) not in classes:+ classes.append(type(layer_list[0]))+ return classes+++def _torch_supported():+ # torch 2.6: blocks fragmented and QLoRA grads moved vs the stock forward.+ from packaging.version import Version+ return Version(torch.__version__.split("+")[0]) >= Version("2.7")+++def _experts_of(module):+ while module is not None:+ if type(module).__name__.endswith("Experts"):+ return module+ module = getattr(module, "base_layer", None)+ return None+++def _lora_would_fold(experts):+ # Traced, zoo folds the LoRA delta into every expert's weights when the forward ignores the stash.+ try:+ from unsloth_zoo.temporary_patches import moe_utils as zoo_moe+ reads = zoo_moe._forward_statically_reads_stash(experts)+ if not reads and zoo_moe._interface_route_reads_stash(experts):+ reads = True+ except Exception:+ return False+ return reads is False+++def _experts_outside(layer):+ # At the layer's first call, after get_peft_model wrapped the experts.+ layer.__dict__["_unsloth_block_moe_done"] = True+ mode = os.environ.get("UNSLOTH_BLOCK_MOE", "auto")+ if mode == "inside":+ return+ for _, module in layer.named_modules():+ if getattr(module, "_unsloth_block_outside", False):+ continue+ experts = _experts_of(module)+ if experts is None: # named_modules() is pre-order: the outermost wrapper comes first+ continue+ if mode != "outside" and not _lora_would_fold(experts):+ continue+ module.forward = torch.compiler.disable(module.forward)+ for inner in module.modules():+ inner._unsloth_block_outside = True+++def patch_block_compile(model):+ """Compile every decoder-layer class of `model` as one region. Returns the classes wrapped."""+ # getattr: PeftModel delegates. The env fills only unstamped layers, so a re-patch keeps the stamp.+ forced = getattr(model, "_unsloth_forced_float32", None)+ env = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"+ for layer_list in _layer_lists(model):+ for layer in layer_list:+ if forced is None:+ layer.__dict__.setdefault("_unsloth_forced_float32", env)+ else:+ layer.__dict__["_unsloth_forced_float32"] = bool(forced)+ forced = env if forced is None else forced+ if forced or not _enabled() or not torch.cuda.is_available() or not _torch_supported():+ return []+ classes = _decoder_layer_classes(model)+ if not classes:+ return []+ inner = {+ type(m)+ for layer_list in _layer_lists(model)+ for layer in layer_list+ for m in layer.modules()+ }+ for cls in inner:+ _rebind_class(cls)+ _flex_split_hook()+ _gemma4_router_hook()+ if any(type(m).__name__ == "Linear4bit" for m in model.modules()):+ _nf4_in_blocks()+ wrapped = []+ for cls in classes:+ if getattr(cls.forward, _MARK, False) or cls.__name__ in _SKIP:+ continue+ _wrap(cls, _COMPILE_OPTIONS)+ wrapped.append(cls.__name__)+ if wrapped:+ print(f"Unsloth: compiling each transformer block as one region ({', '.join(wrapped)}).")+ return wrappeddiff --git a/unsloth/models/_gdn_op.py b/unsloth/models/_gdn_op.pynew file mode 100644index 00000000000..1d2f125d268--- /dev/null+++ b/unsloth/models/_gdn_op.py@@ -0,0 +1,193 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""fla's chunk_gated_delta_rule as an opaque op with a registered backward (training case only),+checked bit-equal to ChunkGatedDeltaRuleFunction before use, so a Gated DeltaNet layer compiles+as one graph."""++import torch+import torch.nn.functional as F+++def _causal_depthwise_conv(x, weight):+ """F.conv1d(x, weight[:, None], groups = C, padding = K - 1)[..., :T] for x (B, C, T), weight+ (C, K), as K shifted multiply-adds accumulated in fp32. Inductor's convolution_backward+ lowering guards on the exact sequence length (a recompile per length); this stays dynamic."""+ seq_len, padding = x.shape[-1], weight.shape[-1] - 1+ xp = F.pad(x, (padding, 0)).float()+ w = weight.float()+ out = xp[:, :, 0:seq_len] * w[:, 0, None]+ for j in range(1, padding + 1):+ out = out + xp[:, :, j : j + seq_len] * w[:, j, None]+ return out.to(x.dtype)+++_FLA = None+++def _fla_ops(make_inputs):+ """fla's (fwd, bwd, l2norm_fwd, l2norm_bwd) once the opaque op matched fla's own Function on+ the first real layer's inputs, else False. Checked on real shapes and run through fla's+ Function first, so its pinned autotune configs are the ones the stock path would pick."""+ global _FLA+ if _FLA is None:+ _FLA = False+ try:+ import fla.ops.gated_delta_rule.chunk as C+ fns = (+ C.chunk_gated_delta_rule_fwd,+ C.chunk_gated_delta_rule_bwd,+ C.l2norm_fwd,+ C.l2norm_bwd,+ )+ # The first call can come from a no_grad checkpoint forward.+ with torch.inference_mode(False):+ with torch.no_grad():+ inputs = [t.detach().clone() for t in make_inputs()]+ with torch.enable_grad():+ _FLA = fns+ _FLA = fns if _self_check(*inputs) else False+ except Exception:+ _FLA = False+ return _FLA+++def _chunk_meta(q, k, v, g):+ # (qn, q_rstd, kn, k_rstd, g_cumsum, A), as fla 0.5 allocates them+ return (+ (q.shape, q.dtype),+ (q.shape[:-1], torch.float32),+ (k.shape, k.dtype),+ (k.shape[:-1], torch.float32),+ (g.shape, torch.float32),+ ((*k.shape[:-1], 64), k.dtype),+ )+++try:++ @torch.library.custom_op("unsloth::qwen3_5_gdn_chunk", mutates_args = ())+ def _gdn_chunk(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ scale: float,+ ) -> tuple[+ torch.Tensor,+ torch.Tensor,+ torch.Tensor,+ torch.Tensor,+ torch.Tensor,+ torch.Tensor,+ torch.Tensor,+ ]:+ fwd, _, l2_fwd, _ = _FLA+ q, k, v, g, beta = (t.contiguous() for t in (q, k, v, g, beta))+ with torch.cuda.device(q.device):+ qn, q_rstd = l2_fwd(q)+ kn, k_rstd = l2_fwd(k)+ g_cum, o, A, _, _, _ = fwd(+ q = qn,+ k = kn,+ v = v,+ g = g,+ beta = beta,+ scale = scale,+ initial_state = None,+ output_final_state = False,+ )+ # Fake metas are contiguous; keep the real outputs the same.+ return tuple(t.contiguous() for t in (o.to(q.dtype), qn, q_rstd, kn, k_rstd, g_cum, A))++ @_gdn_chunk.register_fake+ def _gdn_chunk_fake(q, k, v, g, beta, scale):+ o = v.new_empty(v.shape, dtype = q.dtype)+ return (o,) + tuple(+ q.new_empty(shape, dtype = dtype) for shape, dtype in _chunk_meta(q, k, v, g)+ )++ @torch.library.custom_op("unsloth::qwen3_5_gdn_chunk_bwd", mutates_args = ())+ def _gdn_chunk_bwd(+ qn: torch.Tensor,+ q_rstd: torch.Tensor,+ kn: torch.Tensor,+ k_rstd: torch.Tensor,+ v: torch.Tensor,+ g_cum: torch.Tensor,+ beta: torch.Tensor,+ A: torch.Tensor,+ do: torch.Tensor,+ scale: float,+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ _, bwd, _, l2_bwd = _FLA+ v, beta, do = v.contiguous(), beta.contiguous(), do.contiguous()+ with torch.cuda.device(qn.device):+ dq, dk, dv, db, dg, _, _, _ = bwd(+ q = qn,+ k = kn,+ v = v,+ g = g_cum,+ beta = beta,+ A = A,+ scale = scale,+ initial_state = None,+ do = do,+ dht = None,+ )+ dq = l2_bwd(qn, q_rstd, dq)+ dk = l2_bwd(kn, k_rstd, dk)+ out = (+ dq.to(qn.dtype),+ dk.to(kn.dtype),+ dv.to(v.dtype),+ dg.to(g_cum.dtype),+ db.to(beta.dtype),+ )+ return tuple(t.contiguous() for t in out)++ @_gdn_chunk_bwd.register_fake+ def _gdn_chunk_bwd_fake(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A, do, scale):+ return tuple(qn.new_empty(t.shape, dtype = t.dtype) for t in (qn, kn, v, g_cum, beta))++ def _gdn_chunk_setup(ctx, inputs, output):+ q, k, v, g, beta, scale = inputs+ _, qn, q_rstd, kn, k_rstd, g_cum, A = output+ ctx.save_for_backward(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A)+ ctx.scale = scale+ ctx.dtypes = (q.dtype, k.dtype, g.dtype)++ def _gdn_chunk_backward(ctx, do, *_):+ saved = [t.contiguous() for t in ctx.saved_tensors]+ dq, dk, dv, dg, db = _gdn_chunk_bwd(*saved, do.contiguous(), ctx.scale)+ q_dtype, k_dtype, g_dtype = ctx.dtypes+ return dq.to(q_dtype), dk.to(k_dtype), dv, dg.to(g_dtype), db, None++ _gdn_chunk.register_autograd(_gdn_chunk_backward, setup_context = _gdn_chunk_setup)+except Exception:+ _gdn_chunk = None+++def _self_check(q, k, v, g, beta):+ """Opaque op vs fla's own autograd.Function: outputs, fake metas and grads bit-equal."""+ if _gdn_chunk is None or not q.is_cuda:+ return False+ from fla.ops.gated_delta_rule import chunk_gated_delta_rule++ # Private generator: the default one feeds LoRA dropout, which checkpoint recompute replays.+ gen = torch.Generator(device = v.device).manual_seed(0)+ do = torch.randn(v.shape, dtype = v.dtype, device = v.device, generator = gen)+ ins = [t.contiguous().requires_grad_() for t in (q, k, v, g, beta)]+ ref, _ = chunk_gated_delta_rule(*ins, use_qk_l2norm_in_kernel = True)+ ref_g = torch.autograd.grad(ref, ins, do)+ ins2 = [t.detach().clone().requires_grad_() for t in ins]+ out = _gdn_chunk(*ins2, q.shape[-1] ** -0.5)+ got_g = torch.autograd.grad(out[0], ins2, do)+ metas = [(o.shape, o.dtype) for o in out[1:]]+ want = [(torch.Size(s), d) for s, d in _chunk_meta(q, k, v, g)]+ return (+ metas == want+ and torch.equal(out[0], ref)+ and all(torch.equal(a, b) for a, b in zip(got_g, ref_g))+ )diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex f45d197525c..93a3f1d60e7 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -5100,6 +5100,12 @@ class FastBaseModel:from ..tokenizer_utils import zero_pad_embedding_rowszero_pad_embedding_rows(model, tokenizer)+ # One compiled region per decoder layer (_block_compile.py).+ try:+ from ._block_compile import patch_block_compile+ patch_block_compile(model)+ except Exception as e:+ logger.warning(f"Unsloth: per-block compile not installed: {e}")return model@staticmethod