Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Compile each transformer block as one region for every FastModel model (LoRA and QLoRA) (#10744)

Hexu Zhaoc69969d58d0e
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
new file mode 100644
index 00000000000..63d3b93562e
--- /dev/null
+++ b/tests/test_block_compile.py
@@ -0,0 +1,715 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""One compiled region per decoder layer (unsloth/models/_block_compile.py).
+
+CPU: sequence dims (not widths) are marked dynamic, rebound globals resolve, and a block that
+fails to compile retries with flex_attention outside, then keeps the stock forward.
+GPU: compiled Qwen3 and Qwen3.5 layers match the stock forward and gradients with no graph break
+(Qwen3.5's fla kernel runs as the opaque op inside the block).
+"""
+
+import importlib.util
+import os
+import sys
+import types
+
+import pytest
+from real_accelerator import has_real_cuda
+
+torch = pytest.importorskip("torch")
+
+_DIR = os.path.join(os.path.dirname(__file__), "..", "unsloth", "models")
+
+
[email protected](autouse = True)
+def _block_env(monkeypatch):
+ # Other test modules set these process-wide (e.g. UNSLOTH_COMPILE_DISABLE=1 at import).
+ for key in ("UNSLOTH_BLOCK_COMPILE", "UNSLOTH_COMPILE_DISABLE", "UNSLOTH_FORCE_FLOAT32"):
+ monkeypatch.delenv(key, raising = False)
+
+
+def _load(name = "_block_compile"):
+ # By file path: importing the unsloth package needs an accelerator.
+ spec = importlib.util.spec_from_file_location(f"_u{name}", os.path.join(_DIR, f"{name}.py"))
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = mod
+ spec.loader.exec_module(mod)
+ return mod
+
+
+def test_mark_seq_dynamic_skips_widths(monkeypatch):
+ M = _load()
+ marked = []
+ monkeypatch.setattr(
+ torch._dynamo, "maybe_mark_dynamic", lambda t, d: marked.append((t.dim(), d))
+ )
+ L = 16
+ hidden = torch.zeros(2, L, L) # hidden size equal to the sequence length
+ mask4d = torch.zeros(2, 1, L, L)
+ mask2d = torch.zeros(2, L)
+ mrope_ids = torch.zeros(3, 2, L, dtype = torch.long)
+ M._mark_seq_dynamic(L, (hidden, {"m": mask4d}, [mask2d, "x", None], mrope_ids))
+ assert sorted(marked) == [(2, 1), (3, 1), (3, 2), (4, 2), (4, 3)]
+
+
+def test_rebind_globals_resolves():
+ M = _load()
+ ns = {"__name__": "not_a_registered_module", "OFFSET": 3}
+ exec("def f(x):\n return x + OFFSET\n", ns)
+ f = M._rebind_globals(ns["f"])
+ assert f is not ns["f"] and f(1) == 4
+ assert sys.modules[f.__module__].__dict__ is f.__globals__
+ g = M._rebind_globals(f)
+ assert g is f # already owned by a registered module
+ exec("def h():\n global OFFSET\n OFFSET += 1\n", ns)
+ assert M._rebind_globals(ns["h"]) is ns["h"] # a copied namespace would miss the write
+
+
+class _Boom(torch._dynamo.exc.TorchDynamoException):
+ pass
+
+
+def _toy_class():
+ class ToyDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(
+ self,
+ hidden_states,
+ attention_mask = None,
+ ):
+ return self.proj(hidden_states) + hidden_states
+
+ return ToyDecoderLayer
+
+
[email protected]("fails_in", [("full",), ("full", "split")])
+def test_compile_failure_falls_back(monkeypatch, fails_in):
+ M = _load()
+ calls = []
+
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ mode = {M._FULL: "full", M._SPLIT: "split"}[M._IN_BLOCK[0]]
+ calls.append(mode)
+ if mode in fails_in:
+ raise _Boom("lowering failed")
+ return fn(*args, **kw)
+
+ return compiled
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ cls = _toy_class()
+ original = cls.forward
+ M._wrap(cls, {})
+ layer = cls()
+ x = torch.randn(2, 4, 8)
+ with torch.no_grad():
+ want = original(layer, x)
+ got = layer(x)
+ torch.testing.assert_close(got, want)
+ layer(x)
+ if fails_in == ("full",):
+ assert layer._unsloth_block_mode == M._SPLIT
+ # Grad-mode trial (full fails, split), then the no-grad call twice.
+ assert calls == ["full", "split", "split", "split"]
+ else:
+ assert layer._unsloth_block_mode == M._OFF
+ assert calls == ["full", "split"] # then the stock forward, never compiled again
+ # A sibling of the same kind starts where this layer ended, without retrying what failed.
+ calls.clear()
+ with torch.no_grad():
+ cls()(x)
+ assert calls == (["split", "split"] if fails_in == ("full",) else [])
+ # Eval keeps the stock forward (logits equal transformers bit for bit).
+ calls.clear()
+ sibling = cls().eval()
+ sibling(x)
+ assert calls == []
+
+
+def test_cache_calls_keep_the_stock_forward(monkeypatch):
+ from transformers.cache_utils import DynamicCache
+
+ M = _load()
+ calls = []
+
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ calls.append(1)
+ return fn(*args, **kw)
+
+ return compiled
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+
+ class CacheLayerDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(
+ self,
+ hidden_states,
+ past_key_value = None,
+ **kwargs,
+ ):
+ return self.proj(hidden_states)
+
+ M._wrap(CacheLayerDecoderLayer, {})
+ layer = CacheLayerDecoderLayer()
+ x = torch.randn(2, 4, 8)
+ with torch.no_grad():
+ layer(x, past_key_value = DynamicCache())
+ layer(x, DynamicCache())
+ layer(x, past_key_values = DynamicCache())
+ assert calls == []
+ with torch.no_grad():
+ layer(x)
+ assert calls # no cache: compiled
+
+
+def test_gemma4_band_fast_path_in_blocks(monkeypatch):
+ M = _load()
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+
+ scans, routed, plain_calls = [], [], []
+ band = types.ModuleType("fake_banded_attention")
+ band._mask_is_plain_band = lambda mask, S, w: scans.append((S, w)) or True
+ zoo = types.ModuleType("fake_flash_sliding")
+
+ def plain(module, q, k, v, mask, **kw):
+ plain_calls.append(mask)
+ return "plain", None
+
+ def router(module, q, k, v, mask, **kw):
+ routed.append(mask)
+ return "router", None
+
+ router._unsloth_gemma4_flash = True
+ router.__module__ = zoo.__name__
+ zoo._ORIG_SDPA = [plain]
+ monkeypatch.setitem(sys.modules, band.__name__, band)
+ monkeypatch.setitem(sys.modules, zoo.__name__, zoo)
+ monkeypatch.setattr(M, "_BAND_MODULE", band.__name__)
+ monkeypatch.setitem(ALL_ATTENTION_FUNCTIONS, "sdpa", router)
+ M._gemma4_router_hook()
+ wrapped = ALL_ATTENTION_FUNCTIONS["sdpa"]
+ assert wrapped is not router
+ # Carries zoo's marker, so zoo's patch on a later model load does not wrap it again
+ # with it as the fallback (that loop recursed forever).
+ assert wrapped._unsloth_gemma4_flash
+ M._gemma4_router_hook()
+ assert ALL_ATTENTION_FUNCTIONS["sdpa"] is wrapped
+
+ class Gemma4TextAttention(torch.nn.Module):
+ is_sliding, sliding_window, is_causal = True, 64, True
+
+ layer = torch.nn.Module()
+ layer.self_attn = attn = Gemma4TextAttention()
+ short, long = torch.zeros(1, 1, 32, 32), torch.zeros(1, 1, 96, 96)
+ M._precheck_band(layer, 32, (), {"attention_mask": short})
+ M._precheck_band(layer, 96, (long,), {})
+ assert scans == [(32, 64)] # longer than the window: never scanned
+ assert (short._unsloth_block_band, long._unsloth_block_band) == (True, False)
+
+ q = torch.randn(1, 2, 32, 16)
+ assert wrapped(attn, q, q, q, short)[0] == "router" # eager: the stock router
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ M._IN_BLOCK[0] = M._FULL
+ try:
+ out, _ = wrapped(attn, q, q, q, short, scaling = 0.25)
+ want = torch.nn.functional.scaled_dot_product_attention(q, q, q, is_causal = True, scale = 0.25)
+ torch.testing.assert_close(out, want.transpose(1, 2))
+ ql = torch.randn(1, 2, 96, 16)
+ assert wrapped(attn, ql, ql, ql, long)[0] == "plain"
+ finally:
+ M._IN_BLOCK[0] = M._OFF
+ assert routed == [short] and plain_calls == [long]
+
+
+def test_kill_switch(monkeypatch):
+ M = _load()
+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE", "0")
+ assert M.patch_block_compile(torch.nn.Module()) == []
+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE", "1")
+ monkeypatch.setenv("UNSLOTH_COMPILE_DISABLE", "1")
+ assert not M._enabled()
+ monkeypatch.setenv("UNSLOTH_COMPILE_DISABLE", "0")
+ assert M._enabled()
+ monkeypatch.setattr(torch, "__version__", "2.6.0+cu124")
+ assert not M._torch_supported()
+ monkeypatch.setattr(torch, "__version__", "2.7.1+cu126")
+ assert M._torch_supported()
+
+
+def test_forced_float32_is_decided_per_model(monkeypatch):
+ M = _load()
+ calls = []
+
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ calls.append(1)
+ return fn(*args, **kw)
+
+ return compiled
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(M, "_torch_supported", lambda: True)
+ cls = _toy_class()
+
+ def model_of(forced):
+ model = torch.nn.Module()
+ model.layers = torch.nn.ModuleList([cls(), cls()])
+ if forced is not None:
+ model._unsloth_forced_float32 = forced
+ return model
+
+ # Forced by this load's env (the stamp lands after the first post_patch_model).
+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "1")
+ forced = model_of(None)
+ assert M.patch_block_compile(forced) == []
+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "0")
+ plain = model_of(False)
+ assert M.patch_block_compile(plain) == [cls.__name__]
+ x = torch.randn(2, 4, 8)
+ with torch.no_grad():
+ forced.layers[0](x)
+ assert calls == []
+ with torch.no_grad():
+ plain.layers[0](x)
+ assert len(calls) == 2 # grad-mode trial + the call
+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "1")
+ with torch.no_grad():
+ plain.layers[1](x)
+ assert len(calls) == 4
+ # get_peft_model re-patches a wrapper after another load reset the env: the stamp holds.
+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "0")
+
+ class Delegating(torch.nn.Module):
+ def __init__(self, inner):
+ super().__init__()
+ self.inner = inner
+
+ def __getattr__(self, name):
+ try:
+ return super().__getattr__(name)
+ except AttributeError:
+ return getattr(self.inner, name)
+
+ forced._unsloth_forced_float32 = True
+ wrapper = torch.nn.Module()
+ wrapper.base = forced
+ M.patch_block_compile(wrapper)
+ M.patch_block_compile(Delegating(forced))
+ calls.clear()
+ with torch.no_grad():
+ forced.layers[0](x)
+ assert calls == []
+
+
+def test_checkpoint_recompute_keeps_the_no_grad_mode(monkeypatch):
+ M = _load()
+ calls = []
+
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ mode = {M._FULL: "full", M._SPLIT: "split"}[M._IN_BLOCK[0]]
+ grad = torch.is_grad_enabled()
+ calls.append((mode, grad))
+ if grad:
+ torch.rand(3)
+ if mode == "full":
+ raise _Boom("grad-mode lowering failed")
+ return fn(*args, **kw)
+
+ return compiled
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ cls = _toy_class()
+ M._wrap(cls, {})
+ layer = cls()
+ x = torch.randn(2, 4, 8)
+ rng = torch.get_rng_state()
+ with torch.no_grad():
+ layer(x)
+ # The grad-mode trial demoted first, so the no-grad half already ran in SPLIT.
+ assert calls == [("full", True), ("split", True), ("split", False)]
+ assert torch.equal(torch.get_rng_state(), rng)
+ calls.clear()
+ layer(x.requires_grad_())
+ assert calls == [("split", True)]
+
+
+def test_gdn_self_check_leaves_default_rng_alone():
+ # Checkpoint recompute replays the default generator for LoRA dropout masks.
+ pytest.importorskip("fla.ops.gated_delta_rule.chunk")
+ if not has_real_cuda():
+ pytest.skip("needs a real CUDA device")
+ G = _load("_gdn_op")
+ gen = torch.Generator(device = "cuda").manual_seed(0)
+ B, T, H, D = 1, 70, 2, 128
+ mk = lambda *s, dt = torch.bfloat16: torch.randn(*s, device = "cuda", dtype = dt, generator = gen)
+ q, k, v = mk(B, T, H, D), mk(B, T, H, D), mk(B, T, H, D)
+ g = -mk(B, T, H, dt = torch.float32).abs() * 0.1
+ beta = mk(B, T, H).sigmoid()
+ state = torch.cuda.get_rng_state()
+ assert G._fla_ops(lambda: (q, k, v, g, beta))
+ assert torch.equal(torch.cuda.get_rng_state(), state)
+
+
+def test_sm_count_is_read_outside_the_trace(monkeypatch):
+ # Stands in for Triton's hip_utils.get_device_properties, which Dynamo cannot trace.
+ M = _load()
+ fla = types.ModuleType("fake.fla.modules.fused_norm_gate")
+ calls = []
+
+ @torch._dynamo.disable
+ def get_multiprocessor_count(tensor_idx = 0):
+ calls.append(tensor_idx)
+ return 7
+
+ fla.get_multiprocessor_count = get_multiprocessor_count
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
+ M._route_sm_count({fla.__name__: fla})
+ out = torch.compile(
+ lambda x: x * fla.get_multiprocessor_count(0), fullgraph = True, backend = "eager"
+ )(torch.ones(2))
+ assert out.tolist() == [7.0, 7.0] and calls == [0]
+
+
+def _grads_after_step(layers, run, x):
+ for layer in layers:
+ layer.zero_grad(set_to_none = True)
+ out = x
+ for layer in layers:
+ out = run(layer, out)
+ loss = out.float().pow(2).mean()
+ loss.backward()
+ return loss.detach(), {
+ n: p.grad.clone() for layer in layers for n, p in layer.named_parameters()
+ }
+
+
+def _check_block(layers, run, x, cls):
+ """Stock forward + backward vs the compiled block, two sequence lengths, no graph break."""
+ from torch._dynamo.utils import counters
+
+ import unsloth.models._block_compile as M
+
+ want = [_grads_after_step(layers, run, xi) for xi in x]
+ original = cls.forward
+ try:
+ M._wrap(M._rebind_class(cls) or cls, M._COMPILE_OPTIONS)
+ counters.clear()
+ got = [_grads_after_step(layers, run, xi) for xi in x]
+ assert sum(counters["graph_break"].values()) == 0, dict(counters["graph_break"])
+ finally:
+ cls.forward = original
+ torch._dynamo.reset()
+ for (lw, gw), (lg, gg) in zip(want, got):
+ torch.testing.assert_close(lg, lw, rtol = 1e-4, atol = 1e-5)
+ for k in gw:
+ torch.testing.assert_close(gg[k], gw[k], rtol = 1e-3, atol = 1e-5)
+
+
[email protected](not has_real_cuda(), reason = "compiled block needs CUDA")
+def test_qwen3_block_matches_stock():
+ pytest.importorskip("unsloth")
+ from transformers.models.qwen3 import modeling_qwen3 as modeling
+ from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
+
+ cfg = Qwen3Config(
+ vocab_size = 64,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ )
+ cfg._attn_implementation = "sdpa"
+ torch.manual_seed(0)
+ layers = [modeling.Qwen3DecoderLayer(cfg, layer_idx = i).cuda() for i in range(2)]
+ rotary = modeling.Qwen3RotaryEmbedding(cfg).cuda()
+
+ def run(layer, h):
+ pos = torch.arange(h.shape[1], device = h.device)[None].expand(h.shape[0], -1)
+ out = layer(h, position_embeddings = rotary(h, pos), position_ids = pos)
+ return out[0] if isinstance(out, tuple) else out
+
+ x = [torch.randn(2, L, 64, device = "cuda", requires_grad = True) for L in (24, 40)]
+ _check_block(layers, run, x, modeling.Qwen3DecoderLayer)
+
+
[email protected](not has_real_cuda(), reason = "fla kernels need CUDA")
+def test_qwen3_5_gdn_block_has_no_break():
+ pytest.importorskip("unsloth")
+ pytest.importorskip("fla.ops.gated_delta_rule")
+ from transformers.models.qwen3_5 import modeling_qwen3_5 as modeling
+ from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
+
+ import unsloth.models._block_compile as M
+
+ cfg = Qwen3_5TextConfig(
+ vocab_size = 64,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ linear_num_key_heads = 4,
+ linear_num_value_heads = 16,
+ linear_key_head_dim = 32,
+ linear_value_head_dim = 32,
+ layer_types = ["linear_attention", "full_attention"],
+ )
+ cfg._attn_implementation = "sdpa"
+ torch.manual_seed(0)
+ layers = [
+ modeling.Qwen3_5DecoderLayer(cfg, layer_idx = i).cuda().to(torch.bfloat16) for i in range(2)
+ ]
+ rotary = modeling.Qwen3_5TextRotaryEmbedding(cfg).cuda()
+ x = [torch.randn(2, L, 64, device = "cuda", dtype = torch.bfloat16) for L in (64, 128)]
+ M._prepare_gdn(layers[0], x[0])
+ if not M._GDN_OP[0]:
+ pytest.skip("fla's opaque op failed its self-check on this GPU")
+
+ def run(layer, h):
+ pos = torch.arange(h.shape[1], device = h.device)[None, None].expand(3, h.shape[0], -1)
+ out = layer(h, position_embeddings = rotary(h, pos), position_ids = pos)
+ return out[0] if isinstance(out, tuple) else out
+
+ from torch._dynamo.utils import counters
+
+ cls = modeling.Qwen3_5DecoderLayer
+ original = cls.forward
+ want = [_grads_after_step(layers, run, xi.clone().requires_grad_()) for xi in x]
+ try:
+ M._wrap(cls, M._COMPILE_OPTIONS)
+ counters.clear()
+ got = [_grads_after_step(layers, run, xi.clone().requires_grad_()) for xi in x]
+ assert sum(counters["graph_break"].values()) == 0, dict(counters["graph_break"])
+ finally:
+ cls.forward = original
+ torch._dynamo.reset()
+ for (lw, gw), (lg, gg) in zip(want, got):
+ torch.testing.assert_close(lg, lw, rtol = 2e-2, atol = 1e-3)
+ for k in gw:
+ num = (gg[k].float() - gw[k].float()).norm()
+ assert num <= 0.05 * gw[k].float().norm() + 1e-4, (
+ k,
+ float(num),
+ float(gw[k].float().norm()),
+ )
+
+
[email protected](not has_real_cuda(), reason = "fla kernels need a GPU")
+def test_opaque_fla_op_matches_fla_function():
+ pytest.importorskip("fla.ops.gated_delta_rule.chunk")
+ M = _load("_gdn_op")
+ gen = torch.Generator(device = "cuda").manual_seed(0)
+ B, T, H, D = 2, 150, 4, 128
+ mk = lambda *s, dt = torch.bfloat16: torch.randn(*s, device = "cuda", dtype = dt, generator = gen)
+ q, k, v = mk(B, T, H, D), mk(B, T, H, D), mk(B, T, H, D)
+ g = -mk(B, T, H, dt = torch.float32).abs() * 0.1
+ beta = mk(B, T, H).sigmoid()
+ assert M._fla_ops(lambda: (q, k, v, g, beta))
+ # A wrong fake meta or a numeric mismatch must keep the op out of the block.
+ M._FLA = None
+ orig = M._chunk_meta
+ M._chunk_meta = lambda *a: orig(*a)[:-1] + (((1,), torch.bfloat16),)
+ try:
+ assert M._fla_ops(lambda: (q, k, v, g, beta)) is False
+ finally:
+ M._chunk_meta = orig
+
+
+def test_gdn_adapter_routes_only_the_self_checked_call(monkeypatch):
+ M = _load()
+ seen = []
+ monkeypatch.setattr(M, "_GDN_OP", [True])
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ monkeypatch.setattr(
+ torch.ops,
+ "unsloth",
+ types.SimpleNamespace(qwen3_5_gdn_chunk = lambda *a: seen.append("op") or ("out",)),
+ raising = False,
+ )
+ adapter = M._gdn_chunk_adapter(lambda *a, **kw: seen.append("fla") or ("fla", None))
+ t = torch.zeros(1, 4, 2, 8)
+ gb = torch.zeros(1, 4, 2)
+ stock = dict(
+ use_qk_l2norm_in_kernel = True, initial_state = None, output_final_state = False, cu_seqlens = None
+ )
+ adapter(t, t, t, g = gb, beta = gb, **stock)
+ for extra in (
+ {"head_first": True},
+ {"use_beta_sigmoid_in_kernel": True},
+ {"scale": 0.1},
+ {"output_final_state": True},
+ {"cu_seqlens": torch.tensor([0, 4])},
+ {"use_qk_l2norm_in_kernel": False},
+ ):
+ adapter(t, t, t, g = gb, beta = gb, **dict(stock, **extra))
+ assert seen == ["op"] + ["fla"] * 6
+
+
[email protected]("linear_env", ["auto", "0"])
+def test_nf4_block_route_matches_both_checkpoint_halves(monkeypatch, linear_env):
+ # Package context so `from ..kernels import bnb_override` resolves to the fakes below.
+ pkg = types.ModuleType("_ufake")
+ pkg.__path__ = []
+ models = types.ModuleType("_ufake.models")
+ models.__path__ = []
+ kernels = types.ModuleType("_ufake.kernels")
+ kernels.__path__ = []
+ O = types.ModuleType("_ufake.kernels.bnb_override")
+ O._TRACE, O._TRACE_BF16, O._LINEAR = True, True, linear_env
+ O._eligible = lambda self, x, weight, quant_state: True
+ O._linear = lambda x, weight, quant_state, bias: torch.full_like(x, 7.0)
+ kernels.bnb_override = O
+
+ class Linear4bit(torch.nn.Module):
+ def __init__(self, dtype, bias):
+ super().__init__()
+ self.weight = types.SimpleNamespace(quant_state = types.SimpleNamespace(dtype = dtype))
+ self.bias = torch.nn.Parameter(torch.zeros(8)) if bias else None
+
+ def forward(self, x):
+ return torch.zeros_like(x)
+
+ bnb = types.ModuleType("bitsandbytes")
+ bnb.nn = types.SimpleNamespace(Linear4bit = Linear4bit)
+ for name, mod in (
+ ("_ufake", pkg),
+ ("_ufake.models", models),
+ ("_ufake.kernels", kernels),
+ ("_ufake.kernels.bnb_override", O),
+ ("bitsandbytes", bnb),
+ ):
+ monkeypatch.setitem(sys.modules, name, mod)
+ spec = importlib.util.spec_from_file_location(
+ "_ufake.models._block_compile", os.path.join(_DIR, "_block_compile.py")
+ )
+ M = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, spec.name, M)
+ spec.loader.exec_module(M)
+ M._nf4_in_blocks()
+ if linear_env == "0":
+ assert not getattr(Linear4bit.forward, "_unsloth_block_nf4", False)
+ return
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ monkeypatch.setattr(M, "_IN_BLOCK", [M._FULL])
+
+ def routed(
+ dtype,
+ rows,
+ bias = False,
+ ):
+ return bool(Linear4bit(dtype, bias)(torch.ones(rows, 8, dtype = dtype)).eq(7).all())
+
+ assert routed(torch.bfloat16, 16)
+ assert routed(torch.float32, 8)
+ assert not routed(torch.bfloat16, 1) # one row: fast_gemv only without grad
+ assert not routed(torch.float32, 4) # bitsandbytes' fp32 small-row kernel
+ assert not routed(torch.bfloat16, 16, bias = True) # trainable bias
+
+
+def test_causal_depthwise_conv_matches_conv1d():
+ M = _load("_gdn_op")
+ torch.manual_seed(0)
+ x, w = torch.randn(2, 6, 37), torch.randn(6, 4)
+ want = torch.nn.functional.conv1d(x, w[:, None], groups = 6, padding = 3)[..., :37]
+ torch.testing.assert_close(M._causal_depthwise_conv(x, w), want, rtol = 1e-5, atol = 1e-5)
+
+
+def test_disable_unless_decode_keeps_original_traceable_on_old_torch(monkeypatch):
+ # torch < 2.8: compiler.disable(fn, recursive = False) marks fn itself and returns it.
+ import unsloth.import_fixes as F
+ import unsloth_zoo.temporary_patches.utils as zoo_utils
+
+ marked = []
+
+ def old_disable(
+ fn = None,
+ recursive = True,
+ **kwargs,
+ ):
+ fn._old_disable_marked = True
+ marked.append(fn)
+ return fn
+
+ monkeypatch.setattr(torch.compiler, "disable", old_disable)
+ monkeypatch.setattr(
+ zoo_utils,
+ "torch_compiler_disable_unless_decode",
+ zoo_utils.torch_compiler_disable_unless_decode,
+ )
+ F.fix_torch_compiler_disable_unless_decode()
+ fixed = zoo_utils.torch_compiler_disable_unless_decode
+ assert fixed._unsloth_disable_copy
+
+ def attention(x):
+ return x + 1
+
+ wrapped = fixed(attention)
+ assert wrapped._unsloth_undisabled is attention
+ assert not getattr(attention, "_old_disable_marked", False)
+ disabled_copy = marked[-1]
+ assert disabled_copy is not attention and disabled_copy.__code__ is not attention.__code__
+ assert disabled_copy(1) == attention(1) == 2
+ assert torch.compiler.disable is old_disable
+
+
+def test_moe_experts_run_outside_the_block(monkeypatch):
+ M = _load()
+
+ class ToyExperts(torch.nn.Module):
+ def forward(self, x):
+ return x * 2
+
+ class LoraWrapper(torch.nn.Module): # PEFT-style wrapper around the experts
+ def __init__(self):
+ super().__init__()
+ self.base_layer = ToyExperts()
+
+ def forward(self, x):
+ return self.base_layer(x) + 1
+
+ layer = torch.nn.Module()
+ layer.mlp = torch.nn.Module()
+ layer.mlp.experts = LoraWrapper()
+ layer.self_attn = torch.nn.Linear(4, 4)
+ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: True)
+ M._experts_outside(layer)
+ outer = layer.mlp.experts
+ assert (
+ outer.forward.__func__ is not LoraWrapper.forward
+ if hasattr(outer.forward, "__func__")
+ else True
+ )
+ assert outer._unsloth_block_outside and outer.base_layer._unsloth_block_outside
+ assert not getattr(layer.self_attn, "_unsloth_block_outside", False)
+ from torch._dynamo.utils import counters
+
+ counters.clear()
+ out = torch.compile(lambda x: layer.mlp.experts(x) * 3, backend = "eager")(torch.ones(2))
+ assert out.tolist() == [9.0, 9.0] and sum(counters["graph_break"].values()) >= 1
+ # A stash-reading experts forward (no fold) stays inside; so does everything under "inside".
+ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: False)
+ other = torch.nn.Module()
+ other.experts = LoraWrapper()
+ M._experts_outside(other)
+ assert not getattr(other.experts, "_unsloth_block_outside", False)
+ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: True)
+ monkeypatch.setenv("UNSLOTH_BLOCK_MOE", "inside")
+ third = torch.nn.Module()
+ third.experts = LoraWrapper()
+ M._experts_outside(third)
+ assert not getattr(third.experts, "_unsloth_block_outside", False)
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index f0c7dd8cb8d..5aa2536177f 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -253,6 +253,13 @@ except ModuleNotFoundError:
except:
raise
+try:
+ from .import_fixes import fix_torch_compiler_disable_unless_decode
+ fix_torch_compiler_disable_unless_decode()
+ del fix_torch_compiler_disable_unless_decode
+except Exception:
+ pass
+
# Re-assert after unsloth_zoo's patch_torch_compile, which historically popped
# TORCHINDUCTOR_COMPILE_THREADS.
if os.environ.get("UNSLOTH_FORCE_SINGLE_COMPILE_WORKER", "0") == "1":
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 12da2bc8777..bf0ccc38e75 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -264,6 +264,64 @@ if not UNSLOTH_ENABLE_LOGGING:
)
+def fix_torch_compiler_disable_unless_decode():
+ """torch < 2.8 marks `fn` itself in `compiler.disable(fn, recursive = False)`, so zoo's
+ decode-aware disable disabled the function it runs: hand it a copy to disable."""
+ import functools
+ import types
+
+ try:
+ import torch
+ import unsloth_zoo.temporary_patches.utils as zoo_utils
+ except Exception:
+ return
+ current = getattr(zoo_utils, "torch_compiler_disable_unless_decode", None)
+ if current is None or getattr(current, "_unsloth_disable_copy", False):
+ return
+
+ def _probe(x):
+ return x
+
+ try:
+ if torch.compiler.disable(_probe, recursive = False) is not _probe:
+ return
+ except Exception:
+ return
+ real_disable = torch.compiler.disable
+
+ def _copy(fn):
+ clone = types.FunctionType(
+ fn.__code__.replace(), fn.__globals__, fn.__name__, fn.__defaults__, fn.__closure__
+ )
+ clone.__kwdefaults__ = fn.__kwdefaults__
+ clone.__dict__.update(fn.__dict__)
+ clone.__qualname__ = fn.__qualname__
+ clone.__module__ = fn.__module__
+ return clone
+
+ def disable_copy(
+ fn = None,
+ *args,
+ **kwargs,
+ ):
+ if isinstance(fn, types.FunctionType):
+ fn = _copy(fn)
+ return real_disable(fn, *args, **kwargs)
+
+ @functools.wraps(current)
+ def torch_compiler_disable_unless_decode(func = None, *, recursive = False):
+ if func is None:
+ return functools.partial(torch_compiler_disable_unless_decode, recursive = recursive)
+ torch.compiler.disable = disable_copy
+ try:
+ return current(func, recursive = recursive)
+ finally:
+ torch.compiler.disable = real_disable
+
+ torch_compiler_disable_unless_decode._unsloth_disable_copy = True
+ zoo_utils.torch_compiler_disable_unless_decode = torch_compiler_disable_unless_decode
+
+
def fix_torch_check_is_size():
"""Shim torch._check_is_size if a future torch removes it (bitsandbytes 4-bit
dequant calls it). The FutureWarning is silenced in suppress_cuda_printf."""
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
new file mode 100644
index 00000000000..bbbbab30d3c
--- /dev/null
+++ b/unsloth/models/_block_compile.py
@@ -0,0 +1,771 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""One torch.compile region per decoder layer for training (stock forward: attention, MLP or MoE,
+norms, LoRA, NF4). UNSLOTH_BLOCK_COMPILE=0 or UNSLOTH_COMPILE_DISABLE turns it off."""
+
+import functools
+import os
+import sys
+import types
+
+import torch
+
+__all__ = ["patch_block_compile"]
+
+_MARK = "_unsloth_block_compiled"
+# No max_autotune: it multiplies first-step compile time for every block.
+_COMPILE_OPTIONS = {
+ "epilogue_fusion": True,
+ "shape_padding": True,
+ "triton.cudagraphs": False,
+}
+
+
+# gpt-oss builds its flex mask inside attention: the block broke and recompiled per length.
+_SKIP = {"GptOssDecoderLayer"}
+
+
+def _enabled():
+ return os.environ.get("UNSLOTH_BLOCK_COMPILE", "1") != "0" and os.environ.get(
+ "UNSLOTH_COMPILE_DISABLE", "0"
+ ) not in ("1", "partial")
+
+
+def _forced_float32(module):
+ # Zoo's float32-forced patches round to fp16 on purpose; Inductor drops those roundings.
+ # Stamped per layer at load: the env var only describes the latest load.
+ forced = module.__dict__.get("_unsloth_forced_float32")
+ if forced is None:
+ forced = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"
+ return forced
+
+
+def _writes_globals(code):
+ import dis
+ if any(i.opname in ("STORE_GLOBAL", "DELETE_GLOBAL") for i in dis.get_instructions(code)):
+ return True
+ return any(_writes_globals(c) for c in code.co_consts if isinstance(c, types.CodeType))
+
+
+def _rebind_globals(fn):
+ """dynamo resolves a function's globals through sys.modules[__globals__["__name__"]]; Unsloth's
+ generated modules exec code into a dict that module does not own (Gemma-4: `module
+ 'transformers...modeling_gemma4' has no attribute 'Gemma4RMSNorm_forward'`). Rebuild such a
+ function on a registered module holding a copy of that dict. A copy cannot follow later
+ writes, so functions that assign globals are left alone."""
+ g = getattr(fn, "__globals__", None)
+ if g is None or _writes_globals(fn.__code__):
+ return fn
+ mod = sys.modules.get(g.get("__name__"))
+ if mod is not None and mod.__dict__ is g:
+ return fn
+ name = f"unsloth_block_globals_{id(g)}"
+ holder = sys.modules.get(name)
+ if holder is None:
+ holder = types.ModuleType(name)
+ holder.__dict__.update(g)
+ holder.__dict__["__name__"] = name
+ sys.modules[name] = holder
+ new = types.FunctionType(
+ fn.__code__, holder.__dict__, fn.__name__, fn.__defaults__, fn.__closure__
+ )
+ new.__kwdefaults__ = fn.__kwdefaults__
+ new.__dict__.update(fn.__dict__)
+ new.__qualname__ = fn.__qualname__
+ new.__module__ = name
+ return new
+
+
+def _inline_compiled(fn):
+ """A method that is itself torch.compile'd (zoo's patch_function): inside a block, call the
+ original so dynamo inlines it. Inlining through the compiled wrapper guards on the wrapper's
+ closure as if it were the original's (torch 2.13 / 2.14, Gemma-4 fp16 MLP: `Unexpected type
+ CatchErrorsWrapper`). Outside a block the compiled wrapper runs as before."""
+ original = getattr(fn, "_torchdynamo_orig_callable", None)
+ if (
+ not isinstance(original, types.FunctionType)
+ or getattr(fn, "_torchdynamo_disable", False)
+ or getattr(fn, "_unsloth_block_inline", False)
+ ):
+ return fn
+ compiled = fn
+ original = _rebind_globals(original)
+
+ @functools.wraps(original, updated = ())
+ def dispatch(*args, **kwargs):
+ if torch.compiler.is_compiling():
+ return original(*args, **kwargs)
+ return compiled(*args, **kwargs)
+
+ dispatch._unsloth_block_inline = True
+ return dispatch
+
+
+def _rebind_class(cls):
+ for key, value in list(cls.__dict__.items()):
+ new = _inline_compiled(value) if callable(value) else value
+ if isinstance(new, types.FunctionType):
+ new = _rebind_globals(new)
+ if new is not value:
+ setattr(cls, key, new)
+
+
+def _mark_seq_dynamic(seq_len, obj):
+ if isinstance(obj, torch.Tensor):
+ # A float tensor's last dim is a width even when it equals seq_len (mrope ids are int).
+ last = obj.dim() - 1 if obj.dim() == 3 and obj.is_floating_point() else None
+ for dim, size in enumerate(obj.shape):
+ if size == seq_len and dim != last:
+ torch._dynamo.maybe_mark_dynamic(obj, dim)
+ elif isinstance(obj, (tuple, list)):
+ for o in obj:
+ _mark_seq_dynamic(seq_len, o)
+ elif isinstance(obj, dict):
+ for o in obj.values():
+ _mark_seq_dynamic(seq_len, o)
+
+
+# fla's opaque op state: None until a GatedDeltaNet layer ran its self-check, then True / False.
+_GDN_OP = [None]
+
+
+def _gdn_chunk_adapter(original):
+ # updated = (): the original's __dict__ carries torch.compiler.disable's marker.
+ @functools.wraps(original, updated = ())
+ def chunk_gated_delta_rule(
+ q,
+ k,
+ v,
+ g = None,
+ beta = None,
+ **kwargs,
+ ):
+ if (
+ _GDN_OP[0]
+ and torch.compiler.is_compiling()
+ and kwargs.get("use_qk_l2norm_in_kernel") is True
+ # The op is the self-checked default: no state, varlen, scale, layout or gating flags.
+ and all(
+ value is None or value is False
+ for key, value in kwargs.items()
+ if key != "use_qk_l2norm_in_kernel"
+ )
+ ):
+ # Inductor pins a custom op's input strides; a sliced input fails at the next length.
+ q, k, v, g, beta = (t.contiguous() for t in (q, k, v, g, beta))
+ out = torch.ops.unsloth.qwen3_5_gdn_chunk(q, k, v, g, beta, q.shape[-1] ** -0.5)
+ return out[0], None
+ return original(q, k, v, g, beta, **kwargs)
+
+ chunk_gated_delta_rule._unsloth_block_adapter = True
+ return chunk_gated_delta_rule
+
+
+def _route_fla_kernel():
+ """Point modeling code's references to fla's chunk_gated_delta_rule at the adapter. Callers
+ reach it through module globals, transformers' hub-kernel wrapper (a closure over it) and
+ Unsloth's compiled-module copies, so callers are found from the function object (gc
+ referrers), not by name. Closures are left alone: only namespace entries are swapped."""
+ import gc
+
+ kernels = []
+ for name, module in list(sys.modules.items()):
+ if module is not None and name.endswith("fla.ops.gated_delta_rule.chunk"):
+ fn = getattr(module, "chunk_gated_delta_rule", None)
+ if callable(fn) and fn not in kernels:
+ kernels.append(fn)
+ callers, frontier = list(kernels), list(kernels)
+ for _ in range(4):
+ found = []
+ for inner in frontier:
+ for cell in gc.get_referrers(inner):
+ if type(cell).__name__ != "cell":
+ continue
+ for closure in gc.get_referrers(cell):
+ if not isinstance(closure, tuple):
+ continue
+ for fn in gc.get_referrers(closure):
+ if (
+ isinstance(fn, types.FunctionType)
+ and fn.__closure__ is closure
+ and fn not in callers
+ and not getattr(fn, "_unsloth_block_adapter", False)
+ ):
+ callers.append(fn)
+ found.append(fn)
+ frontier = found
+ if not frontier:
+ break
+ routed = 0
+ for caller in callers:
+ if getattr(caller, "_unsloth_block_adapter", False):
+ continue
+ adapter = _gdn_chunk_adapter(caller)
+ for space in gc.get_referrers(caller):
+ if not isinstance(space, dict) or space is adapter.__dict__:
+ continue
+ for key, value in list(space.items()):
+ if (
+ value is caller
+ and isinstance(key, str)
+ and key.endswith("chunk_gated_delta_rule")
+ ):
+ space[key] = adapter
+ routed += 1
+ return routed
+
+
+def _causal_conv1d_adapter(original, depthwise_conv):
+ # Inductor's conv backward guards on the exact sequence length: recompile per length.
+ @functools.wraps(original, updated = ())
+ def causal_conv1d_fn(
+ hidden_states,
+ weight,
+ bias = None,
+ activation = None,
+ **kwargs,
+ ):
+ if (
+ _IN_BLOCK[0]
+ and torch.compiler.is_compiling()
+ and weight.dim() == 2
+ and activation in (None, "silu", "swish")
+ and all(kwargs.get(key) is None for key in ("seq_idx", "cu_seqlens", "initial_states"))
+ and not kwargs.get("return_final_states")
+ ):
+ out = depthwise_conv(hidden_states.to(weight.dtype), weight)
+ if bias is not None:
+ out = out + bias[:, None]
+ if activation is not None:
+ out = torch.nn.functional.silu(out)
+ return out.to(hidden_states.dtype)
+ return original(hidden_states, weight, bias, activation = activation, **kwargs)
+
+ causal_conv1d_fn._unsloth_block_adapter = True
+ return causal_conv1d_fn
+
+
+def _namespaces(fn, depth = 3):
+ """Global namespaces a forward reaches through wrappers (closures) and module-level helpers."""
+ seen, out, todo = set(), [], [(fn, 0)]
+ while todo:
+ f, d = todo.pop()
+ if not isinstance(f, types.FunctionType) or id(f) in seen:
+ continue
+ seen.add(id(f))
+ if all(f.__globals__ is not g for g in out):
+ out.append(f.__globals__)
+ if d >= depth:
+ continue
+ for cell in f.__closure__ or ():
+ try:
+ todo.append((cell.cell_contents, d + 1))
+ except ValueError:
+ pass
+ for name in f.__code__.co_names:
+ todo.append((f.__globals__.get(name), d + 1))
+ return out
+
+
+def _route_causal_conv(gdn):
+ from ._gdn_op import _causal_depthwise_conv
+ for space in _namespaces(type(gdn).forward):
+ fn = space.get("causal_conv1d_fn")
+ if callable(fn) and not getattr(fn, "_unsloth_block_adapter", False):
+ space["causal_conv1d_fn"] = _causal_conv1d_adapter(fn, _causal_depthwise_conv)
+
+
+def _route_sm_count(modules = None):
+ # ROCm: Triton's hip_utils.get_device_properties is untraceable; the SM count is per device.
+ for name, module in list((sys.modules if modules is None else modules).items()):
+ if module is None or "fla." not in name:
+ continue
+ fn = getattr(module, "get_multiprocessor_count", None)
+ if not callable(fn) or getattr(fn, "_unsloth_block_adapter", False):
+ continue
+ try:
+ counts = {i: int(fn(i)) for i in range(torch.cuda.device_count())}
+ except Exception:
+ continue
+
+ @functools.wraps(fn, updated = ())
+ def get_multiprocessor_count(
+ tensor_idx = 0,
+ _original = fn,
+ _counts = counts,
+ ):
+ count = _counts.get(tensor_idx)
+ return _original(tensor_idx) if count is None else count
+
+ get_multiprocessor_count._unsloth_block_adapter = True
+ module.get_multiprocessor_count = get_multiprocessor_count
+
+
+def _prepare_gdn(layer, hidden_states):
+ """Self-check fla's opaque op on this layer's shapes, then point the GatedDeltaNet forward's
+ chunk kernel at the adapter. Runs eagerly, before the first trace."""
+ gdn = getattr(layer, "linear_attn", None)
+ if gdn is None or _GDN_OP[0] is not None:
+ return
+ _GDN_OP[0] = False
+ try:
+ _route_sm_count()
+ except Exception:
+ pass
+ try:
+ from ._gdn_op import _fla_ops
+
+ heads, dk, dv = gdn.num_v_heads, gdn.head_k_dim, gdn.head_v_dim
+ B, T = hidden_states.shape[0], hidden_states.shape[1]
+ dev, dt = hidden_states.device, hidden_states.dtype
+
+ def make_inputs():
+ gen = torch.Generator(device = dev).manual_seed(0)
+ r = lambda *s, dtype = dt: torch.randn(*s, device = dev, dtype = dtype, generator = gen)
+ g = -torch.rand(B, T, heads, device = dev, generator = gen) * 0.1
+ return (
+ r(B, T, heads, dk),
+ r(B, T, heads, dk),
+ r(B, T, heads, dv),
+ g,
+ r(B, T, heads).sigmoid(),
+ )
+
+ if not _fla_ops(make_inputs):
+ return
+ _GDN_OP[0] = _route_fla_kernel() > 0
+ if _GDN_OP[0]:
+ _route_causal_conv(gdn)
+ except Exception:
+ _GDN_OP[0] = False
+
+
+# zoo's Gemma-4 sliding router scans the mask (data dependent) and declines while compiling, so
+# the block runs the scan eagerly and a router wrapper reads the verdict stored on the mask.
+_BAND_MODULE = "unsloth_zoo.temporary_patches.gemma4_banded_attention"
+_BAND_CHECK = [None]
+
+
+def _gemma4_router_hook():
+ """Inside a block, a sliding layer whose mask is a plain causal band no longer than the
+ window runs is_causal SDPA, as the stock router does; longer bands run masked SDPA. The
+ router itself is not traced: its engagement counters drive a log-once branch dynamo guards
+ on, and its banded kernel traced in a checkpointed block recomputes different saved tensors
+ than its forward (CheckpointError)."""
+ try:
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+ except Exception:
+ return
+ router = ALL_ATTENTION_FUNCTIONS.get("sdpa")
+ if not getattr(router, "_unsloth_gemma4_flash", False) or getattr(
+ router, "_unsloth_block_band", False
+ ):
+ return
+ zoo = sys.modules.get(router.__module__)
+ plain = getattr(zoo, "_ORIG_SDPA", [None])[0]
+ check = getattr(sys.modules.get(_BAND_MODULE), "_mask_is_plain_band", None)
+ if plain is None or not callable(check):
+ return
+ _BAND_CHECK[0] = check
+
+ # wraps() keeps zoo's marker, else a later load wraps this as its own fallback (endless loop).
+ @functools.wraps(router)
+ def sdpa_attention_forward(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = 0.0,
+ scaling = None,
+ is_causal = None,
+ **kwargs,
+ ):
+ if (
+ _IN_BLOCK[0]
+ and torch.compiler.is_compiling()
+ and getattr(module, "is_sliding", False)
+ and type(module).__name__.startswith("Gemma4")
+ and query.dim() == 4
+ and query.shape[-1] <= 256
+ ):
+ window = kwargs.get("sliding_window", None) or getattr(module, "sliding_window", None)
+ causal = is_causal if is_causal is not None else getattr(module, "is_causal", True)
+ if (
+ window
+ and query.shape[2] == key.shape[2]
+ and query.shape[2] <= window
+ and (
+ causal
+ if attention_mask is None
+ else getattr(attention_mask, "_unsloth_block_band", False) is True
+ )
+ ):
+ out = torch.nn.functional.scaled_dot_product_attention(
+ query,
+ key,
+ value,
+ dropout_p = dropout if module.training else 0.0,
+ is_causal = True,
+ scale = scaling,
+ enable_gqa = True,
+ )
+ return out.transpose(1, 2).contiguous(), None
+ return plain(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = dropout,
+ scaling = scaling,
+ is_causal = is_causal,
+ **kwargs,
+ )
+ return router(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = dropout,
+ scaling = scaling,
+ is_causal = is_causal,
+ **kwargs,
+ )
+
+ sdpa_attention_forward._unsloth_block_band = True
+ ALL_ATTENTION_FUNCTIONS["sdpa"] = sdpa_attention_forward
+
+
+def _sliding_window(layer):
+ window = layer.__dict__.get("_unsloth_block_window", 0)
+ if window == 0:
+ window = None
+ for module in layer.modules():
+ if getattr(module, "is_sliding", False) and getattr(module, "sliding_window", None):
+ window = module.sliding_window
+ break
+ layer.__dict__["_unsloth_block_window"] = window
+ return window
+
+
+def _precheck_band(layer, seq_len, args, kwargs):
+ # Every 4D mask gets a verdict: a missing attribute would recompile.
+ check = _BAND_CHECK[0]
+ if check is None:
+ return
+ window = _sliding_window(layer)
+ for obj in (*args, *kwargs.values()):
+ if isinstance(obj, torch.Tensor) and obj.dim() == 4 and not obj.requires_grad:
+ if window is not None and seq_len <= window:
+ try:
+ obj._unsloth_block_band = bool(check(obj, seq_len, window))
+ except Exception:
+ obj._unsloth_block_band = False
+ elif not hasattr(obj, "_unsloth_block_band"):
+ obj._unsloth_block_band = False
+
+
+# Mode of the block being traced (_SPLIT: flex_attention outside); dynamo guards on it.
+_IN_BLOCK = [0]
+_OFF, _FULL, _SPLIT = 0, 1, 2
+_DEMOTION = (_FULL, _SPLIT, _OFF)
+_FIRST_MODE = _SPLIT if os.environ.get("UNSLOTH_BLOCK_FLEX", "inside") == "outside" else _FULL
+
+
+def _flex_split_hook():
+ """Lets a block keep flex_attention out of its graph. Some masks do not lower inside a larger
+ graph (torch 2.13, Gemma-4 12B blockwise mask: `unsupported operand type(s) for &:
+ 'FloorDiv' and 'Add'`); flex then runs as its own compiled region, one break per layer."""
+ try:
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+ except Exception:
+ return
+ flex = ALL_ATTENTION_FUNCTIONS.get("flex_attention")
+ if flex is None or getattr(flex, "_unsloth_block_split", False):
+ return
+ outside = torch.compiler.disable(flex)
+
+ @functools.wraps(flex)
+ def flex_attention_forward(*args, **kwargs):
+ if _IN_BLOCK[0] == _SPLIT and torch.compiler.is_compiling():
+ return outside(*args, **kwargs)
+ return flex(*args, **kwargs)
+
+ flex_attention_forward._unsloth_block_split = True
+ ALL_ATTENTION_FUNCTIONS["flex_attention"] = flex_attention_forward
+
+
+def _nf4_in_blocks():
+ """bitsandbytes NF4 Linear4bit inside a block: Unsloth's Triton dequantize + F.linear (bit-exact
+ to bitsandbytes' dequantize; kernels/bnb_override.py), on every GPU. Outside blocks the
+ override keeps its own per-GPU default."""
+ try:
+ import bitsandbytes as bnb
+
+ from ..kernels import bnb_override as O
+ except Exception:
+ return
+ if not O._TRACE or O._LINEAR == "0":
+ return # torch < 2.11 miscompiles the traced backward; "0" = the user's opt-out
+ Linear4bit = bnb.nn.Linear4bit
+ current = Linear4bit.forward
+ if getattr(current, "_unsloth_block_nf4", False):
+ return
+
+ @functools.wraps(current)
+ def forward(self, x, *args, **kwargs):
+ # Both checkpoint halves must take one kernel: a trainable bias fails _eligible only under
+ # grad, and one row picks fast_gemv only without grad (O._linear).
+ bias = self.bias
+ rows = x.numel() // max(x.shape[-1], 1)
+ if (
+ _IN_BLOCK[0]
+ and torch.compiler.is_compiling()
+ and not args
+ and not kwargs
+ and (bias is None or not bias.requires_grad)
+ and rows > 1
+ ):
+ weight = self.weight
+ quant_state = getattr(weight, "quant_state", None)
+ if (
+ O._eligible(self, x, weight, quant_state)
+ and (quant_state.dtype is not torch.bfloat16 or O._TRACE_BF16)
+ # bitsandbytes' fp32 small-row kernel is more accurate (O._forward keeps it too).
+ and not (quant_state.dtype is torch.float32 and rows < 8)
+ ):
+ dtype = quant_state.dtype
+ bias = None if self.bias is None else self.bias.to(dtype)
+ return O._linear(x.to(dtype), weight, quant_state, bias).to(x.dtype)
+ return current(self, x, *args, **kwargs)
+
+ forward._unsloth_block_nf4 = True
+ Linear4bit.forward = forward
+
+
+_CACHE_KWARGS = ("past_key_values", "past_key_value", "cache_params", "layer_past")
+
+
+def _has_cache(args, kwargs):
+ # transformers 4.x layers take the singular `past_key_value`, sometimes positionally.
+ if any(kwargs.get(key) is not None for key in _CACHE_KWARGS):
+ return True
+ try:
+ from transformers.cache_utils import Cache
+ except Exception:
+ return False
+ return any(isinstance(arg, Cache) for arg in args)
+
+
+def _layer_kind(layer):
+ attn = getattr(layer, "self_attn", None) or getattr(layer, "linear_attn", None)
+ return (
+ type(attn).__name__,
+ getattr(layer, "attention_type", None),
+ getattr(attn, "is_sliding", None),
+ getattr(attn, "sliding_window", None),
+ getattr(attn, "head_dim", None),
+ )
+
+
+def _wrap(cls, compile_options):
+ original = cls.forward
+ compiled = torch.compile(original, fullgraph = False, dynamic = None, options = compile_options)
+ try:
+ from unsloth_zoo.temporary_patches.utils import UNSLOTH_DECODE_COMPILE as inline_flag
+ except Exception:
+ inline_flag = [False]
+ # First call per grad mode compiles with errors raised: on failure retry with flex outside,
+ # then the stock forward. The mode is per layer (a checkpoint recompute must not switch graph);
+ # a failure is also the starting mode for later layers of the same kind.
+ state = {"reported": set(), "start": {}}
+
+ def run(self, hidden_states, args, kwargs, mode):
+ previous = inline_flag[0]
+ inline_flag[0] = True
+ _IN_BLOCK[0] = mode
+ try:
+ return compiled(self, hidden_states, *args, **kwargs)
+ finally:
+ inline_flag[0] = previous
+ _IN_BLOCK[0] = _OFF
+
+ @functools.wraps(original)
+ def forward(self, hidden_states, *args, **kwargs):
+ mode = self.__dict__.get("_unsloth_block_mode")
+ if mode is None:
+ mode = state["start"].get(_layer_kind(self), _FIRST_MODE)
+ self.__dict__["_unsloth_block_mode"] = mode
+ # Training only: eval logits stay bit-equal to transformers.
+ if (
+ mode == _OFF
+ or not self.training
+ or _has_cache(args, kwargs)
+ or not _enabled()
+ or _forced_float32(self)
+ ):
+ return original(self, hidden_states, *args, **kwargs)
+ if not self.__dict__.get("_unsloth_block_moe_done", False):
+ _experts_outside(self)
+ if _GDN_OP[0] is None and hasattr(self, "linear_attn"):
+ _prepare_gdn(self, hidden_states)
+ if hidden_states.dim() == 3 and hidden_states.shape[1] > 1:
+ _mark_seq_dynamic(hidden_states.shape[1], (hidden_states, args, kwargs))
+ _precheck_band(self, hidden_states.shape[1], args, kwargs)
+ bit = 2 if torch.is_grad_enabled() else 1
+ checked = self.__dict__.get("_unsloth_block_checked", 0)
+ if bit == 1 and not checked & 2:
+ # Grad-mode trial first (RNG forked): a checkpoint recompute must not demote below this call.
+ devices = [hidden_states.device.index] if hidden_states.is_cuda else []
+ with torch.random.fork_rng(devices = devices), torch.enable_grad():
+ forward(self, hidden_states.detach().requires_grad_(), *args, **kwargs)
+ mode = self.__dict__["_unsloth_block_mode"]
+ if mode == _OFF:
+ return original(self, hidden_states, *args, **kwargs)
+ checked = self.__dict__.get("_unsloth_block_checked", 0)
+ if checked & bit:
+ return run(self, hidden_states, args, kwargs, mode)
+ while True:
+ try:
+ with torch._dynamo.config.patch(suppress_errors = False):
+ out = run(self, hidden_states, args, kwargs, mode)
+ self.__dict__["_unsloth_block_checked"] = checked | bit
+ return out
+ except torch.OutOfMemoryError:
+ raise
+ except Exception as e:
+ if os.environ.get("UNSLOTH_BLOCK_COMPILE_DEBUG") == "1":
+ import traceback
+ traceback.print_exc()
+ mode = _SPLIT if mode == _FULL else _OFF
+ self.__dict__["_unsloth_block_mode"] = mode
+ kind = _layer_kind(self)
+ state["start"][kind] = max(
+ state["start"].get(kind, _FIRST_MODE), mode, key = _DEMOTION.index
+ )
+ first = str(e).strip().splitlines()[0][:200] if str(e).strip() else type(e).__name__
+ if (first, mode) not in state["reported"]:
+ state["reported"].add((first, mode))
+ print(
+ f"Unsloth: {cls.__name__} block compile failed ({first}); "
+ + (
+ "retrying with flex_attention outside the block."
+ if mode == _SPLIT
+ else "keeping the stock forward for this layer."
+ )
+ )
+ if mode == _OFF:
+ return original(self, hidden_states, *args, **kwargs)
+
+ forward._unsloth_block_original = original
+ forward._unsloth_block_state = state
+ setattr(forward, _MARK, True)
+ cls.forward = forward
+
+
+def _layer_lists(model):
+ """ModuleLists of decoder layers (one class each): the text stack, not vision towers."""
+ return [
+ module
+ for module in model.modules()
+ if isinstance(module, torch.nn.ModuleList)
+ and len(module) > 0
+ and type(module[0]).__name__.endswith("DecoderLayer")
+ and all(type(m) is type(module[0]) for m in module)
+ ]
+
+
+def _decoder_layer_classes(model):
+ classes = []
+ for layer_list in _layer_lists(model):
+ if type(layer_list[0]) not in classes:
+ classes.append(type(layer_list[0]))
+ return classes
+
+
+def _torch_supported():
+ # torch 2.6: blocks fragmented and QLoRA grads moved vs the stock forward.
+ from packaging.version import Version
+ return Version(torch.__version__.split("+")[0]) >= Version("2.7")
+
+
+def _experts_of(module):
+ while module is not None:
+ if type(module).__name__.endswith("Experts"):
+ return module
+ module = getattr(module, "base_layer", None)
+ return None
+
+
+def _lora_would_fold(experts):
+ # Traced, zoo folds the LoRA delta into every expert's weights when the forward ignores the stash.
+ try:
+ from unsloth_zoo.temporary_patches import moe_utils as zoo_moe
+ reads = zoo_moe._forward_statically_reads_stash(experts)
+ if not reads and zoo_moe._interface_route_reads_stash(experts):
+ reads = True
+ except Exception:
+ return False
+ return reads is False
+
+
+def _experts_outside(layer):
+ # At the layer's first call, after get_peft_model wrapped the experts.
+ layer.__dict__["_unsloth_block_moe_done"] = True
+ mode = os.environ.get("UNSLOTH_BLOCK_MOE", "auto")
+ if mode == "inside":
+ return
+ for _, module in layer.named_modules():
+ if getattr(module, "_unsloth_block_outside", False):
+ continue
+ experts = _experts_of(module)
+ if experts is None: # named_modules() is pre-order: the outermost wrapper comes first
+ continue
+ if mode != "outside" and not _lora_would_fold(experts):
+ continue
+ module.forward = torch.compiler.disable(module.forward)
+ for inner in module.modules():
+ inner._unsloth_block_outside = True
+
+
+def patch_block_compile(model):
+ """Compile every decoder-layer class of `model` as one region. Returns the classes wrapped."""
+ # getattr: PeftModel delegates. The env fills only unstamped layers, so a re-patch keeps the stamp.
+ forced = getattr(model, "_unsloth_forced_float32", None)
+ env = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"
+ for layer_list in _layer_lists(model):
+ for layer in layer_list:
+ if forced is None:
+ layer.__dict__.setdefault("_unsloth_forced_float32", env)
+ else:
+ layer.__dict__["_unsloth_forced_float32"] = bool(forced)
+ forced = env if forced is None else forced
+ if forced or not _enabled() or not torch.cuda.is_available() or not _torch_supported():
+ return []
+ classes = _decoder_layer_classes(model)
+ if not classes:
+ return []
+ inner = {
+ type(m)
+ for layer_list in _layer_lists(model)
+ for layer in layer_list
+ for m in layer.modules()
+ }
+ for cls in inner:
+ _rebind_class(cls)
+ _flex_split_hook()
+ _gemma4_router_hook()
+ if any(type(m).__name__ == "Linear4bit" for m in model.modules()):
+ _nf4_in_blocks()
+ wrapped = []
+ for cls in classes:
+ if getattr(cls.forward, _MARK, False) or cls.__name__ in _SKIP:
+ continue
+ _wrap(cls, _COMPILE_OPTIONS)
+ wrapped.append(cls.__name__)
+ if wrapped:
+ print(f"Unsloth: compiling each transformer block as one region ({', '.join(wrapped)}).")
+ return wrapped
diff --git a/unsloth/models/_gdn_op.py b/unsloth/models/_gdn_op.py
new file mode 100644
index 00000000000..1d2f125d268
--- /dev/null
+++ b/unsloth/models/_gdn_op.py
@@ -0,0 +1,193 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""fla's chunk_gated_delta_rule as an opaque op with a registered backward (training case only),
+checked bit-equal to ChunkGatedDeltaRuleFunction before use, so a Gated DeltaNet layer compiles
+as one graph."""
+
+import torch
+import torch.nn.functional as F
+
+
+def _causal_depthwise_conv(x, weight):
+ """F.conv1d(x, weight[:, None], groups = C, padding = K - 1)[..., :T] for x (B, C, T), weight
+ (C, K), as K shifted multiply-adds accumulated in fp32. Inductor's convolution_backward
+ lowering guards on the exact sequence length (a recompile per length); this stays dynamic."""
+ seq_len, padding = x.shape[-1], weight.shape[-1] - 1
+ xp = F.pad(x, (padding, 0)).float()
+ w = weight.float()
+ out = xp[:, :, 0:seq_len] * w[:, 0, None]
+ for j in range(1, padding + 1):
+ out = out + xp[:, :, j : j + seq_len] * w[:, j, None]
+ return out.to(x.dtype)
+
+
+_FLA = None
+
+
+def _fla_ops(make_inputs):
+ """fla's (fwd, bwd, l2norm_fwd, l2norm_bwd) once the opaque op matched fla's own Function on
+ the first real layer's inputs, else False. Checked on real shapes and run through fla's
+ Function first, so its pinned autotune configs are the ones the stock path would pick."""
+ global _FLA
+ if _FLA is None:
+ _FLA = False
+ try:
+ import fla.ops.gated_delta_rule.chunk as C
+ fns = (
+ C.chunk_gated_delta_rule_fwd,
+ C.chunk_gated_delta_rule_bwd,
+ C.l2norm_fwd,
+ C.l2norm_bwd,
+ )
+ # The first call can come from a no_grad checkpoint forward.
+ with torch.inference_mode(False):
+ with torch.no_grad():
+ inputs = [t.detach().clone() for t in make_inputs()]
+ with torch.enable_grad():
+ _FLA = fns
+ _FLA = fns if _self_check(*inputs) else False
+ except Exception:
+ _FLA = False
+ return _FLA
+
+
+def _chunk_meta(q, k, v, g):
+ # (qn, q_rstd, kn, k_rstd, g_cumsum, A), as fla 0.5 allocates them
+ return (
+ (q.shape, q.dtype),
+ (q.shape[:-1], torch.float32),
+ (k.shape, k.dtype),
+ (k.shape[:-1], torch.float32),
+ (g.shape, torch.float32),
+ ((*k.shape[:-1], 64), k.dtype),
+ )
+
+
+try:
+
+ @torch.library.custom_op("unsloth::qwen3_5_gdn_chunk", mutates_args = ())
+ def _gdn_chunk(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor,
+ scale: float,
+ ) -> tuple[
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ ]:
+ fwd, _, l2_fwd, _ = _FLA
+ q, k, v, g, beta = (t.contiguous() for t in (q, k, v, g, beta))
+ with torch.cuda.device(q.device):
+ qn, q_rstd = l2_fwd(q)
+ kn, k_rstd = l2_fwd(k)
+ g_cum, o, A, _, _, _ = fwd(
+ q = qn,
+ k = kn,
+ v = v,
+ g = g,
+ beta = beta,
+ scale = scale,
+ initial_state = None,
+ output_final_state = False,
+ )
+ # Fake metas are contiguous; keep the real outputs the same.
+ return tuple(t.contiguous() for t in (o.to(q.dtype), qn, q_rstd, kn, k_rstd, g_cum, A))
+
+ @_gdn_chunk.register_fake
+ def _gdn_chunk_fake(q, k, v, g, beta, scale):
+ o = v.new_empty(v.shape, dtype = q.dtype)
+ return (o,) + tuple(
+ q.new_empty(shape, dtype = dtype) for shape, dtype in _chunk_meta(q, k, v, g)
+ )
+
+ @torch.library.custom_op("unsloth::qwen3_5_gdn_chunk_bwd", mutates_args = ())
+ def _gdn_chunk_bwd(
+ qn: torch.Tensor,
+ q_rstd: torch.Tensor,
+ kn: torch.Tensor,
+ k_rstd: torch.Tensor,
+ v: torch.Tensor,
+ g_cum: torch.Tensor,
+ beta: torch.Tensor,
+ A: torch.Tensor,
+ do: torch.Tensor,
+ scale: float,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ _, bwd, _, l2_bwd = _FLA
+ v, beta, do = v.contiguous(), beta.contiguous(), do.contiguous()
+ with torch.cuda.device(qn.device):
+ dq, dk, dv, db, dg, _, _, _ = bwd(
+ q = qn,
+ k = kn,
+ v = v,
+ g = g_cum,
+ beta = beta,
+ A = A,
+ scale = scale,
+ initial_state = None,
+ do = do,
+ dht = None,
+ )
+ dq = l2_bwd(qn, q_rstd, dq)
+ dk = l2_bwd(kn, k_rstd, dk)
+ out = (
+ dq.to(qn.dtype),
+ dk.to(kn.dtype),
+ dv.to(v.dtype),
+ dg.to(g_cum.dtype),
+ db.to(beta.dtype),
+ )
+ return tuple(t.contiguous() for t in out)
+
+ @_gdn_chunk_bwd.register_fake
+ def _gdn_chunk_bwd_fake(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A, do, scale):
+ return tuple(qn.new_empty(t.shape, dtype = t.dtype) for t in (qn, kn, v, g_cum, beta))
+
+ def _gdn_chunk_setup(ctx, inputs, output):
+ q, k, v, g, beta, scale = inputs
+ _, qn, q_rstd, kn, k_rstd, g_cum, A = output
+ ctx.save_for_backward(qn, q_rstd, kn, k_rstd, v, g_cum, beta, A)
+ ctx.scale = scale
+ ctx.dtypes = (q.dtype, k.dtype, g.dtype)
+
+ def _gdn_chunk_backward(ctx, do, *_):
+ saved = [t.contiguous() for t in ctx.saved_tensors]
+ dq, dk, dv, dg, db = _gdn_chunk_bwd(*saved, do.contiguous(), ctx.scale)
+ q_dtype, k_dtype, g_dtype = ctx.dtypes
+ return dq.to(q_dtype), dk.to(k_dtype), dv, dg.to(g_dtype), db, None
+
+ _gdn_chunk.register_autograd(_gdn_chunk_backward, setup_context = _gdn_chunk_setup)
+except Exception:
+ _gdn_chunk = None
+
+
+def _self_check(q, k, v, g, beta):
+ """Opaque op vs fla's own autograd.Function: outputs, fake metas and grads bit-equal."""
+ if _gdn_chunk is None or not q.is_cuda:
+ return False
+ from fla.ops.gated_delta_rule import chunk_gated_delta_rule
+
+ # Private generator: the default one feeds LoRA dropout, which checkpoint recompute replays.
+ gen = torch.Generator(device = v.device).manual_seed(0)
+ do = torch.randn(v.shape, dtype = v.dtype, device = v.device, generator = gen)
+ ins = [t.contiguous().requires_grad_() for t in (q, k, v, g, beta)]
+ ref, _ = chunk_gated_delta_rule(*ins, use_qk_l2norm_in_kernel = True)
+ ref_g = torch.autograd.grad(ref, ins, do)
+ ins2 = [t.detach().clone().requires_grad_() for t in ins]
+ out = _gdn_chunk(*ins2, q.shape[-1] ** -0.5)
+ got_g = torch.autograd.grad(out[0], ins2, do)
+ metas = [(o.shape, o.dtype) for o in out[1:]]
+ want = [(torch.Size(s), d) for s, d in _chunk_meta(q, k, v, g)]
+ return (
+ metas == want
+ and torch.equal(out[0], ref)
+ and all(torch.equal(a, b) for a, b in zip(got_g, ref_g))
+ )
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index f45d197525c..93a3f1d60e7 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -5100,6 +5100,12 @@ class FastBaseModel:
from ..tokenizer_utils import zero_pad_embedding_rows
zero_pad_embedding_rows(model, tokenizer)
+ # One compiled region per decoder layer (_block_compile.py).
+ try:
+ from ._block_compile import patch_block_compile
+ patch_block_compile(model)
+ except Exception as e:
+ logger.warning(f"Unsloth: per-block compile not installed: {e}")
return model
@staticmethod