Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block compile: find decoder layers structurally and wrap mixed stacks per class (#14397)

Daniel Hanc663d492ef35
diff --git a/tests/test_block_compile_discovery.py b/tests/test_block_compile_discovery.py
new file mode 100644
index 00000000000..437d826e409
--- /dev/null
+++ b/tests/test_block_compile_discovery.py
@@ -0,0 +1,298 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Block compile decoder-layer discovery on meta-device default-config models (CPU)."""
+
+import importlib.util
+import os
+import sys
+
+import pytest
+
+torch = pytest.importorskip("torch")
+transformers = pytest.importorskip("transformers")
+
+_PATH = os.path.join(os.path.dirname(__file__), "..", "unsloth", "models", "_block_compile.py")
+
+
[email protected](autouse = True)
+def _block_env(monkeypatch):
+ for key in (
+ "UNSLOTH_BLOCK_COMPILE",
+ "UNSLOTH_COMPILE_DISABLE",
+ "UNSLOTH_FORCE_FLOAT32",
+ "UNSLOTH_BLOCK_COMPILE_SSM",
+ ):
+ monkeypatch.delenv(key, raising = False)
+
+
+def _load():
+ spec = importlib.util.spec_from_file_location("_ublock_discovery", _PATH)
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = mod
+ spec.loader.exec_module(mod)
+ return mod
+
+
+def _meta_model(model_type, auto = "AutoModelForCausalLM"):
+ try:
+ config = transformers.AutoConfig.for_model(model_type)
+ except Exception as e:
+ pytest.skip(f"{model_type} not in this transformers: {str(e)[:80]}")
+ try:
+ with torch.device("meta"):
+ return getattr(transformers, auto).from_config(config)
+ except Exception as e: # default config not buildable in this transformers version
+ pytest.skip(f"{model_type}: {type(e).__name__}: {str(e)[:80]}")
+
+
+def _found(M, model):
+ names = {id(m): n for n, m in model.named_modules()}
+ return {
+ names[id(layer_list)]: sorted({type(m).__name__ for m in layer_list})
+ for layer_list in M._layer_lists(model)
+ }
+
+
+def _eager_ssm(M, model):
+ return sorted(
+ {
+ type(layer).__name__
+ for layer, legacy in M._layers(model)
+ if not legacy and M._is_ssm_layer(layer)
+ }
+ )
+
+
+# model_type -> (layer lists found, classes kept eager as Mamba-style SSM layers)
+_EXPECTED = {
+ "llama": ({"model.layers": ["LlamaDecoderLayer"]}, []),
+ "qwen3": ({"model.layers": ["Qwen3DecoderLayer"]}, []),
+ "falcon_h1": ({"model.layers": ["FalconH1DecoderLayer"]}, []),
+ "gpt_neox": ({"gpt_neox.layers": ["GPTNeoXLayer"]}, []),
+ "gpt2": ({"transformer.h": ["GPT2Block"]}, []),
+ "olmo_hybrid": (
+ {
+ "model.layers": [
+ "OlmoHybridAttentionDecoderLayer",
+ "OlmoHybridLinearAttentionDecoderLayer",
+ ]
+ },
+ [],
+ ),
+ "mllama": (
+ {"model.layers": ["MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"]},
+ [],
+ ),
+ "jamba": (
+ {"model.layers": ["JambaAttentionDecoderLayer", "JambaMambaDecoderLayer"]},
+ ["JambaMambaDecoderLayer"],
+ ),
+ "zamba2": (
+ {"model.layers": ["Zamba2HybridLayer", "Zamba2MambaDecoderLayer"]},
+ ["Zamba2HybridLayer", "Zamba2MambaDecoderLayer"],
+ ),
+ "nemotron_h": ({"model.layers": ["NemotronHBlock"]}, ["NemotronHBlock"]),
+ "mamba2": ({"backbone.layers": ["Mamba2Block"]}, ["Mamba2Block"]),
+ "mamba": ({"backbone.layers": ["MambaBlock"]}, ["MambaBlock"]),
+ "falcon_mamba": ({"backbone.layers": ["FalconMambaBlock"]}, ["FalconMambaBlock"]),
+}
+
+
[email protected]("model_type", sorted(_EXPECTED))
+def test_text_decoder_lists_are_found(model_type):
+ M = _load()
+ model = _meta_model(model_type)
+ lists, eager = _EXPECTED[model_type]
+ found = _found(M, model)
+ compiled = sorted(
+ {c for classes in found.values() for c in classes} - set(_eager_ssm(M, model))
+ )
+ assert compiled == sorted({c for classes in lists.values() for c in classes} - set(eager))
+ assert set(_eager_ssm(M, model)) <= set(eager)
+ # transformers < 5 builds zamba2's hybrid layer as a plain Module: the list is not found there.
+ if model_type != "zamba2" or found:
+ assert found == lists
+
+
+ "model_type, lists",
+ [
+ (
+ "mllama",
+ {
+ "model.language_model.layers": [
+ "MllamaCrossAttentionDecoderLayer",
+ "MllamaSelfAttentionDecoderLayer",
+ ]
+ },
+ ),
+ ("qwen2_5_vl", {"model.language_model.layers": ["Qwen2_5_VLDecoderLayer"]}),
+ ("gemma3", {"model.language_model.layers": ["Gemma3DecoderLayer"]}),
+ ("llava", {"model.language_model.layers": ["LlamaDecoderLayer"]}),
+ ("pixtral", {}),
+ ],
+)
+def test_vision_towers_are_excluded(model_type, lists):
+ M = _load()
+ auto = "AutoModel" if model_type == "pixtral" else "AutoModelForImageTextToText"
+ assert _found(M, _meta_model(model_type, auto)) == lists
+
+
+def test_nemotron_h_keeps_only_mamba_blocks_eager():
+ # One class for every block kind: the SSM decision is per layer, not per class.
+ M = _load()
+ model = _meta_model("nemotron_h")
+ kinds = {}
+ for layer, _ in M._layers(model):
+ kinds.setdefault(type(layer.mixer).__name__, set()).add(M._is_ssm_layer(layer))
+ assert kinds.pop("NemotronHMamba2Mixer") == {True}
+ assert kinds and all(v == {False} for v in kinds.values())
+
+
+class _Embed(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed_tokens = torch.nn.Embedding(16, 8)
+
+
+def _layer_class(name, base = None):
+ base = base or transformers.modeling_layers.GradientCheckpointingLayer
+
+ def __init__(self):
+ base.__init__(self)
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(
+ self,
+ hidden_states,
+ attention_mask = None,
+ ):
+ return self.proj(hidden_states) + hidden_states
+
+ return type(name, (base,), {"__init__": __init__, "forward": forward})
+
+
+def _fake_compile(calls):
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ calls.append(type(args[0]).__name__)
+ return fn(*args, **kw)
+
+ return compiled
+
+ return fake_compile
+
+
+def test_mixed_shared_and_tower_layers(monkeypatch):
+ M = _load()
+ calls = []
+ monkeypatch.setattr(torch, "compile", _fake_compile(calls))
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(M, "_torch_supported", lambda: True)
+ Attn, Ssm, Tower = (
+ _layer_class("ToyAttnLayer"),
+ _layer_class("ToySsmLayer"),
+ _layer_class("ToyVisionLayer"),
+ )
+ Plain = _layer_class("ToyPlainBlock", torch.nn.Module)
+
+ class ToyMambaMixer(torch.nn.Module):
+ pass
+
+ shared = Attn()
+ text = _Embed()
+ ssm = Ssm()
+ ssm.mixer = ToyMambaMixer()
+ text.layers = torch.nn.ModuleList([shared, ssm, shared, Attn()])
+ model = torch.nn.Module()
+ model.model = text
+ # GradientCheckpointingLayer subclasses, but no token embedding beside them / a tower path.
+ model.vision_tower = _Embed()
+ model.vision_tower.layers = torch.nn.ModuleList([Tower()])
+ model.blocks_without_embedding = torch.nn.ModuleList([Tower()])
+ model.plain = torch.nn.Module()
+ model.plain.wte = torch.nn.Embedding(4, 8)
+ model.plain.h = torch.nn.ModuleList(
+ [Plain(), Plain()]
+ ) # no GradientCheckpointingLayer: suffix rule
+ assert [c.__name__ for c in M._decoder_layer_classes(model)] == [
+ "ToyAttnLayer",
+ "ToySsmLayer",
+ "ToyPlainBlock",
+ ]
+ assert len(M._layers(model)) == 5
+ limit = (
+ torch._dynamo.config.recompile_limit
+ if hasattr(torch._dynamo.config, "recompile_limit")
+ else None
+ )
+ wrapped = M.patch_block_compile(model)
+ assert wrapped == ["ToyAttnLayer", "ToyPlainBlock"]
+ assert ssm._unsloth_block_mode == M._OFF
+ x = torch.randn(2, 4, 8, requires_grad = True)
+ for layer in list(text.layers) + list(model.plain.h):
+ layer(x).sum().backward()
+ assert "ToySsmLayer" not in calls and {"ToyAttnLayer", "ToyPlainBlock"} <= set(calls)
+ assert not getattr(Tower.forward, M._MARK, False)
+ if limit is not None:
+ torch._dynamo.config.recompile_limit = limit
+
+
+def test_ssm_opt_in(monkeypatch):
+ M = _load()
+ monkeypatch.setattr(torch, "compile", _fake_compile([]))
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(M, "_torch_supported", lambda: True)
+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE_SSM", "1")
+ Ssm = _layer_class("ToySsmOptInLayer")
+ layer = Ssm()
+ layer.ssm_state_size = 4 # Mamba-style mixer attribute
+ text = _Embed()
+ text.layers = torch.nn.ModuleList([layer])
+ assert M.patch_block_compile(text) == ["ToySsmOptInLayer"]
+ assert "_unsloth_block_mode" not in layer.__dict__
+
+
+def test_skip_list_still_applies(monkeypatch):
+ M = _load()
+ monkeypatch.setattr(torch, "compile", _fake_compile([]))
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(M, "_torch_supported", lambda: True)
+ Skip = _layer_class("GptOssDecoderLayer")
+ text = _Embed()
+ text.layers = torch.nn.ModuleList([Skip()])
+ assert M.patch_block_compile(text) == []
+ assert not getattr(Skip.forward, M._MARK, False)
+
+
+def test_cache_limit_covers_every_layer_kind(monkeypatch):
+ M = _load()
+ config = torch._dynamo.config
+ name = "recompile_limit" if hasattr(config, "recompile_limit") else "cache_size_limit"
+ monkeypatch.setattr(config, name, 8)
+ M._raise_cache_limit(5)
+ assert getattr(config, name) == 20
+ M._raise_cache_limit(1)
+ assert getattr(config, name) == 20
+
+
+def test_same_class_outside_the_decoder_stays_stock(monkeypatch):
+ # T5's encoder and decoder share T5Block: only the decoder list is found, the wrap is per class.
+ M = _load()
+ calls = []
+ monkeypatch.setattr(torch, "compile", _fake_compile(calls))
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(M, "_torch_supported", lambda: True)
+ Shared = _layer_class("ToySharedBlockLayer")
+ model = torch.nn.Module()
+ model.encoder = _Embed()
+ model.encoder.layers = torch.nn.ModuleList([Shared()])
+ model.decoder = _Embed()
+ model.decoder.layers = torch.nn.ModuleList([Shared()])
+ assert M.patch_block_compile(model) == ["ToySharedBlockLayer"]
+ x = torch.randn(2, 4, 8, requires_grad = True)
+ model.encoder.layers[0](x).sum().backward()
+ assert calls == []
+ model.decoder.layers[0](x).sum().backward()
+ assert calls
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index a5c35e3449c..41a5770e76b 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -642,7 +642,13 @@ def _has_cache(args, kwargs):
def _layer_kind(layer):
- attn = getattr(layer, "self_attn", None) or getattr(layer, "linear_attn", None)
+ attn = (
+ getattr(layer, "self_attn", None)
+ or getattr(layer, "linear_attn", None)
+ or getattr(layer, "cross_attn", None)
+ or getattr(layer, "attention", None)
+ or getattr(layer, "mixer", None)
+ )
return (
type(attn).__name__,
getattr(layer, "attention_type", None),
@@ -752,23 +758,101 @@ def _wrap(cls, compile_options):
cls.forward = forward
+_TOWER_SEGMENTS = (
+ "vision",
+ "visual",
+ "image",
+ "audio",
+ "speech",
+ "video",
+ "encoder",
+ "tower",
+ "projector",
+ "vqmodel",
+ "vq_model",
+)
+_LAYER_SUFFIXES = ("DecoderLayer", "Block", "Layer")
+# The module holding a text decoder's layer list also holds its token embedding.
+_TEXT_EMBEDDINGS = (
+ "embed_tokens",
+ "wte",
+ "embed_in",
+ "word_embeddings",
+ "tok_embeddings",
+ "embeddings",
+)
+
+
+def _gc_layer_class():
+ try:
+ from transformers.modeling_layers import GradientCheckpointingLayer
+ except Exception:
+ return None
+ return GradientCheckpointingLayer
+
+
+def _in_tower(name):
+ return any(segment in part for part in name.lower().split(".") for segment in _TOWER_SEGMENTS)
+
+
+def _is_ssm_layer(layer):
+ # Traced, transformers swaps Mamba CUDA scan kernels for a torch scan; UNSLOTH_BLOCK_COMPILE_SSM=1.
+ for module in layer.modules():
+ name = type(module).__name__
+ if ("Mamba" in name and name.endswith("Mixer")) or hasattr(module, "ssm_state_size"):
+ return True
+ return False
+
+
+def _find_layer_lists(model):
+ """(ModuleList, legacy) per text decoder list; legacy = the old one-*DecoderLayer-class rule."""
+ gc_layer = _gc_layer_class()
+ found, prefixes = [], []
+ modules = dict(model.named_modules())
+ for name, module in modules.items():
+ if not isinstance(module, torch.nn.ModuleList) or len(module) == 0:
+ continue
+ if any(name.startswith(prefix) for prefix in prefixes):
+ continue
+ classes = {type(m) for m in module}
+ legacy = len(classes) == 1 and type(module[0]).__name__.endswith("DecoderLayer")
+ if not legacy:
+ parent = modules.get(name.rsplit(".", 1)[0] if "." in name else "")
+ if _in_tower(name) or not any(
+ getattr(parent, key, None) is not None for key in _TEXT_EMBEDDINGS
+ ):
+ continue
+ if gc_layer is not None and all(issubclass(c, gc_layer) for c in classes):
+ pass
+ elif len(classes) == 1 and type(module[0]).__name__.endswith(_LAYER_SUFFIXES):
+ pass # remote code without GradientCheckpointingLayer
+ else:
+ continue
+ found.append((module, legacy))
+ prefixes.append(name + "." if name else "")
+ return found
+
+
def _layer_lists(model):
- """ModuleLists of decoder layers (one class each): the text stack, not vision towers."""
- return [
- module
- for module in model.modules()
- if isinstance(module, torch.nn.ModuleList)
- and len(module) > 0
- and type(module[0]).__name__.endswith("DecoderLayer")
- and all(type(m) is type(module[0]) for m in module)
- ]
+ return [layer_list for layer_list, _ in _find_layer_lists(model)]
+
+
+def _layers(model):
+ """(layer, legacy) per decoder layer, shared modules once."""
+ seen, layers = set(), []
+ for layer_list, legacy in _find_layer_lists(model):
+ for layer in layer_list:
+ if id(layer) not in seen:
+ seen.add(id(layer))
+ layers.append((layer, legacy))
+ return layers
def _decoder_layer_classes(model):
classes = []
- for layer_list in _layer_lists(model):
- if type(layer_list[0]) not in classes:
- classes.append(type(layer_list[0]))
+ for layer, _ in _layers(model):
+ if type(layer) not in classes:
+ classes.append(type(layer))
return classes
@@ -834,17 +918,26 @@ def _experts_outside(layer):
inner._unsloth_block_outside = True
+def _raise_cache_limit(kinds):
+ # 4 graphs per kind (grad x flex retry): mixed-kind classes would hit dynamo's cache limit.
+ config = torch._dynamo.config
+ name = "recompile_limit" if hasattr(config, "recompile_limit") else "cache_size_limit"
+ need = 4 * kinds
+ if getattr(config, name, need) < need:
+ setattr(config, name, need)
+
+
def patch_block_compile(model):
"""Compile every decoder-layer class of `model` as one region. Returns the classes wrapped."""
# getattr: PeftModel delegates. The env fills only unstamped layers, so a re-patch keeps the stamp.
forced = getattr(model, "_unsloth_forced_float32", None)
env = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"
- for layer_list in _layer_lists(model):
- for layer in layer_list:
- if forced is None:
- layer.__dict__.setdefault("_unsloth_forced_float32", env)
- else:
- layer.__dict__["_unsloth_forced_float32"] = bool(forced)
+ layers = _layers(model)
+ for layer, _ in layers:
+ if forced is None:
+ layer.__dict__.setdefault("_unsloth_forced_float32", env)
+ else:
+ layer.__dict__["_unsloth_forced_float32"] = bool(forced)
forced = env if forced is None else forced
if (
forced
@@ -854,24 +947,41 @@ def patch_block_compile(model):
or _zero3_partitioned()
):
return []
- classes = _decoder_layer_classes(model)
+ ssm = os.environ.get("UNSLOTH_BLOCK_COMPILE_SSM", "0") == "1"
+ compiled_layers = []
+ for layer, legacy in layers:
+ if type(layer).__name__ in _SKIP:
+ continue
+ # Per layer, not class: NemotronHBlock holds both kinds. Legacy lists compile as before.
+ if not legacy and not ssm and _is_ssm_layer(layer):
+ layer.__dict__["_unsloth_block_mode"] = _OFF
+ continue
+ compiled_layers.append(layer)
+ classes = []
+ for layer in compiled_layers:
+ if type(layer) not in classes:
+ classes.append(type(layer))
if not classes:
return []
- inner = {
- type(m)
- for layer_list in _layer_lists(model)
- for layer in layer_list
- for m in layer.modules()
- }
+ inner = {type(m) for layer in compiled_layers for m in layer.modules()}
for cls in inner:
_rebind_class(cls)
_flex_split_hook()
_gemma4_router_hook()
if any(type(m).__name__ == "Linear4bit" for m in model.modules()):
_nf4_in_blocks()
+ kinds = {}
+ for layer in compiled_layers:
+ kinds.setdefault(type(layer), set()).add(_layer_kind(layer))
+ _raise_cache_limit(max(len(k) for k in kinds.values()))
+ # The wrap is per class: instances outside the decoder lists (T5 encoder) stay stock.
+ found = {id(layer) for layer, _ in layers}
+ for module in model.modules():
+ if type(module) in classes and id(module) not in found:
+ module.__dict__["_unsloth_block_mode"] = _OFF
wrapped = []
for cls in classes:
- if getattr(cls.forward, _MARK, False) or cls.__name__ in _SKIP:
+ if getattr(cls.forward, _MARK, False):
continue
_wrap(cls, _COMPILE_OPTIONS)
wrapped.append(cls.__name__)