Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block compile: find decoder layers structurally and wrap mixed stacks per class (#14397)
Daniel Hanc663d492ef35
diff --git a/tests/test_block_compile_discovery.py b/tests/test_block_compile_discovery.pynew file mode 100644index 00000000000..437d826e409--- /dev/null+++ b/tests/test_block_compile_discovery.py@@ -0,0 +1,298 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Block compile decoder-layer discovery on meta-device default-config models (CPU)."""++import importlib.util+import os+import sys++import pytest++torch = pytest.importorskip("torch")+transformers = pytest.importorskip("transformers")++_PATH = os.path.join(os.path.dirname(__file__), "..", "unsloth", "models", "_block_compile.py")++[email protected](autouse = True)+def _block_env(monkeypatch):+ for key in (+ "UNSLOTH_BLOCK_COMPILE",+ "UNSLOTH_COMPILE_DISABLE",+ "UNSLOTH_FORCE_FLOAT32",+ "UNSLOTH_BLOCK_COMPILE_SSM",+ ):+ monkeypatch.delenv(key, raising = False)+++def _load():+ spec = importlib.util.spec_from_file_location("_ublock_discovery", _PATH)+ mod = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = mod+ spec.loader.exec_module(mod)+ return mod+++def _meta_model(model_type, auto = "AutoModelForCausalLM"):+ try:+ config = transformers.AutoConfig.for_model(model_type)+ except Exception as e:+ pytest.skip(f"{model_type} not in this transformers: {str(e)[:80]}")+ try:+ with torch.device("meta"):+ return getattr(transformers, auto).from_config(config)+ except Exception as e: # default config not buildable in this transformers version+ pytest.skip(f"{model_type}: {type(e).__name__}: {str(e)[:80]}")+++def _found(M, model):+ names = {id(m): n for n, m in model.named_modules()}+ return {+ names[id(layer_list)]: sorted({type(m).__name__ for m in layer_list})+ for layer_list in M._layer_lists(model)+ }+++def _eager_ssm(M, model):+ return sorted(+ {+ type(layer).__name__+ for layer, legacy in M._layers(model)+ if not legacy and M._is_ssm_layer(layer)+ }+ )+++# model_type -> (layer lists found, classes kept eager as Mamba-style SSM layers)+_EXPECTED = {+ "llama": ({"model.layers": ["LlamaDecoderLayer"]}, []),+ "qwen3": ({"model.layers": ["Qwen3DecoderLayer"]}, []),+ "falcon_h1": ({"model.layers": ["FalconH1DecoderLayer"]}, []),+ "gpt_neox": ({"gpt_neox.layers": ["GPTNeoXLayer"]}, []),+ "gpt2": ({"transformer.h": ["GPT2Block"]}, []),+ "olmo_hybrid": (+ {+ "model.layers": [+ "OlmoHybridAttentionDecoderLayer",+ "OlmoHybridLinearAttentionDecoderLayer",+ ]+ },+ [],+ ),+ "mllama": (+ {"model.layers": ["MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"]},+ [],+ ),+ "jamba": (+ {"model.layers": ["JambaAttentionDecoderLayer", "JambaMambaDecoderLayer"]},+ ["JambaMambaDecoderLayer"],+ ),+ "zamba2": (+ {"model.layers": ["Zamba2HybridLayer", "Zamba2MambaDecoderLayer"]},+ ["Zamba2HybridLayer", "Zamba2MambaDecoderLayer"],+ ),+ "nemotron_h": ({"model.layers": ["NemotronHBlock"]}, ["NemotronHBlock"]),+ "mamba2": ({"backbone.layers": ["Mamba2Block"]}, ["Mamba2Block"]),+ "mamba": ({"backbone.layers": ["MambaBlock"]}, ["MambaBlock"]),+ "falcon_mamba": ({"backbone.layers": ["FalconMambaBlock"]}, ["FalconMambaBlock"]),+}++[email protected]("model_type", sorted(_EXPECTED))+def test_text_decoder_lists_are_found(model_type):+ M = _load()+ model = _meta_model(model_type)+ lists, eager = _EXPECTED[model_type]+ found = _found(M, model)+ compiled = sorted(+ {c for classes in found.values() for c in classes} - set(_eager_ssm(M, model))+ )+ assert compiled == sorted({c for classes in lists.values() for c in classes} - set(eager))+ assert set(_eager_ssm(M, model)) <= set(eager)+ # transformers < 5 builds zamba2's hybrid layer as a plain Module: the list is not found there.+ if model_type != "zamba2" or found:+ assert found == lists+++ "model_type, lists",+ [+ (+ "mllama",+ {+ "model.language_model.layers": [+ "MllamaCrossAttentionDecoderLayer",+ "MllamaSelfAttentionDecoderLayer",+ ]+ },+ ),+ ("qwen2_5_vl", {"model.language_model.layers": ["Qwen2_5_VLDecoderLayer"]}),+ ("gemma3", {"model.language_model.layers": ["Gemma3DecoderLayer"]}),+ ("llava", {"model.language_model.layers": ["LlamaDecoderLayer"]}),+ ("pixtral", {}),+ ],+)+def test_vision_towers_are_excluded(model_type, lists):+ M = _load()+ auto = "AutoModel" if model_type == "pixtral" else "AutoModelForImageTextToText"+ assert _found(M, _meta_model(model_type, auto)) == lists+++def test_nemotron_h_keeps_only_mamba_blocks_eager():+ # One class for every block kind: the SSM decision is per layer, not per class.+ M = _load()+ model = _meta_model("nemotron_h")+ kinds = {}+ for layer, _ in M._layers(model):+ kinds.setdefault(type(layer.mixer).__name__, set()).add(M._is_ssm_layer(layer))+ assert kinds.pop("NemotronHMamba2Mixer") == {True}+ assert kinds and all(v == {False} for v in kinds.values())+++class _Embed(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.embed_tokens = torch.nn.Embedding(16, 8)+++def _layer_class(name, base = None):+ base = base or transformers.modeling_layers.GradientCheckpointingLayer++ def __init__(self):+ base.__init__(self)+ self.proj = torch.nn.Linear(8, 8)++ def forward(+ self,+ hidden_states,+ attention_mask = None,+ ):+ return self.proj(hidden_states) + hidden_states++ return type(name, (base,), {"__init__": __init__, "forward": forward})+++def _fake_compile(calls):+ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ calls.append(type(args[0]).__name__)+ return fn(*args, **kw)++ return compiled++ return fake_compile+++def test_mixed_shared_and_tower_layers(monkeypatch):+ M = _load()+ calls = []+ monkeypatch.setattr(torch, "compile", _fake_compile(calls))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(M, "_torch_supported", lambda: True)+ Attn, Ssm, Tower = (+ _layer_class("ToyAttnLayer"),+ _layer_class("ToySsmLayer"),+ _layer_class("ToyVisionLayer"),+ )+ Plain = _layer_class("ToyPlainBlock", torch.nn.Module)++ class ToyMambaMixer(torch.nn.Module):+ pass++ shared = Attn()+ text = _Embed()+ ssm = Ssm()+ ssm.mixer = ToyMambaMixer()+ text.layers = torch.nn.ModuleList([shared, ssm, shared, Attn()])+ model = torch.nn.Module()+ model.model = text+ # GradientCheckpointingLayer subclasses, but no token embedding beside them / a tower path.+ model.vision_tower = _Embed()+ model.vision_tower.layers = torch.nn.ModuleList([Tower()])+ model.blocks_without_embedding = torch.nn.ModuleList([Tower()])+ model.plain = torch.nn.Module()+ model.plain.wte = torch.nn.Embedding(4, 8)+ model.plain.h = torch.nn.ModuleList(+ [Plain(), Plain()]+ ) # no GradientCheckpointingLayer: suffix rule+ assert [c.__name__ for c in M._decoder_layer_classes(model)] == [+ "ToyAttnLayer",+ "ToySsmLayer",+ "ToyPlainBlock",+ ]+ assert len(M._layers(model)) == 5+ limit = (+ torch._dynamo.config.recompile_limit+ if hasattr(torch._dynamo.config, "recompile_limit")+ else None+ )+ wrapped = M.patch_block_compile(model)+ assert wrapped == ["ToyAttnLayer", "ToyPlainBlock"]+ assert ssm._unsloth_block_mode == M._OFF+ x = torch.randn(2, 4, 8, requires_grad = True)+ for layer in list(text.layers) + list(model.plain.h):+ layer(x).sum().backward()+ assert "ToySsmLayer" not in calls and {"ToyAttnLayer", "ToyPlainBlock"} <= set(calls)+ assert not getattr(Tower.forward, M._MARK, False)+ if limit is not None:+ torch._dynamo.config.recompile_limit = limit+++def test_ssm_opt_in(monkeypatch):+ M = _load()+ monkeypatch.setattr(torch, "compile", _fake_compile([]))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(M, "_torch_supported", lambda: True)+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE_SSM", "1")+ Ssm = _layer_class("ToySsmOptInLayer")+ layer = Ssm()+ layer.ssm_state_size = 4 # Mamba-style mixer attribute+ text = _Embed()+ text.layers = torch.nn.ModuleList([layer])+ assert M.patch_block_compile(text) == ["ToySsmOptInLayer"]+ assert "_unsloth_block_mode" not in layer.__dict__+++def test_skip_list_still_applies(monkeypatch):+ M = _load()+ monkeypatch.setattr(torch, "compile", _fake_compile([]))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(M, "_torch_supported", lambda: True)+ Skip = _layer_class("GptOssDecoderLayer")+ text = _Embed()+ text.layers = torch.nn.ModuleList([Skip()])+ assert M.patch_block_compile(text) == []+ assert not getattr(Skip.forward, M._MARK, False)+++def test_cache_limit_covers_every_layer_kind(monkeypatch):+ M = _load()+ config = torch._dynamo.config+ name = "recompile_limit" if hasattr(config, "recompile_limit") else "cache_size_limit"+ monkeypatch.setattr(config, name, 8)+ M._raise_cache_limit(5)+ assert getattr(config, name) == 20+ M._raise_cache_limit(1)+ assert getattr(config, name) == 20+++def test_same_class_outside_the_decoder_stays_stock(monkeypatch):+ # T5's encoder and decoder share T5Block: only the decoder list is found, the wrap is per class.+ M = _load()+ calls = []+ monkeypatch.setattr(torch, "compile", _fake_compile(calls))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(M, "_torch_supported", lambda: True)+ Shared = _layer_class("ToySharedBlockLayer")+ model = torch.nn.Module()+ model.encoder = _Embed()+ model.encoder.layers = torch.nn.ModuleList([Shared()])+ model.decoder = _Embed()+ model.decoder.layers = torch.nn.ModuleList([Shared()])+ assert M.patch_block_compile(model) == ["ToySharedBlockLayer"]+ x = torch.randn(2, 4, 8, requires_grad = True)+ model.encoder.layers[0](x).sum().backward()+ assert calls == []+ model.decoder.layers[0](x).sum().backward()+ assert callsdiff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex a5c35e3449c..41a5770e76b 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -642,7 +642,13 @@ def _has_cache(args, kwargs):def _layer_kind(layer):- attn = getattr(layer, "self_attn", None) or getattr(layer, "linear_attn", None)+ attn = (+ getattr(layer, "self_attn", None)+ or getattr(layer, "linear_attn", None)+ or getattr(layer, "cross_attn", None)+ or getattr(layer, "attention", None)+ or getattr(layer, "mixer", None)+ )return (type(attn).__name__,getattr(layer, "attention_type", None),@@ -752,23 +758,101 @@ def _wrap(cls, compile_options):cls.forward = forward+_TOWER_SEGMENTS = (+ "vision",+ "visual",+ "image",+ "audio",+ "speech",+ "video",+ "encoder",+ "tower",+ "projector",+ "vqmodel",+ "vq_model",+)+_LAYER_SUFFIXES = ("DecoderLayer", "Block", "Layer")+# The module holding a text decoder's layer list also holds its token embedding.+_TEXT_EMBEDDINGS = (+ "embed_tokens",+ "wte",+ "embed_in",+ "word_embeddings",+ "tok_embeddings",+ "embeddings",+)+++def _gc_layer_class():+ try:+ from transformers.modeling_layers import GradientCheckpointingLayer+ except Exception:+ return None+ return GradientCheckpointingLayer+++def _in_tower(name):+ return any(segment in part for part in name.lower().split(".") for segment in _TOWER_SEGMENTS)+++def _is_ssm_layer(layer):+ # Traced, transformers swaps Mamba CUDA scan kernels for a torch scan; UNSLOTH_BLOCK_COMPILE_SSM=1.+ for module in layer.modules():+ name = type(module).__name__+ if ("Mamba" in name and name.endswith("Mixer")) or hasattr(module, "ssm_state_size"):+ return True+ return False+++def _find_layer_lists(model):+ """(ModuleList, legacy) per text decoder list; legacy = the old one-*DecoderLayer-class rule."""+ gc_layer = _gc_layer_class()+ found, prefixes = [], []+ modules = dict(model.named_modules())+ for name, module in modules.items():+ if not isinstance(module, torch.nn.ModuleList) or len(module) == 0:+ continue+ if any(name.startswith(prefix) for prefix in prefixes):+ continue+ classes = {type(m) for m in module}+ legacy = len(classes) == 1 and type(module[0]).__name__.endswith("DecoderLayer")+ if not legacy:+ parent = modules.get(name.rsplit(".", 1)[0] if "." in name else "")+ if _in_tower(name) or not any(+ getattr(parent, key, None) is not None for key in _TEXT_EMBEDDINGS+ ):+ continue+ if gc_layer is not None and all(issubclass(c, gc_layer) for c in classes):+ pass+ elif len(classes) == 1 and type(module[0]).__name__.endswith(_LAYER_SUFFIXES):+ pass # remote code without GradientCheckpointingLayer+ else:+ continue+ found.append((module, legacy))+ prefixes.append(name + "." if name else "")+ return found++def _layer_lists(model):- """ModuleLists of decoder layers (one class each): the text stack, not vision towers."""- return [- module- for module in model.modules()- if isinstance(module, torch.nn.ModuleList)- and len(module) > 0- and type(module[0]).__name__.endswith("DecoderLayer")- and all(type(m) is type(module[0]) for m in module)- ]+ return [layer_list for layer_list, _ in _find_layer_lists(model)]+++def _layers(model):+ """(layer, legacy) per decoder layer, shared modules once."""+ seen, layers = set(), []+ for layer_list, legacy in _find_layer_lists(model):+ for layer in layer_list:+ if id(layer) not in seen:+ seen.add(id(layer))+ layers.append((layer, legacy))+ return layersdef _decoder_layer_classes(model):classes = []- for layer_list in _layer_lists(model):- if type(layer_list[0]) not in classes:- classes.append(type(layer_list[0]))+ for layer, _ in _layers(model):+ if type(layer) not in classes:+ classes.append(type(layer))return classes@@ -834,17 +918,26 @@ def _experts_outside(layer):inner._unsloth_block_outside = True+def _raise_cache_limit(kinds):+ # 4 graphs per kind (grad x flex retry): mixed-kind classes would hit dynamo's cache limit.+ config = torch._dynamo.config+ name = "recompile_limit" if hasattr(config, "recompile_limit") else "cache_size_limit"+ need = 4 * kinds+ if getattr(config, name, need) < need:+ setattr(config, name, need)++def patch_block_compile(model):"""Compile every decoder-layer class of `model` as one region. Returns the classes wrapped."""# getattr: PeftModel delegates. The env fills only unstamped layers, so a re-patch keeps the stamp.forced = getattr(model, "_unsloth_forced_float32", None)env = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"- for layer_list in _layer_lists(model):- for layer in layer_list:- if forced is None:- layer.__dict__.setdefault("_unsloth_forced_float32", env)- else:- layer.__dict__["_unsloth_forced_float32"] = bool(forced)+ layers = _layers(model)+ for layer, _ in layers:+ if forced is None:+ layer.__dict__.setdefault("_unsloth_forced_float32", env)+ else:+ layer.__dict__["_unsloth_forced_float32"] = bool(forced)forced = env if forced is None else forcedif (forced@@ -854,24 +947,41 @@ def patch_block_compile(model):or _zero3_partitioned()):return []- classes = _decoder_layer_classes(model)+ ssm = os.environ.get("UNSLOTH_BLOCK_COMPILE_SSM", "0") == "1"+ compiled_layers = []+ for layer, legacy in layers:+ if type(layer).__name__ in _SKIP:+ continue+ # Per layer, not class: NemotronHBlock holds both kinds. Legacy lists compile as before.+ if not legacy and not ssm and _is_ssm_layer(layer):+ layer.__dict__["_unsloth_block_mode"] = _OFF+ continue+ compiled_layers.append(layer)+ classes = []+ for layer in compiled_layers:+ if type(layer) not in classes:+ classes.append(type(layer))if not classes:return []- inner = {- type(m)- for layer_list in _layer_lists(model)- for layer in layer_list- for m in layer.modules()- }+ inner = {type(m) for layer in compiled_layers for m in layer.modules()}for cls in inner:_rebind_class(cls)_flex_split_hook()_gemma4_router_hook()if any(type(m).__name__ == "Linear4bit" for m in model.modules()):_nf4_in_blocks()+ kinds = {}+ for layer in compiled_layers:+ kinds.setdefault(type(layer), set()).add(_layer_kind(layer))+ _raise_cache_limit(max(len(k) for k in kinds.values()))+ # The wrap is per class: instances outside the decoder lists (T5 encoder) stay stock.+ found = {id(layer) for layer, _ in layers}+ for module in model.modules():+ if type(module) in classes and id(module) not in found:+ module.__dict__["_unsloth_block_mode"] = _OFFwrapped = []for cls in classes:- if getattr(cls.forward, _MARK, False) or cls.__name__ in _SKIP:+ if getattr(cls.forward, _MARK, False):continue_wrap(cls, _COMPILE_OPTIONS)wrapped.append(cls.__name__)