Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE LoRA: keep native per-expert Linears for an explicit target_modules list on transformers 4.x (#13851)
Daniel Handdbbd9b16f2b
diff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.pyindex e6f4e5d38a5..20befc542b9 100644--- a/tests/test_remote_code_class_and_expert_submodules.py+++ b/tests/test_remote_code_class_and_expert_submodules.py@@ -580,3 +580,221 @@ def test_sentence_transformer_probes_use_the_loads_options(monkeypatch):for call in calls:splats = [getattr(k.value, "id", "") for k in call.keywords if k.arg is None]assert "_remote_class_probe_kwargs" in splats, call.lineno+++def _native_qwen3_moe_4x_model(with_visual = False, dense_first = False):+ """Qwen3-MoE / Mixtral on transformers 4.x: native per-expert Linears under mlp.experts.<i>."""++ class _NativeExpert(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.gate_proj = torch.nn.Linear(8, 16, bias = False)+ self.up_proj = torch.nn.Linear(8, 16, bias = False)+ self.down_proj = torch.nn.Linear(16, 8, bias = False)++ _NativeExpert.__module__ = "transformers.models.qwen3_moe.modeling_qwen3_moe"++ class _Attn(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q_proj = torch.nn.Linear(8, 8, bias = False)+ self.o_proj = torch.nn.Linear(8, 8, bias = False)++ class _SparseMoe(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.gate = torch.nn.Linear(8, 4, bias = False)+ self.experts = torch.nn.ModuleList([_NativeExpert() for _ in range(4)])++ class _DecoderLayer(torch.nn.Module):+ def __init__(self, dense = False):+ super().__init__()+ self.self_attn = _Attn()+ self.mlp = _NativeExpert() if dense else _SparseMoe()++ class _Body(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList([_DecoderLayer(dense_first and i == 0) for i in range(2)])++ class _Qwen3Moe(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(num_experts = 4, model_type = "qwen3_moe")+ self.model = _Body()+ if with_visual:+ self.visual = torch.nn.ModuleList([_SparseMoe(), _SparseMoe()])++ return _Qwen3Moe()+++def _explicit_list_regex(model, leaves):+ """FastLanguageModel's regex for an explicit list (skips mlp.experts.<i>)."""+ import importlib++ stub = sys.modules.get("unsloth_zoo.peft_utils")+ if stub is not None and getattr(stub, "__file__", None) is None:+ del sys.modules["unsloth_zoo.peft_utils"]+ peft_utils = importlib.import_module("unsloth_zoo.peft_utils")+ return peft_utils.get_peft_regex(+ model,+ finetune_vision_layers = False,+ finetune_language_layers = True,+ finetune_attention_modules = True,+ finetune_mlp_modules = True,+ target_modules = list(leaves),+ )+++def test_explicit_leaf_list_reaches_native_per_expert_linears():+ """An explicit gate/up/down list must LoRA native experts, as plain PEFT does."""+ U = _utils()+ model = _native_qwen3_moe_4x_model()+ leaves = ["q_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]+ regex = _explicit_list_regex(model, leaves)+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]+ assert not any(re.fullmatch(regex, n) for n in names if ".experts." in n)++ widened, detect, found = U.widen_target_regex_to_expert_submodules(+ model, regex, leaves, auto_regex = True, explicit_leaves = True+ )+ assert found == ["down_proj", "gate_proj", "up_proj"] and detect == leaves+ matched = {n for n in names if re.fullmatch(widened, n)}+ peft_leaf_match = {n for n in names if n.rpartition(".")[2] in leaves}+ assert matched == peft_leaf_match+ assert "model.layers.0.mlp.gate" not in matched++ regex = _explicit_list_regex(model, ["q_proj", "down_proj"])+ widened, _, found = U.widen_target_regex_to_expert_submodules(+ model, regex, ["q_proj", "down_proj"], auto_regex = True, explicit_leaves = True+ )+ assert found == ["down_proj"]+ assert not any(+ re.fullmatch(widened, n) for n in names if n.endswith((".gate_proj", ".up_proj"))+ )++ for dense_first in (False, True):+ model = _native_qwen3_moe_4x_model(dense_first = dense_first)+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]+ for leaves in (["down_proj"], ["gate_proj", "up_proj", "down_proj"]):+ regex = _explicit_list_regex(model, leaves)+ widened, _, _ = U.widen_target_regex_to_expert_submodules(+ model, regex, leaves, auto_regex = True, explicit_leaves = True+ )+ matched = {n for n in names if re.fullmatch(widened, n)}+ assert matched == {n for n in names if n.rpartition(".")[2] in leaves}+ assert any(".experts." in n for n in matched)+++def test_explicit_leaf_list_widening_respects_the_family_filters():+ U = _utils()+ leaves = ["q_proj", "gate_proj", "up_proj", "down_proj"]+ model = _native_qwen3_moe_4x_model()+ regex = _explicit_list_regex(model, ["q_proj", "o_proj"])+ kept, _, found = U.widen_target_regex_to_expert_submodules(+ model, regex, ["q_proj", "o_proj"], auto_regex = True, explicit_leaves = True+ )+ assert kept == regex and found == []+ kept, _, found = U.widen_target_regex_to_expert_submodules(model, regex, regex, auto_regex = True)+ assert kept == regex and found == []+ model = _native_qwen3_moe_4x_model(with_visual = True)+ regex = _explicit_list_regex(model, leaves)+ widened, _, found = U.widen_target_regex_to_expert_submodules(+ model, regex, leaves, auto_regex = True, explicit_leaves = True+ )+ matched = {n for n, _ in model.named_modules() if re.fullmatch(widened, n)}+ assert any(n.startswith("model.layers.") and ".experts." in n for n in matched)+ assert not any(n.startswith("visual.") for n in matched)+++def _native_mixtral_4x_model():+ """Mixtral on transformers 4.x: block_sparse_moe.experts.<i>.w1 / w2 / w3."""++ class _Expert(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.w1 = torch.nn.Linear(8, 16, bias = False)+ self.w2 = torch.nn.Linear(16, 8, bias = False)+ self.w3 = torch.nn.Linear(8, 16, bias = False)++ class _Moe(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.gate = torch.nn.Linear(8, 4, bias = False)+ self.experts = torch.nn.ModuleList([_Expert() for _ in range(4)])++ class _Attn(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q_proj = torch.nn.Linear(8, 8, bias = False)++ class _Layer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.self_attn = _Attn()+ self.block_sparse_moe = _Moe()++ class _Body(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList([_Layer() for _ in range(2)])++ class _Mixtral(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(num_local_experts = 4, model_type = "mixtral")+ self.model = _Body()++ def get_decoder(self):+ return self.model++ return _Mixtral()+++def test_explicit_leaf_list_keeps_mixtral_names_and_the_gate_up_shorthand():+ U = _utils()+ model = _native_mixtral_4x_model()+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]+ leaves = ["q_proj", "w1", "w2", "w3"]+ regex = _explicit_list_regex(model, leaves)+ widened, _, found = U.widen_target_regex_to_expert_submodules(+ model, regex, leaves, auto_regex = True, explicit_leaves = True+ )+ assert found == ["w1", "w2", "w3"]+ assert {n for n in names if re.fullmatch(widened, n)} == {n for n in names if n.rpartition(".")[2] in leaves}++ model = _native_qwen3_moe_4x_model()+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]+ regex = _explicit_list_regex(model, ["q_proj", "gate_up_proj"])+ widened, _, found = U.widen_target_regex_to_expert_submodules(+ model, regex, ["q_proj", "gate_up_proj"], auto_regex = True, explicit_leaves = True+ )+ assert found == ["gate_proj", "up_proj"]+ matched = {n for n in names if re.fullmatch(widened, n) and ".experts." in n}+ assert matched == {n for n in names if ".experts." in n and n.endswith((".gate_proj", ".up_proj"))}+++def test_expert_only_mixtral_list_falls_back_to_the_decoders_experts():+ """[w1, w2, w3] alone: Zoo's regex matches nothing (no MLP tag), plain PEFT reaches every expert."""+ U = _utils()+ model = _native_mixtral_4x_model()+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]+ with pytest.raises(RuntimeError):+ _explicit_list_regex(model, ["w1", "w2", "w3"])+ regex = U.native_expert_only_regex(model, ["w1", "w2", "w3"])+ assert {n for n in names if re.fullmatch(regex, n)} == {n for n in names if n.endswith((".w1", ".w2", ".w3"))}+ assert U.native_expert_only_regex(model, ["q_proj"]) is None+ regex = U.native_expert_only_regex(model, ["w1", "w2", "w3", "gate"])+ want = {n for n in names if n.endswith((".w1", ".w2", ".w3", ".gate"))}+ assert any(n.endswith(".block_sparse_moe.gate") for n in want)+ assert {n for n in names if re.fullmatch(regex, n)} == want+ # Zoo already dropped the other families (finetune_attention_modules=False).+ regex = U.native_expert_only_regex(model, ["q_proj", "w1", "w2", "w3"])+ assert {n for n in names if re.fullmatch(regex, n)} == {n for n in names if n.endswith((".w1", ".w2", ".w3"))}+ model = _native_qwen3_moe_4x_model()+ model.get_decoder = lambda: model.model+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]+ regex = U.native_expert_only_regex(model, ["gate_up_proj"])+ assert {n for n in names if re.fullmatch(regex, n)} == {+ n for n in names if ".experts." in n and n.endswith((".gate_proj", ".up_proj"))+ }diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex b3ad0d2021e..80e94a425ef 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -119,6 +119,7 @@ __all__ = ["get_moe_expert_submodule_leaves","moe_expert_submodule_regex","widen_target_regex_to_expert_submodules",+ "native_expert_only_regex","warn_if_zoo_cannot_merge_moe_experts","_select_moe_detection_targets","EMBEDDING_MODULES",@@ -9208,17 +9209,115 @@ def _remote_expert_parents(model):return parents+def _native_expert_parents(model, target_modules, requested):+ """(parents, requested leaves) of native ``experts.<i>`` blocks inside the regex's language scope."""+ try:+ pattern = re.compile(target_modules)+ except re.error:+ return set(), set()+ modules = list(model.named_modules())+ reached_layers = {+ name.rsplit(".", 2)[0]+ for name, _ in modules+ if ".experts." not in name and pattern.fullmatch(name)+ }+ in_scope, parents, leaves = {}, set(), set()+ for name, module in modules:+ matched = _EXPERT_SUBMODULE_PATTERN.search(name)+ if matched is None or not name[matched.start() :].lstrip(".").startswith("experts."):+ continue+ parent, leaf = name[: matched.start()], matched.group(1)+ if not parent:+ continue+ key = (parent, leaf)+ if key not in in_scope:+ layer = parent.rpartition(".")[0]+ # Mixtral's block_sparse_moe has no MLP keyword; this keeps vision towers out.+ in_scope[key] = any(pattern.fullmatch(f"{parent}.{t}") for t in requested | {leaf}) or bool(+ layer and any(r == layer or r.startswith(layer + ".") for r in reached_layers)+ )+ if not in_scope[key]:+ continue+ parents.add(+ r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in parent.split("."))+ )+ if leaf in requested and (+ isinstance(module, torch.nn.Linear)+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)+ ):+ leaves.add(leaf)+ return parents, leaves+++def native_expert_only_regex(model, target_modules):+ """Regex for the explicit leaves found in the language decoder's native ``experts.<i>`` blocks, or None."""+ try:+ decoder = model.get_decoder()+ except Exception:+ return None+ prefix = next((name for name, module in model.named_modules() if module is decoder), None)+ if prefix is None:+ return None+ requested = {t for t in target_modules if isinstance(t, str) and "." not in t}+ if "gate_up_proj" in requested: # the shorthand reaches split gate / up experts too+ requested |= {"gate_proj", "up_proj"}+ def generic(path):+ return r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in path.split("."))++ linears = [+ name+ for name, module in model.named_modules()+ if (not prefix or name.startswith(prefix + "."))+ and (+ isinstance(module, torch.nn.Linear)+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)+ )+ ]+ blocks, leaves = set(), set()+ for name in linears:+ matched = _EXPERT_SUBMODULE_PATTERN.search(name)+ if matched is not None and matched.group(1) in requested:+ if name[matched.start() :].lstrip(".").startswith("experts."):+ blocks.add(name[: matched.start()])+ leaves.add(matched.group(1))+ if not leaves:+ return None+ parents = {generic(block) for block in blocks}+ # Requested expert siblings (router) too, as PEFT suffix matching would.+ others = {+ generic(name)+ for name in linears+ if name.rpartition(".")[0] in blocks and name.rpartition(".")[2] in requested+ }+ regex = moe_expert_submodule_regex(sorted(leaves), parents)+ return f"(?:{regex})|(?:{'|'.join(sorted(others))})" if others else regex++def widen_target_regex_to_expert_submodules(- model, target_modules, detect_targets, auto_regex: bool+ model,+ target_modules,+ detect_targets,+ auto_regex: bool,+ explicit_leaves: bool = False,):- """Widen a generated regex to ``experts.<i>.<leaf>``; a user-written regex is never widened."""+ """Widen a generated regex to ``experts.<i>.<leaf>``; a user-written regex is never widened.+ explicit_leaves: the caller's list went through get_peft_regex, which skips native experts+ that plain PEFT would match, so widen those too."""if not auto_regex or not isinstance(target_modules, str):return target_modules, detect_targets, []# Remote expert blocks only: widening native ones (Qwen3-MoE on 4.x) LoRAs every routed expert.parents = _remote_expert_parents(model)+ named_leaves = set()+ if explicit_leaves:+ requested = set()+ if isinstance(detect_targets, (list, tuple, set)):+ requested = {t for t in detect_targets if isinstance(t, str) and "." not in t}+ native, named_leaves = _native_expert_parents(model, target_modules, requested)+ parents = parents | nativeif not parents:return target_modules, detect_targets, []- leaves = get_moe_expert_submodule_leaves(model, detect_targets)+ # Requested names used as-is too, as plain PEFT does (Mixtral's w1 / w2 / w3).+ leaves = sorted(set(get_moe_expert_submodule_leaves(model, detect_targets)) | named_leaves)if not leaves:return target_modules, detect_targets, []if any(diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 93a3f1d60e7..f4111afdab1 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -4636,6 +4636,7 @@ class FastBaseModel:# Only a regex generated here may be widened to expert submodules below._target_modules_auto_regex = False+ _target_modules_from_explicit_list = Falseif target_modules is None or target_modules == "all-linear":target_modules = get_peft_regex(model,@@ -4669,16 +4670,26 @@ class FastBaseModel:"finetune_(vision|language|attention|mlp) filters; adapters ""attach only where both select.")- target_modules = get_peft_regex(- model,- finetune_vision_layers = finetune_vision_layers,- finetune_language_layers = finetune_language_layers,- finetune_attention_modules = finetune_attention_modules,- finetune_mlp_modules = finetune_mlp_modules,- target_modules = list(target_modules),- **_audio_kwargs,- )+ try:+ target_modules = get_peft_regex(+ model,+ finetune_vision_layers = finetune_vision_layers,+ finetune_language_layers = finetune_language_layers,+ finetune_attention_modules = finetune_attention_modules,+ finetune_mlp_modules = finetune_mlp_modules,+ target_modules = list(target_modules),+ **_audio_kwargs,+ )+ except RuntimeError:+ # Mixtral's [w1, w2, w3]: block_sparse_moe has no MLP tag, so Zoo matches nothing.+ expert_regex = None+ if finetune_language_layers and finetune_mlp_modules:+ expert_regex = native_expert_only_regex(model, target_modules)+ if expert_regex is None:+ raise+ target_modules = expert_regex_target_modules_auto_regex = True+ _target_modules_from_explicit_list = Trueif hasattr(model, "vllm_engine"):if (@@ -4723,6 +4734,7 @@ class FastBaseModel:target_modules,_moe_module_detect,auto_regex = _target_modules_auto_regex and bool(finetune_language_layers),+ explicit_leaves = _target_modules_from_explicit_list and bool(finetune_mlp_modules),))if _expert_submodule_leaves: