Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE LoRA: keep native per-expert Linears for an explicit target_modules list on transformers 4.x (#13851)

Daniel Handdbbd9b16f2b
diff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.py
index e6f4e5d38a5..20befc542b9 100644
--- a/tests/test_remote_code_class_and_expert_submodules.py
+++ b/tests/test_remote_code_class_and_expert_submodules.py
@@ -580,3 +580,221 @@ def test_sentence_transformer_probes_use_the_loads_options(monkeypatch):
for call in calls:
splats = [getattr(k.value, "id", "") for k in call.keywords if k.arg is None]
assert "_remote_class_probe_kwargs" in splats, call.lineno
+
+
+def _native_qwen3_moe_4x_model(with_visual = False, dense_first = False):
+ """Qwen3-MoE / Mixtral on transformers 4.x: native per-expert Linears under mlp.experts.<i>."""
+
+ class _NativeExpert(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_proj = torch.nn.Linear(8, 16, bias = False)
+ self.up_proj = torch.nn.Linear(8, 16, bias = False)
+ self.down_proj = torch.nn.Linear(16, 8, bias = False)
+
+ _NativeExpert.__module__ = "transformers.models.qwen3_moe.modeling_qwen3_moe"
+
+ class _Attn(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q_proj = torch.nn.Linear(8, 8, bias = False)
+ self.o_proj = torch.nn.Linear(8, 8, bias = False)
+
+ class _SparseMoe(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate = torch.nn.Linear(8, 4, bias = False)
+ self.experts = torch.nn.ModuleList([_NativeExpert() for _ in range(4)])
+
+ class _DecoderLayer(torch.nn.Module):
+ def __init__(self, dense = False):
+ super().__init__()
+ self.self_attn = _Attn()
+ self.mlp = _NativeExpert() if dense else _SparseMoe()
+
+ class _Body(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([_DecoderLayer(dense_first and i == 0) for i in range(2)])
+
+ class _Qwen3Moe(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(num_experts = 4, model_type = "qwen3_moe")
+ self.model = _Body()
+ if with_visual:
+ self.visual = torch.nn.ModuleList([_SparseMoe(), _SparseMoe()])
+
+ return _Qwen3Moe()
+
+
+def _explicit_list_regex(model, leaves):
+ """FastLanguageModel's regex for an explicit list (skips mlp.experts.<i>)."""
+ import importlib
+
+ stub = sys.modules.get("unsloth_zoo.peft_utils")
+ if stub is not None and getattr(stub, "__file__", None) is None:
+ del sys.modules["unsloth_zoo.peft_utils"]
+ peft_utils = importlib.import_module("unsloth_zoo.peft_utils")
+ return peft_utils.get_peft_regex(
+ model,
+ finetune_vision_layers = False,
+ finetune_language_layers = True,
+ finetune_attention_modules = True,
+ finetune_mlp_modules = True,
+ target_modules = list(leaves),
+ )
+
+
+def test_explicit_leaf_list_reaches_native_per_expert_linears():
+ """An explicit gate/up/down list must LoRA native experts, as plain PEFT does."""
+ U = _utils()
+ model = _native_qwen3_moe_4x_model()
+ leaves = ["q_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+ regex = _explicit_list_regex(model, leaves)
+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]
+ assert not any(re.fullmatch(regex, n) for n in names if ".experts." in n)
+
+ widened, detect, found = U.widen_target_regex_to_expert_submodules(
+ model, regex, leaves, auto_regex = True, explicit_leaves = True
+ )
+ assert found == ["down_proj", "gate_proj", "up_proj"] and detect == leaves
+ matched = {n for n in names if re.fullmatch(widened, n)}
+ peft_leaf_match = {n for n in names if n.rpartition(".")[2] in leaves}
+ assert matched == peft_leaf_match
+ assert "model.layers.0.mlp.gate" not in matched
+
+ regex = _explicit_list_regex(model, ["q_proj", "down_proj"])
+ widened, _, found = U.widen_target_regex_to_expert_submodules(
+ model, regex, ["q_proj", "down_proj"], auto_regex = True, explicit_leaves = True
+ )
+ assert found == ["down_proj"]
+ assert not any(
+ re.fullmatch(widened, n) for n in names if n.endswith((".gate_proj", ".up_proj"))
+ )
+
+ for dense_first in (False, True):
+ model = _native_qwen3_moe_4x_model(dense_first = dense_first)
+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]
+ for leaves in (["down_proj"], ["gate_proj", "up_proj", "down_proj"]):
+ regex = _explicit_list_regex(model, leaves)
+ widened, _, _ = U.widen_target_regex_to_expert_submodules(
+ model, regex, leaves, auto_regex = True, explicit_leaves = True
+ )
+ matched = {n for n in names if re.fullmatch(widened, n)}
+ assert matched == {n for n in names if n.rpartition(".")[2] in leaves}
+ assert any(".experts." in n for n in matched)
+
+
+def test_explicit_leaf_list_widening_respects_the_family_filters():
+ U = _utils()
+ leaves = ["q_proj", "gate_proj", "up_proj", "down_proj"]
+ model = _native_qwen3_moe_4x_model()
+ regex = _explicit_list_regex(model, ["q_proj", "o_proj"])
+ kept, _, found = U.widen_target_regex_to_expert_submodules(
+ model, regex, ["q_proj", "o_proj"], auto_regex = True, explicit_leaves = True
+ )
+ assert kept == regex and found == []
+ kept, _, found = U.widen_target_regex_to_expert_submodules(model, regex, regex, auto_regex = True)
+ assert kept == regex and found == []
+ model = _native_qwen3_moe_4x_model(with_visual = True)
+ regex = _explicit_list_regex(model, leaves)
+ widened, _, found = U.widen_target_regex_to_expert_submodules(
+ model, regex, leaves, auto_regex = True, explicit_leaves = True
+ )
+ matched = {n for n, _ in model.named_modules() if re.fullmatch(widened, n)}
+ assert any(n.startswith("model.layers.") and ".experts." in n for n in matched)
+ assert not any(n.startswith("visual.") for n in matched)
+
+
+def _native_mixtral_4x_model():
+ """Mixtral on transformers 4.x: block_sparse_moe.experts.<i>.w1 / w2 / w3."""
+
+ class _Expert(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.w1 = torch.nn.Linear(8, 16, bias = False)
+ self.w2 = torch.nn.Linear(16, 8, bias = False)
+ self.w3 = torch.nn.Linear(8, 16, bias = False)
+
+ class _Moe(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate = torch.nn.Linear(8, 4, bias = False)
+ self.experts = torch.nn.ModuleList([_Expert() for _ in range(4)])
+
+ class _Attn(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q_proj = torch.nn.Linear(8, 8, bias = False)
+
+ class _Layer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.self_attn = _Attn()
+ self.block_sparse_moe = _Moe()
+
+ class _Body(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([_Layer() for _ in range(2)])
+
+ class _Mixtral(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(num_local_experts = 4, model_type = "mixtral")
+ self.model = _Body()
+
+ def get_decoder(self):
+ return self.model
+
+ return _Mixtral()
+
+
+def test_explicit_leaf_list_keeps_mixtral_names_and_the_gate_up_shorthand():
+ U = _utils()
+ model = _native_mixtral_4x_model()
+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]
+ leaves = ["q_proj", "w1", "w2", "w3"]
+ regex = _explicit_list_regex(model, leaves)
+ widened, _, found = U.widen_target_regex_to_expert_submodules(
+ model, regex, leaves, auto_regex = True, explicit_leaves = True
+ )
+ assert found == ["w1", "w2", "w3"]
+ assert {n for n in names if re.fullmatch(widened, n)} == {n for n in names if n.rpartition(".")[2] in leaves}
+
+ model = _native_qwen3_moe_4x_model()
+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]
+ regex = _explicit_list_regex(model, ["q_proj", "gate_up_proj"])
+ widened, _, found = U.widen_target_regex_to_expert_submodules(
+ model, regex, ["q_proj", "gate_up_proj"], auto_regex = True, explicit_leaves = True
+ )
+ assert found == ["gate_proj", "up_proj"]
+ matched = {n for n in names if re.fullmatch(widened, n) and ".experts." in n}
+ assert matched == {n for n in names if ".experts." in n and n.endswith((".gate_proj", ".up_proj"))}
+
+
+def test_expert_only_mixtral_list_falls_back_to_the_decoders_experts():
+ """[w1, w2, w3] alone: Zoo's regex matches nothing (no MLP tag), plain PEFT reaches every expert."""
+ U = _utils()
+ model = _native_mixtral_4x_model()
+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]
+ with pytest.raises(RuntimeError):
+ _explicit_list_regex(model, ["w1", "w2", "w3"])
+ regex = U.native_expert_only_regex(model, ["w1", "w2", "w3"])
+ assert {n for n in names if re.fullmatch(regex, n)} == {n for n in names if n.endswith((".w1", ".w2", ".w3"))}
+ assert U.native_expert_only_regex(model, ["q_proj"]) is None
+ regex = U.native_expert_only_regex(model, ["w1", "w2", "w3", "gate"])
+ want = {n for n in names if n.endswith((".w1", ".w2", ".w3", ".gate"))}
+ assert any(n.endswith(".block_sparse_moe.gate") for n in want)
+ assert {n for n in names if re.fullmatch(regex, n)} == want
+ # Zoo already dropped the other families (finetune_attention_modules=False).
+ regex = U.native_expert_only_regex(model, ["q_proj", "w1", "w2", "w3"])
+ assert {n for n in names if re.fullmatch(regex, n)} == {n for n in names if n.endswith((".w1", ".w2", ".w3"))}
+ model = _native_qwen3_moe_4x_model()
+ model.get_decoder = lambda: model.model
+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]
+ regex = U.native_expert_only_regex(model, ["gate_up_proj"])
+ assert {n for n in names if re.fullmatch(regex, n)} == {
+ n for n in names if ".experts." in n and n.endswith((".gate_proj", ".up_proj"))
+ }
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index b3ad0d2021e..80e94a425ef 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -119,6 +119,7 @@ __all__ = [
"get_moe_expert_submodule_leaves",
"moe_expert_submodule_regex",
"widen_target_regex_to_expert_submodules",
+ "native_expert_only_regex",
"warn_if_zoo_cannot_merge_moe_experts",
"_select_moe_detection_targets",
"EMBEDDING_MODULES",
@@ -9208,17 +9209,115 @@ def _remote_expert_parents(model):
return parents
+def _native_expert_parents(model, target_modules, requested):
+ """(parents, requested leaves) of native ``experts.<i>`` blocks inside the regex's language scope."""
+ try:
+ pattern = re.compile(target_modules)
+ except re.error:
+ return set(), set()
+ modules = list(model.named_modules())
+ reached_layers = {
+ name.rsplit(".", 2)[0]
+ for name, _ in modules
+ if ".experts." not in name and pattern.fullmatch(name)
+ }
+ in_scope, parents, leaves = {}, set(), set()
+ for name, module in modules:
+ matched = _EXPERT_SUBMODULE_PATTERN.search(name)
+ if matched is None or not name[matched.start() :].lstrip(".").startswith("experts."):
+ continue
+ parent, leaf = name[: matched.start()], matched.group(1)
+ if not parent:
+ continue
+ key = (parent, leaf)
+ if key not in in_scope:
+ layer = parent.rpartition(".")[0]
+ # Mixtral's block_sparse_moe has no MLP keyword; this keeps vision towers out.
+ in_scope[key] = any(pattern.fullmatch(f"{parent}.{t}") for t in requested | {leaf}) or bool(
+ layer and any(r == layer or r.startswith(layer + ".") for r in reached_layers)
+ )
+ if not in_scope[key]:
+ continue
+ parents.add(
+ r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in parent.split("."))
+ )
+ if leaf in requested and (
+ isinstance(module, torch.nn.Linear)
+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)
+ ):
+ leaves.add(leaf)
+ return parents, leaves
+
+
+def native_expert_only_regex(model, target_modules):
+ """Regex for the explicit leaves found in the language decoder's native ``experts.<i>`` blocks, or None."""
+ try:
+ decoder = model.get_decoder()
+ except Exception:
+ return None
+ prefix = next((name for name, module in model.named_modules() if module is decoder), None)
+ if prefix is None:
+ return None
+ requested = {t for t in target_modules if isinstance(t, str) and "." not in t}
+ if "gate_up_proj" in requested: # the shorthand reaches split gate / up experts too
+ requested |= {"gate_proj", "up_proj"}
+ def generic(path):
+ return r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in path.split("."))
+
+ linears = [
+ name
+ for name, module in model.named_modules()
+ if (not prefix or name.startswith(prefix + "."))
+ and (
+ isinstance(module, torch.nn.Linear)
+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)
+ )
+ ]
+ blocks, leaves = set(), set()
+ for name in linears:
+ matched = _EXPERT_SUBMODULE_PATTERN.search(name)
+ if matched is not None and matched.group(1) in requested:
+ if name[matched.start() :].lstrip(".").startswith("experts."):
+ blocks.add(name[: matched.start()])
+ leaves.add(matched.group(1))
+ if not leaves:
+ return None
+ parents = {generic(block) for block in blocks}
+ # Requested expert siblings (router) too, as PEFT suffix matching would.
+ others = {
+ generic(name)
+ for name in linears
+ if name.rpartition(".")[0] in blocks and name.rpartition(".")[2] in requested
+ }
+ regex = moe_expert_submodule_regex(sorted(leaves), parents)
+ return f"(?:{regex})|(?:{'|'.join(sorted(others))})" if others else regex
+
+
def widen_target_regex_to_expert_submodules(
- model, target_modules, detect_targets, auto_regex: bool
+ model,
+ target_modules,
+ detect_targets,
+ auto_regex: bool,
+ explicit_leaves: bool = False,
):
- """Widen a generated regex to ``experts.<i>.<leaf>``; a user-written regex is never widened."""
+ """Widen a generated regex to ``experts.<i>.<leaf>``; a user-written regex is never widened.
+ explicit_leaves: the caller's list went through get_peft_regex, which skips native experts
+ that plain PEFT would match, so widen those too."""
if not auto_regex or not isinstance(target_modules, str):
return target_modules, detect_targets, []
# Remote expert blocks only: widening native ones (Qwen3-MoE on 4.x) LoRAs every routed expert.
parents = _remote_expert_parents(model)
+ named_leaves = set()
+ if explicit_leaves:
+ requested = set()
+ if isinstance(detect_targets, (list, tuple, set)):
+ requested = {t for t in detect_targets if isinstance(t, str) and "." not in t}
+ native, named_leaves = _native_expert_parents(model, target_modules, requested)
+ parents = parents | native
if not parents:
return target_modules, detect_targets, []
- leaves = get_moe_expert_submodule_leaves(model, detect_targets)
+ # Requested names used as-is too, as plain PEFT does (Mixtral's w1 / w2 / w3).
+ leaves = sorted(set(get_moe_expert_submodule_leaves(model, detect_targets)) | named_leaves)
if not leaves:
return target_modules, detect_targets, []
if any(
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 93a3f1d60e7..f4111afdab1 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -4636,6 +4636,7 @@ class FastBaseModel:
# Only a regex generated here may be widened to expert submodules below.
_target_modules_auto_regex = False
+ _target_modules_from_explicit_list = False
if target_modules is None or target_modules == "all-linear":
target_modules = get_peft_regex(
model,
@@ -4669,16 +4670,26 @@ class FastBaseModel:
"finetune_(vision|language|attention|mlp) filters; adapters "
"attach only where both select."
)
- target_modules = get_peft_regex(
- model,
- finetune_vision_layers = finetune_vision_layers,
- finetune_language_layers = finetune_language_layers,
- finetune_attention_modules = finetune_attention_modules,
- finetune_mlp_modules = finetune_mlp_modules,
- target_modules = list(target_modules),
- **_audio_kwargs,
- )
+ try:
+ target_modules = get_peft_regex(
+ model,
+ finetune_vision_layers = finetune_vision_layers,
+ finetune_language_layers = finetune_language_layers,
+ finetune_attention_modules = finetune_attention_modules,
+ finetune_mlp_modules = finetune_mlp_modules,
+ target_modules = list(target_modules),
+ **_audio_kwargs,
+ )
+ except RuntimeError:
+ # Mixtral's [w1, w2, w3]: block_sparse_moe has no MLP tag, so Zoo matches nothing.
+ expert_regex = None
+ if finetune_language_layers and finetune_mlp_modules:
+ expert_regex = native_expert_only_regex(model, target_modules)
+ if expert_regex is None:
+ raise
+ target_modules = expert_regex
_target_modules_auto_regex = True
+ _target_modules_from_explicit_list = True
if hasattr(model, "vllm_engine"):
if (
@@ -4723,6 +4734,7 @@ class FastBaseModel:
target_modules,
_moe_module_detect,
auto_regex = _target_modules_auto_regex and bool(finetune_language_layers),
+ explicit_leaves = _target_modules_from_explicit_list and bool(finetune_mlp_modules),
)
)
if _expert_submodule_leaves: