Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE LoRA: widen gate_up_proj to split gate_proj / up_proj experts when down_proj already matches (#14628)

Daniel Han5f13f1271585
diff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.py
index 5cc746a70e1..cebc7ab86b5 100644
--- a/tests/test_remote_code_class_and_expert_submodules.py
+++ b/tests/test_remote_code_class_and_expert_submodules.py
@@ -808,3 +808,89 @@ def test_expert_only_mixtral_list_falls_back_to_the_decoders_experts():
assert {n for n in names if re.fullmatch(regex, n)} == {
n for n in names if ".experts." in n and n.endswith((".gate_proj", ".up_proj"))
}
+
+
+def _expert_counts(names, regex):
+ counts = {"gate_proj": 0, "up_proj": 0, "down_proj": 0}
+ for name in names:
+ leaf = name.rpartition(".")[2]
+ if ".experts." in name and leaf in counts and re.fullmatch(regex, name):
+ counts[leaf] += 1
+ return counts
+
+
+ "leaves",
+ [
+ ["gate_up_proj", "down_proj"],
+ ["q_proj", "gate_up_proj", "down_proj"],
+ ["q_proj", "o_proj", "gate_up_proj", "down_proj"],
+ ],
+)
+def test_gate_up_shorthand_with_down_proj_reaches_split_gate_and_up_experts(leaves):
+ """Zoo's regex already matches experts.<i>.down_proj; gate_proj / up_proj must still be added."""
+ U = _utils()
+ model = _native_qwen3_moe_4x_model()
+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]
+ regex = _explicit_list_regex(model, leaves)
+ widened, _, found = U.widen_target_regex_to_expert_submodules(
+ model, regex, leaves, auto_regex = True, explicit_leaves = True
+ )
+ assert _expert_counts(names, widened) == {"gate_proj": 8, "up_proj": 8, "down_proj": 8}
+ assert "gate_proj" in found and "up_proj" in found
+ assert "model.layers.0.mlp.gate" not in {n for n in names if re.fullmatch(widened, n)}
+ kept, _, found = U.widen_target_regex_to_expert_submodules(
+ model, widened, leaves, auto_regex = True, explicit_leaves = True
+ )
+ assert kept == widened and found == []
+
+
+def test_gate_up_shorthand_with_down_proj_peft_injects_real_hf_split_experts():
+ """Real tiny Qwen3-MoE (text and inside a Llava VLM) with per-expert nn.Linear layers."""
+ from transformers import CLIPVisionConfig, LlavaConfig, LlavaForConditionalGeneration
+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM
+
+ peft = pytest.importorskip("peft")
+ U = _utils()
+ text_config = dict(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ moe_intermediate_size = 64,
+ num_experts = 2,
+ num_experts_per_tok = 1,
+ max_position_embeddings = 128,
+ )
+ text = Qwen3MoeForCausalLM(Qwen3MoeConfig(**text_config))
+ if not any(
+ ".experts." in n and isinstance(m, torch.nn.Linear) for n, m in text.named_modules()
+ ):
+ pytest.skip("this transformers fuses Qwen3-MoE experts into Parameters")
+ vlm = LlavaForConditionalGeneration(
+ LlavaConfig(
+ text_config = Qwen3MoeConfig(**text_config),
+ vision_config = CLIPVisionConfig(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ image_size = 8,
+ patch_size = 2,
+ ),
+ image_token_index = 63,
+ )
+ )
+ for model in (text, vlm):
+ for leaves in (["gate_up_proj", "down_proj"], ["q_proj", "gate_up_proj", "down_proj"]):
+ regex = _explicit_list_regex(model, leaves)
+ widened, _, _ = U.widen_target_regex_to_expert_submodules(
+ model, regex, leaves, auto_regex = True, explicit_leaves = True
+ )
+ peft_model = peft.get_peft_model(model, peft.LoraConfig(r = 2, target_modules = widened))
+ lora = [n for n, m in peft_model.named_modules() if hasattr(m, "lora_A")]
+ assert _expert_counts(lora, r".*") == {"gate_proj": 2, "up_proj": 2, "down_proj": 2}
+ assert not any("vision_tower" in n for n in lora)
+ model = peft_model.unload()
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index ab721351ff7..6aaba14d895 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -10355,14 +10355,13 @@ def widen_target_regex_to_expert_submodules(
leaves = sorted(set(get_moe_expert_submodule_leaves(model, detect_targets)) | named_leaves)
if not leaves:
return target_modules, detect_targets, []
- if any(
- re.fullmatch(target_modules, name)
- for name, _ in model.named_modules()
- if ".experts." in name
- ):
+ expert_regex = moe_expert_submodule_regex(leaves, parents)
+ # Skip only if every expert matches: Zoo may reach down_proj alone, gate_proj / up_proj still need adding.
+ eligible = [name for name, _ in model.named_modules() if re.fullmatch(expert_regex, name)]
+ if eligible and all(re.fullmatch(target_modules, name) for name in eligible):
return target_modules, detect_targets, []
detect_was_the_regex = detect_targets is target_modules
- target_modules = f"(?:{target_modules})|(?:{moe_expert_submodule_regex(leaves, parents)})"
+ target_modules = f"(?:{target_modules})|(?:{expert_regex})"
if detect_was_the_regex:
detect_targets = target_modules
return target_modules, detect_targets, leaves