Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE LoRA: widen gate_up_proj to split gate_proj / up_proj experts when down_proj already matches (#14628)
Daniel Han5f13f1271585
diff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.pyindex 5cc746a70e1..cebc7ab86b5 100644--- a/tests/test_remote_code_class_and_expert_submodules.py+++ b/tests/test_remote_code_class_and_expert_submodules.py@@ -808,3 +808,89 @@ def test_expert_only_mixtral_list_falls_back_to_the_decoders_experts():assert {n for n in names if re.fullmatch(regex, n)} == {n for n in names if ".experts." in n and n.endswith((".gate_proj", ".up_proj"))}+++def _expert_counts(names, regex):+ counts = {"gate_proj": 0, "up_proj": 0, "down_proj": 0}+ for name in names:+ leaf = name.rpartition(".")[2]+ if ".experts." in name and leaf in counts and re.fullmatch(regex, name):+ counts[leaf] += 1+ return counts+++ "leaves",+ [+ ["gate_up_proj", "down_proj"],+ ["q_proj", "gate_up_proj", "down_proj"],+ ["q_proj", "o_proj", "gate_up_proj", "down_proj"],+ ],+)+def test_gate_up_shorthand_with_down_proj_reaches_split_gate_and_up_experts(leaves):+ """Zoo's regex already matches experts.<i>.down_proj; gate_proj / up_proj must still be added."""+ U = _utils()+ model = _native_qwen3_moe_4x_model()+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]+ regex = _explicit_list_regex(model, leaves)+ widened, _, found = U.widen_target_regex_to_expert_submodules(+ model, regex, leaves, auto_regex = True, explicit_leaves = True+ )+ assert _expert_counts(names, widened) == {"gate_proj": 8, "up_proj": 8, "down_proj": 8}+ assert "gate_proj" in found and "up_proj" in found+ assert "model.layers.0.mlp.gate" not in {n for n in names if re.fullmatch(widened, n)}+ kept, _, found = U.widen_target_regex_to_expert_submodules(+ model, widened, leaves, auto_regex = True, explicit_leaves = True+ )+ assert kept == widened and found == []+++def test_gate_up_shorthand_with_down_proj_peft_injects_real_hf_split_experts():+ """Real tiny Qwen3-MoE (text and inside a Llava VLM) with per-expert nn.Linear layers."""+ from transformers import CLIPVisionConfig, LlavaConfig, LlavaForConditionalGeneration+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM++ peft = pytest.importorskip("peft")+ U = _utils()+ text_config = dict(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ moe_intermediate_size = 64,+ num_experts = 2,+ num_experts_per_tok = 1,+ max_position_embeddings = 128,+ )+ text = Qwen3MoeForCausalLM(Qwen3MoeConfig(**text_config))+ if not any(+ ".experts." in n and isinstance(m, torch.nn.Linear) for n, m in text.named_modules()+ ):+ pytest.skip("this transformers fuses Qwen3-MoE experts into Parameters")+ vlm = LlavaForConditionalGeneration(+ LlavaConfig(+ text_config = Qwen3MoeConfig(**text_config),+ vision_config = CLIPVisionConfig(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ image_size = 8,+ patch_size = 2,+ ),+ image_token_index = 63,+ )+ )+ for model in (text, vlm):+ for leaves in (["gate_up_proj", "down_proj"], ["q_proj", "gate_up_proj", "down_proj"]):+ regex = _explicit_list_regex(model, leaves)+ widened, _, _ = U.widen_target_regex_to_expert_submodules(+ model, regex, leaves, auto_regex = True, explicit_leaves = True+ )+ peft_model = peft.get_peft_model(model, peft.LoraConfig(r = 2, target_modules = widened))+ lora = [n for n, m in peft_model.named_modules() if hasattr(m, "lora_A")]+ assert _expert_counts(lora, r".*") == {"gate_proj": 2, "up_proj": 2, "down_proj": 2}+ assert not any("vision_tower" in n for n in lora)+ model = peft_model.unload()diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex ab721351ff7..6aaba14d895 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -10355,14 +10355,13 @@ def widen_target_regex_to_expert_submodules(leaves = sorted(set(get_moe_expert_submodule_leaves(model, detect_targets)) | named_leaves)if not leaves:return target_modules, detect_targets, []- if any(- re.fullmatch(target_modules, name)- for name, _ in model.named_modules()- if ".experts." in name- ):+ expert_regex = moe_expert_submodule_regex(leaves, parents)+ # Skip only if every expert matches: Zoo may reach down_proj alone, gate_proj / up_proj still need adding.+ eligible = [name for name, _ in model.named_modules() if re.fullmatch(expert_regex, name)]+ if eligible and all(re.fullmatch(target_modules, name) for name in eligible):return target_modules, detect_targets, []detect_was_the_regex = detect_targets is target_modules- target_modules = f"(?:{target_modules})|(?:{moe_expert_submodule_regex(leaves, parents)})"+ target_modules = f"(?:{target_modules})|(?:{expert_regex})"if detect_was_the_regex:detect_targets = target_modulesreturn target_modules, detect_targets, leaves