Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE LoRA targets: keep a requested router sibling when attention modules are also listed (#14745)
Daniel Han62a2ec2c09f5
diff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.pyindex cebc7ab86b5..c2fcaa331b6 100644--- a/tests/test_remote_code_class_and_expert_submodules.py+++ b/tests/test_remote_code_class_and_expert_submodules.py@@ -894,3 +894,64 @@ def test_gate_up_shorthand_with_down_proj_peft_injects_real_hf_split_experts():assert _expert_counts(lora, r".*") == {"gate_proj": 2, "up_proj": 2, "down_proj": 2}assert not any("vision_tower" in n for n in lora)model = peft_model.unload()+++ "leaves", [["w1", "w2", "w3", "gate"], ["q_proj", "w1", "w2", "w3", "gate"]]+)+def test_requested_router_sibling_is_kept_when_attention_is_also_listed(leaves):+ """q_proj forces the widen path, which must still keep a requested router."""+ U = _utils()+ model = _native_mixtral_4x_model()+ model.get_decoder = lambda: model.model+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]+ try:+ regex = _explicit_list_regex(model, leaves)+ except RuntimeError:+ regex = U.native_expert_only_regex(model, leaves)+ widened, _, _ = U.widen_target_regex_to_expert_submodules(+ model, regex, leaves, auto_regex = True, explicit_leaves = True+ )+ matched = {n for n in names if re.fullmatch(widened, n)}+ assert matched == {n for n in names if n.rpartition(".")[2] in leaves}+ unrequested = [leaf for leaf in leaves if leaf != "gate"]+ regex = _explicit_list_regex(model, unrequested) if "q_proj" in leaves else None+ if regex is not None:+ widened, _, _ = U.widen_target_regex_to_expert_submodules(+ model, regex, unrequested, auto_regex = True, explicit_leaves = True+ )+ assert not any(n.endswith(".block_sparse_moe.gate") and re.fullmatch(widened, n) for n in names)+++def test_requested_router_sibling_peft_injects_real_hf_mixtral():+ from transformers import MixtralConfig, MixtralForCausalLM++ peft = pytest.importorskip("peft")+ U = _utils()+ config = MixtralConfig(+ vocab_size = 32,+ hidden_size = 8,+ intermediate_size = 16,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ num_local_experts = 4,+ num_experts_per_tok = 2,+ )+ model = MixtralForCausalLM(config)+ if not any(n.endswith(".experts.0.w1") for n, _ in model.named_modules()):+ pytest.skip("this transformers fuses Mixtral experts into Parameters")+ for leaves in (["w1", "w2", "w3", "gate"], ["q_proj", "w1", "w2", "w3", "gate"]):+ try:+ regex = _explicit_list_regex(model, leaves)+ except RuntimeError:+ regex = U.native_expert_only_regex(model, leaves)+ widened, _, _ = U.widen_target_regex_to_expert_submodules(+ model, regex, leaves, auto_regex = True, explicit_leaves = True+ )+ peft_model = peft.get_peft_model(model, peft.LoraConfig(r = 2, target_modules = widened))+ lora = [n for n, m in peft_model.named_modules() if hasattr(m, "lora_A")]+ assert sum(".experts." in n for n in lora) == 24+ assert sum(n.endswith(".block_sparse_moe.gate") for n in lora) == 2+ assert sum(n.endswith(".q_proj") for n in lora) == (2 if "q_proj" in leaves else 0)+ model = peft_model.unload()diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex b2816779774..4e3af86f9ea 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -10370,13 +10370,21 @@ def widen_target_regex_to_expert_submodules(return target_modules, detect_targets, []# Remote expert blocks only: widening native ones (Qwen3-MoE on 4.x) LoRAs every routed expert.parents = _remote_expert_parents(model)- named_leaves = set()+ named_leaves, siblings = set(), set()if explicit_leaves:requested = set()if isinstance(detect_targets, (list, tuple, set)):requested = {t for t in detect_targets if isinstance(t, str) and "." not in t}native, named_leaves = _native_expert_parents(model, target_modules, requested)parents = parents | native+ # Requested expert siblings (Mixtral's router gate), as PEFT suffix matching would.+ for name, module in model.named_modules():+ parent, _, leaf = name.rpartition(".")+ if leaf in requested and any(re.fullmatch(p, parent) for p in native) and (+ isinstance(module, torch.nn.Linear)+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)+ ):+ siblings.add(r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in name.split(".")))if not parents:return target_modules, detect_targets, []# Requested names used as-is too, as plain PEFT does (Mixtral's w1 / w2 / w3).@@ -10384,6 +10392,8 @@ def widen_target_regex_to_expert_submodules(if not leaves:return target_modules, detect_targets, []expert_regex = moe_expert_submodule_regex(leaves, parents)+ if siblings:+ expert_regex = f"(?:{expert_regex})|(?:{'|'.join(sorted(siblings))})"# Skip only if every expert matches: Zoo may reach down_proj alone, gate_proj / up_proj still need adding.eligible = [name for name, _ in model.named_modules() if re.fullmatch(expert_regex, name)]if eligible and all(re.fullmatch(target_modules, name) for name in eligible):