Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE LoRA targets: keep a requested router sibling when attention modules are also listed (#14745)

Daniel Han62a2ec2c09f5
diff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.py
index cebc7ab86b5..c2fcaa331b6 100644
--- a/tests/test_remote_code_class_and_expert_submodules.py
+++ b/tests/test_remote_code_class_and_expert_submodules.py
@@ -894,3 +894,64 @@ def test_gate_up_shorthand_with_down_proj_peft_injects_real_hf_split_experts():
assert _expert_counts(lora, r".*") == {"gate_proj": 2, "up_proj": 2, "down_proj": 2}
assert not any("vision_tower" in n for n in lora)
model = peft_model.unload()
+
+
+ "leaves", [["w1", "w2", "w3", "gate"], ["q_proj", "w1", "w2", "w3", "gate"]]
+)
+def test_requested_router_sibling_is_kept_when_attention_is_also_listed(leaves):
+ """q_proj forces the widen path, which must still keep a requested router."""
+ U = _utils()
+ model = _native_mixtral_4x_model()
+ model.get_decoder = lambda: model.model
+ names = [n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)]
+ try:
+ regex = _explicit_list_regex(model, leaves)
+ except RuntimeError:
+ regex = U.native_expert_only_regex(model, leaves)
+ widened, _, _ = U.widen_target_regex_to_expert_submodules(
+ model, regex, leaves, auto_regex = True, explicit_leaves = True
+ )
+ matched = {n for n in names if re.fullmatch(widened, n)}
+ assert matched == {n for n in names if n.rpartition(".")[2] in leaves}
+ unrequested = [leaf for leaf in leaves if leaf != "gate"]
+ regex = _explicit_list_regex(model, unrequested) if "q_proj" in leaves else None
+ if regex is not None:
+ widened, _, _ = U.widen_target_regex_to_expert_submodules(
+ model, regex, unrequested, auto_regex = True, explicit_leaves = True
+ )
+ assert not any(n.endswith(".block_sparse_moe.gate") and re.fullmatch(widened, n) for n in names)
+
+
+def test_requested_router_sibling_peft_injects_real_hf_mixtral():
+ from transformers import MixtralConfig, MixtralForCausalLM
+
+ peft = pytest.importorskip("peft")
+ U = _utils()
+ config = MixtralConfig(
+ vocab_size = 32,
+ hidden_size = 8,
+ intermediate_size = 16,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ num_local_experts = 4,
+ num_experts_per_tok = 2,
+ )
+ model = MixtralForCausalLM(config)
+ if not any(n.endswith(".experts.0.w1") for n, _ in model.named_modules()):
+ pytest.skip("this transformers fuses Mixtral experts into Parameters")
+ for leaves in (["w1", "w2", "w3", "gate"], ["q_proj", "w1", "w2", "w3", "gate"]):
+ try:
+ regex = _explicit_list_regex(model, leaves)
+ except RuntimeError:
+ regex = U.native_expert_only_regex(model, leaves)
+ widened, _, _ = U.widen_target_regex_to_expert_submodules(
+ model, regex, leaves, auto_regex = True, explicit_leaves = True
+ )
+ peft_model = peft.get_peft_model(model, peft.LoraConfig(r = 2, target_modules = widened))
+ lora = [n for n, m in peft_model.named_modules() if hasattr(m, "lora_A")]
+ assert sum(".experts." in n for n in lora) == 24
+ assert sum(n.endswith(".block_sparse_moe.gate") for n in lora) == 2
+ assert sum(n.endswith(".q_proj") for n in lora) == (2 if "q_proj" in leaves else 0)
+ model = peft_model.unload()
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index b2816779774..4e3af86f9ea 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -10370,13 +10370,21 @@ def widen_target_regex_to_expert_submodules(
return target_modules, detect_targets, []
# Remote expert blocks only: widening native ones (Qwen3-MoE on 4.x) LoRAs every routed expert.
parents = _remote_expert_parents(model)
- named_leaves = set()
+ named_leaves, siblings = set(), set()
if explicit_leaves:
requested = set()
if isinstance(detect_targets, (list, tuple, set)):
requested = {t for t in detect_targets if isinstance(t, str) and "." not in t}
native, named_leaves = _native_expert_parents(model, target_modules, requested)
parents = parents | native
+ # Requested expert siblings (Mixtral's router gate), as PEFT suffix matching would.
+ for name, module in model.named_modules():
+ parent, _, leaf = name.rpartition(".")
+ if leaf in requested and any(re.fullmatch(p, parent) for p in native) and (
+ isinstance(module, torch.nn.Linear)
+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)
+ ):
+ siblings.add(r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in name.split(".")))
if not parents:
return target_modules, detect_targets, []
# Requested names used as-is too, as plain PEFT does (Mixtral's w1 / w2 / w3).
@@ -10384,6 +10392,8 @@ def widen_target_regex_to_expert_submodules(
if not leaves:
return target_modules, detect_targets, []
expert_regex = moe_expert_submodule_regex(leaves, parents)
+ if siblings:
+ expert_regex = f"(?:{expert_regex})|(?:{'|'.join(sorted(siblings))})"
# Skip only if every expert matches: Zoo may reach down_proj alone, gate_proj / up_proj still need adding.
eligible = [name for name, _ in model.named_modules() if re.fullmatch(expert_regex, name)]
if eligible and all(re.fullmatch(target_modules, name) for name in eligible):