Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Target unfused MoE expert parameters instead of dropping them (#11014)

Daniel Hanea017322ddfb
diff --git a/tests/test_moe_lora_targets.py b/tests/test_moe_lora_targets.py
index 66cac2eb289..64abe7c2669 100644
--- a/tests/test_moe_lora_targets.py
+++ b/tests/test_moe_lora_targets.py
@@ -232,3 +232,396 @@ def test_attention_only_list_prefers_original_when_in_scope():
)
assert selected is attn_only_list
assert get_moe_target_parameters(_FakeMoeModel(), selected) is None
+
+
+def test_unfused_expert_parameters_resolve_both_leaves():
+ """NemotronH keeps the expert projections unfused as separate 3D Parameters, so
+ up_proj must resolve alongside down_proj instead of being silently dropped
+ (unsloth#4476)."""
+ import torch
+ from unsloth.models._utils import get_moe_target_parameters
+
+ class _Cfg:
+ model_type = "nemotron_h"
+ n_routed_experts = 4
+
+ class _Fake(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = _Cfg()
+ self.up = torch.nn.Parameter(torch.zeros(4, 8, 8))
+ self.down = torch.nn.Parameter(torch.zeros(4, 8, 8))
+
+ def named_parameters(self, *args, **kwargs):
+ yield "model.layers.1.mixer.experts.up_proj", self.up
+ yield "model.layers.1.mixer.experts.down_proj", self.down
+
+ got = get_moe_target_parameters(_Fake(), target_modules = ["up_proj", "down_proj"])
+ assert got == ["experts.up_proj", "experts.down_proj"], got
+
+
+# ---------------------------------------------------------------------------------------
+# Unfused expert layouts and the no-experts-resolved warning (unsloth#4476).
+#
+# Four expert layouts are in the wild and only the first two were covered before:
+# fused 3D Parameter experts.gate_up_proj / experts.down_proj (transformers 5.x)
+# per-expert Linear list experts.gate_up_projs.<i> (gpt-oss bnb-4bit)
+# unfused 3D Parameter experts.gate_proj / up_proj / down_proj (NemotronH)
+# per-expert submodule experts.<i>.gate_proj (Qwen3-MoE on transformers 4.x)
+# The third resolved only down_proj, silently. The fourth needs no help from us, so it
+# must not trip the warning added for the third.
+# ---------------------------------------------------------------------------------------
+
+ALL_MLP_LEAVES = ["gate_proj", "up_proj", "down_proj"]
+
+
+def _unfused_moe_model(leaves = ("gate_proj", "up_proj", "down_proj"), container = "mixer"):
+ """A MoE layer whose expert projections are separate 3D Parameters, as NemotronH
+ builds them. ``container`` picks the ``experts.*`` (mixer) or ``mlp.experts.*``
+ spelling, which are the two paths _resolve_moe_parameter_name chooses between."""
+ experts = torch.nn.Module()
+ for leaf in leaves:
+ experts.register_parameter(leaf, torch.nn.Parameter(torch.zeros(4, 8, 8)))
+ holder = torch.nn.Module()
+ holder.add_module("experts", experts)
+ model = torch.nn.Module()
+ model.config = SimpleNamespace(n_routed_experts = 4, model_type = "nemotron_h")
+ model.add_module(container, holder)
+ return model
+
+
+def _per_expert_submodule_moe_model(num_experts = 2):
+ """transformers 4.x Qwen3-MoE / Mixtral-with-named-projections: each expert is its own
+ submodule, so PEFT attaches LoRA by ordinary leaf name with no help from Unsloth."""
+ experts = torch.nn.ModuleList()
+ for _ in range(num_experts):
+ expert = torch.nn.Module()
+ for leaf in ALL_MLP_LEAVES:
+ expert.add_module(leaf, torch.nn.Linear(8, 8, bias = False))
+ experts.append(expert)
+ mlp = torch.nn.Module()
+ mlp.add_module("experts", experts)
+ model = torch.nn.Module()
+ model.config = SimpleNamespace(num_experts = num_experts, model_type = "qwen3_moe")
+ model.add_module("mlp", mlp)
+ return model
+
+
+def _per_expert_linear_moe_model(num_experts = 2):
+ """gpt-oss bnb-4bit: ModuleLists of Linear directly under ``experts``, which
+ get_moe_target_modules turns into gate_up_projs.<i> suffixes."""
+ experts = torch.nn.Module()
+ experts.add_module(
+ "gate_up_projs",
+ torch.nn.ModuleList([torch.nn.Linear(8, 16, bias = False) for _ in range(num_experts)]),
+ )
+ experts.add_module(
+ "down_projs",
+ torch.nn.ModuleList([torch.nn.Linear(16, 8, bias = False) for _ in range(num_experts)]),
+ )
+ mlp = torch.nn.Module()
+ mlp.add_module("experts", experts)
+ model = torch.nn.Module()
+ model.config = SimpleNamespace(num_local_experts = num_experts, model_type = "gpt_oss")
+ model.add_module("mlp", mlp)
+ return model
+
+
+def _unreachable_moe_model(num_experts = 2):
+ """transformers 4.x Mixtral: the expert projections are w1/w2/w3, which none of
+ gate_proj/up_proj/down_proj match, so the experts genuinely go untrained."""
+ experts = torch.nn.ModuleList()
+ for _ in range(num_experts):
+ expert = torch.nn.Module()
+ for leaf in ("w1", "w2", "w3"):
+ expert.add_module(leaf, torch.nn.Linear(8, 8, bias = False))
+ experts.append(expert)
+ block = torch.nn.Module()
+ block.add_module("experts", experts)
+ model = torch.nn.Module()
+ model.config = SimpleNamespace(num_local_experts = num_experts, model_type = "mixtral")
+ model.add_module("block_sparse_moe", block)
+ return model
+
+
+ "container,prefix",
+ [("mixer", "experts"), ("mlp", "mlp.experts")],
+)
+def test_unfused_experts_resolve_every_requested_leaf(container, prefix):
+ from unsloth.models._utils import get_moe_target_parameters
+ got = get_moe_target_parameters(
+ _unfused_moe_model(container = container),
+ target_modules = ALL_MLP_LEAVES,
+ )
+ assert got == [f"{prefix}.gate_proj", f"{prefix}.up_proj", f"{prefix}.down_proj"], got
+
+
+def test_unfused_experts_honour_a_down_only_request():
+ from unsloth.models._utils import get_moe_target_parameters
+ got = get_moe_target_parameters(_unfused_moe_model(), target_modules = ["down_proj"])
+ assert got == ["experts.down_proj"], got
+
+
+def test_unfused_experts_honour_a_gate_only_request():
+ from unsloth.models._utils import get_moe_target_parameters
+ got = get_moe_target_parameters(_unfused_moe_model(), target_modules = ["gate_proj"])
+ assert got == ["experts.gate_proj"], got
+
+
+def test_gate_up_proj_shorthand_expands_to_both_unfused_leaves():
+ """A caller who asks for the fused name on a model that has no fused Parameter must
+ still get both halves, not nothing."""
+ from unsloth.models._utils import get_moe_target_parameters
+
+ got = get_moe_target_parameters(_unfused_moe_model(), target_modules = ["gate_up_proj"])
+ assert got == ["experts.gate_proj", "experts.up_proj"], got
+
+
+def test_unfused_experts_skip_a_leaf_the_model_does_not_have():
+ """NemotronH as reported carries up_proj and down_proj only; asking for gate_proj too
+ must not produce a dead path for PEFT to choke on."""
+ from unsloth.models._utils import get_moe_target_parameters
+
+ model = _unfused_moe_model(leaves = ("up_proj", "down_proj"))
+ got = get_moe_target_parameters(model, target_modules = ALL_MLP_LEAVES)
+ assert got == ["experts.up_proj", "experts.down_proj"], got
+
+
+def test_unfused_experts_are_not_touched_by_an_attention_only_request():
+ from unsloth.models._utils import get_moe_target_parameters
+ got = get_moe_target_parameters(
+ _unfused_moe_model(),
+ target_modules = ["q_proj", "k_proj", "v_proj", "o_proj"],
+ )
+ assert got is None
+
+
+def test_fused_layout_is_unchanged_by_the_unfused_fallback():
+ """The fused branch must still win, and must not pick up unfused leaves as well."""
+ from unsloth.models._utils import get_moe_target_parameters
+ assert get_moe_target_parameters(_FakeMoeModel(), ALL_MLP_LEAVES) == [
+ "mlp.experts.gate_up_proj",
+ "mlp.experts.down_proj",
+ ]
+
+
+def _warnings_from(target, *args, **kwargs):
+ """Run get_moe_target_parameters and return what it logged."""
+ from unsloth.models import _utils
+
+ records = []
+ original = _utils.logger.warning
+ _utils.logger.warning = lambda message, *a, **k: records.append(str(message))
+ try:
+ result = _utils.get_moe_target_parameters(target, *args, **kwargs)
+ finally:
+ _utils.logger.warning = original
+ return result, records
+
+
+def test_warns_when_no_route_reaches_the_experts():
+ result, records = _warnings_from(_unreachable_moe_model(), ALL_MLP_LEAVES)
+ assert result is None
+ assert len(records) == 1, records
+ assert "will NOT be trained" in records[0]
+
+
+def test_does_not_warn_when_the_unfused_fallback_resolved_the_experts():
+ result, records = _warnings_from(_unfused_moe_model(), ALL_MLP_LEAVES)
+ assert result == ["experts.gate_proj", "experts.up_proj", "experts.down_proj"]
+ assert records == []
+
+
+def test_does_not_warn_when_an_ordinary_suffix_match_reaches_the_experts():
+ """transformers 4.x builds Qwen3-MoE experts as per-expert submodules. PEFT attaches
+ to them by leaf name, so a warning here would be a false alarm on a model that trains
+ its experts correctly. This is what keeps the fix identical on 4.57.6 and 5.x."""
+ result, records = _warnings_from(_per_expert_submodule_moe_model(), ALL_MLP_LEAVES)
+ assert result is None
+ assert records == []
+
+
+def test_does_not_warn_for_the_per_expert_linear_layout():
+ """gpt-oss bnb-4bit is handled by get_moe_target_modules, not by target_parameters."""
+ result, records = _warnings_from(_per_expert_linear_moe_model(), ALL_MLP_LEAVES)
+ assert result is None
+ assert records == []
+
+
+def test_does_not_warn_for_an_attention_only_request():
+ result, records = _warnings_from(
+ _unreachable_moe_model(),
+ ["q_proj", "k_proj", "v_proj", "o_proj"],
+ )
+ assert result is None
+ assert records == []
+
+
+def test_does_not_warn_for_a_non_moe_model():
+ plain = torch.nn.Module()
+ plain.config = SimpleNamespace(model_type = "llama")
+ result, records = _warnings_from(plain, ALL_MLP_LEAVES)
+ assert result is None
+ assert records == []
+
+
+def _count_calls(model, method_name):
+ calls = []
+ original = getattr(model, method_name)
+
+ def counting(*args, **kwargs):
+ calls.append(1)
+ return original(*args, **kwargs)
+
+ setattr(model, method_name, counting)
+ return calls
+
+
+def test_named_parameters_is_walked_once():
+ """Resolving the fused name, the two unfused leaves and down_proj used to be one walk
+ of named_parameters each. On a real MoE checkpoint that walk is the whole cost."""
+ from unsloth.models._utils import get_moe_target_parameters
+
+ model = _unfused_moe_model()
+ calls = _count_calls(model, "named_parameters")
+ get_moe_target_parameters(model, ALL_MLP_LEAVES)
+ assert len(calls) == 1, len(calls)
+
+
+def _count_module_attribute_calls(name):
+ """Count calls to a module level helper in unsloth.models._utils, keeping its result."""
+ from unsloth.models import _utils
+
+ calls = []
+ original = getattr(_utils, name)
+
+ def counting(*args, **kwargs):
+ calls.append(1)
+ return original(*args, **kwargs)
+
+ setattr(_utils, name, counting)
+ return calls, lambda: setattr(_utils, name, original)
+
+
+def test_supplied_module_targets_are_not_recomputed():
+ """get_peft_model computes get_moe_target_modules anyway, so passing it in means the
+ warning path costs no extra walk of the model."""
+ from unsloth.models import _utils
+
+ calls, restore = _count_module_attribute_calls("get_moe_target_modules")
+ try:
+ result = _utils.get_moe_target_parameters(
+ _unreachable_moe_model(),
+ ALL_MLP_LEAVES,
+ moe_module_targets = ["gate_up_projs.0"],
+ )
+ finally:
+ restore()
+ assert result is None
+ assert calls == [], len(calls)
+
+
+def test_module_targets_are_computed_when_the_caller_does_not_supply_them():
+ """Standalone callers (and the tests above) must keep working unchanged."""
+ from unsloth.models import _utils
+
+ calls, restore = _count_module_attribute_calls("get_moe_target_modules")
+ try:
+ result = _utils.get_moe_target_parameters(_unreachable_moe_model(), ALL_MLP_LEAVES)
+ finally:
+ restore()
+ assert result is None
+ assert len(calls) == 1, len(calls)
+
+
+def test_reachability_probe_runs_once_when_module_targets_are_empty():
+ from unsloth.models import _utils
+
+ calls, restore = _count_module_attribute_calls("_moe_experts_reachable_by_module_name")
+ try:
+ result = _utils.get_moe_target_parameters(
+ _unreachable_moe_model(),
+ ALL_MLP_LEAVES,
+ moe_module_targets = [],
+ )
+ finally:
+ restore()
+ assert result is None
+ assert len(calls) == 1, len(calls)
+
+
+def test_reachability_probe_is_skipped_when_module_targets_resolved():
+ from unsloth.models import _utils
+
+ calls, restore = _count_module_attribute_calls("_moe_experts_reachable_by_module_name")
+ try:
+ _utils.get_moe_target_parameters(
+ _per_expert_linear_moe_model(),
+ ALL_MLP_LEAVES,
+ moe_module_targets = ["gate_up_projs.0"],
+ )
+ finally:
+ restore()
+ assert calls == []
+
+
+def test_supplied_module_targets_suppress_the_warning():
+ """A per-expert Linear model whose targets the caller already resolved must stay quiet
+ even though no Parameter path matched."""
+ model = _per_expert_linear_moe_model()
+ result, records = _warnings_from(
+ model,
+ ALL_MLP_LEAVES,
+ moe_module_targets = ["gate_up_projs.0", "gate_up_projs.1"],
+ )
+ assert result is None
+ assert records == []
+
+
+def _non_mlp_submodule_moe_model(num_experts = 2):
+ """NemotronH-style per-expert submodules under ``mixer``, so a regex anchored on an
+ ``mlp`` path segment reaches none of them."""
+ experts = torch.nn.ModuleList()
+ for _ in range(num_experts):
+ expert = torch.nn.Module()
+ for leaf in ALL_MLP_LEAVES:
+ expert.add_module(leaf, torch.nn.Linear(8, 8, bias = False))
+ experts.append(expert)
+ mixer = torch.nn.Module()
+ mixer.add_module("experts", experts)
+ model = torch.nn.Module()
+ model.config = SimpleNamespace(num_experts = num_experts, model_type = "nemotron_h")
+ model.add_module("mixer", mixer)
+ return model
+
+
+def test_a_regex_that_peft_cannot_match_still_warns():
+ """A str target_modules is a regex PEFT full-matches against the whole module path, so
+ a leaf-only reachability answer suppressed the warning on the very case it exists for:
+ ``.*mlp.*down_proj`` derives {"down_proj"} but matches no ``mixer.experts.N.down_proj``."""
+ result, records = _warnings_from(_non_mlp_submodule_moe_model(), ".*mlp.*down_proj")
+ assert result is None
+ assert len(records) == 1, records
+ assert "will NOT be trained" in records[0]
+
+
+def test_a_regex_that_peft_does_match_stays_quiet():
+ """The control: the same layout with a regex that really reaches the experts."""
+ result, records = _warnings_from(_non_mlp_submodule_moe_model(), ".*experts.*down_proj")
+ assert result is None
+ assert records == []
+
+
+def test_an_uncompilable_regex_does_not_claim_reachability():
+ result, records = _warnings_from(_non_mlp_submodule_moe_model(), ".*mlp.*(down_proj")
+ assert result is None
+ assert len(records) == 1, records
+
+
+def test_a_list_target_modules_still_matches_by_suffix():
+ """PEFT matches a list by whole key or dotted suffix, which is the leaf test; the regex
+ arm must not change that."""
+ result, records = _warnings_from(_non_mlp_submodule_moe_model(), ALL_MLP_LEAVES)
+ assert result is None
+ assert records == []
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index b0b601ecdf2..d8884c7f808 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -3843,17 +3843,34 @@ def is_moe_model(model) -> bool:
return num_experts is not None and num_experts > 0
-def _resolve_moe_parameter_name(model, default_name: str, alternate_name: str) -> str:
+def _moe_parameter_names(model) -> Optional[List[str]]:
+ """Every parameter path on the model, collected ONCE. get_moe_target_parameters asks
+ about up to five candidate paths and used to walk named_parameters for each; on a large
+ MoE checkpoint that walk is the expensive part. None means the model cannot be walked,
+ which the callers treat exactly as they treated a raised exception before."""
+ if not hasattr(model, "named_parameters"):
+ return None
+ try:
+ return [name for name, _ in model.named_parameters()]
+ except Exception:
+ return None
+
+
+def _resolve_moe_parameter_name(
+ model,
+ default_name: str,
+ alternate_name: str,
+ parameter_names: Optional[List[str]] = None,
+) -> str:
"""Resolve the actual parameter path for MoE expert weights. Most Unsloth MoE models expose them under ``mlp.experts.*``; Gemma4 stores them directly under ``experts.*``. Prefers the path that exists on the loaded module."""
- if hasattr(model, "named_parameters"):
- try:
- for name, _ in model.named_parameters():
- if name == default_name or name.endswith("." + default_name):
- return default_name
- if name == alternate_name or name.endswith("." + alternate_name):
- return alternate_name
- except Exception:
- pass
+ if parameter_names is None:
+ parameter_names = _moe_parameter_names(model)
+ if parameter_names is not None:
+ for name in parameter_names:
+ if name == default_name or name.endswith("." + default_name):
+ return default_name
+ if name == alternate_name or name.endswith("." + alternate_name):
+ return alternate_name
config = getattr(model, "config", model)
model_types = {getattr(config, "model_type", None)}
@@ -3897,8 +3914,17 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]:
return set()
-def get_moe_target_parameters(model, target_modules = None) -> Optional[List[str]]:
- """target_parameters for MoE expert layers, or None for a non-MoE model. Returns the expert weight parameter paths (gate_up_proj, down_proj) for PEFT's ``target_parameters``, under whichever layout the model uses (``mlp.experts.*`` or ``experts.*``). Only MoE parameters matching *target_modules* are included: "down_proj" gives "mlp.experts.down_proj", and "gate_proj" or "up_proj" gives "mlp.experts.gate_up_proj"."""
+def get_moe_target_parameters(
+ model,
+ target_modules = None,
+ moe_module_targets = None,
+) -> Optional[List[str]]:
+ """target_parameters for MoE expert layers, or None for a non-MoE model. Returns the expert weight parameter paths (gate_up_proj, down_proj) for PEFT's ``target_parameters``, under whichever layout the model uses (``mlp.experts.*`` or ``experts.*``, fused or unfused). Only MoE parameters matching *target_modules* are included: "down_proj" gives "mlp.experts.down_proj", and "gate_proj" or "up_proj" gives "mlp.experts.gate_up_proj", or the unfused "mlp.experts.gate_proj" / "mlp.experts.up_proj" pair when the model has no fused Parameter.
+
+ *moe_module_targets* is get_moe_target_modules(model, target_modules) when the caller has
+ already computed it. It is only read to decide whether the no-experts-resolved warning
+ applies, and passing it in is what keeps this function from walking named_modules a
+ second time right before the caller does."""
if not is_moe_model(model):
return None
@@ -3929,24 +3955,48 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[str
moe_params = []
+ # One walk of named_parameters for every path this function asks about.
+ parameter_names = _moe_parameter_names(model)
+
gate_up_name = _resolve_moe_parameter_name(
model,
default_name = "mlp.experts.gate_up_proj",
alternate_name = "experts.gate_up_proj",
+ parameter_names = parameter_names,
)
down_name = _resolve_moe_parameter_name(
model,
default_name = "mlp.experts.down_proj",
alternate_name = "experts.down_proj",
+ parameter_names = parameter_names,
)
# gate_up_proj combines gate_proj and up_proj. Target only a fused expert Parameter that exists: per-expert Linear layouts (gpt-oss bnb-4bit) have none and go through get_moe_target_modules, so skip them rather than hand PEFT a dead path.
if "gate_proj" in target_set or "up_proj" in target_set or "gate_up_proj" in target_set:
- if _moe_parameter_exists(model, gate_up_name):
+ if _moe_parameter_exists(model, gate_up_name, parameter_names = parameter_names):
moe_params.append(gate_up_name)
+ else:
+ # NemotronH keeps the expert projections unfused, as separate 3D Parameters.
+ # Without this fallback every gate and up expert weight is silently dropped
+ # while down_proj still resolves, so the model trains with two thirds of each
+ # expert frozen and no error anywhere (unsloth#4476).
+ wanted_leaves = []
+ if "gate_proj" in target_set or "gate_up_proj" in target_set:
+ wanted_leaves.append("gate_proj")
+ if "up_proj" in target_set or "gate_up_proj" in target_set:
+ wanted_leaves.append("up_proj")
+ for leaf in wanted_leaves:
+ unfused_name = _resolve_moe_parameter_name(
+ model,
+ default_name = f"mlp.experts.{leaf}",
+ alternate_name = f"experts.{leaf}",
+ parameter_names = parameter_names,
+ )
+ if _moe_parameter_exists(model, unfused_name, parameter_names = parameter_names):
+ moe_params.append(unfused_name)
if "down_proj" in target_set:
- if _moe_parameter_exists(model, down_name):
+ if _moe_parameter_exists(model, down_name, parameter_names = parameter_names):
moe_params.append(down_name)
if moe_params:
@@ -3955,16 +4005,81 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[str
)
return moe_params
+ # Nothing resolved as a Parameter. That is only a problem if no other route reaches the
+ # experts either: get_moe_target_modules covers the per-expert Linear ModuleList layout
+ # (gpt-oss bnb-4bit) and an ordinary suffix match covers the per-expert submodule layout
+ # transformers 4.x uses for Qwen3-MoE and Mixtral. Warn only when all three miss, which
+ # is the silent no-experts-trained case.
+ if target_set & _MOE_BROAD_MLP_TARGETS:
+ if moe_module_targets is None:
+ moe_module_targets = get_moe_target_modules(model, target_modules)
+ if not moe_module_targets and not _moe_experts_reachable_by_module_name(
+ model,
+ target_set,
+ target_modules,
+ ):
+ logger.warning(
+ f"Unsloth: MoE model with {num_experts = } resolved no expert parameters for "
+ f"{target_modules = }. The expert weights will NOT be trained."
+ )
return None
-def _moe_parameter_exists(model, name: str) -> bool:
+def _moe_parameter_exists(
+ model,
+ name: str,
+ parameter_names: Optional[List[str]] = None,
+) -> bool:
"""True if ``name`` is an exact suffix of some parameter path on the model."""
- if not hasattr(model, "named_parameters"):
+ if parameter_names is None:
+ parameter_names = _moe_parameter_names(model)
+ if parameter_names is None:
+ return False
+ for parameter_name in parameter_names:
+ if parameter_name == name or parameter_name.endswith("." + name):
+ return True
+ return False
+
+
+def _moe_experts_reachable_by_module_name(
+ model,
+ target_set,
+ target_modules = None,
+) -> bool:
+ """True if an ordinary target_modules match already reaches the experts.
+
+ transformers 4.x builds Qwen3-MoE and Mixtral experts as per-expert submodules
+ (``mlp.experts.<i>.gate_proj``), so PEFT attaches LoRA to them by leaf name with no
+ help from us, while transformers 5.x fuses the same weights into 3D Parameters. The
+ unresolved-experts warning must stay quiet in the first case or it fires on a model
+ whose experts are being trained perfectly well.
+
+ PEFT's own two matching rules are mirrored, because the leaf name alone answers only
+ one of them. A str ``target_modules`` is a regex PEFT applies with ``re.fullmatch``
+ against the WHOLE module path (``peft.utils.other.match_target_against_key``), so
+ ``.*mlp.*down_proj`` reaches no expert on a ``mixer.experts.0.down_proj`` layout even
+ though the derived leaf set contains ``down_proj``; answering from the leaf there
+ would silence the warning on exactly the untrained-experts case it exists for. A list
+ is matched by whole key or dotted suffix, which is what the leaf test already does."""
+ if not hasattr(model, "named_modules"):
+ return False
+ pattern = None
+ if isinstance(target_modules, str):
+ try:
+ pattern = re.compile(target_modules)
+ except re.error:
+ # PEFT would raise on this pattern anyway; do not claim reachability for it.
+ return False
+ elif not target_set:
return False
try:
- for parameter_name, _ in model.named_parameters():
- if parameter_name == name or parameter_name.endswith("." + name):
+ for name, _ in model.named_modules():
+ if "experts" not in name:
+ continue
+ if pattern is not None:
+ if pattern.fullmatch(name):
+ return True
+ elif name.rpartition(".")[2] in target_set:
return True
except Exception:
return False
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index a12c1d3d34d..6fe9aa5292a 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -3404,13 +3404,19 @@ class FastLlamaModel:
# Does not get lora yet, so take the name from model, not base model.
is_classification = "Classification" in str(type(model))
+ # Per-expert Linear expert layouts (gpt-oss bnb-4bit) are Linear modules, not fused Parameters,
+ # so target them via target_modules. Resolved before the Parameter detection below so the
+ # two share one walk of the model; neither call mutates anything.
+ _moe_module_targets = get_moe_target_modules(model, target_modules)
+
# Auto-detect MoE models and populate target_parameters for expert layers.
if target_parameters is None:
- target_parameters = get_moe_target_parameters(model, target_modules)
+ target_parameters = get_moe_target_parameters(
+ model,
+ target_modules,
+ moe_module_targets = _moe_module_targets,
+ )
- # Per-expert Linear expert layouts (gpt-oss bnb-4bit) are Linear modules, not fused Parameters,
- # so target them via target_modules.
- _moe_module_targets = get_moe_target_modules(model, target_modules)
if _moe_module_targets:
_added = [t for t in _moe_module_targets if t not in final_modules]
final_modules.extend(_added)
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 6b8c46d11c5..1f6c1fbeb7d 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -2303,24 +2303,25 @@ class FastBaseModel:
loftq_config, lora_dropout, bias, init_lora_weights, model
)
- # Auto-detect MoE models and populate target_parameters for expert layers. Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an attention-only request does not train experts, but only while MLP and language families are both in scope: with finetune_mlp_modules or finetune_language_layers False the scoped regex already dropped the experts.
- if target_parameters is None:
- _moe_targets = _select_moe_detection_targets(
- _moe_detect_target,
- target_modules,
- finetune_mlp_modules = finetune_mlp_modules,
- finetune_language_layers = finetune_language_layers,
- )
- target_parameters = get_moe_target_parameters(model, _moe_targets)
-
- # Per-expert Linear layouts (gpt-oss bnb-4bit) target experts via target_modules, not fused Parameters. Extend either form PEFT accepts: a leaf list, or a regex string.
+ # Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an attention-only request does not train experts, but only while MLP and language families are both in scope: with finetune_mlp_modules or finetune_language_layers False the scoped regex already dropped the experts.
_moe_module_detect = _select_moe_detection_targets(
_moe_detect_target,
target_modules,
finetune_mlp_modules = finetune_mlp_modules,
finetune_language_layers = finetune_language_layers,
)
+
+ # Per-expert Linear layouts (gpt-oss bnb-4bit) target experts via target_modules, not fused Parameters. Extend either form PEFT accepts: a leaf list, or a regex string.
_moe_module_targets = get_moe_target_modules(model, _moe_module_detect)
+
+ # Auto-detect MoE models and populate target_parameters for expert layers.
+ if target_parameters is None:
+ target_parameters = get_moe_target_parameters(
+ model,
+ _moe_module_detect,
+ moe_module_targets = _moe_module_targets,
+ )
+
if _moe_module_targets:
if isinstance(target_modules, (list, tuple)):
target_modules = list(target_modules) + [