Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Target unfused MoE expert parameters instead of dropping them (#11014)
Daniel Hanea017322ddfb
diff --git a/tests/test_moe_lora_targets.py b/tests/test_moe_lora_targets.pyindex 66cac2eb289..64abe7c2669 100644--- a/tests/test_moe_lora_targets.py+++ b/tests/test_moe_lora_targets.py@@ -232,3 +232,396 @@ def test_attention_only_list_prefers_original_when_in_scope():)assert selected is attn_only_listassert get_moe_target_parameters(_FakeMoeModel(), selected) is None+++def test_unfused_expert_parameters_resolve_both_leaves():+ """NemotronH keeps the expert projections unfused as separate 3D Parameters, so+ up_proj must resolve alongside down_proj instead of being silently dropped+ (unsloth#4476)."""+ import torch+ from unsloth.models._utils import get_moe_target_parameters++ class _Cfg:+ model_type = "nemotron_h"+ n_routed_experts = 4++ class _Fake(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = _Cfg()+ self.up = torch.nn.Parameter(torch.zeros(4, 8, 8))+ self.down = torch.nn.Parameter(torch.zeros(4, 8, 8))++ def named_parameters(self, *args, **kwargs):+ yield "model.layers.1.mixer.experts.up_proj", self.up+ yield "model.layers.1.mixer.experts.down_proj", self.down++ got = get_moe_target_parameters(_Fake(), target_modules = ["up_proj", "down_proj"])+ assert got == ["experts.up_proj", "experts.down_proj"], got+++# ---------------------------------------------------------------------------------------+# Unfused expert layouts and the no-experts-resolved warning (unsloth#4476).+#+# Four expert layouts are in the wild and only the first two were covered before:+# fused 3D Parameter experts.gate_up_proj / experts.down_proj (transformers 5.x)+# per-expert Linear list experts.gate_up_projs.<i> (gpt-oss bnb-4bit)+# unfused 3D Parameter experts.gate_proj / up_proj / down_proj (NemotronH)+# per-expert submodule experts.<i>.gate_proj (Qwen3-MoE on transformers 4.x)+# The third resolved only down_proj, silently. The fourth needs no help from us, so it+# must not trip the warning added for the third.+# ---------------------------------------------------------------------------------------++ALL_MLP_LEAVES = ["gate_proj", "up_proj", "down_proj"]+++def _unfused_moe_model(leaves = ("gate_proj", "up_proj", "down_proj"), container = "mixer"):+ """A MoE layer whose expert projections are separate 3D Parameters, as NemotronH+ builds them. ``container`` picks the ``experts.*`` (mixer) or ``mlp.experts.*``+ spelling, which are the two paths _resolve_moe_parameter_name chooses between."""+ experts = torch.nn.Module()+ for leaf in leaves:+ experts.register_parameter(leaf, torch.nn.Parameter(torch.zeros(4, 8, 8)))+ holder = torch.nn.Module()+ holder.add_module("experts", experts)+ model = torch.nn.Module()+ model.config = SimpleNamespace(n_routed_experts = 4, model_type = "nemotron_h")+ model.add_module(container, holder)+ return model+++def _per_expert_submodule_moe_model(num_experts = 2):+ """transformers 4.x Qwen3-MoE / Mixtral-with-named-projections: each expert is its own+ submodule, so PEFT attaches LoRA by ordinary leaf name with no help from Unsloth."""+ experts = torch.nn.ModuleList()+ for _ in range(num_experts):+ expert = torch.nn.Module()+ for leaf in ALL_MLP_LEAVES:+ expert.add_module(leaf, torch.nn.Linear(8, 8, bias = False))+ experts.append(expert)+ mlp = torch.nn.Module()+ mlp.add_module("experts", experts)+ model = torch.nn.Module()+ model.config = SimpleNamespace(num_experts = num_experts, model_type = "qwen3_moe")+ model.add_module("mlp", mlp)+ return model+++def _per_expert_linear_moe_model(num_experts = 2):+ """gpt-oss bnb-4bit: ModuleLists of Linear directly under ``experts``, which+ get_moe_target_modules turns into gate_up_projs.<i> suffixes."""+ experts = torch.nn.Module()+ experts.add_module(+ "gate_up_projs",+ torch.nn.ModuleList([torch.nn.Linear(8, 16, bias = False) for _ in range(num_experts)]),+ )+ experts.add_module(+ "down_projs",+ torch.nn.ModuleList([torch.nn.Linear(16, 8, bias = False) for _ in range(num_experts)]),+ )+ mlp = torch.nn.Module()+ mlp.add_module("experts", experts)+ model = torch.nn.Module()+ model.config = SimpleNamespace(num_local_experts = num_experts, model_type = "gpt_oss")+ model.add_module("mlp", mlp)+ return model+++def _unreachable_moe_model(num_experts = 2):+ """transformers 4.x Mixtral: the expert projections are w1/w2/w3, which none of+ gate_proj/up_proj/down_proj match, so the experts genuinely go untrained."""+ experts = torch.nn.ModuleList()+ for _ in range(num_experts):+ expert = torch.nn.Module()+ for leaf in ("w1", "w2", "w3"):+ expert.add_module(leaf, torch.nn.Linear(8, 8, bias = False))+ experts.append(expert)+ block = torch.nn.Module()+ block.add_module("experts", experts)+ model = torch.nn.Module()+ model.config = SimpleNamespace(num_local_experts = num_experts, model_type = "mixtral")+ model.add_module("block_sparse_moe", block)+ return model+++ "container,prefix",+ [("mixer", "experts"), ("mlp", "mlp.experts")],+)+def test_unfused_experts_resolve_every_requested_leaf(container, prefix):+ from unsloth.models._utils import get_moe_target_parameters+ got = get_moe_target_parameters(+ _unfused_moe_model(container = container),+ target_modules = ALL_MLP_LEAVES,+ )+ assert got == [f"{prefix}.gate_proj", f"{prefix}.up_proj", f"{prefix}.down_proj"], got+++def test_unfused_experts_honour_a_down_only_request():+ from unsloth.models._utils import get_moe_target_parameters+ got = get_moe_target_parameters(_unfused_moe_model(), target_modules = ["down_proj"])+ assert got == ["experts.down_proj"], got+++def test_unfused_experts_honour_a_gate_only_request():+ from unsloth.models._utils import get_moe_target_parameters+ got = get_moe_target_parameters(_unfused_moe_model(), target_modules = ["gate_proj"])+ assert got == ["experts.gate_proj"], got+++def test_gate_up_proj_shorthand_expands_to_both_unfused_leaves():+ """A caller who asks for the fused name on a model that has no fused Parameter must+ still get both halves, not nothing."""+ from unsloth.models._utils import get_moe_target_parameters++ got = get_moe_target_parameters(_unfused_moe_model(), target_modules = ["gate_up_proj"])+ assert got == ["experts.gate_proj", "experts.up_proj"], got+++def test_unfused_experts_skip_a_leaf_the_model_does_not_have():+ """NemotronH as reported carries up_proj and down_proj only; asking for gate_proj too+ must not produce a dead path for PEFT to choke on."""+ from unsloth.models._utils import get_moe_target_parameters++ model = _unfused_moe_model(leaves = ("up_proj", "down_proj"))+ got = get_moe_target_parameters(model, target_modules = ALL_MLP_LEAVES)+ assert got == ["experts.up_proj", "experts.down_proj"], got+++def test_unfused_experts_are_not_touched_by_an_attention_only_request():+ from unsloth.models._utils import get_moe_target_parameters+ got = get_moe_target_parameters(+ _unfused_moe_model(),+ target_modules = ["q_proj", "k_proj", "v_proj", "o_proj"],+ )+ assert got is None+++def test_fused_layout_is_unchanged_by_the_unfused_fallback():+ """The fused branch must still win, and must not pick up unfused leaves as well."""+ from unsloth.models._utils import get_moe_target_parameters+ assert get_moe_target_parameters(_FakeMoeModel(), ALL_MLP_LEAVES) == [+ "mlp.experts.gate_up_proj",+ "mlp.experts.down_proj",+ ]+++def _warnings_from(target, *args, **kwargs):+ """Run get_moe_target_parameters and return what it logged."""+ from unsloth.models import _utils++ records = []+ original = _utils.logger.warning+ _utils.logger.warning = lambda message, *a, **k: records.append(str(message))+ try:+ result = _utils.get_moe_target_parameters(target, *args, **kwargs)+ finally:+ _utils.logger.warning = original+ return result, records+++def test_warns_when_no_route_reaches_the_experts():+ result, records = _warnings_from(_unreachable_moe_model(), ALL_MLP_LEAVES)+ assert result is None+ assert len(records) == 1, records+ assert "will NOT be trained" in records[0]+++def test_does_not_warn_when_the_unfused_fallback_resolved_the_experts():+ result, records = _warnings_from(_unfused_moe_model(), ALL_MLP_LEAVES)+ assert result == ["experts.gate_proj", "experts.up_proj", "experts.down_proj"]+ assert records == []+++def test_does_not_warn_when_an_ordinary_suffix_match_reaches_the_experts():+ """transformers 4.x builds Qwen3-MoE experts as per-expert submodules. PEFT attaches+ to them by leaf name, so a warning here would be a false alarm on a model that trains+ its experts correctly. This is what keeps the fix identical on 4.57.6 and 5.x."""+ result, records = _warnings_from(_per_expert_submodule_moe_model(), ALL_MLP_LEAVES)+ assert result is None+ assert records == []+++def test_does_not_warn_for_the_per_expert_linear_layout():+ """gpt-oss bnb-4bit is handled by get_moe_target_modules, not by target_parameters."""+ result, records = _warnings_from(_per_expert_linear_moe_model(), ALL_MLP_LEAVES)+ assert result is None+ assert records == []+++def test_does_not_warn_for_an_attention_only_request():+ result, records = _warnings_from(+ _unreachable_moe_model(),+ ["q_proj", "k_proj", "v_proj", "o_proj"],+ )+ assert result is None+ assert records == []+++def test_does_not_warn_for_a_non_moe_model():+ plain = torch.nn.Module()+ plain.config = SimpleNamespace(model_type = "llama")+ result, records = _warnings_from(plain, ALL_MLP_LEAVES)+ assert result is None+ assert records == []+++def _count_calls(model, method_name):+ calls = []+ original = getattr(model, method_name)++ def counting(*args, **kwargs):+ calls.append(1)+ return original(*args, **kwargs)++ setattr(model, method_name, counting)+ return calls+++def test_named_parameters_is_walked_once():+ """Resolving the fused name, the two unfused leaves and down_proj used to be one walk+ of named_parameters each. On a real MoE checkpoint that walk is the whole cost."""+ from unsloth.models._utils import get_moe_target_parameters++ model = _unfused_moe_model()+ calls = _count_calls(model, "named_parameters")+ get_moe_target_parameters(model, ALL_MLP_LEAVES)+ assert len(calls) == 1, len(calls)+++def _count_module_attribute_calls(name):+ """Count calls to a module level helper in unsloth.models._utils, keeping its result."""+ from unsloth.models import _utils++ calls = []+ original = getattr(_utils, name)++ def counting(*args, **kwargs):+ calls.append(1)+ return original(*args, **kwargs)++ setattr(_utils, name, counting)+ return calls, lambda: setattr(_utils, name, original)+++def test_supplied_module_targets_are_not_recomputed():+ """get_peft_model computes get_moe_target_modules anyway, so passing it in means the+ warning path costs no extra walk of the model."""+ from unsloth.models import _utils++ calls, restore = _count_module_attribute_calls("get_moe_target_modules")+ try:+ result = _utils.get_moe_target_parameters(+ _unreachable_moe_model(),+ ALL_MLP_LEAVES,+ moe_module_targets = ["gate_up_projs.0"],+ )+ finally:+ restore()+ assert result is None+ assert calls == [], len(calls)+++def test_module_targets_are_computed_when_the_caller_does_not_supply_them():+ """Standalone callers (and the tests above) must keep working unchanged."""+ from unsloth.models import _utils++ calls, restore = _count_module_attribute_calls("get_moe_target_modules")+ try:+ result = _utils.get_moe_target_parameters(_unreachable_moe_model(), ALL_MLP_LEAVES)+ finally:+ restore()+ assert result is None+ assert len(calls) == 1, len(calls)+++def test_reachability_probe_runs_once_when_module_targets_are_empty():+ from unsloth.models import _utils++ calls, restore = _count_module_attribute_calls("_moe_experts_reachable_by_module_name")+ try:+ result = _utils.get_moe_target_parameters(+ _unreachable_moe_model(),+ ALL_MLP_LEAVES,+ moe_module_targets = [],+ )+ finally:+ restore()+ assert result is None+ assert len(calls) == 1, len(calls)+++def test_reachability_probe_is_skipped_when_module_targets_resolved():+ from unsloth.models import _utils++ calls, restore = _count_module_attribute_calls("_moe_experts_reachable_by_module_name")+ try:+ _utils.get_moe_target_parameters(+ _per_expert_linear_moe_model(),+ ALL_MLP_LEAVES,+ moe_module_targets = ["gate_up_projs.0"],+ )+ finally:+ restore()+ assert calls == []+++def test_supplied_module_targets_suppress_the_warning():+ """A per-expert Linear model whose targets the caller already resolved must stay quiet+ even though no Parameter path matched."""+ model = _per_expert_linear_moe_model()+ result, records = _warnings_from(+ model,+ ALL_MLP_LEAVES,+ moe_module_targets = ["gate_up_projs.0", "gate_up_projs.1"],+ )+ assert result is None+ assert records == []+++def _non_mlp_submodule_moe_model(num_experts = 2):+ """NemotronH-style per-expert submodules under ``mixer``, so a regex anchored on an+ ``mlp`` path segment reaches none of them."""+ experts = torch.nn.ModuleList()+ for _ in range(num_experts):+ expert = torch.nn.Module()+ for leaf in ALL_MLP_LEAVES:+ expert.add_module(leaf, torch.nn.Linear(8, 8, bias = False))+ experts.append(expert)+ mixer = torch.nn.Module()+ mixer.add_module("experts", experts)+ model = torch.nn.Module()+ model.config = SimpleNamespace(num_experts = num_experts, model_type = "nemotron_h")+ model.add_module("mixer", mixer)+ return model+++def test_a_regex_that_peft_cannot_match_still_warns():+ """A str target_modules is a regex PEFT full-matches against the whole module path, so+ a leaf-only reachability answer suppressed the warning on the very case it exists for:+ ``.*mlp.*down_proj`` derives {"down_proj"} but matches no ``mixer.experts.N.down_proj``."""+ result, records = _warnings_from(_non_mlp_submodule_moe_model(), ".*mlp.*down_proj")+ assert result is None+ assert len(records) == 1, records+ assert "will NOT be trained" in records[0]+++def test_a_regex_that_peft_does_match_stays_quiet():+ """The control: the same layout with a regex that really reaches the experts."""+ result, records = _warnings_from(_non_mlp_submodule_moe_model(), ".*experts.*down_proj")+ assert result is None+ assert records == []+++def test_an_uncompilable_regex_does_not_claim_reachability():+ result, records = _warnings_from(_non_mlp_submodule_moe_model(), ".*mlp.*(down_proj")+ assert result is None+ assert len(records) == 1, records+++def test_a_list_target_modules_still_matches_by_suffix():+ """PEFT matches a list by whole key or dotted suffix, which is the leaf test; the regex+ arm must not change that."""+ result, records = _warnings_from(_non_mlp_submodule_moe_model(), ALL_MLP_LEAVES)+ assert result is None+ assert records == []diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex b0b601ecdf2..d8884c7f808 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -3843,17 +3843,34 @@ def is_moe_model(model) -> bool:return num_experts is not None and num_experts > 0-def _resolve_moe_parameter_name(model, default_name: str, alternate_name: str) -> str:+def _moe_parameter_names(model) -> Optional[List[str]]:+ """Every parameter path on the model, collected ONCE. get_moe_target_parameters asks+ about up to five candidate paths and used to walk named_parameters for each; on a large+ MoE checkpoint that walk is the expensive part. None means the model cannot be walked,+ which the callers treat exactly as they treated a raised exception before."""+ if not hasattr(model, "named_parameters"):+ return None+ try:+ return [name for name, _ in model.named_parameters()]+ except Exception:+ return None+++def _resolve_moe_parameter_name(+ model,+ default_name: str,+ alternate_name: str,+ parameter_names: Optional[List[str]] = None,+) -> str:"""Resolve the actual parameter path for MoE expert weights. Most Unsloth MoE models expose them under ``mlp.experts.*``; Gemma4 stores them directly under ``experts.*``. Prefers the path that exists on the loaded module."""- if hasattr(model, "named_parameters"):- try:- for name, _ in model.named_parameters():- if name == default_name or name.endswith("." + default_name):- return default_name- if name == alternate_name or name.endswith("." + alternate_name):- return alternate_name- except Exception:- pass+ if parameter_names is None:+ parameter_names = _moe_parameter_names(model)+ if parameter_names is not None:+ for name in parameter_names:+ if name == default_name or name.endswith("." + default_name):+ return default_name+ if name == alternate_name or name.endswith("." + alternate_name):+ return alternate_nameconfig = getattr(model, "config", model)model_types = {getattr(config, "model_type", None)}@@ -3897,8 +3914,17 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]:return set()-def get_moe_target_parameters(model, target_modules = None) -> Optional[List[str]]:- """target_parameters for MoE expert layers, or None for a non-MoE model. Returns the expert weight parameter paths (gate_up_proj, down_proj) for PEFT's ``target_parameters``, under whichever layout the model uses (``mlp.experts.*`` or ``experts.*``). Only MoE parameters matching *target_modules* are included: "down_proj" gives "mlp.experts.down_proj", and "gate_proj" or "up_proj" gives "mlp.experts.gate_up_proj"."""+def get_moe_target_parameters(+ model,+ target_modules = None,+ moe_module_targets = None,+) -> Optional[List[str]]:+ """target_parameters for MoE expert layers, or None for a non-MoE model. Returns the expert weight parameter paths (gate_up_proj, down_proj) for PEFT's ``target_parameters``, under whichever layout the model uses (``mlp.experts.*`` or ``experts.*``, fused or unfused). Only MoE parameters matching *target_modules* are included: "down_proj" gives "mlp.experts.down_proj", and "gate_proj" or "up_proj" gives "mlp.experts.gate_up_proj", or the unfused "mlp.experts.gate_proj" / "mlp.experts.up_proj" pair when the model has no fused Parameter.++ *moe_module_targets* is get_moe_target_modules(model, target_modules) when the caller has+ already computed it. It is only read to decide whether the no-experts-resolved warning+ applies, and passing it in is what keeps this function from walking named_modules a+ second time right before the caller does."""if not is_moe_model(model):return None@@ -3929,24 +3955,48 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[strmoe_params = []+ # One walk of named_parameters for every path this function asks about.+ parameter_names = _moe_parameter_names(model)+gate_up_name = _resolve_moe_parameter_name(model,default_name = "mlp.experts.gate_up_proj",alternate_name = "experts.gate_up_proj",+ parameter_names = parameter_names,)down_name = _resolve_moe_parameter_name(model,default_name = "mlp.experts.down_proj",alternate_name = "experts.down_proj",+ parameter_names = parameter_names,)# gate_up_proj combines gate_proj and up_proj. Target only a fused expert Parameter that exists: per-expert Linear layouts (gpt-oss bnb-4bit) have none and go through get_moe_target_modules, so skip them rather than hand PEFT a dead path.if "gate_proj" in target_set or "up_proj" in target_set or "gate_up_proj" in target_set:- if _moe_parameter_exists(model, gate_up_name):+ if _moe_parameter_exists(model, gate_up_name, parameter_names = parameter_names):moe_params.append(gate_up_name)+ else:+ # NemotronH keeps the expert projections unfused, as separate 3D Parameters.+ # Without this fallback every gate and up expert weight is silently dropped+ # while down_proj still resolves, so the model trains with two thirds of each+ # expert frozen and no error anywhere (unsloth#4476).+ wanted_leaves = []+ if "gate_proj" in target_set or "gate_up_proj" in target_set:+ wanted_leaves.append("gate_proj")+ if "up_proj" in target_set or "gate_up_proj" in target_set:+ wanted_leaves.append("up_proj")+ for leaf in wanted_leaves:+ unfused_name = _resolve_moe_parameter_name(+ model,+ default_name = f"mlp.experts.{leaf}",+ alternate_name = f"experts.{leaf}",+ parameter_names = parameter_names,+ )+ if _moe_parameter_exists(model, unfused_name, parameter_names = parameter_names):+ moe_params.append(unfused_name)if "down_proj" in target_set:- if _moe_parameter_exists(model, down_name):+ if _moe_parameter_exists(model, down_name, parameter_names = parameter_names):moe_params.append(down_name)if moe_params:@@ -3955,16 +4005,81 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[str)return moe_params+ # Nothing resolved as a Parameter. That is only a problem if no other route reaches the+ # experts either: get_moe_target_modules covers the per-expert Linear ModuleList layout+ # (gpt-oss bnb-4bit) and an ordinary suffix match covers the per-expert submodule layout+ # transformers 4.x uses for Qwen3-MoE and Mixtral. Warn only when all three miss, which+ # is the silent no-experts-trained case.+ if target_set & _MOE_BROAD_MLP_TARGETS:+ if moe_module_targets is None:+ moe_module_targets = get_moe_target_modules(model, target_modules)+ if not moe_module_targets and not _moe_experts_reachable_by_module_name(+ model,+ target_set,+ target_modules,+ ):+ logger.warning(+ f"Unsloth: MoE model with {num_experts = } resolved no expert parameters for "+ f"{target_modules = }. The expert weights will NOT be trained."+ )return None-def _moe_parameter_exists(model, name: str) -> bool:+def _moe_parameter_exists(+ model,+ name: str,+ parameter_names: Optional[List[str]] = None,+) -> bool:"""True if ``name`` is an exact suffix of some parameter path on the model."""- if not hasattr(model, "named_parameters"):+ if parameter_names is None:+ parameter_names = _moe_parameter_names(model)+ if parameter_names is None:+ return False+ for parameter_name in parameter_names:+ if parameter_name == name or parameter_name.endswith("." + name):+ return True+ return False+++def _moe_experts_reachable_by_module_name(+ model,+ target_set,+ target_modules = None,+) -> bool:+ """True if an ordinary target_modules match already reaches the experts.++ transformers 4.x builds Qwen3-MoE and Mixtral experts as per-expert submodules+ (``mlp.experts.<i>.gate_proj``), so PEFT attaches LoRA to them by leaf name with no+ help from us, while transformers 5.x fuses the same weights into 3D Parameters. The+ unresolved-experts warning must stay quiet in the first case or it fires on a model+ whose experts are being trained perfectly well.++ PEFT's own two matching rules are mirrored, because the leaf name alone answers only+ one of them. A str ``target_modules`` is a regex PEFT applies with ``re.fullmatch``+ against the WHOLE module path (``peft.utils.other.match_target_against_key``), so+ ``.*mlp.*down_proj`` reaches no expert on a ``mixer.experts.0.down_proj`` layout even+ though the derived leaf set contains ``down_proj``; answering from the leaf there+ would silence the warning on exactly the untrained-experts case it exists for. A list+ is matched by whole key or dotted suffix, which is what the leaf test already does."""+ if not hasattr(model, "named_modules"):+ return False+ pattern = None+ if isinstance(target_modules, str):+ try:+ pattern = re.compile(target_modules)+ except re.error:+ # PEFT would raise on this pattern anyway; do not claim reachability for it.+ return False+ elif not target_set:return Falsetry:- for parameter_name, _ in model.named_parameters():- if parameter_name == name or parameter_name.endswith("." + name):+ for name, _ in model.named_modules():+ if "experts" not in name:+ continue+ if pattern is not None:+ if pattern.fullmatch(name):+ return True+ elif name.rpartition(".")[2] in target_set:return Trueexcept Exception:return Falsediff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex a12c1d3d34d..6fe9aa5292a 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -3404,13 +3404,19 @@ class FastLlamaModel:# Does not get lora yet, so take the name from model, not base model.is_classification = "Classification" in str(type(model))+ # Per-expert Linear expert layouts (gpt-oss bnb-4bit) are Linear modules, not fused Parameters,+ # so target them via target_modules. Resolved before the Parameter detection below so the+ # two share one walk of the model; neither call mutates anything.+ _moe_module_targets = get_moe_target_modules(model, target_modules)+# Auto-detect MoE models and populate target_parameters for expert layers.if target_parameters is None:- target_parameters = get_moe_target_parameters(model, target_modules)+ target_parameters = get_moe_target_parameters(+ model,+ target_modules,+ moe_module_targets = _moe_module_targets,+ )- # Per-expert Linear expert layouts (gpt-oss bnb-4bit) are Linear modules, not fused Parameters,- # so target them via target_modules.- _moe_module_targets = get_moe_target_modules(model, target_modules)if _moe_module_targets:_added = [t for t in _moe_module_targets if t not in final_modules]final_modules.extend(_added)diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 6b8c46d11c5..1f6c1fbeb7d 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -2303,24 +2303,25 @@ class FastBaseModel:loftq_config, lora_dropout, bias, init_lora_weights, model)- # Auto-detect MoE models and populate target_parameters for expert layers. Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an attention-only request does not train experts, but only while MLP and language families are both in scope: with finetune_mlp_modules or finetune_language_layers False the scoped regex already dropped the experts.- if target_parameters is None:- _moe_targets = _select_moe_detection_targets(- _moe_detect_target,- target_modules,- finetune_mlp_modules = finetune_mlp_modules,- finetune_language_layers = finetune_language_layers,- )- target_parameters = get_moe_target_parameters(model, _moe_targets)-- # Per-expert Linear layouts (gpt-oss bnb-4bit) target experts via target_modules, not fused Parameters. Extend either form PEFT accepts: a leaf list, or a regex string.+ # Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an attention-only request does not train experts, but only while MLP and language families are both in scope: with finetune_mlp_modules or finetune_language_layers False the scoped regex already dropped the experts._moe_module_detect = _select_moe_detection_targets(_moe_detect_target,target_modules,finetune_mlp_modules = finetune_mlp_modules,finetune_language_layers = finetune_language_layers,)++ # Per-expert Linear layouts (gpt-oss bnb-4bit) target experts via target_modules, not fused Parameters. Extend either form PEFT accepts: a leaf list, or a regex string._moe_module_targets = get_moe_target_modules(model, _moe_module_detect)++ # Auto-detect MoE models and populate target_parameters for expert layers.+ if target_parameters is None:+ target_parameters = get_moe_target_parameters(+ model,+ _moe_module_detect,+ moe_module_targets = _moe_module_targets,+ )+if _moe_module_targets:if isinstance(target_modules, (list, tuple)):target_modules = list(target_modules) + [