Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Turn off the MoE aux loss for dense models that carry a router config (#12076)
Daniel Han8af84f12cd87
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.ymlindex 1982670eab0..32dbf2890c4 100644--- a/.github/workflows/version-compat-ci.yml+++ b/.github/workflows/version-compat-ci.yml@@ -330,6 +330,7 @@ jobs:tests/version_compat/test_trl_padding_free_max_length.py \tests/version_compat/test_trl_loss_normalization_contract.py \tests/version_compat/test_trl_config_defaults_contract.py \+ tests/version_compat/test_dense_router_aux_loss_off.py \tests/version_compat/test_pref_max_length_clamp.py \tests/version_compat/test_orpo_cpo_long_rows.py \tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \@@ -356,6 +357,7 @@ jobs:tests/version_compat/test_trl_padding_free_max_length.py \tests/version_compat/test_trl_loss_normalization_contract.py \tests/version_compat/test_trl_config_defaults_contract.py \+ tests/version_compat/test_dense_router_aux_loss_off.py \tests/version_compat/test_pref_max_length_clamp.py \tests/version_compat/test_orpo_cpo_long_rows.py \tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \diff --git a/tests/version_compat/test_dense_router_aux_loss_off.py b/tests/version_compat/test_dense_router_aux_loss_off.pynew file mode 100644index 00000000000..3ee2df60467--- /dev/null+++ b/tests/version_compat/test_dense_router_aux_loss_off.py@@ -0,0 +1,55 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""TRL >= 1.7 enables the router aux loss for any config with output_router_logits; a dense one (0 experts,+e.g. granite-4.0-h-350m) has no router logits and crashes. Unsloth turns it back off only for those, offline."""++from __future__ import annotations++import types++import pytest++from tests.version_compat._rl_anchors import RL_PY, _rl_py_constant++transformers = pytest.importorskip("transformers")+++def _trainer_after_trl_init(config):+ # What TRL 1.7+ leaves behind for any config carrying output_router_logits with the default coef.+ config.get_text_config().output_router_logits = True+ self = types.SimpleNamespace(aux_loss_enabled = True, model = types.SimpleNamespace(config = config))+ exec(_rl_py_constant("_DENSE_ROUTER_AUX_LOSS_OFF"), {}, {"self": self})+ return self+++def test_dense_granite_hybrid_turns_the_aux_loss_off():+ config = transformers.GraniteMoeHybridConfig(num_local_experts = 0, num_experts_per_tok = 0)+ trainer = _trainer_after_trl_init(config)+ assert trainer.aux_loss_enabled is False and config.output_router_logits is False+++ "config",+ [+ lambda: transformers.GraniteMoeHybridConfig(num_local_experts = 4, num_experts_per_tok = 2),+ lambda: transformers.Qwen3MoeConfig(),+ lambda: transformers.MixtralConfig(),+ ],+ ids = ["granite_hybrid_moe", "qwen3_moe", "mixtral"],+)+def test_real_moe_keeps_the_aux_loss(config):+ config = config()+ trainer = _trainer_after_trl_init(config)+ assert trainer.aux_loss_enabled is True and config.output_router_logits is True+++def test_trainer_without_aux_loss_is_untouched():+ self = types.SimpleNamespace(model = None)+ exec(_rl_py_constant("_DENSE_ROUTER_AUX_LOSS_OFF"), {}, {"self": self})+ assert not hasattr(self, "aux_loss_enabled")+++def test_rl_py_appends_it_after_trl_init():+ src = RL_PY.read_text("utf-8")+ assert "RLTrainer_post += _DENSE_ROUTER_AUX_LOSS_OFF" in srcdiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 51623e5167c..ef4cf299ff0 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -1112,6 +1112,19 @@ def _pin_pristine_sft_loss_type(config_cls):_UNSLOTH_KBIT_PREP_GUARD_FLAG = "_unsloth_skips_kbit_prep_for_peft_models"+# TRL >= 1.7 enables the aux loss whenever output_router_logits is set; dense configs (every expert count 0, e.g.+# granite-4.0-h-350m) return no router logits and crash on `aux_loss.to`. Real MoE keeps its aux loss.+_DENSE_ROUTER_AUX_LOSS_OFF = (+ "if getattr(self, 'aux_loss_enabled', False) and hasattr(getattr(self, 'model', None), 'config'):\n"+ " _text_config = self.model.config\n"+ " if hasattr(_text_config, 'get_text_config'): _text_config = _text_config.get_text_config()\n"+ " _n_experts = [getattr(_text_config, _k) for _k in ('num_local_experts', 'num_experts', 'n_routed_experts', 'moe_num_experts') if isinstance(getattr(_text_config, _k, None), int)]\n"+ " if _n_experts and all(_n == 0 for _n in _n_experts):\n"+ " self.aux_loss_enabled = False\n"+ " _text_config.output_router_logits = False\n"+ "pass\n"+)+# The one assignment of `self.aux_loss_enabled` in TRL's GRPOTrainer.__init__, whatever its right-hand# side. TRL 1.7.0 wrote `is_moe and args.router_aux_loss_coef != 0.0`; TRL main (#7248) reads the# coefficient from the model config when it is None. Anchoring on the exact expression lost the@@ -2691,6 +2704,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)RLTrainer_post += vllm_chat_template_sync+ if "model" in call_args:+ RLTrainer_post += _DENSE_ROUTER_AUX_LOSS_OFF+# TRL >= 1.7 writes router_aux_loss_coef to the config after MoE CausalLMs cached it at init; nll loss reads the stale copy.if trainer_file == "sft_trainer":RLTrainer_post += (