Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Turn off the MoE aux loss for dense models that carry a router config (#12076)

Daniel Han8af84f12cd87
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
index 1982670eab0..32dbf2890c4 100644
--- a/.github/workflows/version-compat-ci.yml
+++ b/.github/workflows/version-compat-ci.yml
@@ -330,6 +330,7 @@ jobs:
tests/version_compat/test_trl_padding_free_max_length.py \
tests/version_compat/test_trl_loss_normalization_contract.py \
tests/version_compat/test_trl_config_defaults_contract.py \
+ tests/version_compat/test_dense_router_aux_loss_off.py \
tests/version_compat/test_pref_max_length_clamp.py \
tests/version_compat/test_orpo_cpo_long_rows.py \
tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \
@@ -356,6 +357,7 @@ jobs:
tests/version_compat/test_trl_padding_free_max_length.py \
tests/version_compat/test_trl_loss_normalization_contract.py \
tests/version_compat/test_trl_config_defaults_contract.py \
+ tests/version_compat/test_dense_router_aux_loss_off.py \
tests/version_compat/test_pref_max_length_clamp.py \
tests/version_compat/test_orpo_cpo_long_rows.py \
tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \
diff --git a/tests/version_compat/test_dense_router_aux_loss_off.py b/tests/version_compat/test_dense_router_aux_loss_off.py
new file mode 100644
index 00000000000..3ee2df60467
--- /dev/null
+++ b/tests/version_compat/test_dense_router_aux_loss_off.py
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""TRL >= 1.7 enables the router aux loss for any config with output_router_logits; a dense one (0 experts,
+e.g. granite-4.0-h-350m) has no router logits and crashes. Unsloth turns it back off only for those, offline."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+from tests.version_compat._rl_anchors import RL_PY, _rl_py_constant
+
+transformers = pytest.importorskip("transformers")
+
+
+def _trainer_after_trl_init(config):
+ # What TRL 1.7+ leaves behind for any config carrying output_router_logits with the default coef.
+ config.get_text_config().output_router_logits = True
+ self = types.SimpleNamespace(aux_loss_enabled = True, model = types.SimpleNamespace(config = config))
+ exec(_rl_py_constant("_DENSE_ROUTER_AUX_LOSS_OFF"), {}, {"self": self})
+ return self
+
+
+def test_dense_granite_hybrid_turns_the_aux_loss_off():
+ config = transformers.GraniteMoeHybridConfig(num_local_experts = 0, num_experts_per_tok = 0)
+ trainer = _trainer_after_trl_init(config)
+ assert trainer.aux_loss_enabled is False and config.output_router_logits is False
+
+
+ "config",
+ [
+ lambda: transformers.GraniteMoeHybridConfig(num_local_experts = 4, num_experts_per_tok = 2),
+ lambda: transformers.Qwen3MoeConfig(),
+ lambda: transformers.MixtralConfig(),
+ ],
+ ids = ["granite_hybrid_moe", "qwen3_moe", "mixtral"],
+)
+def test_real_moe_keeps_the_aux_loss(config):
+ config = config()
+ trainer = _trainer_after_trl_init(config)
+ assert trainer.aux_loss_enabled is True and config.output_router_logits is True
+
+
+def test_trainer_without_aux_loss_is_untouched():
+ self = types.SimpleNamespace(model = None)
+ exec(_rl_py_constant("_DENSE_ROUTER_AUX_LOSS_OFF"), {}, {"self": self})
+ assert not hasattr(self, "aux_loss_enabled")
+
+
+def test_rl_py_appends_it_after_trl_init():
+ src = RL_PY.read_text("utf-8")
+ assert "RLTrainer_post += _DENSE_ROUTER_AUX_LOSS_OFF" in src
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 51623e5167c..ef4cf299ff0 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -1112,6 +1112,19 @@ def _pin_pristine_sft_loss_type(config_cls):
_UNSLOTH_KBIT_PREP_GUARD_FLAG = "_unsloth_skips_kbit_prep_for_peft_models"
+# TRL >= 1.7 enables the aux loss whenever output_router_logits is set; dense configs (every expert count 0, e.g.
+# granite-4.0-h-350m) return no router logits and crash on `aux_loss.to`. Real MoE keeps its aux loss.
+_DENSE_ROUTER_AUX_LOSS_OFF = (
+ "if getattr(self, 'aux_loss_enabled', False) and hasattr(getattr(self, 'model', None), 'config'):\n"
+ " _text_config = self.model.config\n"
+ " if hasattr(_text_config, 'get_text_config'): _text_config = _text_config.get_text_config()\n"
+ " _n_experts = [getattr(_text_config, _k) for _k in ('num_local_experts', 'num_experts', 'n_routed_experts', 'moe_num_experts') if isinstance(getattr(_text_config, _k, None), int)]\n"
+ " if _n_experts and all(_n == 0 for _n in _n_experts):\n"
+ " self.aux_loss_enabled = False\n"
+ " _text_config.output_router_logits = False\n"
+ "pass\n"
+)
+
# The one assignment of `self.aux_loss_enabled` in TRL's GRPOTrainer.__init__, whatever its right-hand
# side. TRL 1.7.0 wrote `is_moe and args.router_aux_loss_coef != 0.0`; TRL main (#7248) reads the
# coefficient from the model config when it is None. Anchoring on the exact expression lost the
@@ -2691,6 +2704,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
RLTrainer_post += vllm_chat_template_sync
+ if "model" in call_args:
+ RLTrainer_post += _DENSE_ROUTER_AUX_LOSS_OFF
+
# TRL >= 1.7 writes router_aux_loss_coef to the config after MoE CausalLMs cached it at init; nll loss reads the stale copy.
if trainer_file == "sft_trainer":
RLTrainer_post += (