Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
SFT: keep the router aux loss off for single-expert configs (Jamba) (#15010)
Daniel Hanf48b9962b768
diff --git a/tests/version_compat/test_dense_router_aux_loss_off.py b/tests/version_compat/test_dense_router_aux_loss_off.pyindex c63f76756f9..e1e6e0acee4 100644--- a/tests/version_compat/test_dense_router_aux_loss_off.py+++ b/tests/version_compat/test_dense_router_aux_loss_off.py@@ -24,6 +24,14 @@ def _trainer_after_trl_init(config):return self+def test_single_expert_jamba_turns_the_aux_loss_off():+ # AI21-Jamba-Reasoning-3B: num_experts = 1 builds JambaMLP in every layer, so router_logits is () and+ # load_balancing_loss_func raised IndexError at the first SFT step.+ config = transformers.JambaConfig(num_experts = 1, num_experts_per_tok = 1)+ trainer = _trainer_after_trl_init(config)+ assert trainer.aux_loss_enabled is False and config.output_router_logits is False++def test_dense_granite_hybrid_turns_the_aux_loss_off():config = transformers.GraniteMoeHybridConfig(num_local_experts = 0, num_experts_per_tok = 0)trainer = _trainer_after_trl_init(config)@@ -36,8 +44,9 @@ def test_dense_granite_hybrid_turns_the_aux_loss_off():lambda: transformers.GraniteMoeHybridConfig(num_local_experts = 4, num_experts_per_tok = 2),lambda: transformers.Qwen3MoeConfig(),lambda: transformers.MixtralConfig(),+ lambda: transformers.JambaConfig(num_experts = 16, num_experts_per_tok = 2),],- ids = ["granite_hybrid_moe", "qwen3_moe", "mixtral"],+ ids = ["granite_hybrid_moe", "qwen3_moe", "mixtral", "jamba_moe"],)def test_real_moe_keeps_the_aux_loss(config):config = config()diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 98b5aa8413e..7dd73b69332 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -1381,14 +1381,16 @@ def _pin_pristine_sft_loss_type(config_cls):_UNSLOTH_KBIT_PREP_GUARD_FLAG = "_unsloth_skips_kbit_prep_for_peft_models"-# TRL >= 1.7 enables the aux loss whenever output_router_logits is set; dense configs (every expert count 0, e.g.-# granite-4.0-h-350m) return no router logits and crash on `aux_loss.to`. Real MoE keeps its aux loss.+# TRL >= 1.7 enables the aux loss whenever output_router_logits is set; dense configs return no router logits and+# crash (`aux_loss.to`, or Jamba's load_balancing_loss_func indexing an empty tuple). Dense = every expert count+# 0 (granite-4.0-h-350m) or 1 (AI21-Jamba-Reasoning-3B builds a plain MLP for 1 expert; a 1-expert router's aux+# loss is a constant with no gradient anyway). Real MoE keeps its aux loss._DENSE_ROUTER_AUX_LOSS_OFF = ("if getattr(self, 'aux_loss_enabled', False) and hasattr(getattr(self, 'model', None), 'config'):\n"" _text_config = self.model.config\n"" if hasattr(_text_config, 'get_text_config'): _text_config = _text_config.get_text_config()\n"" _n_experts = [getattr(_text_config, _k) for _k in ('num_local_experts', 'num_experts', 'n_routed_experts', 'moe_num_experts') if isinstance(getattr(_text_config, _k, None), int)]\n"- " if _n_experts and all(_n == 0 for _n in _n_experts):\n"+ " if _n_experts and all(_n <= 1 for _n in _n_experts):\n"" self.aux_loss_enabled = False\n"" _text_config.output_router_logits = False\n""pass\n"