Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Apply SFTConfig.router_aux_loss_coef to MoE models that cache it at init (#12006)
Daniel Hanc3c388bafae4
diff --git a/tests/version_compat/test_trl_fake_train_cpu.py b/tests/version_compat/test_trl_fake_train_cpu.pyindex 1e28a60050e..0cfca979d76 100644--- a/tests/version_compat/test_trl_fake_train_cpu.py+++ b/tests/version_compat/test_trl_fake_train_cpu.py@@ -448,3 +448,57 @@ def test_grpo_trains_on_cpu_through_the_patched_batch_sampler(tmp_path):"no batch arrived as a list, so this canary is not exercising the GRPO collator ""shape and would not have caught unsloth-zoo#1217")+++def test_sft_applies_trl_router_aux_loss_coef(tmp_path):+ """The MoE forward must apply SFTConfig.router_aux_loss_coef, not the checkpoint value cached at init."""+ import inspect+ from datasets import Dataset+ from trl import SFTConfig, SFTTrainer+ from transformers import AutoTokenizer, MixtralConfig, MixtralForCausalLM++ if "router_aux_loss_coef" not in inspect.signature(SFTConfig).parameters:+ pytest.skip("SFTConfig has no router_aux_loss_coef before TRL 1.7")+ try:+ tok = AutoTokenizer.from_pretrained(_MODEL)+ except OSError as e:+ pytest.skip(f"could not fetch {_MODEL} (network/hub): {str(e)[:150]}")+ tok.pad_token = tok.pad_token or tok.eos_token+ config = MixtralConfig(+ vocab_size = len(tok),+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ num_local_experts = 4,+ num_experts_per_tok = 2,+ router_aux_loss_coef = 0.02,+ )+ torch.manual_seed(0)+ model = MixtralForCausalLM(config)+ ds = Dataset.from_list([{"text": "The quick brown fox jumps over the lazy dog."}] * 8)+ cfg = SFTConfig(+ output_dir = str(tmp_path / "ci_sft_moe"),+ per_device_train_batch_size = 2,+ max_steps = 1,+ report_to = "none",+ save_strategy = "no",+ use_cpu = True,+ max_length = None,+ padding_free = False,+ dataset_text_field = "text",+ fp16 = False,+ bf16 = False,+ router_aux_loss_coef = 0.05,+ )+ SFTTrainer(model = model, processing_class = tok, args = cfg, train_dataset = ds)+ ids = tok(["The quick brown fox jumps over the lazy dog."], return_tensors = "pt").input_ids+ model.train()+ with torch.no_grad():+ on = model(input_ids = ids, labels = ids, output_router_logits = True)+ off = model(input_ids = ids, labels = ids, output_router_logits = False)+ applied = (float(on.loss) - float(off.loss)) / float(on.aux_loss)+ assert (+ abs(applied - 0.05) < 1e-4+ ), f"applied aux coefficient {applied}, expected SFTConfig's 0.05"diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 25d61082955..b0985bad1de 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2672,6 +2672,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)RLTrainer_post += vllm_chat_template_sync+ # TRL >= 1.7 writes router_aux_loss_coef to the config after MoE CausalLMs cached it at init; nll loss reads the stale copy.+ if trainer_file == "sft_trainer":+ RLTrainer_post += (+ "if hasattr(self, 'aux_loss_enabled') and hasattr(getattr(self, 'model', None), 'modules'):\n"+ " for _module in self.model.modules():\n"+ " _config = getattr(_module, 'config', None)\n"+ " if 'router_aux_loss_coef' in vars(_module) and hasattr(_config, 'get_text_config'):\n"+ " _coef = getattr(_config.get_text_config(), 'router_aux_loss_coef', None)\n"+ " if _coef is not None:\n"+ " _module.router_aux_loss_coef = _coef\n"+ "pass\n"+ )+# TRL >= 0.28 builds SamplingParams inside VLLMGeneration, which never sees args; hand it the user's vllm_sampling_params so the generate wrapper in rl_replacements.py can apply them.if trainer_file == "grpo_trainer":RLTrainer_post += (