Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Turn off Dr GRPO reward scaling under TRL's "group" default (#11951)
Vineeth Sai Varikuntlac0b27d24549d
diff --git a/tests/version_compat/test_trl_loss_normalization_contract.py b/tests/version_compat/test_trl_loss_normalization_contract.pyindex b717de03c39..518277d40fa 100644--- a/tests/version_compat/test_trl_loss_normalization_contract.py+++ b/tests/version_compat/test_trl_loss_normalization_contract.py@@ -101,6 +101,20 @@ def test_explicit_loss_type_still_wins():assert cfg.loss_type == "chunked_nll", "explicit loss_type was clobbered"+def test_dr_grpo_turns_off_reward_scaling_by_default():+ """TRL >= 0.22 defaults scale_rewards to "group" (= True), so dr_grpo must override both."""+ import unsloth # noqa: F401+ import trl++ def scale(**kwargs):+ return trl.GRPOConfig(output_dir = "unused", loss_type = "dr_grpo", **kwargs).scale_rewards++ for kwargs in ({}, {"scale_rewards": True}, {"scale_rewards": "group"}):+ assert scale(**kwargs) in (False, "none"), f"dr_grpo with {kwargs} still scales rewards"+ assert scale(scale_rewards = None) in (True, "group"), "None should keep group scaling"+ assert scale(scale_rewards = "batch") == "batch", "an explicit batch scaling was clobbered"++def _pristine_sft_config_cls():"""TRL's own SFTConfig, not the generated subclass patching rebinds over it."""import trldiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex b0985bad1de..6e1b3b274dd 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2855,6 +2855,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):extra_args += pad_to_multiple_of# Check for loss_type = dr_grpo and scale_rewards for GRPO; DAPO uses per-token loss, so BNPO loss is used. See huggingface/trl#3130 (comment 2746947835).+ # TRL >= 0.22 defaults scale_rewards to "group" (= True).if "loss_type" in call_args and "scale_rewards" in call_args:check_dr_grpo = ("if loss_type.lower() == 'dr_grpo':\n"@@ -2864,8 +2865,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"if loss_type.lower() == 'dr_grpo':\n"" if scale_rewards == None:\n"" scale_rewards = True\n"- " elif scale_rewards == True:\n"- " print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to force False.')\n"+ " elif scale_rewards == True or scale_rewards == 'group':\n"+ " print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to keep scaling.')\n"" scale_rewards = False\n""elif loss_type.lower() == 'dapo':\n"" if mask_truncated_completions != True:\n"