Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Turn off Dr GRPO reward scaling under TRL's "group" default (#11951)

Vineeth Sai Varikuntlac0b27d24549d
diff --git a/tests/version_compat/test_trl_loss_normalization_contract.py b/tests/version_compat/test_trl_loss_normalization_contract.py
index b717de03c39..518277d40fa 100644
--- a/tests/version_compat/test_trl_loss_normalization_contract.py
+++ b/tests/version_compat/test_trl_loss_normalization_contract.py
@@ -101,6 +101,20 @@ def test_explicit_loss_type_still_wins():
assert cfg.loss_type == "chunked_nll", "explicit loss_type was clobbered"
+def test_dr_grpo_turns_off_reward_scaling_by_default():
+ """TRL >= 0.22 defaults scale_rewards to "group" (= True), so dr_grpo must override both."""
+ import unsloth # noqa: F401
+ import trl
+
+ def scale(**kwargs):
+ return trl.GRPOConfig(output_dir = "unused", loss_type = "dr_grpo", **kwargs).scale_rewards
+
+ for kwargs in ({}, {"scale_rewards": True}, {"scale_rewards": "group"}):
+ assert scale(**kwargs) in (False, "none"), f"dr_grpo with {kwargs} still scales rewards"
+ assert scale(scale_rewards = None) in (True, "group"), "None should keep group scaling"
+ assert scale(scale_rewards = "batch") == "batch", "an explicit batch scaling was clobbered"
+
+
def _pristine_sft_config_cls():
"""TRL's own SFTConfig, not the generated subclass patching rebinds over it."""
import trl
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index b0985bad1de..6e1b3b274dd 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2855,6 +2855,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
extra_args += pad_to_multiple_of
# Check for loss_type = dr_grpo and scale_rewards for GRPO; DAPO uses per-token loss, so BNPO loss is used. See huggingface/trl#3130 (comment 2746947835).
+ # TRL >= 0.22 defaults scale_rewards to "group" (= True).
if "loss_type" in call_args and "scale_rewards" in call_args:
check_dr_grpo = (
"if loss_type.lower() == 'dr_grpo':\n"
@@ -2864,8 +2865,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"if loss_type.lower() == 'dr_grpo':\n"
" if scale_rewards == None:\n"
" scale_rewards = True\n"
- " elif scale_rewards == True:\n"
- " print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to force False.')\n"
+ " elif scale_rewards == True or scale_rewards == 'group':\n"
+ " print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to keep scaling.')\n"
" scale_rewards = False\n"
"elif loss_type.lower() == 'dapo':\n"
" if mask_truncated_completions != True:\n"