Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Default GRPO to TRL's dapo loss with beta 0, and cap CISPO weights at 5.0 (#12088)

Daniel Hana9982dcad499
diff --git a/tests/version_compat/test_trl_config_defaults_contract.py b/tests/version_compat/test_trl_config_defaults_contract.py
index 1d54ef4b8b5..4c6310588d9 100644
--- a/tests/version_compat/test_trl_config_defaults_contract.py
+++ b/tests/version_compat/test_trl_config_defaults_contract.py
@@ -151,9 +151,11 @@ def test_trl_fields_the_overrides_key_on_are_unchanged():
assert fields["scale_rewards"].default in (True, "group"), fields["scale_rewards"].default
assert fields["epsilon_high"].default is None, fields["epsilon_high"].default
assert fields["mask_truncated_completions"].default is False
- assert set(("dr_grpo", "dapo")) <= set(
- _documented_loss_types()
- ), "TRL dropped a loss_type Unsloth special-cases"
+ import trl
+ from packaging.version import Version
+
+ special = {"dr_grpo", "dapo"} if Version(trl.__version__) >= Version("0.22.0") else {"dr_grpo"}
+ assert special <= set(_documented_loss_types()), "TRL dropped a loss_type Unsloth special-cases"
def _documented_loss_types():
@@ -164,3 +166,61 @@ def _documented_loss_types():
for lt in ("grpo", "bnpo", "dr_grpo", "dapo", "cispo", "sapo", "luspo", "vespo")
if f'"{lt}"' in src
]
+
+
+def _needs_loss_type(loss_type):
+ if loss_type not in _documented_loss_types():
+ pytest.skip(f"this TRL has no loss_type={loss_type!r}")
+
+
+def test_default_loss_type_is_trls_dapo():
+ cfg = _grpo()
+ if "dapo" not in _documented_loss_types():
+ assert (
+ cfg.loss_type == "bnpo" and cfg.beta == 0.001
+ ), "TRL <= 0.21 has no dapo to default to"
+ return
+ assert cfg.loss_type == "dapo" and cfg.beta == 0.0
+
+
+def test_default_dapo_keeps_trl_clip_and_truncation():
+ """Only an explicit loss_type="dapo" gets the paper's settings: masking truncated rows zeroes every update when all are."""
+ _needs_loss_type("dapo")
+ trl_default = _pristine(_config_cls("GRPOConfig"))(output_dir = "unused")
+ cfg = _grpo()
+ assert cfg.epsilon_high == trl_default.epsilon_high
+ assert cfg.mask_truncated_completions is trl_default.mask_truncated_completions is False
+
+
+ "loss_type, beta", [("dapo", 0.0), ("dr_grpo", 0.0), ("bnpo", 0.001), ("grpo", 0.001)]
+)
+def test_unset_beta_follows_loss_type(loss_type, beta):
+ _needs_loss_type(loss_type)
+ assert _grpo(loss_type = loss_type).beta == beta
+
+
[email protected]("loss_type", ["dapo", "dr_grpo", "bnpo", "grpo"])
[email protected]("beta", [0.0, 0.001, 0.04])
+def test_explicit_beta_is_kept(loss_type, beta):
+ assert _grpo(loss_type = loss_type, beta = beta).beta == beta
+
+
+def test_cispo_caps_the_is_weight_at_scalerl_epsilon_high():
+ """TRL clamps the CISPO weight at epsilon_high itself; its epsilon fallback (0.2) would cap every weight below 1."""
+ _needs_loss_type("cispo")
+ cfg = _grpo(loss_type = "cispo")
+ assert cfg.epsilon_high == 5.0
+ assert cfg.mask_truncated_completions is False, "the dapo recommendations leaked into cispo"
+ assert cfg.beta == 0.001
+ assert _grpo(loss_type = "cispo", epsilon_high = 3.0).epsilon_high == 3.0
+
+
[email protected]("loss_type", ["bnpo", "grpo", "dr_grpo"])
+def test_other_loss_types_keep_trl_epsilon_high(loss_type):
+ assert (
+ _grpo(loss_type = loss_type).epsilon_high
+ == _pristine(_config_cls("GRPOConfig"))(
+ output_dir = "unused", loss_type = loss_type
+ ).epsilon_high
+ )
diff --git a/tests/version_compat/test_trl_loss_normalization_contract.py b/tests/version_compat/test_trl_loss_normalization_contract.py
index 518277d40fa..acd544dfb49 100644
--- a/tests/version_compat/test_trl_loss_normalization_contract.py
+++ b/tests/version_compat/test_trl_loss_normalization_contract.py
@@ -78,7 +78,11 @@ def test_loss_type_replacement_did_not_leak_to_other_trainers():
import unsloth # noqa: F401
import trl
- expected = {"DPOConfig": ["sigmoid"], "KTOConfig": "kto", "GRPOConfig": "bnpo"}
+ from packaging.version import Version
+
+ # Unsloth follows TRL's GRPO default from 0.22, the first TRL with "dapo".
+ grpo = "dapo" if Version(trl.__version__) >= Version("0.22.0") else "bnpo"
+ expected = {"DPOConfig": ["sigmoid"], "KTOConfig": "kto", "GRPOConfig": grpo}
for name, want in expected.items():
cfg_cls = getattr(trl, name, None)
if cfg_cls is None or not hasattr(cfg_cls, "loss_type"):
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 97b21175d83..0e8c01d1620 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2775,11 +2775,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
y = f"{k} = {y},\n"
arguments = re.sub(x, y, arguments)
- # GRPO beta default is 0.001: TRL used 0.04 and now 0.00. See huggingface/trl#3516 and the verl docs.
if trainer_file == "grpo_trainer":
replacements = {
- "loss_type": "bnpo", # TRL <= 0.21 default; the GRPO paper objective is "grpo"
- "beta": 0.001, # Recommended as seen in verl
+ # None = unset: resolved below to TRL's default "dapo" (huggingface/trl#3938, TRL >= 0.22; older TRL passes no num_items_in_batch for it, so "bnpo").
+ "loss_type": None,
+ # None = unset: resolved below to 0.0 for dapo / dr_grpo, else 0.001 (verl's recommendation; TRL used 0.04, now 0.0, huggingface/trl#3516).
+ "beta": None,
"auto_find_batch_size": False, # Cannot work on GRPO
# See fengyao.notion.site/off-policy-rl and huggingface/trl#3867.
"vllm_importance_sampling_correction": False,
@@ -2868,6 +2869,30 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += pad_to_multiple_of
+ # Resolve the None loss_type / beta sentinels before any check reads them.
+ if trainer_file == "grpo_trainer" and "loss_type" in call_args:
+ _default_loss_type = "dapo" if trl_version >= Version("0.22.0") else "bnpo"
+ extra_args += (
+ "_unsloth_default_loss_type = loss_type is None\n"
+ "if _unsloth_default_loss_type:\n"
+ f" loss_type = '{_default_loss_type}'\n"
+ )
+ if _default_loss_type == "dapo" and "beta" in call_args:
+ extra_args += (
+ " if beta is None:\n"
+ " print(\"Unsloth: GRPO now defaults to TRL's loss_type = 'dapo' with beta = 0.0. Set loss_type = 'bnpo', beta = 0.001 for Unsloth's previous default.\")\n"
+ )
+ if trainer_file == "grpo_trainer" and "beta" in call_args:
+ extra_args += (
+ "if beta is None:\n"
+ + (
+ " beta = 0.0 if str(loss_type).lower() in ('dapo', 'dr_grpo') else 0.001\n"
+ if "loss_type" in call_args
+ else " beta = 0.001\n"
+ )
+ + "\n"
+ )
+
# Check for loss_type = dr_grpo and scale_rewards for GRPO; DAPO uses per-token loss, so BNPO loss is used. See huggingface/trl#3130 (comment 2746947835).
# TRL >= 0.22 defaults scale_rewards to "group" (= True).
if "loss_type" in call_args and "scale_rewards" in call_args:
@@ -2882,7 +2907,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" elif scale_rewards == True or scale_rewards == 'group':\n"
" print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to keep scaling.')\n"
" scale_rewards = False\n"
- "elif loss_type.lower() == 'dapo':\n"
+ # Only an explicit dapo gets the paper's settings; the default keeps TRL's (masking every truncated row zeroes the update when all are).
+ "elif loss_type.lower() == 'dapo' and not _unsloth_default_loss_type:\n"
" if mask_truncated_completions is None:\n"
" print('Unsloth: The DAPO paper recommends `mask_truncated_completions = True` - we will set it.')\n"
" mask_truncated_completions = True\n"
@@ -2891,10 +2917,20 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" epsilon_high = 0.28\n"
" if beta != 0.0:\n"
" print(f'[WARNING] Unsloth: The DAPO paper recommends setting `beta = 0.0` to remove the KL term - You have set it to {beta}.')\n"
+ # TRL clamps the CISPO IS weight at epsilon_high itself (not 1 + epsilon_high), so the 0.2 fallback would cap every weight below 1.
+ "elif loss_type.lower() == 'cispo':\n"
+ " loss_type = 'cispo'\n"
+ " if epsilon_high is None:\n"
+ " print('Unsloth: CISPO caps the importance sampling weight at `epsilon_high`; the ScaleRL paper recommends `epsilon_high = 5.0` - we will set it.')\n"
+ " epsilon_high = 5.0\n"
+ " elif epsilon_high < 1.0:\n"
+ " print(f'[WARNING] Unsloth: CISPO caps the importance sampling weight at `epsilon_high` itself, not 1 + epsilon_high, so {epsilon_high} caps every weight below 1. The ScaleRL paper uses 4 to 8.')\n"
"if mask_truncated_completions is None:\n"
" mask_truncated_completions = False\n"
"\n"
)
+ if trainer_file != "grpo_trainer":
+ check_dr_grpo = "_unsloth_default_loss_type = False\n" + check_dr_grpo
extra_args += check_dr_grpo
# GRPO on TRL < 0.20 crashes on the first step (no has_images / images, no reference logps path).