Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Default GRPO to TRL's dapo loss with beta 0, and cap CISPO weights at 5.0 (#12088)
Daniel Hana9982dcad499
diff --git a/tests/version_compat/test_trl_config_defaults_contract.py b/tests/version_compat/test_trl_config_defaults_contract.pyindex 1d54ef4b8b5..4c6310588d9 100644--- a/tests/version_compat/test_trl_config_defaults_contract.py+++ b/tests/version_compat/test_trl_config_defaults_contract.py@@ -151,9 +151,11 @@ def test_trl_fields_the_overrides_key_on_are_unchanged():assert fields["scale_rewards"].default in (True, "group"), fields["scale_rewards"].defaultassert fields["epsilon_high"].default is None, fields["epsilon_high"].defaultassert fields["mask_truncated_completions"].default is False- assert set(("dr_grpo", "dapo")) <= set(- _documented_loss_types()- ), "TRL dropped a loss_type Unsloth special-cases"+ import trl+ from packaging.version import Version++ special = {"dr_grpo", "dapo"} if Version(trl.__version__) >= Version("0.22.0") else {"dr_grpo"}+ assert special <= set(_documented_loss_types()), "TRL dropped a loss_type Unsloth special-cases"def _documented_loss_types():@@ -164,3 +166,61 @@ def _documented_loss_types():for lt in ("grpo", "bnpo", "dr_grpo", "dapo", "cispo", "sapo", "luspo", "vespo")if f'"{lt}"' in src]+++def _needs_loss_type(loss_type):+ if loss_type not in _documented_loss_types():+ pytest.skip(f"this TRL has no loss_type={loss_type!r}")+++def test_default_loss_type_is_trls_dapo():+ cfg = _grpo()+ if "dapo" not in _documented_loss_types():+ assert (+ cfg.loss_type == "bnpo" and cfg.beta == 0.001+ ), "TRL <= 0.21 has no dapo to default to"+ return+ assert cfg.loss_type == "dapo" and cfg.beta == 0.0+++def test_default_dapo_keeps_trl_clip_and_truncation():+ """Only an explicit loss_type="dapo" gets the paper's settings: masking truncated rows zeroes every update when all are."""+ _needs_loss_type("dapo")+ trl_default = _pristine(_config_cls("GRPOConfig"))(output_dir = "unused")+ cfg = _grpo()+ assert cfg.epsilon_high == trl_default.epsilon_high+ assert cfg.mask_truncated_completions is trl_default.mask_truncated_completions is False+++ "loss_type, beta", [("dapo", 0.0), ("dr_grpo", 0.0), ("bnpo", 0.001), ("grpo", 0.001)]+)+def test_unset_beta_follows_loss_type(loss_type, beta):+ _needs_loss_type(loss_type)+ assert _grpo(loss_type = loss_type).beta == beta++[email protected]("loss_type", ["dapo", "dr_grpo", "bnpo", "grpo"])[email protected]("beta", [0.0, 0.001, 0.04])+def test_explicit_beta_is_kept(loss_type, beta):+ assert _grpo(loss_type = loss_type, beta = beta).beta == beta+++def test_cispo_caps_the_is_weight_at_scalerl_epsilon_high():+ """TRL clamps the CISPO weight at epsilon_high itself; its epsilon fallback (0.2) would cap every weight below 1."""+ _needs_loss_type("cispo")+ cfg = _grpo(loss_type = "cispo")+ assert cfg.epsilon_high == 5.0+ assert cfg.mask_truncated_completions is False, "the dapo recommendations leaked into cispo"+ assert cfg.beta == 0.001+ assert _grpo(loss_type = "cispo", epsilon_high = 3.0).epsilon_high == 3.0++[email protected]("loss_type", ["bnpo", "grpo", "dr_grpo"])+def test_other_loss_types_keep_trl_epsilon_high(loss_type):+ assert (+ _grpo(loss_type = loss_type).epsilon_high+ == _pristine(_config_cls("GRPOConfig"))(+ output_dir = "unused", loss_type = loss_type+ ).epsilon_high+ )diff --git a/tests/version_compat/test_trl_loss_normalization_contract.py b/tests/version_compat/test_trl_loss_normalization_contract.pyindex 518277d40fa..acd544dfb49 100644--- a/tests/version_compat/test_trl_loss_normalization_contract.py+++ b/tests/version_compat/test_trl_loss_normalization_contract.py@@ -78,7 +78,11 @@ def test_loss_type_replacement_did_not_leak_to_other_trainers():import unsloth # noqa: F401import trl- expected = {"DPOConfig": ["sigmoid"], "KTOConfig": "kto", "GRPOConfig": "bnpo"}+ from packaging.version import Version++ # Unsloth follows TRL's GRPO default from 0.22, the first TRL with "dapo".+ grpo = "dapo" if Version(trl.__version__) >= Version("0.22.0") else "bnpo"+ expected = {"DPOConfig": ["sigmoid"], "KTOConfig": "kto", "GRPOConfig": grpo}for name, want in expected.items():cfg_cls = getattr(trl, name, None)if cfg_cls is None or not hasattr(cfg_cls, "loss_type"):diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 97b21175d83..0e8c01d1620 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2775,11 +2775,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):y = f"{k} = {y},\n"arguments = re.sub(x, y, arguments)- # GRPO beta default is 0.001: TRL used 0.04 and now 0.00. See huggingface/trl#3516 and the verl docs.if trainer_file == "grpo_trainer":replacements = {- "loss_type": "bnpo", # TRL <= 0.21 default; the GRPO paper objective is "grpo"- "beta": 0.001, # Recommended as seen in verl+ # None = unset: resolved below to TRL's default "dapo" (huggingface/trl#3938, TRL >= 0.22; older TRL passes no num_items_in_batch for it, so "bnpo").+ "loss_type": None,+ # None = unset: resolved below to 0.0 for dapo / dr_grpo, else 0.001 (verl's recommendation; TRL used 0.04, now 0.0, huggingface/trl#3516).+ "beta": None,"auto_find_batch_size": False, # Cannot work on GRPO# See fengyao.notion.site/off-policy-rl and huggingface/trl#3867."vllm_importance_sampling_correction": False,@@ -2868,6 +2869,30 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += pad_to_multiple_of+ # Resolve the None loss_type / beta sentinels before any check reads them.+ if trainer_file == "grpo_trainer" and "loss_type" in call_args:+ _default_loss_type = "dapo" if trl_version >= Version("0.22.0") else "bnpo"+ extra_args += (+ "_unsloth_default_loss_type = loss_type is None\n"+ "if _unsloth_default_loss_type:\n"+ f" loss_type = '{_default_loss_type}'\n"+ )+ if _default_loss_type == "dapo" and "beta" in call_args:+ extra_args += (+ " if beta is None:\n"+ " print(\"Unsloth: GRPO now defaults to TRL's loss_type = 'dapo' with beta = 0.0. Set loss_type = 'bnpo', beta = 0.001 for Unsloth's previous default.\")\n"+ )+ if trainer_file == "grpo_trainer" and "beta" in call_args:+ extra_args += (+ "if beta is None:\n"+ + (+ " beta = 0.0 if str(loss_type).lower() in ('dapo', 'dr_grpo') else 0.001\n"+ if "loss_type" in call_args+ else " beta = 0.001\n"+ )+ + "\n"+ )+# Check for loss_type = dr_grpo and scale_rewards for GRPO; DAPO uses per-token loss, so BNPO loss is used. See huggingface/trl#3130 (comment 2746947835).# TRL >= 0.22 defaults scale_rewards to "group" (= True).if "loss_type" in call_args and "scale_rewards" in call_args:@@ -2882,7 +2907,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" elif scale_rewards == True or scale_rewards == 'group':\n"" print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to keep scaling.')\n"" scale_rewards = False\n"- "elif loss_type.lower() == 'dapo':\n"+ # Only an explicit dapo gets the paper's settings; the default keeps TRL's (masking every truncated row zeroes the update when all are).+ "elif loss_type.lower() == 'dapo' and not _unsloth_default_loss_type:\n"" if mask_truncated_completions is None:\n"" print('Unsloth: The DAPO paper recommends `mask_truncated_completions = True` - we will set it.')\n"" mask_truncated_completions = True\n"@@ -2891,10 +2917,20 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" epsilon_high = 0.28\n"" if beta != 0.0:\n"" print(f'[WARNING] Unsloth: The DAPO paper recommends setting `beta = 0.0` to remove the KL term - You have set it to {beta}.')\n"+ # TRL clamps the CISPO IS weight at epsilon_high itself (not 1 + epsilon_high), so the 0.2 fallback would cap every weight below 1.+ "elif loss_type.lower() == 'cispo':\n"+ " loss_type = 'cispo'\n"+ " if epsilon_high is None:\n"+ " print('Unsloth: CISPO caps the importance sampling weight at `epsilon_high`; the ScaleRL paper recommends `epsilon_high = 5.0` - we will set it.')\n"+ " epsilon_high = 5.0\n"+ " elif epsilon_high < 1.0:\n"+ " print(f'[WARNING] Unsloth: CISPO caps the importance sampling weight at `epsilon_high` itself, not 1 + epsilon_high, so {epsilon_high} caps every weight below 1. The ScaleRL paper uses 4 to 8.')\n""if mask_truncated_completions is None:\n"" mask_truncated_completions = False\n""\n")+ if trainer_file != "grpo_trainer":+ check_dr_grpo = "_unsloth_default_loss_type = False\n" + check_dr_grpoextra_args += check_dr_grpo# GRPO on TRL < 0.20 crashes on the first step (no has_images / images, no reference logps path).