Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep TRL's own RL config defaults and clamp preference max_length (#12069)

Daniel Hanec9cc2c4e0e1
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
index db8ed31a805..5fa53ff28fc 100644
--- a/.github/workflows/version-compat-ci.yml
+++ b/.github/workflows/version-compat-ci.yml
@@ -329,6 +329,8 @@ jobs:
tests/version_compat/test_trl_fake_train_cpu.py \
tests/version_compat/test_trl_padding_free_max_length.py \
tests/version_compat/test_trl_loss_normalization_contract.py \
+ tests/version_compat/test_trl_config_defaults_contract.py \
+ tests/version_compat/test_pref_max_length_clamp.py \
tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \
-v --tb=short
# `main` is scheduled/dispatch-only so PR jobs stay fast and a bleeding-edge
@@ -351,6 +353,8 @@ jobs:
tests/version_compat/test_trl_fake_train_cpu.py \
tests/version_compat/test_trl_padding_free_max_length.py \
tests/version_compat/test_trl_loss_normalization_contract.py \
+ tests/version_compat/test_trl_config_defaults_contract.py \
+ tests/version_compat/test_pref_max_length_clamp.py \
tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \
-v --tb=short
diff --git a/tests/version_compat/test_pref_max_length_clamp.py b/tests/version_compat/test_pref_max_length_clamp.py
new file mode 100644
index 00000000000..65d34828a9b
--- /dev/null
+++ b/tests/version_compat/test_pref_max_length_clamp.py
@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""Preference-trainer rows longer than the model max_seq_length must not crash the log-prob gather.
+
+The fast forward cuts input_ids to model.max_seq_length while TRL builds labels at args.max_length.
+"""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import inspect
+import re
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+
+RL_PY = Path(importlib.util.find_spec("unsloth").origin).parent / "models" / "rl.py"
+
+
+def _clamp_snippet():
+ src = RL_PY.read_text(encoding = "utf-8")
+ m = re.search(
+ r'elif trainer_file in \("dpo_trainer", "kto_trainer", "orpo_trainer", "cpo_trainer"\):\n(.*?)\n \)\n',
+ src,
+ re.S,
+ )
+ assert m, "no preference-trainer max_length clamp in rl.py"
+ body = m.group(1).split("extra_args += (", 1)[1]
+ return "".join(
+ ast.literal_eval(line.strip()) for line in body.splitlines() if line.strip().startswith('"')
+ )
+
+
+def _run(
+ max_seq_length,
+ max_length,
+ max_prompt_length = None,
+):
+ model = SimpleNamespace(max_seq_length = max_seq_length)
+ args = SimpleNamespace(max_length = max_length, max_prompt_length = max_prompt_length)
+ exec(_clamp_snippet(), {"model": model, "args": args, "print": lambda *a: None})
+ return args
+
+
[email protected]("max_length", [1024, None])
+def test_max_length_is_capped_at_the_model_limit(max_length):
+ assert _run(48, max_length).max_length == 48
+
+
+def test_a_shorter_max_length_is_kept():
+ assert _run(2048, 1024).max_length == 1024
+
+
+def test_prompt_length_stays_below_max_length():
+ args = _run(48, 1024, max_prompt_length = 512)
+ assert args.max_length == 48 and args.max_prompt_length == 24
+ assert _run(2048, 1024, max_prompt_length = 512).max_prompt_length == 512
+
+
+def test_prompt_length_is_left_alone_without_a_clamp():
+ args = _run(2048, 1024, max_prompt_length = 1024)
+ assert args.max_length == 1024 and args.max_prompt_length == 1024
+
+
+def test_unset_prompt_length_stays_below_a_small_clamp():
+ # ORPO / CPO / KTO resolve None to 128, which must stay below max_length.
+ assert _run(64, 1024, max_prompt_length = None).max_prompt_length == 32
+ assert _run(512, 1024, max_prompt_length = None).max_prompt_length is None
+
+
+def test_model_without_a_limit_is_untouched():
+ assert _run(None, 1024).max_length == 1024
+
+
[email protected]("trainer", ["DPOTrainer", "KTOTrainer"])
+def test_patched_trainer_carries_the_clamp(trainer):
+ import unsloth # noqa: F401
+ import trl
+
+ cls = getattr(trl, trainer, None)
+ patched = [k for k in getattr(cls, "__mro__", ()) if "Unsloth" in k.__module__ + k.__name__]
+ if not patched:
+ pytest.skip(f"Unsloth does not patch {trainer} on this TRL")
+ assert any("_unsloth_model_msl" in inspect.getsource(k) for k in patched)
diff --git a/tests/version_compat/test_trl_config_defaults_contract.py b/tests/version_compat/test_trl_config_defaults_contract.py
new file mode 100644
index 00000000000..1d54ef4b8b5
--- /dev/null
+++ b/tests/version_compat/test_trl_config_defaults_contract.py
@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""Unsloth's generated TRL configs may only move the defaults Unsloth means to move.
+
+A stale override drifts silently when TRL changes its own default (top_k after trl#4695, data_seed via the seed rewrite).
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import importlib
+import math
+
+import pytest
+
+
+# Fields Unsloth changes on purpose (rl.py replacements, extra_args, or TRL deriving them from those).
+INTENDED = {
+ "auto_find_batch_size",
+ "beta",
+ "bf16",
+ "dataloader_pin_memory",
+ "dataset_num_proc",
+ "eval_accumulation_steps",
+ "fp16",
+ "generation_kwargs",
+ "gradient_accumulation_steps",
+ "gradient_checkpointing_kwargs",
+ "include_num_input_tokens_seen",
+ "include_tokens_per_second",
+ "learning_rate",
+ "logging_nan_inf_filter",
+ "logging_steps",
+ "loss_type",
+ "num_generations",
+ "optim",
+ "padding_free",
+ "per_device_eval_batch_size",
+ "per_device_train_batch_size",
+ "report_to",
+ "router_aux_loss_coef",
+ "seed",
+ "steps_per_generation",
+ "torch_empty_cache_steps",
+ "vllm_importance_sampling_correction",
+ "vllm_mode",
+ "warmup_ratio",
+ "warmup_steps",
+ "weight_decay",
+}
+
+CONFIGS = [
+ "SFTConfig",
+ "DPOConfig",
+ "GRPOConfig",
+ "RLOOConfig",
+ "KTOConfig",
+ "ORPOConfig",
+ "CPOConfig",
+ "RewardConfig",
+ "GKDConfig",
+ "OnlineDPOConfig",
+]
+
+
+def _pristine(cls):
+ while "_unsloth_patched_rl_config" in cls.__dict__ or cls.__name__.startswith("Unsloth"):
+ cls = cls.__mro__[1]
+ return cls
+
+
+def _config_cls(name):
+ import unsloth # noqa: F401
+ import trl
+
+ cls = getattr(trl, name, None)
+ if cls is None:
+ try:
+ cls = getattr(
+ importlib.import_module("trl.experimental." + name[: -len("Config")].lower()), name
+ )
+ except Exception:
+ return None
+ return cls
+
+
+def _same(a, b):
+ if isinstance(a, float) and isinstance(b, float):
+ return math.isclose(a, b)
+ try:
+ return bool(a == b)
+ except Exception:
+ return repr(a) == repr(b)
+
+
[email protected]("name", CONFIGS)
+def test_only_intended_defaults_differ_from_trl(name):
+ cls = _config_cls(name)
+ if cls is None:
+ pytest.skip(f"this TRL has no {name}")
+ pristine = _pristine(cls)
+ if pristine is cls:
+ pytest.skip(f"Unsloth does not patch {name} on this TRL")
+ ours, theirs = cls(output_dir = "unused"), pristine(output_dir = "unused")
+ moved = {
+ f.name: (getattr(ours, f.name, None), getattr(theirs, f.name, None))
+ for f in dataclasses.fields(theirs)
+ if f.name not in INTENDED
+ and not f.name.endswith("_dir")
+ and not _same(getattr(ours, f.name, None), getattr(theirs, f.name, None))
+ }
+ assert not moved, f"{name} defaults moved away from TRL's (unsloth, trl): {moved}"
+
+
[email protected]("name", ["GRPOConfig", "RLOOConfig", "OnlineDPOConfig"])
+def test_top_k_default_is_trls(name):
+ cls = _config_cls(name)
+ if cls is None or not hasattr(_pristine(cls), "top_k"):
+ pytest.skip(f"this TRL has no {name}.top_k")
+ assert cls(output_dir = "unused").top_k == _pristine(cls)(output_dir = "unused").top_k
+
+
+def test_seed_default_does_not_leak_into_data_seed():
+ cls = _config_cls("SFTConfig")
+ assert cls(output_dir = "unused").data_seed is None
+ assert cls(output_dir = "unused", seed = 1).data_seed is None
+
+
+def _grpo(**kwargs):
+ return _config_cls("GRPOConfig")(output_dir = "unused", **kwargs)
+
+
+def test_dapo_fills_only_unset_recommendations():
+ cfg = _grpo(loss_type = "dapo")
+ assert cfg.epsilon_high == 0.28 and cfg.mask_truncated_completions is True
+ cfg = _grpo(loss_type = "dapo", epsilon_high = 0.2, mask_truncated_completions = False)
+ assert cfg.epsilon_high == 0.2, "an explicit epsilon_high was overwritten"
+ assert (
+ cfg.mask_truncated_completions is False
+ ), "an explicit mask_truncated_completions was overwritten"
+
+
[email protected]("loss_type", ["bnpo", "grpo", "dr_grpo"])
+def test_other_loss_types_keep_trl_mask_default(loss_type):
+ assert _grpo(loss_type = loss_type).mask_truncated_completions is False
+
+
+def test_trl_fields_the_overrides_key_on_are_unchanged():
+ """The loss-type overrides compare against these TRL defaults; a new spelling needs the overrides revisited."""
+ fields = {f.name: f for f in dataclasses.fields(_pristine(_config_cls("GRPOConfig")))}
+ assert fields["scale_rewards"].default in (True, "group"), fields["scale_rewards"].default
+ assert fields["epsilon_high"].default is None, fields["epsilon_high"].default
+ assert fields["mask_truncated_completions"].default is False
+ assert set(("dr_grpo", "dapo")) <= set(
+ _documented_loss_types()
+ ), "TRL dropped a loss_type Unsloth special-cases"
+
+
+def _documented_loss_types():
+ import inspect
+ src = inspect.getsource(_pristine(_config_cls("GRPOConfig")))
+ return [
+ lt
+ for lt in ("grpo", "bnpo", "dr_grpo", "dapo", "cispo", "sapo", "luspo", "vespo")
+ if f'"{lt}"' in src
+ ]
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 6e1b3b274dd..97b21175d83 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2571,6 +2571,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" args.padding_free = False\n"
)
extra_args += max_length_check
+ elif trainer_file in ("dpo_trainer", "kto_trainer", "orpo_trainer", "cpo_trainer"):
+ # The fast forward cuts input_ids to model.max_seq_length but these trainers build labels at args.max_length, so a longer row crashes the logps gather.
+ extra_args += (
+ "_unsloth_model_msl = getattr(model, 'max_seq_length', None)\n"
+ "if isinstance(_unsloth_model_msl, int) and _unsloth_model_msl > 0 and hasattr(args, 'max_length'):\n"
+ " if args.max_length is None or args.max_length > _unsloth_model_msl:\n"
+ " print('Unsloth: `max_length = ' + str(args.max_length) + '` exceeds the model max_seq_length of ' + str(_unsloth_model_msl) + ', so it is reduced to ' + str(_unsloth_model_msl) + '.')\n"
+ " args.max_length = _unsloth_model_msl\n"
+ # ORPO / CPO / KTO resolve a None prompt limit to 128, which must stay below max_length.
+ " _unsloth_mpl = getattr(args, 'max_prompt_length', 0)\n"
+ " if (_unsloth_mpl is None and args.max_length <= 128) or (_unsloth_mpl is not None and _unsloth_mpl >= args.max_length):\n"
+ " args.max_prompt_length = args.max_length // 2\n"
+ )
if "model" in call_args:
training_check = (
@@ -2740,7 +2753,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"num_generations": 8,
# "steps_per_generation": 1, # Otherwise defaults to ga_steps, which is wrong
# "generation_batch_size": None, # Useless. If steps_per_generation is set, generation_batch_size clashes
- "top_k": None,
"vllm_mode": "colocate",
"generation_kwargs": {},
"bf16": False,
@@ -2758,7 +2770,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
replacements["warmup_ratio"] = 0.1
for k, v in replacements.items():
- x = f"{k}( = [^,\n]{{1,}})?,\n"
+ x = rf"\b{k}( = [^,\n]{{1,}})?,\n"
y = f"'{v}'" if type(v) is str else f"{v}"
y = f"{k} = {y},\n"
arguments = re.sub(x, y, arguments)
@@ -2766,16 +2778,18 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
# GRPO beta default is 0.001: TRL used 0.04 and now 0.00. See huggingface/trl#3516 and the verl docs.
if trainer_file == "grpo_trainer":
replacements = {
- "loss_type": "bnpo", # Default GRPO paper
+ "loss_type": "bnpo", # TRL <= 0.21 default; the GRPO paper objective is "grpo"
"beta": 0.001, # Recommended as seen in verl
"auto_find_batch_size": False, # Cannot work on GRPO
# See fengyao.notion.site/off-policy-rl and huggingface/trl#3867.
"vllm_importance_sampling_correction": False,
- # TRL >= 1.7.0 enables the MoE router aux loss by default (0.001), but the optimized GRPO forward does not compute it, so default it off; opt in via router_aux_loss_coef > 0.
+ # TRL >= 1.7.0 enables the MoE router aux loss by default (0.001), but the optimized GRPO forward does not compute it, so default it off.
"router_aux_loss_coef": 0.0,
+ # None = unset, so the dapo check below can tell it from an explicit False.
+ "mask_truncated_completions": None,
}
for k, v in replacements.items():
- x = f"{k}( = [^,\n]{{1,}})?,\n"
+ x = rf"\b{k}( = [^,\n]{{1,}})?,\n"
y = f"'{v}'" if type(v) is str else f"{v}"
y = f"{k} = {y},\n"
arguments = re.sub(x, y, arguments)
@@ -2784,7 +2798,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
if trainer_file == "sft_trainer":
replacements = {"loss_type": "nll"}
for k, v in replacements.items():
- x = f"{k}( = [^,\n]{{1,}})?,\n"
+ x = rf"\b{k}( = [^,\n]{{1,}})?,\n"
y = f"'{v}'" if type(v) is str else f"{v}"
y = f"{k} = {y},\n"
arguments = re.sub(x, y, arguments)
@@ -2869,18 +2883,24 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to keep scaling.')\n"
" scale_rewards = False\n"
"elif loss_type.lower() == 'dapo':\n"
- " if mask_truncated_completions != True:\n"
+ " if mask_truncated_completions is None:\n"
" print('Unsloth: The DAPO paper recommends `mask_truncated_completions = True` - we will set it.')\n"
- " if epsilon_high != 0.28:\n"
+ " mask_truncated_completions = True\n"
+ " if epsilon_high is None:\n"
" print('Unsloth: The DAPO paper recommends `epsilon_high = 0.28` - we will set it.')\n"
+ " epsilon_high = 0.28\n"
" if beta != 0.0:\n"
" print(f'[WARNING] Unsloth: The DAPO paper recommends setting `beta = 0.0` to remove the KL term - You have set it to {beta}.')\n"
- " mask_truncated_completions = True\n"
- " epsilon_high = 0.28\n"
+ "if mask_truncated_completions is None:\n"
+ " mask_truncated_completions = False\n"
"\n"
)
extra_args += check_dr_grpo
+ # GRPO on TRL < 0.20 crashes on the first step (no has_images / images, no reference logps path).
+ if trainer_file == "grpo_trainer" and trl_version < Version("0.20.0"):
+ extra_args += f"raise ImportError('Unsloth: GRPO needs trl >= 0.20.0, but trl=={trl_version} is installed. Please do `pip install --upgrade trl`.')\n"
+
if (
"per_device_train_batch_size" in call_args
and "num_generations" in call_args