Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep TRL's own RL config defaults and clamp preference max_length (#12069)
Daniel Hanec9cc2c4e0e1
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.ymlindex db8ed31a805..5fa53ff28fc 100644--- a/.github/workflows/version-compat-ci.yml+++ b/.github/workflows/version-compat-ci.yml@@ -329,6 +329,8 @@ jobs:tests/version_compat/test_trl_fake_train_cpu.py \tests/version_compat/test_trl_padding_free_max_length.py \tests/version_compat/test_trl_loss_normalization_contract.py \+ tests/version_compat/test_trl_config_defaults_contract.py \+ tests/version_compat/test_pref_max_length_clamp.py \tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \-v --tb=short# `main` is scheduled/dispatch-only so PR jobs stay fast and a bleeding-edge@@ -351,6 +353,8 @@ jobs:tests/version_compat/test_trl_fake_train_cpu.py \tests/version_compat/test_trl_padding_free_max_length.py \tests/version_compat/test_trl_loss_normalization_contract.py \+ tests/version_compat/test_trl_config_defaults_contract.py \+ tests/version_compat/test_pref_max_length_clamp.py \tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \-v --tb=shortdiff --git a/tests/version_compat/test_pref_max_length_clamp.py b/tests/version_compat/test_pref_max_length_clamp.pynew file mode 100644index 00000000000..65d34828a9b--- /dev/null+++ b/tests/version_compat/test_pref_max_length_clamp.py@@ -0,0 +1,87 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""Preference-trainer rows longer than the model max_seq_length must not crash the log-prob gather.++The fast forward cuts input_ids to model.max_seq_length while TRL builds labels at args.max_length.+"""++from __future__ import annotations++import ast+import importlib.util+import inspect+import re+from pathlib import Path+from types import SimpleNamespace++import pytest+++RL_PY = Path(importlib.util.find_spec("unsloth").origin).parent / "models" / "rl.py"+++def _clamp_snippet():+ src = RL_PY.read_text(encoding = "utf-8")+ m = re.search(+ r'elif trainer_file in \("dpo_trainer", "kto_trainer", "orpo_trainer", "cpo_trainer"\):\n(.*?)\n \)\n',+ src,+ re.S,+ )+ assert m, "no preference-trainer max_length clamp in rl.py"+ body = m.group(1).split("extra_args += (", 1)[1]+ return "".join(+ ast.literal_eval(line.strip()) for line in body.splitlines() if line.strip().startswith('"')+ )+++def _run(+ max_seq_length,+ max_length,+ max_prompt_length = None,+):+ model = SimpleNamespace(max_seq_length = max_seq_length)+ args = SimpleNamespace(max_length = max_length, max_prompt_length = max_prompt_length)+ exec(_clamp_snippet(), {"model": model, "args": args, "print": lambda *a: None})+ return args++[email protected]("max_length", [1024, None])+def test_max_length_is_capped_at_the_model_limit(max_length):+ assert _run(48, max_length).max_length == 48+++def test_a_shorter_max_length_is_kept():+ assert _run(2048, 1024).max_length == 1024+++def test_prompt_length_stays_below_max_length():+ args = _run(48, 1024, max_prompt_length = 512)+ assert args.max_length == 48 and args.max_prompt_length == 24+ assert _run(2048, 1024, max_prompt_length = 512).max_prompt_length == 512+++def test_prompt_length_is_left_alone_without_a_clamp():+ args = _run(2048, 1024, max_prompt_length = 1024)+ assert args.max_length == 1024 and args.max_prompt_length == 1024+++def test_unset_prompt_length_stays_below_a_small_clamp():+ # ORPO / CPO / KTO resolve None to 128, which must stay below max_length.+ assert _run(64, 1024, max_prompt_length = None).max_prompt_length == 32+ assert _run(512, 1024, max_prompt_length = None).max_prompt_length is None+++def test_model_without_a_limit_is_untouched():+ assert _run(None, 1024).max_length == 1024++[email protected]("trainer", ["DPOTrainer", "KTOTrainer"])+def test_patched_trainer_carries_the_clamp(trainer):+ import unsloth # noqa: F401+ import trl++ cls = getattr(trl, trainer, None)+ patched = [k for k in getattr(cls, "__mro__", ()) if "Unsloth" in k.__module__ + k.__name__]+ if not patched:+ pytest.skip(f"Unsloth does not patch {trainer} on this TRL")+ assert any("_unsloth_model_msl" in inspect.getsource(k) for k in patched)diff --git a/tests/version_compat/test_trl_config_defaults_contract.py b/tests/version_compat/test_trl_config_defaults_contract.pynew file mode 100644index 00000000000..1d54ef4b8b5--- /dev/null+++ b/tests/version_compat/test_trl_config_defaults_contract.py@@ -0,0 +1,166 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""Unsloth's generated TRL configs may only move the defaults Unsloth means to move.++A stale override drifts silently when TRL changes its own default (top_k after trl#4695, data_seed via the seed rewrite).+"""++from __future__ import annotations++import dataclasses+import importlib+import math++import pytest+++# Fields Unsloth changes on purpose (rl.py replacements, extra_args, or TRL deriving them from those).+INTENDED = {+ "auto_find_batch_size",+ "beta",+ "bf16",+ "dataloader_pin_memory",+ "dataset_num_proc",+ "eval_accumulation_steps",+ "fp16",+ "generation_kwargs",+ "gradient_accumulation_steps",+ "gradient_checkpointing_kwargs",+ "include_num_input_tokens_seen",+ "include_tokens_per_second",+ "learning_rate",+ "logging_nan_inf_filter",+ "logging_steps",+ "loss_type",+ "num_generations",+ "optim",+ "padding_free",+ "per_device_eval_batch_size",+ "per_device_train_batch_size",+ "report_to",+ "router_aux_loss_coef",+ "seed",+ "steps_per_generation",+ "torch_empty_cache_steps",+ "vllm_importance_sampling_correction",+ "vllm_mode",+ "warmup_ratio",+ "warmup_steps",+ "weight_decay",+}++CONFIGS = [+ "SFTConfig",+ "DPOConfig",+ "GRPOConfig",+ "RLOOConfig",+ "KTOConfig",+ "ORPOConfig",+ "CPOConfig",+ "RewardConfig",+ "GKDConfig",+ "OnlineDPOConfig",+]+++def _pristine(cls):+ while "_unsloth_patched_rl_config" in cls.__dict__ or cls.__name__.startswith("Unsloth"):+ cls = cls.__mro__[1]+ return cls+++def _config_cls(name):+ import unsloth # noqa: F401+ import trl++ cls = getattr(trl, name, None)+ if cls is None:+ try:+ cls = getattr(+ importlib.import_module("trl.experimental." + name[: -len("Config")].lower()), name+ )+ except Exception:+ return None+ return cls+++def _same(a, b):+ if isinstance(a, float) and isinstance(b, float):+ return math.isclose(a, b)+ try:+ return bool(a == b)+ except Exception:+ return repr(a) == repr(b)++[email protected]("name", CONFIGS)+def test_only_intended_defaults_differ_from_trl(name):+ cls = _config_cls(name)+ if cls is None:+ pytest.skip(f"this TRL has no {name}")+ pristine = _pristine(cls)+ if pristine is cls:+ pytest.skip(f"Unsloth does not patch {name} on this TRL")+ ours, theirs = cls(output_dir = "unused"), pristine(output_dir = "unused")+ moved = {+ f.name: (getattr(ours, f.name, None), getattr(theirs, f.name, None))+ for f in dataclasses.fields(theirs)+ if f.name not in INTENDED+ and not f.name.endswith("_dir")+ and not _same(getattr(ours, f.name, None), getattr(theirs, f.name, None))+ }+ assert not moved, f"{name} defaults moved away from TRL's (unsloth, trl): {moved}"++[email protected]("name", ["GRPOConfig", "RLOOConfig", "OnlineDPOConfig"])+def test_top_k_default_is_trls(name):+ cls = _config_cls(name)+ if cls is None or not hasattr(_pristine(cls), "top_k"):+ pytest.skip(f"this TRL has no {name}.top_k")+ assert cls(output_dir = "unused").top_k == _pristine(cls)(output_dir = "unused").top_k+++def test_seed_default_does_not_leak_into_data_seed():+ cls = _config_cls("SFTConfig")+ assert cls(output_dir = "unused").data_seed is None+ assert cls(output_dir = "unused", seed = 1).data_seed is None+++def _grpo(**kwargs):+ return _config_cls("GRPOConfig")(output_dir = "unused", **kwargs)+++def test_dapo_fills_only_unset_recommendations():+ cfg = _grpo(loss_type = "dapo")+ assert cfg.epsilon_high == 0.28 and cfg.mask_truncated_completions is True+ cfg = _grpo(loss_type = "dapo", epsilon_high = 0.2, mask_truncated_completions = False)+ assert cfg.epsilon_high == 0.2, "an explicit epsilon_high was overwritten"+ assert (+ cfg.mask_truncated_completions is False+ ), "an explicit mask_truncated_completions was overwritten"++[email protected]("loss_type", ["bnpo", "grpo", "dr_grpo"])+def test_other_loss_types_keep_trl_mask_default(loss_type):+ assert _grpo(loss_type = loss_type).mask_truncated_completions is False+++def test_trl_fields_the_overrides_key_on_are_unchanged():+ """The loss-type overrides compare against these TRL defaults; a new spelling needs the overrides revisited."""+ fields = {f.name: f for f in dataclasses.fields(_pristine(_config_cls("GRPOConfig")))}+ assert fields["scale_rewards"].default in (True, "group"), fields["scale_rewards"].default+ assert fields["epsilon_high"].default is None, fields["epsilon_high"].default+ assert fields["mask_truncated_completions"].default is False+ assert set(("dr_grpo", "dapo")) <= set(+ _documented_loss_types()+ ), "TRL dropped a loss_type Unsloth special-cases"+++def _documented_loss_types():+ import inspect+ src = inspect.getsource(_pristine(_config_cls("GRPOConfig")))+ return [+ lt+ for lt in ("grpo", "bnpo", "dr_grpo", "dapo", "cispo", "sapo", "luspo", "vespo")+ if f'"{lt}"' in src+ ]diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 6e1b3b274dd..97b21175d83 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2571,6 +2571,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" args.padding_free = False\n")extra_args += max_length_check+ elif trainer_file in ("dpo_trainer", "kto_trainer", "orpo_trainer", "cpo_trainer"):+ # The fast forward cuts input_ids to model.max_seq_length but these trainers build labels at args.max_length, so a longer row crashes the logps gather.+ extra_args += (+ "_unsloth_model_msl = getattr(model, 'max_seq_length', None)\n"+ "if isinstance(_unsloth_model_msl, int) and _unsloth_model_msl > 0 and hasattr(args, 'max_length'):\n"+ " if args.max_length is None or args.max_length > _unsloth_model_msl:\n"+ " print('Unsloth: `max_length = ' + str(args.max_length) + '` exceeds the model max_seq_length of ' + str(_unsloth_model_msl) + ', so it is reduced to ' + str(_unsloth_model_msl) + '.')\n"+ " args.max_length = _unsloth_model_msl\n"+ # ORPO / CPO / KTO resolve a None prompt limit to 128, which must stay below max_length.+ " _unsloth_mpl = getattr(args, 'max_prompt_length', 0)\n"+ " if (_unsloth_mpl is None and args.max_length <= 128) or (_unsloth_mpl is not None and _unsloth_mpl >= args.max_length):\n"+ " args.max_prompt_length = args.max_length // 2\n"+ )if "model" in call_args:training_check = (@@ -2740,7 +2753,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"num_generations": 8,# "steps_per_generation": 1, # Otherwise defaults to ga_steps, which is wrong# "generation_batch_size": None, # Useless. If steps_per_generation is set, generation_batch_size clashes- "top_k": None,"vllm_mode": "colocate","generation_kwargs": {},"bf16": False,@@ -2758,7 +2770,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):replacements["warmup_ratio"] = 0.1for k, v in replacements.items():- x = f"{k}( = [^,\n]{{1,}})?,\n"+ x = rf"\b{k}( = [^,\n]{{1,}})?,\n"y = f"'{v}'" if type(v) is str else f"{v}"y = f"{k} = {y},\n"arguments = re.sub(x, y, arguments)@@ -2766,16 +2778,18 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):# GRPO beta default is 0.001: TRL used 0.04 and now 0.00. See huggingface/trl#3516 and the verl docs.if trainer_file == "grpo_trainer":replacements = {- "loss_type": "bnpo", # Default GRPO paper+ "loss_type": "bnpo", # TRL <= 0.21 default; the GRPO paper objective is "grpo""beta": 0.001, # Recommended as seen in verl"auto_find_batch_size": False, # Cannot work on GRPO# See fengyao.notion.site/off-policy-rl and huggingface/trl#3867."vllm_importance_sampling_correction": False,- # TRL >= 1.7.0 enables the MoE router aux loss by default (0.001), but the optimized GRPO forward does not compute it, so default it off; opt in via router_aux_loss_coef > 0.+ # TRL >= 1.7.0 enables the MoE router aux loss by default (0.001), but the optimized GRPO forward does not compute it, so default it off."router_aux_loss_coef": 0.0,+ # None = unset, so the dapo check below can tell it from an explicit False.+ "mask_truncated_completions": None,}for k, v in replacements.items():- x = f"{k}( = [^,\n]{{1,}})?,\n"+ x = rf"\b{k}( = [^,\n]{{1,}})?,\n"y = f"'{v}'" if type(v) is str else f"{v}"y = f"{k} = {y},\n"arguments = re.sub(x, y, arguments)@@ -2784,7 +2798,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):if trainer_file == "sft_trainer":replacements = {"loss_type": "nll"}for k, v in replacements.items():- x = f"{k}( = [^,\n]{{1,}})?,\n"+ x = rf"\b{k}( = [^,\n]{{1,}})?,\n"y = f"'{v}'" if type(v) is str else f"{v}"y = f"{k} = {y},\n"arguments = re.sub(x, y, arguments)@@ -2869,18 +2883,24 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to keep scaling.')\n"" scale_rewards = False\n""elif loss_type.lower() == 'dapo':\n"- " if mask_truncated_completions != True:\n"+ " if mask_truncated_completions is None:\n"" print('Unsloth: The DAPO paper recommends `mask_truncated_completions = True` - we will set it.')\n"- " if epsilon_high != 0.28:\n"+ " mask_truncated_completions = True\n"+ " if epsilon_high is None:\n"" print('Unsloth: The DAPO paper recommends `epsilon_high = 0.28` - we will set it.')\n"+ " epsilon_high = 0.28\n"" if beta != 0.0:\n"" print(f'[WARNING] Unsloth: The DAPO paper recommends setting `beta = 0.0` to remove the KL term - You have set it to {beta}.')\n"- " mask_truncated_completions = True\n"- " epsilon_high = 0.28\n"+ "if mask_truncated_completions is None:\n"+ " mask_truncated_completions = False\n""\n")extra_args += check_dr_grpo+ # GRPO on TRL < 0.20 crashes on the first step (no has_images / images, no reference logps path).+ if trainer_file == "grpo_trainer" and trl_version < Version("0.20.0"):+ extra_args += f"raise ImportError('Unsloth: GRPO needs trl >= 0.20.0, but trl=={trl_version} is installed. Please do `pip install --upgrade trl`.')\n"+if ("per_device_train_batch_size" in call_argsand "num_generations" in call_args