Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Patch the TRL trainers that moved to trl.experimental (ORPO, CPO, Online DPO, GKD, ...) (#12097)
Daniel Han69623ef9213e
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.ymlindex 5fa53ff28fc..604c9febad2 100644--- a/.github/workflows/version-compat-ci.yml+++ b/.github/workflows/version-compat-ci.yml@@ -332,6 +332,7 @@ jobs:tests/version_compat/test_trl_config_defaults_contract.py \tests/version_compat/test_pref_max_length_clamp.py \tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \+ tests/version_compat/test_trl_experimental_trainers_patched.py \-v --tb=short# `main` is scheduled/dispatch-only so PR jobs stay fast and a bleeding-edge# TRL break does not red every PR. github.event_name is valid in a step if.@@ -356,6 +357,7 @@ jobs:tests/version_compat/test_trl_config_defaults_contract.py \tests/version_compat/test_pref_max_length_clamp.py \tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \+ tests/version_compat/test_trl_experimental_trainers_patched.py \-v --tb=short- name: Save the pip cachediff --git a/tests/version_compat/test_trl_experimental_trainers_patched.py b/tests/version_compat/test_trl_experimental_trainers_patched.pynew file mode 100644index 00000000000..afb74ef016c--- /dev/null+++ b/tests/version_compat/test_trl_experimental_trainers_patched.py@@ -0,0 +1,132 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""TRL 0.29+ moved ORPO, CPO, GKD, PPO, Online DPO, Nash-MD, XPO, BCO and PRM from trl.trainer to trl.experimental.++Each check imports unsloth in a fresh CPU-spoofed interpreter, since trainer patching is process-wide and import-order+sensitive.+"""++from __future__ import annotations++import importlib.util+import json+import os+import subprocess+import sys+from pathlib import Path++import pytest+++if importlib.util.find_spec("torch") is None or importlib.util.find_spec("trl") is None:+ pytest.skip("torch and trl are needed to patch the trainers", allow_module_level = True)++_TESTS_DIR = Path(__file__).resolve().parents[1]++# (trl.experimental subpackage, class prefix) for every trainer that used to live in trl.trainer.+_MOVED = [+ ("bco", "BCO"),+ ("cpo", "CPO"),+ ("gkd", "GKD"),+ ("nash_md", "NashMD"),+ ("online_dpo", "OnlineDPO"),+ ("orpo", "ORPO"),+ ("ppo", "PPO"),+ ("prm", "PRM"),+ ("xpo", "XPO"),+]++_CHILD = r"""+import importlib, importlib.util, json, os, sys, warnings+sys.path.insert(0, TESTS_DIR)+for k, v in (("UNSLOTH_COMPILE_DISABLE", "1"), ("TORCHDYNAMO_DISABLE", "1"), ("TORCH_COMPILE_DISABLE", "1"),+ ("TRL_EXPERIMENTAL_SILENCE", "1")):+ os.environ.setdefault(k, v)+for name in PRE_IMPORT:+ importlib.import_module(name)+# The same CPU-runner harness pytest loads for every test, then the aggressive spoof, as the fake-train tests do.+spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", os.path.join(TESTS_DIR, "conftest.py"))+spec.loader.exec_module(importlib.util.module_from_spec(spec))+import _zoo_aggressive_cuda_spoof as spoof+spoof.apply()+import unsloth, trl, trl.trainer+in_trainer = set(dir(trl.trainer))+out = {"unsloth": unsloth.__file__, "trl": trl.__version__, "moved": {}}+for pkg, prefix in MOVED:+ if f"{pkg}_trainer" in in_trainer or importlib.util.find_spec("trl.experimental") is None:+ continue+ try:+ if importlib.util.find_spec(f"trl.experimental.{pkg}") is None:+ continue+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ package = importlib.import_module(f"trl.experimental.{pkg}")+ module = importlib.import_module(f"trl.experimental.{pkg}.{pkg}_trainer")+ except Exception as e:+ out["moved"][pkg] = {"import_error": f"{type(e).__name__}: {e}"}+ continue+ trainer = getattr(package, f"{prefix}Trainer")+ config = getattr(package, f"{prefix}Config")+ out["moved"][pkg] = {+ "trainer": trainer.__name__,+ "module_trainer": getattr(module, f"{prefix}Trainer").__name__,+ "config_has_unsloth_fields": "unsloth_num_chunks" in getattr(config, "__dataclass_fields__", {}),+ "mro": [f"{c.__module__}.{c.__name__}" for c in trainer.__mro__],+ "leaked": [x for x in (f"{prefix}Trainer", f"{prefix}Config") if x in vars(trl) or x in vars(trl.trainer)]+ + ([f"trl.trainer.{pkg}_trainer"] if f"{pkg}_trainer" in vars(trl.trainer) else []),+ }+print("RESULT " + json.dumps(out))+"""+++def _run(pre_import = ()):+ code = (+ f"TESTS_DIR = {str(_TESTS_DIR)!r}\nMOVED = {_MOVED!r}\nPRE_IMPORT = {list(pre_import)!r}\n"+ + _CHILD+ )+ env = dict(os.environ)+ env["PYTHONPATH"] = os.pathsep.join(p for p in sys.path if p)+ proc = subprocess.run(+ [sys.executable, "-c", code],+ capture_output = True,+ text = True,+ env = env,+ timeout = 900,+ )+ lines = [x for x in proc.stdout.splitlines() if x.startswith("RESULT ")]+ assert proc.returncode == 0 and lines, proc.stdout[-3000:] + proc.stderr[-3000:]+ result = json.loads(lines[-1][len("RESULT ") :])+ moved = {k: v for k, v in result["moved"].items() if "import_error" not in v}+ if not moved:+ pytest.skip(f"TRL {result['trl']} keeps these trainers in trl.trainer: {result['moved']}")+ return moved++[email protected](scope = "module")+def after_unsloth():+ return _run()+++def test_experimental_trainers_and_configs_are_patched(after_unsloth):+ for pkg, got in after_unsloth.items():+ prefix = dict(_MOVED)[pkg]+ assert got["trainer"] == f"Unsloth{prefix}Trainer", (pkg, got)+ assert got["module_trainer"] == f"Unsloth{prefix}Trainer", (pkg, got)+ assert got["config_has_unsloth_fields"], (pkg, got)+++def test_patching_leaves_no_trl_trainer_aliases(after_unsloth):+ for pkg, got in after_unsloth.items():+ assert got["leaked"] == [], (pkg, got)+++def test_gkd_keeps_trls_own_sft_base_whatever_the_import_order(after_unsloth):+ """GKDTrainer subclasses SFTTrainer; importing it after unsloth must not pick up UnslothSFTTrainer as its base."""+ if "gkd" not in after_unsloth:+ pytest.skip("no trl.experimental.gkd on this TRL")+ before = _run(pre_import = ["trl.experimental.gkd"])["gkd"]+ for got in (after_unsloth["gkd"], before):+ assert got["trainer"] == "UnslothGKDTrainer", got+ assert "trl.trainer.sft_trainer.SFTTrainer" in got["mro"], got+ assert not any(c.endswith("SFTTrainer") and "Unsloth" in c for c in got["mro"]), got+ assert after_unsloth["gkd"]["mro"][2:] == before["mro"][2:]diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 0e8c01d1620..51623e5167c 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -18,11 +18,13 @@ import torchfrom typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Unionimport copyregimport importlib+import importlib.utilimport collectionsimport inspectimport osimport reimport sys+import warningsfrom contextlib import contextmanagerfrom unsloth_zoo.compiler import create_new_functionfrom unsloth_zoo.log import logger@@ -1993,7 +1995,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):_trainer_src = inspect.getsource(RLTrainer)_trainer_module = inspect.getmodule(RLTrainer)_trainer_module_src = inspect.getsource(_trainer_module) if _trainer_module else ""- if "trl.experimental" in _trainer_src or "trl.experimental" in _trainer_module_src:+ if not RLTrainer.__module__.startswith("trl.experimental.") and (+ "trl.experimental" in _trainer_src or "trl.experimental" in _trainer_module_src+ ):for _parent in RLTrainer.__mro__[1:]:if _parent is object:continue@@ -2011,7 +2015,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):_config_src = inspect.getsource(RLConfig)_config_module = inspect.getmodule(RLConfig)_config_module_src = inspect.getsource(_config_module) if _config_module else ""- if "trl.experimental" in _config_src or "trl.experimental" in _config_module_src:+ if not RLConfig.__module__.startswith("trl.experimental.") and (+ "trl.experimental" in _config_src or "trl.experimental" in _config_module_src+ ):for _parent in RLConfig.__mro__[1:]:if _parent is object:continue@@ -3262,7 +3268,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):_resolved_module = _trainer_resolved_module or _config_resolved_module_model_location = (- _resolved_module.__name__ if _resolved_module is not None else f"trl.trainer.{trainer_file}"+ _resolved_module.__name__ if _resolved_module is not None else trainer.__name__)created_module = create_new_function(f"Unsloth{RLTrainer_name}",@@ -3658,6 +3664,85 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importreturn RLTrainer_source+# TRL 0.29+ keeps these only in trl.experimental.<name>.<name>_trainer, which the trl.trainer walk never sees.+_TRL_EXPERIMENTAL_TRAINERS = (+ "bco",+ "cpo",+ "gkd",+ "nash_md",+ "online_dpo",+ "orpo",+ "ppo",+ "prm",+ "xpo",+)+++def _import_trl_experimental_trainers():+ import trl.trainer++ modules = {}+ if importlib.util.find_spec("trl.experimental") is None:+ return modules+ available = set(dir(trl.trainer))+ # Silences TRLExperimentalWarning, which the package raises on first import.+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ for name in _TRL_EXPERIMENTAL_TRAINERS:+ # Still under trl.trainer (a real module, or a 0.26-0.28 shim the walk resolves itself).+ if f"{name}_trainer" in available:+ continue+ try:+ if importlib.util.find_spec(f"trl.experimental.{name}") is None:+ continue+ modules[name] = importlib.import_module(f"trl.experimental.{name}.{name}_trainer")+ except Exception as e:+ logger.info(f"Unsloth: Could not import trl.experimental.{name}: {e}")+ return modules+++def _patch_trl_experimental_trainer(name, module):+ # Run the trl.trainer pipeline on the experimental module through temporary trl.trainer aliases, then rebind the experimental namespaces and drop the aliases so trl's public surface is unchanged.+ import trl+ import trl.trainer++ trainer_file = f"{name}_trainer"+ prefix = name.split("_")[0]+ names = [+ x+ for x in dir(module)+ if (x.endswith("Trainer") or x.endswith("Config"))+ and x not in ("Trainer", "Config")+ and not x.startswith("_")+ and prefix in x.lower()+ ]+ _missing = object()+ aliased = {(trl.trainer, trainer_file): getattr(trl.trainer, trainer_file, _missing)}+ for x in names:+ aliased[(trl.trainer, x)] = trl.trainer.__dict__.get(x, _missing)+ aliased[(trl, x)] = trl.__dict__.get(x, _missing)+ try:+ setattr(trl.trainer, trainer_file, module)+ for x in names:+ setattr(trl.trainer, x, getattr(module, x))+ _patch_trl_rl_trainers(trainer_file)+ patched = {+ x: getattr(module, x) for x in names if _is_unsloth_patched_config(getattr(module, x))+ }+ finally:+ for (owner, attr), value in aliased.items():+ if value is _missing:+ owner.__dict__.pop(attr, None)+ else:+ setattr(owner, attr, value)+ package = sys.modules.get(f"trl.experimental.{name}")+ config_module = sys.modules.get(f"trl.experimental.{name}.{name}_config")+ for owner in (package, config_module):+ for x, cls in patched.items():+ if owner is not None and hasattr(owner, x):+ setattr(owner, x, cls)++def patch_trl_rl_trainers():import trl.trainer@@ -3665,11 +3750,18 @@ def patch_trl_rl_trainers():all_trainers = [x for x in all_trainers if x.islower() and x.endswith("_trainer") and x != "base_trainer"]+ # Imported before the walk rebinds trl.trainer.sft_trainer.SFTTrainer, so GKDTrainer(SFTTrainer) keeps TRL's own base whatever the user's import order.+ experimental = _import_trl_experimental_trainers()for trainer in all_trainers:try:_patch_trl_rl_trainers(trainer)except Exception as e:logger.warning_once(f"Unsloth: Could not patch trl.trainer.{trainer}: {e}")+ for name, module in experimental.items():+ try:+ _patch_trl_experimental_trainer(name, module)+ except Exception as e:+ logger.warning_once(f"Unsloth: Could not patch trl.experimental.{name}: {e}")return