Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Patch the TRL trainers that moved to trl.experimental (ORPO, CPO, Online DPO, GKD, ...) (#12097)

Daniel Han69623ef9213e
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
index 5fa53ff28fc..604c9febad2 100644
--- a/.github/workflows/version-compat-ci.yml
+++ b/.github/workflows/version-compat-ci.yml
@@ -332,6 +332,7 @@ jobs:
tests/version_compat/test_trl_config_defaults_contract.py \
tests/version_compat/test_pref_max_length_clamp.py \
tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \
+ tests/version_compat/test_trl_experimental_trainers_patched.py \
-v --tb=short
# `main` is scheduled/dispatch-only so PR jobs stay fast and a bleeding-edge
# TRL break does not red every PR. github.event_name is valid in a step if.
@@ -356,6 +357,7 @@ jobs:
tests/version_compat/test_trl_config_defaults_contract.py \
tests/version_compat/test_pref_max_length_clamp.py \
tests/version_compat/test_grpo_vllm_top_k_from_fast_inference.py \
+ tests/version_compat/test_trl_experimental_trainers_patched.py \
-v --tb=short
- name: Save the pip cache
diff --git a/tests/version_compat/test_trl_experimental_trainers_patched.py b/tests/version_compat/test_trl_experimental_trainers_patched.py
new file mode 100644
index 00000000000..afb74ef016c
--- /dev/null
+++ b/tests/version_compat/test_trl_experimental_trainers_patched.py
@@ -0,0 +1,132 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""TRL 0.29+ moved ORPO, CPO, GKD, PPO, Online DPO, Nash-MD, XPO, BCO and PRM from trl.trainer to trl.experimental.
+
+Each check imports unsloth in a fresh CPU-spoofed interpreter, since trainer patching is process-wide and import-order
+sensitive.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+
+if importlib.util.find_spec("torch") is None or importlib.util.find_spec("trl") is None:
+ pytest.skip("torch and trl are needed to patch the trainers", allow_module_level = True)
+
+_TESTS_DIR = Path(__file__).resolve().parents[1]
+
+# (trl.experimental subpackage, class prefix) for every trainer that used to live in trl.trainer.
+_MOVED = [
+ ("bco", "BCO"),
+ ("cpo", "CPO"),
+ ("gkd", "GKD"),
+ ("nash_md", "NashMD"),
+ ("online_dpo", "OnlineDPO"),
+ ("orpo", "ORPO"),
+ ("ppo", "PPO"),
+ ("prm", "PRM"),
+ ("xpo", "XPO"),
+]
+
+_CHILD = r"""
+import importlib, importlib.util, json, os, sys, warnings
+sys.path.insert(0, TESTS_DIR)
+for k, v in (("UNSLOTH_COMPILE_DISABLE", "1"), ("TORCHDYNAMO_DISABLE", "1"), ("TORCH_COMPILE_DISABLE", "1"),
+ ("TRL_EXPERIMENTAL_SILENCE", "1")):
+ os.environ.setdefault(k, v)
+for name in PRE_IMPORT:
+ importlib.import_module(name)
+# The same CPU-runner harness pytest loads for every test, then the aggressive spoof, as the fake-train tests do.
+spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", os.path.join(TESTS_DIR, "conftest.py"))
+spec.loader.exec_module(importlib.util.module_from_spec(spec))
+import _zoo_aggressive_cuda_spoof as spoof
+spoof.apply()
+import unsloth, trl, trl.trainer
+in_trainer = set(dir(trl.trainer))
+out = {"unsloth": unsloth.__file__, "trl": trl.__version__, "moved": {}}
+for pkg, prefix in MOVED:
+ if f"{pkg}_trainer" in in_trainer or importlib.util.find_spec("trl.experimental") is None:
+ continue
+ try:
+ if importlib.util.find_spec(f"trl.experimental.{pkg}") is None:
+ continue
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ package = importlib.import_module(f"trl.experimental.{pkg}")
+ module = importlib.import_module(f"trl.experimental.{pkg}.{pkg}_trainer")
+ except Exception as e:
+ out["moved"][pkg] = {"import_error": f"{type(e).__name__}: {e}"}
+ continue
+ trainer = getattr(package, f"{prefix}Trainer")
+ config = getattr(package, f"{prefix}Config")
+ out["moved"][pkg] = {
+ "trainer": trainer.__name__,
+ "module_trainer": getattr(module, f"{prefix}Trainer").__name__,
+ "config_has_unsloth_fields": "unsloth_num_chunks" in getattr(config, "__dataclass_fields__", {}),
+ "mro": [f"{c.__module__}.{c.__name__}" for c in trainer.__mro__],
+ "leaked": [x for x in (f"{prefix}Trainer", f"{prefix}Config") if x in vars(trl) or x in vars(trl.trainer)]
+ + ([f"trl.trainer.{pkg}_trainer"] if f"{pkg}_trainer" in vars(trl.trainer) else []),
+ }
+print("RESULT " + json.dumps(out))
+"""
+
+
+def _run(pre_import = ()):
+ code = (
+ f"TESTS_DIR = {str(_TESTS_DIR)!r}\nMOVED = {_MOVED!r}\nPRE_IMPORT = {list(pre_import)!r}\n"
+ + _CHILD
+ )
+ env = dict(os.environ)
+ env["PYTHONPATH"] = os.pathsep.join(p for p in sys.path if p)
+ proc = subprocess.run(
+ [sys.executable, "-c", code],
+ capture_output = True,
+ text = True,
+ env = env,
+ timeout = 900,
+ )
+ lines = [x for x in proc.stdout.splitlines() if x.startswith("RESULT ")]
+ assert proc.returncode == 0 and lines, proc.stdout[-3000:] + proc.stderr[-3000:]
+ result = json.loads(lines[-1][len("RESULT ") :])
+ moved = {k: v for k, v in result["moved"].items() if "import_error" not in v}
+ if not moved:
+ pytest.skip(f"TRL {result['trl']} keeps these trainers in trl.trainer: {result['moved']}")
+ return moved
+
+
[email protected](scope = "module")
+def after_unsloth():
+ return _run()
+
+
+def test_experimental_trainers_and_configs_are_patched(after_unsloth):
+ for pkg, got in after_unsloth.items():
+ prefix = dict(_MOVED)[pkg]
+ assert got["trainer"] == f"Unsloth{prefix}Trainer", (pkg, got)
+ assert got["module_trainer"] == f"Unsloth{prefix}Trainer", (pkg, got)
+ assert got["config_has_unsloth_fields"], (pkg, got)
+
+
+def test_patching_leaves_no_trl_trainer_aliases(after_unsloth):
+ for pkg, got in after_unsloth.items():
+ assert got["leaked"] == [], (pkg, got)
+
+
+def test_gkd_keeps_trls_own_sft_base_whatever_the_import_order(after_unsloth):
+ """GKDTrainer subclasses SFTTrainer; importing it after unsloth must not pick up UnslothSFTTrainer as its base."""
+ if "gkd" not in after_unsloth:
+ pytest.skip("no trl.experimental.gkd on this TRL")
+ before = _run(pre_import = ["trl.experimental.gkd"])["gkd"]
+ for got in (after_unsloth["gkd"], before):
+ assert got["trainer"] == "UnslothGKDTrainer", got
+ assert "trl.trainer.sft_trainer.SFTTrainer" in got["mro"], got
+ assert not any(c.endswith("SFTTrainer") and "Unsloth" in c for c in got["mro"]), got
+ assert after_unsloth["gkd"]["mro"][2:] == before["mro"][2:]
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 0e8c01d1620..51623e5167c 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -18,11 +18,13 @@ import torch
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
import copyreg
import importlib
+import importlib.util
import collections
import inspect
import os
import re
import sys
+import warnings
from contextlib import contextmanager
from unsloth_zoo.compiler import create_new_function
from unsloth_zoo.log import logger
@@ -1993,7 +1995,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
_trainer_src = inspect.getsource(RLTrainer)
_trainer_module = inspect.getmodule(RLTrainer)
_trainer_module_src = inspect.getsource(_trainer_module) if _trainer_module else ""
- if "trl.experimental" in _trainer_src or "trl.experimental" in _trainer_module_src:
+ if not RLTrainer.__module__.startswith("trl.experimental.") and (
+ "trl.experimental" in _trainer_src or "trl.experimental" in _trainer_module_src
+ ):
for _parent in RLTrainer.__mro__[1:]:
if _parent is object:
continue
@@ -2011,7 +2015,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
_config_src = inspect.getsource(RLConfig)
_config_module = inspect.getmodule(RLConfig)
_config_module_src = inspect.getsource(_config_module) if _config_module else ""
- if "trl.experimental" in _config_src or "trl.experimental" in _config_module_src:
+ if not RLConfig.__module__.startswith("trl.experimental.") and (
+ "trl.experimental" in _config_src or "trl.experimental" in _config_module_src
+ ):
for _parent in RLConfig.__mro__[1:]:
if _parent is object:
continue
@@ -3262,7 +3268,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
_resolved_module = _trainer_resolved_module or _config_resolved_module
_model_location = (
- _resolved_module.__name__ if _resolved_module is not None else f"trl.trainer.{trainer_file}"
+ _resolved_module.__name__ if _resolved_module is not None else trainer.__name__
)
created_module = create_new_function(
f"Unsloth{RLTrainer_name}",
@@ -3658,6 +3664,85 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
return RLTrainer_source
+# TRL 0.29+ keeps these only in trl.experimental.<name>.<name>_trainer, which the trl.trainer walk never sees.
+_TRL_EXPERIMENTAL_TRAINERS = (
+ "bco",
+ "cpo",
+ "gkd",
+ "nash_md",
+ "online_dpo",
+ "orpo",
+ "ppo",
+ "prm",
+ "xpo",
+)
+
+
+def _import_trl_experimental_trainers():
+ import trl.trainer
+
+ modules = {}
+ if importlib.util.find_spec("trl.experimental") is None:
+ return modules
+ available = set(dir(trl.trainer))
+ # Silences TRLExperimentalWarning, which the package raises on first import.
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ for name in _TRL_EXPERIMENTAL_TRAINERS:
+ # Still under trl.trainer (a real module, or a 0.26-0.28 shim the walk resolves itself).
+ if f"{name}_trainer" in available:
+ continue
+ try:
+ if importlib.util.find_spec(f"trl.experimental.{name}") is None:
+ continue
+ modules[name] = importlib.import_module(f"trl.experimental.{name}.{name}_trainer")
+ except Exception as e:
+ logger.info(f"Unsloth: Could not import trl.experimental.{name}: {e}")
+ return modules
+
+
+def _patch_trl_experimental_trainer(name, module):
+ # Run the trl.trainer pipeline on the experimental module through temporary trl.trainer aliases, then rebind the experimental namespaces and drop the aliases so trl's public surface is unchanged.
+ import trl
+ import trl.trainer
+
+ trainer_file = f"{name}_trainer"
+ prefix = name.split("_")[0]
+ names = [
+ x
+ for x in dir(module)
+ if (x.endswith("Trainer") or x.endswith("Config"))
+ and x not in ("Trainer", "Config")
+ and not x.startswith("_")
+ and prefix in x.lower()
+ ]
+ _missing = object()
+ aliased = {(trl.trainer, trainer_file): getattr(trl.trainer, trainer_file, _missing)}
+ for x in names:
+ aliased[(trl.trainer, x)] = trl.trainer.__dict__.get(x, _missing)
+ aliased[(trl, x)] = trl.__dict__.get(x, _missing)
+ try:
+ setattr(trl.trainer, trainer_file, module)
+ for x in names:
+ setattr(trl.trainer, x, getattr(module, x))
+ _patch_trl_rl_trainers(trainer_file)
+ patched = {
+ x: getattr(module, x) for x in names if _is_unsloth_patched_config(getattr(module, x))
+ }
+ finally:
+ for (owner, attr), value in aliased.items():
+ if value is _missing:
+ owner.__dict__.pop(attr, None)
+ else:
+ setattr(owner, attr, value)
+ package = sys.modules.get(f"trl.experimental.{name}")
+ config_module = sys.modules.get(f"trl.experimental.{name}.{name}_config")
+ for owner in (package, config_module):
+ for x, cls in patched.items():
+ if owner is not None and hasattr(owner, x):
+ setattr(owner, x, cls)
+
+
def patch_trl_rl_trainers():
import trl.trainer
@@ -3665,11 +3750,18 @@ def patch_trl_rl_trainers():
all_trainers = [
x for x in all_trainers if x.islower() and x.endswith("_trainer") and x != "base_trainer"
]
+ # Imported before the walk rebinds trl.trainer.sft_trainer.SFTTrainer, so GKDTrainer(SFTTrainer) keeps TRL's own base whatever the user's import order.
+ experimental = _import_trl_experimental_trainers()
for trainer in all_trainers:
try:
_patch_trl_rl_trainers(trainer)
except Exception as e:
logger.warning_once(f"Unsloth: Could not patch trl.trainer.{trainer}: {e}")
+ for name, module in experimental.items():
+ try:
+ _patch_trl_experimental_trainer(name, module)
+ except Exception as e:
+ logger.warning_once(f"Unsloth: Could not patch trl.experimental.{name}: {e}")
return