Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fused preference loss test: take TRL's own trainer as the reference (#15269)

Daniel Han24015fcab3e9
diff --git a/tests/test_fused_preference_loss.py b/tests/test_fused_preference_loss.py
index 9af5a96e12c..1b17972ad6f 100644
--- a/tests/test_fused_preference_loss.py
+++ b/tests/test_fused_preference_loss.py
@@ -39,6 +39,47 @@ def _load(name, path):
return module
+_TRL_TRAINER_MODULES = {
+ "DPOTrainer": ("trl.trainer.dpo_trainer",),
+ "ORPOTrainer": ("trl.experimental.orpo.orpo_trainer", "trl.trainer.orpo_trainer"),
+ "CPOTrainer": ("trl.experimental.cpo.cpo_trainer", "trl.trainer.cpo_trainer"),
+}
+
+
+def _trl_trainer(name):
+ """TRL's own trainer class, loaded from its file.
+
+ Unsloth's RL patch rebinds ``trl.<name>`` and the defining module's attribute to its generated
+ ``Unsloth<name>``, whose concatenated_forward already takes the fused path, so importing the
+ name hands back the code under test as its own reference once anything has patched TRL.
+ """
+ for module_name in _TRL_TRAINER_MODULES[name]:
+ try:
+ spec = importlib.util.find_spec(module_name)
+ except ModuleNotFoundError:
+ continue
+ if spec is None or spec.origin is None:
+ continue
+ # Same package, so TRL's relative imports resolve; a private name, so nothing patched reuses it.
+ pristine_name = f"{module_name}_trl_reference_under_test"
+ module = sys.modules.get(pristine_name)
+ if module is None:
+ module = importlib.util.module_from_spec(
+ importlib.util.spec_from_file_location(pristine_name, spec.origin)
+ )
+ sys.modules[pristine_name] = module
+ try:
+ module.__spec__.loader.exec_module(module)
+ except BaseException:
+ sys.modules.pop(pristine_name, None)
+ raise
+ trainer = getattr(module, name, None)
+ if trainer is not None:
+ assert not trainer.__name__.startswith("Unsloth"), trainer
+ return trainer
+ pytest.skip(f"this TRL has no {name}")
+
+
def _zoo_preference_loss():
spec = importlib.util.find_spec("unsloth_zoo")
if spec is None or not spec.submodule_search_locations:
@@ -176,7 +217,7 @@ def _dpo_batch(seed, n = 3):
def _dpo_trainer(module, model, ref_model, fused, **overrides):
- from trl import DPOTrainer
+ DPOTrainer = _trl_trainer("DPOTrainer")
if not hasattr(DPOTrainer, "concatenated_forward"):
pytest.skip("TRL >= 0.29 DPOTrainer has no concatenated_forward")
@@ -409,16 +450,7 @@ def _orpo_batch(seed, n = 3):
def _orpo_trainer(module, model, method, fused, **overrides):
- if method == "orpo":
- try:
- from trl import ORPOTrainer as Trainer
- except ImportError:
- from trl.experimental.orpo import ORPOTrainer as Trainer
- else:
- try:
- from trl import CPOTrainer as Trainer
- except ImportError:
- from trl.experimental.cpo import CPOTrainer as Trainer
+ Trainer = _trl_trainer("ORPOTrainer" if method == "orpo" else "CPOTrainer")
class Stub:
concatenated_inputs = staticmethod(Trainer.concatenated_inputs)