Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Patch TRL's served-teacher, GOLD and SDFT distillation trainers (#13580)
Daniel Hana43cf778965d
diff --git a/tests/version_compat/test_trl_experimental_trainers_patched.py b/tests/version_compat/test_trl_experimental_trainers_patched.pyindex afb74ef016c..3fe45d10378 100644--- a/tests/version_compat/test_trl_experimental_trainers_patched.py+++ b/tests/version_compat/test_trl_experimental_trainers_patched.py@@ -1,6 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team.-"""TRL 0.29+ moved ORPO, CPO, GKD, PPO, Online DPO, Nash-MD, XPO, BCO and PRM from trl.trainer to trl.experimental.+"""TRL 0.29+ moved ORPO, CPO, GKD, PPO, Online DPO, Nash-MD, XPO, BCO and PRM from trl.trainer to trl.experimental, and the+served-teacher, GOLD and SDFT distillation trainers only ever lived there.Each check imports unsloth in a fresh CPU-spoofed interpreter, since trainer patching is process-wide and import-ordersensitive.@@ -28,11 +29,14 @@ _MOVED = [("bco", "BCO"),("cpo", "CPO"),("gkd", "GKD"),+ ("gold", "GOLD"),("nash_md", "NashMD"),("online_dpo", "OnlineDPO"),("orpo", "ORPO"),("ppo", "PPO"),("prm", "PRM"),+ ("sdft", "SDFT"),+ ("server_distillation", "ServerDistillation"),("xpo", "XPO"),]@@ -49,6 +53,7 @@ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", os.pathspec.loader.exec_module(importlib.util.module_from_spec(spec))import _zoo_aggressive_cuda_spoof as spoofspoof.apply()+import inspect, reimport unsloth, trl, trl.trainerin_trainer = set(dir(trl.trainer))out = {"unsloth": unsloth.__file__, "trl": trl.__version__, "moved": {}}@@ -72,9 +77,29 @@ for pkg, prefix in MOVED:"module_trainer": getattr(module, f"{prefix}Trainer").__name__,"config_has_unsloth_fields": "unsloth_num_chunks" in getattr(config, "__dataclass_fields__", {}),"mro": [f"{c.__module__}.{c.__name__}" for c in trainer.__mro__],+ "num_generations": [+ getattr(inspect.signature(config.__init__).parameters.get("num_generations"), "default", None),+ getattr(getattr(config.__mro__[1], "__dataclass_fields__", {}).get("num_generations"), "default", None),+ ],+ "relative_imports": re.findall(r"^\s*from \.+[\w.]* import .*$", inspect.getsource(sys.modules[trainer.__module__]), re.M),"leaked": [x for x in (f"{prefix}Trainer", f"{prefix}Config") if x in vars(trl) or x in vars(trl.trainer)]+ ([f"trl.trainer.{pkg}_trainer"] if f"{pkg}_trainer" in vars(trl.trainer) else []),}+def _config_probe(pkg, prefix, **kwargs):+ try:+ cfg = getattr(importlib.import_module(f"trl.experimental.{pkg}"), f"{prefix}Config")(+ output_dir = "unused", report_to = "none", **kwargs+ )+ return {"per_device_train_batch_size": cfg.per_device_train_batch_size}+ except Exception as e:+ return {"error": f"{type(e).__name__}: {e}"}+if "gold" in out["moved"]:+ out["gold_config"] = _config_probe(+ "gold", "GOLD", num_generations = 8, per_device_train_batch_size = 4,+ gradient_accumulation_steps = 2, lmbda = 1.0,+ )+if "sdft" in out["moved"]:+ out["sdft_eval_config"] = _config_probe("sdft", "SDFT", eval_strategy = "steps", eval_steps = 1)print("RESULT " + json.dumps(out))"""@@ -97,6 +122,9 @@ def _run(pre_import = ()):assert proc.returncode == 0 and lines, proc.stdout[-3000:] + proc.stderr[-3000:]result = json.loads(lines[-1][len("RESULT ") :])moved = {k: v for k, v in result["moved"].items() if "import_error" not in v}+ for pkg, key in (("gold", "gold_config"), ("sdft", "sdft_eval_config")):+ if pkg in moved:+ moved[pkg]["config_probe"] = result.get(key)if not moved:pytest.skip(f"TRL {result['trl']} keeps these trainers in trl.trainer: {result['moved']}")return moved@@ -115,6 +143,11 @@ def test_experimental_trainers_and_configs_are_patched(after_unsloth):assert got["config_has_unsloth_fields"], (pkg, got)+def test_generated_trainers_have_no_relative_imports(after_unsloth):+ for pkg, got in after_unsloth.items():+ assert got["relative_imports"] == [], (pkg, got["relative_imports"])++def test_patching_leaves_no_trl_trainer_aliases(after_unsloth):for pkg, got in after_unsloth.items():assert got["leaked"] == [], (pkg, got)@@ -130,3 +163,41 @@ def test_gkd_keeps_trls_own_sft_base_whatever_the_import_order(after_unsloth):assert "trl.trainer.sft_trainer.SFTTrainer" in got["mro"], gotassert not any(c.endswith("SFTTrainer") and "Unsloth" in c for c in got["mro"]), gotassert after_unsloth["gkd"]["mro"][2:] == before["mro"][2:]+++ "pkg, parent", [("server_distillation", "DistillationTrainer"), ("gold", "SFTTrainer")]+)+def test_distillation_trainers_keep_trls_own_base_whatever_the_import_order(+ after_unsloth, pkg, parent+):+ if pkg not in after_unsloth:+ pytest.skip(f"no trl.experimental.{pkg} on this TRL")+ before = _run(pre_import = [f"trl.experimental.{pkg}"])[pkg]+ for got in (after_unsloth[pkg], before):+ assert any(+ c.startswith("trl.trainer.") and c.endswith(f".{parent}") for c in got["mro"]+ ), got+ assert not any(c.endswith(parent) and "Unsloth" in c for c in got["mro"][2:]), got+ assert after_unsloth[pkg]["mro"][2:] == before["mro"][2:]+++def test_gold_keeps_trls_num_generations_default(after_unsloth):+ if "gold" not in after_unsloth:+ pytest.skip("no trl.experimental.gold on this TRL")+ patched, pristine = after_unsloth["gold"]["num_generations"]+ assert patched == pristine, after_unsloth["gold"]["num_generations"]+++def test_gold_keeps_a_valid_batch_size(after_unsloth):+ got = after_unsloth.get("gold")+ if got is None:+ pytest.skip("no trl.experimental.gold on this TRL")+ assert got["config_probe"] == {"per_device_train_batch_size": 4}, got["config_probe"]+++def test_sdft_config_with_evaluation_still_builds(after_unsloth):+ got = after_unsloth.get("sdft")+ if got is None:+ pytest.skip("no trl.experimental.sdft on this TRL")+ assert "error" not in got["config_probe"], got["config_probe"]diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 3991e81211c..bbecab934ff 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2359,6 +2359,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):v = re.escape("\n")if v is EMPTY:arguments.append(k)+ elif type(v) is str and v != re.escape("\n") and ("\n" in v or "'" in v):+ # A newline or quote ends the bare '...' literal early (SDFT's teacher_prompt_template).+ arguments.append(f"{k} = {v!r}")elif type(v) is str:arguments.append(f"{k} = '{v}'")elif type(v) in types:@@ -3078,6 +3081,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):replacements["warmup_steps"] = 0.1else:replacements["warmup_ratio"] = 0.1+ # GOLD's num_generations is samples per prompt (default 1), not a GRPO group.+ if trainer_file == "gold_trainer":+ replacements.pop("num_generations")+ # SDFT needs the eval batch divisible by num_generations (8).+ if trainer_file == "sdft_trainer":+ replacements.pop("per_device_eval_batch_size")for k, v in replacements.items():x = rf"\b{k}( = [^,\n]{{1,}})?,\n"@@ -3290,7 +3299,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"\n")extra_args += check_num_generations- elif "per_device_train_batch_size" in call_args and "num_generations" in call_args:+ # GOLD: TRL already checks generation_batch_size * num_generations == batch * grad accum.+ elif (+ "per_device_train_batch_size" in call_args+ and "num_generations" in call_args+ and trainer_file != "gold_trainer"+ ):if "steps_per_generation" not in call_args:print(f"Unsloth: Could not find `steps_per_generation` in {trainer_file}")if "generation_batch_size" not in call_args:@@ -3604,6 +3618,18 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):RLTrainer_source = re.sub(r"[\n]{3,}", "\n", RLTrainer_source)+ # The generated module has no package: make function-level relative imports (ServerDistillationTrainer.__init__) absolute.+ _source_package = getattr(inspect.getmodule(RLTrainer), "__package__", None)+ if _source_package:+ RLTrainer_source = re.sub(+ r"^([ \t]*from[ \t]+)(\.+[\w.]*)([ \t]+import[ \t])",+ lambda m: m.group(1)+ + importlib.util.resolve_name(m.group(2), _source_package)+ + m.group(3),+ RLTrainer_source,+ flags = re.MULTILINE,+ )+_resolved_module = _trainer_resolved_module or _config_resolved_module_model_location = (_resolved_module.__name__ if _resolved_module is not None else trainer.__name__@@ -4051,15 +4077,19 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import# TRL 0.29+ keeps these only in trl.experimental.<name>.<name>_trainer, which the trl.trainer walk never sees.+# Not async_distillation: it loads the student from a model name, so an Unsloth model never reaches it._TRL_EXPERIMENTAL_TRAINERS = ("bco","cpo","gkd",+ "gold","nash_md","online_dpo","orpo","ppo","prm",+ "sdft",+ "server_distillation","xpo",)