Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Patch TRL's served-teacher, GOLD and SDFT distillation trainers (#13580)

Daniel Hana43cf778965d
diff --git a/tests/version_compat/test_trl_experimental_trainers_patched.py b/tests/version_compat/test_trl_experimental_trainers_patched.py
index afb74ef016c..3fe45d10378 100644
--- a/tests/version_compat/test_trl_experimental_trainers_patched.py
+++ b/tests/version_compat/test_trl_experimental_trainers_patched.py
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team.
-"""TRL 0.29+ moved ORPO, CPO, GKD, PPO, Online DPO, Nash-MD, XPO, BCO and PRM from trl.trainer to trl.experimental.
+"""TRL 0.29+ moved ORPO, CPO, GKD, PPO, Online DPO, Nash-MD, XPO, BCO and PRM from trl.trainer to trl.experimental, and the
+served-teacher, GOLD and SDFT distillation trainers only ever lived there.
Each check imports unsloth in a fresh CPU-spoofed interpreter, since trainer patching is process-wide and import-order
sensitive.
@@ -28,11 +29,14 @@ _MOVED = [
("bco", "BCO"),
("cpo", "CPO"),
("gkd", "GKD"),
+ ("gold", "GOLD"),
("nash_md", "NashMD"),
("online_dpo", "OnlineDPO"),
("orpo", "ORPO"),
("ppo", "PPO"),
("prm", "PRM"),
+ ("sdft", "SDFT"),
+ ("server_distillation", "ServerDistillation"),
("xpo", "XPO"),
]
@@ -49,6 +53,7 @@ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", os.path
spec.loader.exec_module(importlib.util.module_from_spec(spec))
import _zoo_aggressive_cuda_spoof as spoof
spoof.apply()
+import inspect, re
import unsloth, trl, trl.trainer
in_trainer = set(dir(trl.trainer))
out = {"unsloth": unsloth.__file__, "trl": trl.__version__, "moved": {}}
@@ -72,9 +77,29 @@ for pkg, prefix in MOVED:
"module_trainer": getattr(module, f"{prefix}Trainer").__name__,
"config_has_unsloth_fields": "unsloth_num_chunks" in getattr(config, "__dataclass_fields__", {}),
"mro": [f"{c.__module__}.{c.__name__}" for c in trainer.__mro__],
+ "num_generations": [
+ getattr(inspect.signature(config.__init__).parameters.get("num_generations"), "default", None),
+ getattr(getattr(config.__mro__[1], "__dataclass_fields__", {}).get("num_generations"), "default", None),
+ ],
+ "relative_imports": re.findall(r"^\s*from \.+[\w.]* import .*$", inspect.getsource(sys.modules[trainer.__module__]), re.M),
"leaked": [x for x in (f"{prefix}Trainer", f"{prefix}Config") if x in vars(trl) or x in vars(trl.trainer)]
+ ([f"trl.trainer.{pkg}_trainer"] if f"{pkg}_trainer" in vars(trl.trainer) else []),
}
+def _config_probe(pkg, prefix, **kwargs):
+ try:
+ cfg = getattr(importlib.import_module(f"trl.experimental.{pkg}"), f"{prefix}Config")(
+ output_dir = "unused", report_to = "none", **kwargs
+ )
+ return {"per_device_train_batch_size": cfg.per_device_train_batch_size}
+ except Exception as e:
+ return {"error": f"{type(e).__name__}: {e}"}
+if "gold" in out["moved"]:
+ out["gold_config"] = _config_probe(
+ "gold", "GOLD", num_generations = 8, per_device_train_batch_size = 4,
+ gradient_accumulation_steps = 2, lmbda = 1.0,
+ )
+if "sdft" in out["moved"]:
+ out["sdft_eval_config"] = _config_probe("sdft", "SDFT", eval_strategy = "steps", eval_steps = 1)
print("RESULT " + json.dumps(out))
"""
@@ -97,6 +122,9 @@ def _run(pre_import = ()):
assert proc.returncode == 0 and lines, proc.stdout[-3000:] + proc.stderr[-3000:]
result = json.loads(lines[-1][len("RESULT ") :])
moved = {k: v for k, v in result["moved"].items() if "import_error" not in v}
+ for pkg, key in (("gold", "gold_config"), ("sdft", "sdft_eval_config")):
+ if pkg in moved:
+ moved[pkg]["config_probe"] = result.get(key)
if not moved:
pytest.skip(f"TRL {result['trl']} keeps these trainers in trl.trainer: {result['moved']}")
return moved
@@ -115,6 +143,11 @@ def test_experimental_trainers_and_configs_are_patched(after_unsloth):
assert got["config_has_unsloth_fields"], (pkg, got)
+def test_generated_trainers_have_no_relative_imports(after_unsloth):
+ for pkg, got in after_unsloth.items():
+ assert got["relative_imports"] == [], (pkg, got["relative_imports"])
+
+
def test_patching_leaves_no_trl_trainer_aliases(after_unsloth):
for pkg, got in after_unsloth.items():
assert got["leaked"] == [], (pkg, got)
@@ -130,3 +163,41 @@ def test_gkd_keeps_trls_own_sft_base_whatever_the_import_order(after_unsloth):
assert "trl.trainer.sft_trainer.SFTTrainer" in got["mro"], got
assert not any(c.endswith("SFTTrainer") and "Unsloth" in c for c in got["mro"]), got
assert after_unsloth["gkd"]["mro"][2:] == before["mro"][2:]
+
+
+ "pkg, parent", [("server_distillation", "DistillationTrainer"), ("gold", "SFTTrainer")]
+)
+def test_distillation_trainers_keep_trls_own_base_whatever_the_import_order(
+ after_unsloth, pkg, parent
+):
+ if pkg not in after_unsloth:
+ pytest.skip(f"no trl.experimental.{pkg} on this TRL")
+ before = _run(pre_import = [f"trl.experimental.{pkg}"])[pkg]
+ for got in (after_unsloth[pkg], before):
+ assert any(
+ c.startswith("trl.trainer.") and c.endswith(f".{parent}") for c in got["mro"]
+ ), got
+ assert not any(c.endswith(parent) and "Unsloth" in c for c in got["mro"][2:]), got
+ assert after_unsloth[pkg]["mro"][2:] == before["mro"][2:]
+
+
+def test_gold_keeps_trls_num_generations_default(after_unsloth):
+ if "gold" not in after_unsloth:
+ pytest.skip("no trl.experimental.gold on this TRL")
+ patched, pristine = after_unsloth["gold"]["num_generations"]
+ assert patched == pristine, after_unsloth["gold"]["num_generations"]
+
+
+def test_gold_keeps_a_valid_batch_size(after_unsloth):
+ got = after_unsloth.get("gold")
+ if got is None:
+ pytest.skip("no trl.experimental.gold on this TRL")
+ assert got["config_probe"] == {"per_device_train_batch_size": 4}, got["config_probe"]
+
+
+def test_sdft_config_with_evaluation_still_builds(after_unsloth):
+ got = after_unsloth.get("sdft")
+ if got is None:
+ pytest.skip("no trl.experimental.sdft on this TRL")
+ assert "error" not in got["config_probe"], got["config_probe"]
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 3991e81211c..bbecab934ff 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2359,6 +2359,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
v = re.escape("\n")
if v is EMPTY:
arguments.append(k)
+ elif type(v) is str and v != re.escape("\n") and ("\n" in v or "'" in v):
+ # A newline or quote ends the bare '...' literal early (SDFT's teacher_prompt_template).
+ arguments.append(f"{k} = {v!r}")
elif type(v) is str:
arguments.append(f"{k} = '{v}'")
elif type(v) in types:
@@ -3078,6 +3081,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
replacements["warmup_steps"] = 0.1
else:
replacements["warmup_ratio"] = 0.1
+ # GOLD's num_generations is samples per prompt (default 1), not a GRPO group.
+ if trainer_file == "gold_trainer":
+ replacements.pop("num_generations")
+ # SDFT needs the eval batch divisible by num_generations (8).
+ if trainer_file == "sdft_trainer":
+ replacements.pop("per_device_eval_batch_size")
for k, v in replacements.items():
x = rf"\b{k}( = [^,\n]{{1,}})?,\n"
@@ -3290,7 +3299,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"\n"
)
extra_args += check_num_generations
- elif "per_device_train_batch_size" in call_args and "num_generations" in call_args:
+ # GOLD: TRL already checks generation_batch_size * num_generations == batch * grad accum.
+ elif (
+ "per_device_train_batch_size" in call_args
+ and "num_generations" in call_args
+ and trainer_file != "gold_trainer"
+ ):
if "steps_per_generation" not in call_args:
print(f"Unsloth: Could not find `steps_per_generation` in {trainer_file}")
if "generation_batch_size" not in call_args:
@@ -3604,6 +3618,18 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
RLTrainer_source = re.sub(r"[\n]{3,}", "\n", RLTrainer_source)
+ # The generated module has no package: make function-level relative imports (ServerDistillationTrainer.__init__) absolute.
+ _source_package = getattr(inspect.getmodule(RLTrainer), "__package__", None)
+ if _source_package:
+ RLTrainer_source = re.sub(
+ r"^([ \t]*from[ \t]+)(\.+[\w.]*)([ \t]+import[ \t])",
+ lambda m: m.group(1)
+ + importlib.util.resolve_name(m.group(2), _source_package)
+ + m.group(3),
+ RLTrainer_source,
+ flags = re.MULTILINE,
+ )
+
_resolved_module = _trainer_resolved_module or _config_resolved_module
_model_location = (
_resolved_module.__name__ if _resolved_module is not None else trainer.__name__
@@ -4051,15 +4077,19 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
# TRL 0.29+ keeps these only in trl.experimental.<name>.<name>_trainer, which the trl.trainer walk never sees.
+# Not async_distillation: it loads the student from a model name, so an Unsloth model never reaches it.
_TRL_EXPERIMENTAL_TRAINERS = (
"bco",
"cpo",
"gkd",
+ "gold",
"nash_md",
"online_dpo",
"orpo",
"ppo",
"prm",
+ "sdft",
+ "server_distillation",
"xpo",
)