Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Stop SFTTrainer downcasting configs that subclass SFTConfig (#11440)
Daniel Hand48ab691fc0e
diff --git a/tests/test_sft_config_subclass_downcast.py b/tests/test_sft_config_subclass_downcast.pynew file mode 100644index 00000000000..47b0d529fc5--- /dev/null+++ b/tests/test_sft_config_subclass_downcast.py@@ -0,0 +1,249 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A config that subclasses ``SFTConfig`` must survive ``SFTTrainer.__init__``.++Replacing ``trl.trainer.sft_trainer.SFTConfig`` leaves two live classes of that+name. TRL guards with ``isinstance(args, TrainingArguments) and not+isinstance(args, SFTConfig)`` and rebuilds the config when it fires; against our+class that test is true for every config still derived from the pristine one, so+a ``GKDConfig`` was rebuilt as a plain SFT config and lost ``lmbda``, ``beta``,+``temperature``, ``teacher_model_name_or_path`` and the rest. See+unslothai/unsloth#1941.++Lifted out of ``unsloth/models/rl.py`` with ``ast`` the same way+``_rl_source`` does, so this stays CPU-only and needs neither torch nor trl.+"""++from __future__ import annotations++import ast+import copyreg+import inspect+import logging+import sys+from pathlib import Path++import pytest+++SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl.py"++NAMES = (+ "_is_unsloth_patched_config",+ "_reduce_pristine_rl_config",+ "_config_reduction_is_safe",+ "_register_config_pickle_fallback",+ "_widen_sft_config_instance_check",+)+CONSTANTS = (+ "_UNSLOTH_PATCHED_CONFIG_FLAG",+ "_UNSLOTH_CONFIG_PICKLE_TARGET",+ "_UNSLOTH_SFT_CONFIG_SHIM_FLAG",+)+++def _load():+ text = SOURCE_PATH.read_text(encoding = "utf-8")+ tree = ast.parse(text, filename = str(SOURCE_PATH))+ wanted = []+ for node in tree.body:+ if isinstance(node, ast.FunctionDef) and node.name in NAMES:+ wanted.append(node)+ elif isinstance(node, ast.Assign) and any(+ isinstance(t, ast.Name) and t.id in CONSTANTS for t in node.targets+ ):+ wanted.append(node)+ found = {n.name for n in wanted if isinstance(n, ast.FunctionDef)}+ missing = set(NAMES) - found+ if missing:+ raise AssertionError(f"missing module-level defs in {SOURCE_PATH}: {sorted(missing)}")+ namespace: dict = {+ "inspect": inspect,+ "logger": logging.getLogger("unsloth-repro"),+ "sys": sys,+ "copyreg": copyreg,+ }+ exec(compile(ast.Module(body = wanted, type_ignores = []), str(SOURCE_PATH), "exec"), namespace)+ return namespace+++NS = _load()+++def trl_like(monkeypatch):+ """The helper resolves ``import trl.trainer.sft_trainer`` through the parent package, so the whole chain has to be stood in, not just the leaf ``sys.modules`` entry."""+ import sys+ import types++ class TrainingArguments:+ pass++ class SFTConfig(TrainingArguments): # the pristine class+ pass++ class GKDConfig(SFTConfig): # a TRL config that builds on SFT+ def __init__(+ self,+ lmbda = 0.5,+ beta = 0.5,+ ):+ self.lmbda = lmbda+ self.beta = beta++ # `_patch_config_pickle_identity` gives the generated class the pristine module and name so it keeps pickling.+ patched = type(+ "SFTConfig",+ (SFTConfig,),+ {NS["_UNSLOTH_PATCHED_CONFIG_FLAG"]: True},+ )+ # The home module (`sft_config`) differs from the guard's (`sft_trainer`); model both, or the pickle test passes vacuously.+ patched.__module__ = "trl.trainer.sft_config"+ patched.__qualname__ = "SFTConfig"++ module = types.ModuleType("trl.trainer.sft_trainer")+ module.SFTConfig = patched+ config_module = types.ModuleType("trl.trainer.sft_config")+ config_module.SFTConfig = patched+ trainer_pkg = types.ModuleType("trl.trainer")+ trainer_pkg.__path__ = []+ trainer_pkg.sft_trainer = module+ trainer_pkg.sft_config = config_module+ trl_pkg = types.ModuleType("trl")+ trl_pkg.__path__ = []+ trl_pkg.trainer = trainer_pkg+ # The top level name is the same object, exactly as after patching.+ trl_pkg.SFTConfig = patched++ monkeypatch.setitem(sys.modules, "trl", trl_pkg)+ monkeypatch.setitem(sys.modules, "trl.trainer", trainer_pkg)+ monkeypatch.setitem(sys.modules, "trl.trainer.sft_trainer", module)+ monkeypatch.setitem(sys.modules, "trl.trainer.sft_config", config_module)+ try:+ yield TrainingArguments, SFTConfig, GKDConfig, patched, module, trl_pkg+ finally:+ for cls in (SFTConfig, GKDConfig, patched):+ copyreg.dispatch_table.pop(cls, None)+++def _guard_fires(args, module):+ """TRL's own predicate, verbatim from ``SFTTrainer.__init__``."""+ TrainingArguments = args.__class__.__mro__[-2]+ return isinstance(args, TrainingArguments) and not isinstance(args, module.SFTConfig)+++def test_subclass_config_is_downcast_without_the_fix(trl_like):+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ gkd = GKDConfig(lmbda = 0.25, beta = 0.75)+ assert isinstance(gkd, TrainingArguments)+ # This is the bug: a GKDConfig is not an instance of the installed SFTConfig.+ assert not isinstance(gkd, module.SFTConfig)+++def test_widening_stops_the_downcast(trl_like):+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ assert NS["_widen_sft_config_instance_check"](patched) is True++ gkd = GKDConfig(lmbda = 0.25, beta = 0.75)+ assert isinstance(gkd, module.SFTConfig), "the guard still downcasts a GKDConfig"+ # The subclass keeps its own fields, which is the whole point.+ assert gkd.lmbda == 0.25 and gkd.beta == 0.75+++def test_plain_training_arguments_are_still_converted(trl_like):+ """The guard exists to convert a bare TrainingArguments; keep that working."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ assert not isinstance(TrainingArguments(), module.SFTConfig)+++def test_instances_of_the_installed_config_still_match(trl_like):+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ assert isinstance(patched(), module.SFTConfig)+ assert isinstance(pristine(), module.SFTConfig)+++def test_calling_the_name_still_builds_the_unsloth_config(trl_like):+ """Widening must not cost the generated config: SFTConfig(...) is still ours."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ built = module.SFTConfig()+ assert isinstance(built, patched)+ assert getattr(built, NS["_UNSLOTH_PATCHED_CONFIG_FLAG"], False)+++def test_is_idempotent(trl_like):+ """``patch_trl_rl_trainers`` can run more than once; do not stack shims."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ assert NS["_widen_sft_config_instance_check"](patched) is True+ first = module.SFTConfig+ assert NS["_widen_sft_config_instance_check"](patched) is False+ assert module.SFTConfig is first+++def test_noop_when_the_module_still_holds_the_pristine_class(trl_like):+ """Nothing was replaced, so the guard already behaves and we leave it alone."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ module.SFTConfig = pristine+ assert NS["_widen_sft_config_instance_check"](patched) is False+ assert module.SFTConfig is pristine+++def test_shim_keeps_the_name_and_answers_to_where_it_lives(trl_like):+ """Same name, but its OWN module: see the comment at the rebinding."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ shim = module.SFTConfig+ assert shim.__name__ == patched.__name__+ assert shim.__qualname__ == patched.__qualname__+ assert shim.__module__ == "trl.trainer.sft_trainer"+ assert patched.__module__ == "trl.trainer.sft_config", "the displaced class must not move"+++def test_the_shim_is_reachable_by_pickle_under_its_own_module_and_name(trl_like):+ """Pickle refuses a class unless the object at its ``__module__`` plus ``__qualname__`` IS it, so a shim advertising the displaced class's home raises PicklingError from ``Trainer._save_checkpoint``."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ shim = module.SFTConfig+ home = sys.modules[shim.__module__]+ # Exactly the identity check pickle performs before it will save the class.+ assert getattr(home, shim.__qualname__) is shim+++def test_only_the_guards_own_module_is_rebound(trl_like):+ """Rebinding every holder of the name is WRONG: the generated class must stay what ``trl.SFTConfig`` resolves to (tests/python/test_rl_config_pickling.py, and the pristine-class walk in tests/version_compat/)."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ assert module.SFTConfig is not patched, "the guard's module was not widened"+ assert trl_pkg.SFTConfig is patched, "the top level binding must not move"+ assert (+ sys.modules["trl.trainer.sft_config"].SFTConfig is patched+ ), "the class's home module must keep holding it, or it stops pickling"+++def test_the_displaced_class_keeps_its_own_home_and_pickles(trl_like):+ """Both classes stay reachable by pickle under their own module and name."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ home = sys.modules[patched.__module__]+ assert getattr(home, patched.__qualname__) is patched+++def test_the_shim_carries_the_patched_config_marker(trl_like):+ """Callers walk back with ``while "_unsloth_patched_rl_config" in cls.__dict__``, so an inherited marker is not enough: the walk would stop on the shim and read Unsloth's field set as TRL's."""+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ cls = module.SFTConfig+ assert NS["_UNSLOTH_PATCHED_CONFIG_FLAG"] in cls.__dict__+ while NS["_UNSLOTH_PATCHED_CONFIG_FLAG"] in cls.__dict__ or cls.__name__.startswith("Unsloth"):+ cls = cls.__bases__[0]+ assert cls is pristine+++def test_subclasscheck_is_widened_too(trl_like):+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like+ NS["_widen_sft_config_instance_check"](patched)+ assert issubclass(GKDConfig, module.SFTConfig)+ assert not issubclass(TrainingArguments, module.SFTConfig)diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex a336ef2c294..ab295a362d9 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -541,6 +541,8 @@ pass_UNSLOTH_PATCHED_CONFIG_FLAG = "_unsloth_patched_rl_config"# Set on the PRISTINE config class, pointing at the Unsloth subclass that has taken over its module attribute._UNSLOTH_CONFIG_PICKLE_TARGET = "_unsloth_config_pickle_target"+# Marks the SFTConfig stand-in whose isinstance test also accepts the pristine class, so a second pass does not stack another one.+_UNSLOTH_SFT_CONFIG_SHIM_FLAG = "_unsloth_sft_config_widened_instance_check"def _is_unsloth_patched_config(config_class):@@ -1004,6 +1006,66 @@ def _pin_pristine_sft_loss_type(config_cls):return True+def _widen_sft_config_instance_check(patched_config):+ """Keep a config that subclasses TRL's own ``SFTConfig`` from being downcast to one.++ Replacing ``trl.trainer.sft_trainer.SFTConfig`` leaves two live classes of that name: ours, and the pristine one every ``SFTConfig`` subclass in TRL still derives from (``GKDConfig``, and any other trainer whose config builds on SFT). ``SFTTrainer.__init__`` guards with ``isinstance(args, TrainingArguments) and not isinstance(args, SFTConfig)`` and rebuilds ``args = SFTConfig(**args.to_dict())`` when it fires. Against our class that test is true for a ``GKDConfig``, so the config is rebuilt as a plain SFT one and every field the subclass added is dropped: ``lmbda``, ``beta``, ``temperature``, ``teacher_model_name_or_path``, ``teacher_model_init_kwargs``, ``disable_dropout`` and ``seq_kd`` all vanish with only an "is not a valid SFTConfig argument" line each, and the rebuilt config carries the mirrored ``eos_token`` placeholder, which then raises. See unslothai/unsloth#1941.++ The guard exists to convert a plain ``TrainingArguments``; a subclass of ``SFTConfig`` already is an SFT config, so widen the test to accept the pristine class while calling the name still builds ours. A plain ``TrainingArguments`` is still converted, so SFT itself is untouched.+ """+ import trl.trainer.sft_trainer as sft_trainer_module++ installed = getattr(sft_trainer_module, "SFTConfig", None)+ if installed is None or getattr(installed, _UNSLOTH_SFT_CONFIG_SHIM_FLAG, False):+ return False+ # Only the class we just installed needs widening; if the module still holds the pristine class the guard already behaves.+ if installed is not patched_config:+ return False+ pristine = None+ for base in getattr(installed, "__mro__", ())[1:]:+ if not _is_unsloth_patched_config(base) and base.__name__ == installed.__name__:+ pristine = base+ break+ if pristine is None:+ return False++ class _WidenedInstanceCheck(type(installed)):+ def __instancecheck__(cls, instance):+ return isinstance(instance, (pristine, installed))++ def __subclasscheck__(cls, subclass):+ return issubclass(subclass, (pristine, installed))++ shim = _WidenedInstanceCheck(+ installed.__name__,+ (installed,),+ # Carry the patched-config marker in the shim's OWN __dict__, not only by+ # inheritance. Callers that walk back to TRL's pristine class do it with+ # `while "_unsloth_patched_rl_config" in cls.__dict__`, which is the right+ # test because the generated subclass is renamed onto TRL's own name; a+ # shim without the marker stops that walk on itself and answers with+ # Unsloth's field set (tests/version_compat/test_trl_padding_free_max_length.py).+ {+ _UNSLOTH_SFT_CONFIG_SHIM_FLAG: True,+ _UNSLOTH_PATCHED_CONFIG_FLAG: True,+ },+ )+ # Answer to the module the shim is actually installed at, NOT to the module+ # the class it stands in for calls home. Pickle stores a class as __module__+ # plus __qualname__ and refuses unless the object living there IS the class,+ # so a shim claiming `trl.trainer.sft_config` while the patched class still+ # sits there is unpicklable, and `torch.save(trainer.args, ...)` from+ # Trainer._save raises PicklingError. That is the failure+ # _patch_config_pickle_identity exists to prevent, so point the shim at its+ # own home and leave the patched class's home alone: both then pickle, and+ # every other binding of the name keeps resolving exactly as it did before.+ shim.__qualname__ = installed.__name__+ shim.__name__ = installed.__name__+ shim.__module__ = getattr(sft_trainer_module, "__name__", installed.__module__)+ sft_trainer_module.SFTConfig = shim+ return True++def _wrap_sft_evaluate_cap(trainer_cls):"""Cap a pre-tokenized split handed to ``evaluate()``/``predict()`` later on. The padding-free branch caps the init-time splits and then clears ``args.max_length`` as TRL's guard demands, so a split supplied afterwards is prepared with ``max_length = None`` and Zoo's prep leaves rows already carrying ``input_ids`` alone: overlength rows reach the collator uncapped. The cap survives on ``args.max_seq_length``, so apply it here. Both entry points, since ``predict(test_dataset = ...)`` reaches the same collator. Up to TRL 1.6 nothing prepares a late split; from 1.7.0 ``SFTTrainer.evaluate`` does, packing included, so ``eval_packing`` must be honoured here exactly as at construction (``_trl_prepares_late_evals``). This has to agree with the construction-time cap on every detail: truncation_mode, refusing a packed split, dropping rows with no supervised token, and handling a stream."""@@ -2967,6 +3029,10 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):breakexcept Exception as e:logger.info(f"Unsloth: Could not pin the {RLConfig_name} loss_type: {e}")+ try:+ _widen_sft_config_instance_check(_patched_config)+ except Exception as e:+ logger.info(f"Unsloth: Could not widen the {RLConfig_name} isinstance check: {e}")try:_wrap_sft_evaluate_cap(getattr(created_module, f"Unsloth{RLTrainer_name}"))except Exception as e: