Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop SFTTrainer downcasting configs that subclass SFTConfig (#11440)

Daniel Hand48ab691fc0e
diff --git a/tests/test_sft_config_subclass_downcast.py b/tests/test_sft_config_subclass_downcast.py
new file mode 100644
index 00000000000..47b0d529fc5
--- /dev/null
+++ b/tests/test_sft_config_subclass_downcast.py
@@ -0,0 +1,249 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A config that subclasses ``SFTConfig`` must survive ``SFTTrainer.__init__``.
+
+Replacing ``trl.trainer.sft_trainer.SFTConfig`` leaves two live classes of that
+name. TRL guards with ``isinstance(args, TrainingArguments) and not
+isinstance(args, SFTConfig)`` and rebuilds the config when it fires; against our
+class that test is true for every config still derived from the pristine one, so
+a ``GKDConfig`` was rebuilt as a plain SFT config and lost ``lmbda``, ``beta``,
+``temperature``, ``teacher_model_name_or_path`` and the rest. See
+unslothai/unsloth#1941.
+
+Lifted out of ``unsloth/models/rl.py`` with ``ast`` the same way
+``_rl_source`` does, so this stays CPU-only and needs neither torch nor trl.
+"""
+
+from __future__ import annotations
+
+import ast
+import copyreg
+import inspect
+import logging
+import sys
+from pathlib import Path
+
+import pytest
+
+
+SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl.py"
+
+NAMES = (
+ "_is_unsloth_patched_config",
+ "_reduce_pristine_rl_config",
+ "_config_reduction_is_safe",
+ "_register_config_pickle_fallback",
+ "_widen_sft_config_instance_check",
+)
+CONSTANTS = (
+ "_UNSLOTH_PATCHED_CONFIG_FLAG",
+ "_UNSLOTH_CONFIG_PICKLE_TARGET",
+ "_UNSLOTH_SFT_CONFIG_SHIM_FLAG",
+)
+
+
+def _load():
+ text = SOURCE_PATH.read_text(encoding = "utf-8")
+ tree = ast.parse(text, filename = str(SOURCE_PATH))
+ wanted = []
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name in NAMES:
+ wanted.append(node)
+ elif isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id in CONSTANTS for t in node.targets
+ ):
+ wanted.append(node)
+ found = {n.name for n in wanted if isinstance(n, ast.FunctionDef)}
+ missing = set(NAMES) - found
+ if missing:
+ raise AssertionError(f"missing module-level defs in {SOURCE_PATH}: {sorted(missing)}")
+ namespace: dict = {
+ "inspect": inspect,
+ "logger": logging.getLogger("unsloth-repro"),
+ "sys": sys,
+ "copyreg": copyreg,
+ }
+ exec(compile(ast.Module(body = wanted, type_ignores = []), str(SOURCE_PATH), "exec"), namespace)
+ return namespace
+
+
+NS = _load()
+
+
+def trl_like(monkeypatch):
+ """The helper resolves ``import trl.trainer.sft_trainer`` through the parent package, so the whole chain has to be stood in, not just the leaf ``sys.modules`` entry."""
+ import sys
+ import types
+
+ class TrainingArguments:
+ pass
+
+ class SFTConfig(TrainingArguments): # the pristine class
+ pass
+
+ class GKDConfig(SFTConfig): # a TRL config that builds on SFT
+ def __init__(
+ self,
+ lmbda = 0.5,
+ beta = 0.5,
+ ):
+ self.lmbda = lmbda
+ self.beta = beta
+
+ # `_patch_config_pickle_identity` gives the generated class the pristine module and name so it keeps pickling.
+ patched = type(
+ "SFTConfig",
+ (SFTConfig,),
+ {NS["_UNSLOTH_PATCHED_CONFIG_FLAG"]: True},
+ )
+ # The home module (`sft_config`) differs from the guard's (`sft_trainer`); model both, or the pickle test passes vacuously.
+ patched.__module__ = "trl.trainer.sft_config"
+ patched.__qualname__ = "SFTConfig"
+
+ module = types.ModuleType("trl.trainer.sft_trainer")
+ module.SFTConfig = patched
+ config_module = types.ModuleType("trl.trainer.sft_config")
+ config_module.SFTConfig = patched
+ trainer_pkg = types.ModuleType("trl.trainer")
+ trainer_pkg.__path__ = []
+ trainer_pkg.sft_trainer = module
+ trainer_pkg.sft_config = config_module
+ trl_pkg = types.ModuleType("trl")
+ trl_pkg.__path__ = []
+ trl_pkg.trainer = trainer_pkg
+ # The top level name is the same object, exactly as after patching.
+ trl_pkg.SFTConfig = patched
+
+ monkeypatch.setitem(sys.modules, "trl", trl_pkg)
+ monkeypatch.setitem(sys.modules, "trl.trainer", trainer_pkg)
+ monkeypatch.setitem(sys.modules, "trl.trainer.sft_trainer", module)
+ monkeypatch.setitem(sys.modules, "trl.trainer.sft_config", config_module)
+ try:
+ yield TrainingArguments, SFTConfig, GKDConfig, patched, module, trl_pkg
+ finally:
+ for cls in (SFTConfig, GKDConfig, patched):
+ copyreg.dispatch_table.pop(cls, None)
+
+
+def _guard_fires(args, module):
+ """TRL's own predicate, verbatim from ``SFTTrainer.__init__``."""
+ TrainingArguments = args.__class__.__mro__[-2]
+ return isinstance(args, TrainingArguments) and not isinstance(args, module.SFTConfig)
+
+
+def test_subclass_config_is_downcast_without_the_fix(trl_like):
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ gkd = GKDConfig(lmbda = 0.25, beta = 0.75)
+ assert isinstance(gkd, TrainingArguments)
+ # This is the bug: a GKDConfig is not an instance of the installed SFTConfig.
+ assert not isinstance(gkd, module.SFTConfig)
+
+
+def test_widening_stops_the_downcast(trl_like):
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ assert NS["_widen_sft_config_instance_check"](patched) is True
+
+ gkd = GKDConfig(lmbda = 0.25, beta = 0.75)
+ assert isinstance(gkd, module.SFTConfig), "the guard still downcasts a GKDConfig"
+ # The subclass keeps its own fields, which is the whole point.
+ assert gkd.lmbda == 0.25 and gkd.beta == 0.75
+
+
+def test_plain_training_arguments_are_still_converted(trl_like):
+ """The guard exists to convert a bare TrainingArguments; keep that working."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ assert not isinstance(TrainingArguments(), module.SFTConfig)
+
+
+def test_instances_of_the_installed_config_still_match(trl_like):
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ assert isinstance(patched(), module.SFTConfig)
+ assert isinstance(pristine(), module.SFTConfig)
+
+
+def test_calling_the_name_still_builds_the_unsloth_config(trl_like):
+ """Widening must not cost the generated config: SFTConfig(...) is still ours."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ built = module.SFTConfig()
+ assert isinstance(built, patched)
+ assert getattr(built, NS["_UNSLOTH_PATCHED_CONFIG_FLAG"], False)
+
+
+def test_is_idempotent(trl_like):
+ """``patch_trl_rl_trainers`` can run more than once; do not stack shims."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ assert NS["_widen_sft_config_instance_check"](patched) is True
+ first = module.SFTConfig
+ assert NS["_widen_sft_config_instance_check"](patched) is False
+ assert module.SFTConfig is first
+
+
+def test_noop_when_the_module_still_holds_the_pristine_class(trl_like):
+ """Nothing was replaced, so the guard already behaves and we leave it alone."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ module.SFTConfig = pristine
+ assert NS["_widen_sft_config_instance_check"](patched) is False
+ assert module.SFTConfig is pristine
+
+
+def test_shim_keeps_the_name_and_answers_to_where_it_lives(trl_like):
+ """Same name, but its OWN module: see the comment at the rebinding."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ shim = module.SFTConfig
+ assert shim.__name__ == patched.__name__
+ assert shim.__qualname__ == patched.__qualname__
+ assert shim.__module__ == "trl.trainer.sft_trainer"
+ assert patched.__module__ == "trl.trainer.sft_config", "the displaced class must not move"
+
+
+def test_the_shim_is_reachable_by_pickle_under_its_own_module_and_name(trl_like):
+ """Pickle refuses a class unless the object at its ``__module__`` plus ``__qualname__`` IS it, so a shim advertising the displaced class's home raises PicklingError from ``Trainer._save_checkpoint``."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ shim = module.SFTConfig
+ home = sys.modules[shim.__module__]
+ # Exactly the identity check pickle performs before it will save the class.
+ assert getattr(home, shim.__qualname__) is shim
+
+
+def test_only_the_guards_own_module_is_rebound(trl_like):
+ """Rebinding every holder of the name is WRONG: the generated class must stay what ``trl.SFTConfig`` resolves to (tests/python/test_rl_config_pickling.py, and the pristine-class walk in tests/version_compat/)."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ assert module.SFTConfig is not patched, "the guard's module was not widened"
+ assert trl_pkg.SFTConfig is patched, "the top level binding must not move"
+ assert (
+ sys.modules["trl.trainer.sft_config"].SFTConfig is patched
+ ), "the class's home module must keep holding it, or it stops pickling"
+
+
+def test_the_displaced_class_keeps_its_own_home_and_pickles(trl_like):
+ """Both classes stay reachable by pickle under their own module and name."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ home = sys.modules[patched.__module__]
+ assert getattr(home, patched.__qualname__) is patched
+
+
+def test_the_shim_carries_the_patched_config_marker(trl_like):
+ """Callers walk back with ``while "_unsloth_patched_rl_config" in cls.__dict__``, so an inherited marker is not enough: the walk would stop on the shim and read Unsloth's field set as TRL's."""
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ cls = module.SFTConfig
+ assert NS["_UNSLOTH_PATCHED_CONFIG_FLAG"] in cls.__dict__
+ while NS["_UNSLOTH_PATCHED_CONFIG_FLAG"] in cls.__dict__ or cls.__name__.startswith("Unsloth"):
+ cls = cls.__bases__[0]
+ assert cls is pristine
+
+
+def test_subclasscheck_is_widened_too(trl_like):
+ TrainingArguments, pristine, GKDConfig, patched, module, trl_pkg = trl_like
+ NS["_widen_sft_config_instance_check"](patched)
+ assert issubclass(GKDConfig, module.SFTConfig)
+ assert not issubclass(TrainingArguments, module.SFTConfig)
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index a336ef2c294..ab295a362d9 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -541,6 +541,8 @@ pass
_UNSLOTH_PATCHED_CONFIG_FLAG = "_unsloth_patched_rl_config"
# Set on the PRISTINE config class, pointing at the Unsloth subclass that has taken over its module attribute.
_UNSLOTH_CONFIG_PICKLE_TARGET = "_unsloth_config_pickle_target"
+# Marks the SFTConfig stand-in whose isinstance test also accepts the pristine class, so a second pass does not stack another one.
+_UNSLOTH_SFT_CONFIG_SHIM_FLAG = "_unsloth_sft_config_widened_instance_check"
def _is_unsloth_patched_config(config_class):
@@ -1004,6 +1006,66 @@ def _pin_pristine_sft_loss_type(config_cls):
return True
+def _widen_sft_config_instance_check(patched_config):
+ """Keep a config that subclasses TRL's own ``SFTConfig`` from being downcast to one.
+
+ Replacing ``trl.trainer.sft_trainer.SFTConfig`` leaves two live classes of that name: ours, and the pristine one every ``SFTConfig`` subclass in TRL still derives from (``GKDConfig``, and any other trainer whose config builds on SFT). ``SFTTrainer.__init__`` guards with ``isinstance(args, TrainingArguments) and not isinstance(args, SFTConfig)`` and rebuilds ``args = SFTConfig(**args.to_dict())`` when it fires. Against our class that test is true for a ``GKDConfig``, so the config is rebuilt as a plain SFT one and every field the subclass added is dropped: ``lmbda``, ``beta``, ``temperature``, ``teacher_model_name_or_path``, ``teacher_model_init_kwargs``, ``disable_dropout`` and ``seq_kd`` all vanish with only an "is not a valid SFTConfig argument" line each, and the rebuilt config carries the mirrored ``eos_token`` placeholder, which then raises. See unslothai/unsloth#1941.
+
+ The guard exists to convert a plain ``TrainingArguments``; a subclass of ``SFTConfig`` already is an SFT config, so widen the test to accept the pristine class while calling the name still builds ours. A plain ``TrainingArguments`` is still converted, so SFT itself is untouched.
+ """
+ import trl.trainer.sft_trainer as sft_trainer_module
+
+ installed = getattr(sft_trainer_module, "SFTConfig", None)
+ if installed is None or getattr(installed, _UNSLOTH_SFT_CONFIG_SHIM_FLAG, False):
+ return False
+ # Only the class we just installed needs widening; if the module still holds the pristine class the guard already behaves.
+ if installed is not patched_config:
+ return False
+ pristine = None
+ for base in getattr(installed, "__mro__", ())[1:]:
+ if not _is_unsloth_patched_config(base) and base.__name__ == installed.__name__:
+ pristine = base
+ break
+ if pristine is None:
+ return False
+
+ class _WidenedInstanceCheck(type(installed)):
+ def __instancecheck__(cls, instance):
+ return isinstance(instance, (pristine, installed))
+
+ def __subclasscheck__(cls, subclass):
+ return issubclass(subclass, (pristine, installed))
+
+ shim = _WidenedInstanceCheck(
+ installed.__name__,
+ (installed,),
+ # Carry the patched-config marker in the shim's OWN __dict__, not only by
+ # inheritance. Callers that walk back to TRL's pristine class do it with
+ # `while "_unsloth_patched_rl_config" in cls.__dict__`, which is the right
+ # test because the generated subclass is renamed onto TRL's own name; a
+ # shim without the marker stops that walk on itself and answers with
+ # Unsloth's field set (tests/version_compat/test_trl_padding_free_max_length.py).
+ {
+ _UNSLOTH_SFT_CONFIG_SHIM_FLAG: True,
+ _UNSLOTH_PATCHED_CONFIG_FLAG: True,
+ },
+ )
+ # Answer to the module the shim is actually installed at, NOT to the module
+ # the class it stands in for calls home. Pickle stores a class as __module__
+ # plus __qualname__ and refuses unless the object living there IS the class,
+ # so a shim claiming `trl.trainer.sft_config` while the patched class still
+ # sits there is unpicklable, and `torch.save(trainer.args, ...)` from
+ # Trainer._save raises PicklingError. That is the failure
+ # _patch_config_pickle_identity exists to prevent, so point the shim at its
+ # own home and leave the patched class's home alone: both then pickle, and
+ # every other binding of the name keeps resolving exactly as it did before.
+ shim.__qualname__ = installed.__name__
+ shim.__name__ = installed.__name__
+ shim.__module__ = getattr(sft_trainer_module, "__name__", installed.__module__)
+ sft_trainer_module.SFTConfig = shim
+ return True
+
+
def _wrap_sft_evaluate_cap(trainer_cls):
"""Cap a pre-tokenized split handed to ``evaluate()``/``predict()`` later on. The padding-free branch caps the init-time splits and then clears ``args.max_length`` as TRL's guard demands, so a split supplied afterwards is prepared with ``max_length = None`` and Zoo's prep leaves rows already carrying ``input_ids`` alone: overlength rows reach the collator uncapped. The cap survives on ``args.max_seq_length``, so apply it here. Both entry points, since ``predict(test_dataset = ...)`` reaches the same collator. Up to TRL 1.6 nothing prepares a late split; from 1.7.0 ``SFTTrainer.evaluate`` does, packing included, so ``eval_packing`` must be honoured here exactly as at construction (``_trl_prepares_late_evals``). This has to agree with the construction-time cap on every detail: truncation_mode, refusing a packed split, dropping rows with no supervised token, and handling a stream."""
@@ -2967,6 +3029,10 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
break
except Exception as e:
logger.info(f"Unsloth: Could not pin the {RLConfig_name} loss_type: {e}")
+ try:
+ _widen_sft_config_instance_check(_patched_config)
+ except Exception as e:
+ logger.info(f"Unsloth: Could not widen the {RLConfig_name} isinstance check: {e}")
try:
_wrap_sft_evaluate_cap(getattr(created_module, f"Unsloth{RLTrainer_name}"))
except Exception as e: