Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Skip TRL's redundant kbit prep for models Unsloth already prepared (#11494)

Daniel Hanc960baad9d64
diff --git a/scripts/exec_literals_baseline.json b/scripts/exec_literals_baseline.json
index 90652d0b3e5..e521c5ee8e2 100644
--- a/scripts/exec_literals_baseline.json
+++ b/scripts/exec_literals_baseline.json
@@ -254,6 +254,13 @@
"count": 1,
"reason": "Installs a patched attention `__init__` generated by this file from a fixed template, and binds it by the name that template defines."
},
+ {
+ "file": "unsloth/models/rl.py",
+ "sink": "compile",
+ "digest": "47706ae142cdfec0",
+ "count": 1,
+ "reason": "Re-executes TRL's own prepare_peft_model with one branch condition corrected, matching the clause TRL itself added in 0.24.0. The source is read with inspect.getsource from the installed trl package, never from user input or a network fetch, and it is only rewritten when the exact expected branch text is present exactly once; otherwise the patch bails and leaves TRL alone. See tests/test_kbit_prep_guard.py."
+ },
{
"file": "unsloth/models/rl.py",
"sink": "eval",
@@ -324,6 +331,13 @@
"count": 1,
"reason": "Re-imports TRL trainer and config classes by name, and installs the RL trainer source this file generates. The names come from this file's own tables."
},
+ {
+ "file": "unsloth/models/rl.py",
+ "sink": "exec",
+ "digest": "792cfde532dac568",
+ "count": 1,
+ "reason": "Re-executes TRL's own prepare_peft_model with one branch condition corrected, matching the clause TRL itself added in 0.24.0. The source is read with inspect.getsource from the installed trl package, never from user input or a network fetch, and it is only rewritten when the exact expected branch text is present exactly once; otherwise the patch bails and leaves TRL alone. See tests/test_kbit_prep_guard.py."
+ },
{
"file": "unsloth/models/rl.py",
"sink": "exec",
diff --git a/tests/test_kbit_prep_guard.py b/tests/test_kbit_prep_guard.py
new file mode 100644
index 00000000000..e658811cb77
--- /dev/null
+++ b/tests/test_kbit_prep_guard.py
@@ -0,0 +1,197 @@
+"""TRL below 0.24.0 re-prepares a model Unsloth already prepared.
+
+``prepare_peft_model`` calls PEFT's ``prepare_model_for_kbit_training``, which
+upcasts every non-``Params4bit`` parameter to float32. The one that matters is
+the dense, frozen ``lm_head``: 4.74 GiB on Qwen3.8, 5.01 GiB on Muse Glimmer,
+enough to OOM a T4 that is already holding the weights. TRL added
+``and not isinstance(model, PeftModel)`` in 0.24.0; these tests pin that we
+apply the same clause below that version, and that we touch nothing at or above
+it.
+
+The fixture is TRL 0.22.2's real function body, not a paraphrase, so a change in
+how the branch is spelled shows up as a failure here rather than as a silent
+no-op in the field.
+"""
+
+import linecache
+import sys
+import types
+import pytest
+
+from unsloth.models.rl import (
+ _guard_kbit_prep_against_peft_models,
+ _UNSLOTH_KBIT_PREP_GUARD_FLAG,
+)
+
+# Verbatim from trl 0.22.2 trl/models/utils.py, trimmed to the branch under test.
+TRL_0_22_2_SOURCE = '''
+def prepare_peft_model(model, peft_config, args):
+ """Prepares a model for PEFT training."""
+ if isinstance(model, PeftModel) and peft_config is not None:
+ model = model.merge_and_unload()
+
+ is_qlora = getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False)
+
+ is_sharded_qlora = False
+ if getattr(model, "is_loaded_in_4bit", False):
+ for _, param in model.named_parameters():
+ if param.__class__.__name__ == "Params4bit":
+ is_sharded_qlora = param.data.device.type in {"cpu", "meta"}
+ break
+
+ if is_qlora and not is_sharded_qlora:
+ model = prepare_model_for_kbit_training(model)
+ CALLS.append("kbit_prep")
+ elif args.gradient_checkpointing:
+ CALLS.append("enable_gc")
+ return model
+'''
+
+
+def _seed_linecache(filename, source):
+ lines = [l + "\n" for l in source.splitlines()]
+ linecache.cache[filename] = (len(source), None, lines, filename)
+
+
+class FakePeftModel:
+ """Stands in for a model Unsloth has already applied LoRA to."""
+
+ is_loaded_in_4bit = True
+
+ def named_parameters(self):
+ return iter(())
+
+
+class PlainModel:
+ is_loaded_in_4bit = True
+
+ def named_parameters(self):
+ return iter(())
+
+
+class Args:
+ gradient_checkpointing = False
+
+
+def trl_modules(monkeypatch):
+ """A trl.models.utils plus the trainer modules that re-export the name."""
+ calls = []
+
+ utils = types.ModuleType("trl.models.utils")
+ utils.__file__ = "<trl-0.22.2-fixture>"
+ utils.PeftModel = FakePeftModel
+ utils.prepare_model_for_kbit_training = lambda m, **kw: m
+ utils.CALLS = calls
+ # inspect.getsource reads through linecache, and an installed TRL has a real
+ # file behind it. Seed the cache so the fixture is readable the same way,
+ # otherwise the guard bails for a reason that never occurs in the field and
+ # every assertion below passes vacuously.
+ _seed_linecache(utils.__file__, TRL_0_22_2_SOURCE)
+ exec(compile(TRL_0_22_2_SOURCE, utils.__file__, "exec"), vars(utils))
+
+ trl = types.ModuleType("trl")
+ trl.__version__ = "0.22.2"
+ models = types.ModuleType("trl.models")
+ models.utils = utils
+ models.prepare_peft_model = utils.prepare_peft_model
+ trl.models = models
+
+ # Every trainer module that does `from ..models import prepare_peft_model`.
+ trainers = {}
+ for name in (
+ "sft_trainer",
+ "grpo_trainer",
+ "rloo_trainer",
+ "prm_trainer",
+ "online_dpo_trainer",
+ "reward_trainer",
+ ):
+ m = types.ModuleType(f"trl.trainer.{name}")
+ m.prepare_peft_model = utils.prepare_peft_model
+ trainers[name] = m
+ monkeypatch.setitem(sys.modules, f"trl.trainer.{name}", m)
+
+ monkeypatch.setitem(sys.modules, "trl", trl)
+ monkeypatch.setitem(sys.modules, "trl.models", models)
+ monkeypatch.setitem(sys.modules, "trl.models.utils", utils)
+ return types.SimpleNamespace(
+ trl = trl, utils = utils, models = models, trainers = trainers, calls = calls
+ )
+
+
+def test_an_already_peft_model_skips_the_upcast(trl_modules):
+ assert _guard_kbit_prep_against_peft_models() is True
+ trl_modules.utils.prepare_peft_model(FakePeftModel(), None, Args())
+ assert "kbit_prep" not in trl_modules.calls
+
+
+def test_a_plain_quantized_model_still_gets_prepared(trl_modules):
+ """The guard must be narrow: a model that has NOT been through PEFT still
+ needs the preparation TRL does for it."""
+ assert _guard_kbit_prep_against_peft_models() is True
+ trl_modules.utils.prepare_peft_model(PlainModel(), None, Args())
+ assert "kbit_prep" in trl_modules.calls
+
+
+def test_every_module_that_re_exports_the_name_is_rebound(trl_modules):
+ """sft_trainer is the one GKD inherits, but rebinding only the definition
+ would leave six other trainers on the original."""
+ original = trl_modules.utils.prepare_peft_model
+ assert _guard_kbit_prep_against_peft_models() is True
+ for name, module in trl_modules.trainers.items():
+ assert module.prepare_peft_model is not original, name
+ assert getattr(module.prepare_peft_model, _UNSLOTH_KBIT_PREP_GUARD_FLAG, False), name
+
+
+def test_it_is_idempotent(trl_modules):
+ assert _guard_kbit_prep_against_peft_models() is True
+ assert _guard_kbit_prep_against_peft_models() is False
+
+
+def test_trl_at_or_above_0_24_is_left_alone(trl_modules):
+ """Upstream already carries the clause; patching it again would mean
+ maintaining a copy of a function we no longer need to correct."""
+ trl_modules.trl.__version__ = "0.24.0"
+ original = trl_modules.utils.prepare_peft_model
+ assert _guard_kbit_prep_against_peft_models() is False
+ assert trl_modules.utils.prepare_peft_model is original
+
+
+def test_an_unrecognised_branch_is_left_alone(trl_modules, monkeypatch):
+ """If TRL respells the branch, bail rather than edit blindly."""
+ src = TRL_0_22_2_SOURCE.replace(
+ "if is_qlora and not is_sharded_qlora:",
+ "if is_qlora and (not is_sharded_qlora):",
+ )
+ _seed_linecache("<respelled>", src)
+ exec(compile(src, "<respelled>", "exec"), vars(trl_modules.utils))
+ original = trl_modules.utils.prepare_peft_model
+ assert _guard_kbit_prep_against_peft_models() is False
+ assert trl_modules.utils.prepare_peft_model is original
+
+
+def test_a_source_checkout_with_no_metadata_is_still_guarded(trl_modules):
+ """TRL run from a source tree sets ``__version__ = "unknown"``, which does
+ not parse. Bailing there would leave exactly the pre-0.24 installs this
+ exists for on the upcast."""
+ trl_modules.trl.__version__ = "unknown"
+ assert _guard_kbit_prep_against_peft_models() is True
+ trl_modules.utils.prepare_peft_model(FakePeftModel(), None, Args())
+ assert "kbit_prep" not in trl_modules.calls
+
+
+def test_an_unparseable_version_does_not_patch_an_already_guarded_trl(trl_modules):
+ """Falling through on an unparseable version is only safe because the
+ source check is self-guarding: 0.24.0 and above spell the branch with the
+ clause already in it, so there is nothing for us to match."""
+ trl_modules.trl.__version__ = "unknown"
+ src = TRL_0_22_2_SOURCE.replace(
+ "if is_qlora and not is_sharded_qlora:",
+ "if is_qlora and not is_sharded_qlora and not isinstance(model, PeftModel):",
+ )
+ _seed_linecache("<trl-0.24-fixture>", src)
+ exec(compile(src, "<trl-0.24-fixture>", "exec"), vars(trl_modules.utils))
+ original = trl_modules.utils.prepare_peft_model
+ assert _guard_kbit_prep_against_peft_models() is False
+ assert trl_modules.utils.prepare_peft_model is original
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index ab295a362d9..d3839b34dfa 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -1006,6 +1006,92 @@ def _pin_pristine_sft_loss_type(config_cls):
return True
+_UNSLOTH_KBIT_PREP_GUARD_FLAG = "_unsloth_skips_kbit_prep_for_peft_models"
+
+
+def _guard_kbit_prep_against_peft_models():
+ """Stop TRL below 0.24.0 re-preparing a model Unsloth already prepared.
+
+ ``prepare_peft_model`` calls PEFT's ``prepare_model_for_kbit_training``, which freezes every parameter and then upcasts each one that is not a ``Params4bit`` to float32. The parameter that matters is the ``lm_head``: Unsloth keeps it out of quantization on purpose so a distillation loss can project through it, so it is dense, it is frozen, and at a large vocabulary it is enormous. Upcasting a frozen tensor buys nothing, since fp32 master weights only matter for parameters an optimizer updates, and autocast already computes in fp32 where precision matters. Measured cost on Kaggle 2x T4: 4.74 GiB for Qwen3.8 (248320 tokens) and 5.01 GiB for Muse Glimmer, both of which OOM with the weights already loaded and sharded across both cards.
+
+ Unsloth's own ``prepare_model_for_kbit_training`` avoids this (it passes ``train_lm_head = False`` and gets fp32 through mixed precision), and ``patch_trl_rl_trainers`` strips the call from each generated trainer. Neither reaches this case: ``GKDTrainer`` subclasses ``SFTTrainer``, the generated ``_UnslothGKDTrainer`` inherits TRL's pristine ``SFTTrainer``, and ``super().__init__`` walks into the unpatched call.
+
+ TRL fixed this themselves in 0.24.0 by adding ``and not isinstance(model, PeftModel)`` to the branch, which is exactly our case because Unsloth has applied LoRA before the trainer is built. So apply that same clause to older TRL rather than inventing a different rule, and leave 0.24.0 and above alone.
+ """
+ try:
+ import trl
+ except Exception:
+ return False
+ try:
+ if Version(trl.__version__) >= Version("0.24.0"):
+ return False
+ except Exception:
+ # A TRL run from a source tree with no distribution metadata sets
+ # __version__ = "unknown", which does not parse. Fall through to the
+ # source check rather than bailing, because bailing leaves exactly the
+ # pre-0.24 installs this exists for on the upcast. The source check is
+ # self-guarding: 0.24.0 and above already carry the clause, so they
+ # spell the branch differently and never match OLD.
+ pass
+ try:
+ from trl.models import utils as trl_models_utils
+ except Exception:
+ return False
+
+ original = getattr(trl_models_utils, "prepare_peft_model", None)
+ if original is None or getattr(original, _UNSLOTH_KBIT_PREP_GUARD_FLAG, False):
+ return False
+
+ OLD = "if is_qlora and not is_sharded_qlora:"
+ NEW = "if is_qlora and not is_sharded_qlora and not isinstance(model, PeftModel):"
+ try:
+ source = inspect.getsource(original)
+ except Exception:
+ return False
+ # Bail rather than guess: if the branch is not spelled the way we expect, a
+ # blind edit is worse than leaving the upcast in place.
+ if source.count(OLD) != 1:
+ return False
+ import functools, textwrap
+
+ source = textwrap.dedent(source).replace(OLD, NEW)
+
+ # exec against the module's LIVE __dict__, not a copy of it. The body closes
+ # over PeftModel, prepare_model_for_kbit_training, get_peft_model,
+ # dataclasses and version, and a copy would freeze those at patch time: a
+ # later rebinding in trl.models.utils (another library patching it, a test
+ # swapping it out) would be invisible to the replacement while the original
+ # would have seen it. The exec defines only this one name, which is the name
+ # we are replacing anyway.
+ namespace = vars(trl_models_utils)
+ exec(compile(source, getattr(trl_models_utils, "__file__", "<unsloth>"), "exec"), namespace)
+ patched = namespace.get("prepare_peft_model")
+ if patched is None:
+ return False
+ functools.update_wrapper(patched, original)
+ setattr(patched, _UNSLOTH_KBIT_PREP_GUARD_FLAG, True)
+
+ # Rebind everywhere, not just at the definition. Each trainer module does
+ # `from ..models import prepare_peft_model` at import time, so it holds its
+ # own reference; in TRL 0.22.2 that is seven modules, and sft_trainer is the
+ # one GKD actually inherits.
+ rebound = 0
+ for module in list(sys.modules.values()):
+ if module is None:
+ continue
+ try:
+ if getattr(module, "prepare_peft_model", None) is original:
+ setattr(module, "prepare_peft_model", patched)
+ rebound += 1
+ except Exception:
+ continue
+ logger.info(
+ f"Unsloth: skipping the redundant float32 upcast in TRL {trl.__version__}'s "
+ f"prepare_peft_model for already-PEFT models ({rebound} binding(s) rebound)."
+ )
+ return True
+
+
def _widen_sft_config_instance_check(patched_config):
"""Keep a config that subclasses TRL's own ``SFTConfig`` from being downcast to one.
@@ -3033,6 +3119,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
_widen_sft_config_instance_check(_patched_config)
except Exception as e:
logger.info(f"Unsloth: Could not widen the {RLConfig_name} isinstance check: {e}")
+ try:
+ # Idempotent: the flag on the replacement makes repeat calls no-ops.
+ # Called from here rather than at import so TRL's trainer modules,
+ # which each hold their own reference to the name, are already
+ # imported and can be rebound in one pass.
+ _guard_kbit_prep_against_peft_models()
+ except Exception as e:
+ logger.info(f"Unsloth: Could not guard the kbit prep against PEFT models: {e}")
try:
_wrap_sft_evaluate_cap(getattr(created_module, f"Unsloth{RLTrainer_name}"))
except Exception as e: