Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Skip TRL's redundant kbit prep for models Unsloth already prepared (#11494)
Daniel Hanc960baad9d64
diff --git a/scripts/exec_literals_baseline.json b/scripts/exec_literals_baseline.jsonindex 90652d0b3e5..e521c5ee8e2 100644--- a/scripts/exec_literals_baseline.json+++ b/scripts/exec_literals_baseline.json@@ -254,6 +254,13 @@"count": 1,"reason": "Installs a patched attention `__init__` generated by this file from a fixed template, and binds it by the name that template defines."},+ {+ "file": "unsloth/models/rl.py",+ "sink": "compile",+ "digest": "47706ae142cdfec0",+ "count": 1,+ "reason": "Re-executes TRL's own prepare_peft_model with one branch condition corrected, matching the clause TRL itself added in 0.24.0. The source is read with inspect.getsource from the installed trl package, never from user input or a network fetch, and it is only rewritten when the exact expected branch text is present exactly once; otherwise the patch bails and leaves TRL alone. See tests/test_kbit_prep_guard.py."+ },{"file": "unsloth/models/rl.py","sink": "eval",@@ -324,6 +331,13 @@"count": 1,"reason": "Re-imports TRL trainer and config classes by name, and installs the RL trainer source this file generates. The names come from this file's own tables."},+ {+ "file": "unsloth/models/rl.py",+ "sink": "exec",+ "digest": "792cfde532dac568",+ "count": 1,+ "reason": "Re-executes TRL's own prepare_peft_model with one branch condition corrected, matching the clause TRL itself added in 0.24.0. The source is read with inspect.getsource from the installed trl package, never from user input or a network fetch, and it is only rewritten when the exact expected branch text is present exactly once; otherwise the patch bails and leaves TRL alone. See tests/test_kbit_prep_guard.py."+ },{"file": "unsloth/models/rl.py","sink": "exec",diff --git a/tests/test_kbit_prep_guard.py b/tests/test_kbit_prep_guard.pynew file mode 100644index 00000000000..e658811cb77--- /dev/null+++ b/tests/test_kbit_prep_guard.py@@ -0,0 +1,197 @@+"""TRL below 0.24.0 re-prepares a model Unsloth already prepared.++``prepare_peft_model`` calls PEFT's ``prepare_model_for_kbit_training``, which+upcasts every non-``Params4bit`` parameter to float32. The one that matters is+the dense, frozen ``lm_head``: 4.74 GiB on Qwen3.8, 5.01 GiB on Muse Glimmer,+enough to OOM a T4 that is already holding the weights. TRL added+``and not isinstance(model, PeftModel)`` in 0.24.0; these tests pin that we+apply the same clause below that version, and that we touch nothing at or above+it.++The fixture is TRL 0.22.2's real function body, not a paraphrase, so a change in+how the branch is spelled shows up as a failure here rather than as a silent+no-op in the field.+"""++import linecache+import sys+import types+import pytest++from unsloth.models.rl import (+ _guard_kbit_prep_against_peft_models,+ _UNSLOTH_KBIT_PREP_GUARD_FLAG,+)++# Verbatim from trl 0.22.2 trl/models/utils.py, trimmed to the branch under test.+TRL_0_22_2_SOURCE = '''+def prepare_peft_model(model, peft_config, args):+ """Prepares a model for PEFT training."""+ if isinstance(model, PeftModel) and peft_config is not None:+ model = model.merge_and_unload()++ is_qlora = getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False)++ is_sharded_qlora = False+ if getattr(model, "is_loaded_in_4bit", False):+ for _, param in model.named_parameters():+ if param.__class__.__name__ == "Params4bit":+ is_sharded_qlora = param.data.device.type in {"cpu", "meta"}+ break++ if is_qlora and not is_sharded_qlora:+ model = prepare_model_for_kbit_training(model)+ CALLS.append("kbit_prep")+ elif args.gradient_checkpointing:+ CALLS.append("enable_gc")+ return model+'''+++def _seed_linecache(filename, source):+ lines = [l + "\n" for l in source.splitlines()]+ linecache.cache[filename] = (len(source), None, lines, filename)+++class FakePeftModel:+ """Stands in for a model Unsloth has already applied LoRA to."""++ is_loaded_in_4bit = True++ def named_parameters(self):+ return iter(())+++class PlainModel:+ is_loaded_in_4bit = True++ def named_parameters(self):+ return iter(())+++class Args:+ gradient_checkpointing = False+++def trl_modules(monkeypatch):+ """A trl.models.utils plus the trainer modules that re-export the name."""+ calls = []++ utils = types.ModuleType("trl.models.utils")+ utils.__file__ = "<trl-0.22.2-fixture>"+ utils.PeftModel = FakePeftModel+ utils.prepare_model_for_kbit_training = lambda m, **kw: m+ utils.CALLS = calls+ # inspect.getsource reads through linecache, and an installed TRL has a real+ # file behind it. Seed the cache so the fixture is readable the same way,+ # otherwise the guard bails for a reason that never occurs in the field and+ # every assertion below passes vacuously.+ _seed_linecache(utils.__file__, TRL_0_22_2_SOURCE)+ exec(compile(TRL_0_22_2_SOURCE, utils.__file__, "exec"), vars(utils))++ trl = types.ModuleType("trl")+ trl.__version__ = "0.22.2"+ models = types.ModuleType("trl.models")+ models.utils = utils+ models.prepare_peft_model = utils.prepare_peft_model+ trl.models = models++ # Every trainer module that does `from ..models import prepare_peft_model`.+ trainers = {}+ for name in (+ "sft_trainer",+ "grpo_trainer",+ "rloo_trainer",+ "prm_trainer",+ "online_dpo_trainer",+ "reward_trainer",+ ):+ m = types.ModuleType(f"trl.trainer.{name}")+ m.prepare_peft_model = utils.prepare_peft_model+ trainers[name] = m+ monkeypatch.setitem(sys.modules, f"trl.trainer.{name}", m)++ monkeypatch.setitem(sys.modules, "trl", trl)+ monkeypatch.setitem(sys.modules, "trl.models", models)+ monkeypatch.setitem(sys.modules, "trl.models.utils", utils)+ return types.SimpleNamespace(+ trl = trl, utils = utils, models = models, trainers = trainers, calls = calls+ )+++def test_an_already_peft_model_skips_the_upcast(trl_modules):+ assert _guard_kbit_prep_against_peft_models() is True+ trl_modules.utils.prepare_peft_model(FakePeftModel(), None, Args())+ assert "kbit_prep" not in trl_modules.calls+++def test_a_plain_quantized_model_still_gets_prepared(trl_modules):+ """The guard must be narrow: a model that has NOT been through PEFT still+ needs the preparation TRL does for it."""+ assert _guard_kbit_prep_against_peft_models() is True+ trl_modules.utils.prepare_peft_model(PlainModel(), None, Args())+ assert "kbit_prep" in trl_modules.calls+++def test_every_module_that_re_exports_the_name_is_rebound(trl_modules):+ """sft_trainer is the one GKD inherits, but rebinding only the definition+ would leave six other trainers on the original."""+ original = trl_modules.utils.prepare_peft_model+ assert _guard_kbit_prep_against_peft_models() is True+ for name, module in trl_modules.trainers.items():+ assert module.prepare_peft_model is not original, name+ assert getattr(module.prepare_peft_model, _UNSLOTH_KBIT_PREP_GUARD_FLAG, False), name+++def test_it_is_idempotent(trl_modules):+ assert _guard_kbit_prep_against_peft_models() is True+ assert _guard_kbit_prep_against_peft_models() is False+++def test_trl_at_or_above_0_24_is_left_alone(trl_modules):+ """Upstream already carries the clause; patching it again would mean+ maintaining a copy of a function we no longer need to correct."""+ trl_modules.trl.__version__ = "0.24.0"+ original = trl_modules.utils.prepare_peft_model+ assert _guard_kbit_prep_against_peft_models() is False+ assert trl_modules.utils.prepare_peft_model is original+++def test_an_unrecognised_branch_is_left_alone(trl_modules, monkeypatch):+ """If TRL respells the branch, bail rather than edit blindly."""+ src = TRL_0_22_2_SOURCE.replace(+ "if is_qlora and not is_sharded_qlora:",+ "if is_qlora and (not is_sharded_qlora):",+ )+ _seed_linecache("<respelled>", src)+ exec(compile(src, "<respelled>", "exec"), vars(trl_modules.utils))+ original = trl_modules.utils.prepare_peft_model+ assert _guard_kbit_prep_against_peft_models() is False+ assert trl_modules.utils.prepare_peft_model is original+++def test_a_source_checkout_with_no_metadata_is_still_guarded(trl_modules):+ """TRL run from a source tree sets ``__version__ = "unknown"``, which does+ not parse. Bailing there would leave exactly the pre-0.24 installs this+ exists for on the upcast."""+ trl_modules.trl.__version__ = "unknown"+ assert _guard_kbit_prep_against_peft_models() is True+ trl_modules.utils.prepare_peft_model(FakePeftModel(), None, Args())+ assert "kbit_prep" not in trl_modules.calls+++def test_an_unparseable_version_does_not_patch_an_already_guarded_trl(trl_modules):+ """Falling through on an unparseable version is only safe because the+ source check is self-guarding: 0.24.0 and above spell the branch with the+ clause already in it, so there is nothing for us to match."""+ trl_modules.trl.__version__ = "unknown"+ src = TRL_0_22_2_SOURCE.replace(+ "if is_qlora and not is_sharded_qlora:",+ "if is_qlora and not is_sharded_qlora and not isinstance(model, PeftModel):",+ )+ _seed_linecache("<trl-0.24-fixture>", src)+ exec(compile(src, "<trl-0.24-fixture>", "exec"), vars(trl_modules.utils))+ original = trl_modules.utils.prepare_peft_model+ assert _guard_kbit_prep_against_peft_models() is False+ assert trl_modules.utils.prepare_peft_model is originaldiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex ab295a362d9..d3839b34dfa 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -1006,6 +1006,92 @@ def _pin_pristine_sft_loss_type(config_cls):return True+_UNSLOTH_KBIT_PREP_GUARD_FLAG = "_unsloth_skips_kbit_prep_for_peft_models"+++def _guard_kbit_prep_against_peft_models():+ """Stop TRL below 0.24.0 re-preparing a model Unsloth already prepared.++ ``prepare_peft_model`` calls PEFT's ``prepare_model_for_kbit_training``, which freezes every parameter and then upcasts each one that is not a ``Params4bit`` to float32. The parameter that matters is the ``lm_head``: Unsloth keeps it out of quantization on purpose so a distillation loss can project through it, so it is dense, it is frozen, and at a large vocabulary it is enormous. Upcasting a frozen tensor buys nothing, since fp32 master weights only matter for parameters an optimizer updates, and autocast already computes in fp32 where precision matters. Measured cost on Kaggle 2x T4: 4.74 GiB for Qwen3.8 (248320 tokens) and 5.01 GiB for Muse Glimmer, both of which OOM with the weights already loaded and sharded across both cards.++ Unsloth's own ``prepare_model_for_kbit_training`` avoids this (it passes ``train_lm_head = False`` and gets fp32 through mixed precision), and ``patch_trl_rl_trainers`` strips the call from each generated trainer. Neither reaches this case: ``GKDTrainer`` subclasses ``SFTTrainer``, the generated ``_UnslothGKDTrainer`` inherits TRL's pristine ``SFTTrainer``, and ``super().__init__`` walks into the unpatched call.++ TRL fixed this themselves in 0.24.0 by adding ``and not isinstance(model, PeftModel)`` to the branch, which is exactly our case because Unsloth has applied LoRA before the trainer is built. So apply that same clause to older TRL rather than inventing a different rule, and leave 0.24.0 and above alone.+ """+ try:+ import trl+ except Exception:+ return False+ try:+ if Version(trl.__version__) >= Version("0.24.0"):+ return False+ except Exception:+ # A TRL run from a source tree with no distribution metadata sets+ # __version__ = "unknown", which does not parse. Fall through to the+ # source check rather than bailing, because bailing leaves exactly the+ # pre-0.24 installs this exists for on the upcast. The source check is+ # self-guarding: 0.24.0 and above already carry the clause, so they+ # spell the branch differently and never match OLD.+ pass+ try:+ from trl.models import utils as trl_models_utils+ except Exception:+ return False++ original = getattr(trl_models_utils, "prepare_peft_model", None)+ if original is None or getattr(original, _UNSLOTH_KBIT_PREP_GUARD_FLAG, False):+ return False++ OLD = "if is_qlora and not is_sharded_qlora:"+ NEW = "if is_qlora and not is_sharded_qlora and not isinstance(model, PeftModel):"+ try:+ source = inspect.getsource(original)+ except Exception:+ return False+ # Bail rather than guess: if the branch is not spelled the way we expect, a+ # blind edit is worse than leaving the upcast in place.+ if source.count(OLD) != 1:+ return False+ import functools, textwrap++ source = textwrap.dedent(source).replace(OLD, NEW)++ # exec against the module's LIVE __dict__, not a copy of it. The body closes+ # over PeftModel, prepare_model_for_kbit_training, get_peft_model,+ # dataclasses and version, and a copy would freeze those at patch time: a+ # later rebinding in trl.models.utils (another library patching it, a test+ # swapping it out) would be invisible to the replacement while the original+ # would have seen it. The exec defines only this one name, which is the name+ # we are replacing anyway.+ namespace = vars(trl_models_utils)+ exec(compile(source, getattr(trl_models_utils, "__file__", "<unsloth>"), "exec"), namespace)+ patched = namespace.get("prepare_peft_model")+ if patched is None:+ return False+ functools.update_wrapper(patched, original)+ setattr(patched, _UNSLOTH_KBIT_PREP_GUARD_FLAG, True)++ # Rebind everywhere, not just at the definition. Each trainer module does+ # `from ..models import prepare_peft_model` at import time, so it holds its+ # own reference; in TRL 0.22.2 that is seven modules, and sft_trainer is the+ # one GKD actually inherits.+ rebound = 0+ for module in list(sys.modules.values()):+ if module is None:+ continue+ try:+ if getattr(module, "prepare_peft_model", None) is original:+ setattr(module, "prepare_peft_model", patched)+ rebound += 1+ except Exception:+ continue+ logger.info(+ f"Unsloth: skipping the redundant float32 upcast in TRL {trl.__version__}'s "+ f"prepare_peft_model for already-PEFT models ({rebound} binding(s) rebound)."+ )+ return True++def _widen_sft_config_instance_check(patched_config):"""Keep a config that subclasses TRL's own ``SFTConfig`` from being downcast to one.@@ -3033,6 +3119,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):_widen_sft_config_instance_check(_patched_config)except Exception as e:logger.info(f"Unsloth: Could not widen the {RLConfig_name} isinstance check: {e}")+ try:+ # Idempotent: the flag on the replacement makes repeat calls no-ops.+ # Called from here rather than at import so TRL's trainer modules,+ # which each hold their own reference to the name, are already+ # imported and can be rebound in one pass.+ _guard_kbit_prep_against_peft_models()+ except Exception as e:+ logger.info(f"Unsloth: Could not guard the kbit prep against PEFT models: {e}")try:_wrap_sft_evaluate_cap(getattr(created_module, f"Unsloth{RLTrainer_name}"))except Exception as e: