Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Switch between training and inference mode in one tree walk (#13374)

Daniel Han0edcd859161f
diff --git a/tests/test_for_training_generation_flag.py b/tests/test_for_training_generation_flag.py
index 2271196c453..94792eba019 100644
--- a/tests/test_for_training_generation_flag.py
+++ b/tests/test_for_training_generation_flag.py
@@ -3,8 +3,10 @@
import ast
import builtins
+import functools
import inspect
import os
+import types
from pathlib import Path
import pytest
@@ -41,6 +43,15 @@ def _helpers_used_by(method, tree):
return list(wanted.values())
+def _torch_or_stub():
+ """The fakes below are not nn.Modules, so without torch the lifted code takes its non-Module path."""
+ try:
+ import torch
+ except ImportError:
+ return types.SimpleNamespace(nn = types.SimpleNamespace(Module = type("_NoTorch", (), {})))
+ return torch
+
+
def _for_training(module, class_name):
path = Path(__file__).parents[1] / "unsloth" / "models" / module
tree = ast.parse(path.read_text(encoding = "utf-8"))
@@ -67,12 +78,15 @@ def _for_training(module, class_name):
"_is_unarmed",
"arm_gradient_checkpointing",
"set_module_gradient_checkpointing",
+ "switch_training_mode",
+ "_mode_switch_class_info",
+ "_has_gradient_checkpointing",
)
]
for node in shared:
node.decorator_list = []
compiled = ast.Module(body = shared + _helpers_used_by(method, tree) + [method], type_ignores = [])
- namespace = _Namespace(os = os, inspect = inspect)
+ namespace = _Namespace(os = os, inspect = inspect, torch = _torch_or_stub(), functools = functools)
exec(compile(ast.fix_missing_locations(compiled), str(path), "exec"), namespace)
return namespace["for_training"]
@@ -142,3 +156,51 @@ def test_for_training_does_not_swallow_unrelated_errors(module, class_name):
with pytest.raises(RuntimeError):
_for_training(module, class_name)(_Exploding())
+
+
[email protected]("module, class_name", SITES)
+def test_a_custom_train_on_the_wrapper_chain_has_the_final_say(module, class_name):
+ torch = pytest.importorskip("torch")
+
+ class _StaysInEval(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = torch.nn.Linear(2, 2)
+
+ def train(self, mode = True):
+ super().train(mode)
+ self.training = False
+ return self
+
+ model = _StaysInEval()
+ _for_training(module, class_name)(model, use_gradient_checkpointing = False)
+ assert model.training is False
+ assert model.model.training is True
+
+
[email protected]("module, class_name", SITES)
+def test_layers_flagged_by_arming_get_the_requested_mode(module, class_name):
+ torch = pytest.importorskip("torch")
+
+ class _Layer(torch.nn.Module):
+ def forward(self, x):
+ return self._gradient_checkpointing_func(x)
+
+ class _Unarmed(_Layer):
+ gradient_checkpointing = False
+
+ class _Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = _Unarmed()
+ self.late = _Layer()
+
+ def gradient_checkpointing_enable(self):
+ # A remote enabler that also flags a layer which had no attribute before.
+ for layer in (self.model, self.late):
+ layer.gradient_checkpointing = True
+ layer._gradient_checkpointing_func = lambda f, *a: f(*a)
+
+ model = _Model()
+ _for_training(module, class_name)(model, use_gradient_checkpointing = "unsloth")
+ assert model.late.gradient_checkpointing == "unsloth"
diff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.py
index 04ad27d4143..ec756f4a891 100644
--- a/tests/test_return_logits_survives_for_training.py
+++ b/tests/test_return_logits_survives_for_training.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import ast
import builtins
+import functools
import inspect
import os
import sys
@@ -27,6 +28,15 @@ class _Namespace(dict):
return getattr(builtins, name, None)
+def _torch_or_stub():
+ """The fakes here are not nn.Modules, so without torch the lifted code takes its non-Module path."""
+ try:
+ import torch
+ except ImportError:
+ return types.SimpleNamespace(nn = types.SimpleNamespace(Module = type("_NoTorch", (), {})))
+ return torch
+
+
def _fast_base_model():
tree = ast.parse((MODELS / "vision.py").read_text(encoding = "utf-8"))
cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "FastBaseModel")
@@ -48,10 +58,13 @@ def _fast_base_model():
"_is_unarmed",
"arm_gradient_checkpointing",
"set_module_gradient_checkpointing",
+ "switch_training_mode",
+ "_mode_switch_class_info",
+ "_has_gradient_checkpointing",
):
node.decorator_list = []
wanted.insert(0, node)
- namespace = _Namespace(os = os, inspect = inspect)
+ namespace = _Namespace(os = os, inspect = inspect, torch = _torch_or_stub(), functools = functools)
module = ast.fix_missing_locations(ast.Module(body = wanted, type_ignores = []))
exec(compile(module, str(MODELS / "vision.py"), "exec"), namespace)
for name in ("for_inference", "for_training"):
diff --git a/tests/test_switch_training_mode.py b/tests/test_switch_training_mode.py
new file mode 100644
index 00000000000..6af73271b7d
--- /dev/null
+++ b/tests/test_switch_training_mode.py
@@ -0,0 +1,379 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""switch_training_mode replaces model.train() / eval() plus three tree scans in for_inference /
+for_training (every GRPO step), so its result must be exactly model.train(mode)'s. Lifted out of
+_utils.py with ``ast`` so this runs without ``import unsloth``.
+"""
+
+from __future__ import annotations
+
+import ast
+import builtins
+import functools
+import inspect
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+nn = torch.nn
+
+UTILS = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_utils.py"
+HELPERS = ("_mode_switch_class_info", "_has_gradient_checkpointing", "switch_training_mode")
+
+
+class _Namespace(dict):
+ def __missing__(self, name):
+ return getattr(builtins, name)
+
+
[email protected](scope = "module")
+def switch():
+ tree = ast.parse(UTILS.read_text(encoding = "utf-8"))
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS]
+ assert {n.name for n in nodes} == set(HELPERS)
+ namespace = _Namespace(torch = torch, functools = functools, inspect = inspect)
+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(UTILS), "exec"), namespace)
+ return namespace["switch_training_mode"]
+
+
+class _Layer(nn.Module):
+ gradient_checkpointing = False # class attribute, as GradientCheckpointingLayer
+
+ def __init__(self):
+ super().__init__()
+ self.proj = nn.Linear(4, 4)
+ self.drop = nn.Dropout(0.1)
+
+
+class _Backbone(nn.Module):
+ def __init__(self, shared):
+ super().__init__()
+ self.gradient_checkpointing = False # instance attribute, as Qwen3Model
+ self.layers = nn.ModuleList([_Layer(), _Layer()])
+ self.shared = shared
+ self.missing = None # an empty submodule slot
+
+
+class _Delegating(nn.Module):
+ """A PEFT-style wrapper: attribute reads fall through to the wrapped model."""
+
+ def __init__(self, model):
+ super().__init__()
+ self.model = model
+
+ def __getattr__(self, name):
+ try:
+ return super().__getattr__(name)
+ except AttributeError:
+ return getattr(self.model, name)
+
+
+def _build():
+ shared = nn.Linear(4, 4)
+ backbone = _Backbone(shared)
+ backbone.head = shared # the same module reachable twice
+ return _Delegating(backbone)
+
+
+def _flags(model):
+ return [(name, m.training) for name, m in model.named_modules(remove_duplicate = False)]
+
+
[email protected]("start", [True, False])
[email protected]("training", [True, False])
+def test_matches_model_train(switch, start, training):
+ ref, ours = _build(), _build()
+ ref.train(start)
+ ours.train(start)
+ ref.train(training)
+ switch(ours, training)
+ assert _flags(ours) == _flags(ref)
+
+
+def test_returns_exactly_the_modules_hasattr_finds(switch):
+ model = _build()
+ expected = [m for m in model.modules() if hasattr(m, "gradient_checkpointing")]
+ got = switch(model, True)
+ assert sorted(map(id, got)) == sorted(map(id, expected))
+ assert model in got # the delegating wrapper answers hasattr through the wrapped model
+
+
+def test_drops_fast_lora_only_when_entering_training(switch):
+ model = _build()
+ weight = model.model.layers[0].proj.weight
+ weight._fast_lora = weight.detach().clone()
+ switch(model, False)
+ assert hasattr(weight, "_fast_lora")
+ switch(model, True)
+ assert not hasattr(weight, "_fast_lora")
+
+
+def test_a_class_with_its_own_train_gets_the_real_call(switch):
+ calls = []
+
+ class _Custom(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.inner = nn.Linear(2, 2)
+
+ def train(self, mode = True):
+ calls.append(mode)
+ return super().train(mode)
+
+ model = nn.Sequential(_Custom(), nn.Dropout())
+ switch(model, False)
+ assert calls == [False]
+ assert not any(m.training for m in model.modules())
+
+
+def test_a_class_with_its_own_setattr_still_sees_the_write(switch):
+ seen = []
+
+ class _Watching(nn.Linear):
+ def __setattr__(self, name, value):
+ if name == "training":
+ seen.append(value)
+ super().__setattr__(name, value)
+
+ model = nn.Sequential(_Watching(2, 2))
+ seen.clear()
+ switch(model, False)
+ assert seen == [False]
+ assert model[0].training is False
+
+
[email protected]("use_kernels", [False, True])
+def test_pretrained_model_recasts_kernels_only_when_they_are_on(switch, monkeypatch, use_kernels):
+ transformers = pytest.importorskip("transformers")
+ if transformers.PreTrainedModel.train is nn.Module.train:
+ pytest.skip("this transformers has no PreTrainedModel.train override")
+
+ class _Config(transformers.PretrainedConfig):
+ model_type = "unsloth_switch_test"
+
+ class _Model(transformers.PreTrainedModel):
+ config_class = _Config
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.lin = nn.Linear(2, 2)
+
+ recasts = []
+ # transformers <= 5.5 re-kernelizes with kernelize(), later ones with set_use_kernels(True).
+ for hook in ("set_use_kernels", "kernelize"):
+ monkeypatch.setattr(_Model, hook, lambda self, *a, **k: recasts.append(1), raising = False)
+
+ def run(step):
+ model = _Model(_Config())
+ model.__dict__["_use_kernels"] = use_kernels
+ model.train(True)
+ recasts.clear()
+ step(model)
+ assert not model.training and not model.lin.training
+ return len(recasts)
+
+ expected = run(lambda model: model.eval())
+ assert run(lambda model: switch(model, False)) == expected
+ assert (expected > 0) is use_kernels
+
+
+def _reference_and_ours(build, switch, training):
+ ref, ours = build(), build()
+ ref.train() if training else ref.eval()
+ switch(ours, training)
+ return ref, ours
+
+
+def _keep_eval(module, mode = True):
+ # What _unsloth_freeze_norm_running_stats installs on frozen running-stat norms.
+ return torch.nn.Module.train(module, False)
+
+
[email protected]("training", [True, False])
+def test_a_per_instance_train_partial_is_honoured(switch, training):
+ def build():
+ model = nn.Sequential(nn.Linear(2, 2), nn.BatchNorm1d(2))
+ model[1].train = functools.partial(_keep_eval, model[1])
+ return model
+
+ ref, ours = _reference_and_ours(build, switch, training)
+ assert _flags(ours) == _flags(ref)
+ assert ours[1].training is False
+
+
+def test_a_root_eval_override_runs_on_inference(switch):
+ calls = []
+
+ class _Root(nn.Sequential):
+ def eval(self):
+ calls.append("eval")
+ return super().eval()
+
+ model = _Root(nn.Linear(2, 2))
+ switch(model, False)
+ assert calls == ["eval"] and not model[0].training
+
+
+def test_a_property_backed_training_flag_goes_through_its_setter(switch):
+ class _Prop(nn.Linear):
+ @property
+ def training(self):
+ return self.__dict__.get("_mode", True)
+
+ @training.setter
+ def training(self, value):
+ self.__dict__["_mode"] = value
+
+ model = nn.Sequential(_Prop(2, 2))
+ switch(model, False)
+ assert model[0].training is False
+
+
+def test_a_custom_child_traversal_is_followed(switch):
+ class _Hiding(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.visible = nn.Dropout()
+ self.hidden = nn.Dropout()
+
+ def named_children(self):
+ yield "visible", self.visible
+
+ for training in (True, False):
+
+ def build():
+ model = nn.Sequential(_Hiding())
+ for module in model.modules():
+ module.training = not training
+ return model
+
+ ref, ours = _reference_and_ours(build, switch, training)
+ assert _flags(ours) == _flags(ref)
+ assert ours[0].hidden.training is (not training)
+
+
+def test_pretrained_train_override_only_recasts_kernels():
+ """Drift guard: the flat path treats PreTrainedModel.train as nn.Module.train unless use_kernels is
+ set. If transformers adds anything else there, this fails and the flat path must learn it."""
+ transformers = pytest.importorskip("transformers")
+ train = transformers.PreTrainedModel.train
+ if train is nn.Module.train:
+ pytest.skip("this transformers has no PreTrainedModel.train override")
+ tree = ast.parse(inspect.cleandoc("\n" + inspect.getsource(train)))
+ attrs = {n.attr for n in ast.walk(tree) if isinstance(n, ast.Attribute)}
+ assert attrs <= {"training", "train", "use_kernels", "set_use_kernels", "kernelize"}, attrs
+ assert transformers.PreTrainedModel.eval is nn.Module.eval or "train" in inspect.getsource(
+ transformers.PreTrainedModel.eval
+ )
+
+
+def test_a_shared_module_under_a_non_recursing_parent_still_switches(switch):
+ class _NoRecurse(nn.Module):
+ def __init__(self, inner):
+ super().__init__()
+ self.inner = inner
+
+ def train(self, mode = True):
+ self.training = mode
+ return self
+
+ for training in (True, False):
+ for shared_first in (True, False):
+
+ def build():
+ shared = nn.Dropout()
+ parts = (
+ [shared, _NoRecurse(shared)] if shared_first else [_NoRecurse(shared), shared]
+ )
+ model = nn.Sequential(*parts)
+ for module in model.modules():
+ module.training = not training
+ return model
+
+ ref, ours = _reference_and_ours(build, switch, training)
+ assert _flags(ours) == _flags(ref)
+
+
+def test_custom_train_calls_run_in_recursion_order(switch):
+ class _Sibling(nn.Module):
+ def __init__(self, other):
+ super().__init__()
+ self.__dict__["other"] = other # not a registered child
+
+ def train(self, mode = True):
+ super().train(mode)
+ self.other.training = not mode
+ return self
+
+ def build():
+ later = nn.Dropout()
+ return nn.Sequential(_Sibling(later), later)
+
+ for training in (True, False):
+ ref, ours = _reference_and_ours(build, switch, training)
+ assert _flags(ours) == _flags(ref)
+
+
+def test_gradient_checkpointing_exposed_by_getattribute_is_found(switch):
+ class _Dynamic(nn.Linear):
+ def __getattribute__(self, name):
+ if name == "gradient_checkpointing":
+ return False
+ return super().__getattribute__(name)
+
+ model = nn.Sequential(_Dynamic(2, 2))
+ got = switch(model, False)
+ assert model[0] in got
+
+
+def test_a_child_registered_twice_in_one_parent_switches_once(switch):
+ calls = []
+
+ class _Counting(nn.Linear):
+ def train(self, mode = True):
+ calls.append(mode)
+ return super().train(mode)
+
+ child = _Counting(2, 2)
+ parent = nn.Module()
+ parent.first = child
+ parent.second = child
+ parent.train(False)
+ calls.clear()
+ parent.train(True)
+ expected = list(calls)
+ parent.train(False)
+ calls.clear()
+ switch(parent, True)
+ assert calls == expected == [True]
+
+
+class _ExposesExtra(nn.Module):
+ """Enumerates a layer and a parameter that are not registered children."""
+
+ def __init__(self):
+ super().__init__()
+ self.body = nn.Linear(2, 2)
+ self.__dict__["extra"] = _Layer()
+
+ def named_modules(self, *args, **kwargs):
+ yield from super().named_modules(*args, **kwargs)
+ yield "extra", self.__dict__["extra"]
+
+ def parameters(self, recurse = True):
+ yield from super().parameters(recurse)
+ yield from self.__dict__["extra"].parameters(recurse)
+
+
+def test_custom_module_enumeration_is_used_for_checkpoint_flags(switch):
+ model = _ExposesExtra()
+ assert model.__dict__["extra"] in switch(model, True)
+
+
+def test_custom_parameter_enumeration_is_used_for_fast_lora(switch):
+ model = _ExposesExtra()
+ weight = model.__dict__["extra"].proj.weight
+ weight._fast_lora = weight.detach().clone()
+ switch(model, True)
+ assert not hasattr(weight, "_fast_lora")
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index ad32446e7d2..c39d148403f 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -8896,22 +8896,27 @@ def _is_unarmed(module):
)
-def resolve_training_gradient_checkpointing(model, use_gradient_checkpointing):
+def resolve_training_gradient_checkpointing(
+ model,
+ use_gradient_checkpointing,
+ modules = None,
+):
"""`None` keeps the mode chosen at load / get_peft_model instead of forcing it on."""
if use_gradient_checkpointing is not None:
return use_gradient_checkpointing
if hasattr(model, "_unsloth_gradient_checkpointing"):
return model._unsloth_gradient_checkpointing
# Nothing recorded (no adapter, full finetuning): loading's choice is whether it armed the layers.
- checkpointing = [m for m in model.modules() if _calls_checkpoint_function(m)]
+ modules = model.modules() if modules is None else modules
+ checkpointing = [m for m in modules if _calls_checkpoint_function(m)]
if checkpointing:
return any(not _is_unarmed(m) for m in checkpointing)
return True
-def arm_gradient_checkpointing(model):
+def arm_gradient_checkpointing(model, modules = None):
"""Install the checkpoint function a load with checkpointing off never gave the layers."""
- if not any(_is_unarmed(m) for m in model.modules()):
+ if not any(_is_unarmed(m) for m in (model.modules() if modules is None else modules)):
return False
# Outer model, not get_base_model(): Gemma 3N / 4 and DeepSeek-V4.1 install their override there.
try:
@@ -8937,3 +8942,151 @@ def set_module_gradient_checkpointing(module, value):
return False
module.gradient_checkpointing = value
return True
+
+
[email protected]_cache(maxsize = None)
+def _mode_switch_class_info(cls):
+ """(train, eval, gc, setattr, walk) for switch_training_mode, per class.
+
+ train: True when train() and the child traversal are nn.Module's, "kernels" for PreTrainedModel.train
+ (adds a kernel re-cast only under `use_kernels`), False otherwise. eval: eval() is nn.Module's or
+ PreTrainedModel's. gc: the class has gradient_checkpointing, None for classes with their own
+ __getattr__ / __getattribute__ (PEFT wrappers delegate it). setattr: `training` is a plain instance
+ attribute. walk: modules() / parameters() enumerate the registered tree, as the raw _modules walk does.
+ """
+ try:
+ from transformers import PreTrainedModel
+ pretrained = (PreTrainedModel.train, PreTrainedModel.eval)
+ except Exception:
+ pretrained = (None, None)
+ module = torch.nn.Module
+ children = cls.children is module.children and cls.named_children is module.named_children
+ if not children:
+ train = False
+ elif cls.train is module.train:
+ train = True
+ else:
+ train = "kernels" if cls.train is pretrained[0] else False
+ plain_eval = cls.eval is module.eval or cls.eval is pretrained[1]
+ plain_lookup = (
+ cls.__getattr__ is module.__getattr__ and cls.__getattribute__ is module.__getattribute__
+ )
+ gc = hasattr(cls, "gradient_checkpointing") if plain_lookup else None
+ descriptor = inspect.getattr_static(cls, "training", None)
+ plain_setattr = cls.__setattr__ is module.__setattr__ and not hasattr(
+ type(descriptor), "__set__"
+ )
+ plain_walk = all(
+ getattr(cls, name) is getattr(module, name)
+ for name in ("modules", "named_modules", "parameters", "named_parameters", "_named_members")
+ )
+ return train, plain_eval, gc, plain_setattr, plain_walk
+
+
+def _has_gradient_checkpointing(module, on_class):
+ # hasattr() for a plain nn.Module, without its slow AttributeError path.
+ if on_class is None:
+ return hasattr(module, "gradient_checkpointing")
+ return (
+ on_class
+ or "gradient_checkpointing" in module.__dict__
+ or "gradient_checkpointing" in module._parameters
+ or "gradient_checkpointing" in module._buffers
+ or "gradient_checkpointing" in module._modules
+ )
+
+
+def switch_training_mode(
+ model,
+ training,
+ defer = False,
+):
+ """for_inference / for_training's model.train() / eval() plus their gradient_checkpointing and
+ `_fast_lora` scans (every GRPO step) in one walk; returns the modules with a gradient_checkpointing flag.
+
+ `training` is written directly only where nn.Module.train would do nothing else; any other train()
+ (class or instance override such as the frozen-norm partial, kernels on, own child traversal) runs for
+ its subtree, and a differing root or eval() gets model.train() / model.eval(). defer=True returns
+ (modules, apply) so for_training can still set the mode last.
+ """
+ if not isinstance(model, torch.nn.Module):
+ if training:
+ for param in model.parameters():
+ if hasattr(param, "_fast_lora"):
+ del param._fast_lora
+ checkpointing = [m for m in model.modules() if hasattr(m, "gradient_checkpointing")]
+ apply = model.train if training else model.eval
+ if defer:
+ return checkpointing, apply
+ apply()
+ return checkpointing
+ # Mode writes in nn.Module.train's preorder, shared modules once per path, so custom train() calls run
+ # where the recursion would run them. The gc / _fast_lora scan sees each module once, as modules() does.
+ ops, checkpointing = [], []
+ seen = set()
+ plain_walk = True
+ stack = [(model, False)]
+ while stack:
+ module, covered = stack.pop()
+ if module is None:
+ continue
+ train, plain_eval, gc, plain_setattr, walk = _mode_switch_class_info(type(module))
+ plain_walk = plain_walk and walk
+ if not covered:
+ if (
+ train is False
+ or "train" in module.__dict__
+ or (train == "kernels" and getattr(module, "use_kernels", False))
+ or (
+ module is model
+ and not training
+ and (not plain_eval or "eval" in module.__dict__)
+ )
+ ):
+ ops.append((module, None))
+ covered = True
+ else:
+ ops.append((module, plain_setattr))
+ if id(module) in seen:
+ if covered or len(ops) > 64 * len(seen): # nothing left to record, or a cycle
+ continue
+ else:
+ seen.add(id(module))
+ if _has_gradient_checkpointing(module, gc):
+ checkpointing.append(module)
+ if training:
+ for param in module._parameters.values():
+ if param is not None and "_fast_lora" in param.__dict__:
+ del param._fast_lora
+ if module._modules:
+ children = [child for child in module._modules.values() if child is not None]
+ if len(children) > 1:
+ # named_children() yields a module registered under two names once per parent.
+ children = list({id(child): child for child in children}.values())
+ children.reverse()
+ stack.extend([(child, covered) for child in children])
+ if not plain_walk:
+ # Some class enumerates modules / parameters its own way: scan as the old code did.
+ checkpointing = [m for m in model.modules() if hasattr(m, "gradient_checkpointing")]
+ if training:
+ for param in model.parameters():
+ if hasattr(param, "_fast_lora"):
+ del param._fast_lora
+
+ def apply():
+ if (ops and ops[0][0] is model and ops[0][1] is None) or len(ops) > 64 * len(seen):
+ model.train() if training else model.eval()
+ return
+ for module, plain_setattr in ops:
+ if plain_setattr is None:
+ module.train(training)
+ # nn.Module.__setattr__ only routes Parameters / Modules / buffers; a bool lands in __dict__.
+ elif plain_setattr:
+ module.__dict__["training"] = training
+ else:
+ module.training = training
+
+ if defer:
+ return checkpointing, apply
+ apply()
+ return checkpointing
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 5fce9674afb..6abad517777 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -35,6 +35,7 @@ from ._utils import (
arm_gradient_checkpointing,
resolve_training_gradient_checkpointing,
set_module_gradient_checkpointing,
+ switch_training_mode,
)
from .loader_utils import (
DEFAULT_DEVICE_MAP,
@@ -4666,12 +4667,9 @@ class FastLlamaModel:
_for_inference(m)
m = m.model
_for_inference(m)
- model.eval() # to turn off training on modules deeper in
-
- # Since transformers 4.53, this must be turned off explicitly.
- for module in model.modules():
- if hasattr(module, "gradient_checkpointing"):
- module.gradient_checkpointing = False
+ # Since transformers 4.53, checkpointing must be turned off explicitly.
+ for module in switch_training_mode(model, False):
+ module.gradient_checkpointing = False
if hasattr(model, "get_input_embeddings"):
embeddings = model.get_input_embeddings()
@@ -4697,15 +4695,14 @@ class FastLlamaModel:
raise TypeError(
"Unsloth: I think you're passing a tokenizer, not the model to for_training!"
)
+ # Mode last, as model.train() was: a custom train() keeps the final say over the chain below.
+ checkpointing, apply_training_mode = switch_training_mode(model, True, defer = True)
use_gradient_checkpointing = resolve_training_gradient_checkpointing(
- model, use_gradient_checkpointing
+ model, use_gradient_checkpointing, checkpointing
)
- if use_gradient_checkpointing:
- arm_gradient_checkpointing(model)
-
- for param in model.parameters():
- if hasattr(param, "_fast_lora"):
- del param._fast_lora
+ if use_gradient_checkpointing and arm_gradient_checkpointing(model, checkpointing):
+ # Arming runs the model's enabler once (remote code can flag layers the walk did not see).
+ checkpointing = [m for m in model.modules() if hasattr(m, "gradient_checkpointing")]
def _for_training(m):
if hasattr(m, "gradient_checkpointing"):
@@ -4726,12 +4723,10 @@ class FastLlamaModel:
_for_training(m)
m = m.model
_for_training(m)
- model.train() # to turn on training on modules deeper in
-
+ apply_training_mode()
# Since transformers 4.53, this must be turned on explicitly.
- for module in model.modules():
- if hasattr(module, "gradient_checkpointing"):
- set_module_gradient_checkpointing(module, use_gradient_checkpointing)
+ for module in checkpointing:
+ set_module_gradient_checkpointing(module, use_gradient_checkpointing)
if hasattr(model, "get_input_embeddings"):
embeddings = model.get_input_embeddings()
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 99c1b6005b9..610db015f7b 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -128,6 +128,7 @@ from ._utils import (
arm_gradient_checkpointing,
resolve_training_gradient_checkpointing,
set_module_gradient_checkpointing,
+ switch_training_mode,
_is_seq2seq_lm_config,
)
from ._utils import *
@@ -5033,12 +5034,9 @@ class FastBaseModel:
_for_inference(m)
m = m.model
_for_inference(m)
- model.eval() # to turn off training on modules deeper in
-
- # Since transformers 4.53, this must be turned off explicitly.
- for module in model.modules():
- if hasattr(module, "gradient_checkpointing"):
- module.gradient_checkpointing = False
+ # Since transformers 4.53, checkpointing must be turned off explicitly.
+ for module in switch_training_mode(model, False):
+ module.gradient_checkpointing = False
for _getter in ("get_input_embeddings", "get_output_embeddings"):
embeddings = _embeddings_or_none(model, _getter)
@@ -5068,15 +5066,14 @@ class FastBaseModel:
raise TypeError(
"Unsloth: I think you're passing a tokenizer, not the model to for_training!"
)
+ # Mode last, as model.train() was: a custom train() keeps the final say over the chain below.
+ checkpointing, apply_training_mode = switch_training_mode(model, True, defer = True)
use_gradient_checkpointing = resolve_training_gradient_checkpointing(
- model, use_gradient_checkpointing
+ model, use_gradient_checkpointing, checkpointing
)
- if use_gradient_checkpointing:
- arm_gradient_checkpointing(model)
-
- for param in model.parameters():
- if hasattr(param, "_fast_lora"):
- del param._fast_lora
+ if use_gradient_checkpointing and arm_gradient_checkpointing(model, checkpointing):
+ # Arming runs the model's enabler once (remote code can flag layers the walk did not see).
+ checkpointing = [m for m in model.modules() if hasattr(m, "gradient_checkpointing")]
def _for_training(m):
if hasattr(m, "gradient_checkpointing"):
@@ -5097,12 +5094,10 @@ class FastBaseModel:
_for_training(m)
m = m.model
_for_training(m)
- model.train() # to turn on training on modules deeper in
-
+ apply_training_mode()
# Since transformers 4.53, this must be turned on explicitly.
- for module in model.modules():
- if hasattr(module, "gradient_checkpointing"):
- set_module_gradient_checkpointing(module, use_gradient_checkpointing)
+ for module in checkpointing:
+ set_module_gradient_checkpointing(module, use_gradient_checkpointing)
for _getter in ("get_input_embeddings", "get_output_embeddings"):
embeddings = _embeddings_or_none(model, _getter)