Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
CPU backend: install the every_n_layers gradient checkpointing router (#16730)
Daniel Hanf450da37e775
diff --git a/tests/test_block_swap_install.py b/tests/test_block_swap_install.pyindex c2c7564274d..3e8fc7f9236 100644--- a/tests/test_block_swap_install.py+++ b/tests/test_block_swap_install.py@@ -538,7 +538,10 @@ def test_host_load_runs_for_layers_or_streamed_embeddings_only():def _load_skip():- src = open(UTILS, encoding = "utf-8").read()+ gc_kwargs = os.path.join(+ os.path.dirname(os.path.dirname(UTILS)), "_gc_kwargs.py"+ ) # _skip_aware_flag+ src = open(UTILS, encoding = "utf-8").read() + "\n" + open(gc_kwargs, encoding = "utf-8").read()mod = ast.parse(src)names = ("_skip_aware_flag", "skip_checkpointing", "_gc_keep_layers_env")ns = {"find_decoder_layers": lambda m: m.layers, "os": os}diff --git a/tests/test_gc_enable_kwargs.py b/tests/test_gc_enable_kwargs.pyindex 22010cfe7b1..590dd70d167 100644--- a/tests/test_gc_enable_kwargs.py+++ b/tests/test_gc_enable_kwargs.py@@ -12,7 +12,7 @@ import transformers.modeling_utils as hf_modeling_utilsfrom transformers import PreTrainedModel, Qwen3Config, Qwen3ForCausalLMHERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))-UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py")+UTILS = os.path.join(HERE, "unsloth", "_gc_kwargs.py")VISION = os.path.join(HERE, "unsloth", "models", "vision.py")LOADER_UTILS = os.path.join(HERE, "unsloth", "models", "loader_utils.py")FUNCS = (diff --git a/tests/test_gc_every_n_cpu_route.py b/tests/test_gc_every_n_cpu_route.pynew file mode 100644index 00000000000..d96d6e2c9a9--- /dev/null+++ b/tests/test_gc_every_n_cpu_route.py@@ -0,0 +1,115 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""every_n_layers on a CPU-only host, where `import unsloth` loads _cpu_init.py instead of models/_utils.py."""++from __future__ import annotations++import json+import os+import subprocess+import sys+from pathlib import Path++import pytest++_ROOT = Path(__file__).resolve().parents[1]++_PROBE = r"""+import json, sys+import unsloth+import torch+from transformers import PreTrainedModel, Qwen3Config, Qwen3ForCausalLM++def model():+ torch.manual_seed(0)+ cfg = Qwen3Config(vocab_size = 64, hidden_size = 16, intermediate_size = 32, num_hidden_layers = 4,+ num_attention_heads = 2, num_key_value_heads = 1, head_dim = 8, max_position_embeddings = 64)+ m = Qwen3ForCausalLM(cfg).train()+ m.config.use_cache = False+ return m++def layers(m):+ return [x for x in m.modules() if type(x).__name__ == "Qwen3DecoderLayer"]++def run(m):+ calls = [0] * len(layers(m))+ for i, layer in enumerate(layers(m)):+ f = layer.__dict__.get("_gradient_checkpointing_func")+ if f is not None:+ def w(*a, _f = f, _i = i, **k):+ calls[_i] += 1+ return _f(*a, **k)+ layer._gradient_checkpointing_func = w+ ids = torch.randint(0, 64, (2, 8))+ m(input_ids = ids, labels = ids).loss.backward()+ return calls++out = {"device_type": unsloth.DEVICE_TYPE,+ "models_loaded": sorted(n for n in sys.modules if n == "unsloth.models" or n.startswith("unsloth.models.")),+ "patched": bool(getattr(PreTrainedModel.gradient_checkpointing_enable, "_unsloth_gc_kwargs", False))}+res = {}+# dict form (TrainingArguments on transformers < 5.17, PEFT prepare_model_for_kbit_training, direct calls)+m = model()+try:+ m.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False, "every_n_layers": 2})+ res["dict"] = run(m)+except Exception as e:+ res["dict"] = repr(e)[:200]+# keyword form, then every flag rewritten to True (what for_training / the trainer do): skipped layers stay out+m = model()+m.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False}, every_n_layers = 2)+for x in m.modules():+ if hasattr(x, "gradient_checkpointing"):+ x.gradient_checkpointing = True+res["flag_rewrite"] = run(m)+# plain enable stays every layer+m = model()+m.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})+res["plain"] = run(m)+out["calls"] = res+print("PROBE " + json.dumps(out))+"""+++def _zoo_has_cpu_backend():+ import importlib.util+ spec = importlib.util.find_spec("unsloth_zoo")+ return bool(spec and spec.submodule_search_locations) and any(+ (Path(p) / "cpu" / "loader.py").is_file() for p in spec.submodule_search_locations+ )++[email protected](not _zoo_has_cpu_backend(), reason = "unsloth_zoo without the CPU backend")+def test_every_n_layers_on_the_cpu_route():+ torch = pytest.importorskip("torch")+ pytest.importorskip("transformers")+ for name in ("xpu", "npu"):+ try:+ if getattr(torch, name).is_available():+ pytest.skip(f"{name} host: hiding CUDA leaves that accelerator selected")+ except Exception:+ pass+ env = {+ k: v+ for k, v in os.environ.items()+ if k not in ("UNSLOTH_ALLOW_CPU", "UNSLOTH_FORCE_CPU", "UNSLOTH_DISABLE_CPU_FALLBACK")+ }+ env.update({"CUDA_VISIBLE_DEVICES": "", "UNSLOTH_FORCE_GPU_PATH": "1"})+ env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(_ROOT), env.get("PYTHONPATH")]))+ out = subprocess.run(+ [sys.executable, "-c", _PROBE], capture_output = True, text = True, env = env, timeout = 1200+ )+ probe = next(+ (json.loads(x[6:]) for x in out.stdout.splitlines() if x.startswith("PROBE ")), None+ )+ if probe is None and "ROCm" in out.stderr:+ pytest.skip("ROCm runtime host without a ROCm torch keeps its install hint")+ assert probe is not None, out.stderr[-3000:]+ assert probe["device_type"] == "cpu"+ assert probe["models_loaded"] == [], "the CPU route must not import the GPU model stack"+ assert probe["patched"]+ calls = probe["calls"]+ assert calls["plain"] == [1, 1, 1, 1]+ # transformers' rule: layers 0, N, 2N... stay checkpointed+ assert calls["dict"] == [1, 0, 1, 0], calls["dict"]+ assert calls["flag_rewrite"] == [1, 0, 1, 0], calls["flag_rewrite"]diff --git a/tests/test_gc_skip_auto.py b/tests/test_gc_skip_auto.pyindex 142f7af6f17..f58b4f9a452 100644--- a/tests/test_gc_skip_auto.py+++ b/tests/test_gc_skip_auto.py@@ -12,6 +12,7 @@ from transformers import Qwen3Config, Qwen3ForCausalLMHERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py")+GC_KWARGS = os.path.join(HERE, "unsloth", "_gc_kwargs.py") # _skip_aware_flagNAMES = ("_skip_aware_flag","skip_checkpointing",@@ -24,7 +25,7 @@ NAMES = (def _load():- src = open(UTILS, encoding = "utf-8").read()+ src = open(UTILS, encoding = "utf-8").read() + "\n" + open(GC_KWARGS, encoding = "utf-8").read()ns = {"torch": torch,"os": os,diff --git a/unsloth/_cpu_init.py b/unsloth/_cpu_init.pyindex da627c7448b..ab0231e6498 100644--- a/unsloth/_cpu_init.py+++ b/unsloth/_cpu_init.py@@ -22,6 +22,11 @@ DEVICE_COUNT = 0install_cpu_training_defaults()+# Same every_n_layers / offload router the GPU route installs from models/_utils.py.+from ._gc_kwargs import patch_gradient_checkpointing_enable_kwargs++patch_gradient_checkpointing_enable_kwargs()+print(f"Unsloth {__version__}: no GPU found, training on CPU ({__import__('platform').machine()}, "f"{torch.get_num_threads()} threads, bfloat16 = {cpu_bf16_supported()}). CPU training is much "diff --git a/unsloth/_gc_kwargs.py b/unsloth/_gc_kwargs.pynew file mode 100644index 00000000000..466b7376cec--- /dev/null+++ b/unsloth/_gc_kwargs.py@@ -0,0 +1,157 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""gradient_checkpointing_enable(every_n_layers, offload) for both import routes (GPU _utils.py, CPU _cpu_init.py).+Imports transformers lazily so the CPU route avoids the GPU model stack."""++import functools+import inspect+++def _skip_aware_flag(cls):+ # for_training, gradient_checkpointing_enable and the trainer rewrite the flag; a marked layer keeps reading False.+ if cls.__dict__.get("_unsloth_skip_aware", False):+ return++ def _get(self):+ if self.__dict__.get("_unsloth_skip_checkpoint", False) or self.__dict__.get(+ "_unsloth_every_n_skip", False+ ):+ return False+ # Unmarked layers of the class still hold the flag in their own dict.+ return self.__dict__.get(+ "_unsloth_gc_flag", self.__dict__.get("gradient_checkpointing", False)+ )++ def _set(self, value):+ self.__dict__["_unsloth_gc_flag"] = value++ cls.gradient_checkpointing = property(_get, _set)+ cls._unsloth_skip_aware = True+++def _apply_every_n_layers(model, every_n_layers):+ """Checkpoint GradientCheckpointingLayer 0, N, 2N...; held by the skip-aware flag so for_training keeps it."""+ try:+ from transformers.modeling_layers import GradientCheckpointingLayer+ except Exception:+ return []+ layers = [+ m+ for m in model.modules()+ if isinstance(m, GradientCheckpointingLayer)+ or "_unsloth_gradient_checkpointing_holder" in m.__dict__+ ]+ skipped = []+ for i, layer in enumerate(layers):+ if every_n_layers > 1 and i % every_n_layers != 0:+ if not isinstance(layer, GradientCheckpointingLayer):+ layer.__dict__["_unsloth_every_n_skip"] = True+ skipped.append(i)+ continue+ if not hasattr(layer, "gradient_checkpointing"):+ continue+ current = layer.__dict__.pop("gradient_checkpointing", None)+ _skip_aware_flag(type(layer))+ if current is not None:+ layer.__dict__["_unsloth_gc_flag"] = current+ layer.__dict__["_unsloth_every_n_skip"] = True+ skipped.append(i)+ else:+ layer.__dict__.pop("_unsloth_every_n_skip", None)+ return skipped+++_OFFLOADING_CHECKPOINT_SHIMS = frozenset(+ {"unsloth_checkpoint", "unsloth_offloaded_gradient_checkpoint"}+)+++def _checkpoint_offloads(fn):+ # Unsloth checkpoint functions that offload set `_unsloth_offloads`; the zoo shims are known by name.+ return bool(getattr(fn, "_unsloth_offloads", False)) or (+ getattr(fn, "__name__", "") in _OFFLOADING_CHECKPOINT_SHIMS+ )+++_GC_KWARGS_WARNED = set()+++def _gc_kwargs_note(key, message):+ if key not in _GC_KWARGS_WARNED:+ _GC_KWARGS_WARNED.add(key)+ print(message)+++def patch_gradient_checkpointing_enable_kwargs():+ """Pop every_n_layers / offload from gradient_checkpointing_kwargs; offload under Unsloth's offloading shim would double-offload."""+ try:+ from transformers import PreTrainedModel+ import transformers.modeling_utils as hf_modeling_utils+ except Exception:+ return+ original = PreTrainedModel.gradient_checkpointing_enable+ if getattr(original, "_unsloth_gc_kwargs", False):+ return+ try:+ params = list(inspect.signature(original).parameters)+ except Exception:+ return+ positional = [p for p in params if p not in ("self", "gradient_checkpointing_kwargs")]+ native_every = "every_n_layers" in params+ native_offload = "offload" in params++ @functools.wraps(original)+ def gradient_checkpointing_enable(+ self,+ gradient_checkpointing_kwargs = None,+ *args,+ **kwargs,+ ):+ kwargs.update(zip(positional, args))+ every = kwargs.pop("every_n_layers", None)+ offload = kwargs.pop("offload", None)+ if gradient_checkpointing_kwargs is not None:+ gc_kwargs = dict(gradient_checkpointing_kwargs)+ dict_every = gc_kwargs.pop("every_n_layers", None)+ dict_offload = gc_kwargs.pop("offload", None)+ every = dict_every if every is None else every+ offload = dict_offload if offload is None else offload+ if len(gc_kwargs) != len(gradient_checkpointing_kwargs):+ gradient_checkpointing_kwargs = gc_kwargs or None+ every = 1 if every is None else int(every)+ if every < 1:+ raise ValueError(f"Unsloth: every_n_layers must be >= 1, got {every}.")+ if offload:+ shim = getattr(hf_modeling_utils, "checkpoint", None)+ if _checkpoint_offloads(shim):+ _gc_kwargs_note(+ "offload",+ "Unsloth: gradient checkpointing already offloads activations to host memory, "+ "so transformers' offload=True is ignored.",+ )+ offload = False+ elif not native_offload:+ _gc_kwargs_note(+ "offload_native",+ "Unsloth: gradient checkpointing offload=True needs transformers >= 5.17; ignored. "+ 'use_gradient_checkpointing = "unsloth" offloads activations on any version.',+ )+ offload = False+ if native_offload and offload:+ kwargs["offload"] = True+ if native_every:+ kwargs["every_n_layers"] = 1+ out = original(self, gradient_checkpointing_kwargs, **kwargs)+ if every > 1 or any("_unsloth_every_n_skip" in m.__dict__ for m in self.modules()):+ skipped = _apply_every_n_layers(self, every)+ if every > 1 and not skipped:+ _gc_kwargs_note(+ "every_n_layers",+ f"Unsloth: every_n_layers = {every} found no GradientCheckpointingLayer to skip; "+ "every layer stays checkpointed.",+ )+ return out++ gradient_checkpointing_enable._unsloth_gc_kwargs = True+ PreTrainedModel.gradient_checkpointing_enable = gradient_checkpointing_enablediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 1d3d159c240..c2f383b0e5a 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -10733,26 +10733,7 @@ def _training_reserve_bytes(return reserve, seq_len-def _skip_aware_flag(cls):- # for_training, gradient_checkpointing_enable and the trainer rewrite the flag; a marked layer keeps reading False.- if cls.__dict__.get("_unsloth_skip_aware", False):- return-- def _get(self):- if self.__dict__.get("_unsloth_skip_checkpoint", False) or self.__dict__.get(- "_unsloth_every_n_skip", False- ):- return False- # Unmarked layers of the class still hold the flag in their own dict.- return self.__dict__.get(- "_unsloth_gc_flag", self.__dict__.get("gradient_checkpointing", False)- )-- def _set(self, value):- self.__dict__["_unsloth_gc_flag"] = value-- cls.gradient_checkpointing = property(_get, _set)- cls._unsloth_skip_aware = True+from .._gc_kwargs import _skip_aware_flagdef _gc_keep_layers_env():@@ -11185,132 +11166,7 @@ def skip_checkpointing(model, layers = 0):return chosen-def _apply_every_n_layers(model, every_n_layers):- """Checkpoint GradientCheckpointingLayer 0, N, 2N...; held by the skip-aware flag so for_training keeps it."""- try:- from transformers.modeling_layers import GradientCheckpointingLayer- except Exception:- return []- layers = [- m- for m in model.modules()- if isinstance(m, GradientCheckpointingLayer)- or "_unsloth_gradient_checkpointing_holder" in m.__dict__- ]- skipped = []- for i, layer in enumerate(layers):- if every_n_layers > 1 and i % every_n_layers != 0:- if not isinstance(layer, GradientCheckpointingLayer):- layer.__dict__["_unsloth_every_n_skip"] = True- skipped.append(i)- continue- if not hasattr(layer, "gradient_checkpointing"):- continue- current = layer.__dict__.pop("gradient_checkpointing", None)- _skip_aware_flag(type(layer))- if current is not None:- layer.__dict__["_unsloth_gc_flag"] = current- layer.__dict__["_unsloth_every_n_skip"] = True- skipped.append(i)- else:- layer.__dict__.pop("_unsloth_every_n_skip", None)- return skipped---_OFFLOADING_CHECKPOINT_SHIMS = frozenset(- {"unsloth_checkpoint", "unsloth_offloaded_gradient_checkpoint"}-)---def _checkpoint_offloads(fn):- # Unsloth checkpoint functions that offload set `_unsloth_offloads`; the zoo shims are known by name.- return bool(getattr(fn, "_unsloth_offloads", False)) or (- getattr(fn, "__name__", "") in _OFFLOADING_CHECKPOINT_SHIMS- )---_GC_KWARGS_WARNED = set()---def _gc_kwargs_note(key, message):- if key not in _GC_KWARGS_WARNED:- _GC_KWARGS_WARNED.add(key)- print(message)---def patch_gradient_checkpointing_enable_kwargs():- """Pop every_n_layers / offload from gradient_checkpointing_kwargs; offload under Unsloth's offloading shim would double-offload."""- try:- from transformers import PreTrainedModel- import transformers.modeling_utils as hf_modeling_utils- except Exception:- return- original = PreTrainedModel.gradient_checkpointing_enable- if getattr(original, "_unsloth_gc_kwargs", False):- return- try:- params = list(inspect.signature(original).parameters)- except Exception:- return- positional = [p for p in params if p not in ("self", "gradient_checkpointing_kwargs")]- native_every = "every_n_layers" in params- native_offload = "offload" in params-- @functools.wraps(original)- def gradient_checkpointing_enable(- self,- gradient_checkpointing_kwargs = None,- *args,- **kwargs,- ):- kwargs.update(zip(positional, args))- every = kwargs.pop("every_n_layers", None)- offload = kwargs.pop("offload", None)- if gradient_checkpointing_kwargs is not None:- gc_kwargs = dict(gradient_checkpointing_kwargs)- dict_every = gc_kwargs.pop("every_n_layers", None)- dict_offload = gc_kwargs.pop("offload", None)- every = dict_every if every is None else every- offload = dict_offload if offload is None else offload- if len(gc_kwargs) != len(gradient_checkpointing_kwargs):- gradient_checkpointing_kwargs = gc_kwargs or None- every = 1 if every is None else int(every)- if every < 1:- raise ValueError(f"Unsloth: every_n_layers must be >= 1, got {every}.")- if offload:- shim = getattr(hf_modeling_utils, "checkpoint", None)- if _checkpoint_offloads(shim):- _gc_kwargs_note(- "offload",- "Unsloth: gradient checkpointing already offloads activations to host memory, "- "so transformers' offload=True is ignored.",- )- offload = False- elif not native_offload:- _gc_kwargs_note(- "offload_native",- "Unsloth: gradient checkpointing offload=True needs transformers >= 5.17; ignored. "- 'use_gradient_checkpointing = "unsloth" offloads activations on any version.',- )- offload = False- if native_offload and offload:- kwargs["offload"] = True- if native_every:- kwargs["every_n_layers"] = 1- out = original(self, gradient_checkpointing_kwargs, **kwargs)- if every > 1 or any("_unsloth_every_n_skip" in m.__dict__ for m in self.modules()):- skipped = _apply_every_n_layers(self, every)- if every > 1 and not skipped:- _gc_kwargs_note(- "every_n_layers",- f"Unsloth: every_n_layers = {every} found no GradientCheckpointingLayer to skip; "- "every layer stays checkpointed.",- )- return out-- gradient_checkpointing_enable._unsloth_gc_kwargs = True- PreTrainedModel.gradient_checkpointing_enable = gradient_checkpointing_enable-+from .._gc_kwargs import patch_gradient_checkpointing_enable_kwargspatch_gradient_checkpointing_enable_kwargs()