Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

CPU backend: install the every_n_layers gradient checkpointing router (#16730)

Daniel Hanf450da37e775
diff --git a/tests/test_block_swap_install.py b/tests/test_block_swap_install.py
index c2c7564274d..3e8fc7f9236 100644
--- a/tests/test_block_swap_install.py
+++ b/tests/test_block_swap_install.py
@@ -538,7 +538,10 @@ def test_host_load_runs_for_layers_or_streamed_embeddings_only():
def _load_skip():
- src = open(UTILS, encoding = "utf-8").read()
+ gc_kwargs = os.path.join(
+ os.path.dirname(os.path.dirname(UTILS)), "_gc_kwargs.py"
+ ) # _skip_aware_flag
+ src = open(UTILS, encoding = "utf-8").read() + "\n" + open(gc_kwargs, encoding = "utf-8").read()
mod = ast.parse(src)
names = ("_skip_aware_flag", "skip_checkpointing", "_gc_keep_layers_env")
ns = {"find_decoder_layers": lambda m: m.layers, "os": os}
diff --git a/tests/test_gc_enable_kwargs.py b/tests/test_gc_enable_kwargs.py
index 22010cfe7b1..590dd70d167 100644
--- a/tests/test_gc_enable_kwargs.py
+++ b/tests/test_gc_enable_kwargs.py
@@ -12,7 +12,7 @@ import transformers.modeling_utils as hf_modeling_utils
from transformers import PreTrainedModel, Qwen3Config, Qwen3ForCausalLM
HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py")
+UTILS = os.path.join(HERE, "unsloth", "_gc_kwargs.py")
VISION = os.path.join(HERE, "unsloth", "models", "vision.py")
LOADER_UTILS = os.path.join(HERE, "unsloth", "models", "loader_utils.py")
FUNCS = (
diff --git a/tests/test_gc_every_n_cpu_route.py b/tests/test_gc_every_n_cpu_route.py
new file mode 100644
index 00000000000..d96d6e2c9a9
--- /dev/null
+++ b/tests/test_gc_every_n_cpu_route.py
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""every_n_layers on a CPU-only host, where `import unsloth` loads _cpu_init.py instead of models/_utils.py."""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+_ROOT = Path(__file__).resolve().parents[1]
+
+_PROBE = r"""
+import json, sys
+import unsloth
+import torch
+from transformers import PreTrainedModel, Qwen3Config, Qwen3ForCausalLM
+
+def model():
+ torch.manual_seed(0)
+ cfg = Qwen3Config(vocab_size = 64, hidden_size = 16, intermediate_size = 32, num_hidden_layers = 4,
+ num_attention_heads = 2, num_key_value_heads = 1, head_dim = 8, max_position_embeddings = 64)
+ m = Qwen3ForCausalLM(cfg).train()
+ m.config.use_cache = False
+ return m
+
+def layers(m):
+ return [x for x in m.modules() if type(x).__name__ == "Qwen3DecoderLayer"]
+
+def run(m):
+ calls = [0] * len(layers(m))
+ for i, layer in enumerate(layers(m)):
+ f = layer.__dict__.get("_gradient_checkpointing_func")
+ if f is not None:
+ def w(*a, _f = f, _i = i, **k):
+ calls[_i] += 1
+ return _f(*a, **k)
+ layer._gradient_checkpointing_func = w
+ ids = torch.randint(0, 64, (2, 8))
+ m(input_ids = ids, labels = ids).loss.backward()
+ return calls
+
+out = {"device_type": unsloth.DEVICE_TYPE,
+ "models_loaded": sorted(n for n in sys.modules if n == "unsloth.models" or n.startswith("unsloth.models.")),
+ "patched": bool(getattr(PreTrainedModel.gradient_checkpointing_enable, "_unsloth_gc_kwargs", False))}
+res = {}
+# dict form (TrainingArguments on transformers < 5.17, PEFT prepare_model_for_kbit_training, direct calls)
+m = model()
+try:
+ m.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False, "every_n_layers": 2})
+ res["dict"] = run(m)
+except Exception as e:
+ res["dict"] = repr(e)[:200]
+# keyword form, then every flag rewritten to True (what for_training / the trainer do): skipped layers stay out
+m = model()
+m.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False}, every_n_layers = 2)
+for x in m.modules():
+ if hasattr(x, "gradient_checkpointing"):
+ x.gradient_checkpointing = True
+res["flag_rewrite"] = run(m)
+# plain enable stays every layer
+m = model()
+m.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})
+res["plain"] = run(m)
+out["calls"] = res
+print("PROBE " + json.dumps(out))
+"""
+
+
+def _zoo_has_cpu_backend():
+ import importlib.util
+ spec = importlib.util.find_spec("unsloth_zoo")
+ return bool(spec and spec.submodule_search_locations) and any(
+ (Path(p) / "cpu" / "loader.py").is_file() for p in spec.submodule_search_locations
+ )
+
+
[email protected](not _zoo_has_cpu_backend(), reason = "unsloth_zoo without the CPU backend")
+def test_every_n_layers_on_the_cpu_route():
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("transformers")
+ for name in ("xpu", "npu"):
+ try:
+ if getattr(torch, name).is_available():
+ pytest.skip(f"{name} host: hiding CUDA leaves that accelerator selected")
+ except Exception:
+ pass
+ env = {
+ k: v
+ for k, v in os.environ.items()
+ if k not in ("UNSLOTH_ALLOW_CPU", "UNSLOTH_FORCE_CPU", "UNSLOTH_DISABLE_CPU_FALLBACK")
+ }
+ env.update({"CUDA_VISIBLE_DEVICES": "", "UNSLOTH_FORCE_GPU_PATH": "1"})
+ env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(_ROOT), env.get("PYTHONPATH")]))
+ out = subprocess.run(
+ [sys.executable, "-c", _PROBE], capture_output = True, text = True, env = env, timeout = 1200
+ )
+ probe = next(
+ (json.loads(x[6:]) for x in out.stdout.splitlines() if x.startswith("PROBE ")), None
+ )
+ if probe is None and "ROCm" in out.stderr:
+ pytest.skip("ROCm runtime host without a ROCm torch keeps its install hint")
+ assert probe is not None, out.stderr[-3000:]
+ assert probe["device_type"] == "cpu"
+ assert probe["models_loaded"] == [], "the CPU route must not import the GPU model stack"
+ assert probe["patched"]
+ calls = probe["calls"]
+ assert calls["plain"] == [1, 1, 1, 1]
+ # transformers' rule: layers 0, N, 2N... stay checkpointed
+ assert calls["dict"] == [1, 0, 1, 0], calls["dict"]
+ assert calls["flag_rewrite"] == [1, 0, 1, 0], calls["flag_rewrite"]
diff --git a/tests/test_gc_skip_auto.py b/tests/test_gc_skip_auto.py
index 142f7af6f17..f58b4f9a452 100644
--- a/tests/test_gc_skip_auto.py
+++ b/tests/test_gc_skip_auto.py
@@ -12,6 +12,7 @@ from transformers import Qwen3Config, Qwen3ForCausalLM
HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py")
+GC_KWARGS = os.path.join(HERE, "unsloth", "_gc_kwargs.py") # _skip_aware_flag
NAMES = (
"_skip_aware_flag",
"skip_checkpointing",
@@ -24,7 +25,7 @@ NAMES = (
def _load():
- src = open(UTILS, encoding = "utf-8").read()
+ src = open(UTILS, encoding = "utf-8").read() + "\n" + open(GC_KWARGS, encoding = "utf-8").read()
ns = {
"torch": torch,
"os": os,
diff --git a/unsloth/_cpu_init.py b/unsloth/_cpu_init.py
index da627c7448b..ab0231e6498 100644
--- a/unsloth/_cpu_init.py
+++ b/unsloth/_cpu_init.py
@@ -22,6 +22,11 @@ DEVICE_COUNT = 0
install_cpu_training_defaults()
+# Same every_n_layers / offload router the GPU route installs from models/_utils.py.
+from ._gc_kwargs import patch_gradient_checkpointing_enable_kwargs
+
+patch_gradient_checkpointing_enable_kwargs()
+
print(
f"Unsloth {__version__}: no GPU found, training on CPU ({__import__('platform').machine()}, "
f"{torch.get_num_threads()} threads, bfloat16 = {cpu_bf16_supported()}). CPU training is much "
diff --git a/unsloth/_gc_kwargs.py b/unsloth/_gc_kwargs.py
new file mode 100644
index 00000000000..466b7376cec
--- /dev/null
+++ b/unsloth/_gc_kwargs.py
@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""gradient_checkpointing_enable(every_n_layers, offload) for both import routes (GPU _utils.py, CPU _cpu_init.py).
+Imports transformers lazily so the CPU route avoids the GPU model stack."""
+
+import functools
+import inspect
+
+
+def _skip_aware_flag(cls):
+ # for_training, gradient_checkpointing_enable and the trainer rewrite the flag; a marked layer keeps reading False.
+ if cls.__dict__.get("_unsloth_skip_aware", False):
+ return
+
+ def _get(self):
+ if self.__dict__.get("_unsloth_skip_checkpoint", False) or self.__dict__.get(
+ "_unsloth_every_n_skip", False
+ ):
+ return False
+ # Unmarked layers of the class still hold the flag in their own dict.
+ return self.__dict__.get(
+ "_unsloth_gc_flag", self.__dict__.get("gradient_checkpointing", False)
+ )
+
+ def _set(self, value):
+ self.__dict__["_unsloth_gc_flag"] = value
+
+ cls.gradient_checkpointing = property(_get, _set)
+ cls._unsloth_skip_aware = True
+
+
+def _apply_every_n_layers(model, every_n_layers):
+ """Checkpoint GradientCheckpointingLayer 0, N, 2N...; held by the skip-aware flag so for_training keeps it."""
+ try:
+ from transformers.modeling_layers import GradientCheckpointingLayer
+ except Exception:
+ return []
+ layers = [
+ m
+ for m in model.modules()
+ if isinstance(m, GradientCheckpointingLayer)
+ or "_unsloth_gradient_checkpointing_holder" in m.__dict__
+ ]
+ skipped = []
+ for i, layer in enumerate(layers):
+ if every_n_layers > 1 and i % every_n_layers != 0:
+ if not isinstance(layer, GradientCheckpointingLayer):
+ layer.__dict__["_unsloth_every_n_skip"] = True
+ skipped.append(i)
+ continue
+ if not hasattr(layer, "gradient_checkpointing"):
+ continue
+ current = layer.__dict__.pop("gradient_checkpointing", None)
+ _skip_aware_flag(type(layer))
+ if current is not None:
+ layer.__dict__["_unsloth_gc_flag"] = current
+ layer.__dict__["_unsloth_every_n_skip"] = True
+ skipped.append(i)
+ else:
+ layer.__dict__.pop("_unsloth_every_n_skip", None)
+ return skipped
+
+
+_OFFLOADING_CHECKPOINT_SHIMS = frozenset(
+ {"unsloth_checkpoint", "unsloth_offloaded_gradient_checkpoint"}
+)
+
+
+def _checkpoint_offloads(fn):
+ # Unsloth checkpoint functions that offload set `_unsloth_offloads`; the zoo shims are known by name.
+ return bool(getattr(fn, "_unsloth_offloads", False)) or (
+ getattr(fn, "__name__", "") in _OFFLOADING_CHECKPOINT_SHIMS
+ )
+
+
+_GC_KWARGS_WARNED = set()
+
+
+def _gc_kwargs_note(key, message):
+ if key not in _GC_KWARGS_WARNED:
+ _GC_KWARGS_WARNED.add(key)
+ print(message)
+
+
+def patch_gradient_checkpointing_enable_kwargs():
+ """Pop every_n_layers / offload from gradient_checkpointing_kwargs; offload under Unsloth's offloading shim would double-offload."""
+ try:
+ from transformers import PreTrainedModel
+ import transformers.modeling_utils as hf_modeling_utils
+ except Exception:
+ return
+ original = PreTrainedModel.gradient_checkpointing_enable
+ if getattr(original, "_unsloth_gc_kwargs", False):
+ return
+ try:
+ params = list(inspect.signature(original).parameters)
+ except Exception:
+ return
+ positional = [p for p in params if p not in ("self", "gradient_checkpointing_kwargs")]
+ native_every = "every_n_layers" in params
+ native_offload = "offload" in params
+
+ @functools.wraps(original)
+ def gradient_checkpointing_enable(
+ self,
+ gradient_checkpointing_kwargs = None,
+ *args,
+ **kwargs,
+ ):
+ kwargs.update(zip(positional, args))
+ every = kwargs.pop("every_n_layers", None)
+ offload = kwargs.pop("offload", None)
+ if gradient_checkpointing_kwargs is not None:
+ gc_kwargs = dict(gradient_checkpointing_kwargs)
+ dict_every = gc_kwargs.pop("every_n_layers", None)
+ dict_offload = gc_kwargs.pop("offload", None)
+ every = dict_every if every is None else every
+ offload = dict_offload if offload is None else offload
+ if len(gc_kwargs) != len(gradient_checkpointing_kwargs):
+ gradient_checkpointing_kwargs = gc_kwargs or None
+ every = 1 if every is None else int(every)
+ if every < 1:
+ raise ValueError(f"Unsloth: every_n_layers must be >= 1, got {every}.")
+ if offload:
+ shim = getattr(hf_modeling_utils, "checkpoint", None)
+ if _checkpoint_offloads(shim):
+ _gc_kwargs_note(
+ "offload",
+ "Unsloth: gradient checkpointing already offloads activations to host memory, "
+ "so transformers' offload=True is ignored.",
+ )
+ offload = False
+ elif not native_offload:
+ _gc_kwargs_note(
+ "offload_native",
+ "Unsloth: gradient checkpointing offload=True needs transformers >= 5.17; ignored. "
+ 'use_gradient_checkpointing = "unsloth" offloads activations on any version.',
+ )
+ offload = False
+ if native_offload and offload:
+ kwargs["offload"] = True
+ if native_every:
+ kwargs["every_n_layers"] = 1
+ out = original(self, gradient_checkpointing_kwargs, **kwargs)
+ if every > 1 or any("_unsloth_every_n_skip" in m.__dict__ for m in self.modules()):
+ skipped = _apply_every_n_layers(self, every)
+ if every > 1 and not skipped:
+ _gc_kwargs_note(
+ "every_n_layers",
+ f"Unsloth: every_n_layers = {every} found no GradientCheckpointingLayer to skip; "
+ "every layer stays checkpointed.",
+ )
+ return out
+
+ gradient_checkpointing_enable._unsloth_gc_kwargs = True
+ PreTrainedModel.gradient_checkpointing_enable = gradient_checkpointing_enable
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 1d3d159c240..c2f383b0e5a 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -10733,26 +10733,7 @@ def _training_reserve_bytes(
return reserve, seq_len
-def _skip_aware_flag(cls):
- # for_training, gradient_checkpointing_enable and the trainer rewrite the flag; a marked layer keeps reading False.
- if cls.__dict__.get("_unsloth_skip_aware", False):
- return
-
- def _get(self):
- if self.__dict__.get("_unsloth_skip_checkpoint", False) or self.__dict__.get(
- "_unsloth_every_n_skip", False
- ):
- return False
- # Unmarked layers of the class still hold the flag in their own dict.
- return self.__dict__.get(
- "_unsloth_gc_flag", self.__dict__.get("gradient_checkpointing", False)
- )
-
- def _set(self, value):
- self.__dict__["_unsloth_gc_flag"] = value
-
- cls.gradient_checkpointing = property(_get, _set)
- cls._unsloth_skip_aware = True
+from .._gc_kwargs import _skip_aware_flag
def _gc_keep_layers_env():
@@ -11185,132 +11166,7 @@ def skip_checkpointing(model, layers = 0):
return chosen
-def _apply_every_n_layers(model, every_n_layers):
- """Checkpoint GradientCheckpointingLayer 0, N, 2N...; held by the skip-aware flag so for_training keeps it."""
- try:
- from transformers.modeling_layers import GradientCheckpointingLayer
- except Exception:
- return []
- layers = [
- m
- for m in model.modules()
- if isinstance(m, GradientCheckpointingLayer)
- or "_unsloth_gradient_checkpointing_holder" in m.__dict__
- ]
- skipped = []
- for i, layer in enumerate(layers):
- if every_n_layers > 1 and i % every_n_layers != 0:
- if not isinstance(layer, GradientCheckpointingLayer):
- layer.__dict__["_unsloth_every_n_skip"] = True
- skipped.append(i)
- continue
- if not hasattr(layer, "gradient_checkpointing"):
- continue
- current = layer.__dict__.pop("gradient_checkpointing", None)
- _skip_aware_flag(type(layer))
- if current is not None:
- layer.__dict__["_unsloth_gc_flag"] = current
- layer.__dict__["_unsloth_every_n_skip"] = True
- skipped.append(i)
- else:
- layer.__dict__.pop("_unsloth_every_n_skip", None)
- return skipped
-
-
-_OFFLOADING_CHECKPOINT_SHIMS = frozenset(
- {"unsloth_checkpoint", "unsloth_offloaded_gradient_checkpoint"}
-)
-
-
-def _checkpoint_offloads(fn):
- # Unsloth checkpoint functions that offload set `_unsloth_offloads`; the zoo shims are known by name.
- return bool(getattr(fn, "_unsloth_offloads", False)) or (
- getattr(fn, "__name__", "") in _OFFLOADING_CHECKPOINT_SHIMS
- )
-
-
-_GC_KWARGS_WARNED = set()
-
-
-def _gc_kwargs_note(key, message):
- if key not in _GC_KWARGS_WARNED:
- _GC_KWARGS_WARNED.add(key)
- print(message)
-
-
-def patch_gradient_checkpointing_enable_kwargs():
- """Pop every_n_layers / offload from gradient_checkpointing_kwargs; offload under Unsloth's offloading shim would double-offload."""
- try:
- from transformers import PreTrainedModel
- import transformers.modeling_utils as hf_modeling_utils
- except Exception:
- return
- original = PreTrainedModel.gradient_checkpointing_enable
- if getattr(original, "_unsloth_gc_kwargs", False):
- return
- try:
- params = list(inspect.signature(original).parameters)
- except Exception:
- return
- positional = [p for p in params if p not in ("self", "gradient_checkpointing_kwargs")]
- native_every = "every_n_layers" in params
- native_offload = "offload" in params
-
- @functools.wraps(original)
- def gradient_checkpointing_enable(
- self,
- gradient_checkpointing_kwargs = None,
- *args,
- **kwargs,
- ):
- kwargs.update(zip(positional, args))
- every = kwargs.pop("every_n_layers", None)
- offload = kwargs.pop("offload", None)
- if gradient_checkpointing_kwargs is not None:
- gc_kwargs = dict(gradient_checkpointing_kwargs)
- dict_every = gc_kwargs.pop("every_n_layers", None)
- dict_offload = gc_kwargs.pop("offload", None)
- every = dict_every if every is None else every
- offload = dict_offload if offload is None else offload
- if len(gc_kwargs) != len(gradient_checkpointing_kwargs):
- gradient_checkpointing_kwargs = gc_kwargs or None
- every = 1 if every is None else int(every)
- if every < 1:
- raise ValueError(f"Unsloth: every_n_layers must be >= 1, got {every}.")
- if offload:
- shim = getattr(hf_modeling_utils, "checkpoint", None)
- if _checkpoint_offloads(shim):
- _gc_kwargs_note(
- "offload",
- "Unsloth: gradient checkpointing already offloads activations to host memory, "
- "so transformers' offload=True is ignored.",
- )
- offload = False
- elif not native_offload:
- _gc_kwargs_note(
- "offload_native",
- "Unsloth: gradient checkpointing offload=True needs transformers >= 5.17; ignored. "
- 'use_gradient_checkpointing = "unsloth" offloads activations on any version.',
- )
- offload = False
- if native_offload and offload:
- kwargs["offload"] = True
- if native_every:
- kwargs["every_n_layers"] = 1
- out = original(self, gradient_checkpointing_kwargs, **kwargs)
- if every > 1 or any("_unsloth_every_n_skip" in m.__dict__ for m in self.modules()):
- skipped = _apply_every_n_layers(self, every)
- if every > 1 and not skipped:
- _gc_kwargs_note(
- "every_n_layers",
- f"Unsloth: every_n_layers = {every} found no GradientCheckpointingLayer to skip; "
- "every layer stays checkpointed.",
- )
- return out
-
- gradient_checkpointing_enable._unsloth_gc_kwargs = True
- PreTrainedModel.gradient_checkpointing_enable = gradient_checkpointing_enable
-
+from .._gc_kwargs import patch_gradient_checkpointing_enable_kwargs
patch_gradient_checkpointing_enable_kwargs()