Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Point a 4-bit load that does not fit the GPU at offload_layers = "auto" (#13052)

Daniel Han0bafe1df755f
diff --git a/tests/test_bnb_cpu_spill_message.py b/tests/test_bnb_cpu_spill_message.py
new file mode 100644
index 00000000000..8b47f996bfd
--- /dev/null
+++ b/tests/test_bnb_cpu_spill_message.py
@@ -0,0 +1,152 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""A 4-bit / 8-bit load that does not fit the GPU should say what to do in Unsloth (#1629).
+
+transformers' bitsandbytes quantizers refuse a map that spills to the CPU and suggest
+`llm_int8_enable_fp32_cpu_offload`, which is no route to training. Unsloth's
+`offload_layers = "auto"` is, so the error points there instead.
+"""
+
+import inspect
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from unsloth.models.loader_utils import ( # noqa: E402
+ _BNB_CPU_SPILL_PREFIX,
+ raise_if_bnb_cpu_spill,
+)
+
+
[email protected]("module", ["quantizer_bnb_4bit", "quantizer_bnb_8bit"])
+def test_prefix_matches_the_installed_transformers(module):
+ quantizer = pytest.importorskip(f"transformers.quantizers.{module}")
+ assert _BNB_CPU_SPILL_PREFIX in inspect.getsource(quantizer)
+
+
+def _spill_error():
+ return ValueError(
+ _BNB_CPU_SPILL_PREFIX + ". Make sure you have enough GPU RAM to fit the quantized model."
+ )
+
+
+def test_spill_points_at_offload_layers():
+ original = _spill_error()
+ with pytest.raises(ValueError) as info:
+ raise_if_bnb_cpu_spill(original, "unsloth/Mistral-Small-3.2-24B-Instruct-2506")
+ message = str(info.value)
+ assert message.startswith("Unsloth: unsloth/Mistral-Small-3.2-24B-Instruct-2506 does not fit")
+ assert 'offload_layers = "auto"' in message
+ assert "llm_int8_enable_fp32_cpu_offload" not in message
+ assert info.value.__cause__ is original
+
+
+def test_spill_with_offload_auto_already_requested():
+ with pytest.raises(ValueError) as info:
+ raise_if_bnb_cpu_spill(_spill_error(), "m", "auto")
+ assert "offload_layers" not in str(info.value)
+ assert "max_seq_length" in str(info.value)
+
+
+def test_spill_with_a_layer_count_suggests_more_layers():
+ with pytest.raises(ValueError) as info:
+ raise_if_bnb_cpu_spill(_spill_error(), "m", 4)
+ assert "Raise offload_layers" in str(info.value)
+
+
+def test_eight_bit_is_not_pointed_at_offload_layers():
+ with pytest.raises(ValueError) as info:
+ raise_if_bnb_cpu_spill(_spill_error(), "m", None, load_in_8bit = True)
+ assert "offload_layers" not in str(info.value)
+ assert "load_in_4bit = True" in str(info.value)
+
+
+def test_a_callers_quantization_config_gets_a_hint_that_works():
+ from types import SimpleNamespace
+
+ with pytest.raises(ValueError) as info:
+ raise_if_bnb_cpu_spill(
+ _spill_error(), "m", quantization_config = SimpleNamespace(load_in_8bit = True)
+ )
+ assert "offload_layers" not in str(info.value)
+ with pytest.raises(ValueError) as info:
+ raise_if_bnb_cpu_spill(
+ _spill_error(), "m", quantization_config = SimpleNamespace(load_in_8bit = False)
+ )
+ assert "instead of a quantization_config" in str(info.value)
+
+
+def test_a_callers_own_dict_map_keeps_the_transformers_error():
+ original = _spill_error()
+ assert raise_if_bnb_cpu_spill(original, "m", device_map = {"model": 0, "lm_head": "cpu"}) is None
+
+
[email protected]("device_map", ["cpu", "cpu:0"])
+def test_a_callers_cpu_map_keeps_the_transformers_error(device_map):
+ import torch
+ assert raise_if_bnb_cpu_spill(_spill_error(), "m", device_map = device_map) is None
+ assert raise_if_bnb_cpu_spill(_spill_error(), "m", device_map = torch.device(device_map)) is None
+
+
+ "error",
+ [ValueError("Unrecognized configuration class"), RuntimeError(_BNB_CPU_SPILL_PREFIX)],
+)
+def test_other_errors_are_left_alone(error):
+ assert raise_if_bnb_cpu_spill(error, "m") is None
+
+
+ "path, cls",
+ [("unsloth/models/llama.py", "FastLlamaModel"), ("unsloth/models/vision.py", "FastBaseModel")],
+)
+def test_both_loaders_route_load_errors_through_it(path, cls):
+ import ast
+
+ tree = ast.parse((ROOT / path).read_text(encoding = "utf-8"))
+ method = next(
+ node
+ for c in tree.body
+ if isinstance(c, ast.ClassDef) and c.name == cls
+ for node in c.body
+ if isinstance(node, ast.FunctionDef) and node.name == "from_pretrained"
+ )
+ handlers = [
+ h
+ for node in ast.walk(method)
+ if isinstance(node, ast.Try)
+ for h in node.handlers
+ if isinstance(h.type, ast.Name) and h.type.id == "ValueError"
+ ]
+ routed = [
+ h
+ for h in handlers
+ if any(
+ isinstance(s, ast.Expr)
+ and isinstance(s.value, ast.Call)
+ and getattr(s.value.func, "id", None) == "raise_if_bnb_cpu_spill"
+ and getattr(s.value.args[2], "id", None).endswith("_offload_layers_requested")
+ for s in h.body
+ )
+ and isinstance(h.body[-1], ast.Raise)
+ and h.body[-1].exc is None
+ ]
+ assert len(routed) == 1
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index e00d6ebfe97..aa95aa9d703 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -51,6 +51,7 @@ from .loader_utils import (
planner_hub_kwargs,
planner_kwargs_with_max_memory,
planner_quantization_kwargs,
+ raise_if_bnb_cpu_spill,
requested_device_map,
resolve_unsloth_device_map,
resolve_auto_block_swap,
@@ -2781,6 +2782,8 @@ class FastLlamaModel:
# HF gets it again through **kwargs alongside our config= and fails with a duplicate kwarg.
user_config = kwargs.pop("config", None)
offload_layers = legacy_offload_layers(kwargs, kwargs.pop("offload_layers", None))
+ _offload_layers_requested = offload_layers
+ _quantization_config_requested = kwargs.get("quantization_config")
offload_embedding = kwargs.pop("offload_embedding", False)
if offload_embedding and fast_inference:
if offload_embedding != OFFLOAD_EMBEDDING_AUTO:
@@ -3411,6 +3414,17 @@ class FastLlamaModel:
llm.shared_weights = True
model.fast_generate = model.vllm_engine.generate
model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)
+ except ValueError as error:
+ raise_if_bnb_cpu_spill(
+ error,
+ model_name,
+ _offload_layers_requested,
+ device_map = device_map,
+ load_in_8bit = load_in_8bit,
+ quantization_config = _quantization_config_requested,
+ max_memory = kwargs.get("max_memory"),
+ )
+ raise
finally:
raise_handler.remove()
_undo_block_swap_keys()
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index e94e612847a..81f885dbefe 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -223,6 +223,63 @@ def planner_kwargs_with_max_memory(planner_kwargs, loader_kwargs):
return merged
+# transformers' bitsandbytes quantizers (4-bit and 8-bit) raise this when the automatic map spills past the GPU.
+_BNB_CPU_SPILL_PREFIX = "Some modules are dispatched on the CPU or the disk"
+
+
+def raise_if_bnb_cpu_spill(
+ error,
+ model_name,
+ offload_layers = None,
+ device_map = None,
+ load_in_8bit = False,
+ quantization_config = None,
+ max_memory = None,
+):
+ """Replace transformers' bitsandbytes CPU-spill error with what to do in Unsloth (#1629). Its advice, `llm_int8_enable_fp32_cpu_offload`, keeps the spilled weights unquantized in fp32 on the CPU, which is no route to training; `offload_layers = "auto"` is. Returns for any other error, and for a caller's own dict or CPU map (that placement is a choice, and transformers' advice is the relevant one), so the caller re-raises it unchanged. `offload_layers`, `quantization_config` and `max_memory` are what the caller passed, before the loader added its own."""
+ if not isinstance(error, ValueError) or not str(error).startswith(_BNB_CPU_SPILL_PREFIX):
+ return
+ # On transformers 4.x an all-CPU map raises this too unless bitsandbytes' multi-backend is on.
+ if isinstance(device_map, dict) or str(device_map).split(":")[0] in ("cpu", "disk"):
+ return
+ free = ""
+ try:
+ # The current card only: probing others would open a CUDA context on cards the caller may have withheld.
+ # A caller's max_memory may withhold this card; then the figure would be about the wrong one.
+ if DEVICE_TYPE_TORCH == "cuda" and torch.cuda.is_available() and not max_memory:
+ device = torch.cuda.current_device()
+ free = (
+ f" (cuda:{device} has {torch.cuda.mem_get_info(device)[0] / 1024**3:.2f} GB free)"
+ )
+ except Exception:
+ free = ""
+ if load_in_8bit or getattr(quantization_config, "load_in_8bit", False):
+ # offload_layers supports 16-bit and 4-bit loads only.
+ hint = "Load in 4-bit (load_in_4bit = True) to halve the weights, or load a smaller model."
+ elif quantization_config is not None:
+ # offload_layers refuses a quantization_config.
+ hint = (
+ 'Pass load_in_4bit = True with `offload_layers = "auto"` instead of a '
+ "quantization_config to stream the layers the GPU cannot hold from host RAM, "
+ "or load a smaller model."
+ )
+ elif offload_layers == "auto":
+ hint = "Lower max_seq_length or the batch size, load a smaller model, or add a GPU."
+ elif offload_layers:
+ hint = 'Raise offload_layers or pass `offload_layers = "auto"`, load a smaller model, or add a GPU.'
+ else:
+ hint = (
+ 'Pass `offload_layers = "auto"` to from_pretrained to keep the decoder layers the GPU '
+ "cannot hold in host RAM and stream them in during training (slower, and it needs that "
+ "much free system RAM), "
+ "load a smaller model, or free GPU memory held by other programs."
+ )
+ raise ValueError(
+ f"Unsloth: {model_name} does not fit in GPU memory{free}, so transformers placed "
+ f"part of it on the CPU, which bitsandbytes cannot quantize. {hint}"
+ ) from error
+
+
def unmarked_device_map(device_map):
"""The default with its marker removed; anything else exactly as it came in. For a nested load that must not re-read the value as "nobody chose this". A bare `str()` would also flatten a caller's `{"": 0}` into text transformers reads as a device name."""
return str(device_map) if isinstance(device_map, _DefaultDeviceMap) else device_map
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index a67076ee64a..7f921857e62 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -153,6 +153,7 @@ from .loader_utils import (
planner_model_class,
exclude_no_placement_params,
planner_quantization_kwargs,
+ raise_if_bnb_cpu_spill,
requested_device_map,
resolve_auto_block_swap,
resolve_unsloth_device_map,
@@ -2784,6 +2785,8 @@ class FastBaseModel:
if auto_config is None and user_config is not None:
auto_config = user_config
_offload_layers = legacy_offload_layers(kwargs, kwargs.pop("offload_layers", None))
+ _offload_layers_requested = _offload_layers
+ _quantization_config_requested = kwargs.get("quantization_config")
if _offload_layers and load_layers_to_host is None:
_offload_layers = refuse_block_swap_load(
_offload_layers,
@@ -3749,6 +3752,17 @@ class FastBaseModel:
model.fast_generate = model.vllm_engine.generate
model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)
+ except ValueError as error:
+ raise_if_bnb_cpu_spill(
+ error,
+ model_name,
+ _offload_layers_requested,
+ device_map = device_map,
+ load_in_8bit = load_in_8bit,
+ quantization_config = _quantization_config_requested,
+ max_memory = kwargs.get("max_memory"),
+ )
+ raise
finally:
raise_handler.remove()
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer