Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Point a 4-bit load that does not fit the GPU at offload_layers = "auto" (#13052)
Daniel Han0bafe1df755f
diff --git a/tests/test_bnb_cpu_spill_message.py b/tests/test_bnb_cpu_spill_message.pynew file mode 100644index 00000000000..8b47f996bfd--- /dev/null+++ b/tests/test_bnb_cpu_spill_message.py@@ -0,0 +1,152 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""A 4-bit / 8-bit load that does not fit the GPU should say what to do in Unsloth (#1629).++transformers' bitsandbytes quantizers refuse a map that spills to the CPU and suggest+`llm_int8_enable_fp32_cpu_offload`, which is no route to training. Unsloth's+`offload_layers = "auto"` is, so the error points there instead.+"""++import inspect+import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))++from unsloth.models.loader_utils import ( # noqa: E402+ _BNB_CPU_SPILL_PREFIX,+ raise_if_bnb_cpu_spill,+)++[email protected]("module", ["quantizer_bnb_4bit", "quantizer_bnb_8bit"])+def test_prefix_matches_the_installed_transformers(module):+ quantizer = pytest.importorskip(f"transformers.quantizers.{module}")+ assert _BNB_CPU_SPILL_PREFIX in inspect.getsource(quantizer)+++def _spill_error():+ return ValueError(+ _BNB_CPU_SPILL_PREFIX + ". Make sure you have enough GPU RAM to fit the quantized model."+ )+++def test_spill_points_at_offload_layers():+ original = _spill_error()+ with pytest.raises(ValueError) as info:+ raise_if_bnb_cpu_spill(original, "unsloth/Mistral-Small-3.2-24B-Instruct-2506")+ message = str(info.value)+ assert message.startswith("Unsloth: unsloth/Mistral-Small-3.2-24B-Instruct-2506 does not fit")+ assert 'offload_layers = "auto"' in message+ assert "llm_int8_enable_fp32_cpu_offload" not in message+ assert info.value.__cause__ is original+++def test_spill_with_offload_auto_already_requested():+ with pytest.raises(ValueError) as info:+ raise_if_bnb_cpu_spill(_spill_error(), "m", "auto")+ assert "offload_layers" not in str(info.value)+ assert "max_seq_length" in str(info.value)+++def test_spill_with_a_layer_count_suggests_more_layers():+ with pytest.raises(ValueError) as info:+ raise_if_bnb_cpu_spill(_spill_error(), "m", 4)+ assert "Raise offload_layers" in str(info.value)+++def test_eight_bit_is_not_pointed_at_offload_layers():+ with pytest.raises(ValueError) as info:+ raise_if_bnb_cpu_spill(_spill_error(), "m", None, load_in_8bit = True)+ assert "offload_layers" not in str(info.value)+ assert "load_in_4bit = True" in str(info.value)+++def test_a_callers_quantization_config_gets_a_hint_that_works():+ from types import SimpleNamespace++ with pytest.raises(ValueError) as info:+ raise_if_bnb_cpu_spill(+ _spill_error(), "m", quantization_config = SimpleNamespace(load_in_8bit = True)+ )+ assert "offload_layers" not in str(info.value)+ with pytest.raises(ValueError) as info:+ raise_if_bnb_cpu_spill(+ _spill_error(), "m", quantization_config = SimpleNamespace(load_in_8bit = False)+ )+ assert "instead of a quantization_config" in str(info.value)+++def test_a_callers_own_dict_map_keeps_the_transformers_error():+ original = _spill_error()+ assert raise_if_bnb_cpu_spill(original, "m", device_map = {"model": 0, "lm_head": "cpu"}) is None++[email protected]("device_map", ["cpu", "cpu:0"])+def test_a_callers_cpu_map_keeps_the_transformers_error(device_map):+ import torch+ assert raise_if_bnb_cpu_spill(_spill_error(), "m", device_map = device_map) is None+ assert raise_if_bnb_cpu_spill(_spill_error(), "m", device_map = torch.device(device_map)) is None+++ "error",+ [ValueError("Unrecognized configuration class"), RuntimeError(_BNB_CPU_SPILL_PREFIX)],+)+def test_other_errors_are_left_alone(error):+ assert raise_if_bnb_cpu_spill(error, "m") is None+++ "path, cls",+ [("unsloth/models/llama.py", "FastLlamaModel"), ("unsloth/models/vision.py", "FastBaseModel")],+)+def test_both_loaders_route_load_errors_through_it(path, cls):+ import ast++ tree = ast.parse((ROOT / path).read_text(encoding = "utf-8"))+ method = next(+ node+ for c in tree.body+ if isinstance(c, ast.ClassDef) and c.name == cls+ for node in c.body+ if isinstance(node, ast.FunctionDef) and node.name == "from_pretrained"+ )+ handlers = [+ h+ for node in ast.walk(method)+ if isinstance(node, ast.Try)+ for h in node.handlers+ if isinstance(h.type, ast.Name) and h.type.id == "ValueError"+ ]+ routed = [+ h+ for h in handlers+ if any(+ isinstance(s, ast.Expr)+ and isinstance(s.value, ast.Call)+ and getattr(s.value.func, "id", None) == "raise_if_bnb_cpu_spill"+ and getattr(s.value.args[2], "id", None).endswith("_offload_layers_requested")+ for s in h.body+ )+ and isinstance(h.body[-1], ast.Raise)+ and h.body[-1].exc is None+ ]+ assert len(routed) == 1diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex e00d6ebfe97..aa95aa9d703 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -51,6 +51,7 @@ from .loader_utils import (planner_hub_kwargs,planner_kwargs_with_max_memory,planner_quantization_kwargs,+ raise_if_bnb_cpu_spill,requested_device_map,resolve_unsloth_device_map,resolve_auto_block_swap,@@ -2781,6 +2782,8 @@ class FastLlamaModel:# HF gets it again through **kwargs alongside our config= and fails with a duplicate kwarg.user_config = kwargs.pop("config", None)offload_layers = legacy_offload_layers(kwargs, kwargs.pop("offload_layers", None))+ _offload_layers_requested = offload_layers+ _quantization_config_requested = kwargs.get("quantization_config")offload_embedding = kwargs.pop("offload_embedding", False)if offload_embedding and fast_inference:if offload_embedding != OFFLOAD_EMBEDDING_AUTO:@@ -3411,6 +3414,17 @@ class FastLlamaModel:llm.shared_weights = Truemodel.fast_generate = model.vllm_engine.generatemodel.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)+ except ValueError as error:+ raise_if_bnb_cpu_spill(+ error,+ model_name,+ _offload_layers_requested,+ device_map = device_map,+ load_in_8bit = load_in_8bit,+ quantization_config = _quantization_config_requested,+ max_memory = kwargs.get("max_memory"),+ )+ raisefinally:raise_handler.remove()_undo_block_swap_keys()diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex e94e612847a..81f885dbefe 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -223,6 +223,63 @@ def planner_kwargs_with_max_memory(planner_kwargs, loader_kwargs):return merged+# transformers' bitsandbytes quantizers (4-bit and 8-bit) raise this when the automatic map spills past the GPU.+_BNB_CPU_SPILL_PREFIX = "Some modules are dispatched on the CPU or the disk"+++def raise_if_bnb_cpu_spill(+ error,+ model_name,+ offload_layers = None,+ device_map = None,+ load_in_8bit = False,+ quantization_config = None,+ max_memory = None,+):+ """Replace transformers' bitsandbytes CPU-spill error with what to do in Unsloth (#1629). Its advice, `llm_int8_enable_fp32_cpu_offload`, keeps the spilled weights unquantized in fp32 on the CPU, which is no route to training; `offload_layers = "auto"` is. Returns for any other error, and for a caller's own dict or CPU map (that placement is a choice, and transformers' advice is the relevant one), so the caller re-raises it unchanged. `offload_layers`, `quantization_config` and `max_memory` are what the caller passed, before the loader added its own."""+ if not isinstance(error, ValueError) or not str(error).startswith(_BNB_CPU_SPILL_PREFIX):+ return+ # On transformers 4.x an all-CPU map raises this too unless bitsandbytes' multi-backend is on.+ if isinstance(device_map, dict) or str(device_map).split(":")[0] in ("cpu", "disk"):+ return+ free = ""+ try:+ # The current card only: probing others would open a CUDA context on cards the caller may have withheld.+ # A caller's max_memory may withhold this card; then the figure would be about the wrong one.+ if DEVICE_TYPE_TORCH == "cuda" and torch.cuda.is_available() and not max_memory:+ device = torch.cuda.current_device()+ free = (+ f" (cuda:{device} has {torch.cuda.mem_get_info(device)[0] / 1024**3:.2f} GB free)"+ )+ except Exception:+ free = ""+ if load_in_8bit or getattr(quantization_config, "load_in_8bit", False):+ # offload_layers supports 16-bit and 4-bit loads only.+ hint = "Load in 4-bit (load_in_4bit = True) to halve the weights, or load a smaller model."+ elif quantization_config is not None:+ # offload_layers refuses a quantization_config.+ hint = (+ 'Pass load_in_4bit = True with `offload_layers = "auto"` instead of a '+ "quantization_config to stream the layers the GPU cannot hold from host RAM, "+ "or load a smaller model."+ )+ elif offload_layers == "auto":+ hint = "Lower max_seq_length or the batch size, load a smaller model, or add a GPU."+ elif offload_layers:+ hint = 'Raise offload_layers or pass `offload_layers = "auto"`, load a smaller model, or add a GPU.'+ else:+ hint = (+ 'Pass `offload_layers = "auto"` to from_pretrained to keep the decoder layers the GPU '+ "cannot hold in host RAM and stream them in during training (slower, and it needs that "+ "much free system RAM), "+ "load a smaller model, or free GPU memory held by other programs."+ )+ raise ValueError(+ f"Unsloth: {model_name} does not fit in GPU memory{free}, so transformers placed "+ f"part of it on the CPU, which bitsandbytes cannot quantize. {hint}"+ ) from error++def unmarked_device_map(device_map):"""The default with its marker removed; anything else exactly as it came in. For a nested load that must not re-read the value as "nobody chose this". A bare `str()` would also flatten a caller's `{"": 0}` into text transformers reads as a device name."""return str(device_map) if isinstance(device_map, _DefaultDeviceMap) else device_mapdiff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex a67076ee64a..7f921857e62 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -153,6 +153,7 @@ from .loader_utils import (planner_model_class,exclude_no_placement_params,planner_quantization_kwargs,+ raise_if_bnb_cpu_spill,requested_device_map,resolve_auto_block_swap,resolve_unsloth_device_map,@@ -2784,6 +2785,8 @@ class FastBaseModel:if auto_config is None and user_config is not None:auto_config = user_config_offload_layers = legacy_offload_layers(kwargs, kwargs.pop("offload_layers", None))+ _offload_layers_requested = _offload_layers+ _quantization_config_requested = kwargs.get("quantization_config")if _offload_layers and load_layers_to_host is None:_offload_layers = refuse_block_swap_load(_offload_layers,@@ -3749,6 +3752,17 @@ class FastBaseModel:model.fast_generate = model.vllm_engine.generatemodel.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)+ except ValueError as error:+ raise_if_bnb_cpu_spill(+ error,+ model_name,+ _offload_layers_requested,+ device_map = device_map,+ load_in_8bit = load_in_8bit,+ quantization_config = _quantization_config_requested,+ max_memory = kwargs.get("max_memory"),+ )+ raisefinally:raise_handler.remove()os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer