Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep a model's no-placement parameters on CPU (Qwen3.8-Flash-Next n-gram table) (#12141)
Daniel Han431ce2d45493
diff --git a/tests/test_no_placement_params_device_map.py b/tests/test_no_placement_params_device_map.pynew file mode 100644index 00000000000..457ac47b5ab--- /dev/null+++ b/tests/test_no_placement_params_device_map.py@@ -0,0 +1,265 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""`exclude_no_placement_params`: `_no_placement_params` stay off the device map; maps of other models are untouched."""++import pytest+import torch+from torch import nn++import unsloth # noqa: F401+from real_accelerator import has_real_cuda+from unsloth.models.loader_utils import exclude_no_placement_params+++class ScaledEmbedding(nn.Embedding):+ def __init__(self, n, d):+ super().__init__(n, d)+ self.weight_scale = nn.Parameter(torch.ones(1))+++class Table(nn.Module):+ def __init__(self):+ super().__init__()+ self.ngram_embedding = ScaledEmbedding(1000, 8)+ self.register_buffer("offsets", torch.zeros(4, dtype = torch.long))+++class Ple(nn.Module):+ def __init__(self):+ super().__init__()+ self.ple_embedding = Table()+ self.key_proj = nn.Linear(8, 8)+++class Layer(nn.Module):+ def __init__(self, with_ple):+ super().__init__()+ self.mlp = nn.Linear(8, 8)+ self.ple = Ple() if with_ple else None+ self.norm_weight = nn.Parameter(torch.ones(8))+++class Inner(nn.Module):+ def __init__(self):+ super().__init__()+ self.embed_tokens = nn.Embedding(10, 8)+ self.layers = nn.ModuleList([Layer(False), Layer(True), Layer(False)])+++class Model(nn.Module):+ _no_placement_params = ["ple.ple_embedding.ngram_embedding.weight"]++ def __init__(self, config = None):+ super().__init__()+ self.model = Inner()+ self.lm_head = nn.Linear(8, 10)++ @classmethod+ def _from_config(cls, config):+ return cls(config)+++class Plain(Model):+ _no_placement_params = None+++def covered(device_map, name):+ return any(k == "" or name == k or name.startswith(k + ".") for k in device_map)+++def check(device_map, device):+ names = [n for n, _ in list(Model().named_parameters()) + list(Model().named_buffers())]+ table = "model.layers.1.ple.ple_embedding.ngram_embedding."+ assert not covered(device_map, table + "weight") and not covered(+ device_map, table + "weight_scale"+ )+ for n in names:+ if not n.startswith(table):+ assert covered(device_map, n), n+ assert set(device_map.values()) == {device}+ # No entry is a prefix of another (accelerate needs a disjoint map).+ keys = list(device_map)+ assert not any(a != b and b.startswith(a + ".") for a in keys for b in keys)+++def test_dict_map_splits_only_the_owner():+ out = exclude_no_placement_params({"": 0}, Model, None)+ check(out, 0)+ assert "model.layers.0" in out and "model.layers.2" in out and "lm_head" in out+ assert "model.layers.1.ple.ple_embedding.offsets" in out+++def test_single_gpu_string_map(monkeypatch):+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)+ check(exclude_no_placement_params("sequential", Model, None), 0)+ check(exclude_no_placement_params("cuda:0", Model, None), 0)+++def test_untouched_without_attribute_or_with_kill_switch(monkeypatch):+ assert exclude_no_placement_params("sequential", Plain, None) == "sequential"+ assert exclude_no_placement_params({"": 0}, None, None) == {"": 0}+ monkeypatch.setenv("UNSLOTH_PLACE_NO_PLACEMENT_PARAMS", "1")+ assert exclude_no_placement_params("sequential", Model, None) == "sequential"+++def test_multi_gpu_string_left_to_transformers(monkeypatch):+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ assert exclude_no_placement_params("sequential", Model, None) == "sequential"+++def test_real_qwen4_exp_on_meta(monkeypatch):+ mod = pytest.importorskip("transformers.models.qwen4_exp.modeling_qwen4_exp")+ from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig++ cfg = Qwen4ExpTextConfig(+ hidden_size = 64,+ num_hidden_layers = 4,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 16,+ indexer_n_heads = 2,+ indexer_kv_heads = 1,+ indexer_head_dim = 16,+ indexer_budget = 16,+ indexer_compress_ratio = 4,+ linear_num_key_heads = 2,+ linear_num_value_heads = 2,+ linear_key_head_dim = 16,+ linear_value_head_dim = 16,+ num_experts = 4,+ num_experts_per_tok = 2,+ moe_intermediate_size = 16,+ shared_expert_intermediate_size = 16,+ vocab_size = 128,+ hc_count = 2,+ hc_lowrank = 8,+ ple_layer_ids = [2],+ ngram_vocab_size_base = 100,+ eos_token_id = 1,+ )+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)+ out = exclude_no_placement_params("sequential", mod.Qwen4ExpForCausalLM, cfg)+ assert isinstance(out, dict)+ assert not covered(out, "model.layers.1.ple.ple_embedding.ngram_embedding.weight")+ assert covered(out, "model.layers.1.ple.key_proj.weight") and covered(+ out, "model.layers.3.mlp.experts.down_proj"+ )++[email protected](not has_real_cuda(), reason = "needs a CUDA device")+def test_bnb_hooks_leave_the_cpu_table_alone():+ """4bit hooks must not offload the CPU table (was copied to GPU every forward, 110 GB peak)."""+ from unsloth.models.vision import _attach_bnb_multidevice_hooks++ model = Model()+ model.is_loaded_in_4bit = True+ model.to("cuda:0")+ table = model.model.layers[1].ple.ple_embedding.ngram_embedding+ table.to("cpu")+ _attach_bnb_multidevice_hooks(model, True, False, False, False)+ assert not any(hasattr(m, "_hf_hook") for m in model.modules())+ assert table.weight.device.type == "cpu"+++ not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs 2 CUDA devices"+)+def test_split_ancestors_get_input_hooks():+ """Split model: the table's ancestors need input hooks, else ids meet buffers on another card."""+ from unsloth.models.vision import _hook_no_placement_ancestors++ class HashTable(Table):+ def forward(self, ids):+ mixed = ids * self.offsets[:1]+ return self.ngram_embedding(mixed.to(self.ngram_embedding.weight.device)).to(ids.device)++ model = Model()+ model.model.layers[1].ple.ple_embedding = HashTable()+ model.model.embed_tokens.to("cuda:0")+ model.model.layers[0].to("cuda:0")+ model.model.layers[1].to("cuda:1")+ model.model.layers[2].to("cuda:1")+ model.lm_head.to("cuda:1")+ table = model.model.layers[1].ple.ple_embedding+ table.ngram_embedding.to("cpu")+ ids = torch.ones(1, 4, dtype = torch.long, device = "cuda:0")+ with pytest.raises(RuntimeError):+ table(ids)+ assert _hook_no_placement_ancestors(model) == 3 # layer 1, its ple, the n-gram module+ assert not hasattr(table.ngram_embedding, "_hf_hook")+ out = table(ids)+ assert out.device == torch.device("cuda:1")+ assert _hook_no_placement_ancestors(model) == 0 # idempotent+++def test_unindexed_torch_device_uses_current_device(monkeypatch):+ from unsloth.models.loader_utils import _single_device_index++ monkeypatch.setattr(torch.cuda, "current_device", lambda: 1)+ assert _single_device_index(torch.device("cuda")) == 1+ assert _single_device_index(torch.device("cuda", 0)) == 0+ assert _single_device_index(torch.device("cpu")) is None+++def test_inferred_map_never_covers_the_skipped_table():+ """One device for the rest must not yield `{"": dev}` / a layer key that dispatch_model would push the table under."""+ from unsloth.models.loader_utils import no_placement_tensor_names+ from unsloth.models.vision import _infer_device_map_from_loaded_model++ model = Model()+ skip = no_placement_tensor_names(model)+ inferred = _infer_device_map_from_loaded_model(model, skip = skip)+ assert inferred+ for key in inferred:+ assert not any(key == "" or s == key or s.startswith(key + ".") for s in skip), key+ assert any(k.startswith("model.layers.1.") for k in inferred)+ assert "model.layers.0" in inferred and "lm_head" in inferred+++def test_bare_cuda_string_is_one_device_on_multi_gpu_hosts(monkeypatch):+ from unsloth.models.loader_utils import _single_device_index++ monkeypatch.setattr(torch.cuda, "device_count", lambda: 4)+ monkeypatch.delenv("LOCAL_RANK", raising = False)+ assert _single_device_index("cuda") == 0+ monkeypatch.setenv("LOCAL_RANK", "2")+ assert _single_device_index("cuda") == 2+ assert _single_device_index("sequential") is None+++def test_split_keeps_more_specific_entries():+ out = exclude_no_placement_params({"": 0, "lm_head": 1}, Model, None)+ assert out["lm_head"] == 1+ assert out["model.embed_tokens"] == 0+ assert not covered(out, "model.layers.1.ple.ple_embedding.ngram_embedding.weight")+++def test_split_every_covering_ancestor():+ out = exclude_no_placement_params({"": 0, "model.layers.1": 1}, Model, None)+ assert not covered(out, "model.layers.1.ple.ple_embedding.ngram_embedding.weight")+ assert out["model.layers.1.mlp"] == 1 and out["model.layers.1.ple.key_proj"] == 1+ assert out["lm_head"] == 0++[email protected](not has_real_cuda(), reason = "needs a CUDA device")+def test_disk_offloaded_ancestors_get_no_meta_hooks():+ from unsloth.models.vision import _hook_no_placement_ancestors++ model = Model().to("cuda:0")+ layer = model.model.layers[1]+ layer.mlp.to("meta")+ layer.ple.key_proj.to("meta")+ layer.norm_weight = nn.Parameter(torch.empty(8, device = "meta"))+ layer.ple.ple_embedding.to("cpu")+ layer.ple.ple_embedding.offsets = torch.zeros(4, dtype = torch.long, device = "meta")+ _hook_no_placement_ancestors(model)+ hooked = [m for m in model.modules() if hasattr(m, "_hf_hook")]+ assert not any(m._hf_hook.execution_device == torch.device("meta") for m in hooked)+++def test_offloaded_covering_entry_is_left_whole():+ out = exclude_no_placement_params({"": 0, "model.layers.1": "disk"}, Model, None)+ assert out["model.layers.1"] == "disk" and out[""] == 0+ assert not any(k.startswith("model.layers.1.") for k in out)+ assert exclude_no_placement_params({"": "cpu"}, Model, None) == {"": "cpu"}diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 1606805b291..668f4fd69e4 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -284,6 +284,148 @@ def planner_quantization_kwargs(return kwargs+def _single_device_index(device_map):+ """The one CUDA device a string/int device map resolves to on this host, else None."""+ if isinstance(device_map, bool):+ return None+ if isinstance(device_map, int):+ return device_map+ if isinstance(device_map, torch.device):+ if device_map.type != "cuda":+ return None+ # An unindexed torch.device means the current device (set_device(local_rank)), not 0.+ return device_map.index if device_map.index is not None else torch.cuda.current_device()+ if not isinstance(device_map, str):+ return None+ if device_map.startswith("cuda:"):+ try:+ return int(device_map.split(":", 1)[1])+ except ValueError:+ return None+ if device_map == "cuda":+ # transformers maps a bare "cuda" to cuda:{LOCAL_RANK}, one device on any host.+ try:+ return int(os.environ.get("LOCAL_RANK", 0))+ except ValueError:+ return None+ if (+ device_map in ("auto", "sequential", "balanced", "balanced_low_0")+ or isinstance(device_map, _DefaultDeviceMap)+ or device_map in AUTOMATIC_DEVICE_MAPS+ ):+ try:+ if DEVICE_TYPE_TORCH == "cuda" and torch.cuda.device_count() == 1:+ return 0+ except Exception:+ return None+ return None+++def no_placement_tensor_names(model):+ """Every tensor name under a module owning one of the model's `_no_placement_params`."""+ names = getattr(model, "_no_placement_params", None)+ if not names:+ return set()+ tensors = list(model.named_parameters(remove_duplicate = False)) + list(+ model.named_buffers(remove_duplicate = False)+ )+ owners = {+ name.rsplit(".", 1)[0]+ for name, _ in tensors+ if any(name == n or name.endswith("." + n) for n in names)+ }+ return {name for name, _ in tensors if any(name.startswith(o + ".") for o in owners)}+++def exclude_no_placement_params(device_map, model_class, config):+ """Keep `_no_placement_params` off the device map (on CPU): transformers' handling sent all of+ Qwen4Exp to CPU on one GPU. `UNSLOTH_PLACE_NO_PLACEMENT_PARAMS=1` restores it."""+ names = getattr(model_class, "_no_placement_params", None) if model_class is not None else None+ if not names or os.environ.get("UNSLOTH_PLACE_NO_PLACEMENT_PARAMS", "0") == "1":+ return device_map+ if isinstance(device_map, dict):+ base = dict(device_map)+ else:+ index = _single_device_index(device_map)+ if index is None:+ print(+ f"Unsloth: {model_class.__name__} keeps {', '.join(names)} off the device map, but "+ f"device_map = {device_map!r} spans several devices; leaving the placement to transformers."+ )+ return device_map+ base = {"": index}+ try:+ from accelerate import init_empty_weights+ with init_empty_weights():+ meta = model_class._from_config(config)+ except Exception as error:+ print(+ f"Unsloth: could not build {model_class.__name__} on meta to place {names} ({error})."+ )+ return device_map+ matched = {+ name+ for name, _ in list(meta.named_parameters()) + list(meta.named_buffers())+ if any(name == n or name.endswith("." + n) for n in names)+ }+ if not matched:+ return device_map+ # Whole owning module: FP8Embedding multiplies by a sibling weight_scale on the same device.+ excluded_modules = sorted({name.rsplit(".", 1)[0] for name in matched})+ excluded = {+ name+ for name, _ in list(meta.named_parameters()) + list(meta.named_buffers())+ if any(name.startswith(module + ".") for module in excluded_modules)+ }++ def owner(path):+ best = None+ for key in out:+ if key == "" or path == key or path.startswith(key + "."):+ if best is None or len(key) > len(best):+ best = key+ return best++ out = dict(base)+ for path in excluded_modules:+ while (key := owner(path)) is not None:+ device = out[key]+ # Already off the GPU: keep the module key so accelerate's offload hooks still cover it.+ if str(device).split(":")[0] in ("cpu", "disk", "meta"):+ break+ out.pop(key)+ if key == path:+ continue+ module = meta.get_submodule(key) if key else meta+ prefix = key+ parts = path[len(key) + 1 :].split(".") if key else path.split(".")+ for part in parts:+ for child_name, _ in module.named_children():+ if child_name != part:+ out.setdefault(f"{prefix}.{child_name}" if prefix else child_name, device)+ for tensor_name, _ in list(module.named_parameters(recurse = False)) + list(+ module.named_buffers(recurse = False)+ ):+ if tensor_name != part:+ out.setdefault(f"{prefix}.{tensor_name}" if prefix else tensor_name, device)+ module = getattr(module, part)+ prefix = f"{prefix}.{part}" if prefix else part+ billions = (+ sum(+ t.numel()+ for n, t in list(meta.named_parameters()) + list(meta.named_buffers())+ if n in excluded+ )+ / 1e9+ )+ print(+ f"Unsloth: keeping {', '.join(excluded_modules)} ({billions:.1f}B parameters, frozen) on CPU; "+ f"set UNSLOTH_PLACE_NO_PLACEMENT_PARAMS=1 to place it on the GPU instead."+ )+ del meta+ return out++def compressed_tensors_planner_bits(model_config, load_in_4bit, load_in_8bit):"""(load_in_4bit, load_in_8bit) to size an armed compressed-tensors load at: the packed route keeps INT8 at 8 bits."""if not load_in_4bit or compressed_tensors_prepared_config(model_config) is None:diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex fb2821065f3..bbc83eabbec 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -113,6 +113,7 @@ from .loader_utils import (_prepare_compressed_tensors_model,planner_class_mismatch_reason,planner_model_class,+ exclude_no_placement_params,planner_quantization_kwargs,requested_device_map,resolve_unsloth_device_map,@@ -204,19 +205,34 @@ __all__ = []-def _infer_device_map_from_loaded_model(model):- """Build a compact device_map by inspecting actual parameter placements."""+def _infer_device_map_from_loaded_model(model, skip = ()):+ """Build a compact device_map from actual parameter placements, leaving `skip` names off."""device_map = {}def _assign(module, prefix):- params = list(module.named_parameters(remove_duplicate = False))+ params = [+ (n, p)+ for n, p in module.named_parameters(remove_duplicate = False)+ if (f"{prefix}.{n}" if prefix else n) not in skip+ ]+ if (+ not params+ and skip+ and any(+ (f"{prefix}.{n}" if prefix else n) in skip+ for n, _ in module.named_parameters(remove_duplicate = False)+ )+ ):+ returnif not params:bufs = list(module.named_buffers())if bufs:device_map[prefix] = bufs[0][1].devicereturndevices = {p.device for _, p in params}- if len(devices) == 1:+ # A key covering a skipped tensor would make dispatch_model move it too: recurse instead.+ holds_skipped = bool(skip) and any(not prefix or s.startswith(prefix + ".") for s in skip)+ if len(devices) == 1 and not holds_skipped:device_map[prefix] = next(iter(devices))else:for child_name, child in module.named_children():@@ -225,6 +241,8 @@ def _infer_device_map_from_loaded_model(model):for pname, param in module.named_parameters(remove_duplicate = False):if "." not in pname:full = f"{prefix}.{pname}" if prefix else pname+ if full in skip:+ continueif not any(full == k or full.startswith(k + ".") for k in device_map):device_map[full] = param.device@@ -415,6 +433,58 @@ def _align_root_hook_with_input_embeddings(model):return target+def _hook_no_placement_ancestors(model):+ """Input-align hooks on single-card ancestors of a CPU-kept no-placement table (split model)."""+ from .loader_utils import no_placement_tensor_names++ unplaced = no_placement_tensor_names(model)+ if not unplaced:+ return 0+ try:+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module+ except ImportError:+ return 0+ placed_devices = {+ p.device+ for n, p in model.named_parameters()+ if n not in unplaced and p.device.type not in ("cpu", "meta")+ }+ if len(placed_devices) < 2:+ return 0+ names = getattr(model, "_no_placement_params", None) or []+ owners = {+ n.rsplit(".", 1)[0] for n in unplaced if any(n == x or n.endswith("." + x) for x in names)+ }+ skip_keys = getattr(model, "_skip_keys_device_placement", None)+ hooked = 0+ for owner in owners:+ parts = owner.split(".")+ for depth in range(1, len(parts)):+ path = ".".join(parts[:depth])+ module = model.get_submodule(path)+ if hasattr(module, "_hf_hook"):+ continue+ devices = {+ t.device+ for n, t in list(module.named_parameters(prefix = path))+ + list(module.named_buffers(prefix = path))+ if n not in unplaced+ }+ if len(devices) != 1:+ continue+ device = next(iter(devices))+ if device.type in ("cpu", "meta"): # meta = disk-offloaded, has its own hook+ continue+ add_hook_to_module(+ module,+ AlignDevicesHook(+ execution_device = device, io_same_device = False, skip_keys = skip_keys+ ),+ )+ hooked += 1+ return hooked++def _attach_bnb_multidevice_hooks(model, load_in_4bit, load_in_8bit, offload_embedding, fast_inference):@@ -443,8 +513,14 @@ def _attach_bnb_multidevice_hooks(if getattr(model, "hf_device_map", None) is not None:return # already dispatched+ # Unplaceable tables stay on CPU unhooked; hooking copies ~102 GB to the GPU every forward.+ try:+ from .loader_utils import no_placement_tensor_names+ _unplaced = no_placement_tensor_names(model)+ except Exception:+ _unplaced = set()try:- all_devs = {p.device for p in model.parameters()}+ all_devs = {p.device for n, p in model.named_parameters() if n not in _unplaced}except Exception as exc:warnings.warn("Unsloth: Failed to determine device placement from model parameters, "@@ -468,7 +544,7 @@ def _attach_bnb_multidevice_hooks(return # accelerate not availabletry:- inferred_map = _infer_device_map_from_loaded_model(model)+ inferred_map = _infer_device_map_from_loaded_model(model, skip = _unplaced)if not inferred_map:return@@ -494,13 +570,21 @@ def _attach_bnb_multidevice_hooks((d for d in device_map_int.values() if d not in ("cpu", "disk")),None,)- dispatch_model(- model,- device_map = device_map_int,- main_device = main_device,- skip_keys = getattr(model, "_skip_keys_device_placement", None),- force_hooks = True,- )+ _skip_check = contextlib.nullcontext()+ if _unplaced:+ try:+ from transformers.integrations.accelerate import skip_device_map_check+ _skip_check = skip_device_map_check()+ except Exception:+ pass+ with _skip_check:+ dispatch_model(+ model,+ device_map = device_map_int,+ main_device = main_device,+ skip_keys = getattr(model, "_skip_keys_device_placement", None),+ force_hooks = True,+ )desc = f"{len(inferred_map)} block(s) across {len(cuda_devs)} device(s)"finally:for param, key, val in _stripped:@@ -2550,6 +2634,8 @@ class FastBaseModel:),)+ device_map = exclude_no_placement_params(device_map, model_class, auto_config)+if int(load_in_4bit) + int(load_in_8bit) + int(load_in_16bit) >= 2:raise RuntimeError("Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!"@@ -2793,6 +2879,12 @@ class FastBaseModel:offload_embedding = offload_embedding,fast_inference = fast_inference,)+ _no_placement_hooked = _hook_no_placement_ancestors(model)+ if _no_placement_hooked:+ logger.info(+ f"Unsloth: hooked {_no_placement_hooked} module(s) above the CPU-kept "+ "no-placement table so their inputs follow the map."+ )_aligned_root_device = _align_root_hook_with_input_embeddings(model)if _aligned_root_device is not None:logger.info(