Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep a model's no-placement parameters on CPU (Qwen3.8-Flash-Next n-gram table) (#12141)

Daniel Han431ce2d45493
diff --git a/tests/test_no_placement_params_device_map.py b/tests/test_no_placement_params_device_map.py
new file mode 100644
index 00000000000..457ac47b5ab
--- /dev/null
+++ b/tests/test_no_placement_params_device_map.py
@@ -0,0 +1,265 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""`exclude_no_placement_params`: `_no_placement_params` stay off the device map; maps of other models are untouched."""
+
+import pytest
+import torch
+from torch import nn
+
+import unsloth # noqa: F401
+from real_accelerator import has_real_cuda
+from unsloth.models.loader_utils import exclude_no_placement_params
+
+
+class ScaledEmbedding(nn.Embedding):
+ def __init__(self, n, d):
+ super().__init__(n, d)
+ self.weight_scale = nn.Parameter(torch.ones(1))
+
+
+class Table(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.ngram_embedding = ScaledEmbedding(1000, 8)
+ self.register_buffer("offsets", torch.zeros(4, dtype = torch.long))
+
+
+class Ple(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.ple_embedding = Table()
+ self.key_proj = nn.Linear(8, 8)
+
+
+class Layer(nn.Module):
+ def __init__(self, with_ple):
+ super().__init__()
+ self.mlp = nn.Linear(8, 8)
+ self.ple = Ple() if with_ple else None
+ self.norm_weight = nn.Parameter(torch.ones(8))
+
+
+class Inner(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed_tokens = nn.Embedding(10, 8)
+ self.layers = nn.ModuleList([Layer(False), Layer(True), Layer(False)])
+
+
+class Model(nn.Module):
+ _no_placement_params = ["ple.ple_embedding.ngram_embedding.weight"]
+
+ def __init__(self, config = None):
+ super().__init__()
+ self.model = Inner()
+ self.lm_head = nn.Linear(8, 10)
+
+ @classmethod
+ def _from_config(cls, config):
+ return cls(config)
+
+
+class Plain(Model):
+ _no_placement_params = None
+
+
+def covered(device_map, name):
+ return any(k == "" or name == k or name.startswith(k + ".") for k in device_map)
+
+
+def check(device_map, device):
+ names = [n for n, _ in list(Model().named_parameters()) + list(Model().named_buffers())]
+ table = "model.layers.1.ple.ple_embedding.ngram_embedding."
+ assert not covered(device_map, table + "weight") and not covered(
+ device_map, table + "weight_scale"
+ )
+ for n in names:
+ if not n.startswith(table):
+ assert covered(device_map, n), n
+ assert set(device_map.values()) == {device}
+ # No entry is a prefix of another (accelerate needs a disjoint map).
+ keys = list(device_map)
+ assert not any(a != b and b.startswith(a + ".") for a in keys for b in keys)
+
+
+def test_dict_map_splits_only_the_owner():
+ out = exclude_no_placement_params({"": 0}, Model, None)
+ check(out, 0)
+ assert "model.layers.0" in out and "model.layers.2" in out and "lm_head" in out
+ assert "model.layers.1.ple.ple_embedding.offsets" in out
+
+
+def test_single_gpu_string_map(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
+ check(exclude_no_placement_params("sequential", Model, None), 0)
+ check(exclude_no_placement_params("cuda:0", Model, None), 0)
+
+
+def test_untouched_without_attribute_or_with_kill_switch(monkeypatch):
+ assert exclude_no_placement_params("sequential", Plain, None) == "sequential"
+ assert exclude_no_placement_params({"": 0}, None, None) == {"": 0}
+ monkeypatch.setenv("UNSLOTH_PLACE_NO_PLACEMENT_PARAMS", "1")
+ assert exclude_no_placement_params("sequential", Model, None) == "sequential"
+
+
+def test_multi_gpu_string_left_to_transformers(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ assert exclude_no_placement_params("sequential", Model, None) == "sequential"
+
+
+def test_real_qwen4_exp_on_meta(monkeypatch):
+ mod = pytest.importorskip("transformers.models.qwen4_exp.modeling_qwen4_exp")
+ from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig
+
+ cfg = Qwen4ExpTextConfig(
+ hidden_size = 64,
+ num_hidden_layers = 4,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 16,
+ indexer_n_heads = 2,
+ indexer_kv_heads = 1,
+ indexer_head_dim = 16,
+ indexer_budget = 16,
+ indexer_compress_ratio = 4,
+ linear_num_key_heads = 2,
+ linear_num_value_heads = 2,
+ linear_key_head_dim = 16,
+ linear_value_head_dim = 16,
+ num_experts = 4,
+ num_experts_per_tok = 2,
+ moe_intermediate_size = 16,
+ shared_expert_intermediate_size = 16,
+ vocab_size = 128,
+ hc_count = 2,
+ hc_lowrank = 8,
+ ple_layer_ids = [2],
+ ngram_vocab_size_base = 100,
+ eos_token_id = 1,
+ )
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
+ out = exclude_no_placement_params("sequential", mod.Qwen4ExpForCausalLM, cfg)
+ assert isinstance(out, dict)
+ assert not covered(out, "model.layers.1.ple.ple_embedding.ngram_embedding.weight")
+ assert covered(out, "model.layers.1.ple.key_proj.weight") and covered(
+ out, "model.layers.3.mlp.experts.down_proj"
+ )
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA device")
+def test_bnb_hooks_leave_the_cpu_table_alone():
+ """4bit hooks must not offload the CPU table (was copied to GPU every forward, 110 GB peak)."""
+ from unsloth.models.vision import _attach_bnb_multidevice_hooks
+
+ model = Model()
+ model.is_loaded_in_4bit = True
+ model.to("cuda:0")
+ table = model.model.layers[1].ple.ple_embedding.ngram_embedding
+ table.to("cpu")
+ _attach_bnb_multidevice_hooks(model, True, False, False, False)
+ assert not any(hasattr(m, "_hf_hook") for m in model.modules())
+ assert table.weight.device.type == "cpu"
+
+
+ not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs 2 CUDA devices"
+)
+def test_split_ancestors_get_input_hooks():
+ """Split model: the table's ancestors need input hooks, else ids meet buffers on another card."""
+ from unsloth.models.vision import _hook_no_placement_ancestors
+
+ class HashTable(Table):
+ def forward(self, ids):
+ mixed = ids * self.offsets[:1]
+ return self.ngram_embedding(mixed.to(self.ngram_embedding.weight.device)).to(ids.device)
+
+ model = Model()
+ model.model.layers[1].ple.ple_embedding = HashTable()
+ model.model.embed_tokens.to("cuda:0")
+ model.model.layers[0].to("cuda:0")
+ model.model.layers[1].to("cuda:1")
+ model.model.layers[2].to("cuda:1")
+ model.lm_head.to("cuda:1")
+ table = model.model.layers[1].ple.ple_embedding
+ table.ngram_embedding.to("cpu")
+ ids = torch.ones(1, 4, dtype = torch.long, device = "cuda:0")
+ with pytest.raises(RuntimeError):
+ table(ids)
+ assert _hook_no_placement_ancestors(model) == 3 # layer 1, its ple, the n-gram module
+ assert not hasattr(table.ngram_embedding, "_hf_hook")
+ out = table(ids)
+ assert out.device == torch.device("cuda:1")
+ assert _hook_no_placement_ancestors(model) == 0 # idempotent
+
+
+def test_unindexed_torch_device_uses_current_device(monkeypatch):
+ from unsloth.models.loader_utils import _single_device_index
+
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 1)
+ assert _single_device_index(torch.device("cuda")) == 1
+ assert _single_device_index(torch.device("cuda", 0)) == 0
+ assert _single_device_index(torch.device("cpu")) is None
+
+
+def test_inferred_map_never_covers_the_skipped_table():
+ """One device for the rest must not yield `{"": dev}` / a layer key that dispatch_model would push the table under."""
+ from unsloth.models.loader_utils import no_placement_tensor_names
+ from unsloth.models.vision import _infer_device_map_from_loaded_model
+
+ model = Model()
+ skip = no_placement_tensor_names(model)
+ inferred = _infer_device_map_from_loaded_model(model, skip = skip)
+ assert inferred
+ for key in inferred:
+ assert not any(key == "" or s == key or s.startswith(key + ".") for s in skip), key
+ assert any(k.startswith("model.layers.1.") for k in inferred)
+ assert "model.layers.0" in inferred and "lm_head" in inferred
+
+
+def test_bare_cuda_string_is_one_device_on_multi_gpu_hosts(monkeypatch):
+ from unsloth.models.loader_utils import _single_device_index
+
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 4)
+ monkeypatch.delenv("LOCAL_RANK", raising = False)
+ assert _single_device_index("cuda") == 0
+ monkeypatch.setenv("LOCAL_RANK", "2")
+ assert _single_device_index("cuda") == 2
+ assert _single_device_index("sequential") is None
+
+
+def test_split_keeps_more_specific_entries():
+ out = exclude_no_placement_params({"": 0, "lm_head": 1}, Model, None)
+ assert out["lm_head"] == 1
+ assert out["model.embed_tokens"] == 0
+ assert not covered(out, "model.layers.1.ple.ple_embedding.ngram_embedding.weight")
+
+
+def test_split_every_covering_ancestor():
+ out = exclude_no_placement_params({"": 0, "model.layers.1": 1}, Model, None)
+ assert not covered(out, "model.layers.1.ple.ple_embedding.ngram_embedding.weight")
+ assert out["model.layers.1.mlp"] == 1 and out["model.layers.1.ple.key_proj"] == 1
+ assert out["lm_head"] == 0
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA device")
+def test_disk_offloaded_ancestors_get_no_meta_hooks():
+ from unsloth.models.vision import _hook_no_placement_ancestors
+
+ model = Model().to("cuda:0")
+ layer = model.model.layers[1]
+ layer.mlp.to("meta")
+ layer.ple.key_proj.to("meta")
+ layer.norm_weight = nn.Parameter(torch.empty(8, device = "meta"))
+ layer.ple.ple_embedding.to("cpu")
+ layer.ple.ple_embedding.offsets = torch.zeros(4, dtype = torch.long, device = "meta")
+ _hook_no_placement_ancestors(model)
+ hooked = [m for m in model.modules() if hasattr(m, "_hf_hook")]
+ assert not any(m._hf_hook.execution_device == torch.device("meta") for m in hooked)
+
+
+def test_offloaded_covering_entry_is_left_whole():
+ out = exclude_no_placement_params({"": 0, "model.layers.1": "disk"}, Model, None)
+ assert out["model.layers.1"] == "disk" and out[""] == 0
+ assert not any(k.startswith("model.layers.1.") for k in out)
+ assert exclude_no_placement_params({"": "cpu"}, Model, None) == {"": "cpu"}
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 1606805b291..668f4fd69e4 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -284,6 +284,148 @@ def planner_quantization_kwargs(
return kwargs
+def _single_device_index(device_map):
+ """The one CUDA device a string/int device map resolves to on this host, else None."""
+ if isinstance(device_map, bool):
+ return None
+ if isinstance(device_map, int):
+ return device_map
+ if isinstance(device_map, torch.device):
+ if device_map.type != "cuda":
+ return None
+ # An unindexed torch.device means the current device (set_device(local_rank)), not 0.
+ return device_map.index if device_map.index is not None else torch.cuda.current_device()
+ if not isinstance(device_map, str):
+ return None
+ if device_map.startswith("cuda:"):
+ try:
+ return int(device_map.split(":", 1)[1])
+ except ValueError:
+ return None
+ if device_map == "cuda":
+ # transformers maps a bare "cuda" to cuda:{LOCAL_RANK}, one device on any host.
+ try:
+ return int(os.environ.get("LOCAL_RANK", 0))
+ except ValueError:
+ return None
+ if (
+ device_map in ("auto", "sequential", "balanced", "balanced_low_0")
+ or isinstance(device_map, _DefaultDeviceMap)
+ or device_map in AUTOMATIC_DEVICE_MAPS
+ ):
+ try:
+ if DEVICE_TYPE_TORCH == "cuda" and torch.cuda.device_count() == 1:
+ return 0
+ except Exception:
+ return None
+ return None
+
+
+def no_placement_tensor_names(model):
+ """Every tensor name under a module owning one of the model's `_no_placement_params`."""
+ names = getattr(model, "_no_placement_params", None)
+ if not names:
+ return set()
+ tensors = list(model.named_parameters(remove_duplicate = False)) + list(
+ model.named_buffers(remove_duplicate = False)
+ )
+ owners = {
+ name.rsplit(".", 1)[0]
+ for name, _ in tensors
+ if any(name == n or name.endswith("." + n) for n in names)
+ }
+ return {name for name, _ in tensors if any(name.startswith(o + ".") for o in owners)}
+
+
+def exclude_no_placement_params(device_map, model_class, config):
+ """Keep `_no_placement_params` off the device map (on CPU): transformers' handling sent all of
+ Qwen4Exp to CPU on one GPU. `UNSLOTH_PLACE_NO_PLACEMENT_PARAMS=1` restores it."""
+ names = getattr(model_class, "_no_placement_params", None) if model_class is not None else None
+ if not names or os.environ.get("UNSLOTH_PLACE_NO_PLACEMENT_PARAMS", "0") == "1":
+ return device_map
+ if isinstance(device_map, dict):
+ base = dict(device_map)
+ else:
+ index = _single_device_index(device_map)
+ if index is None:
+ print(
+ f"Unsloth: {model_class.__name__} keeps {', '.join(names)} off the device map, but "
+ f"device_map = {device_map!r} spans several devices; leaving the placement to transformers."
+ )
+ return device_map
+ base = {"": index}
+ try:
+ from accelerate import init_empty_weights
+ with init_empty_weights():
+ meta = model_class._from_config(config)
+ except Exception as error:
+ print(
+ f"Unsloth: could not build {model_class.__name__} on meta to place {names} ({error})."
+ )
+ return device_map
+ matched = {
+ name
+ for name, _ in list(meta.named_parameters()) + list(meta.named_buffers())
+ if any(name == n or name.endswith("." + n) for n in names)
+ }
+ if not matched:
+ return device_map
+ # Whole owning module: FP8Embedding multiplies by a sibling weight_scale on the same device.
+ excluded_modules = sorted({name.rsplit(".", 1)[0] for name in matched})
+ excluded = {
+ name
+ for name, _ in list(meta.named_parameters()) + list(meta.named_buffers())
+ if any(name.startswith(module + ".") for module in excluded_modules)
+ }
+
+ def owner(path):
+ best = None
+ for key in out:
+ if key == "" or path == key or path.startswith(key + "."):
+ if best is None or len(key) > len(best):
+ best = key
+ return best
+
+ out = dict(base)
+ for path in excluded_modules:
+ while (key := owner(path)) is not None:
+ device = out[key]
+ # Already off the GPU: keep the module key so accelerate's offload hooks still cover it.
+ if str(device).split(":")[0] in ("cpu", "disk", "meta"):
+ break
+ out.pop(key)
+ if key == path:
+ continue
+ module = meta.get_submodule(key) if key else meta
+ prefix = key
+ parts = path[len(key) + 1 :].split(".") if key else path.split(".")
+ for part in parts:
+ for child_name, _ in module.named_children():
+ if child_name != part:
+ out.setdefault(f"{prefix}.{child_name}" if prefix else child_name, device)
+ for tensor_name, _ in list(module.named_parameters(recurse = False)) + list(
+ module.named_buffers(recurse = False)
+ ):
+ if tensor_name != part:
+ out.setdefault(f"{prefix}.{tensor_name}" if prefix else tensor_name, device)
+ module = getattr(module, part)
+ prefix = f"{prefix}.{part}" if prefix else part
+ billions = (
+ sum(
+ t.numel()
+ for n, t in list(meta.named_parameters()) + list(meta.named_buffers())
+ if n in excluded
+ )
+ / 1e9
+ )
+ print(
+ f"Unsloth: keeping {', '.join(excluded_modules)} ({billions:.1f}B parameters, frozen) on CPU; "
+ f"set UNSLOTH_PLACE_NO_PLACEMENT_PARAMS=1 to place it on the GPU instead."
+ )
+ del meta
+ return out
+
+
def compressed_tensors_planner_bits(model_config, load_in_4bit, load_in_8bit):
"""(load_in_4bit, load_in_8bit) to size an armed compressed-tensors load at: the packed route keeps INT8 at 8 bits."""
if not load_in_4bit or compressed_tensors_prepared_config(model_config) is None:
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index fb2821065f3..bbc83eabbec 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -113,6 +113,7 @@ from .loader_utils import (
_prepare_compressed_tensors_model,
planner_class_mismatch_reason,
planner_model_class,
+ exclude_no_placement_params,
planner_quantization_kwargs,
requested_device_map,
resolve_unsloth_device_map,
@@ -204,19 +205,34 @@ __all__ = [
]
-def _infer_device_map_from_loaded_model(model):
- """Build a compact device_map by inspecting actual parameter placements."""
+def _infer_device_map_from_loaded_model(model, skip = ()):
+ """Build a compact device_map from actual parameter placements, leaving `skip` names off."""
device_map = {}
def _assign(module, prefix):
- params = list(module.named_parameters(remove_duplicate = False))
+ params = [
+ (n, p)
+ for n, p in module.named_parameters(remove_duplicate = False)
+ if (f"{prefix}.{n}" if prefix else n) not in skip
+ ]
+ if (
+ not params
+ and skip
+ and any(
+ (f"{prefix}.{n}" if prefix else n) in skip
+ for n, _ in module.named_parameters(remove_duplicate = False)
+ )
+ ):
+ return
if not params:
bufs = list(module.named_buffers())
if bufs:
device_map[prefix] = bufs[0][1].device
return
devices = {p.device for _, p in params}
- if len(devices) == 1:
+ # A key covering a skipped tensor would make dispatch_model move it too: recurse instead.
+ holds_skipped = bool(skip) and any(not prefix or s.startswith(prefix + ".") for s in skip)
+ if len(devices) == 1 and not holds_skipped:
device_map[prefix] = next(iter(devices))
else:
for child_name, child in module.named_children():
@@ -225,6 +241,8 @@ def _infer_device_map_from_loaded_model(model):
for pname, param in module.named_parameters(remove_duplicate = False):
if "." not in pname:
full = f"{prefix}.{pname}" if prefix else pname
+ if full in skip:
+ continue
if not any(full == k or full.startswith(k + ".") for k in device_map):
device_map[full] = param.device
@@ -415,6 +433,58 @@ def _align_root_hook_with_input_embeddings(model):
return target
+def _hook_no_placement_ancestors(model):
+ """Input-align hooks on single-card ancestors of a CPU-kept no-placement table (split model)."""
+ from .loader_utils import no_placement_tensor_names
+
+ unplaced = no_placement_tensor_names(model)
+ if not unplaced:
+ return 0
+ try:
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+ except ImportError:
+ return 0
+ placed_devices = {
+ p.device
+ for n, p in model.named_parameters()
+ if n not in unplaced and p.device.type not in ("cpu", "meta")
+ }
+ if len(placed_devices) < 2:
+ return 0
+ names = getattr(model, "_no_placement_params", None) or []
+ owners = {
+ n.rsplit(".", 1)[0] for n in unplaced if any(n == x or n.endswith("." + x) for x in names)
+ }
+ skip_keys = getattr(model, "_skip_keys_device_placement", None)
+ hooked = 0
+ for owner in owners:
+ parts = owner.split(".")
+ for depth in range(1, len(parts)):
+ path = ".".join(parts[:depth])
+ module = model.get_submodule(path)
+ if hasattr(module, "_hf_hook"):
+ continue
+ devices = {
+ t.device
+ for n, t in list(module.named_parameters(prefix = path))
+ + list(module.named_buffers(prefix = path))
+ if n not in unplaced
+ }
+ if len(devices) != 1:
+ continue
+ device = next(iter(devices))
+ if device.type in ("cpu", "meta"): # meta = disk-offloaded, has its own hook
+ continue
+ add_hook_to_module(
+ module,
+ AlignDevicesHook(
+ execution_device = device, io_same_device = False, skip_keys = skip_keys
+ ),
+ )
+ hooked += 1
+ return hooked
+
+
def _attach_bnb_multidevice_hooks(
model, load_in_4bit, load_in_8bit, offload_embedding, fast_inference
):
@@ -443,8 +513,14 @@ def _attach_bnb_multidevice_hooks(
if getattr(model, "hf_device_map", None) is not None:
return # already dispatched
+ # Unplaceable tables stay on CPU unhooked; hooking copies ~102 GB to the GPU every forward.
+ try:
+ from .loader_utils import no_placement_tensor_names
+ _unplaced = no_placement_tensor_names(model)
+ except Exception:
+ _unplaced = set()
try:
- all_devs = {p.device for p in model.parameters()}
+ all_devs = {p.device for n, p in model.named_parameters() if n not in _unplaced}
except Exception as exc:
warnings.warn(
"Unsloth: Failed to determine device placement from model parameters, "
@@ -468,7 +544,7 @@ def _attach_bnb_multidevice_hooks(
return # accelerate not available
try:
- inferred_map = _infer_device_map_from_loaded_model(model)
+ inferred_map = _infer_device_map_from_loaded_model(model, skip = _unplaced)
if not inferred_map:
return
@@ -494,13 +570,21 @@ def _attach_bnb_multidevice_hooks(
(d for d in device_map_int.values() if d not in ("cpu", "disk")),
None,
)
- dispatch_model(
- model,
- device_map = device_map_int,
- main_device = main_device,
- skip_keys = getattr(model, "_skip_keys_device_placement", None),
- force_hooks = True,
- )
+ _skip_check = contextlib.nullcontext()
+ if _unplaced:
+ try:
+ from transformers.integrations.accelerate import skip_device_map_check
+ _skip_check = skip_device_map_check()
+ except Exception:
+ pass
+ with _skip_check:
+ dispatch_model(
+ model,
+ device_map = device_map_int,
+ main_device = main_device,
+ skip_keys = getattr(model, "_skip_keys_device_placement", None),
+ force_hooks = True,
+ )
desc = f"{len(inferred_map)} block(s) across {len(cuda_devs)} device(s)"
finally:
for param, key, val in _stripped:
@@ -2550,6 +2634,8 @@ class FastBaseModel:
),
)
+ device_map = exclude_no_placement_params(device_map, model_class, auto_config)
+
if int(load_in_4bit) + int(load_in_8bit) + int(load_in_16bit) >= 2:
raise RuntimeError(
"Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!"
@@ -2793,6 +2879,12 @@ class FastBaseModel:
offload_embedding = offload_embedding,
fast_inference = fast_inference,
)
+ _no_placement_hooked = _hook_no_placement_ancestors(model)
+ if _no_placement_hooked:
+ logger.info(
+ f"Unsloth: hooked {_no_placement_hooked} module(s) above the CPU-kept "
+ "no-placement table so their inputs follow the map."
+ )
_aligned_root_device = _align_root_hook_with_input_embeddings(model)
if _aligned_root_device is not None:
logger.info(