Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix MoE adapter setup and reload with offloaded layers (#15159)

Leo Borcherding810c932ae2ee
diff --git a/tests/test_block_swap_shaped.py b/tests/test_block_swap_shaped.py
new file mode 100644
index 00000000000..860bda5be93
--- /dev/null
+++ b/tests/test_block_swap_shaped.py
@@ -0,0 +1,269 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import ast, contextlib, os, types
+import torch
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py")
+NAMES = ("block_swap_shaped", "_block_swap_shaped_fallback")
+
+
+def _load():
+ src = open(UTILS, encoding = "utf-8").read()
+ nodes = {
+ n.name: n for n in ast.parse(src).body if isinstance(n, ast.FunctionDef) and n.name in NAMES
+ }
+ assert set(nodes) == set(NAMES)
+ ns = {"contextlib": contextlib}
+ for name in NAMES:
+ exec(ast.get_source_segment(src, nodes[name]), ns)
+ ns["_block_swap_shaped_fallback"] = contextlib.contextmanager(ns["_block_swap_shaped_fallback"])
+ return ns
+
+
+def _evicted_swapper(shape = (4, 16, 8)):
+ w = torch.nn.Parameter(torch.randn(shape), requires_grad = False)
+ empty = torch.empty(0)
+ w.data = empty
+ block = types.SimpleNamespace(
+ resident = False, params = [w], empties = [empty], sig = ((shape, w.dtype, "cpu", 0),)
+ )
+ return types.SimpleNamespace(blocks = [block]), w
+
+
+def test_older_zoo_still_gets_the_real_shape_while_peft_wraps():
+ ns = _load()
+ swapper, w = _evicted_swapper()
+ with ns["block_swap_shaped"](types.SimpleNamespace(_unsloth_block_swap = swapper)):
+ assert w.shape == (4, 16, 8)
+ assert w.untyped_storage().nbytes() == w.element_size()
+ assert w.numel() == 0
+
+
+def test_zoo_shaped_is_used_when_present():
+ ns = _load()
+ entered = []
+
+ @contextlib.contextmanager
+ def shaped():
+ entered.append(True)
+ yield
+
+ model = types.SimpleNamespace(_unsloth_block_swap = types.SimpleNamespace(shaped = shaped))
+ with ns["block_swap_shaped"](model):
+ pass
+ assert entered == [True]
+
+
+def test_no_swap_and_unknown_swappers_are_no_ops():
+ ns = _load()
+ with ns["block_swap_shaped"](types.SimpleNamespace()):
+ pass
+ with ns["block_swap_shaped"](types.SimpleNamespace(_unsloth_block_swap = object())):
+ pass
+
+
+def test_data_dependent_inits_never_see_the_placeholder():
+ ns = _load()
+ for init in ("pissa", "pissa_niter_4", "olora", "loftq", "corda", "orthogonal"):
+ swapper, w = _evicted_swapper()
+ with ns["block_swap_shaped"](types.SimpleNamespace(_unsloth_block_swap = swapper), init):
+ assert w.numel() == 0, init
+ for init in ("gaussian", "eva"):
+ swapper, w = _evicted_swapper()
+ with ns["block_swap_shaped"](types.SimpleNamespace(_unsloth_block_swap = swapper), init):
+ assert w.shape == (4, 16, 8), init
+
+
+def test_fallback_leaves_a_block_fetched_inside_the_context():
+ w = torch.nn.Parameter(torch.empty(0), requires_grad = False)
+ blk = types.SimpleNamespace(resident = False, params = [w], empties = [w.data], sig = [((4, 8),)])
+ with _load()["_block_swap_shaped_fallback"](types.SimpleNamespace(blocks = [blk])):
+ w.data, blk.resident = torch.ones(4, 8), True
+ assert w.shape == (4, 8)
+
+
+def test_dora_rejects_evicted_weights_before_initializing_magnitudes():
+ import inspect
+ import pytest
+ from peft import LoraConfig, get_peft_model
+
+ model = torch.nn.Sequential(torch.nn.Linear(8, 4, bias = False))
+ model.requires_grad_(False)
+ swapper, weight = _evicted_swapper((4, 8))
+ model[0].weight = weight
+ model._unsloth_block_swap = swapper
+ config = LoraConfig(r = 2, target_modules = ["0"], use_dora = True)
+ shaped = _load()["block_swap_shaped"]
+ options = (
+ {"use_dora": config.use_dora} if "use_dora" in inspect.signature(shaped).parameters else {}
+ )
+ with pytest.raises(ValueError, match = "DoRA"):
+ with shaped(model, **options):
+ get_peft_model(model, config)
+ assert weight.numel() == 0
+
+
+def test_dora_without_block_swap_keeps_real_magnitudes():
+ from peft import LoraConfig, get_peft_model
+ for resident_swap in (False, True):
+ model = torch.nn.Sequential(torch.nn.Linear(8, 4, bias = False))
+ expected = model[0].weight.detach().norm(dim = 1)
+ if resident_swap:
+ model._unsloth_block_swap = types.SimpleNamespace(
+ blocks = [types.SimpleNamespace(resident = True)]
+ )
+ config = LoraConfig(r = 2, target_modules = ["0"], use_dora = True)
+ with _load()["block_swap_shaped"](model, use_dora = config.use_dora):
+ model = get_peft_model(model, config)
+ magnitude = model.base_model.model[0].lora_magnitude_vector["default"].weight
+ torch.testing.assert_close(magnitude, expected)
+
+
+def test_modules_to_save_rejects_copying_evicted_weights():
+ import inspect
+ import pytest
+ from peft import LoraConfig, get_peft_model
+
+ model = torch.nn.Sequential(
+ torch.nn.Linear(8, 4, bias = False), torch.nn.Linear(4, 4, bias = False)
+ )
+ model.requires_grad_(False)
+ swapper, weight = _evicted_swapper((4, 8))
+ model[0].weight = weight
+ model._unsloth_block_swap = swapper
+ config = LoraConfig(r = 2, target_modules = ["1"], modules_to_save = ["0"])
+ shaped = _load()["block_swap_shaped"]
+ options = (
+ {"modules_to_save": config.modules_to_save}
+ if "modules_to_save" in inspect.signature(shaped).parameters
+ else {}
+ )
+ with pytest.raises(ValueError, match = "modules_to_save"):
+ with shaped(model, **options):
+ get_peft_model(model, config)
+ assert weight.numel() == 0
+
+
+def test_modules_to_save_preserves_resident_modules_beside_evicted_weights():
+ from peft import LoraConfig, get_peft_model
+
+ model = torch.nn.Sequential(
+ torch.nn.Linear(8, 4, bias = False), torch.nn.Linear(4, 4, bias = False)
+ )
+ expected = model[1].weight.detach().clone()
+ swapper, weight = _evicted_swapper((4, 8))
+ model[0].weight = weight
+ model._unsloth_block_swap = swapper
+ config = LoraConfig(r = 2, target_modules = ["0"], modules_to_save = ["1"])
+ with _load()["block_swap_shaped"](model, modules_to_save = config.modules_to_save):
+ model = get_peft_model(model, config)
+ saved = model.base_model.model[1].modules_to_save["default"].weight
+ torch.testing.assert_close(saved, expected)
+
+
+def test_dora_targets_can_stay_resident_beside_evicted_blocks():
+ import inspect
+ from peft import LoraConfig, get_peft_model
+
+ model = torch.nn.Sequential(
+ torch.nn.Linear(8, 4, bias = False), torch.nn.Linear(4, 4, bias = False)
+ )
+ expected = model[1].weight.detach().norm(dim = 1)
+ swapper, weight = _evicted_swapper((4, 8))
+ model[0].weight = weight
+ model._unsloth_block_swap = swapper
+ config = LoraConfig(r = 2, target_modules = ["1"], use_dora = True)
+ shaped = _load()["block_swap_shaped"]
+ options = (
+ {"lora_config": config} if "lora_config" in inspect.signature(shaped).parameters else {}
+ )
+ with shaped(model, use_dora = True, **options):
+ model = get_peft_model(model, config)
+ torch.testing.assert_close(
+ model.base_model.model[1].lora_magnitude_vector["default"].weight, expected
+ )
+
+
+def test_kasa_never_truncates_placeholder_weights():
+ import inspect
+ import pytest
+ import peft
+
+ if not hasattr(peft, "KasaConfig"):
+ pytest.skip("Installed PEFT has no KaSA")
+ model = torch.nn.Sequential(torch.nn.Linear(8, 6, bias = False))
+ model.requires_grad_(False)
+ swapper, weight = _evicted_swapper((6, 8))
+ model[0].weight = weight
+ model._unsloth_block_swap = swapper
+ config = peft.LoraConfig(r = 2, target_modules = ["0"], kasa_config = peft.KasaConfig())
+ shaped = _load()["block_swap_shaped"]
+ options = (
+ {"lora_config": config} if "lora_config" in inspect.signature(shaped).parameters else {}
+ )
+ with pytest.raises(ValueError, match = "KaSA"):
+ with shaped(model, **options):
+ peft.get_peft_model(model, config)
+ assert weight.numel() == 0
+
+
+def test_both_adapter_reload_paths_wrap_evicted_experts(tmp_path):
+ from peft import LoraConfig, PeftModel, get_peft_model
+
+ class Experts(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = torch.nn.Parameter(torch.randn(4, 16, 8))
+
+ config = LoraConfig(
+ r = 2, target_modules = [], target_parameters = ["0.experts"], modules_to_save = ["1"]
+ )
+ original = torch.nn.Sequential(Experts(), torch.nn.Linear(8, 8, bias = False))
+ expected = original[1].weight.detach().clone()
+ saved = get_peft_model(original, config)
+ saved.save_pretrained(tmp_path)
+ source = open(os.path.join(HERE, "unsloth", "models", "loader.py"), encoding = "utf-8").read()
+ contexts = [
+ node
+ for node in ast.walk(ast.parse(source))
+ if isinstance(node, ast.With)
+ and any(
+ isinstance(child, ast.Call)
+ and isinstance(child.func, ast.Attribute)
+ and isinstance(child.func.value, ast.Name)
+ and child.func.value.id == "PeftModel"
+ and child.func.attr == "from_pretrained"
+ for statement in node.body
+ for child in ast.walk(statement)
+ )
+ ]
+ assert len(contexts) == 2
+ for node in contexts:
+ model = torch.nn.Sequential(Experts(), torch.nn.Linear(8, 8, bias = False))
+ swapper, weight = _evicted_swapper()
+ model[0].experts = weight
+ saved_swapper, saved_weight = _evicted_swapper((8, 8))
+ model[1].weight = saved_weight
+ swapper.blocks.extend(saved_swapper.blocks)
+ model._unsloth_block_swap = swapper
+ ns = dict(
+ _load(),
+ model = model,
+ PeftModel = PeftModel,
+ contextlib = contextlib,
+ fast_pissa = False,
+ old_model_name = str(tmp_path),
+ token = None,
+ revision = None,
+ local_files_only = True,
+ trust_remote_code = False,
+ peft_load_kwargs = {"config": config},
+ load_config = config,
+ )
+ exec(compile(ast.Module(body = [node], type_ignores = []), "loader_reload", "exec"), ns)
+ assert isinstance(ns["model"], PeftModel)
+ assert weight.numel() == 0
+ assert saved_weight.numel() == 0
+ torch.testing.assert_close(
+ ns["model"].base_model.model[1].modules_to_save["default"].weight, expected
+ )
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 590ca93823a..c1ec368f2ea 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -10532,6 +10532,83 @@ def planned_prefetch_depth(device_map_planner_kwargs):
return depth if depth == "auto" else int(depth)
+def block_swap_shaped(
+ model,
+ init_lora_weights = True,
+ use_dora = False,
+ modules_to_save = None,
+ lora_config = None,
+ loading_adapter = False,
+):
+ swapper = getattr(model, "_unsloth_block_swap", None)
+ use_dora = getattr(lora_config, "use_dora", use_dora)
+ modules_to_save = getattr(lora_config, "modules_to_save", modules_to_save)
+ use_kasa = getattr(lora_config, "kasa_config", None) is not None
+ if use_dora or use_kasa or modules_to_save:
+ evicted = {
+ id(param)
+ for block in getattr(swapper, "blocks", ())
+ if not block.resident
+ for param in block.params
+ }
+ if evicted and (use_dora or use_kasa):
+ targeted = True
+ if lora_config is not None:
+ from copy import copy
+ from peft.tuners.tuners_utils import (
+ _maybe_include_all_linear_layers,
+ check_target_module_exists,
+ )
+
+ config = _maybe_include_all_linear_layers(copy(lora_config), model)
+ targeted = any(
+ check_target_module_exists(config, name)
+ and any(id(param) in evicted for param in module.parameters())
+ for name, module in model.named_modules()
+ )
+ if targeted:
+ variant = "DoRA" if use_dora else "KaSA"
+ raise ValueError(
+ f"Unsloth: {variant} needs real base weights to initialize, "
+ "but offload_layers evicted its targets. Load without offload_layers."
+ )
+ if evicted and modules_to_save and not loading_adapter:
+ for name, module in model.named_modules():
+ if any(name.endswith(target) for target in modules_to_save) and any(
+ id(param) in evicted for param in module.parameters()
+ ):
+ raise ValueError(
+ f"Unsloth: modules_to_save needs the real weights of '{name}', "
+ "but offload_layers evicted them. Load without offload_layers."
+ )
+ # PiSSA, OLoRA, LoftQ, CorDA, MiCA... read the weights: a placeholder would silently init them from zeros.
+ # EVA only zeroes lora_B here; its activation pass runs after get_peft_model, through the swap.
+ if swapper is None or init_lora_weights not in (True, False, "gaussian", "eva"):
+ return contextlib.nullcontext()
+ shaped = getattr(swapper, "shaped", None)
+ return shaped() if shaped is not None else _block_swap_shaped_fallback(swapper)
+
+
+def _block_swap_shaped_fallback(swapper):
+ held = []
+ try:
+ try:
+ for b in swapper.blocks:
+ if not b.resident:
+ for p, e, sig in zip(b.params, b.empties, b.sig):
+ held.append((b, p, e))
+ p.data = e.new_zeros(()).expand(sig[0])
+ except AttributeError:
+ pass
+ yield
+ finally:
+ # A block fetched inside the context holds real weights now: leave it.
+ for b, p, e in held:
+ if not b.resident:
+ p.data = e
+
+
def finish_block_swap_load(
model,
state,
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index f2b9341e40f..cc25b3c52eb 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -5195,8 +5195,16 @@ class FastLlamaModel:
collect_garbage(after = clean_gpu_cache)
from .lora_init import fast_lora_init, record_fast_pissa
+ from ._utils import block_swap_shaped
- with fast_lora_init() as fast:
+ with (
+ fast_lora_init() as fast,
+ block_swap_shaped(
+ model,
+ init_lora_weights,
+ lora_config = lora_config,
+ ),
+ ):
model = _get_peft_model(model, lora_config)
if fast["pissa"]:
record_fast_pissa(model)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 28537aa521a..0e6a48accea 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -187,6 +187,7 @@ from ._utils import (
_mark_forced_float32,
_drop_float16_overflowing_attention_dropout,
_ignore_block_swap_params_for_ddp,
+ block_swap_shaped,
_mark_full_finetuning,
_get_text_only_config,
resolve_model_class,
@@ -2112,7 +2113,16 @@ class FastLanguageModel(FastLlamaModel):
local_files_only = local_files_only,
cache_dir = kwargs.get("cache_dir"),
)
- with fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext():
+ load_config = peft_load_kwargs.get("config", peft_config)
+ with (
+ fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext(),
+ block_swap_shaped(
+ model,
+ getattr(load_config, "init_lora_weights", True),
+ lora_config = load_config,
+ loading_adapter = True,
+ ),
+ ):
model = PeftModel.from_pretrained(
model,
old_model_name,
@@ -3594,7 +3604,16 @@ class FastModel(FastBaseModel):
cache_dir = kwargs.get("cache_dir"),
)
try:
- with fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext():
+ load_config = peft_load_kwargs.get("config", peft_config)
+ with (
+ fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext(),
+ block_swap_shaped(
+ model,
+ getattr(load_config, "init_lora_weights", True),
+ lora_config = load_config,
+ loading_adapter = True,
+ ),
+ ):
model = PeftModel.from_pretrained(
model,
old_model_name,
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 0936b0f3698..731d29bbf66 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -5754,8 +5754,16 @@ class FastBaseModel:
_LoraModel._create_and_replace = _patched_car
from .lora_init import fast_lora_init, record_fast_pissa
+ from ._utils import block_swap_shaped
- with fast_lora_init() as fast:
+ with (
+ fast_lora_init() as fast,
+ block_swap_shaped(
+ model,
+ init_lora_weights,
+ lora_config = lora_config,
+ ),
+ ):
model = _get_peft_model(model, lora_config)
if fast["pissa"]:
record_fast_pissa(model)