Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix MoE adapter setup and reload with offloaded layers (#15159)
Leo Borcherding810c932ae2ee
diff --git a/tests/test_block_swap_shaped.py b/tests/test_block_swap_shaped.pynew file mode 100644index 00000000000..860bda5be93--- /dev/null+++ b/tests/test_block_swap_shaped.py@@ -0,0 +1,269 @@+# SPDX-License-Identifier: AGPL-3.0-only+import ast, contextlib, os, types+import torch++HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py")+NAMES = ("block_swap_shaped", "_block_swap_shaped_fallback")+++def _load():+ src = open(UTILS, encoding = "utf-8").read()+ nodes = {+ n.name: n for n in ast.parse(src).body if isinstance(n, ast.FunctionDef) and n.name in NAMES+ }+ assert set(nodes) == set(NAMES)+ ns = {"contextlib": contextlib}+ for name in NAMES:+ exec(ast.get_source_segment(src, nodes[name]), ns)+ ns["_block_swap_shaped_fallback"] = contextlib.contextmanager(ns["_block_swap_shaped_fallback"])+ return ns+++def _evicted_swapper(shape = (4, 16, 8)):+ w = torch.nn.Parameter(torch.randn(shape), requires_grad = False)+ empty = torch.empty(0)+ w.data = empty+ block = types.SimpleNamespace(+ resident = False, params = [w], empties = [empty], sig = ((shape, w.dtype, "cpu", 0),)+ )+ return types.SimpleNamespace(blocks = [block]), w+++def test_older_zoo_still_gets_the_real_shape_while_peft_wraps():+ ns = _load()+ swapper, w = _evicted_swapper()+ with ns["block_swap_shaped"](types.SimpleNamespace(_unsloth_block_swap = swapper)):+ assert w.shape == (4, 16, 8)+ assert w.untyped_storage().nbytes() == w.element_size()+ assert w.numel() == 0+++def test_zoo_shaped_is_used_when_present():+ ns = _load()+ entered = []++ @contextlib.contextmanager+ def shaped():+ entered.append(True)+ yield++ model = types.SimpleNamespace(_unsloth_block_swap = types.SimpleNamespace(shaped = shaped))+ with ns["block_swap_shaped"](model):+ pass+ assert entered == [True]+++def test_no_swap_and_unknown_swappers_are_no_ops():+ ns = _load()+ with ns["block_swap_shaped"](types.SimpleNamespace()):+ pass+ with ns["block_swap_shaped"](types.SimpleNamespace(_unsloth_block_swap = object())):+ pass+++def test_data_dependent_inits_never_see_the_placeholder():+ ns = _load()+ for init in ("pissa", "pissa_niter_4", "olora", "loftq", "corda", "orthogonal"):+ swapper, w = _evicted_swapper()+ with ns["block_swap_shaped"](types.SimpleNamespace(_unsloth_block_swap = swapper), init):+ assert w.numel() == 0, init+ for init in ("gaussian", "eva"):+ swapper, w = _evicted_swapper()+ with ns["block_swap_shaped"](types.SimpleNamespace(_unsloth_block_swap = swapper), init):+ assert w.shape == (4, 16, 8), init+++def test_fallback_leaves_a_block_fetched_inside_the_context():+ w = torch.nn.Parameter(torch.empty(0), requires_grad = False)+ blk = types.SimpleNamespace(resident = False, params = [w], empties = [w.data], sig = [((4, 8),)])+ with _load()["_block_swap_shaped_fallback"](types.SimpleNamespace(blocks = [blk])):+ w.data, blk.resident = torch.ones(4, 8), True+ assert w.shape == (4, 8)+++def test_dora_rejects_evicted_weights_before_initializing_magnitudes():+ import inspect+ import pytest+ from peft import LoraConfig, get_peft_model++ model = torch.nn.Sequential(torch.nn.Linear(8, 4, bias = False))+ model.requires_grad_(False)+ swapper, weight = _evicted_swapper((4, 8))+ model[0].weight = weight+ model._unsloth_block_swap = swapper+ config = LoraConfig(r = 2, target_modules = ["0"], use_dora = True)+ shaped = _load()["block_swap_shaped"]+ options = (+ {"use_dora": config.use_dora} if "use_dora" in inspect.signature(shaped).parameters else {}+ )+ with pytest.raises(ValueError, match = "DoRA"):+ with shaped(model, **options):+ get_peft_model(model, config)+ assert weight.numel() == 0+++def test_dora_without_block_swap_keeps_real_magnitudes():+ from peft import LoraConfig, get_peft_model+ for resident_swap in (False, True):+ model = torch.nn.Sequential(torch.nn.Linear(8, 4, bias = False))+ expected = model[0].weight.detach().norm(dim = 1)+ if resident_swap:+ model._unsloth_block_swap = types.SimpleNamespace(+ blocks = [types.SimpleNamespace(resident = True)]+ )+ config = LoraConfig(r = 2, target_modules = ["0"], use_dora = True)+ with _load()["block_swap_shaped"](model, use_dora = config.use_dora):+ model = get_peft_model(model, config)+ magnitude = model.base_model.model[0].lora_magnitude_vector["default"].weight+ torch.testing.assert_close(magnitude, expected)+++def test_modules_to_save_rejects_copying_evicted_weights():+ import inspect+ import pytest+ from peft import LoraConfig, get_peft_model++ model = torch.nn.Sequential(+ torch.nn.Linear(8, 4, bias = False), torch.nn.Linear(4, 4, bias = False)+ )+ model.requires_grad_(False)+ swapper, weight = _evicted_swapper((4, 8))+ model[0].weight = weight+ model._unsloth_block_swap = swapper+ config = LoraConfig(r = 2, target_modules = ["1"], modules_to_save = ["0"])+ shaped = _load()["block_swap_shaped"]+ options = (+ {"modules_to_save": config.modules_to_save}+ if "modules_to_save" in inspect.signature(shaped).parameters+ else {}+ )+ with pytest.raises(ValueError, match = "modules_to_save"):+ with shaped(model, **options):+ get_peft_model(model, config)+ assert weight.numel() == 0+++def test_modules_to_save_preserves_resident_modules_beside_evicted_weights():+ from peft import LoraConfig, get_peft_model++ model = torch.nn.Sequential(+ torch.nn.Linear(8, 4, bias = False), torch.nn.Linear(4, 4, bias = False)+ )+ expected = model[1].weight.detach().clone()+ swapper, weight = _evicted_swapper((4, 8))+ model[0].weight = weight+ model._unsloth_block_swap = swapper+ config = LoraConfig(r = 2, target_modules = ["0"], modules_to_save = ["1"])+ with _load()["block_swap_shaped"](model, modules_to_save = config.modules_to_save):+ model = get_peft_model(model, config)+ saved = model.base_model.model[1].modules_to_save["default"].weight+ torch.testing.assert_close(saved, expected)+++def test_dora_targets_can_stay_resident_beside_evicted_blocks():+ import inspect+ from peft import LoraConfig, get_peft_model++ model = torch.nn.Sequential(+ torch.nn.Linear(8, 4, bias = False), torch.nn.Linear(4, 4, bias = False)+ )+ expected = model[1].weight.detach().norm(dim = 1)+ swapper, weight = _evicted_swapper((4, 8))+ model[0].weight = weight+ model._unsloth_block_swap = swapper+ config = LoraConfig(r = 2, target_modules = ["1"], use_dora = True)+ shaped = _load()["block_swap_shaped"]+ options = (+ {"lora_config": config} if "lora_config" in inspect.signature(shaped).parameters else {}+ )+ with shaped(model, use_dora = True, **options):+ model = get_peft_model(model, config)+ torch.testing.assert_close(+ model.base_model.model[1].lora_magnitude_vector["default"].weight, expected+ )+++def test_kasa_never_truncates_placeholder_weights():+ import inspect+ import pytest+ import peft++ if not hasattr(peft, "KasaConfig"):+ pytest.skip("Installed PEFT has no KaSA")+ model = torch.nn.Sequential(torch.nn.Linear(8, 6, bias = False))+ model.requires_grad_(False)+ swapper, weight = _evicted_swapper((6, 8))+ model[0].weight = weight+ model._unsloth_block_swap = swapper+ config = peft.LoraConfig(r = 2, target_modules = ["0"], kasa_config = peft.KasaConfig())+ shaped = _load()["block_swap_shaped"]+ options = (+ {"lora_config": config} if "lora_config" in inspect.signature(shaped).parameters else {}+ )+ with pytest.raises(ValueError, match = "KaSA"):+ with shaped(model, **options):+ peft.get_peft_model(model, config)+ assert weight.numel() == 0+++def test_both_adapter_reload_paths_wrap_evicted_experts(tmp_path):+ from peft import LoraConfig, PeftModel, get_peft_model++ class Experts(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.experts = torch.nn.Parameter(torch.randn(4, 16, 8))++ config = LoraConfig(+ r = 2, target_modules = [], target_parameters = ["0.experts"], modules_to_save = ["1"]+ )+ original = torch.nn.Sequential(Experts(), torch.nn.Linear(8, 8, bias = False))+ expected = original[1].weight.detach().clone()+ saved = get_peft_model(original, config)+ saved.save_pretrained(tmp_path)+ source = open(os.path.join(HERE, "unsloth", "models", "loader.py"), encoding = "utf-8").read()+ contexts = [+ node+ for node in ast.walk(ast.parse(source))+ if isinstance(node, ast.With)+ and any(+ isinstance(child, ast.Call)+ and isinstance(child.func, ast.Attribute)+ and isinstance(child.func.value, ast.Name)+ and child.func.value.id == "PeftModel"+ and child.func.attr == "from_pretrained"+ for statement in node.body+ for child in ast.walk(statement)+ )+ ]+ assert len(contexts) == 2+ for node in contexts:+ model = torch.nn.Sequential(Experts(), torch.nn.Linear(8, 8, bias = False))+ swapper, weight = _evicted_swapper()+ model[0].experts = weight+ saved_swapper, saved_weight = _evicted_swapper((8, 8))+ model[1].weight = saved_weight+ swapper.blocks.extend(saved_swapper.blocks)+ model._unsloth_block_swap = swapper+ ns = dict(+ _load(),+ model = model,+ PeftModel = PeftModel,+ contextlib = contextlib,+ fast_pissa = False,+ old_model_name = str(tmp_path),+ token = None,+ revision = None,+ local_files_only = True,+ trust_remote_code = False,+ peft_load_kwargs = {"config": config},+ load_config = config,+ )+ exec(compile(ast.Module(body = [node], type_ignores = []), "loader_reload", "exec"), ns)+ assert isinstance(ns["model"], PeftModel)+ assert weight.numel() == 0+ assert saved_weight.numel() == 0+ torch.testing.assert_close(+ ns["model"].base_model.model[1].modules_to_save["default"].weight, expected+ )diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 590ca93823a..c1ec368f2ea 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -10532,6 +10532,83 @@ def planned_prefetch_depth(device_map_planner_kwargs):return depth if depth == "auto" else int(depth)+def block_swap_shaped(+ model,+ init_lora_weights = True,+ use_dora = False,+ modules_to_save = None,+ lora_config = None,+ loading_adapter = False,+):+ swapper = getattr(model, "_unsloth_block_swap", None)+ use_dora = getattr(lora_config, "use_dora", use_dora)+ modules_to_save = getattr(lora_config, "modules_to_save", modules_to_save)+ use_kasa = getattr(lora_config, "kasa_config", None) is not None+ if use_dora or use_kasa or modules_to_save:+ evicted = {+ id(param)+ for block in getattr(swapper, "blocks", ())+ if not block.resident+ for param in block.params+ }+ if evicted and (use_dora or use_kasa):+ targeted = True+ if lora_config is not None:+ from copy import copy+ from peft.tuners.tuners_utils import (+ _maybe_include_all_linear_layers,+ check_target_module_exists,+ )++ config = _maybe_include_all_linear_layers(copy(lora_config), model)+ targeted = any(+ check_target_module_exists(config, name)+ and any(id(param) in evicted for param in module.parameters())+ for name, module in model.named_modules()+ )+ if targeted:+ variant = "DoRA" if use_dora else "KaSA"+ raise ValueError(+ f"Unsloth: {variant} needs real base weights to initialize, "+ "but offload_layers evicted its targets. Load without offload_layers."+ )+ if evicted and modules_to_save and not loading_adapter:+ for name, module in model.named_modules():+ if any(name.endswith(target) for target in modules_to_save) and any(+ id(param) in evicted for param in module.parameters()+ ):+ raise ValueError(+ f"Unsloth: modules_to_save needs the real weights of '{name}', "+ "but offload_layers evicted them. Load without offload_layers."+ )+ # PiSSA, OLoRA, LoftQ, CorDA, MiCA... read the weights: a placeholder would silently init them from zeros.+ # EVA only zeroes lora_B here; its activation pass runs after get_peft_model, through the swap.+ if swapper is None or init_lora_weights not in (True, False, "gaussian", "eva"):+ return contextlib.nullcontext()+ shaped = getattr(swapper, "shaped", None)+ return shaped() if shaped is not None else _block_swap_shaped_fallback(swapper)+++def _block_swap_shaped_fallback(swapper):+ held = []+ try:+ try:+ for b in swapper.blocks:+ if not b.resident:+ for p, e, sig in zip(b.params, b.empties, b.sig):+ held.append((b, p, e))+ p.data = e.new_zeros(()).expand(sig[0])+ except AttributeError:+ pass+ yield+ finally:+ # A block fetched inside the context holds real weights now: leave it.+ for b, p, e in held:+ if not b.resident:+ p.data = e++def finish_block_swap_load(model,state,diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex f2b9341e40f..cc25b3c52eb 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -5195,8 +5195,16 @@ class FastLlamaModel:collect_garbage(after = clean_gpu_cache)from .lora_init import fast_lora_init, record_fast_pissa+ from ._utils import block_swap_shaped- with fast_lora_init() as fast:+ with (+ fast_lora_init() as fast,+ block_swap_shaped(+ model,+ init_lora_weights,+ lora_config = lora_config,+ ),+ ):model = _get_peft_model(model, lora_config)if fast["pissa"]:record_fast_pissa(model)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 28537aa521a..0e6a48accea 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -187,6 +187,7 @@ from ._utils import (_mark_forced_float32,_drop_float16_overflowing_attention_dropout,_ignore_block_swap_params_for_ddp,+ block_swap_shaped,_mark_full_finetuning,_get_text_only_config,resolve_model_class,@@ -2112,7 +2113,16 @@ class FastLanguageModel(FastLlamaModel):local_files_only = local_files_only,cache_dir = kwargs.get("cache_dir"),)- with fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext():+ load_config = peft_load_kwargs.get("config", peft_config)+ with (+ fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext(),+ block_swap_shaped(+ model,+ getattr(load_config, "init_lora_weights", True),+ lora_config = load_config,+ loading_adapter = True,+ ),+ ):model = PeftModel.from_pretrained(model,old_model_name,@@ -3594,7 +3604,16 @@ class FastModel(FastBaseModel):cache_dir = kwargs.get("cache_dir"),)try:- with fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext():+ load_config = peft_load_kwargs.get("config", peft_config)+ with (+ fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext(),+ block_swap_shaped(+ model,+ getattr(load_config, "init_lora_weights", True),+ lora_config = load_config,+ loading_adapter = True,+ ),+ ):model = PeftModel.from_pretrained(model,old_model_name,diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 0936b0f3698..731d29bbf66 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -5754,8 +5754,16 @@ class FastBaseModel:_LoraModel._create_and_replace = _patched_carfrom .lora_init import fast_lora_init, record_fast_pissa+ from ._utils import block_swap_shaped- with fast_lora_init() as fast:+ with (+ fast_lora_init() as fast,+ block_swap_shaped(+ model,+ init_lora_weights,+ lora_config = lora_config,+ ),+ ):model = _get_peft_model(model, lora_config)if fast["pissa"]:record_fast_pissa(model)