Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Save LoRA-GA adapters relative to the original base so they reload correctly (#13322)
Daniel Han14b634ba09f0
diff --git a/tests/test_lora_init.py b/tests/test_lora_init.pyindex be502cb2482..9478d0f356e 100644--- a/tests/test_lora_init.py+++ b/tests/test_lora_init.py@@ -236,6 +236,34 @@ def test_calibration_patch_does_not_trigger_lazy_module_getattr(monkeypatch):assert names and not set(asked) & set(names)[email protected]("has_set_stance", [True, False])+def test_loraga_calibration_keeps_the_padding_mask(monkeypatch, has_set_stance):+ loraga = pytest.importorskip("peft.tuners.lora.loraga")+ fn = loraga.preprocess_loraga+ monkeypatch.setattr(loraga, "preprocess_loraga", getattr(fn, "__wrapped__", fn))+ if not has_set_stance:+ monkeypatch.delattr(torch.compiler, "set_stance", raising = False)+ lora_init.patch_peft_calibration_eager()++ model = torch.nn.Sequential(torch.nn.Linear(8, 8))+ model[0]._unsloth_keep_padding_mask = False+ seen = []++ def train_step():+ seen.append(model[0]._unsloth_keep_padding_mask)+ model(torch.randn(2, 8)).sum().backward()++ config = LoraConfig(+ r = 2,+ target_modules = ["0"],+ init_lora_weights = "lora_ga",+ lora_ga_config = peft.LoraGAConfig(),+ )+ loraga.preprocess_loraga(model, config, train_step)+ assert seen == [True]+ assert model[0]._unsloth_keep_padding_mask is False++def test_fast_init_kill_switch_and_force(monkeypatch):monkeypatch.setenv("UNSLOTH_FAST_LORA_INIT", "0")original = LoraLayer.pissa_initdiff --git a/tests/utils/test_lora_ga_save_conversion.py b/tests/utils/test_lora_ga_save_conversion.pynew file mode 100644index 00000000000..6a75652c951--- /dev/null+++ b/tests/utils/test_lora_ga_save_conversion.py@@ -0,0 +1,225 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++import json+import os+import shutil+import sys++import pytest+import torch++import unsloth # noqa: F401+from unsloth.models._utils import snapshot_residual_lora_init++peft = pytest.importorskip("peft")+if not hasattr(peft, "preprocess_loraga"):+ pytest.skip("PEFT without LoRA-GA", allow_module_level = True)++from peft import LoraConfig, LoraGAConfig, PeftModel, get_peft_model, preprocess_loraga+from transformers import AutoModelForCausalLM++MODEL = "trl-internal-testing/tiny-Qwen3ForCausalLM"+R = 4+++def _base():+ return AutoModelForCausalLM.from_pretrained(MODEL, dtype = torch.float32)+++def _lora_ga_model(+ seed,+ extra_targets = (),+ **config_kwargs,+):+ torch.manual_seed(seed)+ model = _base()+ config = LoraConfig(+ r = R,+ lora_alpha = 8,+ lora_dropout = 0.0,+ target_modules = ["q_proj", "v_proj", "down_proj", *extra_targets],+ init_lora_weights = "lora_ga",+ lora_ga_config = LoraGAConfig(),+ **config_kwargs,+ )+ ids = torch.randint(+ 0, model.config.vocab_size, (2, 12), generator = torch.Generator().manual_seed(0)+ )++ def train_step():+ model(input_ids = ids, labels = ids).loss.backward()++ preprocess_loraga(model, config, train_step)+ peft_model = get_peft_model(model, config)+ snapshot_residual_lora_init(peft_model, "lora_ga")+ return peft_model+++def _train(model, seed = 1):+ g = torch.Generator().manual_seed(seed)+ with torch.no_grad():+ for n, p in model.named_parameters():+ if "lora_" in n:+ p.add_(0.05 * torch.randn(p.shape, generator = g))+++def _logits(model):+ ids = torch.arange(1, 17).unsqueeze(0)+ model.eval()+ with torch.no_grad():+ return model(input_ids = ids).logits+++def _lora_state(model):+ return {n: p.detach().clone() for n, p in model.named_parameters() if "lora_" in n}++[email protected](scope = "module")+def trained():+ model = _lora_ga_model(seed = 3407)+ _train(model)+ return model, _logits(model)+++def test_saved_adapter_reloads_on_original_base(trained, tmp_path):+ model, reference = trained+ model.save_pretrained(tmp_path)+ config = json.load(open(tmp_path / "adapter_config.json"))+ assert (+ config["r"] == 2 * R and config["lora_alpha"] == 16 and config["init_lora_weights"] is True+ )+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")+ torch.testing.assert_close(_logits(reloaded), reference, atol = 1e-4, rtol = 1e-4)+++def test_embedding_targets_keep_their_rank(tmp_path):+ # PEFT gives the embedding a default init; doubling only the global r would break its reload.+ model = _lora_ga_model(seed = 3407, extra_targets = ["embed_tokens"])+ _train(model)+ reference = _logits(model)+ model.save_pretrained(tmp_path)+ config = json.load(open(tmp_path / "adapter_config.json"))+ assert config["rank_pattern"] == {r"model\.embed_tokens": R}+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")+ torch.testing.assert_close(_logits(reloaded), reference, atol = 1e-4, rtol = 1e-4)+++def test_user_pattern_does_not_shadow_the_embedding_pin(tmp_path):+ # "embed_tokens" sorts before the pin and would resolve the embedding to the doubled rank.+ model = _lora_ga_model(+ seed = 3407, extra_targets = ["embed_tokens"], rank_pattern = {"embed_tokens": R}+ )+ _train(model)+ reference = _logits(model)+ model.save_pretrained(tmp_path)+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")+ torch.testing.assert_close(_logits(reloaded), reference, atol = 1e-4, rtol = 1e-4)+++def test_empty_adapter_selection_saves_nothing(trained, tmp_path):+ model, _ = trained+ model.save_pretrained(tmp_path, selected_adapters = [])+ assert not (tmp_path / "adapter_config.json").exists()+++def test_native_save_is_the_bug(trained, tmp_path):+ model, reference = trained+ model.save_pretrained.__wrapped__(tmp_path)+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")+ assert (+ _logits(reloaded) - reference+ ).abs().max() > 1e-3 # 30x the tolerance the converted save meets+++def test_trainer_state_dict_save_is_converted(trained, tmp_path):+ model, reference = trained+ model.save_pretrained(tmp_path, state_dict = model.state_dict(), safe_serialization = True)+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")+ torch.testing.assert_close(_logits(reloaded), reference, atol = 1e-4, rtol = 1e-4)++[email protected]("as_path", [False, True])+def test_resume_into_live_model_restores_rank_r(trained, tmp_path, as_path):+ model, reference = trained+ model.save_pretrained(tmp_path)+ saved = _lora_state(model)+ _train(model, seed = 2)+ model.load_adapter(tmp_path if as_path else str(tmp_path), "default", is_trainable = True)+ for n, p in model.named_parameters():+ if "lora_" in n:+ assert p.shape == saved[n].shape and p.requires_grad+ torch.testing.assert_close(p.detach(), saved[n], atol = 0, rtol = 0)+ torch.testing.assert_close(_logits(model), reference, atol = 1e-5, rtol = 1e-5)+++def test_resume_in_fresh_run_rebases_onto_saved_init(trained, tmp_path):+ model, reference = trained+ model.save_pretrained(tmp_path)+ fresh = _lora_ga_model(+ seed = 7+ ) # svd_lowrank draws differ: another B0 @ A0 subtracted from the base+ model_init = next(m for m in model.modules() if hasattr(m, "_unsloth_initial_lora"))+ fresh_init = next(m for m in fresh.modules() if hasattr(m, "_unsloth_initial_lora"))+ assert not torch.equal(+ model_init._unsloth_initial_lora["default"][0],+ fresh_init._unsloth_initial_lora["default"][0],+ )+ fresh.load_adapter(str(tmp_path), "default", is_trainable = True)+ torch.testing.assert_close(_logits(fresh), reference, atol = 1e-4, rtol = 1e-4)+++def test_native_checkpoint_still_loads(trained, tmp_path):+ model, reference = trained+ model.save_pretrained.__wrapped__(tmp_path) # what earlier Unsloth versions wrote+ _train(model, seed = 3)+ model.load_adapter(str(tmp_path), "default", is_trainable = True)+ torch.testing.assert_close(_logits(model), reference, atol = 1e-5, rtol = 1e-5)+++def test_wrapping_is_idempotent(tmp_path):+ import types++ model = _lora_ga_model(seed = 3407)+ inner = model.save_pretrained+ # patch_saving_functions rebinds save_pretrained around the wrapper with types.MethodType.+ model.save_pretrained = types.MethodType(lambda self, *a, **k: inner(*a, **k), model)+ outer = model.save_pretrained+ snapshot_residual_lora_init(model, "lora_ga")+ assert model.save_pretrained is outer+ model.save_pretrained(tmp_path)+ assert json.load(open(tmp_path / "adapter_config.json"))["r"] == 2 * R+++ sys.platform == "win32" and shutil.which("cl") is None,+ reason = "the Unsloth training step compiles with inductor, which needs MSVC on Windows",+)+def test_trainer_checkpoints_resume_and_reload(tmp_path):+ from datasets import Dataset+ from transformers import Trainer, TrainingArguments++ g = torch.Generator().manual_seed(1)+ rows = [torch.randint(0, 1000, (16,), generator = g).tolist() for _ in range(16)]+ data = Dataset.from_list([{"input_ids": r, "labels": r} for r in rows])++ def train(model, resume = None):+ args = TrainingArguments(+ str(tmp_path),+ max_steps = 4,+ save_steps = 2,+ per_device_train_batch_size = 2,+ learning_rate = 1e-2,+ use_cpu = True,+ report_to = [],+ seed = 3407,+ )+ Trainer(model = model, args = args, train_dataset = data).train(resume_from_checkpoint = resume)+ return _logits(model)++ reference = train(_lora_ga_model(seed = 3407))+ plain = PeftModel.from_pretrained(_base(), tmp_path / "checkpoint-4", torch_device = "cpu")+ torch.testing.assert_close(_logits(plain), reference, atol = 1e-4, rtol = 1e-4)+ # A fresh run draws another LoRA-GA init, then resumes steps 3-4 from the converted checkpoint.+ resumed = train(_lora_ga_model(seed = 7), resume = str(tmp_path / "checkpoint-2"))+ torch.testing.assert_close(resumed, reference, atol = 1e-3, rtol = 1e-3)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex c8c564a1212..cda7c8a2316 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -6752,6 +6752,190 @@ def snapshot_residual_lora_init(model, init_lora_weights):}if initial:module._unsloth_initial_lora = initial+ if init_lora_weights in CONVERT_ON_SAVE_LORA_WEIGHTS:+ save_lora_relative_to_original_base(model)+++# PEFT cannot rebuild these residual bases when it loads an adapter (LoRA-GA needs the training gradients and+# silently falls back to a random init), so a native adapter reloads onto the wrong base.+CONVERT_ON_SAVE_LORA_WEIGHTS = ("lora_ga",)+++def _residual_lora_rows(model, adapters):+ from transformers.pytorch_utils import Conv1D+ for name, module in model.named_modules():+ initial = getattr(module, "_unsloth_initial_lora", None)+ if not initial or not isinstance(getattr(module, "lora_A", None), torch.nn.ModuleDict):+ continue+ # LoRA-GA only rewrites Linear / Conv1D bases; embeddings and convolutions keep PEFT's default init.+ if not isinstance(module.get_base_layer(), (torch.nn.Linear, Conv1D)):+ continue+ for k in initial:+ if k in adapters and k in module.lora_A:+ yield module, k, name+++def _unconverted_patterns(model, converted, adapter):+ import re+ from peft.tuners.lora import LoraLayer++ ranks, alphas = {}, {}+ for name, module in model.named_modules():+ if id(module) in converted or not isinstance(module, LoraLayer):+ continue+ if adapter not in getattr(module, "r", {}):+ continue+ key = re.escape(name.removeprefix("base_model.model."))+ ranks[key] = module.r[adapter]+ alphas[key] = module.lora_alpha[adapter]+ return ranks, alphas+++def _convertible_adapters(model, selected):+ names = set()+ for k in selected:+ config = model.peft_config.get(k)+ if config is None or getattr(config, "use_dora", False):+ continue+ if config.use_rslora and (config.rank_pattern or config.alpha_pattern):+ continue # PEFT refuses this conversion too+ names.add(k)+ return names+++def save_lora_relative_to_original_base(model):+ """Saves (and pushes) write PEFT's `path_initial_model_for_weight_conversion` form: a plain rank-2r LoRA on the+ original base, `s0 * [B * s / s0, -B0] @ [A; A0]`, loadable anywhere. `load_adapter` into this live model, as+ Trainer resume and load_best_model_at_end do, slices it back to rank r."""+ import inspect+ import json+ import os++ # On the model: patch_saving_functions later rebinds save_pretrained around this wrapper.+ if getattr(model, "_unsloth_residual_save", False):+ return+ original_save = model.save_pretrained+ original_load = model.load_adapter+ save_signature = inspect.signature(original_save)++ def _converted_state_dict(rows, state_dict):+ state_dict = dict(model.state_dict() if state_dict is None else state_dict)+ for module, k, name in rows:+ key_A, key_B = f"{name}.lora_A.{k}.weight", f"{name}.lora_B.{k}.weight"+ A0, B0, scaling0 = module._unsloth_initial_lora[k]+ if key_A not in state_dict or key_B not in state_dict or not scaling0:+ return None+ A, B = state_dict[key_A].detach(), state_dict[key_B].detach()+ state_dict[key_A] = torch.cat([A, A0.to(A)], 0)+ state_dict[key_B] = torch.cat([B * (module.scaling[k] / scaling0), -B0.to(B)], 1)+ return state_dict++ @functools.wraps(original_save)+ def save_pretrained(save_directory, *args, **kwargs):+ bound = save_signature.bind_partial(save_directory, *args, **kwargs).arguments+ if bound.get("path_initial_model_for_weight_conversion") is not None:+ return original_save(save_directory, *args, **kwargs)+ extra = bound.get("kwargs", {})+ selected = bound.get("selected_adapters")+ adapters = _convertible_adapters(+ model, list(model.peft_config) if selected is None else selected+ )+ rows = list(_residual_lora_rows(model, adapters))+ state_dict = _converted_state_dict(rows, extra.get("state_dict")) if rows else None+ if state_dict is None:+ return original_save(save_directory, *args, **kwargs)+ out = original_save(save_directory, *args, **{**kwargs, "state_dict": state_dict})+ if bound.get("is_main_process", True):+ for k in {k for _, k, _ in rows}:+ folder = save_directory if k == "default" else os.path.join(save_directory, k)+ path = os.path.join(folder, "adapter_config.json")+ with open(path, encoding = "utf-8") as f:+ config = json.load(f)+ # Same rewrite as PEFT's save_pretrained(path_initial_model_for_weight_conversion = ...).+ config["init_lora_weights"] = True+ config["r"] *= 2+ config["lora_alpha"] *= 2**0.5 if config.get("use_rslora") else 2+ kept = _unconverted_patterns(model, {id(m) for m, j, _ in rows if j == k}, k)+ for pattern, pinned in zip(("rank_pattern", "alpha_pattern"), kept):+ doubled = {key: 2 * v for key, v in (config.get(pattern) or {}).items()}+ if pinned or doubled:+ config[pattern] = {**pinned, **doubled}+ config.pop("lora_ga_config", None)+ with open(path, "w", encoding = "utf-8") as f:+ # No sort_keys: PEFT matches patterns first-come, and the exact pins go first.+ json.dump(config, f, indent = 2)+ return out++ @functools.wraps(original_load)+ def load_adapter(+ model_id,+ adapter_name = None,+ *args,+ **kwargs,+ ):+ native = None+ if adapter_name in model.peft_config and isinstance(model_id, (str, os.PathLike)):+ folder = os.path.join(os.fspath(model_id), kwargs.get("subfolder") or "")+ if os.path.isdir(folder):+ native = _native_from_converted(model, adapter_name, folder)+ if native is None:+ return original_load(model_id, adapter_name, *args, **kwargs)+ import shutil+ import tempfile+ from safetensors.torch import save_file++ with tempfile.TemporaryDirectory() as tmp:+ save_file(+ native, os.path.join(tmp, "adapter_model.safetensors"), metadata = {"format": "pt"}+ )+ shutil.copy(os.path.join(folder, "adapter_config.json"), tmp)+ kwargs.pop("subfolder", None)+ return original_load(tmp, adapter_name, *args, **kwargs)++ model._unsloth_residual_save = True+ model.save_pretrained = save_pretrained+ model.load_adapter = load_adapter++[email protected]_grad()+def _native_from_converted(model, adapter_name, folder):+ """Rank-r weights of a converted adapter for the live residual model, or None if `folder` is not one.+ A differing snapshot (a fresh run re-initialized) has its base re-based onto the saved A0 / B0."""+ import os++ if not any(+ os.path.isfile(os.path.join(folder, n))+ for n in ("adapter_model.safetensors", "adapter_model.bin")+ ):+ return None+ rows = list(_residual_lora_rows(model, {adapter_name}))+ if not rows:+ return None+ from peft.utils import load_peft_weights++ weights = load_peft_weights(folder, device = "cpu")+ plan = []+ for module, k, name in rows:+ key_A, key_B = f"{name}.lora_A.weight", f"{name}.lora_B.weight"+ A, B = weights.get(key_A), weights.get(key_B)+ r = module.lora_A[k].weight.shape[0]+ if A is None or B is None or A.shape[0] != 2 * r or B.shape[1] != 2 * r:+ return None+ plan.append((module, k, key_A, key_B, r))+ for module, k, key_A, key_B, r in plan:+ A, B = weights[key_A], weights[key_B]+ A0_live, B0_live, scaling0 = module._unsloth_initial_lora[k]+ A0, B0 = A[r:].to(A0_live), -B[:, r:].to(B0_live)+ weights[key_A] = A[:r].contiguous()+ weights[key_B] = (B[:, :r] * (scaling0 / module.scaling[k])).contiguous()+ if not (torch.equal(A0, A0_live) and torch.equal(B0, B0_live)):+ base = module.get_base_layer().weight+ delta = scaling0 * (B0_live.float() @ A0_live.float() - B0.float() @ A0.float())+ if getattr(module, "fan_in_fan_out", False):+ delta = delta.T+ base.data.add_(delta.to(base))+ module._unsloth_initial_lora[k] = (A0.clone(), B0.clone(), scaling0)+ return weights@contextlib.contextmanagerdiff --git a/unsloth/models/lora_init.py b/unsloth/models/lora_init.pyindex 08d072a90c0..37a37300546 100644--- a/unsloth/models/lora_init.py+++ b/unsloth/models/lora_init.py@@ -427,9 +427,6 @@ def _calibration_functions():def patch_peft_calibration_eager():import functoolsimport sys-- if not hasattr(torch.compiler, "set_stance"):- returnfor module, name in _calibration_functions():original = getattr(module, name, None)if original is None or getattr(original, "_unsloth_eager", False):@@ -441,8 +438,27 @@ def patch_peft_calibration_eager():__original = original,**kwargs,):- with torch.compiler.set_stance("force_eager"):- return __original(*args, **kwargs)+ # Runs before get_peft_model, so the tokenizer still pads left; labelled training forwards drop the mask.+ model = args[0] if args else kwargs.get("model")+ masked = []+ if isinstance(model, torch.nn.Module):+ masked = [+ m+ for m in model.modules()+ if not getattr(m, "_unsloth_keep_padding_mask", False)+ ]+ for m in masked:+ m._unsloth_keep_padding_mask = True+ try:+ with (+ torch.compiler.set_stance("force_eager")+ if hasattr(torch.compiler, "set_stance")+ else contextlib.nullcontext()+ ):+ return __original(*args, **kwargs)+ finally:+ for m in masked:+ m._unsloth_keep_padding_mask = Falseeager._unsloth_eager = True# vars(), not getattr: getattr on lazy modules (transformers) imports submodules.