Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Save LoRA-GA adapters relative to the original base so they reload correctly (#13322)

Daniel Han14b634ba09f0
diff --git a/tests/test_lora_init.py b/tests/test_lora_init.py
index be502cb2482..9478d0f356e 100644
--- a/tests/test_lora_init.py
+++ b/tests/test_lora_init.py
@@ -236,6 +236,34 @@ def test_calibration_patch_does_not_trigger_lazy_module_getattr(monkeypatch):
assert names and not set(asked) & set(names)
[email protected]("has_set_stance", [True, False])
+def test_loraga_calibration_keeps_the_padding_mask(monkeypatch, has_set_stance):
+ loraga = pytest.importorskip("peft.tuners.lora.loraga")
+ fn = loraga.preprocess_loraga
+ monkeypatch.setattr(loraga, "preprocess_loraga", getattr(fn, "__wrapped__", fn))
+ if not has_set_stance:
+ monkeypatch.delattr(torch.compiler, "set_stance", raising = False)
+ lora_init.patch_peft_calibration_eager()
+
+ model = torch.nn.Sequential(torch.nn.Linear(8, 8))
+ model[0]._unsloth_keep_padding_mask = False
+ seen = []
+
+ def train_step():
+ seen.append(model[0]._unsloth_keep_padding_mask)
+ model(torch.randn(2, 8)).sum().backward()
+
+ config = LoraConfig(
+ r = 2,
+ target_modules = ["0"],
+ init_lora_weights = "lora_ga",
+ lora_ga_config = peft.LoraGAConfig(),
+ )
+ loraga.preprocess_loraga(model, config, train_step)
+ assert seen == [True]
+ assert model[0]._unsloth_keep_padding_mask is False
+
+
def test_fast_init_kill_switch_and_force(monkeypatch):
monkeypatch.setenv("UNSLOTH_FAST_LORA_INIT", "0")
original = LoraLayer.pissa_init
diff --git a/tests/utils/test_lora_ga_save_conversion.py b/tests/utils/test_lora_ga_save_conversion.py
new file mode 100644
index 00000000000..6a75652c951
--- /dev/null
+++ b/tests/utils/test_lora_ga_save_conversion.py
@@ -0,0 +1,225 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+import json
+import os
+import shutil
+import sys
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth.models._utils import snapshot_residual_lora_init
+
+peft = pytest.importorskip("peft")
+if not hasattr(peft, "preprocess_loraga"):
+ pytest.skip("PEFT without LoRA-GA", allow_module_level = True)
+
+from peft import LoraConfig, LoraGAConfig, PeftModel, get_peft_model, preprocess_loraga
+from transformers import AutoModelForCausalLM
+
+MODEL = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+R = 4
+
+
+def _base():
+ return AutoModelForCausalLM.from_pretrained(MODEL, dtype = torch.float32)
+
+
+def _lora_ga_model(
+ seed,
+ extra_targets = (),
+ **config_kwargs,
+):
+ torch.manual_seed(seed)
+ model = _base()
+ config = LoraConfig(
+ r = R,
+ lora_alpha = 8,
+ lora_dropout = 0.0,
+ target_modules = ["q_proj", "v_proj", "down_proj", *extra_targets],
+ init_lora_weights = "lora_ga",
+ lora_ga_config = LoraGAConfig(),
+ **config_kwargs,
+ )
+ ids = torch.randint(
+ 0, model.config.vocab_size, (2, 12), generator = torch.Generator().manual_seed(0)
+ )
+
+ def train_step():
+ model(input_ids = ids, labels = ids).loss.backward()
+
+ preprocess_loraga(model, config, train_step)
+ peft_model = get_peft_model(model, config)
+ snapshot_residual_lora_init(peft_model, "lora_ga")
+ return peft_model
+
+
+def _train(model, seed = 1):
+ g = torch.Generator().manual_seed(seed)
+ with torch.no_grad():
+ for n, p in model.named_parameters():
+ if "lora_" in n:
+ p.add_(0.05 * torch.randn(p.shape, generator = g))
+
+
+def _logits(model):
+ ids = torch.arange(1, 17).unsqueeze(0)
+ model.eval()
+ with torch.no_grad():
+ return model(input_ids = ids).logits
+
+
+def _lora_state(model):
+ return {n: p.detach().clone() for n, p in model.named_parameters() if "lora_" in n}
+
+
[email protected](scope = "module")
+def trained():
+ model = _lora_ga_model(seed = 3407)
+ _train(model)
+ return model, _logits(model)
+
+
+def test_saved_adapter_reloads_on_original_base(trained, tmp_path):
+ model, reference = trained
+ model.save_pretrained(tmp_path)
+ config = json.load(open(tmp_path / "adapter_config.json"))
+ assert (
+ config["r"] == 2 * R and config["lora_alpha"] == 16 and config["init_lora_weights"] is True
+ )
+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")
+ torch.testing.assert_close(_logits(reloaded), reference, atol = 1e-4, rtol = 1e-4)
+
+
+def test_embedding_targets_keep_their_rank(tmp_path):
+ # PEFT gives the embedding a default init; doubling only the global r would break its reload.
+ model = _lora_ga_model(seed = 3407, extra_targets = ["embed_tokens"])
+ _train(model)
+ reference = _logits(model)
+ model.save_pretrained(tmp_path)
+ config = json.load(open(tmp_path / "adapter_config.json"))
+ assert config["rank_pattern"] == {r"model\.embed_tokens": R}
+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")
+ torch.testing.assert_close(_logits(reloaded), reference, atol = 1e-4, rtol = 1e-4)
+
+
+def test_user_pattern_does_not_shadow_the_embedding_pin(tmp_path):
+ # "embed_tokens" sorts before the pin and would resolve the embedding to the doubled rank.
+ model = _lora_ga_model(
+ seed = 3407, extra_targets = ["embed_tokens"], rank_pattern = {"embed_tokens": R}
+ )
+ _train(model)
+ reference = _logits(model)
+ model.save_pretrained(tmp_path)
+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")
+ torch.testing.assert_close(_logits(reloaded), reference, atol = 1e-4, rtol = 1e-4)
+
+
+def test_empty_adapter_selection_saves_nothing(trained, tmp_path):
+ model, _ = trained
+ model.save_pretrained(tmp_path, selected_adapters = [])
+ assert not (tmp_path / "adapter_config.json").exists()
+
+
+def test_native_save_is_the_bug(trained, tmp_path):
+ model, reference = trained
+ model.save_pretrained.__wrapped__(tmp_path)
+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")
+ assert (
+ _logits(reloaded) - reference
+ ).abs().max() > 1e-3 # 30x the tolerance the converted save meets
+
+
+def test_trainer_state_dict_save_is_converted(trained, tmp_path):
+ model, reference = trained
+ model.save_pretrained(tmp_path, state_dict = model.state_dict(), safe_serialization = True)
+ reloaded = PeftModel.from_pretrained(_base(), tmp_path, torch_device = "cpu")
+ torch.testing.assert_close(_logits(reloaded), reference, atol = 1e-4, rtol = 1e-4)
+
+
[email protected]("as_path", [False, True])
+def test_resume_into_live_model_restores_rank_r(trained, tmp_path, as_path):
+ model, reference = trained
+ model.save_pretrained(tmp_path)
+ saved = _lora_state(model)
+ _train(model, seed = 2)
+ model.load_adapter(tmp_path if as_path else str(tmp_path), "default", is_trainable = True)
+ for n, p in model.named_parameters():
+ if "lora_" in n:
+ assert p.shape == saved[n].shape and p.requires_grad
+ torch.testing.assert_close(p.detach(), saved[n], atol = 0, rtol = 0)
+ torch.testing.assert_close(_logits(model), reference, atol = 1e-5, rtol = 1e-5)
+
+
+def test_resume_in_fresh_run_rebases_onto_saved_init(trained, tmp_path):
+ model, reference = trained
+ model.save_pretrained(tmp_path)
+ fresh = _lora_ga_model(
+ seed = 7
+ ) # svd_lowrank draws differ: another B0 @ A0 subtracted from the base
+ model_init = next(m for m in model.modules() if hasattr(m, "_unsloth_initial_lora"))
+ fresh_init = next(m for m in fresh.modules() if hasattr(m, "_unsloth_initial_lora"))
+ assert not torch.equal(
+ model_init._unsloth_initial_lora["default"][0],
+ fresh_init._unsloth_initial_lora["default"][0],
+ )
+ fresh.load_adapter(str(tmp_path), "default", is_trainable = True)
+ torch.testing.assert_close(_logits(fresh), reference, atol = 1e-4, rtol = 1e-4)
+
+
+def test_native_checkpoint_still_loads(trained, tmp_path):
+ model, reference = trained
+ model.save_pretrained.__wrapped__(tmp_path) # what earlier Unsloth versions wrote
+ _train(model, seed = 3)
+ model.load_adapter(str(tmp_path), "default", is_trainable = True)
+ torch.testing.assert_close(_logits(model), reference, atol = 1e-5, rtol = 1e-5)
+
+
+def test_wrapping_is_idempotent(tmp_path):
+ import types
+
+ model = _lora_ga_model(seed = 3407)
+ inner = model.save_pretrained
+ # patch_saving_functions rebinds save_pretrained around the wrapper with types.MethodType.
+ model.save_pretrained = types.MethodType(lambda self, *a, **k: inner(*a, **k), model)
+ outer = model.save_pretrained
+ snapshot_residual_lora_init(model, "lora_ga")
+ assert model.save_pretrained is outer
+ model.save_pretrained(tmp_path)
+ assert json.load(open(tmp_path / "adapter_config.json"))["r"] == 2 * R
+
+
+ sys.platform == "win32" and shutil.which("cl") is None,
+ reason = "the Unsloth training step compiles with inductor, which needs MSVC on Windows",
+)
+def test_trainer_checkpoints_resume_and_reload(tmp_path):
+ from datasets import Dataset
+ from transformers import Trainer, TrainingArguments
+
+ g = torch.Generator().manual_seed(1)
+ rows = [torch.randint(0, 1000, (16,), generator = g).tolist() for _ in range(16)]
+ data = Dataset.from_list([{"input_ids": r, "labels": r} for r in rows])
+
+ def train(model, resume = None):
+ args = TrainingArguments(
+ str(tmp_path),
+ max_steps = 4,
+ save_steps = 2,
+ per_device_train_batch_size = 2,
+ learning_rate = 1e-2,
+ use_cpu = True,
+ report_to = [],
+ seed = 3407,
+ )
+ Trainer(model = model, args = args, train_dataset = data).train(resume_from_checkpoint = resume)
+ return _logits(model)
+
+ reference = train(_lora_ga_model(seed = 3407))
+ plain = PeftModel.from_pretrained(_base(), tmp_path / "checkpoint-4", torch_device = "cpu")
+ torch.testing.assert_close(_logits(plain), reference, atol = 1e-4, rtol = 1e-4)
+ # A fresh run draws another LoRA-GA init, then resumes steps 3-4 from the converted checkpoint.
+ resumed = train(_lora_ga_model(seed = 7), resume = str(tmp_path / "checkpoint-2"))
+ torch.testing.assert_close(resumed, reference, atol = 1e-3, rtol = 1e-3)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c8c564a1212..cda7c8a2316 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -6752,6 +6752,190 @@ def snapshot_residual_lora_init(model, init_lora_weights):
}
if initial:
module._unsloth_initial_lora = initial
+ if init_lora_weights in CONVERT_ON_SAVE_LORA_WEIGHTS:
+ save_lora_relative_to_original_base(model)
+
+
+# PEFT cannot rebuild these residual bases when it loads an adapter (LoRA-GA needs the training gradients and
+# silently falls back to a random init), so a native adapter reloads onto the wrong base.
+CONVERT_ON_SAVE_LORA_WEIGHTS = ("lora_ga",)
+
+
+def _residual_lora_rows(model, adapters):
+ from transformers.pytorch_utils import Conv1D
+ for name, module in model.named_modules():
+ initial = getattr(module, "_unsloth_initial_lora", None)
+ if not initial or not isinstance(getattr(module, "lora_A", None), torch.nn.ModuleDict):
+ continue
+ # LoRA-GA only rewrites Linear / Conv1D bases; embeddings and convolutions keep PEFT's default init.
+ if not isinstance(module.get_base_layer(), (torch.nn.Linear, Conv1D)):
+ continue
+ for k in initial:
+ if k in adapters and k in module.lora_A:
+ yield module, k, name
+
+
+def _unconverted_patterns(model, converted, adapter):
+ import re
+ from peft.tuners.lora import LoraLayer
+
+ ranks, alphas = {}, {}
+ for name, module in model.named_modules():
+ if id(module) in converted or not isinstance(module, LoraLayer):
+ continue
+ if adapter not in getattr(module, "r", {}):
+ continue
+ key = re.escape(name.removeprefix("base_model.model."))
+ ranks[key] = module.r[adapter]
+ alphas[key] = module.lora_alpha[adapter]
+ return ranks, alphas
+
+
+def _convertible_adapters(model, selected):
+ names = set()
+ for k in selected:
+ config = model.peft_config.get(k)
+ if config is None or getattr(config, "use_dora", False):
+ continue
+ if config.use_rslora and (config.rank_pattern or config.alpha_pattern):
+ continue # PEFT refuses this conversion too
+ names.add(k)
+ return names
+
+
+def save_lora_relative_to_original_base(model):
+ """Saves (and pushes) write PEFT's `path_initial_model_for_weight_conversion` form: a plain rank-2r LoRA on the
+ original base, `s0 * [B * s / s0, -B0] @ [A; A0]`, loadable anywhere. `load_adapter` into this live model, as
+ Trainer resume and load_best_model_at_end do, slices it back to rank r."""
+ import inspect
+ import json
+ import os
+
+ # On the model: patch_saving_functions later rebinds save_pretrained around this wrapper.
+ if getattr(model, "_unsloth_residual_save", False):
+ return
+ original_save = model.save_pretrained
+ original_load = model.load_adapter
+ save_signature = inspect.signature(original_save)
+
+ def _converted_state_dict(rows, state_dict):
+ state_dict = dict(model.state_dict() if state_dict is None else state_dict)
+ for module, k, name in rows:
+ key_A, key_B = f"{name}.lora_A.{k}.weight", f"{name}.lora_B.{k}.weight"
+ A0, B0, scaling0 = module._unsloth_initial_lora[k]
+ if key_A not in state_dict or key_B not in state_dict or not scaling0:
+ return None
+ A, B = state_dict[key_A].detach(), state_dict[key_B].detach()
+ state_dict[key_A] = torch.cat([A, A0.to(A)], 0)
+ state_dict[key_B] = torch.cat([B * (module.scaling[k] / scaling0), -B0.to(B)], 1)
+ return state_dict
+
+ @functools.wraps(original_save)
+ def save_pretrained(save_directory, *args, **kwargs):
+ bound = save_signature.bind_partial(save_directory, *args, **kwargs).arguments
+ if bound.get("path_initial_model_for_weight_conversion") is not None:
+ return original_save(save_directory, *args, **kwargs)
+ extra = bound.get("kwargs", {})
+ selected = bound.get("selected_adapters")
+ adapters = _convertible_adapters(
+ model, list(model.peft_config) if selected is None else selected
+ )
+ rows = list(_residual_lora_rows(model, adapters))
+ state_dict = _converted_state_dict(rows, extra.get("state_dict")) if rows else None
+ if state_dict is None:
+ return original_save(save_directory, *args, **kwargs)
+ out = original_save(save_directory, *args, **{**kwargs, "state_dict": state_dict})
+ if bound.get("is_main_process", True):
+ for k in {k for _, k, _ in rows}:
+ folder = save_directory if k == "default" else os.path.join(save_directory, k)
+ path = os.path.join(folder, "adapter_config.json")
+ with open(path, encoding = "utf-8") as f:
+ config = json.load(f)
+ # Same rewrite as PEFT's save_pretrained(path_initial_model_for_weight_conversion = ...).
+ config["init_lora_weights"] = True
+ config["r"] *= 2
+ config["lora_alpha"] *= 2**0.5 if config.get("use_rslora") else 2
+ kept = _unconverted_patterns(model, {id(m) for m, j, _ in rows if j == k}, k)
+ for pattern, pinned in zip(("rank_pattern", "alpha_pattern"), kept):
+ doubled = {key: 2 * v for key, v in (config.get(pattern) or {}).items()}
+ if pinned or doubled:
+ config[pattern] = {**pinned, **doubled}
+ config.pop("lora_ga_config", None)
+ with open(path, "w", encoding = "utf-8") as f:
+ # No sort_keys: PEFT matches patterns first-come, and the exact pins go first.
+ json.dump(config, f, indent = 2)
+ return out
+
+ @functools.wraps(original_load)
+ def load_adapter(
+ model_id,
+ adapter_name = None,
+ *args,
+ **kwargs,
+ ):
+ native = None
+ if adapter_name in model.peft_config and isinstance(model_id, (str, os.PathLike)):
+ folder = os.path.join(os.fspath(model_id), kwargs.get("subfolder") or "")
+ if os.path.isdir(folder):
+ native = _native_from_converted(model, adapter_name, folder)
+ if native is None:
+ return original_load(model_id, adapter_name, *args, **kwargs)
+ import shutil
+ import tempfile
+ from safetensors.torch import save_file
+
+ with tempfile.TemporaryDirectory() as tmp:
+ save_file(
+ native, os.path.join(tmp, "adapter_model.safetensors"), metadata = {"format": "pt"}
+ )
+ shutil.copy(os.path.join(folder, "adapter_config.json"), tmp)
+ kwargs.pop("subfolder", None)
+ return original_load(tmp, adapter_name, *args, **kwargs)
+
+ model._unsloth_residual_save = True
+ model.save_pretrained = save_pretrained
+ model.load_adapter = load_adapter
+
+
+def _native_from_converted(model, adapter_name, folder):
+ """Rank-r weights of a converted adapter for the live residual model, or None if `folder` is not one.
+ A differing snapshot (a fresh run re-initialized) has its base re-based onto the saved A0 / B0."""
+ import os
+
+ if not any(
+ os.path.isfile(os.path.join(folder, n))
+ for n in ("adapter_model.safetensors", "adapter_model.bin")
+ ):
+ return None
+ rows = list(_residual_lora_rows(model, {adapter_name}))
+ if not rows:
+ return None
+ from peft.utils import load_peft_weights
+
+ weights = load_peft_weights(folder, device = "cpu")
+ plan = []
+ for module, k, name in rows:
+ key_A, key_B = f"{name}.lora_A.weight", f"{name}.lora_B.weight"
+ A, B = weights.get(key_A), weights.get(key_B)
+ r = module.lora_A[k].weight.shape[0]
+ if A is None or B is None or A.shape[0] != 2 * r or B.shape[1] != 2 * r:
+ return None
+ plan.append((module, k, key_A, key_B, r))
+ for module, k, key_A, key_B, r in plan:
+ A, B = weights[key_A], weights[key_B]
+ A0_live, B0_live, scaling0 = module._unsloth_initial_lora[k]
+ A0, B0 = A[r:].to(A0_live), -B[:, r:].to(B0_live)
+ weights[key_A] = A[:r].contiguous()
+ weights[key_B] = (B[:, :r] * (scaling0 / module.scaling[k])).contiguous()
+ if not (torch.equal(A0, A0_live) and torch.equal(B0, B0_live)):
+ base = module.get_base_layer().weight
+ delta = scaling0 * (B0_live.float() @ A0_live.float() - B0.float() @ A0.float())
+ if getattr(module, "fan_in_fan_out", False):
+ delta = delta.T
+ base.data.add_(delta.to(base))
+ module._unsloth_initial_lora[k] = (A0.clone(), B0.clone(), scaling0)
+ return weights
@contextlib.contextmanager
diff --git a/unsloth/models/lora_init.py b/unsloth/models/lora_init.py
index 08d072a90c0..37a37300546 100644
--- a/unsloth/models/lora_init.py
+++ b/unsloth/models/lora_init.py
@@ -427,9 +427,6 @@ def _calibration_functions():
def patch_peft_calibration_eager():
import functools
import sys
-
- if not hasattr(torch.compiler, "set_stance"):
- return
for module, name in _calibration_functions():
original = getattr(module, name, None)
if original is None or getattr(original, "_unsloth_eager", False):
@@ -441,8 +438,27 @@ def patch_peft_calibration_eager():
__original = original,
**kwargs,
):
- with torch.compiler.set_stance("force_eager"):
- return __original(*args, **kwargs)
+ # Runs before get_peft_model, so the tokenizer still pads left; labelled training forwards drop the mask.
+ model = args[0] if args else kwargs.get("model")
+ masked = []
+ if isinstance(model, torch.nn.Module):
+ masked = [
+ m
+ for m in model.modules()
+ if not getattr(m, "_unsloth_keep_padding_mask", False)
+ ]
+ for m in masked:
+ m._unsloth_keep_padding_mask = True
+ try:
+ with (
+ torch.compiler.set_stance("force_eager")
+ if hasattr(torch.compiler, "set_stance")
+ else contextlib.nullcontext()
+ ):
+ return __original(*args, **kwargs)
+ finally:
+ for m in masked:
+ m._unsloth_keep_padding_mask = False
eager._unsloth_eager = True
# vars(), not getattr: getattr on lazy modules (transformers) imports submodules.