Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Support every PEFT init_lora_weights option, with fast PiSSA and MiCA init (#6879)

Suchitra Malimbadac17ca646c7cc
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.py
index 2b1c75aa352..da1aadfb79d 100644
--- a/tests/saving/test_save_method_lora_routing.py
+++ b/tests/saving/test_save_method_lora_routing.py
@@ -27,6 +27,7 @@ This file therefore runs on Linux, macOS and Windows with no accelerator and no
from __future__ import annotations
import ast
+import contextlib
import sys
import types
from pathlib import Path
@@ -56,7 +57,12 @@ def _load(*names, **env):
"""Exec the named top-level functions against `env` and return the namespace."""
# save.py imports this from models.mistral_format (#12144) and calls it on every merge and
# GGUF path; none of these fixtures is a Mistral-format view, so it never refuses here.
- namespace = {"raise_if_merging_mistral_format_view": lambda model, save_method: None, **env}
+ namespace = {
+ "raise_if_merging_mistral_format_view": lambda model, save_method: None,
+ "lora_relative_to_original_base": lambda model: contextlib.nullcontext(),
+ "nullcontext": contextlib.nullcontext,
+ **env,
+ }
for name in names:
exec(compile(_function_source(name), str(_SAVE_PY), "exec"), namespace)
return namespace
@@ -239,7 +245,7 @@ def _not_reached(*args, **kwargs):
def _routing_environment(monkeypatch, model):
- calls = {"merge": [], "adapter": [], "prewarm": []}
+ calls = {"merge": [], "adapter": [], "prewarm": [], "convert": []}
zoo = types.ModuleType("unsloth_zoo.saving_utils")
zoo.merge_and_overwrite_lora = lambda *args, **kwargs: calls["merge"].append(kwargs)
@@ -260,6 +266,9 @@ def _routing_environment(monkeypatch, model):
_qwen3_5_vlm_state_dict_for_save = _not_reached,
_determine_username = lambda repo, old, token: (repo, "owner"),
unsloth_save_model = lambda *args, **kwargs: calls["adapter"].append(kwargs),
+ lora_relative_to_original_base = lambda model: (
+ calls["convert"].append(model) or contextlib.nullcontext()
+ ),
logger = types.SimpleNamespace(warning_once = lambda *a, **k: None),
gc = types.SimpleNamespace(collect = lambda: None),
torch = types.SimpleNamespace(
@@ -348,6 +357,7 @@ def test_every_other_method_still_merges(monkeypatch, tmp_path, save_method):
assert calls["adapter"] == []
assert len(calls["merge"]) == 1
assert len(calls["prewarm"]) == 1
+ assert len(calls["convert"]) == (save_method != "merged_4bit_forced")
def test_a_model_with_no_adapter_is_unchanged(monkeypatch, tmp_path):
diff --git a/tests/test_lora_init.py b/tests/test_lora_init.py
new file mode 100644
index 00000000000..be502cb2482
--- /dev/null
+++ b/tests/test_lora_init.py
@@ -0,0 +1,395 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+import copy
+import importlib.util
+from pathlib import Path
+
+import pytest
+import torch
+
+peft = pytest.importorskip("peft")
+from peft import LoraConfig, get_peft_model
+from peft.tuners.lora.layer import LoraLayer
+
+_spec = importlib.util.spec_from_file_location(
+ "unsloth_lora_init", Path(__file__).resolve().parents[1] / "unsloth" / "models" / "lora_init.py"
+)
+lora_init = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(lora_init)
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+def _weight(
+ out_features,
+ in_features,
+ seed = 0,
+):
+ g = torch.Generator().manual_seed(seed)
+ k = min(out_features, in_features)
+ U = torch.linalg.qr(torch.randn(out_features, k, generator = g, dtype = torch.float64)).Q
+ V = torch.linalg.qr(torch.randn(in_features, k, generator = g, dtype = torch.float64)).Q
+ S = torch.logspace(0, -2, k, dtype = torch.float64) + 0.01 * torch.rand(
+ k, generator = g, dtype = torch.float64
+ )
+ return (U * S) @ V.T
+
+
[email protected]("shape", [(96, 48), (48, 96), (64, 64)])
[email protected]("rank", [1, 8])
+def test_randomized_svd_near_optimal(shape, rank):
+ W = _weight(*shape)
+ U, S, Vh = lora_init.randomized_svd(W.float().to(DEVICE), rank)
+ Ue, Se, Vhe = torch.linalg.svd(W, full_matrices = False)
+ best = (W - (Ue[:, :rank] * Se[:rank]) @ Vhe[:rank]).norm()
+ err = (W - (U.double().cpu() * S.double().cpu()) @ Vh.double().cpu()).norm()
+ assert err / best < 1.001
+ assert torch.allclose(S.double().cpu(), Se[:rank], rtol = 1e-3)
+ assert U.dtype == torch.float32 and U.shape == (shape[0], rank) and Vh.shape == (rank, shape[1])
+
+
+def _adversarial():
+ g = torch.Generator().manual_seed(1)
+ return {
+ "zero": torch.zeros(64, 40),
+ "rank1": torch.randn(64, 1, generator = g) @ torch.randn(1, 40, generator = g),
+ "duplicate_columns": torch.randn(64, 4, generator = g).repeat(1, 10),
+ # Column norms above ~1e19 overflow a plain fp32 vector_norm.
+ "column_scales": torch.randn(64, 40, generator = g) * torch.logspace(-20, 20, 40),
+ "row_scales": torch.randn(64, 40, generator = g) * torch.logspace(-20, 20, 64)[:, None],
+ "one_huge_entry": torch.randn(64, 40, generator = g).index_put_(
+ (torch.tensor([3]), torch.tensor([7])), torch.tensor(1e30)
+ ),
+ }
+
+
[email protected]("name", list(_adversarial()))
[email protected]("rank", [4, 16])
+def test_randomized_svd_adversarial_finite_and_near_optimal(name, rank):
+ W = _adversarial()[name]
+ U, S, Vh = lora_init.randomized_svd(W.to(DEVICE), rank)
+ assert torch.isfinite(U).all() and torch.isfinite(S).all() and torch.isfinite(Vh).all()
+ Wd = W.double()
+ S0 = torch.linalg.svdvals(Wd)
+ err = (Wd - (U.double().cpu() * S.double().cpu()) @ Vh.double().cpu()).norm()
+ best = (S0[rank:] ** 2).sum().sqrt()
+ assert err <= best + 1e-4 * Wd.norm() + 1e-30
+ assert (S.double().cpu() - S0[:rank]).abs().max() <= 1e-4 * S0[0] + 1e-30
+
+
[email protected]("true_rank", [0, 1, 8])
+def test_randomized_svd_rank_deficient_wide_sketch(true_rank):
+ # q = 128 takes the eigh path, which cuSOLVER refuses on the NaNs a CholeskyQR breakdown leaves.
+ g = torch.Generator().manual_seed(2)
+ W = torch.randn(512, true_rank, generator = g) @ torch.randn(true_rank, 384, generator = g)
+ U, S, Vh = lora_init.randomized_svd(W.to(DEVICE), 64)
+ assert torch.isfinite(U).all() and torch.isfinite(S).all() and torch.isfinite(Vh).all()
+ err = (W.double() - (U.double().cpu() * S.double().cpu()) @ Vh.double().cpu()).norm()
+ assert err <= 1e-5 * W.norm() + 1e-30
+
+
[email protected]("name", ["column_scales", "row_scales", "one_huge_entry"])
[email protected]("rank", [4, 16])
+def test_randomized_svd_survives_unscaled_solver_norms(monkeypatch, name, rank):
+ # rocSOLVER's geqrf / gesvd return NaN where a squared column norm overflows fp32 (gfx1151).
+ qr, svd = torch.linalg.qr, torch.linalg.svd
+
+ def overflows(X):
+ return X.dtype == torch.float32 and not torch.isfinite(X.pow(2).sum(-2)).all()
+
+ def unscaled_qr(X, *args, **kwargs):
+ out = qr(X, *args, **kwargs)
+ return torch.return_types.linalg_qr((out.Q * float("nan"), out.R)) if overflows(X) else out
+
+ def unscaled_svd(X, *args, **kwargs):
+ return svd(X * float("nan") if overflows(X) else X, *args, **kwargs)
+
+ monkeypatch.setattr(torch.linalg, "qr", unscaled_qr)
+ monkeypatch.setattr(torch.linalg, "svd", unscaled_svd)
+ monkeypatch.setattr(torch.version, "hip", "7.0", raising = False)
+ test_randomized_svd_adversarial_finite_and_near_optimal(name, rank)
+
+
+def test_randomized_svd_is_fp32_only(monkeypatch):
+ # Nothing may run in float64: consumer GPUs run it at 1/64 rate.
+ calls = []
+ original = torch.Tensor.double
+ monkeypatch.setattr(torch.Tensor, "double", lambda self: calls.append(1) or original(self))
+ U, S, Vh = lora_init.randomized_svd(_weight(96, 48).float().to(DEVICE), 8)
+ assert not calls and U.dtype == S.dtype == Vh.dtype == torch.float32
+
+
[email protected]("shape", [(96, 48), (48, 96), (64, 64)])
+def test_mica_basis_matches_fp64_svd(shape):
+ W = _weight(*shape)
+ r = 8
+ B = lora_init.mica_basis(W.float(), r)
+ U = torch.linalg.svd(W, full_matrices = False)[0][:, -r:]
+ assert torch.linalg.matrix_norm(U @ U.T - B.double() @ B.double().T, ord = 2) < 1e-4
+ assert torch.nn.functional.cosine_similarity(B.double(), U, dim = 0).abs().min() > 0.999
+
+
[email protected]("shape", [(96, 48), (48, 96)])
+def test_mica_basis_rank_deficient_is_finite(shape):
+ # One dead input / output dimension: a single zero eigenvalue, below the r smallest's largest.
+ W = _weight(*shape)
+ W[:, 0] = 0
+ W[0] = 0
+ B = lora_init.mica_basis(W.float(), 8).double()
+ assert torch.isfinite(B).all()
+ torch.testing.assert_close(B.T @ B, torch.eye(8, dtype = B.dtype), atol = 1e-4, rtol = 0)
+
+
+class _Model(torch.nn.Module):
+ def __init__(self, W):
+ super().__init__()
+ self.q = torch.nn.Linear(W.shape[1], W.shape[0], bias = False)
+ self.q.weight.data = W.float()
+
+ def forward(self, x):
+ return self.q(x)
+
+
[email protected]("init", ["pissa", "pissa_niter_4"])
+def test_fast_pissa_preserves_output(init):
+ W = _weight(96, 64)
+ base = _Model(W).to(DEVICE)
+ x = torch.randn(3, 64, device = DEVICE)
+ with lora_init.fast_lora_init():
+ assert LoraLayer.pissa_init is lora_init._pissa_init
+ model = get_peft_model(
+ copy.deepcopy(base),
+ LoraConfig(r = 8, lora_alpha = 16, target_modules = ["q"], init_lora_weights = init),
+ )
+ assert LoraLayer.pissa_init is not lora_init._pissa_init
+ assert torch.allclose(model(x), base(x), atol = 1e-5)
+ layer = model.base_model.model.q
+ BA = (
+ layer.scaling["default"]
+ * layer.lora_B["default"].weight.double().cpu()
+ @ layer.lora_A["default"].weight.double().cpu()
+ )
+ Ue, Se, Vhe = torch.linalg.svd(W, full_matrices = False)
+ # "pissa" stands in for an exact SVD; pissa_niter_N keeps PEFT's svd_lowrank accuracy class.
+ bound = 1.001 if init == "pissa" else 1.05
+ assert (W - BA).norm() / (W - (Ue[:, :8] * Se[:8]) @ Vhe[:8]).norm() < bound
+
+
+def test_kill_switch(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_FAST_LORA_INIT", "0")
+ original = LoraLayer.pissa_init
+ # An earlier fast run must not make a kill-switched one report the fast path.
+ monkeypatch.setitem(lora_init._STATE, "pissa", True)
+ with lora_init.fast_lora_init() as fast:
+ assert LoraLayer.pissa_init is original
+ assert not fast["pissa"]
+
+
+def test_fast_pissa_marker_lookup_fails_closed_only_when_online(monkeypatch, tmp_path):
+ import huggingface_hub
+ from huggingface_hub import constants
+ from huggingface_hub.utils import EntryNotFoundError, LocalEntryNotFoundError
+
+ def raising(error):
+ def download(*args, **kwargs):
+ raise error
+
+ return download
+
+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", False)
+ _save_fast_pissa(tmp_path)
+ monkeypatch.setattr(
+ huggingface_hub, "hf_hub_download", lambda repo, name, **k: str(tmp_path / name)
+ )
+ assert lora_init.adapter_used_fast_pissa("user/adapter")
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", raising(EntryNotFoundError("404")))
+ assert not lora_init.adapter_used_fast_pissa("user/adapter")
+ monkeypatch.setattr(
+ huggingface_hub, "hf_hub_download", raising(LocalEntryNotFoundError("down"))
+ )
+ with pytest.raises(LocalEntryNotFoundError):
+ lora_init.adapter_used_fast_pissa("user/adapter")
+ assert not lora_init.adapter_used_fast_pissa("user/adapter", local_files_only = True)
+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True)
+ assert not lora_init.adapter_used_fast_pissa("user/adapter")
+
+
+def test_calibration_patch_does_not_trigger_lazy_module_getattr(monkeypatch):
+ # transformers' lazy module imports submodules (torchvision for aria) on any getattr.
+ import sys
+ import types
+
+ asked = []
+ lazy = types.ModuleType("unsloth_test_lazy_module")
+
+ def __getattr__(name):
+ asked.append(name)
+ raise AttributeError(name)
+
+ lazy.__getattr__ = __getattr__
+ monkeypatch.setitem(sys.modules, lazy.__name__, lazy)
+ names = [name for _, name in lora_init._calibration_functions()]
+ for module, name in lora_init._calibration_functions():
+ fn = getattr(module, name)
+ monkeypatch.setattr(module, name, getattr(fn, "__wrapped__", fn))
+ lora_init.patch_peft_calibration_eager()
+ assert names and not set(asked) & set(names)
+
+
+def test_fast_init_kill_switch_and_force(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_FAST_LORA_INIT", "0")
+ original = LoraLayer.pissa_init
+ with lora_init.fast_lora_init():
+ assert LoraLayer.pissa_init is original
+ with lora_init.fast_lora_init(force = True):
+ assert LoraLayer.pissa_init is lora_init._pissa_init
+ with lora_init.fast_lora_init():
+ assert LoraLayer.pissa_init is lora_init._pissa_init
+ assert LoraLayer.pissa_init is original
+
+
+def _save_fast_pissa(folder):
+ from peft import LoraConfig, get_peft_model
+
+ base = torch.nn.Sequential(torch.nn.Linear(64, 48, bias = False))
+ with lora_init.fast_lora_init():
+ model = get_peft_model(
+ base, LoraConfig(r = 4, target_modules = ["0"], init_lora_weights = "pissa")
+ )
+ lora_init.record_fast_pissa(model)
+ model.save_pretrained(str(folder))
+ return model
+
+
+def test_a_stale_marker_does_not_follow_a_later_ordinary_save(tmp_path):
+ from peft import LoraConfig, get_peft_model
+
+ _save_fast_pissa(tmp_path)
+ assert lora_init.adapter_used_fast_pissa(str(tmp_path))
+ base = torch.nn.Sequential(torch.nn.Linear(64, 48, bias = False))
+ get_peft_model(base, LoraConfig(r = 4, target_modules = ["0"])).save_pretrained(str(tmp_path))
+ assert (tmp_path / lora_init.SIDECAR).is_file()
+ assert not lora_init.adapter_used_fast_pissa(str(tmp_path))
+
+
+def test_fast_pissa_saves_record_their_algorithm(tmp_path):
+ from peft import LoraConfig, get_peft_model
+
+ def config():
+ return LoraConfig(r = 4, target_modules = ["0"], init_lora_weights = "pissa")
+
+ base = torch.nn.Sequential(torch.nn.Linear(64, 48, bias = False))
+ assert not lora_init.adapter_used_fast_pissa(str(tmp_path))
+ with lora_init.fast_lora_init() as fast:
+ model = get_peft_model(base, config())
+ assert fast["pissa"]
+ lora_init.record_fast_pissa(model)
+ lora_init.record_fast_pissa(model)
+ # A second PiSSA adapter through plain PEFT must keep PEFT's initializer on reload.
+ model.add_adapter("plain", config())
+ model.save_pretrained(str(tmp_path))
+ assert (tmp_path / "plain" / "adapter_config.json").is_file()
+ assert lora_init.adapter_used_fast_pissa(str(tmp_path))
+ assert not lora_init.adapter_used_fast_pissa(str(tmp_path / "plain"))
+
+
+def test_sketch_ignores_default_dtype():
+ previous = torch.get_default_dtype()
+ lora_init._sketch.cache_clear()
+ try:
+ torch.set_default_dtype(torch.float64)
+ U, S, Vh = lora_init.randomized_svd(_weight(96, 48).float(), 8)
+ finally:
+ torch.set_default_dtype(previous)
+ lora_init._sketch.cache_clear()
+ assert U.dtype == S.dtype == Vh.dtype == torch.float32
+
+
+def test_other_threads_keep_peft_init(monkeypatch):
+ import threading
+
+ calls = []
+ with lora_init.fast_lora_init(force = True):
+ monkeypatch.setitem(lora_init._ORIGINAL_ANY, "pissa_init", lambda *args: calls.append(args))
+ thread = threading.Thread(target = lora_init._pissa_init, args = (None, "a", "pissa"))
+ thread.start()
+ thread.join()
+ assert calls == [(None, "a", "pissa")]
+
+
+def test_packed_quantized_base_is_refused():
+ layer = torch.nn.Linear(16, 16)
+ Params4bit = type("Params4bit", (torch.nn.Parameter,), {})
+ layer.weight = Params4bit(layer.weight.data.to(torch.bfloat16), requires_grad = False)
+ with pytest.raises(TypeError, match = "load_in_4bit = False"):
+ lora_init._check_float_weight(layer.weight, "pissa")
+
+
+def test_pissa_niter_width_is_device_independent(monkeypatch):
+ # Loading re-runs the init, possibly on another device: the sketch width must not depend on it.
+ seen = []
+ real = lora_init.randomized_svd
+
+ def spy(W, rank, **kwargs):
+ seen.append(kwargs["n_oversamples"])
+ return real(W, rank, **kwargs)
+
+ monkeypatch.setattr(lora_init, "randomized_svd", spy)
+ model = torch.nn.Sequential(torch.nn.Linear(160, 160))
+ with lora_init.fast_lora_init(force = True):
+ get_peft_model(
+ model, LoraConfig(r = 64, target_modules = ["0"], init_lora_weights = "pissa_niter_4")
+ )
+ assert seen == [0]
+
+
+def test_swap_state_is_per_call():
+ with lora_init.fast_lora_init(force = True) as first:
+ first["pissa"] = True
+ with lora_init.fast_lora_init(force = True) as second:
+ assert not second["pissa"]
+ assert first["pissa"]
+
+
[email protected]("alpha", [0, 16])
+def test_merge_conversion_matches_the_live_adapter(alpha):
+ from peft import LoraConfig, get_peft_model
+ from unsloth.models._utils import lora_relative_to_original_base, snapshot_residual_lora_init
+
+ torch.manual_seed(0)
+ linear = torch.nn.Linear(64, 48, bias = False)
+ W = linear.weight.detach().clone()
+ config = LoraConfig(r = 8, lora_alpha = alpha, target_modules = ["0"], init_lora_weights = "olora")
+ model = get_peft_model(torch.nn.Sequential(linear), config)
+ snapshot_residual_lora_init(model, "olora")
+ layer = model.base_model.model[0]
+ with torch.no_grad():
+ layer.lora_B["default"].weight.add_(0.05 * torch.randn_like(layer.lora_B["default"].weight))
+ x = torch.randn(5, 64)
+ live = model(x)
+ with lora_relative_to_original_base(model):
+ s = layer.scaling["default"]
+ merged = W + s * layer.lora_B["default"].weight @ layer.lora_A["default"].weight
+ assert torch.isfinite(merged).all()
+ torch.testing.assert_close(x @ merged.T, live, atol = 1e-4, rtol = 1e-4)
+
+
[email protected]("layer", ["linear", "embedding"])
+def test_untrained_residual_init_merges_back_to_the_original_weight(layer):
+ # Factors set by hand: PEFT >= 0.20's Embedding.loftq_init is broken (0.18 / 0.19 reach this).
+ from peft import LoraConfig, get_peft_model
+ from unsloth.models._utils import lora_relative_to_original_base, snapshot_residual_lora_init
+
+ torch.manual_seed(0)
+ base = torch.nn.Linear(64, 48, bias = False) if layer == "linear" else torch.nn.Embedding(48, 64)
+ model = get_peft_model(torch.nn.Sequential(base), LoraConfig(r = 8, target_modules = ["0"]))
+ module = model.base_model.model[0]
+ with torch.no_grad():
+ for p in module.parameters():
+ if p.requires_grad:
+ p.normal_()
+ snapshot_residual_lora_init(model, "loftq")
+ assert module.get_delta_weight("default").abs().max() > 0
+ with lora_relative_to_original_base(model):
+ delta = module.get_delta_weight("default")
+ torch.testing.assert_close(delta, torch.zeros_like(delta), atol = 1e-4, rtol = 0)
diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.py
index 1c9633a51ef..a2afd983b74 100644
--- a/tests/test_merged_hub_destination.py
+++ b/tests/test_merged_hub_destination.py
@@ -2,6 +2,7 @@
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
import ast
+import contextlib
import gc
import json
import os
@@ -167,6 +168,8 @@ def saving(monkeypatch, tmp_path):
_is_qwen3_5_vlm = lambda model: False,
# A Mistral-format view (#12144) refuses merged saves; these fixtures are not one.
raise_if_merging_mistral_format_view = lambda model, save_method: None,
+ lora_relative_to_original_base = lambda model: contextlib.nullcontext(),
+ nullcontext = contextlib.nullcontext,
logger = SimpleNamespace(warning_once = lambda *args: None),
# save_method="lora" leaves this module for the adapter save rather than the merge,
# so record the handover instead of re-implementing it.
diff --git a/tests/utils/test_attention_dispatch_eval_backward.py b/tests/utils/test_attention_dispatch_eval_backward.py
new file mode 100644
index 00000000000..5ac8e987d6c
--- /dev/null
+++ b/tests/utils/test_attention_dispatch_eval_backward.py
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""An eval-mode forward that will backpropagate (LoRA-GA calibration) must not get the GQA layout that
+xformers can only run forward."""
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth.utils import attention_dispatch as ad
+
+
+def _run(monkeypatch, requires_grad, grad_enabled):
+ seen = {}
+
+ def fake_kernel(
+ query,
+ key,
+ value,
+ attn_bias = None,
+ **kwargs,
+ ):
+ seen["ndim"] = query.dim()
+ return torch.zeros_like(query)
+
+ monkeypatch.setattr(ad, "xformers_attention", fake_kernel, raising = False)
+ context = ad.AttentionContext(
+ bsz = 1,
+ q_len = 3,
+ kv_seq_len = 3,
+ n_heads = 4,
+ head_dim = 8,
+ requires_grad = requires_grad,
+ seq_info = None,
+ attention_mask = None,
+ causal_mask = None,
+ )
+ config = ad.AttentionConfig(backend = ad.XFORMERS, n_kv_heads = 2, n_groups = 2)
+ Q = torch.randn(1, 4, 3, 8, requires_grad = True)
+ K = torch.randn(1, 2, 3, 8, requires_grad = True)
+ V = torch.randn(1, 2, 3, 8, requires_grad = True)
+ with torch.set_grad_enabled(grad_enabled):
+ ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)
+ return seen["ndim"]
+
+
+ "requires_grad, grad_enabled, ndim",
+ [(False, True, 4), (False, False, 5), (True, True, 4)],
+)
+def test_xformers_gqa_layout_follows_backward(monkeypatch, requires_grad, grad_enabled, ndim):
+ assert _run(monkeypatch, requires_grad, grad_enabled) == ndim
diff --git a/tests/utils/test_init_lora_validation.py b/tests/utils/test_init_lora_validation.py
new file mode 100644
index 00000000000..c8ed84c5e16
--- /dev/null
+++ b/tests/utils/test_init_lora_validation.py
@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth.models._utils import (
+ _has_quantized_linears,
+ validate_init_lora_weights,
+ validate_init_target_parameters,
+)
+
+
+class _UnslothNVFP4Linear(torch.nn.Linear):
+ pass
+
+
+def _model(*layers):
+ return torch.nn.Sequential(torch.nn.Embedding(8, 4), *layers)
+
+
+def test_dense_model_is_not_quantized():
+ assert not _has_quantized_linears(_model(torch.nn.Linear(4, 4)), routed_ok = True)
+
+
[email protected]("dtype", [torch.uint8, torch.int8, torch.float8_e4m3fn])
+def test_non_float_linear_is_quantized(dtype):
+ layer = torch.nn.Linear(4, 4)
+ layer.weight = torch.nn.Parameter(layer.weight.data.to(dtype), requires_grad = False)
+ assert _has_quantized_linears(_model(layer), routed_ok = True)
+
+
+def test_packed_params4bit_in_float_storage_is_quantized():
+ layer = torch.nn.Linear(4, 4)
+ Params4bit = type("Params4bit", (torch.nn.Parameter,), {})
+ layer.weight = Params4bit(layer.weight.data.to(torch.bfloat16), requires_grad = False)
+ assert _has_quantized_linears(_model(layer), routed_ok = True)
+
+
[email protected]("packed", ["qweight", "W_q", "weight_packed"])
+def test_packed_non_linear_projection_is_quantized(packed):
+ # GPTQ / AWQ (qweight), HQQ (W_q), packed MXFP4 / INT4 (weight_packed): no dense .weight.
+ layer = torch.nn.Module()
+ layer.register_buffer(packed, torch.zeros(4, 1, dtype = torch.int32))
+ assert _has_quantized_linears(_model(layer), routed_ok = True)
+
+
+def test_packed_mxfp4_linear_is_quantized():
+ from unsloth.models.mxfp4_compressed_linear import Mxfp4PackedLinear
+
+ layer = Mxfp4PackedLinear(4, 4, bias = False)
+ del layer.weight
+ layer.register_buffer("weight_packed", torch.zeros(4, 2, dtype = torch.uint8))
+ assert _has_quantized_linears(_model(layer), routed_ok = True)
+
+
+def test_olora_accepts_only_bitsandbytes_quantization():
+ # PEFT's olora_init dequantizes / requantizes bitsandbytes weights only.
+ bnb = torch.nn.Linear(4, 4)
+ Params4bit = type("Params4bit", (torch.nn.Parameter,), {})
+ bnb.weight = Params4bit(bnb.weight.data.to(torch.bfloat16), requires_grad = False)
+ validate_init_lora_weights("olora", _model(bnb))
+ with pytest.raises(ValueError, match = "quantized"):
+ validate_init_lora_weights("pissa", _model(bnb))
+ fp8 = torch.nn.Linear(4, 4)
+ fp8.weight = torch.nn.Parameter(fp8.weight.data.to(torch.float8_e4m3fn), requires_grad = False)
+ gptq = torch.nn.Module()
+ gptq.register_buffer("qweight", torch.zeros(4, 1, dtype = torch.int32))
+ for layer in (fp8, gptq):
+ with pytest.raises(ValueError, match = "quantized"):
+ validate_init_lora_weights("olora", _model(layer))
+
+
+def test_routed_compressed_linears_pass_except_for_mica():
+ nvfp4 = _UnslothNVFP4Linear(4, 4)
+ nvfp4.register_buffer("weight_packed", torch.zeros(4, 2, dtype = torch.uint8))
+ fp8 = torch.nn.Linear(4, 4)
+ fp8.weight = torch.nn.Parameter(fp8.weight.data.to(torch.float8_e4m3fn), requires_grad = False)
+ fp8._unsloth_compressed_tensors_fp8 = True
+ model = _model(nvfp4, fp8)
+ assert not _has_quantized_linears(model, routed_ok = True)
+ assert _has_quantized_linears(model, routed_ok = False)
+
+
+ "init", ["pissa", "pissa_niter_4", "olora", "orthogonal", "corda", "loftq", "lora_ga", "mica"]
+)
+def test_weight_reading_inits_reject_moe_expert_parameters(init):
+ with pytest.raises(ValueError, match = "target_parameters = \\[\\]"):
+ validate_init_target_parameters(init, ["gate_up_proj", "down_proj"])
+
+
[email protected]("init", [True, False, "gaussian", "eva"])
+def test_other_inits_allow_moe_expert_parameters(init):
+ validate_init_target_parameters(init, ["gate_up_proj"])
+
+
+def test_no_expert_parameters_allows_pissa():
+ validate_init_target_parameters("pissa", None)
+ validate_init_target_parameters("pissa", [])
diff --git a/tests/version_compat/test_peft_pinned_symbols.py b/tests/version_compat/test_peft_pinned_symbols.py
index 60ef23d3025..561c8fc6d07 100644
--- a/tests/version_compat/test_peft_pinned_symbols.py
+++ b/tests/version_compat/test_peft_pinned_symbols.py
@@ -21,6 +21,7 @@ PEFT_TAGS = [
"v0.18.1",
"v0.19.0",
"v0.19.1",
+ "v0.20.0",
"main",
]
@@ -289,3 +290,37 @@ def test_peft_version_parseable(tag: str):
assert (
has_literal or has_subimport or has_metadata
), f"{tag}: peft.__version__ not exported via any known mechanism"
+
+
+# 11. init_lora_weights="mica": check_mica_init imports MiCALinearVariant; PEFT inits via LoraLayer.mica_init
+# and freezes lora_B via _freeze_non_trainable_peft_weights (freeze_peft_variant_weights). PEFT >= 0.20.0.
+def test_peft_mica_variant_and_init(tag: str):
+ variants_src = fetch_text("huggingface/peft", tag, "src/peft/tuners/lora/variants.py")
+ if variants_src is None or not has_def(variants_src, "MiCALinearVariant", "class"):
+ pytest.skip(f"{tag}: MiCA not yet introduced (peft 0.20)")
+ layer_src = fetch_text("huggingface/peft", tag, "src/peft/tuners/lora/layer.py")
+ tuners_src = fetch_text("huggingface/peft", tag, "src/peft/tuners/tuners_utils.py")
+ assert layer_src is not None and has_def(
+ layer_src, "mica_init", "func"
+ ), f"{tag}: mica_init missing"
+ assert tuners_src is not None and has_def(
+ tuners_src, "_freeze_non_trainable_peft_weights", "func"
+ ), f"{tag}: _freeze_non_trainable_peft_weights missing"
+
+
+# 12. unsloth/models/lora_init.py replaces LoraLayer.pissa_init(self, adapter_name, init_lora_weights) and
+# mica_init(self, adapter_name) during get_peft_model and calls layer.py's `transpose`.
+def test_peft_lora_init_hooks(tag: str):
+ src = fetch_text("huggingface/peft", tag, "src/peft/tuners/lora/layer.py")
+ if src is None:
+ pytest.skip(f"{tag}: layer.py missing")
+ assert re.search(
+ r"def pissa_init\(self, adapter_name, init_lora_weights\)", src
+ ), f"{tag}: pissa_init signature moved"
+ assert re.search(
+ r"^from peft\.utils\.other import transpose$", src, re.M
+ ), f"{tag}: transpose import moved"
+ if has_def(src, "mica_init", "func"):
+ assert re.search(
+ r"def mica_init\(self, adapter_name\)", src
+ ), f"{tag}: mica_init signature moved"
diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.py
index cdbecea261a..2ef7fec7dfa 100644
--- a/unsloth/models/__init__.py
+++ b/unsloth/models/__init__.py
@@ -27,3 +27,7 @@ except:
from .dpo import PatchDPOTrainer, PatchKTOTrainer
from ._utils import is_bfloat16_supported, is_vLLM_available, __version__
from .rl import PatchFastRL, vLLMSamplingParams
+from .lora_init import patch_peft_calibration_eager as _patch_peft_calibration_eager
+
+_patch_peft_calibration_eager()
+del _patch_peft_calibration_eager
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 220c7c27317..5930706ea8c 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -75,6 +75,12 @@ __all__ = [
"set_task_config_attr",
"patch_fast_lora",
"validate_loftq_config",
+ "validate_init_lora_weights",
+ "validate_init_target_parameters",
+ "RESIDUAL_INIT_LORA_WEIGHTS",
+ "snapshot_residual_lora_init",
+ "lora_relative_to_original_base",
+ "freeze_peft_variant_weights",
"RaiseUninitialized",
"fast_inference_setup",
"patch_peft_fast_inference",
@@ -2675,6 +2681,7 @@ _ROOT_AUX_PREFETCH_PATTERNS = (
_ADAPTER_PREFETCH_PATTERNS = (
"adapter_config.json",
"adapter_model*",
+ "unsloth_lora_init.json", # lora_init.SIDECAR: which PiSSA algorithm rebuilds the residual base
)
@@ -5811,7 +5818,237 @@ for function in ("__reduce__", "__reduce_ex__", "__getstate__", "__setstate__"):
pass
-def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, model):
+_INIT_LORA_WEIGHTS = (
+ "gaussian",
+ "eva",
+ "olora",
+ "pissa",
+ "corda",
+ "loftq",
+ "orthogonal",
+ "lora_ga",
+ "mica",
+)
+# Inits that rewrite the base weight to W - scaling * B @ A (save_pretrained_merged must not use the original W).
+RESIDUAL_INIT_LORA_WEIGHTS = ("pissa", "olora", "corda", "loftq", "lora_ga")
+
+
+def _has_quantized_linears(
+ model,
+ routed_ok,
+ bnb_ok = False,
+):
+ for module in model.modules():
+ routed = type(module).__name__ == "_UnslothNVFP4Linear" or getattr(
+ module, "_unsloth_compressed_tensors_fp8", False
+ )
+ if routed:
+ if not routed_ok:
+ return True
+ continue
+ # No dense .weight: GPTQ / AWQ (qweight), HQQ (W_q), packed MXFP4 / INT4 (weight_packed).
+ if any(hasattr(module, name) for name in ("qweight", "qzeros", "W_q", "weight_packed")):
+ return True
+ if not isinstance(module, torch.nn.Linear):
+ continue
+ weight = getattr(module, "weight", None)
+ # FSDP-QLoRA packs Params4bit into a float quant_storage, so the dtype alone looks dense.
+ if type(weight).__name__ in ("Params4bit", "Int8Params") or hasattr(weight, "quant_state"):
+ if bnb_ok:
+ continue
+ return True
+ if isinstance(weight, torch.Tensor) and weight.dtype not in (
+ torch.float32,
+ torch.float16,
+ torch.bfloat16,
+ ):
+ return True
+ return False
+
+
+def validate_init_target_parameters(init_lora_weights, target_parameters):
+ # PEFT's ParamWrapper reads a .weight fused experts lack and refuses MiCA, failing mid get_peft_model.
+ if not target_parameters or not isinstance(init_lora_weights, str):
+ return
+ if init_lora_weights.split("_niter_")[0] in (
+ "pissa",
+ "olora",
+ "orthogonal",
+ "corda",
+ "loftq",
+ "lora_ga",
+ "mica",
+ ):
+ raise ValueError(
+ f"Unsloth: `init_lora_weights = {init_lora_weights!r}` cannot initialize fused MoE expert "
+ f"parameters ({target_parameters}).\n"
+ "Pass `target_parameters = []` to apply it to the other layers only, or use another init."
+ )
+
+
+def validate_init_lora_weights(
+ init_lora_weights,
+ model,
+ r = None,
+):
+ if type(init_lora_weights) is bool:
+ return
+ name = init_lora_weights if isinstance(init_lora_weights, str) else None
+ if name is not None and name.startswith("pissa_niter_"):
+ if not name[len("pissa_niter_") :].isdigit():
+ raise ValueError(
+ f"Unsloth: `init_lora_weights = {name!r}` must be `pissa_niter_<non-negative int>`."
+ )
+ name = "pissa"
+ if name not in _INIT_LORA_WEIGHTS:
+ raise ValueError(
+ "Unsloth: `init_lora_weights` must be True, False, `pissa_niter_<int>` or one of "
+ f"{list(_INIT_LORA_WEIGHTS)}, got {init_lora_weights!r}."
+ )
+ import peft
+ from peft.tuners.lora import LoraLayer
+
+ def _require(supported, version):
+ if not supported:
+ raise RuntimeError(
+ f"Unsloth: Your PEFT version of {peft.__version__} does not support "
+ f"`init_lora_weights = {init_lora_weights!r}`.\n"
+ f"Please install PEFT {version} or higher: `pip install --upgrade peft`"
+ )
+
+ if name == "mica":
+ try:
+ from peft.tuners.lora.variants import MiCALinearVariant
+ except ImportError:
+ _require(False, "0.20.0")
+ elif name == "lora_ga":
+ _require(hasattr(LoraLayer, "lora_ga_init"), "0.19.0")
+
+ # PEFT's olora handles bitsandbytes only; loader_utils densifies routed NVFP4 / FP8 for all but MiCA.
+ base = name.split("_niter_")[0] if name is not None else None
+ if base in (
+ "pissa",
+ "olora",
+ "corda",
+ "loftq",
+ "lora_ga",
+ "mica",
+ "orthogonal",
+ ) and _has_quantized_linears(model, routed_ok = base != "mica", bnb_ok = base == "olora"):
+ raise ValueError(
+ f"Unsloth: `init_lora_weights = {init_lora_weights!r}` needs float32/float16/bfloat16 base weights, "
+ "yet your model is quantized.\n"
+ "Reload your model with `load_in_4bit = False` and `load_in_8bit = False`."
+ )
+ if name == "orthogonal" and r is not None and r % 2 != 0:
+ raise ValueError(
+ f"Unsloth: `init_lora_weights = 'orthogonal'` needs an even rank, got r = {r}."
+ )
+ if name in ("corda", "lora_ga"):
+ attr = "eigens" if name == "corda" else "_peft_loraga_grad"
+ if not any(hasattr(module, attr) for module in model.modules()):
+ preprocess = (
+ "peft.tuners.lora.corda.preprocess_corda"
+ if name == "corda"
+ else "peft.preprocess_loraga"
+ )
+ # lora_ga silently falls back to the default init without it.
+ raise ValueError(
+ f"Unsloth: `init_lora_weights = {name!r}` needs `{preprocess}(model, lora_config, ...)` "
+ "to be run on the model before `get_peft_model`."
+ )
+ if name == "eva":
+ logger.warning_once(
+ "Unsloth: `init_lora_weights = 'eva'` only zeroes lora_B. Call "
+ "`peft.initialize_lora_eva_weights(model, dataloader)` after `get_peft_model` to run EVA."
+ )
+
+
+def _lora_factors(module):
+ """(adapter, A holder, A attr, B holder, B attr): Linear / Conv keep factors as `.weight` of lora_A / lora_B
+ modules, Embedding as entries of the lora_embedding_A / lora_embedding_B ParameterDicts."""
+ lora_A = getattr(module, "lora_A", None)
+ if isinstance(lora_A, torch.nn.ModuleDict):
+ for k in lora_A:
+ yield k, lora_A[k], "weight", module.lora_B[k], "weight"
+ lora_embedding_A = getattr(module, "lora_embedding_A", None)
+ if isinstance(lora_embedding_A, torch.nn.ParameterDict):
+ for k in lora_embedding_A:
+ yield k, lora_embedding_A, k, module.lora_embedding_B, k
+
+
+def snapshot_residual_lora_init(model, init_lora_weights):
+ if not isinstance(init_lora_weights, str):
+ return
+ if init_lora_weights.split("_niter_")[0] not in RESIDUAL_INIT_LORA_WEIGHTS:
+ return
+ # The scale the base rewrite used: LoftQ fits B0 @ A0 to W - Q unscaled, the others subtract s * B0 @ A0.
+ unscaled = init_lora_weights == "loftq"
+ for module in model.modules():
+ initial = {
+ k: (
+ getattr(a, a_name).detach().clone(),
+ getattr(b, b_name).detach().clone(),
+ 1.0 if unscaled else module.scaling[k],
+ )
+ for k, a, a_name, b, b_name in _lora_factors(module)
+ }
+ if initial:
+ module._unsloth_initial_lora = initial
+
+
+def lora_relative_to_original_base(model):
+ # Merge reads the original W: s0 * [B * s / s0, -B0] @ [A; A0] (PEFT's path_initial_model_for_weight_conversion).
+ swapped = []
+ try:
+ for module in model.modules():
+ initial = getattr(module, "_unsloth_initial_lora", None)
+ if not initial:
+ continue
+ for k, a, a_name, b, b_name in list(_lora_factors(module)):
+ if k not in initial:
+ continue
+ A0, B0, scaling0 = initial[k]
+ if not scaling0:
+ continue # lora_alpha = 0: the base rewrite was a no-op
+ A, B = getattr(a, a_name), getattr(b, b_name)
+ swapped.append((module, k, module.scaling[k], a, a_name, A, b, b_name, B))
+ B_new = B.detach() * (module.scaling[k] / scaling0)
+ module.scaling[k] = scaling0
+ setattr(
+ a,
+ a_name,
+ torch.nn.Parameter(torch.cat([A.detach(), A0.to(A)], 0), requires_grad = False),
+ )
+ setattr(
+ b,
+ b_name,
+ torch.nn.Parameter(torch.cat([B_new, -B0.to(B_new)], 1), requires_grad = False),
+ )
+ yield
+ finally:
+ for module, k, scaling, a, a_name, A, b, b_name, B in swapped:
+ module.scaling[k] = scaling
+ setattr(a, a_name, A)
+ setattr(b, b_name, B)
+
+
+def freeze_peft_variant_weights(model):
+ # prepare_model_for_training re-enables every lora_A/lora_B; PEFT variants such as MiCA freeze lora_B.
+ for module in model.modules():
+ if getattr(module, "frozen_peft_weight_names", None):
+ module._freeze_non_trainable_peft_weights()
+
+
+def validate_loftq_config(
+ loftq_config,
+ lora_dropout,
+ bias,
+ init_lora_weights,
+ model,
+ r = None,
+):
from peft import LoraConfig
if loftq_config is None:
@@ -5832,15 +6069,7 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m
f"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit."
)
- if not (
- type(init_lora_weights) is bool
- or init_lora_weights == "gaussian"
- or init_lora_weights == "loftq"
- or init_lora_weights == "corda"
- ):
- raise ValueError(
- 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'
- )
+ validate_init_lora_weights(init_lora_weights, model, r)
if init_lora_weights == "loftq":
if not SUPPORTS_LOFTQ:
@@ -5859,12 +6088,6 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m
)
loftq_config = LoftQConfig(loftq_bits = 4, loftq_iter = 1)
- if hasattr(model.config, "quantization_config"):
- raise ValueError(
- "Unsloth: You are using `loftq` init, yet `load_in_4bit = True` was set.\n"
- "Reload your model without any quantization by setting `load_in_4bit = False`."
- )
-
return loftq_config
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 07c24bfaa1c..1fb4e3cda58 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -3848,14 +3848,12 @@ class FastLlamaModel:
f"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit."
)
- if not (
- type(init_lora_weights) is bool
- or init_lora_weights == "gaussian"
- or init_lora_weights == "loftq"
- or init_lora_weights == "corda"
- ):
- raise ValueError(
- 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'
+ validate_init_lora_weights(init_lora_weights, model, r)
+ if init_lora_weights == "eva":
+ # EVA collects layer inputs with LoRA module forward hooks, which the fused LoRA kernels never call.
+ raise NotImplementedError(
+ "Unsloth: `init_lora_weights = 'eva'` is not supported by FastLanguageModel's fused LoRA path.\n"
+ "Use `FastModel.from_pretrained` and `FastModel.get_peft_model` instead."
)
if init_lora_weights == "loftq":
@@ -3875,12 +3873,6 @@ class FastLlamaModel:
)
loftq_config = LoftQConfig(loftq_bits = 4, loftq_iter = 1)
- if hasattr(model.config, "quantization_config"):
- raise ValueError(
- "Unsloth: You are using `loftq` init, yet `load_in_4bit = True` was set.\n"
- "Reload your model without any quantization by setting `load_in_4bit = False`."
- )
-
assert type(use_rslora) is bool
if use_rslora:
if not SUPPORTS_RSLORA:
@@ -4053,6 +4045,8 @@ class FastLlamaModel:
_n = max(1, min(int(finetune_last_n_layers), _total_layers))
layers_to_transform = list(range(_total_layers - _n, _total_layers))
+ validate_init_target_parameters(init_lora_weights, target_parameters)
+
arguments = dict(
r = r,
lora_alpha = lora_alpha,
@@ -4105,7 +4099,13 @@ class FastLlamaModel:
gc.collect()
clean_gpu_cache()
- model = _get_peft_model(model, lora_config)
+ from .lora_init import fast_lora_init, record_fast_pissa
+
+ with fast_lora_init() as fast:
+ model = _get_peft_model(model, lora_config)
+ if fast["pissa"]:
+ record_fast_pissa(model)
+ snapshot_residual_lora_init(model, init_lora_weights)
try:
from .vision import _lift_endpoint_hooks_onto_adapters
@@ -4279,6 +4279,7 @@ class FastLlamaModel:
use_gradient_checkpointing = use_gradient_checkpointing,
use_reentrant = True,
)
+ freeze_peft_variant_weights(model)
for active_adapter in model.peft_config.keys():
if False:
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 8a4c01fc8ec..a2c6301cb30 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -41,6 +41,7 @@ from .mistral_format import (
mistral_format_redirect,
prepare_mistral_format_checkpoint,
)
+from .lora_init import adapter_used_fast_pissa, fast_lora_init, record_fast_pissa
from .loader_utils import (
DEFAULT_DEVICE_MAP,
OFFLOAD_EMBEDDING_AUTO,
@@ -1620,16 +1621,27 @@ class FastLanguageModel(FastLlamaModel):
local_files_only = local_files_only,
cache_dir = kwargs.get("cache_dir"),
)
- model = PeftModel.from_pretrained(
- model,
+ # PEFT re-runs PiSSA at load: rebuild the residual with the algorithm that made the adapter.
+ fast_pissa = adapter_used_fast_pissa(
old_model_name,
token = token,
revision = revision,
local_files_only = local_files_only,
- is_trainable = True,
- trust_remote_code = trust_remote_code,
- **peft_load_kwargs,
+ cache_dir = kwargs.get("cache_dir"),
)
+ with fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext():
+ model = PeftModel.from_pretrained(
+ model,
+ old_model_name,
+ token = token,
+ revision = revision,
+ local_files_only = local_files_only,
+ is_trainable = True,
+ trust_remote_code = trust_remote_code,
+ **peft_load_kwargs,
+ )
+ if fast_pissa:
+ record_fast_pissa(model)
model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing)
try:
from .vision import _lift_endpoint_hooks_onto_adapters
@@ -2856,17 +2868,25 @@ class FastModel(FastBaseModel):
local_files_only = local_files_only,
cache_dir = kwargs.get("cache_dir"),
)
+ fast_pissa = adapter_used_fast_pissa(
+ old_model_name,
+ token = token,
+ revision = revision,
+ local_files_only = local_files_only,
+ cache_dir = kwargs.get("cache_dir"),
+ )
try:
- model = PeftModel.from_pretrained(
- model,
- old_model_name,
- token = token,
- revision = revision,
- local_files_only = local_files_only,
- is_trainable = True,
- trust_remote_code = trust_remote_code,
- **peft_load_kwargs,
- )
+ with fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext():
+ model = PeftModel.from_pretrained(
+ model,
+ old_model_name,
+ token = token,
+ revision = revision,
+ local_files_only = local_files_only,
+ is_trainable = True,
+ trust_remote_code = trust_remote_code,
+ **peft_load_kwargs,
+ )
finally:
# Always restore the original PEFT method, even if loading fails.
if _clippable_linear_cls is not None:
@@ -2875,6 +2895,8 @@ class FastModel(FastBaseModel):
model = FastBaseModel.post_patch_model(
model, use_gradient_checkpointing, trust_remote_code = trust_remote_code
)
+ if fast_pissa:
+ record_fast_pissa(model)
try:
from .vision import _lift_endpoint_hooks_onto_adapters
_lift_endpoint_hooks_onto_adapters(model)
diff --git a/unsloth/models/lora_init.py b/unsloth/models/lora_init.py
new file mode 100644
index 00000000000..08d072a90c0
--- /dev/null
+++ b/unsloth/models/lora_init.py
@@ -0,0 +1,451 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+import contextlib
+import functools
+import json
+import os
+import threading
+import torch
+
+__all__ = [
+ "randomized_svd",
+ "mica_basis",
+ "fast_lora_init",
+]
+
+_FLOATS = (torch.float32, torch.float16, torch.bfloat16)
+
+
+def _tf32(enabled):
+ flags = torch.backends.cuda.matmul
+ old = flags.allow_tf32
+ flags.allow_tf32 = enabled
+ try:
+ yield
+ finally:
+ flags.allow_tf32 = old
+
+
+_EPS32 = torch.finfo(torch.float32).eps / 2 # unit roundoff u
+
+
+def _orthonormalize_(Y, shift, failures, householder):
+ """Y <- Y R^-1 in place, fp32 shifted CholeskyQR (Fukaya et al. 2020); no host sync.
+
+ Columns are scaled to unit norm first (Gram diagonal = 1, no overflow). The shift is Rump-Ogita's
+ breakdown bound 2.2 (n + 1) u tr(G) plus a sqrt(m) u n allowance for the Gram's rounding error.
+ Breakdown flags are appended to `failures` and checked once per call; any breakdown reruns the
+ whole call with Householder QR, which cannot fail on finite input.
+ """
+ if householder:
+ Y.copy_(torch.linalg.qr(Y).Q)
+ return
+ m, n = Y.shape
+ tiny = torch.finfo(torch.float32).tiny
+ # max-abs first: vector_norm squares entries and overflows fp32 above ~1e19.
+ Y.div_(Y.abs().amax(0).clamp_min_(tiny))
+ Y.div_(torch.linalg.vector_norm(Y, dim = 0).clamp_min_(tiny))
+ with _tf32(False):
+ G = Y.mT @ Y
+ if shift:
+ G.diagonal().add_((2.2 * (n + 1) * n + 11.0 * (m**0.5 * n + n * (n + 1))) * _EPS32)
+ R, info = torch.linalg.cholesky_ex(G, upper = True)
+ failures.append(info)
+ torch.linalg.solve_triangular(R, Y, upper = True, left = False, out = Y)
+
+
[email protected]_cache(maxsize = 8)
+def _sketch(N, q):
+ # Seeded CPU draw: reloading re-runs this init and must rebuild the same residual on any device.
+ return torch.randn(N, q, dtype = torch.float32, generator = torch.Generator().manual_seed(3407))
+
+
+def randomized_svd(
+ W,
+ rank,
+ n_oversamples = None,
+ n_iter = 6,
+ final_passes = 1,
+ generator = None,
+ _safe = False,
+):
+ """Truncated W ~= U diag(S) Vh by randomized subspace iteration (Halko et al. 2011, Alg. 4.4 + 5.1).
+
+ fp32 throughout, one host sync per call. Oversampling defaults to max(rank, 10): LLM spectra decay
+ slowly, so a wider sketch buys more accuracy per ms than iterations. Shifted CholeskyQR replaces
+ cuSOLVER QR for sketches 32+ wide (geqrf + ormqr was no faster: ormqr applies the full m x m Q);
+ the final basis gets `final_passes` unshifted passes after the shifted one.
+ """
+ m, n = W.shape
+ A = W if m >= n else W.mT
+ A = A.float()
+ scale = None
+ if _safe:
+ # rocSOLVER's geqrf / gesvd square entries without LAPACK's scaling and overflow above ~1e19.
+ scale = A.abs().amax().clamp_min_(torch.finfo(torch.float32).tiny)
+ A = A / scale
+ M, N = A.shape
+ rank = min(rank, N)
+ q = min(rank + (max(rank, 10) if n_oversamples is None else n_oversamples), N)
+ if generator is None:
+ Z = _sketch(N, q).to(A.device, A.dtype, copy = True)
+ else:
+ Z = torch.randn(N, q, device = A.device, dtype = A.dtype, generator = generator)
+ failures = []
+ # Below q = 32 one Householder QR beats CholeskyQR's ~8 launches.
+ householder = _safe or q < 32
+
+ def _power(k):
+ # Both half steps: skipping one squares cond(Y) (~2e3 on real r = 128 weights), past fp32.
+ for _ in range(k):
+ _orthonormalize_(Y, True, failures, householder)
+ torch.matmul(A.mT, Y, out = Z)
+ _orthonormalize_(Z, True, failures, householder)
+ torch.matmul(A, Z, out = Y)
+
+ with _tf32(A.is_cuda and n_iter > 1):
+ Y = torch.matmul(A, Z)
+ _power(n_iter - 1)
+ # Last iteration in fp32 removes the TF32 error when sigma_1 / sigma_rank is large.
+ _power(min(n_iter, 1))
+ _orthonormalize_(Y, True, failures, householder)
+ if not householder:
+ for _ in range(final_passes):
+ _orthonormalize_(Y, False, failures, householder)
+ with _tf32(False):
+ torch.matmul(A.mT, Y, out = Z) # Z = (Q^T A)^T, N x q
+ # From q = 128, eigh of [[0, R2], [R2^T, 0]] (eigenvalues +-sigma) beats cuSOLVER's fp32 SVD (~60x accuracy).
+ # cuSOLVER syevd / rocSOLVER gesvd refuse degenerate R2 (rank 0 / 1): the safe rerun uses LAPACK.
+ Q2, R2 = torch.linalg.qr(Z)
+ try:
+ if q < 128 or _safe:
+ Ur, S, Vrh = torch.linalg.svd(R2.cpu() if _safe else R2)
+ Ur, S, Vr = (x.to(R2.device) for x in (Ur[:, :rank], S[:rank], Vrh[:rank].mT))
+ else:
+ J = R2.new_zeros(2 * q, 2 * q)
+ J[:q, q:] = R2
+ J[q:, :q] = R2.mT
+ L, X = torch.linalg.eigh(J)
+ X = X[:, -rank:].flip(1)
+ S = L[-rank:].flip(0).clamp_min_(0)
+ tiny = torch.finfo(torch.float32).tiny
+ Ur = X[:q].div_(torch.linalg.vector_norm(X[:q], dim = 0).clamp_min_(tiny))
+ Vr = X[q:].div_(torch.linalg.vector_norm(X[q:], dim = 0).clamp_min_(tiny))
+ except torch.linalg.LinAlgError:
+ if _safe:
+ raise
+ return randomized_svd(W, rank, n_oversamples, n_iter, final_passes, generator, _safe = True)
+ # A^T ~= Z Y^T = Q2 Ur S Vr^T Y^T, so A ~= (Y Vr) S (Q2 Ur)^T.
+ V = Q2 @ Ur
+ U = Y @ Vr
+ if scale is not None:
+ S = S * scale
+ # cuSOLVER's Householder QR cannot fail on finite input; ROCm's can overflow, and CPU syncs for free.
+ if not _safe and (not householder or A.device.type != "cuda" or torch.version.hip is not None):
+ bad = torch.stack(failures).ne(0).any() if failures else S.new_zeros((), dtype = torch.bool)
+ bad = bad | ~torch.isfinite(S).all() | ~torch.isfinite(U).all() | ~torch.isfinite(V).all()
+ if bad.item():
+ del A, Y, Z, U, V
+ return randomized_svd(
+ W, rank, n_oversamples, n_iter, final_passes, generator, _safe = True
+ )
+ if m < n:
+ return V, S, U.mT
+ return U, S, V.mT
+
+
+def _fast_fp64(device):
+ if device.type == "cpu":
+ return True
+ if device.type != "cuda":
+ return False
+ if getattr(torch.version, "hip", None):
+ return "gfx9" in getattr(torch.cuda.get_device_properties(device), "gcnArchName", "")
+ # Full-rate fp64: P100, V100, A100, H100, B200. B300 (10.3) and consumer / L4 / T4 parts run fp64 at 1/64.
+ return torch.cuda.get_device_capability(device) in ((6, 0), (7, 0), (8, 0), (9, 0), (10, 0))
+
+
+def mica_basis(W, r):
+ """U[:, -r:] of the reduced SVD of W (out x in) as fp32, via fp64 eigh of the smaller Gram matrix.
+
+ fp64 Gram squaring costs ~1e-16 * cond^2, below fp32 SVD's own error on LLM weights. lobpcg does not
+ converge here: the relative gaps between the smallest singular values are ~1e-5..1e-3.
+ """
+ out_features, in_features = W.shape
+ A = W.to(torch.float64)
+ wide = out_features <= in_features
+ L, V = torch.linalg.eigh(A @ A.T if wide else A.T @ A)
+ # L ascends: L[0] is the smallest divisor below.
+ if L[0] <= 1e-12 * L[-1]:
+ U = torch.linalg.svd(A, full_matrices = False)[0]
+ return U[:, -r:].float().contiguous()
+ V = V[:, :r].flip(1)
+ if wide:
+ return V.float().contiguous()
+ U = A @ V
+ U /= L[:r].flip(0).sqrt()
+ return U.float().contiguous()
+
+
+def _routed(base):
+ # Routed compressed-tensors linears: loader_utils' wrapper (the saved original) densifies them first.
+ return type(base).__name__ == "_UnslothNVFP4Linear" or getattr(
+ base, "_unsloth_compressed_tensors_fp8", False
+ )
+
+
+def _check_float_weight(weight, init):
+ # FSDP-QLoRA packs Params4bit into a float quant_storage: its dtype alone looks dense.
+ if type(weight).__name__ in ("Params4bit", "Int8Params") or hasattr(weight, "quant_state"):
+ raise TypeError(
+ f"Unsloth: `init_lora_weights = {init!r}` re-runs on the base weights when an adapter is created "
+ "or loaded, and they are quantized here. Load the model with `load_in_4bit = False` and "
+ "`load_in_8bit = False`."
+ )
+
+
+def _pissa_init(self, adapter_name, init_lora_weights):
+ from peft.tuners.lora.layer import transpose
+
+ if _OWNER.get("thread") != threading.get_ident():
+ return _ORIGINAL_ANY["pissa_init"](self, adapter_name, init_lora_weights)
+ if _routed(self.get_base_layer()):
+ return _ORIGINAL["pissa_init"](self, adapter_name, init_lora_weights)
+ weight = self.get_base_layer().weight
+ _check_float_weight(weight, init_lora_weights)
+ dtype = weight.dtype
+ r = self.r[adapter_name]
+ if (
+ dtype not in _FLOATS
+ or r > min(weight.shape)
+ or weight.device.type == "meta"
+ or not torch.isfinite(weight).all()
+ ):
+ return _ORIGINAL["pissa_init"](self, adapter_name, init_lora_weights)
+ if init_lora_weights == "pissa":
+ n_iter, n_oversamples = 6, None
+ else:
+ parts = init_lora_weights.split("_niter_")
+ if len(parts) != 2:
+ return _ORIGINAL["pissa_init"](self, adapter_name, init_lora_weights)
+ # PEFT's svd_lowrank(q = r, niter = N), device-independent so reloads rebuild the same residual.
+ n_iter, n_oversamples = int(parts[-1]), 0
+ _STATE["pissa"] = True
+ self._unsloth_fast_pissa = getattr(self, "_unsloth_fast_pissa", set()) | {adapter_name}
+ W = transpose(weight.to(torch.float32), self.fan_in_fan_out)
+ U, S, Vh = randomized_svd(W, r, n_oversamples = n_oversamples, n_iter = n_iter)
+ scaling = self.scaling[adapter_name]
+ S.div_(scaling).sqrt_()
+ lora_A = Vh.mul_(S.unsqueeze(1)).contiguous()
+ lora_B = U.mul_(S).contiguous()
+ self.lora_A[adapter_name].weight.data = lora_A
+ self.lora_B[adapter_name].weight.data = lora_B
+ W = W if W.data_ptr() != weight.data_ptr() else W.clone()
+ W.addmm_(lora_B, lora_A, alpha = -scaling)
+ self.get_base_layer().weight.data = transpose(W.to(dtype), self.fan_in_fan_out)
+
+
+def _mica_init(self, adapter_name):
+ from peft.tuners.lora.layer import transpose
+
+ if _OWNER.get("thread") != threading.get_ident():
+ return _ORIGINAL_ANY["mica_init"](self, adapter_name)
+ if _routed(self.get_base_layer()):
+ return _ORIGINAL["mica_init"](self, adapter_name)
+ weight = self.get_base_layer().weight
+ _check_float_weight(weight, "mica")
+ r = self.r[adapter_name]
+ if (
+ self.lora_B[adapter_name].weight.device.type == "meta"
+ or weight.dtype not in _FLOATS
+ or r > min(weight.shape)
+ or not _fast_fp64(weight.device)
+ ):
+ return _ORIGINAL["mica_init"](self, adapter_name)
+ dtype = weight.dtype
+ W = transpose(weight, self.fan_in_fan_out)
+ self.lora_B[adapter_name].weight.data = mica_basis(W, r).to(dtype)
+ self.lora_A[adapter_name].weight.data = torch.zeros(
+ r, W.shape[1], device = weight.device, dtype = dtype
+ )
+
+
+_ORIGINAL = {}
+# PEFT's own methods, kept past the swap: another thread calling PEFT directly must not get ours.
+_ORIGINAL_ANY = {}
+_OWNER = {}
+_STATE = {"pissa": False}
+# One owner at a time: the swap is process-wide, so a second thread must not build layers mid-restore.
+_LOCK = threading.RLock()
+SIDECAR = "unsloth_lora_init.json"
+
+
+def fast_lora_init(force = False):
+ """Swap PEFT's LoraLayer.pissa_init / mica_init for the fast versions during get_peft_model.
+
+ Yields a dict whose "pissa" is True once a layer took the fast PiSSA path. UNSLOTH_FAST_LORA_INIT=0
+ keeps PEFT's own SVD unless `force` (reloading an adapter that recorded the fast path).
+ """
+ global _STATE
+ with _LOCK:
+ if _ORIGINAL:
+ # Nested use keeps the outer swap.
+ yield _STATE
+ return
+ if not force and os.environ.get("UNSLOTH_FAST_LORA_INIT", "1") == "0":
+ yield {"pissa": False}
+ return
+ from peft.tuners.lora.layer import LoraLayer
+
+ swapped = []
+ for name, fn in (("pissa_init", _pissa_init), ("mica_init", _mica_init)):
+ original = LoraLayer.__dict__.get(name)
+ if original is None:
+ continue
+ _ORIGINAL[name] = _ORIGINAL_ANY[name] = original
+ setattr(LoraLayer, name, fn)
+ swapped.append((name, original))
+ # A fresh dict per swap: callers read it after the lock is released.
+ _STATE = {"pissa": False}
+ _OWNER["thread"] = threading.get_ident()
+ try:
+ yield _STATE
+ finally:
+ _OWNER.clear()
+ for name, original in swapped:
+ setattr(LoraLayer, name, original)
+ _ORIGINAL.clear()
+
+
+def record_fast_pissa(model):
+ """PEFT re-runs PiSSA when loading an adapter, and only the same algorithm rebuilds the residual base
+ training saw: saves of this model mark their adapter folders so Unsloth's loaders can tell."""
+ original = model.save_pretrained
+ if getattr(original, "_unsloth_fast_pissa", False):
+ return
+
+ @functools.wraps(original)
+ def save_pretrained(save_directory, *args, **kwargs):
+ out = original(save_directory, *args, **kwargs)
+ if kwargs.get("is_main_process", True):
+ # Only fast-path adapters; PEFT saves "default" at the root, others in a subfolder.
+ fast = set()
+ for module in model.modules():
+ fast |= getattr(module, "_unsloth_fast_pissa", set())
+ for name in fast:
+ folder = save_directory if name == "default" else os.path.join(save_directory, name)
+ weights = _adapter_weights(folder)
+ if weights is not None:
+ # Digest: a later ordinary save into this folder leaves the marker but not a match.
+ with open(os.path.join(folder, SIDECAR), "w", encoding = "utf-8") as f:
+ json.dump(
+ {"pissa": "unsloth_randomized_svd", "sha256": _sha256(weights)}, f
+ )
+ return out
+
+ save_pretrained._unsloth_fast_pissa = True
+ model.save_pretrained = save_pretrained
+
+
+_WEIGHT_FILES = ("adapter_model.safetensors", "adapter_model.bin")
+
+
+def _adapter_weights(folder):
+ for name in _WEIGHT_FILES:
+ if os.path.isfile(os.path.join(folder, name)):
+ return os.path.join(folder, name)
+ return None
+
+
+def _sha256(path):
+ import hashlib
+
+ digest = hashlib.sha256()
+ with open(path, "rb") as f:
+ for chunk in iter(lambda: f.read(1 << 20), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _marker_matches(sidecar, weights):
+ with open(sidecar, encoding = "utf-8") as f:
+ expected = json.load(f).get("sha256")
+ return weights is not None and expected == _sha256(weights)
+
+
+def adapter_used_fast_pissa(path, **hub_kwargs):
+ if os.path.isdir(path):
+ sidecar = os.path.join(path, SIDECAR)
+ return os.path.isfile(sidecar) and _marker_matches(sidecar, _adapter_weights(path))
+ from huggingface_hub import constants, hf_hub_download
+ from huggingface_hub.utils import EntryNotFoundError, HFValidationError, LocalEntryNotFoundError
+
+ hub_kwargs = {k: v for k, v in hub_kwargs.items() if v is not None}
+ try:
+ sidecar = hf_hub_download(path, SIDECAR, **hub_kwargs)
+ except LocalEntryNotFoundError:
+ # Offline: uncached = absent (the prefetch fetches it). Online: unreachable Hub, never guess.
+ if hub_kwargs.get("local_files_only") or constants.HF_HUB_OFFLINE:
+ return False
+ raise
+ except (EntryNotFoundError, HFValidationError):
+ return False
+ for name in _WEIGHT_FILES:
+ try:
+ return _marker_matches(sidecar, hf_hub_download(path, name, **hub_kwargs))
+ except EntryNotFoundError:
+ continue
+ return False
+
+
+# Compiled modules skip hooks added later (skip_nnmodule_hook_guards): CorDA then divides by zero.
+def _calibration_functions():
+ found = []
+ try:
+ from peft.tuners.lora import corda
+ found.append((corda, "preprocess_corda"))
+ except Exception:
+ pass
+ try:
+ from peft.tuners.lora import eva
+ found.append((eva, "initialize_lora_eva_weights"))
+ except Exception:
+ pass
+ try:
+ from peft.tuners.lora import loraga
+ found.append((loraga, "preprocess_loraga"))
+ except Exception:
+ pass
+ return found
+
+
+def patch_peft_calibration_eager():
+ import functools
+ import sys
+
+ if not hasattr(torch.compiler, "set_stance"):
+ return
+ for module, name in _calibration_functions():
+ original = getattr(module, name, None)
+ if original is None or getattr(original, "_unsloth_eager", False):
+ continue
+
+ @functools.wraps(original)
+ def eager(
+ *args,
+ __original = original,
+ **kwargs,
+ ):
+ with torch.compiler.set_stance("force_eager"):
+ return __original(*args, **kwargs)
+
+ eager._unsloth_eager = True
+ # vars(), not getattr: getattr on lazy modules (transformers) imports submodules.
+ for loaded in list(sys.modules.values()):
+ if getattr(loaded, "__dict__", {}).get(name) is original:
+ setattr(loaded, name, eager)
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 2f3082321c9..ca4fa1495c3 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -4342,7 +4342,7 @@ class FastBaseModel:
max_seq_length = model.max_seq_length
# Passing loftq_config = None gives an error.
loftq_config = validate_loftq_config(
- loftq_config, lora_dropout, bias, init_lora_weights, model
+ loftq_config, lora_dropout, bias, init_lora_weights, model, r
)
# Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an attention-only request does not train experts, but only while MLP and language families are both in scope: with finetune_mlp_modules or finetune_language_layers False the scoped regex already dropped the experts.
@@ -4433,6 +4433,8 @@ class FastBaseModel:
n = max(1, min(int(finetune_last_n_layers), _total_layers))
layers_to_transform = list(range(_total_layers - n, _total_layers))
+ validate_init_target_parameters(init_lora_weights, target_parameters)
+
local_variables = {
**locals(),
**kwargs,
@@ -4506,7 +4508,13 @@ class FastBaseModel:
_LoraModel._create_and_replace = _patched_car
- model = _get_peft_model(model, lora_config)
+ from .lora_init import fast_lora_init, record_fast_pissa
+
+ with fast_lora_init() as fast:
+ model = _get_peft_model(model, lora_config)
+ if fast["pissa"]:
+ record_fast_pissa(model)
+ snapshot_residual_lora_init(model, init_lora_weights)
# PEFT may have wrapped an endpoint this load repaired; the hook stays on base_layer and the adapter branch reads the caller's tensor.
try:
@@ -4632,6 +4640,7 @@ class FastBaseModel:
float32_mixed_precision = float32_mixed_precision,
patch_modules_to_save = True,
)
+ freeze_peft_variant_weights(model)
if full_finetuning:
# prepare_model_for_training re-enabled every parameter, a kept wrapper's siblings too.
_freeze_unused_siblings(model)
diff --git a/unsloth/save.py b/unsloth/save.py
index 4e656255344..cfde2aec85e 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -13,7 +13,7 @@
from unsloth_zoo.utils import Version
from importlib.metadata import version as importlib_version
from unsloth_zoo.hf_utils import dtype_from_config, HAS_TORCH_DTYPE
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
from unsloth_zoo.llama_cpp import (
convert_to_gguf,
quantize_gguf,
@@ -67,7 +67,7 @@ from .models.loader_utils import (
_tokenizer_revision,
_tokenizer_wants_local_only,
)
-from .models._utils import _convert_torchao_model
+from .models._utils import _convert_torchao_model, lora_relative_to_original_base
from .models.mistral_format import raise_if_merging_mistral_format_view
from .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPER
from .device_type import clean_gpu_cache
@@ -5702,19 +5702,22 @@ def unsloth_generic_save(
_prewarm_base_model_hub_cache(model, save_method = save_method, token = token)
from unsloth_zoo.saving_utils import merge_and_overwrite_lora
- merge_and_overwrite_lora(
- get_model_name,
- model = model,
- tokenizer = tokenizer,
- save_directory = save_directory,
- push_to_hub = push_to_hub,
- private = private,
- token = token,
- save_method = save_method,
- output_dtype = None,
- low_disk_space_usage = True,
- use_temp_file = False,
- )
+ # merged_4bit merges into the loaded (already residual) weights, so it needs no conversion.
+ in_place = save_method in ("merged_4bit", "forced_merged_4bit")
+ with nullcontext() if in_place else lora_relative_to_original_base(model):
+ merge_and_overwrite_lora(
+ get_model_name,
+ model = model,
+ tokenizer = tokenizer,
+ save_directory = save_directory,
+ push_to_hub = push_to_hub,
+ private = private,
+ token = token,
+ save_method = save_method,
+ output_dtype = None,
+ low_disk_space_usage = True,
+ use_temp_file = False,
+ )
if push_to_hub and datasets:
try:
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 3b87034cddf..4fdccd2979a 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -452,7 +452,10 @@ def run_attention(
q_len = context.q_len
head_dim = context.head_dim
kv_seq_len = context.kv_seq_len
- requires_grad = context.requires_grad
+ # Eval forwards can backpropagate (LoRA-GA calibration); xformers' inference GQA layout cannot.
+ requires_grad = context.requires_grad or (
+ torch.is_grad_enabled() and (Q.requires_grad or K.requires_grad or V.requires_grad)
+ )
sliding_window = context.sliding_window
# A non-positive window means "no local attention", not "a window of nothing": a config spelling
# it 0 would put the mask's lower bound above its causal upper bound.