Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Support every PEFT init_lora_weights option, with fast PiSSA and MiCA init (#6879)
Suchitra Malimbadac17ca646c7cc
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.pyindex 2b1c75aa352..da1aadfb79d 100644--- a/tests/saving/test_save_method_lora_routing.py+++ b/tests/saving/test_save_method_lora_routing.py@@ -27,6 +27,7 @@ This file therefore runs on Linux, macOS and Windows with no accelerator and nofrom __future__ import annotationsimport ast+import contextlibimport sysimport typesfrom pathlib import Path@@ -56,7 +57,12 @@ def _load(*names, **env):"""Exec the named top-level functions against `env` and return the namespace."""# save.py imports this from models.mistral_format (#12144) and calls it on every merge and# GGUF path; none of these fixtures is a Mistral-format view, so it never refuses here.- namespace = {"raise_if_merging_mistral_format_view": lambda model, save_method: None, **env}+ namespace = {+ "raise_if_merging_mistral_format_view": lambda model, save_method: None,+ "lora_relative_to_original_base": lambda model: contextlib.nullcontext(),+ "nullcontext": contextlib.nullcontext,+ **env,+ }for name in names:exec(compile(_function_source(name), str(_SAVE_PY), "exec"), namespace)return namespace@@ -239,7 +245,7 @@ def _not_reached(*args, **kwargs):def _routing_environment(monkeypatch, model):- calls = {"merge": [], "adapter": [], "prewarm": []}+ calls = {"merge": [], "adapter": [], "prewarm": [], "convert": []}zoo = types.ModuleType("unsloth_zoo.saving_utils")zoo.merge_and_overwrite_lora = lambda *args, **kwargs: calls["merge"].append(kwargs)@@ -260,6 +266,9 @@ def _routing_environment(monkeypatch, model):_qwen3_5_vlm_state_dict_for_save = _not_reached,_determine_username = lambda repo, old, token: (repo, "owner"),unsloth_save_model = lambda *args, **kwargs: calls["adapter"].append(kwargs),+ lora_relative_to_original_base = lambda model: (+ calls["convert"].append(model) or contextlib.nullcontext()+ ),logger = types.SimpleNamespace(warning_once = lambda *a, **k: None),gc = types.SimpleNamespace(collect = lambda: None),torch = types.SimpleNamespace(@@ -348,6 +357,7 @@ def test_every_other_method_still_merges(monkeypatch, tmp_path, save_method):assert calls["adapter"] == []assert len(calls["merge"]) == 1assert len(calls["prewarm"]) == 1+ assert len(calls["convert"]) == (save_method != "merged_4bit_forced")def test_a_model_with_no_adapter_is_unchanged(monkeypatch, tmp_path):diff --git a/tests/test_lora_init.py b/tests/test_lora_init.pynew file mode 100644index 00000000000..be502cb2482--- /dev/null+++ b/tests/test_lora_init.py@@ -0,0 +1,395 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++import copy+import importlib.util+from pathlib import Path++import pytest+import torch++peft = pytest.importorskip("peft")+from peft import LoraConfig, get_peft_model+from peft.tuners.lora.layer import LoraLayer++_spec = importlib.util.spec_from_file_location(+ "unsloth_lora_init", Path(__file__).resolve().parents[1] / "unsloth" / "models" / "lora_init.py"+)+lora_init = importlib.util.module_from_spec(_spec)+_spec.loader.exec_module(lora_init)+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"+++def _weight(+ out_features,+ in_features,+ seed = 0,+):+ g = torch.Generator().manual_seed(seed)+ k = min(out_features, in_features)+ U = torch.linalg.qr(torch.randn(out_features, k, generator = g, dtype = torch.float64)).Q+ V = torch.linalg.qr(torch.randn(in_features, k, generator = g, dtype = torch.float64)).Q+ S = torch.logspace(0, -2, k, dtype = torch.float64) + 0.01 * torch.rand(+ k, generator = g, dtype = torch.float64+ )+ return (U * S) @ V.T++[email protected]("shape", [(96, 48), (48, 96), (64, 64)])[email protected]("rank", [1, 8])+def test_randomized_svd_near_optimal(shape, rank):+ W = _weight(*shape)+ U, S, Vh = lora_init.randomized_svd(W.float().to(DEVICE), rank)+ Ue, Se, Vhe = torch.linalg.svd(W, full_matrices = False)+ best = (W - (Ue[:, :rank] * Se[:rank]) @ Vhe[:rank]).norm()+ err = (W - (U.double().cpu() * S.double().cpu()) @ Vh.double().cpu()).norm()+ assert err / best < 1.001+ assert torch.allclose(S.double().cpu(), Se[:rank], rtol = 1e-3)+ assert U.dtype == torch.float32 and U.shape == (shape[0], rank) and Vh.shape == (rank, shape[1])+++def _adversarial():+ g = torch.Generator().manual_seed(1)+ return {+ "zero": torch.zeros(64, 40),+ "rank1": torch.randn(64, 1, generator = g) @ torch.randn(1, 40, generator = g),+ "duplicate_columns": torch.randn(64, 4, generator = g).repeat(1, 10),+ # Column norms above ~1e19 overflow a plain fp32 vector_norm.+ "column_scales": torch.randn(64, 40, generator = g) * torch.logspace(-20, 20, 40),+ "row_scales": torch.randn(64, 40, generator = g) * torch.logspace(-20, 20, 64)[:, None],+ "one_huge_entry": torch.randn(64, 40, generator = g).index_put_(+ (torch.tensor([3]), torch.tensor([7])), torch.tensor(1e30)+ ),+ }++[email protected]("name", list(_adversarial()))[email protected]("rank", [4, 16])+def test_randomized_svd_adversarial_finite_and_near_optimal(name, rank):+ W = _adversarial()[name]+ U, S, Vh = lora_init.randomized_svd(W.to(DEVICE), rank)+ assert torch.isfinite(U).all() and torch.isfinite(S).all() and torch.isfinite(Vh).all()+ Wd = W.double()+ S0 = torch.linalg.svdvals(Wd)+ err = (Wd - (U.double().cpu() * S.double().cpu()) @ Vh.double().cpu()).norm()+ best = (S0[rank:] ** 2).sum().sqrt()+ assert err <= best + 1e-4 * Wd.norm() + 1e-30+ assert (S.double().cpu() - S0[:rank]).abs().max() <= 1e-4 * S0[0] + 1e-30++[email protected]("true_rank", [0, 1, 8])+def test_randomized_svd_rank_deficient_wide_sketch(true_rank):+ # q = 128 takes the eigh path, which cuSOLVER refuses on the NaNs a CholeskyQR breakdown leaves.+ g = torch.Generator().manual_seed(2)+ W = torch.randn(512, true_rank, generator = g) @ torch.randn(true_rank, 384, generator = g)+ U, S, Vh = lora_init.randomized_svd(W.to(DEVICE), 64)+ assert torch.isfinite(U).all() and torch.isfinite(S).all() and torch.isfinite(Vh).all()+ err = (W.double() - (U.double().cpu() * S.double().cpu()) @ Vh.double().cpu()).norm()+ assert err <= 1e-5 * W.norm() + 1e-30++[email protected]("name", ["column_scales", "row_scales", "one_huge_entry"])[email protected]("rank", [4, 16])+def test_randomized_svd_survives_unscaled_solver_norms(monkeypatch, name, rank):+ # rocSOLVER's geqrf / gesvd return NaN where a squared column norm overflows fp32 (gfx1151).+ qr, svd = torch.linalg.qr, torch.linalg.svd++ def overflows(X):+ return X.dtype == torch.float32 and not torch.isfinite(X.pow(2).sum(-2)).all()++ def unscaled_qr(X, *args, **kwargs):+ out = qr(X, *args, **kwargs)+ return torch.return_types.linalg_qr((out.Q * float("nan"), out.R)) if overflows(X) else out++ def unscaled_svd(X, *args, **kwargs):+ return svd(X * float("nan") if overflows(X) else X, *args, **kwargs)++ monkeypatch.setattr(torch.linalg, "qr", unscaled_qr)+ monkeypatch.setattr(torch.linalg, "svd", unscaled_svd)+ monkeypatch.setattr(torch.version, "hip", "7.0", raising = False)+ test_randomized_svd_adversarial_finite_and_near_optimal(name, rank)+++def test_randomized_svd_is_fp32_only(monkeypatch):+ # Nothing may run in float64: consumer GPUs run it at 1/64 rate.+ calls = []+ original = torch.Tensor.double+ monkeypatch.setattr(torch.Tensor, "double", lambda self: calls.append(1) or original(self))+ U, S, Vh = lora_init.randomized_svd(_weight(96, 48).float().to(DEVICE), 8)+ assert not calls and U.dtype == S.dtype == Vh.dtype == torch.float32++[email protected]("shape", [(96, 48), (48, 96), (64, 64)])+def test_mica_basis_matches_fp64_svd(shape):+ W = _weight(*shape)+ r = 8+ B = lora_init.mica_basis(W.float(), r)+ U = torch.linalg.svd(W, full_matrices = False)[0][:, -r:]+ assert torch.linalg.matrix_norm(U @ U.T - B.double() @ B.double().T, ord = 2) < 1e-4+ assert torch.nn.functional.cosine_similarity(B.double(), U, dim = 0).abs().min() > 0.999++[email protected]("shape", [(96, 48), (48, 96)])+def test_mica_basis_rank_deficient_is_finite(shape):+ # One dead input / output dimension: a single zero eigenvalue, below the r smallest's largest.+ W = _weight(*shape)+ W[:, 0] = 0+ W[0] = 0+ B = lora_init.mica_basis(W.float(), 8).double()+ assert torch.isfinite(B).all()+ torch.testing.assert_close(B.T @ B, torch.eye(8, dtype = B.dtype), atol = 1e-4, rtol = 0)+++class _Model(torch.nn.Module):+ def __init__(self, W):+ super().__init__()+ self.q = torch.nn.Linear(W.shape[1], W.shape[0], bias = False)+ self.q.weight.data = W.float()++ def forward(self, x):+ return self.q(x)++[email protected]("init", ["pissa", "pissa_niter_4"])+def test_fast_pissa_preserves_output(init):+ W = _weight(96, 64)+ base = _Model(W).to(DEVICE)+ x = torch.randn(3, 64, device = DEVICE)+ with lora_init.fast_lora_init():+ assert LoraLayer.pissa_init is lora_init._pissa_init+ model = get_peft_model(+ copy.deepcopy(base),+ LoraConfig(r = 8, lora_alpha = 16, target_modules = ["q"], init_lora_weights = init),+ )+ assert LoraLayer.pissa_init is not lora_init._pissa_init+ assert torch.allclose(model(x), base(x), atol = 1e-5)+ layer = model.base_model.model.q+ BA = (+ layer.scaling["default"]+ * layer.lora_B["default"].weight.double().cpu()+ @ layer.lora_A["default"].weight.double().cpu()+ )+ Ue, Se, Vhe = torch.linalg.svd(W, full_matrices = False)+ # "pissa" stands in for an exact SVD; pissa_niter_N keeps PEFT's svd_lowrank accuracy class.+ bound = 1.001 if init == "pissa" else 1.05+ assert (W - BA).norm() / (W - (Ue[:, :8] * Se[:8]) @ Vhe[:8]).norm() < bound+++def test_kill_switch(monkeypatch):+ monkeypatch.setenv("UNSLOTH_FAST_LORA_INIT", "0")+ original = LoraLayer.pissa_init+ # An earlier fast run must not make a kill-switched one report the fast path.+ monkeypatch.setitem(lora_init._STATE, "pissa", True)+ with lora_init.fast_lora_init() as fast:+ assert LoraLayer.pissa_init is original+ assert not fast["pissa"]+++def test_fast_pissa_marker_lookup_fails_closed_only_when_online(monkeypatch, tmp_path):+ import huggingface_hub+ from huggingface_hub import constants+ from huggingface_hub.utils import EntryNotFoundError, LocalEntryNotFoundError++ def raising(error):+ def download(*args, **kwargs):+ raise error++ return download++ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", False)+ _save_fast_pissa(tmp_path)+ monkeypatch.setattr(+ huggingface_hub, "hf_hub_download", lambda repo, name, **k: str(tmp_path / name)+ )+ assert lora_init.adapter_used_fast_pissa("user/adapter")+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", raising(EntryNotFoundError("404")))+ assert not lora_init.adapter_used_fast_pissa("user/adapter")+ monkeypatch.setattr(+ huggingface_hub, "hf_hub_download", raising(LocalEntryNotFoundError("down"))+ )+ with pytest.raises(LocalEntryNotFoundError):+ lora_init.adapter_used_fast_pissa("user/adapter")+ assert not lora_init.adapter_used_fast_pissa("user/adapter", local_files_only = True)+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True)+ assert not lora_init.adapter_used_fast_pissa("user/adapter")+++def test_calibration_patch_does_not_trigger_lazy_module_getattr(monkeypatch):+ # transformers' lazy module imports submodules (torchvision for aria) on any getattr.+ import sys+ import types++ asked = []+ lazy = types.ModuleType("unsloth_test_lazy_module")++ def __getattr__(name):+ asked.append(name)+ raise AttributeError(name)++ lazy.__getattr__ = __getattr__+ monkeypatch.setitem(sys.modules, lazy.__name__, lazy)+ names = [name for _, name in lora_init._calibration_functions()]+ for module, name in lora_init._calibration_functions():+ fn = getattr(module, name)+ monkeypatch.setattr(module, name, getattr(fn, "__wrapped__", fn))+ lora_init.patch_peft_calibration_eager()+ assert names and not set(asked) & set(names)+++def test_fast_init_kill_switch_and_force(monkeypatch):+ monkeypatch.setenv("UNSLOTH_FAST_LORA_INIT", "0")+ original = LoraLayer.pissa_init+ with lora_init.fast_lora_init():+ assert LoraLayer.pissa_init is original+ with lora_init.fast_lora_init(force = True):+ assert LoraLayer.pissa_init is lora_init._pissa_init+ with lora_init.fast_lora_init():+ assert LoraLayer.pissa_init is lora_init._pissa_init+ assert LoraLayer.pissa_init is original+++def _save_fast_pissa(folder):+ from peft import LoraConfig, get_peft_model++ base = torch.nn.Sequential(torch.nn.Linear(64, 48, bias = False))+ with lora_init.fast_lora_init():+ model = get_peft_model(+ base, LoraConfig(r = 4, target_modules = ["0"], init_lora_weights = "pissa")+ )+ lora_init.record_fast_pissa(model)+ model.save_pretrained(str(folder))+ return model+++def test_a_stale_marker_does_not_follow_a_later_ordinary_save(tmp_path):+ from peft import LoraConfig, get_peft_model++ _save_fast_pissa(tmp_path)+ assert lora_init.adapter_used_fast_pissa(str(tmp_path))+ base = torch.nn.Sequential(torch.nn.Linear(64, 48, bias = False))+ get_peft_model(base, LoraConfig(r = 4, target_modules = ["0"])).save_pretrained(str(tmp_path))+ assert (tmp_path / lora_init.SIDECAR).is_file()+ assert not lora_init.adapter_used_fast_pissa(str(tmp_path))+++def test_fast_pissa_saves_record_their_algorithm(tmp_path):+ from peft import LoraConfig, get_peft_model++ def config():+ return LoraConfig(r = 4, target_modules = ["0"], init_lora_weights = "pissa")++ base = torch.nn.Sequential(torch.nn.Linear(64, 48, bias = False))+ assert not lora_init.adapter_used_fast_pissa(str(tmp_path))+ with lora_init.fast_lora_init() as fast:+ model = get_peft_model(base, config())+ assert fast["pissa"]+ lora_init.record_fast_pissa(model)+ lora_init.record_fast_pissa(model)+ # A second PiSSA adapter through plain PEFT must keep PEFT's initializer on reload.+ model.add_adapter("plain", config())+ model.save_pretrained(str(tmp_path))+ assert (tmp_path / "plain" / "adapter_config.json").is_file()+ assert lora_init.adapter_used_fast_pissa(str(tmp_path))+ assert not lora_init.adapter_used_fast_pissa(str(tmp_path / "plain"))+++def test_sketch_ignores_default_dtype():+ previous = torch.get_default_dtype()+ lora_init._sketch.cache_clear()+ try:+ torch.set_default_dtype(torch.float64)+ U, S, Vh = lora_init.randomized_svd(_weight(96, 48).float(), 8)+ finally:+ torch.set_default_dtype(previous)+ lora_init._sketch.cache_clear()+ assert U.dtype == S.dtype == Vh.dtype == torch.float32+++def test_other_threads_keep_peft_init(monkeypatch):+ import threading++ calls = []+ with lora_init.fast_lora_init(force = True):+ monkeypatch.setitem(lora_init._ORIGINAL_ANY, "pissa_init", lambda *args: calls.append(args))+ thread = threading.Thread(target = lora_init._pissa_init, args = (None, "a", "pissa"))+ thread.start()+ thread.join()+ assert calls == [(None, "a", "pissa")]+++def test_packed_quantized_base_is_refused():+ layer = torch.nn.Linear(16, 16)+ Params4bit = type("Params4bit", (torch.nn.Parameter,), {})+ layer.weight = Params4bit(layer.weight.data.to(torch.bfloat16), requires_grad = False)+ with pytest.raises(TypeError, match = "load_in_4bit = False"):+ lora_init._check_float_weight(layer.weight, "pissa")+++def test_pissa_niter_width_is_device_independent(monkeypatch):+ # Loading re-runs the init, possibly on another device: the sketch width must not depend on it.+ seen = []+ real = lora_init.randomized_svd++ def spy(W, rank, **kwargs):+ seen.append(kwargs["n_oversamples"])+ return real(W, rank, **kwargs)++ monkeypatch.setattr(lora_init, "randomized_svd", spy)+ model = torch.nn.Sequential(torch.nn.Linear(160, 160))+ with lora_init.fast_lora_init(force = True):+ get_peft_model(+ model, LoraConfig(r = 64, target_modules = ["0"], init_lora_weights = "pissa_niter_4")+ )+ assert seen == [0]+++def test_swap_state_is_per_call():+ with lora_init.fast_lora_init(force = True) as first:+ first["pissa"] = True+ with lora_init.fast_lora_init(force = True) as second:+ assert not second["pissa"]+ assert first["pissa"]++[email protected]("alpha", [0, 16])+def test_merge_conversion_matches_the_live_adapter(alpha):+ from peft import LoraConfig, get_peft_model+ from unsloth.models._utils import lora_relative_to_original_base, snapshot_residual_lora_init++ torch.manual_seed(0)+ linear = torch.nn.Linear(64, 48, bias = False)+ W = linear.weight.detach().clone()+ config = LoraConfig(r = 8, lora_alpha = alpha, target_modules = ["0"], init_lora_weights = "olora")+ model = get_peft_model(torch.nn.Sequential(linear), config)+ snapshot_residual_lora_init(model, "olora")+ layer = model.base_model.model[0]+ with torch.no_grad():+ layer.lora_B["default"].weight.add_(0.05 * torch.randn_like(layer.lora_B["default"].weight))+ x = torch.randn(5, 64)+ live = model(x)+ with lora_relative_to_original_base(model):+ s = layer.scaling["default"]+ merged = W + s * layer.lora_B["default"].weight @ layer.lora_A["default"].weight+ assert torch.isfinite(merged).all()+ torch.testing.assert_close(x @ merged.T, live, atol = 1e-4, rtol = 1e-4)++[email protected]("layer", ["linear", "embedding"])+def test_untrained_residual_init_merges_back_to_the_original_weight(layer):+ # Factors set by hand: PEFT >= 0.20's Embedding.loftq_init is broken (0.18 / 0.19 reach this).+ from peft import LoraConfig, get_peft_model+ from unsloth.models._utils import lora_relative_to_original_base, snapshot_residual_lora_init++ torch.manual_seed(0)+ base = torch.nn.Linear(64, 48, bias = False) if layer == "linear" else torch.nn.Embedding(48, 64)+ model = get_peft_model(torch.nn.Sequential(base), LoraConfig(r = 8, target_modules = ["0"]))+ module = model.base_model.model[0]+ with torch.no_grad():+ for p in module.parameters():+ if p.requires_grad:+ p.normal_()+ snapshot_residual_lora_init(model, "loftq")+ assert module.get_delta_weight("default").abs().max() > 0+ with lora_relative_to_original_base(model):+ delta = module.get_delta_weight("default")+ torch.testing.assert_close(delta, torch.zeros_like(delta), atol = 1e-4, rtol = 0)diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.pyindex 1c9633a51ef..a2afd983b74 100644--- a/tests/test_merged_hub_destination.py+++ b/tests/test_merged_hub_destination.py@@ -2,6 +2,7 @@# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.import ast+import contextlibimport gcimport jsonimport os@@ -167,6 +168,8 @@ def saving(monkeypatch, tmp_path):_is_qwen3_5_vlm = lambda model: False,# A Mistral-format view (#12144) refuses merged saves; these fixtures are not one.raise_if_merging_mistral_format_view = lambda model, save_method: None,+ lora_relative_to_original_base = lambda model: contextlib.nullcontext(),+ nullcontext = contextlib.nullcontext,logger = SimpleNamespace(warning_once = lambda *args: None),# save_method="lora" leaves this module for the adapter save rather than the merge,# so record the handover instead of re-implementing it.diff --git a/tests/utils/test_attention_dispatch_eval_backward.py b/tests/utils/test_attention_dispatch_eval_backward.pynew file mode 100644index 00000000000..5ac8e987d6c--- /dev/null+++ b/tests/utils/test_attention_dispatch_eval_backward.py@@ -0,0 +1,51 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""An eval-mode forward that will backpropagate (LoRA-GA calibration) must not get the GQA layout that+xformers can only run forward."""++import pytest+import torch++import unsloth # noqa: F401+from unsloth.utils import attention_dispatch as ad+++def _run(monkeypatch, requires_grad, grad_enabled):+ seen = {}++ def fake_kernel(+ query,+ key,+ value,+ attn_bias = None,+ **kwargs,+ ):+ seen["ndim"] = query.dim()+ return torch.zeros_like(query)++ monkeypatch.setattr(ad, "xformers_attention", fake_kernel, raising = False)+ context = ad.AttentionContext(+ bsz = 1,+ q_len = 3,+ kv_seq_len = 3,+ n_heads = 4,+ head_dim = 8,+ requires_grad = requires_grad,+ seq_info = None,+ attention_mask = None,+ causal_mask = None,+ )+ config = ad.AttentionConfig(backend = ad.XFORMERS, n_kv_heads = 2, n_groups = 2)+ Q = torch.randn(1, 4, 3, 8, requires_grad = True)+ K = torch.randn(1, 2, 3, 8, requires_grad = True)+ V = torch.randn(1, 2, 3, 8, requires_grad = True)+ with torch.set_grad_enabled(grad_enabled):+ ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)+ return seen["ndim"]+++ "requires_grad, grad_enabled, ndim",+ [(False, True, 4), (False, False, 5), (True, True, 4)],+)+def test_xformers_gqa_layout_follows_backward(monkeypatch, requires_grad, grad_enabled, ndim):+ assert _run(monkeypatch, requires_grad, grad_enabled) == ndimdiff --git a/tests/utils/test_init_lora_validation.py b/tests/utils/test_init_lora_validation.pynew file mode 100644index 00000000000..c8ed84c5e16--- /dev/null+++ b/tests/utils/test_init_lora_validation.py@@ -0,0 +1,101 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++import pytest+import torch++import unsloth # noqa: F401+from unsloth.models._utils import (+ _has_quantized_linears,+ validate_init_lora_weights,+ validate_init_target_parameters,+)+++class _UnslothNVFP4Linear(torch.nn.Linear):+ pass+++def _model(*layers):+ return torch.nn.Sequential(torch.nn.Embedding(8, 4), *layers)+++def test_dense_model_is_not_quantized():+ assert not _has_quantized_linears(_model(torch.nn.Linear(4, 4)), routed_ok = True)++[email protected]("dtype", [torch.uint8, torch.int8, torch.float8_e4m3fn])+def test_non_float_linear_is_quantized(dtype):+ layer = torch.nn.Linear(4, 4)+ layer.weight = torch.nn.Parameter(layer.weight.data.to(dtype), requires_grad = False)+ assert _has_quantized_linears(_model(layer), routed_ok = True)+++def test_packed_params4bit_in_float_storage_is_quantized():+ layer = torch.nn.Linear(4, 4)+ Params4bit = type("Params4bit", (torch.nn.Parameter,), {})+ layer.weight = Params4bit(layer.weight.data.to(torch.bfloat16), requires_grad = False)+ assert _has_quantized_linears(_model(layer), routed_ok = True)++[email protected]("packed", ["qweight", "W_q", "weight_packed"])+def test_packed_non_linear_projection_is_quantized(packed):+ # GPTQ / AWQ (qweight), HQQ (W_q), packed MXFP4 / INT4 (weight_packed): no dense .weight.+ layer = torch.nn.Module()+ layer.register_buffer(packed, torch.zeros(4, 1, dtype = torch.int32))+ assert _has_quantized_linears(_model(layer), routed_ok = True)+++def test_packed_mxfp4_linear_is_quantized():+ from unsloth.models.mxfp4_compressed_linear import Mxfp4PackedLinear++ layer = Mxfp4PackedLinear(4, 4, bias = False)+ del layer.weight+ layer.register_buffer("weight_packed", torch.zeros(4, 2, dtype = torch.uint8))+ assert _has_quantized_linears(_model(layer), routed_ok = True)+++def test_olora_accepts_only_bitsandbytes_quantization():+ # PEFT's olora_init dequantizes / requantizes bitsandbytes weights only.+ bnb = torch.nn.Linear(4, 4)+ Params4bit = type("Params4bit", (torch.nn.Parameter,), {})+ bnb.weight = Params4bit(bnb.weight.data.to(torch.bfloat16), requires_grad = False)+ validate_init_lora_weights("olora", _model(bnb))+ with pytest.raises(ValueError, match = "quantized"):+ validate_init_lora_weights("pissa", _model(bnb))+ fp8 = torch.nn.Linear(4, 4)+ fp8.weight = torch.nn.Parameter(fp8.weight.data.to(torch.float8_e4m3fn), requires_grad = False)+ gptq = torch.nn.Module()+ gptq.register_buffer("qweight", torch.zeros(4, 1, dtype = torch.int32))+ for layer in (fp8, gptq):+ with pytest.raises(ValueError, match = "quantized"):+ validate_init_lora_weights("olora", _model(layer))+++def test_routed_compressed_linears_pass_except_for_mica():+ nvfp4 = _UnslothNVFP4Linear(4, 4)+ nvfp4.register_buffer("weight_packed", torch.zeros(4, 2, dtype = torch.uint8))+ fp8 = torch.nn.Linear(4, 4)+ fp8.weight = torch.nn.Parameter(fp8.weight.data.to(torch.float8_e4m3fn), requires_grad = False)+ fp8._unsloth_compressed_tensors_fp8 = True+ model = _model(nvfp4, fp8)+ assert not _has_quantized_linears(model, routed_ok = True)+ assert _has_quantized_linears(model, routed_ok = False)+++ "init", ["pissa", "pissa_niter_4", "olora", "orthogonal", "corda", "loftq", "lora_ga", "mica"]+)+def test_weight_reading_inits_reject_moe_expert_parameters(init):+ with pytest.raises(ValueError, match = "target_parameters = \\[\\]"):+ validate_init_target_parameters(init, ["gate_up_proj", "down_proj"])++[email protected]("init", [True, False, "gaussian", "eva"])+def test_other_inits_allow_moe_expert_parameters(init):+ validate_init_target_parameters(init, ["gate_up_proj"])+++def test_no_expert_parameters_allows_pissa():+ validate_init_target_parameters("pissa", None)+ validate_init_target_parameters("pissa", [])diff --git a/tests/version_compat/test_peft_pinned_symbols.py b/tests/version_compat/test_peft_pinned_symbols.pyindex 60ef23d3025..561c8fc6d07 100644--- a/tests/version_compat/test_peft_pinned_symbols.py+++ b/tests/version_compat/test_peft_pinned_symbols.py@@ -21,6 +21,7 @@ PEFT_TAGS = ["v0.18.1","v0.19.0","v0.19.1",+ "v0.20.0","main",]@@ -289,3 +290,37 @@ def test_peft_version_parseable(tag: str):assert (has_literal or has_subimport or has_metadata), f"{tag}: peft.__version__ not exported via any known mechanism"+++# 11. init_lora_weights="mica": check_mica_init imports MiCALinearVariant; PEFT inits via LoraLayer.mica_init+# and freezes lora_B via _freeze_non_trainable_peft_weights (freeze_peft_variant_weights). PEFT >= 0.20.0.+def test_peft_mica_variant_and_init(tag: str):+ variants_src = fetch_text("huggingface/peft", tag, "src/peft/tuners/lora/variants.py")+ if variants_src is None or not has_def(variants_src, "MiCALinearVariant", "class"):+ pytest.skip(f"{tag}: MiCA not yet introduced (peft 0.20)")+ layer_src = fetch_text("huggingface/peft", tag, "src/peft/tuners/lora/layer.py")+ tuners_src = fetch_text("huggingface/peft", tag, "src/peft/tuners/tuners_utils.py")+ assert layer_src is not None and has_def(+ layer_src, "mica_init", "func"+ ), f"{tag}: mica_init missing"+ assert tuners_src is not None and has_def(+ tuners_src, "_freeze_non_trainable_peft_weights", "func"+ ), f"{tag}: _freeze_non_trainable_peft_weights missing"+++# 12. unsloth/models/lora_init.py replaces LoraLayer.pissa_init(self, adapter_name, init_lora_weights) and+# mica_init(self, adapter_name) during get_peft_model and calls layer.py's `transpose`.+def test_peft_lora_init_hooks(tag: str):+ src = fetch_text("huggingface/peft", tag, "src/peft/tuners/lora/layer.py")+ if src is None:+ pytest.skip(f"{tag}: layer.py missing")+ assert re.search(+ r"def pissa_init\(self, adapter_name, init_lora_weights\)", src+ ), f"{tag}: pissa_init signature moved"+ assert re.search(+ r"^from peft\.utils\.other import transpose$", src, re.M+ ), f"{tag}: transpose import moved"+ if has_def(src, "mica_init", "func"):+ assert re.search(+ r"def mica_init\(self, adapter_name\)", src+ ), f"{tag}: mica_init signature moved"diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.pyindex cdbecea261a..2ef7fec7dfa 100644--- a/unsloth/models/__init__.py+++ b/unsloth/models/__init__.py@@ -27,3 +27,7 @@ except:from .dpo import PatchDPOTrainer, PatchKTOTrainerfrom ._utils import is_bfloat16_supported, is_vLLM_available, __version__from .rl import PatchFastRL, vLLMSamplingParams+from .lora_init import patch_peft_calibration_eager as _patch_peft_calibration_eager++_patch_peft_calibration_eager()+del _patch_peft_calibration_eagerdiff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 220c7c27317..5930706ea8c 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -75,6 +75,12 @@ __all__ = ["set_task_config_attr","patch_fast_lora","validate_loftq_config",+ "validate_init_lora_weights",+ "validate_init_target_parameters",+ "RESIDUAL_INIT_LORA_WEIGHTS",+ "snapshot_residual_lora_init",+ "lora_relative_to_original_base",+ "freeze_peft_variant_weights","RaiseUninitialized","fast_inference_setup","patch_peft_fast_inference",@@ -2675,6 +2681,7 @@ _ROOT_AUX_PREFETCH_PATTERNS = (_ADAPTER_PREFETCH_PATTERNS = ("adapter_config.json","adapter_model*",+ "unsloth_lora_init.json", # lora_init.SIDECAR: which PiSSA algorithm rebuilds the residual base)@@ -5811,7 +5818,237 @@ for function in ("__reduce__", "__reduce_ex__", "__getstate__", "__setstate__"):pass-def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, model):+_INIT_LORA_WEIGHTS = (+ "gaussian",+ "eva",+ "olora",+ "pissa",+ "corda",+ "loftq",+ "orthogonal",+ "lora_ga",+ "mica",+)+# Inits that rewrite the base weight to W - scaling * B @ A (save_pretrained_merged must not use the original W).+RESIDUAL_INIT_LORA_WEIGHTS = ("pissa", "olora", "corda", "loftq", "lora_ga")+++def _has_quantized_linears(+ model,+ routed_ok,+ bnb_ok = False,+):+ for module in model.modules():+ routed = type(module).__name__ == "_UnslothNVFP4Linear" or getattr(+ module, "_unsloth_compressed_tensors_fp8", False+ )+ if routed:+ if not routed_ok:+ return True+ continue+ # No dense .weight: GPTQ / AWQ (qweight), HQQ (W_q), packed MXFP4 / INT4 (weight_packed).+ if any(hasattr(module, name) for name in ("qweight", "qzeros", "W_q", "weight_packed")):+ return True+ if not isinstance(module, torch.nn.Linear):+ continue+ weight = getattr(module, "weight", None)+ # FSDP-QLoRA packs Params4bit into a float quant_storage, so the dtype alone looks dense.+ if type(weight).__name__ in ("Params4bit", "Int8Params") or hasattr(weight, "quant_state"):+ if bnb_ok:+ continue+ return True+ if isinstance(weight, torch.Tensor) and weight.dtype not in (+ torch.float32,+ torch.float16,+ torch.bfloat16,+ ):+ return True+ return False+++def validate_init_target_parameters(init_lora_weights, target_parameters):+ # PEFT's ParamWrapper reads a .weight fused experts lack and refuses MiCA, failing mid get_peft_model.+ if not target_parameters or not isinstance(init_lora_weights, str):+ return+ if init_lora_weights.split("_niter_")[0] in (+ "pissa",+ "olora",+ "orthogonal",+ "corda",+ "loftq",+ "lora_ga",+ "mica",+ ):+ raise ValueError(+ f"Unsloth: `init_lora_weights = {init_lora_weights!r}` cannot initialize fused MoE expert "+ f"parameters ({target_parameters}).\n"+ "Pass `target_parameters = []` to apply it to the other layers only, or use another init."+ )+++def validate_init_lora_weights(+ init_lora_weights,+ model,+ r = None,+):+ if type(init_lora_weights) is bool:+ return+ name = init_lora_weights if isinstance(init_lora_weights, str) else None+ if name is not None and name.startswith("pissa_niter_"):+ if not name[len("pissa_niter_") :].isdigit():+ raise ValueError(+ f"Unsloth: `init_lora_weights = {name!r}` must be `pissa_niter_<non-negative int>`."+ )+ name = "pissa"+ if name not in _INIT_LORA_WEIGHTS:+ raise ValueError(+ "Unsloth: `init_lora_weights` must be True, False, `pissa_niter_<int>` or one of "+ f"{list(_INIT_LORA_WEIGHTS)}, got {init_lora_weights!r}."+ )+ import peft+ from peft.tuners.lora import LoraLayer++ def _require(supported, version):+ if not supported:+ raise RuntimeError(+ f"Unsloth: Your PEFT version of {peft.__version__} does not support "+ f"`init_lora_weights = {init_lora_weights!r}`.\n"+ f"Please install PEFT {version} or higher: `pip install --upgrade peft`"+ )++ if name == "mica":+ try:+ from peft.tuners.lora.variants import MiCALinearVariant+ except ImportError:+ _require(False, "0.20.0")+ elif name == "lora_ga":+ _require(hasattr(LoraLayer, "lora_ga_init"), "0.19.0")++ # PEFT's olora handles bitsandbytes only; loader_utils densifies routed NVFP4 / FP8 for all but MiCA.+ base = name.split("_niter_")[0] if name is not None else None+ if base in (+ "pissa",+ "olora",+ "corda",+ "loftq",+ "lora_ga",+ "mica",+ "orthogonal",+ ) and _has_quantized_linears(model, routed_ok = base != "mica", bnb_ok = base == "olora"):+ raise ValueError(+ f"Unsloth: `init_lora_weights = {init_lora_weights!r}` needs float32/float16/bfloat16 base weights, "+ "yet your model is quantized.\n"+ "Reload your model with `load_in_4bit = False` and `load_in_8bit = False`."+ )+ if name == "orthogonal" and r is not None and r % 2 != 0:+ raise ValueError(+ f"Unsloth: `init_lora_weights = 'orthogonal'` needs an even rank, got r = {r}."+ )+ if name in ("corda", "lora_ga"):+ attr = "eigens" if name == "corda" else "_peft_loraga_grad"+ if not any(hasattr(module, attr) for module in model.modules()):+ preprocess = (+ "peft.tuners.lora.corda.preprocess_corda"+ if name == "corda"+ else "peft.preprocess_loraga"+ )+ # lora_ga silently falls back to the default init without it.+ raise ValueError(+ f"Unsloth: `init_lora_weights = {name!r}` needs `{preprocess}(model, lora_config, ...)` "+ "to be run on the model before `get_peft_model`."+ )+ if name == "eva":+ logger.warning_once(+ "Unsloth: `init_lora_weights = 'eva'` only zeroes lora_B. Call "+ "`peft.initialize_lora_eva_weights(model, dataloader)` after `get_peft_model` to run EVA."+ )+++def _lora_factors(module):+ """(adapter, A holder, A attr, B holder, B attr): Linear / Conv keep factors as `.weight` of lora_A / lora_B+ modules, Embedding as entries of the lora_embedding_A / lora_embedding_B ParameterDicts."""+ lora_A = getattr(module, "lora_A", None)+ if isinstance(lora_A, torch.nn.ModuleDict):+ for k in lora_A:+ yield k, lora_A[k], "weight", module.lora_B[k], "weight"+ lora_embedding_A = getattr(module, "lora_embedding_A", None)+ if isinstance(lora_embedding_A, torch.nn.ParameterDict):+ for k in lora_embedding_A:+ yield k, lora_embedding_A, k, module.lora_embedding_B, k+++def snapshot_residual_lora_init(model, init_lora_weights):+ if not isinstance(init_lora_weights, str):+ return+ if init_lora_weights.split("_niter_")[0] not in RESIDUAL_INIT_LORA_WEIGHTS:+ return+ # The scale the base rewrite used: LoftQ fits B0 @ A0 to W - Q unscaled, the others subtract s * B0 @ A0.+ unscaled = init_lora_weights == "loftq"+ for module in model.modules():+ initial = {+ k: (+ getattr(a, a_name).detach().clone(),+ getattr(b, b_name).detach().clone(),+ 1.0 if unscaled else module.scaling[k],+ )+ for k, a, a_name, b, b_name in _lora_factors(module)+ }+ if initial:+ module._unsloth_initial_lora = initial+++def lora_relative_to_original_base(model):+ # Merge reads the original W: s0 * [B * s / s0, -B0] @ [A; A0] (PEFT's path_initial_model_for_weight_conversion).+ swapped = []+ try:+ for module in model.modules():+ initial = getattr(module, "_unsloth_initial_lora", None)+ if not initial:+ continue+ for k, a, a_name, b, b_name in list(_lora_factors(module)):+ if k not in initial:+ continue+ A0, B0, scaling0 = initial[k]+ if not scaling0:+ continue # lora_alpha = 0: the base rewrite was a no-op+ A, B = getattr(a, a_name), getattr(b, b_name)+ swapped.append((module, k, module.scaling[k], a, a_name, A, b, b_name, B))+ B_new = B.detach() * (module.scaling[k] / scaling0)+ module.scaling[k] = scaling0+ setattr(+ a,+ a_name,+ torch.nn.Parameter(torch.cat([A.detach(), A0.to(A)], 0), requires_grad = False),+ )+ setattr(+ b,+ b_name,+ torch.nn.Parameter(torch.cat([B_new, -B0.to(B_new)], 1), requires_grad = False),+ )+ yield+ finally:+ for module, k, scaling, a, a_name, A, b, b_name, B in swapped:+ module.scaling[k] = scaling+ setattr(a, a_name, A)+ setattr(b, b_name, B)+++def freeze_peft_variant_weights(model):+ # prepare_model_for_training re-enables every lora_A/lora_B; PEFT variants such as MiCA freeze lora_B.+ for module in model.modules():+ if getattr(module, "frozen_peft_weight_names", None):+ module._freeze_non_trainable_peft_weights()+++def validate_loftq_config(+ loftq_config,+ lora_dropout,+ bias,+ init_lora_weights,+ model,+ r = None,+):from peft import LoraConfigif loftq_config is None:@@ -5832,15 +6069,7 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, mf"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit.")- if not (- type(init_lora_weights) is bool- or init_lora_weights == "gaussian"- or init_lora_weights == "loftq"- or init_lora_weights == "corda"- ):- raise ValueError(- 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'- )+ validate_init_lora_weights(init_lora_weights, model, r)if init_lora_weights == "loftq":if not SUPPORTS_LOFTQ:@@ -5859,12 +6088,6 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m)loftq_config = LoftQConfig(loftq_bits = 4, loftq_iter = 1)- if hasattr(model.config, "quantization_config"):- raise ValueError(- "Unsloth: You are using `loftq` init, yet `load_in_4bit = True` was set.\n"- "Reload your model without any quantization by setting `load_in_4bit = False`."- )-return loftq_configdiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 07c24bfaa1c..1fb4e3cda58 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -3848,14 +3848,12 @@ class FastLlamaModel:f"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit.")- if not (- type(init_lora_weights) is bool- or init_lora_weights == "gaussian"- or init_lora_weights == "loftq"- or init_lora_weights == "corda"- ):- raise ValueError(- 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'+ validate_init_lora_weights(init_lora_weights, model, r)+ if init_lora_weights == "eva":+ # EVA collects layer inputs with LoRA module forward hooks, which the fused LoRA kernels never call.+ raise NotImplementedError(+ "Unsloth: `init_lora_weights = 'eva'` is not supported by FastLanguageModel's fused LoRA path.\n"+ "Use `FastModel.from_pretrained` and `FastModel.get_peft_model` instead.")if init_lora_weights == "loftq":@@ -3875,12 +3873,6 @@ class FastLlamaModel:)loftq_config = LoftQConfig(loftq_bits = 4, loftq_iter = 1)- if hasattr(model.config, "quantization_config"):- raise ValueError(- "Unsloth: You are using `loftq` init, yet `load_in_4bit = True` was set.\n"- "Reload your model without any quantization by setting `load_in_4bit = False`."- )-assert type(use_rslora) is boolif use_rslora:if not SUPPORTS_RSLORA:@@ -4053,6 +4045,8 @@ class FastLlamaModel:_n = max(1, min(int(finetune_last_n_layers), _total_layers))layers_to_transform = list(range(_total_layers - _n, _total_layers))+ validate_init_target_parameters(init_lora_weights, target_parameters)+arguments = dict(r = r,lora_alpha = lora_alpha,@@ -4105,7 +4099,13 @@ class FastLlamaModel:gc.collect()clean_gpu_cache()- model = _get_peft_model(model, lora_config)+ from .lora_init import fast_lora_init, record_fast_pissa++ with fast_lora_init() as fast:+ model = _get_peft_model(model, lora_config)+ if fast["pissa"]:+ record_fast_pissa(model)+ snapshot_residual_lora_init(model, init_lora_weights)try:from .vision import _lift_endpoint_hooks_onto_adapters@@ -4279,6 +4279,7 @@ class FastLlamaModel:use_gradient_checkpointing = use_gradient_checkpointing,use_reentrant = True,)+ freeze_peft_variant_weights(model)for active_adapter in model.peft_config.keys():if False:diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 8a4c01fc8ec..a2c6301cb30 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -41,6 +41,7 @@ from .mistral_format import (mistral_format_redirect,prepare_mistral_format_checkpoint,)+from .lora_init import adapter_used_fast_pissa, fast_lora_init, record_fast_pissafrom .loader_utils import (DEFAULT_DEVICE_MAP,OFFLOAD_EMBEDDING_AUTO,@@ -1620,16 +1621,27 @@ class FastLanguageModel(FastLlamaModel):local_files_only = local_files_only,cache_dir = kwargs.get("cache_dir"),)- model = PeftModel.from_pretrained(- model,+ # PEFT re-runs PiSSA at load: rebuild the residual with the algorithm that made the adapter.+ fast_pissa = adapter_used_fast_pissa(old_model_name,token = token,revision = revision,local_files_only = local_files_only,- is_trainable = True,- trust_remote_code = trust_remote_code,- **peft_load_kwargs,+ cache_dir = kwargs.get("cache_dir"),)+ with fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext():+ model = PeftModel.from_pretrained(+ model,+ old_model_name,+ token = token,+ revision = revision,+ local_files_only = local_files_only,+ is_trainable = True,+ trust_remote_code = trust_remote_code,+ **peft_load_kwargs,+ )+ if fast_pissa:+ record_fast_pissa(model)model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing)try:from .vision import _lift_endpoint_hooks_onto_adapters@@ -2856,17 +2868,25 @@ class FastModel(FastBaseModel):local_files_only = local_files_only,cache_dir = kwargs.get("cache_dir"),)+ fast_pissa = adapter_used_fast_pissa(+ old_model_name,+ token = token,+ revision = revision,+ local_files_only = local_files_only,+ cache_dir = kwargs.get("cache_dir"),+ )try:- model = PeftModel.from_pretrained(- model,- old_model_name,- token = token,- revision = revision,- local_files_only = local_files_only,- is_trainable = True,- trust_remote_code = trust_remote_code,- **peft_load_kwargs,- )+ with fast_lora_init(force = True) if fast_pissa else contextlib.nullcontext():+ model = PeftModel.from_pretrained(+ model,+ old_model_name,+ token = token,+ revision = revision,+ local_files_only = local_files_only,+ is_trainable = True,+ trust_remote_code = trust_remote_code,+ **peft_load_kwargs,+ )finally:# Always restore the original PEFT method, even if loading fails.if _clippable_linear_cls is not None:@@ -2875,6 +2895,8 @@ class FastModel(FastBaseModel):model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing, trust_remote_code = trust_remote_code)+ if fast_pissa:+ record_fast_pissa(model)try:from .vision import _lift_endpoint_hooks_onto_adapters_lift_endpoint_hooks_onto_adapters(model)diff --git a/unsloth/models/lora_init.py b/unsloth/models/lora_init.pynew file mode 100644index 00000000000..08d072a90c0--- /dev/null+++ b/unsloth/models/lora_init.py@@ -0,0 +1,451 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++import contextlib+import functools+import json+import os+import threading+import torch++__all__ = [+ "randomized_svd",+ "mica_basis",+ "fast_lora_init",+]++_FLOATS = (torch.float32, torch.float16, torch.bfloat16)+++def _tf32(enabled):+ flags = torch.backends.cuda.matmul+ old = flags.allow_tf32+ flags.allow_tf32 = enabled+ try:+ yield+ finally:+ flags.allow_tf32 = old+++_EPS32 = torch.finfo(torch.float32).eps / 2 # unit roundoff u+++def _orthonormalize_(Y, shift, failures, householder):+ """Y <- Y R^-1 in place, fp32 shifted CholeskyQR (Fukaya et al. 2020); no host sync.++ Columns are scaled to unit norm first (Gram diagonal = 1, no overflow). The shift is Rump-Ogita's+ breakdown bound 2.2 (n + 1) u tr(G) plus a sqrt(m) u n allowance for the Gram's rounding error.+ Breakdown flags are appended to `failures` and checked once per call; any breakdown reruns the+ whole call with Householder QR, which cannot fail on finite input.+ """+ if householder:+ Y.copy_(torch.linalg.qr(Y).Q)+ return+ m, n = Y.shape+ tiny = torch.finfo(torch.float32).tiny+ # max-abs first: vector_norm squares entries and overflows fp32 above ~1e19.+ Y.div_(Y.abs().amax(0).clamp_min_(tiny))+ Y.div_(torch.linalg.vector_norm(Y, dim = 0).clamp_min_(tiny))+ with _tf32(False):+ G = Y.mT @ Y+ if shift:+ G.diagonal().add_((2.2 * (n + 1) * n + 11.0 * (m**0.5 * n + n * (n + 1))) * _EPS32)+ R, info = torch.linalg.cholesky_ex(G, upper = True)+ failures.append(info)+ torch.linalg.solve_triangular(R, Y, upper = True, left = False, out = Y)++[email protected]_cache(maxsize = 8)+def _sketch(N, q):+ # Seeded CPU draw: reloading re-runs this init and must rebuild the same residual on any device.+ return torch.randn(N, q, dtype = torch.float32, generator = torch.Generator().manual_seed(3407))++[email protected]_grad()+def randomized_svd(+ W,+ rank,+ n_oversamples = None,+ n_iter = 6,+ final_passes = 1,+ generator = None,+ _safe = False,+):+ """Truncated W ~= U diag(S) Vh by randomized subspace iteration (Halko et al. 2011, Alg. 4.4 + 5.1).++ fp32 throughout, one host sync per call. Oversampling defaults to max(rank, 10): LLM spectra decay+ slowly, so a wider sketch buys more accuracy per ms than iterations. Shifted CholeskyQR replaces+ cuSOLVER QR for sketches 32+ wide (geqrf + ormqr was no faster: ormqr applies the full m x m Q);+ the final basis gets `final_passes` unshifted passes after the shifted one.+ """+ m, n = W.shape+ A = W if m >= n else W.mT+ A = A.float()+ scale = None+ if _safe:+ # rocSOLVER's geqrf / gesvd square entries without LAPACK's scaling and overflow above ~1e19.+ scale = A.abs().amax().clamp_min_(torch.finfo(torch.float32).tiny)+ A = A / scale+ M, N = A.shape+ rank = min(rank, N)+ q = min(rank + (max(rank, 10) if n_oversamples is None else n_oversamples), N)+ if generator is None:+ Z = _sketch(N, q).to(A.device, A.dtype, copy = True)+ else:+ Z = torch.randn(N, q, device = A.device, dtype = A.dtype, generator = generator)+ failures = []+ # Below q = 32 one Householder QR beats CholeskyQR's ~8 launches.+ householder = _safe or q < 32++ def _power(k):+ # Both half steps: skipping one squares cond(Y) (~2e3 on real r = 128 weights), past fp32.+ for _ in range(k):+ _orthonormalize_(Y, True, failures, householder)+ torch.matmul(A.mT, Y, out = Z)+ _orthonormalize_(Z, True, failures, householder)+ torch.matmul(A, Z, out = Y)++ with _tf32(A.is_cuda and n_iter > 1):+ Y = torch.matmul(A, Z)+ _power(n_iter - 1)+ # Last iteration in fp32 removes the TF32 error when sigma_1 / sigma_rank is large.+ _power(min(n_iter, 1))+ _orthonormalize_(Y, True, failures, householder)+ if not householder:+ for _ in range(final_passes):+ _orthonormalize_(Y, False, failures, householder)+ with _tf32(False):+ torch.matmul(A.mT, Y, out = Z) # Z = (Q^T A)^T, N x q+ # From q = 128, eigh of [[0, R2], [R2^T, 0]] (eigenvalues +-sigma) beats cuSOLVER's fp32 SVD (~60x accuracy).+ # cuSOLVER syevd / rocSOLVER gesvd refuse degenerate R2 (rank 0 / 1): the safe rerun uses LAPACK.+ Q2, R2 = torch.linalg.qr(Z)+ try:+ if q < 128 or _safe:+ Ur, S, Vrh = torch.linalg.svd(R2.cpu() if _safe else R2)+ Ur, S, Vr = (x.to(R2.device) for x in (Ur[:, :rank], S[:rank], Vrh[:rank].mT))+ else:+ J = R2.new_zeros(2 * q, 2 * q)+ J[:q, q:] = R2+ J[q:, :q] = R2.mT+ L, X = torch.linalg.eigh(J)+ X = X[:, -rank:].flip(1)+ S = L[-rank:].flip(0).clamp_min_(0)+ tiny = torch.finfo(torch.float32).tiny+ Ur = X[:q].div_(torch.linalg.vector_norm(X[:q], dim = 0).clamp_min_(tiny))+ Vr = X[q:].div_(torch.linalg.vector_norm(X[q:], dim = 0).clamp_min_(tiny))+ except torch.linalg.LinAlgError:+ if _safe:+ raise+ return randomized_svd(W, rank, n_oversamples, n_iter, final_passes, generator, _safe = True)+ # A^T ~= Z Y^T = Q2 Ur S Vr^T Y^T, so A ~= (Y Vr) S (Q2 Ur)^T.+ V = Q2 @ Ur+ U = Y @ Vr+ if scale is not None:+ S = S * scale+ # cuSOLVER's Householder QR cannot fail on finite input; ROCm's can overflow, and CPU syncs for free.+ if not _safe and (not householder or A.device.type != "cuda" or torch.version.hip is not None):+ bad = torch.stack(failures).ne(0).any() if failures else S.new_zeros((), dtype = torch.bool)+ bad = bad | ~torch.isfinite(S).all() | ~torch.isfinite(U).all() | ~torch.isfinite(V).all()+ if bad.item():+ del A, Y, Z, U, V+ return randomized_svd(+ W, rank, n_oversamples, n_iter, final_passes, generator, _safe = True+ )+ if m < n:+ return V, S, U.mT+ return U, S, V.mT+++def _fast_fp64(device):+ if device.type == "cpu":+ return True+ if device.type != "cuda":+ return False+ if getattr(torch.version, "hip", None):+ return "gfx9" in getattr(torch.cuda.get_device_properties(device), "gcnArchName", "")+ # Full-rate fp64: P100, V100, A100, H100, B200. B300 (10.3) and consumer / L4 / T4 parts run fp64 at 1/64.+ return torch.cuda.get_device_capability(device) in ((6, 0), (7, 0), (8, 0), (9, 0), (10, 0))++[email protected]_grad()+def mica_basis(W, r):+ """U[:, -r:] of the reduced SVD of W (out x in) as fp32, via fp64 eigh of the smaller Gram matrix.++ fp64 Gram squaring costs ~1e-16 * cond^2, below fp32 SVD's own error on LLM weights. lobpcg does not+ converge here: the relative gaps between the smallest singular values are ~1e-5..1e-3.+ """+ out_features, in_features = W.shape+ A = W.to(torch.float64)+ wide = out_features <= in_features+ L, V = torch.linalg.eigh(A @ A.T if wide else A.T @ A)+ # L ascends: L[0] is the smallest divisor below.+ if L[0] <= 1e-12 * L[-1]:+ U = torch.linalg.svd(A, full_matrices = False)[0]+ return U[:, -r:].float().contiguous()+ V = V[:, :r].flip(1)+ if wide:+ return V.float().contiguous()+ U = A @ V+ U /= L[:r].flip(0).sqrt()+ return U.float().contiguous()+++def _routed(base):+ # Routed compressed-tensors linears: loader_utils' wrapper (the saved original) densifies them first.+ return type(base).__name__ == "_UnslothNVFP4Linear" or getattr(+ base, "_unsloth_compressed_tensors_fp8", False+ )+++def _check_float_weight(weight, init):+ # FSDP-QLoRA packs Params4bit into a float quant_storage: its dtype alone looks dense.+ if type(weight).__name__ in ("Params4bit", "Int8Params") or hasattr(weight, "quant_state"):+ raise TypeError(+ f"Unsloth: `init_lora_weights = {init!r}` re-runs on the base weights when an adapter is created "+ "or loaded, and they are quantized here. Load the model with `load_in_4bit = False` and "+ "`load_in_8bit = False`."+ )+++def _pissa_init(self, adapter_name, init_lora_weights):+ from peft.tuners.lora.layer import transpose++ if _OWNER.get("thread") != threading.get_ident():+ return _ORIGINAL_ANY["pissa_init"](self, adapter_name, init_lora_weights)+ if _routed(self.get_base_layer()):+ return _ORIGINAL["pissa_init"](self, adapter_name, init_lora_weights)+ weight = self.get_base_layer().weight+ _check_float_weight(weight, init_lora_weights)+ dtype = weight.dtype+ r = self.r[adapter_name]+ if (+ dtype not in _FLOATS+ or r > min(weight.shape)+ or weight.device.type == "meta"+ or not torch.isfinite(weight).all()+ ):+ return _ORIGINAL["pissa_init"](self, adapter_name, init_lora_weights)+ if init_lora_weights == "pissa":+ n_iter, n_oversamples = 6, None+ else:+ parts = init_lora_weights.split("_niter_")+ if len(parts) != 2:+ return _ORIGINAL["pissa_init"](self, adapter_name, init_lora_weights)+ # PEFT's svd_lowrank(q = r, niter = N), device-independent so reloads rebuild the same residual.+ n_iter, n_oversamples = int(parts[-1]), 0+ _STATE["pissa"] = True+ self._unsloth_fast_pissa = getattr(self, "_unsloth_fast_pissa", set()) | {adapter_name}+ W = transpose(weight.to(torch.float32), self.fan_in_fan_out)+ U, S, Vh = randomized_svd(W, r, n_oversamples = n_oversamples, n_iter = n_iter)+ scaling = self.scaling[adapter_name]+ S.div_(scaling).sqrt_()+ lora_A = Vh.mul_(S.unsqueeze(1)).contiguous()+ lora_B = U.mul_(S).contiguous()+ self.lora_A[adapter_name].weight.data = lora_A+ self.lora_B[adapter_name].weight.data = lora_B+ W = W if W.data_ptr() != weight.data_ptr() else W.clone()+ W.addmm_(lora_B, lora_A, alpha = -scaling)+ self.get_base_layer().weight.data = transpose(W.to(dtype), self.fan_in_fan_out)+++def _mica_init(self, adapter_name):+ from peft.tuners.lora.layer import transpose++ if _OWNER.get("thread") != threading.get_ident():+ return _ORIGINAL_ANY["mica_init"](self, adapter_name)+ if _routed(self.get_base_layer()):+ return _ORIGINAL["mica_init"](self, adapter_name)+ weight = self.get_base_layer().weight+ _check_float_weight(weight, "mica")+ r = self.r[adapter_name]+ if (+ self.lora_B[adapter_name].weight.device.type == "meta"+ or weight.dtype not in _FLOATS+ or r > min(weight.shape)+ or not _fast_fp64(weight.device)+ ):+ return _ORIGINAL["mica_init"](self, adapter_name)+ dtype = weight.dtype+ W = transpose(weight, self.fan_in_fan_out)+ self.lora_B[adapter_name].weight.data = mica_basis(W, r).to(dtype)+ self.lora_A[adapter_name].weight.data = torch.zeros(+ r, W.shape[1], device = weight.device, dtype = dtype+ )+++_ORIGINAL = {}+# PEFT's own methods, kept past the swap: another thread calling PEFT directly must not get ours.+_ORIGINAL_ANY = {}+_OWNER = {}+_STATE = {"pissa": False}+# One owner at a time: the swap is process-wide, so a second thread must not build layers mid-restore.+_LOCK = threading.RLock()+SIDECAR = "unsloth_lora_init.json"+++def fast_lora_init(force = False):+ """Swap PEFT's LoraLayer.pissa_init / mica_init for the fast versions during get_peft_model.++ Yields a dict whose "pissa" is True once a layer took the fast PiSSA path. UNSLOTH_FAST_LORA_INIT=0+ keeps PEFT's own SVD unless `force` (reloading an adapter that recorded the fast path).+ """+ global _STATE+ with _LOCK:+ if _ORIGINAL:+ # Nested use keeps the outer swap.+ yield _STATE+ return+ if not force and os.environ.get("UNSLOTH_FAST_LORA_INIT", "1") == "0":+ yield {"pissa": False}+ return+ from peft.tuners.lora.layer import LoraLayer++ swapped = []+ for name, fn in (("pissa_init", _pissa_init), ("mica_init", _mica_init)):+ original = LoraLayer.__dict__.get(name)+ if original is None:+ continue+ _ORIGINAL[name] = _ORIGINAL_ANY[name] = original+ setattr(LoraLayer, name, fn)+ swapped.append((name, original))+ # A fresh dict per swap: callers read it after the lock is released.+ _STATE = {"pissa": False}+ _OWNER["thread"] = threading.get_ident()+ try:+ yield _STATE+ finally:+ _OWNER.clear()+ for name, original in swapped:+ setattr(LoraLayer, name, original)+ _ORIGINAL.clear()+++def record_fast_pissa(model):+ """PEFT re-runs PiSSA when loading an adapter, and only the same algorithm rebuilds the residual base+ training saw: saves of this model mark their adapter folders so Unsloth's loaders can tell."""+ original = model.save_pretrained+ if getattr(original, "_unsloth_fast_pissa", False):+ return++ @functools.wraps(original)+ def save_pretrained(save_directory, *args, **kwargs):+ out = original(save_directory, *args, **kwargs)+ if kwargs.get("is_main_process", True):+ # Only fast-path adapters; PEFT saves "default" at the root, others in a subfolder.+ fast = set()+ for module in model.modules():+ fast |= getattr(module, "_unsloth_fast_pissa", set())+ for name in fast:+ folder = save_directory if name == "default" else os.path.join(save_directory, name)+ weights = _adapter_weights(folder)+ if weights is not None:+ # Digest: a later ordinary save into this folder leaves the marker but not a match.+ with open(os.path.join(folder, SIDECAR), "w", encoding = "utf-8") as f:+ json.dump(+ {"pissa": "unsloth_randomized_svd", "sha256": _sha256(weights)}, f+ )+ return out++ save_pretrained._unsloth_fast_pissa = True+ model.save_pretrained = save_pretrained+++_WEIGHT_FILES = ("adapter_model.safetensors", "adapter_model.bin")+++def _adapter_weights(folder):+ for name in _WEIGHT_FILES:+ if os.path.isfile(os.path.join(folder, name)):+ return os.path.join(folder, name)+ return None+++def _sha256(path):+ import hashlib++ digest = hashlib.sha256()+ with open(path, "rb") as f:+ for chunk in iter(lambda: f.read(1 << 20), b""):+ digest.update(chunk)+ return digest.hexdigest()+++def _marker_matches(sidecar, weights):+ with open(sidecar, encoding = "utf-8") as f:+ expected = json.load(f).get("sha256")+ return weights is not None and expected == _sha256(weights)+++def adapter_used_fast_pissa(path, **hub_kwargs):+ if os.path.isdir(path):+ sidecar = os.path.join(path, SIDECAR)+ return os.path.isfile(sidecar) and _marker_matches(sidecar, _adapter_weights(path))+ from huggingface_hub import constants, hf_hub_download+ from huggingface_hub.utils import EntryNotFoundError, HFValidationError, LocalEntryNotFoundError++ hub_kwargs = {k: v for k, v in hub_kwargs.items() if v is not None}+ try:+ sidecar = hf_hub_download(path, SIDECAR, **hub_kwargs)+ except LocalEntryNotFoundError:+ # Offline: uncached = absent (the prefetch fetches it). Online: unreachable Hub, never guess.+ if hub_kwargs.get("local_files_only") or constants.HF_HUB_OFFLINE:+ return False+ raise+ except (EntryNotFoundError, HFValidationError):+ return False+ for name in _WEIGHT_FILES:+ try:+ return _marker_matches(sidecar, hf_hub_download(path, name, **hub_kwargs))+ except EntryNotFoundError:+ continue+ return False+++# Compiled modules skip hooks added later (skip_nnmodule_hook_guards): CorDA then divides by zero.+def _calibration_functions():+ found = []+ try:+ from peft.tuners.lora import corda+ found.append((corda, "preprocess_corda"))+ except Exception:+ pass+ try:+ from peft.tuners.lora import eva+ found.append((eva, "initialize_lora_eva_weights"))+ except Exception:+ pass+ try:+ from peft.tuners.lora import loraga+ found.append((loraga, "preprocess_loraga"))+ except Exception:+ pass+ return found+++def patch_peft_calibration_eager():+ import functools+ import sys++ if not hasattr(torch.compiler, "set_stance"):+ return+ for module, name in _calibration_functions():+ original = getattr(module, name, None)+ if original is None or getattr(original, "_unsloth_eager", False):+ continue++ @functools.wraps(original)+ def eager(+ *args,+ __original = original,+ **kwargs,+ ):+ with torch.compiler.set_stance("force_eager"):+ return __original(*args, **kwargs)++ eager._unsloth_eager = True+ # vars(), not getattr: getattr on lazy modules (transformers) imports submodules.+ for loaded in list(sys.modules.values()):+ if getattr(loaded, "__dict__", {}).get(name) is original:+ setattr(loaded, name, eager)diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 2f3082321c9..ca4fa1495c3 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -4342,7 +4342,7 @@ class FastBaseModel:max_seq_length = model.max_seq_length# Passing loftq_config = None gives an error.loftq_config = validate_loftq_config(- loftq_config, lora_dropout, bias, init_lora_weights, model+ loftq_config, lora_dropout, bias, init_lora_weights, model, r)# Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an attention-only request does not train experts, but only while MLP and language families are both in scope: with finetune_mlp_modules or finetune_language_layers False the scoped regex already dropped the experts.@@ -4433,6 +4433,8 @@ class FastBaseModel:n = max(1, min(int(finetune_last_n_layers), _total_layers))layers_to_transform = list(range(_total_layers - n, _total_layers))+ validate_init_target_parameters(init_lora_weights, target_parameters)+local_variables = {**locals(),**kwargs,@@ -4506,7 +4508,13 @@ class FastBaseModel:_LoraModel._create_and_replace = _patched_car- model = _get_peft_model(model, lora_config)+ from .lora_init import fast_lora_init, record_fast_pissa++ with fast_lora_init() as fast:+ model = _get_peft_model(model, lora_config)+ if fast["pissa"]:+ record_fast_pissa(model)+ snapshot_residual_lora_init(model, init_lora_weights)# PEFT may have wrapped an endpoint this load repaired; the hook stays on base_layer and the adapter branch reads the caller's tensor.try:@@ -4632,6 +4640,7 @@ class FastBaseModel:float32_mixed_precision = float32_mixed_precision,patch_modules_to_save = True,)+ freeze_peft_variant_weights(model)if full_finetuning:# prepare_model_for_training re-enabled every parameter, a kept wrapper's siblings too._freeze_unused_siblings(model)diff --git a/unsloth/save.py b/unsloth/save.pyindex 4e656255344..cfde2aec85e 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -13,7 +13,7 @@from unsloth_zoo.utils import Versionfrom importlib.metadata import version as importlib_versionfrom unsloth_zoo.hf_utils import dtype_from_config, HAS_TORCH_DTYPE-from contextlib import contextmanager+from contextlib import contextmanager, nullcontextfrom unsloth_zoo.llama_cpp import (convert_to_gguf,quantize_gguf,@@ -67,7 +67,7 @@ from .models.loader_utils import (_tokenizer_revision,_tokenizer_wants_local_only,)-from .models._utils import _convert_torchao_model+from .models._utils import _convert_torchao_model, lora_relative_to_original_basefrom .models.mistral_format import raise_if_merging_mistral_format_viewfrom .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPERfrom .device_type import clean_gpu_cache@@ -5702,19 +5702,22 @@ def unsloth_generic_save(_prewarm_base_model_hub_cache(model, save_method = save_method, token = token)from unsloth_zoo.saving_utils import merge_and_overwrite_lora- merge_and_overwrite_lora(- get_model_name,- model = model,- tokenizer = tokenizer,- save_directory = save_directory,- push_to_hub = push_to_hub,- private = private,- token = token,- save_method = save_method,- output_dtype = None,- low_disk_space_usage = True,- use_temp_file = False,- )+ # merged_4bit merges into the loaded (already residual) weights, so it needs no conversion.+ in_place = save_method in ("merged_4bit", "forced_merged_4bit")+ with nullcontext() if in_place else lora_relative_to_original_base(model):+ merge_and_overwrite_lora(+ get_model_name,+ model = model,+ tokenizer = tokenizer,+ save_directory = save_directory,+ push_to_hub = push_to_hub,+ private = private,+ token = token,+ save_method = save_method,+ output_dtype = None,+ low_disk_space_usage = True,+ use_temp_file = False,+ )if push_to_hub and datasets:try:diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 3b87034cddf..4fdccd2979a 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -452,7 +452,10 @@ def run_attention(q_len = context.q_lenhead_dim = context.head_dimkv_seq_len = context.kv_seq_len- requires_grad = context.requires_grad+ # Eval forwards can backpropagate (LoRA-GA calibration); xformers' inference GQA layout cannot.+ requires_grad = context.requires_grad or (+ torch.is_grad_enabled() and (Q.requires_grad or K.requires_grad or V.requires_grad)+ )sliding_window = context.sliding_window# A non-positive window means "no local attention", not "a window of nothing": a config spelling# it 0 would put the mask's lower bound above its causal upper bound.