Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run LoftQ when the LoftQ variant is selected (#13535)
Daniel Han882293f503ca
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex 62405a6ed8b..7828e7dbc44 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -411,6 +411,28 @@ def _tokenizer_is_empty(tokenizer) -> bool:return False+LOFTQ_SKIPPED_ON_QUANTIZED = (+ "LoftQ needs 16-bit weights and this model is loaded quantized, so it trains without LoftQ. "+ "Choose LoRA (16-bit) to use LoftQ."+)+++def _has_quantized_weights(model) -> bool:+ try:+ from unsloth.models._utils import _has_quantized_linears+ except ImportError:+ return False+ # routed_ok mirrors Unsloth's own LoftQ check: its loader densifies routed NVFP4 / FP8 weights.+ return _has_quantized_linears(model, routed_ok = True)+++def _loftq_peft_kwargs(use_loftq) -> dict:+ # PEFT ignores loftq_config unless init_lora_weights is "loftq", so the toggle alone trained plain LoRA.+ if not use_loftq:+ return {"loftq_config": None}+ return {"init_lora_weights": "loftq", "loftq_config": {"loftq_bits": 4, "loftq_iter": 1}}++def _load_adapter_aware_tokenizer(lookup_name, hf_token, **kwargs):"""AutoTokenizer for *lookup_name*. A PEFT adapter saved without tokenizer files falls backto the base it names, as Unsloth's own adapter loader does (#13140)."""@@ -1621,6 +1643,12 @@ class UnslothTrainer:)return True+ if use_loftq and _has_quantized_weights(self.model):+ # Unsloth refuses LoftQ on quantized weights; train as before rather than fail a QLoRA run or its resume.+ logger.warning(LOFTQ_SKIPPED_ON_QUANTIZED)+ self._update_progress(status_message = LOFTQ_SKIPPED_ON_QUANTIZED)+ use_loftq = False+# LoRA/QLoRA. "all-linear" is a PEFT keyword targeting every linear layer.if isinstance(target_modules, list) and "all-linear" in target_modules:if len(target_modules) == 1:@@ -1683,7 +1711,7 @@ class UnslothTrainer:random_state = 3407,use_rslora = use_rslora,use_dora = use_dora,- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,+ **_loftq_peft_kwargs(use_loftq),**self._offload_peft_kwargs(),)if self.is_audio_vlm:@@ -1715,7 +1743,7 @@ class UnslothTrainer:random_state = 3407,use_rslora = use_rslora,use_dora = use_dora,- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,+ **_loftq_peft_kwargs(use_loftq),task_type = None,)@@ -1735,7 +1763,7 @@ class UnslothTrainer:random_state = 3407,use_rslora = use_rslora,use_dora = use_dora,- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,+ **_loftq_peft_kwargs(use_loftq),)elif self.is_vlm:@@ -1760,7 +1788,7 @@ class UnslothTrainer:random_state = 3407,use_rslora = use_rslora,use_dora = use_dora,- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,+ **_loftq_peft_kwargs(use_loftq),modules_to_save = modules_to_save,**self._offload_peft_kwargs(),)@@ -1781,7 +1809,7 @@ class UnslothTrainer:random_state = 3407,use_rslora = use_rslora,use_dora = use_dora,- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,+ **_loftq_peft_kwargs(use_loftq),modules_to_save = modules_to_save,**self._offload_peft_kwargs(),)diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 9c01fed3f93..b20d3c60637 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -203,6 +203,26 @@ def _untrainable_model_format_error(config: dict) -> str | None:return None+def _lora_uses_loftq(config: dict) -> bool:+ """A resume rebuilds its adapter before loading the checkpoint, so it reruns LoftQ only when the checkpoint was+ trained with it: runs saved while Studio never applied LoftQ kept the original base weights."""+ if not config.get("use_loftq"):+ return False+ resume = config.get("resume_from_checkpoint")+ if not resume:+ return True+ import json++ from core.training.resume import get_resume_checkpoint_path++ checkpoint = Path(get_resume_checkpoint_path(resume) or resume)+ try:+ saved = json.loads((checkpoint / "adapter_config.json").read_text(encoding = "utf-8"))+ except (OSError, ValueError):+ return False+ return isinstance(saved, dict) and saved.get("init_lora_weights") == "loftq"++def _resolve_cached_model_load_name(config: dict) -> str:return config.get("model_snapshot_path") or config["model_name"]@@ -4804,7 +4824,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->lora_dropout = config.get("lora_dropout", 0.0),use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),use_rslora = config.get("use_rslora", False),- use_loftq = config.get("use_loftq", False),+ use_loftq = _lora_uses_loftq(config),use_dora = config.get("use_dora", False),)elif use_lora:@@ -4821,7 +4841,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->lora_dropout = config.get("lora_dropout", 0.0),use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),use_rslora = config.get("use_rslora", False),- use_loftq = config.get("use_loftq", False),+ use_loftq = _lora_uses_loftq(config),use_dora = config.get("use_dora", False),)else:@@ -5410,6 +5430,18 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->if gradient_checkpointing in ("none", "", None):gradient_checkpointing = False+ from core.training.trainer import (+ LOFTQ_SKIPPED_ON_QUANTIZED,+ _has_quantized_weights,+ _loftq_peft_kwargs,+ )++ use_loftq = _lora_uses_loftq(config)+ # The fast encoder prepares an intrinsically quantized repo for k-bit training, and LoftQ fails on it.+ if use_loftq and _has_quantized_weights(model):+ _send_status(event_queue, LOFTQ_SKIPPED_ON_QUANTIZED)+ use_loftq = False+model = FastSentenceTransformer.get_peft_model(model,r = config.get("lora_r", 32),@@ -5422,9 +5454,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->random_state = config.get("random_seed", 3407),use_rslora = config.get("use_rslora", False),use_dora = config.get("use_dora", False),- loftq_config = {"loftq_bits": 4, "loftq_iter": 1}- if config.get("use_loftq")- else None,+ **_loftq_peft_kwargs(use_loftq),task_type = "FEATURE_EXTRACTION",)except Exception as e:diff --git a/studio/backend/tests/test_audio_lora_logging.py b/studio/backend/tests/test_audio_lora_logging.pyindex 13cc9a62472..5d9ba7eaa2f 100644--- a/studio/backend/tests/test_audio_lora_logging.py+++ b/studio/backend/tests/test_audio_lora_logging.py@@ -16,11 +16,14 @@ import structlogdef _module_level_dependencies(tree):"""Defaults and annotations are evaluated at def time, so exec needs the module-level- names the signature reads."""+ names the signature reads, plus the helpers the LoRA setup calls."""return [nodefor node in tree.body- if (isinstance(node, ast.FunctionDef) and node.name == "normalize_gradient_checkpointing")+ if (+ isinstance(node, ast.FunctionDef)+ and node.name in ("normalize_gradient_checkpointing", "_loftq_peft_kwargs")+ )or (isinstance(node, ast.Assign)and any(isinstance(t, ast.Name) and t.id == "_UNSET" for t in node.targets)diff --git a/studio/backend/tests/test_training_loftq.py b/studio/backend/tests/test_training_loftq.pynew file mode 100644index 00000000000..9318e6e1386--- /dev/null+++ b/studio/backend/tests/test_training_loftq.py@@ -0,0 +1,193 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The LoftQ variant must initialize the adapter with LoftQ wherever the weights allow it.++PEFT drops loftq_config unless init_lora_weights is "loftq", so Studio sent the config alone and+every LoftQ run trained plain LoRA.+"""++from __future__ import annotations++import ast+import json+import sys+from pathlib import Path+from types import ModuleType, SimpleNamespace+from typing import Union+from unittest.mock import Mock++import pytest++_BACKEND_ROOT = Path(__file__).resolve().parent.parent+_TRAINER = _BACKEND_ROOT / "core" / "training" / "trainer.py"+_WORKER = _BACKEND_ROOT / "core" / "training" / "worker.py"+_SENTENCE_TRANSFORMER = _BACKEND_ROOT.parents[1] / "unsloth" / "models" / "sentence_transformer.py"+++def _functions(+ path,+ *names,+ namespace = None,+):+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ body = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]+ namespace = {} if namespace is None else namespace+ exec(compile(ast.Module(body = body, type_ignores = []), str(path), "exec"), namespace)+ return namespace+++def test_loftq_turns_on_the_loftq_initialization():+ kwargs = _functions(_TRAINER, "_loftq_peft_kwargs")["_loftq_peft_kwargs"](True)++ assert kwargs["init_lora_weights"] == "loftq"+ assert kwargs["loftq_config"] == {"loftq_bits": 4, "loftq_iter": 1}+++def test_plain_lora_keeps_the_default_initialization():+ loftq_kwargs = _functions(_TRAINER, "_loftq_peft_kwargs")["_loftq_peft_kwargs"]++ assert loftq_kwargs(False) == {"loftq_config": None}++[email protected]("path", [_TRAINER, _WORKER], ids = lambda p: p.name)+def test_no_get_peft_model_call_passes_loftq_config_alone(path):+ for node in ast.walk(ast.parse(path.read_text(encoding = "utf-8"))):+ if (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "get_peft_model"+ ):+ names = {k.arg for k in node.keywords}+ assert (+ "loftq_config" not in names or "init_lora_weights" in names+ ), f"{path.name}:{node.lineno} passes loftq_config without init_lora_weights"+++def test_no_worker_call_reads_the_toggle_directly():+ # The persisted toggle alone would rerun LoftQ on a resume saved before LoftQ applied.+ tree = ast.parse(_WORKER.read_text(encoding = "utf-8"))+ helper = next(+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_lora_uses_loftq"+ )+ inside = {id(n) for n in ast.walk(helper)}+ for node in ast.walk(tree):+ if not isinstance(node, ast.Call) or id(node) in inside:+ continue+ for value in [*node.args, *(k.value for k in node.keywords)]:+ for sub in ast.walk(value):+ reads_toggle = (+ isinstance(sub, ast.Call)+ and getattr(sub.func, "attr", None) == "get"+ and any(+ isinstance(a, ast.Constant) and a.value == "use_loftq" for a in sub.args+ )+ )+ assert (+ not reads_toggle+ ), f"worker.py:{node.lineno} passes use_loftq on without _lora_uses_loftq"+++def test_the_fast_encoder_lora_config_forwards_the_initialization():+ # Encoder embedding models (MiniLM, BGE, ModernBERT) build their own LoraConfig.+ for node in ast.walk(ast.parse(_SENTENCE_TRANSFORMER.read_text(encoding = "utf-8"))):+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "LoraConfig":+ names = {k.arg for k in node.keywords}+ assert {"init_lora_weights", "loftq_config"} <= names, f"line {node.lineno}"+++def _prepare(monkeypatch, quantized):+ tree = ast.parse(_TRAINER.read_text(encoding = "utf-8"))+ trainer = next(+ n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "UnslothTrainer"+ )+ method = next(+ n+ for n in trainer.body+ if isinstance(n, ast.FunctionDef) and n.name == "prepare_model_for_training"+ )+ helpers = [+ n+ for n in tree.body+ if (+ isinstance(n, ast.FunctionDef)+ and n.name in ("normalize_gradient_checkpointing", "_loftq_peft_kwargs")+ )+ or (+ isinstance(n, ast.Assign)+ and any(+ isinstance(t, ast.Name) and t.id in ("_UNSET", "LOFTQ_SKIPPED_ON_QUANTIZED")+ for t in n.targets+ )+ )+ ]+ adapter = Mock(return_value = SimpleNamespace(config = object()))+ namespace = {+ "logger": Mock(),+ "Union": Union,+ "FastLanguageModel": SimpleNamespace(get_peft_model = adapter),+ "_has_quantized_weights": lambda model: quantized,+ }+ monkeypatch.setitem(sys.modules, "unsloth", ModuleType("unsloth"))+ exec(+ compile(ast.Module(body = helpers + [method], type_ignores = []), str(_TRAINER), "exec"),+ namespace,+ )+ progress = Mock()+ instance = SimpleNamespace(+ model = SimpleNamespace(config = object()),+ _audio_type = None,+ is_audio_vlm = False,+ is_vlm = False,+ should_stop = False,+ _update_progress = progress,+ _use_gradient_checkpointing = "unsloth",+ _offload_peft_kwargs = lambda: {},+ )+ assert namespace["prepare_model_for_training"](instance, use_loftq = True) is True+ statuses = [c.kwargs.get("status_message") or "" for c in progress.call_args_list]+ return adapter.call_args.kwargs, statuses+++def test_loftq_runs_on_dense_weights(monkeypatch):+ kwargs, statuses = _prepare(monkeypatch, quantized = False)++ assert kwargs["init_lora_weights"] == "loftq"+ assert not any("without LoftQ" in s for s in statuses)+++def test_a_quantized_load_trains_without_loftq_and_says_so(monkeypatch):+ kwargs, statuses = _prepare(monkeypatch, quantized = True)++ assert "init_lora_weights" not in kwargs+ assert kwargs["loftq_config"] is None+ assert any("without LoftQ" in s for s in statuses)+++def _uses_loftq(config):+ worker = _functions(_WORKER, "_lora_uses_loftq", namespace = {"Path": Path})+ return worker["_lora_uses_loftq"](config)+++def _checkpoint(folder, init):+ folder.mkdir(parents = True)+ (folder / "adapter_config.json").write_text(+ json.dumps({"init_lora_weights": init}), encoding = "utf-8"+ )+ return str(folder)+++def test_a_new_run_follows_the_toggle():+ assert _uses_loftq({"use_loftq": True}) is True+ assert _uses_loftq({"use_loftq": False}) is False+++def test_a_resume_reruns_loftq_only_for_a_loftq_checkpoint(tmp_path):+ loftq = _checkpoint(tmp_path / "loftq" / "checkpoint-30", "loftq")+ plain = _checkpoint(tmp_path / "plain" / "checkpoint-30", True)++ assert _uses_loftq({"use_loftq": True, "resume_from_checkpoint": loftq}) is True+ # Saved before Studio applied LoftQ: the checkpoint trained on the original base weights.+ assert _uses_loftq({"use_loftq": True, "resume_from_checkpoint": plain}) is False+ gone = str(tmp_path / "gone")+ assert _uses_loftq({"use_loftq": True, "resume_from_checkpoint": gone}) is Falsediff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex 99ad8940d70..66667906698 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -3025,6 +3025,8 @@ class FastSentenceTransformer(FastModel):lora_dropout = lora_dropout,bias = bias,use_dora = use_dora,+ init_lora_weights = init_lora_weights,+ loftq_config = loftq_config or {},task_type = kwargs.get("task_type", "FEATURE_EXTRACTION"),)