Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run LoftQ when the LoftQ variant is selected (#13535)

Daniel Han882293f503ca
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index 62405a6ed8b..7828e7dbc44 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -411,6 +411,28 @@ def _tokenizer_is_empty(tokenizer) -> bool:
return False
+LOFTQ_SKIPPED_ON_QUANTIZED = (
+ "LoftQ needs 16-bit weights and this model is loaded quantized, so it trains without LoftQ. "
+ "Choose LoRA (16-bit) to use LoftQ."
+)
+
+
+def _has_quantized_weights(model) -> bool:
+ try:
+ from unsloth.models._utils import _has_quantized_linears
+ except ImportError:
+ return False
+ # routed_ok mirrors Unsloth's own LoftQ check: its loader densifies routed NVFP4 / FP8 weights.
+ return _has_quantized_linears(model, routed_ok = True)
+
+
+def _loftq_peft_kwargs(use_loftq) -> dict:
+ # PEFT ignores loftq_config unless init_lora_weights is "loftq", so the toggle alone trained plain LoRA.
+ if not use_loftq:
+ return {"loftq_config": None}
+ return {"init_lora_weights": "loftq", "loftq_config": {"loftq_bits": 4, "loftq_iter": 1}}
+
+
def _load_adapter_aware_tokenizer(lookup_name, hf_token, **kwargs):
"""AutoTokenizer for *lookup_name*. A PEFT adapter saved without tokenizer files falls back
to the base it names, as Unsloth's own adapter loader does (#13140)."""
@@ -1621,6 +1643,12 @@ class UnslothTrainer:
)
return True
+ if use_loftq and _has_quantized_weights(self.model):
+ # Unsloth refuses LoftQ on quantized weights; train as before rather than fail a QLoRA run or its resume.
+ logger.warning(LOFTQ_SKIPPED_ON_QUANTIZED)
+ self._update_progress(status_message = LOFTQ_SKIPPED_ON_QUANTIZED)
+ use_loftq = False
+
# LoRA/QLoRA. "all-linear" is a PEFT keyword targeting every linear layer.
if isinstance(target_modules, list) and "all-linear" in target_modules:
if len(target_modules) == 1:
@@ -1683,7 +1711,7 @@ class UnslothTrainer:
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
+ **_loftq_peft_kwargs(use_loftq),
**self._offload_peft_kwargs(),
)
if self.is_audio_vlm:
@@ -1715,7 +1743,7 @@ class UnslothTrainer:
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
+ **_loftq_peft_kwargs(use_loftq),
task_type = None,
)
@@ -1735,7 +1763,7 @@ class UnslothTrainer:
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
+ **_loftq_peft_kwargs(use_loftq),
)
elif self.is_vlm:
@@ -1760,7 +1788,7 @@ class UnslothTrainer:
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
+ **_loftq_peft_kwargs(use_loftq),
modules_to_save = modules_to_save,
**self._offload_peft_kwargs(),
)
@@ -1781,7 +1809,7 @@ class UnslothTrainer:
random_state = 3407,
use_rslora = use_rslora,
use_dora = use_dora,
- loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
+ **_loftq_peft_kwargs(use_loftq),
modules_to_save = modules_to_save,
**self._offload_peft_kwargs(),
)
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 9c01fed3f93..b20d3c60637 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -203,6 +203,26 @@ def _untrainable_model_format_error(config: dict) -> str | None:
return None
+def _lora_uses_loftq(config: dict) -> bool:
+ """A resume rebuilds its adapter before loading the checkpoint, so it reruns LoftQ only when the checkpoint was
+ trained with it: runs saved while Studio never applied LoftQ kept the original base weights."""
+ if not config.get("use_loftq"):
+ return False
+ resume = config.get("resume_from_checkpoint")
+ if not resume:
+ return True
+ import json
+
+ from core.training.resume import get_resume_checkpoint_path
+
+ checkpoint = Path(get_resume_checkpoint_path(resume) or resume)
+ try:
+ saved = json.loads((checkpoint / "adapter_config.json").read_text(encoding = "utf-8"))
+ except (OSError, ValueError):
+ return False
+ return isinstance(saved, dict) and saved.get("init_lora_weights") == "loftq"
+
+
def _resolve_cached_model_load_name(config: dict) -> str:
return config.get("model_snapshot_path") or config["model_name"]
@@ -4804,7 +4824,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
lora_dropout = config.get("lora_dropout", 0.0),
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
use_rslora = config.get("use_rslora", False),
- use_loftq = config.get("use_loftq", False),
+ use_loftq = _lora_uses_loftq(config),
use_dora = config.get("use_dora", False),
)
elif use_lora:
@@ -4821,7 +4841,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
lora_dropout = config.get("lora_dropout", 0.0),
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
use_rslora = config.get("use_rslora", False),
- use_loftq = config.get("use_loftq", False),
+ use_loftq = _lora_uses_loftq(config),
use_dora = config.get("use_dora", False),
)
else:
@@ -5410,6 +5430,18 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->
if gradient_checkpointing in ("none", "", None):
gradient_checkpointing = False
+ from core.training.trainer import (
+ LOFTQ_SKIPPED_ON_QUANTIZED,
+ _has_quantized_weights,
+ _loftq_peft_kwargs,
+ )
+
+ use_loftq = _lora_uses_loftq(config)
+ # The fast encoder prepares an intrinsically quantized repo for k-bit training, and LoftQ fails on it.
+ if use_loftq and _has_quantized_weights(model):
+ _send_status(event_queue, LOFTQ_SKIPPED_ON_QUANTIZED)
+ use_loftq = False
+
model = FastSentenceTransformer.get_peft_model(
model,
r = config.get("lora_r", 32),
@@ -5422,9 +5454,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->
random_state = config.get("random_seed", 3407),
use_rslora = config.get("use_rslora", False),
use_dora = config.get("use_dora", False),
- loftq_config = {"loftq_bits": 4, "loftq_iter": 1}
- if config.get("use_loftq")
- else None,
+ **_loftq_peft_kwargs(use_loftq),
task_type = "FEATURE_EXTRACTION",
)
except Exception as e:
diff --git a/studio/backend/tests/test_audio_lora_logging.py b/studio/backend/tests/test_audio_lora_logging.py
index 13cc9a62472..5d9ba7eaa2f 100644
--- a/studio/backend/tests/test_audio_lora_logging.py
+++ b/studio/backend/tests/test_audio_lora_logging.py
@@ -16,11 +16,14 @@ import structlog
def _module_level_dependencies(tree):
"""Defaults and annotations are evaluated at def time, so exec needs the module-level
- names the signature reads."""
+ names the signature reads, plus the helpers the LoRA setup calls."""
return [
node
for node in tree.body
- if (isinstance(node, ast.FunctionDef) and node.name == "normalize_gradient_checkpointing")
+ if (
+ isinstance(node, ast.FunctionDef)
+ and node.name in ("normalize_gradient_checkpointing", "_loftq_peft_kwargs")
+ )
or (
isinstance(node, ast.Assign)
and any(isinstance(t, ast.Name) and t.id == "_UNSET" for t in node.targets)
diff --git a/studio/backend/tests/test_training_loftq.py b/studio/backend/tests/test_training_loftq.py
new file mode 100644
index 00000000000..9318e6e1386
--- /dev/null
+++ b/studio/backend/tests/test_training_loftq.py
@@ -0,0 +1,193 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The LoftQ variant must initialize the adapter with LoftQ wherever the weights allow it.
+
+PEFT drops loftq_config unless init_lora_weights is "loftq", so Studio sent the config alone and
+every LoftQ run trained plain LoRA.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+from typing import Union
+from unittest.mock import Mock
+
+import pytest
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+_TRAINER = _BACKEND_ROOT / "core" / "training" / "trainer.py"
+_WORKER = _BACKEND_ROOT / "core" / "training" / "worker.py"
+_SENTENCE_TRANSFORMER = _BACKEND_ROOT.parents[1] / "unsloth" / "models" / "sentence_transformer.py"
+
+
+def _functions(
+ path,
+ *names,
+ namespace = None,
+):
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ body = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]
+ namespace = {} if namespace is None else namespace
+ exec(compile(ast.Module(body = body, type_ignores = []), str(path), "exec"), namespace)
+ return namespace
+
+
+def test_loftq_turns_on_the_loftq_initialization():
+ kwargs = _functions(_TRAINER, "_loftq_peft_kwargs")["_loftq_peft_kwargs"](True)
+
+ assert kwargs["init_lora_weights"] == "loftq"
+ assert kwargs["loftq_config"] == {"loftq_bits": 4, "loftq_iter": 1}
+
+
+def test_plain_lora_keeps_the_default_initialization():
+ loftq_kwargs = _functions(_TRAINER, "_loftq_peft_kwargs")["_loftq_peft_kwargs"]
+
+ assert loftq_kwargs(False) == {"loftq_config": None}
+
+
[email protected]("path", [_TRAINER, _WORKER], ids = lambda p: p.name)
+def test_no_get_peft_model_call_passes_loftq_config_alone(path):
+ for node in ast.walk(ast.parse(path.read_text(encoding = "utf-8"))):
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "get_peft_model"
+ ):
+ names = {k.arg for k in node.keywords}
+ assert (
+ "loftq_config" not in names or "init_lora_weights" in names
+ ), f"{path.name}:{node.lineno} passes loftq_config without init_lora_weights"
+
+
+def test_no_worker_call_reads_the_toggle_directly():
+ # The persisted toggle alone would rerun LoftQ on a resume saved before LoftQ applied.
+ tree = ast.parse(_WORKER.read_text(encoding = "utf-8"))
+ helper = next(
+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_lora_uses_loftq"
+ )
+ inside = {id(n) for n in ast.walk(helper)}
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call) or id(node) in inside:
+ continue
+ for value in [*node.args, *(k.value for k in node.keywords)]:
+ for sub in ast.walk(value):
+ reads_toggle = (
+ isinstance(sub, ast.Call)
+ and getattr(sub.func, "attr", None) == "get"
+ and any(
+ isinstance(a, ast.Constant) and a.value == "use_loftq" for a in sub.args
+ )
+ )
+ assert (
+ not reads_toggle
+ ), f"worker.py:{node.lineno} passes use_loftq on without _lora_uses_loftq"
+
+
+def test_the_fast_encoder_lora_config_forwards_the_initialization():
+ # Encoder embedding models (MiniLM, BGE, ModernBERT) build their own LoraConfig.
+ for node in ast.walk(ast.parse(_SENTENCE_TRANSFORMER.read_text(encoding = "utf-8"))):
+ if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "LoraConfig":
+ names = {k.arg for k in node.keywords}
+ assert {"init_lora_weights", "loftq_config"} <= names, f"line {node.lineno}"
+
+
+def _prepare(monkeypatch, quantized):
+ tree = ast.parse(_TRAINER.read_text(encoding = "utf-8"))
+ trainer = next(
+ n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "UnslothTrainer"
+ )
+ method = next(
+ n
+ for n in trainer.body
+ if isinstance(n, ast.FunctionDef) and n.name == "prepare_model_for_training"
+ )
+ helpers = [
+ n
+ for n in tree.body
+ if (
+ isinstance(n, ast.FunctionDef)
+ and n.name in ("normalize_gradient_checkpointing", "_loftq_peft_kwargs")
+ )
+ or (
+ isinstance(n, ast.Assign)
+ and any(
+ isinstance(t, ast.Name) and t.id in ("_UNSET", "LOFTQ_SKIPPED_ON_QUANTIZED")
+ for t in n.targets
+ )
+ )
+ ]
+ adapter = Mock(return_value = SimpleNamespace(config = object()))
+ namespace = {
+ "logger": Mock(),
+ "Union": Union,
+ "FastLanguageModel": SimpleNamespace(get_peft_model = adapter),
+ "_has_quantized_weights": lambda model: quantized,
+ }
+ monkeypatch.setitem(sys.modules, "unsloth", ModuleType("unsloth"))
+ exec(
+ compile(ast.Module(body = helpers + [method], type_ignores = []), str(_TRAINER), "exec"),
+ namespace,
+ )
+ progress = Mock()
+ instance = SimpleNamespace(
+ model = SimpleNamespace(config = object()),
+ _audio_type = None,
+ is_audio_vlm = False,
+ is_vlm = False,
+ should_stop = False,
+ _update_progress = progress,
+ _use_gradient_checkpointing = "unsloth",
+ _offload_peft_kwargs = lambda: {},
+ )
+ assert namespace["prepare_model_for_training"](instance, use_loftq = True) is True
+ statuses = [c.kwargs.get("status_message") or "" for c in progress.call_args_list]
+ return adapter.call_args.kwargs, statuses
+
+
+def test_loftq_runs_on_dense_weights(monkeypatch):
+ kwargs, statuses = _prepare(monkeypatch, quantized = False)
+
+ assert kwargs["init_lora_weights"] == "loftq"
+ assert not any("without LoftQ" in s for s in statuses)
+
+
+def test_a_quantized_load_trains_without_loftq_and_says_so(monkeypatch):
+ kwargs, statuses = _prepare(monkeypatch, quantized = True)
+
+ assert "init_lora_weights" not in kwargs
+ assert kwargs["loftq_config"] is None
+ assert any("without LoftQ" in s for s in statuses)
+
+
+def _uses_loftq(config):
+ worker = _functions(_WORKER, "_lora_uses_loftq", namespace = {"Path": Path})
+ return worker["_lora_uses_loftq"](config)
+
+
+def _checkpoint(folder, init):
+ folder.mkdir(parents = True)
+ (folder / "adapter_config.json").write_text(
+ json.dumps({"init_lora_weights": init}), encoding = "utf-8"
+ )
+ return str(folder)
+
+
+def test_a_new_run_follows_the_toggle():
+ assert _uses_loftq({"use_loftq": True}) is True
+ assert _uses_loftq({"use_loftq": False}) is False
+
+
+def test_a_resume_reruns_loftq_only_for_a_loftq_checkpoint(tmp_path):
+ loftq = _checkpoint(tmp_path / "loftq" / "checkpoint-30", "loftq")
+ plain = _checkpoint(tmp_path / "plain" / "checkpoint-30", True)
+
+ assert _uses_loftq({"use_loftq": True, "resume_from_checkpoint": loftq}) is True
+ # Saved before Studio applied LoftQ: the checkpoint trained on the original base weights.
+ assert _uses_loftq({"use_loftq": True, "resume_from_checkpoint": plain}) is False
+ gone = str(tmp_path / "gone")
+ assert _uses_loftq({"use_loftq": True, "resume_from_checkpoint": gone}) is False
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index 99ad8940d70..66667906698 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -3025,6 +3025,8 @@ class FastSentenceTransformer(FastModel):
lora_dropout = lora_dropout,
bias = bias,
use_dora = use_dora,
+ init_lora_weights = init_lora_weights,
+ loftq_config = loftq_config or {},
task_type = kwargs.get("task_type", "FEATURE_EXTRACTION"),
)