Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: seed the LoRA init with the Random Seed setting (#13637)
Nilay9f606d6fa1cc
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex d85b591cae7..f90825623be 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -1771,6 +1771,7 @@ class UnslothTrainer:use_rslora,use_loftq,use_dora,+ random_state = 3407,) -> bool:"""LoRA for a text-diffusion model through its profile. Studio's default target list namesautoregressive projections (LLaDA has none of o/gate/down_proj, and the MoE families must keep their@@ -1819,7 +1820,7 @@ class UnslothTrainer:lora_dropout = lora_dropout,bias = "none",use_gradient_checkpointing = bool(use_gradient_checkpointing),- random_state = 3407,+ random_state = random_state,)if modules_to_save:peft_kwargs["modules_to_save"] = modules_to_save@@ -1849,6 +1850,7 @@ class UnslothTrainer:use_loftq: bool = False,use_dora: bool = False,modules_to_save: list = None,+ random_state: int = 3407,) -> bool:try:if self.model is None:@@ -1916,6 +1918,7 @@ class UnslothTrainer:use_rslora = use_rslora,use_loftq = use_loftq,use_dora = use_dora,+ random_state = random_state,)# LoRA/QLoRA. "all-linear" is a PEFT keyword targeting every linear layer.@@ -1977,7 +1980,7 @@ class UnslothTrainer:lora_dropout = lora_dropout,bias = "none",use_gradient_checkpointing = use_gradient_checkpointing,- random_state = 3407,+ random_state = random_state,use_rslora = use_rslora,use_dora = use_dora,**_loftq_peft_kwargs(use_loftq),@@ -2009,7 +2012,7 @@ class UnslothTrainer:lora_dropout = lora_dropout,bias = "none",use_gradient_checkpointing = use_gradient_checkpointing,- random_state = 3407,+ random_state = random_state,use_rslora = use_rslora,use_dora = use_dora,**_loftq_peft_kwargs(use_loftq),@@ -2029,7 +2032,7 @@ class UnslothTrainer:lora_dropout = lora_dropout,bias = "none",use_gradient_checkpointing = use_gradient_checkpointing,- random_state = 3407,+ random_state = random_state,use_rslora = use_rslora,use_dora = use_dora,**_loftq_peft_kwargs(use_loftq),@@ -2054,7 +2057,7 @@ class UnslothTrainer:lora_dropout = lora_dropout,bias = "none",use_gradient_checkpointing = use_gradient_checkpointing,- random_state = 3407,+ random_state = random_state,use_rslora = use_rslora,use_dora = use_dora,**_loftq_peft_kwargs(use_loftq),@@ -2075,7 +2078,7 @@ class UnslothTrainer:lora_dropout = lora_dropout,bias = "none",use_gradient_checkpointing = use_gradient_checkpointing,- random_state = 3407,+ random_state = random_state,use_rslora = use_rslora,use_dora = use_dora,**_loftq_peft_kwargs(use_loftq),diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex dd85adf29ad..cf090737c7d 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -845,6 +845,7 @@ class _MLXTrainerAdapter:use_rslora: bool = False,use_loftq: bool = False,use_dora: bool = False,+ random_state: int = 3407,) -> bool:if use_gradient_checkpointing is _UNSET:# This entry overrides load_model's, so default to the mode recorded there.diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 01ea659d1bb..d21d37c5865 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -4940,6 +4940,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->use_rslora = config.get("use_rslora", False),use_loftq = _lora_uses_loftq(config),use_dora = config.get("use_dora", False),+ random_state = config.get("random_seed", 3407),)elif use_lora:_send_status(event_queue, "Configuring LoRA adapters...")@@ -4957,6 +4958,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->use_rslora = config.get("use_rslora", False),use_loftq = _lora_uses_loftq(config),use_dora = config.get("use_dora", False),+ random_state = config.get("random_seed", 3407),)else:_send_status(event_queue, "Preparing model for full finetuning...")diff --git a/studio/backend/tests/test_lora_init_random_seed.py b/studio/backend/tests/test_lora_init_random_seed.pynew file mode 100644index 00000000000..33a29901a5f--- /dev/null+++ b/studio/backend/tests/test_lora_init_random_seed.py@@ -0,0 +1,123 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import sys+from pathlib import Path++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++import ast # noqa: E402+import importlib # noqa: E402+import types # noqa: E402+from unittest.mock import MagicMock # noqa: E402++import pytest # noqa: E402+++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++from core.training import trainer as tmod # noqa: E402++for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)++WORKER = _BACKEND / "core" / "training" / "worker.py"++BRANCHES = {+ "text": {},+ "vlm": {"is_vlm": True},+ "audio_vlm": {"is_audio_vlm": True},+ "csm": {"_audio_type": "csm"},+ "whisper": {"_audio_type": "whisper"},+ "snac": {"_audio_type": "snac"},+ "diffusion": {"diffusion_lm_profile": "llada"},+}+++def peft_calls(monkeypatch):+ calls = []++ class _FakeFast:+ @staticmethod+ def get_peft_model(model, **kwargs):+ calls.append(kwargs)+ return model++ fake_unsloth = types.ModuleType("unsloth")+ fake_unsloth.__spec__ = None+ fake_unsloth.FastModel = _FakeFast+ monkeypatch.setitem(sys.modules, "unsloth", fake_unsloth)+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)+ monkeypatch.setattr(tmod, "FastLanguageModel", _FakeFast)+ monkeypatch.setattr(tmod, "FastVisionModel", _FakeFast)+ return calls++[email protected]("branch", list(BRANCHES))+def test_lora_init_uses_the_requested_seed(peft_calls, branch):+ t = tmod.UnslothTrainer()+ t.model = types.SimpleNamespace(config = types.SimpleNamespace(), named_modules = lambda: iter(()))+ for attr, value in BRANCHES[branch].items():+ setattr(t, attr, value)++ assert t.prepare_model_for_training(use_lora = True, random_state = 42)++ assert [c["random_state"] for c in peft_calls] == [42]+++def test_lora_init_seed_defaults_to_3407(peft_calls):+ t = tmod.UnslothTrainer()+ t.model = types.SimpleNamespace(config = types.SimpleNamespace())++ assert t.prepare_model_for_training(use_lora = True)++ assert [c["random_state"] for c in peft_calls] == [3407]+++def test_worker_seeds_every_lora_adapter_from_random_seed():+ tree = ast.parse(WORKER.read_text(encoding = "utf-8"))+ lora_calls = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and getattr(node.func, "attr", None) == "prepare_model_for_training"+ and any(+ k.arg == "use_lora" and isinstance(k.value, ast.Constant) and k.value.value is True+ for k in node.keywords+ )+ ]+ assert len(lora_calls) >= 2+ for call in lora_calls:+ seed = next((k.value for k in call.keywords if k.arg == "random_state"), None)+ assert seed is not None and "random_seed" in ast.unparse(+ seed+ ), f"worker.py:{call.lineno} builds a LoRA adapter without the run's random_seed"diff --git a/unsloth_cli/config.py b/unsloth_cli/config.pyindex fe025928ba6..91d4fbe1191 100644--- a/unsloth_cli/config.py+++ b/unsloth_cli/config.py@@ -144,6 +144,7 @@ class Config(BaseModel):"use_rslora": self.lora.use_rslora,"use_loftq": self.lora.use_loftq,"use_dora": self.lora.use_dora,+ "random_state": self.training.random_seed,}def training_kwargs(self) -> dict:diff --git a/unsloth_cli/tests/test_train_random_seed.py b/unsloth_cli/tests/test_train_random_seed.pynew file mode 100644index 00000000000..9b25042e727--- /dev/null+++ b/unsloth_cli/tests/test_train_random_seed.py@@ -0,0 +1,71 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import sys+from pathlib import Path+from types import SimpleNamespace++import typer+from typer.testing import CliRunner++_REPO_ROOT = Path(__file__).resolve().parents[2]+if str(_REPO_ROOT) not in sys.path:+ sys.path.insert(0, str(_REPO_ROOT))++import unsloth_cli.commands.train as train_module # noqa: E402+from unsloth_cli.config import Config # noqa: E402+++class _RecordingTrainer:+ is_vlm = False+ training_thread = None++ def __init__(self):+ self.prepare_kwargs = None++ def load_model(self, **kwargs):+ return True++ def prepare_model_for_training(self, **kwargs):+ self.prepare_kwargs = kwargs+ return True++ def load_and_format_dataset(self, **kwargs):+ return [], None++ def start_training(self, **kwargs):+ return True++ def get_training_progress(self):+ return SimpleNamespace(error = None, is_completed = True)+++def test_random_seed_flag_seeds_the_lora_init(tmp_path, monkeypatch):+ config = tmp_path / "config.yaml"+ config.write_text(+ "model: unsloth/Qwen3-0.6B\ndata:\n dataset: yahma/alpaca-cleaned\n",+ encoding = "utf-8",+ )+ trainer = _RecordingTrainer()+ monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *args: trainer)+ app = typer.Typer()+ app.command()(train_module.train)++ result = CliRunner().invoke(app, ["--config", str(config), "--random-seed", "42"])++ assert result.exit_code == 0, result.output+ assert trainer.prepare_kwargs["random_state"] == 42+++def test_mlx_adapter_accepts_the_cli_lora_kwargs():+ backend = str(_REPO_ROOT / "studio" / "backend")+ if backend not in sys.path:+ sys.path.insert(0, backend)+ from core.training.training import create_mlx_trainer_adapter++ adapter = create_mlx_trainer_adapter()+ adapter._model_config = {}++ assert adapter.prepare_model_for_training(**Config().model_kwargs(True, False))