Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: seed the LoRA init with the Random Seed setting (#13637)

Nilay9f606d6fa1cc
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index d85b591cae7..f90825623be 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -1771,6 +1771,7 @@ class UnslothTrainer:
use_rslora,
use_loftq,
use_dora,
+ random_state = 3407,
) -> bool:
"""LoRA for a text-diffusion model through its profile. Studio's default target list names
autoregressive projections (LLaDA has none of o/gate/down_proj, and the MoE families must keep their
@@ -1819,7 +1820,7 @@ class UnslothTrainer:
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = bool(use_gradient_checkpointing),
- random_state = 3407,
+ random_state = random_state,
)
if modules_to_save:
peft_kwargs["modules_to_save"] = modules_to_save
@@ -1849,6 +1850,7 @@ class UnslothTrainer:
use_loftq: bool = False,
use_dora: bool = False,
modules_to_save: list = None,
+ random_state: int = 3407,
) -> bool:
try:
if self.model is None:
@@ -1916,6 +1918,7 @@ class UnslothTrainer:
use_rslora = use_rslora,
use_loftq = use_loftq,
use_dora = use_dora,
+ random_state = random_state,
)
# LoRA/QLoRA. "all-linear" is a PEFT keyword targeting every linear layer.
@@ -1977,7 +1980,7 @@ class UnslothTrainer:
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
- random_state = 3407,
+ random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
@@ -2009,7 +2012,7 @@ class UnslothTrainer:
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
- random_state = 3407,
+ random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
@@ -2029,7 +2032,7 @@ class UnslothTrainer:
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
- random_state = 3407,
+ random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
@@ -2054,7 +2057,7 @@ class UnslothTrainer:
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
- random_state = 3407,
+ random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
@@ -2075,7 +2078,7 @@ class UnslothTrainer:
lora_dropout = lora_dropout,
bias = "none",
use_gradient_checkpointing = use_gradient_checkpointing,
- random_state = 3407,
+ random_state = random_state,
use_rslora = use_rslora,
use_dora = use_dora,
**_loftq_peft_kwargs(use_loftq),
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index dd85adf29ad..cf090737c7d 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -845,6 +845,7 @@ class _MLXTrainerAdapter:
use_rslora: bool = False,
use_loftq: bool = False,
use_dora: bool = False,
+ random_state: int = 3407,
) -> bool:
if use_gradient_checkpointing is _UNSET:
# This entry overrides load_model's, so default to the mode recorded there.
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 01ea659d1bb..d21d37c5865 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -4940,6 +4940,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
use_rslora = config.get("use_rslora", False),
use_loftq = _lora_uses_loftq(config),
use_dora = config.get("use_dora", False),
+ random_state = config.get("random_seed", 3407),
)
elif use_lora:
_send_status(event_queue, "Configuring LoRA adapters...")
@@ -4957,6 +4958,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
use_rslora = config.get("use_rslora", False),
use_loftq = _lora_uses_loftq(config),
use_dora = config.get("use_dora", False),
+ random_state = config.get("random_seed", 3407),
)
else:
_send_status(event_queue, "Preparing model for full finetuning...")
diff --git a/studio/backend/tests/test_lora_init_random_seed.py b/studio/backend/tests/test_lora_init_random_seed.py
new file mode 100644
index 00000000000..33a29901a5f
--- /dev/null
+++ b/studio/backend/tests/test_lora_init_random_seed.py
@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+import ast # noqa: E402
+import importlib # noqa: E402
+import types # noqa: E402
+from unittest.mock import MagicMock # noqa: E402
+
+import pytest # noqa: E402
+
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+from core.training import trainer as tmod # noqa: E402
+
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+WORKER = _BACKEND / "core" / "training" / "worker.py"
+
+BRANCHES = {
+ "text": {},
+ "vlm": {"is_vlm": True},
+ "audio_vlm": {"is_audio_vlm": True},
+ "csm": {"_audio_type": "csm"},
+ "whisper": {"_audio_type": "whisper"},
+ "snac": {"_audio_type": "snac"},
+ "diffusion": {"diffusion_lm_profile": "llada"},
+}
+
+
+def peft_calls(monkeypatch):
+ calls = []
+
+ class _FakeFast:
+ @staticmethod
+ def get_peft_model(model, **kwargs):
+ calls.append(kwargs)
+ return model
+
+ fake_unsloth = types.ModuleType("unsloth")
+ fake_unsloth.__spec__ = None
+ fake_unsloth.FastModel = _FakeFast
+ monkeypatch.setitem(sys.modules, "unsloth", fake_unsloth)
+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)
+ monkeypatch.setattr(tmod, "FastLanguageModel", _FakeFast)
+ monkeypatch.setattr(tmod, "FastVisionModel", _FakeFast)
+ return calls
+
+
[email protected]("branch", list(BRANCHES))
+def test_lora_init_uses_the_requested_seed(peft_calls, branch):
+ t = tmod.UnslothTrainer()
+ t.model = types.SimpleNamespace(config = types.SimpleNamespace(), named_modules = lambda: iter(()))
+ for attr, value in BRANCHES[branch].items():
+ setattr(t, attr, value)
+
+ assert t.prepare_model_for_training(use_lora = True, random_state = 42)
+
+ assert [c["random_state"] for c in peft_calls] == [42]
+
+
+def test_lora_init_seed_defaults_to_3407(peft_calls):
+ t = tmod.UnslothTrainer()
+ t.model = types.SimpleNamespace(config = types.SimpleNamespace())
+
+ assert t.prepare_model_for_training(use_lora = True)
+
+ assert [c["random_state"] for c in peft_calls] == [3407]
+
+
+def test_worker_seeds_every_lora_adapter_from_random_seed():
+ tree = ast.parse(WORKER.read_text(encoding = "utf-8"))
+ lora_calls = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "attr", None) == "prepare_model_for_training"
+ and any(
+ k.arg == "use_lora" and isinstance(k.value, ast.Constant) and k.value.value is True
+ for k in node.keywords
+ )
+ ]
+ assert len(lora_calls) >= 2
+ for call in lora_calls:
+ seed = next((k.value for k in call.keywords if k.arg == "random_state"), None)
+ assert seed is not None and "random_seed" in ast.unparse(
+ seed
+ ), f"worker.py:{call.lineno} builds a LoRA adapter without the run's random_seed"
diff --git a/unsloth_cli/config.py b/unsloth_cli/config.py
index fe025928ba6..91d4fbe1191 100644
--- a/unsloth_cli/config.py
+++ b/unsloth_cli/config.py
@@ -144,6 +144,7 @@ class Config(BaseModel):
"use_rslora": self.lora.use_rslora,
"use_loftq": self.lora.use_loftq,
"use_dora": self.lora.use_dora,
+ "random_state": self.training.random_seed,
}
def training_kwargs(self) -> dict:
diff --git a/unsloth_cli/tests/test_train_random_seed.py b/unsloth_cli/tests/test_train_random_seed.py
new file mode 100644
index 00000000000..9b25042e727
--- /dev/null
+++ b/unsloth_cli/tests/test_train_random_seed.py
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import typer
+from typer.testing import CliRunner
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+import unsloth_cli.commands.train as train_module # noqa: E402
+from unsloth_cli.config import Config # noqa: E402
+
+
+class _RecordingTrainer:
+ is_vlm = False
+ training_thread = None
+
+ def __init__(self):
+ self.prepare_kwargs = None
+
+ def load_model(self, **kwargs):
+ return True
+
+ def prepare_model_for_training(self, **kwargs):
+ self.prepare_kwargs = kwargs
+ return True
+
+ def load_and_format_dataset(self, **kwargs):
+ return [], None
+
+ def start_training(self, **kwargs):
+ return True
+
+ def get_training_progress(self):
+ return SimpleNamespace(error = None, is_completed = True)
+
+
+def test_random_seed_flag_seeds_the_lora_init(tmp_path, monkeypatch):
+ config = tmp_path / "config.yaml"
+ config.write_text(
+ "model: unsloth/Qwen3-0.6B\ndata:\n dataset: yahma/alpaca-cleaned\n",
+ encoding = "utf-8",
+ )
+ trainer = _RecordingTrainer()
+ monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *args: trainer)
+ app = typer.Typer()
+ app.command()(train_module.train)
+
+ result = CliRunner().invoke(app, ["--config", str(config), "--random-seed", "42"])
+
+ assert result.exit_code == 0, result.output
+ assert trainer.prepare_kwargs["random_state"] == 42
+
+
+def test_mlx_adapter_accepts_the_cli_lora_kwargs():
+ backend = str(_REPO_ROOT / "studio" / "backend")
+ if backend not in sys.path:
+ sys.path.insert(0, backend)
+ from core.training.training import create_mlx_trainer_adapter
+
+ adapter = create_mlx_trainer_adapter()
+ adapter._model_config = {}
+
+ assert adapter.prepare_model_for_training(**Config().model_kwargs(True, False))