Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Save each unsloth train run in its own folder (#13484)
Nilaya2aa74a56172
diff --git a/unsloth_cli/commands/train.py b/unsloth_cli/commands/train.pyindex 67106f60c7a..053cd7109a4 100644--- a/unsloth_cli/commands/train.py+++ b/unsloth_cli/commands/train.py@@ -1,6 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import osimport timefrom pathlib import Pathfrom typing import Optional@@ -80,6 +81,96 @@ def _optimizer_for_host(requested: Optional[str] = None) -> str:return normalize_training_optimizer_for_device(optimizer, device_backend = device_backend)+def _launch_rank() -> Optional[int]:+ # A size alone is not a launch: a stale PMI_SIZE or WORLD_SIZE in a shell would pin every run to one folder.+ rank = os.environ.get("RANK")+ size = os.environ.get("WORLD_SIZE")+ if rank is not None and size is not None:+ try:+ rank, size = int(rank), int(size)+ except ValueError:+ return None+ return rank if 0 <= rank < size and size > 1 else None+ is_launch, rank, _size = mlx_distributed_info()+ return rank if is_launch and (_size or 0) > 1 else None+++_RUN_DIR_STORE = None+++def _torch_launch_store(rank: int):+ # torchrun's agent hosts a store at MASTER_ADDR:MASTER_PORT; a plain env:// launch has rank 0 host it, and+ # multi_tenant lets init_process_group reuse this same server instead of failing to bind the port.+ global _RUN_DIR_STORE+ if (+ "RANK" not in os.environ+ or "MASTER_ADDR" not in os.environ+ or "MASTER_PORT" not in os.environ+ ):+ return None+ try:+ from datetime import timedelta++ from torch.distributed import TCPStore++ host = rank == 0 and os.environ.get("TORCHELASTIC_USE_AGENT_STORE") != "True"+ store = TCPStore(+ os.environ["MASTER_ADDR"],+ int(os.environ["MASTER_PORT"]),+ is_master = host,+ timeout = timedelta(minutes = 30),+ multi_tenant = True,+ )+ except Exception: # noqa: BLE001 -- no store reachable: fall back to the launcher's start time+ return None+ if host:+ # Keep the server alive for the other ranks until init_process_group takes it over.+ _RUN_DIR_STORE = store+ return store+++def _launcher_start_time() -> int:+ # Ranks reach this seconds apart after their own model load; their shared launcher started once.+ try:+ import psutil+ return int(psutil.Process(os.getppid()).create_time())+ except Exception: # noqa: BLE001 -- no launcher to ask: fall back to this rank's clock+ return int(time.time())+++def _reserve_run_dir(name: str) -> str:+ from utils.paths import ensure_dir, resolve_output_dir+ stamp = int(time.time())+ while True:+ path = resolve_output_dir(f"{name}_{stamp}")+ ensure_dir(path.parent)+ try:+ path.mkdir()+ return str(path)+ except FileExistsError:+ stamp += 1+++def _default_output_dir(model_name: str) -> str:+ ensure_studio_backend_path()+ from utils.paths import default_run_dir_name, resolve_output_dir++ name = default_run_dir_name(model_name)+ rank = _launch_rank()+ if rank is None:+ return _reserve_run_dir(name)+ store = _torch_launch_store(rank)+ if store is None:+ # mlx.launch / mpirun ranks never write the folder, so rank 0 reserves one alone.+ if rank == 0 and "RANK" not in os.environ:+ return _reserve_run_dir(name)+ return str(resolve_output_dir(f"{name}_{_launcher_start_time()}"))+ key = "unsloth_cli/output_dir"+ if rank == 0 and not store.check([key]):+ store.set(key, _reserve_run_dir(name))+ return store.get(key).decode()++@add_options_from_config(Config)def train(config: Optional[Path] = typer.Option(@@ -187,6 +278,8 @@ def train(ds, eval_ds = resulttraining_kwargs = cfg.training_kwargs()+ if "output_dir" not in cfg.training.model_fields_set:+ training_kwargs["output_dir"] = _default_output_dir(cfg.model)training_kwargs["wandb_token"] = wandb_tokentraining_kwargs["optim"] = _optimizer_for_host(training_kwargs.get("optim"))started = trainer.start_training(dataset = ds, eval_dataset = eval_ds, **training_kwargs)diff --git a/unsloth_cli/tests/test_train_ctrl_c_exit_code.py b/unsloth_cli/tests/test_train_ctrl_c_exit_code.pyindex 53b133ecfca..571363e1e90 100644--- a/unsloth_cli/tests/test_train_ctrl_c_exit_code.py+++ b/unsloth_cli/tests/test_train_ctrl_c_exit_code.py@@ -76,8 +76,9 @@ def _run(tmp_path, monkeypatch, trainer, sleep):"model: unsloth/Qwen3-0.6B\ndata:\n dataset: yahma/alpaca-cleaned\n",encoding = "utf-8",)+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *args: trainer)- monkeypatch.setattr(train_module, "time", SimpleNamespace(sleep = sleep))+ monkeypatch.setattr(train_module, "time", SimpleNamespace(sleep = sleep, time = time.time))app = typer.Typer()app.command()(train_module.train)return CliRunner().invoke(app, ["--config", str(config)])diff --git a/unsloth_cli/tests/test_train_default_output_dir.py b/unsloth_cli/tests/test_train_default_output_dir.pynew file mode 100644index 00000000000..ecfc0b8bbff--- /dev/null+++ b/unsloth_cli/tests/test_train_default_output_dir.py@@ -0,0 +1,236 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import sys+from pathlib import Path+from types import SimpleNamespace++import typer+from typer.testing import CliRunner++_REPO_ROOT = Path(__file__).resolve().parents[2]+if str(_REPO_ROOT) not in sys.path:+ sys.path.insert(0, str(_REPO_ROOT))++import unsloth_cli.commands.train as train_module # noqa: E402+++class _SavingTrainer:+ is_vlm = False+ training_thread = None++ def __init__(self):+ self.output_dirs = []+ self.requested = []++ def load_model(self, **kwargs):+ return True++ def prepare_model_for_training(self, **kwargs):+ return True++ def load_and_format_dataset(self, **kwargs):+ return [], None++ def start_training(self, **kwargs):+ from utils.paths import ensure_dir, resolve_output_dir++ self.requested.append(kwargs.get("output_dir"))+ output_dir = resolve_output_dir(kwargs.get("output_dir"))+ ensure_dir(output_dir)+ (output_dir / "adapter_config.json").write_text(+ '{"base_model_name_or_path": "unsloth/Qwen3-0.6B", "peft_type": "LORA", "r": 16}',+ encoding = "utf-8",+ )+ (output_dir / "adapter_model.safetensors").write_bytes(b"\0")+ self.output_dirs.append(output_dir)+ return True++ def get_training_progress(self):+ return SimpleNamespace(error = None, is_completed = True)+++def _train(monkeypatch, trainer, *args):+ monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *a: trainer)+ app = typer.Typer()+ app.command()(train_module.train)+ return CliRunner().invoke(+ app,+ ["--model", "unsloth/Qwen3-0.6B", "--dataset", "yahma/alpaca-cleaned", *args],+ )+++def test_default_runs_get_their_own_listed_folder(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ clock = iter([1000, 2000])+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: next(clock)))+ trainer = _SavingTrainer()++ assert _train(monkeypatch, trainer).exit_code == 0+ assert _train(monkeypatch, trainer).exit_code == 0++ from utils.models.checkpoints import scan_checkpoints+ from utils.paths import outputs_root++ first, second = trainer.output_dirs+ assert first != second+ assert first.parent == outputs_root() and second.parent == outputs_root()+ listed = {Path(entry[1][0][1]) for entry in scan_checkpoints()}+ assert listed == {first, second}+++def test_explicit_output_dir_is_kept(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ trainer = _SavingTrainer()++ assert _train(monkeypatch, trainer, "--output-dir", "my_run").exit_code == 0++ from utils.paths import outputs_root++ assert trainer.output_dirs == [outputs_root() / "my_run"]+++def test_explicit_outputs_is_not_treated_as_default(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ trainer = _SavingTrainer()++ assert _train(monkeypatch, trainer, "--output-dir", "./outputs").exit_code == 0++ assert trainer.requested == ["outputs"]+++def test_runs_started_in_the_same_second_get_separate_folders(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: 1000))+ trainer = _SavingTrainer()++ assert _train(monkeypatch, trainer).exit_code == 0+ assert _train(monkeypatch, trainer).exit_code == 0++ first, second = trainer.output_dirs+ assert first != second+ assert first.name.endswith("_1000") and second.name.endswith("_1001")+++class _AgentStore:+ def __init__(self):+ self.values = {}++ def check(self, keys):+ return all(key in self.values for key in keys)++ def set(self, key, value):+ self.values[key] = value.encode()++ def get(self, key):+ return self.values[key]+++def test_torchrun_ranks_take_rank_zeros_reserved_folder(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ monkeypatch.setenv("WORLD_SIZE", "2")+ # Two nodes: different launchers and clocks, one agent store.+ clock = iter([1000, 1048])+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: next(clock)))+ store = _AgentStore()+ monkeypatch.setattr(train_module, "_torch_launch_store", lambda rank: store)+ trainer = _SavingTrainer()++ for rank in ("0", "1"):+ monkeypatch.setenv("RANK", rank)+ assert _train(monkeypatch, trainer).exit_code == 0++ first, second = trainer.output_dirs+ assert first == second+ assert first.name.endswith("_1000")+++def test_stale_world_size_without_a_rank_still_reserves_per_run(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ monkeypatch.setenv("PMI_SIZE", "8")+ monkeypatch.setenv("WORLD_SIZE", "8")+ for name in ("RANK", "PMI_RANK", "MLX_RANK", "OMPI_COMM_WORLD_RANK"):+ monkeypatch.delenv(name, raising = False)+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: 1000))+ trainer = _SavingTrainer()++ assert _train(monkeypatch, trainer).exit_code == 0+ assert _train(monkeypatch, trainer).exit_code == 0++ first, second = trainer.output_dirs+ assert first != second+++def test_launches_without_a_store_reserve_on_rank_zero(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ for name in ("RANK", "WORLD_SIZE"):+ monkeypatch.delenv(name, raising = False)+ monkeypatch.setenv("OMPI_COMM_WORLD_RANK", "0")+ monkeypatch.setenv("OMPI_COMM_WORLD_SIZE", "2")+ monkeypatch.delenv("TORCHELASTIC_USE_AGENT_STORE", raising = False)+ # Two launches whose rank 0 starts in the same second.+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: 1000))+ trainer = _SavingTrainer()++ assert _train(monkeypatch, trainer).exit_code == 0+ assert _train(monkeypatch, trainer).exit_code == 0++ first, second = trainer.output_dirs+ assert first != second+++def test_torch_ranks_without_a_store_share_the_launcher_folder(tmp_path, monkeypatch):+ import psutil++ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.delenv("MASTER_ADDR", raising = False)+ # Each rank finishes its model load at a different second; the launcher started once.+ clock = iter([1000, 1048])+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: next(clock)))+ monkeypatch.setattr(psutil, "Process", lambda pid: SimpleNamespace(create_time = lambda: 900.5))+ trainer = _SavingTrainer()++ for rank in ("0", "1"):+ monkeypatch.setenv("RANK", rank)+ assert _train(monkeypatch, trainer).exit_code == 0++ first, second = trainer.output_dirs+ assert first == second+ assert first.name.endswith("_900")+++def test_env_launch_ranks_share_rank_zeros_folder_and_port(tmp_path, monkeypatch):+ import socket++ import torch.distributed as dist++ with socket.socket() as sock:+ sock.bind(("127.0.0.1", 0))+ port = sock.getsockname()[1]+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ monkeypatch.setenv("MASTER_ADDR", "127.0.0.1")+ monkeypatch.setenv("MASTER_PORT", str(port))+ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.delenv("TORCHELASTIC_USE_AGENT_STORE", raising = False)+ clock = iter([1000, 1048])+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: next(clock)))+ trainer = _SavingTrainer()++ for rank in ("0", "1"):+ monkeypatch.setenv("RANK", rank)+ assert _train(monkeypatch, trainer).exit_code == 0++ first, second = trainer.output_dirs+ assert first == second+ assert first.name.endswith("_1000")+ # init_process_group's env:// store must still bind rank 0's port next to the one above.+ monkeypatch.setenv("RANK", "0")+ monkeypatch.setenv("WORLD_SIZE", "1")+ dist.init_process_group("gloo", rank = 0, world_size = 1)+ try:+ assert dist.get_world_size() == 1+ finally:+ dist.destroy_process_group()