Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Save each unsloth train run in its own folder (#13484)

Nilaya2aa74a56172
diff --git a/unsloth_cli/commands/train.py b/unsloth_cli/commands/train.py
index 67106f60c7a..053cd7109a4 100644
--- a/unsloth_cli/commands/train.py
+++ b/unsloth_cli/commands/train.py
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import os
import time
from pathlib import Path
from typing import Optional
@@ -80,6 +81,96 @@ def _optimizer_for_host(requested: Optional[str] = None) -> str:
return normalize_training_optimizer_for_device(optimizer, device_backend = device_backend)
+def _launch_rank() -> Optional[int]:
+ # A size alone is not a launch: a stale PMI_SIZE or WORLD_SIZE in a shell would pin every run to one folder.
+ rank = os.environ.get("RANK")
+ size = os.environ.get("WORLD_SIZE")
+ if rank is not None and size is not None:
+ try:
+ rank, size = int(rank), int(size)
+ except ValueError:
+ return None
+ return rank if 0 <= rank < size and size > 1 else None
+ is_launch, rank, _size = mlx_distributed_info()
+ return rank if is_launch and (_size or 0) > 1 else None
+
+
+_RUN_DIR_STORE = None
+
+
+def _torch_launch_store(rank: int):
+ # torchrun's agent hosts a store at MASTER_ADDR:MASTER_PORT; a plain env:// launch has rank 0 host it, and
+ # multi_tenant lets init_process_group reuse this same server instead of failing to bind the port.
+ global _RUN_DIR_STORE
+ if (
+ "RANK" not in os.environ
+ or "MASTER_ADDR" not in os.environ
+ or "MASTER_PORT" not in os.environ
+ ):
+ return None
+ try:
+ from datetime import timedelta
+
+ from torch.distributed import TCPStore
+
+ host = rank == 0 and os.environ.get("TORCHELASTIC_USE_AGENT_STORE") != "True"
+ store = TCPStore(
+ os.environ["MASTER_ADDR"],
+ int(os.environ["MASTER_PORT"]),
+ is_master = host,
+ timeout = timedelta(minutes = 30),
+ multi_tenant = True,
+ )
+ except Exception: # noqa: BLE001 -- no store reachable: fall back to the launcher's start time
+ return None
+ if host:
+ # Keep the server alive for the other ranks until init_process_group takes it over.
+ _RUN_DIR_STORE = store
+ return store
+
+
+def _launcher_start_time() -> int:
+ # Ranks reach this seconds apart after their own model load; their shared launcher started once.
+ try:
+ import psutil
+ return int(psutil.Process(os.getppid()).create_time())
+ except Exception: # noqa: BLE001 -- no launcher to ask: fall back to this rank's clock
+ return int(time.time())
+
+
+def _reserve_run_dir(name: str) -> str:
+ from utils.paths import ensure_dir, resolve_output_dir
+ stamp = int(time.time())
+ while True:
+ path = resolve_output_dir(f"{name}_{stamp}")
+ ensure_dir(path.parent)
+ try:
+ path.mkdir()
+ return str(path)
+ except FileExistsError:
+ stamp += 1
+
+
+def _default_output_dir(model_name: str) -> str:
+ ensure_studio_backend_path()
+ from utils.paths import default_run_dir_name, resolve_output_dir
+
+ name = default_run_dir_name(model_name)
+ rank = _launch_rank()
+ if rank is None:
+ return _reserve_run_dir(name)
+ store = _torch_launch_store(rank)
+ if store is None:
+ # mlx.launch / mpirun ranks never write the folder, so rank 0 reserves one alone.
+ if rank == 0 and "RANK" not in os.environ:
+ return _reserve_run_dir(name)
+ return str(resolve_output_dir(f"{name}_{_launcher_start_time()}"))
+ key = "unsloth_cli/output_dir"
+ if rank == 0 and not store.check([key]):
+ store.set(key, _reserve_run_dir(name))
+ return store.get(key).decode()
+
+
@add_options_from_config(Config)
def train(
config: Optional[Path] = typer.Option(
@@ -187,6 +278,8 @@ def train(
ds, eval_ds = result
training_kwargs = cfg.training_kwargs()
+ if "output_dir" not in cfg.training.model_fields_set:
+ training_kwargs["output_dir"] = _default_output_dir(cfg.model)
training_kwargs["wandb_token"] = wandb_token
training_kwargs["optim"] = _optimizer_for_host(training_kwargs.get("optim"))
started = trainer.start_training(dataset = ds, eval_dataset = eval_ds, **training_kwargs)
diff --git a/unsloth_cli/tests/test_train_ctrl_c_exit_code.py b/unsloth_cli/tests/test_train_ctrl_c_exit_code.py
index 53b133ecfca..571363e1e90 100644
--- a/unsloth_cli/tests/test_train_ctrl_c_exit_code.py
+++ b/unsloth_cli/tests/test_train_ctrl_c_exit_code.py
@@ -76,8 +76,9 @@ def _run(tmp_path, monkeypatch, trainer, sleep):
"model: unsloth/Qwen3-0.6B\ndata:\n dataset: yahma/alpaca-cleaned\n",
encoding = "utf-8",
)
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *args: trainer)
- monkeypatch.setattr(train_module, "time", SimpleNamespace(sleep = sleep))
+ monkeypatch.setattr(train_module, "time", SimpleNamespace(sleep = sleep, time = time.time))
app = typer.Typer()
app.command()(train_module.train)
return CliRunner().invoke(app, ["--config", str(config)])
diff --git a/unsloth_cli/tests/test_train_default_output_dir.py b/unsloth_cli/tests/test_train_default_output_dir.py
new file mode 100644
index 00000000000..ecfc0b8bbff
--- /dev/null
+++ b/unsloth_cli/tests/test_train_default_output_dir.py
@@ -0,0 +1,236 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import typer
+from typer.testing import CliRunner
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+import unsloth_cli.commands.train as train_module # noqa: E402
+
+
+class _SavingTrainer:
+ is_vlm = False
+ training_thread = None
+
+ def __init__(self):
+ self.output_dirs = []
+ self.requested = []
+
+ def load_model(self, **kwargs):
+ return True
+
+ def prepare_model_for_training(self, **kwargs):
+ return True
+
+ def load_and_format_dataset(self, **kwargs):
+ return [], None
+
+ def start_training(self, **kwargs):
+ from utils.paths import ensure_dir, resolve_output_dir
+
+ self.requested.append(kwargs.get("output_dir"))
+ output_dir = resolve_output_dir(kwargs.get("output_dir"))
+ ensure_dir(output_dir)
+ (output_dir / "adapter_config.json").write_text(
+ '{"base_model_name_or_path": "unsloth/Qwen3-0.6B", "peft_type": "LORA", "r": 16}',
+ encoding = "utf-8",
+ )
+ (output_dir / "adapter_model.safetensors").write_bytes(b"\0")
+ self.output_dirs.append(output_dir)
+ return True
+
+ def get_training_progress(self):
+ return SimpleNamespace(error = None, is_completed = True)
+
+
+def _train(monkeypatch, trainer, *args):
+ monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *a: trainer)
+ app = typer.Typer()
+ app.command()(train_module.train)
+ return CliRunner().invoke(
+ app,
+ ["--model", "unsloth/Qwen3-0.6B", "--dataset", "yahma/alpaca-cleaned", *args],
+ )
+
+
+def test_default_runs_get_their_own_listed_folder(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ clock = iter([1000, 2000])
+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: next(clock)))
+ trainer = _SavingTrainer()
+
+ assert _train(monkeypatch, trainer).exit_code == 0
+ assert _train(monkeypatch, trainer).exit_code == 0
+
+ from utils.models.checkpoints import scan_checkpoints
+ from utils.paths import outputs_root
+
+ first, second = trainer.output_dirs
+ assert first != second
+ assert first.parent == outputs_root() and second.parent == outputs_root()
+ listed = {Path(entry[1][0][1]) for entry in scan_checkpoints()}
+ assert listed == {first, second}
+
+
+def test_explicit_output_dir_is_kept(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ trainer = _SavingTrainer()
+
+ assert _train(monkeypatch, trainer, "--output-dir", "my_run").exit_code == 0
+
+ from utils.paths import outputs_root
+
+ assert trainer.output_dirs == [outputs_root() / "my_run"]
+
+
+def test_explicit_outputs_is_not_treated_as_default(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ trainer = _SavingTrainer()
+
+ assert _train(monkeypatch, trainer, "--output-dir", "./outputs").exit_code == 0
+
+ assert trainer.requested == ["outputs"]
+
+
+def test_runs_started_in_the_same_second_get_separate_folders(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: 1000))
+ trainer = _SavingTrainer()
+
+ assert _train(monkeypatch, trainer).exit_code == 0
+ assert _train(monkeypatch, trainer).exit_code == 0
+
+ first, second = trainer.output_dirs
+ assert first != second
+ assert first.name.endswith("_1000") and second.name.endswith("_1001")
+
+
+class _AgentStore:
+ def __init__(self):
+ self.values = {}
+
+ def check(self, keys):
+ return all(key in self.values for key in keys)
+
+ def set(self, key, value):
+ self.values[key] = value.encode()
+
+ def get(self, key):
+ return self.values[key]
+
+
+def test_torchrun_ranks_take_rank_zeros_reserved_folder(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ # Two nodes: different launchers and clocks, one agent store.
+ clock = iter([1000, 1048])
+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: next(clock)))
+ store = _AgentStore()
+ monkeypatch.setattr(train_module, "_torch_launch_store", lambda rank: store)
+ trainer = _SavingTrainer()
+
+ for rank in ("0", "1"):
+ monkeypatch.setenv("RANK", rank)
+ assert _train(monkeypatch, trainer).exit_code == 0
+
+ first, second = trainer.output_dirs
+ assert first == second
+ assert first.name.endswith("_1000")
+
+
+def test_stale_world_size_without_a_rank_still_reserves_per_run(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ monkeypatch.setenv("PMI_SIZE", "8")
+ monkeypatch.setenv("WORLD_SIZE", "8")
+ for name in ("RANK", "PMI_RANK", "MLX_RANK", "OMPI_COMM_WORLD_RANK"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: 1000))
+ trainer = _SavingTrainer()
+
+ assert _train(monkeypatch, trainer).exit_code == 0
+ assert _train(monkeypatch, trainer).exit_code == 0
+
+ first, second = trainer.output_dirs
+ assert first != second
+
+
+def test_launches_without_a_store_reserve_on_rank_zero(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ for name in ("RANK", "WORLD_SIZE"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setenv("OMPI_COMM_WORLD_RANK", "0")
+ monkeypatch.setenv("OMPI_COMM_WORLD_SIZE", "2")
+ monkeypatch.delenv("TORCHELASTIC_USE_AGENT_STORE", raising = False)
+ # Two launches whose rank 0 starts in the same second.
+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: 1000))
+ trainer = _SavingTrainer()
+
+ assert _train(monkeypatch, trainer).exit_code == 0
+ assert _train(monkeypatch, trainer).exit_code == 0
+
+ first, second = trainer.output_dirs
+ assert first != second
+
+
+def test_torch_ranks_without_a_store_share_the_launcher_folder(tmp_path, monkeypatch):
+ import psutil
+
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.delenv("MASTER_ADDR", raising = False)
+ # Each rank finishes its model load at a different second; the launcher started once.
+ clock = iter([1000, 1048])
+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: next(clock)))
+ monkeypatch.setattr(psutil, "Process", lambda pid: SimpleNamespace(create_time = lambda: 900.5))
+ trainer = _SavingTrainer()
+
+ for rank in ("0", "1"):
+ monkeypatch.setenv("RANK", rank)
+ assert _train(monkeypatch, trainer).exit_code == 0
+
+ first, second = trainer.output_dirs
+ assert first == second
+ assert first.name.endswith("_900")
+
+
+def test_env_launch_ranks_share_rank_zeros_folder_and_port(tmp_path, monkeypatch):
+ import socket
+
+ import torch.distributed as dist
+
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ port = sock.getsockname()[1]
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ monkeypatch.setenv("MASTER_ADDR", "127.0.0.1")
+ monkeypatch.setenv("MASTER_PORT", str(port))
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.delenv("TORCHELASTIC_USE_AGENT_STORE", raising = False)
+ clock = iter([1000, 1048])
+ monkeypatch.setattr(train_module, "time", SimpleNamespace(time = lambda: next(clock)))
+ trainer = _SavingTrainer()
+
+ for rank in ("0", "1"):
+ monkeypatch.setenv("RANK", rank)
+ assert _train(monkeypatch, trainer).exit_code == 0
+
+ first, second = trainer.output_dirs
+ assert first == second
+ assert first.name.endswith("_1000")
+ # init_process_group's env:// store must still bind rank 0's port next to the one above.
+ monkeypatch.setenv("RANK", "0")
+ monkeypatch.setenv("WORLD_SIZE", "1")
+ dist.init_process_group("gloo", rank = 0, world_size = 1)
+ try:
+ assert dist.get_world_size() == 1
+ finally:
+ dist.destroy_process_group()