Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

18.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import json
import os
import sqlite3
import sys
import time
import types as _types
from pathlib import Path

import pytest

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
sys.modules.setdefault("structlog", _types.ModuleType("structlog"))

from utils.models import checkpoints as checkpoints_module
from utils.training_runs import build_default_output_dir_name


_PLAIN_FEATURES = {
    "dora": False,
    "full_state": None,
    "moe_target_parameters": False,
    "non_uniform": False,
}


def _make_history_connection(db_path: Path) -> sqlite3.Connection:
    conn = sqlite3.connect(str(db_path))
    conn.row_factory = sqlite3.Row
    return conn


def _setup_training_runs_table(db_path: Path) -> None:
    conn = _make_history_connection(db_path)
    try:
        conn.execute(
            """
            CREATE TABLE training_runs (
                id TEXT PRIMARY KEY,
                model_name TEXT NOT NULL,
                config_json TEXT NOT NULL,
                output_dir TEXT,
                started_at TEXT NOT NULL
            )
            """
        )
        conn.commit()
    finally:
        conn.close()


def _make_outputs_dir(tmp_path, monkeypatch) -> Path:
    studio_home = tmp_path / "studio-home"
    outputs_dir = studio_home / "outputs"
    outputs_dir.mkdir(parents = True)
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(studio_home))
    return outputs_dir


def test_scan_checkpoints_uses_output_dir_history_for_base_model(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_dir = outputs_dir / "custom-run"
    run_dir.mkdir()
    (run_dir / "config.json").write_text("{}")

    db_path = tmp_path / "studio.db"
    _setup_training_runs_table(db_path)
    conn = _make_history_connection(db_path)
    try:
        conn.execute(
            """
            INSERT INTO training_runs (id, model_name, config_json, output_dir, started_at)
            VALUES (?, ?, ?, ?, ?)
            """,
            (
                "run-1",
                "unsloth/Llama-3.2-3B-Instruct",
                "{}",
                str(run_dir.resolve()),
                "2026-04-09T00:00:00Z",
            ),
        )
        conn.commit()
    finally:
        conn.close()

    monkeypatch.setattr(
        checkpoints_module,
        "get_connection",
        lambda: _make_history_connection(db_path),
    )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models[0][2]["base_model"] == "unsloth/Llama-3.2-3B-Instruct"


def test_scan_checkpoints_matches_project_suffixed_default_dir_against_history(
    tmp_path, monkeypatch
):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_name = build_default_output_dir_name(
        "unsloth/Llama-3.2-3B-Instruct",
        "Customer Support",
        timestamp = 1771227800,
    )
    run_dir = outputs_dir / run_name
    run_dir.mkdir()
    (run_dir / "config.json").write_text("{}")

    db_path = tmp_path / "studio.db"
    _setup_training_runs_table(db_path)
    conn = _make_history_connection(db_path)
    try:
        conn.execute(
            """
            INSERT INTO training_runs (id, model_name, config_json, output_dir, started_at)
            VALUES (?, ?, ?, ?, ?)
            """,
            (
                "run-2",
                "unsloth/Llama-3.2-3B-Instruct",
                json.dumps({"project_name": "Customer Support"}),
                None,
                "2026-04-09T00:00:00Z",
            ),
        )
        conn.commit()
    finally:
        conn.close()

    monkeypatch.setattr(
        checkpoints_module,
        "get_connection",
        lambda: _make_history_connection(db_path),
    )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models[0][2]["base_model"] == "unsloth/Llama-3.2-3B-Instruct"


def test_scan_checkpoints_strips_project_suffix_without_history(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_name = build_default_output_dir_name(
        "unsloth/Llama-3.2-3B-Instruct",
        "Customer Support",
        timestamp = 1771227800,
    )
    run_dir = outputs_dir / run_name
    run_dir.mkdir()
    (run_dir / "config.json").write_text("{}")

    db_path = tmp_path / "studio.db"
    _setup_training_runs_table(db_path)
    monkeypatch.setattr(
        checkpoints_module,
        "get_connection",
        lambda: _make_history_connection(db_path),
    )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models[0][2]["base_model"] == "unsloth/Llama-3.2-3B-Instruct"


def test_scan_checkpoints_preserves_project_marker_in_model_without_history(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_name = build_default_output_dir_name(
        "org/foo__project-bar",
        timestamp = 1771227800,
    )
    run_dir = outputs_dir / run_name
    run_dir.mkdir()
    (run_dir / "config.json").write_text("{}")

    db_path = tmp_path / "studio.db"
    _setup_training_runs_table(db_path)
    monkeypatch.setattr(
        checkpoints_module,
        "get_connection",
        lambda: _make_history_connection(db_path),
    )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models[0][2]["base_model"] == "org/foo__project-bar"


def test_scan_checkpoints_preserves_legacy_folder_name_fallback(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_dir = outputs_dir / "unsloth_Llama-3.2-3B-Instruct_1771227800"
    run_dir.mkdir()
    (run_dir / "config.json").write_text("{}")

    db_path = tmp_path / "studio.db"
    _setup_training_runs_table(db_path)
    monkeypatch.setattr(
        checkpoints_module,
        "get_connection",
        lambda: _make_history_connection(db_path),
    )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models[0][2]["base_model"] == "unsloth/Llama-3.2-3B-Instruct"


def test_scan_checkpoints_prefers_exact_history_match_over_newer_suffix(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_dir = outputs_dir / "unsloth_Test_1771227800"
    run_dir.mkdir()
    (run_dir / "config.json").write_text("{}")

    copied_dir = tmp_path / "copied" / run_dir.name
    copied_dir.mkdir(parents = True)

    db_path = tmp_path / "studio.db"
    _setup_training_runs_table(db_path)
    conn = _make_history_connection(db_path)
    try:
        conn.execute(
            """
            INSERT INTO training_runs (id, model_name, config_json, output_dir, started_at)
            VALUES (?, ?, ?, ?, ?)
            """,
            (
                "run-exact",
                "correct/base",
                "{}",
                str(run_dir.resolve()),
                "2026-04-09T00:00:00Z",
            ),
        )
        conn.execute(
            """
            INSERT INTO training_runs (id, model_name, config_json, output_dir, started_at)
            VALUES (?, ?, ?, ?, ?)
            """,
            (
                "run-suffix",
                "wrong/base",
                "{}",
                str(copied_dir.resolve()),
                "2026-04-10T00:00:00Z",
            ),
        )
        conn.commit()
    finally:
        conn.close()

    monkeypatch.setattr(
        checkpoints_module,
        "get_connection",
        lambda: _make_history_connection(db_path),
    )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models[0][2]["base_model"] == "correct/base"


def test_scan_checkpoints_lists_run_with_only_intermediate_checkpoints(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_dir = outputs_dir / "run_x"
    for step, loss, adapter_config in (
        (50, 0.75, {}),
        (100, 0.5, {"base_model_name_or_path": "unsloth/X", "peft_type": "LORA", "r": 16}),
    ):
        checkpoint_dir = run_dir / f"checkpoint-{step}"
        checkpoint_dir.mkdir(parents = True)
        (checkpoint_dir / "adapter_config.json").write_text(json.dumps(adapter_config))
        (checkpoint_dir / "trainer_state.json").write_text(
            json.dumps({"log_history": [{"loss": loss}]})
        )
    partial_dir = outputs_dir / "run_partial" / "tmp-checkpoint-5"
    partial_dir.mkdir(parents = True)
    (partial_dir / "adapter_config.json").write_text("{}")

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models == [
        (
            "run_x",
            [
                ("checkpoint-100", str(run_dir / "checkpoint-100"), 0.5),
                ("checkpoint-50", str(run_dir / "checkpoint-50"), 0.75),
            ],
            {
                "base_model": "unsloth/X",
                "peft_type": "LORA",
                "lora_rank": 16,
                "adapter_features": _PLAIN_FEATURES,
            },
        )
    ]

    targets = checkpoints_module.list_preview_targets(str(outputs_dir))

    assert [(t["ref"], t["is_latest"]) for t in targets] == [
        ("run_x/checkpoint-100", False),
        ("run_x/checkpoint-50", False),
    ]


def test_scan_checkpoints_skips_unreadable_folder(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_dir = outputs_dir / "run_done"
    run_dir.mkdir()
    (run_dir / "adapter_config.json").write_text("{}")
    (outputs_dir / "locked").mkdir()
    (outputs_dir / "locked_checkpoint" / "checkpoint-100").mkdir(parents = True)
    real_iterdir, real_exists = Path.iterdir, Path.exists

    def iterdir(self):
        if self.name == "locked":
            raise PermissionError(13, "Permission denied", str(self))
        return real_iterdir(self)

    def exists(self, *args, **kwargs):
        if self.parent.name in ("locked", "checkpoint-100"):
            raise PermissionError(13, "Permission denied", str(self))
        return real_exists(self, *args, **kwargs)

    monkeypatch.setattr(Path, "iterdir", iterdir)
    monkeypatch.setattr(Path, "exists", exists)

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert [name for name, _checkpoints, _metadata in models] == ["run_done"]


def test_scan_checkpoints_keeps_root_adapter_first(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_dir = outputs_dir / "run_root"
    run_dir.mkdir()
    (run_dir / "adapter_config.json").write_text(
        json.dumps({"base_model_name_or_path": "unsloth/X", "peft_type": "LORA", "r": 16})
    )
    for step, loss in ((50, 0.25), (100, 0.125)):
        checkpoint_dir = run_dir / f"checkpoint-{step}"
        checkpoint_dir.mkdir()
        (checkpoint_dir / "adapter_config.json").write_text("{}")
        (checkpoint_dir / "trainer_state.json").write_text(
            json.dumps({"log_history": [{"loss": loss}]})
        )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models == [
        (
            "run_root",
            [
                ("run_root", str(run_dir), 0.125),
                ("checkpoint-100", str(run_dir / "checkpoint-100"), 0.125),
                ("checkpoint-50", str(run_dir / "checkpoint-50"), 0.25),
            ],
            {
                "base_model": "unsloth/X",
                "peft_type": "LORA",
                "lora_rank": 16,
                "adapter_features": _PLAIN_FEATURES,
            },
        )
    ]


def test_scan_checkpoints_reads_training_loss_past_an_eval_entry(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_dir = outputs_dir / "run_eval"
    checkpoint_dir = run_dir / "checkpoint-50"
    checkpoint_dir.mkdir(parents = True)
    (checkpoint_dir / "adapter_config.json").write_text("{}")
    # A step that both evaluates and saves: the Trainer logs the eval metrics last.
    (checkpoint_dir / "trainer_state.json").write_text(
        json.dumps(
            {
                "log_history": [
                    {"epoch": 0.25, "loss": 0.5, "step": 25},
                    {"epoch": 0.5, "grad_norm": 1.5, "loss": 0.25, "step": 50},
                    {"epoch": 0.5, "eval_loss": 0.75, "eval_runtime": 2.0, "step": 50},
                ]
            }
        )
    )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models[0][1] == [("checkpoint-50", str(checkpoint_dir), 0.25)]


def test_scan_checkpoints_reads_history_once_for_many_fallback_runs(tmp_path, monkeypatch):
    # Full fine-tunes save config.json without _name_or_path, so every such run falls back to history.
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    db_path = tmp_path / "studio.db"
    _setup_training_runs_table(db_path)
    expected = {}
    conn = _make_history_connection(db_path)
    try:
        for index in range(12):
            model = f"org/model-{index % 3}"
            project = "Support" if index % 2 else None
            run_name = build_default_output_dir_name(model, project, timestamp = 1771227800 + index)
            run_dir = outputs_dir / run_name
            run_dir.mkdir()
            (run_dir / "config.json").write_text("{}")
            expected[run_name] = model
            conn.execute(
                "INSERT INTO training_runs (id, model_name, config_json, output_dir, started_at)"
                " VALUES (?, ?, ?, ?, ?)",
                (
                    f"run-{index}",
                    model,
                    json.dumps({"project_name": project}),
                    str(run_dir.resolve()) if index % 3 == 0 else None,
                    f"2026-04-09T00:00:{index:02d}Z",
                ),
            )
        conn.commit()
    finally:
        conn.close()

    opened = []

    def connect():
        conn = _make_history_connection(db_path)
        opened.append(conn)
        return conn

    parsed = []
    real_loads = json.loads

    class _CountingJson:
        JSONDecodeError = json.JSONDecodeError
        load = staticmethod(json.load)

        @staticmethod
        def loads(*args, **kwargs):
            parsed.append(args[0] if args else None)
            return real_loads(*args, **kwargs)

    monkeypatch.setattr(checkpoints_module, "get_connection", connect)
    monkeypatch.setattr(checkpoints_module, "json", _CountingJson)

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert {name: metadata["base_model"] for name, _, metadata in models} == expected
    assert len(opened) == 1
    assert len(parsed) <= 12 + len(models)


def test_scan_checkpoints_tied_exact_match_follows_sqlite_order(tmp_path, monkeypatch):
    outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
    run_dir = outputs_dir / "custom-run"
    run_dir.mkdir()
    (run_dir / "config.json").write_text("{}")
    db_path = tmp_path / "studio.db"
    _setup_training_runs_table(db_path)
    conn = _make_history_connection(db_path)
    try:
        for run_id, model in (("a", "first/base"), ("b", ""), ("c", "second/base")):
            conn.execute(
                "INSERT INTO training_runs (id, model_name, config_json, output_dir, started_at)"
                " VALUES (?, ?, ?, ?, ?)",
                (run_id, model, "{}", str(run_dir.resolve()), "2026-04-09T00:00:00Z"),
            )
        conn.commit()
        want = conn.execute(
            "SELECT model_name FROM training_runs WHERE output_dir IN (?, ?) AND model_name != ''"
            " ORDER BY started_at DESC",
            (str(run_dir.resolve()), str(run_dir)),
        ).fetchone()["model_name"]
    finally:
        conn.close()
    monkeypatch.setattr(
        checkpoints_module, "get_connection", lambda: _make_history_connection(db_path)
    )

    models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))

    assert models[0][2]["base_model"] == want


def _write_loss_state(path: Path, loss: str) -> None:
    path.write_text('{"log_history": [{"loss": ' + loss + ', "step": 10}]}')


def _counting_json_load(monkeypatch) -> list:
    calls = []
    real_load = checkpoints_module.json.load

    def load(f, *args, **kwargs):
        calls.append(f.name)
        return real_load(f, *args, **kwargs)

    monkeypatch.setattr(checkpoints_module.json, "load", load)
    return calls


def _skip_settle_window(monkeypatch) -> None:
    real_time_ns = time.time_ns
    monkeypatch.setattr(time, "time_ns", lambda: real_time_ns() + 3600 * 10**9)


def test_checkpoint_loss_cache_skips_reparse_of_unchanged_file(tmp_path, monkeypatch):
    _skip_settle_window(monkeypatch)
    checkpoint_dir = tmp_path / "checkpoint-10"
    checkpoint_dir.mkdir()
    _write_loss_state(checkpoint_dir / "trainer_state.json", "1.25")
    calls = _counting_json_load(monkeypatch)

    assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25
    assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25
    assert len(calls) == 1


@pytest.mark.skipif(
    sys.platform == "win32",
    reason = "st_ctime is the creation time on Windows, so it does not move on a rewrite",
)
def test_checkpoint_loss_cache_sees_same_size_rewrite_with_restored_mtime(tmp_path, monkeypatch):
    _skip_settle_window(monkeypatch)
    checkpoint_dir = tmp_path / "checkpoint-10"
    checkpoint_dir.mkdir()
    state = checkpoint_dir / "trainer_state.json"
    _write_loss_state(state, "1.25")
    assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25

    before = state.stat()
    with open(state, "r+") as f:
        f.write('{"log_history": [{"loss": 2.75, "step": 10}]}')
    os.utime(state, ns = (before.st_atime_ns, before.st_mtime_ns))
    assert state.stat().st_size == before.st_size

    assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 2.75


def test_checkpoint_loss_cache_ignores_recently_written_files(tmp_path, monkeypatch):
    checkpoint_dir = tmp_path / "checkpoint-10"
    checkpoint_dir.mkdir()
    _write_loss_state(checkpoint_dir / "trainer_state.json", "1.25")
    calls = _counting_json_load(monkeypatch)

    checkpoints_module._read_checkpoint_loss(checkpoint_dir)
    checkpoints_module._read_checkpoint_loss(checkpoint_dir)

    assert len(calls) == 2