Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

18.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import json
import subprocess
import sys
import textwrap
import time
from contextvars import ContextVar
from pathlib import Path
from queue import Queue
from threading import Event, current_thread
from types import ModuleType, SimpleNamespace

import pytest

from core.training import decision_trainer as decision
from utils.account_context import AccountContext, bind_account, current_account_id, reset_account


def _no_stop():
    return None


def test_prefetch_keeps_account_and_scoped_resource_context(monkeypatch):
    resource_scope = ContextVar("decision_test_resources", default = None)
    expected = ([{"state": "example"}], None)
    calls = []

    def load(config, should_stop, status, warn):
        calls.append((current_account_id(), resource_scope.get(), current_thread()))
        assert config is configuration
        return expected

    monkeypatch.setattr(decision, "_load_rows", load)
    configuration = {"dataset_revision": "pinned-sha", "require_exact_resume_resources": True}
    account = bind_account(AccountContext("alice", "Alice"))
    scope = resource_scope.set("private-snapshot")
    try:
        with decision._DecisionRows(
            configuration, lambda: False, lambda _m: None, lambda _m: None
        ) as rows:
            assert rows.result(_no_stop) is expected
            assert rows.result(_no_stop) is expected
    finally:
        reset_account(account)
        resource_scope.reset(scope)
    assert len(calls) == 1
    assert calls[0][:2] == ("alice", "private-snapshot")
    assert calls[0][2] is not current_thread()
    assert not calls[0][2].is_alive()


def test_prefetch_delivers_dataset_error_in_foreground(monkeypatch):
    failure = PermissionError("private dataset access refused")

    def load(*_a):
        raise failure

    monkeypatch.setattr(decision, "_load_rows", load)
    with decision._DecisionRows({}, lambda: False, lambda _m: None, lambda _m: None) as rows:
        with pytest.raises(PermissionError) as caught:
            rows.result(_no_stop)
    assert caught.value is failure


def test_stop_cancels_acquisition_and_mutes_late_notifications(monkeypatch):
    started, stop, closed = Event(), Event(), Event()
    events, callback, threads = [], {}, []

    def load(config, should_stop, status, warn):
        threads.append(current_thread())
        callback.update(status = status, warn = warn)
        started.set()
        while not should_stop():
            stop.wait(0.01)
        try:
            status("late dataset update")
            warn("late dataset warning")
            raise decision._Stopped("Training cancelled")
        finally:
            closed.set()

    def check_stop():
        if stop.is_set():
            raise decision._Stopped("Training cancelled")

    monkeypatch.setattr(decision, "_load_rows", load)
    with pytest.raises(decision._Stopped, match = "Training cancelled"):
        with decision._DecisionRows({}, stop.is_set, events.append, events.append) as rows:
            assert started.wait(3)
            stop.set()
            rows.result(check_stop)
    assert closed.wait(3)
    assert not threads[0].is_alive()
    events.clear()
    callback["status"]("after close")
    callback["warn"]("after close")
    assert events == []


def test_model_error_cooperatively_closes_dataset_task(monkeypatch):
    started, cleaned = Event(), Event()

    def load(config, should_stop, status, warn):
        started.set()
        while not should_stop():
            cleaned.wait(0.01)
        cleaned.set()
        return [], None

    monkeypatch.setattr(decision, "_load_rows", load)
    with pytest.raises(RuntimeError, match = "model failed"):
        with decision._DecisionRows({}, lambda: False, lambda _m: None, lambda _m: None):
            assert started.wait(3)
            raise RuntimeError("model failed")
    assert cleaned.is_set()


@pytest.fixture
def fake_decision_runtime(monkeypatch):
    from core import import_guards
    from core.systemone import laya_runtime

    monkeypatch.setenv("HF_TOKEN", "fixture-token")
    monkeypatch.setattr(import_guards, "ensure_real_packages", lambda *_a: None)
    monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda *_a: "local-checkpoint")
    model = SimpleNamespace(rows_problem = lambda *_a: None)
    unsloth = ModuleType("unsloth")
    unsloth.DecisionTrainer = object
    unsloth.FastDecisionModel = model
    unsloth.is_bfloat16_supported = lambda: False
    monkeypatch.setitem(sys.modules, "unsloth", unsloth)
    monkeypatch.setitem(sys.modules, "torch", ModuleType("torch"))
    transformers = ModuleType("transformers")
    transformers.TrainingArguments = object
    monkeypatch.setitem(sys.modules, "transformers", transformers)
    trainer = ModuleType("core.training.trainer")
    trainer._drop_hf_stdout_callbacks = lambda *_a: None
    trainer._hf_stdout_progress_disabled = lambda: True
    trainer.normalize_gradient_checkpointing = lambda value: value
    monkeypatch.setitem(sys.modules, "core.training.trainer", trainer)
    worker = ModuleType("core.training.worker")
    worker._create_embedding_progress_callback = lambda *_a, **_kw: None
    worker._download_decision_checkpoint = lambda *_a: None
    worker._emit_output_dir = lambda *_a: None
    worker._send_status = lambda events, message: events.put({"type": "status", "message": message})
    worker._start_worker_stop_poller = lambda *_a: None
    worker._worker_hf_token = lambda config: config.get("hf_token")
    monkeypatch.setitem(sys.modules, "core.training.worker", worker)
    return model, worker


@pytest.mark.parametrize("layout", ["llm", "clef", "laya"])
def test_raw_rows_overlap_checkpoint_and_model_load_but_preparation_waits(
    fake_decision_runtime, monkeypatch, layout
):
    model_api, worker = fake_decision_runtime
    raw_started, model_started, model_ready = Event(), Event(), Event()
    train_rows, eval_rows = [{"state": "train"}], [{"state": "eval"}]
    seen = []
    model, tokenizer = object(), object()
    events = Queue()
    config = {
        "model_name": "example/decision",
        "decision_layout": layout,
        "training_type": "Full Finetuning",
        "random_seed": 7,
        "gradient_checkpointing": "none",
        "hf_token": "job-token",
    }

    def load_rows(cfg, should_stop, status, warn):
        assert cfg is config
        assert not should_stop()
        seen.append("rows")
        status("dataset download must not replace model status")
        raw_started.set()
        assert model_started.wait(3), "row acquisition must overlap the model load"
        return train_rows, eval_rows

    def download(queue, cfg):
        assert raw_started.wait(3), "dataset IO must begin before checkpoint download finishes"
        seen.append("checkpoint")

    def load_model(*_a, **_kw):
        assert raw_started.wait(3)
        assert all(
            event.get("message") != "dataset download must not replace model status"
            for event in events.queue
        )
        model_started.set()
        model_ready.set()
        seen.append("model")
        return model, tokenizer

    class ReachedBuild(Exception):
        pass

    def build(rows, tok, loaded_model, validate):
        assert model_ready.is_set()
        assert rows == train_rows and tok is tokenizer and loaded_model is model
        assert seen.count("rows") == 1
        raise ReachedBuild

    monkeypatch.setattr(decision, "_load_rows", load_rows)
    worker._download_decision_checkpoint = download
    model_api.from_pretrained = load_model
    model_api.build_dataset = build
    with pytest.raises(ReachedBuild):
        decision._run(events, Queue(), config, "unused-output")
    messages = [event.get("message") for event in events.queue]
    if "dataset download must not replace model status" in messages:
        assert messages.index("dataset download must not replace model status") > messages.index(
            "Loading dataset..."
        )
    assert messages[-1] == "Preparing decision rows..."


def _decision_config(**overrides):
    return {
        "model_name": "example/decision",
        "decision_layout": "clef",
        "training_type": "Full Finetuning",
        "random_seed": 7,
        "gradient_checkpointing": "none",
        **overrides,
    }


def test_unusable_local_rows_fail_before_the_model_is_touched(fake_decision_runtime, tmp_path):
    model_api, worker = fake_decision_runtime
    touched = []
    dataset = tmp_path / "decisions.jsonl"
    dataset.write_text('{"state": "only a state"}\n', encoding = "utf-8")
    model_api.rows_problem = lambda rows, clef, validate: "row 1 needs state, questions and gold"
    worker._load_embedding_hf_dataset = None
    worker._download_decision_checkpoint = lambda *_a: touched.append("download")
    model_api.from_pretrained = lambda *_a, **_kw: touched.append("load")

    config = _decision_config(local_datasets = [str(dataset)], eval_steps = 0)
    with pytest.raises(ValueError, match = "No usable decisions in the dataset .row 1 needs state"):
        decision._run(Queue(), Queue(), config, "unused-output")
    assert touched == []


def test_unusable_hub_rows_end_a_model_download_in_progress(fake_decision_runtime, monkeypatch):
    model_api, worker = fake_decision_runtime
    downloading, release = Event(), Event()
    loaded = []

    def load_rows(*_a):
        assert downloading.wait(3)
        return [{"state": "only a state"}], None

    def download(*_a):
        downloading.set()
        release.wait(3)

    monkeypatch.setattr(decision, "_load_rows", load_rows)
    model_api.rows_problem = lambda rows, clef, validate: "row 1 has no gold"
    worker._download_decision_checkpoint = download
    model_api.from_pretrained = lambda *_a, **_kw: loaded.append(release.is_set())
    try:
        with pytest.raises(ValueError, match = "row 1 has no gold"):
            decision._run(Queue(), Queue(), _decision_config(), "unused-output")
        assert not release.is_set() and loaded == []
        assert decision._abandoned_downloads[-1].is_alive()
    finally:
        release.set()
        while decision._abandoned_downloads:
            decision._abandoned_downloads.pop().join(3)


def test_an_abandoned_download_does_not_hold_the_worker_open(tmp_path):
    script = textwrap.dedent(
        """
        import multiprocessing, threading, time
        from concurrent.futures import ThreadPoolExecutor
        from core.training import decision_trainer, worker

        def download():
            with ThreadPoolExecutor(2) as pool:
                list(pool.map(time.sleep, [30] * 4))

        thread = threading.Thread(target = download, daemon = True)
        thread.start()
        decision_trainer._abandoned_downloads.append(thread)
        events = multiprocessing.get_context("spawn").Queue()
        events.put({"type": "error", "error": "bad rows"})
        # The in-process MLX adapter: a plain queue, or any call off the main thread, must return.
        import queue
        worker._end_abandoned_decision_download(queue.Queue())
        caller = threading.Thread(target = worker._end_abandoned_decision_download, args = (events,))
        caller.start()
        caller.join()
        print("host survived", flush = True)
        worker._end_abandoned_decision_download(events)
        """
    )
    backend = Path(__file__).resolve().parents[1]
    started = time.monotonic()
    # A file, not a pipe: the queue's resource tracker would hold a pipe open past the child.
    log = tmp_path / "child.log"
    with log.open("w") as out:
        proc = subprocess.Popen(
            [sys.executable, "-c", script], cwd = backend, stdout = out, stderr = subprocess.STDOUT
        )
        try:
            returncode = proc.wait(timeout = 25)
        finally:
            proc.kill()
    assert returncode == 0, log.read_text()[-2000:]
    assert "host survived" in log.read_text()
    assert time.monotonic() - started < 25


def test_a_bad_jsonl_line_names_its_file_and_line(tmp_path):
    good = json.dumps({"state": "fine"})
    dataset = tmp_path / "decisions.jsonl"
    dataset.write_text(f"{good}\n\n{good}\n{{broken\n", encoding = "utf-8")
    with pytest.raises(ValueError, match = r"Invalid JSON at decisions\.jsonl:4: "):
        decision._read_local_rows([str(dataset)], None, _no_stop)


def test_an_empty_jsonl_shard_beside_real_rows_is_skipped(tmp_path):
    (tmp_path / "a.jsonl").write_text(json.dumps({"state": "fine"}) + "\n", encoding = "utf-8")
    (tmp_path / "b.jsonl").write_text("", encoding = "utf-8")
    (tmp_path / "c.jsonl").write_text("\n", encoding = "utf-8")
    rows = decision._read_local_rows([str(tmp_path)], None, _no_stop)
    assert rows == [{"state": "fine"}]


def test_a_bad_line_in_an_upload_names_the_file_as_it_was_uploaded(tmp_path):
    dataset = tmp_path / f"{'a1' * 16}_decisions.jsonl"
    dataset.write_text('{"state": "fine"}\n{broken\n', encoding = "utf-8")
    with pytest.raises(ValueError, match = r"Invalid JSON at decisions\.jsonl:2: "):
        decision._read_local_rows([str(dataset)], None, _no_stop)


def test_a_bad_first_line_and_a_pretty_printed_file_report_the_real_line(tmp_path):
    first = tmp_path / "first.jsonl"
    first.write_text('{broken\n{"state": "fine"}\n', encoding = "utf-8")
    pretty = tmp_path / "pretty.json"
    pretty.write_text('[\n  {"state": "fine"},\n  {"state": }\n]\n', encoding = "utf-8")
    with pytest.raises(ValueError, match = r"first\.jsonl:1: "):
        decision._read_local_rows([str(first)], None, _no_stop)
    with pytest.raises(ValueError, match = r"pretty\.json:3: "):
        decision._read_local_rows([str(pretty)], None, _no_stop)


def test_prefetch_keeps_strict_cached_splits_bounds_revision_and_token(monkeypatch):
    import importlib
    from utils.datasets import cache_safe

    worker = importlib.import_module("core.training.worker")
    reads = []
    config = {
        "hf_dataset": "org/decisions",
        "dataset_snapshot_path": "/verified/cache",
        "dataset_revision": "pinned-commit",
        "require_exact_resume_resources": True,
        "train_split": "train",
        "eval_split": "validation",
        "eval_steps": 0.1,
        "dataset_slice_start": 8,
        "dataset_slice_end": 32,
        "random_seed": 7,
        "hf_token": "test-job-token",
    }

    def cached(
        cfg,
        split,
        token,
        *,
        row_limit = None,
    ):
        reads.append(
            (
                split,
                row_limit,
                cfg["dataset_revision"],
                token,
                cfg["require_exact_resume_resources"],
            )
        )
        return [{"state": str(i)} for i in range(33 if split == "train" else 10)]

    monkeypatch.setattr(worker, "_load_cached_dataset_for_config", cached)
    monkeypatch.setattr(
        cache_safe,
        "load_dataset_cache_safe",
        lambda *_a, **_kw: pytest.fail("strict cache must not read the network"),
    )
    with decision._DecisionRows(config, lambda: False, lambda _m: None, lambda _m: None) as pending:
        rows, held = pending.result(_no_stop)
    assert [row["state"] for row in rows] == [str(i) for i in range(8, 33)]
    assert len(held) == 10
    assert reads == [
        ("train", 33, "pinned-commit", "test-job-token", True),
        ("validation", None, "pinned-commit", "test-job-token", True),
    ]


def test_prefetch_strict_missing_snapshot_never_falls_back_to_hub(monkeypatch):
    import importlib
    from utils.datasets import cache_safe

    worker = importlib.import_module("core.training.worker")
    monkeypatch.setattr(worker, "_load_cached_dataset_for_config", lambda *_a, **_kw: None)
    monkeypatch.setattr(
        cache_safe,
        "load_dataset_cache_safe",
        lambda *_a, **_kw: pytest.fail("strict cache must not read the network"),
    )
    config = {
        "hf_dataset": "org/decisions",
        "dataset_snapshot_path": "/missing/cache",
        "dataset_revision": "pinned-commit",
        "require_exact_resume_resources": True,
        "train_split": "train",
        "eval_steps": 0,
        "random_seed": 7,
    }
    with decision._DecisionRows(config, lambda: False, lambda _m: None, lambda _m: None) as pending:
        with pytest.raises(FileNotFoundError, match = "exact cached dataset split 'train'"):
            pending.result(_no_stop)


def _frontend_regex(name):
    import re
    from pathlib import Path

    source = (
        Path(__file__).resolve().parents[2] / "frontend/src/features/studio/preparation-progress.ts"
    ).read_text(encoding = "utf-8")
    pattern, flags = re.search(rf"const {name} =\s*/(.+)/(\w*);", source).groups()
    return re.compile(pattern.replace("(?<", "(?P<"), re.I if "i" in flags else 0)


def test_preparing_rows_reports_counted_dataset_progress(monkeypatch):
    clock = iter(range(0, 10_000, 2))
    monkeypatch.setattr(decision.time, "monotonic", lambda: next(clock))
    messages, stops = [], []
    rows = decision._ProgressRows(
        [{"state": i} for i in range(5)],
        "Preparing decision rows",
        messages.append,
        lambda: stops.append(1),
    )

    assert (
        list(rows) == [{"state": i} for i in range(5)]
        and len(rows) == 5
        and rows[2] == {"state": 2}
    )
    assert len(stops) == 5
    assert messages[0] == "Preparing decision rows 0% (0/5)"
    assert messages[-1] == "Preparing decision rows 100% (5/5)"
    counted, dataset = (
        _frontend_regex("COUNTED_PREPARATION_RE"),
        _frontend_regex("DATASET_PREPARATION_RE"),
    )
    for message in messages + ["Preparing evaluation rows 50% (1,000/2,000)"]:
        groups = counted.match(message).groupdict()
        assert dataset.search(groups["label"])
        assert int(groups["current"].replace(",", "")) <= int(groups["total"].replace(",", ""))
    assert dataset.search("Preparing decision rows...")
    assert (
        list(decision._ProgressRows([], "Preparing decision rows", messages.append, _no_stop)) == []
    )


def test_stop_while_preparing_rows_ends_the_read():
    def stop():
        raise decision._Stopped("Training cancelled")

    with pytest.raises(decision._Stopped):
        list(
            decision._ProgressRows([{"state": 1}], "Preparing decision rows", lambda _m: None, stop)
        )