Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import json
import subprocess
import sys
import textwrap
import time
from contextvars import ContextVar
from pathlib import Path
from queue import Queue
from threading import Event, current_thread
from types import ModuleType, SimpleNamespace
import pytest
from core.training import decision_trainer as decision
from utils.account_context import AccountContext, bind_account, current_account_id, reset_account
def _no_stop():
return None
def test_prefetch_keeps_account_and_scoped_resource_context(monkeypatch):
resource_scope = ContextVar("decision_test_resources", default = None)
expected = ([{"state": "example"}], None)
calls = []
def load(config, should_stop, status, warn):
calls.append((current_account_id(), resource_scope.get(), current_thread()))
assert config is configuration
return expected
monkeypatch.setattr(decision, "_load_rows", load)
configuration = {"dataset_revision": "pinned-sha", "require_exact_resume_resources": True}
account = bind_account(AccountContext("alice", "Alice"))
scope = resource_scope.set("private-snapshot")
try:
with decision._DecisionRows(
configuration, lambda: False, lambda _m: None, lambda _m: None
) as rows:
assert rows.result(_no_stop) is expected
assert rows.result(_no_stop) is expected
finally:
reset_account(account)
resource_scope.reset(scope)
assert len(calls) == 1
assert calls[0][:2] == ("alice", "private-snapshot")
assert calls[0][2] is not current_thread()
assert not calls[0][2].is_alive()
def test_prefetch_delivers_dataset_error_in_foreground(monkeypatch):
failure = PermissionError("private dataset access refused")
def load(*_a):
raise failure
monkeypatch.setattr(decision, "_load_rows", load)
with decision._DecisionRows({}, lambda: False, lambda _m: None, lambda _m: None) as rows:
with pytest.raises(PermissionError) as caught:
rows.result(_no_stop)
assert caught.value is failure
def test_stop_cancels_acquisition_and_mutes_late_notifications(monkeypatch):
started, stop, closed = Event(), Event(), Event()
events, callback, threads = [], {}, []
def load(config, should_stop, status, warn):
threads.append(current_thread())
callback.update(status = status, warn = warn)
started.set()
while not should_stop():
stop.wait(0.01)
try:
status("late dataset update")
warn("late dataset warning")
raise decision._Stopped("Training cancelled")
finally:
closed.set()
def check_stop():
if stop.is_set():
raise decision._Stopped("Training cancelled")
monkeypatch.setattr(decision, "_load_rows", load)
with pytest.raises(decision._Stopped, match = "Training cancelled"):
with decision._DecisionRows({}, stop.is_set, events.append, events.append) as rows:
assert started.wait(3)
stop.set()
rows.result(check_stop)
assert closed.wait(3)
assert not threads[0].is_alive()
events.clear()
callback["status"]("after close")
callback["warn"]("after close")
assert events == []
def test_model_error_cooperatively_closes_dataset_task(monkeypatch):
started, cleaned = Event(), Event()
def load(config, should_stop, status, warn):
started.set()
while not should_stop():
cleaned.wait(0.01)
cleaned.set()
return [], None
monkeypatch.setattr(decision, "_load_rows", load)
with pytest.raises(RuntimeError, match = "model failed"):
with decision._DecisionRows({}, lambda: False, lambda _m: None, lambda _m: None):
assert started.wait(3)
raise RuntimeError("model failed")
assert cleaned.is_set()
@pytest.fixture
def fake_decision_runtime(monkeypatch):
from core import import_guards
from core.systemone import laya_runtime
monkeypatch.setenv("HF_TOKEN", "fixture-token")
monkeypatch.setattr(import_guards, "ensure_real_packages", lambda *_a: None)
monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda *_a: "local-checkpoint")
model = SimpleNamespace(rows_problem = lambda *_a: None)
unsloth = ModuleType("unsloth")
unsloth.DecisionTrainer = object
unsloth.FastDecisionModel = model
unsloth.is_bfloat16_supported = lambda: False
monkeypatch.setitem(sys.modules, "unsloth", unsloth)
monkeypatch.setitem(sys.modules, "torch", ModuleType("torch"))
transformers = ModuleType("transformers")
transformers.TrainingArguments = object
monkeypatch.setitem(sys.modules, "transformers", transformers)
trainer = ModuleType("core.training.trainer")
trainer._drop_hf_stdout_callbacks = lambda *_a: None
trainer._hf_stdout_progress_disabled = lambda: True
trainer.normalize_gradient_checkpointing = lambda value: value
monkeypatch.setitem(sys.modules, "core.training.trainer", trainer)
worker = ModuleType("core.training.worker")
worker._create_embedding_progress_callback = lambda *_a, **_kw: None
worker._download_decision_checkpoint = lambda *_a: None
worker._emit_output_dir = lambda *_a: None
worker._send_status = lambda events, message: events.put({"type": "status", "message": message})
worker._start_worker_stop_poller = lambda *_a: None
worker._worker_hf_token = lambda config: config.get("hf_token")
monkeypatch.setitem(sys.modules, "core.training.worker", worker)
return model, worker
@pytest.mark.parametrize("layout", ["llm", "clef", "laya"])
def test_raw_rows_overlap_checkpoint_and_model_load_but_preparation_waits(
fake_decision_runtime, monkeypatch, layout
):
model_api, worker = fake_decision_runtime
raw_started, model_started, model_ready = Event(), Event(), Event()
train_rows, eval_rows = [{"state": "train"}], [{"state": "eval"}]
seen = []
model, tokenizer = object(), object()
events = Queue()
config = {
"model_name": "example/decision",
"decision_layout": layout,
"training_type": "Full Finetuning",
"random_seed": 7,
"gradient_checkpointing": "none",
"hf_token": "job-token",
}
def load_rows(cfg, should_stop, status, warn):
assert cfg is config
assert not should_stop()
seen.append("rows")
status("dataset download must not replace model status")
raw_started.set()
assert model_started.wait(3), "row acquisition must overlap the model load"
return train_rows, eval_rows
def download(queue, cfg):
assert raw_started.wait(3), "dataset IO must begin before checkpoint download finishes"
seen.append("checkpoint")
def load_model(*_a, **_kw):
assert raw_started.wait(3)
assert all(
event.get("message") != "dataset download must not replace model status"
for event in events.queue
)
model_started.set()
model_ready.set()
seen.append("model")
return model, tokenizer
class ReachedBuild(Exception):
pass
def build(rows, tok, loaded_model, validate):
assert model_ready.is_set()
assert rows == train_rows and tok is tokenizer and loaded_model is model
assert seen.count("rows") == 1
raise ReachedBuild
monkeypatch.setattr(decision, "_load_rows", load_rows)
worker._download_decision_checkpoint = download
model_api.from_pretrained = load_model
model_api.build_dataset = build
with pytest.raises(ReachedBuild):
decision._run(events, Queue(), config, "unused-output")
messages = [event.get("message") for event in events.queue]
if "dataset download must not replace model status" in messages:
assert messages.index("dataset download must not replace model status") > messages.index(
"Loading dataset..."
)
assert messages[-1] == "Preparing decision rows..."
def _decision_config(**overrides):
return {
"model_name": "example/decision",
"decision_layout": "clef",
"training_type": "Full Finetuning",
"random_seed": 7,
"gradient_checkpointing": "none",
**overrides,
}
def test_unusable_local_rows_fail_before_the_model_is_touched(fake_decision_runtime, tmp_path):
model_api, worker = fake_decision_runtime
touched = []
dataset = tmp_path / "decisions.jsonl"
dataset.write_text('{"state": "only a state"}\n', encoding = "utf-8")
model_api.rows_problem = lambda rows, clef, validate: "row 1 needs state, questions and gold"
worker._load_embedding_hf_dataset = None
worker._download_decision_checkpoint = lambda *_a: touched.append("download")
model_api.from_pretrained = lambda *_a, **_kw: touched.append("load")
config = _decision_config(local_datasets = [str(dataset)], eval_steps = 0)
with pytest.raises(ValueError, match = "No usable decisions in the dataset .row 1 needs state"):
decision._run(Queue(), Queue(), config, "unused-output")
assert touched == []
def test_unusable_hub_rows_end_a_model_download_in_progress(fake_decision_runtime, monkeypatch):
model_api, worker = fake_decision_runtime
downloading, release = Event(), Event()
loaded = []
def load_rows(*_a):
assert downloading.wait(3)
return [{"state": "only a state"}], None
def download(*_a):
downloading.set()
release.wait(3)
monkeypatch.setattr(decision, "_load_rows", load_rows)
model_api.rows_problem = lambda rows, clef, validate: "row 1 has no gold"
worker._download_decision_checkpoint = download
model_api.from_pretrained = lambda *_a, **_kw: loaded.append(release.is_set())
try:
with pytest.raises(ValueError, match = "row 1 has no gold"):
decision._run(Queue(), Queue(), _decision_config(), "unused-output")
assert not release.is_set() and loaded == []
assert decision._abandoned_downloads[-1].is_alive()
finally:
release.set()
while decision._abandoned_downloads:
decision._abandoned_downloads.pop().join(3)
def test_an_abandoned_download_does_not_hold_the_worker_open(tmp_path):
script = textwrap.dedent(
"""
import multiprocessing, threading, time
from concurrent.futures import ThreadPoolExecutor
from core.training import decision_trainer, worker
def download():
with ThreadPoolExecutor(2) as pool:
list(pool.map(time.sleep, [30] * 4))
thread = threading.Thread(target = download, daemon = True)
thread.start()
decision_trainer._abandoned_downloads.append(thread)
events = multiprocessing.get_context("spawn").Queue()
events.put({"type": "error", "error": "bad rows"})
# The in-process MLX adapter: a plain queue, or any call off the main thread, must return.
import queue
worker._end_abandoned_decision_download(queue.Queue())
caller = threading.Thread(target = worker._end_abandoned_decision_download, args = (events,))
caller.start()
caller.join()
print("host survived", flush = True)
worker._end_abandoned_decision_download(events)
"""
)
backend = Path(__file__).resolve().parents[1]
started = time.monotonic()
# A file, not a pipe: the queue's resource tracker would hold a pipe open past the child.
log = tmp_path / "child.log"
with log.open("w") as out:
proc = subprocess.Popen(
[sys.executable, "-c", script], cwd = backend, stdout = out, stderr = subprocess.STDOUT
)
try:
returncode = proc.wait(timeout = 25)
finally:
proc.kill()
assert returncode == 0, log.read_text()[-2000:]
assert "host survived" in log.read_text()
assert time.monotonic() - started < 25
def test_a_bad_jsonl_line_names_its_file_and_line(tmp_path):
good = json.dumps({"state": "fine"})
dataset = tmp_path / "decisions.jsonl"
dataset.write_text(f"{good}\n\n{good}\n{{broken\n", encoding = "utf-8")
with pytest.raises(ValueError, match = r"Invalid JSON at decisions\.jsonl:4: "):
decision._read_local_rows([str(dataset)], None, _no_stop)
def test_an_empty_jsonl_shard_beside_real_rows_is_skipped(tmp_path):
(tmp_path / "a.jsonl").write_text(json.dumps({"state": "fine"}) + "\n", encoding = "utf-8")
(tmp_path / "b.jsonl").write_text("", encoding = "utf-8")
(tmp_path / "c.jsonl").write_text("\n", encoding = "utf-8")
rows = decision._read_local_rows([str(tmp_path)], None, _no_stop)
assert rows == [{"state": "fine"}]
def test_a_bad_line_in_an_upload_names_the_file_as_it_was_uploaded(tmp_path):
dataset = tmp_path / f"{'a1' * 16}_decisions.jsonl"
dataset.write_text('{"state": "fine"}\n{broken\n', encoding = "utf-8")
with pytest.raises(ValueError, match = r"Invalid JSON at decisions\.jsonl:2: "):
decision._read_local_rows([str(dataset)], None, _no_stop)
def test_a_bad_first_line_and_a_pretty_printed_file_report_the_real_line(tmp_path):
first = tmp_path / "first.jsonl"
first.write_text('{broken\n{"state": "fine"}\n', encoding = "utf-8")
pretty = tmp_path / "pretty.json"
pretty.write_text('[\n {"state": "fine"},\n {"state": }\n]\n', encoding = "utf-8")
with pytest.raises(ValueError, match = r"first\.jsonl:1: "):
decision._read_local_rows([str(first)], None, _no_stop)
with pytest.raises(ValueError, match = r"pretty\.json:3: "):
decision._read_local_rows([str(pretty)], None, _no_stop)
def test_prefetch_keeps_strict_cached_splits_bounds_revision_and_token(monkeypatch):
import importlib
from utils.datasets import cache_safe
worker = importlib.import_module("core.training.worker")
reads = []
config = {
"hf_dataset": "org/decisions",
"dataset_snapshot_path": "/verified/cache",
"dataset_revision": "pinned-commit",
"require_exact_resume_resources": True,
"train_split": "train",
"eval_split": "validation",
"eval_steps": 0.1,
"dataset_slice_start": 8,
"dataset_slice_end": 32,
"random_seed": 7,
"hf_token": "test-job-token",
}
def cached(
cfg,
split,
token,
*,
row_limit = None,
):
reads.append(
(
split,
row_limit,
cfg["dataset_revision"],
token,
cfg["require_exact_resume_resources"],
)
)
return [{"state": str(i)} for i in range(33 if split == "train" else 10)]
monkeypatch.setattr(worker, "_load_cached_dataset_for_config", cached)
monkeypatch.setattr(
cache_safe,
"load_dataset_cache_safe",
lambda *_a, **_kw: pytest.fail("strict cache must not read the network"),
)
with decision._DecisionRows(config, lambda: False, lambda _m: None, lambda _m: None) as pending:
rows, held = pending.result(_no_stop)
assert [row["state"] for row in rows] == [str(i) for i in range(8, 33)]
assert len(held) == 10
assert reads == [
("train", 33, "pinned-commit", "test-job-token", True),
("validation", None, "pinned-commit", "test-job-token", True),
]
def test_prefetch_strict_missing_snapshot_never_falls_back_to_hub(monkeypatch):
import importlib
from utils.datasets import cache_safe
worker = importlib.import_module("core.training.worker")
monkeypatch.setattr(worker, "_load_cached_dataset_for_config", lambda *_a, **_kw: None)
monkeypatch.setattr(
cache_safe,
"load_dataset_cache_safe",
lambda *_a, **_kw: pytest.fail("strict cache must not read the network"),
)
config = {
"hf_dataset": "org/decisions",
"dataset_snapshot_path": "/missing/cache",
"dataset_revision": "pinned-commit",
"require_exact_resume_resources": True,
"train_split": "train",
"eval_steps": 0,
"random_seed": 7,
}
with decision._DecisionRows(config, lambda: False, lambda _m: None, lambda _m: None) as pending:
with pytest.raises(FileNotFoundError, match = "exact cached dataset split 'train'"):
pending.result(_no_stop)
def _frontend_regex(name):
import re
from pathlib import Path
source = (
Path(__file__).resolve().parents[2] / "frontend/src/features/studio/preparation-progress.ts"
).read_text(encoding = "utf-8")
pattern, flags = re.search(rf"const {name} =\s*/(.+)/(\w*);", source).groups()
return re.compile(pattern.replace("(?<", "(?P<"), re.I if "i" in flags else 0)
def test_preparing_rows_reports_counted_dataset_progress(monkeypatch):
clock = iter(range(0, 10_000, 2))
monkeypatch.setattr(decision.time, "monotonic", lambda: next(clock))
messages, stops = [], []
rows = decision._ProgressRows(
[{"state": i} for i in range(5)],
"Preparing decision rows",
messages.append,
lambda: stops.append(1),
)
assert (
list(rows) == [{"state": i} for i in range(5)]
and len(rows) == 5
and rows[2] == {"state": 2}
)
assert len(stops) == 5
assert messages[0] == "Preparing decision rows 0% (0/5)"
assert messages[-1] == "Preparing decision rows 100% (5/5)"
counted, dataset = (
_frontend_regex("COUNTED_PREPARATION_RE"),
_frontend_regex("DATASET_PREPARATION_RE"),
)
for message in messages + ["Preparing evaluation rows 50% (1,000/2,000)"]:
groups = counted.match(message).groupdict()
assert dataset.search(groups["label"])
assert int(groups["current"].replace(",", "")) <= int(groups["total"].replace(",", ""))
assert dataset.search("Preparing decision rows...")
assert (
list(decision._ProgressRows([], "Preparing decision rows", messages.append, _no_stop)) == []
)
def test_stop_while_preparing_rows_ends_the_read():
def stop():
raise decision._Stopped("Training cancelled")
with pytest.raises(decision._Stopped):
list(
decision._ProgressRows([{"state": 1}], "Preparing decision rows", lambda _m: None, stop)
)