Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import importlib.util
import json
import shutil
import sys
import time
from pathlib import Path
import pytest
torch = pytest.importorskip("torch")
from fastapi import FastAPI
from fastapi.testclient import TestClient
from safetensors import safe_open
from auth.authentication import get_current_subject
from core.systemone import catalog, laya_runtime
WORDS = (
"the server is down again refund my card charge twice please help now "
"angry calm fine login broken slow outage billing account password"
).split()
QUESTIONS = {
"urgent": {"type": "noul", "instructions": "Does this need a reply now?"},
"team": {
"type": "choice",
"instructions": "Which team should handle it?",
"criteria": {"outage": "service down", "billing": "charges and refunds"},
},
"mood": {
"type": "score",
"instructions": "How upset is the customer?",
"criteria": ["calm", "annoyed", "angry"],
},
}
def _row(i):
outage = i % 2 == 0
return {
"state": "the server is down again help now" if outage else "refund my card charge twice",
"questions": json.dumps(QUESTIONS),
"gold": json.dumps(
{
"urgent": {"label": "true" if outage else "false"},
"team": {
"label": "outage" if outage else "billing",
"probabilities": {"outage": 0.9, "billing": 0.1}
if outage
else {"outage": 0.2, "billing": 0.8},
},
"mood": {"label": 2 if outage else 0},
}
),
}
def _base_checkpoint(folder):
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import ModernBertConfig, PreTrainedTokenizerFast
from safetensors.torch import save_file
laya = laya_runtime._laya()
specials = ["[PAD]", "[SEP]", "[CLS]", "[UNK]", "[MASK]"]
vocab = {token: i for i, token in enumerate(specials + sorted(set(WORDS)))}
tokenizer = Tokenizer(models.WordLevel(vocab, unk_token = "[UNK]"))
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
PreTrainedTokenizerFast(
tokenizer_object = tokenizer,
pad_token = "[PAD]",
sep_token = "[SEP]",
cls_token = "[CLS]",
unk_token = "[UNK]",
mask_token = "[MASK]",
).save_pretrained(str(folder / "tokenizer"))
ModernBertConfig(
vocab_size = 64,
hidden_size = 64,
intermediate_size = 96,
num_hidden_layers = 2,
num_attention_heads = 4,
pad_token_id = 0,
cls_token_id = 2,
sep_token_id = 1,
global_attn_every_n_layers = 2,
local_attention = 16,
).save_pretrained(str(folder / "encoder"))
cfg = {
"encoder": "tiny",
"head_layers": 1,
"act_costs": {"escalate": 0.5},
"max_len": 96,
"head_max_len": 48,
"temperature": [1.2, 1.1, 1.3],
"temperature_by_options": {"noul:2": 0.1},
}
torch.manual_seed(0)
model = laya.common.build_model(cfg, encoder_dir = str(folder / "encoder"))
save_file(
{k: v.half().contiguous() for k, v in model.state_dict().items()},
str(folder / "model.safetensors"),
)
(folder / "rl_agent_config.json").write_text(json.dumps(cfg), encoding = "utf-8")
return folder
@pytest.fixture
def studio_home(tmp_path, monkeypatch):
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))
for name in (
"UNSLOTH_SYSTEMONE_MODEL",
"UNSLOTH_SYSTEMONE_DISABLE",
"UNSLOTH_SYSTEMONE_DEVICE",
):
monkeypatch.delenv(name, raising = False)
for name in ("_agent", "_loaded", "_device_name", "_loader", "_loading", "_failure"):
monkeypatch.setattr(laya_runtime, name, None)
yield tmp_path
if laya_runtime._loader is not None:
laya_runtime._loader.join(30)
@pytest.fixture
def base(studio_home):
return _base_checkpoint(studio_home / "base")
def _dataset(folder, rows):
path = folder / "decisions.jsonl"
path.write_text("\n".join(json.dumps(row) for row in rows), encoding = "utf-8")
return str(path)
def _config(base, dataset, **overrides):
return {
"model_name": str(base),
"model_subfolder": None,
"is_decision": True,
"hf_dataset": "",
"local_datasets": [dataset],
"batch_size": 8,
"gradient_accumulation_steps": 2,
"num_epochs": 1,
"max_steps": 3,
"learning_rate": "1e-3",
"warmup_steps": 0,
"weight_decay": 0.01,
"lr_scheduler_type": "cosine",
"optim": "adamw_torch",
"random_seed": 3407,
"gradient_checkpointing": "none",
"eval_steps": 0,
"project_name": None,
**overrides,
}
# Worker imports unsloth; CI installs unsloth_zoo only in this step; Apple Silicon uses MLX.
_WORKER = "unsloth_zoo.mlx.decision" if sys.platform == "darwin" else "unsloth_zoo"
needs_worker = pytest.mark.skipif(
importlib.util.find_spec("unsloth_zoo") is None or importlib.util.find_spec(_WORKER) is None,
reason = "trains in a worker that imports unsloth",
)
def _train(
config,
stop = None,
when = lambda event: event["type"] == "progress",
):
import multiprocessing as mp
from core.training.training import _build_training_worker_config
from core.training.worker import run_training_process
context = mp.get_context("spawn")
events, stops = context.Queue(), context.Queue()
process = context.Process(
target = run_training_process,
kwargs = {
"event_queue": events,
"stop_queue": stops,
"config": _build_training_worker_config({"training_type": "Full Finetuning", **config}),
},
)
process.start()
received = []
try:
while not received or received[-1]["type"] not in ("complete", "error"):
received.append(events.get(timeout = 600))
if stop is not None and when(received[-1]):
stops.put({"type": "stop", "save": stop})
stop = None
finally:
process.join(60)
return received
def _of(events, kind):
return [e for e in events if e["type"] == kind]
@pytest.fixture
def client(studio_home):
from routes import systemone
from routes.settings import router as settings_router
app = FastAPI()
app.include_router(systemone.router, prefix = "/v1")
app.include_router(settings_router, prefix = "/api/settings")
app.dependency_overrides[get_current_subject] = lambda: "tester"
return TestClient(app)
def _decide(client, model = "default"):
for _ in range(120):
response = client.post(
"/v1/systemone",
json = {"model": model, "state": "the server is down again", "questions": QUESTIONS},
)
if response.status_code != 503:
return response
time.sleep(0.5)
return response
@needs_worker
def test_fine_tune_is_calibrated_saved_and_served(base, studio_home, client):
rows = [_row(i) for i in range(200)]
rows += [
{"state": "slow login", "questions": "not json", "gold": "{}"},
{"state": "slow login", "questions": json.dumps(QUESTIONS), "gold": json.dumps({})},
{
"state": "slow login",
"questions": json.dumps({"q": {"type": "maybe", "instructions": "?"}}),
"gold": json.dumps({"q": "yes"}),
},
]
events = _train(_config(base, _dataset(studio_home, rows)))
assert not _of(events, "error"), _of(events, "error")
progress = [e for e in _of(events, "progress") if e["loss"] is not None]
assert [e["step"] for e in progress] == [1, 2, 3]
assert all(e["total_steps"] == 3 and e["loss"] > 0 for e in progress)
assert any(e["eval_loss"] is not None for e in _of(events, "progress"))
assert _of(events, "warning")[0]["message"] == (
'Skipped 5 of 605 decisions: row 202: "urgent" has no gold (and 2 more like it).'
)
complete = _of(events, "complete")[-1]
assert complete["status_message"].startswith("Held-out accuracy ")
output = complete["output_dir"]
assert _of(events, "output_dir")[0]["output_dir"] == output
from utils.paths import outputs_root
folder = outputs_root() / output.rsplit("/", 1)[-1]
assert str(folder) == output
cfg = json.loads((folder / "rl_agent_config.json").read_text(encoding = "utf-8"))
assert cfg["fine_tuned"] is True
assert "temperature_by_options" not in cfg
assert cfg["max_len"] == 1024 and cfg["head_max_len"] == 256
assert all(0.5 <= t <= 5.0 for t in cfg["temperature"])
assert cfg["temperature"] != [1.2, 1.1, 1.3]
assert cfg["training"]["heldout_decisions"] == 60
assert cfg["training"]["objective"] == "soft_cross_entropy"
assert cfg["training"]["steps"] == 3
with safe_open(str(folder / "model.safetensors"), "pt") as weights:
assert {weights.get_tensor(k).dtype for k in weights.keys()} == {torch.float16}
name = catalog.FINE_TUNE_PREFIX + folder.name
settings = client.get("/api/settings/systemone").json()
assert {"name": name, "kind": "fine_tune", "label": folder.name}.items() <= next(
m for m in settings["models"] if m["name"] == name
).items()
response = client.put("/api/settings/systemone", json = {"enabled": True, "model": name})
assert response.status_code == 200, response.text
assert response.json()["model"] == name
plan = client.get("/api/settings/systemone/resolve").json()
assert plan["cached"] is True and plan["repo"] is None
response = _decide(client)
assert response.status_code == 200, response.text
body = response.json()
assert body["model"] == name
assert set(body["answers"]) == set(QUESTIONS)
assert body["answers"]["team"]["choice"] in ("outage", "billing")
assert _decide(client, name).json()["model"] == name
@needs_worker
def test_lora_run_merges_into_the_served_layout(base, studio_home, client, monkeypatch):
from safetensors.torch import load_file
# The spawned worker inherits sys.path and imports this wandb, whose init fails like a bad key.
offline = studio_home / "offline" / "wandb"
offline.mkdir(parents = True)
(offline / "__init__.py").write_text(
'def init(**kwargs):\n raise RuntimeError("cannot reach W&B")\n', encoding = "utf-8"
)
monkeypatch.syspath_prepend(str(offline.parent))
rows = [_row(i) for i in range(120)]
events = _train(
_config(
base,
_dataset(studio_home, rows),
training_type = "LoRA/QLoRA",
lora_r = 4,
lora_alpha = 4,
enable_wandb = True,
)
)
assert not _of(events, "error"), _of(events, "error")
assert [e["message"] for e in _of(events, "warning")] == [
"Weights & Biases logging is off: cannot reach W&B"
]
checkpoint = catalog.fine_tune(
catalog.FINE_TUNE_PREFIX + _of(events, "complete")[-1]["output_dir"].rsplit("/", 1)[-1]
)
folder = Path(checkpoint.source)
cfg = json.loads((folder / "rl_agent_config.json").read_text(encoding = "utf-8"))
assert cfg["training"]["method"] == "lora"
tuned = load_file(str(folder / "model.safetensors"))
original = load_file(str(base / "model.safetensors"))
assert tuned.keys() == original.keys()
assert {t.dtype for t in tuned.values()} == {torch.float16}
assert not torch.equal(
tuned["encoder.layers.0.attn.Wqkv.weight"], original["encoder.layers.0.attn.Wqkv.weight"]
)
assert torch.equal(
tuned["encoder.embeddings.tok_embeddings.weight"],
original["encoder.embeddings.tok_embeddings.weight"],
)
name = catalog.FINE_TUNE_PREFIX + folder.name
response = client.put("/api/settings/systemone", json = {"enabled": True, "model": name})
assert response.status_code == 200, response.text
response = _decide(client)
assert response.status_code == 200, response.text
assert response.json()["model"] == name
@needs_worker
@pytest.mark.skipif(
importlib.util.find_spec("tensorboard") is None
and importlib.util.find_spec("tensorboardX") is None,
reason = "needs tensorboard",
)
def test_tensorboard_logs_a_decision_run(base, studio_home):
from utils.paths import tensorboard_root
rows = [_row(i) for i in range(60)]
events = _train(
_config(
base,
_dataset(studio_home, rows),
max_steps = 2,
enable_tensorboard = True,
tensorboard_dir = "decision-tb",
)
)
assert not _of(events, "error"), _of(events, "error")
assert list((tensorboard_root() / "decision-tb").rglob("events.out.tfevents.*"))
@needs_worker
def test_an_unusable_eval_file_falls_back_to_the_hold_out(base, studio_home):
held = studio_home / "held"
held.mkdir()
rows = [_row(i) for i in range(120)]
events = _train(
_config(
base,
_dataset(studio_home, rows),
eval_steps = 0.5,
local_eval_datasets = [_dataset(held, [{"state": "slow"}] * 4)],
)
)
assert not _of(events, "error"), _of(events, "error")
assert "Holding out 36 decisions to calibrate confidence..." in [
e["message"] for e in _of(events, "status")
]
folder = Path(_of(events, "complete")[-1]["output_dir"])
cfg = json.loads((folder / "rl_agent_config.json").read_text(encoding = "utf-8"))
assert cfg["training"]["heldout_decisions"] == 36
@needs_worker
def test_all_invalid_rows_are_an_error_and_save_nothing(base, studio_home):
from utils.paths import outputs_root
rows = [{"state": "slow", "questions": json.dumps(QUESTIONS)}] * 4
events = _train(_config(base, _dataset(studio_home, rows)))
assert "No usable decisions" in _of(events, "error")[0]["error"]
assert "Loading decision model..." not in [e["message"] for e in _of(events, "status")]
assert not _of(events, "complete")
assert not outputs_root().exists() or not any(outputs_root().iterdir())
@needs_worker
@pytest.mark.parametrize("training_type", ["Full Finetuning", "LoRA/QLoRA"])
def test_stop_with_save_leaves_a_servable_checkpoint(base, studio_home, training_type):
rows = [_row(i) for i in range(120)]
events = _train(
_config(
base,
_dataset(studio_home, rows),
max_steps = 0,
num_epochs = 3,
training_type = training_type,
),
True,
)
complete = _of(events, "complete")[-1]
last = _of(events, "progress")[-1]
assert last["step"] < last["total_steps"]
checkpoint = catalog.fine_tune(
catalog.FINE_TUNE_PREFIX + complete["output_dir"].rsplit("/", 1)[-1]
)
assert checkpoint is not None and laya_runtime.is_cached(checkpoint)
folder = Path(checkpoint.source)
with (
safe_open(str(folder / "model.safetensors"), "pt") as tuned,
safe_open(str(base / "model.safetensors"), "pt") as original,
):
assert set(tuned.keys()) == set(original.keys())
@needs_worker
def test_cancel_leaves_no_output_folder(base, studio_home):
from utils.paths import outputs_root
rows = [_row(i) for i in range(120)]
events = _train(_config(base, _dataset(studio_home, rows), max_steps = 0, num_epochs = 3), False)
complete = _of(events, "complete")[-1]
assert complete["output_dir"] is None
assert complete["status_message"] == "Training cancelled"
assert _of(events, "output_dir")
assert not outputs_root().exists() or not any(outputs_root().iterdir())
@needs_worker
def test_stop_while_preparing_decisions_ends_the_run(base, studio_home):
from utils.paths import outputs_root
# Seconds of preparation; the bad row comes last, so a skip warning means it ran to the end.
rows = [_row(i) for i in range(6000)] + [{"state": "slow", "questions": QUESTIONS, "gold": {}}]
events = _train(
_config(base, _dataset(studio_home, rows)),
False,
lambda event: event.get("message") == "Preparing decision rows...",
)
assert _of(events, "complete")[-1]["status_message"] == "Training cancelled"
assert not _of(events, "warning") and not _of(events, "output_dir")
assert not outputs_root().exists() or not any(outputs_root().iterdir())
def _fake_output(
root,
name,
complete = True,
):
folder = root / name
for sub in ("encoder", "tokenizer"):
(folder / sub).mkdir(parents = True)
(folder / "model.safetensors").write_bytes(b"x")
if complete:
(folder / "rl_agent_config.json").write_text("{}", encoding = "utf-8")
return folder
def test_mlx_runs_report_to_tensorboard_and_wandb(monkeypatch, tmp_path):
from types import SimpleNamespace
from core.training.decision_trainer import _mlx_tracking_callback
# Built first: importing transformers looks for the real reporting packages.
callback, state, seen = _mlx_tracking_callback(["tensorboard", "wandb"]), SimpleNamespace(), []
state.global_step = 7
writer = SimpleNamespace(add_scalar = lambda *row: seen.append(row), close = lambda: None)
boards = SimpleNamespace(SummaryWriter = lambda log_dir: seen.append(log_dir) or writer)
monkeypatch.setitem(sys.modules, "tensorboardX", boards)
wandb = SimpleNamespace(log = lambda scalars, step: seen.append((scalars, step)))
monkeypatch.setitem(sys.modules, "wandb", wandb)
monkeypatch.setenv("TENSORBOARD_LOGGING_DIR", str(tmp_path))
callback.on_train_begin(None, state, None)
callback.on_log(None, state, None, logs = {"loss": 0.5, "eval_loss": 0.25, "note": "skipped"})
scalars = {"train/loss": 0.5, "eval/loss": 0.25}
assert seen == [str(tmp_path), ("train/loss", 0.5, 7), ("eval/loss", 0.25, 7), (scalars, 7)]
def test_settings_accept_only_complete_owner_fine_tunes(studio_home, client, tmp_path):
from utils.paths import outputs_root
root = outputs_root()
_fake_output(root, "laya_done_1")
_fake_output(root, "laya_half_2", complete = False)
outside = _fake_output(tmp_path, "elsewhere")
(root / "linked").symlink_to(outside, target_is_directory = True)
names = [m["name"] for m in client.get("/api/settings/systemone").json()["models"]]
assert "laya-ft:laya_done_1" in names
assert not any(n in names for n in ("laya-ft:laya_half_2", "laya-ft:linked"))
for bad in (
"laya-ft:laya_half_2",
"laya-ft:linked",
"laya-ft:../elsewhere",
f"laya-ft:{outside}",
"laya-ft:",
):
response = client.put("/api/settings/systemone", json = {"model": bad})
assert response.status_code == 400, bad
response = client.put("/api/settings/systemone", json = {"model": "laya-ft:laya_done_1"})
assert response.status_code == 200
assert client.get("/api/settings/systemone").json()["model"] == "laya-ft:laya_done_1"
shutil.rmtree(root / "laya_done_1")
assert client.get("/api/settings/systemone").json()["model"] == "laya-multilingual"
def test_llm_output_scans_skip_decision_outputs(studio_home):
from utils.models.checkpoints import scan_checkpoints
from utils.models.model_config import scan_trained_models
from utils.paths import outputs_root
root = outputs_root()
_fake_output(root, "laya_done_1")
merged = root / "llama_merged_1"
merged.mkdir()
(merged / "config.json").write_text("{}", encoding = "utf-8")
(merged / "model.safetensors").write_bytes(b"x")
assert [name for name, _, _ in scan_trained_models(str(root))] == ["llama_merged_1"]
assert [name for name, _, _ in scan_checkpoints(str(root))] == ["llama_merged_1"]
def test_export_checkpoint_list_includes_decision_runs(studio_home):
from routes import models as models_routes
from utils.models.checkpoints import list_preview_targets
from utils.paths import outputs_root
root = outputs_root()
laya = _fake_output(root, "laya_done_1")
(laya / "rl_agent_config.json").write_text(
json.dumps({"encoder": "answerdotai/ModernBERT-base", "training": {"base": "laya-tiny"}}),
encoding = "utf-8",
)
_fake_output(root, "laya_half_2", complete = False)
clef = root / "clef_merged_3"
clef.mkdir()
for name in ("config.json", "joint_head_config.json", "joint_head.safetensors"):
(clef / name).write_text("{}", encoding = "utf-8")
app = FastAPI()
app.include_router(models_routes.router, prefix = "/api/models")
app.dependency_overrides[get_current_subject] = lambda: "unsloth"
response = TestClient(app).get("/api/models/checkpoints", params = {"outputs_dir": str(root)})
assert response.status_code == 200, response.text
listed = {m["name"]: m for m in response.json()["models"]}
assert sorted(listed) == ["clef_merged_3", "laya_done_1"]
assert listed["laya_done_1"]["base_model"] == "laya-tiny"
assert listed["laya_done_1"]["checkpoints"][0]["path"] == str(laya)
# Chat preview still lists only what chat can load.
assert [t["run"] for t in list_preview_targets(str(root))] == ["clef_merged_3"]
def test_model_config_offers_an_llm_as_a_decision_model(studio_home):
import asyncio
from routes.models import get_model_config
from utils.models.model_config import load_llm_decision_defaults
llm = studio_home / "llm"
llm.mkdir()
config = {
"model_type": "llama",
"architectures": ["LlamaForCausalLM"],
"hidden_size": 64,
"num_hidden_layers": 2,
"num_attention_heads": 4,
"vocab_size": 64,
}
(llm / "config.json").write_text(json.dumps(config), encoding = "utf-8")
(llm / "model.safetensors").write_bytes(b"x")
def fetch(**kwargs):
return asyncio.run(
get_model_config(model_name = str(llm), hf_token = None, current_subject = "tester", **kwargs)
)
plain = fetch()
assert plain.model_type == "text" and plain.decision_layout is None
decision = fetch(as_decision = True)
assert decision.model_type == "decision" and decision.is_decision is True
assert decision.decision_layout == "llm" and decision.decision_checkpoints is None
assert decision.config == load_llm_decision_defaults()
def test_model_config_classifies_a_local_laya_folder(base):
import asyncio
from routes.models import get_model_config
result = asyncio.run(
get_model_config(model_name = str(base), hf_token = None, current_subject = "tester")
)
assert result.model_type == "decision" and result.is_decision is True
assert float(result.config["training"]["learning_rate"]) == 8e-4
assert result.config["training"]["optim"] == "adamw_torch"
assert result.config["lora"]["lora_r"] == 64
assert result.decision_checkpoints is None
def test_start_preflight_accepts_a_local_laya_folder(base):
from models.training import TrainingStartRequest
from routes.training import _reject_untrainable_model_request
request = TrainingStartRequest(
model_name = str(base),
training_type = "Full Finetuning",
is_decision = True,
hf_dataset = "org/decisions",
format_type = "auto",
)
assert _reject_untrainable_model_request(request).model_name == str(base.resolve())
def test_arrow_merged_criteria_are_not_trained_as_options(tmp_path):
pa = pytest.importorskip("pyarrow")
import pyarrow.parquet as pq
from core.training.decision_trainer import STRUCT_COLUMNS_WARNING, _read_local_rows
from utils.datasets.cache_safe import load_dataset_cache_safe
rows = [
{
"state": "refund my card",
"questions": {"q": {"type": "choice", "instructions": "Team?", "criteria": crit}},
"gold": {"q": {"label": label, "probabilities": probs}},
}
for crit, label, probs in (
({"billing": "charges", "outage": ""}, "billing", {"billing": 0.9, "outage": 0.1}),
({"login": "accounts", "slow": "speed"}, "slow", {"login": 0.2, "slow": 0.8}),
)
]
path = tmp_path / "decisions.parquet"
pq.write_table(pa.Table.from_pylist(rows), str(path))
warnings = []
loaded = _read_local_rows([str(path)], load_dataset_cache_safe, warnings.append)
assert [row["questions"]["q"]["criteria"] for row in loaded] == [
{"billing": "charges", "outage": ""},
{"login": "accounts", "slow": "speed"},
]
assert loaded[1]["gold"]["q"]["probabilities"] == {"login": 0.2, "slow": 0.8}
assert warnings == [STRUCT_COLUMNS_WARNING]
strings = tmp_path / "strings.parquet"
pq.write_table(
pa.Table.from_pylist(
[{key: json.dumps(value) for key, value in row.items()} for row in rows]
),
str(strings),
)
warnings.clear()
_read_local_rows([str(strings)], load_dataset_cache_safe, warnings.append)
assert not warnings
def test_jsonl_rows_keep_unicode_separators_inside_strings(tmp_path):
from core.training.decision_trainer import _read_local_rows
rows = [
{"state": "one\u2028two\u2029three\u0085four", "gold": "a"},
{"state": "plain", "gold": "b"},
]
path = tmp_path / "decisions.jsonl"
path.write_text(
"\n".join(json.dumps(row, ensure_ascii = False) for row in rows) + "\n",
encoding = "utf-8",
)
assert _read_local_rows([str(path)], None, lambda message: None) == rows
def _load(should_stop = lambda: False, **config):
from core.training.decision_trainer import _load_rows
warnings = []
rows, eval_rows = _load_rows(
{"random_seed": 3407, "eval_steps": 0, **config},
should_stop,
lambda message: None,
warnings.append,
)
return rows, eval_rows, warnings
def test_local_files_are_read_before_a_hub_dataset(tmp_path):
rows, _, _ = _load(
local_datasets = [_dataset(tmp_path, [_row(0)])], hf_dataset = "nobody/not-a-dataset"
)
assert rows == [_row(0)]
def test_the_train_split_is_never_the_held_out_set(tmp_path):
hub = tmp_path / "decisions"
hub.mkdir()
for split, count in (("train", 30), ("test", 10)):
(hub / f"{split}.jsonl").write_text(
"\n".join(json.dumps(_row(i)) for i in range(count)), encoding = "utf-8"
)
rows, eval_rows, warnings = _load(
hf_dataset = str(hub), train_split = "train", eval_split = "train", eval_steps = 0.1
)
assert len(rows) == 30 and eval_rows is None
assert warnings == [
"The evaluation split is the training split, so part of it is held out "
"of training for evaluation instead."
]
rows, eval_rows, warnings = _load(
hf_dataset = str(hub), train_split = "train", eval_split = "test", eval_steps = 0.1
)
assert len(rows) == 30 and len(eval_rows) == 10 and not warnings
def test_eval_rows_are_sampled_before_they_are_prepared(tmp_path):
from core.training.decision_trainer import EVAL_MAX
held = tmp_path / "held"
held.mkdir()
_, eval_rows, _ = _load(
local_datasets = [_dataset(tmp_path, [_row(0)])],
local_eval_datasets = [_dataset(held, [_row(i) for i in range(EVAL_MAX + 500)])],
eval_steps = 0.1,
)
assert len(eval_rows) == EVAL_MAX
def test_csv_cells_are_read_as_text(tmp_path):
import csv
path = tmp_path / "decisions.csv"
with path.open("w", newline = "", encoding = "utf-8") as file:
writer = csv.writer(file)
writer.writerow(["state", "questions", "gold"])
for state in ("NA", "00501", "null"):
writer.writerow([state, _row(0)["questions"], _row(0)["gold"]])
rows, _, _ = _load(local_datasets = [str(path)])
assert [row["state"] for row in rows] == ["NA", "00501", "null"]
def test_a_stop_during_the_s3_download_cancels_the_run(monkeypatch):
from core.training import s3_dataset
from core.training.decision_trainer import _Stopped
def download(s3_config, cancel_callback):
assert cancel_callback()
raise s3_dataset.S3DownloadCancelled("S3 dataset download cancelled")
monkeypatch.setattr(s3_dataset, "prepare_s3_dataset_download", download)
with pytest.raises(_Stopped, match = "Training cancelled"):
_load(lambda: True, s3_config = {"bucket": "b"})
def test_s3_audio_rows_keep_their_clips_after_the_download_is_cleaned_up(monkeypatch, tmp_path):
import shutil
from types import SimpleNamespace
from core.training import s3_dataset
clips = [tmp_path / "a.wav", tmp_path / "b.wav", tmp_path / "c.wav"]
for index, clip in enumerate(clips):
clip.write_bytes(b"RIFF" + bytes([index]))
manifest = tmp_path / "rows.jsonl"
rows = [
{**_row(0), "audio": {"bytes": None, "path": str(clips[0])}},
{**_row(1), "audio": [{"bytes": None, "path": str(clips[1])}]},
# A plain path string, as the S3 rewrite leaves a manifest's usual audio cell.
{**_row(2), "audio": str(clips[2])},
_row(3),
]
manifest.write_text("\n".join(json.dumps(row) for row in rows), encoding = "utf-8")
copy = tmp_path / "download"
copy.mkdir()
shutil.copy(manifest, copy)
def download(s3_config, cancel_callback):
return SimpleNamespace(
files = [str(copy / "rows.jsonl")], cleanup = lambda: [clip.unlink() for clip in clips]
)
monkeypatch.setattr(s3_dataset, "prepare_s3_dataset_download", download)
loaded, _, _ = _load(s3_config = {"bucket": "b"})
assert [
loaded[0]["audio"]["bytes"],
loaded[1]["audio"][0]["bytes"],
loaded[2]["audio"]["bytes"],
] == [b"RIFF\x00", b"RIFF\x01", b"RIFF\x02"]
assert "audio" not in loaded[3]
assert not any(clip.exists() for clip in clips)
# A slice reads only the clips of the rows it keeps.
for index, clip in enumerate(clips):
clip.write_bytes(b"RIFF" + bytes([index]))
read, read_bytes = [], Path.read_bytes
monkeypatch.setattr(Path, "read_bytes", lambda path: read.append(path.name) or read_bytes(path))
loaded, _, _ = _load(s3_config = {"bucket": "b"}, dataset_slice_start = 1, dataset_slice_end = 1)
assert [row["audio"][0]["bytes"] for row in loaded] == [b"RIFF\x01"] and read == ["b.wav"]
def test_other_accounts_reach_only_the_configured_fine_tune(studio_home, client):
from utils.account_context import AccountContext, bind_account, reset_account
from utils.paths import outputs_root
root = outputs_root()
_fake_output(root, "laya_done_1")
_fake_output(root, "laya_done_2")
assert (
client.put("/api/settings/systemone", json = {"model": "laya-ft:laya_done_1"}).status_code
== 200
)
token = bind_account(AccountContext("b" * 32, "bob"))
try:
assert catalog.resolve("laya-ft:laya_done_1") is not None
assert catalog.resolve("laya-ft:laya_done_2") is None
finally:
reset_account(token)
assert catalog.resolve("laya-ft:laya_done_2") is not None
def test_only_the_owner_can_start_decision_training():
from fastapi import HTTPException
from models.training import TrainingStartRequest
from routes.training import _validate_decision_request
from utils.account_context import AccountContext, bind_account, reset_account
request = TrainingStartRequest(
model_name = catalog.LAYA_REPO,
model_subfolder = "multilingual",
training_type = "Full Finetuning",
is_decision = True,
hf_dataset = "org/decisions",
format_type = "auto",
)
_validate_decision_request(request)
token = bind_account(AccountContext("b" * 32, "bob"))
try:
with pytest.raises(HTTPException) as refused:
_validate_decision_request(request)
finally:
reset_account(token)
assert refused.value.status_code == 403
def test_eval_files_that_are_training_files_are_never_the_held_out_set(tmp_path):
train = _dataset(tmp_path, [_row(i) for i in range(30)])
for eval_files in ([train], [str(tmp_path)], [str(tmp_path / "." / "decisions.jsonl")]):
rows, eval_rows, warnings = _load(
local_datasets = [train], local_eval_datasets = eval_files, eval_steps = 0.1
)
assert len(rows) == 30 and eval_rows is None, eval_files
assert warnings == [
"The evaluation files include training files, so part of the training data is "
"held out for evaluation instead."
]
@needs_worker
def test_cancel_during_calibration_saves_nothing(base, studio_home):
from utils.paths import outputs_root
rows = [_row(i) for i in range(120)]
events = _train(
_config(base, _dataset(studio_home, rows)),
False,
when = lambda event: event.get("message") == "Calibrating confidence...",
)
complete = _of(events, "complete")[-1]
assert (complete["output_dir"], complete["status_message"]) == (None, "Training cancelled")
assert not outputs_root().exists() or not any(outputs_root().iterdir())
def test_held_out_decisions_count_every_clef_question():
from core.training.decision_trainer import _decision_count
laya = [{"row": 0}, {"row": 1}]
clef = [{"row": 0, "labels": [0, 1, 2]}, {"row": 1, "labels": [1]}]
assert (_decision_count(laya), _decision_count(clef)) == (2, 4)
@pytest.mark.parametrize(
"model_type, mlx, zoo_d1, head",
[
("lfm3_omni", True, True, "d1"),
("lfm3_vl", True, True, "d1"),
("lfm3", True, True, "clef"),
("qwen3_5", True, True, "clef"),
("lfm3_omni", False, True, "clef"),
("lfm3_vl", True, False, "clef"),
],
)
def test_lfm3_vision_and_omni_train_as_a_d1_on_mlx(
tmp_path, monkeypatch, model_type, mlx, zoo_d1, head
):
from core.training import decision_trainer
monkeypatch.setattr(decision_trainer, "_is_mlx", lambda: mlx)
monkeypatch.setattr(decision_trainer, "_zoo_trains_d1", lambda: zoo_d1)
(tmp_path / "config.json").write_text(json.dumps({"model_type": model_type}), encoding = "utf-8")
assert decision_trainer._lm_decision_head(str(tmp_path), None) == head
def test_an_unreadable_language_model_keeps_the_clef_head(tmp_path, monkeypatch):
from core.training import decision_trainer
monkeypatch.setattr(decision_trainer, "_is_mlx", lambda: True)
assert decision_trainer._lm_decision_head(str(tmp_path), None) == "clef"