Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: report bad decision datasets before the model downloads (#16498)
Nilay096fa323e9c0
diff --git a/studio/backend/core/training/decision_trainer.py b/studio/backend/core/training/decision_trainer.pyindex d0769fa3063..c27ac9274d7 100644--- a/studio/backend/core/training/decision_trainer.py+++ b/studio/backend/core/training/decision_trainer.py@@ -8,6 +8,8 @@ import jsonimport mathimport osimport random+import re+import sysimport threadingimport timefrom contextvars import copy_context@@ -34,6 +36,35 @@ class _Stopped(Exception):pass+_abandoned_downloads: list[threading.Thread] = []+++def end_abandoned_download(event_queue: Any) -> None:+ # Shutdown joins the hub executor threads, so the worker would keep downloading and refuse the next run.+ # Only a worker process owns its exit; the in-process MLX adapter passes a plain queue from a thread.+ owns_process = (+ hasattr(event_queue, "join_thread")+ and threading.current_thread() is threading.main_thread()+ )+ if not owns_process or not any(thread.is_alive() for thread in _abandoned_downloads):+ return+ with contextlib.suppress(Exception):+ event_queue.close()+ event_queue.join_thread()+ for stream in (sys.stdout, sys.stderr):+ with contextlib.suppress(Exception):+ stream.flush()+ with contextlib.suppress(Exception):+ from utils.worker_stderr import stop_worker_stderr_mirror+ stop_worker_stderr_mirror()+ os._exit(0)+++def _no_decisions_message(reason: str | None) -> str:+ detail = f" ({reason})" if reason else ""+ return f"No usable decisions in the dataset{detail}. Each row needs state, questions and gold."++def _decision_count(items: list) -> int:# A Clef item holds every question of its row, as split_holdout counts them.return sum(len(item.get("labels", (None,))) for item in items)@@ -101,6 +132,11 @@ def _local_files(paths: list[str]) -> list[Path]:return files+def _bad_json(path: Path, line: int, exc: json.JSONDecodeError) -> ValueError:+ name = re.sub(r"^[0-9a-f]{32}_", "", path.name)+ return ValueError(f"Invalid JSON at {name}:{line}: {exc.msg}")++def _read_local_rows(paths: list[str], load_dataset, warn: Callable[[str], None]) -> list[dict]:from utils.datasets.cells import csv_as_text_kwargs@@ -113,11 +149,22 @@ def _read_local_rows(paths: list[str], load_dataset, warn: Callable[[str], None]if suffix in (".json", ".jsonl"):# Read directly: Arrow would merge every question's criteria keys into one struct.text = path.read_text(encoding = "utf-8-sig")+ if not text.strip():+ continuetry:data = json.loads(text)rows.extend(data if isinstance(data, list) else [data])- except ValueError:- rows.extend(json.loads(line) for line in text.splitlines() if line.strip())+ except json.JSONDecodeError as exc:+ # "Extra data" means the first line parsed, so this is JSONL.+ if exc.msg != "Extra data":+ raise _bad_json(path, exc.lineno, exc) from None+ for number, line in enumerate(text.splitlines(), 1):+ if not line.strip():+ continue+ try:+ rows.append(json.loads(line))+ except json.JSONDecodeError as bad_line:+ raise _bad_json(path, number, bad_line) from Noneelif suffix in (".csv", ".parquet"):dataset = load_dataset(suffix[1:],@@ -239,11 +286,19 @@ def _load_rows(class _DecisionRows:"""Reads raw rows on a thread, in this job's context, while the model loads."""- def __init__(self, config, should_stop, status, warn):+ def __init__(+ self,+ config,+ should_stop,+ status,+ warn,+ check = None,+ ):self._config = configself._should_stop = should_stopself._status = statusself._warn = warn+ self._check = checkself._cancelled = threading.Event()self._done = threading.Event()self._lock = threading.Lock()@@ -272,9 +327,12 @@ class _DecisionRows:try:if self._stopped():raise _Stopped("Training cancelled")- self._result = _load_rows(+ result = _load_rows(self._config, self._stopped, self._report_status, self._report_warning)+ if self._check is not None:+ self._check(result[0])+ self._result = resultexcept BaseException as exc:self._error = excfinally:@@ -300,6 +358,32 @@ class _DecisionRows:raise self._errorreturn self._result+ def alongside(self, work, check_stop):+ from utils.account_context import account_thread++ failure = []++ def run():+ try:+ work()+ except BaseException as exc:+ failure.append(exc)++ context = copy_context()+ thread = account_thread(target = context.run, args = (run,), daemon = True)+ thread.start()+ try:+ while thread.is_alive():+ thread.join(0.1)+ check_stop()+ if self._done.is_set() and self._error is not None:+ raise self._error+ except BaseException:+ _abandoned_downloads.append(thread)+ raise+ if failure:+ raise failure[0]+def __exit__(self, *_exc):with self._lock:self._cancelled.set()@@ -513,9 +597,18 @@ def _run(event_queue: Any, stop_queue: Any, config: dict, output_dir: str) -> Nouse_lora = config["training_type"] == "LoRA/QLoRA"gradient_checkpointing = normalize_gradient_checkpointing(config["gradient_checkpointing"])- with _DecisionRows(config, lambda: stop["requested"], status, warn) as pending_rows:+ def check_rows(rows: list) -> None:+ problem = FastDecisionModel.rows_problem(rows, clef, validate)+ if problem is not None:+ raise ValueError(_no_decisions_message(problem))++ with _DecisionRows(config, lambda: stop["requested"], status, warn, check_rows) as pending_rows:check_stop()- _download_decision_checkpoint(event_queue, config)+ if config.get("local_datasets"):+ pending_rows.result(check_stop)+ pending_rows.alongside(+ lambda: _download_decision_checkpoint(event_queue, config), check_stop+ )check_stop()status("Loading decision model...")if llm:@@ -593,12 +686,7 @@ def _run(event_queue: Any, stop_queue: Any, config: dict, output_dir: str) -> No"reason": report["reason"] or eval_report["reason"],}if not items:- reason = f" ({report['reason']})" if report["reason"] else ""- send(- "error",- error = f"No usable decisions in the dataset{reason}. Each row needs state, questions and gold.",- stack = "",- )+ send("error", error = _no_decisions_message(report["reason"]), stack = "")returnif report["skipped"]:warn(f"Skipped {report['skipped']:,} of {report['total']:,} decisions: {report['reason']}.")diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex b9c5af781ae..a6a3d5103b9 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -2511,7 +2511,12 @@ class TrainingBackend:"""First model-load stall respawns the worker over HTTP; later stalls surface as errors."""msg = event.get("message", "Download stalled")with self._lock:- recover = self._in_model_load and not self._xet_fallback_used+ # A stall queued behind an error or completion must not respawn a finished run.+ recover = (+ self._in_model_load+ and not self._xet_fallback_used+ and not self._complete_seen.is_set()+ )proc = self._procrun_id = self.current_job_idif recover:diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 72a1b20a8cd..a4b7f3ed5be 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -3930,6 +3930,13 @@ def run_mlx_training_process("ts": time.time(),})+ _end_abandoned_decision_download(event_queue)+++def _end_abandoned_decision_download(event_queue: Any) -> None:+ trainer = sys.modules.get("core.training.decision_trainer")+ if trainer is not None:+ trainer.end_abandoned_download(event_queue)def _training_job_is_local(config) -> bool:@@ -4768,6 +4775,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->"ts": time.time(),})+ _end_abandoned_decision_download(event_queue)returnif config.get("batch_members"):diff --git a/studio/backend/tests/test_decision_dataset_prefetch.py b/studio/backend/tests/test_decision_dataset_prefetch.pyindex 007fca1c770..49e7221789d 100644--- a/studio/backend/tests/test_decision_dataset_prefetch.py+++ b/studio/backend/tests/test_decision_dataset_prefetch.py@@ -1,8 +1,13 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import json+import subprocessimport sys+import textwrap+import timefrom contextvars import ContextVar+from pathlib import Pathfrom queue import Queuefrom threading import Event, current_threadfrom types import ModuleType, SimpleNamespace@@ -120,7 +125,7 @@ def fake_decision_runtime(monkeypatch):monkeypatch.setenv("HF_TOKEN", "fixture-token")monkeypatch.setattr(import_guards, "ensure_real_packages", lambda *_a: None)monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda *_a: "local-checkpoint")- model = SimpleNamespace()+ model = SimpleNamespace(rows_problem = lambda *_a: None)unsloth = ModuleType("unsloth")unsloth.DecisionTrainer = objectunsloth.FastDecisionModel = model@@ -212,6 +217,138 @@ def test_raw_rows_overlap_checkpoint_and_model_load_but_preparation_waits(assert messages[-1] == "Preparing decision rows..."+def _decision_config(**overrides):+ return {+ "model_name": "example/decision",+ "decision_layout": "clef",+ "training_type": "Full Finetuning",+ "random_seed": 7,+ "gradient_checkpointing": "none",+ **overrides,+ }+++def test_unusable_local_rows_fail_before_the_model_is_touched(fake_decision_runtime, tmp_path):+ model_api, worker = fake_decision_runtime+ touched = []+ dataset = tmp_path / "decisions.jsonl"+ dataset.write_text('{"state": "only a state"}\n', encoding = "utf-8")+ model_api.rows_problem = lambda rows, clef, validate: "row 1 needs state, questions and gold"+ worker._load_embedding_hf_dataset = None+ worker._download_decision_checkpoint = lambda *_a: touched.append("download")+ model_api.from_pretrained = lambda *_a, **_kw: touched.append("load")++ config = _decision_config(local_datasets = [str(dataset)], eval_steps = 0)+ with pytest.raises(ValueError, match = "No usable decisions in the dataset .row 1 needs state"):+ decision._run(Queue(), Queue(), config, "unused-output")+ assert touched == []+++def test_unusable_hub_rows_end_a_model_download_in_progress(fake_decision_runtime, monkeypatch):+ model_api, worker = fake_decision_runtime+ downloading, release = Event(), Event()+ loaded = []++ def load_rows(*_a):+ assert downloading.wait(3)+ return [{"state": "only a state"}], None++ def download(*_a):+ downloading.set()+ release.wait(3)++ monkeypatch.setattr(decision, "_load_rows", load_rows)+ model_api.rows_problem = lambda rows, clef, validate: "row 1 has no gold"+ worker._download_decision_checkpoint = download+ model_api.from_pretrained = lambda *_a, **_kw: loaded.append(release.is_set())+ try:+ with pytest.raises(ValueError, match = "row 1 has no gold"):+ decision._run(Queue(), Queue(), _decision_config(), "unused-output")+ assert not release.is_set() and loaded == []+ assert decision._abandoned_downloads[-1].is_alive()+ finally:+ release.set()+ while decision._abandoned_downloads:+ decision._abandoned_downloads.pop().join(3)+++def test_an_abandoned_download_does_not_hold_the_worker_open(tmp_path):+ script = textwrap.dedent(+ """+ import multiprocessing, threading, time+ from concurrent.futures import ThreadPoolExecutor+ from core.training import decision_trainer, worker++ def download():+ with ThreadPoolExecutor(2) as pool:+ list(pool.map(time.sleep, [30] * 4))++ thread = threading.Thread(target = download, daemon = True)+ thread.start()+ decision_trainer._abandoned_downloads.append(thread)+ events = multiprocessing.get_context("spawn").Queue()+ events.put({"type": "error", "error": "bad rows"})+ # The in-process MLX adapter: a plain queue, or any call off the main thread, must return.+ import queue+ worker._end_abandoned_decision_download(queue.Queue())+ caller = threading.Thread(target = worker._end_abandoned_decision_download, args = (events,))+ caller.start()+ caller.join()+ print("host survived", flush = True)+ worker._end_abandoned_decision_download(events)+ """+ )+ backend = Path(__file__).resolve().parents[1]+ started = time.monotonic()+ # A file, not a pipe: the queue's resource tracker would hold a pipe open past the child.+ log = tmp_path / "child.log"+ with log.open("w") as out:+ proc = subprocess.Popen(+ [sys.executable, "-c", script], cwd = backend, stdout = out, stderr = subprocess.STDOUT+ )+ try:+ returncode = proc.wait(timeout = 25)+ finally:+ proc.kill()+ assert returncode == 0, log.read_text()[-2000:]+ assert "host survived" in log.read_text()+ assert time.monotonic() - started < 25+++def test_a_bad_jsonl_line_names_its_file_and_line(tmp_path):+ good = json.dumps({"state": "fine"})+ dataset = tmp_path / "decisions.jsonl"+ dataset.write_text(f"{good}\n\n{good}\n{{broken\n", encoding = "utf-8")+ with pytest.raises(ValueError, match = r"Invalid JSON at decisions\.jsonl:4: "):+ decision._read_local_rows([str(dataset)], None, _no_stop)+++def test_an_empty_jsonl_shard_beside_real_rows_is_skipped(tmp_path):+ (tmp_path / "a.jsonl").write_text(json.dumps({"state": "fine"}) + "\n", encoding = "utf-8")+ (tmp_path / "b.jsonl").write_text("", encoding = "utf-8")+ (tmp_path / "c.jsonl").write_text("\n", encoding = "utf-8")+ rows = decision._read_local_rows([str(tmp_path)], None, _no_stop)+ assert rows == [{"state": "fine"}]+++def test_a_bad_line_in_an_upload_names_the_file_as_it_was_uploaded(tmp_path):+ dataset = tmp_path / f"{'a1' * 16}_decisions.jsonl"+ dataset.write_text('{"state": "fine"}\n{broken\n', encoding = "utf-8")+ with pytest.raises(ValueError, match = r"Invalid JSON at decisions\.jsonl:2: "):+ decision._read_local_rows([str(dataset)], None, _no_stop)+++def test_a_bad_first_line_and_a_pretty_printed_file_report_the_real_line(tmp_path):+ first = tmp_path / "first.jsonl"+ first.write_text('{broken\n{"state": "fine"}\n', encoding = "utf-8")+ pretty = tmp_path / "pretty.json"+ pretty.write_text('[\n {"state": "fine"},\n {"state": }\n]\n', encoding = "utf-8")+ with pytest.raises(ValueError, match = r"first\.jsonl:1: "):+ decision._read_local_rows([str(first)], None, _no_stop)+ with pytest.raises(ValueError, match = r"pretty\.json:3: "):+ decision._read_local_rows([str(pretty)], None, _no_stop)++def test_prefetch_keeps_strict_cached_splits_bounds_revision_and_token(monkeypatch):import importlibfrom utils.datasets import cache_safediff --git a/studio/backend/tests/test_decision_training.py b/studio/backend/tests/test_decision_training.pyindex 234ce943831..6195feffa14 100644--- a/studio/backend/tests/test_decision_training.py+++ b/studio/backend/tests/test_decision_training.py@@ -398,6 +398,7 @@ def test_all_invalid_rows_are_an_error_and_save_nothing(base, studio_home):events = _train(_config(base, _dataset(studio_home, rows)))assert "No usable decisions" in _of(events, "error")[0]["error"]+ assert "Loading decision model..." not in [e["message"] for e in _of(events, "status")]assert not _of(events, "complete")assert not outputs_root().exists() or not any(outputs_root().iterdir())diff --git a/studio/backend/tests/test_training_xet_fallback.py b/studio/backend/tests/test_training_xet_fallback.pyindex c7b360ed1e4..488900baf56 100644--- a/studio/backend/tests/test_training_xet_fallback.py+++ b/studio/backend/tests/test_training_xet_fallback.py@@ -322,6 +322,15 @@ def test_second_stall_surfaces_error_without_respawn():assert proc.is_alive() is False+def test_stall_queued_behind_an_error_does_not_respawn():+ b, proc = _backend_mid_load()+ b._handle_event({"type": "error", "error": "No usable decisions in the dataset", "stack": ""})+ b._handle_event({"type": "stall", "message": "no progress for 180s"})+ assert b._needs_xet_respawn is False+ assert b._progress.error == "No usable decisions in the dataset"+ assert proc.is_alive() is False++def test_model_load_completed_disarms_recovery():b, _ = _backend_mid_load()b._handle_event({"type": "model_load_completed"})diff --git a/tests/test_decision_model.py b/tests/test_decision_model.pyindex ded9c396e5a..de756c545f8 100644--- a/tests/test_decision_model.py+++ b/tests/test_decision_model.py@@ -251,6 +251,35 @@ def test_dataset_skips_bad_rows_and_names_the_most_common_reason(checkpoint):assert torch.allclose(batch["target"].sum(-1), torch.ones(3))[email protected]("clef", [False, True])+def test_rows_problem_needs_no_model_and_stops_at_the_first_usable_row(clef):+ assert FastDecisionModel.rows_problem([], clef) == ""+ no_gold = {**_row(0), "gold": {}}+ no_questions = {"state": "only a state"}+ assert FastDecisionModel.rows_problem([no_questions], clef) == (+ "row 1 needs state, questions and gold"+ )+ assert FastDecisionModel.rows_problem([no_questions, no_questions], clef) == (+ "row 1 needs state, questions and gold (and 1 more like it)"+ )+ assert FastDecisionModel.rows_problem([no_gold], clef).startswith('row 1: "')++ def record(name, question):+ seen.append(name)++ seen = []+ assert FastDecisionModel.rows_problem([no_gold, _row(1), no_questions], clef, record) is None+ assert len(seen) == 1+++def test_rows_problem_applies_the_callers_validation():+ def refuse(name, question):+ raise ValueError("is over the limit")++ problem = FastDecisionModel.rows_problem([_row(0)], False, refuse)+ assert problem.endswith("is over the limit (and 2 more like it)")++def test_holdout_takes_whole_rows_and_never_overshoots():items = [{"row": row} for row in range(50) for _ in range(3)]train, held = FastDecisionModel.split_holdout(items, 3407)diff --git a/tests/test_decision_model_mlx.py b/tests/test_decision_model_mlx.pyindex bc45421ea53..f5be50aa3af 100644--- a/tests/test_decision_model_mlx.py+++ b/tests/test_decision_model_mlx.py@@ -200,6 +200,14 @@ def test_decision_helpers_are_one_torch_free_module_for_both_backends():assert [a.name for n in taken for a in n.names] == ["*"][email protected]("clef", [False, True])+def test_rows_problem_checks_rows_without_a_model(clef):+ assert FastDecisionModel.rows_problem([{"state": "s"}, _row(0)], clef) is None+ assert FastDecisionModel.rows_problem([{"state": "s"}], clef) == (+ "row 1 needs state, questions and gold"+ )++@pytest.fixturedef clef_checkpoint(tmp_path, monkeypatch):from types import SimpleNamespacediff --git a/unsloth/models/_decision_common.py b/unsloth/models/_decision_common.pyindex 0ecb7f28b99..01ba53cfbea 100644--- a/unsloth/models/_decision_common.py+++ b/unsloth/models/_decision_common.py@@ -35,6 +35,7 @@ __all__ = ["_target","_target_for","_clef_question",+ "_rows_problem","_predicted","_metrics","_fit_temperature",@@ -50,7 +51,7 @@ import jsonimport mathimport sysfrom pathlib import Path-from typing import Optional+from typing import Callable, OptionalTRAIN_MAX_LEN, TRAIN_HEAD_MAX_LEN = 1024, 256HOLDOUT_MAX = 400@@ -360,6 +361,53 @@ def _clef_question(question) -> dict:return clef_question+def _rows_problem(+ rows,+ clef: bool,+ validate: Optional[Callable] = None,+) -> Optional[str]:+ # None at the first decision that can train, else why none can ("" when there are no rows).+ skips = {}++ def skip(+ index,+ reason,+ name = None,+ ):+ where = f"row {index + 1}" if name is None else f'row {index + 1}: "{name}"'+ skips.setdefault(reason, [0, f"{where} {reason}"])[0] += 1++ for index, row in enumerate(rows):+ row = row if isinstance(row, dict) else {}+ state = _parsed(row.get("state"))+ questions = _parsed(row.get("questions"))+ gold = _parsed(row["gold"] if row.get("gold") is not None else row.get("answers"))+ if state is None or not isinstance(questions, dict) or not isinstance(gold, dict):+ skip(index, "needs state, questions and gold")+ continue+ for name, question in questions.items():+ if name not in gold:+ skip(index, "has no gold", name)+ continue+ try:+ if validate is not None:+ validate(name, question)+ if clef:+ parsed = _clef_question(question)+ internal = {"t": parsed["type"], "crit": parsed.get("criteria")}+ else:+ internal = _internal(question)+ _target(internal, _parsed(gold[name]))+ except (TypeError, ValueError, KeyError) as exc:+ skip(index, str(exc) or "is not a valid question", name)+ continue+ return None+ if not skips:+ return ""+ count, example = max(skips.values(), key = lambda skipped: skipped[0])+ return example if count == 1 else f"{example} (and {count - 1:,} more like it)"++def _predicted(question: dict, answer: dict, probabilities: dict) -> dict:# The Decision API answer plus "answer": the option (choice), True / False (noul) or level number (score).kind = question["type"]diff --git a/unsloth/models/decision.py b/unsloth/models/decision.pyindex 3ac49d0d7b5..f8e87e1b3ad 100644--- a/unsloth/models/decision.py+++ b/unsloth/models/decision.py@@ -1832,6 +1832,8 @@ class FastDecisionModel:)return items, report+ rows_problem = staticmethod(_rows_problem)+@staticmethoddef split_holdout(items: list,diff --git a/unsloth/models/decision_mlx.py b/unsloth/models/decision_mlx.pyindex 7c6b205fd7f..72588c99b66 100644--- a/unsloth/models/decision_mlx.py+++ b/unsloth/models/decision_mlx.py@@ -50,6 +50,7 @@ from unsloth._decision_common import (_parsed,_predicted,_row_images,+ _rows_problem,_served_lengths,_served_temperatures,_target,@@ -871,6 +872,7 @@ class FastDecisionModel:clef = functools.partial(_clef_items, model._unsloth_pipeline) if _is_clef(model) else Nonereturn _decision_dataset(rows, tokenizer, model.decision_config, validate, clef)+ rows_problem = staticmethod(_rows_problem)split_holdout = staticmethod(_decision_holdout)@staticmethod