Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: report bad decision datasets before the model downloads (#16498)

Nilay096fa323e9c0
diff --git a/studio/backend/core/training/decision_trainer.py b/studio/backend/core/training/decision_trainer.py
index d0769fa3063..c27ac9274d7 100644
--- a/studio/backend/core/training/decision_trainer.py
+++ b/studio/backend/core/training/decision_trainer.py
@@ -8,6 +8,8 @@ import json
import math
import os
import random
+import re
+import sys
import threading
import time
from contextvars import copy_context
@@ -34,6 +36,35 @@ class _Stopped(Exception):
pass
+_abandoned_downloads: list[threading.Thread] = []
+
+
+def end_abandoned_download(event_queue: Any) -> None:
+ # Shutdown joins the hub executor threads, so the worker would keep downloading and refuse the next run.
+ # Only a worker process owns its exit; the in-process MLX adapter passes a plain queue from a thread.
+ owns_process = (
+ hasattr(event_queue, "join_thread")
+ and threading.current_thread() is threading.main_thread()
+ )
+ if not owns_process or not any(thread.is_alive() for thread in _abandoned_downloads):
+ return
+ with contextlib.suppress(Exception):
+ event_queue.close()
+ event_queue.join_thread()
+ for stream in (sys.stdout, sys.stderr):
+ with contextlib.suppress(Exception):
+ stream.flush()
+ with contextlib.suppress(Exception):
+ from utils.worker_stderr import stop_worker_stderr_mirror
+ stop_worker_stderr_mirror()
+ os._exit(0)
+
+
+def _no_decisions_message(reason: str | None) -> str:
+ detail = f" ({reason})" if reason else ""
+ return f"No usable decisions in the dataset{detail}. Each row needs state, questions and gold."
+
+
def _decision_count(items: list) -> int:
# A Clef item holds every question of its row, as split_holdout counts them.
return sum(len(item.get("labels", (None,))) for item in items)
@@ -101,6 +132,11 @@ def _local_files(paths: list[str]) -> list[Path]:
return files
+def _bad_json(path: Path, line: int, exc: json.JSONDecodeError) -> ValueError:
+ name = re.sub(r"^[0-9a-f]{32}_", "", path.name)
+ return ValueError(f"Invalid JSON at {name}:{line}: {exc.msg}")
+
+
def _read_local_rows(paths: list[str], load_dataset, warn: Callable[[str], None]) -> list[dict]:
from utils.datasets.cells import csv_as_text_kwargs
@@ -113,11 +149,22 @@ def _read_local_rows(paths: list[str], load_dataset, warn: Callable[[str], None]
if suffix in (".json", ".jsonl"):
# Read directly: Arrow would merge every question's criteria keys into one struct.
text = path.read_text(encoding = "utf-8-sig")
+ if not text.strip():
+ continue
try:
data = json.loads(text)
rows.extend(data if isinstance(data, list) else [data])
- except ValueError:
- rows.extend(json.loads(line) for line in text.splitlines() if line.strip())
+ except json.JSONDecodeError as exc:
+ # "Extra data" means the first line parsed, so this is JSONL.
+ if exc.msg != "Extra data":
+ raise _bad_json(path, exc.lineno, exc) from None
+ for number, line in enumerate(text.splitlines(), 1):
+ if not line.strip():
+ continue
+ try:
+ rows.append(json.loads(line))
+ except json.JSONDecodeError as bad_line:
+ raise _bad_json(path, number, bad_line) from None
elif suffix in (".csv", ".parquet"):
dataset = load_dataset(
suffix[1:],
@@ -239,11 +286,19 @@ def _load_rows(
class _DecisionRows:
"""Reads raw rows on a thread, in this job's context, while the model loads."""
- def __init__(self, config, should_stop, status, warn):
+ def __init__(
+ self,
+ config,
+ should_stop,
+ status,
+ warn,
+ check = None,
+ ):
self._config = config
self._should_stop = should_stop
self._status = status
self._warn = warn
+ self._check = check
self._cancelled = threading.Event()
self._done = threading.Event()
self._lock = threading.Lock()
@@ -272,9 +327,12 @@ class _DecisionRows:
try:
if self._stopped():
raise _Stopped("Training cancelled")
- self._result = _load_rows(
+ result = _load_rows(
self._config, self._stopped, self._report_status, self._report_warning
)
+ if self._check is not None:
+ self._check(result[0])
+ self._result = result
except BaseException as exc:
self._error = exc
finally:
@@ -300,6 +358,32 @@ class _DecisionRows:
raise self._error
return self._result
+ def alongside(self, work, check_stop):
+ from utils.account_context import account_thread
+
+ failure = []
+
+ def run():
+ try:
+ work()
+ except BaseException as exc:
+ failure.append(exc)
+
+ context = copy_context()
+ thread = account_thread(target = context.run, args = (run,), daemon = True)
+ thread.start()
+ try:
+ while thread.is_alive():
+ thread.join(0.1)
+ check_stop()
+ if self._done.is_set() and self._error is not None:
+ raise self._error
+ except BaseException:
+ _abandoned_downloads.append(thread)
+ raise
+ if failure:
+ raise failure[0]
+
def __exit__(self, *_exc):
with self._lock:
self._cancelled.set()
@@ -513,9 +597,18 @@ def _run(event_queue: Any, stop_queue: Any, config: dict, output_dir: str) -> No
use_lora = config["training_type"] == "LoRA/QLoRA"
gradient_checkpointing = normalize_gradient_checkpointing(config["gradient_checkpointing"])
- with _DecisionRows(config, lambda: stop["requested"], status, warn) as pending_rows:
+ def check_rows(rows: list) -> None:
+ problem = FastDecisionModel.rows_problem(rows, clef, validate)
+ if problem is not None:
+ raise ValueError(_no_decisions_message(problem))
+
+ with _DecisionRows(config, lambda: stop["requested"], status, warn, check_rows) as pending_rows:
check_stop()
- _download_decision_checkpoint(event_queue, config)
+ if config.get("local_datasets"):
+ pending_rows.result(check_stop)
+ pending_rows.alongside(
+ lambda: _download_decision_checkpoint(event_queue, config), check_stop
+ )
check_stop()
status("Loading decision model...")
if llm:
@@ -593,12 +686,7 @@ def _run(event_queue: Any, stop_queue: Any, config: dict, output_dir: str) -> No
"reason": report["reason"] or eval_report["reason"],
}
if not items:
- reason = f" ({report['reason']})" if report["reason"] else ""
- send(
- "error",
- error = f"No usable decisions in the dataset{reason}. Each row needs state, questions and gold.",
- stack = "",
- )
+ send("error", error = _no_decisions_message(report["reason"]), stack = "")
return
if report["skipped"]:
warn(f"Skipped {report['skipped']:,} of {report['total']:,} decisions: {report['reason']}.")
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index b9c5af781ae..a6a3d5103b9 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -2511,7 +2511,12 @@ class TrainingBackend:
"""First model-load stall respawns the worker over HTTP; later stalls surface as errors."""
msg = event.get("message", "Download stalled")
with self._lock:
- recover = self._in_model_load and not self._xet_fallback_used
+ # A stall queued behind an error or completion must not respawn a finished run.
+ recover = (
+ self._in_model_load
+ and not self._xet_fallback_used
+ and not self._complete_seen.is_set()
+ )
proc = self._proc
run_id = self.current_job_id
if recover:
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 72a1b20a8cd..a4b7f3ed5be 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -3930,6 +3930,13 @@ def run_mlx_training_process(
"ts": time.time(),
}
)
+ _end_abandoned_decision_download(event_queue)
+
+
+def _end_abandoned_decision_download(event_queue: Any) -> None:
+ trainer = sys.modules.get("core.training.decision_trainer")
+ if trainer is not None:
+ trainer.end_abandoned_download(event_queue)
def _training_job_is_local(config) -> bool:
@@ -4768,6 +4775,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
"ts": time.time(),
}
)
+ _end_abandoned_decision_download(event_queue)
return
if config.get("batch_members"):
diff --git a/studio/backend/tests/test_decision_dataset_prefetch.py b/studio/backend/tests/test_decision_dataset_prefetch.py
index 007fca1c770..49e7221789d 100644
--- a/studio/backend/tests/test_decision_dataset_prefetch.py
+++ b/studio/backend/tests/test_decision_dataset_prefetch.py
@@ -1,8 +1,13 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import json
+import subprocess
import sys
+import textwrap
+import time
from contextvars import ContextVar
+from pathlib import Path
from queue import Queue
from threading import Event, current_thread
from types import ModuleType, SimpleNamespace
@@ -120,7 +125,7 @@ def fake_decision_runtime(monkeypatch):
monkeypatch.setenv("HF_TOKEN", "fixture-token")
monkeypatch.setattr(import_guards, "ensure_real_packages", lambda *_a: None)
monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda *_a: "local-checkpoint")
- model = SimpleNamespace()
+ model = SimpleNamespace(rows_problem = lambda *_a: None)
unsloth = ModuleType("unsloth")
unsloth.DecisionTrainer = object
unsloth.FastDecisionModel = model
@@ -212,6 +217,138 @@ def test_raw_rows_overlap_checkpoint_and_model_load_but_preparation_waits(
assert messages[-1] == "Preparing decision rows..."
+def _decision_config(**overrides):
+ return {
+ "model_name": "example/decision",
+ "decision_layout": "clef",
+ "training_type": "Full Finetuning",
+ "random_seed": 7,
+ "gradient_checkpointing": "none",
+ **overrides,
+ }
+
+
+def test_unusable_local_rows_fail_before_the_model_is_touched(fake_decision_runtime, tmp_path):
+ model_api, worker = fake_decision_runtime
+ touched = []
+ dataset = tmp_path / "decisions.jsonl"
+ dataset.write_text('{"state": "only a state"}\n', encoding = "utf-8")
+ model_api.rows_problem = lambda rows, clef, validate: "row 1 needs state, questions and gold"
+ worker._load_embedding_hf_dataset = None
+ worker._download_decision_checkpoint = lambda *_a: touched.append("download")
+ model_api.from_pretrained = lambda *_a, **_kw: touched.append("load")
+
+ config = _decision_config(local_datasets = [str(dataset)], eval_steps = 0)
+ with pytest.raises(ValueError, match = "No usable decisions in the dataset .row 1 needs state"):
+ decision._run(Queue(), Queue(), config, "unused-output")
+ assert touched == []
+
+
+def test_unusable_hub_rows_end_a_model_download_in_progress(fake_decision_runtime, monkeypatch):
+ model_api, worker = fake_decision_runtime
+ downloading, release = Event(), Event()
+ loaded = []
+
+ def load_rows(*_a):
+ assert downloading.wait(3)
+ return [{"state": "only a state"}], None
+
+ def download(*_a):
+ downloading.set()
+ release.wait(3)
+
+ monkeypatch.setattr(decision, "_load_rows", load_rows)
+ model_api.rows_problem = lambda rows, clef, validate: "row 1 has no gold"
+ worker._download_decision_checkpoint = download
+ model_api.from_pretrained = lambda *_a, **_kw: loaded.append(release.is_set())
+ try:
+ with pytest.raises(ValueError, match = "row 1 has no gold"):
+ decision._run(Queue(), Queue(), _decision_config(), "unused-output")
+ assert not release.is_set() and loaded == []
+ assert decision._abandoned_downloads[-1].is_alive()
+ finally:
+ release.set()
+ while decision._abandoned_downloads:
+ decision._abandoned_downloads.pop().join(3)
+
+
+def test_an_abandoned_download_does_not_hold_the_worker_open(tmp_path):
+ script = textwrap.dedent(
+ """
+ import multiprocessing, threading, time
+ from concurrent.futures import ThreadPoolExecutor
+ from core.training import decision_trainer, worker
+
+ def download():
+ with ThreadPoolExecutor(2) as pool:
+ list(pool.map(time.sleep, [30] * 4))
+
+ thread = threading.Thread(target = download, daemon = True)
+ thread.start()
+ decision_trainer._abandoned_downloads.append(thread)
+ events = multiprocessing.get_context("spawn").Queue()
+ events.put({"type": "error", "error": "bad rows"})
+ # The in-process MLX adapter: a plain queue, or any call off the main thread, must return.
+ import queue
+ worker._end_abandoned_decision_download(queue.Queue())
+ caller = threading.Thread(target = worker._end_abandoned_decision_download, args = (events,))
+ caller.start()
+ caller.join()
+ print("host survived", flush = True)
+ worker._end_abandoned_decision_download(events)
+ """
+ )
+ backend = Path(__file__).resolve().parents[1]
+ started = time.monotonic()
+ # A file, not a pipe: the queue's resource tracker would hold a pipe open past the child.
+ log = tmp_path / "child.log"
+ with log.open("w") as out:
+ proc = subprocess.Popen(
+ [sys.executable, "-c", script], cwd = backend, stdout = out, stderr = subprocess.STDOUT
+ )
+ try:
+ returncode = proc.wait(timeout = 25)
+ finally:
+ proc.kill()
+ assert returncode == 0, log.read_text()[-2000:]
+ assert "host survived" in log.read_text()
+ assert time.monotonic() - started < 25
+
+
+def test_a_bad_jsonl_line_names_its_file_and_line(tmp_path):
+ good = json.dumps({"state": "fine"})
+ dataset = tmp_path / "decisions.jsonl"
+ dataset.write_text(f"{good}\n\n{good}\n{{broken\n", encoding = "utf-8")
+ with pytest.raises(ValueError, match = r"Invalid JSON at decisions\.jsonl:4: "):
+ decision._read_local_rows([str(dataset)], None, _no_stop)
+
+
+def test_an_empty_jsonl_shard_beside_real_rows_is_skipped(tmp_path):
+ (tmp_path / "a.jsonl").write_text(json.dumps({"state": "fine"}) + "\n", encoding = "utf-8")
+ (tmp_path / "b.jsonl").write_text("", encoding = "utf-8")
+ (tmp_path / "c.jsonl").write_text("\n", encoding = "utf-8")
+ rows = decision._read_local_rows([str(tmp_path)], None, _no_stop)
+ assert rows == [{"state": "fine"}]
+
+
+def test_a_bad_line_in_an_upload_names_the_file_as_it_was_uploaded(tmp_path):
+ dataset = tmp_path / f"{'a1' * 16}_decisions.jsonl"
+ dataset.write_text('{"state": "fine"}\n{broken\n', encoding = "utf-8")
+ with pytest.raises(ValueError, match = r"Invalid JSON at decisions\.jsonl:2: "):
+ decision._read_local_rows([str(dataset)], None, _no_stop)
+
+
+def test_a_bad_first_line_and_a_pretty_printed_file_report_the_real_line(tmp_path):
+ first = tmp_path / "first.jsonl"
+ first.write_text('{broken\n{"state": "fine"}\n', encoding = "utf-8")
+ pretty = tmp_path / "pretty.json"
+ pretty.write_text('[\n {"state": "fine"},\n {"state": }\n]\n', encoding = "utf-8")
+ with pytest.raises(ValueError, match = r"first\.jsonl:1: "):
+ decision._read_local_rows([str(first)], None, _no_stop)
+ with pytest.raises(ValueError, match = r"pretty\.json:3: "):
+ decision._read_local_rows([str(pretty)], None, _no_stop)
+
+
def test_prefetch_keeps_strict_cached_splits_bounds_revision_and_token(monkeypatch):
import importlib
from utils.datasets import cache_safe
diff --git a/studio/backend/tests/test_decision_training.py b/studio/backend/tests/test_decision_training.py
index 234ce943831..6195feffa14 100644
--- a/studio/backend/tests/test_decision_training.py
+++ b/studio/backend/tests/test_decision_training.py
@@ -398,6 +398,7 @@ def test_all_invalid_rows_are_an_error_and_save_nothing(base, studio_home):
events = _train(_config(base, _dataset(studio_home, rows)))
assert "No usable decisions" in _of(events, "error")[0]["error"]
+ assert "Loading decision model..." not in [e["message"] for e in _of(events, "status")]
assert not _of(events, "complete")
assert not outputs_root().exists() or not any(outputs_root().iterdir())
diff --git a/studio/backend/tests/test_training_xet_fallback.py b/studio/backend/tests/test_training_xet_fallback.py
index c7b360ed1e4..488900baf56 100644
--- a/studio/backend/tests/test_training_xet_fallback.py
+++ b/studio/backend/tests/test_training_xet_fallback.py
@@ -322,6 +322,15 @@ def test_second_stall_surfaces_error_without_respawn():
assert proc.is_alive() is False
+def test_stall_queued_behind_an_error_does_not_respawn():
+ b, proc = _backend_mid_load()
+ b._handle_event({"type": "error", "error": "No usable decisions in the dataset", "stack": ""})
+ b._handle_event({"type": "stall", "message": "no progress for 180s"})
+ assert b._needs_xet_respawn is False
+ assert b._progress.error == "No usable decisions in the dataset"
+ assert proc.is_alive() is False
+
+
def test_model_load_completed_disarms_recovery():
b, _ = _backend_mid_load()
b._handle_event({"type": "model_load_completed"})
diff --git a/tests/test_decision_model.py b/tests/test_decision_model.py
index ded9c396e5a..de756c545f8 100644
--- a/tests/test_decision_model.py
+++ b/tests/test_decision_model.py
@@ -251,6 +251,35 @@ def test_dataset_skips_bad_rows_and_names_the_most_common_reason(checkpoint):
assert torch.allclose(batch["target"].sum(-1), torch.ones(3))
[email protected]("clef", [False, True])
+def test_rows_problem_needs_no_model_and_stops_at_the_first_usable_row(clef):
+ assert FastDecisionModel.rows_problem([], clef) == ""
+ no_gold = {**_row(0), "gold": {}}
+ no_questions = {"state": "only a state"}
+ assert FastDecisionModel.rows_problem([no_questions], clef) == (
+ "row 1 needs state, questions and gold"
+ )
+ assert FastDecisionModel.rows_problem([no_questions, no_questions], clef) == (
+ "row 1 needs state, questions and gold (and 1 more like it)"
+ )
+ assert FastDecisionModel.rows_problem([no_gold], clef).startswith('row 1: "')
+
+ def record(name, question):
+ seen.append(name)
+
+ seen = []
+ assert FastDecisionModel.rows_problem([no_gold, _row(1), no_questions], clef, record) is None
+ assert len(seen) == 1
+
+
+def test_rows_problem_applies_the_callers_validation():
+ def refuse(name, question):
+ raise ValueError("is over the limit")
+
+ problem = FastDecisionModel.rows_problem([_row(0)], False, refuse)
+ assert problem.endswith("is over the limit (and 2 more like it)")
+
+
def test_holdout_takes_whole_rows_and_never_overshoots():
items = [{"row": row} for row in range(50) for _ in range(3)]
train, held = FastDecisionModel.split_holdout(items, 3407)
diff --git a/tests/test_decision_model_mlx.py b/tests/test_decision_model_mlx.py
index bc45421ea53..f5be50aa3af 100644
--- a/tests/test_decision_model_mlx.py
+++ b/tests/test_decision_model_mlx.py
@@ -200,6 +200,14 @@ def test_decision_helpers_are_one_torch_free_module_for_both_backends():
assert [a.name for n in taken for a in n.names] == ["*"]
[email protected]("clef", [False, True])
+def test_rows_problem_checks_rows_without_a_model(clef):
+ assert FastDecisionModel.rows_problem([{"state": "s"}, _row(0)], clef) is None
+ assert FastDecisionModel.rows_problem([{"state": "s"}], clef) == (
+ "row 1 needs state, questions and gold"
+ )
+
+
@pytest.fixture
def clef_checkpoint(tmp_path, monkeypatch):
from types import SimpleNamespace
diff --git a/unsloth/models/_decision_common.py b/unsloth/models/_decision_common.py
index 0ecb7f28b99..01ba53cfbea 100644
--- a/unsloth/models/_decision_common.py
+++ b/unsloth/models/_decision_common.py
@@ -35,6 +35,7 @@ __all__ = [
"_target",
"_target_for",
"_clef_question",
+ "_rows_problem",
"_predicted",
"_metrics",
"_fit_temperature",
@@ -50,7 +51,7 @@ import json
import math
import sys
from pathlib import Path
-from typing import Optional
+from typing import Callable, Optional
TRAIN_MAX_LEN, TRAIN_HEAD_MAX_LEN = 1024, 256
HOLDOUT_MAX = 400
@@ -360,6 +361,53 @@ def _clef_question(question) -> dict:
return clef_question
+def _rows_problem(
+ rows,
+ clef: bool,
+ validate: Optional[Callable] = None,
+) -> Optional[str]:
+ # None at the first decision that can train, else why none can ("" when there are no rows).
+ skips = {}
+
+ def skip(
+ index,
+ reason,
+ name = None,
+ ):
+ where = f"row {index + 1}" if name is None else f'row {index + 1}: "{name}"'
+ skips.setdefault(reason, [0, f"{where} {reason}"])[0] += 1
+
+ for index, row in enumerate(rows):
+ row = row if isinstance(row, dict) else {}
+ state = _parsed(row.get("state"))
+ questions = _parsed(row.get("questions"))
+ gold = _parsed(row["gold"] if row.get("gold") is not None else row.get("answers"))
+ if state is None or not isinstance(questions, dict) or not isinstance(gold, dict):
+ skip(index, "needs state, questions and gold")
+ continue
+ for name, question in questions.items():
+ if name not in gold:
+ skip(index, "has no gold", name)
+ continue
+ try:
+ if validate is not None:
+ validate(name, question)
+ if clef:
+ parsed = _clef_question(question)
+ internal = {"t": parsed["type"], "crit": parsed.get("criteria")}
+ else:
+ internal = _internal(question)
+ _target(internal, _parsed(gold[name]))
+ except (TypeError, ValueError, KeyError) as exc:
+ skip(index, str(exc) or "is not a valid question", name)
+ continue
+ return None
+ if not skips:
+ return ""
+ count, example = max(skips.values(), key = lambda skipped: skipped[0])
+ return example if count == 1 else f"{example} (and {count - 1:,} more like it)"
+
+
def _predicted(question: dict, answer: dict, probabilities: dict) -> dict:
# The Decision API answer plus "answer": the option (choice), True / False (noul) or level number (score).
kind = question["type"]
diff --git a/unsloth/models/decision.py b/unsloth/models/decision.py
index 3ac49d0d7b5..f8e87e1b3ad 100644
--- a/unsloth/models/decision.py
+++ b/unsloth/models/decision.py
@@ -1832,6 +1832,8 @@ class FastDecisionModel:
)
return items, report
+ rows_problem = staticmethod(_rows_problem)
+
@staticmethod
def split_holdout(
items: list,
diff --git a/unsloth/models/decision_mlx.py b/unsloth/models/decision_mlx.py
index 7c6b205fd7f..72588c99b66 100644
--- a/unsloth/models/decision_mlx.py
+++ b/unsloth/models/decision_mlx.py
@@ -50,6 +50,7 @@ from unsloth._decision_common import (
_parsed,
_predicted,
_row_images,
+ _rows_problem,
_served_lengths,
_served_temperatures,
_target,
@@ -871,6 +872,7 @@ class FastDecisionModel:
clef = functools.partial(_clef_items, model._unsloth_pipeline) if _is_clef(model) else None
return _decision_dataset(rows, tokenizer, model.decision_config, validate, clef)
+ rows_problem = staticmethod(_rows_problem)
split_holdout = staticmethod(_decision_holdout)
@staticmethod