Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): load the MLX training dataset before the model (#16618)

Long Yixingd63d05692099
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index a4b7f3ed5be..ce56cf08684 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -3081,7 +3081,6 @@ def _run_mlx_training(event_queue, stop_queue, config):
lr_scheduler_type = _normalize_mlx_studio_scheduler(config.get("lr_scheduler_type", "linear"))
# Force text-only for non-image datasets even on vision-capable models (e.g. Qwen3.5-VL on alpaca).
- _send("status", status_message = f"Loading {model_name}...")
# Forward resume_from_checkpoint to MLXTrainer.train(); else Resume silently restarts from step 0.
resume_from_checkpoint = config.get("resume_from_checkpoint") or None
is_dataset_image = bool(config.get("is_dataset_image", False))
@@ -3104,6 +3103,74 @@ def _run_mlx_training(event_queue, stop_queue, config):
_send("error", **security_error)
return
+ # Fetch the dataset before the model so a dataset error does not wait on the model download.
+ _send("status", status_message = "Loading dataset...")
+ hf_dataset = config.get("hf_dataset", "")
+ config["_dataset_loaded_from_exact_snapshot"] = False
+
+ def _load_local(file_paths):
+ from utils.datasets.saved import load_saved_dataset
+
+ if len(file_paths) == 1:
+ saved = load_saved_dataset(_resolve_mlx_local_dataset_path(file_paths[0]))
+ if saved is not None:
+ return saved
+ all_files = _resolve_mlx_local_dataset_files(file_paths)
+ if not all_files:
+ raise ValueError("No local dataset files found")
+ from utils.datasets.cells import csv_as_text_kwargs
+
+ loader = _mlx_local_dataset_loader_for_files(all_files)
+ return load_dataset(
+ loader,
+ data_files = all_files,
+ split = "train",
+ **csv_as_text_kwargs(all_files),
+ )
+
+ eval_dataset = None
+ if hf_dataset:
+ dataset, eval_dataset = _load_hf_train_and_eval_datasets(
+ config,
+ hf_token,
+ load_dataset,
+ lambda message: _send("status", status_message = message),
+ lambda message: _send("warning", message = message),
+ )
+ elif config.get("local_datasets"):
+ dataset = _load_local(config["local_datasets"])
+ elif config.get("s3_config"):
+ from core.training.s3_dataset import (
+ S3DownloadCancelled,
+ prepare_s3_dataset_download,
+ )
+ _send("status", status_message = "Downloading dataset from S3...")
+ try:
+ s3_download = prepare_s3_dataset_download(
+ config["s3_config"],
+ cancel_callback = _is_stop_requested,
+ )
+ try:
+ dataset = _load_local(s3_download.files)
+ finally:
+ s3_download.cleanup()
+ except S3DownloadCancelled:
+ _send("complete", output_dir = None, status_message = "Training cancelled")
+ return
+ else:
+ raise ValueError("No dataset specified")
+
+ from core.training.eval_dataset import evaluation_enabled
+
+ eval_enabled = evaluation_enabled(config.get("eval_steps"))
+ if eval_enabled and not hf_dataset and config.get("local_eval_datasets"):
+ eval_dataset = _load_local(config["local_eval_datasets"])
+
+ if _is_stop_requested():
+ _send("complete", output_dir = None, status_message = "Training cancelled")
+ return
+
+ _send("status", status_message = f"Loading {model_name}...")
try:
model, tokenizer = FastMLXModel.from_pretrained(
model_load_name,
@@ -3238,11 +3305,8 @@ def _run_mlx_training(event_queue, stop_queue, config):
peft_kwargs["finetune_vision_layers"] = finetune_vision
model = FastMLXModel.get_peft_model(model, **peft_kwargs)
- _send("status", status_message = "Loading dataset...")
- hf_dataset = config.get("hf_dataset", "")
slice_start = config.get("dataset_slice_start")
slice_end = config.get("dataset_slice_end")
- config["_dataset_loaded_from_exact_snapshot"] = False
# max_steps runs cannot reach the whole dataset; bound rows before mapping.
mlx_raw_text_mode = (
@@ -3285,61 +3349,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
on_bound = _on_bound,
)
- def _load_local(file_paths):
- from utils.datasets.saved import load_saved_dataset
-
- if len(file_paths) == 1:
- saved = load_saved_dataset(_resolve_mlx_local_dataset_path(file_paths[0]))
- if saved is not None:
- return saved
- all_files = _resolve_mlx_local_dataset_files(file_paths)
- if not all_files:
- raise ValueError("No local dataset files found")
- from utils.datasets.cells import csv_as_text_kwargs
-
- loader = _mlx_local_dataset_loader_for_files(all_files)
- return load_dataset(
- loader,
- data_files = all_files,
- split = "train",
- **csv_as_text_kwargs(all_files),
- )
-
- eval_dataset = None
- if hf_dataset:
- dataset, eval_dataset = _load_hf_train_and_eval_datasets(
- config,
- hf_token,
- load_dataset,
- lambda message: _send("status", status_message = message),
- lambda message: _send("warning", message = message),
- )
- dataset = _slice(dataset)
- elif config.get("local_datasets"):
- dataset = _load_local(config["local_datasets"])
- dataset = _slice(dataset)
- elif config.get("s3_config"):
- from core.training.s3_dataset import (
- S3DownloadCancelled,
- prepare_s3_dataset_download,
- )
-
- _send("status", status_message = "Downloading dataset from S3...")
- try:
- s3_download = prepare_s3_dataset_download(
- config["s3_config"],
- cancel_callback = _is_stop_requested,
- )
- try:
- dataset = _load_local(s3_download.files)
- finally:
- s3_download.cleanup()
- except S3DownloadCancelled:
- _send("complete", output_dir = None, status_message = "Training cancelled")
- return
- dataset = _slice(dataset)
- else:
- raise ValueError("No dataset specified")
+ dataset = _slice(dataset)
_emit_resource_provenance(
event_queue,
@@ -3350,12 +3360,6 @@ def _run_mlx_training(event_queue, stop_queue, config):
dataset_loaded_from_exact_snapshot = bool(config.get("_dataset_loaded_from_exact_snapshot")),
)
- from core.training.eval_dataset import evaluation_enabled
-
- eval_enabled = evaluation_enabled(config.get("eval_steps"))
- if eval_enabled and not hf_dataset and config.get("local_eval_datasets"):
- eval_dataset = _load_local(config["local_eval_datasets"])
-
# Reuse the GPU format pipeline for VLM (OCR/caption/llava/sharegpt+images) and text.
format_type = config.get("format_type", "")
custom_format_mapping = config.get("custom_format_mapping")
diff --git a/studio/backend/tests/test_mlx_dataset_format_failure.py b/studio/backend/tests/test_mlx_dataset_format_failure.py
index 1791fdcbcae..1eddefda896 100644
--- a/studio/backend/tests/test_mlx_dataset_format_failure.py
+++ b/studio/backend/tests/test_mlx_dataset_format_failure.py
@@ -5,7 +5,9 @@
import ast
import inspect
+import sys
import textwrap
+import types
import pytest
@@ -126,3 +128,78 @@ def test_an_eval_split_formatted_to_nothing_disables_evaluation():
assert result["warnings"] == [
"The eval dataset is empty after preprocessing, so this run has no evaluation."
]
+
+
+def _run_mlx_until_model_load(monkeypatch, config, stop_requested):
+ """Runs the real _run_mlx_training with MLX stubbed; a model load fails the test."""
+ loaded = []
+ events = []
+
+ class _FastMLXModel:
+ @staticmethod
+ def from_pretrained(*args, **kwargs):
+ loaded.append(args)
+ raise AssertionError("the model loaded")
+
+ @staticmethod
+ def get_peft_model(model, **kwargs):
+ return model
+
+ mlx = types.ModuleType("mlx")
+ mlx_core = types.ModuleType("mlx.core")
+ mlx_core.metal = types.SimpleNamespace(is_available = lambda: False)
+ mlx.core = mlx_core
+ loader = types.ModuleType("unsloth_zoo.mlx.loader")
+ loader.FastMLXModel = _FastMLXModel
+ trainer = types.ModuleType("unsloth_zoo.mlx.trainer")
+ trainer.MLXTrainer = trainer.MLXTrainingConfig = trainer.train_on_responses_only = object
+ s3_dataset = types.ModuleType("core.training.s3_dataset")
+ s3_dataset.S3DownloadCancelled = type("S3DownloadCancelled", (Exception,), {})
+
+ def _reject_manifest(*_args, **_kwargs):
+ raise ValueError("ambiguous audio reference")
+
+ s3_dataset.prepare_s3_dataset_download = _reject_manifest
+ for name, module in {
+ "mlx": mlx,
+ "mlx.core": mlx_core,
+ "unsloth_zoo.mlx.loader": loader,
+ "unsloth_zoo.mlx.trainer": trainer,
+ "core.training.s3_dataset": s3_dataset,
+ }.items():
+ monkeypatch.setitem(sys.modules, name, module)
+ monkeypatch.setattr(_worker, "_model_load_security_error", lambda *_args: None)
+ monkeypatch.setattr(
+ _worker,
+ "_start_mlx_stop_poller",
+ lambda _queue: ([True], stop_requested, [None], lambda: stop_requested[0], None),
+ )
+
+ class _Queue:
+ def put(self, event):
+ events.append(event)
+
+ _worker._run_mlx_training(_Queue(), None, {"model_name": "org/model", **config})
+ return loaded, events
+
+
+def test_mlx_dataset_error_surfaces_before_the_model_loads(monkeypatch):
+ with pytest.raises(ValueError, match = "ambiguous audio reference"):
+ _run_mlx_until_model_load(monkeypatch, {"s3_config": {"bucket": "b"}}, [False])
+
+
+def test_mlx_stop_during_dataset_load_skips_the_model_load(monkeypatch):
+ stop_requested = [False]
+
+ def _load_and_stop(*_args):
+ stop_requested[0] = True
+ return ["row"], None
+
+ monkeypatch.setattr(_worker, "_load_hf_train_and_eval_datasets", _load_and_stop)
+ loaded, events = _run_mlx_until_model_load(
+ monkeypatch, {"hf_dataset": "org/data"}, stop_requested
+ )
+
+ assert loaded == []
+ assert events[-1]["type"] == "complete"
+ assert events[-1]["status_message"] == "Training cancelled"