Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: use the eval dataset for TTS and codec audio training (#10259)
Nilay41d2ff40a622
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex 7a1e5ec777b..05c1d33fedd 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -1323,7 +1323,6 @@ class UnslothTrainer:logger.info(f" - Finetune language layers: {finetune_language_layers}")logger.info(f" - Finetune attention modules: {finetune_attention_modules}")logger.info(f" - Finetune MLP modules: {finetune_mlp_modules}")- logger.info()peft_kwargs = dict(r = lora_r,@@ -2363,6 +2362,37 @@ class UnslothTrainer:logger.info(f"Sample text (first 200 chars): {sample[:200]}...\n")return result_dataset+ def _preprocess_audio_eval_split(self, eval_dataset, preprocess, custom_format_mapping):+ """Preprocess eval data, warning and dropping it on failure."""+ if eval_dataset is None:+ return None+ try:+ return preprocess(eval_dataset, custom_format_mapping)+ except Exception as e:+ self._record_warning(+ "The eval dataset could not be prepared for this audio model, so this run has "+ f"no evaluation: {e}"+ )+ return None++ def _audio_eval_config(self, training_args):+ """Build audio evaluation arguments and return the eval dataset."""+ eval_dataset = training_args.get("eval_dataset", None)+ eval_steps = training_args.get("eval_steps", 0.00)+ if eval_dataset is None:+ return {}, None+ if not eval_steps or eval_steps <= 0:+ logger.info(f"⚠️ Eval dataset provided but eval_steps={eval_steps} (disabled)\n")+ return {}, None+ rows = len(eval_dataset) if hasattr(eval_dataset, "__len__") else "?"+ logger.info(f"✅ Evaluation enabled: eval_steps={eval_steps}, eval rows={rows}\n")+ return {+ "eval_strategy": "steps",+ "eval_steps": eval_steps,+ # Avoid HF's default of 8, which can OOM audio runs.+ "per_device_eval_batch_size": training_args.get("batch_size", 2),+ }, eval_dataset+def _preprocess_whisper_dataset(self,dataset,@@ -3015,7 +3045,12 @@ class UnslothTrainer:)if self._audio_type == "csm":processed = self._preprocess_csm_dataset(dataset, custom_format_mapping)- return (processed, None)+ return (+ processed,+ self._preprocess_audio_eval_split(+ eval_dataset, self._preprocess_csm_dataset, custom_format_mapping+ ),+ )elif self._audio_type == "whisper":train_data, eval_data = self._preprocess_whisper_dataset(@@ -3027,15 +3062,30 @@ class UnslothTrainer:elif self._audio_type == "snac":processed = self._preprocess_snac_dataset(dataset, custom_format_mapping)- return (processed, None)+ return (+ processed,+ self._preprocess_audio_eval_split(+ eval_dataset, self._preprocess_snac_dataset, custom_format_mapping+ ),+ )elif self._audio_type == "bicodec":processed = self._preprocess_bicodec_dataset(dataset, custom_format_mapping)- return ({"dataset": processed, "final_format": "audio_bicodec"}, None)+ return (+ {"dataset": processed, "final_format": "audio_bicodec"},+ self._preprocess_audio_eval_split(+ eval_dataset, self._preprocess_bicodec_dataset, custom_format_mapping+ ),+ )elif self._audio_type == "dac":processed = self._preprocess_dac_dataset(dataset, custom_format_mapping)- return ({"dataset": processed, "final_format": "audio_dac"}, None)+ return (+ {"dataset": processed, "final_format": "audio_dac"},+ self._preprocess_audio_eval_split(+ eval_dataset, self._preprocess_dac_dataset, custom_format_mapping+ ),+ )# ========== RAW TEXT BYPASS ==========if raw_text_mode:@@ -3664,18 +3714,23 @@ class UnslothTrainer:self._apply_csm_forward_fix()+ eval_args, eval_dataset = self._audio_eval_config(training_args)config = self._build_audio_training_args(training_args,output_dir,extra_args = {"remove_unused_columns": False,+ **eval_args,},)- self.trainer = HFTrainer(- model = self.model,- train_dataset = dataset,- args = TrainingArguments(**config),- )+ trainer_kwargs = {+ "model": self.model,+ "train_dataset": dataset,+ "args": TrainingArguments(**config),+ }+ if eval_dataset is not None:+ trainer_kwargs["eval_dataset"] = eval_dataset+ self.trainer = HFTrainer(**trainer_kwargs)self.trainer.add_callback(self._create_progress_callback())# Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that has# no terminal; --verbose keeps them.@@ -3707,17 +3762,23 @@ class UnslothTrainer:DataCollatorForSeq2Seq,)- config = self._build_audio_training_args(training_args, output_dir)- self.trainer = HFTrainer(- model = self.model,- train_dataset = dataset,- args = TrainingArguments(**config),- data_collator = DataCollatorForSeq2Seq(+ eval_args, eval_dataset = self._audio_eval_config(training_args)+ config = self._build_audio_training_args(+ training_args, output_dir, extra_args = eval_args+ )+ trainer_kwargs = {+ "model": self.model,+ "train_dataset": dataset,+ "args": TrainingArguments(**config),+ "data_collator": DataCollatorForSeq2Seq(tokenizer = self.tokenizer,padding = True,pad_to_multiple_of = 8,),- )+ }+ if eval_dataset is not None:+ trainer_kwargs["eval_dataset"] = eval_dataset+ self.trainer = HFTrainer(**trainer_kwargs)self.trainer.add_callback(self._create_progress_callback())_drop_hf_stdout_callbacks(self.trainer)diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.pyindex 464b146acf9..cc1ee228bc9 100644--- a/studio/backend/routes/training.py+++ b/studio/backend/routes/training.py@@ -267,6 +267,11 @@ def _validate_local_dataset_paths(paths: list[str], label: str = "Local dataset"validated = []missing = []for dataset_path in paths:+ if not dataset_path.strip():+ raise HTTPException(+ status_code = 400,+ detail = f"{label} path must not be blank",+ )dataset_file = resolve_dataset_path(dataset_path)if not dataset_file.exists():missing.append(f"{dataset_path} (resolved: {dataset_file})")diff --git a/studio/backend/tests/test_audio_eval_dataset.py b/studio/backend/tests/test_audio_eval_dataset.pynew file mode 100644index 00000000000..afc0624beaa--- /dev/null+++ b/studio/backend/tests/test_audio_eval_dataset.py@@ -0,0 +1,150 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Regression tests for preserving codec-audio evaluation datasets."""++from __future__ import annotations++import sys+from pathlib import Path++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++import importlib # noqa: E402+import json # noqa: E402+import types # noqa: E402+from unittest.mock import MagicMock # noqa: E402++import pytest # noqa: E402+++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ """Stub dependencies missing from the backend test environment."""+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001 - stub unusable imports+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++from core.training import trainer as tmod # noqa: E402++for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)+++CODEC_TYPES = ("csm", "snac", "bicodec", "dac")+++def audio_trainer(monkeypatch):+ # Avoid MLX substitution on Apple silicon.+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)+ t = tmod.UnslothTrainer()+ t.model_name = "unsloth/csm-1b"+ return t+++def _rows(path: Path, text: str) -> str:+ path.write_text(json.dumps({"audio": text, "text": text}) + "\n", encoding = "utf-8")+ return str(path)++[email protected]("audio_type", CODEC_TYPES)+def test_codec_branches_return_the_uploaded_eval_split(+ audio_trainer, tmp_path, monkeypatch, audio_type+):+ audio_trainer._audio_type = audio_type+ monkeypatch.setattr(tmod, "ensure_audio_decoding", lambda: True)+ seen = []++ def fake_preprocess(dataset, custom_format_mapping = None):+ seen.append(len(dataset))+ return dataset++ monkeypatch.setattr(+ audio_trainer, f"_preprocess_{audio_type}_dataset", fake_preprocess, raising = True+ )++ result = audio_trainer.load_and_format_dataset(+ None,+ local_datasets = [_rows(tmp_path / "train.jsonl", "train")],+ local_eval_datasets = [_rows(tmp_path / "eval.jsonl", "eval")],+ eval_steps = 0.1,+ )++ assert result is not None+ _train, eval_dataset = result+ assert eval_dataset is not None, "the uploaded eval split was dropped"+ assert len(seen) == 2, "the eval split must go through the same preprocessing as the train one"++[email protected]("audio_type", CODEC_TYPES)+def test_no_eval_upload_still_returns_no_eval_split(+ audio_trainer, tmp_path, monkeypatch, audio_type+):+ audio_trainer._audio_type = audio_type+ monkeypatch.setattr(tmod, "ensure_audio_decoding", lambda: True)+ monkeypatch.setattr(+ audio_trainer, f"_preprocess_{audio_type}_dataset", lambda ds, m = None: ds, raising = True+ )++ _train, eval_dataset = audio_trainer.load_and_format_dataset(+ None,+ local_datasets = [_rows(tmp_path / "train.jsonl", "train")],+ eval_steps = 0.1,+ )++ assert eval_dataset is None+++def test_an_unpreparable_eval_split_warns_instead_of_failing_the_run(audio_trainer):+ def explode(dataset, custom_format_mapping = None):+ raise ValueError("no audio column found in dataset")++ assert audio_trainer._preprocess_audio_eval_split(object(), explode, None) is None+ assert any("no evaluation" in w for w in audio_trainer.training_progress.warnings)+ assert any("no audio column found" in w for w in audio_trainer.training_progress.warnings)+++def test_eval_args_enable_evaluation_when_eval_steps_is_set(audio_trainer):+ dataset = ["a", "b"]+ args, eval_dataset = audio_trainer._audio_eval_config(+ {"eval_dataset": dataset, "eval_steps": 0.1, "batch_size": 2}+ )+ assert eval_dataset is dataset+ assert args["eval_strategy"] == "steps"+ assert args["eval_steps"] == 0.1+ assert args["per_device_eval_batch_size"] == 2+++def test_eval_steps_zero_disables_evaluation(audio_trainer):+ args, eval_dataset = audio_trainer._audio_eval_config(+ {"eval_dataset": ["a"], "eval_steps": 0.0}+ )+ assert (args, eval_dataset) == ({}, None)+++def test_no_eval_dataset_disables_evaluation(audio_trainer):+ args, eval_dataset = audio_trainer._audio_eval_config({"eval_dataset": None, "eval_steps": 0.1})+ assert (args, eval_dataset) == ({}, None)diff --git a/studio/backend/tests/test_audio_lora_logging.py b/studio/backend/tests/test_audio_lora_logging.pynew file mode 100644index 00000000000..84b1f7a63c0--- /dev/null+++ b/studio/backend/tests/test_audio_lora_logging.py@@ -0,0 +1,51 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Exercise audio adapter setup with a real logger without loading GPU models."""++import ast+import sys+from pathlib import Path+from types import ModuleType, SimpleNamespace+from unittest.mock import Mock++import pytest+import structlog++[email protected]("audio_type", ["csm", "bicodec", "dac", "snac", "audio_vlm"])+def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):+ source = Path(__file__).resolve().parents[1] / "core/training/trainer.py"+ tree = ast.parse(source.read_text(encoding = "utf-8"))+ trainer = next(+ n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "UnslothTrainer"+ )+ method = next(+ n+ for n in trainer.body+ if isinstance(n, ast.FunctionDef) and n.name == "prepare_model_for_training"+ )+ namespace = {"logger": structlog.get_logger()}+ adapter = Mock(return_value = SimpleNamespace(config = object()))+ fast_model = SimpleNamespace(get_peft_model = adapter)+ module = ModuleType("unsloth")+ module.FastModel = fast_model+ monkeypatch.setitem(sys.modules, "unsloth", module)+ namespace["FastLanguageModel"] = fast_model+ exec(compile(ast.Module(body = [method], type_ignores = []), str(source), "exec"), namespace)+ model = SimpleNamespace(config = object())+ progress = Mock()+ instance = SimpleNamespace(+ model = model,+ _audio_type = None if audio_type == "audio_vlm" else audio_type,+ is_audio_vlm = audio_type == "audio_vlm",+ is_vlm = False,+ should_stop = False,+ _update_progress = progress,+ )++ assert namespace["prepare_model_for_training"](instance) is True+ adapter.assert_called_once()+ assert adapter.call_args.args == (model,)+ assert instance.model is adapter.return_value+ progress.assert_called_once_with(status_message = "LoRA adapters configured")diff --git a/studio/backend/tests/test_training_cached_start.py b/studio/backend/tests/test_training_cached_start.pyindex 6deb615e14b..7b1af0d2e0c 100644--- a/studio/backend/tests/test_training_cached_start.py+++ b/studio/backend/tests/test_training_cached_start.py@@ -45,6 +45,24 @@ def _request(**overrides) -> TrainingStartRequest:return TrainingStartRequest(**payload)[email protected]("dataset_path", ["", " ", "\t"])+def test_local_dataset_path_validation_rejects_blank_values(dataset_path):+ route = _load_route_module("training_route_rejects_blank_local_dataset_path")++ with (+ patch.object(+ route,+ "resolve_dataset_path",+ side_effect = AssertionError("blank path must be rejected before resolution"),+ ),+ pytest.raises(HTTPException) as exc_info,+ ):+ route._validate_local_dataset_paths([dataset_path], "Local eval dataset")++ assert exc_info.value.status_code == 400+ assert exc_info.value.detail == "Local eval dataset path must not be blank"++@pytest.mark.parametrize("repo_id", ["_team/dataset_", "dataset_"])def test_training_request_accepts_hugging_face_underscore_boundaries(repo_id):assert _request(hf_dataset = repo_id).hf_dataset == repo_id