Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: use the eval dataset for TTS and codec audio training (#10259)

Nilay41d2ff40a622
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index 7a1e5ec777b..05c1d33fedd 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -1323,7 +1323,6 @@ class UnslothTrainer:
logger.info(f" - Finetune language layers: {finetune_language_layers}")
logger.info(f" - Finetune attention modules: {finetune_attention_modules}")
logger.info(f" - Finetune MLP modules: {finetune_mlp_modules}")
- logger.info()
peft_kwargs = dict(
r = lora_r,
@@ -2363,6 +2362,37 @@ class UnslothTrainer:
logger.info(f"Sample text (first 200 chars): {sample[:200]}...\n")
return result_dataset
+ def _preprocess_audio_eval_split(self, eval_dataset, preprocess, custom_format_mapping):
+ """Preprocess eval data, warning and dropping it on failure."""
+ if eval_dataset is None:
+ return None
+ try:
+ return preprocess(eval_dataset, custom_format_mapping)
+ except Exception as e:
+ self._record_warning(
+ "The eval dataset could not be prepared for this audio model, so this run has "
+ f"no evaluation: {e}"
+ )
+ return None
+
+ def _audio_eval_config(self, training_args):
+ """Build audio evaluation arguments and return the eval dataset."""
+ eval_dataset = training_args.get("eval_dataset", None)
+ eval_steps = training_args.get("eval_steps", 0.00)
+ if eval_dataset is None:
+ return {}, None
+ if not eval_steps or eval_steps <= 0:
+ logger.info(f"⚠️ Eval dataset provided but eval_steps={eval_steps} (disabled)\n")
+ return {}, None
+ rows = len(eval_dataset) if hasattr(eval_dataset, "__len__") else "?"
+ logger.info(f"✅ Evaluation enabled: eval_steps={eval_steps}, eval rows={rows}\n")
+ return {
+ "eval_strategy": "steps",
+ "eval_steps": eval_steps,
+ # Avoid HF's default of 8, which can OOM audio runs.
+ "per_device_eval_batch_size": training_args.get("batch_size", 2),
+ }, eval_dataset
+
def _preprocess_whisper_dataset(
self,
dataset,
@@ -3015,7 +3045,12 @@ class UnslothTrainer:
)
if self._audio_type == "csm":
processed = self._preprocess_csm_dataset(dataset, custom_format_mapping)
- return (processed, None)
+ return (
+ processed,
+ self._preprocess_audio_eval_split(
+ eval_dataset, self._preprocess_csm_dataset, custom_format_mapping
+ ),
+ )
elif self._audio_type == "whisper":
train_data, eval_data = self._preprocess_whisper_dataset(
@@ -3027,15 +3062,30 @@ class UnslothTrainer:
elif self._audio_type == "snac":
processed = self._preprocess_snac_dataset(dataset, custom_format_mapping)
- return (processed, None)
+ return (
+ processed,
+ self._preprocess_audio_eval_split(
+ eval_dataset, self._preprocess_snac_dataset, custom_format_mapping
+ ),
+ )
elif self._audio_type == "bicodec":
processed = self._preprocess_bicodec_dataset(dataset, custom_format_mapping)
- return ({"dataset": processed, "final_format": "audio_bicodec"}, None)
+ return (
+ {"dataset": processed, "final_format": "audio_bicodec"},
+ self._preprocess_audio_eval_split(
+ eval_dataset, self._preprocess_bicodec_dataset, custom_format_mapping
+ ),
+ )
elif self._audio_type == "dac":
processed = self._preprocess_dac_dataset(dataset, custom_format_mapping)
- return ({"dataset": processed, "final_format": "audio_dac"}, None)
+ return (
+ {"dataset": processed, "final_format": "audio_dac"},
+ self._preprocess_audio_eval_split(
+ eval_dataset, self._preprocess_dac_dataset, custom_format_mapping
+ ),
+ )
# ========== RAW TEXT BYPASS ==========
if raw_text_mode:
@@ -3664,18 +3714,23 @@ class UnslothTrainer:
self._apply_csm_forward_fix()
+ eval_args, eval_dataset = self._audio_eval_config(training_args)
config = self._build_audio_training_args(
training_args,
output_dir,
extra_args = {
"remove_unused_columns": False,
+ **eval_args,
},
)
- self.trainer = HFTrainer(
- model = self.model,
- train_dataset = dataset,
- args = TrainingArguments(**config),
- )
+ trainer_kwargs = {
+ "model": self.model,
+ "train_dataset": dataset,
+ "args": TrainingArguments(**config),
+ }
+ if eval_dataset is not None:
+ trainer_kwargs["eval_dataset"] = eval_dataset
+ self.trainer = HFTrainer(**trainer_kwargs)
self.trainer.add_callback(self._create_progress_callback())
# Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that has
# no terminal; --verbose keeps them.
@@ -3707,17 +3762,23 @@ class UnslothTrainer:
DataCollatorForSeq2Seq,
)
- config = self._build_audio_training_args(training_args, output_dir)
- self.trainer = HFTrainer(
- model = self.model,
- train_dataset = dataset,
- args = TrainingArguments(**config),
- data_collator = DataCollatorForSeq2Seq(
+ eval_args, eval_dataset = self._audio_eval_config(training_args)
+ config = self._build_audio_training_args(
+ training_args, output_dir, extra_args = eval_args
+ )
+ trainer_kwargs = {
+ "model": self.model,
+ "train_dataset": dataset,
+ "args": TrainingArguments(**config),
+ "data_collator": DataCollatorForSeq2Seq(
tokenizer = self.tokenizer,
padding = True,
pad_to_multiple_of = 8,
),
- )
+ }
+ if eval_dataset is not None:
+ trainer_kwargs["eval_dataset"] = eval_dataset
+ self.trainer = HFTrainer(**trainer_kwargs)
self.trainer.add_callback(self._create_progress_callback())
_drop_hf_stdout_callbacks(self.trainer)
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 464b146acf9..cc1ee228bc9 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -267,6 +267,11 @@ def _validate_local_dataset_paths(paths: list[str], label: str = "Local dataset"
validated = []
missing = []
for dataset_path in paths:
+ if not dataset_path.strip():
+ raise HTTPException(
+ status_code = 400,
+ detail = f"{label} path must not be blank",
+ )
dataset_file = resolve_dataset_path(dataset_path)
if not dataset_file.exists():
missing.append(f"{dataset_path} (resolved: {dataset_file})")
diff --git a/studio/backend/tests/test_audio_eval_dataset.py b/studio/backend/tests/test_audio_eval_dataset.py
new file mode 100644
index 00000000000..afc0624beaa
--- /dev/null
+++ b/studio/backend/tests/test_audio_eval_dataset.py
@@ -0,0 +1,150 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Regression tests for preserving codec-audio evaluation datasets."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+import importlib # noqa: E402
+import json # noqa: E402
+import types # noqa: E402
+from unittest.mock import MagicMock # noqa: E402
+
+import pytest # noqa: E402
+
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ """Stub dependencies missing from the backend test environment."""
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001 - stub unusable imports
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+from core.training import trainer as tmod # noqa: E402
+
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+
+CODEC_TYPES = ("csm", "snac", "bicodec", "dac")
+
+
+def audio_trainer(monkeypatch):
+ # Avoid MLX substitution on Apple silicon.
+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)
+ t = tmod.UnslothTrainer()
+ t.model_name = "unsloth/csm-1b"
+ return t
+
+
+def _rows(path: Path, text: str) -> str:
+ path.write_text(json.dumps({"audio": text, "text": text}) + "\n", encoding = "utf-8")
+ return str(path)
+
+
[email protected]("audio_type", CODEC_TYPES)
+def test_codec_branches_return_the_uploaded_eval_split(
+ audio_trainer, tmp_path, monkeypatch, audio_type
+):
+ audio_trainer._audio_type = audio_type
+ monkeypatch.setattr(tmod, "ensure_audio_decoding", lambda: True)
+ seen = []
+
+ def fake_preprocess(dataset, custom_format_mapping = None):
+ seen.append(len(dataset))
+ return dataset
+
+ monkeypatch.setattr(
+ audio_trainer, f"_preprocess_{audio_type}_dataset", fake_preprocess, raising = True
+ )
+
+ result = audio_trainer.load_and_format_dataset(
+ None,
+ local_datasets = [_rows(tmp_path / "train.jsonl", "train")],
+ local_eval_datasets = [_rows(tmp_path / "eval.jsonl", "eval")],
+ eval_steps = 0.1,
+ )
+
+ assert result is not None
+ _train, eval_dataset = result
+ assert eval_dataset is not None, "the uploaded eval split was dropped"
+ assert len(seen) == 2, "the eval split must go through the same preprocessing as the train one"
+
+
[email protected]("audio_type", CODEC_TYPES)
+def test_no_eval_upload_still_returns_no_eval_split(
+ audio_trainer, tmp_path, monkeypatch, audio_type
+):
+ audio_trainer._audio_type = audio_type
+ monkeypatch.setattr(tmod, "ensure_audio_decoding", lambda: True)
+ monkeypatch.setattr(
+ audio_trainer, f"_preprocess_{audio_type}_dataset", lambda ds, m = None: ds, raising = True
+ )
+
+ _train, eval_dataset = audio_trainer.load_and_format_dataset(
+ None,
+ local_datasets = [_rows(tmp_path / "train.jsonl", "train")],
+ eval_steps = 0.1,
+ )
+
+ assert eval_dataset is None
+
+
+def test_an_unpreparable_eval_split_warns_instead_of_failing_the_run(audio_trainer):
+ def explode(dataset, custom_format_mapping = None):
+ raise ValueError("no audio column found in dataset")
+
+ assert audio_trainer._preprocess_audio_eval_split(object(), explode, None) is None
+ assert any("no evaluation" in w for w in audio_trainer.training_progress.warnings)
+ assert any("no audio column found" in w for w in audio_trainer.training_progress.warnings)
+
+
+def test_eval_args_enable_evaluation_when_eval_steps_is_set(audio_trainer):
+ dataset = ["a", "b"]
+ args, eval_dataset = audio_trainer._audio_eval_config(
+ {"eval_dataset": dataset, "eval_steps": 0.1, "batch_size": 2}
+ )
+ assert eval_dataset is dataset
+ assert args["eval_strategy"] == "steps"
+ assert args["eval_steps"] == 0.1
+ assert args["per_device_eval_batch_size"] == 2
+
+
+def test_eval_steps_zero_disables_evaluation(audio_trainer):
+ args, eval_dataset = audio_trainer._audio_eval_config(
+ {"eval_dataset": ["a"], "eval_steps": 0.0}
+ )
+ assert (args, eval_dataset) == ({}, None)
+
+
+def test_no_eval_dataset_disables_evaluation(audio_trainer):
+ args, eval_dataset = audio_trainer._audio_eval_config({"eval_dataset": None, "eval_steps": 0.1})
+ assert (args, eval_dataset) == ({}, None)
diff --git a/studio/backend/tests/test_audio_lora_logging.py b/studio/backend/tests/test_audio_lora_logging.py
new file mode 100644
index 00000000000..84b1f7a63c0
--- /dev/null
+++ b/studio/backend/tests/test_audio_lora_logging.py
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Exercise audio adapter setup with a real logger without loading GPU models."""
+
+import ast
+import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+import structlog
+
+
[email protected]("audio_type", ["csm", "bicodec", "dac", "snac", "audio_vlm"])
+def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):
+ source = Path(__file__).resolve().parents[1] / "core/training/trainer.py"
+ tree = ast.parse(source.read_text(encoding = "utf-8"))
+ trainer = next(
+ n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "UnslothTrainer"
+ )
+ method = next(
+ n
+ for n in trainer.body
+ if isinstance(n, ast.FunctionDef) and n.name == "prepare_model_for_training"
+ )
+ namespace = {"logger": structlog.get_logger()}
+ adapter = Mock(return_value = SimpleNamespace(config = object()))
+ fast_model = SimpleNamespace(get_peft_model = adapter)
+ module = ModuleType("unsloth")
+ module.FastModel = fast_model
+ monkeypatch.setitem(sys.modules, "unsloth", module)
+ namespace["FastLanguageModel"] = fast_model
+ exec(compile(ast.Module(body = [method], type_ignores = []), str(source), "exec"), namespace)
+ model = SimpleNamespace(config = object())
+ progress = Mock()
+ instance = SimpleNamespace(
+ model = model,
+ _audio_type = None if audio_type == "audio_vlm" else audio_type,
+ is_audio_vlm = audio_type == "audio_vlm",
+ is_vlm = False,
+ should_stop = False,
+ _update_progress = progress,
+ )
+
+ assert namespace["prepare_model_for_training"](instance) is True
+ adapter.assert_called_once()
+ assert adapter.call_args.args == (model,)
+ assert instance.model is adapter.return_value
+ progress.assert_called_once_with(status_message = "LoRA adapters configured")
diff --git a/studio/backend/tests/test_training_cached_start.py b/studio/backend/tests/test_training_cached_start.py
index 6deb615e14b..7b1af0d2e0c 100644
--- a/studio/backend/tests/test_training_cached_start.py
+++ b/studio/backend/tests/test_training_cached_start.py
@@ -45,6 +45,24 @@ def _request(**overrides) -> TrainingStartRequest:
return TrainingStartRequest(**payload)
[email protected]("dataset_path", ["", " ", "\t"])
+def test_local_dataset_path_validation_rejects_blank_values(dataset_path):
+ route = _load_route_module("training_route_rejects_blank_local_dataset_path")
+
+ with (
+ patch.object(
+ route,
+ "resolve_dataset_path",
+ side_effect = AssertionError("blank path must be rejected before resolution"),
+ ),
+ pytest.raises(HTTPException) as exc_info,
+ ):
+ route._validate_local_dataset_paths([dataset_path], "Local eval dataset")
+
+ assert exc_info.value.status_code == 400
+ assert exc_info.value.detail == "Local eval dataset path must not be blank"
+
+
@pytest.mark.parametrize("repo_id", ["_team/dataset_", "dataset_"])
def test_training_request_accepts_hugging_face_underscore_boundaries(repo_id):
assert _request(hf_dataset = repo_id).hf_dataset == repo_id