Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop audio models learning to answer None for blank transcripts (#17257)

Nilayfa115cd429c3
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index b67e5a461a4..233d9006117 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -83,7 +83,7 @@ from utils.models.model_identity import restore_hf_cache_repo_identity
from utils.models.unsloth_mirror import unsloth_public_mirror
from utils.models.model_config import _env_offline
from utils.datasets import format_and_template_dataset
-from utils.datasets.cells import csv_as_text_kwargs
+from utils.datasets.cells import cell_text, csv_as_text_kwargs
from utils.datasets.chat_templates import get_training_chat_template
from utils.datasets.completion_masking import (
apply_completion_masking,
@@ -2341,6 +2341,17 @@ class UnslothTrainer:
)
self._audio_vlm_image_col = image_col
+ total = len(dataset)
+ dataset = dataset.filter(
+ lambda text: bool(cell_text(text).strip()), input_columns = [text_col]
+ )
+ if not len(dataset):
+ raise ValueError(f"Every transcript in the '{text_col}' column is blank.")
+ if len(dataset) < total:
+ self._record_warning(
+ f"Skipped {total - len(dataset):,} of {total:,} audio rows with a blank transcript."
+ )
+
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 16000))
if image_col:
dataset = prepare_audio_vlm_image_column(dataset, image_col)
@@ -2353,7 +2364,7 @@ class UnslothTrainer:
formatted["messages"].append(
audio_transcription_messages(
samples[audio_col][idx]["array"],
- str(samples[text_col][idx]),
+ cell_text(samples[text_col][idx]),
image_slot_count(row_images),
)
)
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 41b0f0266cf..74d22444cd6 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -2552,6 +2552,7 @@ def _mlx_audio_rows(
processor,
custom_format_mapping = None,
image_col = _RESOLVE_IMAGE_COLUMN,
+ warn = None,
) -> tuple:
"""The CUDA transcription rows, with each clip left decoded so zoo can check its rate, and
the row's images (a mixed audio + image dataset) beside them. Returns (rows, image column);
@@ -2566,6 +2567,7 @@ def _mlx_audio_rows(
resolve_audio_columns,
resolve_audio_vlm_image_column,
)
+ from utils.datasets.cells import cell_text
if not ensure_audio_decoding():
raise RuntimeError(
@@ -2577,7 +2579,8 @@ def _mlx_audio_rows(
raise ValueError(
f"Audio dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
- if image_col is _RESOLVE_IMAGE_COLUMN:
+ train_split = image_col is _RESOLVE_IMAGE_COLUMN
+ if train_split:
image_col = resolve_audio_vlm_image_column(
dataset, custom_format_mapping, audio_col, text_col, columns["speaker_col"]
)
@@ -2588,19 +2591,26 @@ def _mlx_audio_rows(
if image_col:
dataset = prepare_audio_vlm_image_column(dataset, image_col)
rows = []
+ blank = 0
for row in dataset:
+ text = cell_text(row[text_col])
+ if not text.strip():
+ blank += 1
+ continue
images = row[image_col] if image_col else None
if not isinstance(images, list):
images = [] if images is None else [images]
images = [image for image in images if image is not None]
- item = {
- "messages": audio_transcription_messages(
- row[audio_col], str(row[text_col]), len(images)
- )
- }
+ item = {"messages": audio_transcription_messages(row[audio_col], text, len(images))}
if images:
item["images"] = images
rows.append(item)
+ if not rows and train_split:
+ raise ValueError(f"Every transcript in the '{text_col}' column is blank.")
+ if blank:
+ (warn or logger.warning)(
+ f"Skipped {blank:,} of {blank + len(rows):,} audio rows with a blank transcript."
+ )
return rows, image_col
@@ -3538,11 +3548,21 @@ def _run_mlx_training(event_queue, stop_queue, config):
if is_dataset_audio:
_send("status", status_message = "Formatting audio dataset...")
- dataset, audio_image_col = _mlx_audio_rows(dataset, tokenizer, custom_format_mapping)
+ _warn = lambda message: _send("warning", message = message)
+ dataset, audio_image_col = _mlx_audio_rows(
+ dataset, tokenizer, custom_format_mapping, warn = _warn
+ )
if eval_dataset is not None:
eval_dataset, _ = _mlx_audio_rows(
- eval_dataset, tokenizer, custom_format_mapping, audio_image_col
+ eval_dataset, tokenizer, custom_format_mapping, audio_image_col, warn = _warn
)
+ if not eval_dataset:
+ _send(
+ "warning",
+ message = "The eval dataset is empty after preprocessing, so this run has no evaluation.",
+ )
+ eval_dataset = None
+ eval_enabled = False
if audio_image_col:
image_layout = (
_mlx_vlm_resized_image_layout(tokenizer)
diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.py
index 36e6aacbfbf..6d7c9b03f5a 100644
--- a/studio/backend/tests/test_mlx_training_worker_config.py
+++ b/studio/backend/tests/test_mlx_training_worker_config.py
@@ -921,7 +921,9 @@ def test_mlx_audio_dataset_formats_both_splits_as_audio_rows(image_col):
"custom_format_mapping": None,
"_send": lambda *_a, **_kw: None,
"vision_image_size": None,
- "_mlx_audio_rows": lambda ds, processor, mapping, *column: formatted.append((ds, *column))
+ "_mlx_audio_rows": lambda ds, processor, mapping, *column, **_kw: formatted.append(
+ (ds, *column)
+ )
or ([ds], image_col),
"_adapt_for_mlx_vlm": lambda rows, **_kw: adapted.append(rows) or [*rows, "lifted"],
"format_and_template_dataset": lambda *_a, **_kw: pytest.fail(
@@ -1028,6 +1030,52 @@ def test_mlx_audio_rows_carry_each_rows_images(monkeypatch):
assert eval_image_col is None and "images" not in eval_rows[0]
+def test_mlx_audio_rows_skip_blank_transcripts(monkeypatch):
+ from utils.datasets import audio_decode
+
+ monkeypatch.setattr(audio_decode, "ensure_audio_decoding", lambda: True)
+ monkeypatch.setattr(_worker, "logger", types.SimpleNamespace(warning = lambda *a: None))
+ zoo_utils = types.ModuleType("unsloth_zoo.mlx.utils")
+ zoo_utils.audio_extractor_sampling_rate = lambda processor: 16000
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.utils", zoo_utils)
+ clip = {"array": [0.1], "sampling_rate": 16000}
+
+ def dataset(texts):
+ return _AudioRowsDataset([{"audio": clip, "text": text} for text in texts])
+
+ rows, _ = _worker._mlx_audio_rows(dataset(["hi", None, " "]), "processor")
+
+ assert [row["messages"][2]["content"][0]["text"] for row in rows] == ["hi"]
+ with pytest.raises(ValueError, match = "blank"):
+ _worker._mlx_audio_rows(dataset([None, ""]), "processor")
+ assert _worker._mlx_audio_rows(dataset([None, ""]), "processor", None, None) == ([], None)
+
+
+def test_mlx_blank_audio_eval_split_disables_eval_not_training():
+ tree = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training)))
+ branch = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.If) and ast.unparse(n.test) == "is_dataset_audio"
+ )
+ events = []
+ namespace = {
+ "is_dataset_audio": True,
+ "_send": lambda kind, **kw: events.append((kind, kw)),
+ "_mlx_audio_rows": lambda ds, *a, **kw: (ds, None),
+ "tokenizer": None,
+ "custom_format_mapping": None,
+ "dataset": [{"messages": []}],
+ "eval_dataset": [],
+ "eval_enabled": True,
+ }
+ exec(compile(ast.Module(body = [branch], type_ignores = []), "<audio>", "exec"), namespace)
+
+ assert namespace["eval_dataset"] is None and namespace["eval_enabled"] is False
+ assert namespace["dataset"] == [{"messages": []}]
+ assert [kind for kind, _ in events].count("warning") == 1
+
+
@pytest.mark.parametrize("is_dataset_audio", [True, False])
def test_mlx_audio_dataset_loads_and_trains_as_a_vlm(is_dataset_audio):
names = {"is_dataset_image": False, "is_dataset_audio": is_dataset_audio}
diff --git a/studio/backend/tests/test_trainer_audio_vlm_image_and_audio.py b/studio/backend/tests/test_trainer_audio_vlm_image_and_audio.py
index 0dba918e954..41b0711bc88 100644
--- a/studio/backend/tests/test_trainer_audio_vlm_image_and_audio.py
+++ b/studio/backend/tests/test_trainer_audio_vlm_image_and_audio.py
@@ -68,6 +68,9 @@ class _NoAudioCast:
kwargs.pop("num_proc", None)
return _NoAudioCast(self._ds.map(fn, **kwargs))
+ def filter(self, fn, **kwargs):
+ return _NoAudioCast(self._ds.filter(fn, **kwargs))
+
def __getattr__(self, name):
return getattr(self._ds, name)
@@ -502,3 +505,19 @@ def test_a_pixel_array_column_does_not_break_column_resolution(audio_vlm_trainer
ds = Rows()
audio_vlm_trainer._resolve_audio_vlm_image_column(ds, None, "audio")
+
+
+def _transcripts(texts):
+ rows = {"audio": [{"array": [0.0] * 160, "sampling_rate": 16000}] * len(texts), "text": texts}
+ return _NoAudioCast(datasets.Dataset.from_dict(rows))
+
+
+def test_blank_transcripts_are_skipped_not_trained(audio_vlm_trainer):
+ formatted = audio_vlm_trainer._format_audio_vlm_dataset(_transcripts(["hi", None, " "]))
+
+ assert [row["messages"][-1]["content"][0]["text"] for row in formatted] == ["hi"]
+ assert audio_vlm_trainer.training_progress.warnings == [
+ "Skipped 2 of 3 audio rows with a blank transcript."
+ ]
+ with pytest.raises(ValueError, match = "blank"):
+ audio_vlm_trainer._format_audio_vlm_dataset(_transcripts([None, ""]))