Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop audio models learning to answer None for blank transcripts (#17257)
Nilayfa115cd429c3
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex b67e5a461a4..233d9006117 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -83,7 +83,7 @@ from utils.models.model_identity import restore_hf_cache_repo_identityfrom utils.models.unsloth_mirror import unsloth_public_mirrorfrom utils.models.model_config import _env_offlinefrom utils.datasets import format_and_template_dataset-from utils.datasets.cells import csv_as_text_kwargs+from utils.datasets.cells import cell_text, csv_as_text_kwargsfrom utils.datasets.chat_templates import get_training_chat_templatefrom utils.datasets.completion_masking import (apply_completion_masking,@@ -2341,6 +2341,17 @@ class UnslothTrainer:)self._audio_vlm_image_col = image_col+ total = len(dataset)+ dataset = dataset.filter(+ lambda text: bool(cell_text(text).strip()), input_columns = [text_col]+ )+ if not len(dataset):+ raise ValueError(f"Every transcript in the '{text_col}' column is blank.")+ if len(dataset) < total:+ self._record_warning(+ f"Skipped {total - len(dataset):,} of {total:,} audio rows with a blank transcript."+ )+dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 16000))if image_col:dataset = prepare_audio_vlm_image_column(dataset, image_col)@@ -2353,7 +2364,7 @@ class UnslothTrainer:formatted["messages"].append(audio_transcription_messages(samples[audio_col][idx]["array"],- str(samples[text_col][idx]),+ cell_text(samples[text_col][idx]),image_slot_count(row_images),))diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 41b0f0266cf..74d22444cd6 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -2552,6 +2552,7 @@ def _mlx_audio_rows(processor,custom_format_mapping = None,image_col = _RESOLVE_IMAGE_COLUMN,+ warn = None,) -> tuple:"""The CUDA transcription rows, with each clip left decoded so zoo can check its rate, andthe row's images (a mixed audio + image dataset) beside them. Returns (rows, image column);@@ -2566,6 +2567,7 @@ def _mlx_audio_rows(resolve_audio_columns,resolve_audio_vlm_image_column,)+ from utils.datasets.cells import cell_textif not ensure_audio_decoding():raise RuntimeError(@@ -2577,7 +2579,8 @@ def _mlx_audio_rows(raise ValueError(f"Audio dataset needs 'audio' and 'text' columns, got: {dataset.column_names}")- if image_col is _RESOLVE_IMAGE_COLUMN:+ train_split = image_col is _RESOLVE_IMAGE_COLUMN+ if train_split:image_col = resolve_audio_vlm_image_column(dataset, custom_format_mapping, audio_col, text_col, columns["speaker_col"])@@ -2588,19 +2591,26 @@ def _mlx_audio_rows(if image_col:dataset = prepare_audio_vlm_image_column(dataset, image_col)rows = []+ blank = 0for row in dataset:+ text = cell_text(row[text_col])+ if not text.strip():+ blank += 1+ continueimages = row[image_col] if image_col else Noneif not isinstance(images, list):images = [] if images is None else [images]images = [image for image in images if image is not None]- item = {- "messages": audio_transcription_messages(- row[audio_col], str(row[text_col]), len(images)- )- }+ item = {"messages": audio_transcription_messages(row[audio_col], text, len(images))}if images:item["images"] = imagesrows.append(item)+ if not rows and train_split:+ raise ValueError(f"Every transcript in the '{text_col}' column is blank.")+ if blank:+ (warn or logger.warning)(+ f"Skipped {blank:,} of {blank + len(rows):,} audio rows with a blank transcript."+ )return rows, image_col@@ -3538,11 +3548,21 @@ def _run_mlx_training(event_queue, stop_queue, config):if is_dataset_audio:_send("status", status_message = "Formatting audio dataset...")- dataset, audio_image_col = _mlx_audio_rows(dataset, tokenizer, custom_format_mapping)+ _warn = lambda message: _send("warning", message = message)+ dataset, audio_image_col = _mlx_audio_rows(+ dataset, tokenizer, custom_format_mapping, warn = _warn+ )if eval_dataset is not None:eval_dataset, _ = _mlx_audio_rows(- eval_dataset, tokenizer, custom_format_mapping, audio_image_col+ eval_dataset, tokenizer, custom_format_mapping, audio_image_col, warn = _warn)+ if not eval_dataset:+ _send(+ "warning",+ message = "The eval dataset is empty after preprocessing, so this run has no evaluation.",+ )+ eval_dataset = None+ eval_enabled = Falseif audio_image_col:image_layout = (_mlx_vlm_resized_image_layout(tokenizer)diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.pyindex 36e6aacbfbf..6d7c9b03f5a 100644--- a/studio/backend/tests/test_mlx_training_worker_config.py+++ b/studio/backend/tests/test_mlx_training_worker_config.py@@ -921,7 +921,9 @@ def test_mlx_audio_dataset_formats_both_splits_as_audio_rows(image_col):"custom_format_mapping": None,"_send": lambda *_a, **_kw: None,"vision_image_size": None,- "_mlx_audio_rows": lambda ds, processor, mapping, *column: formatted.append((ds, *column))+ "_mlx_audio_rows": lambda ds, processor, mapping, *column, **_kw: formatted.append(+ (ds, *column)+ )or ([ds], image_col),"_adapt_for_mlx_vlm": lambda rows, **_kw: adapted.append(rows) or [*rows, "lifted"],"format_and_template_dataset": lambda *_a, **_kw: pytest.fail(@@ -1028,6 +1030,52 @@ def test_mlx_audio_rows_carry_each_rows_images(monkeypatch):assert eval_image_col is None and "images" not in eval_rows[0]+def test_mlx_audio_rows_skip_blank_transcripts(monkeypatch):+ from utils.datasets import audio_decode++ monkeypatch.setattr(audio_decode, "ensure_audio_decoding", lambda: True)+ monkeypatch.setattr(_worker, "logger", types.SimpleNamespace(warning = lambda *a: None))+ zoo_utils = types.ModuleType("unsloth_zoo.mlx.utils")+ zoo_utils.audio_extractor_sampling_rate = lambda processor: 16000+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.utils", zoo_utils)+ clip = {"array": [0.1], "sampling_rate": 16000}++ def dataset(texts):+ return _AudioRowsDataset([{"audio": clip, "text": text} for text in texts])++ rows, _ = _worker._mlx_audio_rows(dataset(["hi", None, " "]), "processor")++ assert [row["messages"][2]["content"][0]["text"] for row in rows] == ["hi"]+ with pytest.raises(ValueError, match = "blank"):+ _worker._mlx_audio_rows(dataset([None, ""]), "processor")+ assert _worker._mlx_audio_rows(dataset([None, ""]), "processor", None, None) == ([], None)+++def test_mlx_blank_audio_eval_split_disables_eval_not_training():+ tree = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training)))+ branch = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.If) and ast.unparse(n.test) == "is_dataset_audio"+ )+ events = []+ namespace = {+ "is_dataset_audio": True,+ "_send": lambda kind, **kw: events.append((kind, kw)),+ "_mlx_audio_rows": lambda ds, *a, **kw: (ds, None),+ "tokenizer": None,+ "custom_format_mapping": None,+ "dataset": [{"messages": []}],+ "eval_dataset": [],+ "eval_enabled": True,+ }+ exec(compile(ast.Module(body = [branch], type_ignores = []), "<audio>", "exec"), namespace)++ assert namespace["eval_dataset"] is None and namespace["eval_enabled"] is False+ assert namespace["dataset"] == [{"messages": []}]+ assert [kind for kind, _ in events].count("warning") == 1++@pytest.mark.parametrize("is_dataset_audio", [True, False])def test_mlx_audio_dataset_loads_and_trains_as_a_vlm(is_dataset_audio):names = {"is_dataset_image": False, "is_dataset_audio": is_dataset_audio}diff --git a/studio/backend/tests/test_trainer_audio_vlm_image_and_audio.py b/studio/backend/tests/test_trainer_audio_vlm_image_and_audio.pyindex 0dba918e954..41b0711bc88 100644--- a/studio/backend/tests/test_trainer_audio_vlm_image_and_audio.py+++ b/studio/backend/tests/test_trainer_audio_vlm_image_and_audio.py@@ -68,6 +68,9 @@ class _NoAudioCast:kwargs.pop("num_proc", None)return _NoAudioCast(self._ds.map(fn, **kwargs))+ def filter(self, fn, **kwargs):+ return _NoAudioCast(self._ds.filter(fn, **kwargs))+def __getattr__(self, name):return getattr(self._ds, name)@@ -502,3 +505,19 @@ def test_a_pixel_array_column_does_not_break_column_resolution(audio_vlm_trainerds = Rows()audio_vlm_trainer._resolve_audio_vlm_image_column(ds, None, "audio")+++def _transcripts(texts):+ rows = {"audio": [{"array": [0.0] * 160, "sampling_rate": 16000}] * len(texts), "text": texts}+ return _NoAudioCast(datasets.Dataset.from_dict(rows))+++def test_blank_transcripts_are_skipped_not_trained(audio_vlm_trainer):+ formatted = audio_vlm_trainer._format_audio_vlm_dataset(_transcripts(["hi", None, " "]))++ assert [row["messages"][-1]["content"][0]["text"] for row in formatted] == ["hi"]+ assert audio_vlm_trainer.training_progress.warnings == [+ "Skipped 2 of 3 audio rows with a blank transcript."+ ]+ with pytest.raises(ValueError, match = "blank"):+ audio_vlm_trainer._format_audio_vlm_dataset(_transcripts([None, ""]))