Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: honor completion-only training for audio VLMs (#14402)
cherry77-cloud5641b2aafe2a
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex 605d8f39a94..e98ff90e848 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -4612,6 +4612,7 @@ class UnslothTrainer:)data_collator = None+ audio_response_masker = Noneif is_deepseek_ocr:logger.info("Detected DeepSeek OCR model\n")if not _ensure_deepseek_ocr_installed():@@ -4670,6 +4671,32 @@ class UnslothTrainer:processor = self.tokenizeraudio_col_name = getattr(self, "_audio_vlm_audio_col", "audio")+ if training_args.get("train_on_completions", False) and not training_args.get(+ "is_cpt", False+ ):+ from unsloth.chat_templates import train_on_responses_only++ def _build_response_masker(holder, **kwargs):+ holder.masker = train_on_responses_only(+ None, tokenizer = processor, return_function = True, **kwargs+ )+ return holder++ def _notify_masking(level, message):+ if level == "warning":+ logger.warning(message)+ else:+ logger.info(f"{message}\n")++ # Same marker policy as the text path: auto-detect, template table, else full sequences.+ holder, audio_masking_applied = apply_completion_masking(+ types.SimpleNamespace(processing_class = processor, masker = None),+ self.model_name,+ _build_response_masker,+ notify = _notify_masking,+ )+ if audio_masking_applied:+ audio_response_masker = holder.maskerdef audio_vlm_collate_fn(examples):texts = []@@ -4697,6 +4724,8 @@ class UnslothTrainer:if token_id is not None:labels[labels == token_id] = -100batch["labels"] = labels+ if audio_response_masker is not None:+ batch["labels"] = audio_response_masker(batch)["labels"]return batchdata_collator = audio_vlm_collate_fn@@ -5166,6 +5195,8 @@ class UnslothTrainer:else:if train_on_responses_enabled and is_deepseek_ocr:logger.info("Train on responses handled by DeepSeek OCR collator\n")+ elif train_on_responses_enabled and audio_response_masker is not None:+ logger.info("Train on responses handled by audio VLM collator\n")else:logger.info("Training on full sequences (including prompts)\n")diff --git a/studio/backend/tests/test_audio_vlm_completion_masking.py b/studio/backend/tests/test_audio_vlm_completion_masking.pynew file mode 100644index 00000000000..78131c81602--- /dev/null+++ b/studio/backend/tests/test_audio_vlm_completion_masking.py@@ -0,0 +1,180 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""'Assistant completions only' for audio VLMs (Gemma 3n / Gemma 4 audio).++The audio VLM collator builds its labels per batch, so the shared dataset-level masking never reaches it. These+tests exec the collator block of core/training/trainer.py (importing the trainer drags in the whole torch/unsloth+stack) against a small fake processor.+"""++from __future__ import annotations++import logging+import sys+import textwrap+import types+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")++from utils.datasets.completion_masking import apply_completion_masking++PAD, AUDIO, USER, MODEL, END = 0, 5, 1, 2, 3+VOCAB = {"<pad>": PAD, "<user>": USER, "<model>": MODEL, "<end>": END, "<audio>": AUDIO}+++def _collator_block():+ lines = (+ (Path(__file__).resolve().parents[1] / "core" / "training" / "trainer.py")+ .read_text(encoding = "utf-8")+ .splitlines()+ )+ start = next(+ i+ for i, line in enumerate(lines)+ if line.strip() == 'audio_col_name = getattr(self, "_audio_vlm_audio_col", "audio")'+ )+ end = next(+ i+ for i in range(start, len(lines))+ if lines[i].strip() == "data_collator = audio_vlm_collate_fn"+ )+ return textwrap.dedent("\n".join(lines[start:end]))+++class _Tokenizer:+ pad_token_id = PAD+ audio_token_id = AUDIO++ def __init__(self, preset_markers):+ if preset_markers:+ self._unsloth_input_part = "<user>"+ self._unsloth_output_part = "<model>"++ def id(self, word):+ return VOCAB.setdefault(word, len(VOCAB) + 10)+++class _Processor:+ def __init__(self, preset_markers = True):+ self.tokenizer = _Tokenizer(preset_markers)++ def apply_chat_template(self, messages, tokenize, add_generation_prompt):+ parts = []+ for message in messages:+ marker = "<model>" if message["role"] == "assistant" else "<user>"+ parts.append(f"{marker} {message['content']} <end>")+ return " ".join(parts)++ def __call__(self, text, audio, return_tensors, padding):+ rows = [[self.tokenizer.id(word) for word in t.split()] for t in text]+ width = max(len(row) for row in rows)+ return {"input_ids": torch.tensor([[PAD] * (width - len(row)) + row for row in rows])}+++def _fake_train_on_responses_only(+ trainer,+ tokenizer = None,+ return_function = False,+ **kwargs,+):+ """Keeps old labels from each <model> marker to its <end>, as unsloth_zoo does."""+ assert trainer is None and return_function++ def mask(batch):+ out = []+ for ids, old in zip(batch["input_ids"].tolist(), batch["labels"].tolist()):+ new, keep = [-100] * len(ids), False+ for i, token in enumerate(ids):+ if keep:+ new[i] = old[i]+ keep = (keep and token != END) or token == MODEL+ out.append(new)+ return {"labels": torch.tensor(out)}++ return mask+++def fake_unsloth(monkeypatch):+ chat_templates = types.ModuleType("unsloth.chat_templates")+ chat_templates.train_on_responses_only = _fake_train_on_responses_only+ monkeypatch.setitem(sys.modules, "unsloth.chat_templates", chat_templates)+ monkeypatch.setitem(sys.modules, "unsloth", types.ModuleType("unsloth"))+++def _collate(+ processor,+ model_name = "unmapped/audio-model",+ **training_args,+):+ namespace = {+ "processor": processor,+ "training_args": training_args,+ "self": types.SimpleNamespace(_audio_vlm_audio_col = "speech", model_name = model_name),+ "logger": logging.getLogger("test_audio_vlm_completion_masking"),+ "types": types,+ "apply_completion_masking": apply_completion_masking,+ "audio_response_masker": None,+ }+ exec(_collator_block(), namespace)+ examples = [+ {+ "messages": [+ {"role": "system", "content": "transcribe accurately"},+ {"role": "user", "content": "<audio> <audio> please transcribe"},+ {"role": "assistant", "content": answer},+ ],+ "speech": {"array": [0.0]},+ }+ for answer in ("hello world", "a longer answer here")+ ]+ return namespace["audio_vlm_collate_fn"](examples)+++def _targets(batch):+ return [+ [token for token, label in zip(ids, labels) if label != -100]+ for ids, labels in zip(batch["input_ids"].tolist(), batch["labels"].tolist())+ ]+++def _words(*words):+ return [VOCAB[w] for w in words]+++def test_completion_only_trains_on_the_assistant_turn_only(fake_unsloth):+ batch = _collate(_Processor(), train_on_completions = True)+ assert _targets(batch) == [+ _words("hello", "world", "<end>"),+ _words("a", "longer", "answer", "here", "<end>"),+ ]+++ "training_args",+ [{}, {"train_on_completions": False}, {"train_on_completions": True, "is_cpt": True}],+)+def test_disabled_option_and_cpt_keep_full_sequence_labels(fake_unsloth, training_args):+ batch = _collate(_Processor(), **training_args)+ for ids, labels in zip(batch["input_ids"].tolist(), batch["labels"].tolist()):+ assert labels == [-100 if token in (PAD, AUDIO) else token for token in ids]+++def test_undetectable_markers_warn_and_train_full_sequences(fake_unsloth, monkeypatch, caplog):+ dataset_utils = types.ModuleType("unsloth_zoo.dataset_utils")++ def fail(processor):+ raise ValueError("no chat template")++ dataset_utils.get_chat_template_parts = fail+ monkeypatch.setitem(sys.modules, "unsloth_zoo", types.ModuleType("unsloth_zoo"))+ monkeypatch.setitem(sys.modules, "unsloth_zoo.dataset_utils", dataset_utils)+ with caplog.at_level(logging.WARNING):+ batch = _collate(_Processor(preset_markers = False), train_on_completions = True)+ assert "could not be applied" in caplog.text+ for ids, labels in zip(batch["input_ids"].tolist(), batch["labels"].tolist()):+ assert labels == [-100 if token in (PAD, AUDIO) else token for token in ids]