Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: name a too-short context length when completion masking drops rows (#16000)
Daniel Hanbebfa2b0ba9c
diff --git a/studio/backend/core/training/multi_lora_trainer.py b/studio/backend/core/training/multi_lora_trainer.pyindex f1fe973f081..ffd3bb5fb40 100644--- a/studio/backend/core/training/multi_lora_trainer.py+++ b/studio/backend/core/training/multi_lora_trainer.py@@ -389,7 +389,11 @@ def _tokenize_member("""(tokenizer to save, [(input_ids, labels)]) with the single SFT path's tokenization and masking."""from datasets import Dataset- from utils.datasets.completion_masking import apply_completion_masking+ from utils.datasets.completion_masking import (+ apply_completion_masking,+ count_truncated_drops,+ truncated_drop_cause,+ )from utils.datasets.online_tokenization import (build_tokenizing_transform,first_sample_text,@@ -461,12 +465,16 @@ def _tokenize_member(dropped = len(labels) - len(kept)drop_pct = round(100 * dropped / len(labels), 1) if labels else 0if not kept or drop_pct > 30:+ cut = count_truncated_drops(input_ids, [input_ids[i] for i in kept], max_length)+ cause = truncated_drop_cause(cut, dropped, max_length) or (+ "The usual cause is that this model's response template was not found in the "+ "formatted samples, so 'Train on completions' should be turned off for this "+ "dataset."+ )raise ValueError(f"{dropped}/{len(labels)} samples ({drop_pct}%) were dropped after applying "f"'Train on completions': after masking, those rows had no trainable response "- f"tokens left. The usual cause is that this model's response template was not "- f"found in the formatted samples, so 'Train on completions' should be turned "- f"off for this dataset."+ f"tokens left. {cause}")input_ids = [input_ids[i] for i in kept]labels = [labels[i] for i in kept]diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex 93f6a44d0f5..4769ade5130 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -78,7 +78,11 @@ from utils.models.model_config import _env_offlinefrom utils.datasets import format_and_template_datasetfrom utils.datasets.cells import csv_as_text_kwargsfrom utils.datasets.chat_templates import get_training_chat_template-from utils.datasets.completion_masking import apply_completion_masking+from utils.datasets.completion_masking import (+ apply_completion_masking,+ count_truncated_drops,+ truncated_drop_cause,+)from utils.datasets.iterable import is_streaming_dataset as detect_streaming_datasetfrom utils.datasets.pretokenized import (PRETOKENIZED_FORMAT,@@ -5093,19 +5097,40 @@ class UnslothTrainer:if filtered_len == 0 or drop_pct > 30:max_seq = training_args.get("max_seq_length", 2048)+ cause = None+ # Packed rows all reach the cap, so a truncated prompt cannot be told apart there.+ if not config_args.get("packing"):+ try:+ cap = (+ getattr(self.trainer.args, "max_length", None)+ or max_seq+ )+ cut = count_truncated_drops(+ pre_mask_dataset["input_ids"],+ self.trainer.train_dataset["input_ids"],+ cap,+ )+ cause = truncated_drop_cause(cut, dropped, cap)+ except Exception as e: # noqa: BLE001 - keep the generic message+ logger.info(f"Could not count truncated rows: {e}\n")error_msg = (f"{dropped}/{original_len} samples ({drop_pct}%) were "f"dropped after applying 'Train on completions': after "f"masking, those rows had no trainable response tokens "- f"left. The usual cause is that this model's response "- f"template was not found in the formatted samples, so "- f"every token was masked out. That typically means the "- f"dataset is already formatted, or its structure does "- f"not match the model's chat template, so 'Train on "- f"completions' should be turned off for this dataset. "- f"Less commonly, a max_seq_length ({max_seq}) shorter "- f"than the prompt can truncate the response away; only "- f"raise it if your samples are actually longer than that."+ f"left. "+ ) + (+ cause+ or (+ f"The usual cause is that this model's response "+ f"template was not found in the formatted samples, so "+ f"every token was masked out. That typically means the "+ f"dataset is already formatted, or its structure does "+ f"not match the model's chat template, so 'Train on "+ f"completions' should be turned off for this dataset. "+ f"Less commonly, a max_seq_length ({max_seq}) shorter "+ f"than the prompt can truncate the response away; only "+ f"raise it if your samples are actually longer than that."+ ))logger.error(error_msg)self._update_progress(error = error_msg, is_training = False)diff --git a/studio/backend/tests/test_completion_masking.py b/studio/backend/tests/test_completion_masking.pyindex a68471d4d3a..d401811509b 100644--- a/studio/backend/tests/test_completion_masking.py+++ b/studio/backend/tests/test_completion_masking.py@@ -9,7 +9,12 @@ from pathlib import Pathimport pytest-from utils.datasets.completion_masking import apply_completion_masking, lookup_manual_markers+from utils.datasets.completion_masking import (+ apply_completion_masking,+ count_truncated_drops,+ lookup_manual_markers,+ truncated_drop_cause,+)from utils.datasets.model_mappings import TEMPLATE_TO_RESPONSES_MAPPER@@ -514,3 +519,22 @@ def test_gemma4_turn_text_inside_tool_output_stays_masked():)assert "Station" not in trainedassert "Here is the prompt." in trained+++def test_truncated_drops_count_only_dropped_rows_at_the_cap():+ rows = [[1] * 8, [2] * 8, [3] * 3, [1] * 8]+ assert count_truncated_drops(rows, [[2] * 8, [1] * 8], 8) == 1+ assert count_truncated_drops(rows, [[2] * 8], 8) == 2+ assert count_truncated_drops(rows, rows, 8) == 0+++def test_truncated_drop_cause_names_the_context_length():+ assert truncated_drop_cause(0, 5, 2048) is None+ only_cut = truncated_drop_cause(5, 5, 2048)+ assert "5 of them reach the context length (2048 tokens)" in only_cut+ assert "If they are not that long" in only_cut+ mixed = truncated_drop_cause(3, 5, 2048)+ assert (+ "For the other 2, this model's response template was not found" in mixed+ and "turned off" in mixed+ )diff --git a/studio/backend/tests/test_multi_lora_trainer.py b/studio/backend/tests/test_multi_lora_trainer.pyindex 4ecdde2ff07..45739ab185b 100644--- a/studio/backend/tests/test_multi_lora_trainer.py+++ b/studio/backend/tests/test_multi_lora_trainer.py@@ -376,6 +376,34 @@ def test_completion_masking_masks_the_prompt():assert labels[response:] == ids[response:]+def test_completion_drops_from_truncation_name_the_context_length():+ tokenizer = _tokenizer()+ tokenizer._unsloth_input_part = "[U]"+ tokenizer._unsloth_output_part = "[B]"+ long_prompt = "[U] " + " ".join(WORDS[:20]) + " [B] w4 w5 [EOS]"+ info = {+ "dataset": Dataset.from_dict({"text": [long_prompt] * 3 + ["[U] w1 [B] w2 [EOS]"]}),+ "final_format": "",+ }+ with pytest.raises(ValueError) as exc:+ mlt._tokenize_member(info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 16)+ message = str(exc.value)+ assert "3/4 samples (75.0%) were dropped" in message+ assert "3 of them reach the context length (16 tokens)" in message+ assert "response template was not found in the formatted samples" not in message+++def test_completion_drops_without_a_template_keep_the_template_advice():+ tokenizer = _tokenizer()+ tokenizer._unsloth_input_part = "[U]"+ tokenizer._unsloth_output_part = "[B]"+ info = {"dataset": Dataset.from_dict({"text": ["[U] w1 w2 [EOS]"] * 3}), "final_format": ""}+ with pytest.raises(ValueError) as exc:+ mlt._tokenize_member(info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 64)+ assert "response template was not found" in str(exc.value)+ assert "context length" not in str(exc.value)++def test_batch_trains_through_sft_trainer_and_saves_checkpoints(monkeypatch, tmp_path):from trl import SFTTrainerdiff --git a/studio/backend/utils/datasets/completion_masking.py b/studio/backend/utils/datasets/completion_masking.pyindex 586064e3acc..4d32a5f7e16 100644--- a/studio/backend/utils/datasets/completion_masking.py+++ b/studio/backend/utils/datasets/completion_masking.py@@ -23,6 +23,43 @@ def lookup_manual_markers(model_name):return template, None, None+def count_truncated_drops(rows, kept_rows, max_length):+ """Rows masking dropped (rows minus kept_rows) that had been truncated at max_length."""+ from collections import Counter++ kept = Counter(map(tuple, kept_rows))+ cut = 0+ for row in rows:+ key = tuple(row)+ if kept[key]:+ kept[key] -= 1+ elif len(row) >= max_length:+ cut += 1+ return cut+++def truncated_drop_cause(cut, dropped, max_length):+ """Error text when dropped rows reached the context length, or None when none did."""+ if not cut:+ return None+ # Reaching the cap suggests truncation but does not prove it, so the template advice stays as the fallback.+ cause = (+ f"{cut} of them reach the context length ({max_length} tokens), so truncation most likely "+ f"cut off their response: raise the context length or shorten those rows."+ )+ if dropped > cut:+ cause += (+ f" For the other {dropped - cut}, this model's response template was not found, so "+ f"'Train on completions' should be turned off for this dataset."+ )+ else:+ cause += (+ " If they are not that long, this model's response template was not found, so "+ "'Train on completions' should be turned off for this dataset."+ )+ return cause++def _mask_tool_responses(trainer, start_id, end_id, turn_id):def mask(batch):all_labels = []