Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: name a too-short context length when completion masking drops rows (#16000)

Daniel Hanbebfa2b0ba9c
diff --git a/studio/backend/core/training/multi_lora_trainer.py b/studio/backend/core/training/multi_lora_trainer.py
index f1fe973f081..ffd3bb5fb40 100644
--- a/studio/backend/core/training/multi_lora_trainer.py
+++ b/studio/backend/core/training/multi_lora_trainer.py
@@ -389,7 +389,11 @@ def _tokenize_member(
"""(tokenizer to save, [(input_ids, labels)]) with the single SFT path's tokenization and masking."""
from datasets import Dataset
- from utils.datasets.completion_masking import apply_completion_masking
+ from utils.datasets.completion_masking import (
+ apply_completion_masking,
+ count_truncated_drops,
+ truncated_drop_cause,
+ )
from utils.datasets.online_tokenization import (
build_tokenizing_transform,
first_sample_text,
@@ -461,12 +465,16 @@ def _tokenize_member(
dropped = len(labels) - len(kept)
drop_pct = round(100 * dropped / len(labels), 1) if labels else 0
if not kept or drop_pct > 30:
+ cut = count_truncated_drops(input_ids, [input_ids[i] for i in kept], max_length)
+ cause = truncated_drop_cause(cut, dropped, max_length) or (
+ "The usual cause is that this model's response template was not found in the "
+ "formatted samples, so 'Train on completions' should be turned off for this "
+ "dataset."
+ )
raise ValueError(
f"{dropped}/{len(labels)} samples ({drop_pct}%) were dropped after applying "
f"'Train on completions': after masking, those rows had no trainable response "
- f"tokens left. The usual cause is that this model's response template was not "
- f"found in the formatted samples, so 'Train on completions' should be turned "
- f"off for this dataset."
+ f"tokens left. {cause}"
)
input_ids = [input_ids[i] for i in kept]
labels = [labels[i] for i in kept]
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index 93f6a44d0f5..4769ade5130 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -78,7 +78,11 @@ from utils.models.model_config import _env_offline
from utils.datasets import format_and_template_dataset
from utils.datasets.cells import csv_as_text_kwargs
from utils.datasets.chat_templates import get_training_chat_template
-from utils.datasets.completion_masking import apply_completion_masking
+from utils.datasets.completion_masking import (
+ apply_completion_masking,
+ count_truncated_drops,
+ truncated_drop_cause,
+)
from utils.datasets.iterable import is_streaming_dataset as detect_streaming_dataset
from utils.datasets.pretokenized import (
PRETOKENIZED_FORMAT,
@@ -5093,19 +5097,40 @@ class UnslothTrainer:
if filtered_len == 0 or drop_pct > 30:
max_seq = training_args.get("max_seq_length", 2048)
+ cause = None
+ # Packed rows all reach the cap, so a truncated prompt cannot be told apart there.
+ if not config_args.get("packing"):
+ try:
+ cap = (
+ getattr(self.trainer.args, "max_length", None)
+ or max_seq
+ )
+ cut = count_truncated_drops(
+ pre_mask_dataset["input_ids"],
+ self.trainer.train_dataset["input_ids"],
+ cap,
+ )
+ cause = truncated_drop_cause(cut, dropped, cap)
+ except Exception as e: # noqa: BLE001 - keep the generic message
+ logger.info(f"Could not count truncated rows: {e}\n")
error_msg = (
f"{dropped}/{original_len} samples ({drop_pct}%) were "
f"dropped after applying 'Train on completions': after "
f"masking, those rows had no trainable response tokens "
- f"left. The usual cause is that this model's response "
- f"template was not found in the formatted samples, so "
- f"every token was masked out. That typically means the "
- f"dataset is already formatted, or its structure does "
- f"not match the model's chat template, so 'Train on "
- f"completions' should be turned off for this dataset. "
- f"Less commonly, a max_seq_length ({max_seq}) shorter "
- f"than the prompt can truncate the response away; only "
- f"raise it if your samples are actually longer than that."
+ f"left. "
+ ) + (
+ cause
+ or (
+ f"The usual cause is that this model's response "
+ f"template was not found in the formatted samples, so "
+ f"every token was masked out. That typically means the "
+ f"dataset is already formatted, or its structure does "
+ f"not match the model's chat template, so 'Train on "
+ f"completions' should be turned off for this dataset. "
+ f"Less commonly, a max_seq_length ({max_seq}) shorter "
+ f"than the prompt can truncate the response away; only "
+ f"raise it if your samples are actually longer than that."
+ )
)
logger.error(error_msg)
self._update_progress(error = error_msg, is_training = False)
diff --git a/studio/backend/tests/test_completion_masking.py b/studio/backend/tests/test_completion_masking.py
index a68471d4d3a..d401811509b 100644
--- a/studio/backend/tests/test_completion_masking.py
+++ b/studio/backend/tests/test_completion_masking.py
@@ -9,7 +9,12 @@ from pathlib import Path
import pytest
-from utils.datasets.completion_masking import apply_completion_masking, lookup_manual_markers
+from utils.datasets.completion_masking import (
+ apply_completion_masking,
+ count_truncated_drops,
+ lookup_manual_markers,
+ truncated_drop_cause,
+)
from utils.datasets.model_mappings import TEMPLATE_TO_RESPONSES_MAPPER
@@ -514,3 +519,22 @@ def test_gemma4_turn_text_inside_tool_output_stays_masked():
)
assert "Station" not in trained
assert "Here is the prompt." in trained
+
+
+def test_truncated_drops_count_only_dropped_rows_at_the_cap():
+ rows = [[1] * 8, [2] * 8, [3] * 3, [1] * 8]
+ assert count_truncated_drops(rows, [[2] * 8, [1] * 8], 8) == 1
+ assert count_truncated_drops(rows, [[2] * 8], 8) == 2
+ assert count_truncated_drops(rows, rows, 8) == 0
+
+
+def test_truncated_drop_cause_names_the_context_length():
+ assert truncated_drop_cause(0, 5, 2048) is None
+ only_cut = truncated_drop_cause(5, 5, 2048)
+ assert "5 of them reach the context length (2048 tokens)" in only_cut
+ assert "If they are not that long" in only_cut
+ mixed = truncated_drop_cause(3, 5, 2048)
+ assert (
+ "For the other 2, this model's response template was not found" in mixed
+ and "turned off" in mixed
+ )
diff --git a/studio/backend/tests/test_multi_lora_trainer.py b/studio/backend/tests/test_multi_lora_trainer.py
index 4ecdde2ff07..45739ab185b 100644
--- a/studio/backend/tests/test_multi_lora_trainer.py
+++ b/studio/backend/tests/test_multi_lora_trainer.py
@@ -376,6 +376,34 @@ def test_completion_masking_masks_the_prompt():
assert labels[response:] == ids[response:]
+def test_completion_drops_from_truncation_name_the_context_length():
+ tokenizer = _tokenizer()
+ tokenizer._unsloth_input_part = "[U]"
+ tokenizer._unsloth_output_part = "[B]"
+ long_prompt = "[U] " + " ".join(WORDS[:20]) + " [B] w4 w5 [EOS]"
+ info = {
+ "dataset": Dataset.from_dict({"text": [long_prompt] * 3 + ["[U] w1 [B] w2 [EOS]"]}),
+ "final_format": "",
+ }
+ with pytest.raises(ValueError) as exc:
+ mlt._tokenize_member(info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 16)
+ message = str(exc.value)
+ assert "3/4 samples (75.0%) were dropped" in message
+ assert "3 of them reach the context length (16 tokens)" in message
+ assert "response template was not found in the formatted samples" not in message
+
+
+def test_completion_drops_without_a_template_keep_the_template_advice():
+ tokenizer = _tokenizer()
+ tokenizer._unsloth_input_part = "[U]"
+ tokenizer._unsloth_output_part = "[B]"
+ info = {"dataset": Dataset.from_dict({"text": ["[U] w1 w2 [EOS]"] * 3}), "final_format": ""}
+ with pytest.raises(ValueError) as exc:
+ mlt._tokenize_member(info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 64)
+ assert "response template was not found" in str(exc.value)
+ assert "context length" not in str(exc.value)
+
+
def test_batch_trains_through_sft_trainer_and_saves_checkpoints(monkeypatch, tmp_path):
from trl import SFTTrainer
diff --git a/studio/backend/utils/datasets/completion_masking.py b/studio/backend/utils/datasets/completion_masking.py
index 586064e3acc..4d32a5f7e16 100644
--- a/studio/backend/utils/datasets/completion_masking.py
+++ b/studio/backend/utils/datasets/completion_masking.py
@@ -23,6 +23,43 @@ def lookup_manual_markers(model_name):
return template, None, None
+def count_truncated_drops(rows, kept_rows, max_length):
+ """Rows masking dropped (rows minus kept_rows) that had been truncated at max_length."""
+ from collections import Counter
+
+ kept = Counter(map(tuple, kept_rows))
+ cut = 0
+ for row in rows:
+ key = tuple(row)
+ if kept[key]:
+ kept[key] -= 1
+ elif len(row) >= max_length:
+ cut += 1
+ return cut
+
+
+def truncated_drop_cause(cut, dropped, max_length):
+ """Error text when dropped rows reached the context length, or None when none did."""
+ if not cut:
+ return None
+ # Reaching the cap suggests truncation but does not prove it, so the template advice stays as the fallback.
+ cause = (
+ f"{cut} of them reach the context length ({max_length} tokens), so truncation most likely "
+ f"cut off their response: raise the context length or shorten those rows."
+ )
+ if dropped > cut:
+ cause += (
+ f" For the other {dropped - cut}, this model's response template was not found, so "
+ f"'Train on completions' should be turned off for this dataset."
+ )
+ else:
+ cause += (
+ " If they are not that long, this model's response template was not found, so "
+ "'Train on completions' should be turned off for this dataset."
+ )
+ return cause
+
+
def _mask_tool_responses(trainer, start_id, end_id, turn_id):
def mask(batch):
all_labels = []