Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

check_dataset: measure the text SFT trains on (#17047)

Shubhankar Tripathy5bec89fc24fe
diff --git a/tests/test_check_dataset.py b/tests/test_check_dataset.py
index 6f9a7f9b17d..576506b1147 100644
--- a/tests/test_check_dataset.py
+++ b/tests/test_check_dataset.py
@@ -226,6 +226,7 @@ def test_prompt_completion_rows():
rows = [
{"prompt": "hi there", "completion": " hello"},
{"prompt": "hi", "completion": " " + LONG},
+ {"prompt": "", "completion": " "},
{
"prompt": [{"role": "user", "content": "hi"}],
"completion": [{"role": "assistant", "content": "hello"}],
@@ -234,9 +235,11 @@ def test_prompt_completion_rows():
report = check_dataset(rows, tok, max_seq_length = 8)
assert report["malformed"] == 0
+ assert report["empty"] == 1
assert report["over_max_seq_length"] == 1
assert report["response_marker_found"] == 1
- assert report["lengths"] == [3, 31] + _lengths([_chat("hi", "hello")], tok)
+ # SFT appends EOS to a string completion: "hi there hello<|im_end|>" is 4 tokens.
+ assert report["lengths"] == [4, 32, 1] + _lengths([_chat("hi", "hello")], tok)
def test_unusable_prompt_completion_and_alpaca_rows_stay_malformed(capsys):
@@ -323,6 +326,27 @@ def test_text_column_wins_over_conversations():
assert _lengths(rows, _tokenizer()) == _lengths([{"text": LONG}], _tokenizer())
+def test_text_column_wins_over_messages():
+ rows = [{"messages": _chat("hi", "hello")["messages"], "text": LONG}]
+ report = check_dataset(rows, _tokenizer(), max_seq_length = 8)
+ assert report["lengths"] == _lengths([{"text": LONG}], _tokenizer())
+ assert report["over_max_seq_length"] == 1
+
+
+def test_prompt_completion_wins_over_text_and_messages():
+ rows = [
+ {"prompt": "hi", "completion": " hello", "text": LONG},
+ {"prompt": "hi", "completion": " hello", **_chat("hi", LONG)},
+ ]
+ assert _lengths(rows, _tokenizer()) == [3, 3]
+
+
+def test_processor_prompt_completion_gets_inner_eos():
+ tok = _tokenizer()
+ rows = [{"prompt": "hi", "completion": " hello"}]
+ assert _lengths(rows, _ImagesFirstProcessor(tok)) == _lengths(rows, tok) == [3]
+
+
def test_assistant_tool_call_turn_without_content():
template = (
"{% for m in messages %}{{ m['role'] }} "
diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index 97624a04119..beb0f440f26 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -86,8 +86,8 @@ else:
trainer = arguments.pop("trainer", None)
return _zoo_train_on_responses_only(trainer, **arguments)
-# TRL trains on "messages" when present, else on "text"; "conversations" is only read without "text".
-_DATASET_CHECK_COLUMNS = ("messages", "text", "conversations")
+# Order of unsloth_zoo sft_prepare_dataset: prompt/completion, then "text", then "messages".
+_DATASET_CHECK_COLUMNS = ("text", "messages", "conversations")
_MEDIA_TYPES = ("image", "image_url", "video", "audio", "input_audio")
@@ -129,16 +129,20 @@ def _render_dataset_row(row, tokenizer):
if not isinstance(row, dict):
raise _MalformedRow("row is not a dict")
column = next((c for c in _DATASET_CHECK_COLUMNS if c in row), None)
- if column is not None:
- value = row[column]
- elif "prompt" in row and "completion" in row:
+ if "prompt" in row and "completion" in row:
prompt, completion = row["prompt"], row["completion"]
if isinstance(prompt, str) and isinstance(completion, str):
- text = prompt + completion
- return text, text.strip() == "", True, _has_media(row, ())
+ empty = (prompt + completion).strip() == ""
+ text_tokenizer = tokenizer.tokenizer if isinstance(tokenizer, ProcessorMixin) else tokenizer
+ eos = getattr(text_tokenizer, "eos_token", None)
+ if eos and not completion.endswith(eos):
+ completion += eos
+ return prompt + completion, empty, True, _has_media(row, ())
if not isinstance(prompt, list) or not isinstance(completion, list):
raise _MalformedRow("prompt and completion are not both strings or both message lists")
column, value = "prompt/completion", prompt + completion
+ elif column is not None:
+ value = row[column]
elif "instruction" in row and "output" in row:
instruction, extra, output = row["instruction"], row.get("input"), row["output"]
extra = "" if extra is None else extra