Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
check_dataset: measure the text SFT trains on (#17047)
Shubhankar Tripathy5bec89fc24fe
diff --git a/tests/test_check_dataset.py b/tests/test_check_dataset.pyindex 6f9a7f9b17d..576506b1147 100644--- a/tests/test_check_dataset.py+++ b/tests/test_check_dataset.py@@ -226,6 +226,7 @@ def test_prompt_completion_rows():rows = [{"prompt": "hi there", "completion": " hello"},{"prompt": "hi", "completion": " " + LONG},+ {"prompt": "", "completion": " "},{"prompt": [{"role": "user", "content": "hi"}],"completion": [{"role": "assistant", "content": "hello"}],@@ -234,9 +235,11 @@ def test_prompt_completion_rows():report = check_dataset(rows, tok, max_seq_length = 8)assert report["malformed"] == 0+ assert report["empty"] == 1assert report["over_max_seq_length"] == 1assert report["response_marker_found"] == 1- assert report["lengths"] == [3, 31] + _lengths([_chat("hi", "hello")], tok)+ # SFT appends EOS to a string completion: "hi there hello<|im_end|>" is 4 tokens.+ assert report["lengths"] == [4, 32, 1] + _lengths([_chat("hi", "hello")], tok)def test_unusable_prompt_completion_and_alpaca_rows_stay_malformed(capsys):@@ -323,6 +326,27 @@ def test_text_column_wins_over_conversations():assert _lengths(rows, _tokenizer()) == _lengths([{"text": LONG}], _tokenizer())+def test_text_column_wins_over_messages():+ rows = [{"messages": _chat("hi", "hello")["messages"], "text": LONG}]+ report = check_dataset(rows, _tokenizer(), max_seq_length = 8)+ assert report["lengths"] == _lengths([{"text": LONG}], _tokenizer())+ assert report["over_max_seq_length"] == 1+++def test_prompt_completion_wins_over_text_and_messages():+ rows = [+ {"prompt": "hi", "completion": " hello", "text": LONG},+ {"prompt": "hi", "completion": " hello", **_chat("hi", LONG)},+ ]+ assert _lengths(rows, _tokenizer()) == [3, 3]+++def test_processor_prompt_completion_gets_inner_eos():+ tok = _tokenizer()+ rows = [{"prompt": "hi", "completion": " hello"}]+ assert _lengths(rows, _ImagesFirstProcessor(tok)) == _lengths(rows, tok) == [3]++def test_assistant_tool_call_turn_without_content():template = ("{% for m in messages %}{{ m['role'] }} "diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.pyindex 97624a04119..beb0f440f26 100644--- a/unsloth/chat_templates.py+++ b/unsloth/chat_templates.py@@ -86,8 +86,8 @@ else:trainer = arguments.pop("trainer", None)return _zoo_train_on_responses_only(trainer, **arguments)-# TRL trains on "messages" when present, else on "text"; "conversations" is only read without "text".-_DATASET_CHECK_COLUMNS = ("messages", "text", "conversations")+# Order of unsloth_zoo sft_prepare_dataset: prompt/completion, then "text", then "messages".+_DATASET_CHECK_COLUMNS = ("text", "messages", "conversations")_MEDIA_TYPES = ("image", "image_url", "video", "audio", "input_audio")@@ -129,16 +129,20 @@ def _render_dataset_row(row, tokenizer):if not isinstance(row, dict):raise _MalformedRow("row is not a dict")column = next((c for c in _DATASET_CHECK_COLUMNS if c in row), None)- if column is not None:- value = row[column]- elif "prompt" in row and "completion" in row:+ if "prompt" in row and "completion" in row:prompt, completion = row["prompt"], row["completion"]if isinstance(prompt, str) and isinstance(completion, str):- text = prompt + completion- return text, text.strip() == "", True, _has_media(row, ())+ empty = (prompt + completion).strip() == ""+ text_tokenizer = tokenizer.tokenizer if isinstance(tokenizer, ProcessorMixin) else tokenizer+ eos = getattr(text_tokenizer, "eos_token", None)+ if eos and not completion.endswith(eos):+ completion += eos+ return prompt + completion, empty, True, _has_media(row, ())if not isinstance(prompt, list) or not isinstance(completion, list):raise _MalformedRow("prompt and completion are not both strings or both message lists")column, value = "prompt/completion", prompt + completion+ elif column is not None:+ value = row[column]elif "instruction" in row and "output" in row:instruction, extra, output = row["instruction"], row.get("input"), row["output"]extra = "" if extra is None else extra