Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Check a dataset for bad rows and over-length examples before training (#16818)

Nilayd06ee9d733bb
diff --git a/tests/test_check_dataset.py b/tests/test_check_dataset.py
new file mode 100644
index 00000000000..6f9a7f9b17d
--- /dev/null
+++ b/tests/test_check_dataset.py
@@ -0,0 +1,410 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+
+tokenizers = pytest.importorskip("tokenizers")
+from transformers import PreTrainedTokenizerFast, ProcessorMixin
+
+from unsloth.chat_templates import check_dataset, get_chat_template
+
+TEMPLATE = (
+ "{% for message in messages %}"
+ "{% if message['role'] not in ['user', 'assistant'] %}{{ raise_exception('bad role') }}{% endif %}"
+ "{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>\n' }}"
+ "{% endfor %}"
+)
+MARKER = "<|im_start|>assistant\n"
+
+
+def _tokenizer(template = TEMPLATE):
+ from tokenizers import Tokenizer, models, pre_tokenizers
+
+ words = ["<|im_start|>", "<|im_end|>", "hi", "hello", "there"] + [f"w{i}" for i in range(40)]
+ tk = Tokenizer(
+ models.WordLevel({"[UNK]": 0, **{w: i + 1 for i, w in enumerate(words)}}, unk_token = "[UNK]")
+ )
+ tk.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
+ tok = PreTrainedTokenizerFast(tokenizer_object = tk, unk_token = "[UNK]", eos_token = "<|im_end|>")
+ tok.chat_template = template
+ tok.name_or_path = "tiny/chat-tokenizer"
+ return tok
+
+
+def _chat(user, assistant):
+ return {
+ "messages": [{"role": "user", "content": user}, {"role": "assistant", "content": assistant}]
+ }
+
+
+LONG = " ".join(f"w{i}" for i in range(30))
+ROWS = [
+ _chat("hi", "hello"),
+ _chat("hi there", "hello there"),
+ _chat("hi", "hello"),
+ _chat("hi", LONG),
+ _chat("hi there", LONG),
+ _chat(" ", ""),
+ "not a row",
+ {"prompt": "hi"},
+ {"messages": "hi hello"},
+ {"messages": [{"role": "user"}]},
+ {"messages": [{"role": "tool", "content": "hi"}]},
+]
+
+
+def test_counts_malformed_empty_and_over_length_rows(capsys):
+ report = check_dataset(ROWS, _tokenizer(), max_seq_length = 8)
+
+ assert report["rows"] == 11
+ assert report["malformed"] == 5
+ assert report["empty"] == 1
+ assert report["over_max_seq_length"] == 2
+ assert report["max_seq_length"] == 8
+ assert sorted(report["lengths"]) == [4, 6, 6, 8, 35, 36]
+ assert report["response_marker_found"] == 6
+
+ out = capsys.readouterr().out
+ assert "Rows: 11 | Malformed: 5 | Empty: 1" in out
+ assert "row 6: row is not a dict" in out
+ assert (
+ "row 7: no messages, conversations, text, prompt/completion or instruction/output column"
+ in out
+ )
+ assert "row 10: chat template failed: bad role" in out
+ assert "Token lengths: min 4 | median 7 | p95 35 | max 36" in out
+ assert "Over max_seq_length (8): 2 (18.2%)" in out
+ assert (
+ "Template: tiny/chat-tokenizer (tokenizer's own) | Response marker found in: 6 / 11" in out
+ )
+
+
+def test_hf_dataset_and_conversations_column():
+ datasets = pytest.importorskip("datasets")
+ rows = [{"conversations": r["messages"]} for r in ROWS[:6]]
+ report = check_dataset(datasets.Dataset.from_list(rows), _tokenizer(), max_seq_length = 8)
+
+ assert report["rows"] == 6
+ assert report["malformed"] == 0
+ assert report["empty"] == 1
+ assert report["over_max_seq_length"] == 2
+ assert report["response_marker_found"] == 6
+
+
+def test_text_column_with_explicit_marker():
+ tok = _tokenizer(template = None)
+ rows = [
+ {"text": "hi there ### Response: hello"},
+ {"text": "hi ### Response: " + LONG},
+ {"text": "hi hello"},
+ {"text": " "},
+ {"text": 42},
+ ]
+ report = check_dataset(rows, tok, max_seq_length = 8, response_part = "### Response:")
+
+ assert report["malformed"] == 1
+ assert report["empty"] == 1
+ assert report["over_max_seq_length"] == 1
+ assert report["response_marker_found"] == 2
+ assert sorted(report["lengths"]) == [0, 2, 5, 33]
+
+
+def test_no_chat_template_makes_message_rows_malformed(capsys):
+ report = check_dataset(ROWS[:2], _tokenizer(template = None), max_seq_length = 8)
+
+ assert report["malformed"] == 2
+ assert "Template: none" in capsys.readouterr().out
+
+
+def test_get_chat_template_names_the_template(capsys):
+ tok = get_chat_template(
+ _tokenizer(), chat_template = "chatml", map_eos_token = False, patch_saving = False
+ )
+ report = check_dataset(ROWS[:3], tok, max_seq_length = 8)
+
+ assert report["template"] == "chatml"
+ assert report["response_marker_found"] == 3
+ assert "Template: chatml | Response marker found in: 3 / 3" in capsys.readouterr().out
+
+
+def test_sharegpt_from_value_rows_are_not_malformed():
+ expected = check_dataset([_chat("hi", "hello")], _tokenizer(), max_seq_length = 8)["lengths"]
+ rows = [
+ {"conversations": [{"from": "human", "value": "hi"}, {"from": "gpt", "value": "hello"}]},
+ {"conversations": [{"from": "human", "value": " "}, {"from": "gpt", "value": ""}]},
+ ]
+ report = check_dataset(rows, _tokenizer(), max_seq_length = 8)
+ assert report["malformed"] == 0
+ assert report["empty"] == 1
+ assert report["lengths"][:1] == expected
+
+ tok = get_chat_template(
+ _tokenizer(),
+ chat_template = "chatml",
+ mapping = {"role": "from", "content": "value", "user": "human", "assistant": "gpt"},
+ map_eos_token = False,
+ patch_saving = False,
+ )
+ report = check_dataset(rows, tok, max_seq_length = 8)
+ assert report["malformed"] == 0
+ assert report["empty"] == 1
+
+
+def test_row_tools_are_rendered_into_length():
+ import json
+
+ template = (
+ "{% if tools %}{% for t in tools %}{{ t['name'] }} {% endfor %}{% endif %}" + TEMPLATE
+ )
+ row = _chat("hi", "hello")
+ plain = check_dataset([row], _tokenizer(template), max_seq_length = 8)["lengths"][0]
+ tools = [{"name": f"w{i}"} for i in range(10)]
+ as_list = check_dataset([{**row, "tools": tools}], _tokenizer(template), max_seq_length = 8)
+ as_str = check_dataset(
+ [{**row, "tools": json.dumps(tools)}], _tokenizer(template), max_seq_length = 8
+ )
+ assert as_list["lengths"][0] == as_str["lengths"][0] == plain + 10
+ assert as_list["over_max_seq_length"] == 1
+
+
+class _ImagesFirstProcessor(ProcessorMixin):
+ def __init__(self, tokenizer):
+ self.tokenizer = tokenizer
+ self.chat_template = tokenizer.chat_template
+
+ def apply_chat_template(self, *args, **kwargs):
+ return self.tokenizer.apply_chat_template(*args, **kwargs)
+
+ def __call__(
+ self,
+ images = None,
+ text = None,
+ **kwargs,
+ ):
+ if images is not None:
+ raise ValueError("Incorrect image source")
+ return self.tokenizer(text, **kwargs)
+
+
+def test_vision_processor_measures_with_inner_tokenizer():
+ tok = _tokenizer()
+ rows = ROWS[:5]
+ report = check_dataset(rows, _ImagesFirstProcessor(tok), max_seq_length = 8)
+ assert report["lengths"] == check_dataset(rows, tok, max_seq_length = 8)["lengths"]
+
+
+def test_tokenizer_with_backend_attribute_is_not_unwrapped():
+ tok = _tokenizer()
+ expected = check_dataset(ROWS[:5], tok, max_seq_length = 8)["lengths"]
+ tok.tokenizer = object()
+ assert check_dataset(ROWS[:5], tok, max_seq_length = 8)["lengths"] == expected
+
+
+def _lengths(rows, tok):
+ return check_dataset(rows, tok, max_seq_length = 8)["lengths"]
+
+
+def test_alpaca_rows():
+ tok = _tokenizer()
+ rows = [
+ {"instruction": "hi", "input": "there", "output": "hello"},
+ {"instruction": "hi", "input": "", "output": LONG},
+ {"instruction": "hi", "output": "hello"},
+ ]
+ report = check_dataset(rows, tok, max_seq_length = 8)
+
+ assert report["malformed"] == 0
+ assert report["over_max_seq_length"] == 1
+ assert report["response_marker_found"] == 3
+ assert report["lengths"] == _lengths(
+ [_chat("hi\n\nthere", "hello"), _chat("hi", LONG), _chat("hi", "hello")], tok
+ )
+
+
+def test_prompt_completion_rows():
+ tok = _tokenizer()
+ rows = [
+ {"prompt": "hi there", "completion": " hello"},
+ {"prompt": "hi", "completion": " " + LONG},
+ {
+ "prompt": [{"role": "user", "content": "hi"}],
+ "completion": [{"role": "assistant", "content": "hello"}],
+ },
+ ]
+ report = check_dataset(rows, tok, max_seq_length = 8)
+
+ assert report["malformed"] == 0
+ assert report["over_max_seq_length"] == 1
+ assert report["response_marker_found"] == 1
+ assert report["lengths"] == [3, 31] + _lengths([_chat("hi", "hello")], tok)
+
+
+def test_unusable_prompt_completion_and_alpaca_rows_stay_malformed(capsys):
+ rows = [
+ {"prompt": "hi", "completion": [{"role": "assistant", "content": "hello"}]},
+ {"instruction": "hi", "output": 42},
+ {"instruction": "hi", "input": [], "output": "hello"},
+ ]
+ report = check_dataset(rows, _tokenizer(), max_seq_length = 8)
+
+ assert report["malformed"] == 3
+ assert report["lengths"] == []
+ out = capsys.readouterr().out
+ assert "row 0: prompt and completion are not both strings or both message lists" in out
+ assert "row 1: instruction, input or output is not a string" in out
+
+
+def test_median_of_an_even_number_of_rows(capsys):
+ check_dataset([{"text": "hi"}, {"text": "hi hello"}], _tokenizer(), max_seq_length = 8)
+
+ assert "Token lengths: min 1 | median 1.5 | p95 1 | max 2" in capsys.readouterr().out
+
+
+class TokenizerWrapper:
+ # Same forwarding as mlx_lm.tokenizer_utils.TokenizerWrapper, which defines no __call__.
+ def __init__(self, tok):
+ self._tokenizer = tok
+
+ def __getattr__(self, attr):
+ if attr.startswith("_"):
+ return self.__getattribute__(attr)
+ return getattr(self._tokenizer, attr)
+
+
+def test_mlx_tokenizer_wrapper_is_unwrapped():
+ tok = get_chat_template(
+ _tokenizer(), chat_template = "chatml", map_eos_token = False, patch_saving = False
+ )
+ report = check_dataset(ROWS[:3], TokenizerWrapper(tok), max_seq_length = 8)
+ assert report["lengths"] == _lengths(ROWS[:3], tok)
+ assert report["template"] == "chatml"
+
+
+def test_replaced_template_is_not_reported_by_old_name():
+ tok = get_chat_template(
+ _tokenizer(), chat_template = "chatml", map_eos_token = False, patch_saving = False
+ )
+ tok.chat_template = TEMPLATE
+ assert check_dataset(ROWS[:1], tok, max_seq_length = 8)["template"] != "chatml"
+
+
+def test_preformatted_text_gets_no_second_bos():
+ from tokenizers import processors
+
+ tok = _tokenizer(template = None)
+ tok.add_special_tokens({"bos_token": "<s>"})
+ tok._tokenizer.post_processor = processors.TemplateProcessing(
+ single = "<s> $A", special_tokens = [("<s>", tok.convert_tokens_to_ids("<s>"))]
+ )
+ assert _lengths([{"text": "<s> hi hello"}, {"text": "hi hello"}], tok) == [3, 3]
+
+
+def test_media_rows_are_counted_but_not_measured(capsys):
+ template = (
+ "{% for m in messages %}{% if m['content'] is string %}{{ m['content'] }}"
+ "{% else %}{% for c in m['content'] %}{{ c.get('text', '<image>') }} {% endfor %}"
+ "{% endif %}{% endfor %}"
+ )
+ image_row = {
+ "messages": [
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "hi"}]},
+ {"role": "assistant", "content": [{"type": "text", "text": "hello"}]},
+ ]
+ }
+ tok = _tokenizer(template)
+ report = check_dataset([image_row, _chat("hi", "hello")], tok, max_seq_length = 8)
+ assert report["media"] == 1
+ assert report["lengths"] == _lengths([_chat("hi", "hello")], tok)
+ assert "Not measured: 1 rows with images, video or audio" in capsys.readouterr().out
+
+
+def test_text_column_wins_over_conversations():
+ rows = [{"conversations": _chat("hi", "hello")["messages"], "text": LONG}]
+ assert _lengths(rows, _tokenizer()) == _lengths([{"text": LONG}], _tokenizer())
+
+
+def test_assistant_tool_call_turn_without_content():
+ template = (
+ "{% for m in messages %}{{ m['role'] }} "
+ "{% if m.get('tool_calls') %}{{ m['tool_calls'][0]['name'] }}{% else %}{{ m['content'] }}{% endif %}"
+ " {% endfor %}"
+ )
+ row = {
+ "messages": [
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "tool_calls": [{"name": "w1"}]},
+ ]
+ }
+ report = check_dataset([row], _tokenizer(template), max_seq_length = 8)
+ assert report["malformed"] == 0
+ assert report["empty"] == 0
+
+
+def test_sharegpt_bad_role_is_malformed_and_extra_fields_render():
+ template = "{% for m in messages %}{{ m['content'] }} {{ m.get('thinking', '') }} {% endfor %}"
+ rows = [
+ {"conversations": [{"from": ["human"], "value": "hi"}]},
+ {"conversations": [{"from": "gpt", "value": "hi", "thinking": LONG}]},
+ ]
+ report = check_dataset(rows, _tokenizer(template), max_seq_length = 8)
+ assert report["rows"] == 2
+ assert report["over_max_seq_length"] == 1
+
+
+def test_plain_text_rows_with_images_are_not_measured():
+ rows = [
+ {"text": "hi hello", "images": ["a.png"]},
+ {"prompt": "hi", "completion": " hello", "image": "a.png"},
+ {"text": "hi hello"},
+ ]
+ report = check_dataset(rows, _tokenizer(), max_seq_length = 8)
+ assert report["media"] == 2
+ assert report["lengths"] == [2]
+
+
+def test_lengths_keep_row_order_across_batches():
+ rows = [{"text": " ".join(["hi"] * (i % 7 + 1))} for i in range(2500)]
+ rows[1500] = {"messages": _chat("hi", "hello")["messages"]}
+ tok = _tokenizer()
+ expected = [i % 7 + 1 for i in range(2500)]
+ expected[1500] = _lengths(rows[1500:1501], tok)[0]
+ assert check_dataset(rows, tok, max_seq_length = 8)["lengths"] == expected
+
+
+def test_structured_blank_content_is_empty_and_sharegpt_media_is_flagged():
+ template = "{% for m in messages %}{{ m['role'] }} {% endfor %}"
+ blank = {
+ "messages": [
+ {"role": "user", "content": [{"type": "text", "text": " "}]},
+ {"role": "assistant", "content": [{"type": "text", "text": ""}]},
+ ]
+ }
+ image = {"messages": [{"role": "user", "content": [{"type": "image"}]}]}
+ report = check_dataset([blank, image], _tokenizer(template), max_seq_length = 8)
+ assert report["empty"] == 1
+ assert report["media"] == 1
+
+ tok = get_chat_template(
+ _tokenizer(),
+ chat_template = "chatml",
+ mapping = {"role": "from", "content": "value", "user": "human", "assistant": "gpt"},
+ map_eos_token = False,
+ patch_saving = False,
+ )
+ tok.chat_template = tok.chat_template.replace("message['content']", "''")
+ row = {"conversations": [{"from": "human", "value": [{"type": "image"}]}]}
+ assert check_dataset([row], tok, max_seq_length = 8)["media"] == 1
+
+
+def test_array_media_fields_and_openai_media_parts():
+ np = pytest.importorskip("numpy")
+ template = "{% for m in messages %}{{ m['role'] }} {% endfor %}"
+ rows = [
+ {"text": "hi", "image": np.zeros((2, 2))},
+ {"text": "hi", "image": np.zeros((0,))},
+ {"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": "a"}]}]},
+ {"messages": [{"role": "user", "content": [{"type": "input_audio"}]}]},
+ ]
+ report = check_dataset(rows, _tokenizer(template), max_seq_length = 8)
+ assert report["rows"] == 4
+ assert report["media"] == 3
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index dada94b19b6..4e4f2646a3f 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -1680,6 +1680,10 @@ if _IS_MLX:
)
return _train_on_responses_only(*args, **kwargs)
+ def check_dataset(*args, **kwargs):
+ from .chat_templates import check_dataset as _check_dataset
+ return _check_dataset(*args, **kwargs)
+
def PatchDPOTrainer():
"""No-op on MLX: trl's preference trainers already resolve to the MLX ones."""
diff --git a/unsloth/_cpu_init.py b/unsloth/_cpu_init.py
index e55162fb4ee..238a65830d9 100644
--- a/unsloth/_cpu_init.py
+++ b/unsloth/_cpu_init.py
@@ -97,6 +97,11 @@ def train_on_responses_only(*args, **kwargs):
return _train_on_responses_only(*args, **kwargs)
+def check_dataset(*args, **kwargs):
+ from .chat_templates import check_dataset as _check_dataset
+ return _check_dataset(*args, **kwargs)
+
+
try:
from trl import SFTConfig as _SFTConfig, SFTTrainer as _SFTTrainer
except ImportError:
@@ -333,6 +338,7 @@ __all__ = [
"standardize_data_formats",
"standardize_sharegpt",
"train_on_responses_only",
+ "check_dataset",
"UnslothTrainer",
"UnslothTrainingArguments",
"UnslothVisionDataCollator",
diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index 6dca6642994..dd899e1a655 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -20,6 +20,7 @@ __all__ = [
"standardize_data_formats",
"apply_chat_template",
"train_on_responses_only",
+ "check_dataset",
"test_construct_chat_template",
]
@@ -32,6 +33,7 @@ try:
except ImportError:
LongTensor = FloatTensor = None
logger = logging.get_logger(__name__)
+import json
import os
import shutil
import re
@@ -84,6 +86,196 @@ else:
trainer = arguments.pop("trainer", None)
return _zoo_train_on_responses_only(trainer, **arguments)
+# TRL trains on "messages" when present, else on "text"; "conversations" is only read without "text".
+_DATASET_CHECK_COLUMNS = ("messages", "text", "conversations")
+_MEDIA_TYPES = ("image", "image_url", "video", "audio", "input_audio")
+
+
+class _MalformedRow(Exception):
+ pass
+
+
+def _is_present(value):
+ if value is None:
+ return False
+ try:
+ return len(value) > 0 # never bool(): numpy / torch payloads raise on truth tests
+ except TypeError:
+ return True
+
+
+def _has_media(row, messages):
+ if any(_is_present(row.get(k)) for k in ("images", "image", "videos", "video", "audios", "audio")):
+ return True
+ contents = (m.get("content", m.get("value")) for m in messages)
+ return any(
+ isinstance(part, dict) and part.get("type") in _MEDIA_TYPES
+ for c in contents
+ if isinstance(c, list)
+ for part in c
+ )
+
+
+def _is_blank(content):
+ if isinstance(content, str):
+ return content.strip() == ""
+ return isinstance(content, list) and all(
+ isinstance(part, dict) and part.get("type") == "text" and not str(part.get("text") or "").strip()
+ for part in content
+ )
+
+
+def _render_dataset_row(row, tokenizer):
+ if not isinstance(row, dict):
+ raise _MalformedRow("row is not a dict")
+ column = next((c for c in _DATASET_CHECK_COLUMNS if c in row), None)
+ if column is not None:
+ value = row[column]
+ elif "prompt" in row and "completion" in row:
+ prompt, completion = row["prompt"], row["completion"]
+ if isinstance(prompt, str) and isinstance(completion, str):
+ text = prompt + completion
+ return text, text.strip() == "", True, _has_media(row, ())
+ if not isinstance(prompt, list) or not isinstance(completion, list):
+ raise _MalformedRow("prompt and completion are not both strings or both message lists")
+ column, value = "prompt/completion", prompt + completion
+ elif "instruction" in row and "output" in row:
+ instruction, extra, output = row["instruction"], row.get("input"), row["output"]
+ extra = "" if extra is None else extra
+ if not all(isinstance(x, str) for x in (instruction, extra, output)):
+ raise _MalformedRow("instruction, input or output is not a string")
+ user = f"{instruction}\n\n{extra}".strip() if extra.strip() else instruction
+ column = "instruction/output"
+ value = [{"role": "user", "content": user}, {"role": "assistant", "content": output}]
+ else:
+ raise _MalformedRow(
+ "no messages, conversations, text, prompt/completion or instruction/output column"
+ )
+ if column == "text":
+ if not isinstance(value, str):
+ raise _MalformedRow("text is not a string")
+ return value, value.strip() == "", True, _has_media(row, ())
+ if not isinstance(value, list) or not value or not all(isinstance(m, dict) for m in value):
+ raise _MalformedRow(f"{column} is not a list of role/content messages")
+ sharegpt = all("from" in m and "value" in m for m in value)
+ if not sharegpt and not all("role" in m and ("content" in m or "tool_calls" in m) for m in value):
+ raise _MalformedRow(f"{column} is not a list of role/content messages")
+ tools = row.get("tools")
+ try:
+ # get_chat_template(mapping = ...) templates read ShareGPT keys themselves.
+ if sharegpt and "'role' not in messages[0]" not in (tokenizer.chat_template or ""):
+ roles = {"human": "user", "gpt": "assistant"}
+ value = [
+ dict(m, role = roles.get(m["from"], m["from"]), content = m["value"]) for m in value
+ ]
+ tools = json.loads(tools) if isinstance(tools, str) else tools
+ text = tokenizer.apply_chat_template(
+ value, tools = tools or None, tokenize = False, add_generation_prompt = False
+ )
+ except Exception as e:
+ raise _MalformedRow(f"chat template failed: {e}") from e
+ contents = (m["value"] if sharegpt else m.get("content") for m in value)
+ empty = all(_is_blank(c) for c in contents)
+ return text, empty, False, _has_media(row, value)
+
+
+def check_dataset(dataset, tokenizer, max_seq_length, response_part = None):
+ # mlx-lm's TokenizerWrapper is not callable; Unsloth's attributes live on the HF tokenizer.
+ if tokenizer.__class__.__name__ == "TokenizerWrapper" and hasattr(tokenizer, "_tokenizer"):
+ tokenizer = tokenizer._tokenizer
+ chat_template = getattr(tokenizer, "chat_template", None)
+ if response_part is None:
+ response_part = getattr(tokenizer, "_unsloth_output_part", None)
+ if response_part is None and chat_template is not None:
+ try:
+ from unsloth_zoo.dataset_utils import get_chat_template_parts
+ response_part = get_chat_template_parts(tokenizer)[1]
+ except Exception:
+ pass
+
+ text_tokenizer = tokenizer.tokenizer if isinstance(tokenizer, ProcessorMixin) else tokenizer
+ bos = getattr(text_tokenizer, "bos_token", None)
+ bos_in_template = bos is not None and bos in (chat_template or "")
+ rows = malformed = empty = found = media = 0
+ problems = []
+ pending = {True: [], False: []}
+ measured = []
+
+ # Flush as we go: holding every rendered row would not fit a large corpus.
+ def flush(add_special_tokens):
+ items = pending[add_special_tokens]
+ if items:
+ ids = text_tokenizer(
+ [text for _, text in items], add_special_tokens = add_special_tokens
+ ).input_ids
+ measured.extend((order, len(x)) for (order, _), x in zip(items, ids))
+ items.clear()
+
+ for i, row in enumerate(dataset):
+ rows += 1
+ try:
+ text, is_empty, is_plain_text, has_media = _render_dataset_row(row, tokenizer)
+ except _MalformedRow as e:
+ malformed += 1
+ if len(problems) < 5:
+ problems.append(f" row {i}: {e}")
+ continue
+ if is_empty:
+ empty += 1
+ if response_part is not None and response_part in text:
+ found += 1
+ if has_media:
+ media += 1
+ continue
+ # Same rule as the SFT dataset prep in models/rl_replacements.py: no second BOS.
+ add_special_tokens = is_plain_text and not (
+ bos is not None and (bos_in_template or text.startswith(bos))
+ )
+ pending[add_special_tokens].append((i, text))
+ if len(pending[add_special_tokens]) >= 1024:
+ flush(add_special_tokens)
+ flush(True)
+ flush(False)
+ lengths = [length for _, length in sorted(measured)]
+
+ over = sum(length > max_seq_length for length in lengths)
+ template = getattr(tokenizer, "_unsloth_chat_template_name", None)
+ template = template[0] if template and template[1] == chat_template else None
+ if template is None:
+ if chat_template is None:
+ template = "none"
+ else:
+ template = f"{getattr(tokenizer, 'name_or_path', 'tokenizer')} (tokenizer's own)"
+ marker = f"{found:,} / {rows:,}" if response_part is not None else "none detected"
+
+ print("Unsloth: Dataset check")
+ print(f"Rows: {rows:,} | Malformed: {malformed:,} | Empty: {empty:,}")
+ for problem in problems:
+ print(problem)
+ if lengths:
+ s = sorted(lengths)
+ median = f"{(s[(len(s) - 1) // 2] + s[len(s) // 2]) / 2:,.1f}".removesuffix(".0")
+ print(
+ f"Token lengths: min {s[0]:,} | median {median} | "
+ f"p95 {s[int(0.95 * (len(s) - 1))]:,} | max {s[-1]:,}"
+ )
+ if media:
+ print(f"Not measured: {media:,} rows with images, video or audio (expanded at training time)")
+ print(f"Over max_seq_length ({max_seq_length:,}): {over:,} ({100 * over / max(rows, 1):.1f}%)")
+ print(f"Template: {template} | Response marker found in: {marker}")
+ return {
+ "rows": rows,
+ "malformed": malformed,
+ "empty": empty,
+ "media": media,
+ "lengths": lengths,
+ "max_seq_length": max_seq_length,
+ "over_max_seq_length": over,
+ "template": template,
+ "response_part": response_part,
+ "response_marker_found": found,
+ }
+
standardize_sharegpt = standardize_data_formats
CHAT_TEMPLATES = {}
DEFAULT_SYSTEM_MESSAGE = {}
@@ -2513,6 +2705,7 @@ def get_chat_template(
same_padding_token = False
type_chat_template = None
+ template_name = None
if type(chat_template) in (list, tuple,):
# For changing system message later Since it's not supported yet, we will raise an error first!
@@ -2525,6 +2718,7 @@ def get_chat_template(
elif type(chat_template) is str:
# For changing system message later
type_chat_template = chat_template.lower()
+ template_name = chat_template
chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template]
@@ -2743,6 +2937,9 @@ def get_chat_template(
tokenizer._ollama_modelfile = ollama_modelfile
tokenizer._system_message = system_message
+ tokenizer._unsloth_chat_template_name = (
+ (template_name, chat_template) if template_name is not None else None
+ )
# Re-wrap so the trainer gets the same TokenizerWrapper type back.
if _mlx_tokenizer_wrapper is not None: