Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Check a dataset for bad rows and over-length examples before training (#16818)
Nilayd06ee9d733bb
diff --git a/tests/test_check_dataset.py b/tests/test_check_dataset.pynew file mode 100644index 00000000000..6f9a7f9b17d--- /dev/null+++ b/tests/test_check_dataset.py@@ -0,0 +1,410 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest++tokenizers = pytest.importorskip("tokenizers")+from transformers import PreTrainedTokenizerFast, ProcessorMixin++from unsloth.chat_templates import check_dataset, get_chat_template++TEMPLATE = (+ "{% for message in messages %}"+ "{% if message['role'] not in ['user', 'assistant'] %}{{ raise_exception('bad role') }}{% endif %}"+ "{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>\n' }}"+ "{% endfor %}"+)+MARKER = "<|im_start|>assistant\n"+++def _tokenizer(template = TEMPLATE):+ from tokenizers import Tokenizer, models, pre_tokenizers++ words = ["<|im_start|>", "<|im_end|>", "hi", "hello", "there"] + [f"w{i}" for i in range(40)]+ tk = Tokenizer(+ models.WordLevel({"[UNK]": 0, **{w: i + 1 for i, w in enumerate(words)}}, unk_token = "[UNK]")+ )+ tk.pre_tokenizer = pre_tokenizers.WhitespaceSplit()+ tok = PreTrainedTokenizerFast(tokenizer_object = tk, unk_token = "[UNK]", eos_token = "<|im_end|>")+ tok.chat_template = template+ tok.name_or_path = "tiny/chat-tokenizer"+ return tok+++def _chat(user, assistant):+ return {+ "messages": [{"role": "user", "content": user}, {"role": "assistant", "content": assistant}]+ }+++LONG = " ".join(f"w{i}" for i in range(30))+ROWS = [+ _chat("hi", "hello"),+ _chat("hi there", "hello there"),+ _chat("hi", "hello"),+ _chat("hi", LONG),+ _chat("hi there", LONG),+ _chat(" ", ""),+ "not a row",+ {"prompt": "hi"},+ {"messages": "hi hello"},+ {"messages": [{"role": "user"}]},+ {"messages": [{"role": "tool", "content": "hi"}]},+]+++def test_counts_malformed_empty_and_over_length_rows(capsys):+ report = check_dataset(ROWS, _tokenizer(), max_seq_length = 8)++ assert report["rows"] == 11+ assert report["malformed"] == 5+ assert report["empty"] == 1+ assert report["over_max_seq_length"] == 2+ assert report["max_seq_length"] == 8+ assert sorted(report["lengths"]) == [4, 6, 6, 8, 35, 36]+ assert report["response_marker_found"] == 6++ out = capsys.readouterr().out+ assert "Rows: 11 | Malformed: 5 | Empty: 1" in out+ assert "row 6: row is not a dict" in out+ assert (+ "row 7: no messages, conversations, text, prompt/completion or instruction/output column"+ in out+ )+ assert "row 10: chat template failed: bad role" in out+ assert "Token lengths: min 4 | median 7 | p95 35 | max 36" in out+ assert "Over max_seq_length (8): 2 (18.2%)" in out+ assert (+ "Template: tiny/chat-tokenizer (tokenizer's own) | Response marker found in: 6 / 11" in out+ )+++def test_hf_dataset_and_conversations_column():+ datasets = pytest.importorskip("datasets")+ rows = [{"conversations": r["messages"]} for r in ROWS[:6]]+ report = check_dataset(datasets.Dataset.from_list(rows), _tokenizer(), max_seq_length = 8)++ assert report["rows"] == 6+ assert report["malformed"] == 0+ assert report["empty"] == 1+ assert report["over_max_seq_length"] == 2+ assert report["response_marker_found"] == 6+++def test_text_column_with_explicit_marker():+ tok = _tokenizer(template = None)+ rows = [+ {"text": "hi there ### Response: hello"},+ {"text": "hi ### Response: " + LONG},+ {"text": "hi hello"},+ {"text": " "},+ {"text": 42},+ ]+ report = check_dataset(rows, tok, max_seq_length = 8, response_part = "### Response:")++ assert report["malformed"] == 1+ assert report["empty"] == 1+ assert report["over_max_seq_length"] == 1+ assert report["response_marker_found"] == 2+ assert sorted(report["lengths"]) == [0, 2, 5, 33]+++def test_no_chat_template_makes_message_rows_malformed(capsys):+ report = check_dataset(ROWS[:2], _tokenizer(template = None), max_seq_length = 8)++ assert report["malformed"] == 2+ assert "Template: none" in capsys.readouterr().out+++def test_get_chat_template_names_the_template(capsys):+ tok = get_chat_template(+ _tokenizer(), chat_template = "chatml", map_eos_token = False, patch_saving = False+ )+ report = check_dataset(ROWS[:3], tok, max_seq_length = 8)++ assert report["template"] == "chatml"+ assert report["response_marker_found"] == 3+ assert "Template: chatml | Response marker found in: 3 / 3" in capsys.readouterr().out+++def test_sharegpt_from_value_rows_are_not_malformed():+ expected = check_dataset([_chat("hi", "hello")], _tokenizer(), max_seq_length = 8)["lengths"]+ rows = [+ {"conversations": [{"from": "human", "value": "hi"}, {"from": "gpt", "value": "hello"}]},+ {"conversations": [{"from": "human", "value": " "}, {"from": "gpt", "value": ""}]},+ ]+ report = check_dataset(rows, _tokenizer(), max_seq_length = 8)+ assert report["malformed"] == 0+ assert report["empty"] == 1+ assert report["lengths"][:1] == expected++ tok = get_chat_template(+ _tokenizer(),+ chat_template = "chatml",+ mapping = {"role": "from", "content": "value", "user": "human", "assistant": "gpt"},+ map_eos_token = False,+ patch_saving = False,+ )+ report = check_dataset(rows, tok, max_seq_length = 8)+ assert report["malformed"] == 0+ assert report["empty"] == 1+++def test_row_tools_are_rendered_into_length():+ import json++ template = (+ "{% if tools %}{% for t in tools %}{{ t['name'] }} {% endfor %}{% endif %}" + TEMPLATE+ )+ row = _chat("hi", "hello")+ plain = check_dataset([row], _tokenizer(template), max_seq_length = 8)["lengths"][0]+ tools = [{"name": f"w{i}"} for i in range(10)]+ as_list = check_dataset([{**row, "tools": tools}], _tokenizer(template), max_seq_length = 8)+ as_str = check_dataset(+ [{**row, "tools": json.dumps(tools)}], _tokenizer(template), max_seq_length = 8+ )+ assert as_list["lengths"][0] == as_str["lengths"][0] == plain + 10+ assert as_list["over_max_seq_length"] == 1+++class _ImagesFirstProcessor(ProcessorMixin):+ def __init__(self, tokenizer):+ self.tokenizer = tokenizer+ self.chat_template = tokenizer.chat_template++ def apply_chat_template(self, *args, **kwargs):+ return self.tokenizer.apply_chat_template(*args, **kwargs)++ def __call__(+ self,+ images = None,+ text = None,+ **kwargs,+ ):+ if images is not None:+ raise ValueError("Incorrect image source")+ return self.tokenizer(text, **kwargs)+++def test_vision_processor_measures_with_inner_tokenizer():+ tok = _tokenizer()+ rows = ROWS[:5]+ report = check_dataset(rows, _ImagesFirstProcessor(tok), max_seq_length = 8)+ assert report["lengths"] == check_dataset(rows, tok, max_seq_length = 8)["lengths"]+++def test_tokenizer_with_backend_attribute_is_not_unwrapped():+ tok = _tokenizer()+ expected = check_dataset(ROWS[:5], tok, max_seq_length = 8)["lengths"]+ tok.tokenizer = object()+ assert check_dataset(ROWS[:5], tok, max_seq_length = 8)["lengths"] == expected+++def _lengths(rows, tok):+ return check_dataset(rows, tok, max_seq_length = 8)["lengths"]+++def test_alpaca_rows():+ tok = _tokenizer()+ rows = [+ {"instruction": "hi", "input": "there", "output": "hello"},+ {"instruction": "hi", "input": "", "output": LONG},+ {"instruction": "hi", "output": "hello"},+ ]+ report = check_dataset(rows, tok, max_seq_length = 8)++ assert report["malformed"] == 0+ assert report["over_max_seq_length"] == 1+ assert report["response_marker_found"] == 3+ assert report["lengths"] == _lengths(+ [_chat("hi\n\nthere", "hello"), _chat("hi", LONG), _chat("hi", "hello")], tok+ )+++def test_prompt_completion_rows():+ tok = _tokenizer()+ rows = [+ {"prompt": "hi there", "completion": " hello"},+ {"prompt": "hi", "completion": " " + LONG},+ {+ "prompt": [{"role": "user", "content": "hi"}],+ "completion": [{"role": "assistant", "content": "hello"}],+ },+ ]+ report = check_dataset(rows, tok, max_seq_length = 8)++ assert report["malformed"] == 0+ assert report["over_max_seq_length"] == 1+ assert report["response_marker_found"] == 1+ assert report["lengths"] == [3, 31] + _lengths([_chat("hi", "hello")], tok)+++def test_unusable_prompt_completion_and_alpaca_rows_stay_malformed(capsys):+ rows = [+ {"prompt": "hi", "completion": [{"role": "assistant", "content": "hello"}]},+ {"instruction": "hi", "output": 42},+ {"instruction": "hi", "input": [], "output": "hello"},+ ]+ report = check_dataset(rows, _tokenizer(), max_seq_length = 8)++ assert report["malformed"] == 3+ assert report["lengths"] == []+ out = capsys.readouterr().out+ assert "row 0: prompt and completion are not both strings or both message lists" in out+ assert "row 1: instruction, input or output is not a string" in out+++def test_median_of_an_even_number_of_rows(capsys):+ check_dataset([{"text": "hi"}, {"text": "hi hello"}], _tokenizer(), max_seq_length = 8)++ assert "Token lengths: min 1 | median 1.5 | p95 1 | max 2" in capsys.readouterr().out+++class TokenizerWrapper:+ # Same forwarding as mlx_lm.tokenizer_utils.TokenizerWrapper, which defines no __call__.+ def __init__(self, tok):+ self._tokenizer = tok++ def __getattr__(self, attr):+ if attr.startswith("_"):+ return self.__getattribute__(attr)+ return getattr(self._tokenizer, attr)+++def test_mlx_tokenizer_wrapper_is_unwrapped():+ tok = get_chat_template(+ _tokenizer(), chat_template = "chatml", map_eos_token = False, patch_saving = False+ )+ report = check_dataset(ROWS[:3], TokenizerWrapper(tok), max_seq_length = 8)+ assert report["lengths"] == _lengths(ROWS[:3], tok)+ assert report["template"] == "chatml"+++def test_replaced_template_is_not_reported_by_old_name():+ tok = get_chat_template(+ _tokenizer(), chat_template = "chatml", map_eos_token = False, patch_saving = False+ )+ tok.chat_template = TEMPLATE+ assert check_dataset(ROWS[:1], tok, max_seq_length = 8)["template"] != "chatml"+++def test_preformatted_text_gets_no_second_bos():+ from tokenizers import processors++ tok = _tokenizer(template = None)+ tok.add_special_tokens({"bos_token": "<s>"})+ tok._tokenizer.post_processor = processors.TemplateProcessing(+ single = "<s> $A", special_tokens = [("<s>", tok.convert_tokens_to_ids("<s>"))]+ )+ assert _lengths([{"text": "<s> hi hello"}, {"text": "hi hello"}], tok) == [3, 3]+++def test_media_rows_are_counted_but_not_measured(capsys):+ template = (+ "{% for m in messages %}{% if m['content'] is string %}{{ m['content'] }}"+ "{% else %}{% for c in m['content'] %}{{ c.get('text', '<image>') }} {% endfor %}"+ "{% endif %}{% endfor %}"+ )+ image_row = {+ "messages": [+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "hi"}]},+ {"role": "assistant", "content": [{"type": "text", "text": "hello"}]},+ ]+ }+ tok = _tokenizer(template)+ report = check_dataset([image_row, _chat("hi", "hello")], tok, max_seq_length = 8)+ assert report["media"] == 1+ assert report["lengths"] == _lengths([_chat("hi", "hello")], tok)+ assert "Not measured: 1 rows with images, video or audio" in capsys.readouterr().out+++def test_text_column_wins_over_conversations():+ rows = [{"conversations": _chat("hi", "hello")["messages"], "text": LONG}]+ assert _lengths(rows, _tokenizer()) == _lengths([{"text": LONG}], _tokenizer())+++def test_assistant_tool_call_turn_without_content():+ template = (+ "{% for m in messages %}{{ m['role'] }} "+ "{% if m.get('tool_calls') %}{{ m['tool_calls'][0]['name'] }}{% else %}{{ m['content'] }}{% endif %}"+ " {% endfor %}"+ )+ row = {+ "messages": [+ {"role": "user", "content": "hi"},+ {"role": "assistant", "tool_calls": [{"name": "w1"}]},+ ]+ }+ report = check_dataset([row], _tokenizer(template), max_seq_length = 8)+ assert report["malformed"] == 0+ assert report["empty"] == 0+++def test_sharegpt_bad_role_is_malformed_and_extra_fields_render():+ template = "{% for m in messages %}{{ m['content'] }} {{ m.get('thinking', '') }} {% endfor %}"+ rows = [+ {"conversations": [{"from": ["human"], "value": "hi"}]},+ {"conversations": [{"from": "gpt", "value": "hi", "thinking": LONG}]},+ ]+ report = check_dataset(rows, _tokenizer(template), max_seq_length = 8)+ assert report["rows"] == 2+ assert report["over_max_seq_length"] == 1+++def test_plain_text_rows_with_images_are_not_measured():+ rows = [+ {"text": "hi hello", "images": ["a.png"]},+ {"prompt": "hi", "completion": " hello", "image": "a.png"},+ {"text": "hi hello"},+ ]+ report = check_dataset(rows, _tokenizer(), max_seq_length = 8)+ assert report["media"] == 2+ assert report["lengths"] == [2]+++def test_lengths_keep_row_order_across_batches():+ rows = [{"text": " ".join(["hi"] * (i % 7 + 1))} for i in range(2500)]+ rows[1500] = {"messages": _chat("hi", "hello")["messages"]}+ tok = _tokenizer()+ expected = [i % 7 + 1 for i in range(2500)]+ expected[1500] = _lengths(rows[1500:1501], tok)[0]+ assert check_dataset(rows, tok, max_seq_length = 8)["lengths"] == expected+++def test_structured_blank_content_is_empty_and_sharegpt_media_is_flagged():+ template = "{% for m in messages %}{{ m['role'] }} {% endfor %}"+ blank = {+ "messages": [+ {"role": "user", "content": [{"type": "text", "text": " "}]},+ {"role": "assistant", "content": [{"type": "text", "text": ""}]},+ ]+ }+ image = {"messages": [{"role": "user", "content": [{"type": "image"}]}]}+ report = check_dataset([blank, image], _tokenizer(template), max_seq_length = 8)+ assert report["empty"] == 1+ assert report["media"] == 1++ tok = get_chat_template(+ _tokenizer(),+ chat_template = "chatml",+ mapping = {"role": "from", "content": "value", "user": "human", "assistant": "gpt"},+ map_eos_token = False,+ patch_saving = False,+ )+ tok.chat_template = tok.chat_template.replace("message['content']", "''")+ row = {"conversations": [{"from": "human", "value": [{"type": "image"}]}]}+ assert check_dataset([row], tok, max_seq_length = 8)["media"] == 1+++def test_array_media_fields_and_openai_media_parts():+ np = pytest.importorskip("numpy")+ template = "{% for m in messages %}{{ m['role'] }} {% endfor %}"+ rows = [+ {"text": "hi", "image": np.zeros((2, 2))},+ {"text": "hi", "image": np.zeros((0,))},+ {"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": "a"}]}]},+ {"messages": [{"role": "user", "content": [{"type": "input_audio"}]}]},+ ]+ report = check_dataset(rows, _tokenizer(template), max_seq_length = 8)+ assert report["rows"] == 4+ assert report["media"] == 3diff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex dada94b19b6..4e4f2646a3f 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -1680,6 +1680,10 @@ if _IS_MLX:)return _train_on_responses_only(*args, **kwargs)+ def check_dataset(*args, **kwargs):+ from .chat_templates import check_dataset as _check_dataset+ return _check_dataset(*args, **kwargs)+def PatchDPOTrainer():"""No-op on MLX: trl's preference trainers already resolve to the MLX ones."""diff --git a/unsloth/_cpu_init.py b/unsloth/_cpu_init.pyindex e55162fb4ee..238a65830d9 100644--- a/unsloth/_cpu_init.py+++ b/unsloth/_cpu_init.py@@ -97,6 +97,11 @@ def train_on_responses_only(*args, **kwargs):return _train_on_responses_only(*args, **kwargs)+def check_dataset(*args, **kwargs):+ from .chat_templates import check_dataset as _check_dataset+ return _check_dataset(*args, **kwargs)++try:from trl import SFTConfig as _SFTConfig, SFTTrainer as _SFTTrainerexcept ImportError:@@ -333,6 +338,7 @@ __all__ = ["standardize_data_formats","standardize_sharegpt","train_on_responses_only",+ "check_dataset","UnslothTrainer","UnslothTrainingArguments","UnslothVisionDataCollator",diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.pyindex 6dca6642994..dd899e1a655 100644--- a/unsloth/chat_templates.py+++ b/unsloth/chat_templates.py@@ -20,6 +20,7 @@ __all__ = ["standardize_data_formats","apply_chat_template","train_on_responses_only",+ "check_dataset","test_construct_chat_template",]@@ -32,6 +33,7 @@ try:except ImportError:LongTensor = FloatTensor = Nonelogger = logging.get_logger(__name__)+import jsonimport osimport shutilimport re@@ -84,6 +86,196 @@ else:trainer = arguments.pop("trainer", None)return _zoo_train_on_responses_only(trainer, **arguments)+# TRL trains on "messages" when present, else on "text"; "conversations" is only read without "text".+_DATASET_CHECK_COLUMNS = ("messages", "text", "conversations")+_MEDIA_TYPES = ("image", "image_url", "video", "audio", "input_audio")+++class _MalformedRow(Exception):+ pass+++def _is_present(value):+ if value is None:+ return False+ try:+ return len(value) > 0 # never bool(): numpy / torch payloads raise on truth tests+ except TypeError:+ return True+++def _has_media(row, messages):+ if any(_is_present(row.get(k)) for k in ("images", "image", "videos", "video", "audios", "audio")):+ return True+ contents = (m.get("content", m.get("value")) for m in messages)+ return any(+ isinstance(part, dict) and part.get("type") in _MEDIA_TYPES+ for c in contents+ if isinstance(c, list)+ for part in c+ )+++def _is_blank(content):+ if isinstance(content, str):+ return content.strip() == ""+ return isinstance(content, list) and all(+ isinstance(part, dict) and part.get("type") == "text" and not str(part.get("text") or "").strip()+ for part in content+ )+++def _render_dataset_row(row, tokenizer):+ if not isinstance(row, dict):+ raise _MalformedRow("row is not a dict")+ column = next((c for c in _DATASET_CHECK_COLUMNS if c in row), None)+ if column is not None:+ value = row[column]+ elif "prompt" in row and "completion" in row:+ prompt, completion = row["prompt"], row["completion"]+ if isinstance(prompt, str) and isinstance(completion, str):+ text = prompt + completion+ return text, text.strip() == "", True, _has_media(row, ())+ if not isinstance(prompt, list) or not isinstance(completion, list):+ raise _MalformedRow("prompt and completion are not both strings or both message lists")+ column, value = "prompt/completion", prompt + completion+ elif "instruction" in row and "output" in row:+ instruction, extra, output = row["instruction"], row.get("input"), row["output"]+ extra = "" if extra is None else extra+ if not all(isinstance(x, str) for x in (instruction, extra, output)):+ raise _MalformedRow("instruction, input or output is not a string")+ user = f"{instruction}\n\n{extra}".strip() if extra.strip() else instruction+ column = "instruction/output"+ value = [{"role": "user", "content": user}, {"role": "assistant", "content": output}]+ else:+ raise _MalformedRow(+ "no messages, conversations, text, prompt/completion or instruction/output column"+ )+ if column == "text":+ if not isinstance(value, str):+ raise _MalformedRow("text is not a string")+ return value, value.strip() == "", True, _has_media(row, ())+ if not isinstance(value, list) or not value or not all(isinstance(m, dict) for m in value):+ raise _MalformedRow(f"{column} is not a list of role/content messages")+ sharegpt = all("from" in m and "value" in m for m in value)+ if not sharegpt and not all("role" in m and ("content" in m or "tool_calls" in m) for m in value):+ raise _MalformedRow(f"{column} is not a list of role/content messages")+ tools = row.get("tools")+ try:+ # get_chat_template(mapping = ...) templates read ShareGPT keys themselves.+ if sharegpt and "'role' not in messages[0]" not in (tokenizer.chat_template or ""):+ roles = {"human": "user", "gpt": "assistant"}+ value = [+ dict(m, role = roles.get(m["from"], m["from"]), content = m["value"]) for m in value+ ]+ tools = json.loads(tools) if isinstance(tools, str) else tools+ text = tokenizer.apply_chat_template(+ value, tools = tools or None, tokenize = False, add_generation_prompt = False+ )+ except Exception as e:+ raise _MalformedRow(f"chat template failed: {e}") from e+ contents = (m["value"] if sharegpt else m.get("content") for m in value)+ empty = all(_is_blank(c) for c in contents)+ return text, empty, False, _has_media(row, value)+++def check_dataset(dataset, tokenizer, max_seq_length, response_part = None):+ # mlx-lm's TokenizerWrapper is not callable; Unsloth's attributes live on the HF tokenizer.+ if tokenizer.__class__.__name__ == "TokenizerWrapper" and hasattr(tokenizer, "_tokenizer"):+ tokenizer = tokenizer._tokenizer+ chat_template = getattr(tokenizer, "chat_template", None)+ if response_part is None:+ response_part = getattr(tokenizer, "_unsloth_output_part", None)+ if response_part is None and chat_template is not None:+ try:+ from unsloth_zoo.dataset_utils import get_chat_template_parts+ response_part = get_chat_template_parts(tokenizer)[1]+ except Exception:+ pass++ text_tokenizer = tokenizer.tokenizer if isinstance(tokenizer, ProcessorMixin) else tokenizer+ bos = getattr(text_tokenizer, "bos_token", None)+ bos_in_template = bos is not None and bos in (chat_template or "")+ rows = malformed = empty = found = media = 0+ problems = []+ pending = {True: [], False: []}+ measured = []++ # Flush as we go: holding every rendered row would not fit a large corpus.+ def flush(add_special_tokens):+ items = pending[add_special_tokens]+ if items:+ ids = text_tokenizer(+ [text for _, text in items], add_special_tokens = add_special_tokens+ ).input_ids+ measured.extend((order, len(x)) for (order, _), x in zip(items, ids))+ items.clear()++ for i, row in enumerate(dataset):+ rows += 1+ try:+ text, is_empty, is_plain_text, has_media = _render_dataset_row(row, tokenizer)+ except _MalformedRow as e:+ malformed += 1+ if len(problems) < 5:+ problems.append(f" row {i}: {e}")+ continue+ if is_empty:+ empty += 1+ if response_part is not None and response_part in text:+ found += 1+ if has_media:+ media += 1+ continue+ # Same rule as the SFT dataset prep in models/rl_replacements.py: no second BOS.+ add_special_tokens = is_plain_text and not (+ bos is not None and (bos_in_template or text.startswith(bos))+ )+ pending[add_special_tokens].append((i, text))+ if len(pending[add_special_tokens]) >= 1024:+ flush(add_special_tokens)+ flush(True)+ flush(False)+ lengths = [length for _, length in sorted(measured)]++ over = sum(length > max_seq_length for length in lengths)+ template = getattr(tokenizer, "_unsloth_chat_template_name", None)+ template = template[0] if template and template[1] == chat_template else None+ if template is None:+ if chat_template is None:+ template = "none"+ else:+ template = f"{getattr(tokenizer, 'name_or_path', 'tokenizer')} (tokenizer's own)"+ marker = f"{found:,} / {rows:,}" if response_part is not None else "none detected"++ print("Unsloth: Dataset check")+ print(f"Rows: {rows:,} | Malformed: {malformed:,} | Empty: {empty:,}")+ for problem in problems:+ print(problem)+ if lengths:+ s = sorted(lengths)+ median = f"{(s[(len(s) - 1) // 2] + s[len(s) // 2]) / 2:,.1f}".removesuffix(".0")+ print(+ f"Token lengths: min {s[0]:,} | median {median} | "+ f"p95 {s[int(0.95 * (len(s) - 1))]:,} | max {s[-1]:,}"+ )+ if media:+ print(f"Not measured: {media:,} rows with images, video or audio (expanded at training time)")+ print(f"Over max_seq_length ({max_seq_length:,}): {over:,} ({100 * over / max(rows, 1):.1f}%)")+ print(f"Template: {template} | Response marker found in: {marker}")+ return {+ "rows": rows,+ "malformed": malformed,+ "empty": empty,+ "media": media,+ "lengths": lengths,+ "max_seq_length": max_seq_length,+ "over_max_seq_length": over,+ "template": template,+ "response_part": response_part,+ "response_marker_found": found,+ }+standardize_sharegpt = standardize_data_formatsCHAT_TEMPLATES = {}DEFAULT_SYSTEM_MESSAGE = {}@@ -2513,6 +2705,7 @@ def get_chat_template(same_padding_token = Falsetype_chat_template = None+ template_name = Noneif type(chat_template) in (list, tuple,):# For changing system message later Since it's not supported yet, we will raise an error first!@@ -2525,6 +2718,7 @@ def get_chat_template(elif type(chat_template) is str:# For changing system message latertype_chat_template = chat_template.lower()+ template_name = chat_templatechat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template]@@ -2743,6 +2937,9 @@ def get_chat_template(tokenizer._ollama_modelfile = ollama_modelfiletokenizer._system_message = system_message+ tokenizer._unsloth_chat_template_name = (+ (template_name, chat_template) if template_name is not None else None+ )# Re-wrap so the trainer gets the same TokenizerWrapper type back.if _mlx_tokenizer_wrapper is not None: