Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: drop rows the chat template can't format and tell the user (#11293)
Nilay8265caeadc9b
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex 97e5f0d6fdb..6cb64cbd2e9 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -3279,6 +3279,9 @@ class UnslothTrainer:self._update_progress(error = error_msg)return None+ if dataset_info.get("dropped_rows_warning"):+ self._record_warning(dataset_info["dropped_rows_warning"])+detected = dataset_info.get("detected_format", "unknown")final_ds = dataset_info.get("dataset")final_n = len(final_ds) if hasattr(final_ds, "__len__") else "?"@@ -3301,6 +3304,14 @@ class UnslothTrainer:dataset_name = dataset_source,custom_format_mapping = custom_format_mapping,)+ if not eval_info.get("success", True):+ eval_errors = eval_info.get("errors", [])+ error_msg = "; ".join(eval_errors) or "Eval dataset formatting failed"+ logger.error(f"Eval dataset conversion failed: {error_msg}")+ self._update_progress(error = error_msg)+ return None+ if eval_info.get("dropped_rows_warning"):+ self._record_warning(f"Eval dataset: {eval_info['dropped_rows_warning']}")eval_dataset = eval_info["dataset"]logger.info("Eval dataset formatted successfully\n")elif eval_enabled and not has_separate_eval_source and not dataset_streaming:diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 2e4405c9ea1..d3721e4342a 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -2835,6 +2835,11 @@ def _run_mlx_training(event_queue, stop_queue, config):)if info.get("success", True):dataset = info.get("dataset", dataset)+ else:+ errors = info.get("errors", [])+ raise ValueError(f"Dataset format conversion failed: {'; '.join(errors)}")+ if info.get("dropped_rows_warning"):+ _send("warning", message = info["dropped_rows_warning"])dataset_final_format = str(info.get("final_format", "") or "").lower()if eval_dataset is not None:ev = format_and_template_dataset(@@ -2848,6 +2853,13 @@ def _run_mlx_training(event_queue, stop_queue, config):)if ev.get("success", True):eval_dataset = ev.get("dataset", eval_dataset)+ else:+ eval_errors = ev.get("errors", [])+ raise ValueError(+ f"Eval dataset format conversion failed: {'; '.join(eval_errors)}"+ )+ if ev.get("dropped_rows_warning"):+ _send("warning", message = f"Eval dataset: {ev['dropped_rows_warning']}")except ImportError:_send("status", status_message = "Format helper unavailable, using raw dataset")diff --git a/studio/backend/tests/test_chat_template_failed_rows.py b/studio/backend/tests/test_chat_template_failed_rows.pynew file mode 100644index 00000000000..37e5a647303--- /dev/null+++ b/studio/backend/tests/test_chat_template_failed_rows.py@@ -0,0 +1,193 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import pytest+from datasets import Dataset, IterableDataset++import utils.hardware+from utils.datasets import chat_templates+from utils.datasets import apply_chat_template_to_dataset, format_and_template_dataset+++class _StrictTokenizer:+ chat_template = "{{ messages }}"+ eos_token = "</s>"++ def apply_chat_template(self, conversation, **_kwargs):+ if any(turn["role"] == "system" for turn in conversation):+ raise ValueError("System role not supported")+ return "\n".join(f"{turn['role']}: {turn['content']}" for turn in conversation)+++def _convo(index, with_system = False):+ turns = [+ {"role": "user", "content": f"question {index}"},+ {"role": "assistant", "content": f"answer {index}"},+ ]+ if with_system:+ turns.insert(0, {"role": "system", "content": "be brief"})+ return turns+++def _dataset_info(dataset):+ return {+ "dataset": dataset,+ "detected_format": "chatml_messages",+ "final_format": "chatml_messages",+ "chat_column": "messages",+ "is_standardized": True,+ "warnings": [],+ }+++def _mixed_dataset():+ return Dataset.from_dict(+ {"messages": [_convo(i, with_system = i in (1, 4, 5)) for i in range(8)]}+ )++[email protected]("num_proc", [None, 2])+def test_rows_whose_template_raised_are_dropped_and_counted(monkeypatch, num_proc):+ monkeypatch.setattr(utils.hardware, "dataset_map_num_proc", lambda *_a, **_k: num_proc)++ result = apply_chat_template_to_dataset(_dataset_info(_mixed_dataset()), _StrictTokenizer())++ assert result["success"] is True+ formatted = result["dataset"]+ assert len(formatted) == 5+ assert [text.splitlines()[0] for text in formatted["text"]] == [+ f"user: question {i}" for i in (0, 2, 3, 6, 7)+ ]+ assert formatted.column_names == ["messages", "text"]+ assert result["dropped_rows_warning"] == (+ "Dropped 3 of 8 rows because the chat template failed: System role not supported"+ )+ assert result["dropped_rows_warning"] in result["warnings"]+++def test_clean_dataset_reports_no_dropped_rows():+ dataset = Dataset.from_dict({"messages": [_convo(i) for i in range(3)]})++ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())++ assert result["success"] is True+ assert len(result["dataset"]) == 3+ assert result["dataset"].column_names == ["messages", "text"]+ assert result.get("dropped_rows_warning") is None+++def test_every_row_failing_is_an_error():+ dataset = Dataset.from_dict({"messages": [_convo(i, with_system = True) for i in range(3)]})++ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())++ assert result["success"] is False+ assert result["errors"] == ["Chat template failed on all 3 rows: System role not supported"]+ assert result["dataset"] is dataset+++def test_streaming_rows_whose_template_raised_are_filtered():+ result = apply_chat_template_to_dataset(+ _dataset_info(_mixed_dataset().to_iterable_dataset()), _StrictTokenizer()+ )++ assert result["success"] is True+ rows = list(result["dataset"])+ assert len(rows) == 5+ assert all(set(row) == {"messages", "text"} and row["text"] for row in rows)+++def test_format_and_template_dataset_passes_the_dropped_rows_warning_through():+ result = format_and_template_dataset(+ _mixed_dataset(), model_name = "stub-model", tokenizer = _StrictTokenizer()+ )++ assert result["success"] is True+ assert len(result["dataset"]) == 5+ assert result["dropped_rows_warning"].startswith("Dropped 3 of 8 rows")+++def test_a_column_named_like_the_marker_is_not_clobbered():+ """A dataset may already carry a column named like the internal marker."""+ dataset = _mixed_dataset().add_column(+ "__chat_template_error", [f"user-data-{i}" for i in range(8)]+ )++ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())++ assert result["success"] is True+ formatted = result["dataset"]+ assert len(formatted) == 5+ assert sorted(formatted.column_names) == ["__chat_template_error", "messages", "text"]+ assert formatted["__chat_template_error"] == [f"user-data-{i}" for i in (0, 2, 3, 6, 7)]+++def test_a_marker_named_column_survives_a_schema_less_stream():+ """A generator-backed IterableDataset reports column_names AND features as None.++ Taking that as "no columns" picks the plain marker name, and remove_columns then+ deletes the user's own column from every row that survived.+ """++ def rows():+ for i in range(8):+ yield {+ "messages": _convo(i, with_system = i in (1, 4, 5)),+ "__chat_template_error": f"user-data-{i}",+ }++ dataset = IterableDataset.from_generator(rows)+ assert dataset.column_names is None and dataset.features is None++ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())++ assert result["success"] is True+ kept = list(result["dataset"])+ assert [row["__chat_template_error"] for row in kept] == [+ f"user-data-{i}" for i in (0, 2, 3, 6, 7)+ ]+++def test_failures_spread_across_scan_batches_are_all_counted(monkeypatch):+ """The error column is scanned and filtered in batches, so a batch boundary is a seam.++ Counting or first-error logic that only looked at one batch would under-report here.+ """+ monkeypatch.setattr(chat_templates, "_ERROR_SCAN_BATCH", 3)+ n = 20+ bad = {0, 4, 5, 11, 19}+ dataset = Dataset.from_dict({"messages": [_convo(i, with_system = i in bad) for i in range(n)]})++ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())++ assert result["success"] is True+ assert len(result["dataset"]) == n - len(bad)+ assert result["dropped_rows_warning"] == (+ "Dropped 5 of 20 rows because the chat template failed: System role not supported"+ )+ assert result["dataset"].column_names == ["messages", "text"]+++def test_all_rows_failing_still_returns_the_dropped_rows_key():+ dataset = Dataset.from_dict({"messages": [_convo(i, with_system = True) for i in range(3)]})++ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())++ assert result["success"] is False+ assert result["dropped_rows_warning"] is None+++def test_a_template_that_renders_empty_is_not_treated_as_a_failure():+ """Only a raised exception drops a row; a legitimately empty render stays."""++ class _BlankTokenizer(_StrictTokenizer):+ def apply_chat_template(self, conversation, **_kwargs):+ return ""++ dataset = Dataset.from_dict({"messages": [_convo(i) for i in range(3)]})++ result = apply_chat_template_to_dataset(_dataset_info(dataset), _BlankTokenizer())++ assert result["success"] is True+ assert len(result["dataset"]) == 3+ assert result["dropped_rows_warning"] is Nonediff --git a/studio/backend/tests/test_mlx_dataset_format_failure.py b/studio/backend/tests/test_mlx_dataset_format_failure.pynew file mode 100644index 00000000000..7dd94d27baf--- /dev/null+++ b/studio/backend/tests/test_mlx_dataset_format_failure.py@@ -0,0 +1,122 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The MLX text path must react to a dataset format failure the way the GPU trainer does.++format_and_template_dataset returns success=False when the chat template raised on every+row. Ignoring that leaves the raw, untemplated dataset in place and trains on it.+"""++import ast+import inspect+import textwrap++import pytest++import core.training.worker as _worker+++def _format_block():+ """The `elif format_type:` branch of _run_mlx_training, which formats the dataset."""+ tree = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training)))+ for node in ast.walk(tree):+ if not isinstance(node, ast.If):+ continue+ branches = [node] + [o for o in node.orelse if isinstance(o, ast.If)]+ for branch in branches:+ if ast.unparse(branch.test) == "format_type":+ return branch+ raise AssertionError("could not locate the `elif format_type:` block")+++def _run(+ train_success = True,+ train_warning = None,+ eval_success = True,+ eval_warning = None,+):+ """Execute the real block from _run_mlx_training and report what it did.++ _run_mlx_training only runs on Apple Silicon, so the block is lifted out and executed+ directly, the same technique test_mlx_training_worker_config.py uses.+ """+ block = compile(ast.Module(body = _format_block().body, type_ignores = []), "<fmt>", "exec")+ events = []+ calls = {"n": 0}++ def _fake_format(dataset, **_kwargs):+ calls["n"] += 1+ first = calls["n"] == 1+ return {+ "success": train_success if first else eval_success,+ "dataset": "FORMATTED_TRAIN" if first else "FORMATTED_EVAL",+ "final_format": "chatml_messages",+ "dropped_rows_warning": train_warning if first else eval_warning,+ }++ namespace = dict(vars(_worker))+ namespace.update(+ {+ "format_type": "chatml_messages",+ "dataset": "RAW_TRAIN",+ "eval_dataset": "RAW_EVAL",+ "model_name": "org/model",+ "tokenizer": object(),+ "hf_dataset": "local",+ "custom_format_mapping": None,+ "_fmt_progress": lambda **_kw: None,+ "_send": lambda kind, **kw: events.append((kind, kw)),+ "format_and_template_dataset": _fake_format,+ }+ )++ error = None+ try:+ exec(block, namespace)+ except ValueError as exc:+ error = str(exc)++ return {+ "dataset": namespace.get("dataset"),+ "eval_dataset": namespace.get("eval_dataset"),+ "warnings": [kw["message"] for kind, kw in events if kind == "warning"],+ "error": error,+ }+++def test_a_failed_train_dataset_stops_the_run():+ result = _run(train_success = False)++ assert result["error"] is not None+ assert result["dataset"] == "RAW_TRAIN", "the untemplated dataset must not reach training"+++def test_a_failed_eval_dataset_stops_the_run():+ result = _run(eval_success = False)++ assert result["error"] is not None+ assert result["eval_dataset"] == "RAW_EVAL"+++def test_dropped_row_warnings_reach_the_user_for_both_datasets():+ result = _run(+ train_warning = "Dropped 3 of 8 rows because the chat template failed: boom",+ eval_warning = "Dropped 1 of 4 rows because the chat template failed: boom",+ )++ assert result["error"] is None+ assert result["dataset"] == "FORMATTED_TRAIN"+ assert result["eval_dataset"] == "FORMATTED_EVAL"+ assert result["warnings"] == [+ "Dropped 3 of 8 rows because the chat template failed: boom",+ "Eval dataset: Dropped 1 of 4 rows because the chat template failed: boom",+ ]+++def test_a_clean_run_is_unchanged():+ result = _run()++ assert result["error"] is None+ assert result["dataset"] == "FORMATTED_TRAIN"+ assert result["eval_dataset"] == "FORMATTED_EVAL"+ assert result["warnings"] == []diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.pyindex 9fe3d451022..60b1d72c49c 100644--- a/studio/backend/utils/datasets/chat_templates.py+++ b/studio/backend/utils/datasets/chat_templates.py@@ -26,6 +26,12 @@ DEFAULT_ALPACA_TEMPLATE = """Below is an instruction that describes a task, pair### Response:{}"""+_TEMPLATE_ERROR_COLUMN = "__chat_template_error"++# Rows per batch when scanning or filtering the error column, so neither pass+# materialises the whole column in Python.+_ERROR_SCAN_BATCH = 10_000+_CUSTOM_PROMPT_TEMPLATE_ERROR = ("custom_prompt_template is deprecated and unsupported because Unsloth Studio cannot persist a ""matching template for inference. Pass None to continue without a custom prompt template."@@ -333,9 +339,23 @@ def apply_chat_template_to_dataset(if model_name:tokenizer = get_tokenizer_chat_template(tokenizer, model_name)+ streamed_failures = []++ # Never clobber a real column: a dataset is allowed to already carry one named+ # like our marker, and remove_columns would then delete the user's own data.+ # A generator-backed IterableDataset reports column_names AND features as None,+ # so resolve_column_names' first-row probe is what sees the column there.+ from .raw_text import resolve_column_names++ existing_columns = set(resolve_column_names(dataset))+ error_column = _TEMPLATE_ERROR_COLUMN+ while error_column in existing_columns:+ error_column += "_"+def _format_chatml(examples):convos = examples[chat_column]texts = []+ row_errors = []for convo in convos:try:@@ -350,12 +370,18 @@ def apply_chat_template_to_dataset(text += eos_tokentexts.append(text)+ row_errors.append("")except Exception as e:- if len(texts) == 0:- warnings.append(f"Chat template failed: {e}")texts.append("")+ row_errors.append(str(e) or type(e).__name__)- return {"text": texts}+ return {"text": texts, error_column: row_errors}++ def _keep_streamed_row(row_error):+ if row_error and not streamed_failures:+ streamed_failures.append(row_error)+ logger.warning(f"Dropping rows whose chat template failed: {row_error}")+ return not row_errortry:is_iterable = is_streaming_dataset(dataset)@@ -405,11 +431,64 @@ def apply_chat_template_to_dataset(if _tqdm_monitor_stop is not None:_tqdm_monitor_stop.set()+ dropped_rows_warning = None+ if is_iterable:+ formatted_dataset = formatted_dataset.filter(+ _keep_streamed_row, input_columns = [error_column]+ ).remove_columns(error_column)+ elif len(formatted_dataset):+ # Everything here stays Arrow-side and batched. Reading the error column+ # into a Python list, or building one index per surviving row, costs a+ # measured 85 MB at 2M rows (70 MB of it the index list) and scales+ # linearly, so a dataset of tens of millions of mostly valid rows could be+ # killed during formatting.+ n_total = len(formatted_dataset)+ kept = formatted_dataset.filter(+ lambda row_errors: [not row_error for row_error in row_errors],+ input_columns = [error_column],+ batched = True,+ batch_size = _ERROR_SCAN_BATCH,+ desc = "Dropping rows whose chat template failed",+ )+ n_failed = n_total - len(kept)+ if n_failed:+ first_error = next(+ (+ row_error+ for batch in formatted_dataset.select_columns(+ [error_column]+ ).iter(batch_size = _ERROR_SCAN_BATCH)+ for row_error in batch[error_column]+ if row_error+ ),+ "",+ )+ if n_failed == n_total:+ errors.append(+ f"Chat template failed on all {n_total:,} rows: {first_error}"+ )+ return {+ "dataset": dataset,+ "success": False,+ "warnings": warnings,+ "errors": errors,+ "dropped_rows_warning": None,+ }+ if n_failed:+ formatted_dataset = kept+ dropped_rows_warning = (+ f"Dropped {n_failed:,} of {n_total:,} rows because the "+ f"chat template failed: {first_error}"+ )+ warnings.append(dropped_rows_warning)+ formatted_dataset = formatted_dataset.remove_columns(error_column)+return {"dataset": formatted_dataset,"success": True,"warnings": warnings,- "errors": errors+ "errors": errors,+ "dropped_rows_warning": dropped_rows_warning,}except Exception as e:errors.append(f"Failed to format ChatML dataset: {e}")diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.pyindex 4865ee719ed..8f0c3b79e78 100644--- a/studio/backend/utils/datasets/dataset_utils.py+++ b/studio/backend/utils/datasets/dataset_utils.py@@ -1098,5 +1098,6 @@ def format_and_template_dataset("requires_manual_mapping": requires_manual,"warnings": all_warnings,"errors": all_errors,+ "dropped_rows_warning": template_result.get("dropped_rows_warning"),"summary": summary,}