Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: drop rows the chat template can't format and tell the user (#11293)

Nilay8265caeadc9b
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index 97e5f0d6fdb..6cb64cbd2e9 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -3279,6 +3279,9 @@ class UnslothTrainer:
self._update_progress(error = error_msg)
return None
+ if dataset_info.get("dropped_rows_warning"):
+ self._record_warning(dataset_info["dropped_rows_warning"])
+
detected = dataset_info.get("detected_format", "unknown")
final_ds = dataset_info.get("dataset")
final_n = len(final_ds) if hasattr(final_ds, "__len__") else "?"
@@ -3301,6 +3304,14 @@ class UnslothTrainer:
dataset_name = dataset_source,
custom_format_mapping = custom_format_mapping,
)
+ if not eval_info.get("success", True):
+ eval_errors = eval_info.get("errors", [])
+ error_msg = "; ".join(eval_errors) or "Eval dataset formatting failed"
+ logger.error(f"Eval dataset conversion failed: {error_msg}")
+ self._update_progress(error = error_msg)
+ return None
+ if eval_info.get("dropped_rows_warning"):
+ self._record_warning(f"Eval dataset: {eval_info['dropped_rows_warning']}")
eval_dataset = eval_info["dataset"]
logger.info("Eval dataset formatted successfully\n")
elif eval_enabled and not has_separate_eval_source and not dataset_streaming:
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 2e4405c9ea1..d3721e4342a 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -2835,6 +2835,11 @@ def _run_mlx_training(event_queue, stop_queue, config):
)
if info.get("success", True):
dataset = info.get("dataset", dataset)
+ else:
+ errors = info.get("errors", [])
+ raise ValueError(f"Dataset format conversion failed: {'; '.join(errors)}")
+ if info.get("dropped_rows_warning"):
+ _send("warning", message = info["dropped_rows_warning"])
dataset_final_format = str(info.get("final_format", "") or "").lower()
if eval_dataset is not None:
ev = format_and_template_dataset(
@@ -2848,6 +2853,13 @@ def _run_mlx_training(event_queue, stop_queue, config):
)
if ev.get("success", True):
eval_dataset = ev.get("dataset", eval_dataset)
+ else:
+ eval_errors = ev.get("errors", [])
+ raise ValueError(
+ f"Eval dataset format conversion failed: {'; '.join(eval_errors)}"
+ )
+ if ev.get("dropped_rows_warning"):
+ _send("warning", message = f"Eval dataset: {ev['dropped_rows_warning']}")
except ImportError:
_send("status", status_message = "Format helper unavailable, using raw dataset")
diff --git a/studio/backend/tests/test_chat_template_failed_rows.py b/studio/backend/tests/test_chat_template_failed_rows.py
new file mode 100644
index 00000000000..37e5a647303
--- /dev/null
+++ b/studio/backend/tests/test_chat_template_failed_rows.py
@@ -0,0 +1,193 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import pytest
+from datasets import Dataset, IterableDataset
+
+import utils.hardware
+from utils.datasets import chat_templates
+from utils.datasets import apply_chat_template_to_dataset, format_and_template_dataset
+
+
+class _StrictTokenizer:
+ chat_template = "{{ messages }}"
+ eos_token = "</s>"
+
+ def apply_chat_template(self, conversation, **_kwargs):
+ if any(turn["role"] == "system" for turn in conversation):
+ raise ValueError("System role not supported")
+ return "\n".join(f"{turn['role']}: {turn['content']}" for turn in conversation)
+
+
+def _convo(index, with_system = False):
+ turns = [
+ {"role": "user", "content": f"question {index}"},
+ {"role": "assistant", "content": f"answer {index}"},
+ ]
+ if with_system:
+ turns.insert(0, {"role": "system", "content": "be brief"})
+ return turns
+
+
+def _dataset_info(dataset):
+ return {
+ "dataset": dataset,
+ "detected_format": "chatml_messages",
+ "final_format": "chatml_messages",
+ "chat_column": "messages",
+ "is_standardized": True,
+ "warnings": [],
+ }
+
+
+def _mixed_dataset():
+ return Dataset.from_dict(
+ {"messages": [_convo(i, with_system = i in (1, 4, 5)) for i in range(8)]}
+ )
+
+
[email protected]("num_proc", [None, 2])
+def test_rows_whose_template_raised_are_dropped_and_counted(monkeypatch, num_proc):
+ monkeypatch.setattr(utils.hardware, "dataset_map_num_proc", lambda *_a, **_k: num_proc)
+
+ result = apply_chat_template_to_dataset(_dataset_info(_mixed_dataset()), _StrictTokenizer())
+
+ assert result["success"] is True
+ formatted = result["dataset"]
+ assert len(formatted) == 5
+ assert [text.splitlines()[0] for text in formatted["text"]] == [
+ f"user: question {i}" for i in (0, 2, 3, 6, 7)
+ ]
+ assert formatted.column_names == ["messages", "text"]
+ assert result["dropped_rows_warning"] == (
+ "Dropped 3 of 8 rows because the chat template failed: System role not supported"
+ )
+ assert result["dropped_rows_warning"] in result["warnings"]
+
+
+def test_clean_dataset_reports_no_dropped_rows():
+ dataset = Dataset.from_dict({"messages": [_convo(i) for i in range(3)]})
+
+ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())
+
+ assert result["success"] is True
+ assert len(result["dataset"]) == 3
+ assert result["dataset"].column_names == ["messages", "text"]
+ assert result.get("dropped_rows_warning") is None
+
+
+def test_every_row_failing_is_an_error():
+ dataset = Dataset.from_dict({"messages": [_convo(i, with_system = True) for i in range(3)]})
+
+ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())
+
+ assert result["success"] is False
+ assert result["errors"] == ["Chat template failed on all 3 rows: System role not supported"]
+ assert result["dataset"] is dataset
+
+
+def test_streaming_rows_whose_template_raised_are_filtered():
+ result = apply_chat_template_to_dataset(
+ _dataset_info(_mixed_dataset().to_iterable_dataset()), _StrictTokenizer()
+ )
+
+ assert result["success"] is True
+ rows = list(result["dataset"])
+ assert len(rows) == 5
+ assert all(set(row) == {"messages", "text"} and row["text"] for row in rows)
+
+
+def test_format_and_template_dataset_passes_the_dropped_rows_warning_through():
+ result = format_and_template_dataset(
+ _mixed_dataset(), model_name = "stub-model", tokenizer = _StrictTokenizer()
+ )
+
+ assert result["success"] is True
+ assert len(result["dataset"]) == 5
+ assert result["dropped_rows_warning"].startswith("Dropped 3 of 8 rows")
+
+
+def test_a_column_named_like_the_marker_is_not_clobbered():
+ """A dataset may already carry a column named like the internal marker."""
+ dataset = _mixed_dataset().add_column(
+ "__chat_template_error", [f"user-data-{i}" for i in range(8)]
+ )
+
+ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())
+
+ assert result["success"] is True
+ formatted = result["dataset"]
+ assert len(formatted) == 5
+ assert sorted(formatted.column_names) == ["__chat_template_error", "messages", "text"]
+ assert formatted["__chat_template_error"] == [f"user-data-{i}" for i in (0, 2, 3, 6, 7)]
+
+
+def test_a_marker_named_column_survives_a_schema_less_stream():
+ """A generator-backed IterableDataset reports column_names AND features as None.
+
+ Taking that as "no columns" picks the plain marker name, and remove_columns then
+ deletes the user's own column from every row that survived.
+ """
+
+ def rows():
+ for i in range(8):
+ yield {
+ "messages": _convo(i, with_system = i in (1, 4, 5)),
+ "__chat_template_error": f"user-data-{i}",
+ }
+
+ dataset = IterableDataset.from_generator(rows)
+ assert dataset.column_names is None and dataset.features is None
+
+ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())
+
+ assert result["success"] is True
+ kept = list(result["dataset"])
+ assert [row["__chat_template_error"] for row in kept] == [
+ f"user-data-{i}" for i in (0, 2, 3, 6, 7)
+ ]
+
+
+def test_failures_spread_across_scan_batches_are_all_counted(monkeypatch):
+ """The error column is scanned and filtered in batches, so a batch boundary is a seam.
+
+ Counting or first-error logic that only looked at one batch would under-report here.
+ """
+ monkeypatch.setattr(chat_templates, "_ERROR_SCAN_BATCH", 3)
+ n = 20
+ bad = {0, 4, 5, 11, 19}
+ dataset = Dataset.from_dict({"messages": [_convo(i, with_system = i in bad) for i in range(n)]})
+
+ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())
+
+ assert result["success"] is True
+ assert len(result["dataset"]) == n - len(bad)
+ assert result["dropped_rows_warning"] == (
+ "Dropped 5 of 20 rows because the chat template failed: System role not supported"
+ )
+ assert result["dataset"].column_names == ["messages", "text"]
+
+
+def test_all_rows_failing_still_returns_the_dropped_rows_key():
+ dataset = Dataset.from_dict({"messages": [_convo(i, with_system = True) for i in range(3)]})
+
+ result = apply_chat_template_to_dataset(_dataset_info(dataset), _StrictTokenizer())
+
+ assert result["success"] is False
+ assert result["dropped_rows_warning"] is None
+
+
+def test_a_template_that_renders_empty_is_not_treated_as_a_failure():
+ """Only a raised exception drops a row; a legitimately empty render stays."""
+
+ class _BlankTokenizer(_StrictTokenizer):
+ def apply_chat_template(self, conversation, **_kwargs):
+ return ""
+
+ dataset = Dataset.from_dict({"messages": [_convo(i) for i in range(3)]})
+
+ result = apply_chat_template_to_dataset(_dataset_info(dataset), _BlankTokenizer())
+
+ assert result["success"] is True
+ assert len(result["dataset"]) == 3
+ assert result["dropped_rows_warning"] is None
diff --git a/studio/backend/tests/test_mlx_dataset_format_failure.py b/studio/backend/tests/test_mlx_dataset_format_failure.py
new file mode 100644
index 00000000000..7dd94d27baf
--- /dev/null
+++ b/studio/backend/tests/test_mlx_dataset_format_failure.py
@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The MLX text path must react to a dataset format failure the way the GPU trainer does.
+
+format_and_template_dataset returns success=False when the chat template raised on every
+row. Ignoring that leaves the raw, untemplated dataset in place and trains on it.
+"""
+
+import ast
+import inspect
+import textwrap
+
+import pytest
+
+import core.training.worker as _worker
+
+
+def _format_block():
+ """The `elif format_type:` branch of _run_mlx_training, which formats the dataset."""
+ tree = ast.parse(textwrap.dedent(inspect.getsource(_worker._run_mlx_training)))
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.If):
+ continue
+ branches = [node] + [o for o in node.orelse if isinstance(o, ast.If)]
+ for branch in branches:
+ if ast.unparse(branch.test) == "format_type":
+ return branch
+ raise AssertionError("could not locate the `elif format_type:` block")
+
+
+def _run(
+ train_success = True,
+ train_warning = None,
+ eval_success = True,
+ eval_warning = None,
+):
+ """Execute the real block from _run_mlx_training and report what it did.
+
+ _run_mlx_training only runs on Apple Silicon, so the block is lifted out and executed
+ directly, the same technique test_mlx_training_worker_config.py uses.
+ """
+ block = compile(ast.Module(body = _format_block().body, type_ignores = []), "<fmt>", "exec")
+ events = []
+ calls = {"n": 0}
+
+ def _fake_format(dataset, **_kwargs):
+ calls["n"] += 1
+ first = calls["n"] == 1
+ return {
+ "success": train_success if first else eval_success,
+ "dataset": "FORMATTED_TRAIN" if first else "FORMATTED_EVAL",
+ "final_format": "chatml_messages",
+ "dropped_rows_warning": train_warning if first else eval_warning,
+ }
+
+ namespace = dict(vars(_worker))
+ namespace.update(
+ {
+ "format_type": "chatml_messages",
+ "dataset": "RAW_TRAIN",
+ "eval_dataset": "RAW_EVAL",
+ "model_name": "org/model",
+ "tokenizer": object(),
+ "hf_dataset": "local",
+ "custom_format_mapping": None,
+ "_fmt_progress": lambda **_kw: None,
+ "_send": lambda kind, **kw: events.append((kind, kw)),
+ "format_and_template_dataset": _fake_format,
+ }
+ )
+
+ error = None
+ try:
+ exec(block, namespace)
+ except ValueError as exc:
+ error = str(exc)
+
+ return {
+ "dataset": namespace.get("dataset"),
+ "eval_dataset": namespace.get("eval_dataset"),
+ "warnings": [kw["message"] for kind, kw in events if kind == "warning"],
+ "error": error,
+ }
+
+
+def test_a_failed_train_dataset_stops_the_run():
+ result = _run(train_success = False)
+
+ assert result["error"] is not None
+ assert result["dataset"] == "RAW_TRAIN", "the untemplated dataset must not reach training"
+
+
+def test_a_failed_eval_dataset_stops_the_run():
+ result = _run(eval_success = False)
+
+ assert result["error"] is not None
+ assert result["eval_dataset"] == "RAW_EVAL"
+
+
+def test_dropped_row_warnings_reach_the_user_for_both_datasets():
+ result = _run(
+ train_warning = "Dropped 3 of 8 rows because the chat template failed: boom",
+ eval_warning = "Dropped 1 of 4 rows because the chat template failed: boom",
+ )
+
+ assert result["error"] is None
+ assert result["dataset"] == "FORMATTED_TRAIN"
+ assert result["eval_dataset"] == "FORMATTED_EVAL"
+ assert result["warnings"] == [
+ "Dropped 3 of 8 rows because the chat template failed: boom",
+ "Eval dataset: Dropped 1 of 4 rows because the chat template failed: boom",
+ ]
+
+
+def test_a_clean_run_is_unchanged():
+ result = _run()
+
+ assert result["error"] is None
+ assert result["dataset"] == "FORMATTED_TRAIN"
+ assert result["eval_dataset"] == "FORMATTED_EVAL"
+ assert result["warnings"] == []
diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.py
index 9fe3d451022..60b1d72c49c 100644
--- a/studio/backend/utils/datasets/chat_templates.py
+++ b/studio/backend/utils/datasets/chat_templates.py
@@ -26,6 +26,12 @@ DEFAULT_ALPACA_TEMPLATE = """Below is an instruction that describes a task, pair
### Response:
{}"""
+_TEMPLATE_ERROR_COLUMN = "__chat_template_error"
+
+# Rows per batch when scanning or filtering the error column, so neither pass
+# materialises the whole column in Python.
+_ERROR_SCAN_BATCH = 10_000
+
_CUSTOM_PROMPT_TEMPLATE_ERROR = (
"custom_prompt_template is deprecated and unsupported because Unsloth Studio cannot persist a "
"matching template for inference. Pass None to continue without a custom prompt template."
@@ -333,9 +339,23 @@ def apply_chat_template_to_dataset(
if model_name:
tokenizer = get_tokenizer_chat_template(tokenizer, model_name)
+ streamed_failures = []
+
+ # Never clobber a real column: a dataset is allowed to already carry one named
+ # like our marker, and remove_columns would then delete the user's own data.
+ # A generator-backed IterableDataset reports column_names AND features as None,
+ # so resolve_column_names' first-row probe is what sees the column there.
+ from .raw_text import resolve_column_names
+
+ existing_columns = set(resolve_column_names(dataset))
+ error_column = _TEMPLATE_ERROR_COLUMN
+ while error_column in existing_columns:
+ error_column += "_"
+
def _format_chatml(examples):
convos = examples[chat_column]
texts = []
+ row_errors = []
for convo in convos:
try:
@@ -350,12 +370,18 @@ def apply_chat_template_to_dataset(
text += eos_token
texts.append(text)
+ row_errors.append("")
except Exception as e:
- if len(texts) == 0:
- warnings.append(f"Chat template failed: {e}")
texts.append("")
+ row_errors.append(str(e) or type(e).__name__)
- return {"text": texts}
+ return {"text": texts, error_column: row_errors}
+
+ def _keep_streamed_row(row_error):
+ if row_error and not streamed_failures:
+ streamed_failures.append(row_error)
+ logger.warning(f"Dropping rows whose chat template failed: {row_error}")
+ return not row_error
try:
is_iterable = is_streaming_dataset(dataset)
@@ -405,11 +431,64 @@ def apply_chat_template_to_dataset(
if _tqdm_monitor_stop is not None:
_tqdm_monitor_stop.set()
+ dropped_rows_warning = None
+ if is_iterable:
+ formatted_dataset = formatted_dataset.filter(
+ _keep_streamed_row, input_columns = [error_column]
+ ).remove_columns(error_column)
+ elif len(formatted_dataset):
+ # Everything here stays Arrow-side and batched. Reading the error column
+ # into a Python list, or building one index per surviving row, costs a
+ # measured 85 MB at 2M rows (70 MB of it the index list) and scales
+ # linearly, so a dataset of tens of millions of mostly valid rows could be
+ # killed during formatting.
+ n_total = len(formatted_dataset)
+ kept = formatted_dataset.filter(
+ lambda row_errors: [not row_error for row_error in row_errors],
+ input_columns = [error_column],
+ batched = True,
+ batch_size = _ERROR_SCAN_BATCH,
+ desc = "Dropping rows whose chat template failed",
+ )
+ n_failed = n_total - len(kept)
+ if n_failed:
+ first_error = next(
+ (
+ row_error
+ for batch in formatted_dataset.select_columns(
+ [error_column]
+ ).iter(batch_size = _ERROR_SCAN_BATCH)
+ for row_error in batch[error_column]
+ if row_error
+ ),
+ "",
+ )
+ if n_failed == n_total:
+ errors.append(
+ f"Chat template failed on all {n_total:,} rows: {first_error}"
+ )
+ return {
+ "dataset": dataset,
+ "success": False,
+ "warnings": warnings,
+ "errors": errors,
+ "dropped_rows_warning": None,
+ }
+ if n_failed:
+ formatted_dataset = kept
+ dropped_rows_warning = (
+ f"Dropped {n_failed:,} of {n_total:,} rows because the "
+ f"chat template failed: {first_error}"
+ )
+ warnings.append(dropped_rows_warning)
+ formatted_dataset = formatted_dataset.remove_columns(error_column)
+
return {
"dataset": formatted_dataset,
"success": True,
"warnings": warnings,
- "errors": errors
+ "errors": errors,
+ "dropped_rows_warning": dropped_rows_warning,
}
except Exception as e:
errors.append(f"Failed to format ChatML dataset: {e}")
diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py
index 4865ee719ed..8f0c3b79e78 100644
--- a/studio/backend/utils/datasets/dataset_utils.py
+++ b/studio/backend/utils/datasets/dataset_utils.py
@@ -1098,5 +1098,6 @@ def format_and_template_dataset(
"requires_manual_mapping": requires_manual,
"warnings": all_warnings,
"errors": all_errors,
+ "dropped_rows_warning": template_result.get("dropped_rows_warning"),
"summary": summary,
}