Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: train on pre-tokenized datasets (input_ids / attention_mask / labels) (#13816)

Daniel Hanc48d442c083e
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index f90825623be..69939778ebd 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -85,6 +85,11 @@ from utils.datasets.cells import csv_as_text_kwargs
from utils.datasets.chat_templates import get_training_chat_template
from utils.datasets.completion_masking import apply_completion_masking
from utils.datasets.iterable import is_streaming_dataset as detect_streaming_dataset
+from utils.datasets.pretokenized import (
+ PRETOKENIZED_FORMAT,
+ has_custom_labels,
+ prepare_pretokenized_dataset,
+)
from utils.datasets.raw_text import prepare_raw_text_dataset, resolve_column_names
from utils.paths import (
dataset_files_in_dir,
@@ -4591,6 +4596,9 @@ class UnslothTrainer:
str(dataset.get("final_format", "")).lower() if isinstance(dataset, dict) else ""
)
raw_text_mode = dataset_final_format == "raw_text"
+ pretokenized_mode = dataset_final_format == PRETOKENIZED_FORMAT
+ # Without a labels column, "Train on responses only" can still mask prompts from input_ids.
+ user_labels = pretokenized_mode and has_custom_labels(dataset["dataset"])
self.tokenizer = get_training_chat_template(
self.tokenizer, self.model_name, dataset_final_format
)
@@ -4877,6 +4885,23 @@ class UnslothTrainer:
# Packed rows share one bidirectional attention window, so documents would see each other.
config_args["packing"] = False
logger.info("Text-diffusion model: sequence packing turned off\n")
+ if pretokenized_mode:
+ # Packing on TRL < 1.7 keeps only input_ids, silently replacing the user's labels.
+ if config_args.get("packing"):
+ logger.info("Pre-tokenized dataset: sequence packing turned off\n")
+ config_args["packing"] = False
+ try:
+ vocab_size = int(self.model.get_input_embeddings().num_embeddings)
+ except Exception:
+ vocab_size = None
+ max_seq = training_args.get("max_seq_length", 2048)
+ dataset["dataset"] = prepare_pretokenized_dataset(
+ dataset["dataset"], max_seq_length = max_seq, vocab_size = vocab_size
+ )
+ if eval_dataset is not None:
+ eval_dataset = prepare_pretokenized_dataset(
+ eval_dataset, max_seq_length = max_seq, vocab_size = vocab_size
+ )
# Plain-text single-pass runs tokenize in the DataLoader workers instead of a blocking .map(); everything
# else stays eager.
@@ -5035,7 +5060,7 @@ class UnslothTrainer:
is_cpt = training_args.get("is_cpt", False)
train_on_responses_enabled = (
False
- if (is_cpt or raw_text_mode)
+ if (is_cpt or raw_text_mode or user_labels)
else training_args.get("train_on_completions", False)
)
@@ -5045,6 +5070,10 @@ class UnslothTrainer:
logger.info(
"Raw-text mode: skipping train_on_responses_only — training on all tokens\n"
)
+ elif user_labels:
+ logger.info(
+ "Pre-tokenized dataset: its labels define the loss, no response masking\n"
+ )
# Markers auto-detected from the chat template first, with the manual table as fallback; gpt-oss stays on
# manual markers. See apply_completion_masking.
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index d21d37c5865..9e45d19225a 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -3452,10 +3452,25 @@ def _run_mlx_training(event_queue, stop_queue, config):
# Auto-detect markers from the chat template first, manual table as fallback. Mirror the CUDA skips: raw/CPT text
# has no chat turns. Check the resolved format too, since format_type="auto" can land on alpaca or raw.
+ from utils.datasets.pretokenized import has_custom_labels
+
+ if (
+ config.get("train_on_completions", False)
+ and dataset_final_format == "pretokenized"
+ and not has_custom_labels(dataset)
+ ):
+ # zoo's MLX masker re-tokenizes text, so token-id rows have nothing it can mask (CUDA masks input_ids).
+ _send(
+ "warning",
+ message = (
+ "'Train on completions' is not available for pre-tokenized datasets on Mac: "
+ "training will run on full sequences. Add a labels column to choose the trained tokens."
+ ),
+ )
if (
config.get("train_on_completions", False)
and not raw_text_mode
- and dataset_final_format != "raw_text"
+ and dataset_final_format not in ("raw_text", "pretokenized")
):
_send("status", status_message = "Configuring response-only training...")
# No catch: the helper handles detection failures and double misses, so an exception here is a real masking
diff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.py
index 933d66fe77e..656be159e46 100644
--- a/studio/backend/hub/utils/dataset_format.py
+++ b/studio/backend/hub/utils/dataset_format.py
@@ -8,6 +8,7 @@ from typing import Any, Optional
from utils.datasets.cells import text_cell_check
from utils.datasets.format_detection import detect_dataset_format, has_message_prompt_completion
+from utils.datasets.pretokenized import PRETOKENIZED_FORMAT, is_pretokenized_dataset
def _first_row(dataset) -> Optional[dict]:
@@ -795,6 +796,18 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict:
"multimodal_columns": multimodal_info.get("multimodal_columns"),
**audio_fields,
}
+ if is_pretokenized_dataset(dataset):
+ return {
+ "requires_manual_mapping": False,
+ "detected_format": PRETOKENIZED_FORMAT,
+ "columns": columns,
+ "suggested_mapping": None,
+ "detected_image_column": None,
+ "detected_text_column": None,
+ "is_image": multimodal_info["is_image"],
+ "multimodal_columns": multimodal_info.get("multimodal_columns"),
+ **audio_fields,
+ }
return {
"requires_manual_mapping": True,
"detected_format": "unknown",
diff --git a/studio/backend/tests/test_pretokenized_dataset.py b/studio/backend/tests/test_pretokenized_dataset.py
new file mode 100644
index 00000000000..2e01c0ab624
--- /dev/null
+++ b/studio/backend/tests/test_pretokenized_dataset.py
@@ -0,0 +1,165 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Pre-tokenized datasets (input_ids / attention_mask / labels) route to training as-is (#7139), and nothing else moves."""
+
+import inspect
+import sys
+from pathlib import Path
+
+import pytest
+from datasets import Dataset
+
+_BACKEND_ROOT = Path(__file__).resolve().parents[1]
+if str(_BACKEND_ROOT) not in sys.path:
+ sys.path.insert(0, str(_BACKEND_ROOT))
+
+from hub.utils.dataset_format import check_dataset_format as hub_check_dataset_format # noqa: E402
+from utils.datasets.dataset_utils import ( # noqa: E402
+ _format_dataset,
+ check_dataset_format,
+ format_and_template_dataset,
+ format_dataset,
+)
+from utils.datasets.pretokenized import ( # noqa: E402
+ has_custom_labels,
+ is_pretokenized_dataset,
+ prepare_pretokenized_dataset,
+)
+
+
+def _pretokenized(with_labels = True, n = 4):
+ rows = {
+ "input_ids": [[1, 2, 3, 4, 5][: 3 + i % 3] for i in range(n)],
+ "attention_mask": [[1] * (3 + i % 3) for i in range(n)],
+ }
+ if with_labels:
+ rows["labels"] = [[-100] + ids[1:] for ids in rows["input_ids"]]
+ return Dataset.from_dict(rows)
+
+
+NORMAL_DATASETS = {
+ "alpaca": (
+ Dataset.from_dict({"instruction": ["Say hi"], "input": [""], "output": ["hi"]}),
+ "alpaca",
+ ),
+ "chatml": (
+ Dataset.from_dict(
+ {
+ "messages": [
+ [
+ {"role": "user", "content": "Say hi"},
+ {"role": "assistant", "content": "hi"},
+ ]
+ ]
+ }
+ ),
+ "chatml",
+ ),
+ "sharegpt": (
+ Dataset.from_dict(
+ {
+ "conversations": [
+ [{"from": "human", "value": "Say hi"}, {"from": "gpt", "value": "hi"}]
+ ]
+ }
+ ),
+ "sharegpt",
+ ),
+ # A text dataset that also ships token ids keeps its text route.
+ "question_answer_with_ids": (
+ Dataset.from_dict({"question": ["Say hi"], "answer": ["hi"], "input_ids": [[1, 2, 3]]}),
+ "custom_heuristic",
+ ),
+}
+
+
[email protected]("with_labels", [True, False])
+def test_pretokenized_is_detected_and_needs_no_mapping(with_labels):
+ ds = _pretokenized(with_labels)
+ assert is_pretokenized_dataset(ds)
+ for check in (check_dataset_format, hub_check_dataset_format):
+ result = check(ds)
+ assert result["requires_manual_mapping"] is False
+ assert result["detected_format"] == "pretokenized"
+
+
+def test_format_passes_pretokenized_rows_through_untouched():
+ ds = _pretokenized()
+ info = format_and_template_dataset(ds, model_name = "unsloth/Qwen3-0.6B", tokenizer = None)
+ assert info["success"] is True
+ assert info["final_format"] == "pretokenized"
+ assert info["dataset"].column_names == ds.column_names
+ assert info["dataset"].to_dict() == ds.to_dict()
+
+
[email protected]("name", sorted(NORMAL_DATASETS))
+def test_normal_datasets_keep_their_route(name):
+ ds, expected = NORMAL_DATASETS[name]
+ result = hub_check_dataset_format(ds)
+ assert result["detected_format"] == expected
+ assert check_dataset_format(ds)["detected_format"] == expected
+ assert format_dataset(ds, format_type = "auto")["final_format"] != "pretokenized"
+
+
+ "rows",
+ [
+ {"input_ids": [["a", "b"]]},
+ {"input_ids": [[]]},
+ {"input_ids": [[1, 2, 3]], "labels": [[1, 2]]},
+ {"tokens": [[1, 2, 3]]},
+ ],
+)
+def test_malformed_token_rows_still_ask_for_mapping(rows):
+ ds = Dataset.from_dict(rows)
+ assert not is_pretokenized_dataset(ds)
+ assert hub_check_dataset_format(ds)["requires_manual_mapping"] is True
+
+
+def test_prepare_truncates_every_token_column_and_is_idempotent():
+ ds = _pretokenized()
+ once = prepare_pretokenized_dataset(ds, max_seq_length = 3, vocab_size = 100)
+ for column in ("input_ids", "attention_mask", "labels"):
+ assert max(len(v) for v in once[column]) == 3
+ assert once[0]["labels"] == [-100, 2, 3]
+ twice = prepare_pretokenized_dataset(once, max_seq_length = 3, vocab_size = 100)
+ assert twice.to_dict() == once.to_dict()
+
+
+def test_prepare_leaves_short_rows_alone():
+ ds = _pretokenized()
+ assert prepare_pretokenized_dataset(ds, max_seq_length = 2048, vocab_size = 100) is ds
+
+
+def test_prepare_rejects_ids_outside_the_vocabulary():
+ with pytest.raises(ValueError, match = "outside this model's vocabulary"):
+ prepare_pretokenized_dataset(_pretokenized(), max_seq_length = 2048, vocab_size = 4)
+
+
+def test_prepare_rejects_labels_of_another_length():
+ ds = Dataset.from_dict({"input_ids": [[1, 2, 3]], "labels": [[1, 2]]})
+ with pytest.raises(ValueError, match = "same length"):
+ prepare_pretokenized_dataset(ds, max_seq_length = 2048, vocab_size = 100)
+
+
+def test_only_user_labels_turn_off_response_masking():
+ assert has_custom_labels(_pretokenized(with_labels = True))
+ assert not has_custom_labels(_pretokenized(with_labels = False))
+ assert not has_custom_labels(_pretokenized(with_labels = False).to_iterable_dataset())
+
+
[email protected]("format_type", ["auto", "alpaca", "chatml", "sharegpt"])
+def test_any_format_but_raw_routes_token_ids(format_type):
+ info = format_dataset(_pretokenized(), format_type = format_type)
+ assert info["final_format"] == "pretokenized"
+ assert info["requires_manual_mapping"] is False
+
+
+def test_positional_custom_format_mapping_still_binds():
+ params = inspect.signature(_format_dataset).parameters
+ names = list(params)
+ defaults = [params[n].default for n in names[1 : names.index("custom_format_mapping")]]
+ assert format_dataset(_pretokenized(), *defaults, None)["final_format"] == "pretokenized"
+ mapped = format_dataset(_pretokenized(), *defaults, {"input_ids": "user"})
+ assert mapped["final_format"] != "pretokenized"
diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.py
index ed812fb5abd..b8c2eac2008 100644
--- a/studio/backend/utils/datasets/chat_templates.py
+++ b/studio/backend/utils/datasets/chat_templates.py
@@ -488,6 +488,7 @@ def get_dataset_info_summary(dataset_info):
"sharegpt": "ShareGPT format (needs standardization)",
"chatml_messages": "ChatML format (messages column) - OpenAI compatible",
"chatml_conversations": "ChatML format (conversations column) - HuggingFace standard",
+ "pretokenized": "Pre-tokenized (input_ids / labels)",
"unknown": "Unknown format"
}
diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py
index 9745ced5af2..77bee72f0ec 100644
--- a/studio/backend/utils/datasets/dataset_utils.py
+++ b/studio/backend/utils/datasets/dataset_utils.py
@@ -3,6 +3,8 @@
"""Dataset utilities for format detection, conversion, and template application. Entry points: check_dataset_format (lightweight check whether manual mapping is needed, for the frontend), format_dataset (detects and normalizes dataset formats) and format_and_template_dataset (end-to-end processing with chat template). Internal utilities live in format_detection, format_conversion, chat_templates, vlm_processing, data_collators and model_mappings."""
+import inspect
+
from .format_detection import (
detect_dataset_format,
detect_multimodal_dataset,
@@ -26,6 +28,7 @@ from .chat_templates import (
)
from .cells import cell_text, cell_turns, message_list_columns
from .raw_text import prepare_raw_text_dataset
+from .pretokenized import PRETOKENIZED_FORMAT, is_pretokenized_dataset
from .vlm_processing import generate_smart_vlm_instruction
from .data_collators import DeepSeekOCRDataCollator, VLMDataCollator
from .model_mappings import TEMPLATE_TO_MODEL_MAPPER
@@ -115,6 +118,18 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict:
"multimodal_columns": multimodal_info.get("multimodal_columns"),
**audio_fields,
}
+ elif is_pretokenized_dataset(dataset):
+ return {
+ "requires_manual_mapping": False,
+ "detected_format": PRETOKENIZED_FORMAT,
+ "columns": columns,
+ "suggested_mapping": None,
+ "detected_image_column": None,
+ "detected_text_column": None,
+ "is_image": multimodal_info["is_image"],
+ "multimodal_columns": multimodal_info.get("multimodal_columns"),
+ **audio_fields,
+ }
else:
return {
"requires_manual_mapping": True,
@@ -358,7 +373,31 @@ def _apply_user_mapping_alpaca(
)
-def format_dataset(
+def format_dataset(dataset, *args, **kwargs):
+ """``_format_dataset``, but token-id rows no format matched route as pre-tokenized under any format except raw."""
+ bound = _FORMAT_DATASET_SIGNATURE.bind(dataset, *args, **kwargs)
+ bound.apply_defaults()
+ info = _format_dataset(*bound.args, **bound.kwargs)
+ if (
+ info["final_format"] == "unknown"
+ and bound.arguments["format_type"] != "raw"
+ and not bound.arguments["custom_format_mapping"]
+ and is_pretokenized_dataset(dataset)
+ ):
+ info = {
+ **info,
+ "dataset": dataset,
+ "detected_format": PRETOKENIZED_FORMAT,
+ "final_format": PRETOKENIZED_FORMAT,
+ "chat_column": None,
+ "is_standardized": True,
+ "requires_manual_mapping": False,
+ "warnings": [],
+ }
+ return info
+
+
+def _format_dataset(
dataset,
format_type = "auto",
tokenizer = None,
@@ -798,6 +837,9 @@ def format_dataset(
raise ValueError(f"Unknown format_type: {format_type}")
+_FORMAT_DATASET_SIGNATURE = inspect.signature(_format_dataset)
+
+
def format_and_template_dataset(
dataset,
model_name,
@@ -1077,7 +1119,7 @@ def format_and_template_dataset(
raw_text_column = raw_text_column,
)
- if dataset_info["final_format"] == "raw_text":
+ if dataset_info["final_format"] in ("raw_text", PRETOKENIZED_FORMAT):
summary = get_dataset_info_summary(dataset_info)
return {
"dataset": dataset_info["dataset"],
diff --git a/studio/backend/utils/datasets/pretokenized.py b/studio/backend/utils/datasets/pretokenized.py
new file mode 100644
index 00000000000..5793a4f31b4
--- /dev/null
+++ b/studio/backend/utils/datasets/pretokenized.py
@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Pre-tokenized text datasets: rows that already hold ``input_ids`` (optionally ``attention_mask`` and ``labels``). They train as-is, without a chat template; user ``labels`` are the objective, so response masking applies only to rows without them. Only datasets no other format matched are claimed, so every dataset that trained before keeps its route."""
+
+PRETOKENIZED_FORMAT = "pretokenized"
+_TOKEN_COLUMNS = ("input_ids", "attention_mask", "labels", "token_type_ids")
+_SAMPLE_ROWS = 20
+
+
+def _is_token_list(value) -> bool:
+ return (
+ isinstance(value, (list, tuple))
+ and len(value) > 0
+ and all(isinstance(x, int) and not isinstance(x, bool) for x in value)
+ )
+
+
+def is_pretokenized_dataset(dataset) -> bool:
+ from .format_detection import _sample_dataset_rows
+
+ rows = _sample_dataset_rows(dataset, limit = _SAMPLE_ROWS)
+ if not rows:
+ return False
+ for row in rows:
+ if not isinstance(row, dict) or not _is_token_list(row.get("input_ids")):
+ return False
+ labels = row.get("labels")
+ if labels is not None and (
+ not _is_token_list(labels) or len(labels) != len(row["input_ids"])
+ ):
+ return False
+ return True
+
+
+def _column_names(dataset) -> list[str]:
+ names = getattr(dataset, "column_names", None)
+ if names:
+ return list(names)
+ return list(next(iter(dataset)).keys())
+
+
+def has_custom_labels(dataset) -> bool:
+ return "labels" in _column_names(dataset)
+
+
+def _check_arrow_table(table, max_seq_length, vocab_size) -> bool:
+ import pyarrow.compute as pc
+
+ ids = table.column("input_ids")
+ lengths = pc.list_value_length(ids)
+ if "labels" in table.column_names:
+ labels = table.column("labels")
+ if not pc.all(pc.equal(lengths, pc.list_value_length(labels))).as_py():
+ raise ValueError(
+ "Pre-tokenized dataset: 'labels' must have the same length as 'input_ids' in every row."
+ )
+ if vocab_size:
+ flat = pc.list_flatten(ids)
+ bounds = pc.min_max(flat).as_py() if len(flat) else {"min": 0, "max": 0}
+ if bounds["min"] < 0 or bounds["max"] >= vocab_size:
+ bad = bounds["min"] if bounds["min"] < 0 else bounds["max"]
+ raise ValueError(
+ f"Pre-tokenized dataset: token id {bad} is outside this model's vocabulary "
+ f"(size {vocab_size}). The dataset was likely "
+ "tokenized for a different model; re-tokenize it with this model's tokenizer."
+ )
+ longest = pc.max(lengths).as_py() or 0
+ return longest > max_seq_length
+
+
+def prepare_pretokenized_dataset(
+ dataset,
+ *,
+ max_seq_length,
+ vocab_size = None,
+):
+ """Unsloth's SFT preparation passes tokenized rows through untouched: an over-long row would reach the model whole and an id past the embedding table is a device-side assert mid-run. Idempotent."""
+ columns = [c for c in _TOKEN_COLUMNS if c in _column_names(dataset)]
+ table = getattr(dataset, "data", None)
+ needs_truncation = True
+ if table is not None and hasattr(table, "column"):
+ # Covers the whole backing table, rows a select() hid included: a bad id there is still a bad dataset.
+ needs_truncation = _check_arrow_table(table, max_seq_length, vocab_size)
+ if not needs_truncation or not max_seq_length:
+ return dataset
+
+ if table is None:
+
+ def _truncate(batch):
+ return {
+ c: [v[:max_seq_length] if v is not None else v for v in batch[c]] for c in columns
+ }
+
+ return dataset.map(_truncate, batched = True)
+
+ import pyarrow.compute as pc
+
+ def _truncate_arrow(batch):
+ # Arrow slices in C: 46x faster than slicing Python lists on 52M tokens.
+ for c in columns:
+ batch = batch.set_column(
+ batch.column_names.index(c), c, pc.list_slice(batch.column(c), 0, max_seq_length)
+ )
+ return batch
+
+ out = dataset.with_format("arrow").map(
+ _truncate_arrow, batched = True, desc = "Truncating pre-tokenized rows"
+ )
+ return out.with_format(dataset.format["type"])