Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: train on pre-tokenized datasets (input_ids / attention_mask / labels) (#13816)
Daniel Hanc48d442c083e
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex f90825623be..69939778ebd 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -85,6 +85,11 @@ from utils.datasets.cells import csv_as_text_kwargsfrom utils.datasets.chat_templates import get_training_chat_templatefrom utils.datasets.completion_masking import apply_completion_maskingfrom utils.datasets.iterable import is_streaming_dataset as detect_streaming_dataset+from utils.datasets.pretokenized import (+ PRETOKENIZED_FORMAT,+ has_custom_labels,+ prepare_pretokenized_dataset,+)from utils.datasets.raw_text import prepare_raw_text_dataset, resolve_column_namesfrom utils.paths import (dataset_files_in_dir,@@ -4591,6 +4596,9 @@ class UnslothTrainer:str(dataset.get("final_format", "")).lower() if isinstance(dataset, dict) else "")raw_text_mode = dataset_final_format == "raw_text"+ pretokenized_mode = dataset_final_format == PRETOKENIZED_FORMAT+ # Without a labels column, "Train on responses only" can still mask prompts from input_ids.+ user_labels = pretokenized_mode and has_custom_labels(dataset["dataset"])self.tokenizer = get_training_chat_template(self.tokenizer, self.model_name, dataset_final_format)@@ -4877,6 +4885,23 @@ class UnslothTrainer:# Packed rows share one bidirectional attention window, so documents would see each other.config_args["packing"] = Falselogger.info("Text-diffusion model: sequence packing turned off\n")+ if pretokenized_mode:+ # Packing on TRL < 1.7 keeps only input_ids, silently replacing the user's labels.+ if config_args.get("packing"):+ logger.info("Pre-tokenized dataset: sequence packing turned off\n")+ config_args["packing"] = False+ try:+ vocab_size = int(self.model.get_input_embeddings().num_embeddings)+ except Exception:+ vocab_size = None+ max_seq = training_args.get("max_seq_length", 2048)+ dataset["dataset"] = prepare_pretokenized_dataset(+ dataset["dataset"], max_seq_length = max_seq, vocab_size = vocab_size+ )+ if eval_dataset is not None:+ eval_dataset = prepare_pretokenized_dataset(+ eval_dataset, max_seq_length = max_seq, vocab_size = vocab_size+ )# Plain-text single-pass runs tokenize in the DataLoader workers instead of a blocking .map(); everything# else stays eager.@@ -5035,7 +5060,7 @@ class UnslothTrainer:is_cpt = training_args.get("is_cpt", False)train_on_responses_enabled = (False- if (is_cpt or raw_text_mode)+ if (is_cpt or raw_text_mode or user_labels)else training_args.get("train_on_completions", False))@@ -5045,6 +5070,10 @@ class UnslothTrainer:logger.info("Raw-text mode: skipping train_on_responses_only — training on all tokens\n")+ elif user_labels:+ logger.info(+ "Pre-tokenized dataset: its labels define the loss, no response masking\n"+ )# Markers auto-detected from the chat template first, with the manual table as fallback; gpt-oss stays on# manual markers. See apply_completion_masking.diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex d21d37c5865..9e45d19225a 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -3452,10 +3452,25 @@ def _run_mlx_training(event_queue, stop_queue, config):# Auto-detect markers from the chat template first, manual table as fallback. Mirror the CUDA skips: raw/CPT text# has no chat turns. Check the resolved format too, since format_type="auto" can land on alpaca or raw.+ from utils.datasets.pretokenized import has_custom_labels++ if (+ config.get("train_on_completions", False)+ and dataset_final_format == "pretokenized"+ and not has_custom_labels(dataset)+ ):+ # zoo's MLX masker re-tokenizes text, so token-id rows have nothing it can mask (CUDA masks input_ids).+ _send(+ "warning",+ message = (+ "'Train on completions' is not available for pre-tokenized datasets on Mac: "+ "training will run on full sequences. Add a labels column to choose the trained tokens."+ ),+ )if (config.get("train_on_completions", False)and not raw_text_mode- and dataset_final_format != "raw_text"+ and dataset_final_format not in ("raw_text", "pretokenized")):_send("status", status_message = "Configuring response-only training...")# No catch: the helper handles detection failures and double misses, so an exception here is a real maskingdiff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.pyindex 933d66fe77e..656be159e46 100644--- a/studio/backend/hub/utils/dataset_format.py+++ b/studio/backend/hub/utils/dataset_format.py@@ -8,6 +8,7 @@ from typing import Any, Optionalfrom utils.datasets.cells import text_cell_checkfrom utils.datasets.format_detection import detect_dataset_format, has_message_prompt_completion+from utils.datasets.pretokenized import PRETOKENIZED_FORMAT, is_pretokenized_datasetdef _first_row(dataset) -> Optional[dict]:@@ -795,6 +796,18 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict:"multimodal_columns": multimodal_info.get("multimodal_columns"),**audio_fields,}+ if is_pretokenized_dataset(dataset):+ return {+ "requires_manual_mapping": False,+ "detected_format": PRETOKENIZED_FORMAT,+ "columns": columns,+ "suggested_mapping": None,+ "detected_image_column": None,+ "detected_text_column": None,+ "is_image": multimodal_info["is_image"],+ "multimodal_columns": multimodal_info.get("multimodal_columns"),+ **audio_fields,+ }return {"requires_manual_mapping": True,"detected_format": "unknown",diff --git a/studio/backend/tests/test_pretokenized_dataset.py b/studio/backend/tests/test_pretokenized_dataset.pynew file mode 100644index 00000000000..2e01c0ab624--- /dev/null+++ b/studio/backend/tests/test_pretokenized_dataset.py@@ -0,0 +1,165 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Pre-tokenized datasets (input_ids / attention_mask / labels) route to training as-is (#7139), and nothing else moves."""++import inspect+import sys+from pathlib import Path++import pytest+from datasets import Dataset++_BACKEND_ROOT = Path(__file__).resolve().parents[1]+if str(_BACKEND_ROOT) not in sys.path:+ sys.path.insert(0, str(_BACKEND_ROOT))++from hub.utils.dataset_format import check_dataset_format as hub_check_dataset_format # noqa: E402+from utils.datasets.dataset_utils import ( # noqa: E402+ _format_dataset,+ check_dataset_format,+ format_and_template_dataset,+ format_dataset,+)+from utils.datasets.pretokenized import ( # noqa: E402+ has_custom_labels,+ is_pretokenized_dataset,+ prepare_pretokenized_dataset,+)+++def _pretokenized(with_labels = True, n = 4):+ rows = {+ "input_ids": [[1, 2, 3, 4, 5][: 3 + i % 3] for i in range(n)],+ "attention_mask": [[1] * (3 + i % 3) for i in range(n)],+ }+ if with_labels:+ rows["labels"] = [[-100] + ids[1:] for ids in rows["input_ids"]]+ return Dataset.from_dict(rows)+++NORMAL_DATASETS = {+ "alpaca": (+ Dataset.from_dict({"instruction": ["Say hi"], "input": [""], "output": ["hi"]}),+ "alpaca",+ ),+ "chatml": (+ Dataset.from_dict(+ {+ "messages": [+ [+ {"role": "user", "content": "Say hi"},+ {"role": "assistant", "content": "hi"},+ ]+ ]+ }+ ),+ "chatml",+ ),+ "sharegpt": (+ Dataset.from_dict(+ {+ "conversations": [+ [{"from": "human", "value": "Say hi"}, {"from": "gpt", "value": "hi"}]+ ]+ }+ ),+ "sharegpt",+ ),+ # A text dataset that also ships token ids keeps its text route.+ "question_answer_with_ids": (+ Dataset.from_dict({"question": ["Say hi"], "answer": ["hi"], "input_ids": [[1, 2, 3]]}),+ "custom_heuristic",+ ),+}++[email protected]("with_labels", [True, False])+def test_pretokenized_is_detected_and_needs_no_mapping(with_labels):+ ds = _pretokenized(with_labels)+ assert is_pretokenized_dataset(ds)+ for check in (check_dataset_format, hub_check_dataset_format):+ result = check(ds)+ assert result["requires_manual_mapping"] is False+ assert result["detected_format"] == "pretokenized"+++def test_format_passes_pretokenized_rows_through_untouched():+ ds = _pretokenized()+ info = format_and_template_dataset(ds, model_name = "unsloth/Qwen3-0.6B", tokenizer = None)+ assert info["success"] is True+ assert info["final_format"] == "pretokenized"+ assert info["dataset"].column_names == ds.column_names+ assert info["dataset"].to_dict() == ds.to_dict()++[email protected]("name", sorted(NORMAL_DATASETS))+def test_normal_datasets_keep_their_route(name):+ ds, expected = NORMAL_DATASETS[name]+ result = hub_check_dataset_format(ds)+ assert result["detected_format"] == expected+ assert check_dataset_format(ds)["detected_format"] == expected+ assert format_dataset(ds, format_type = "auto")["final_format"] != "pretokenized"+++ "rows",+ [+ {"input_ids": [["a", "b"]]},+ {"input_ids": [[]]},+ {"input_ids": [[1, 2, 3]], "labels": [[1, 2]]},+ {"tokens": [[1, 2, 3]]},+ ],+)+def test_malformed_token_rows_still_ask_for_mapping(rows):+ ds = Dataset.from_dict(rows)+ assert not is_pretokenized_dataset(ds)+ assert hub_check_dataset_format(ds)["requires_manual_mapping"] is True+++def test_prepare_truncates_every_token_column_and_is_idempotent():+ ds = _pretokenized()+ once = prepare_pretokenized_dataset(ds, max_seq_length = 3, vocab_size = 100)+ for column in ("input_ids", "attention_mask", "labels"):+ assert max(len(v) for v in once[column]) == 3+ assert once[0]["labels"] == [-100, 2, 3]+ twice = prepare_pretokenized_dataset(once, max_seq_length = 3, vocab_size = 100)+ assert twice.to_dict() == once.to_dict()+++def test_prepare_leaves_short_rows_alone():+ ds = _pretokenized()+ assert prepare_pretokenized_dataset(ds, max_seq_length = 2048, vocab_size = 100) is ds+++def test_prepare_rejects_ids_outside_the_vocabulary():+ with pytest.raises(ValueError, match = "outside this model's vocabulary"):+ prepare_pretokenized_dataset(_pretokenized(), max_seq_length = 2048, vocab_size = 4)+++def test_prepare_rejects_labels_of_another_length():+ ds = Dataset.from_dict({"input_ids": [[1, 2, 3]], "labels": [[1, 2]]})+ with pytest.raises(ValueError, match = "same length"):+ prepare_pretokenized_dataset(ds, max_seq_length = 2048, vocab_size = 100)+++def test_only_user_labels_turn_off_response_masking():+ assert has_custom_labels(_pretokenized(with_labels = True))+ assert not has_custom_labels(_pretokenized(with_labels = False))+ assert not has_custom_labels(_pretokenized(with_labels = False).to_iterable_dataset())++[email protected]("format_type", ["auto", "alpaca", "chatml", "sharegpt"])+def test_any_format_but_raw_routes_token_ids(format_type):+ info = format_dataset(_pretokenized(), format_type = format_type)+ assert info["final_format"] == "pretokenized"+ assert info["requires_manual_mapping"] is False+++def test_positional_custom_format_mapping_still_binds():+ params = inspect.signature(_format_dataset).parameters+ names = list(params)+ defaults = [params[n].default for n in names[1 : names.index("custom_format_mapping")]]+ assert format_dataset(_pretokenized(), *defaults, None)["final_format"] == "pretokenized"+ mapped = format_dataset(_pretokenized(), *defaults, {"input_ids": "user"})+ assert mapped["final_format"] != "pretokenized"diff --git a/studio/backend/utils/datasets/chat_templates.py b/studio/backend/utils/datasets/chat_templates.pyindex ed812fb5abd..b8c2eac2008 100644--- a/studio/backend/utils/datasets/chat_templates.py+++ b/studio/backend/utils/datasets/chat_templates.py@@ -488,6 +488,7 @@ def get_dataset_info_summary(dataset_info):"sharegpt": "ShareGPT format (needs standardization)","chatml_messages": "ChatML format (messages column) - OpenAI compatible","chatml_conversations": "ChatML format (conversations column) - HuggingFace standard",+ "pretokenized": "Pre-tokenized (input_ids / labels)","unknown": "Unknown format"}diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.pyindex 9745ced5af2..77bee72f0ec 100644--- a/studio/backend/utils/datasets/dataset_utils.py+++ b/studio/backend/utils/datasets/dataset_utils.py@@ -3,6 +3,8 @@"""Dataset utilities for format detection, conversion, and template application. Entry points: check_dataset_format (lightweight check whether manual mapping is needed, for the frontend), format_dataset (detects and normalizes dataset formats) and format_and_template_dataset (end-to-end processing with chat template). Internal utilities live in format_detection, format_conversion, chat_templates, vlm_processing, data_collators and model_mappings."""+import inspect+from .format_detection import (detect_dataset_format,detect_multimodal_dataset,@@ -26,6 +28,7 @@ from .chat_templates import ()from .cells import cell_text, cell_turns, message_list_columnsfrom .raw_text import prepare_raw_text_dataset+from .pretokenized import PRETOKENIZED_FORMAT, is_pretokenized_datasetfrom .vlm_processing import generate_smart_vlm_instructionfrom .data_collators import DeepSeekOCRDataCollator, VLMDataCollatorfrom .model_mappings import TEMPLATE_TO_MODEL_MAPPER@@ -115,6 +118,18 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict:"multimodal_columns": multimodal_info.get("multimodal_columns"),**audio_fields,}+ elif is_pretokenized_dataset(dataset):+ return {+ "requires_manual_mapping": False,+ "detected_format": PRETOKENIZED_FORMAT,+ "columns": columns,+ "suggested_mapping": None,+ "detected_image_column": None,+ "detected_text_column": None,+ "is_image": multimodal_info["is_image"],+ "multimodal_columns": multimodal_info.get("multimodal_columns"),+ **audio_fields,+ }else:return {"requires_manual_mapping": True,@@ -358,7 +373,31 @@ def _apply_user_mapping_alpaca()-def format_dataset(+def format_dataset(dataset, *args, **kwargs):+ """``_format_dataset``, but token-id rows no format matched route as pre-tokenized under any format except raw."""+ bound = _FORMAT_DATASET_SIGNATURE.bind(dataset, *args, **kwargs)+ bound.apply_defaults()+ info = _format_dataset(*bound.args, **bound.kwargs)+ if (+ info["final_format"] == "unknown"+ and bound.arguments["format_type"] != "raw"+ and not bound.arguments["custom_format_mapping"]+ and is_pretokenized_dataset(dataset)+ ):+ info = {+ **info,+ "dataset": dataset,+ "detected_format": PRETOKENIZED_FORMAT,+ "final_format": PRETOKENIZED_FORMAT,+ "chat_column": None,+ "is_standardized": True,+ "requires_manual_mapping": False,+ "warnings": [],+ }+ return info+++def _format_dataset(dataset,format_type = "auto",tokenizer = None,@@ -798,6 +837,9 @@ def format_dataset(raise ValueError(f"Unknown format_type: {format_type}")+_FORMAT_DATASET_SIGNATURE = inspect.signature(_format_dataset)++def format_and_template_dataset(dataset,model_name,@@ -1077,7 +1119,7 @@ def format_and_template_dataset(raw_text_column = raw_text_column,)- if dataset_info["final_format"] == "raw_text":+ if dataset_info["final_format"] in ("raw_text", PRETOKENIZED_FORMAT):summary = get_dataset_info_summary(dataset_info)return {"dataset": dataset_info["dataset"],diff --git a/studio/backend/utils/datasets/pretokenized.py b/studio/backend/utils/datasets/pretokenized.pynew file mode 100644index 00000000000..5793a4f31b4--- /dev/null+++ b/studio/backend/utils/datasets/pretokenized.py@@ -0,0 +1,110 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Pre-tokenized text datasets: rows that already hold ``input_ids`` (optionally ``attention_mask`` and ``labels``). They train as-is, without a chat template; user ``labels`` are the objective, so response masking applies only to rows without them. Only datasets no other format matched are claimed, so every dataset that trained before keeps its route."""++PRETOKENIZED_FORMAT = "pretokenized"+_TOKEN_COLUMNS = ("input_ids", "attention_mask", "labels", "token_type_ids")+_SAMPLE_ROWS = 20+++def _is_token_list(value) -> bool:+ return (+ isinstance(value, (list, tuple))+ and len(value) > 0+ and all(isinstance(x, int) and not isinstance(x, bool) for x in value)+ )+++def is_pretokenized_dataset(dataset) -> bool:+ from .format_detection import _sample_dataset_rows++ rows = _sample_dataset_rows(dataset, limit = _SAMPLE_ROWS)+ if not rows:+ return False+ for row in rows:+ if not isinstance(row, dict) or not _is_token_list(row.get("input_ids")):+ return False+ labels = row.get("labels")+ if labels is not None and (+ not _is_token_list(labels) or len(labels) != len(row["input_ids"])+ ):+ return False+ return True+++def _column_names(dataset) -> list[str]:+ names = getattr(dataset, "column_names", None)+ if names:+ return list(names)+ return list(next(iter(dataset)).keys())+++def has_custom_labels(dataset) -> bool:+ return "labels" in _column_names(dataset)+++def _check_arrow_table(table, max_seq_length, vocab_size) -> bool:+ import pyarrow.compute as pc++ ids = table.column("input_ids")+ lengths = pc.list_value_length(ids)+ if "labels" in table.column_names:+ labels = table.column("labels")+ if not pc.all(pc.equal(lengths, pc.list_value_length(labels))).as_py():+ raise ValueError(+ "Pre-tokenized dataset: 'labels' must have the same length as 'input_ids' in every row."+ )+ if vocab_size:+ flat = pc.list_flatten(ids)+ bounds = pc.min_max(flat).as_py() if len(flat) else {"min": 0, "max": 0}+ if bounds["min"] < 0 or bounds["max"] >= vocab_size:+ bad = bounds["min"] if bounds["min"] < 0 else bounds["max"]+ raise ValueError(+ f"Pre-tokenized dataset: token id {bad} is outside this model's vocabulary "+ f"(size {vocab_size}). The dataset was likely "+ "tokenized for a different model; re-tokenize it with this model's tokenizer."+ )+ longest = pc.max(lengths).as_py() or 0+ return longest > max_seq_length+++def prepare_pretokenized_dataset(+ dataset,+ *,+ max_seq_length,+ vocab_size = None,+):+ """Unsloth's SFT preparation passes tokenized rows through untouched: an over-long row would reach the model whole and an id past the embedding table is a device-side assert mid-run. Idempotent."""+ columns = [c for c in _TOKEN_COLUMNS if c in _column_names(dataset)]+ table = getattr(dataset, "data", None)+ needs_truncation = True+ if table is not None and hasattr(table, "column"):+ # Covers the whole backing table, rows a select() hid included: a bad id there is still a bad dataset.+ needs_truncation = _check_arrow_table(table, max_seq_length, vocab_size)+ if not needs_truncation or not max_seq_length:+ return dataset++ if table is None:++ def _truncate(batch):+ return {+ c: [v[:max_seq_length] if v is not None else v for v in batch[c]] for c in columns+ }++ return dataset.map(_truncate, batched = True)++ import pyarrow.compute as pc++ def _truncate_arrow(batch):+ # Arrow slices in C: 46x faster than slicing Python lists on 52M tokens.+ for c in columns:+ batch = batch.set_column(+ batch.column_names.index(c), c, pc.list_slice(batch.column(c), 0, max_seq_length)+ )+ return batch++ out = dataset.with_format("arrow").map(+ _truncate_arrow, batched = True, desc = "Truncating pre-tokenized rows"+ )+ return out.with_format(dataset.format["type"])