Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Dataset utilities for format detection, conversion, and template application. Entry points: check_dataset_format (lightweight check whether manual mapping is needed, for the frontend), format_dataset (detects and normalizes dataset formats) and format_and_template_dataset (end-to-end processing with chat template). Internal utilities live in format_detection, format_conversion, chat_templates, vlm_processing, data_collators and model_mappings."""
import inspect
from .format_detection import (
detect_dataset_format,
detect_multimodal_dataset,
detect_vlm_dataset_structure,
detect_custom_format_heuristic,
is_plain_text_dataset,
detect_video_columns,
detect_video_structure,
video_sample,
_VIDEO_TEXT_KEYWORDS,
)
from .format_conversion import (
standardize_chat_format,
convert_chatml_to_alpaca,
convert_alpaca_to_chatml,
convert_to_vlm_format,
convert_llava_to_vlm_format,
convert_sharegpt_with_images_to_vlm_format,
convert_video_to_vlm_format,
own_message_videos,
normalize_chatml_roles,
)
from .chat_templates import (
_custom_prompt_template_error,
apply_chat_template_to_dataset,
get_dataset_info_summary,
get_tokenizer_chat_template,
DEFAULT_ALPACA_TEMPLATE,
)
from .cells import cell_text, cell_turns, message_list_columns, unwrap_message_column
from .raw_text import prepare_raw_text_dataset
from .pretokenized import PRETOKENIZED_FORMAT, is_pretokenized_dataset
from .vlm_processing import generate_smart_vlm_instruction
from .data_collators import DeepSeekOCRDataCollator, VLMDataCollator
from .model_mappings import TEMPLATE_TO_MODEL_MAPPER
from loggers import get_logger
logger = get_logger(__name__)
def check_dataset_format(dataset, is_vlm: bool = False) -> dict:
"""check whether frontend column mapping is required without processing the dataset."""
columns = (
list(dataset.column_names)
if hasattr(dataset, "column_names")
else list(next(iter(dataset)).keys())
)
multimodal_info = detect_multimodal_dataset(dataset)
is_audio = multimodal_info.get("is_audio", False)
audio_fields = {
"is_audio": is_audio,
"detected_audio_column": multimodal_info.get("detected_audio_column"),
"detected_speaker_column": multimodal_info.get("detected_speaker_column"),
}
if is_vlm:
vlm_structure = detect_vlm_dataset_structure(dataset)
requires_mapping = vlm_structure["format"] == "unknown"
warning = None
if requires_mapping:
img_col = vlm_structure.get("image_column")
txt_col = vlm_structure.get("text_column")
missing = []
if not img_col:
missing.append("image")
if not txt_col:
missing.append("text")
if missing:
warning = (
f"Could not auto-detect {' or '.join(missing)} column. "
"Please assign image and text columns manually."
)
return {
"requires_manual_mapping": requires_mapping,
"detected_format": vlm_structure["format"],
"columns": columns,
"suggested_mapping": None,
"detected_image_column": vlm_structure.get("image_column"),
"detected_text_column": vlm_structure.get("text_column"),
"is_image": multimodal_info["is_image"],
"multimodal_columns": multimodal_info.get("multimodal_columns"),
"warning": warning,
**audio_fields,
}
if is_audio:
detected_audio = multimodal_info.get("detected_audio_column")
detected_text = multimodal_info.get("detected_text_column")
needs_mapping = not detected_audio or not detected_text
return {
"requires_manual_mapping": needs_mapping,
"detected_format": "audio",
"columns": columns,
"suggested_mapping": None,
"detected_image_column": None,
"detected_text_column": multimodal_info.get("detected_text_column"),
"is_image": False,
"multimodal_columns": multimodal_info.get("audio_columns"),
**audio_fields,
}
detected = detect_dataset_format(dataset)
if detected["format"] == "unknown":
heuristic_mapping = detect_custom_format_heuristic(dataset)
if heuristic_mapping:
return {
"requires_manual_mapping": False,
"detected_format": "custom_heuristic",
"columns": columns,
"suggested_mapping": heuristic_mapping,
"detected_image_column": None,
"detected_text_column": None,
"is_image": multimodal_info["is_image"],
"multimodal_columns": multimodal_info.get("multimodal_columns"),
**audio_fields,
}
elif is_pretokenized_dataset(dataset):
return {
"requires_manual_mapping": False,
"detected_format": PRETOKENIZED_FORMAT,
"columns": columns,
"suggested_mapping": None,
"detected_image_column": None,
"detected_text_column": None,
"is_image": multimodal_info["is_image"],
"multimodal_columns": multimodal_info.get("multimodal_columns"),
**audio_fields,
}
else:
return {
"requires_manual_mapping": True,
"detected_format": "unknown",
"columns": columns,
"suggested_mapping": None,
"detected_image_column": None,
"detected_text_column": None,
"is_image": multimodal_info["is_image"],
"multimodal_columns": multimodal_info.get("multimodal_columns"),
"warning": (
f"Could not auto-detect column roles for columns: {columns}. "
"Please assign roles manually, or use AI Assist."
),
**audio_fields,
}
return {
"requires_manual_mapping": False,
"detected_format": detected["format"],
"columns": columns,
"suggested_mapping": None,
"detected_image_column": None,
"detected_text_column": None,
"chat_column": detected.get("chat_column"),
"is_image": multimodal_info["is_image"],
"multimodal_columns": multimodal_info.get("multimodal_columns"),
**audio_fields,
}
# Normalise any format-specific role to canonical chatml (user/assistant/system)
_TO_CHATML = {
"user": "user",
"human": "user",
"instruction": "user",
"assistant": "assistant",
"gpt": "assistant",
"output": "assistant",
"system": "system",
"input": "system",
}
_CHATML_ROLE_ORDER = ("system", "user", "assistant")
_CHATML_TO_ALPACA = {"user": "instruction", "system": "input", "assistant": "output"}
_KNOWN_CHAT_COLUMNS = {"messages", "conversations", "texts"}
def _mapped_message_columns(dataset, columns):
messages = message_list_columns(dataset)
prompt_completion = {"prompt", "completion"}
if prompt_completion <= set(columns) and messages & prompt_completion:
messages.update(prompt_completion)
return messages
def _chatml_final_format(chat_column: str | None) -> str:
return "chatml_messages" if chat_column == "messages" else "chatml_conversations"
def _chatml_detected_format_label(chat_column: str | None) -> str:
if chat_column in _KNOWN_CHAT_COLUMNS:
return f"chatml_{chat_column}"
return "chatml_conversations"
def _apply_user_mapping(
dataset,
mapping: dict,
batch_size: int = 1000,
):
"""normalize chatml, ShareGPT, or Alpaca roles; ``__`` metadata selects template conversion."""
meta = {k: v for k, v in mapping.items() if k.startswith("__")}
column_roles = {k: v for k, v in mapping.items() if not k.startswith("__")}
if meta:
return _apply_template_mapping(dataset, column_roles, meta, batch_size)
role_groups: dict[str, list[str]] = {r: [] for r in _CHATML_ROLE_ORDER}
for col_name, role in column_roles.items():
canonical = _TO_CHATML.get(role)
if canonical:
role_groups[canonical].append(col_name)
message_columns = _mapped_message_columns(dataset, column_roles)
def _convert(examples):
num = len(next(iter(examples.values())))
conversations = []
for i in range(num):
convo = []
for chatml_role in _CHATML_ROLE_ORDER:
for col in role_groups[chatml_role]:
if col in examples:
convo.extend(
cell_turns(
examples[col][i],
chatml_role,
empty_is_messages = col in message_columns,
)
)
conversations.append(convo)
mapped = {"conversations": conversations}
if "tools" in examples:
mapped["tools"] = examples["tools"]
return mapped
return dataset.map(
_convert,
batched = True,
batch_size = batch_size,
remove_columns = dataset.column_names,
)
def _extract_column_value(val, col: str, label_mapping: dict) -> str:
if isinstance(val, dict):
if "text" not in val:
return cell_text(val)
inner = val["text"]
val = inner[0] if isinstance(inner, list) and inner else inner
if isinstance(val, list) and any(isinstance(item, (dict, list)) for item in val):
return cell_text(val)
names = label_mapping.get(col)
if not isinstance(names, dict):
names = {}
values = val if isinstance(val, list) else [val]
return ", ".join(names.get(text, text) for text in map(cell_text, values))
def _apply_template_mapping(
dataset,
column_roles: dict,
meta: dict,
batch_size: int = 1000,
):
"""Apply advisor-driven mapping for non-conversational datasets: groups columns by assigned role (system/user/assistant), concatenates values within each role into one message, and injects an optional system prompt ahead of any system column. Label mapping converts integer labels to human-readable strings. Returns a dataset with a single 'conversations' column."""
system_prompt = meta.get("__system_prompt", "")
label_mapping = meta.get("__label_mapping", {}) # {col: {int_str: label_str}}
role_groups: dict[str, list[str]] = {"system": [], "user": [], "assistant": []}
for col, role in column_roles.items():
canonical = _TO_CHATML.get(role, role)
if canonical in role_groups:
role_groups[canonical].append(col)
import logging as _log
_log.getLogger(__name__).info(
f"Applying role mapping: sys={bool(system_prompt)}, "
f"user_cols={role_groups['user']}, asst_cols={role_groups['assistant']}, "
f"label_map={list(label_mapping.keys())}"
)
def _convert(examples):
num = len(next(iter(examples.values())))
conversations = []
for i in range(num):
convo = []
# The system prompt leads, then system columns, so neither is dropped.
system_parts = [
_extract_column_value(examples[col][i], col, label_mapping)
for col in role_groups["system"]
if col in examples
]
system_text = "\n\n".join(
part for part in (system_prompt, "\n".join(p for p in system_parts if p)) if part
)
if system_text:
convo.append({"role": "system", "content": system_text})
user_parts = []
for col in role_groups["user"]:
if col in examples:
user_parts.append(_extract_column_value(examples[col][i], col, label_mapping))
if user_parts:
convo.append({"role": "user", "content": "\n".join(user_parts)})
asst_parts = []
for col in role_groups["assistant"]:
if col in examples:
asst_parts.append(_extract_column_value(examples[col][i], col, label_mapping))
if asst_parts:
convo.append({"role": "assistant", "content": "\n".join(asst_parts)})
conversations.append(convo)
mapped = {"conversations": conversations}
if "tools" in examples:
mapped["tools"] = examples["tools"]
return mapped
return dataset.map(
_convert,
batched = True,
batch_size = batch_size,
remove_columns = dataset.column_names,
)
def _apply_user_mapping_alpaca(
dataset,
mapping: dict,
batch_size: int = 1000,
):
"""__system_prompt is prepended to instruction, since a mapped system column already fills input."""
meta = {k: v for k, v in mapping.items() if k.startswith("__")}
column_roles = {k: v for k, v in mapping.items() if not k.startswith("__")}
system_prompt = meta.get("__system_prompt", "")
label_mapping = meta.get("__label_mapping", {})
cols_for: dict[str, list[str]] = {
"instruction": [],
"input": [],
"output": [],
}
for col_name, role in column_roles.items():
canonical = _TO_CHATML.get(role)
alpaca_field = _CHATML_TO_ALPACA.get(canonical) if canonical else None
if alpaca_field:
cols_for[alpaca_field].append(col_name)
def _convert(examples):
num = len(next(iter(examples.values())))
instructions, inputs, outputs = [], [], []
for i in range(num):
for field, dest in (
("instruction", instructions),
("input", inputs),
("output", outputs),
):
val = "\n".join(
_extract_column_value(examples[col][i], col, label_mapping)
for col in cols_for[field]
if col in examples
)
if field == "instruction" and system_prompt:
val = "\n\n".join(part for part in (system_prompt, val) if part)
dest.append(val)
return {"instruction": instructions, "input": inputs, "output": outputs}
return dataset.map(
_convert,
batched = True,
batch_size = batch_size,
remove_columns = dataset.column_names,
)
def format_dataset(dataset, *args, **kwargs):
"""``_format_dataset``, but token-id rows no format matched route as pre-tokenized under any format except raw."""
bound = _FORMAT_DATASET_SIGNATURE.bind(dataset, *args, **kwargs)
bound.apply_defaults()
info = _format_dataset(*bound.args, **bound.kwargs)
if (
info["final_format"] == "unknown"
and bound.arguments["format_type"] != "raw"
and not bound.arguments["custom_format_mapping"]
and is_pretokenized_dataset(dataset)
):
info = {
**info,
"dataset": dataset,
"detected_format": PRETOKENIZED_FORMAT,
"final_format": PRETOKENIZED_FORMAT,
"chat_column": None,
"is_standardized": True,
"requires_manual_mapping": False,
"warnings": [],
}
if info["final_format"] in ("chatml_messages", "chatml_conversations") and info["chat_column"]:
info["dataset"] = normalize_chatml_roles(
info["dataset"], info["chat_column"], bound.arguments["batch_size"]
)
return info
def _format_dataset(
dataset,
format_type = "auto",
tokenizer = None,
aliases_for_system = [
"system",
],
aliases_for_user = [
"user",
"human",
"input",
],
aliases_for_assistant = [
"gpt",
"assistant",
"output",
],
batch_size = 1000,
num_proc = None,
auto_detect_custom = True,
custom_format_mapping = None,
split_name = None,
raw_text_column = None,
):
"""Formats dataset and returns {dataset, detected_format, final_format, chat_column, is_standardized, requires_manual_mapping, warnings}."""
multimodal_info = detect_multimodal_dataset(dataset)
auto_raw = (
format_type == "auto" and not custom_format_mapping and is_plain_text_dataset(dataset)
)
if auto_raw:
# Matches the preview's "raw" verdict; auto would otherwise fail as "unknown".
format_type = "raw"
if format_type == "raw":
# Explicit "raw" already gets EOS from the trainer / MLXTrainingConfig.append_eos.
raw_result = prepare_raw_text_dataset(
dataset,
split_name = split_name,
# A processor without Unsloth's patch (MLX VLM) keeps eos_token on its inner tokenizer.
eos_token = getattr(tokenizer, "eos_token", None)
or getattr(getattr(tokenizer, "tokenizer", None), "eos_token", None),
append_eos = auto_raw,
text_column = raw_text_column,
)
return {
"dataset": raw_result.dataset,
"detected_format": "raw_text",
"final_format": "raw_text",
"chat_column": "text",
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [notice.message for notice in raw_result.notices],
"raw_text_column": raw_result.source_column,
"run_warnings": [
notice.message
for notice in raw_result.notices
if notice.level == "warning" and notice.update_status
],
}
if custom_format_mapping:
try:
if format_type == "alpaca":
mapped_dataset = _apply_user_mapping_alpaca(
dataset, custom_format_mapping, batch_size
)
final_format = "alpaca"
chat_column = None
else:
# auto / chatml / sharegpt / conversational all produce chatml conversations (sharegpt standardized to role/content internally)
mapped_dataset = _apply_user_mapping(dataset, custom_format_mapping, batch_size)
final_format = "chatml_conversations"
chat_column = "conversations"
return {
"dataset": mapped_dataset,
"detected_format": "user_mapped",
"final_format": final_format,
"chat_column": chat_column,
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [
f"Applied user-provided column mapping ({format_type}): {custom_format_mapping}"
],
}
except Exception as e:
return {
"dataset": dataset,
"detected_format": "user_mapped",
"final_format": "unknown",
"chat_column": None,
"is_standardized": False,
"requires_manual_mapping": True,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [f"Failed to apply user mapping: {e}"],
}
detected = detect_dataset_format(dataset)
dataset = unwrap_message_column(dataset, detected["chat_column"], batch_size)
warnings = []
if multimodal_info["is_image"]:
warnings.append(
f"Multimodal dataset detected. Found columns: {multimodal_info['multimodal_columns']}"
)
if format_type == "auto":
if detected["format"] == "alpaca":
return {
"dataset": dataset,
"detected_format": "alpaca",
"final_format": "alpaca",
"chat_column": None,
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [],
}
elif detected["format"] == "sharegpt" and detected.get("chat_column"):
try:
standardized = standardize_chat_format(
dataset,
tokenizer,
aliases_for_system,
aliases_for_user,
aliases_for_assistant,
batch_size,
num_proc,
chat_column = detected["chat_column"],
)
return {
"dataset": standardized,
"detected_format": "sharegpt",
"final_format": _chatml_final_format(detected["chat_column"]),
"chat_column": detected["chat_column"],
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [],
}
except Exception as e:
warnings.append(f"Failed to standardize ShareGPT format: {e}")
return {
"dataset": dataset,
"detected_format": "sharegpt",
"final_format": "sharegpt",
"chat_column": detected["chat_column"],
"is_standardized": False,
"requires_manual_mapping": True,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
elif detected["format"] == "chatml" and detected.get("chat_column"):
return {
"dataset": dataset,
"detected_format": _chatml_detected_format_label(detected["chat_column"]),
"final_format": _chatml_final_format(detected["chat_column"]),
"chat_column": detected["chat_column"],
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
else:
warnings.append(f"Unknown format detected. Keys found: {detected['sample_keys']}")
if auto_detect_custom:
custom_mapping = detect_custom_format_heuristic(dataset)
if custom_mapping:
warnings.append(f"Auto-detected column mapping: {custom_mapping}")
message_columns = _mapped_message_columns(dataset, custom_mapping)
def _apply_auto_mapping(examples):
conversations = []
num_examples = len(examples[list(examples.keys())[0]])
all_columns = set(examples.keys())
mapped_columns = set(custom_mapping.keys())
preserved_columns = {
col: examples[col]
for col in all_columns - mapped_columns
if col != "conversations"
}
for i in range(num_examples):
convo = []
for target_role in ["system", "user", "assistant"]:
for col_name, role in custom_mapping.items():
if role == target_role and col_name in examples:
convo.extend(
turn
for turn in cell_turns(
examples[col_name][i],
role,
empty_is_messages = col_name in message_columns,
)
if turn.get("tool_calls") or turn["content"].strip()
)
conversations.append(convo)
return {**preserved_columns, "conversations": conversations}
try:
dataset = dataset.map(
_apply_auto_mapping, batched = True, batch_size = batch_size
)
return {
"dataset": dataset,
"detected_format": "unknown",
"final_format": "chatml_conversations",
"chat_column": "conversations",
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
except Exception as e:
warnings.append(f"Auto-detection failed: {e}")
if detected["chat_column"]:
try:
standardized = standardize_chat_format(
dataset,
tokenizer,
aliases_for_system,
aliases_for_user,
aliases_for_assistant,
batch_size,
num_proc,
chat_column = detected["chat_column"],
)
warnings.append("Successfully standardized unknown format")
return {
"dataset": standardized,
"detected_format": "unknown",
"final_format": _chatml_final_format(detected["chat_column"]),
"chat_column": detected["chat_column"],
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
except Exception as e:
warnings.append(f"Could not standardize: {e}. Passing dataset as-is.")
return {
"dataset": dataset,
"detected_format": "unknown",
"final_format": "unknown",
"chat_column": detected["chat_column"],
"is_standardized": False,
"requires_manual_mapping": True,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
elif format_type == "alpaca":
if detected["format"] == "alpaca":
return {
"dataset": dataset,
"detected_format": "alpaca",
"final_format": "alpaca",
"chat_column": None,
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [],
}
elif detected["format"] in ["sharegpt", "chatml"] and detected.get("chat_column"):
try:
if detected["format"] == "sharegpt":
dataset = standardize_chat_format(
dataset,
tokenizer,
aliases_for_system,
aliases_for_user,
aliases_for_assistant,
batch_size,
num_proc,
chat_column = detected["chat_column"],
)
converted = convert_chatml_to_alpaca(
dataset,
batch_size,
num_proc,
chat_column = detected["chat_column"],
)
return {
"dataset": converted,
"detected_format": detected["format"],
"final_format": "alpaca",
"chat_column": None,
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [],
}
except Exception as e:
warnings.append(f"Failed to convert chat dataset to Alpaca: {e}")
return {
"dataset": dataset,
"detected_format": detected["format"],
"final_format": "unknown",
"chat_column": detected["chat_column"],
"is_standardized": False,
"requires_manual_mapping": True,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
else:
warnings.append(f"Cannot convert unknown format to Alpaca")
return {
"dataset": dataset,
"detected_format": "unknown",
"final_format": "unknown",
"chat_column": detected["chat_column"],
"is_standardized": False,
"requires_manual_mapping": True,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
elif format_type in ["chatml", "conversational", "sharegpt"]:
if detected["format"] == "alpaca":
converted = convert_alpaca_to_chatml(dataset, batch_size, num_proc)
return {
"dataset": converted,
"detected_format": "alpaca",
"final_format": "chatml_conversations",
"chat_column": "conversations",
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [],
}
elif detected["format"] == "sharegpt" and detected.get("chat_column"):
try:
standardized = standardize_chat_format(
dataset,
tokenizer,
aliases_for_system,
aliases_for_user,
aliases_for_assistant,
batch_size,
num_proc,
chat_column = detected["chat_column"],
)
return {
"dataset": standardized,
"detected_format": "sharegpt",
"final_format": _chatml_final_format(detected["chat_column"]),
"chat_column": detected["chat_column"],
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [],
}
except Exception as e:
warnings.append(f"Failed to standardize ShareGPT format: {e}")
return {
"dataset": dataset,
"detected_format": "sharegpt",
"final_format": "sharegpt",
"chat_column": detected["chat_column"],
"is_standardized": False,
"requires_manual_mapping": True,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
elif detected["format"] == "chatml" and detected.get("chat_column"):
return {
"dataset": dataset,
"detected_format": _chatml_detected_format_label(detected["chat_column"]),
"final_format": _chatml_final_format(detected["chat_column"]),
"chat_column": detected["chat_column"],
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": [],
}
else:
warnings.append(f"Unknown format, attempting standardization")
if detected["chat_column"]:
try:
standardized = standardize_chat_format(
dataset,
tokenizer,
aliases_for_system,
aliases_for_user,
aliases_for_assistant,
batch_size,
num_proc,
chat_column = detected["chat_column"],
)
return {
"dataset": standardized,
"detected_format": "unknown",
"final_format": _chatml_final_format(detected["chat_column"]),
"chat_column": detected["chat_column"],
"is_standardized": True,
"requires_manual_mapping": False,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
except Exception as e:
warnings.append(f"Standardization failed: {e}")
return {
"dataset": dataset,
"detected_format": "unknown",
"final_format": "unknown",
"chat_column": detected["chat_column"],
"is_standardized": False,
"requires_manual_mapping": True,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"warnings": warnings,
}
else:
raise ValueError(f"Unknown format_type: {format_type}")
_FORMAT_DATASET_SIGNATURE = inspect.signature(_format_dataset)
def format_and_template_dataset(
dataset,
model_name,
tokenizer,
is_vlm = False,
format_type = "auto",
vlm_instruction = None,
vlm_text_column = None,
vlm_image_column = None,
dataset_name = None,
custom_prompt_template = None,
add_eos_token = False,
remove_bos_prefix = False,
custom_format_mapping = None,
auto_detect_custom = True,
auto_detect_mapping = True,
aliases_for_system = [
"system",
],
aliases_for_user = [
"user",
"human",
"input",
],
aliases_for_assistant = [
"gpt",
"assistant",
"output",
],
batch_size = 1000,
num_proc = None,
progress_callback = None,
split_name = None,
raw_text_column = None,
dataset_revision = None,
):
"""Combines format_dataset and apply_chat_template_to_dataset, for UI workflows where one call does everything. custom_prompt_template is retained for signature compatibility; non-None values are rejected because Studio cannot persist a matching inference template. Returns {dataset (with a 'text' column), detected_format, final_format, success, requires_manual_mapping, warnings, errors, summary}."""
from .tokenizer_fingerprint import install_tokenizer_fingerprint
install_tokenizer_fingerprint()
custom_prompt_error = _custom_prompt_template_error(custom_prompt_template)
if custom_prompt_error:
return {
"dataset": dataset,
"detected_format": "unknown",
"final_format": "unknown",
"chat_column": None,
"is_vlm": is_vlm,
"success": False,
"requires_manual_mapping": False,
"warnings": [],
"errors": [custom_prompt_error],
"summary": None,
}
if is_vlm:
warnings = []
errors = []
multimodal_info = detect_multimodal_dataset(dataset)
if custom_format_mapping:
user_vlm_image_column = None
user_vlm_text_column = None
for col, role in custom_format_mapping.items():
if role == "image":
user_vlm_image_column = col
elif role in ["text", "user", "caption", "assistant"]:
user_vlm_text_column = col
# The text-role loop above keeps the last one; a clip needs the answer and question kept apart.
video_answer_column = next(
(
c
for c, r in custom_format_mapping.items()
if r in ("assistant", "text", "caption")
),
user_vlm_text_column,
)
video_question_column = next(
(
c
for c, r in custom_format_mapping.items()
if r == "user" and c != video_answer_column
),
None,
)
first = next(iter(dataset), None)
probe = video_sample(dataset, first) if isinstance(first, dict) else {}
mapped_video = (
user_vlm_image_column
and user_vlm_text_column
and user_vlm_image_column in detect_video_columns(probe)
)
if mapped_video and video_question_column is None:
# The VLM mapping UI offers only image and text roles, so take the question from detection.
detected_question = (detect_video_structure(probe) or {}).get("question_column")
if detected_question not in (None, video_answer_column, user_vlm_image_column):
video_question_column = detected_question
if mapped_video:
# The image role on a clip column: the image converter would try to open the clip.
try:
dataset = convert_video_to_vlm_format(
dataset,
video_column = user_vlm_image_column,
text_column = video_answer_column,
question_column = video_question_column,
instruction = vlm_instruction,
dataset_name = dataset_name,
dataset_revision = dataset_revision,
)
except Exception as e:
errors.append(f"Failed to convert the mapped video + text columns: {e}")
return {
"dataset": dataset,
"detected_format": "user_mapped",
"final_format": "vlm_conversion_failed",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
warnings.append(
f"Applied user VLM mapping: video='{user_vlm_image_column}', text='{video_answer_column}'"
)
return {
"dataset": dataset,
"detected_format": "user_mapped",
"final_format": "vlm_messages",
"chat_column": "messages",
"is_vlm": True,
"is_image": True,
"multimodal_info": multimodal_info,
"success": True,
"requires_manual_mapping": False,
"warnings": warnings,
"errors": [],
}
if user_vlm_image_column and user_vlm_text_column:
try:
dataset = convert_to_vlm_format(
dataset,
instruction = vlm_instruction,
text_column = user_vlm_text_column,
image_column = user_vlm_image_column,
dataset_name = dataset_name,
progress_callback = progress_callback,
)
warnings.append(
f"Applied user VLM mapping: image='{user_vlm_image_column}', text='{user_vlm_text_column}'"
)
return {
"dataset": dataset,
"detected_format": "user_mapped",
"final_format": "vlm_messages",
"chat_column": "messages",
"is_vlm": True,
"is_image": True,
"multimodal_info": multimodal_info,
"success": True,
"requires_manual_mapping": False,
"warnings": warnings,
"errors": [],
}
except Exception as e:
# User mapping failed; fall back to auto-detection (handles stale cached mappings).
warnings.append(
f"User VLM mapping (image='{user_vlm_image_column}', "
f"text='{user_vlm_text_column}') failed: {e} — "
f"falling back to auto-detection"
)
logger.info(f"⚠️ User VLM mapping failed, falling back to auto-detection...")
custom_format_mapping = None
else:
errors.append(
f"Invalid VLM mapping: need 'image' and 'text' roles. Got: {custom_format_mapping}"
)
return {
"dataset": dataset,
"detected_format": "user_mapped",
"final_format": "vlm_unknown",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
vlm_structure = detect_vlm_dataset_structure(dataset)
if vlm_structure["format"] == "vlm_messages_llava":
try:
dataset = convert_llava_to_vlm_format(dataset)
warnings.append(
"Converted from Llava format (image indices) to standard VLM format"
)
except Exception as e:
errors.append(f"Failed to convert Llava format: {e}")
import traceback
traceback.print_exc()
return {
"dataset": dataset,
"detected_format": "vlm_messages_llava",
"final_format": "vlm_conversion_failed",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
elif vlm_structure["format"] == "sharegpt_with_images":
try:
dataset = convert_sharegpt_with_images_to_vlm_format(
dataset,
image_column = vlm_structure["image_column"],
messages_column = vlm_structure["messages_column"],
dataset_name = dataset_name,
progress_callback = progress_callback,
)
warnings.append("Converted from ShareGPT+image format to standard VLM format")
except Exception as e:
errors.append(f"Failed to convert ShareGPT+image format: {e}")
import traceback
traceback.print_exc()
return {
"dataset": dataset,
"detected_format": "sharegpt_with_images",
"final_format": "vlm_conversion_failed",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
elif vlm_structure["format"] == "video_text" and vlm_structure.get("image_columns"):
errors.append(
f"Rows mix video ('{vlm_structure['video_column']}') with image columns "
f"{vlm_structure['image_columns']}; video conversion would drop the images, so remove "
"those columns or split the dataset."
)
return {
"dataset": dataset,
"detected_format": "video_text",
"final_format": "vlm_conversion_failed",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
elif vlm_structure["format"] == "video_text":
try:
dataset = convert_video_to_vlm_format(
dataset,
video_column = vlm_structure["video_column"],
text_column = vlm_structure["text_column"],
question_column = vlm_structure.get("question_column"),
instruction = vlm_instruction,
video_columns = vlm_structure.get("video_columns"),
dataset_name = dataset_name,
dataset_revision = dataset_revision,
)
except Exception as e:
errors.append(f"Failed to convert video + text rows to VLM format: {e}")
return {
"dataset": dataset,
"detected_format": "video_text",
"final_format": "vlm_conversion_failed",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
elif vlm_structure["format"] == "unknown" and vlm_structure.get("video_column"):
errors.append(
f"Found videos in '{vlm_structure['video_column']}' but no answer column; name the answer column with one of these words: "
+ ", ".join(_VIDEO_TEXT_KEYWORDS)
)
return {
"dataset": dataset,
"detected_format": "unknown",
"final_format": "vlm_unknown",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
elif vlm_structure["needs_conversion"]:
if vlm_text_column is None:
vlm_text_column = vlm_structure["text_column"]
if vlm_image_column is None:
vlm_image_column = vlm_structure["image_column"]
if vlm_text_column is None or vlm_image_column is None:
columns = list(next(iter(dataset)).keys()) if dataset else []
issues = [
f"Could not auto-detect image and text columns from: {columns}",
f"VLM structure detected: {vlm_structure.get('format', 'unknown')}",
]
friendly = None
try:
from .llm_assist import llm_generate_dataset_warning
friendly = llm_generate_dataset_warning(
issues,
dataset_name = dataset_name,
modality = "vision",
column_names = columns,
)
except Exception:
pass
errors.append(
friendly
or f"Could not auto-detect image/text columns. Found: {vlm_structure}. "
)
return {
"dataset": dataset,
"detected_format": "vlm_unknown",
"final_format": "vlm_unknown",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
try:
dataset = convert_to_vlm_format(
dataset,
instruction = vlm_instruction,
text_column = vlm_text_column,
image_column = vlm_image_column,
dataset_name = dataset_name,
progress_callback = progress_callback,
)
if vlm_instruction:
warnings.append(f"Using user-provided instruction: '{vlm_instruction}'")
else:
warnings.append("Auto-generated instruction based on dataset analysis")
except Exception as e:
errors.append(f"Failed to convert to VLM format: {e}")
import traceback
traceback.print_exc()
return {
"dataset": dataset,
"detected_format": vlm_structure["format"],
"final_format": "vlm_conversion_failed",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
elif vlm_structure["format"] == "vlm_messages":
try:
dataset = own_message_videos(
dataset, dataset_name = dataset_name, dataset_revision = dataset_revision
)
except Exception as e:
errors.append(f"Failed to read the video parts of VLM messages rows: {e}")
return {
"dataset": dataset,
"detected_format": "vlm_messages",
"final_format": "vlm_conversion_failed",
"is_vlm": True,
"success": False,
"requires_manual_mapping": True,
"warnings": warnings,
"errors": errors,
}
warnings.append("Dataset already in standard VLM messages format")
return {
"dataset": dataset,
"detected_format": vlm_structure["format"],
"final_format": "vlm_messages",
"chat_column": "messages",
"is_vlm": True,
"is_image": multimodal_info["is_image"],
"multimodal_info": multimodal_info,
"vlm_structure": vlm_structure,
"success": True,
"requires_manual_mapping": False,
"warnings": warnings,
"errors": errors,
}
else:
n_rows = len(dataset) if hasattr(dataset, "__len__") else None
if progress_callback and n_rows:
progress_callback(status_message = f"Formatting dataset ({n_rows:,} rows)...")
dataset_info = format_dataset(
dataset,
format_type = format_type,
tokenizer = tokenizer,
auto_detect_custom = auto_detect_custom,
custom_format_mapping = custom_format_mapping,
aliases_for_system = aliases_for_system,
aliases_for_user = aliases_for_user,
aliases_for_assistant = aliases_for_assistant,
batch_size = batch_size,
num_proc = num_proc,
split_name = split_name,
raw_text_column = raw_text_column,
)
if dataset_info["final_format"] in ("raw_text", PRETOKENIZED_FORMAT):
summary = get_dataset_info_summary(dataset_info)
return {
"dataset": dataset_info["dataset"],
"detected_format": dataset_info["detected_format"],
"final_format": dataset_info["final_format"],
"chat_column": dataset_info.get("chat_column"),
"is_vlm": False,
"success": True,
"requires_manual_mapping": False,
"warnings": dataset_info.get("warnings", []),
"errors": [],
"summary": summary,
"raw_text_column": dataset_info.get("raw_text_column"),
"run_warnings": dataset_info.get("run_warnings", []),
}
detected = dataset_info.get("detected_format", "unknown")
if progress_callback and n_rows:
progress_callback(
status_message = f"Applying chat template to {detected} ({n_rows:,} rows)..."
)
# Gemma emits a leading <bos>, stripped for text-only chatml/sharegpt.
is_alpaca = format_type == "alpaca" or (
format_type == "auto" and dataset_info["detected_format"] == "alpaca"
)
is_gemma = "gemma" in model_name.lower()
if is_gemma and not dataset_info["is_image"] and not is_alpaca:
remove_bos_prefix = True
template_result = apply_chat_template_to_dataset(
dataset_info = dataset_info,
tokenizer = tokenizer,
model_name = model_name,
custom_prompt_template = custom_prompt_template,
add_eos_token = add_eos_token,
remove_bos_prefix = remove_bos_prefix,
custom_format_mapping = custom_format_mapping,
auto_detect_mapping = auto_detect_mapping,
batch_size = batch_size,
num_proc = num_proc,
progress_callback = progress_callback,
)
summary = get_dataset_info_summary(dataset_info)
all_warnings = dataset_info.get("warnings", []) + template_result.get("warnings", [])
all_errors = template_result.get("errors", [])
# If apply_chat_template rescued an "unknown" format, update final_format.
final_format = dataset_info["final_format"]
requires_manual = dataset_info.get("requires_manual_mapping", False)
if final_format == "unknown" and template_result["success"]:
out_ds = template_result["dataset"]
# IterableDataset.column_names can be None after .map() loses features; guard to avoid `"text" in None` -> TypeError on streaming datasets.
out_columns = getattr(out_ds, "column_names", None)
if out_columns is not None and "text" in out_columns:
final_format = "chatml_conversations"
requires_manual = False
return {
"dataset": template_result["dataset"],
"detected_format": dataset_info["detected_format"],
"final_format": final_format,
"chat_column": dataset_info.get("chat_column"),
"is_vlm": False,
"success": template_result["success"],
"requires_manual_mapping": requires_manual,
"warnings": all_warnings,
"errors": all_errors,
"dropped_rows_warning": template_result.get("dropped_rows_warning"),
"summary": summary,
}