Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

54.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Dataset utilities for format detection, conversion, and template application. Entry points: check_dataset_format (lightweight check whether manual mapping is needed, for the frontend), format_dataset (detects and normalizes dataset formats) and format_and_template_dataset (end-to-end processing with chat template). Internal utilities live in format_detection, format_conversion, chat_templates, vlm_processing, data_collators and model_mappings."""

import inspect

from .format_detection import (
    detect_dataset_format,
    detect_multimodal_dataset,
    detect_vlm_dataset_structure,
    detect_custom_format_heuristic,
    is_plain_text_dataset,
    detect_video_columns,
    detect_video_structure,
    video_sample,
    _VIDEO_TEXT_KEYWORDS,
)
from .format_conversion import (
    standardize_chat_format,
    convert_chatml_to_alpaca,
    convert_alpaca_to_chatml,
    convert_to_vlm_format,
    convert_llava_to_vlm_format,
    convert_sharegpt_with_images_to_vlm_format,
    convert_video_to_vlm_format,
    own_message_videos,
    normalize_chatml_roles,
)
from .chat_templates import (
    _custom_prompt_template_error,
    apply_chat_template_to_dataset,
    get_dataset_info_summary,
    get_tokenizer_chat_template,
    DEFAULT_ALPACA_TEMPLATE,
)
from .cells import cell_text, cell_turns, message_list_columns, unwrap_message_column
from .raw_text import prepare_raw_text_dataset
from .pretokenized import PRETOKENIZED_FORMAT, is_pretokenized_dataset
from .vlm_processing import generate_smart_vlm_instruction
from .data_collators import DeepSeekOCRDataCollator, VLMDataCollator
from .model_mappings import TEMPLATE_TO_MODEL_MAPPER
from loggers import get_logger

logger = get_logger(__name__)


def check_dataset_format(dataset, is_vlm: bool = False) -> dict:
    """check whether frontend column mapping is required without processing the dataset."""
    columns = (
        list(dataset.column_names)
        if hasattr(dataset, "column_names")
        else list(next(iter(dataset)).keys())
    )

    multimodal_info = detect_multimodal_dataset(dataset)
    is_audio = multimodal_info.get("is_audio", False)

    audio_fields = {
        "is_audio": is_audio,
        "detected_audio_column": multimodal_info.get("detected_audio_column"),
        "detected_speaker_column": multimodal_info.get("detected_speaker_column"),
    }

    if is_vlm:
        vlm_structure = detect_vlm_dataset_structure(dataset)
        requires_mapping = vlm_structure["format"] == "unknown"

        warning = None
        if requires_mapping:
            img_col = vlm_structure.get("image_column")
            txt_col = vlm_structure.get("text_column")
            missing = []
            if not img_col:
                missing.append("image")
            if not txt_col:
                missing.append("text")
            if missing:
                warning = (
                    f"Could not auto-detect {' or '.join(missing)} column. "
                    "Please assign image and text columns manually."
                )

        return {
            "requires_manual_mapping": requires_mapping,
            "detected_format": vlm_structure["format"],
            "columns": columns,
            "suggested_mapping": None,
            "detected_image_column": vlm_structure.get("image_column"),
            "detected_text_column": vlm_structure.get("text_column"),
            "is_image": multimodal_info["is_image"],
            "multimodal_columns": multimodal_info.get("multimodal_columns"),
            "warning": warning,
            **audio_fields,
        }

    if is_audio:
        detected_audio = multimodal_info.get("detected_audio_column")
        detected_text = multimodal_info.get("detected_text_column")
        needs_mapping = not detected_audio or not detected_text
        return {
            "requires_manual_mapping": needs_mapping,
            "detected_format": "audio",
            "columns": columns,
            "suggested_mapping": None,
            "detected_image_column": None,
            "detected_text_column": multimodal_info.get("detected_text_column"),
            "is_image": False,
            "multimodal_columns": multimodal_info.get("audio_columns"),
            **audio_fields,
        }

    detected = detect_dataset_format(dataset)

    if detected["format"] == "unknown":
        heuristic_mapping = detect_custom_format_heuristic(dataset)
        if heuristic_mapping:
            return {
                "requires_manual_mapping": False,
                "detected_format": "custom_heuristic",
                "columns": columns,
                "suggested_mapping": heuristic_mapping,
                "detected_image_column": None,
                "detected_text_column": None,
                "is_image": multimodal_info["is_image"],
                "multimodal_columns": multimodal_info.get("multimodal_columns"),
                **audio_fields,
            }
        elif is_pretokenized_dataset(dataset):
            return {
                "requires_manual_mapping": False,
                "detected_format": PRETOKENIZED_FORMAT,
                "columns": columns,
                "suggested_mapping": None,
                "detected_image_column": None,
                "detected_text_column": None,
                "is_image": multimodal_info["is_image"],
                "multimodal_columns": multimodal_info.get("multimodal_columns"),
                **audio_fields,
            }
        else:
            return {
                "requires_manual_mapping": True,
                "detected_format": "unknown",
                "columns": columns,
                "suggested_mapping": None,
                "detected_image_column": None,
                "detected_text_column": None,
                "is_image": multimodal_info["is_image"],
                "multimodal_columns": multimodal_info.get("multimodal_columns"),
                "warning": (
                    f"Could not auto-detect column roles for columns: {columns}. "
                    "Please assign roles manually, or use AI Assist."
                ),
                **audio_fields,
            }

    return {
        "requires_manual_mapping": False,
        "detected_format": detected["format"],
        "columns": columns,
        "suggested_mapping": None,
        "detected_image_column": None,
        "detected_text_column": None,
        "chat_column": detected.get("chat_column"),
        "is_image": multimodal_info["is_image"],
        "multimodal_columns": multimodal_info.get("multimodal_columns"),
        **audio_fields,
    }


# Normalise any format-specific role to canonical chatml (user/assistant/system)
_TO_CHATML = {
    "user": "user",
    "human": "user",
    "instruction": "user",
    "assistant": "assistant",
    "gpt": "assistant",
    "output": "assistant",
    "system": "system",
    "input": "system",
}
_CHATML_ROLE_ORDER = ("system", "user", "assistant")
_CHATML_TO_ALPACA = {"user": "instruction", "system": "input", "assistant": "output"}
_KNOWN_CHAT_COLUMNS = {"messages", "conversations", "texts"}


def _mapped_message_columns(dataset, columns):
    messages = message_list_columns(dataset)
    prompt_completion = {"prompt", "completion"}
    if prompt_completion <= set(columns) and messages & prompt_completion:
        messages.update(prompt_completion)
    return messages


def _chatml_final_format(chat_column: str | None) -> str:
    return "chatml_messages" if chat_column == "messages" else "chatml_conversations"


def _chatml_detected_format_label(chat_column: str | None) -> str:
    if chat_column in _KNOWN_CHAT_COLUMNS:
        return f"chatml_{chat_column}"
    return "chatml_conversations"


def _apply_user_mapping(
    dataset,
    mapping: dict,
    batch_size: int = 1000,
):
    """normalize chatml, ShareGPT, or Alpaca roles; ``__`` metadata selects template conversion."""
    meta = {k: v for k, v in mapping.items() if k.startswith("__")}
    column_roles = {k: v for k, v in mapping.items() if not k.startswith("__")}

    if meta:
        return _apply_template_mapping(dataset, column_roles, meta, batch_size)

    role_groups: dict[str, list[str]] = {r: [] for r in _CHATML_ROLE_ORDER}
    for col_name, role in column_roles.items():
        canonical = _TO_CHATML.get(role)
        if canonical:
            role_groups[canonical].append(col_name)
    message_columns = _mapped_message_columns(dataset, column_roles)

    def _convert(examples):
        num = len(next(iter(examples.values())))
        conversations = []
        for i in range(num):
            convo = []
            for chatml_role in _CHATML_ROLE_ORDER:
                for col in role_groups[chatml_role]:
                    if col in examples:
                        convo.extend(
                            cell_turns(
                                examples[col][i],
                                chatml_role,
                                empty_is_messages = col in message_columns,
                            )
                        )
            conversations.append(convo)
        mapped = {"conversations": conversations}
        if "tools" in examples:
            mapped["tools"] = examples["tools"]
        return mapped

    return dataset.map(
        _convert,
        batched = True,
        batch_size = batch_size,
        remove_columns = dataset.column_names,
    )


def _extract_column_value(val, col: str, label_mapping: dict) -> str:
    if isinstance(val, dict):
        if "text" not in val:
            return cell_text(val)
        inner = val["text"]
        val = inner[0] if isinstance(inner, list) and inner else inner
    if isinstance(val, list) and any(isinstance(item, (dict, list)) for item in val):
        return cell_text(val)

    names = label_mapping.get(col)
    if not isinstance(names, dict):
        names = {}
    values = val if isinstance(val, list) else [val]
    return ", ".join(names.get(text, text) for text in map(cell_text, values))


def _apply_template_mapping(
    dataset,
    column_roles: dict,
    meta: dict,
    batch_size: int = 1000,
):
    """Apply advisor-driven mapping for non-conversational datasets: groups columns by assigned role (system/user/assistant), concatenates values within each role into one message, and injects an optional system prompt ahead of any system column. Label mapping converts integer labels to human-readable strings. Returns a dataset with a single 'conversations' column."""
    system_prompt = meta.get("__system_prompt", "")
    label_mapping = meta.get("__label_mapping", {})  # {col: {int_str: label_str}}

    role_groups: dict[str, list[str]] = {"system": [], "user": [], "assistant": []}
    for col, role in column_roles.items():
        canonical = _TO_CHATML.get(role, role)
        if canonical in role_groups:
            role_groups[canonical].append(col)

    import logging as _log

    _log.getLogger(__name__).info(
        f"Applying role mapping: sys={bool(system_prompt)}, "
        f"user_cols={role_groups['user']}, asst_cols={role_groups['assistant']}, "
        f"label_map={list(label_mapping.keys())}"
    )

    def _convert(examples):
        num = len(next(iter(examples.values())))
        conversations = []
        for i in range(num):
            convo = []

            # The system prompt leads, then system columns, so neither is dropped.
            system_parts = [
                _extract_column_value(examples[col][i], col, label_mapping)
                for col in role_groups["system"]
                if col in examples
            ]
            system_text = "\n\n".join(
                part for part in (system_prompt, "\n".join(p for p in system_parts if p)) if part
            )
            if system_text:
                convo.append({"role": "system", "content": system_text})

            user_parts = []
            for col in role_groups["user"]:
                if col in examples:
                    user_parts.append(_extract_column_value(examples[col][i], col, label_mapping))
            if user_parts:
                convo.append({"role": "user", "content": "\n".join(user_parts)})

            asst_parts = []
            for col in role_groups["assistant"]:
                if col in examples:
                    asst_parts.append(_extract_column_value(examples[col][i], col, label_mapping))
            if asst_parts:
                convo.append({"role": "assistant", "content": "\n".join(asst_parts)})

            conversations.append(convo)
        mapped = {"conversations": conversations}
        if "tools" in examples:
            mapped["tools"] = examples["tools"]
        return mapped

    return dataset.map(
        _convert,
        batched = True,
        batch_size = batch_size,
        remove_columns = dataset.column_names,
    )


def _apply_user_mapping_alpaca(
    dataset,
    mapping: dict,
    batch_size: int = 1000,
):
    """__system_prompt is prepended to instruction, since a mapped system column already fills input."""
    meta = {k: v for k, v in mapping.items() if k.startswith("__")}
    column_roles = {k: v for k, v in mapping.items() if not k.startswith("__")}
    system_prompt = meta.get("__system_prompt", "")
    label_mapping = meta.get("__label_mapping", {})

    cols_for: dict[str, list[str]] = {
        "instruction": [],
        "input": [],
        "output": [],
    }
    for col_name, role in column_roles.items():
        canonical = _TO_CHATML.get(role)
        alpaca_field = _CHATML_TO_ALPACA.get(canonical) if canonical else None
        if alpaca_field:
            cols_for[alpaca_field].append(col_name)

    def _convert(examples):
        num = len(next(iter(examples.values())))
        instructions, inputs, outputs = [], [], []
        for i in range(num):
            for field, dest in (
                ("instruction", instructions),
                ("input", inputs),
                ("output", outputs),
            ):
                val = "\n".join(
                    _extract_column_value(examples[col][i], col, label_mapping)
                    for col in cols_for[field]
                    if col in examples
                )
                if field == "instruction" and system_prompt:
                    val = "\n\n".join(part for part in (system_prompt, val) if part)
                dest.append(val)
        return {"instruction": instructions, "input": inputs, "output": outputs}

    return dataset.map(
        _convert,
        batched = True,
        batch_size = batch_size,
        remove_columns = dataset.column_names,
    )


def format_dataset(dataset, *args, **kwargs):
    """``_format_dataset``, but token-id rows no format matched route as pre-tokenized under any format except raw."""
    bound = _FORMAT_DATASET_SIGNATURE.bind(dataset, *args, **kwargs)
    bound.apply_defaults()
    info = _format_dataset(*bound.args, **bound.kwargs)
    if (
        info["final_format"] == "unknown"
        and bound.arguments["format_type"] != "raw"
        and not bound.arguments["custom_format_mapping"]
        and is_pretokenized_dataset(dataset)
    ):
        info = {
            **info,
            "dataset": dataset,
            "detected_format": PRETOKENIZED_FORMAT,
            "final_format": PRETOKENIZED_FORMAT,
            "chat_column": None,
            "is_standardized": True,
            "requires_manual_mapping": False,
            "warnings": [],
        }
    if info["final_format"] in ("chatml_messages", "chatml_conversations") and info["chat_column"]:
        info["dataset"] = normalize_chatml_roles(
            info["dataset"], info["chat_column"], bound.arguments["batch_size"]
        )
    return info


def _format_dataset(
    dataset,
    format_type = "auto",
    tokenizer = None,
    aliases_for_system = [
        "system",
    ],
    aliases_for_user = [
        "user",
        "human",
        "input",
    ],
    aliases_for_assistant = [
        "gpt",
        "assistant",
        "output",
    ],
    batch_size = 1000,
    num_proc = None,
    auto_detect_custom = True,
    custom_format_mapping = None,
    split_name = None,
    raw_text_column = None,
):
    """Formats dataset and returns {dataset, detected_format, final_format, chat_column, is_standardized, requires_manual_mapping, warnings}."""

    multimodal_info = detect_multimodal_dataset(dataset)

    auto_raw = (
        format_type == "auto" and not custom_format_mapping and is_plain_text_dataset(dataset)
    )
    if auto_raw:
        # Matches the preview's "raw" verdict; auto would otherwise fail as "unknown".
        format_type = "raw"

    if format_type == "raw":
        # Explicit "raw" already gets EOS from the trainer / MLXTrainingConfig.append_eos.
        raw_result = prepare_raw_text_dataset(
            dataset,
            split_name = split_name,
            # A processor without Unsloth's patch (MLX VLM) keeps eos_token on its inner tokenizer.
            eos_token = getattr(tokenizer, "eos_token", None)
            or getattr(getattr(tokenizer, "tokenizer", None), "eos_token", None),
            append_eos = auto_raw,
            text_column = raw_text_column,
        )
        return {
            "dataset": raw_result.dataset,
            "detected_format": "raw_text",
            "final_format": "raw_text",
            "chat_column": "text",
            "is_standardized": True,
            "requires_manual_mapping": False,
            "is_image": multimodal_info["is_image"],
            "multimodal_info": multimodal_info,
            "warnings": [notice.message for notice in raw_result.notices],
            "raw_text_column": raw_result.source_column,
            "run_warnings": [
                notice.message
                for notice in raw_result.notices
                if notice.level == "warning" and notice.update_status
            ],
        }

    if custom_format_mapping:
        try:
            if format_type == "alpaca":
                mapped_dataset = _apply_user_mapping_alpaca(
                    dataset, custom_format_mapping, batch_size
                )
                final_format = "alpaca"
                chat_column = None
            else:
                # auto / chatml / sharegpt / conversational all produce chatml conversations (sharegpt standardized to role/content internally)
                mapped_dataset = _apply_user_mapping(dataset, custom_format_mapping, batch_size)
                final_format = "chatml_conversations"
                chat_column = "conversations"

            return {
                "dataset": mapped_dataset,
                "detected_format": "user_mapped",
                "final_format": final_format,
                "chat_column": chat_column,
                "is_standardized": True,
                "requires_manual_mapping": False,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": [
                    f"Applied user-provided column mapping ({format_type}): {custom_format_mapping}"
                ],
            }
        except Exception as e:
            return {
                "dataset": dataset,
                "detected_format": "user_mapped",
                "final_format": "unknown",
                "chat_column": None,
                "is_standardized": False,
                "requires_manual_mapping": True,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": [f"Failed to apply user mapping: {e}"],
            }

    detected = detect_dataset_format(dataset)
    dataset = unwrap_message_column(dataset, detected["chat_column"], batch_size)
    warnings = []

    if multimodal_info["is_image"]:
        warnings.append(
            f"Multimodal dataset detected. Found columns: {multimodal_info['multimodal_columns']}"
        )

    if format_type == "auto":
        if detected["format"] == "alpaca":
            return {
                "dataset": dataset,
                "detected_format": "alpaca",
                "final_format": "alpaca",
                "chat_column": None,
                "is_standardized": True,
                "requires_manual_mapping": False,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": [],
            }

        elif detected["format"] == "sharegpt" and detected.get("chat_column"):
            try:
                standardized = standardize_chat_format(
                    dataset,
                    tokenizer,
                    aliases_for_system,
                    aliases_for_user,
                    aliases_for_assistant,
                    batch_size,
                    num_proc,
                    chat_column = detected["chat_column"],
                )
                return {
                    "dataset": standardized,
                    "detected_format": "sharegpt",
                    "final_format": _chatml_final_format(detected["chat_column"]),
                    "chat_column": detected["chat_column"],
                    "is_standardized": True,
                    "requires_manual_mapping": False,
                    "is_image": multimodal_info["is_image"],
                    "multimodal_info": multimodal_info,
                    "warnings": [],
                }
            except Exception as e:
                warnings.append(f"Failed to standardize ShareGPT format: {e}")
                return {
                    "dataset": dataset,
                    "detected_format": "sharegpt",
                    "final_format": "sharegpt",
                    "chat_column": detected["chat_column"],
                    "is_standardized": False,
                    "requires_manual_mapping": True,
                    "is_image": multimodal_info["is_image"],
                    "multimodal_info": multimodal_info,
                    "warnings": warnings,
                }

        elif detected["format"] == "chatml" and detected.get("chat_column"):
            return {
                "dataset": dataset,
                "detected_format": _chatml_detected_format_label(detected["chat_column"]),
                "final_format": _chatml_final_format(detected["chat_column"]),
                "chat_column": detected["chat_column"],
                "is_standardized": True,
                "requires_manual_mapping": False,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": warnings,
            }

        else:
            warnings.append(f"Unknown format detected. Keys found: {detected['sample_keys']}")

            if auto_detect_custom:
                custom_mapping = detect_custom_format_heuristic(dataset)
                if custom_mapping:
                    warnings.append(f"Auto-detected column mapping: {custom_mapping}")
                    message_columns = _mapped_message_columns(dataset, custom_mapping)

                    def _apply_auto_mapping(examples):
                        conversations = []
                        num_examples = len(examples[list(examples.keys())[0]])

                        all_columns = set(examples.keys())
                        mapped_columns = set(custom_mapping.keys())
                        preserved_columns = {
                            col: examples[col]
                            for col in all_columns - mapped_columns
                            if col != "conversations"
                        }

                        for i in range(num_examples):
                            convo = []
                            for target_role in ["system", "user", "assistant"]:
                                for col_name, role in custom_mapping.items():
                                    if role == target_role and col_name in examples:
                                        convo.extend(
                                            turn
                                            for turn in cell_turns(
                                                examples[col_name][i],
                                                role,
                                                empty_is_messages = col_name in message_columns,
                                            )
                                            if turn.get("tool_calls") or turn["content"].strip()
                                        )
                            conversations.append(convo)

                        return {**preserved_columns, "conversations": conversations}

                    try:
                        dataset = dataset.map(
                            _apply_auto_mapping, batched = True, batch_size = batch_size
                        )
                        return {
                            "dataset": dataset,
                            "detected_format": "unknown",
                            "final_format": "chatml_conversations",
                            "chat_column": "conversations",
                            "is_standardized": True,
                            "requires_manual_mapping": False,
                            "is_image": multimodal_info["is_image"],
                            "multimodal_info": multimodal_info,
                            "warnings": warnings,
                        }
                    except Exception as e:
                        warnings.append(f"Auto-detection failed: {e}")

            if detected["chat_column"]:
                try:
                    standardized = standardize_chat_format(
                        dataset,
                        tokenizer,
                        aliases_for_system,
                        aliases_for_user,
                        aliases_for_assistant,
                        batch_size,
                        num_proc,
                        chat_column = detected["chat_column"],
                    )
                    warnings.append("Successfully standardized unknown format")
                    return {
                        "dataset": standardized,
                        "detected_format": "unknown",
                        "final_format": _chatml_final_format(detected["chat_column"]),
                        "chat_column": detected["chat_column"],
                        "is_standardized": True,
                        "requires_manual_mapping": False,
                        "is_image": multimodal_info["is_image"],
                        "multimodal_info": multimodal_info,
                        "warnings": warnings,
                    }
                except Exception as e:
                    warnings.append(f"Could not standardize: {e}. Passing dataset as-is.")

            return {
                "dataset": dataset,
                "detected_format": "unknown",
                "final_format": "unknown",
                "chat_column": detected["chat_column"],
                "is_standardized": False,
                "requires_manual_mapping": True,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": warnings,
            }

    elif format_type == "alpaca":
        if detected["format"] == "alpaca":
            return {
                "dataset": dataset,
                "detected_format": "alpaca",
                "final_format": "alpaca",
                "chat_column": None,
                "is_standardized": True,
                "requires_manual_mapping": False,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": [],
            }

        elif detected["format"] in ["sharegpt", "chatml"] and detected.get("chat_column"):
            try:
                if detected["format"] == "sharegpt":
                    dataset = standardize_chat_format(
                        dataset,
                        tokenizer,
                        aliases_for_system,
                        aliases_for_user,
                        aliases_for_assistant,
                        batch_size,
                        num_proc,
                        chat_column = detected["chat_column"],
                    )

                converted = convert_chatml_to_alpaca(
                    dataset,
                    batch_size,
                    num_proc,
                    chat_column = detected["chat_column"],
                )
                return {
                    "dataset": converted,
                    "detected_format": detected["format"],
                    "final_format": "alpaca",
                    "chat_column": None,
                    "is_standardized": True,
                    "requires_manual_mapping": False,
                    "is_image": multimodal_info["is_image"],
                    "multimodal_info": multimodal_info,
                    "warnings": [],
                }
            except Exception as e:
                warnings.append(f"Failed to convert chat dataset to Alpaca: {e}")
                return {
                    "dataset": dataset,
                    "detected_format": detected["format"],
                    "final_format": "unknown",
                    "chat_column": detected["chat_column"],
                    "is_standardized": False,
                    "requires_manual_mapping": True,
                    "is_image": multimodal_info["is_image"],
                    "multimodal_info": multimodal_info,
                    "warnings": warnings,
                }

        else:
            warnings.append(f"Cannot convert unknown format to Alpaca")
            return {
                "dataset": dataset,
                "detected_format": "unknown",
                "final_format": "unknown",
                "chat_column": detected["chat_column"],
                "is_standardized": False,
                "requires_manual_mapping": True,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": warnings,
            }

    elif format_type in ["chatml", "conversational", "sharegpt"]:
        if detected["format"] == "alpaca":
            converted = convert_alpaca_to_chatml(dataset, batch_size, num_proc)
            return {
                "dataset": converted,
                "detected_format": "alpaca",
                "final_format": "chatml_conversations",
                "chat_column": "conversations",
                "is_standardized": True,
                "requires_manual_mapping": False,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": [],
            }

        elif detected["format"] == "sharegpt" and detected.get("chat_column"):
            try:
                standardized = standardize_chat_format(
                    dataset,
                    tokenizer,
                    aliases_for_system,
                    aliases_for_user,
                    aliases_for_assistant,
                    batch_size,
                    num_proc,
                    chat_column = detected["chat_column"],
                )
                return {
                    "dataset": standardized,
                    "detected_format": "sharegpt",
                    "final_format": _chatml_final_format(detected["chat_column"]),
                    "chat_column": detected["chat_column"],
                    "is_standardized": True,
                    "requires_manual_mapping": False,
                    "is_image": multimodal_info["is_image"],
                    "multimodal_info": multimodal_info,
                    "warnings": [],
                }
            except Exception as e:
                warnings.append(f"Failed to standardize ShareGPT format: {e}")
                return {
                    "dataset": dataset,
                    "detected_format": "sharegpt",
                    "final_format": "sharegpt",
                    "chat_column": detected["chat_column"],
                    "is_standardized": False,
                    "requires_manual_mapping": True,
                    "is_image": multimodal_info["is_image"],
                    "multimodal_info": multimodal_info,
                    "warnings": warnings,
                }

        elif detected["format"] == "chatml" and detected.get("chat_column"):
            return {
                "dataset": dataset,
                "detected_format": _chatml_detected_format_label(detected["chat_column"]),
                "final_format": _chatml_final_format(detected["chat_column"]),
                "chat_column": detected["chat_column"],
                "is_standardized": True,
                "requires_manual_mapping": False,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": [],
            }

        else:
            warnings.append(f"Unknown format, attempting standardization")
            if detected["chat_column"]:
                try:
                    standardized = standardize_chat_format(
                        dataset,
                        tokenizer,
                        aliases_for_system,
                        aliases_for_user,
                        aliases_for_assistant,
                        batch_size,
                        num_proc,
                        chat_column = detected["chat_column"],
                    )
                    return {
                        "dataset": standardized,
                        "detected_format": "unknown",
                        "final_format": _chatml_final_format(detected["chat_column"]),
                        "chat_column": detected["chat_column"],
                        "is_standardized": True,
                        "requires_manual_mapping": False,
                        "is_image": multimodal_info["is_image"],
                        "multimodal_info": multimodal_info,
                        "warnings": warnings,
                    }
                except Exception as e:
                    warnings.append(f"Standardization failed: {e}")

            return {
                "dataset": dataset,
                "detected_format": "unknown",
                "final_format": "unknown",
                "chat_column": detected["chat_column"],
                "is_standardized": False,
                "requires_manual_mapping": True,
                "is_image": multimodal_info["is_image"],
                "multimodal_info": multimodal_info,
                "warnings": warnings,
            }

    else:
        raise ValueError(f"Unknown format_type: {format_type}")


_FORMAT_DATASET_SIGNATURE = inspect.signature(_format_dataset)


def format_and_template_dataset(
    dataset,
    model_name,
    tokenizer,
    is_vlm = False,
    format_type = "auto",
    vlm_instruction = None,
    vlm_text_column = None,
    vlm_image_column = None,
    dataset_name = None,
    custom_prompt_template = None,
    add_eos_token = False,
    remove_bos_prefix = False,
    custom_format_mapping = None,
    auto_detect_custom = True,
    auto_detect_mapping = True,
    aliases_for_system = [
        "system",
    ],
    aliases_for_user = [
        "user",
        "human",
        "input",
    ],
    aliases_for_assistant = [
        "gpt",
        "assistant",
        "output",
    ],
    batch_size = 1000,
    num_proc = None,
    progress_callback = None,
    split_name = None,
    raw_text_column = None,
    dataset_revision = None,
):
    """Combines format_dataset and apply_chat_template_to_dataset, for UI workflows where one call does everything. custom_prompt_template is retained for signature compatibility; non-None values are rejected because Studio cannot persist a matching inference template. Returns {dataset (with a 'text' column), detected_format, final_format, success, requires_manual_mapping, warnings, errors, summary}."""
    from .tokenizer_fingerprint import install_tokenizer_fingerprint

    install_tokenizer_fingerprint()

    custom_prompt_error = _custom_prompt_template_error(custom_prompt_template)
    if custom_prompt_error:
        return {
            "dataset": dataset,
            "detected_format": "unknown",
            "final_format": "unknown",
            "chat_column": None,
            "is_vlm": is_vlm,
            "success": False,
            "requires_manual_mapping": False,
            "warnings": [],
            "errors": [custom_prompt_error],
            "summary": None,
        }

    if is_vlm:
        warnings = []
        errors = []

        multimodal_info = detect_multimodal_dataset(dataset)

        if custom_format_mapping:
            user_vlm_image_column = None
            user_vlm_text_column = None

            for col, role in custom_format_mapping.items():
                if role == "image":
                    user_vlm_image_column = col
                elif role in ["text", "user", "caption", "assistant"]:
                    user_vlm_text_column = col

            # The text-role loop above keeps the last one; a clip needs the answer and question kept apart.
            video_answer_column = next(
                (
                    c
                    for c, r in custom_format_mapping.items()
                    if r in ("assistant", "text", "caption")
                ),
                user_vlm_text_column,
            )
            video_question_column = next(
                (
                    c
                    for c, r in custom_format_mapping.items()
                    if r == "user" and c != video_answer_column
                ),
                None,
            )
            first = next(iter(dataset), None)
            probe = video_sample(dataset, first) if isinstance(first, dict) else {}
            mapped_video = (
                user_vlm_image_column
                and user_vlm_text_column
                and user_vlm_image_column in detect_video_columns(probe)
            )
            if mapped_video and video_question_column is None:
                # The VLM mapping UI offers only image and text roles, so take the question from detection.
                detected_question = (detect_video_structure(probe) or {}).get("question_column")
                if detected_question not in (None, video_answer_column, user_vlm_image_column):
                    video_question_column = detected_question
            if mapped_video:
                # The image role on a clip column: the image converter would try to open the clip.
                try:
                    dataset = convert_video_to_vlm_format(
                        dataset,
                        video_column = user_vlm_image_column,
                        text_column = video_answer_column,
                        question_column = video_question_column,
                        instruction = vlm_instruction,
                        dataset_name = dataset_name,
                        dataset_revision = dataset_revision,
                    )
                except Exception as e:
                    errors.append(f"Failed to convert the mapped video + text columns: {e}")
                    return {
                        "dataset": dataset,
                        "detected_format": "user_mapped",
                        "final_format": "vlm_conversion_failed",
                        "is_vlm": True,
                        "success": False,
                        "requires_manual_mapping": True,
                        "warnings": warnings,
                        "errors": errors,
                    }
                warnings.append(
                    f"Applied user VLM mapping: video='{user_vlm_image_column}', text='{video_answer_column}'"
                )
                return {
                    "dataset": dataset,
                    "detected_format": "user_mapped",
                    "final_format": "vlm_messages",
                    "chat_column": "messages",
                    "is_vlm": True,
                    "is_image": True,
                    "multimodal_info": multimodal_info,
                    "success": True,
                    "requires_manual_mapping": False,
                    "warnings": warnings,
                    "errors": [],
                }

            if user_vlm_image_column and user_vlm_text_column:
                try:
                    dataset = convert_to_vlm_format(
                        dataset,
                        instruction = vlm_instruction,
                        text_column = user_vlm_text_column,
                        image_column = user_vlm_image_column,
                        dataset_name = dataset_name,
                        progress_callback = progress_callback,
                    )
                    warnings.append(
                        f"Applied user VLM mapping: image='{user_vlm_image_column}', text='{user_vlm_text_column}'"
                    )

                    return {
                        "dataset": dataset,
                        "detected_format": "user_mapped",
                        "final_format": "vlm_messages",
                        "chat_column": "messages",
                        "is_vlm": True,
                        "is_image": True,
                        "multimodal_info": multimodal_info,
                        "success": True,
                        "requires_manual_mapping": False,
                        "warnings": warnings,
                        "errors": [],
                    }
                except Exception as e:
                    # User mapping failed; fall back to auto-detection (handles stale cached mappings).
                    warnings.append(
                        f"User VLM mapping (image='{user_vlm_image_column}', "
                        f"text='{user_vlm_text_column}') failed: {e} — "
                        f"falling back to auto-detection"
                    )
                    logger.info(f"⚠️ User VLM mapping failed, falling back to auto-detection...")
                    custom_format_mapping = None
            else:
                errors.append(
                    f"Invalid VLM mapping: need 'image' and 'text' roles. Got: {custom_format_mapping}"
                )
                return {
                    "dataset": dataset,
                    "detected_format": "user_mapped",
                    "final_format": "vlm_unknown",
                    "is_vlm": True,
                    "success": False,
                    "requires_manual_mapping": True,
                    "warnings": warnings,
                    "errors": errors,
                }

        vlm_structure = detect_vlm_dataset_structure(dataset)

        if vlm_structure["format"] == "vlm_messages_llava":
            try:
                dataset = convert_llava_to_vlm_format(dataset)
                warnings.append(
                    "Converted from Llava format (image indices) to standard VLM format"
                )
            except Exception as e:
                errors.append(f"Failed to convert Llava format: {e}")
                import traceback

                traceback.print_exc()

                return {
                    "dataset": dataset,
                    "detected_format": "vlm_messages_llava",
                    "final_format": "vlm_conversion_failed",
                    "is_vlm": True,
                    "success": False,
                    "requires_manual_mapping": True,
                    "warnings": warnings,
                    "errors": errors,
                }

        elif vlm_structure["format"] == "sharegpt_with_images":
            try:
                dataset = convert_sharegpt_with_images_to_vlm_format(
                    dataset,
                    image_column = vlm_structure["image_column"],
                    messages_column = vlm_structure["messages_column"],
                    dataset_name = dataset_name,
                    progress_callback = progress_callback,
                )
                warnings.append("Converted from ShareGPT+image format to standard VLM format")
            except Exception as e:
                errors.append(f"Failed to convert ShareGPT+image format: {e}")
                import traceback

                traceback.print_exc()

                return {
                    "dataset": dataset,
                    "detected_format": "sharegpt_with_images",
                    "final_format": "vlm_conversion_failed",
                    "is_vlm": True,
                    "success": False,
                    "requires_manual_mapping": True,
                    "warnings": warnings,
                    "errors": errors,
                }

        elif vlm_structure["format"] == "video_text" and vlm_structure.get("image_columns"):
            errors.append(
                f"Rows mix video ('{vlm_structure['video_column']}') with image columns "
                f"{vlm_structure['image_columns']}; video conversion would drop the images, so remove "
                "those columns or split the dataset."
            )
            return {
                "dataset": dataset,
                "detected_format": "video_text",
                "final_format": "vlm_conversion_failed",
                "is_vlm": True,
                "success": False,
                "requires_manual_mapping": True,
                "warnings": warnings,
                "errors": errors,
            }

        elif vlm_structure["format"] == "video_text":
            try:
                dataset = convert_video_to_vlm_format(
                    dataset,
                    video_column = vlm_structure["video_column"],
                    text_column = vlm_structure["text_column"],
                    question_column = vlm_structure.get("question_column"),
                    instruction = vlm_instruction,
                    video_columns = vlm_structure.get("video_columns"),
                    dataset_name = dataset_name,
                    dataset_revision = dataset_revision,
                )
            except Exception as e:
                errors.append(f"Failed to convert video + text rows to VLM format: {e}")
                return {
                    "dataset": dataset,
                    "detected_format": "video_text",
                    "final_format": "vlm_conversion_failed",
                    "is_vlm": True,
                    "success": False,
                    "requires_manual_mapping": True,
                    "warnings": warnings,
                    "errors": errors,
                }

        elif vlm_structure["format"] == "unknown" and vlm_structure.get("video_column"):
            errors.append(
                f"Found videos in '{vlm_structure['video_column']}' but no answer column; name the answer column with one of these words: "
                + ", ".join(_VIDEO_TEXT_KEYWORDS)
            )
            return {
                "dataset": dataset,
                "detected_format": "unknown",
                "final_format": "vlm_unknown",
                "is_vlm": True,
                "success": False,
                "requires_manual_mapping": True,
                "warnings": warnings,
                "errors": errors,
            }

        elif vlm_structure["needs_conversion"]:
            if vlm_text_column is None:
                vlm_text_column = vlm_structure["text_column"]
            if vlm_image_column is None:
                vlm_image_column = vlm_structure["image_column"]

            if vlm_text_column is None or vlm_image_column is None:
                columns = list(next(iter(dataset)).keys()) if dataset else []
                issues = [
                    f"Could not auto-detect image and text columns from: {columns}",
                    f"VLM structure detected: {vlm_structure.get('format', 'unknown')}",
                ]
                friendly = None
                try:
                    from .llm_assist import llm_generate_dataset_warning
                    friendly = llm_generate_dataset_warning(
                        issues,
                        dataset_name = dataset_name,
                        modality = "vision",
                        column_names = columns,
                    )
                except Exception:
                    pass
                errors.append(
                    friendly
                    or f"Could not auto-detect image/text columns. Found: {vlm_structure}. "
                )
                return {
                    "dataset": dataset,
                    "detected_format": "vlm_unknown",
                    "final_format": "vlm_unknown",
                    "is_vlm": True,
                    "success": False,
                    "requires_manual_mapping": True,
                    "warnings": warnings,
                    "errors": errors,
                }

            try:
                dataset = convert_to_vlm_format(
                    dataset,
                    instruction = vlm_instruction,
                    text_column = vlm_text_column,
                    image_column = vlm_image_column,
                    dataset_name = dataset_name,
                    progress_callback = progress_callback,
                )

                if vlm_instruction:
                    warnings.append(f"Using user-provided instruction: '{vlm_instruction}'")
                else:
                    warnings.append("Auto-generated instruction based on dataset analysis")

            except Exception as e:
                errors.append(f"Failed to convert to VLM format: {e}")
                import traceback

                traceback.print_exc()

                return {
                    "dataset": dataset,
                    "detected_format": vlm_structure["format"],
                    "final_format": "vlm_conversion_failed",
                    "is_vlm": True,
                    "success": False,
                    "requires_manual_mapping": True,
                    "warnings": warnings,
                    "errors": errors,
                }

        elif vlm_structure["format"] == "vlm_messages":
            try:
                dataset = own_message_videos(
                    dataset, dataset_name = dataset_name, dataset_revision = dataset_revision
                )
            except Exception as e:
                errors.append(f"Failed to read the video parts of VLM messages rows: {e}")
                return {
                    "dataset": dataset,
                    "detected_format": "vlm_messages",
                    "final_format": "vlm_conversion_failed",
                    "is_vlm": True,
                    "success": False,
                    "requires_manual_mapping": True,
                    "warnings": warnings,
                    "errors": errors,
                }
            warnings.append("Dataset already in standard VLM messages format")

        return {
            "dataset": dataset,
            "detected_format": vlm_structure["format"],
            "final_format": "vlm_messages",
            "chat_column": "messages",
            "is_vlm": True,
            "is_image": multimodal_info["is_image"],
            "multimodal_info": multimodal_info,
            "vlm_structure": vlm_structure,
            "success": True,
            "requires_manual_mapping": False,
            "warnings": warnings,
            "errors": errors,
        }

    else:
        n_rows = len(dataset) if hasattr(dataset, "__len__") else None
        if progress_callback and n_rows:
            progress_callback(status_message = f"Formatting dataset ({n_rows:,} rows)...")
        dataset_info = format_dataset(
            dataset,
            format_type = format_type,
            tokenizer = tokenizer,
            auto_detect_custom = auto_detect_custom,
            custom_format_mapping = custom_format_mapping,
            aliases_for_system = aliases_for_system,
            aliases_for_user = aliases_for_user,
            aliases_for_assistant = aliases_for_assistant,
            batch_size = batch_size,
            num_proc = num_proc,
            split_name = split_name,
            raw_text_column = raw_text_column,
        )

        if dataset_info["final_format"] in ("raw_text", PRETOKENIZED_FORMAT):
            summary = get_dataset_info_summary(dataset_info)
            return {
                "dataset": dataset_info["dataset"],
                "detected_format": dataset_info["detected_format"],
                "final_format": dataset_info["final_format"],
                "chat_column": dataset_info.get("chat_column"),
                "is_vlm": False,
                "success": True,
                "requires_manual_mapping": False,
                "warnings": dataset_info.get("warnings", []),
                "errors": [],
                "summary": summary,
                "raw_text_column": dataset_info.get("raw_text_column"),
                "run_warnings": dataset_info.get("run_warnings", []),
            }

        detected = dataset_info.get("detected_format", "unknown")
        if progress_callback and n_rows:
            progress_callback(
                status_message = f"Applying chat template to {detected} ({n_rows:,} rows)..."
            )
        # Gemma emits a leading <bos>, stripped for text-only chatml/sharegpt.
        is_alpaca = format_type == "alpaca" or (
            format_type == "auto" and dataset_info["detected_format"] == "alpaca"
        )
        is_gemma = "gemma" in model_name.lower()
        if is_gemma and not dataset_info["is_image"] and not is_alpaca:
            remove_bos_prefix = True
        template_result = apply_chat_template_to_dataset(
            dataset_info = dataset_info,
            tokenizer = tokenizer,
            model_name = model_name,
            custom_prompt_template = custom_prompt_template,
            add_eos_token = add_eos_token,
            remove_bos_prefix = remove_bos_prefix,
            custom_format_mapping = custom_format_mapping,
            auto_detect_mapping = auto_detect_mapping,
            batch_size = batch_size,
            num_proc = num_proc,
            progress_callback = progress_callback,
        )

        summary = get_dataset_info_summary(dataset_info)

        all_warnings = dataset_info.get("warnings", []) + template_result.get("warnings", [])
        all_errors = template_result.get("errors", [])

        # If apply_chat_template rescued an "unknown" format, update final_format.
        final_format = dataset_info["final_format"]
        requires_manual = dataset_info.get("requires_manual_mapping", False)
        if final_format == "unknown" and template_result["success"]:
            out_ds = template_result["dataset"]
            # IterableDataset.column_names can be None after .map() loses features; guard to avoid `"text" in None` -> TypeError on streaming datasets.
            out_columns = getattr(out_ds, "column_names", None)
            if out_columns is not None and "text" in out_columns:
                final_format = "chatml_conversations"
                requires_manual = False

        return {
            "dataset": template_result["dataset"],
            "detected_format": dataset_info["detected_format"],
            "final_format": final_format,
            "chat_column": dataset_info.get("chat_column"),
            "is_vlm": False,
            "success": template_result["success"],
            "requires_manual_mapping": requires_manual,
            "warnings": all_warnings,
            "errors": all_errors,
            "dropped_rows_warning": template_result.get("dropped_rows_warning"),
            "summary": summary,
        }