Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

54.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Dataset format conversion between Alpaca, ShareGPT, and ChatML."""

import contextvars
import functools
import os

from .cells import cell_text
from .iterable import is_streaming_dataset
from .remote_images import (
    fetch_opted_in_image_bytes,
    fetch_untrusted_image_bytes,
    is_network_share,
    private_sources_allowed,
)
from loggers import get_logger

logger = get_logger(__name__)


def _quiet_bar_kwargs() -> dict:
    """Send our own conversion bars to a null stream while the log is quiet. They still count (the UI status poller reads tqdm._instances), they just do not write carriage-return fragments beside the worker's structured records."""
    try:
        from loggers.config import quiet_bar_kwargs
        return quiet_bar_kwargs()
    except Exception:  # noqa: BLE001 - a bar is never worth failing a conversion for
        return {}


def _own_image_path(path, account):
    """A local image a managed account names must be one it may read (a no-op for the owner). ``account`` is
    captured when the conversion starts: URL datasets convert in a thread pool, which does not carry it."""
    from utils.account_context import run_as
    from utils.security.dataset_media import require_readable_media

    if isinstance(path, str) and is_network_share(path) and not private_sources_allowed():
        raise OSError(
            f"Refusing an image path on a network share: {path}. "
            "Set UNSLOTH_ALLOW_PRIVATE_URL_FETCH=1 to load images from shares you trust."
        )
    run_as(account, require_readable_media, path)
    return path


def _normalize_role_alias(role) -> str:
    """Stripped and lowercased, as `standardize_data_formats` and the preview do it."""
    if role is None:
        return ""
    return str(role).strip().lower()


def standardize_chat_format(
    dataset,
    tokenizer = None,
    aliases_for_system = [
        "system",
    ],
    aliases_for_user = [
        "user",
        "human",
        "input",
    ],
    aliases_for_assistant = [
        "gpt",
        "assistant",
        "output",
    ],
    batch_size = 1000,
    num_proc = None,
    chat_column: str | None = None,
):
    """Standardize BOTH messages and conversations: map non-standard role names and keys to the standard format."""
    import collections
    import itertools

    is_vlm = False
    if tokenizer is not None:
        if hasattr(tokenizer, "image_processor") or hasattr(tokenizer, "tokenizer"):
            is_vlm = True

    column_names = set(next(iter(dataset)).keys())

    if chat_column:
        if chat_column not in column_names:
            return dataset
    elif "conversations" in column_names:
        chat_column = "conversations"
    elif "messages" in column_names:
        chat_column = "messages"
    elif "texts" in column_names:
        chat_column = "texts"
    else:
        return dataset

    def _iter_probe_rows():
        try:
            total = min(len(dataset), 100)
            for index in range(total):
                yield dataset[index]
            return
        except Exception:
            pass
        for example in itertools.islice(dataset, 100):
            yield example

    uniques = collections.defaultdict(list)
    for example in _iter_probe_rows():
        chat_data = example.get(chat_column)
        if not isinstance(chat_data, list) or len(chat_data) == 0:
            continue
        for message in chat_data:
            if not isinstance(message, dict):
                continue
            for key, value in message.items():
                if type(value) is not str:
                    continue
                uniques[key].append(value)

    if "from" in uniques and "value" in uniques:
        role_key = "from"
        content_key = "value"
    elif "role" in uniques and "content" in uniques:
        role_key = "role"
        content_key = "content"
    elif len(uniques.keys()) == 2:
        keys = list(uniques.keys())
        length_first = len(set(uniques[keys[0]]))
        length_second = len(set(uniques[keys[1]]))
        if length_first < length_second:
            role_key = keys[0]
            content_key = keys[1]
        else:
            role_key = keys[1]
            content_key = keys[0]
    else:
        raise ValueError(f"Could not infer role/content keys for chat column '{chat_column}'")

    # Keyed on the normalised alias: "Human" / " user " would reach the template raw.
    aliases_mapping = {}
    for x in aliases_for_system:
        aliases_mapping[_normalize_role_alias(x)] = "system"
    for x in aliases_for_user:
        aliases_mapping[_normalize_role_alias(x)] = "user"
    for x in aliases_for_assistant:
        aliases_mapping[_normalize_role_alias(x)] = "assistant"

    def _standardize_dataset(examples):
        convos = examples[chat_column]
        all_convos = []
        for convo in convos:
            if not isinstance(convo, list):
                all_convos.append([])
                continue

            new_convo = []
            for message in convo:
                if not isinstance(message, dict):
                    continue

                # Use the inferred keys first, falling back per-message so mixed ShareGPT/ChatML rows keep valid turns.
                original_role = message.get(role_key)
                original_content = message.get(content_key)
                # Blank role counts as absent, as in the preview; an is None check trained a blank-role gpt turn as a user turn. Blank content is legitimate.
                if not _normalize_role_alias(original_role):
                    original_role = message.get("role") or message.get("from") or ""
                if original_content is None:
                    original_content = message.get("content")
                if original_content is None:
                    original_content = message.get("value")
                if original_content is None and not message.get("tool_calls"):
                    original_content = ""

                # Unknown alias left as written; blank is "user", as most templates reject one.
                normalized_role = _normalize_role_alias(original_role)
                standard_role = (
                    aliases_mapping.get(normalized_role, original_role)
                    if normalized_role
                    else "user"
                )

                if is_vlm and original_content is not None:
                    original_content = [{"type": "text", "text": original_content}]

                new_message = {
                    key: value
                    for key, value in message.items()
                    if key not in (role_key, content_key, "from", "value")
                }
                new_message.update(role = standard_role, content = original_content)
                new_convo.append(new_message)

            all_convos.append(new_convo)

        return {chat_column: all_convos}

    dataset_map_kwargs = {
        "batched": True,
        "batch_size": batch_size,
    }

    if not is_streaming_dataset(dataset):
        from utils.hardware import light_map_num_proc

        num_proc = light_map_num_proc(dataset, num_proc)

        dataset_map_kwargs["num_proc"] = num_proc
        dataset_map_kwargs["desc"] = "Standardizing chat format"

    result = dataset.map(_standardize_dataset, **dataset_map_kwargs)

    # For streaming, force the first mapped row through now so column/format errors surface before training rather than mid-iteration; IterableDataset re-iterates from the generator source, so this is safe.
    if is_streaming_dataset(dataset):
        try:
            next(iter(result))
        except Exception as exc:
            raise ValueError(
                f"Streaming chat-format standardization failed on the first row: {exc}"
            ) from exc

    return result


_CHATML_ROLES = {"system", "user", "assistant", "tool"}


def normalize_chatml_roles(
    dataset,
    chat_column,
    batch_size = 1000,
):
    def _fold_message(message):
        if not isinstance(message, dict):
            return message
        role = _normalize_role_alias(message.get("role"))
        if role in _CHATML_ROLES and message.get("role") != role:
            return {**message, "role": role}
        return message

    try:
        import pyarrow.compute as pc
        roles = pc.unique(
            pc.struct_field(pc.list_flatten(dataset.data.column(chat_column)), "role")
        )
        if not any(
            _normalize_role_alias(r) in _CHATML_ROLES and _normalize_role_alias(r) != r
            for r in roles.to_pylist()
        ):
            return dataset
    except Exception:
        pass

    def _fold(examples):
        return {
            chat_column: [
                [_fold_message(message) for message in convo] if isinstance(convo, list) else convo
                for convo in examples[chat_column]
            ]
        }

    return dataset.map(
        _fold,
        batched = True,
        batch_size = batch_size,
        features = getattr(dataset, "features", None),
    )


def _content_text(content):
    if isinstance(content, list):
        return "\n".join(
            part["text"]
            for part in content
            if isinstance(part, dict) and part.get("type") == "text" and part.get("text")
        )
    return cell_text(content)


def convert_chatml_to_alpaca(
    dataset,
    batch_size = 1000,
    num_proc = None,
    chat_column: str | None = None,
):
    """Convert ChatML to Alpaca format. Accepts a "messages" or "conversations" column with either standard "role"/"content" or ShareGPT "from"/"value" keys."""
    is_iterable = is_streaming_dataset(dataset)
    roles = {
        "system": "system",
        "user": "user",
        "human": "user",
        "input": "user",
        "assistant": "assistant",
        "gpt": "assistant",
        "output": "assistant",
    }

    def _convert(examples):
        chatml_data = examples.get(chat_column) if chat_column else None
        if chatml_data is None:
            chatml_data = (
                examples.get("messages") or examples.get("conversations") or examples.get("texts")
            )

        if chatml_data is None:
            raise ValueError("No 'messages' or 'conversations' or 'texts' column found.")

        instructions = []
        outputs = []
        inputs = []
        column_systems = examples.get("system") or [None] * len(chatml_data)

        for convo, column_system in zip(chatml_data, column_systems):
            turns = []
            for msg in convo or []:
                role = roles.get(_normalize_role_alias(msg.get("role") or msg.get("from")))
                content = _content_text(msg.get("content") or msg.get("value"))
                if role is None or not content:
                    continue
                if turns and turns[-1][0] == role:
                    turns[-1][1] = f"{turns[-1][1]}\n\n{content}"
                else:
                    turns.append([role, content])
            # A separate system column fills in only when the conversation has no leading system turn.
            if (
                isinstance(column_system, str)
                and column_system.strip()
                and not (turns and turns[0][0] == "system")
            ):
                turns.insert(0, ["system", column_system])

            system = ""
            context = []
            instruction = None
            for role, content in turns:
                if role == "system":
                    system = f"{system}\n\n{content}" if system else content
                elif role == "user":
                    instruction = content
                elif instruction is not None:
                    instructions.append(instruction)
                    inputs.append(
                        "\n\n".join(part for part in (system, "\n".join(context)) if part)
                    )
                    outputs.append(content)
                    context += [f"User: {instruction}", f"Assistant: {content}"]
                    instruction = None

        return {"instruction": instructions, "input": inputs, "output": outputs}

    dataset_map_kwargs = {
        "batched": True,
        "batch_size": batch_size,
        "remove_columns": dataset.column_names or list(next(iter(dataset), {})),
    }

    if not is_iterable:
        from utils.hardware import light_map_num_proc

        num_proc = light_map_num_proc(dataset, num_proc)

        dataset_map_kwargs["num_proc"] = num_proc
        dataset_map_kwargs["desc"] = "Converting ChatML to Alpaca format"

    result = dataset.map(_convert, **dataset_map_kwargs)

    # For streaming, force the first mapped row through now so column/format errors surface before training rather than mid-iteration; IterableDataset re-iterates from the generator source, so this is safe.
    if is_iterable:
        try:
            next(iter(result))
        except Exception as exc:
            raise ValueError(
                f"Streaming ChatML-to-Alpaca conversion failed on the first row: {exc}"
            ) from exc

    return result


def convert_alpaca_to_chatml(
    dataset,
    batch_size = 1000,
    num_proc = None,
):
    """Convert Alpaca format to ChatML: a 'conversations' column of standard 'role'/'content' dicts."""
    is_iterable = is_streaming_dataset(dataset)

    def _convert(examples):
        conversations = []

        for i in range(len(examples["instruction"])):
            instruction = examples["instruction"][i]
            input_text = examples.get("input", [""] * len(examples["instruction"]))[i]
            output = examples["output"][i]
            instruction, input_text, output = (
                cell_text(value) for value in (instruction, input_text, output)
            )

            if input_text and input_text.strip():
                user_content = f"{instruction}\n\n{input_text}".strip()
            else:
                user_content = instruction

            convo = [
                {"role": "user", "content": user_content},
                {"role": "assistant", "content": output},
            ]
            conversations.append(convo)

        return {"conversations": conversations}

    dataset_map_kwargs = {
        "batched": True,
        "batch_size": batch_size,
    }

    if not is_iterable:
        from utils.hardware import light_map_num_proc

        num_proc = light_map_num_proc(dataset, num_proc)

        dataset_map_kwargs["num_proc"] = num_proc
        dataset_map_kwargs["desc"] = "Converting Alpaca to ChatML format"

    result = dataset.map(_convert, **dataset_map_kwargs)

    # For streaming, force the first mapped row through now so column/format errors surface before training rather than mid-iteration; IterableDataset re-iterates from the generator source, so this is safe.
    if is_iterable:
        try:
            next(iter(result))
        except Exception as exc:
            raise ValueError(
                f"Streaming Alpaca-to-ChatML conversion failed on the first row: {exc}"
            ) from exc

    return result


def _format_eta(seconds):
    """Format seconds into a human-readable ETA string."""
    if seconds < 60:
        return f"{seconds:.0f}s"
    elif seconds < 3600:
        m, s = divmod(int(seconds), 60)
        return f"{m}m {s}s"
    else:
        h, remainder = divmod(int(seconds), 3600)
        m, _ = divmod(remainder, 60)
        return f"{h}h {m}m"


# huggingface_hub snapshot_download's default; caps parallel requests to one image host or the Hub.
_MAX_ROW_FETCH_WORKERS = 8


def _fetch_row_image(url):
    from io import BytesIO

    from PIL import Image

    fetch = fetch_opted_in_image_bytes if private_sources_allowed() else fetch_untrusted_image_bytes
    return Image.open(BytesIO(fetch(url))).convert("RGB")


def _settle_in_order(rows, task, record, num_workers):
    """Calls ``record(row, future)`` in row order; ``future`` runs ``task(row)`` on a pool (None if serial or ``task`` gives None)."""
    if num_workers <= 1:
        for row in rows:
            record(row, None)
        return
    from collections import deque
    from concurrent.futures import ThreadPoolExecutor

    # Settling here keeps output, random caption picks and the first-failure log as serial; 2x workers bounds images held.
    pending = deque()
    with ThreadPoolExecutor(max_workers = num_workers) as executor:
        try:
            for row in rows:
                fn = task(row)
                # A fresh context copy per task: it sees the caller's account / HF cache settings.
                future = None if fn is None else executor.submit(contextvars.copy_context().run, fn)
                pending.append((row, future))
                if len(pending) > 2 * num_workers:
                    record(*pending.popleft())
            while pending:
                record(*pending.popleft())
        except BaseException:
            executor.shutdown(wait = False, cancel_futures = True)
            raise


def _non_null_rows(dataset, column):
    """Row positions whose ``column`` is not None, read from Arrow validity so cells are never built or decoded."""
    try:
        # numpy / pandas / torch / custom formats turn nulls into values (NaN, pd.NA) the walk keeps.
        if dataset.format["type"] not in (None, "python"):
            raise TypeError("formatted dataset")
        values = dataset.with_format("arrow")[column]
        if values.null_count == 0:
            return range(len(values))
        import numpy as np
        import pyarrow as pa
        import pyarrow.compute as pc

        valid = pc.is_valid(values)
        if isinstance(valid, pa.ChunkedArray):
            valid = valid.combine_chunks()
        return np.flatnonzero(valid.to_numpy(zero_copy_only = False)).tolist()
    except Exception:
        return [i for i, value in enumerate(dataset[column]) if value is not None]


def convert_to_vlm_format(
    dataset,
    instruction = None,
    text_column = "text",
    image_column = "image",
    dataset_name = None,
    progress_callback = None,
):
    """Returns a list, not a Dataset, so PIL Images are preserved; URL datasets are probed first."""
    from PIL import Image
    from utils.account_context import current_account
    from .vlm_processing import generate_smart_vlm_instruction

    account = current_account()

    def _notify(msg):
        """Send a status update to the training overlay if callback set."""
        if progress_callback:
            progress_callback(status_message = msg)

    if instruction is None:
        instruction_info = generate_smart_vlm_instruction(
            dataset,
            text_column = text_column,
            image_column = image_column,
            dataset_name = dataset_name,
        )

        instruction = instruction_info["instruction"]
        instruction_column = instruction_info.get("instruction_column")
        uses_dynamic = instruction_info["uses_dynamic_instruction"]

        logger.info(f"📝 Auto-detected instruction type: {instruction_info['instruction_type']}")
        logger.info(f"📝 Confidence: {instruction_info['confidence']:.2f}")
        if not uses_dynamic:
            logger.info(f"📝 Using instruction: '{instruction}'")
        else:
            logger.info(f"📝 Using dynamic instructions from column: '{instruction_column}'")
    else:
        instruction_column = None
        uses_dynamic = False

    def _fetch_repo_image(name):
        from huggingface_hub import hf_hub_download
        from utils.hf_cache_settings import active_hf_hub_cache

        local_path = hf_hub_download(
            dataset_name,
            _image_lookup[name],
            repo_type = "dataset",
            cache_dir = active_hf_hub_cache(),
        )
        return Image.open(local_path).convert("RGB")

    def _convert_single_sample(sample, fetched = None):
        """Image may be a PIL Image, local path, URL, or bare filename; ``fetched`` = the URL's or repo file's already-downloaded image."""
        image_data = sample[image_column]

        if isinstance(image_data, str):
            if image_data.startswith(("http://", "https://")):
                image_data = fetched if fetched is not None else _fetch_row_image(image_data)
            elif _image_lookup is not None and image_data in _image_lookup:
                # Bare filename → resolve via HF repo lookup
                image_data = fetched if fetched is not None else _fetch_repo_image(image_data)
            else:
                image_data = Image.open(_own_image_path(image_data, account)).convert("RGB")
        elif image_data is None:
            raise ValueError("Row has no image")

        text_data = sample[text_column]
        if isinstance(text_data, list) and len(text_data) > 0:
            import random
            text_data = random.choice(text_data)

        if uses_dynamic and instruction_column:
            current_instruction = sample[instruction_column]
            if not isinstance(current_instruction, str) or not current_instruction.strip():
                current_instruction = "Describe this image in detail."
        else:
            current_instruction = instruction

        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": current_instruction},
                    {"type": "image", "image": image_data},
                ],
            },
            {"role": "assistant", "content": [{"type": "text", "text": text_data}]},
        ]

        return {"messages": messages}

    total = len(dataset)
    first_image = next(
        (row[image_column] for row in dataset if row[image_column] is not None), None
    )
    has_urls = isinstance(first_image, str) and first_image.startswith(("http://", "https://"))

    if has_urls:
        with_image = _non_null_rows(dataset, image_column)
        if len(with_image) < total:
            logger.info(f"Skipping {total - len(with_image)}/{total} rows without an image")
            dataset = dataset.select(with_image)
            total = len(with_image)

    # A null caption would render as the word "None" in the assistant turn, so leave those rows out like rows without an image.
    with_text = _non_null_rows(dataset, text_column)
    if len(with_text) < total:
        logger.info(f"Skipping {total - len(with_text)}/{total} rows without text")
        dataset = dataset.select(with_text)
        total = len(with_text)

    _image_lookup = None
    _IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff")
    if (
        not has_urls
        and isinstance(first_image, str)
        and not is_network_share(first_image)
        and not os.path.exists(first_image)
        and dataset_name
    ):
        try:
            from huggingface_hub import HfApi

            _notify("Resolving image filenames from HF repo...")
            logger.info(
                f"🔍 Image column contains bare filenames (e.g. '{first_image}') — building repo lookup..."
            )
            repo_files = HfApi().list_repo_files(dataset_name, repo_type = "dataset")
            _image_lookup = {
                os.path.basename(f): f
                for f in repo_files
                if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS)
            }
            # Also key by the full relative path, e.g. "images/0001.jpg", as the ShareGPT converter does.
            for f in repo_files:
                if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS):
                    _image_lookup[f] = f
            if first_image in _image_lookup:
                logger.info(
                    f"✅ Matched {len(_image_lookup)} image files in repo (e.g. '{first_image}' → '{_image_lookup[first_image]}')"
                )
            else:
                logger.info(
                    f"⚠️ Built lookup with {len(_image_lookup)} images but '{first_image}' not found — falling back to local open"
                )
                _image_lookup = None
        except Exception as e:
            logger.info(f"⚠️ Failed to build HF repo image lookup: {e}")
            _image_lookup = None

    # URL probe: 200 parallel samples to estimate speed and failure rate.
    PROBE_SIZE = 200
    MAX_FAIL_RATE = 0.3

    # Successful probe downloads by row, reused by the conversion.
    probe_images = {}

    if has_urls and total > PROBE_SIZE:
        import time
        from concurrent.futures import ThreadPoolExecutor, as_completed
        from utils.hardware import safe_thread_num_proc

        num_workers = safe_thread_num_proc()
        _notify(f"Probing {PROBE_SIZE} image URLs with {num_workers} workers...")
        logger.info(f"🔍 Probing {PROBE_SIZE}/{total} image URLs with {num_workers} workers...")

        probe_samples = [dataset[i] for i in range(PROBE_SIZE)]
        probe_ok = 0
        probe_fail = 0
        probe_start = time.time()

        def _probe_sample(index, sample):
            url = sample[image_column]
            if isinstance(url, str) and url.startswith(("http://", "https://")):
                # A failed download is not kept, so the conversion retries it.
                probe_images[index] = _fetch_row_image(url)
            return _convert_single_sample(sample, probe_images.get(index))

        with ThreadPoolExecutor(max_workers = num_workers) as executor:
            futures = {executor.submit(_probe_sample, i, s): s for i, s in enumerate(probe_samples)}
            for future in as_completed(futures):
                try:
                    future.result()
                    probe_ok += 1
                except Exception:
                    probe_fail += 1

        probe_elapsed = time.time() - probe_start
        probe_total = probe_ok + probe_fail
        fail_rate = probe_fail / probe_total if probe_total > 0 else 0
        throughput = probe_total / probe_elapsed if probe_elapsed > 0 else 0

        if fail_rate >= MAX_FAIL_RATE:
            issues = [
                f"{fail_rate:.0%} of the first {PROBE_SIZE} image URLs failed to download ({probe_fail}/{probe_total})",
                "Images are external URLs, not embedded in the dataset",
            ]
            friendly = None
            try:
                from .llm_assist import llm_generate_dataset_warning
                friendly = llm_generate_dataset_warning(
                    issues,
                    dataset_name = dataset_name,
                    modality = "vision",
                    column_names = [image_column, text_column],
                )
            except Exception:
                pass
            msg = friendly or (
                f"⚠️ {fail_rate:.0%} of the first {PROBE_SIZE} images failed to download "
                f"({probe_fail}/{probe_total}). "
                "This dataset has too many broken or unreachable image URLs. "
                "Consider using a dataset with embedded images instead."
            )
            logger.info(msg)
            _notify(msg)
            raise ValueError(msg)

        remaining = total - PROBE_SIZE
        estimated_seconds = remaining / throughput if throughput > 0 else 0
        eta_str = _format_eta(estimated_seconds)

        info_msg = (
            f"Downloading {total:,} images ({num_workers} workers, ~{throughput:.1f} img/s). "
            f"Estimated time: ~{eta_str}"
        )
        if probe_fail > 0:
            info_msg += f" | {fail_rate:.0%} broken URLs will be skipped"

        logger.info(
            f"✅ Probe passed: {probe_ok}/{probe_total} ok, {probe_fail} failed ({fail_rate:.0%}), {throughput:.1f} img/s"
        )
        logger.info(f"⏱️ Estimated time for {total:,} samples: ~{eta_str}")
        _notify(info_msg)

    from tqdm import tqdm

    logger.info(f"🔄 Converting {total} samples to VLM format...")
    converted_list = []
    failed_count = 0

    if has_urls:
        import time
        from concurrent.futures import ThreadPoolExecutor, as_completed
        from utils.hardware import safe_thread_num_proc

        num_workers = safe_thread_num_proc()
        batch_size = 500
        start_time = time.time()

        for batch_start in range(0, total, batch_size):
            batch_end = min(batch_start + batch_size, total)
            batch_samples = [dataset[i] for i in range(batch_start, batch_end)]

            with ThreadPoolExecutor(max_workers = num_workers) as executor:
                futures = {
                    executor.submit(
                        _convert_single_sample, s, probe_images.pop(batch_start + i, None)
                    ): i
                    for i, s in enumerate(batch_samples)
                }
                batch_results = [None] * len(batch_samples)
                for future in as_completed(futures):
                    idx = futures[future]
                    try:
                        batch_results[idx] = future.result()
                    except Exception as e:
                        failed_count += 1
                        if failed_count == 1:
                            logger.info(f"First VLM conversion failure: {type(e).__name__}: {e}")

            converted_list.extend(r for r in batch_results if r is not None)

            elapsed = time.time() - start_time
            done = batch_end
            rate = done / elapsed if elapsed > 0 else 0
            remaining_time = (total - done) / rate if rate > 0 else 0
            eta_str = _format_eta(remaining_time)
            progress_msg = f"Downloading images: {done:,}/{total:,} ({done*100//total}%) | ~{eta_str} remaining | {failed_count} skipped"
            logger.info(
                f"  [{done}/{total}] {rate:.1f} img/s, {failed_count} failed, ETA {eta_str}"
            )
            _notify(progress_msg)
    else:
        pbar = tqdm(
            dataset,
            total = total,
            desc = "Converting VLM samples",
            unit = "sample",
            **_quiet_bar_kwargs(),
        )

        def _record(sample, future = None):
            nonlocal failed_count
            try:
                fetched = future.result() if future is not None else None
                converted_list.append(_convert_single_sample(sample, fetched))
            except Exception as e:
                failed_count += 1
                if failed_count == 1:
                    logger.info(f"First VLM conversion failure: {type(e).__name__}: {e}")
            pbar.set_postfix(ok = len(converted_list), failed = failed_count, refresh = False)

        from utils.hardware import safe_thread_num_proc

        num_workers = (
            min(safe_thread_num_proc(), _MAX_ROW_FETCH_WORKERS) if _image_lookup is not None else 1
        )

        def _repo_fetch(sample):
            name = sample[image_column]
            if isinstance(name, str) and name in _image_lookup:
                return functools.partial(_fetch_repo_image, name)
            return None

        try:
            _settle_in_order(pbar, _repo_fetch, _record, num_workers)
        finally:
            pbar.close()

    if failed_count > 0:
        fail_rate = failed_count / total
        logger.info(
            f"⚠️ Skipped {failed_count}/{total} ({fail_rate:.0%}) samples with broken/unreachable images"
        )
        # Small URL datasets skip the probe, so check the fail rate here.
        if has_urls and fail_rate >= MAX_FAIL_RATE:
            issues = [
                f"{fail_rate:.0%} of images failed to download ({failed_count}/{total})",
                "Images are external URLs, not embedded in the dataset",
            ]
            friendly = None
            try:
                from .llm_assist import llm_generate_dataset_warning
                friendly = llm_generate_dataset_warning(
                    issues,
                    dataset_name = dataset_name,
                    modality = "vision",
                    column_names = [image_column, text_column],
                )
            except Exception:
                pass
            msg = friendly or (
                f"⚠️ {fail_rate:.0%} of images failed to download ({failed_count}/{total}). "
                "This dataset has too many broken or unreachable image URLs. "
                "Consider using a dataset with embedded images instead."
            )
            _notify(msg)
            raise ValueError(msg)

    if len(converted_list) == 0:
        issues = [
            f"All {total} samples failed during VLM conversion — no usable images found",
            f"Image column '{image_column}' may contain URLs that are no longer accessible, "
            "or local file paths that don't exist",
        ]
        friendly = None
        try:
            from .llm_assist import llm_generate_dataset_warning
            friendly = llm_generate_dataset_warning(
                issues,
                dataset_name = dataset_name,
                modality = "vision",
                column_names = [image_column, text_column],
            )
        except Exception:
            pass
        raise ValueError(
            friendly
            or (
                f"All {total} samples failed during VLM conversion — no usable images found. "
                "This dataset may contain only image URLs that are no longer accessible."
            )
        )

    logger.info(f"✅ Converted {len(converted_list)}/{total} samples")
    _notify(f"Converted {len(converted_list):,}/{total:,} images successfully")

    return converted_list


# A real video container never starts like this. A playlist / manifest saved as clip.mp4 would make
# FFmpeg open the URLs inside it itself, outside the guarded remote-media fetch.
_PLAYLIST_HEADS = (b"#EXTM3U", b"<", b"ffconcat", b"\xef\xbb\xbf", b" ", b"\t", b"\r", b"\n")


def _refuse_hub_playlist(local_path, name):
    try:
        with open(local_path, "rb") as handle:
            head = handle.read(64)
    except OSError:
        return local_path  # unreadable: the decoder reports it
    if head.startswith(_PLAYLIST_HEADS):
        raise ValueError(f"Video '{name}' is a playlist or text document, not a video file.")
    return local_path


class _HubClips:
    """Repo-relative clip names of a Hub dataset, resolved by download the way bare image filenames are."""

    def __init__(
        self,
        dataset_name,
        revision = None,
    ):
        self.repo, self.revision, self.files, self.listed = dataset_name, revision, None, False
        if (
            not isinstance(dataset_name, str)
            or dataset_name.count("/") != 1
            or os.path.exists(dataset_name)
        ):
            self.repo = None

    def resolve(self, path):
        if self.repo is None or os.path.exists(path) or "://" in path or os.path.isabs(path):
            return None
        if self.files is None:
            from huggingface_hub import HfApi

            from .format_detection import _VIDEO_EXTENSIONS

            self.files, ambiguous = {}, {}
            try:
                repo_files = HfApi().list_repo_files(
                    self.repo, repo_type = "dataset", revision = self.revision
                )
                self.listed = True
            except Exception as e:  # noqa: BLE001 -- not a Hub repo or offline: paths stay local
                logger.info(f"Could not list {self.repo} to resolve video filenames: {e}")
                repo_files = []
            for f in repo_files:
                if f.lower().endswith(_VIDEO_EXTENSIONS + (".webm",)):
                    ambiguous.setdefault(os.path.basename(f), []).append(f)
            for f in repo_files:
                if f.lower().endswith(_VIDEO_EXTENSIONS + (".webm",)):
                    name = os.path.basename(f)
                    # A name in several folders resolves to its list, refused below: one copy is a guess.
                    self.files.setdefault(name, f if len(ambiguous[name]) == 1 else ambiguous[name])
                    self.files[f] = f
        from huggingface_hub import hf_hub_download

        from utils.hf_cache_settings import active_hf_hub_cache

        if path not in self.files:
            if self.listed:
                return None
            # Offline: an exact repo path may still be in the cache.
            try:
                cached = hf_hub_download(
                    self.repo,
                    path,
                    repo_type = "dataset",
                    revision = self.revision,
                    cache_dir = active_hf_hub_cache(),
                    local_files_only = True,
                )
            except Exception:  # noqa: BLE001 -- not cached: the path stays local
                return None
            return _refuse_hub_playlist(cached, path)
        if isinstance(self.files[path], list):
            raise ValueError(
                f"Video '{path}' matches several files in {self.repo} ({', '.join(self.files[path])}); "
                "use the repo-relative path."
            )
        return _refuse_hub_playlist(
            hf_hub_download(
                self.repo,
                self.files[path],
                repo_type = "dataset",
                revision = self.revision,
                cache_dir = active_hf_hub_cache(),
            ),
            path,
        )


def _own_video_part(
    video,
    account,
    hub = None,
):
    from .format_detection import encoded_video

    video = encoded_video(video)
    embedded = isinstance(video, dict) and bool(video.get("bytes"))
    if isinstance(video, dict):
        video = video.get("path")
    # Before any stat: probing a UNC path can already contact the share.
    if isinstance(video, str) and is_network_share(video) and not private_sources_allowed():
        raise OSError(
            f"Refusing a video path on a network share: {video}. "
            "Set UNSLOTH_ALLOW_PRIVATE_URL_FETCH=1 to load videos from shares you trust."
        )
    # An embedded clip's path is often just its original filename, with no file behind it.
    if embedded and not (isinstance(video, str) and os.path.isfile(video)):
        resolved = (
            hub.resolve(video)
            if hub is not None and isinstance(video, str) and video.strip()
            else None
        )
        if resolved is not None:
            return resolved
        raise ValueError(
            "A video is stored as embedded bytes with no readable file path; training reads clips from "
            "files, so save the clips to files and point the column at their paths."
        )
    if not isinstance(video, str) or not video.strip():
        raise ValueError("Row has no video")
    if video.strip().lower().startswith(("http://", "https://")):
        # The collator would fetch it unguarded; the image fetchers' 64 MB cap does not fit clips.
        raise ValueError(
            f"Remote video URLs are not fetched for training: {video}. "
            "Download the clips and point the column at local files or a Hub dataset."
        )
    resolved = hub.resolve(video) if hub is not None else None
    if resolved is not None:
        return resolved
    return _own_image_path(video, account)


def own_message_videos(
    rows,
    dataset_name = None,
    dataset_revision = None,
):
    """VLM messages rows as a list, every video path checked like an image path."""
    from utils.account_context import current_account
    from .format_detection import video_parts

    account = current_account()
    hub = _HubClips(dataset_name, dataset_revision)
    rows = [row for row in rows]
    for row in rows:
        if not isinstance(row.get("messages"), list):
            continue
        for message in row["messages"]:
            content = message.get("content") if isinstance(message, dict) else None
            if not isinstance(content, list):
                continue
            # Arrow fills absent keys with None; templates and the collator read key presence.
            content[:] = [
                {k: v for k, v in part.items() if v is not None} if isinstance(part, dict) else part
                for part in content
            ]
        for part in video_parts(row["messages"]):
            part["video"] = _own_video_part(part["video"], account, hub)
    return rows


def convert_video_to_vlm_format(
    dataset,
    video_column,
    text_column,
    question_column = None,
    instruction = None,
    video_columns = None,
    dataset_name = None,
    dataset_revision = None,
):
    """Flat video + text rows to VLM messages; clips stay paths for the collator to decode."""
    from utils.account_context import current_account

    from .format_detection import _holds_images, _is_image_value

    account = current_account()
    hub = _HubClips(dataset_name, dataset_revision)
    answer_feature = (getattr(dataset, "features", None) or {}).get(text_column)
    as_label = getattr(answer_feature, "int2str", None)
    default_instruction = instruction or "Describe this video."
    converted = []
    for row in dataset:
        answer = row.get(text_column)
        clip_columns = video_columns or [video_column]
        # Detection probes a few rows; an image further down would otherwise be dropped silently.
        images = [
            c
            for c, v in row.items()
            if c not in clip_columns and (_is_image_value(v) or _holds_images(v))
        ]
        if images:
            raise ValueError(
                f"Rows mix video with image columns {images}; video conversion would drop the images, "
                "so remove those columns or split the dataset."
            )
        clips = [
            row[c]
            for c in clip_columns
            if row.get(c) is not None and not (isinstance(row[c], str) and not row[c].strip())
        ]
        if not clips or answer is None or not str(answer).strip():
            continue
        answer = as_label(answer) if as_label and isinstance(answer, int) else str(answer)
        question = row.get(question_column) if question_column else None
        if not isinstance(question, str) or not question.strip():
            question = default_instruction
        converted.append(
            {
                "messages": [
                    {
                        "role": "user",
                        "content": [
                            *(
                                {"type": "video", "video": _own_video_part(clip, account, hub)}
                                for clip in clips
                            ),
                            {"type": "text", "text": question},
                        ],
                    },
                    {"role": "assistant", "content": [{"type": "text", "text": answer}]},
                ]
            }
        )
    if not converted:
        raise ValueError(
            f"No rows have both a video in '{video_column}' and text in '{text_column}'"
        )
    skipped = len(dataset) - len(converted) if hasattr(dataset, "__len__") else 0
    if skipped:
        logger.info(f"Skipping {skipped} rows without a video or text")
    return converted


def convert_sharegpt_with_images_to_vlm_format(
    dataset,
    image_column = "image",
    messages_column = "conversations",
    dataset_name = None,
    progress_callback = None,
):
    """Convert ShareGPT/ChatML datasets carrying a separate image column and ``<image>`` placeholders in the conversation text, e.g. {"image": "sam/images/sa_545504.jpg", "conversations": [{"from": "human", "value": "<image> What is this photo about?"}, ...]}. Image lists match placeholders in conversation order. Returns a list of dicts in standard VLM messages format, PIL Images inline."""
    from PIL import Image
    from tqdm import tqdm
    from utils.account_context import current_account

    account = current_account()
    _IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff")
    _ROLE_MAP = {
        "human": "user",
        "user": "user",
        "gpt": "assistant",
        "assistant": "assistant",
        "system": "system",
    }

    def _notify(msg):
        if progress_callback:
            progress_callback(status_message = msg)

    # Resolve image loading, the same 3 tiers as convert_to_vlm_format.
    total = len(dataset)
    first_image = None
    for sample in dataset:
        image_data = sample.get(image_column)
        if isinstance(image_data, list):
            image_data = next((image for image in image_data if image is not None), None)
        if image_data is not None:
            first_image = image_data
            break

    _image_lookup = None
    if (
        isinstance(first_image, str)
        and not first_image.startswith(("http://", "https://"))
        and not is_network_share(first_image)
        and not os.path.exists(first_image)
        and dataset_name
    ):
        try:
            from huggingface_hub import HfApi

            _notify("Resolving image filenames from HF repo...")
            logger.info(
                f"🔍 Image column contains bare filenames (e.g. '{first_image}') — building repo lookup..."
            )
            repo_files = HfApi().list_repo_files(dataset_name, repo_type = "dataset")
            _image_lookup = {
                os.path.basename(f): f
                for f in repo_files
                if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS)
            }
            # Also key by the full relative path, e.g. "sam/images/sa_545504.jpg".
            for f in repo_files:
                if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS):
                    _image_lookup[f] = f
            if first_image in _image_lookup:
                logger.info(
                    f"✅ Matched {len(_image_lookup)} image files in repo (e.g. '{first_image}' → '{_image_lookup[first_image]}')"
                )
            else:
                logger.info(
                    f"⚠️ Built lookup with {len(_image_lookup)} images but '{first_image}' not found; unmatched filenames use local paths"
                )
        except Exception as e:
            logger.info(f"⚠️ Failed to build HF repo image lookup: {e}")
            _image_lookup = None

    def _resolve_image(image_data):
        """Resolve image data to a PIL Image."""
        if hasattr(image_data, "size") and hasattr(image_data, "mode"):
            return image_data
        if isinstance(image_data, str):
            if image_data.startswith(("http://", "https://")):
                return _fetch_row_image(image_data)
            elif _image_lookup is not None and image_data in _image_lookup:
                from huggingface_hub import hf_hub_download
                from utils.hf_cache_settings import active_hf_hub_cache

                local_path = hf_hub_download(
                    dataset_name,
                    _image_lookup[image_data],
                    repo_type = "dataset",
                    cache_dir = active_hf_hub_cache(),
                )
                return Image.open(local_path).convert("RGB")
            else:
                return Image.open(_own_image_path(image_data, account)).convert("RGB")
        if isinstance(image_data, dict) and ("bytes" in image_data or "path" in image_data):
            if image_data.get("bytes"):
                from io import BytesIO
                return Image.open(BytesIO(image_data["bytes"])).convert("RGB")
            if image_data.get("path"):
                return Image.open(_own_image_path(image_data["path"], account)).convert("RGB")
        raise ValueError(f"Cannot resolve image: {type(image_data)}")

    def _convert_single_sample(sample):
        """Convert one ShareGPT+image sample to standard VLM format."""
        image_data = sample[image_column]
        conversation = sample[messages_column]
        is_image_list = isinstance(image_data, list)
        if is_image_list:
            marker_count = sum(
                (msg.get("value") or msg.get("content") or "").count("<image>")
                for msg in conversation
            )
            if not image_data or len(image_data) != marker_count:
                raise ValueError(
                    f"Image list has {len(image_data)} images but conversation has "
                    f"{marker_count} <image> placeholders"
                )
            pil_images = [_resolve_image(image) for image in image_data]
        else:
            pil_images = [_resolve_image(image_data)]

        new_messages = []
        image_index = 0
        for msg in conversation:
            role_raw = msg.get("from") or msg.get("role", "user")
            role = _ROLE_MAP.get(role_raw.lower(), role_raw.lower())
            text = msg.get("value") or msg.get("content") or ""

            if "<image>" in text:
                parts = text.split("<image>")
                content = []
                for i, part in enumerate(parts):
                    part = part.strip()
                    if part:
                        content.append({"type": "text", "text": part})
                    if i < len(parts) - 1:
                        content.append(
                            {
                                "type": "image",
                                "image": pil_images[image_index if is_image_list else 0],
                            }
                        )
                        image_index += 1
                if not content:
                    content.append({"type": "image", "image": pil_images[0]})
            else:
                content = [{"type": "text", "text": text}]

            new_messages.append({"role": role, "content": content})

        return {"messages": new_messages}

    logger.info(f"🔄 Converting {total} samples from ShareGPT+image format...")
    converted_list = []
    failed_count = 0

    def _record(sample, future):
        nonlocal failed_count
        try:
            converted_list.append(
                future.result() if future is not None else _convert_single_sample(sample)
            )
        except Exception as e:
            failed_count += 1
            if failed_count == 1:
                logger.info(f"⚠️ First conversion failure: {type(e).__name__}: {e}")
        pbar.set_postfix(ok = len(converted_list), failed = failed_count, refresh = False)

    num_workers = 1
    if _image_lookup is not None or (
        isinstance(first_image, str) and first_image.startswith(("http://", "https://"))
    ):
        from utils.hardware import safe_thread_num_proc
        num_workers = min(safe_thread_num_proc(), _MAX_ROW_FETCH_WORKERS)

    pbar = tqdm(
        dataset,
        total = total,
        desc = "Converting ShareGPT+image",
        unit = "sample",
        **_quiet_bar_kwargs(),
    )
    try:
        _settle_in_order(
            pbar,
            lambda sample: functools.partial(_convert_single_sample, sample),
            _record,
            num_workers,
        )
    finally:
        pbar.close()

    if failed_count > 0:
        logger.info(f"⚠️ Skipped {failed_count}/{total} ({failed_count*100//total}%) samples")

    if len(converted_list) == 0:
        raise ValueError(
            f"All {total} samples failed during ShareGPT+image conversion — "
            "no usable samples found."
        )

    logger.info(f"✅ Converted {len(converted_list)}/{total} samples")
    _notify(f"Converted {len(converted_list):,}/{total:,} samples successfully")
    return converted_list


def convert_llava_to_vlm_format(dataset):
    """Convert Llava format to standard VLM format: Llava carries messages whose content blocks name an image by {'type': 'image', 'index': 0} plus a parallel images list, while the standard form inlines the PIL object as {'type': 'image', 'image': PIL_Image}."""
    from PIL import Image
    from utils.account_context import current_account

    account = current_account()

    is_iterable = is_streaming_dataset(dataset)
    if is_iterable:
        logger.info("🔄 Converting streaming samples from Llava format to standard VLM format...")
    else:
        logger.info(
            f"🔄 Converting {len(dataset)} samples from Llava format to standard VLM format..."
        )

    def _convert_single_sample(sample):
        """Convert one llava sample to standard VLM format."""
        messages = sample["messages"]
        images = sample.get("images", [])
        unindexed_image_index = 0
        explicit_image_indices = set()
        for message in messages:
            content = message.get("content")
            if not isinstance(content, list):
                continue
            for item in content:
                if not isinstance(item, dict) or item.get("type") != "image":
                    continue
                image_index = item.get("index")
                if isinstance(image_index, int) and not isinstance(image_index, bool):
                    explicit_image_indices.add(image_index)

        new_messages = []
        for msg in messages:
            new_content = []
            message_content = msg["content"]

            if isinstance(message_content, str):
                new_messages.append(
                    {
                        "role": msg["role"],
                        "content": [{"type": "text", "text": message_content}],
                    }
                )
                continue

            for item in message_content:
                if item["type"] == "image":
                    img_idx = item.get("index")
                    if img_idx is None:
                        while unindexed_image_index in explicit_image_indices:
                            unindexed_image_index += 1
                        img_idx = unindexed_image_index
                        unindexed_image_index += 1
                    if not isinstance(img_idx, int) or isinstance(img_idx, bool):
                        raise ValueError(f"Invalid Llava image index: {img_idx!r}")
                    if not 0 <= img_idx < len(images):
                        raise ValueError(
                            f"Llava image index {img_idx} is missing from a row with "
                            f"{len(images)} images"
                        )

                    pil_image = images[img_idx]
                    if isinstance(pil_image, str):
                        pil_image = Image.open(_own_image_path(pil_image, account)).convert("RGB")
                    elif not isinstance(pil_image, Image.Image):
                        raise ValueError(
                            f"Unsupported Llava image value at index {img_idx}: "
                            f"{type(pil_image).__name__}"
                        )

                    new_content.append({"type": "image", "image": pil_image})

                elif item["type"] == "text":
                    new_content.append({"type": "text", "text": item.get("text", "")})

            new_messages.append({"role": msg["role"], "content": new_content})

        return {"messages": new_messages}

    if is_iterable:
        remove_columns = dataset.column_names or list(next(iter(dataset), {}))
        converted_dataset = dataset.map(
            _convert_single_sample,
            remove_columns = remove_columns,
        )
        try:
            next(iter(converted_dataset), None)
        except Exception as exc:
            raise ValueError(f"Streaming Llava conversion failed on the first row: {exc}") from exc
        logger.info("✅ Configured streaming Llava conversion")
        return converted_dataset

    converted_list = [_convert_single_sample(sample) for sample in dataset]
    logger.info(f"✅ Converted {len(converted_list)} samples")
    return converted_list