Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Dataset format conversion between Alpaca, ShareGPT, and ChatML."""
import contextvars
import functools
import os
from .cells import cell_text
from .iterable import is_streaming_dataset
from .remote_images import (
fetch_opted_in_image_bytes,
fetch_untrusted_image_bytes,
is_network_share,
private_sources_allowed,
)
from loggers import get_logger
logger = get_logger(__name__)
def _quiet_bar_kwargs() -> dict:
"""Send our own conversion bars to a null stream while the log is quiet. They still count (the UI status poller reads tqdm._instances), they just do not write carriage-return fragments beside the worker's structured records."""
try:
from loggers.config import quiet_bar_kwargs
return quiet_bar_kwargs()
except Exception: # noqa: BLE001 - a bar is never worth failing a conversion for
return {}
def _own_image_path(path, account):
"""A local image a managed account names must be one it may read (a no-op for the owner). ``account`` is
captured when the conversion starts: URL datasets convert in a thread pool, which does not carry it."""
from utils.account_context import run_as
from utils.security.dataset_media import require_readable_media
if isinstance(path, str) and is_network_share(path) and not private_sources_allowed():
raise OSError(
f"Refusing an image path on a network share: {path}. "
"Set UNSLOTH_ALLOW_PRIVATE_URL_FETCH=1 to load images from shares you trust."
)
run_as(account, require_readable_media, path)
return path
def _normalize_role_alias(role) -> str:
"""Stripped and lowercased, as `standardize_data_formats` and the preview do it."""
if role is None:
return ""
return str(role).strip().lower()
def standardize_chat_format(
dataset,
tokenizer = None,
aliases_for_system = [
"system",
],
aliases_for_user = [
"user",
"human",
"input",
],
aliases_for_assistant = [
"gpt",
"assistant",
"output",
],
batch_size = 1000,
num_proc = None,
chat_column: str | None = None,
):
"""Standardize BOTH messages and conversations: map non-standard role names and keys to the standard format."""
import collections
import itertools
is_vlm = False
if tokenizer is not None:
if hasattr(tokenizer, "image_processor") or hasattr(tokenizer, "tokenizer"):
is_vlm = True
column_names = set(next(iter(dataset)).keys())
if chat_column:
if chat_column not in column_names:
return dataset
elif "conversations" in column_names:
chat_column = "conversations"
elif "messages" in column_names:
chat_column = "messages"
elif "texts" in column_names:
chat_column = "texts"
else:
return dataset
def _iter_probe_rows():
try:
total = min(len(dataset), 100)
for index in range(total):
yield dataset[index]
return
except Exception:
pass
for example in itertools.islice(dataset, 100):
yield example
uniques = collections.defaultdict(list)
for example in _iter_probe_rows():
chat_data = example.get(chat_column)
if not isinstance(chat_data, list) or len(chat_data) == 0:
continue
for message in chat_data:
if not isinstance(message, dict):
continue
for key, value in message.items():
if type(value) is not str:
continue
uniques[key].append(value)
if "from" in uniques and "value" in uniques:
role_key = "from"
content_key = "value"
elif "role" in uniques and "content" in uniques:
role_key = "role"
content_key = "content"
elif len(uniques.keys()) == 2:
keys = list(uniques.keys())
length_first = len(set(uniques[keys[0]]))
length_second = len(set(uniques[keys[1]]))
if length_first < length_second:
role_key = keys[0]
content_key = keys[1]
else:
role_key = keys[1]
content_key = keys[0]
else:
raise ValueError(f"Could not infer role/content keys for chat column '{chat_column}'")
# Keyed on the normalised alias: "Human" / " user " would reach the template raw.
aliases_mapping = {}
for x in aliases_for_system:
aliases_mapping[_normalize_role_alias(x)] = "system"
for x in aliases_for_user:
aliases_mapping[_normalize_role_alias(x)] = "user"
for x in aliases_for_assistant:
aliases_mapping[_normalize_role_alias(x)] = "assistant"
def _standardize_dataset(examples):
convos = examples[chat_column]
all_convos = []
for convo in convos:
if not isinstance(convo, list):
all_convos.append([])
continue
new_convo = []
for message in convo:
if not isinstance(message, dict):
continue
# Use the inferred keys first, falling back per-message so mixed ShareGPT/ChatML rows keep valid turns.
original_role = message.get(role_key)
original_content = message.get(content_key)
# Blank role counts as absent, as in the preview; an is None check trained a blank-role gpt turn as a user turn. Blank content is legitimate.
if not _normalize_role_alias(original_role):
original_role = message.get("role") or message.get("from") or ""
if original_content is None:
original_content = message.get("content")
if original_content is None:
original_content = message.get("value")
if original_content is None and not message.get("tool_calls"):
original_content = ""
# Unknown alias left as written; blank is "user", as most templates reject one.
normalized_role = _normalize_role_alias(original_role)
standard_role = (
aliases_mapping.get(normalized_role, original_role)
if normalized_role
else "user"
)
if is_vlm and original_content is not None:
original_content = [{"type": "text", "text": original_content}]
new_message = {
key: value
for key, value in message.items()
if key not in (role_key, content_key, "from", "value")
}
new_message.update(role = standard_role, content = original_content)
new_convo.append(new_message)
all_convos.append(new_convo)
return {chat_column: all_convos}
dataset_map_kwargs = {
"batched": True,
"batch_size": batch_size,
}
if not is_streaming_dataset(dataset):
from utils.hardware import light_map_num_proc
num_proc = light_map_num_proc(dataset, num_proc)
dataset_map_kwargs["num_proc"] = num_proc
dataset_map_kwargs["desc"] = "Standardizing chat format"
result = dataset.map(_standardize_dataset, **dataset_map_kwargs)
# For streaming, force the first mapped row through now so column/format errors surface before training rather than mid-iteration; IterableDataset re-iterates from the generator source, so this is safe.
if is_streaming_dataset(dataset):
try:
next(iter(result))
except Exception as exc:
raise ValueError(
f"Streaming chat-format standardization failed on the first row: {exc}"
) from exc
return result
_CHATML_ROLES = {"system", "user", "assistant", "tool"}
def normalize_chatml_roles(
dataset,
chat_column,
batch_size = 1000,
):
def _fold_message(message):
if not isinstance(message, dict):
return message
role = _normalize_role_alias(message.get("role"))
if role in _CHATML_ROLES and message.get("role") != role:
return {**message, "role": role}
return message
try:
import pyarrow.compute as pc
roles = pc.unique(
pc.struct_field(pc.list_flatten(dataset.data.column(chat_column)), "role")
)
if not any(
_normalize_role_alias(r) in _CHATML_ROLES and _normalize_role_alias(r) != r
for r in roles.to_pylist()
):
return dataset
except Exception:
pass
def _fold(examples):
return {
chat_column: [
[_fold_message(message) for message in convo] if isinstance(convo, list) else convo
for convo in examples[chat_column]
]
}
return dataset.map(
_fold,
batched = True,
batch_size = batch_size,
features = getattr(dataset, "features", None),
)
def _content_text(content):
if isinstance(content, list):
return "\n".join(
part["text"]
for part in content
if isinstance(part, dict) and part.get("type") == "text" and part.get("text")
)
return cell_text(content)
def convert_chatml_to_alpaca(
dataset,
batch_size = 1000,
num_proc = None,
chat_column: str | None = None,
):
"""Convert ChatML to Alpaca format. Accepts a "messages" or "conversations" column with either standard "role"/"content" or ShareGPT "from"/"value" keys."""
is_iterable = is_streaming_dataset(dataset)
roles = {
"system": "system",
"user": "user",
"human": "user",
"input": "user",
"assistant": "assistant",
"gpt": "assistant",
"output": "assistant",
}
def _convert(examples):
chatml_data = examples.get(chat_column) if chat_column else None
if chatml_data is None:
chatml_data = (
examples.get("messages") or examples.get("conversations") or examples.get("texts")
)
if chatml_data is None:
raise ValueError("No 'messages' or 'conversations' or 'texts' column found.")
instructions = []
outputs = []
inputs = []
column_systems = examples.get("system") or [None] * len(chatml_data)
for convo, column_system in zip(chatml_data, column_systems):
turns = []
for msg in convo or []:
role = roles.get(_normalize_role_alias(msg.get("role") or msg.get("from")))
content = _content_text(msg.get("content") or msg.get("value"))
if role is None or not content:
continue
if turns and turns[-1][0] == role:
turns[-1][1] = f"{turns[-1][1]}\n\n{content}"
else:
turns.append([role, content])
# A separate system column fills in only when the conversation has no leading system turn.
if (
isinstance(column_system, str)
and column_system.strip()
and not (turns and turns[0][0] == "system")
):
turns.insert(0, ["system", column_system])
system = ""
context = []
instruction = None
for role, content in turns:
if role == "system":
system = f"{system}\n\n{content}" if system else content
elif role == "user":
instruction = content
elif instruction is not None:
instructions.append(instruction)
inputs.append(
"\n\n".join(part for part in (system, "\n".join(context)) if part)
)
outputs.append(content)
context += [f"User: {instruction}", f"Assistant: {content}"]
instruction = None
return {"instruction": instructions, "input": inputs, "output": outputs}
dataset_map_kwargs = {
"batched": True,
"batch_size": batch_size,
"remove_columns": dataset.column_names or list(next(iter(dataset), {})),
}
if not is_iterable:
from utils.hardware import light_map_num_proc
num_proc = light_map_num_proc(dataset, num_proc)
dataset_map_kwargs["num_proc"] = num_proc
dataset_map_kwargs["desc"] = "Converting ChatML to Alpaca format"
result = dataset.map(_convert, **dataset_map_kwargs)
# For streaming, force the first mapped row through now so column/format errors surface before training rather than mid-iteration; IterableDataset re-iterates from the generator source, so this is safe.
if is_iterable:
try:
next(iter(result))
except Exception as exc:
raise ValueError(
f"Streaming ChatML-to-Alpaca conversion failed on the first row: {exc}"
) from exc
return result
def convert_alpaca_to_chatml(
dataset,
batch_size = 1000,
num_proc = None,
):
"""Convert Alpaca format to ChatML: a 'conversations' column of standard 'role'/'content' dicts."""
is_iterable = is_streaming_dataset(dataset)
def _convert(examples):
conversations = []
for i in range(len(examples["instruction"])):
instruction = examples["instruction"][i]
input_text = examples.get("input", [""] * len(examples["instruction"]))[i]
output = examples["output"][i]
instruction, input_text, output = (
cell_text(value) for value in (instruction, input_text, output)
)
if input_text and input_text.strip():
user_content = f"{instruction}\n\n{input_text}".strip()
else:
user_content = instruction
convo = [
{"role": "user", "content": user_content},
{"role": "assistant", "content": output},
]
conversations.append(convo)
return {"conversations": conversations}
dataset_map_kwargs = {
"batched": True,
"batch_size": batch_size,
}
if not is_iterable:
from utils.hardware import light_map_num_proc
num_proc = light_map_num_proc(dataset, num_proc)
dataset_map_kwargs["num_proc"] = num_proc
dataset_map_kwargs["desc"] = "Converting Alpaca to ChatML format"
result = dataset.map(_convert, **dataset_map_kwargs)
# For streaming, force the first mapped row through now so column/format errors surface before training rather than mid-iteration; IterableDataset re-iterates from the generator source, so this is safe.
if is_iterable:
try:
next(iter(result))
except Exception as exc:
raise ValueError(
f"Streaming Alpaca-to-ChatML conversion failed on the first row: {exc}"
) from exc
return result
def _format_eta(seconds):
"""Format seconds into a human-readable ETA string."""
if seconds < 60:
return f"{seconds:.0f}s"
elif seconds < 3600:
m, s = divmod(int(seconds), 60)
return f"{m}m {s}s"
else:
h, remainder = divmod(int(seconds), 3600)
m, _ = divmod(remainder, 60)
return f"{h}h {m}m"
# huggingface_hub snapshot_download's default; caps parallel requests to one image host or the Hub.
_MAX_ROW_FETCH_WORKERS = 8
def _fetch_row_image(url):
from io import BytesIO
from PIL import Image
fetch = fetch_opted_in_image_bytes if private_sources_allowed() else fetch_untrusted_image_bytes
return Image.open(BytesIO(fetch(url))).convert("RGB")
def _settle_in_order(rows, task, record, num_workers):
"""Calls ``record(row, future)`` in row order; ``future`` runs ``task(row)`` on a pool (None if serial or ``task`` gives None)."""
if num_workers <= 1:
for row in rows:
record(row, None)
return
from collections import deque
from concurrent.futures import ThreadPoolExecutor
# Settling here keeps output, random caption picks and the first-failure log as serial; 2x workers bounds images held.
pending = deque()
with ThreadPoolExecutor(max_workers = num_workers) as executor:
try:
for row in rows:
fn = task(row)
# A fresh context copy per task: it sees the caller's account / HF cache settings.
future = None if fn is None else executor.submit(contextvars.copy_context().run, fn)
pending.append((row, future))
if len(pending) > 2 * num_workers:
record(*pending.popleft())
while pending:
record(*pending.popleft())
except BaseException:
executor.shutdown(wait = False, cancel_futures = True)
raise
def _non_null_rows(dataset, column):
"""Row positions whose ``column`` is not None, read from Arrow validity so cells are never built or decoded."""
try:
# numpy / pandas / torch / custom formats turn nulls into values (NaN, pd.NA) the walk keeps.
if dataset.format["type"] not in (None, "python"):
raise TypeError("formatted dataset")
values = dataset.with_format("arrow")[column]
if values.null_count == 0:
return range(len(values))
import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
valid = pc.is_valid(values)
if isinstance(valid, pa.ChunkedArray):
valid = valid.combine_chunks()
return np.flatnonzero(valid.to_numpy(zero_copy_only = False)).tolist()
except Exception:
return [i for i, value in enumerate(dataset[column]) if value is not None]
def convert_to_vlm_format(
dataset,
instruction = None,
text_column = "text",
image_column = "image",
dataset_name = None,
progress_callback = None,
):
"""Returns a list, not a Dataset, so PIL Images are preserved; URL datasets are probed first."""
from PIL import Image
from utils.account_context import current_account
from .vlm_processing import generate_smart_vlm_instruction
account = current_account()
def _notify(msg):
"""Send a status update to the training overlay if callback set."""
if progress_callback:
progress_callback(status_message = msg)
if instruction is None:
instruction_info = generate_smart_vlm_instruction(
dataset,
text_column = text_column,
image_column = image_column,
dataset_name = dataset_name,
)
instruction = instruction_info["instruction"]
instruction_column = instruction_info.get("instruction_column")
uses_dynamic = instruction_info["uses_dynamic_instruction"]
logger.info(f"📝 Auto-detected instruction type: {instruction_info['instruction_type']}")
logger.info(f"📝 Confidence: {instruction_info['confidence']:.2f}")
if not uses_dynamic:
logger.info(f"📝 Using instruction: '{instruction}'")
else:
logger.info(f"📝 Using dynamic instructions from column: '{instruction_column}'")
else:
instruction_column = None
uses_dynamic = False
def _fetch_repo_image(name):
from huggingface_hub import hf_hub_download
from utils.hf_cache_settings import active_hf_hub_cache
local_path = hf_hub_download(
dataset_name,
_image_lookup[name],
repo_type = "dataset",
cache_dir = active_hf_hub_cache(),
)
return Image.open(local_path).convert("RGB")
def _convert_single_sample(sample, fetched = None):
"""Image may be a PIL Image, local path, URL, or bare filename; ``fetched`` = the URL's or repo file's already-downloaded image."""
image_data = sample[image_column]
if isinstance(image_data, str):
if image_data.startswith(("http://", "https://")):
image_data = fetched if fetched is not None else _fetch_row_image(image_data)
elif _image_lookup is not None and image_data in _image_lookup:
# Bare filename → resolve via HF repo lookup
image_data = fetched if fetched is not None else _fetch_repo_image(image_data)
else:
image_data = Image.open(_own_image_path(image_data, account)).convert("RGB")
elif image_data is None:
raise ValueError("Row has no image")
text_data = sample[text_column]
if isinstance(text_data, list) and len(text_data) > 0:
import random
text_data = random.choice(text_data)
if uses_dynamic and instruction_column:
current_instruction = sample[instruction_column]
if not isinstance(current_instruction, str) or not current_instruction.strip():
current_instruction = "Describe this image in detail."
else:
current_instruction = instruction
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": current_instruction},
{"type": "image", "image": image_data},
],
},
{"role": "assistant", "content": [{"type": "text", "text": text_data}]},
]
return {"messages": messages}
total = len(dataset)
first_image = next(
(row[image_column] for row in dataset if row[image_column] is not None), None
)
has_urls = isinstance(first_image, str) and first_image.startswith(("http://", "https://"))
if has_urls:
with_image = _non_null_rows(dataset, image_column)
if len(with_image) < total:
logger.info(f"Skipping {total - len(with_image)}/{total} rows without an image")
dataset = dataset.select(with_image)
total = len(with_image)
# A null caption would render as the word "None" in the assistant turn, so leave those rows out like rows without an image.
with_text = _non_null_rows(dataset, text_column)
if len(with_text) < total:
logger.info(f"Skipping {total - len(with_text)}/{total} rows without text")
dataset = dataset.select(with_text)
total = len(with_text)
_image_lookup = None
_IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff")
if (
not has_urls
and isinstance(first_image, str)
and not is_network_share(first_image)
and not os.path.exists(first_image)
and dataset_name
):
try:
from huggingface_hub import HfApi
_notify("Resolving image filenames from HF repo...")
logger.info(
f"🔍 Image column contains bare filenames (e.g. '{first_image}') — building repo lookup..."
)
repo_files = HfApi().list_repo_files(dataset_name, repo_type = "dataset")
_image_lookup = {
os.path.basename(f): f
for f in repo_files
if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS)
}
# Also key by the full relative path, e.g. "images/0001.jpg", as the ShareGPT converter does.
for f in repo_files:
if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS):
_image_lookup[f] = f
if first_image in _image_lookup:
logger.info(
f"✅ Matched {len(_image_lookup)} image files in repo (e.g. '{first_image}' → '{_image_lookup[first_image]}')"
)
else:
logger.info(
f"⚠️ Built lookup with {len(_image_lookup)} images but '{first_image}' not found — falling back to local open"
)
_image_lookup = None
except Exception as e:
logger.info(f"⚠️ Failed to build HF repo image lookup: {e}")
_image_lookup = None
# URL probe: 200 parallel samples to estimate speed and failure rate.
PROBE_SIZE = 200
MAX_FAIL_RATE = 0.3
# Successful probe downloads by row, reused by the conversion.
probe_images = {}
if has_urls and total > PROBE_SIZE:
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from utils.hardware import safe_thread_num_proc
num_workers = safe_thread_num_proc()
_notify(f"Probing {PROBE_SIZE} image URLs with {num_workers} workers...")
logger.info(f"🔍 Probing {PROBE_SIZE}/{total} image URLs with {num_workers} workers...")
probe_samples = [dataset[i] for i in range(PROBE_SIZE)]
probe_ok = 0
probe_fail = 0
probe_start = time.time()
def _probe_sample(index, sample):
url = sample[image_column]
if isinstance(url, str) and url.startswith(("http://", "https://")):
# A failed download is not kept, so the conversion retries it.
probe_images[index] = _fetch_row_image(url)
return _convert_single_sample(sample, probe_images.get(index))
with ThreadPoolExecutor(max_workers = num_workers) as executor:
futures = {executor.submit(_probe_sample, i, s): s for i, s in enumerate(probe_samples)}
for future in as_completed(futures):
try:
future.result()
probe_ok += 1
except Exception:
probe_fail += 1
probe_elapsed = time.time() - probe_start
probe_total = probe_ok + probe_fail
fail_rate = probe_fail / probe_total if probe_total > 0 else 0
throughput = probe_total / probe_elapsed if probe_elapsed > 0 else 0
if fail_rate >= MAX_FAIL_RATE:
issues = [
f"{fail_rate:.0%} of the first {PROBE_SIZE} image URLs failed to download ({probe_fail}/{probe_total})",
"Images are external URLs, not embedded in the dataset",
]
friendly = None
try:
from .llm_assist import llm_generate_dataset_warning
friendly = llm_generate_dataset_warning(
issues,
dataset_name = dataset_name,
modality = "vision",
column_names = [image_column, text_column],
)
except Exception:
pass
msg = friendly or (
f"⚠️ {fail_rate:.0%} of the first {PROBE_SIZE} images failed to download "
f"({probe_fail}/{probe_total}). "
"This dataset has too many broken or unreachable image URLs. "
"Consider using a dataset with embedded images instead."
)
logger.info(msg)
_notify(msg)
raise ValueError(msg)
remaining = total - PROBE_SIZE
estimated_seconds = remaining / throughput if throughput > 0 else 0
eta_str = _format_eta(estimated_seconds)
info_msg = (
f"Downloading {total:,} images ({num_workers} workers, ~{throughput:.1f} img/s). "
f"Estimated time: ~{eta_str}"
)
if probe_fail > 0:
info_msg += f" | {fail_rate:.0%} broken URLs will be skipped"
logger.info(
f"✅ Probe passed: {probe_ok}/{probe_total} ok, {probe_fail} failed ({fail_rate:.0%}), {throughput:.1f} img/s"
)
logger.info(f"⏱️ Estimated time for {total:,} samples: ~{eta_str}")
_notify(info_msg)
from tqdm import tqdm
logger.info(f"🔄 Converting {total} samples to VLM format...")
converted_list = []
failed_count = 0
if has_urls:
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from utils.hardware import safe_thread_num_proc
num_workers = safe_thread_num_proc()
batch_size = 500
start_time = time.time()
for batch_start in range(0, total, batch_size):
batch_end = min(batch_start + batch_size, total)
batch_samples = [dataset[i] for i in range(batch_start, batch_end)]
with ThreadPoolExecutor(max_workers = num_workers) as executor:
futures = {
executor.submit(
_convert_single_sample, s, probe_images.pop(batch_start + i, None)
): i
for i, s in enumerate(batch_samples)
}
batch_results = [None] * len(batch_samples)
for future in as_completed(futures):
idx = futures[future]
try:
batch_results[idx] = future.result()
except Exception as e:
failed_count += 1
if failed_count == 1:
logger.info(f"First VLM conversion failure: {type(e).__name__}: {e}")
converted_list.extend(r for r in batch_results if r is not None)
elapsed = time.time() - start_time
done = batch_end
rate = done / elapsed if elapsed > 0 else 0
remaining_time = (total - done) / rate if rate > 0 else 0
eta_str = _format_eta(remaining_time)
progress_msg = f"Downloading images: {done:,}/{total:,} ({done*100//total}%) | ~{eta_str} remaining | {failed_count} skipped"
logger.info(
f" [{done}/{total}] {rate:.1f} img/s, {failed_count} failed, ETA {eta_str}"
)
_notify(progress_msg)
else:
pbar = tqdm(
dataset,
total = total,
desc = "Converting VLM samples",
unit = "sample",
**_quiet_bar_kwargs(),
)
def _record(sample, future = None):
nonlocal failed_count
try:
fetched = future.result() if future is not None else None
converted_list.append(_convert_single_sample(sample, fetched))
except Exception as e:
failed_count += 1
if failed_count == 1:
logger.info(f"First VLM conversion failure: {type(e).__name__}: {e}")
pbar.set_postfix(ok = len(converted_list), failed = failed_count, refresh = False)
from utils.hardware import safe_thread_num_proc
num_workers = (
min(safe_thread_num_proc(), _MAX_ROW_FETCH_WORKERS) if _image_lookup is not None else 1
)
def _repo_fetch(sample):
name = sample[image_column]
if isinstance(name, str) and name in _image_lookup:
return functools.partial(_fetch_repo_image, name)
return None
try:
_settle_in_order(pbar, _repo_fetch, _record, num_workers)
finally:
pbar.close()
if failed_count > 0:
fail_rate = failed_count / total
logger.info(
f"⚠️ Skipped {failed_count}/{total} ({fail_rate:.0%}) samples with broken/unreachable images"
)
# Small URL datasets skip the probe, so check the fail rate here.
if has_urls and fail_rate >= MAX_FAIL_RATE:
issues = [
f"{fail_rate:.0%} of images failed to download ({failed_count}/{total})",
"Images are external URLs, not embedded in the dataset",
]
friendly = None
try:
from .llm_assist import llm_generate_dataset_warning
friendly = llm_generate_dataset_warning(
issues,
dataset_name = dataset_name,
modality = "vision",
column_names = [image_column, text_column],
)
except Exception:
pass
msg = friendly or (
f"⚠️ {fail_rate:.0%} of images failed to download ({failed_count}/{total}). "
"This dataset has too many broken or unreachable image URLs. "
"Consider using a dataset with embedded images instead."
)
_notify(msg)
raise ValueError(msg)
if len(converted_list) == 0:
issues = [
f"All {total} samples failed during VLM conversion — no usable images found",
f"Image column '{image_column}' may contain URLs that are no longer accessible, "
"or local file paths that don't exist",
]
friendly = None
try:
from .llm_assist import llm_generate_dataset_warning
friendly = llm_generate_dataset_warning(
issues,
dataset_name = dataset_name,
modality = "vision",
column_names = [image_column, text_column],
)
except Exception:
pass
raise ValueError(
friendly
or (
f"All {total} samples failed during VLM conversion — no usable images found. "
"This dataset may contain only image URLs that are no longer accessible."
)
)
logger.info(f"✅ Converted {len(converted_list)}/{total} samples")
_notify(f"Converted {len(converted_list):,}/{total:,} images successfully")
return converted_list
# A real video container never starts like this. A playlist / manifest saved as clip.mp4 would make
# FFmpeg open the URLs inside it itself, outside the guarded remote-media fetch.
_PLAYLIST_HEADS = (b"#EXTM3U", b"<", b"ffconcat", b"\xef\xbb\xbf", b" ", b"\t", b"\r", b"\n")
def _refuse_hub_playlist(local_path, name):
try:
with open(local_path, "rb") as handle:
head = handle.read(64)
except OSError:
return local_path # unreadable: the decoder reports it
if head.startswith(_PLAYLIST_HEADS):
raise ValueError(f"Video '{name}' is a playlist or text document, not a video file.")
return local_path
class _HubClips:
"""Repo-relative clip names of a Hub dataset, resolved by download the way bare image filenames are."""
def __init__(
self,
dataset_name,
revision = None,
):
self.repo, self.revision, self.files, self.listed = dataset_name, revision, None, False
if (
not isinstance(dataset_name, str)
or dataset_name.count("/") != 1
or os.path.exists(dataset_name)
):
self.repo = None
def resolve(self, path):
if self.repo is None or os.path.exists(path) or "://" in path or os.path.isabs(path):
return None
if self.files is None:
from huggingface_hub import HfApi
from .format_detection import _VIDEO_EXTENSIONS
self.files, ambiguous = {}, {}
try:
repo_files = HfApi().list_repo_files(
self.repo, repo_type = "dataset", revision = self.revision
)
self.listed = True
except Exception as e: # noqa: BLE001 -- not a Hub repo or offline: paths stay local
logger.info(f"Could not list {self.repo} to resolve video filenames: {e}")
repo_files = []
for f in repo_files:
if f.lower().endswith(_VIDEO_EXTENSIONS + (".webm",)):
ambiguous.setdefault(os.path.basename(f), []).append(f)
for f in repo_files:
if f.lower().endswith(_VIDEO_EXTENSIONS + (".webm",)):
name = os.path.basename(f)
# A name in several folders resolves to its list, refused below: one copy is a guess.
self.files.setdefault(name, f if len(ambiguous[name]) == 1 else ambiguous[name])
self.files[f] = f
from huggingface_hub import hf_hub_download
from utils.hf_cache_settings import active_hf_hub_cache
if path not in self.files:
if self.listed:
return None
# Offline: an exact repo path may still be in the cache.
try:
cached = hf_hub_download(
self.repo,
path,
repo_type = "dataset",
revision = self.revision,
cache_dir = active_hf_hub_cache(),
local_files_only = True,
)
except Exception: # noqa: BLE001 -- not cached: the path stays local
return None
return _refuse_hub_playlist(cached, path)
if isinstance(self.files[path], list):
raise ValueError(
f"Video '{path}' matches several files in {self.repo} ({', '.join(self.files[path])}); "
"use the repo-relative path."
)
return _refuse_hub_playlist(
hf_hub_download(
self.repo,
self.files[path],
repo_type = "dataset",
revision = self.revision,
cache_dir = active_hf_hub_cache(),
),
path,
)
def _own_video_part(
video,
account,
hub = None,
):
from .format_detection import encoded_video
video = encoded_video(video)
embedded = isinstance(video, dict) and bool(video.get("bytes"))
if isinstance(video, dict):
video = video.get("path")
# Before any stat: probing a UNC path can already contact the share.
if isinstance(video, str) and is_network_share(video) and not private_sources_allowed():
raise OSError(
f"Refusing a video path on a network share: {video}. "
"Set UNSLOTH_ALLOW_PRIVATE_URL_FETCH=1 to load videos from shares you trust."
)
# An embedded clip's path is often just its original filename, with no file behind it.
if embedded and not (isinstance(video, str) and os.path.isfile(video)):
resolved = (
hub.resolve(video)
if hub is not None and isinstance(video, str) and video.strip()
else None
)
if resolved is not None:
return resolved
raise ValueError(
"A video is stored as embedded bytes with no readable file path; training reads clips from "
"files, so save the clips to files and point the column at their paths."
)
if not isinstance(video, str) or not video.strip():
raise ValueError("Row has no video")
if video.strip().lower().startswith(("http://", "https://")):
# The collator would fetch it unguarded; the image fetchers' 64 MB cap does not fit clips.
raise ValueError(
f"Remote video URLs are not fetched for training: {video}. "
"Download the clips and point the column at local files or a Hub dataset."
)
resolved = hub.resolve(video) if hub is not None else None
if resolved is not None:
return resolved
return _own_image_path(video, account)
def own_message_videos(
rows,
dataset_name = None,
dataset_revision = None,
):
"""VLM messages rows as a list, every video path checked like an image path."""
from utils.account_context import current_account
from .format_detection import video_parts
account = current_account()
hub = _HubClips(dataset_name, dataset_revision)
rows = [row for row in rows]
for row in rows:
if not isinstance(row.get("messages"), list):
continue
for message in row["messages"]:
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
# Arrow fills absent keys with None; templates and the collator read key presence.
content[:] = [
{k: v for k, v in part.items() if v is not None} if isinstance(part, dict) else part
for part in content
]
for part in video_parts(row["messages"]):
part["video"] = _own_video_part(part["video"], account, hub)
return rows
def convert_video_to_vlm_format(
dataset,
video_column,
text_column,
question_column = None,
instruction = None,
video_columns = None,
dataset_name = None,
dataset_revision = None,
):
"""Flat video + text rows to VLM messages; clips stay paths for the collator to decode."""
from utils.account_context import current_account
from .format_detection import _holds_images, _is_image_value
account = current_account()
hub = _HubClips(dataset_name, dataset_revision)
answer_feature = (getattr(dataset, "features", None) or {}).get(text_column)
as_label = getattr(answer_feature, "int2str", None)
default_instruction = instruction or "Describe this video."
converted = []
for row in dataset:
answer = row.get(text_column)
clip_columns = video_columns or [video_column]
# Detection probes a few rows; an image further down would otherwise be dropped silently.
images = [
c
for c, v in row.items()
if c not in clip_columns and (_is_image_value(v) or _holds_images(v))
]
if images:
raise ValueError(
f"Rows mix video with image columns {images}; video conversion would drop the images, "
"so remove those columns or split the dataset."
)
clips = [
row[c]
for c in clip_columns
if row.get(c) is not None and not (isinstance(row[c], str) and not row[c].strip())
]
if not clips or answer is None or not str(answer).strip():
continue
answer = as_label(answer) if as_label and isinstance(answer, int) else str(answer)
question = row.get(question_column) if question_column else None
if not isinstance(question, str) or not question.strip():
question = default_instruction
converted.append(
{
"messages": [
{
"role": "user",
"content": [
*(
{"type": "video", "video": _own_video_part(clip, account, hub)}
for clip in clips
),
{"type": "text", "text": question},
],
},
{"role": "assistant", "content": [{"type": "text", "text": answer}]},
]
}
)
if not converted:
raise ValueError(
f"No rows have both a video in '{video_column}' and text in '{text_column}'"
)
skipped = len(dataset) - len(converted) if hasattr(dataset, "__len__") else 0
if skipped:
logger.info(f"Skipping {skipped} rows without a video or text")
return converted
def convert_sharegpt_with_images_to_vlm_format(
dataset,
image_column = "image",
messages_column = "conversations",
dataset_name = None,
progress_callback = None,
):
"""Convert ShareGPT/ChatML datasets carrying a separate image column and ``<image>`` placeholders in the conversation text, e.g. {"image": "sam/images/sa_545504.jpg", "conversations": [{"from": "human", "value": "<image> What is this photo about?"}, ...]}. Image lists match placeholders in conversation order. Returns a list of dicts in standard VLM messages format, PIL Images inline."""
from PIL import Image
from tqdm import tqdm
from utils.account_context import current_account
account = current_account()
_IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff")
_ROLE_MAP = {
"human": "user",
"user": "user",
"gpt": "assistant",
"assistant": "assistant",
"system": "system",
}
def _notify(msg):
if progress_callback:
progress_callback(status_message = msg)
# Resolve image loading, the same 3 tiers as convert_to_vlm_format.
total = len(dataset)
first_image = None
for sample in dataset:
image_data = sample.get(image_column)
if isinstance(image_data, list):
image_data = next((image for image in image_data if image is not None), None)
if image_data is not None:
first_image = image_data
break
_image_lookup = None
if (
isinstance(first_image, str)
and not first_image.startswith(("http://", "https://"))
and not is_network_share(first_image)
and not os.path.exists(first_image)
and dataset_name
):
try:
from huggingface_hub import HfApi
_notify("Resolving image filenames from HF repo...")
logger.info(
f"🔍 Image column contains bare filenames (e.g. '{first_image}') — building repo lookup..."
)
repo_files = HfApi().list_repo_files(dataset_name, repo_type = "dataset")
_image_lookup = {
os.path.basename(f): f
for f in repo_files
if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS)
}
# Also key by the full relative path, e.g. "sam/images/sa_545504.jpg".
for f in repo_files:
if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS):
_image_lookup[f] = f
if first_image in _image_lookup:
logger.info(
f"✅ Matched {len(_image_lookup)} image files in repo (e.g. '{first_image}' → '{_image_lookup[first_image]}')"
)
else:
logger.info(
f"⚠️ Built lookup with {len(_image_lookup)} images but '{first_image}' not found; unmatched filenames use local paths"
)
except Exception as e:
logger.info(f"⚠️ Failed to build HF repo image lookup: {e}")
_image_lookup = None
def _resolve_image(image_data):
"""Resolve image data to a PIL Image."""
if hasattr(image_data, "size") and hasattr(image_data, "mode"):
return image_data
if isinstance(image_data, str):
if image_data.startswith(("http://", "https://")):
return _fetch_row_image(image_data)
elif _image_lookup is not None and image_data in _image_lookup:
from huggingface_hub import hf_hub_download
from utils.hf_cache_settings import active_hf_hub_cache
local_path = hf_hub_download(
dataset_name,
_image_lookup[image_data],
repo_type = "dataset",
cache_dir = active_hf_hub_cache(),
)
return Image.open(local_path).convert("RGB")
else:
return Image.open(_own_image_path(image_data, account)).convert("RGB")
if isinstance(image_data, dict) and ("bytes" in image_data or "path" in image_data):
if image_data.get("bytes"):
from io import BytesIO
return Image.open(BytesIO(image_data["bytes"])).convert("RGB")
if image_data.get("path"):
return Image.open(_own_image_path(image_data["path"], account)).convert("RGB")
raise ValueError(f"Cannot resolve image: {type(image_data)}")
def _convert_single_sample(sample):
"""Convert one ShareGPT+image sample to standard VLM format."""
image_data = sample[image_column]
conversation = sample[messages_column]
is_image_list = isinstance(image_data, list)
if is_image_list:
marker_count = sum(
(msg.get("value") or msg.get("content") or "").count("<image>")
for msg in conversation
)
if not image_data or len(image_data) != marker_count:
raise ValueError(
f"Image list has {len(image_data)} images but conversation has "
f"{marker_count} <image> placeholders"
)
pil_images = [_resolve_image(image) for image in image_data]
else:
pil_images = [_resolve_image(image_data)]
new_messages = []
image_index = 0
for msg in conversation:
role_raw = msg.get("from") or msg.get("role", "user")
role = _ROLE_MAP.get(role_raw.lower(), role_raw.lower())
text = msg.get("value") or msg.get("content") or ""
if "<image>" in text:
parts = text.split("<image>")
content = []
for i, part in enumerate(parts):
part = part.strip()
if part:
content.append({"type": "text", "text": part})
if i < len(parts) - 1:
content.append(
{
"type": "image",
"image": pil_images[image_index if is_image_list else 0],
}
)
image_index += 1
if not content:
content.append({"type": "image", "image": pil_images[0]})
else:
content = [{"type": "text", "text": text}]
new_messages.append({"role": role, "content": content})
return {"messages": new_messages}
logger.info(f"🔄 Converting {total} samples from ShareGPT+image format...")
converted_list = []
failed_count = 0
def _record(sample, future):
nonlocal failed_count
try:
converted_list.append(
future.result() if future is not None else _convert_single_sample(sample)
)
except Exception as e:
failed_count += 1
if failed_count == 1:
logger.info(f"⚠️ First conversion failure: {type(e).__name__}: {e}")
pbar.set_postfix(ok = len(converted_list), failed = failed_count, refresh = False)
num_workers = 1
if _image_lookup is not None or (
isinstance(first_image, str) and first_image.startswith(("http://", "https://"))
):
from utils.hardware import safe_thread_num_proc
num_workers = min(safe_thread_num_proc(), _MAX_ROW_FETCH_WORKERS)
pbar = tqdm(
dataset,
total = total,
desc = "Converting ShareGPT+image",
unit = "sample",
**_quiet_bar_kwargs(),
)
try:
_settle_in_order(
pbar,
lambda sample: functools.partial(_convert_single_sample, sample),
_record,
num_workers,
)
finally:
pbar.close()
if failed_count > 0:
logger.info(f"⚠️ Skipped {failed_count}/{total} ({failed_count*100//total}%) samples")
if len(converted_list) == 0:
raise ValueError(
f"All {total} samples failed during ShareGPT+image conversion — "
"no usable samples found."
)
logger.info(f"✅ Converted {len(converted_list)}/{total} samples")
_notify(f"Converted {len(converted_list):,}/{total:,} samples successfully")
return converted_list
def convert_llava_to_vlm_format(dataset):
"""Convert Llava format to standard VLM format: Llava carries messages whose content blocks name an image by {'type': 'image', 'index': 0} plus a parallel images list, while the standard form inlines the PIL object as {'type': 'image', 'image': PIL_Image}."""
from PIL import Image
from utils.account_context import current_account
account = current_account()
is_iterable = is_streaming_dataset(dataset)
if is_iterable:
logger.info("🔄 Converting streaming samples from Llava format to standard VLM format...")
else:
logger.info(
f"🔄 Converting {len(dataset)} samples from Llava format to standard VLM format..."
)
def _convert_single_sample(sample):
"""Convert one llava sample to standard VLM format."""
messages = sample["messages"]
images = sample.get("images", [])
unindexed_image_index = 0
explicit_image_indices = set()
for message in messages:
content = message.get("content")
if not isinstance(content, list):
continue
for item in content:
if not isinstance(item, dict) or item.get("type") != "image":
continue
image_index = item.get("index")
if isinstance(image_index, int) and not isinstance(image_index, bool):
explicit_image_indices.add(image_index)
new_messages = []
for msg in messages:
new_content = []
message_content = msg["content"]
if isinstance(message_content, str):
new_messages.append(
{
"role": msg["role"],
"content": [{"type": "text", "text": message_content}],
}
)
continue
for item in message_content:
if item["type"] == "image":
img_idx = item.get("index")
if img_idx is None:
while unindexed_image_index in explicit_image_indices:
unindexed_image_index += 1
img_idx = unindexed_image_index
unindexed_image_index += 1
if not isinstance(img_idx, int) or isinstance(img_idx, bool):
raise ValueError(f"Invalid Llava image index: {img_idx!r}")
if not 0 <= img_idx < len(images):
raise ValueError(
f"Llava image index {img_idx} is missing from a row with "
f"{len(images)} images"
)
pil_image = images[img_idx]
if isinstance(pil_image, str):
pil_image = Image.open(_own_image_path(pil_image, account)).convert("RGB")
elif not isinstance(pil_image, Image.Image):
raise ValueError(
f"Unsupported Llava image value at index {img_idx}: "
f"{type(pil_image).__name__}"
)
new_content.append({"type": "image", "image": pil_image})
elif item["type"] == "text":
new_content.append({"type": "text", "text": item.get("text", "")})
new_messages.append({"role": msg["role"], "content": new_content})
return {"messages": new_messages}
if is_iterable:
remove_columns = dataset.column_names or list(next(iter(dataset), {}))
converted_dataset = dataset.map(
_convert_single_sample,
remove_columns = remove_columns,
)
try:
next(iter(converted_dataset), None)
except Exception as exc:
raise ValueError(f"Streaming Llava conversion failed on the first row: {exc}") from exc
logger.info("✅ Configured streaming Llava conversion")
return converted_dataset
converted_list = [_convert_single_sample(sample) for sample in dataset]
logger.info(f"✅ Converted {len(converted_list)} samples")
return converted_list