Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: train vision models on images in a list column or inside the messages (#12484)
Nilayaff8dfbba023
diff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.pyindex 2934c79cf00..33523a454bb 100644--- a/studio/backend/hub/utils/dataset_format.py+++ b/studio/backend/hub/utils/dataset_format.py@@ -380,15 +380,21 @@ def _has_image_header(data: bytes) -> bool:)-def _is_image_value(value) -> bool:- if value is None:- return False+def _is_decoded_image_value(value) -> bool:try:from PIL.Image import Image as PILImageif isinstance(value, PILImage):return Trueexcept ImportError:pass+ return False+++def _is_image_value(value) -> bool:+ if value is None:+ return False+ if _is_decoded_image_value(value):+ return Trueif isinstance(value, dict):if "array" in value and "sampling_rate" in value:return False@@ -410,6 +416,31 @@ def _is_image_value(value) -> bool:return False+def _is_image_list_item(value) -> bool:+ if isinstance(value, (dict, bytes, bytearray)):+ return False+ if isinstance(value, str):+ normalized = value.strip().lower()+ if normalized.startswith(("http://", "https://")) or normalized.endswith(".svg"):+ return False+ return _is_image_value(value)+++def _holds_images(value) -> bool:+ if not isinstance(value, list) or not value:+ return False+ if all(_is_image_list_item(item) for item in value):+ return True+ for item in value:+ content = item.get("content") if isinstance(item, dict) else None+ if isinstance(content, list) and any(+ isinstance(part, dict) and part.get("type") == "image" and part.get("image") is not None+ for part in content+ ):+ return True+ return False++def detect_multimodal_dataset(dataset):sample = _first_row(dataset)if sample is None:@@ -452,7 +483,8 @@ def detect_multimodal_dataset(dataset):multimodal_columns.append(col_name)modality_types.add("image")for col_name in column_names:- if col_name not in multimodal_columns and _is_image_value(sample[col_name]):+ value = sample[col_name]+ if col_name not in multimodal_columns and (_is_image_value(value) or _holds_images(value)):multimodal_columns.append(col_name)modality_types.add("image")for col_name in column_names:@@ -513,34 +545,43 @@ def detect_vlm_dataset_structure(dataset):is_text = text_cell_check(dataset)if "messages" in column_names:messages = sample["messages"]- if messages and len(messages) > 0:- first_msg = messages[0]- if "content" in first_msg:- content = first_msg["content"]+ message_rows = messages if isinstance(messages, list) else []+ image_parts = [+ part+ for message in message_rows+ if isinstance(message, dict) and isinstance(message.get("content"), list)+ for part in message["content"]+ if isinstance(part, dict) and part.get("type") == "image"+ ]+ if image_parts:+ embedded_images = [+ part.get("image") for part in image_parts if part.get("image") is not None+ ]+ if embedded_images:+ if len(embedded_images) == len(image_parts) and all(+ _is_decoded_image_value(image) for image in embedded_images+ ):+ return {+ "format": "vlm_messages",+ "needs_conversion": False,+ "messages_column": "messages",+ "image_column": None,+ "text_column": None,+ }+ else:+ images = sample.get("images")if (- isinstance(content, list)- and content- and isinstance(content[0], dict)- and "type" in content[0]+ isinstance(images, list)+ and images+ and all(_is_image_list_item(image) for image in images)):- has_index = any("index" in item for item in content if isinstance(item, dict))- if has_index and "images" in column_names:- return {- "format": "vlm_messages_llava",- "needs_conversion": True,- "messages_column": "messages",- "image_column": "images",- "text_column": None,- }- has_image = any("image" in item for item in content if isinstance(item, dict))- if has_image:- return {- "format": "vlm_messages",- "needs_conversion": False,- "messages_column": "messages",- "image_column": None,- "text_column": None,- }+ return {+ "format": "vlm_messages_llava",+ "needs_conversion": True,+ "messages_column": "messages",+ "image_column": "images",+ "text_column": None,+ }for chat_col in ("conversations", "messages"):if chat_col not in column_names:diff --git a/studio/backend/tests/test_dataset_image_lists.py b/studio/backend/tests/test_dataset_image_lists.pynew file mode 100644index 00000000000..26ea9351ab0--- /dev/null+++ b/studio/backend/tests/test_dataset_image_lists.py@@ -0,0 +1,322 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import io+import sys+from pathlib import Path++import pytest+from datasets import Dataset, IterableDataset+from PIL import Image++_BACKEND_ROOT = Path(__file__).resolve().parents[1]+if str(_BACKEND_ROOT) not in sys.path:+ sys.path.insert(0, str(_BACKEND_ROOT))++from hub.utils import dataset_format # noqa: E402+from utils.datasets.format_conversion import convert_llava_to_vlm_format # noqa: E402+from utils.datasets import format_detection # noqa: E402+++def _llava_row():+ return {+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "image", "index": 0, "text": None},+ {"type": "text", "index": None, "text": "What is in the picture?"},+ ],+ },+ {+ "role": "assistant",+ "content": [{"type": "text", "index": None, "text": "A red square."}],+ },+ ],+ "images": [Image.new("RGB", (8, 8), "red")],+ }+++def _llava_row_without_index():+ row = _llava_row()+ for message in row["messages"]:+ for part in message["content"]:+ part.pop("index", None)+ return row+++def _llava_rows():+ yield _llava_row_without_index()+++def _image_part_row():+ return {+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "image", "image": Image.new("RGB", (8, 8), "red"), "text": None},+ {"type": "text", "image": None, "text": "Describe this."},+ ],+ },+ {+ "role": "assistant",+ "content": [{"type": "text", "image": None, "text": "A dog."}],+ },+ ]+ }+++def _image_part_row_after_system_message():+ row = _image_part_row()+ row["messages"].insert(+ 0,+ {+ "role": "system",+ "content": [{"type": "text", "image": None, "text": "Answer accurately."}],+ },+ )+ return row+++def _text_parts_row():+ return {+ "messages": [+ {"role": "user", "content": [{"type": "text", "text": "Hi"}]},+ {"role": "assistant", "content": [{"type": "text", "text": "Hello"}]},+ ]+ }+++def _encoded_image_row():+ encoded = io.BytesIO()+ Image.new("RGB", (8, 8), "red").save(encoded, format = "PNG")+ return {"media": [{"bytes": encoded.getvalue(), "path": None}]}+++def _encoded_audio_row():+ return {"clips": [{"bytes": b"RIFF\x00\x00\x00\x00WAVEfmt ", "path": None}]}+++ "row, expected_vlm_format",+ [+ pytest.param(_llava_row(), "vlm_messages_llava", id = "images-list"),+ pytest.param(+ _llava_row_without_index(),+ "vlm_messages_llava",+ id = "images-list-without-index",+ ),+ pytest.param(_image_part_row(), "vlm_messages", id = "image-parts"),+ pytest.param(+ _image_part_row_after_system_message(),+ "vlm_messages",+ id = "image-parts-after-system-message",+ ),+ ],+)+def test_images_in_a_list_or_in_messages_are_detected(row, expected_vlm_format):+ dataset = [row] if expected_vlm_format == "vlm_messages" else Dataset.from_list([row])++ assert dataset_format.check_dataset_format(dataset, is_vlm = False)["is_image"] is True+ vlm = dataset_format.check_dataset_format(dataset, is_vlm = True)+ assert vlm["is_image"] is True+ assert vlm["detected_format"] == expected_vlm_format+ assert vlm["requires_manual_mapping"] is False+ assert format_detection.detect_multimodal_dataset(dataset)["is_image"] is True+++def test_unindexed_llava_images_are_consumed_in_order():+ dataset = Dataset.from_list(+ [+ {+ "messages": [+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "image"}],+ }+ ],+ "images": [+ Image.new("RGB", (8, 8), "red"),+ Image.new("RGB", (8, 8), "blue"),+ ],+ }+ ]+ )++ converted = convert_llava_to_vlm_format(dataset)+ images = [part["image"] for part in converted[0]["messages"][0]["content"]]++ assert [image.getpixel((0, 0)) for image in images] == [(255, 0, 0), (0, 0, 255)]+++def test_unindexed_llava_images_skip_explicit_indices():+ dataset = [+ {+ "messages": [+ {+ "role": "user",+ "content": [{"type": "image", "index": 0}, {"type": "image"}],+ }+ ],+ "images": [+ Image.new("RGB", (8, 8), "red"),+ Image.new("RGB", (8, 8), "blue"),+ ],+ }+ ]++ converted = convert_llava_to_vlm_format(dataset)+ images = [part["image"] for part in converted[0]["messages"][0]["content"]]++ assert [image.getpixel((0, 0)) for image in images] == [(255, 0, 0), (0, 0, 255)]+++def test_llava_conversion_rejects_missing_unindexed_image():+ dataset = [+ {+ "messages": [+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "image"}],+ }+ ],+ "images": [Image.new("RGB", (8, 8), "red")],+ }+ ]++ with pytest.raises(ValueError, match = "image index 1"):+ convert_llava_to_vlm_format(dataset)+++def test_llava_conversion_rejects_undecoded_image_dictionary():+ row = _llava_row_without_index()+ row["images"] = [_encoded_image_row()["media"][0]]++ with pytest.raises(ValueError, match = "Unsupported Llava image value"):+ convert_llava_to_vlm_format([row])+++def test_llava_conversion_preserves_string_turns():+ row = _llava_row_without_index()+ row["messages"].insert(0, {"role": "system", "content": "Answer accurately."})+ dataset = [row]++ assert dataset_format.detect_vlm_dataset_structure(dataset)["format"] == "vlm_messages_llava"+ assert format_detection.detect_vlm_dataset_structure(dataset)["format"] == "vlm_messages_llava"++ converted = convert_llava_to_vlm_format(dataset)++ assert converted[0]["messages"][0] == {+ "role": "system",+ "content": [{"type": "text", "text": "Answer accurately."}],+ }+++def test_llava_conversion_stays_lazy_for_streaming_dataset():+ dataset = IterableDataset.from_generator(_llava_rows)++ converted = convert_llava_to_vlm_format(dataset)++ assert isinstance(converted, IterableDataset)+ first = next(iter(converted))+ assert first["messages"][0]["content"][0]["image"].getpixel((0, 0)) == (255, 0, 0)+++ "row",+ [+ pytest.param(_text_parts_row(), id = "text-parts"),+ pytest.param(+ {+ "messages": [+ {"role": "user", "content": "Hi"},+ {"role": "assistant", "content": "Yo"},+ ]+ },+ id = "chatml",+ ),+ pytest.param({"tags": ["news", "sports"], "text": "Final score 2-1."}, id = "string-list"),+ pytest.param(+ {+ "messages": [+ {"role": "user", "content": "Which files changed?"},+ {"role": "assistant", "content": "The entry point and the logo."},+ ],+ "files": ["src/main.py", "docs/logo.png"],+ },+ id = "file-list",+ ),+ ],+)+def test_text_only_datasets_stay_text(row):+ dataset = Dataset.from_list([row])++ assert dataset_format.check_dataset_format(dataset, is_vlm = False)["is_image"] is False+ assert format_detection.detect_multimodal_dataset(dataset)["is_image"] is False++[email protected]("detector", [dataset_format, format_detection])+def test_image_list_detection_requires_values_the_converter_can_open(detector, tmp_path):+ png_path = tmp_path / "image.png"+ Image.new("RGB", (8, 8), "red").save(png_path)+ svg_path = tmp_path / "image.svg"+ svg_path.write_text('<svg xmlns="http://www.w3.org/2000/svg"/>')++ encoded_image_row = _encoded_image_row()+ image_dataset = Dataset.from_list([encoded_image_row])+ image_bytes_dataset = Dataset.from_list([{"media": [encoded_image_row["media"][0]["bytes"]]}])+ audio_dataset = Dataset.from_list([_encoded_audio_row()])+ local_image_dataset = Dataset.from_list([{"media": [str(png_path)]}])+ remote_image_dataset = Dataset.from_list([{"media": ["https://example.com/image.png"]}])+ svg_image_dataset = Dataset.from_list([{"media": [str(svg_path)]}])++ assert detector.detect_multimodal_dataset(image_dataset)["is_image"] is False+ assert detector.detect_multimodal_dataset(image_bytes_dataset)["is_image"] is False+ assert detector.detect_multimodal_dataset(audio_dataset)["is_image"] is False+ assert detector.detect_multimodal_dataset(local_image_dataset)["is_image"] is True+ assert detector.detect_multimodal_dataset(remote_image_dataset)["is_image"] is False+ assert detector.detect_multimodal_dataset(svg_image_dataset)["is_image"] is False++[email protected]("detector", [dataset_format, format_detection])+def test_llava_placeholders_require_convertible_images(detector):+ row = _llava_row_without_index()+ row["images"] = [_encoded_image_row()["media"][0]]+ dataset = Dataset.from_list([row])++ result = detector.detect_vlm_dataset_structure(dataset)++ assert result["format"] != "vlm_messages_llava"++[email protected]("detector", [dataset_format, format_detection])+def test_embedded_images_require_convertible_values(detector):+ row = _image_part_row()+ row["messages"][0]["content"][0]["image"] = _encoded_image_row()["media"][0]+ dataset = Dataset.from_list([row])++ result = detector.detect_vlm_dataset_structure(dataset)++ assert result["format"] != "vlm_messages"++[email protected]("detector", [dataset_format, format_detection])+def test_embedded_image_paths_require_conversion(detector):+ row = _image_part_row()+ row["messages"][0]["content"][0]["image"] = "/data/photos/a.jpg"++ result = detector.detect_vlm_dataset_structure([row])++ assert result["format"] != "vlm_messages"++[email protected]("detector", [dataset_format, format_detection])+def test_embedded_images_do_not_hide_top_level_placeholders(detector):+ row = _image_part_row()+ row["messages"][0]["content"].append({"type": "image", "image": None, "text": None})+ row["images"] = [Image.new("RGB", (8, 8), "blue")]++ result = detector.detect_vlm_dataset_structure([row])++ assert result["format"] not in {"vlm_messages", "vlm_messages_llava"}diff --git a/studio/backend/utils/datasets/format_conversion.py b/studio/backend/utils/datasets/format_conversion.pyindex a315a53805a..e5714187efe 100644--- a/studio/backend/utils/datasets/format_conversion.py+++ b/studio/backend/utils/datasets/format_conversion.py@@ -872,39 +872,71 @@ def convert_llava_to_vlm_format(dataset):"""Convert Llava format to standard VLM format: Llava carries messages whose content blocks name an image by {'type': 'image', 'index': 0} plus a parallel images list, while the standard form inlines the PIL object as {'type': 'image', 'image': PIL_Image}."""from PIL import Image- logger.info(f"🔄 Converting {len(dataset)} samples from Llava format to standard VLM format...")+ is_iterable = is_streaming_dataset(dataset)+ if is_iterable:+ logger.info("🔄 Converting streaming samples from Llava format to standard VLM format...")+ else:+ logger.info(+ f"🔄 Converting {len(dataset)} samples from Llava format to standard VLM format..."+ )def _convert_single_sample(sample):"""Convert one llava sample to standard VLM format."""messages = sample["messages"]images = sample.get("images", [])+ unindexed_image_index = 0+ explicit_image_indices = set()+ for message in messages:+ content = message.get("content")+ if not isinstance(content, list):+ continue+ for item in content:+ if not isinstance(item, dict) or item.get("type") != "image":+ continue+ image_index = item.get("index")+ if isinstance(image_index, int) and not isinstance(image_index, bool):+ explicit_image_indices.add(image_index)new_messages = []for msg in messages:new_content = []+ message_content = msg["content"]++ if isinstance(message_content, str):+ new_messages.append(+ {+ "role": msg["role"],+ "content": [{"type": "text", "text": message_content}],+ }+ )+ continue- for item in msg["content"]:+ for item in message_content:if item["type"] == "image":- if "index" in item and item["index"] is not None:- img_idx = item["index"]- if img_idx < len(images):- pil_image = images[img_idx]- if isinstance(pil_image, str):- pil_image = Image.open(pil_image).convert("RGB")-- new_content.append(- {- "type": "image",- "image": pil_image, # Actual PIL object- }- )- else:- if len(images) > 0:- pil_image = images[0]- if isinstance(pil_image, str):- pil_image = Image.open(pil_image).convert("RGB")-- new_content.append({"type": "image", "image": pil_image})+ img_idx = item.get("index")+ if img_idx is None:+ while unindexed_image_index in explicit_image_indices:+ unindexed_image_index += 1+ img_idx = unindexed_image_index+ unindexed_image_index += 1+ if not isinstance(img_idx, int) or isinstance(img_idx, bool):+ raise ValueError(f"Invalid Llava image index: {img_idx!r}")+ if not 0 <= img_idx < len(images):+ raise ValueError(+ f"Llava image index {img_idx} is missing from a row with "+ f"{len(images)} images"+ )++ pil_image = images[img_idx]+ if isinstance(pil_image, str):+ pil_image = Image.open(pil_image).convert("RGB")+ elif not isinstance(pil_image, Image.Image):+ raise ValueError(+ f"Unsupported Llava image value at index {img_idx}: "+ f"{type(pil_image).__name__}"+ )++ new_content.append({"type": "image", "image": pil_image})elif item["type"] == "text":new_content.append({"type": "text", "text": item.get("text", "")})@@ -913,7 +945,19 @@ def convert_llava_to_vlm_format(dataset):return {"messages": new_messages}- converted_list = [_convert_single_sample(sample) for sample in dataset]+ if is_iterable:+ remove_columns = dataset.column_names or list(next(iter(dataset), {}))+ converted_dataset = dataset.map(+ _convert_single_sample,+ remove_columns = remove_columns,+ )+ try:+ next(iter(converted_dataset), None)+ except Exception as exc:+ raise ValueError(f"Streaming Llava conversion failed on the first row: {exc}") from exc+ logger.info("✅ Configured streaming Llava conversion")+ return converted_dataset+ converted_list = [_convert_single_sample(sample) for sample in dataset]logger.info(f"✅ Converted {len(converted_list)} samples")return converted_listdiff --git a/studio/backend/utils/datasets/format_detection.py b/studio/backend/utils/datasets/format_detection.pyindex 12d2a884ced..bddaa035656 100644--- a/studio/backend/utils/datasets/format_detection.py+++ b/studio/backend/utils/datasets/format_detection.py@@ -572,7 +572,7 @@ def detect_multimodal_dataset(dataset):if col_name in already_detected:continuevalue = sample[col_name]- if _is_image_value(value):+ if _is_image_value(value) or _holds_images(value):multimodal_columns.append(col_name)modality_types.add("image")@@ -630,17 +630,22 @@ def detect_multimodal_dataset(dataset):}-def _is_image_value(value) -> bool:- """Check if a single sample value looks like image data."""- if value is None:- return False-+def _is_decoded_image_value(value) -> bool:try:from PIL.Image import Image as PILImageif isinstance(value, PILImage):return Trueexcept ImportError:pass+ return False+++def _is_image_value(value) -> bool:+ """Check if a single sample value looks like image data."""+ if value is None:+ return False+ if _is_decoded_image_value(value):+ return True# HF Image feature: decoded as PIL, or {"bytes", "path"} when undecoded. Exclude audio dicts, whose decoded form has "array" + "sampling_rate".if isinstance(value, dict):@@ -671,6 +676,31 @@ def _is_image_value(value) -> bool:return False+def _is_image_list_item(value) -> bool:+ if isinstance(value, (dict, bytes, bytearray)):+ return False+ if isinstance(value, str):+ normalized = value.strip().lower()+ if normalized.startswith(("http://", "https://")) or normalized.endswith(".svg"):+ return False+ return _is_image_value(value)+++def _holds_images(value) -> bool:+ if not isinstance(value, list) or not value:+ return False+ if all(_is_image_list_item(item) for item in value):+ return True+ for item in value:+ content = item.get("content") if isinstance(item, dict) else None+ if isinstance(content, list) and any(+ isinstance(part, dict) and part.get("type") == "image" and part.get("image") is not None+ for part in content+ ):+ return True+ return False++_AUDIO_EXTENSIONS = (".wav",".mp3",@@ -741,39 +771,43 @@ def detect_vlm_dataset_structure(dataset):if "messages" in column_names:messages = sample["messages"]-- if messages and len(messages) > 0:- first_msg = messages[0]- if "content" in first_msg:- content = first_msg["content"]-- if isinstance(content, list) and len(content) > 0:- if isinstance(content[0], dict) and "type" in content[0]:- has_index = any(- "index" in item for item in content if isinstance(item, dict)- )- has_images_column = "images" in column_names-- if has_index and has_images_column:- return {- "format": "vlm_messages_llava",- "needs_conversion": True,- "messages_column": "messages",- "image_column": "images",- "text_column": None,- }-- has_image = any(- "image" in item for item in content if isinstance(item, dict)- )- if has_image:- return {- "format": "vlm_messages",- "needs_conversion": False,- "messages_column": "messages",- "image_column": None,- "text_column": None,- }+ message_rows = messages if isinstance(messages, list) else []+ image_parts = [+ part+ for message in message_rows+ if isinstance(message, dict) and isinstance(message.get("content"), list)+ for part in message["content"]+ if isinstance(part, dict) and part.get("type") == "image"+ ]+ if image_parts:+ embedded_images = [+ part.get("image") for part in image_parts if part.get("image") is not None+ ]+ if embedded_images:+ if len(embedded_images) == len(image_parts) and all(+ _is_decoded_image_value(image) for image in embedded_images+ ):+ return {+ "format": "vlm_messages",+ "needs_conversion": False,+ "messages_column": "messages",+ "image_column": None,+ "text_column": None,+ }+ else:+ images = sample.get("images")+ if (+ isinstance(images, list)+ and images+ and all(_is_image_list_item(image) for image in images)+ ):+ return {+ "format": "vlm_messages_llava",+ "needs_conversion": True,+ "messages_column": "messages",+ "image_column": "images",+ "text_column": None,+ }# ShareGPT/ChatML conversations with an <image> placeholder plus a companion image column, e.g. Lin-Chen/ShareGPT4V and LLaVA-style datasets.for chat_col in ("conversations", "messages"):