Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: train vision models on images in a list column or inside the messages (#12484)

Nilayaff8dfbba023
diff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.py
index 2934c79cf00..33523a454bb 100644
--- a/studio/backend/hub/utils/dataset_format.py
+++ b/studio/backend/hub/utils/dataset_format.py
@@ -380,15 +380,21 @@ def _has_image_header(data: bytes) -> bool:
)
-def _is_image_value(value) -> bool:
- if value is None:
- return False
+def _is_decoded_image_value(value) -> bool:
try:
from PIL.Image import Image as PILImage
if isinstance(value, PILImage):
return True
except ImportError:
pass
+ return False
+
+
+def _is_image_value(value) -> bool:
+ if value is None:
+ return False
+ if _is_decoded_image_value(value):
+ return True
if isinstance(value, dict):
if "array" in value and "sampling_rate" in value:
return False
@@ -410,6 +416,31 @@ def _is_image_value(value) -> bool:
return False
+def _is_image_list_item(value) -> bool:
+ if isinstance(value, (dict, bytes, bytearray)):
+ return False
+ if isinstance(value, str):
+ normalized = value.strip().lower()
+ if normalized.startswith(("http://", "https://")) or normalized.endswith(".svg"):
+ return False
+ return _is_image_value(value)
+
+
+def _holds_images(value) -> bool:
+ if not isinstance(value, list) or not value:
+ return False
+ if all(_is_image_list_item(item) for item in value):
+ return True
+ for item in value:
+ content = item.get("content") if isinstance(item, dict) else None
+ if isinstance(content, list) and any(
+ isinstance(part, dict) and part.get("type") == "image" and part.get("image") is not None
+ for part in content
+ ):
+ return True
+ return False
+
+
def detect_multimodal_dataset(dataset):
sample = _first_row(dataset)
if sample is None:
@@ -452,7 +483,8 @@ def detect_multimodal_dataset(dataset):
multimodal_columns.append(col_name)
modality_types.add("image")
for col_name in column_names:
- if col_name not in multimodal_columns and _is_image_value(sample[col_name]):
+ value = sample[col_name]
+ if col_name not in multimodal_columns and (_is_image_value(value) or _holds_images(value)):
multimodal_columns.append(col_name)
modality_types.add("image")
for col_name in column_names:
@@ -513,34 +545,43 @@ def detect_vlm_dataset_structure(dataset):
is_text = text_cell_check(dataset)
if "messages" in column_names:
messages = sample["messages"]
- if messages and len(messages) > 0:
- first_msg = messages[0]
- if "content" in first_msg:
- content = first_msg["content"]
+ message_rows = messages if isinstance(messages, list) else []
+ image_parts = [
+ part
+ for message in message_rows
+ if isinstance(message, dict) and isinstance(message.get("content"), list)
+ for part in message["content"]
+ if isinstance(part, dict) and part.get("type") == "image"
+ ]
+ if image_parts:
+ embedded_images = [
+ part.get("image") for part in image_parts if part.get("image") is not None
+ ]
+ if embedded_images:
+ if len(embedded_images) == len(image_parts) and all(
+ _is_decoded_image_value(image) for image in embedded_images
+ ):
+ return {
+ "format": "vlm_messages",
+ "needs_conversion": False,
+ "messages_column": "messages",
+ "image_column": None,
+ "text_column": None,
+ }
+ else:
+ images = sample.get("images")
if (
- isinstance(content, list)
- and content
- and isinstance(content[0], dict)
- and "type" in content[0]
+ isinstance(images, list)
+ and images
+ and all(_is_image_list_item(image) for image in images)
):
- has_index = any("index" in item for item in content if isinstance(item, dict))
- if has_index and "images" in column_names:
- return {
- "format": "vlm_messages_llava",
- "needs_conversion": True,
- "messages_column": "messages",
- "image_column": "images",
- "text_column": None,
- }
- has_image = any("image" in item for item in content if isinstance(item, dict))
- if has_image:
- return {
- "format": "vlm_messages",
- "needs_conversion": False,
- "messages_column": "messages",
- "image_column": None,
- "text_column": None,
- }
+ return {
+ "format": "vlm_messages_llava",
+ "needs_conversion": True,
+ "messages_column": "messages",
+ "image_column": "images",
+ "text_column": None,
+ }
for chat_col in ("conversations", "messages"):
if chat_col not in column_names:
diff --git a/studio/backend/tests/test_dataset_image_lists.py b/studio/backend/tests/test_dataset_image_lists.py
new file mode 100644
index 00000000000..26ea9351ab0
--- /dev/null
+++ b/studio/backend/tests/test_dataset_image_lists.py
@@ -0,0 +1,322 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import io
+import sys
+from pathlib import Path
+
+import pytest
+from datasets import Dataset, IterableDataset
+from PIL import Image
+
+_BACKEND_ROOT = Path(__file__).resolve().parents[1]
+if str(_BACKEND_ROOT) not in sys.path:
+ sys.path.insert(0, str(_BACKEND_ROOT))
+
+from hub.utils import dataset_format # noqa: E402
+from utils.datasets.format_conversion import convert_llava_to_vlm_format # noqa: E402
+from utils.datasets import format_detection # noqa: E402
+
+
+def _llava_row():
+ return {
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image", "index": 0, "text": None},
+ {"type": "text", "index": None, "text": "What is in the picture?"},
+ ],
+ },
+ {
+ "role": "assistant",
+ "content": [{"type": "text", "index": None, "text": "A red square."}],
+ },
+ ],
+ "images": [Image.new("RGB", (8, 8), "red")],
+ }
+
+
+def _llava_row_without_index():
+ row = _llava_row()
+ for message in row["messages"]:
+ for part in message["content"]:
+ part.pop("index", None)
+ return row
+
+
+def _llava_rows():
+ yield _llava_row_without_index()
+
+
+def _image_part_row():
+ return {
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image", "image": Image.new("RGB", (8, 8), "red"), "text": None},
+ {"type": "text", "image": None, "text": "Describe this."},
+ ],
+ },
+ {
+ "role": "assistant",
+ "content": [{"type": "text", "image": None, "text": "A dog."}],
+ },
+ ]
+ }
+
+
+def _image_part_row_after_system_message():
+ row = _image_part_row()
+ row["messages"].insert(
+ 0,
+ {
+ "role": "system",
+ "content": [{"type": "text", "image": None, "text": "Answer accurately."}],
+ },
+ )
+ return row
+
+
+def _text_parts_row():
+ return {
+ "messages": [
+ {"role": "user", "content": [{"type": "text", "text": "Hi"}]},
+ {"role": "assistant", "content": [{"type": "text", "text": "Hello"}]},
+ ]
+ }
+
+
+def _encoded_image_row():
+ encoded = io.BytesIO()
+ Image.new("RGB", (8, 8), "red").save(encoded, format = "PNG")
+ return {"media": [{"bytes": encoded.getvalue(), "path": None}]}
+
+
+def _encoded_audio_row():
+ return {"clips": [{"bytes": b"RIFF\x00\x00\x00\x00WAVEfmt ", "path": None}]}
+
+
+ "row, expected_vlm_format",
+ [
+ pytest.param(_llava_row(), "vlm_messages_llava", id = "images-list"),
+ pytest.param(
+ _llava_row_without_index(),
+ "vlm_messages_llava",
+ id = "images-list-without-index",
+ ),
+ pytest.param(_image_part_row(), "vlm_messages", id = "image-parts"),
+ pytest.param(
+ _image_part_row_after_system_message(),
+ "vlm_messages",
+ id = "image-parts-after-system-message",
+ ),
+ ],
+)
+def test_images_in_a_list_or_in_messages_are_detected(row, expected_vlm_format):
+ dataset = [row] if expected_vlm_format == "vlm_messages" else Dataset.from_list([row])
+
+ assert dataset_format.check_dataset_format(dataset, is_vlm = False)["is_image"] is True
+ vlm = dataset_format.check_dataset_format(dataset, is_vlm = True)
+ assert vlm["is_image"] is True
+ assert vlm["detected_format"] == expected_vlm_format
+ assert vlm["requires_manual_mapping"] is False
+ assert format_detection.detect_multimodal_dataset(dataset)["is_image"] is True
+
+
+def test_unindexed_llava_images_are_consumed_in_order():
+ dataset = Dataset.from_list(
+ [
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": [{"type": "image"}, {"type": "image"}],
+ }
+ ],
+ "images": [
+ Image.new("RGB", (8, 8), "red"),
+ Image.new("RGB", (8, 8), "blue"),
+ ],
+ }
+ ]
+ )
+
+ converted = convert_llava_to_vlm_format(dataset)
+ images = [part["image"] for part in converted[0]["messages"][0]["content"]]
+
+ assert [image.getpixel((0, 0)) for image in images] == [(255, 0, 0), (0, 0, 255)]
+
+
+def test_unindexed_llava_images_skip_explicit_indices():
+ dataset = [
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": [{"type": "image", "index": 0}, {"type": "image"}],
+ }
+ ],
+ "images": [
+ Image.new("RGB", (8, 8), "red"),
+ Image.new("RGB", (8, 8), "blue"),
+ ],
+ }
+ ]
+
+ converted = convert_llava_to_vlm_format(dataset)
+ images = [part["image"] for part in converted[0]["messages"][0]["content"]]
+
+ assert [image.getpixel((0, 0)) for image in images] == [(255, 0, 0), (0, 0, 255)]
+
+
+def test_llava_conversion_rejects_missing_unindexed_image():
+ dataset = [
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": [{"type": "image"}, {"type": "image"}],
+ }
+ ],
+ "images": [Image.new("RGB", (8, 8), "red")],
+ }
+ ]
+
+ with pytest.raises(ValueError, match = "image index 1"):
+ convert_llava_to_vlm_format(dataset)
+
+
+def test_llava_conversion_rejects_undecoded_image_dictionary():
+ row = _llava_row_without_index()
+ row["images"] = [_encoded_image_row()["media"][0]]
+
+ with pytest.raises(ValueError, match = "Unsupported Llava image value"):
+ convert_llava_to_vlm_format([row])
+
+
+def test_llava_conversion_preserves_string_turns():
+ row = _llava_row_without_index()
+ row["messages"].insert(0, {"role": "system", "content": "Answer accurately."})
+ dataset = [row]
+
+ assert dataset_format.detect_vlm_dataset_structure(dataset)["format"] == "vlm_messages_llava"
+ assert format_detection.detect_vlm_dataset_structure(dataset)["format"] == "vlm_messages_llava"
+
+ converted = convert_llava_to_vlm_format(dataset)
+
+ assert converted[0]["messages"][0] == {
+ "role": "system",
+ "content": [{"type": "text", "text": "Answer accurately."}],
+ }
+
+
+def test_llava_conversion_stays_lazy_for_streaming_dataset():
+ dataset = IterableDataset.from_generator(_llava_rows)
+
+ converted = convert_llava_to_vlm_format(dataset)
+
+ assert isinstance(converted, IterableDataset)
+ first = next(iter(converted))
+ assert first["messages"][0]["content"][0]["image"].getpixel((0, 0)) == (255, 0, 0)
+
+
+ "row",
+ [
+ pytest.param(_text_parts_row(), id = "text-parts"),
+ pytest.param(
+ {
+ "messages": [
+ {"role": "user", "content": "Hi"},
+ {"role": "assistant", "content": "Yo"},
+ ]
+ },
+ id = "chatml",
+ ),
+ pytest.param({"tags": ["news", "sports"], "text": "Final score 2-1."}, id = "string-list"),
+ pytest.param(
+ {
+ "messages": [
+ {"role": "user", "content": "Which files changed?"},
+ {"role": "assistant", "content": "The entry point and the logo."},
+ ],
+ "files": ["src/main.py", "docs/logo.png"],
+ },
+ id = "file-list",
+ ),
+ ],
+)
+def test_text_only_datasets_stay_text(row):
+ dataset = Dataset.from_list([row])
+
+ assert dataset_format.check_dataset_format(dataset, is_vlm = False)["is_image"] is False
+ assert format_detection.detect_multimodal_dataset(dataset)["is_image"] is False
+
+
[email protected]("detector", [dataset_format, format_detection])
+def test_image_list_detection_requires_values_the_converter_can_open(detector, tmp_path):
+ png_path = tmp_path / "image.png"
+ Image.new("RGB", (8, 8), "red").save(png_path)
+ svg_path = tmp_path / "image.svg"
+ svg_path.write_text('<svg xmlns="http://www.w3.org/2000/svg"/>')
+
+ encoded_image_row = _encoded_image_row()
+ image_dataset = Dataset.from_list([encoded_image_row])
+ image_bytes_dataset = Dataset.from_list([{"media": [encoded_image_row["media"][0]["bytes"]]}])
+ audio_dataset = Dataset.from_list([_encoded_audio_row()])
+ local_image_dataset = Dataset.from_list([{"media": [str(png_path)]}])
+ remote_image_dataset = Dataset.from_list([{"media": ["https://example.com/image.png"]}])
+ svg_image_dataset = Dataset.from_list([{"media": [str(svg_path)]}])
+
+ assert detector.detect_multimodal_dataset(image_dataset)["is_image"] is False
+ assert detector.detect_multimodal_dataset(image_bytes_dataset)["is_image"] is False
+ assert detector.detect_multimodal_dataset(audio_dataset)["is_image"] is False
+ assert detector.detect_multimodal_dataset(local_image_dataset)["is_image"] is True
+ assert detector.detect_multimodal_dataset(remote_image_dataset)["is_image"] is False
+ assert detector.detect_multimodal_dataset(svg_image_dataset)["is_image"] is False
+
+
[email protected]("detector", [dataset_format, format_detection])
+def test_llava_placeholders_require_convertible_images(detector):
+ row = _llava_row_without_index()
+ row["images"] = [_encoded_image_row()["media"][0]]
+ dataset = Dataset.from_list([row])
+
+ result = detector.detect_vlm_dataset_structure(dataset)
+
+ assert result["format"] != "vlm_messages_llava"
+
+
[email protected]("detector", [dataset_format, format_detection])
+def test_embedded_images_require_convertible_values(detector):
+ row = _image_part_row()
+ row["messages"][0]["content"][0]["image"] = _encoded_image_row()["media"][0]
+ dataset = Dataset.from_list([row])
+
+ result = detector.detect_vlm_dataset_structure(dataset)
+
+ assert result["format"] != "vlm_messages"
+
+
[email protected]("detector", [dataset_format, format_detection])
+def test_embedded_image_paths_require_conversion(detector):
+ row = _image_part_row()
+ row["messages"][0]["content"][0]["image"] = "/data/photos/a.jpg"
+
+ result = detector.detect_vlm_dataset_structure([row])
+
+ assert result["format"] != "vlm_messages"
+
+
[email protected]("detector", [dataset_format, format_detection])
+def test_embedded_images_do_not_hide_top_level_placeholders(detector):
+ row = _image_part_row()
+ row["messages"][0]["content"].append({"type": "image", "image": None, "text": None})
+ row["images"] = [Image.new("RGB", (8, 8), "blue")]
+
+ result = detector.detect_vlm_dataset_structure([row])
+
+ assert result["format"] not in {"vlm_messages", "vlm_messages_llava"}
diff --git a/studio/backend/utils/datasets/format_conversion.py b/studio/backend/utils/datasets/format_conversion.py
index a315a53805a..e5714187efe 100644
--- a/studio/backend/utils/datasets/format_conversion.py
+++ b/studio/backend/utils/datasets/format_conversion.py
@@ -872,39 +872,71 @@ def convert_llava_to_vlm_format(dataset):
"""Convert Llava format to standard VLM format: Llava carries messages whose content blocks name an image by {'type': 'image', 'index': 0} plus a parallel images list, while the standard form inlines the PIL object as {'type': 'image', 'image': PIL_Image}."""
from PIL import Image
- logger.info(f"🔄 Converting {len(dataset)} samples from Llava format to standard VLM format...")
+ is_iterable = is_streaming_dataset(dataset)
+ if is_iterable:
+ logger.info("🔄 Converting streaming samples from Llava format to standard VLM format...")
+ else:
+ logger.info(
+ f"🔄 Converting {len(dataset)} samples from Llava format to standard VLM format..."
+ )
def _convert_single_sample(sample):
"""Convert one llava sample to standard VLM format."""
messages = sample["messages"]
images = sample.get("images", [])
+ unindexed_image_index = 0
+ explicit_image_indices = set()
+ for message in messages:
+ content = message.get("content")
+ if not isinstance(content, list):
+ continue
+ for item in content:
+ if not isinstance(item, dict) or item.get("type") != "image":
+ continue
+ image_index = item.get("index")
+ if isinstance(image_index, int) and not isinstance(image_index, bool):
+ explicit_image_indices.add(image_index)
new_messages = []
for msg in messages:
new_content = []
+ message_content = msg["content"]
+
+ if isinstance(message_content, str):
+ new_messages.append(
+ {
+ "role": msg["role"],
+ "content": [{"type": "text", "text": message_content}],
+ }
+ )
+ continue
- for item in msg["content"]:
+ for item in message_content:
if item["type"] == "image":
- if "index" in item and item["index"] is not None:
- img_idx = item["index"]
- if img_idx < len(images):
- pil_image = images[img_idx]
- if isinstance(pil_image, str):
- pil_image = Image.open(pil_image).convert("RGB")
-
- new_content.append(
- {
- "type": "image",
- "image": pil_image, # Actual PIL object
- }
- )
- else:
- if len(images) > 0:
- pil_image = images[0]
- if isinstance(pil_image, str):
- pil_image = Image.open(pil_image).convert("RGB")
-
- new_content.append({"type": "image", "image": pil_image})
+ img_idx = item.get("index")
+ if img_idx is None:
+ while unindexed_image_index in explicit_image_indices:
+ unindexed_image_index += 1
+ img_idx = unindexed_image_index
+ unindexed_image_index += 1
+ if not isinstance(img_idx, int) or isinstance(img_idx, bool):
+ raise ValueError(f"Invalid Llava image index: {img_idx!r}")
+ if not 0 <= img_idx < len(images):
+ raise ValueError(
+ f"Llava image index {img_idx} is missing from a row with "
+ f"{len(images)} images"
+ )
+
+ pil_image = images[img_idx]
+ if isinstance(pil_image, str):
+ pil_image = Image.open(pil_image).convert("RGB")
+ elif not isinstance(pil_image, Image.Image):
+ raise ValueError(
+ f"Unsupported Llava image value at index {img_idx}: "
+ f"{type(pil_image).__name__}"
+ )
+
+ new_content.append({"type": "image", "image": pil_image})
elif item["type"] == "text":
new_content.append({"type": "text", "text": item.get("text", "")})
@@ -913,7 +945,19 @@ def convert_llava_to_vlm_format(dataset):
return {"messages": new_messages}
- converted_list = [_convert_single_sample(sample) for sample in dataset]
+ if is_iterable:
+ remove_columns = dataset.column_names or list(next(iter(dataset), {}))
+ converted_dataset = dataset.map(
+ _convert_single_sample,
+ remove_columns = remove_columns,
+ )
+ try:
+ next(iter(converted_dataset), None)
+ except Exception as exc:
+ raise ValueError(f"Streaming Llava conversion failed on the first row: {exc}") from exc
+ logger.info("✅ Configured streaming Llava conversion")
+ return converted_dataset
+ converted_list = [_convert_single_sample(sample) for sample in dataset]
logger.info(f"✅ Converted {len(converted_list)} samples")
return converted_list
diff --git a/studio/backend/utils/datasets/format_detection.py b/studio/backend/utils/datasets/format_detection.py
index 12d2a884ced..bddaa035656 100644
--- a/studio/backend/utils/datasets/format_detection.py
+++ b/studio/backend/utils/datasets/format_detection.py
@@ -572,7 +572,7 @@ def detect_multimodal_dataset(dataset):
if col_name in already_detected:
continue
value = sample[col_name]
- if _is_image_value(value):
+ if _is_image_value(value) or _holds_images(value):
multimodal_columns.append(col_name)
modality_types.add("image")
@@ -630,17 +630,22 @@ def detect_multimodal_dataset(dataset):
}
-def _is_image_value(value) -> bool:
- """Check if a single sample value looks like image data."""
- if value is None:
- return False
-
+def _is_decoded_image_value(value) -> bool:
try:
from PIL.Image import Image as PILImage
if isinstance(value, PILImage):
return True
except ImportError:
pass
+ return False
+
+
+def _is_image_value(value) -> bool:
+ """Check if a single sample value looks like image data."""
+ if value is None:
+ return False
+ if _is_decoded_image_value(value):
+ return True
# HF Image feature: decoded as PIL, or {"bytes", "path"} when undecoded. Exclude audio dicts, whose decoded form has "array" + "sampling_rate".
if isinstance(value, dict):
@@ -671,6 +676,31 @@ def _is_image_value(value) -> bool:
return False
+def _is_image_list_item(value) -> bool:
+ if isinstance(value, (dict, bytes, bytearray)):
+ return False
+ if isinstance(value, str):
+ normalized = value.strip().lower()
+ if normalized.startswith(("http://", "https://")) or normalized.endswith(".svg"):
+ return False
+ return _is_image_value(value)
+
+
+def _holds_images(value) -> bool:
+ if not isinstance(value, list) or not value:
+ return False
+ if all(_is_image_list_item(item) for item in value):
+ return True
+ for item in value:
+ content = item.get("content") if isinstance(item, dict) else None
+ if isinstance(content, list) and any(
+ isinstance(part, dict) and part.get("type") == "image" and part.get("image") is not None
+ for part in content
+ ):
+ return True
+ return False
+
+
_AUDIO_EXTENSIONS = (
".wav",
".mp3",
@@ -741,39 +771,43 @@ def detect_vlm_dataset_structure(dataset):
if "messages" in column_names:
messages = sample["messages"]
-
- if messages and len(messages) > 0:
- first_msg = messages[0]
- if "content" in first_msg:
- content = first_msg["content"]
-
- if isinstance(content, list) and len(content) > 0:
- if isinstance(content[0], dict) and "type" in content[0]:
- has_index = any(
- "index" in item for item in content if isinstance(item, dict)
- )
- has_images_column = "images" in column_names
-
- if has_index and has_images_column:
- return {
- "format": "vlm_messages_llava",
- "needs_conversion": True,
- "messages_column": "messages",
- "image_column": "images",
- "text_column": None,
- }
-
- has_image = any(
- "image" in item for item in content if isinstance(item, dict)
- )
- if has_image:
- return {
- "format": "vlm_messages",
- "needs_conversion": False,
- "messages_column": "messages",
- "image_column": None,
- "text_column": None,
- }
+ message_rows = messages if isinstance(messages, list) else []
+ image_parts = [
+ part
+ for message in message_rows
+ if isinstance(message, dict) and isinstance(message.get("content"), list)
+ for part in message["content"]
+ if isinstance(part, dict) and part.get("type") == "image"
+ ]
+ if image_parts:
+ embedded_images = [
+ part.get("image") for part in image_parts if part.get("image") is not None
+ ]
+ if embedded_images:
+ if len(embedded_images) == len(image_parts) and all(
+ _is_decoded_image_value(image) for image in embedded_images
+ ):
+ return {
+ "format": "vlm_messages",
+ "needs_conversion": False,
+ "messages_column": "messages",
+ "image_column": None,
+ "text_column": None,
+ }
+ else:
+ images = sample.get("images")
+ if (
+ isinstance(images, list)
+ and images
+ and all(_is_image_list_item(image) for image in images)
+ ):
+ return {
+ "format": "vlm_messages_llava",
+ "needs_conversion": True,
+ "messages_column": "messages",
+ "image_column": "images",
+ "text_column": None,
+ }
# ShareGPT/ChatML conversations with an <image> placeholder plus a companion image column, e.g. Lin-Chen/ShareGPT4V and LLaVA-style datasets.
for chat_col in ("conversations", "messages"):