Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio VLM conversion: fetch repo image files on a pool, settle rows in order (#17029)

Daniel Han4e260ac6d9e4
diff --git a/studio/backend/tests/test_vlm_repo_image_fetch_pool.py b/studio/backend/tests/test_vlm_repo_image_fetch_pool.py
new file mode 100644
index 00000000000..fea89f3c7a1
--- /dev/null
+++ b/studio/backend/tests/test_vlm_repo_image_fetch_pool.py
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Image + text rows naming repo files fetch them on a pool, with the same output as one at a time."""
+
+import random
+import threading
+import time
+
+from datasets import Dataset
+from PIL import Image
+
+from utils import hardware
+from utils.datasets import format_conversion
+
+N = 48
+
+
+def _fake_hub(monkeypatch, tmp_path):
+ tmp_path.mkdir(parents = True, exist_ok = True)
+ paths = {}
+ for i in range(N):
+ path = tmp_path / f"{i}.png"
+ Image.new("RGB", (i + 1, 2)).save(path)
+ paths[f"images/{i}.png"] = str(path)
+ state = {"live": 0, "peak": 0, "calls": []}
+ lock = threading.Lock()
+
+ def _download(repo_id, filename, **kwargs):
+ with lock:
+ state["live"] += 1
+ state["peak"] = max(state["peak"], state["live"])
+ state["calls"].append(filename)
+ try:
+ time.sleep(0.02)
+ if filename == "images/7.png":
+ raise OSError("404 for images/7.png")
+ return paths[filename]
+ finally:
+ with lock:
+ state["live"] -= 1
+
+ monkeypatch.setattr(
+ "huggingface_hub.HfApi.list_repo_files", lambda self, *a, **k: ["README.md", *paths]
+ )
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", _download)
+ from utils.datasets import llm_assist
+
+ monkeypatch.setattr(llm_assist, "llm_generate_dataset_warning", lambda *a, **k: None)
+ return state
+
+
+def _convert(workers, monkeypatch, tmp_path):
+ state = _fake_hub(monkeypatch, tmp_path)
+ monkeypatch.setattr(hardware, "safe_thread_num_proc", lambda *a, **k: workers)
+ ds = Dataset.from_dict(
+ {
+ "image": [f"{i}.png" if i % 5 else f"images/{i}.png" for i in range(N)],
+ # A list caption is picked with the global random module, so the pick order must not move.
+ "text": [[f"cap {i} a", f"cap {i} b", f"cap {i} c"] for i in range(N)],
+ }
+ )
+ random.seed(1234)
+ out = format_conversion.convert_to_vlm_format(
+ ds, instruction = "Describe.", dataset_name = "org/ds"
+ )
+ rows = [
+ (
+ sample["messages"][0]["content"][1]["image"].size,
+ sample["messages"][1]["content"][0]["text"],
+ )
+ for sample in out
+ ]
+ return rows, state
+
+
+def test_pooled_fetch_matches_serial(monkeypatch, tmp_path):
+ serial, serial_state = _convert(1, monkeypatch, tmp_path / "serial")
+ pooled, pooled_state = _convert(8, monkeypatch, tmp_path / "pooled")
+
+ assert len(serial) == N - 1
+ assert pooled == serial
+ assert sorted(pooled_state["calls"]) == sorted(serial_state["calls"])
+ assert serial_state["peak"] == 1
+ assert pooled_state["peak"] > 1
diff --git a/studio/backend/utils/datasets/format_conversion.py b/studio/backend/utils/datasets/format_conversion.py
index 65785916adc..017f9b5e9f0 100644
--- a/studio/backend/utils/datasets/format_conversion.py
+++ b/studio/backend/utils/datasets/format_conversion.py
@@ -4,6 +4,7 @@
"""Dataset format conversion between Alpaca, ShareGPT, and ChatML."""
import contextvars
+import functools
import os
from .cells import cell_text
@@ -460,6 +461,33 @@ def _fetch_row_image(url):
return Image.open(BytesIO(fetch(url))).convert("RGB")
+def _settle_in_order(rows, task, record, num_workers):
+ """Calls ``record(row, future)`` in row order; ``future`` runs ``task(row)`` on a pool (None if serial or ``task`` gives None)."""
+ if num_workers <= 1:
+ for row in rows:
+ record(row, None)
+ return
+ from collections import deque
+ from concurrent.futures import ThreadPoolExecutor
+
+ # Settling here keeps output, random caption picks and the first-failure log as serial; 2x workers bounds images held.
+ pending = deque()
+ with ThreadPoolExecutor(max_workers = num_workers) as executor:
+ try:
+ for row in rows:
+ fn = task(row)
+ # A fresh context copy per task: it sees the caller's account / HF cache settings.
+ future = None if fn is None else executor.submit(contextvars.copy_context().run, fn)
+ pending.append((row, future))
+ if len(pending) > 2 * num_workers:
+ record(*pending.popleft())
+ while pending:
+ record(*pending.popleft())
+ except BaseException:
+ executor.shutdown(wait = False, cancel_futures = True)
+ raise
+
+
def _non_null_rows(dataset, column):
"""Row positions whose ``column`` is not None, read from Arrow validity so cells are never built or decoded."""
try:
@@ -523,8 +551,20 @@ def convert_to_vlm_format(
instruction_column = None
uses_dynamic = False
+ def _fetch_repo_image(name):
+ from huggingface_hub import hf_hub_download
+ from utils.hf_cache_settings import active_hf_hub_cache
+
+ local_path = hf_hub_download(
+ dataset_name,
+ _image_lookup[name],
+ repo_type = "dataset",
+ cache_dir = active_hf_hub_cache(),
+ )
+ return Image.open(local_path).convert("RGB")
+
def _convert_single_sample(sample, fetched = None):
- """Image may be a PIL Image, local path, URL, or bare filename; ``fetched`` = the URL's already-downloaded image."""
+ """Image may be a PIL Image, local path, URL, or bare filename; ``fetched`` = the URL's or repo file's already-downloaded image."""
image_data = sample[image_column]
if isinstance(image_data, str):
@@ -532,16 +572,7 @@ def convert_to_vlm_format(
image_data = fetched if fetched is not None else _fetch_row_image(image_data)
elif _image_lookup is not None and image_data in _image_lookup:
# Bare filename → resolve via HF repo lookup
- from huggingface_hub import hf_hub_download
- from utils.hf_cache_settings import active_hf_hub_cache
-
- local_path = hf_hub_download(
- dataset_name,
- _image_lookup[image_data],
- repo_type = "dataset",
- cache_dir = active_hf_hub_cache(),
- )
- image_data = Image.open(local_path).convert("RGB")
+ image_data = fetched if fetched is not None else _fetch_repo_image(image_data)
else:
image_data = Image.open(_own_image_path(image_data, account)).convert("RGB")
elif image_data is None:
@@ -772,15 +803,34 @@ def convert_to_vlm_format(
unit = "sample",
**_quiet_bar_kwargs(),
)
- for sample in pbar:
+
+ def _record(sample, future = None):
+ nonlocal failed_count
try:
- converted_list.append(_convert_single_sample(sample))
+ fetched = future.result() if future is not None else None
+ converted_list.append(_convert_single_sample(sample, fetched))
except Exception as e:
failed_count += 1
if failed_count == 1:
logger.info(f"First VLM conversion failure: {type(e).__name__}: {e}")
pbar.set_postfix(ok = len(converted_list), failed = failed_count, refresh = False)
- pbar.close()
+
+ from utils.hardware import safe_thread_num_proc
+
+ num_workers = (
+ min(safe_thread_num_proc(), _MAX_ROW_FETCH_WORKERS) if _image_lookup is not None else 1
+ )
+
+ def _repo_fetch(sample):
+ name = sample[image_column]
+ if isinstance(name, str) and name in _image_lookup:
+ return functools.partial(_fetch_repo_image, name)
+ return None
+
+ try:
+ _settle_in_order(pbar, _repo_fetch, _record, num_workers)
+ finally:
+ pbar.close()
if failed_count > 0:
fail_rate = failed_count / total
@@ -1220,21 +1270,18 @@ def convert_sharegpt_with_images_to_vlm_format(
converted_list = []
failed_count = 0
- def _record(future_or_sample):
+ def _record(sample, future):
nonlocal failed_count
try:
- if isinstance(future_or_sample, Future):
- converted_list.append(future_or_sample.result())
- else:
- converted_list.append(_convert_single_sample(future_or_sample))
+ converted_list.append(
+ future.result() if future is not None else _convert_single_sample(sample)
+ )
except Exception as e:
failed_count += 1
if failed_count == 1:
logger.info(f"⚠️ First conversion failure: {type(e).__name__}: {e}")
pbar.set_postfix(ok = len(converted_list), failed = failed_count, refresh = False)
- from concurrent.futures import Future
-
num_workers = 1
if _image_lookup is not None or (
isinstance(first_image, str) and first_image.startswith(("http://", "https://"))
@@ -1250,31 +1297,12 @@ def convert_sharegpt_with_images_to_vlm_format(
**_quiet_bar_kwargs(),
)
try:
- if num_workers <= 1:
- for sample in pbar:
- _record(sample)
- else:
- from collections import deque
- from concurrent.futures import ThreadPoolExecutor
-
- # Settled in row order (same output and first-failure log as serial); 2x workers bounds images held in memory.
- pending = deque()
- with ThreadPoolExecutor(max_workers = num_workers) as executor:
- try:
- for sample in pbar:
- # A fresh context copy per task: the fetch sees the caller's account / HF cache settings.
- pending.append(
- executor.submit(
- contextvars.copy_context().run, _convert_single_sample, sample
- )
- )
- if len(pending) > 2 * num_workers:
- _record(pending.popleft())
- while pending:
- _record(pending.popleft())
- except BaseException:
- executor.shutdown(wait = False, cancel_futures = True)
- raise
+ _settle_in_order(
+ pbar,
+ lambda sample: functools.partial(_convert_single_sample, sample),
+ _record,
+ num_workers,
+ )
finally:
pbar.close()