Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Train vision and audio models from unsloth train (#14389)

Nilaybee45c294d85
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index e98ff90e848..1af08085a11 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -3194,20 +3194,28 @@ class UnslothTrainer:
return (train_data, eval_data)
+ @staticmethod
+ def _resolve_local_path(dataset_file: str) -> Path:
+ dataset_file = os.path.expanduser(dataset_file)
+ if os.path.isabs(dataset_file):
+ return Path(dataset_file)
+ if os.path.exists(dataset_file):
+ return Path(os.path.abspath(dataset_file))
+ return Path(str(resolve_dataset_path(dataset_file)))
+
+ @staticmethod
+ def _load_saved_dataset(file_paths: list):
+ if len(file_paths) != 1:
+ return None
+ from utils.datasets.saved import load_saved_dataset
+ return load_saved_dataset(UnslothTrainer._resolve_local_path(file_paths[0]))
+
@staticmethod
def _resolve_local_files(file_paths: list) -> list[str]:
"""Resolve a list of local dataset paths to concrete file paths."""
all_files: list[str] = []
for dataset_file in file_paths:
- if os.path.isabs(dataset_file):
- file_path = dataset_file
- elif os.path.exists(dataset_file):
- # A path relative to the current working directory (CLI usage)
- file_path = os.path.abspath(dataset_file)
- else:
- file_path = str(resolve_dataset_path(dataset_file))
-
- file_path_obj = Path(file_path)
+ file_path_obj = UnslothTrainer._resolve_local_path(dataset_file)
if file_path_obj.is_dir():
all_files.extend(str(p) for p in dataset_files_in_dir(file_path_obj))
@@ -3358,7 +3366,11 @@ class UnslothTrainer:
if local_datasets:
# load_dataset() is Arrow-backed; an in-memory Dataset.from_list() has no cache and forces num_proc=1
# during tokenization.
- all_files = self._resolve_local_files(local_datasets)
+ saved = self._load_saved_dataset(local_datasets)
+ all_files = [] if saved is not None else self._resolve_local_files(local_datasets)
+ if saved is not None:
+ dataset = saved
+ logger.info(f"Loaded {len(dataset)} samples from a saved dataset folder\n")
if all_files:
loader = self._loader_for_files(all_files)
@@ -3380,7 +3392,18 @@ class UnslothTrainer:
logger.info(f"[DEBUG] Dataset cache_files: {dataset.cache_files}\n")
if local_eval_datasets and eval_enabled:
- eval_all_files = self._resolve_local_files(local_eval_datasets)
+ eval_saved = self._load_saved_dataset(local_eval_datasets)
+ eval_all_files = (
+ []
+ if eval_saved is not None
+ else self._resolve_local_files(local_eval_datasets)
+ )
+ if eval_saved is not None:
+ eval_dataset = eval_saved
+ has_separate_eval_source = True
+ logger.info(
+ f"Loaded {len(eval_dataset)} eval samples from a saved dataset folder\n"
+ )
if eval_all_files:
eval_loader = self._loader_for_files(eval_all_files)
eval_dataset = load_dataset(
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 2bbcc8108ee..dcd054a1cca 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -2720,20 +2720,24 @@ def _normalize_mlx_studio_scheduler(value):
return raw
+def _resolve_mlx_local_dataset_path(dataset_file) -> Path:
+ from utils.paths import resolve_dataset_path
+
+ dataset_path = Path(os.path.expanduser(str(dataset_file)))
+ if dataset_path.is_absolute():
+ return dataset_path
+ if dataset_path.exists():
+ return dataset_path.resolve()
+ return Path(str(resolve_dataset_path(str(dataset_file))))
+
+
def _resolve_mlx_local_dataset_files(file_paths: list) -> list[str]:
"""Resolve CLI paths and Unsloth local dataset uploads without importing the GPU trainer."""
- from utils.paths import dataset_files_in_dir, resolve_dataset_path
+ from utils.paths import dataset_files_in_dir
all_files: list[str] = []
for dataset_file in file_paths or []:
- dataset_path = Path(os.path.expanduser(str(dataset_file)))
- if dataset_path.is_absolute():
- file_path = str(dataset_path)
- elif dataset_path.exists():
- file_path = str(dataset_path.resolve())
- else:
- file_path = str(resolve_dataset_path(str(dataset_file)))
- file_path_obj = Path(file_path)
+ file_path_obj = _resolve_mlx_local_dataset_path(dataset_file)
if file_path_obj.is_dir():
all_files.extend(str(p) for p in dataset_files_in_dir(file_path_obj))
@@ -3387,12 +3391,12 @@ def _run_mlx_training(event_queue, stop_queue, config):
)
def _load_local(file_paths):
- from datasets import load_from_disk
+ from utils.datasets.saved import load_saved_dataset
if len(file_paths) == 1:
- p = Path(file_paths[0])
- if p.is_dir() and ((p / "dataset_info.json").exists() or (p / "state.json").exists()):
- return load_from_disk(str(p))
+ saved = load_saved_dataset(_resolve_mlx_local_dataset_path(file_paths[0]))
+ if saved is not None:
+ return saved
all_files = _resolve_mlx_local_dataset_files(file_paths)
if not all_files:
raise ValueError("No local dataset files found")
diff --git a/studio/backend/tests/test_trainer_saved_dataset_dir.py b/studio/backend/tests/test_trainer_saved_dataset_dir.py
new file mode 100644
index 00000000000..2e22095843f
--- /dev/null
+++ b/studio/backend/tests/test_trainer_saved_dataset_dir.py
@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+import importlib # noqa: E402
+import types # noqa: E402
+from unittest.mock import MagicMock # noqa: E402
+
+import pytest # noqa: E402
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001 - stub unusable imports
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+from core.training import trainer as tmod # noqa: E402
+
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+
+class _Tokenizer:
+ chat_template = "{{ messages }}"
+ eos_token = "</s>"
+
+
+def trainer(monkeypatch):
+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)
+ monkeypatch.setattr(tmod, "ensure_audio_decoding", lambda: True)
+ t = tmod.UnslothTrainer()
+ t.model_name = "Qwen2ForCausalLM"
+ t.tokenizer = _Tokenizer()
+ return t
+
+
+_ROWS = [{"instruction": f"Say {w}", "input": "", "output": w} for w in ("hi", "yes", "no")]
+
+
[email protected]("as_dict", [False, True])
+def test_a_saved_to_disk_folder_trains_as_saved(trainer, tmp_path, as_dict):
+ from datasets import Dataset, DatasetDict
+
+ saved = tmp_path / "saved"
+ dataset = Dataset.from_list(_ROWS)
+ (DatasetDict(train = dataset) if as_dict else dataset).save_to_disk(str(saved))
+
+ result = trainer.load_and_format_dataset(None, local_datasets = [str(saved)])
+
+ assert result is not None
+ dataset_info, _ = result
+ assert dataset_info["final_format"] == "alpaca"
+ assert len(dataset_info["dataset"]) == 3
+
+
[email protected]("name", ["state.json", "dataset_dict.json"])
+def test_a_folder_with_a_data_file_named_like_a_manifest_still_loads_its_rows(
+ trainer, tmp_path, name
+):
+ folder = tmp_path / "rows"
+ folder.mkdir()
+ (folder / name).write_text("\n".join(json.dumps(row) for row in _ROWS))
+
+ result = trainer.load_and_format_dataset(None, local_datasets = [str(folder)])
+
+ assert result is not None
+ assert len(result[0]["dataset"]) == 3
+
+
[email protected]("layout", ["dataset", "dict", "state_json_rows"])
+def test_the_saved_dataset_loader_shared_with_the_mlx_worker(tmp_path, layout):
+ from datasets import Dataset, DatasetDict
+ from utils.datasets.saved import load_saved_dataset
+
+ folder = tmp_path / layout
+ if layout == "state_json_rows":
+ folder.mkdir()
+ (folder / "state.json").write_text("\n".join(json.dumps(row) for row in _ROWS))
+ assert load_saved_dataset(folder) is None
+ return
+ dataset = Dataset.from_list(_ROWS)
+ (DatasetDict(train = dataset) if layout == "dict" else dataset).save_to_disk(str(folder))
+
+ assert load_saved_dataset(folder)["output"] == ["hi", "yes", "no"]
+
+
+def test_the_mlx_worker_resolves_a_studio_dataset_name_before_the_saved_probe(
+ tmp_path, monkeypatch
+):
+ from datasets import Dataset
+ import utils.paths
+ from core.training.worker import _resolve_mlx_local_dataset_path
+ from utils.datasets.saved import load_saved_dataset
+
+ saved = tmp_path / "datasets" / "my_saved"
+ Dataset.from_list(_ROWS).save_to_disk(str(saved))
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(utils.paths, "resolve_dataset_path", lambda name: saved)
+
+ assert load_saved_dataset("my_saved") is None
+ assert load_saved_dataset(_resolve_mlx_local_dataset_path("my_saved"))["output"] == [
+ "hi",
+ "yes",
+ "no",
+ ]
+
+
+def test_a_saved_to_disk_eval_folder_loads_as_saved(trainer, tmp_path):
+ from datasets import Dataset
+
+ train, evals = tmp_path / "train", tmp_path / "eval"
+ Dataset.from_list(_ROWS).save_to_disk(str(train))
+ Dataset.from_list(_ROWS[:2]).save_to_disk(str(evals))
+
+ result = trainer.load_and_format_dataset(
+ None, local_datasets = [str(train)], local_eval_datasets = [str(evals)], eval_steps = 0.5
+ )
+
+ assert result is not None
+ assert len(result[1]) == 2
+
+
[email protected]("escape", ["state_file", "dict_split"])
+def test_a_saved_manifest_pointing_outside_its_folder_is_refused(tmp_path, escape):
+ from datasets import Dataset, DatasetDict
+ from utils.datasets.saved import load_saved_dataset
+
+ Dataset.from_list(_ROWS).save_to_disk(str(tmp_path / "outside"))
+ folder = tmp_path / "saved"
+ if escape == "state_file":
+ Dataset.from_list(_ROWS).save_to_disk(str(folder))
+ state = json.loads((folder / "state.json").read_text())
+ arrow = next((tmp_path / "outside").glob("*.arrow")).name
+ state["_data_files"] = [{"filename": f"../outside/{arrow}"}]
+ (folder / "state.json").write_text(json.dumps(state))
+ else:
+ DatasetDict(train = Dataset.from_list(_ROWS)).save_to_disk(str(folder))
+ (folder / "dataset_dict.json").write_text(json.dumps({"splits": ["../outside"]}))
+
+ with pytest.raises(ValueError, match = "outside its folder"):
+ load_saved_dataset(folder)
diff --git a/studio/backend/utils/datasets/saved.py b/studio/backend/utils/datasets/saved.py
new file mode 100644
index 00000000000..c4de24dbf29
--- /dev/null
+++ b/studio/backend/utils/datasets/saved.py
@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import json
+from pathlib import Path
+
+
+def load_saved_dataset(path):
+ """A `save_to_disk` folder as one Dataset (a DatasetDict's train split, else its first); None for any other path."""
+ path = Path(path)
+ if not _is_saved_dataset(path) and not _is_saved_dataset_dict(path):
+ return None
+ _require_manifest_inside(path)
+ from datasets import DatasetDict, load_from_disk
+
+ dataset = load_from_disk(str(path))
+ if isinstance(dataset, DatasetDict):
+ dataset = dataset["train"] if "train" in dataset else next(iter(dataset.values()))
+ return dataset
+
+
+def _is_saved_dataset(path: Path) -> bool:
+ return (path / "dataset_info.json").is_file() and (path / "state.json").is_file()
+
+
+def _is_saved_dataset_dict(path: Path) -> bool:
+ # A data file can be named dataset_dict.json too, so the manifest must also list split folders that are saves.
+ try:
+ splits = json.loads((path / "dataset_dict.json").read_text(encoding = "utf-8")).get("splits")
+ except (OSError, ValueError, AttributeError):
+ return False
+ return (
+ isinstance(splits, list)
+ and bool(splits)
+ and all(isinstance(split, str) and _is_saved_dataset(path / split) for split in splits)
+ )
+
+
+def _require_manifest_inside(path: Path) -> None:
+ # load_from_disk joins manifest paths onto the folder unchecked: a crafted manifest could read another account's files.
+ root = path.resolve()
+
+ def check(entry: Path) -> Path:
+ resolved = entry.resolve()
+ if resolved != root and root not in resolved.parents:
+ raise ValueError(f"saved dataset at {path} refers to files outside its folder")
+ return resolved
+
+ folders = [path]
+ if _is_saved_dataset_dict(path):
+ splits = (
+ json.loads((path / "dataset_dict.json").read_text(encoding = "utf-8")).get("splits") or []
+ )
+ folders = [check(path / str(split)) for split in splits]
+ for folder in folders:
+ state = folder / "state.json"
+ if state.is_file():
+ for data_file in json.loads(state.read_text(encoding = "utf-8")).get("_data_files") or []:
+ check(folder / str(data_file.get("filename", "")))
diff --git a/studio/backend/utils/models/__init__.py b/studio/backend/utils/models/__init__.py
index d7f6a366b0c..38f8acdbe73 100644
--- a/studio/backend/utils/models/__init__.py
+++ b/studio/backend/utils/models/__init__.py
@@ -8,6 +8,7 @@ from .model_config import (
ModelConfig,
GgufVariantInfo,
is_vision_model,
+ is_vision_model_checked,
is_embedding_model,
detect_audio_type,
detect_audio_type_checked,
@@ -32,6 +33,7 @@ __all__ = [
"ModelConfig",
"GgufVariantInfo",
"is_vision_model",
+ "is_vision_model_checked",
"is_embedding_model",
"detect_audio_type",
"detect_audio_type_checked",
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 39fe6d06fd9..7a31a7bda2b 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -1274,7 +1274,7 @@ _vision_detection_cache: Dict[_CapabilityCacheKey, bool] = {}
_vision_cache_lock = threading.Lock()
-def is_vision_model(
+def _is_vision_model_or_none(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
@@ -1282,7 +1282,7 @@ def is_vision_model(
gguf_variant: Optional[str] = None,
require_image: bool = True,
gguf_companion_roots: Optional[Tuple[str, ...]] = None,
-) -> bool:
+) -> Optional[bool]:
"""Detect VLMs via the config architecture (works for fine-tunes); transformers-5.x
models are checked in a .venv_t5/ subprocess. Cached per (model_name, token,
local_files_only, revision) minus transient failures; local_files_only is in the
@@ -1376,7 +1376,41 @@ def is_vision_model(
with _vision_cache_lock:
_vision_detection_cache[cache_key] = result
return result
- return False
+ return None
+
+
+def is_vision_model(
+ model_name: str,
+ hf_token: Optional[str] = None,
+ local_files_only: bool = False,
+ revision: Optional[str] = None,
+ gguf_variant: Optional[str] = None,
+ require_image: bool = True,
+ gguf_companion_roots: Optional[Tuple[str, ...]] = None,
+) -> bool:
+ """Whether model_name takes images; a transient probe failure reads as False (see _is_vision_model_or_none)."""
+ return bool(
+ _is_vision_model_or_none(
+ model_name,
+ hf_token,
+ local_files_only,
+ revision,
+ gguf_variant,
+ require_image,
+ gguf_companion_roots,
+ )
+ )
+
+
+def is_vision_model_checked(
+ model_name: str,
+ hf_token: Optional[str] = None,
+ local_files_only: bool = False,
+ revision: Optional[str] = None,
+) -> Tuple[bool, bool]:
+ """is_vision_model, plus whether the answer is definitive (False after a transient probe failure)."""
+ result = _is_vision_model_or_none(model_name, hf_token, local_files_only, revision)
+ return bool(result), result is not None
def _is_vision_model_uncached(
diff --git a/unsloth_cli/commands/train.py b/unsloth_cli/commands/train.py
index 053cd7109a4..82cfe3e7d6f 100644
--- a/unsloth_cli/commands/train.py
+++ b/unsloth_cli/commands/train.py
@@ -81,6 +81,142 @@ def _optimizer_for_host(requested: Optional[str] = None) -> str:
return normalize_training_optimizer_for_device(optimizer, device_backend = device_backend)
+_MEDIA_SNIFF_ROWS = 16
+# Containers the PyAV audio decoder reads beyond the detector's extension list.
+_MORE_AUDIO_EXTENSIONS = (".amr", ".aif", ".aiff", ".aifc", ".caf")
+
+
+def _dataset_modalities(
+ cfg: Config, hf_token: Optional[str], load_in_4bit: bool
+) -> tuple[bool, bool]:
+ ensure_studio_backend_path()
+ with studio_backend_imports("unsloth train"):
+ from hub.services.datasets.local import _load_local_preview_slice
+ from hub.utils.dataset_format import detect_multimodal_dataset
+ from utils.datasets.audio_decode import ensure_audio_decoding
+ from utils.datasets.cache_safe import load_dataset_cache_safe
+ from utils.datasets.saved import load_saved_dataset
+ from utils.models import detect_audio_type_checked, is_vision_model_checked
+ from utils.models.unsloth_mirror import unsloth_public_mirror
+ from utils.paths import resolve_dataset_path
+
+ model_name = unsloth_public_mirror(cfg.model, load_in_4bit) or cfg.model
+ audio_type, audio_type_known = detect_audio_type_checked(model_name, hf_token)
+ if audio_type is None and audio_type_known:
+ vision, vision_known = is_vision_model_checked(model_name, hf_token = hf_token)
+ if not vision and vision_known:
+ return False, False
+
+ def preview(path):
+ path = os.path.expanduser(path)
+ if not os.path.isabs(path) and not os.path.exists(path):
+ path = resolve_dataset_path(path)
+ saved = load_saved_dataset(path)
+ if saved is not None:
+ return saved
+ dataset, _ = _load_local_preview_slice(
+ dataset_path = Path(path), train_split = "train", preview_size = _MEDIA_SNIFF_ROWS
+ )
+ return dataset
+
+ image = audio = False
+ try:
+ ensure_audio_decoding()
+ if cfg.data.local_dataset:
+ previews = [preview(path) for path in cfg.data.local_dataset]
+ else:
+ previews = [load_dataset_cache_safe(cfg.data.dataset, split = "train", token = hf_token)]
+ for dataset in previews:
+ detected = detect_multimodal_dataset(dataset)
+ has_image, has_audio = _media_columns_hold_media(dataset, detected)
+ image = image or (detected["is_image"] and has_image)
+ audio = audio or (detected["is_audio"] and has_audio)
+ except Exception as exc: # noqa: BLE001
+ typer.echo(f"Warning: could not check the dataset for images or audio: {exc}", err = True)
+ return False, False
+ return image, audio
+
+
+def _media_columns_hold_media(dataset, detected) -> tuple[bool, bool]:
+ # The detector also flags columns by name alone, so a text column called `filename` or `speech` must hold media.
+ from itertools import islice
+
+ from datasets import Audio, Image
+ from hub.utils.dataset_format import (
+ _AUDIO_EXTENSIONS,
+ _has_image_header,
+ _holds_images,
+ _is_audio_value,
+ _is_image_value,
+ )
+
+ features = getattr(dataset, "features", None) or {}
+ rows = []
+
+ def values(column):
+ if not rows:
+ rows.extend(row or {} for row in islice(iter(dataset), _MEDIA_SNIFF_ROWS))
+ return [row.get(column) for row in rows]
+
+ def bare(value):
+ return value.strip().split("?")[0] if isinstance(value, str) else ""
+
+ def sniff(head, kind):
+ if kind == "image":
+ return _has_image_header(head) or head[:4] in (b"II*\x00", b"MM\x00*")
+ return (
+ head[:4] in (b"fLaC", b"OggS", b"\x1aE\xdf\xa3", b"0&\xb2u", b"caff")
+ or head[:3] == b"ID3"
+ or head[:5] == b"#!AMR"
+ or (head[:4] == b"FORM" and head[8:12] in (b"AIFF", b"AIFC"))
+ or head[4:8] == b"ftyp"
+ or (head[:4] == b"RIFF" and head[8:12] == b"WAVE")
+ or (len(head) > 1 and head[0] == 0xFF and head[1] & 0xE0 == 0xE0)
+ )
+
+ def reference(value, kind):
+ if isinstance(value, dict):
+ value = value.get("bytes") or value.get("path")
+ if isinstance(value, (bytes, bytearray)):
+ return sniff(bytes(value[:12]), kind)
+ path = bare(value)
+ if not path:
+ return False
+ if path.lower().startswith(("http://", "https://")):
+ from urllib.request import Request, urlopen
+ try:
+ request = Request(value.strip(), headers = {"Range": "bytes=0-11"})
+ with urlopen(request, timeout = 5) as response:
+ if response.headers.get_content_type().startswith(f"{kind}/"):
+ return True
+ return sniff(response.read(12), kind)
+ except Exception: # noqa: BLE001
+ return False
+ if not os.path.isfile(path):
+ return False
+ with open(path, "rb") as file:
+ return sniff(file.read(12), kind)
+
+ def image(column):
+ return isinstance(features.get(column), Image) or any(
+ _is_image_value(value) or _holds_images(value) or reference(value, "image")
+ for value in values(column)
+ )
+
+ def audio(column):
+ return isinstance(features.get(column), Audio) or any(
+ _is_audio_value(value)
+ or bare(value).lower().endswith(_AUDIO_EXTENSIONS + _MORE_AUDIO_EXTENSIONS)
+ or reference(value, "audio")
+ for value in values(column)
+ )
+
+ return (
+ any(image(column) for column in detected.get("multimodal_columns", [])),
+ any(audio(column) for column in detected.get("audio_columns", [])),
+ )
+
+
def _launch_rank() -> Optional[int]:
# A size alone is not a launch: a stale PMI_SIZE or WORLD_SIZE in a shell would pin every run to one folder.
rank = os.environ.get("RANK")
@@ -247,6 +383,9 @@ def train(
raise typer.Exit(code = 2)
trainer = _create_cli_trainer(cfg.model, hf_token)
+ is_dataset_image, is_dataset_audio = _dataset_modalities(
+ cfg, hf_token, use_lora and cfg.training.load_in_4bit
+ )
if not trainer.load_model(
model_name = cfg.model,
@@ -254,6 +393,8 @@ def train(
load_in_4bit = cfg.training.load_in_4bit if use_lora else False,
full_finetuning = not use_lora,
hf_token = hf_token,
+ is_dataset_image = is_dataset_image,
+ is_dataset_audio = is_dataset_audio,
use_gradient_checkpointing = cfg.training.gradient_checkpointing,
):
typer.echo("Model load failed", err = True)
diff --git a/unsloth_cli/tests/test_train_ctrl_c_exit_code.py b/unsloth_cli/tests/test_train_ctrl_c_exit_code.py
index 571363e1e90..c275a0c7cdd 100644
--- a/unsloth_cli/tests/test_train_ctrl_c_exit_code.py
+++ b/unsloth_cli/tests/test_train_ctrl_c_exit_code.py
@@ -78,6 +78,9 @@ def _run(tmp_path, monkeypatch, trainer, sleep):
)
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *args: trainer)
+ monkeypatch.setattr(
+ train_module, "_dataset_modalities", lambda *args: (False, False), raising = False
+ )
monkeypatch.setattr(train_module, "time", SimpleNamespace(sleep = sleep, time = time.time))
app = typer.Typer()
app.command()(train_module.train)
diff --git a/unsloth_cli/tests/test_train_default_output_dir.py b/unsloth_cli/tests/test_train_default_output_dir.py
index ecfc0b8bbff..0b7c4e04d1b 100644
--- a/unsloth_cli/tests/test_train_default_output_dir.py
+++ b/unsloth_cli/tests/test_train_default_output_dir.py
@@ -54,6 +54,9 @@ class _SavingTrainer:
def _train(monkeypatch, trainer, *args):
monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *a: trainer)
+ monkeypatch.setattr(
+ train_module, "_dataset_modalities", lambda *args: (False, False), raising = False
+ )
app = typer.Typer()
app.command()(train_module.train)
return CliRunner().invoke(
diff --git a/unsloth_cli/tests/test_train_random_seed.py b/unsloth_cli/tests/test_train_random_seed.py
index 9b25042e727..aaa36e5338b 100644
--- a/unsloth_cli/tests/test_train_random_seed.py
+++ b/unsloth_cli/tests/test_train_random_seed.py
@@ -50,6 +50,9 @@ def test_random_seed_flag_seeds_the_lora_init(tmp_path, monkeypatch):
)
trainer = _RecordingTrainer()
monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *args: trainer)
+ monkeypatch.setattr(
+ train_module, "_dataset_modalities", lambda *args: (False, False), raising = False
+ )
app = typer.Typer()
app.command()(train_module.train)
diff --git a/unsloth_cli/tests/test_train_vision_dataset.py b/unsloth_cli/tests/test_train_vision_dataset.py
new file mode 100644
index 00000000000..5b4dc5ebc46
--- /dev/null
+++ b/unsloth_cli/tests/test_train_vision_dataset.py
@@ -0,0 +1,408 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import typer
+from typer.testing import CliRunner
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+import unsloth_cli.commands.train as train_module # noqa: E402
+
+_VISION_LORA_YAML = _REPO_ROOT / "studio" / "backend" / "assets" / "configs" / "vision_lora.yaml"
+_VISION_CONFIG = {
+ "model_type": "qwen2_vl",
+ "architectures": ["Qwen2VLForConditionalGeneration"],
+ "vision_config": {},
+}
+_TEXT_CONFIG = {"model_type": "qwen3", "architectures": ["Qwen3ForCausalLM"]}
+_TEXT_TOKENIZER = {"added_tokens_decoder": {"151643": {"content": "<|endoftext|>"}}}
+_AUDIO_VLM_TOKENIZER = {"added_tokens_decoder": {"262273": {"content": "<audio_soft_token>"}}}
+_IMAGE_ROWS = [{"image": "photos/cat.png", "text": "a cat"}]
+_TEXT_ROWS = [{"instruction": "Say hi", "output": "hi"}]
+_AUDIO_ROWS = [{"audio": "clips/hello.wav", "text": "hello"}]
+_FILE_NAME_ROWS = [{"instruction": "Summarise", "output": "A readme", "file_name": "README.md"}]
+_PROSE_IMAGE_ROWS = [{"instruction": "Describe", "output": "A cat", "image": "a cat"}]
+_PROSE_SPEECH_ROWS = [
+ {"instruction": "Rewrite", "output": "Hello!", "speech": "Welcome to our meeting."}
+]
+
+
+class _RecordingTrainer:
+ training_thread = None
+
+ def __init__(self):
+ self.is_vlm = False
+ self.load_kwargs = None
+ self.prepare_kwargs = None
+
+ def load_model(self, **kwargs):
+ self.load_kwargs = kwargs
+ self.is_vlm = kwargs.get("is_dataset_image", False)
+ return True
+
+ def prepare_model_for_training(self, **kwargs):
+ self.prepare_kwargs = kwargs
+ return True
+
+ def load_and_format_dataset(self, **kwargs):
+ return [], None
+
+ def start_training(self, **kwargs):
+ return True
+
+ def get_training_progress(self):
+ return SimpleNamespace(error = None, is_completed = True)
+
+
+def _train(tmp_path, monkeypatch, *args):
+ import datasets
+
+ monkeypatch.setattr(datasets.config, "HF_UPDATE_DOWNLOAD_COUNTS", False)
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ trainer = _RecordingTrainer()
+ monkeypatch.setattr(train_module, "_create_cli_trainer", lambda *a: trainer)
+ app = typer.Typer()
+ app.command()(train_module.train)
+
+ result = CliRunner().invoke(app, [*args, "--output-dir", str(tmp_path / "out")])
+
+ assert result.exit_code == 0, result.output
+ return trainer
+
+
+def _local_model(
+ tmp_path,
+ config,
+ tokenizer_config = None,
+):
+ model = tmp_path / "model"
+ model.mkdir()
+ (model / "config.json").write_text(json.dumps(config), encoding = "utf-8")
+ (model / "tokenizer_config.json").write_text(json.dumps(tokenizer_config), encoding = "utf-8")
+ return str(model)
+
+
+def _local_parquet(tmp_path, rows):
+ from datasets import Dataset
+
+ path = tmp_path / "data.parquet"
+ Dataset.from_list(rows).to_parquet(str(path))
+ return str(path)
+
+
+def _hub(monkeypatch, vision_repos, rows):
+ train_module.ensure_studio_backend_path()
+ import utils.datasets.cache_safe as cache_safe
+ import utils.models as models
+ from datasets import Dataset
+
+ loaded = []
+
+ def load_dataset(path, **kwargs):
+ loaded.append((path, kwargs.get("split")))
+ return Dataset.from_list(rows)
+
+ monkeypatch.setattr(cache_safe, "load_dataset_cache_safe", load_dataset)
+ monkeypatch.setattr(
+ models, "is_vision_model_checked", lambda name, **kwargs: (name in vision_repos, True)
+ )
+ monkeypatch.setattr(models, "detect_audio_type_checked", lambda *args, **kwargs: (None, True))
+ return loaded
+
+
+ "config, tokenizer_config, rows, image, audio",
+ [
+ (_VISION_CONFIG, _TEXT_TOKENIZER, _IMAGE_ROWS, True, False),
+ (_VISION_CONFIG, _TEXT_TOKENIZER, _TEXT_ROWS, False, False),
+ (_TEXT_CONFIG, _TEXT_TOKENIZER, _IMAGE_ROWS, False, False),
+ (_TEXT_CONFIG, _AUDIO_VLM_TOKENIZER, _AUDIO_ROWS, False, True),
+ (_VISION_CONFIG, _TEXT_TOKENIZER, _FILE_NAME_ROWS, False, False),
+ (_VISION_CONFIG, _TEXT_TOKENIZER, _PROSE_IMAGE_ROWS, False, False),
+ (_TEXT_CONFIG, _AUDIO_VLM_TOKENIZER, _PROSE_SPEECH_ROWS, False, False),
+ ],
+)
+def test_train_tells_the_trainer_whether_a_local_dataset_has_images_or_audio(
+ tmp_path, monkeypatch, config, tokenizer_config, rows, image, audio
+):
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, config, tokenizer_config),
+ "--local-dataset",
+ _local_parquet(tmp_path, rows),
+ )
+
+ assert trainer.load_kwargs.get("is_dataset_image", False) is image
+ assert trainer.load_kwargs.get("is_dataset_audio", False) is audio
+
+
[email protected]("header, image", [(b"\x89PNG\r\n\x1a\n", True), (b"# Read me\n", False)])
+def test_an_extensionless_file_counts_as_an_image_only_when_it_is_one(
+ tmp_path, monkeypatch, header, image
+):
+ referenced = tmp_path / "README"
+ referenced.write_bytes(header)
+ rows = [{"instruction": "Summarise", "output": "A readme", "file_name": str(referenced)}]
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, _VISION_CONFIG, _TEXT_TOKENIZER),
+ "--local-dataset",
+ _local_parquet(tmp_path, rows),
+ )
+
+ assert trainer.load_kwargs.get("is_dataset_image", False) is image
+
+
+ "header, audio",
+ [
+ (b"\x00\x00\x00\x20ftypM4A ", True),
+ (b"\x1a\x45\xdf\xa3\x01\x00", True),
+ (b"\xff\xfb\x90\x00", True),
+ (b"\xff\xf1\x50\x80", True),
+ (b"0&\xb2u\x8ef\xcf\x11", True),
+ (b"#!AMR\n\x3c", True),
+ (b"FORM\x00\x00\x00\x00AIFF", True),
+ (b"caff\x00\x01\x00\x00", True),
+ (b"Hello there\n", False),
+ ],
+)
+def test_an_extensionless_audio_file_is_recognised_by_its_container(
+ tmp_path, monkeypatch, header, audio
+):
+ clip = tmp_path / "clip"
+ clip.write_bytes(header)
+ rows = [{"audio": str(clip), "text": "hello"}]
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, _TEXT_CONFIG, _AUDIO_VLM_TOKENIZER),
+ "--local-dataset",
+ _local_parquet(tmp_path, rows),
+ )
+
+ assert trainer.load_kwargs.get("is_dataset_audio", False) is audio
+
+
+def test_an_image_set_with_a_leading_empty_row_still_trains_the_vision_model(tmp_path, monkeypatch):
+ rows = [{"image": None, "text": "no picture"}, {"image": "photos/cat.png", "text": "a cat"}]
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, _VISION_CONFIG, _TEXT_TOKENIZER),
+ "--local-dataset",
+ _local_parquet(tmp_path, rows),
+ )
+
+ assert trainer.load_kwargs["is_dataset_image"] is True
+
+
+ "clip", [b"RIFF\x24\x00\x00\x00WAVEfmt ", {"bytes": b"fLaC\x00\x00\x00\x22", "path": None}]
+)
+def test_byte_backed_audio_trains_the_audio_model(tmp_path, monkeypatch, clip):
+ rows = [{"audio": clip, "text": "hello"}]
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, _TEXT_CONFIG, _AUDIO_VLM_TOKENIZER),
+ "--local-dataset",
+ _local_parquet(tmp_path, rows),
+ )
+
+ assert trainer.load_kwargs["is_dataset_audio"] is True
+
+
+def test_an_extensionless_image_url_is_checked_by_what_it_serves(tmp_path, monkeypatch):
+ import http.server
+ import threading
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ self.send_response(200)
+ self.send_header("Content-Type", "application/octet-stream")
+ self.end_headers()
+ self.wfile.write(b"\x89PNG\r\n\x1a\n" if self.path == "/cat" else b"plain text")
+
+ def log_message(self, *args):
+ pass
+
+ server = http.server.HTTPServer(("127.0.0.1", 0), Handler)
+ threading.Thread(target = server.serve_forever, daemon = True).start()
+ base = f"http://127.0.0.1:{server.server_port}"
+ try:
+ results = []
+ for name in ("cat", "readme"):
+ directory = tmp_path / name
+ directory.mkdir()
+ trainer = _train(
+ directory,
+ monkeypatch,
+ "--model",
+ _local_model(directory, _VISION_CONFIG, _TEXT_TOKENIZER),
+ "--local-dataset",
+ _local_parquet(directory, [{"image": f"{base}/{name}", "text": "x"}]),
+ )
+ results.append(trainer.load_kwargs.get("is_dataset_image", False))
+ finally:
+ server.shutdown()
+
+ assert results == [True, False]
+
+
+def test_a_native_audio_model_is_told_its_dataset_has_audio(tmp_path, monkeypatch):
+ train_module.ensure_studio_backend_path()
+ import utils.models as models
+
+ monkeypatch.setattr(models, "detect_audio_type_checked", lambda *a, **k: ("whisper", True))
+ monkeypatch.setattr(models, "is_vision_model_checked", lambda *a, **k: (False, True))
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ "openai/whisper-tiny",
+ "--local-dataset",
+ _local_parquet(tmp_path, _AUDIO_ROWS),
+ )
+
+ assert trainer.load_kwargs["is_dataset_audio"] is True
+
+
[email protected]("name", ["clip.amr", "clip.aiff", "clip.caf", "clip.bin"])
+def test_audio_paths_with_other_suffixes_and_path_dicts_count_as_audio(tmp_path, monkeypatch, name):
+ clip = tmp_path / name
+ clip.write_bytes(b"#!AMR\n\x3c\x00")
+ rows = [{"audio": {"bytes": None, "path": str(clip)}, "text": "hello"}]
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, _TEXT_CONFIG, _AUDIO_VLM_TOKENIZER),
+ "--local-dataset",
+ _local_parquet(tmp_path, rows),
+ )
+
+ assert trainer.load_kwargs["is_dataset_audio"] is True
+
+
+def test_every_local_dataset_path_is_checked_for_images(tmp_path, monkeypatch):
+ from datasets import Dataset
+
+ text = tmp_path / "text.parquet"
+ Dataset.from_list(_TEXT_ROWS).to_parquet(str(text))
+ images = tmp_path / "images.parquet"
+ Dataset.from_list(_IMAGE_ROWS).to_parquet(str(images))
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, _VISION_CONFIG, _TEXT_TOKENIZER),
+ "--local-dataset",
+ str(text),
+ "--local-dataset",
+ str(images),
+ )
+
+ assert trainer.load_kwargs["is_dataset_image"] is True
+
+
+def test_a_transient_vision_probe_still_checks_the_dataset(tmp_path, monkeypatch):
+ train_module.ensure_studio_backend_path()
+ import utils.models as models
+
+ monkeypatch.setattr(models, "detect_audio_type_checked", lambda *a, **k: (None, True))
+ monkeypatch.setattr(models, "is_vision_model_checked", lambda *a, **k: (False, False))
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ "Qwen/Qwen2-VL-2B-Instruct",
+ "--local-dataset",
+ _local_parquet(tmp_path, _IMAGE_ROWS),
+ )
+
+ assert trainer.load_kwargs["is_dataset_image"] is True
+
+
[email protected]("as_dict", [False, True])
+def test_a_saved_to_disk_image_dataset_trains_the_vision_model(tmp_path, monkeypatch, as_dict):
+ from datasets import Dataset, DatasetDict
+ from PIL import Image
+
+ saved = tmp_path / "saved"
+ dataset = Dataset.from_list([{"image": Image.new("RGB", (8, 8)), "text": "a square"}])
+ (DatasetDict(train = dataset) if as_dict else dataset).save_to_disk(str(saved))
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, _VISION_CONFIG, _TEXT_TOKENIZER),
+ "--local-dataset",
+ str(saved),
+ )
+
+ assert trainer.load_kwargs["is_dataset_image"] is True
+
+
+def test_shipped_vision_lora_config_trains_the_vision_lora(tmp_path, monkeypatch):
+ loaded = _hub(monkeypatch, {"unsloth/Qwen2-VL-2B-Instruct-bnb-4bit"}, _IMAGE_ROWS)
+
+ trainer = _train(tmp_path, monkeypatch, "--config", str(_VISION_LORA_YAML))
+
+ assert loaded == [("philschmid/amazon-product-descriptions-vlm", "train")]
+ assert trainer.load_kwargs["is_dataset_image"] is True
+ assert trainer.prepare_kwargs["target_modules"] == "all-linear"
+
+
+def test_a_gated_upstream_vlm_is_checked_through_its_public_mirror(tmp_path, monkeypatch):
+ _hub(monkeypatch, {"unsloth/gemma-3-4b-it-unsloth-bnb-4bit"}, _IMAGE_ROWS)
+
+ trainer = _train(
+ tmp_path, monkeypatch, "--model", "google/gemma-3-4b-it", "--dataset", "unsloth/LaTeX_OCR"
+ )
+
+ assert trainer.load_kwargs["is_dataset_image"] is True
+
+
+def test_a_home_relative_local_dataset_is_checked_for_images(tmp_path, monkeypatch):
+ from datasets import Dataset
+
+ Dataset.from_list(_IMAGE_ROWS).to_parquet(str(tmp_path / "images.parquet"))
+ monkeypatch.setenv("HOME", str(tmp_path))
+
+ trainer = _train(
+ tmp_path,
+ monkeypatch,
+ "--model",
+ _local_model(tmp_path, _VISION_CONFIG, _TEXT_TOKENIZER),
+ "--local-dataset",
+ "~/images.parquet",
+ )
+
+ assert trainer.load_kwargs["is_dataset_image"] is True