Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""One string column is raw training text: the preview and the formatter must agree on it."""
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from datasets import Dataset
_BACKEND_ROOT = Path(__file__).resolve().parents[1]
if str(_BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(_BACKEND_ROOT))
from hub.utils import dataset_format # noqa: E402
from utils.datasets import format_detection # noqa: E402
from utils.datasets.dataset_utils import format_and_template_dataset, format_dataset # noqa: E402
def _text_dataset(column: str = "text"):
return Dataset.from_list(
[
{column: "## What happened\nWrote the fix."},
{column: "## What happened\nReviewed the diff."},
]
)
@pytest.mark.parametrize("column", ["text", "content", "body"], ids = ["text", "content", "body"])
@pytest.mark.parametrize("is_vlm", [False, True], ids = ["text_model", "vision_model"])
def test_single_string_column_needs_no_mapping(column, is_vlm):
result = dataset_format.check_dataset_format(_text_dataset(column), is_vlm = is_vlm)
assert result["requires_manual_mapping"] is False
assert result["detected_format"] == "raw"
assert result["suggested_mapping"] is None
assert result["columns"] == [column]
def test_text_column_keeps_detection_flags_for_other_branches():
result = dataset_format.check_dataset_format(_text_dataset())
assert result["is_image"] is False
assert result["is_audio"] is False
assert result["detected_image_column"] is None
def test_media_named_text_column_is_not_routed_as_media():
result = dataset_format.check_dataset_format(_text_dataset("speech"), is_vlm = True)
assert result["detected_format"] == "raw"
assert result["is_audio"] is False
assert result["detected_audio_column"] is None
assert result["is_image"] is False
@pytest.mark.parametrize(
"rows",
[
pytest.param([{"text": "a", "id": 1}], id = "text_plus_metadata"),
pytest.param([{"text": " "}], id = "blank_text"),
pytest.param([{"text": 5}], id = "non_string_text"),
],
)
def test_other_shapes_still_require_mapping(rows):
result = dataset_format.check_dataset_format(Dataset.from_list(rows))
assert result["detected_format"] == "unknown"
assert result["requires_manual_mapping"] is True
@pytest.mark.parametrize(
"row", [{"image": "photos/sloth.jpg"}, {"audio": "clips/sloth.wav"}], ids = ["image", "audio"]
)
def test_media_path_column_is_not_raw_text(row):
result = dataset_format.check_dataset_format(Dataset.from_list([row]))
assert result["detected_format"] != "raw"
assert result["requires_manual_mapping"] is True
def test_plain_text_dataset_helper_rejects_other_shapes():
assert format_detection.is_plain_text_dataset(_text_dataset()) is True
assert format_detection.is_plain_text_dataset(_text_dataset("content")) is True
assert (
format_detection.is_plain_text_dataset(Dataset.from_list([{"text": "a", "id": 1}])) is False
)
@pytest.mark.parametrize("column", ["text", "content"], ids = ["text", "content"])
def test_auto_resolves_a_single_string_column_to_raw_text(column):
info = format_dataset(_text_dataset(column), format_type = "auto")
assert info["final_format"] == "raw_text"
assert info["requires_manual_mapping"] is False
assert info["dataset"].column_names == ["text"]
assert info["dataset"]["text"] == [
"## What happened\nWrote the fix.",
"## What happened\nReviewed the diff.",
]
def test_renamed_column_is_reported():
info = format_dataset(_text_dataset("content"), format_type = "auto")
assert any("content" in warning for warning in info["warnings"])
def test_auto_leaves_a_mapped_dataset_alone():
info = format_dataset(
Dataset.from_list([{"text": "a", "id": 1}]),
format_type = "auto",
custom_format_mapping = {"text": "user"},
)
assert info["final_format"] != "raw_text"
@pytest.mark.parametrize("column", ["text", "content"], ids = ["text", "content"])
def test_auto_raw_text_rows_end_in_eos(column):
tokenizer = SimpleNamespace(eos_token = "</s>")
train = format_and_template_dataset(
_text_dataset(column), model_name = "unsloth/test", tokenizer = tokenizer
)
eval_split = format_and_template_dataset(
_text_dataset(column),
model_name = "unsloth/test",
tokenizer = tokenizer,
split_name = "eval",
raw_text_column = train["raw_text_column"],
)
for info in (train, eval_split):
assert info["final_format"] == "raw_text"
assert info["dataset"]["text"] == [
"## What happened\nWrote the fix.</s>",
"## What happened\nReviewed the diff.</s>",
]
def test_auto_raw_text_reads_eos_from_a_processors_tokenizer():
processor = SimpleNamespace(tokenizer = SimpleNamespace(eos_token = "<|im_end|>"))
info = format_and_template_dataset(
_text_dataset(), model_name = "unsloth/test", tokenizer = processor
)
assert all(text.endswith("<|im_end|>") for text in info["dataset"]["text"])
def test_explicit_raw_leaves_eos_to_the_trainer():
info = format_and_template_dataset(
_text_dataset(),
model_name = "unsloth/test",
tokenizer = SimpleNamespace(eos_token = "</s>"),
format_type = "raw",
)
assert not any(text.endswith("</s>") for text in info["dataset"]["text"])