Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: train single-string-column datasets as raw text (#15899)
Yakshithffd1a184c683
diff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.pyindex fa558e06c61..0a8507b8cf9 100644--- a/studio/backend/hub/utils/dataset_format.py+++ b/studio/backend/hub/utils/dataset_format.py@@ -7,7 +7,11 @@ import refrom typing import Any, Optionalfrom utils.datasets.cells import text_cell_check, unwrap_message_column-from utils.datasets.format_detection import detect_dataset_format, has_message_prompt_completion+from utils.datasets.format_detection import (+ detect_dataset_format,+ has_message_prompt_completion,+ is_plain_text_dataset,+)from utils.datasets.pretokenized import PRETOKENIZED_FORMAT, is_pretokenized_dataset@@ -734,6 +738,23 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict:"detected_speaker_column": multimodal_info.get("detected_speaker_column"),}+ if is_plain_text_dataset(dataset):+ # Before the VLM branch: text-only data trains as raw text on a vision model too.+ # Media flags off: a name match (a "speech" column) would route an audio model to its audio path.+ return {+ "requires_manual_mapping": False,+ "detected_format": "raw",+ "columns": columns,+ "suggested_mapping": None,+ "detected_image_column": None,+ "detected_text_column": None,+ "is_image": False,+ "multimodal_columns": None,+ "is_audio": False,+ "detected_audio_column": None,+ "detected_speaker_column": None,+ }+if is_vlm:vlm_structure = detect_vlm_dataset_structure(dataset)requires_mapping = vlm_structure["format"] == "unknown"diff --git a/studio/backend/tests/test_plain_text_dataset.py b/studio/backend/tests/test_plain_text_dataset.pynew file mode 100644index 00000000000..8d52d8728bb--- /dev/null+++ b/studio/backend/tests/test_plain_text_dataset.py@@ -0,0 +1,117 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""One string column is raw training text: the preview and the formatter must agree on it."""++import sys+from pathlib import Path++import pytest+from datasets import Dataset++_BACKEND_ROOT = Path(__file__).resolve().parents[1]+if str(_BACKEND_ROOT) not in sys.path:+ sys.path.insert(0, str(_BACKEND_ROOT))++from hub.utils import dataset_format # noqa: E402+from utils.datasets import format_detection # noqa: E402+from utils.datasets.dataset_utils import format_dataset # noqa: E402+++def _text_dataset(column: str = "text"):+ return Dataset.from_list(+ [+ {column: "## What happened\nWrote the fix."},+ {column: "## What happened\nReviewed the diff."},+ ]+ )++[email protected]("column", ["text", "content", "body"], ids = ["text", "content", "body"])[email protected]("is_vlm", [False, True], ids = ["text_model", "vision_model"])+def test_single_string_column_needs_no_mapping(column, is_vlm):+ result = dataset_format.check_dataset_format(_text_dataset(column), is_vlm = is_vlm)++ assert result["requires_manual_mapping"] is False+ assert result["detected_format"] == "raw"+ assert result["suggested_mapping"] is None+ assert result["columns"] == [column]+++def test_text_column_keeps_detection_flags_for_other_branches():+ result = dataset_format.check_dataset_format(_text_dataset())++ assert result["is_image"] is False+ assert result["is_audio"] is False+ assert result["detected_image_column"] is None+++def test_media_named_text_column_is_not_routed_as_media():+ result = dataset_format.check_dataset_format(_text_dataset("speech"), is_vlm = True)++ assert result["detected_format"] == "raw"+ assert result["is_audio"] is False+ assert result["detected_audio_column"] is None+ assert result["is_image"] is False+++ "rows",+ [+ pytest.param([{"text": "a", "id": 1}], id = "text_plus_metadata"),+ pytest.param([{"text": " "}], id = "blank_text"),+ pytest.param([{"text": 5}], id = "non_string_text"),+ ],+)+def test_other_shapes_still_require_mapping(rows):+ result = dataset_format.check_dataset_format(Dataset.from_list(rows))++ assert result["detected_format"] == "unknown"+ assert result["requires_manual_mapping"] is True+++ "row", [{"image": "photos/sloth.jpg"}, {"audio": "clips/sloth.wav"}], ids = ["image", "audio"]+)+def test_media_path_column_is_not_raw_text(row):+ result = dataset_format.check_dataset_format(Dataset.from_list([row]))++ assert result["detected_format"] != "raw"+ assert result["requires_manual_mapping"] is True+++def test_plain_text_dataset_helper_rejects_other_shapes():+ assert format_detection.is_plain_text_dataset(_text_dataset()) is True+ assert format_detection.is_plain_text_dataset(_text_dataset("content")) is True+ assert (+ format_detection.is_plain_text_dataset(Dataset.from_list([{"text": "a", "id": 1}])) is False+ )++[email protected]("column", ["text", "content"], ids = ["text", "content"])+def test_auto_resolves_a_single_string_column_to_raw_text(column):+ info = format_dataset(_text_dataset(column), format_type = "auto")++ assert info["final_format"] == "raw_text"+ assert info["requires_manual_mapping"] is False+ assert info["dataset"].column_names == ["text"]+ assert info["dataset"]["text"] == [+ "## What happened\nWrote the fix.",+ "## What happened\nReviewed the diff.",+ ]+++def test_renamed_column_is_reported():+ info = format_dataset(_text_dataset("content"), format_type = "auto")++ assert any("content" in warning for warning in info["warnings"])+++def test_auto_leaves_a_mapped_dataset_alone():+ info = format_dataset(+ Dataset.from_list([{"text": "a", "id": 1}]),+ format_type = "auto",+ custom_format_mapping = {"text": "user"},+ )++ assert info["final_format"] != "raw_text"diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.pyindex 038f9875e87..eef0f6e24b0 100644--- a/studio/backend/utils/datasets/dataset_utils.py+++ b/studio/backend/utils/datasets/dataset_utils.py@@ -10,6 +10,7 @@ from .format_detection import (detect_multimodal_dataset,detect_vlm_dataset_structure,detect_custom_format_heuristic,+ is_plain_text_dataset,)from .format_conversion import (standardize_chat_format,@@ -435,6 +436,10 @@ def _format_dataset(multimodal_info = detect_multimodal_dataset(dataset)+ if format_type == "auto" and not custom_format_mapping and is_plain_text_dataset(dataset):+ # Matches the preview's "raw" verdict; auto would otherwise fail as "unknown".+ format_type = "raw"+if format_type == "raw":raw_result = prepare_raw_text_dataset(dataset, split_name = split_name, text_column = raw_text_columndiff --git a/studio/backend/utils/datasets/format_detection.py b/studio/backend/utils/datasets/format_detection.pyindex 2ff229bd5d4..30fd5ebd345 100644--- a/studio/backend/utils/datasets/format_detection.py+++ b/studio/backend/utils/datasets/format_detection.py@@ -17,6 +17,7 @@ CONVERSATION_COLUMNS = ("messages", "conversations", "texts")_CHATML_KEYS = frozenset({"role", "content"})_SHAREGPT_KEYS = frozenset({"from", "value"})_TRACE_SUFFIXES = ("__trace", "_trace")+_PLAIN_TEXT_SAMPLE_ROWS = 10def _sample_dataset_rows(dataset, limit: int = 100) -> list[dict]:@@ -159,6 +160,27 @@ def _detect_conversation_column(return None+def is_plain_text_dataset(dataset) -> bool:+ """One column of non-blank strings: raw text, whatever the column is called (prepare_raw_text_dataset renames it)."""+ column_names = getattr(dataset, "column_names", None)+ if column_names is not None and len(list(column_names)) != 1:+ return False+ rows = _sample_dataset_rows(dataset, limit = _PLAIN_TEXT_SAMPLE_ROWS)+ if not rows:+ return False+ return all(_is_single_string_row(row) for row in rows)+++def _is_single_string_row(row) -> bool:+ if not isinstance(row, dict) or len(row) != 1:+ return False+ value = next(iter(row.values()))+ if not isinstance(value, str) or not value.strip() or _is_image_value(value):+ return False+ # A column of media paths still needs its other roles mapped.+ return not value.strip().lower().endswith(_AUDIO_EXTENSIONS)++def detect_dataset_format(dataset):"""Detect dataset format by inspecting structure: {"format": alpaca/sharegpt/chatml/unknown, "chat_column": str or None, "needs_standardization": bool, "sample_keys": keys found in messages}."""sample_rows = _sample_dataset_rows(dataset)