Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import pytest
from datasets import Dataset, IterableDataset
from utils.datasets.raw_text import prepare_raw_text_dataset
def test_streaming_raw_text_drops_invalid_prefix_lazily_then_appends_eos():
visited: list[int] = []
def rows():
for index, text in enumerate([None, " ", "valid"]):
visited.append(index)
yield {"text": text}
result = prepare_raw_text_dataset(
IterableDataset.from_generator(rows),
mode_label = "CPT",
split_name = "train",
eos_token = "<eos>",
append_eos = True,
)
# Column discovery reads one row; the filter must not scan further until training.
assert visited == [0]
assert next(iter(result.dataset))["text"] == "valid<eos>"
@pytest.mark.parametrize("streaming", [False, True])
def test_raw_blanks_are_dropped_before_eos_append(streaming):
texts = ["", " ", None, "hello", "\n", "world"]
dataset = (
IterableDataset.from_generator(lambda: ({"text": text} for text in texts))
if streaming
else Dataset.from_dict({"text": texts})
)
result = prepare_raw_text_dataset(
dataset,
mode_label = "CPT",
split_name = "train",
eos_token = "<eos>",
append_eos = True,
)
assert [row["text"] for row in result.dataset] == ["hello<eos>", "world<eos>"]
assert any("blank" in notice.message for notice in result.notices)
def test_all_blank_raw_eval_split_is_left_for_the_trainer_to_skip():
result = prepare_raw_text_dataset(
Dataset.from_dict({"text": ["", " "]}),
mode_label = "CPT",
split_name = "eval",
eos_token = "<eos>",
append_eos = True,
)
assert len(result.dataset) == 0
assert any("blank" in notice.message for notice in result.notices)
def test_all_blank_raw_train_split_still_fails():
with pytest.raises(ValueError, match = "at least one non-blank string"):
prepare_raw_text_dataset(Dataset.from_dict({"text": ["", " "]}), split_name = "train")
def test_shared_formatter_passes_the_eval_split_to_raw_prep():
from utils.datasets import format_and_template_dataset
blank = Dataset.from_dict({"text": ["", " "]})
kwargs = dict(model_name = "m", tokenizer = None, format_type = "raw")
ev = format_and_template_dataset(blank, split_name = "eval", **kwargs)
assert ev["success"] and len(ev["dataset"]) == 0
with pytest.raises(ValueError, match = "at least one non-blank string"):
format_and_template_dataset(blank, **kwargs)