Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""A wrong online-tokenization gate either crashes (VLM, pre-tokenized) or trains on different rows."""
import sys
from types import SimpleNamespace
import pytest
sys.path.insert(0, "studio/backend")
from utils.datasets.online_tokenization import ( # noqa: E402
ENV_FLAG,
MIN_ROWS_FOR_ONLINE,
TRUNCATION_ATTESTATION_ATTR,
OnlineTokenizationDecision,
attach_online_tokenization,
build_tokenizing_transform,
dataset_column_names,
dataset_supports_with_transform,
decide_online_tokenization,
env_override,
is_processor,
online_config_args,
prewarm_batch_count,
resolve_add_special_tokens,
text_column_defect,
trl_supports_skip_prepare_dataset,
)
datasets = pytest.importorskip("datasets")
ROWS = MIN_ROWS_FOR_ONLINE + 5
def _text_dataset(n = ROWS, extra_columns = None):
data = {
"text": [f"row {i}" for i in range(n)],
"conversations": [[{"role": "user", "content": str(i)}] for i in range(n)],
}
data.update(extra_columns or {})
return datasets.Dataset.from_dict(data)
class _Tokenizer:
"""The narrowest thing the online path needs: callable, no ``.tokenizer``."""
bos_token = "<s>"
chat_template = "{{ messages }}"
def __call__(
self,
texts,
truncation = True,
max_length = 8,
add_special_tokens = True,
):
if isinstance(texts, str):
texts = [texts]
ids = [[len(t)] * min(len(t), max_length if truncation else len(t)) for t in texts]
return {"input_ids": ids}
class _Processor(_Tokenizer):
tokenizer = _Tokenizer()
def _base_kwargs(**overrides):
kwargs = dict(
dataset = _text_dataset(),
eval_dataset = None,
processing_class = _Tokenizer(),
model = SimpleNamespace(),
text_field = "text",
packing = False,
num_train_epochs = 1,
max_steps = 0,
grad_accum = 4,
workers = 4,
)
kwargs.update(overrides)
return kwargs
@pytest.fixture(autouse = True)
def _no_env_override(monkeypatch):
monkeypatch.delenv(ENV_FLAG, raising = False)
# The gate refuses on spawn platforms; other platforms are simulated explicitly.
monkeypatch.setattr(sys, "platform", "linux")
# Pin the TRL hook: the CPU job has no TRL. The detector is covered separately.
monkeypatch.setattr(
"utils.datasets.online_tokenization.trl_supports_skip_prepare_dataset",
lambda: True,
)
def test_plain_text_single_epoch_run_goes_online():
decision = decide_online_tokenization(**_base_kwargs())
assert decision.enabled, decision.reason
assert decision.workers == 4
assert decision.prewarm_batches == max(4, 4 * decision.prefetch_factor)
def test_online_config_args_are_the_four_keys_the_mechanism_needs():
decision = decide_online_tokenization(**_base_kwargs())
args = online_config_args(decision)
assert args["dataset_kwargs"] == {"skip_prepare_dataset": True}
# _remove_unused_columns reads the backing table and would strip the transform's text column.
assert args["remove_unused_columns"] is False
assert args["dataloader_num_workers"] == 4
assert args["dataloader_prefetch_factor"] > 0
assert args["dataloader_persistent_workers"] is True
@pytest.mark.parametrize("platform", ["win32", "darwin"])
def test_spawn_platforms_keep_the_eager_path(monkeypatch, platform):
"""Unsloth already forces `dataloader_num_workers = 0` there, because a
modified `sys.path` does not survive the spawn. Never a crash: just off."""
monkeypatch.setattr(sys, "platform", platform)
decision = decide_online_tokenization(**_base_kwargs())
assert not decision.enabled
assert platform in decision.reason
@pytest.mark.parametrize(
"flag, reason_fragment",
[
({"is_vlm": True}, "multimodal"),
({"is_audio_vlm": True}, "multimodal"),
({"is_deepseek_ocr": True}, "multimodal"),
({"is_audio": True}, "audio"),
({"is_cpt": True}, "continued pretraining"),
({"raw_text_mode": True}, "raw-text"),
({"has_custom_collator": True}, "custom data collator"),
({"packing": True}, "packing"),
({"train_on_completions": True}, "train on completions"),
({"dataset_streaming": True}, "streaming"),
],
)
def test_each_excluded_shape_takes_the_old_path(flag, reason_fragment):
decision = decide_online_tokenization(**_base_kwargs(**flag))
assert not decision.enabled
assert reason_fragment in decision.reason
def test_streaming_dataset_object_is_refused_even_without_the_flag():
"""`IterableDataset` also has `with_transform` in recent `datasets`, so the
check is an isinstance and not a `hasattr`."""
stream = datasets.Dataset.from_dict({"text": ["a", "b"]}).to_iterable_dataset()
decision = decide_online_tokenization(**_base_kwargs(dataset = stream))
assert not decision.enabled
assert "map-style" in decision.reason
def test_a_plain_list_dataset_is_refused():
decision = decide_online_tokenization(**_base_kwargs(dataset = [{"text": "a"}] * ROWS))
assert not decision.enabled
assert "map-style" in decision.reason
@pytest.mark.parametrize("column", ["input_ids", "labels", "prompt", "completion"])
def test_an_already_tokenized_dataset_is_refused(column):
dataset = _text_dataset(extra_columns = {column: [[1, 2, 3]] * ROWS})
decision = decide_online_tokenization(**_base_kwargs(dataset = dataset))
assert not decision.enabled
assert column in decision.reason
def test_a_processor_is_refused():
decision = decide_online_tokenization(**_base_kwargs(processing_class = _Processor()))
assert not decision.enabled
assert "processor" in decision.reason
def test_a_model_needing_token_type_ids_is_refused(monkeypatch):
"""Gemma-family modules build their causal mask from `token_type_ids`, and
the zoo's tokenize asks for them. Rather than reproduce that column lazily,
those models keep the eager path."""
module = SimpleNamespace(**{"create_" + "causal_mask_mapping": lambda: None})
monkeypatch.setitem(sys.modules, "fake_gemma_modelling", module)
class _GemmaLike:
pass
_GemmaLike.__module__ = "fake_gemma_modelling"
decision = decide_online_tokenization(**_base_kwargs(model = _GemmaLike()))
assert not decision.enabled
assert "token_type_ids" in decision.reason
def test_missing_text_column_is_refused():
dataset = datasets.Dataset.from_dict({"conversations": [[]] * ROWS})
decision = decide_online_tokenization(**_base_kwargs(dataset = dataset))
assert not decision.enabled
assert "text" in decision.reason
def test_a_null_text_row_is_refused():
"""One None text row crashes the eager map at construction; the lazy view dies hours in."""
texts = [f"row {i}" for i in range(ROWS)]
texts[137] = None
dataset = datasets.Dataset.from_dict({"text": texts})
decision = decide_online_tokenization(**_base_kwargs(dataset = dataset))
assert not decision.enabled
assert "null" in decision.reason
@pytest.mark.parametrize(
"column",
[
[7] * ROWS,
[[f"row {i}"] for i in range(ROWS)],
[{"content": "x"}] * ROWS,
],
ids = ["ints", "lists", "structs"],
)
def test_a_text_column_that_is_not_strings_is_refused(column):
dataset = datasets.Dataset.from_dict({"text": column})
decision = decide_online_tokenization(**_base_kwargs(dataset = dataset))
assert not decision.enabled
assert "not strings" in decision.reason
def test_a_null_text_row_in_the_eval_split_is_refused():
texts = [f"row {i}" for i in range(64)]
texts[7] = None
eval_dataset = datasets.Dataset.from_dict({"text": texts})
decision = decide_online_tokenization(**_base_kwargs(eval_dataset = eval_dataset))
assert not decision.enabled
assert "eval split" in decision.reason and "null" in decision.reason
def test_the_text_column_check_reads_metadata_and_never_a_row():
"""The gate uses schema and Arrow null counts, never a row; reading rows is the eager pass."""
class _Unreadable(type(_text_dataset(16))):
def __getitem__(self, key):
raise AssertionError("the gate read a row")
dataset = _text_dataset()
unreadable = _Unreadable(dataset.data, info = dataset.info)
assert text_column_defect(unreadable, "text") is None
def test_a_spawn_start_method_keeps_the_eager_path_on_linux(monkeypatch):
"""The gate is named for Windows and macOS, but the hazard it describes is
`spawn` re-importing the entry point against a `sys.path` Unsloth modified in
process. A Linux host set to spawn is the same hazard."""
import multiprocessing
monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none = False: "spawn")
decision = decide_online_tokenization(**_base_kwargs())
assert not decision.enabled
assert "spawn" in decision.reason and "fork" in decision.reason
def test_an_unset_start_method_falls_back_to_the_platform_default(monkeypatch):
"""`get_start_method()` with no argument pins the context and makes a later
`set_start_method()` raise, so the default is read off the method list."""
import multiprocessing
monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none = False: None)
monkeypatch.setattr(multiprocessing, "get_all_start_methods", lambda: ["fork", "spawn"])
assert decide_online_tokenization(**_base_kwargs()).enabled
monkeypatch.setattr(multiprocessing, "get_all_start_methods", lambda: ["forkserver"])
assert not decide_online_tokenization(**_base_kwargs()).enabled
def test_a_trl_without_the_hook_keeps_the_eager_path(monkeypatch):
"""The veto the autouse fixture pins away, exercised on its own: without
`skip_prepare_dataset` TRL would run its own tokenizing map over the lazy
view, which is the whole pass the online path exists to avoid."""
monkeypatch.setattr(
"utils.datasets.online_tokenization.trl_supports_skip_prepare_dataset",
lambda: False,
)
decision = decide_online_tokenization(**_base_kwargs())
assert not decision.enabled
assert "skip_prepare_dataset" in decision.reason
def test_the_hook_detector_reads_the_installed_trl(monkeypatch):
"""No TRL means no SFT run, so the detector says no rather than raising; a
`SFTConfig` without `dataset_kwargs` says no too, since the key would be
dropped and TRL would tokenize the view."""
import builtins
real_import = builtins.__import__
def no_trl(name, *args, **kwargs):
if name == "trl" or name.startswith("trl."):
raise ImportError("No module named 'trl'")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", no_trl)
assert trl_supports_skip_prepare_dataset() is False
def test_too_few_workers_is_refused():
decision = decide_online_tokenization(**_base_kwargs(workers = 1))
assert not decision.enabled
assert "workers" in decision.reason
def test_a_small_dataset_keeps_the_eager_path():
decision = decide_online_tokenization(**_base_kwargs(dataset = _text_dataset(100)))
assert not decision.enabled
assert "smaller than" in decision.reason
def test_multi_epoch_runs_keep_the_eager_path():
"""The lazy view re-tokenizes every pass, so anything past a single pass stays eager."""
decision = decide_online_tokenization(**_base_kwargs(num_train_epochs = 3))
assert not decision.enabled
assert "one pass" in decision.reason
def test_a_step_capped_run_of_unknown_length_keeps_the_eager_path():
decision = decide_online_tokenization(**_base_kwargs(max_steps = 500))
assert not decision.enabled
assert "unknown length" in decision.reason
def test_a_resolved_sub_epoch_step_cap_may_go_online():
"""max_steps alone says nothing about passes; a resolved sub-epoch fraction answers it."""
decision = decide_online_tokenization(
**_base_kwargs(max_steps = 60, resolved_max_steps_epochs = 0.02)
)
assert decision.enabled, decision.reason
def test_a_raw_eval_split_is_transformed_alongside_the_train_split():
decision = decide_online_tokenization(**_base_kwargs(eval_dataset = _text_dataset(64)))
assert decision.enabled, decision.reason
def test_an_eval_split_the_transform_cannot_serve_disables_the_feature():
"""`skip_prepare_dataset` skips the EVAL prep too, so an eval split the
online path cannot tokenize would reach the model as raw text."""
bad_eval = datasets.Dataset.from_dict({"something_else": ["x"] * 8})
decision = decide_online_tokenization(**_base_kwargs(eval_dataset = bad_eval))
assert not decision.enabled
assert "eval" in decision.reason
def test_an_already_tokenized_eval_split_disables_the_feature():
tokenized_eval = datasets.Dataset.from_dict({"text": ["x"] * 8, "input_ids": [[1, 2]] * 8})
decision = decide_online_tokenization(**_base_kwargs(eval_dataset = tokenized_eval))
assert not decision.enabled
assert "eval" in decision.reason
def test_env_flag_zero_forces_the_eager_path(monkeypatch):
monkeypatch.setenv(ENV_FLAG, "0")
assert env_override() is False
decision = decide_online_tokenization(**_base_kwargs())
assert not decision.enabled
assert ENV_FLAG in decision.reason
def test_env_flag_one_overrides_the_cost_gates_only(monkeypatch):
monkeypatch.setenv(ENV_FLAG, "1")
forced = decide_online_tokenization(
**_base_kwargs(dataset = _text_dataset(10), num_train_epochs = 5)
)
assert forced.enabled, forced.reason
assert not decide_online_tokenization(**_base_kwargs(is_vlm = True)).enabled
def test_an_unrecognised_env_value_is_not_an_override(monkeypatch):
monkeypatch.setenv(ENV_FLAG, "maybe")
assert env_override() is None
assert decide_online_tokenization(**_base_kwargs()).enabled
def test_the_transform_returns_input_ids_for_the_whole_batch():
transform = build_tokenizing_transform(_Tokenizer(), "text", 8, True)
out = transform({"text": ["abc", "de"]})
assert "input_ids" in out
assert len(out["input_ids"]) == 2
def test_the_transform_passes_the_whole_tokenizer_output_through():
"""The eager map keeps `attention_mask` too (`remove_columns` drops only the
ORIGINAL columns), and both the collator and the attention dispatcher branch
on which keys are present."""
class _WithMask(_Tokenizer):
def __call__(self, texts, **kwargs):
out = super().__call__(texts, **kwargs)
out["attention_mask"] = [[1] * len(ids) for ids in out["input_ids"]]
return out
transform = build_tokenizing_transform(_WithMask(), "text", 8, True)
out = transform({"text": ["abc", "de"]})
assert sorted(out) == ["attention_mask", "input_ids"]
def test_the_view_is_immutable_and_leaves_the_original_alone():
"""`with_transform`, never `set_transform`: the caller's object is also held
by the preview and the row-count checks."""
dataset = _text_dataset(32)
view = attach_online_tokenization(
dataset,
tokenizer = _Tokenizer(),
text_field = "text",
max_length = 8,
add_special_tokens = True,
)
assert view is not dataset
assert "input_ids" in view[0]
assert "input_ids" not in dataset[0]
assert dataset[0]["text"] == "row 0"
def test_the_view_yields_the_same_row_count_and_order():
dataset = _text_dataset(32)
view = attach_online_tokenization(
dataset,
tokenizer = _Tokenizer(),
text_field = "text",
max_length = 8,
add_special_tokens = True,
)
assert len(view) == len(dataset)
assert view[5]["input_ids"] == _Tokenizer()(["row 5"], max_length = 8)["input_ids"][0]
def test_the_view_attests_its_truncation_width():
"""max_length enforcement reads this, since scanning a lazy split re-tokenizes every row."""
view = attach_online_tokenization(
_text_dataset(32),
tokenizer = _Tokenizer(),
text_field = "text",
max_length = 1234,
add_special_tokens = True,
)
assert view.__dict__[TRUNCATION_ATTESTATION_ATTR] == 1234
def test_the_transformed_view_still_reports_its_backing_columns():
"""Pinned because two consumers depend on it: `Trainer._remove_unused_columns`
(hence `remove_unused_columns = False`) and unsloth's tokenized-split probe,
which is why that probe reads a row rather than the metadata."""
view = attach_online_tokenization(
_text_dataset(32),
tokenizer = _Tokenizer(),
text_field = "text",
max_length = 8,
add_special_tokens = True,
)
assert "text" in dataset_column_names(view)
assert "input_ids" not in dataset_column_names(view)
def test_add_special_tokens_is_off_when_the_template_emits_a_bos():
tokenizer = SimpleNamespace(bos_token = "<s>", chat_template = "<s>{{ x }}")
assert resolve_add_special_tokens(tokenizer, "hello") is False
def test_add_special_tokens_is_off_when_the_text_already_starts_with_bos():
tokenizer = SimpleNamespace(bos_token = "<s>", chat_template = "{{ x }}")
assert resolve_add_special_tokens(tokenizer, "<s>hello") is False
def test_add_special_tokens_stays_on_otherwise():
tokenizer = SimpleNamespace(bos_token = "<s>", chat_template = "{{ x }}")
assert resolve_add_special_tokens(tokenizer, "hello") is True
def test_no_bos_token_means_add_special_tokens_stays_on():
tokenizer = SimpleNamespace(bos_token = None, chat_template = "")
assert resolve_add_special_tokens(tokenizer, "hello") is True
@pytest.mark.parametrize(
"grad_accum, workers, prefetch, expected",
[(4, 4, 4, 16), (32, 2, 2, 32), (1, 0, 0, 1), (0, 0, 0, 1)],
)
def test_prewarm_depth_covers_the_first_step_and_the_queue(grad_accum, workers, prefetch, expected):
assert prewarm_batch_count(grad_accum, workers, prefetch) == expected
def test_is_processor_spots_a_wrapped_tokenizer():
assert is_processor(_Processor()) is True
assert is_processor(_Tokenizer()) is False
def test_dataset_supports_with_transform_rejects_none_and_streams():
assert dataset_supports_with_transform(None) is False
assert dataset_supports_with_transform(_text_dataset(4)) is True
def test_a_disabled_decision_never_carries_worker_settings():
decision = OnlineTokenizationDecision(enabled = False, reason = "test")
assert decision.workers == 0
assert decision.prewarm_batches == 0
assert "off" in decision.as_log_line()