Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Legacy CLI: apply --chunk_size and --stride to a text file passed as --dataset (#16424)
Vineeth Sai Varikuntla047030f4ca4a
diff --git a/tests/test_cli_dataset_file_chunk_flags.py b/tests/test_cli_dataset_file_chunk_flags.pynew file mode 100644index 00000000000..c56495a4ec2--- /dev/null+++ b/tests/test_cli_dataset_file_chunk_flags.py@@ -0,0 +1,141 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""unsloth-cli.py: --chunk_size and --stride also apply to a text file passed as --dataset."""++import argparse+import importlib.util+import sys+import types+from pathlib import Path++import torch++ROOT = Path(__file__).resolve().parents[1]+RAW_TEXT_PATH = ROOT / "unsloth" / "dataprep" / "raw_text.py"+++class _Tokenizer:+ eos_token = "</s>"+ eos_token_id = 0++ def __call__(self, text, **kwargs):+ # ids from 1: EOS is 0.+ ids = torch.arange(1, len(text.split()) + 1)+ return {"input_ids": ids.unsqueeze(0)}+++def _run_cli(monkeypatch, **overrides):+ class _Dataset:+ @staticmethod+ def from_dict(data):+ return data++ datasets = types.ModuleType("datasets")+ datasets.Dataset, datasets.load_dataset = _Dataset, None+ # transformers' find_spec("datasets") raises on a module without __spec__.+ datasets.__spec__ = importlib.util.spec_from_loader("datasets", loader = None)+ monkeypatch.setitem(sys.modules, "datasets", datasets)+ raw_text = types.ModuleType("raw_text_under_test")+ source = RAW_TEXT_PATH.read_text(encoding = "utf-8")+ exec(compile(source, str(RAW_TEXT_PATH), "exec"), raw_text.__dict__)++ trained = {}++ class _Trainer:+ def __init__(self, **kwargs):+ trained.update(kwargs)++ def train(self):+ pass++ class _Model:+ @staticmethod+ def from_pretrained(**kwargs):+ return object(), _Tokenizer()++ @staticmethod+ def get_peft_model(model, **kwargs):+ return model++ unsloth = types.ModuleType("unsloth")+ unsloth.FastLanguageModel = _Model+ unsloth.is_bfloat16_supported = lambda: True+ unsloth.RawTextDataLoader = raw_text.RawTextDataLoader+ models = types.ModuleType("unsloth.models")+ loader_utils = types.ModuleType("unsloth.models.loader_utils")+ loader_utils.prepare_device_map = lambda: (None, False)+ trl = types.ModuleType("trl")+ trl.SFTTrainer = _Trainer+ trl.SFTConfig = lambda **kwargs: types.SimpleNamespace(**kwargs)+ for name, module in (+ ("unsloth", unsloth),+ ("unsloth.models", models),+ ("unsloth.models.loader_utils", loader_utils),+ ("trl", trl),+ ):+ monkeypatch.setitem(sys.modules, name, module)++ path = ROOT / "unsloth-cli.py"+ spec = importlib.util.spec_from_file_location("unsloth_cli_chunk_flags", path)+ cli = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(cli)++ args = dict(+ model_name = "m",+ max_seq_length = 32,+ dtype = None,+ load_in_4bit = False,+ dataset = "unused",+ raw_text_file = None,+ chunk_size = 32,+ stride = 8,+ r = 8,+ lora_alpha = 8,+ lora_dropout = 0.0,+ bias = "none",+ use_gradient_checkpointing = "unsloth",+ random_state = 0,+ use_rslora = False,+ loftq_config = None,+ use_dora = False,+ per_device_train_batch_size = 1,+ gradient_accumulation_steps = 1,+ warmup_steps = 0,+ max_steps = 1,+ learning_rate = 1e-4,+ logging_steps = 1,+ optim = "adamw_torch",+ weight_decay = 0.0,+ lr_scheduler_type = "linear",+ seed = 0,+ output_dir = "unused",+ report_to = "none",+ packing = False,+ per_device_eval_batch_size = 4,+ context_parallel_size = 1,+ save_model = False,+ )+ args.update(overrides)+ cli.run(argparse.Namespace(**args))+ return trained["train_dataset"]+++def _longest_row(dataset):+ return max(len(ids) for ids in dataset["input_ids"])+++def test_dataset_text_file_honours_chunk_size_and_stride(tmp_path, monkeypatch):+ path = tmp_path / "corpus.txt"+ path.write_text(" ".join(f"w{i}" for i in range(100)), encoding = "utf-8")+ dataset = _run_cli(monkeypatch, dataset = str(path), chunk_size = 32, stride = 8)+ # 101 tokens at chunk 32 / step 24 = 4 windows; the ignored-flag default (2048) gives 1.+ assert len(dataset["input_ids"]) == 4+ assert _longest_row(dataset) <= 32+++def test_raw_text_file_flag_still_honours_chunk_size(tmp_path, monkeypatch):+ path = tmp_path / "corpus.txt"+ path.write_text(" ".join(f"w{i}" for i in range(100)), encoding = "utf-8")+ dataset = _run_cli(monkeypatch, raw_text_file = str(path), chunk_size = 32, stride = 8)+ assert _longest_row(dataset) <= 32diff --git a/unsloth-cli.py b/unsloth-cli.pyindex 92a85e7e2e3..a50cb011009 100644--- a/unsloth-cli.py+++ b/unsloth-cli.py@@ -80,6 +80,13 @@ def _raw_text_loader_for_backend()+def _load_raw_text_file(RawTextDataLoader, tokenizer, is_mlx, args, path):+ loader = _raw_text_loader_for_backend(+ RawTextDataLoader, tokenizer, is_mlx, args.chunk_size, args.stride+ )+ return loader.load_from_file(path)++def _train_with_legacy_save_control(trainer, is_mlx):if not is_mlx:return trainer.train()@@ -279,13 +286,11 @@ def run(args):def load_dataset_smart(args):from transformers.utils import strtoboolif args.raw_text_file:- loader = _raw_text_loader_for_backend(- RawTextDataLoader, tokenizer, is_mlx, args.chunk_size, args.stride+ dataset = _load_raw_text_file(+ RawTextDataLoader, tokenizer, is_mlx, args, args.raw_text_file)- dataset = loader.load_from_file(args.raw_text_file)elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")):- loader = _raw_text_loader_for_backend(RawTextDataLoader, tokenizer, is_mlx)- dataset = loader.load_from_file(args.dataset)+ dataset = _load_raw_text_file(RawTextDataLoader, tokenizer, is_mlx, args, args.dataset)else:use_modelscope = strtobool(os.environ.get("UNSLOTH_USE_MODELSCOPE", "False"))if use_modelscope: