Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Legacy CLI: apply --chunk_size and --stride to a text file passed as --dataset (#16424)

Vineeth Sai Varikuntla047030f4ca4a
diff --git a/tests/test_cli_dataset_file_chunk_flags.py b/tests/test_cli_dataset_file_chunk_flags.py
new file mode 100644
index 00000000000..c56495a4ec2
--- /dev/null
+++ b/tests/test_cli_dataset_file_chunk_flags.py
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""unsloth-cli.py: --chunk_size and --stride also apply to a text file passed as --dataset."""
+
+import argparse
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import torch
+
+ROOT = Path(__file__).resolve().parents[1]
+RAW_TEXT_PATH = ROOT / "unsloth" / "dataprep" / "raw_text.py"
+
+
+class _Tokenizer:
+ eos_token = "</s>"
+ eos_token_id = 0
+
+ def __call__(self, text, **kwargs):
+ # ids from 1: EOS is 0.
+ ids = torch.arange(1, len(text.split()) + 1)
+ return {"input_ids": ids.unsqueeze(0)}
+
+
+def _run_cli(monkeypatch, **overrides):
+ class _Dataset:
+ @staticmethod
+ def from_dict(data):
+ return data
+
+ datasets = types.ModuleType("datasets")
+ datasets.Dataset, datasets.load_dataset = _Dataset, None
+ # transformers' find_spec("datasets") raises on a module without __spec__.
+ datasets.__spec__ = importlib.util.spec_from_loader("datasets", loader = None)
+ monkeypatch.setitem(sys.modules, "datasets", datasets)
+ raw_text = types.ModuleType("raw_text_under_test")
+ source = RAW_TEXT_PATH.read_text(encoding = "utf-8")
+ exec(compile(source, str(RAW_TEXT_PATH), "exec"), raw_text.__dict__)
+
+ trained = {}
+
+ class _Trainer:
+ def __init__(self, **kwargs):
+ trained.update(kwargs)
+
+ def train(self):
+ pass
+
+ class _Model:
+ @staticmethod
+ def from_pretrained(**kwargs):
+ return object(), _Tokenizer()
+
+ @staticmethod
+ def get_peft_model(model, **kwargs):
+ return model
+
+ unsloth = types.ModuleType("unsloth")
+ unsloth.FastLanguageModel = _Model
+ unsloth.is_bfloat16_supported = lambda: True
+ unsloth.RawTextDataLoader = raw_text.RawTextDataLoader
+ models = types.ModuleType("unsloth.models")
+ loader_utils = types.ModuleType("unsloth.models.loader_utils")
+ loader_utils.prepare_device_map = lambda: (None, False)
+ trl = types.ModuleType("trl")
+ trl.SFTTrainer = _Trainer
+ trl.SFTConfig = lambda **kwargs: types.SimpleNamespace(**kwargs)
+ for name, module in (
+ ("unsloth", unsloth),
+ ("unsloth.models", models),
+ ("unsloth.models.loader_utils", loader_utils),
+ ("trl", trl),
+ ):
+ monkeypatch.setitem(sys.modules, name, module)
+
+ path = ROOT / "unsloth-cli.py"
+ spec = importlib.util.spec_from_file_location("unsloth_cli_chunk_flags", path)
+ cli = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(cli)
+
+ args = dict(
+ model_name = "m",
+ max_seq_length = 32,
+ dtype = None,
+ load_in_4bit = False,
+ dataset = "unused",
+ raw_text_file = None,
+ chunk_size = 32,
+ stride = 8,
+ r = 8,
+ lora_alpha = 8,
+ lora_dropout = 0.0,
+ bias = "none",
+ use_gradient_checkpointing = "unsloth",
+ random_state = 0,
+ use_rslora = False,
+ loftq_config = None,
+ use_dora = False,
+ per_device_train_batch_size = 1,
+ gradient_accumulation_steps = 1,
+ warmup_steps = 0,
+ max_steps = 1,
+ learning_rate = 1e-4,
+ logging_steps = 1,
+ optim = "adamw_torch",
+ weight_decay = 0.0,
+ lr_scheduler_type = "linear",
+ seed = 0,
+ output_dir = "unused",
+ report_to = "none",
+ packing = False,
+ per_device_eval_batch_size = 4,
+ context_parallel_size = 1,
+ save_model = False,
+ )
+ args.update(overrides)
+ cli.run(argparse.Namespace(**args))
+ return trained["train_dataset"]
+
+
+def _longest_row(dataset):
+ return max(len(ids) for ids in dataset["input_ids"])
+
+
+def test_dataset_text_file_honours_chunk_size_and_stride(tmp_path, monkeypatch):
+ path = tmp_path / "corpus.txt"
+ path.write_text(" ".join(f"w{i}" for i in range(100)), encoding = "utf-8")
+ dataset = _run_cli(monkeypatch, dataset = str(path), chunk_size = 32, stride = 8)
+ # 101 tokens at chunk 32 / step 24 = 4 windows; the ignored-flag default (2048) gives 1.
+ assert len(dataset["input_ids"]) == 4
+ assert _longest_row(dataset) <= 32
+
+
+def test_raw_text_file_flag_still_honours_chunk_size(tmp_path, monkeypatch):
+ path = tmp_path / "corpus.txt"
+ path.write_text(" ".join(f"w{i}" for i in range(100)), encoding = "utf-8")
+ dataset = _run_cli(monkeypatch, raw_text_file = str(path), chunk_size = 32, stride = 8)
+ assert _longest_row(dataset) <= 32
diff --git a/unsloth-cli.py b/unsloth-cli.py
index 92a85e7e2e3..a50cb011009 100644
--- a/unsloth-cli.py
+++ b/unsloth-cli.py
@@ -80,6 +80,13 @@ def _raw_text_loader_for_backend(
)
+def _load_raw_text_file(RawTextDataLoader, tokenizer, is_mlx, args, path):
+ loader = _raw_text_loader_for_backend(
+ RawTextDataLoader, tokenizer, is_mlx, args.chunk_size, args.stride
+ )
+ return loader.load_from_file(path)
+
+
def _train_with_legacy_save_control(trainer, is_mlx):
if not is_mlx:
return trainer.train()
@@ -279,13 +286,11 @@ def run(args):
def load_dataset_smart(args):
from transformers.utils import strtobool
if args.raw_text_file:
- loader = _raw_text_loader_for_backend(
- RawTextDataLoader, tokenizer, is_mlx, args.chunk_size, args.stride
+ dataset = _load_raw_text_file(
+ RawTextDataLoader, tokenizer, is_mlx, args, args.raw_text_file
)
- dataset = loader.load_from_file(args.raw_text_file)
elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")):
- loader = _raw_text_loader_for_backend(RawTextDataLoader, tokenizer, is_mlx)
- dataset = loader.load_from_file(args.dataset)
+ dataset = _load_raw_text_file(RawTextDataLoader, tokenizer, is_mlx, args, args.dataset)
else:
use_modelscope = strtobool(os.environ.get("UNSLOTH_USE_MODELSCOPE", "False"))
if use_modelscope: