Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

CPU backend: export load_ebook_dataset, RawTextDataLoader and TextPreprocessor (#15501)

Daniel Han3bce1d107202
diff --git a/tests/test_cpu_backend_init.py b/tests/test_cpu_backend_init.py
index 24a8c5c7966..d5ccaa4abd0 100644
--- a/tests/test_cpu_backend_init.py
+++ b/tests/test_cpu_backend_init.py
@@ -40,6 +40,7 @@ print("PROBE " + json.dumps({
"fast": [c.__name__ for c in unsloth.FastLanguageModel.__mro__],
"modules": mods,
"trainer_file": getattr(sys.modules.get("unsloth.trainer"), "__file__", None),
+ "dataprep": [n for n in ("load_ebook_dataset", "RawTextDataLoader", "TextPreprocessor") if hasattr(unsloth, n)],
}))
"""
@@ -100,6 +101,55 @@ def test_no_accelerator_loads_the_cpu_backend():
assert (
probe["trainer_file"] is None
), "unsloth.trainer must be the CPU shim, not unsloth/trainer.py"
+ # The dataprep helpers the GPU backend exports, without loading synthetic.py.
+ assert probe["dataprep"] == ["load_ebook_dataset", "RawTextDataLoader", "TextPreprocessor"]
+
+
+_PICKLE_PROBE = r"""
+import json, multiprocessing, pickle, sys
+import unsloth
+from unsloth import RawTextDataLoader, TextPreprocessor, load_ebook_dataset
+objs = [TextPreprocessor(), RawTextDataLoader, load_ebook_dataset]
+roundtrip = [pickle.loads(pickle.dumps(o)) is not None for o in objs]
+# A spawned worker (datasets num_proc / DataLoader workers on macOS) imports the classes back fresh.
+with multiprocessing.get_context("spawn").Pool(1) as pool:
+ spawned = pool.apply(len, (objs,))
+print("PROBE " + json.dumps({
+ "device_type": unsloth.DEVICE_TYPE,
+ "roundtrip": roundtrip,
+ "spawned": spawned,
+ "synthetic": "unsloth.dataprep.synthetic" in sys.modules,
+}))
+"""
+
+
[email protected](not _zoo_has_cpu_backend(), reason = "unsloth_zoo without the CPU backend")
+def test_cpu_dataprep_exports_pickle_into_spawned_workers():
+ pytest.importorskip("torch")
+ env = {
+ k: v
+ for k, v in os.environ.items()
+ if k not in ("UNSLOTH_ALLOW_CPU", "UNSLOTH_DISABLE_CPU_FALLBACK")
+ }
+ env.update(
+ {
+ "UNSLOTH_FORCE_CPU": "1",
+ "UNSLOTH_FORCE_GPU_PATH": "1",
+ "PYTHONPATH": os.pathsep.join(filter(None, [str(_ROOT), env.get("PYTHONPATH")])),
+ }
+ )
+ out = subprocess.run(
+ [sys.executable, "-c", _PICKLE_PROBE], capture_output = True, text = True, env = env, timeout = 1200
+ )
+ probe = next(
+ (json.loads(line[6:]) for line in out.stdout.splitlines() if line.startswith("PROBE ")),
+ None,
+ )
+ assert probe is not None, out.stderr[-3000:]
+ assert probe["device_type"] == "cpu"
+ assert probe["roundtrip"] == [True, True, True]
+ assert probe["spawned"] == 3
+ assert probe["synthetic"] is False
def test_gpu_host_never_loads_the_cpu_backend():
diff --git a/unsloth/_cpu_init.py b/unsloth/_cpu_init.py
index dc72fb420f6..8a92329875c 100644
--- a/unsloth/_cpu_init.py
+++ b/unsloth/_cpu_init.py
@@ -193,6 +193,34 @@ def _install_cpu_trainer_module():
_install_cpu_trainer_module()
del _install_cpu_trainer_module
+
+def _load_dataprep(name: str):
+ # By file: dataprep/__init__ also imports synthetic.py, which needs a GPU device type.
+ import sys
+ import importlib.util
+ from pathlib import Path
+
+ path = Path(__file__).resolve().parent / "dataprep" / f"{name}.py"
+ module_name = f"{__name__.rsplit('.', 1)[0]}._cpu_{name}"
+ spec = importlib.util.spec_from_file_location(module_name, path)
+ module = importlib.util.module_from_spec(spec)
+ # Registered so pickle (spawn workers, datasets num_proc) can import the classes back; a fresh
+ # process resolves the name because importing the package runs this loader again.
+ sys.modules[module_name] = module
+ try:
+ spec.loader.exec_module(module)
+ except BaseException:
+ sys.modules.pop(module_name, None)
+ raise
+ return module
+
+
+_raw_text = _load_dataprep("raw_text")
+RawTextDataLoader = _raw_text.RawTextDataLoader
+TextPreprocessor = _raw_text.TextPreprocessor
+load_ebook_dataset = _load_dataprep("ebooks").load_ebook_dataset
+del _raw_text, _load_dataprep
+
__all__ = [
"FastLanguageModel",
"FastModel",
@@ -208,6 +236,9 @@ __all__ = [
"UnslothTrainer",
"UnslothTrainingArguments",
"UnslothVisionDataCollator",
+ "RawTextDataLoader",
+ "TextPreprocessor",
+ "load_ebook_dataset",
"DEVICE_TYPE",
"DEVICE_TYPE_TORCH",
"DEVICE_COUNT",