Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

feat: add EPUB and optional MOBI dataset loading (#14453)

Hugh Scott8a7d528faff8
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.json
index 88a9ead6d83..c423b9ab833 100644
--- a/scripts/risky_loaders_baseline.json
+++ b/scripts/risky_loaders_baseline.json
@@ -458,6 +458,14 @@
"count": 1,
"reason": "Loads unsloth/models/decision_mlx.py from a fixed path inside this package on Apple Silicon, where importing unsloth.models would import torch; never a downloaded or user-supplied file."
},
+ {
+ "file": "unsloth/__init__.py",
+ "rule": "dynamic-import",
+ "sink": "spec_from_file_location",
+ "digest": "8840de300c0740e6",
+ "count": 1,
+ "reason": "Loads unsloth/dataprep/ebooks.py from a fixed path derived from this package's __file__ on MLX to avoid importing torch through dataprep; never an ebook-provided or downloaded code path."
+ },
{
"file": "unsloth/_gpu_init.py",
"rule": "dynamic-import",
diff --git a/tests/test_ebook_dataset.py b/tests/test_ebook_dataset.py
new file mode 100644
index 00000000000..7fbb36a4e5f
--- /dev/null
+++ b/tests/test_ebook_dataset.py
@@ -0,0 +1,440 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""CPU-only ebook fixtures; no model weights or network access are needed."""
+
+import importlib.util
+import ast
+import shutil
+import subprocess
+import zipfile
+from pathlib import Path
+from xml.sax.saxutils import escape
+
+import pytest
+
+
+def ebooks():
+ # Loading by path avoids Unsloth's accelerator initialization, like test_raw_text.py.
+ path = Path(__file__).resolve().parents[1] / "unsloth/dataprep/ebooks.py"
+ spec = importlib.util.spec_from_file_location("ebook_dataset", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def tokenizer():
+ pytest.importorskip("tokenizers")
+ pytest.importorskip("transformers")
+ from tokenizers import Tokenizer, models, pre_tokenizers
+ from transformers import PreTrainedTokenizerFast
+
+ words = "Chapter One Two First Second paragraph hello world café Fiction Ada Book".split()
+ words += [f"word{i}" for i in range(50)]
+ backend = Tokenizer(
+ models.WordLevel({"[UNK]": 0, **{w: i + 1 for i, w in enumerate(words)}}, unk_token = "[UNK]")
+ )
+ backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
+ return PreTrainedTokenizerFast(tokenizer_object = backend, unk_token = "[UNK]")
+
+
+def byte_tokenizer():
+ pytest.importorskip("tokenizers")
+ pytest.importorskip("transformers")
+ from tokenizers import Tokenizer, decoders, models, pre_tokenizers
+ from transformers import PreTrainedTokenizerFast
+
+ vocab = {char: i for i, char in enumerate(pre_tokenizers.ByteLevel.alphabet())}
+ backend = Tokenizer(models.BPE(vocab = vocab, merges = []))
+ backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space = False)
+ backend.decoder = decoders.ByteLevel()
+ return PreTrainedTokenizerFast(tokenizer_object = backend)
+
+
+def make_epub(
+ path,
+ *,
+ chapters = None,
+ spine = None,
+ extra = None,
+ encoding = "utf-8",
+):
+ if chapters is None:
+ chapters = {
+ "second.xhtml": "<h1>Chapter Two</h1><p>Second paragraph.</p>",
+ "first.xhtml": "<h1>Chapter One</h1><p>First <em>paragraph</em>.</p>",
+ }
+ if spine is None:
+ spine = list(reversed(chapters))
+ manifest = "".join(
+ f'<item id="c{i}" href="{escape(name)}" media-type="application/xhtml+xml"/>'
+ for i, name in enumerate(chapters)
+ )
+ ids = {name: f"c{i}" for i, name in enumerate(chapters)}
+ reading_order = "".join(f'<itemref idref="{ids[name]}"/>' for name in spine)
+ opf = f"""<?xml version="1.0" encoding="UTF-8"?>
+ <package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid">
+ <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
+ <dc:identifier id="uid">urn:uuid:ebook-test</dc:identifier>
+ <dc:title>Book</dc:title><dc:creator>Ada</dc:creator><dc:subject>Fiction</dc:subject>
+ <dc:language>en</dc:language>
+ </metadata><manifest>{manifest}</manifest><spine>{reading_order}</spine>
+ </package>"""
+ with zipfile.ZipFile(path, "w") as archive:
+ archive.writestr("mimetype", "application/epub+zip")
+ archive.writestr(
+ "META-INF/container.xml",
+ """<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container" version="1.0"><rootfiles><rootfile full-path="Books/content.opf" media-type="application/oebps-package+xml"/></rootfiles></container>""",
+ )
+ archive.writestr("Books/content.opf", opf)
+ for name, body in chapters.items():
+ html = f'<?xml version="1.0" encoding="{encoding}"?><html xmlns="http://www.w3.org/1999/xhtml"><head><title>Clutter</title><style>unwanted CSS</style></head><body>{body}</body></html>'
+ archive.writestr("Books/" + name.replace("%20", " "), html.encode(encoding))
+ for name, data in (extra or {}).items():
+ archive.writestr(name, data)
+ return path
+
+
+def test_epub_reads_spine_and_preserves_paragraphs(ebooks, tokenizer, tmp_path):
+ book = make_epub(tmp_path / "book.epub")
+ dataset = ebooks.load_ebook_dataset(book, tokenizer = tokenizer)
+ assert dataset.column_names == ["text"]
+ assert (
+ dataset[0]["text"] == "Chapter One\n\nFirst paragraph.\n\nChapter Two\n\nSecond paragraph."
+ )
+
+
+def test_cleaning_and_unicode(ebooks, tokenizer, tmp_path):
+ body = '<p>hello&nbsp;<em>world</em> café &amp; tea.</p><script>bad script</script><nav>bad navigation</nav><svg><text>bad image</text></svg><p hidden="hidden">bad hidden</p><p>Next<br/>line.</p>'
+ book = make_epub(
+ tmp_path / "unicode.epub", chapters = {"chapter%20one.xhtml": body}, encoding = "utf-16"
+ )
+ dataset = ebooks.load_ebook_dataset(book, tokenizer = tokenizer)
+ assert dataset[0]["text"] == "hello world café & tea.\n\nNext\nline."
+
+
+ "content", ["The setting encoding=auto selects café.", "Use encoding=latin1 for café."]
+)
+def test_encoding_words_in_prose_do_not_change_decoding(ebooks, content):
+ data = ("<html><body><p>" + content + "</p></body></html>").encode("utf-8")
+ assert ebooks._html_text(data) == content
+
+
+def test_cdata_in_xhtml_keeps_literal_text(ebooks):
+ assert (
+ ebooks._html_text(b"<html><body><p><![CDATA[Actual <book> & text]]></p></body></html>")
+ == "Actual <book> & text"
+ )
+
+
+def test_preformatted_text_keeps_its_layout(ebooks):
+ data = b"<html><body><p>Intro text</p><pre><code>def f():\n return 1</code></pre></body></html>"
+ assert ebooks._html_text(data) == "Intro text\n\ndef f():\n return 1"
+
+
+def test_definition_lists_and_figures_do_not_merge_words(ebooks):
+ data = (
+ b"<html><body><dl><dt>Term</dt><dd>Definition</dd></dl>"
+ b"<figure><img src='a.png'/><figcaption>Caption</figcaption></figure><aside>Note</aside></body></html>"
+ )
+ assert ebooks._html_text(data) == "Term\n\nDefinition\n\nCaption\n\nNote"
+
+
+def test_bom_is_dropped_when_xml_declares_utf8(ebooks):
+ data = b'\xef\xbb\xbf<?xml version="1.0" encoding="UTF-8"?><html><body><p>Chapter</p></body></html>'
+ assert ebooks._html_text(data) == "Chapter"
+
+
+def test_declared_html_charset_is_honored(ebooks):
+ data = (
+ '<html><head><meta charset="windows-1252"/></head><body><p>café</p></body></html>'.encode(
+ "windows-1252"
+ )
+ )
+ assert ebooks._html_text(data) == "café"
+
+
+def test_html_body_closes_implicit_head(ebooks):
+ assert ebooks._html_text(b"<head><title>Book</title><body><p>hello</p>") == "hello"
+
+
+ "meta",
+ [
+ """<meta http-equiv="Content-Type" content='text/html; charset="windows-1252"'>""",
+ """<meta http-equiv="Content-Type" content="text/html; charset=&quot;windows-1252&quot;">""",
+ ],
+)
+def test_quoted_mime_charset_is_honored(ebooks, meta):
+ html = f"<html><head>{meta}</head><body><p>café</p></body></html>"
+ assert ebooks._html_text(html.encode("windows-1252")) == "café"
+
+
+def test_optional_metadata_and_chapter_markers(ebooks, tokenizer, tmp_path):
+ book = make_epub(tmp_path / "book.epub")
+ row = ebooks.load_ebook_dataset(
+ book, tokenizer = tokenizer, include_metadata = True, chapter_markers = True
+ )[0]
+ assert {key: row[key] for key in ("title", "author", "genre")} == {
+ "title": "Book",
+ "author": "Ada",
+ "genre": "Fiction",
+ }
+ assert row["source"] == str(book.resolve())
+ assert row["chunk_id"] == 0
+ assert row["text"].count("<|chapter_start|>") == 2
+ assert row["text"].count("<|chapter_end|>") == 2
+
+
+def test_chunk_size_overlap_and_complete_coverage(ebooks, tokenizer, tmp_path):
+ words = [f"word{i}" for i in range(23)]
+ book = make_epub(
+ tmp_path / "book.epub", chapters = {"one.xhtml": "<p>" + " ".join(words) + "</p>"}
+ )
+ rows = ebooks.load_ebook_dataset(book, tokenizer = tokenizer, chunk_size = 8, overlap = 2)["text"]
+ assert [row.split() for row in rows] == [words[0:8], words[6:14], words[12:20], words[18:23]]
+ assert all(len(tokenizer.encode(row, add_special_tokens = False)) <= 8 for row in rows)
+
+
+def test_prefers_sentence_boundary(ebooks, tokenizer, tmp_path):
+ book = make_epub(
+ tmp_path / "book.epub",
+ chapters = {
+ "one.xhtml": "<p>One Two First paragraph. Second paragraph hello world café.</p>"
+ },
+ )
+ rows = ebooks.load_ebook_dataset(book, tokenizer = tokenizer, chunk_size = 7, overlap = 1)["text"]
+ assert rows[0] == "One Two First paragraph."
+ assert rows[1].startswith("paragraph. Second")
+
+
+def test_library_is_recursive_sorted_and_books_never_mix(ebooks, tokenizer, tmp_path):
+ (tmp_path / "nested").mkdir()
+ make_epub(tmp_path / "z.EPUB", chapters = {"a.xhtml": "<p>Second</p>"})
+ make_epub(tmp_path / "nested/a.epub", chapters = {"b.xhtml": "<p>First</p>"})
+ (tmp_path / "ignored.txt").write_text("not an ebook")
+ rows = ebooks.load_ebook_dataset(tmp_path, tokenizer = tokenizer, file_types = [".EPUB"])["text"]
+ assert rows == ["First", "Second"]
+
+
[email protected]("file_types", [None, ["notes"]])
+def test_new_reader_reuses_discovery_chunking_and_metadata(
+ ebooks, tokenizer, tmp_path, monkeypatch, file_types
+):
+ def read_notes(path, chapter_markers):
+ return path.read_text(encoding = "utf-8"), {
+ "title": "Notes",
+ "author": "Ada",
+ "genre": "Fiction",
+ }
+
+ # A future format supplies only its reader and one extension mapping entry.
+ monkeypatch.setattr(ebooks, "_READERS", {".notes": read_notes}, raising = False)
+ (tmp_path / "book.notes").write_text("word0 word1 word2 word3 word4")
+ kwargs = {} if file_types is None else {"file_types": file_types}
+ dataset = ebooks.load_ebook_dataset(
+ tmp_path, tokenizer = tokenizer, chunk_size = 3, overlap = 1, include_metadata = True, **kwargs
+ )
+ assert dataset["text"] == ["word0 word1 word2", "word2 word3 word4"]
+ assert dataset["title"] == ["Notes", "Notes"]
+ assert dataset["chunk_id"] == [0, 1]
+
+
+ "kwargs",
+ [
+ {"chunk_size": 0},
+ {"chunk_size": 3, "overlap": 3},
+ {"overlap": -1},
+ {"chunk_size": 1.5},
+ {"file_types": ["pdf"]},
+ {"file_types": []},
+ ],
+)
+def test_invalid_options(ebooks, tokenizer, tmp_path, kwargs):
+ with pytest.raises((ValueError, TypeError)):
+ ebooks.load_ebook_dataset(tmp_path, tokenizer = tokenizer, **kwargs)
+
+
+def test_requires_tokenizer_instead_of_guessing_token_counts(ebooks, tmp_path):
+ with pytest.raises(ValueError, match = "tokenizer"):
+ ebooks.load_ebook_dataset(tmp_path)
+
+
+def test_empty_missing_unsupported_and_corrupt_inputs(ebooks, tokenizer, tmp_path):
+ with pytest.raises(FileNotFoundError):
+ ebooks.load_ebook_dataset(tmp_path / "missing.epub", tokenizer = tokenizer)
+ with pytest.raises(ValueError, match = "No .*ebook"):
+ ebooks.load_ebook_dataset(tmp_path, tokenizer = tokenizer)
+ other = tmp_path / "file.txt"
+ other.write_text("not a book")
+ with pytest.raises(ValueError, match = "Unsupported"):
+ ebooks.load_ebook_dataset(other, tokenizer = tokenizer)
+ bad = tmp_path / "bad.epub"
+ bad.write_bytes(b"not a zip")
+ with pytest.raises(ValueError, match = "bad.epub"):
+ ebooks.load_ebook_dataset(bad, tokenizer = tokenizer)
+ empty = make_epub(tmp_path / "empty.epub", chapters = {"empty.xhtml": "<p> </p>"})
+ with pytest.raises(ValueError, match = "readable text"):
+ ebooks.load_ebook_dataset(empty, tokenizer = tokenizer)
+
+
+def test_non_linear_and_nav_documents_are_not_training_text(ebooks, tokenizer, tmp_path):
+ book = make_epub(tmp_path / "book.epub")
+ with zipfile.ZipFile(book) as archive:
+ files = {n: archive.read(n) for n in archive.namelist()}
+ files["Books/content.opf"] = files["Books/content.opf"].replace(
+ b'idref="c0"', b'idref="c0" linear="no"'
+ )
+ with zipfile.ZipFile(book, "w") as archive:
+ for name, data in files.items():
+ archive.writestr(name, data)
+ assert "Chapter Two" not in ebooks.load_ebook_dataset(book, tokenizer = tokenizer)[0]["text"]
+
+
+def test_encrypted_chapter_is_rejected_but_obfuscated_fonts_are_allowed(
+ ebooks, tokenizer, tmp_path
+):
+ encryption = """<encryption xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><EncryptedData xmlns="http://www.w3.org/2001/04/xmlenc#"><CipherData><CipherReference URI="Books/first.xhtml"/></CipherData></EncryptedData></encryption>"""
+ book = make_epub(tmp_path / "drm.epub", extra = {"META-INF/encryption.xml": encryption})
+ with pytest.raises(ValueError, match = "[Ee]ncrypt|DRM"):
+ ebooks.load_ebook_dataset(book, tokenizer = tokenizer)
+ book = make_epub(
+ tmp_path / "font.epub",
+ extra = {
+ "META-INF/encryption.xml": encryption.replace("Books/first.xhtml", "Fonts/font.otf")
+ },
+ )
+ assert len(ebooks.load_ebook_dataset(book, tokenizer = tokenizer)) == 1
+
+
+def test_archive_size_limit(ebooks, tokenizer, tmp_path, monkeypatch):
+ book = make_epub(tmp_path / "book.epub")
+ monkeypatch.setattr(ebooks, "_MAX_MEMBER_BYTES", 32)
+ with pytest.raises(ValueError, match = "size limit"):
+ ebooks.load_ebook_dataset(book, tokenizer = tokenizer)
+
+
+def test_missing_mobi_converter_has_actionable_error(ebooks, tokenizer, tmp_path, monkeypatch):
+ book = tmp_path / "book.mobi"
+ book.write_bytes(b"placeholder")
+ monkeypatch.setattr(shutil, "which", lambda name: None)
+ with pytest.raises(ImportError, match = "[Cc]alibre.*ebook-convert"):
+ ebooks.load_ebook_dataset(book, tokenizer = tokenizer)
+
+
+def test_mobi_conversion_failure_reports_source(ebooks, tokenizer, tmp_path):
+ if not shutil.which("ebook-convert"):
+ pytest.skip("Calibre is optional")
+ book = tmp_path / "broken.mobi"
+ book.write_bytes(b"not a mobi")
+ with pytest.raises(ValueError, match = "broken.mobi"):
+ ebooks.load_ebook_dataset(book, tokenizer = tokenizer)
+
+
+def test_real_mobi_round_trip(ebooks, tokenizer, tmp_path):
+ converter = shutil.which("ebook-convert")
+ if not converter:
+ pytest.skip("Calibre is optional")
+ epub = make_epub(tmp_path / "source.epub")
+ mobi = tmp_path / "book with spaces.mobi"
+ subprocess.run(
+ [converter, str(epub), str(mobi), "--no-inline-toc"],
+ check = True,
+ capture_output = True,
+ timeout = 120,
+ )
+ dataset = ebooks.load_ebook_dataset(mobi, tokenizer = tokenizer, include_metadata = True)
+ text = dataset[0]["text"]
+ assert text.index("Chapter One") < text.index("Chapter Two")
+ assert "First paragraph." in text and "Second paragraph." in text
+ assert dataset[0]["title"] == "Book" and dataset[0]["author"] == "Ada"
+ assert dataset[0]["source"] == str(mobi.resolve())
+
+
+def test_public_exports_cover_gpu_mlx_and_dataprep(ebooks):
+ root = Path(__file__).resolve().parents[1] / "unsloth"
+ # Execute the MLX loading block without importing its unrelated model backends.
+ tree = ast.parse((root / "__init__.py").read_text(encoding = "utf-8"))
+ branch = next(
+ n
+ for n in tree.body
+ if isinstance(n, ast.If) and isinstance(n.test, ast.Name) and n.test.id == "_IS_MLX"
+ )
+ statements = [n for n in branch.body if "_ebook" in ast.unparse(n)]
+ namespace = {
+ "__file__": str(root / "__init__.py"),
+ "importlib": __import__("importlib"),
+ "_Path": Path,
+ }
+ exec(
+ compile(ast.Module(body = statements, type_ignores = []), "mlx_ebook_export", "exec"), namespace
+ )
+ assert callable(namespace.get("load_ebook_dataset"))
+ for name in ("_gpu_init.py", "dataprep/__init__.py"):
+ imports = [
+ n
+ for n in ast.walk(ast.parse((root / name).read_text(encoding = "utf-8")))
+ if isinstance(n, ast.ImportFrom) and n.module
+ ]
+ assert any(
+ (
+ n.module.endswith("ebooks")
+ and any(a.name in {"load_ebook_dataset", "*"} for a in n.names)
+ )
+ or (n.module.endswith("dataprep") and any(a.name == "ebooks" for a in n.names))
+ for n in imports
+ )
+
+
[email protected]("chunk_size", [10, 13, 16, 25])
+def test_byte_bpe_chunks_preserve_unicode_and_fit_budget(
+ ebooks, byte_tokenizer, tmp_path, chunk_size
+):
+ tokenizer = byte_tokenizer
+ text = "café 🦥 中文 hello world. " * 8
+ book = make_epub(tmp_path / "unicode.epub", chapters = {"one.xhtml": "<p>" + text + "</p>"})
+ rows = ebooks.load_ebook_dataset(book, tokenizer = tokenizer, chunk_size = chunk_size, overlap = 3)[
+ "text"
+ ]
+ assert all(
+ "\ufffd" not in row and len(tokenizer.encode(row, add_special_tokens = False)) <= chunk_size
+ for row in rows
+ )
+ for char in "é🦥中文":
+ assert sum(row.count(char) for row in rows) >= text.count(char)
+ # A zero-overlap split must not lose or duplicate even a multibyte character.
+ rows = ebooks.load_ebook_dataset(book, tokenizer = tokenizer, chunk_size = chunk_size, overlap = 0)[
+ "text"
+ ]
+ assert "".join("".join(rows).split()) == "".join(text.split())
+
+
+ "html, chunk_size, overlap, expected",
+ [
+ ("<p>aaa.</p><p>bbb</p>", 4, 0, ["aaa.", "bbb"]),
+ ("<p>a b</p>", 1, 0, ["a", "b"]),
+ ("<p>a</p><p>b</p>", 1, 0, ["a", "b"]),
+ ("<p>a b</p>", 2, 1, ["a", "b"]),
+ ],
+)
+def test_whitespace_windows_do_not_abort_books(
+ ebooks, byte_tokenizer, html, chunk_size, overlap, expected
+):
+ text = ebooks._html_text(html.encode())
+ chunks = list(ebooks._chunk_text(text, byte_tokenizer, chunk_size, overlap))
+ assert chunks == expected
+ assert all(
+ len(byte_tokenizer.encode(chunk, add_special_tokens = False)) <= chunk_size
+ for chunk in chunks
+ )
+
+
+def test_tiny_budget_still_rejects_an_incomplete_unicode_character(ebooks, byte_tokenizer):
+ with pytest.raises(ValueError, match = "complete character"):
+ list(ebooks._chunk_text("🦥", byte_tokenizer, 1, 0))
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index 016ea4bd124..14aa836b937 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -372,6 +372,12 @@ if _IS_MLX:
_raw_text_spec.loader.exec_module(_raw_text)
RawTextDataLoader = _raw_text.RawTextDataLoader
TextPreprocessor = _raw_text.TextPreprocessor
+ _ebook_path = _Path(__file__).resolve().parent / "dataprep" / "ebooks.py"
+ _ebook_spec = importlib.util.spec_from_file_location("unsloth._mlx_ebooks", _ebook_path)
+ _ebook = importlib.util.module_from_spec(_ebook_spec)
+ _ebook_spec.loader.exec_module(_ebook)
+ load_ebook_dataset = _ebook.load_ebook_dataset
+ del _ebook, _ebook_spec, _ebook_path
del _raw_text, _raw_text_spec, _raw_text_path, _Path
def _mlx_from_pretrained(*args, **kwargs):
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 8bdccf49aaf..3879fc6316c 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -806,6 +806,9 @@ from .trainer import *
# Export dataprep utilities for CLI and downstream users
from .dataprep.raw_text import RawTextDataLoader, TextPreprocessor
+from .dataprep import ebooks as _ebooks
+
+load_ebook_dataset = _ebooks.load_ebook_dataset
from unsloth_zoo.rl_environments import (
check_python_modules,
create_locked_down_function,
diff --git a/unsloth/dataprep/__init__.py b/unsloth/dataprep/__init__.py
index 2c0ab5c22b4..8427ffab3f7 100644
--- a/unsloth/dataprep/__init__.py
+++ b/unsloth/dataprep/__init__.py
@@ -11,3 +11,4 @@
from .synthetic import *
from .raw_text import *
+from .ebooks import *
diff --git a/unsloth/dataprep/ebooks.py b/unsloth/dataprep/ebooks.py
new file mode 100644
index 00000000000..3e506cdcad4
--- /dev/null
+++ b/unsloth/dataprep/ebooks.py
@@ -0,0 +1,426 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Load EPUB books and, with Calibre installed, MOBI books on CPU.
+
+Pass a Hugging Face fast tokenizer, not model weights. Token counts exclude
+special tokens added later by a trainer; reserve space for those in chunk_size.
+"""
+
+from bisect import bisect_right
+from html.parser import HTMLParser
+from pathlib import Path
+import posixpath
+import re
+import shutil
+import subprocess
+import tempfile
+import unicodedata
+from urllib.parse import unquote, urlsplit
+from xml.etree import ElementTree as ET
+from zipfile import BadZipFile, ZipFile
+
+__all__ = ["load_ebook_dataset"]
+
+_MAX_MEMBER_BYTES = 32 * 1024 * 1024
+_MAX_BOOK_BYTES = 256 * 1024 * 1024
+_BLOCKS = {
+ "p",
+ "div",
+ "section",
+ "article",
+ "blockquote",
+ "li",
+ "tr",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "pre",
+ "ul",
+ "ol",
+ "dl",
+ "dt",
+ "dd",
+ "table",
+ "caption",
+ "figure",
+ "figcaption",
+ "header",
+ "footer",
+ "main",
+ "aside",
+ "address",
+ "details",
+ "summary",
+}
+_VOID = {
+ "area",
+ "base",
+ "br",
+ "col",
+ "embed",
+ "hr",
+ "img",
+ "input",
+ "link",
+ "meta",
+ "param",
+ "source",
+ "track",
+ "wbr",
+}
+_IGNORE = {"head", "script", "style", "nav", "svg"}
+
+
+class _BookTextParser(HTMLParser):
+ def __init__(self):
+ super().__init__(convert_charrefs = True)
+ self.parts = []
+ self.stack = []
+ self.encoding = None
+ # <pre> text is kept verbatim and spliced back after whitespace cleanup.
+ self.preformatted = []
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "body":
+ self.handle_endtag("head")
+ attrs = dict(attrs)
+ if tag == "meta" and self.encoding is None:
+ self.encoding = attrs.get("charset")
+ if not self.encoding and attrs.get("http-equiv", "").lower() == "content-type":
+ charset = re.search(r"charset\s*=\s*[\"']?([\w-]+)", attrs.get("content", ""), re.I)
+ if charset:
+ self.encoding = charset[1]
+ hidden = (
+ bool(self.stack and self.stack[-1][1])
+ or tag in _IGNORE
+ or "hidden" in attrs
+ or attrs.get("aria-hidden", "").lower() == "true"
+ )
+ if not hidden:
+ if tag in _BLOCKS or tag == "hr":
+ self.parts.append("\n\n")
+ elif tag == "br":
+ self.parts.append("\n")
+ elif tag in {"td", "th"}:
+ self.parts.append(" ")
+ if tag not in _VOID:
+ self.stack.append((tag, hidden))
+
+ def handle_startendtag(self, tag, attrs):
+ self.handle_starttag(tag, attrs)
+ if tag not in _VOID:
+ self.handle_endtag(tag)
+
+ def handle_endtag(self, tag):
+ hidden = bool(self.stack and self.stack[-1][1])
+ for i in range(len(self.stack) - 1, -1, -1):
+ if self.stack[i][0] == tag:
+ del self.stack[i:]
+ break
+ if not hidden and tag in _BLOCKS:
+ self.parts.append("\n\n")
+
+ def handle_data(self, data):
+ if not self.stack or not self.stack[-1][1]:
+ if any(tag == "pre" for tag, _ in self.stack):
+ self.parts.append(f"\ue000{len(self.preformatted)}\ue001")
+ self.preformatted.append(data)
+ else:
+ # Source newlines inside an HTML paragraph render as spaces.
+ self.parts.append(re.sub(r"\s+", " ", data))
+
+ def unknown_decl(self, data):
+ if data.startswith("CDATA["):
+ self.handle_data(data[6:])
+
+
+def _html_text(data):
+ if data.startswith((b"\xff\xfe", b"\xfe\xff")):
+ encoding = "utf-16"
+ elif data.startswith(b"\x00<"):
+ encoding = "utf-16-be"
+ elif data.startswith(b"<\x00"):
+ encoding = "utf-16-le"
+ else:
+ declaration = re.match(
+ rb'(?:\xef\xbb\xbf)?\s*<\?xml\b[^?]*\bencoding\s*=\s*["\']([\w-]+)', data[:1024], re.I
+ )
+ if declaration:
+ encoding = declaration[1].decode("ascii")
+ else:
+ header = _BookTextParser()
+ header.feed(data[:1024].decode("latin-1"))
+ encoding = header.encoding or "utf-8-sig"
+ parser = _BookTextParser()
+ # A declared UTF-8 encoding keeps a leading BOM, which str.strip() leaves in place.
+ parser.feed(data.decode(encoding).lstrip("\ufeff"))
+ parser.close()
+ text = unicodedata.normalize("NFC", "".join(parser.parts))
+ text = re.sub(r"[^\S\n]+", " ", text)
+ text = re.sub(r" *\n *", "\n", text)
+ text = re.sub(r"\n{3,}", "\n\n", text).strip()
+ return re.sub(
+ "\ue000(\\d+)\ue001",
+ lambda m: unicodedata.normalize("NFC", parser.preformatted[int(m[1])]),
+ text,
+ )
+
+
+def _archive_path(base, href):
+ url = urlsplit(href)
+ if url.scheme or url.netloc or url.path.startswith("/") or "\\" in url.path:
+ raise ValueError(f"Invalid EPUB resource path: {href!r}")
+ name = posixpath.normpath(posixpath.join(base, unquote(url.path)))
+ if name == ".." or name.startswith("../") or name.startswith("/"):
+ raise ValueError(f"Invalid EPUB resource path: {href!r}")
+ return name
+
+
+def _read_epub(path, chapter_markers):
+ with ZipFile(path) as archive:
+ total = 0
+
+ def read(name):
+ nonlocal total
+ info = archive.getinfo(name)
+ total += info.file_size
+ if info.file_size > _MAX_MEMBER_BYTES or total > _MAX_BOOK_BYTES:
+ raise ValueError("EPUB exceeds the uncompressed size limit")
+ return archive.read(info)
+
+ container = ET.fromstring(read("META-INF/container.xml"))
+ rootfile = container.find(".//{*}rootfile")
+ if rootfile is None or not rootfile.get("full-path"):
+ raise ValueError("EPUB container has no package document")
+ package_path = _archive_path("", rootfile.get("full-path"))
+ package = ET.fromstring(read(package_path))
+ base = posixpath.dirname(package_path)
+ manifest = package.find("{*}manifest")
+ spine = package.find("{*}spine")
+ if manifest is None or spine is None:
+ raise ValueError("EPUB package is missing its manifest or reading order")
+ items = {item.get("id"): item for item in manifest}
+ encrypted = set()
+ if "META-INF/encryption.xml" in archive.namelist():
+ encryption = ET.fromstring(read("META-INF/encryption.xml"))
+ encrypted = {
+ _archive_path("", ref.get("URI", ""))
+ for ref in encryption.iter("{http://www.w3.org/2001/04/xmlenc#}CipherReference")
+ }
+ chapters = []
+ for ref in spine:
+ if ref.get("linear", "yes") == "no":
+ continue
+ item = items.get(ref.get("idref"))
+ if item is None:
+ raise ValueError("EPUB reading order references a missing manifest item")
+ if "nav" in item.get("properties", "").split():
+ continue
+ if item.get("media-type") not in {"application/xhtml+xml", "text/html"}:
+ continue
+ name = _archive_path(base, item.get("href", ""))
+ if name in encrypted:
+ raise ValueError("Encrypted/DRM-protected EPUB text is not supported")
+ text = _html_text(read(name))
+ if text:
+ if chapter_markers:
+ text = "<|chapter_start|>\n" + text + "\n<|chapter_end|>"
+ chapters.append(text)
+ metadata = package.find("{*}metadata")
+ fields = {"title": "title", "author": "creator", "genre": "subject"}
+ values = {}
+ for key, field in fields.items():
+ elements = (
+ []
+ if metadata is None
+ else metadata.findall("{http://purl.org/dc/elements/1.1/}" + field)
+ )
+ values[key] = "; ".join(
+ " ".join("".join(element.itertext()).split()) for element in elements
+ )
+ return "\n\n".join(chapters), values
+
+
+def _read_mobi(path, chapter_markers):
+ converter = shutil.which("ebook-convert")
+ if converter is None:
+ raise ImportError(
+ "MOBI support requires Calibre's ebook-convert on PATH. Install Calibre or use EPUB files."
+ )
+ with tempfile.TemporaryDirectory(prefix = "unsloth-ebook-") as directory:
+ output = Path(directory) / "book.epub"
+ subprocess.run(
+ [converter, str(path), str(output)], check = True, capture_output = True, timeout = 120
+ )
+ return _read_epub(output, chapter_markers)
+
+
+# Add formats here. Readers return (clean_text, {"title": str, "author": str, "genre": str}).
+_READERS = {".epub": _read_epub, ".mobi": _read_mobi}
+
+
+def _read_book(path, chapter_markers):
+ try:
+ return _READERS[path.suffix.lower()](path, chapter_markers)
+ except (
+ BadZipFile,
+ ET.ParseError,
+ LookupError,
+ OSError,
+ ValueError,
+ subprocess.SubprocessError,
+ ) as exc:
+ raise ValueError(f"Could not read ebook '{path}': {exc}") from exc
+
+
+def _chunk_text(text, tokenizer, chunk_size, overlap):
+ encoded = tokenizer(
+ text,
+ add_special_tokens = False,
+ return_offsets_mapping = True,
+ return_attention_mask = False,
+ truncation = False,
+ verbose = False,
+ )
+ offsets = [(a, b) for a, b in encoded["offset_mapping"] if b > a]
+ if not offsets:
+ raise ValueError("The tokenizer produced no tokens for the ebook text")
+ ends = [b for _, b in offsets]
+ starts = [a for a, _ in offsets]
+ boundaries = [
+ m.end() for m in re.finditer(r'(?<=[.!?。!?])["\u201d\u2019\)\]]*\s+|\n\s*\n', text)
+ ]
+ start = 0
+ char_start = 0
+ while start < len(offsets):
+ while char_start < len(text) and text[char_start].isspace():
+ char_start += 1
+ start = max(start, bisect_right(ends, char_start))
+ if start == len(offsets):
+ break
+ end = min(start + chunk_size, len(offsets))
+ char_end = len(text) if end == len(offsets) else ends[end - 1]
+ if end < len(offsets):
+ boundary_idx = bisect_right(boundaries, char_end) - 1
+ if boundary_idx >= 0:
+ boundary = boundaries[boundary_idx]
+ boundary_token = bisect_right(ends, boundary)
+ if (
+ boundary_token - start >= max(overlap + 1, chunk_size // 2)
+ and text[char_start:boundary].strip()
+ ):
+ end, char_end = boundary_token, boundary
+ # BPE offsets may share a Unicode character; re-encoding also checks the new left boundary.
+ while end > start:
+ chunk = text[char_start:char_end].strip()
+ if chunk and len(tokenizer.encode(chunk, add_special_tokens = False)) <= chunk_size:
+ break
+ end -= 1
+ char_end = ends[end - 1] if end > start else char_start
+ if end == start:
+ raise ValueError(
+ "chunk_size is too small to represent a complete character with this tokenizer"
+ )
+ yield chunk
+ # Consume every byte token of the final character, even when it spans the token window.
+ consumed = bisect_right(ends, char_end)
+ if consumed == len(offsets):
+ break
+ if overlap:
+ start = max(consumed - overlap, bisect_right(starts, char_start))
+ char_start = starts[start]
+ else:
+ start, char_start = consumed, char_end
+
+
+def load_ebook_dataset(
+ path,
+ file_types = None,
+ chunk_size = 4096,
+ overlap = 256,
+ *,
+ tokenizer = None,
+ include_metadata = False,
+ chapter_markers = False,
+):
+ """Return a Hugging Face Dataset with a ``text`` column from a book or library.
+
+ ``tokenizer`` must be a Hugging Face fast tokenizer (``use_fast=True``).
+ ``chunk_size`` and ``overlap`` count its tokens, excluding trainer-added
+ special tokens. Chunks prefer sentence/paragraph boundaries in the latter
+ half of a window; overlap uses source-token offsets and may expand to keep
+ Unicode characters intact. No model is loaded and nothing is downloaded.
+
+ Directories are searched recursively in sorted path order. Each book is
+ chunked separately and any unreadable book raises an error naming its path.
+ ``file_types=None`` selects all formats in the internal ``_READERS`` mapping.
+ EPUB uses the package's primary reading order, skipping navigation and
+ non-linear content. MOBI requires Calibre's ``ebook-convert`` on PATH.
+ DRM-protected text is unsupported. Images, CSS and scripts are discarded;
+ paragraphs and chapter breaks remain. ``chapter_markers=True`` inserts
+ ``<|chapter_start|>`` and ``<|chapter_end|>`` around each content document.
+
+ ``include_metadata=True`` adds ``source`` (absolute original filename),
+ ``chunk_id`` (zero-based within each book), ``title``, ``author`` and
+ ``genre``. Missing metadata is an empty string; multiple values use ``; ``.
+
+ Example::
+
+ from unsloth import load_ebook_dataset
+ dataset = load_ebook_dataset(
+ "./my_library/", tokenizer=tokenizer,
+ chunk_size=4096, overlap=256, include_metadata=True,
+ )
+
+ Reserve room for EOS/BOS or chat-template tokens in ``chunk_size`` when
+ passing the text dataset to a trainer.
+ """
+ if not isinstance(chunk_size, int) or isinstance(chunk_size, bool) or chunk_size <= 0:
+ raise ValueError("chunk_size must be a positive integer")
+ if not isinstance(overlap, int) or isinstance(overlap, bool) or not 0 <= overlap < chunk_size:
+ raise ValueError("overlap must be a non-negative integer smaller than chunk_size")
+ if tokenizer is None or not getattr(tokenizer, "is_fast", False):
+ raise ValueError(
+ "Pass a Hugging Face fast tokenizer (use_fast=True) to measure token counts without loading a model"
+ )
+ if isinstance(file_types, str):
+ file_types = [file_types]
+ extensions = (
+ set(_READERS)
+ if file_types is None
+ else {"." + extension.lower().lstrip(".") for extension in file_types}
+ )
+ if not extensions or not extensions <= _READERS.keys():
+ raise ValueError(
+ f"file_types must select supported ebook formats: {', '.join(sorted(_READERS))}"
+ )
+ path = Path(path).expanduser().resolve()
+ if not path.exists():
+ raise FileNotFoundError(path)
+ if path.is_dir():
+ files = sorted(p for p in path.rglob("*") if p.is_file() and p.suffix.lower() in extensions)
+ if not files:
+ raise ValueError(f"No matching ebook files found in '{path}'")
+ else:
+ if path.suffix.lower() not in extensions:
+ raise ValueError(f"Unsupported ebook file type: '{path.suffix}'")
+ files = [path]
+ columns = {"text": []}
+ if include_metadata:
+ columns.update({key: [] for key in ("source", "chunk_id", "title", "author", "genre")})
+ for book in files:
+ text, metadata = _read_book(book, chapter_markers)
+ if not text:
+ raise ValueError(f"Ebook '{book}' contains no readable text")
+ for index, chunk in enumerate(_chunk_text(text, tokenizer, chunk_size, overlap)):
+ columns["text"].append(chunk)
+ if include_metadata:
+ for key, value in {"source": str(book), "chunk_id": index, **metadata}.items():
+ columns[key].append(value)
+ from datasets import Dataset
+
+ return Dataset.from_dict(columns)