Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
feat: add EPUB and optional MOBI dataset loading (#14453)
Hugh Scott8a7d528faff8
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.jsonindex 88a9ead6d83..c423b9ab833 100644--- a/scripts/risky_loaders_baseline.json+++ b/scripts/risky_loaders_baseline.json@@ -458,6 +458,14 @@"count": 1,"reason": "Loads unsloth/models/decision_mlx.py from a fixed path inside this package on Apple Silicon, where importing unsloth.models would import torch; never a downloaded or user-supplied file."},+ {+ "file": "unsloth/__init__.py",+ "rule": "dynamic-import",+ "sink": "spec_from_file_location",+ "digest": "8840de300c0740e6",+ "count": 1,+ "reason": "Loads unsloth/dataprep/ebooks.py from a fixed path derived from this package's __file__ on MLX to avoid importing torch through dataprep; never an ebook-provided or downloaded code path."+ },{"file": "unsloth/_gpu_init.py","rule": "dynamic-import",diff --git a/tests/test_ebook_dataset.py b/tests/test_ebook_dataset.pynew file mode 100644index 00000000000..7fbb36a4e5f--- /dev/null+++ b/tests/test_ebook_dataset.py@@ -0,0 +1,440 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""CPU-only ebook fixtures; no model weights or network access are needed."""++import importlib.util+import ast+import shutil+import subprocess+import zipfile+from pathlib import Path+from xml.sax.saxutils import escape++import pytest+++def ebooks():+ # Loading by path avoids Unsloth's accelerator initialization, like test_raw_text.py.+ path = Path(__file__).resolve().parents[1] / "unsloth/dataprep/ebooks.py"+ spec = importlib.util.spec_from_file_location("ebook_dataset", path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def tokenizer():+ pytest.importorskip("tokenizers")+ pytest.importorskip("transformers")+ from tokenizers import Tokenizer, models, pre_tokenizers+ from transformers import PreTrainedTokenizerFast++ words = "Chapter One Two First Second paragraph hello world café Fiction Ada Book".split()+ words += [f"word{i}" for i in range(50)]+ backend = Tokenizer(+ models.WordLevel({"[UNK]": 0, **{w: i + 1 for i, w in enumerate(words)}}, unk_token = "[UNK]")+ )+ backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()+ return PreTrainedTokenizerFast(tokenizer_object = backend, unk_token = "[UNK]")+++def byte_tokenizer():+ pytest.importorskip("tokenizers")+ pytest.importorskip("transformers")+ from tokenizers import Tokenizer, decoders, models, pre_tokenizers+ from transformers import PreTrainedTokenizerFast++ vocab = {char: i for i, char in enumerate(pre_tokenizers.ByteLevel.alphabet())}+ backend = Tokenizer(models.BPE(vocab = vocab, merges = []))+ backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space = False)+ backend.decoder = decoders.ByteLevel()+ return PreTrainedTokenizerFast(tokenizer_object = backend)+++def make_epub(+ path,+ *,+ chapters = None,+ spine = None,+ extra = None,+ encoding = "utf-8",+):+ if chapters is None:+ chapters = {+ "second.xhtml": "<h1>Chapter Two</h1><p>Second paragraph.</p>",+ "first.xhtml": "<h1>Chapter One</h1><p>First <em>paragraph</em>.</p>",+ }+ if spine is None:+ spine = list(reversed(chapters))+ manifest = "".join(+ f'<item id="c{i}" href="{escape(name)}" media-type="application/xhtml+xml"/>'+ for i, name in enumerate(chapters)+ )+ ids = {name: f"c{i}" for i, name in enumerate(chapters)}+ reading_order = "".join(f'<itemref idref="{ids[name]}"/>' for name in spine)+ opf = f"""<?xml version="1.0" encoding="UTF-8"?>+ <package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid">+ <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">+ <dc:identifier id="uid">urn:uuid:ebook-test</dc:identifier>+ <dc:title>Book</dc:title><dc:creator>Ada</dc:creator><dc:subject>Fiction</dc:subject>+ <dc:language>en</dc:language>+ </metadata><manifest>{manifest}</manifest><spine>{reading_order}</spine>+ </package>"""+ with zipfile.ZipFile(path, "w") as archive:+ archive.writestr("mimetype", "application/epub+zip")+ archive.writestr(+ "META-INF/container.xml",+ """<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container" version="1.0"><rootfiles><rootfile full-path="Books/content.opf" media-type="application/oebps-package+xml"/></rootfiles></container>""",+ )+ archive.writestr("Books/content.opf", opf)+ for name, body in chapters.items():+ html = f'<?xml version="1.0" encoding="{encoding}"?><html xmlns="http://www.w3.org/1999/xhtml"><head><title>Clutter</title><style>unwanted CSS</style></head><body>{body}</body></html>'+ archive.writestr("Books/" + name.replace("%20", " "), html.encode(encoding))+ for name, data in (extra or {}).items():+ archive.writestr(name, data)+ return path+++def test_epub_reads_spine_and_preserves_paragraphs(ebooks, tokenizer, tmp_path):+ book = make_epub(tmp_path / "book.epub")+ dataset = ebooks.load_ebook_dataset(book, tokenizer = tokenizer)+ assert dataset.column_names == ["text"]+ assert (+ dataset[0]["text"] == "Chapter One\n\nFirst paragraph.\n\nChapter Two\n\nSecond paragraph."+ )+++def test_cleaning_and_unicode(ebooks, tokenizer, tmp_path):+ body = '<p>hello <em>world</em> café & tea.</p><script>bad script</script><nav>bad navigation</nav><svg><text>bad image</text></svg><p hidden="hidden">bad hidden</p><p>Next<br/>line.</p>'+ book = make_epub(+ tmp_path / "unicode.epub", chapters = {"chapter%20one.xhtml": body}, encoding = "utf-16"+ )+ dataset = ebooks.load_ebook_dataset(book, tokenizer = tokenizer)+ assert dataset[0]["text"] == "hello world café & tea.\n\nNext\nline."+++ "content", ["The setting encoding=auto selects café.", "Use encoding=latin1 for café."]+)+def test_encoding_words_in_prose_do_not_change_decoding(ebooks, content):+ data = ("<html><body><p>" + content + "</p></body></html>").encode("utf-8")+ assert ebooks._html_text(data) == content+++def test_cdata_in_xhtml_keeps_literal_text(ebooks):+ assert (+ ebooks._html_text(b"<html><body><p><![CDATA[Actual <book> & text]]></p></body></html>")+ == "Actual <book> & text"+ )+++def test_preformatted_text_keeps_its_layout(ebooks):+ data = b"<html><body><p>Intro text</p><pre><code>def f():\n return 1</code></pre></body></html>"+ assert ebooks._html_text(data) == "Intro text\n\ndef f():\n return 1"+++def test_definition_lists_and_figures_do_not_merge_words(ebooks):+ data = (+ b"<html><body><dl><dt>Term</dt><dd>Definition</dd></dl>"+ b"<figure><img src='a.png'/><figcaption>Caption</figcaption></figure><aside>Note</aside></body></html>"+ )+ assert ebooks._html_text(data) == "Term\n\nDefinition\n\nCaption\n\nNote"+++def test_bom_is_dropped_when_xml_declares_utf8(ebooks):+ data = b'\xef\xbb\xbf<?xml version="1.0" encoding="UTF-8"?><html><body><p>Chapter</p></body></html>'+ assert ebooks._html_text(data) == "Chapter"+++def test_declared_html_charset_is_honored(ebooks):+ data = (+ '<html><head><meta charset="windows-1252"/></head><body><p>café</p></body></html>'.encode(+ "windows-1252"+ )+ )+ assert ebooks._html_text(data) == "café"+++def test_html_body_closes_implicit_head(ebooks):+ assert ebooks._html_text(b"<head><title>Book</title><body><p>hello</p>") == "hello"+++ "meta",+ [+ """<meta http-equiv="Content-Type" content='text/html; charset="windows-1252"'>""",+ """<meta http-equiv="Content-Type" content="text/html; charset="windows-1252"">""",+ ],+)+def test_quoted_mime_charset_is_honored(ebooks, meta):+ html = f"<html><head>{meta}</head><body><p>café</p></body></html>"+ assert ebooks._html_text(html.encode("windows-1252")) == "café"+++def test_optional_metadata_and_chapter_markers(ebooks, tokenizer, tmp_path):+ book = make_epub(tmp_path / "book.epub")+ row = ebooks.load_ebook_dataset(+ book, tokenizer = tokenizer, include_metadata = True, chapter_markers = True+ )[0]+ assert {key: row[key] for key in ("title", "author", "genre")} == {+ "title": "Book",+ "author": "Ada",+ "genre": "Fiction",+ }+ assert row["source"] == str(book.resolve())+ assert row["chunk_id"] == 0+ assert row["text"].count("<|chapter_start|>") == 2+ assert row["text"].count("<|chapter_end|>") == 2+++def test_chunk_size_overlap_and_complete_coverage(ebooks, tokenizer, tmp_path):+ words = [f"word{i}" for i in range(23)]+ book = make_epub(+ tmp_path / "book.epub", chapters = {"one.xhtml": "<p>" + " ".join(words) + "</p>"}+ )+ rows = ebooks.load_ebook_dataset(book, tokenizer = tokenizer, chunk_size = 8, overlap = 2)["text"]+ assert [row.split() for row in rows] == [words[0:8], words[6:14], words[12:20], words[18:23]]+ assert all(len(tokenizer.encode(row, add_special_tokens = False)) <= 8 for row in rows)+++def test_prefers_sentence_boundary(ebooks, tokenizer, tmp_path):+ book = make_epub(+ tmp_path / "book.epub",+ chapters = {+ "one.xhtml": "<p>One Two First paragraph. Second paragraph hello world café.</p>"+ },+ )+ rows = ebooks.load_ebook_dataset(book, tokenizer = tokenizer, chunk_size = 7, overlap = 1)["text"]+ assert rows[0] == "One Two First paragraph."+ assert rows[1].startswith("paragraph. Second")+++def test_library_is_recursive_sorted_and_books_never_mix(ebooks, tokenizer, tmp_path):+ (tmp_path / "nested").mkdir()+ make_epub(tmp_path / "z.EPUB", chapters = {"a.xhtml": "<p>Second</p>"})+ make_epub(tmp_path / "nested/a.epub", chapters = {"b.xhtml": "<p>First</p>"})+ (tmp_path / "ignored.txt").write_text("not an ebook")+ rows = ebooks.load_ebook_dataset(tmp_path, tokenizer = tokenizer, file_types = [".EPUB"])["text"]+ assert rows == ["First", "Second"]++[email protected]("file_types", [None, ["notes"]])+def test_new_reader_reuses_discovery_chunking_and_metadata(+ ebooks, tokenizer, tmp_path, monkeypatch, file_types+):+ def read_notes(path, chapter_markers):+ return path.read_text(encoding = "utf-8"), {+ "title": "Notes",+ "author": "Ada",+ "genre": "Fiction",+ }++ # A future format supplies only its reader and one extension mapping entry.+ monkeypatch.setattr(ebooks, "_READERS", {".notes": read_notes}, raising = False)+ (tmp_path / "book.notes").write_text("word0 word1 word2 word3 word4")+ kwargs = {} if file_types is None else {"file_types": file_types}+ dataset = ebooks.load_ebook_dataset(+ tmp_path, tokenizer = tokenizer, chunk_size = 3, overlap = 1, include_metadata = True, **kwargs+ )+ assert dataset["text"] == ["word0 word1 word2", "word2 word3 word4"]+ assert dataset["title"] == ["Notes", "Notes"]+ assert dataset["chunk_id"] == [0, 1]+++ "kwargs",+ [+ {"chunk_size": 0},+ {"chunk_size": 3, "overlap": 3},+ {"overlap": -1},+ {"chunk_size": 1.5},+ {"file_types": ["pdf"]},+ {"file_types": []},+ ],+)+def test_invalid_options(ebooks, tokenizer, tmp_path, kwargs):+ with pytest.raises((ValueError, TypeError)):+ ebooks.load_ebook_dataset(tmp_path, tokenizer = tokenizer, **kwargs)+++def test_requires_tokenizer_instead_of_guessing_token_counts(ebooks, tmp_path):+ with pytest.raises(ValueError, match = "tokenizer"):+ ebooks.load_ebook_dataset(tmp_path)+++def test_empty_missing_unsupported_and_corrupt_inputs(ebooks, tokenizer, tmp_path):+ with pytest.raises(FileNotFoundError):+ ebooks.load_ebook_dataset(tmp_path / "missing.epub", tokenizer = tokenizer)+ with pytest.raises(ValueError, match = "No .*ebook"):+ ebooks.load_ebook_dataset(tmp_path, tokenizer = tokenizer)+ other = tmp_path / "file.txt"+ other.write_text("not a book")+ with pytest.raises(ValueError, match = "Unsupported"):+ ebooks.load_ebook_dataset(other, tokenizer = tokenizer)+ bad = tmp_path / "bad.epub"+ bad.write_bytes(b"not a zip")+ with pytest.raises(ValueError, match = "bad.epub"):+ ebooks.load_ebook_dataset(bad, tokenizer = tokenizer)+ empty = make_epub(tmp_path / "empty.epub", chapters = {"empty.xhtml": "<p> </p>"})+ with pytest.raises(ValueError, match = "readable text"):+ ebooks.load_ebook_dataset(empty, tokenizer = tokenizer)+++def test_non_linear_and_nav_documents_are_not_training_text(ebooks, tokenizer, tmp_path):+ book = make_epub(tmp_path / "book.epub")+ with zipfile.ZipFile(book) as archive:+ files = {n: archive.read(n) for n in archive.namelist()}+ files["Books/content.opf"] = files["Books/content.opf"].replace(+ b'idref="c0"', b'idref="c0" linear="no"'+ )+ with zipfile.ZipFile(book, "w") as archive:+ for name, data in files.items():+ archive.writestr(name, data)+ assert "Chapter Two" not in ebooks.load_ebook_dataset(book, tokenizer = tokenizer)[0]["text"]+++def test_encrypted_chapter_is_rejected_but_obfuscated_fonts_are_allowed(+ ebooks, tokenizer, tmp_path+):+ encryption = """<encryption xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><EncryptedData xmlns="http://www.w3.org/2001/04/xmlenc#"><CipherData><CipherReference URI="Books/first.xhtml"/></CipherData></EncryptedData></encryption>"""+ book = make_epub(tmp_path / "drm.epub", extra = {"META-INF/encryption.xml": encryption})+ with pytest.raises(ValueError, match = "[Ee]ncrypt|DRM"):+ ebooks.load_ebook_dataset(book, tokenizer = tokenizer)+ book = make_epub(+ tmp_path / "font.epub",+ extra = {+ "META-INF/encryption.xml": encryption.replace("Books/first.xhtml", "Fonts/font.otf")+ },+ )+ assert len(ebooks.load_ebook_dataset(book, tokenizer = tokenizer)) == 1+++def test_archive_size_limit(ebooks, tokenizer, tmp_path, monkeypatch):+ book = make_epub(tmp_path / "book.epub")+ monkeypatch.setattr(ebooks, "_MAX_MEMBER_BYTES", 32)+ with pytest.raises(ValueError, match = "size limit"):+ ebooks.load_ebook_dataset(book, tokenizer = tokenizer)+++def test_missing_mobi_converter_has_actionable_error(ebooks, tokenizer, tmp_path, monkeypatch):+ book = tmp_path / "book.mobi"+ book.write_bytes(b"placeholder")+ monkeypatch.setattr(shutil, "which", lambda name: None)+ with pytest.raises(ImportError, match = "[Cc]alibre.*ebook-convert"):+ ebooks.load_ebook_dataset(book, tokenizer = tokenizer)+++def test_mobi_conversion_failure_reports_source(ebooks, tokenizer, tmp_path):+ if not shutil.which("ebook-convert"):+ pytest.skip("Calibre is optional")+ book = tmp_path / "broken.mobi"+ book.write_bytes(b"not a mobi")+ with pytest.raises(ValueError, match = "broken.mobi"):+ ebooks.load_ebook_dataset(book, tokenizer = tokenizer)+++def test_real_mobi_round_trip(ebooks, tokenizer, tmp_path):+ converter = shutil.which("ebook-convert")+ if not converter:+ pytest.skip("Calibre is optional")+ epub = make_epub(tmp_path / "source.epub")+ mobi = tmp_path / "book with spaces.mobi"+ subprocess.run(+ [converter, str(epub), str(mobi), "--no-inline-toc"],+ check = True,+ capture_output = True,+ timeout = 120,+ )+ dataset = ebooks.load_ebook_dataset(mobi, tokenizer = tokenizer, include_metadata = True)+ text = dataset[0]["text"]+ assert text.index("Chapter One") < text.index("Chapter Two")+ assert "First paragraph." in text and "Second paragraph." in text+ assert dataset[0]["title"] == "Book" and dataset[0]["author"] == "Ada"+ assert dataset[0]["source"] == str(mobi.resolve())+++def test_public_exports_cover_gpu_mlx_and_dataprep(ebooks):+ root = Path(__file__).resolve().parents[1] / "unsloth"+ # Execute the MLX loading block without importing its unrelated model backends.+ tree = ast.parse((root / "__init__.py").read_text(encoding = "utf-8"))+ branch = next(+ n+ for n in tree.body+ if isinstance(n, ast.If) and isinstance(n.test, ast.Name) and n.test.id == "_IS_MLX"+ )+ statements = [n for n in branch.body if "_ebook" in ast.unparse(n)]+ namespace = {+ "__file__": str(root / "__init__.py"),+ "importlib": __import__("importlib"),+ "_Path": Path,+ }+ exec(+ compile(ast.Module(body = statements, type_ignores = []), "mlx_ebook_export", "exec"), namespace+ )+ assert callable(namespace.get("load_ebook_dataset"))+ for name in ("_gpu_init.py", "dataprep/__init__.py"):+ imports = [+ n+ for n in ast.walk(ast.parse((root / name).read_text(encoding = "utf-8")))+ if isinstance(n, ast.ImportFrom) and n.module+ ]+ assert any(+ (+ n.module.endswith("ebooks")+ and any(a.name in {"load_ebook_dataset", "*"} for a in n.names)+ )+ or (n.module.endswith("dataprep") and any(a.name == "ebooks" for a in n.names))+ for n in imports+ )++[email protected]("chunk_size", [10, 13, 16, 25])+def test_byte_bpe_chunks_preserve_unicode_and_fit_budget(+ ebooks, byte_tokenizer, tmp_path, chunk_size+):+ tokenizer = byte_tokenizer+ text = "café 🦥 中文 hello world. " * 8+ book = make_epub(tmp_path / "unicode.epub", chapters = {"one.xhtml": "<p>" + text + "</p>"})+ rows = ebooks.load_ebook_dataset(book, tokenizer = tokenizer, chunk_size = chunk_size, overlap = 3)[+ "text"+ ]+ assert all(+ "\ufffd" not in row and len(tokenizer.encode(row, add_special_tokens = False)) <= chunk_size+ for row in rows+ )+ for char in "é🦥中文":+ assert sum(row.count(char) for row in rows) >= text.count(char)+ # A zero-overlap split must not lose or duplicate even a multibyte character.+ rows = ebooks.load_ebook_dataset(book, tokenizer = tokenizer, chunk_size = chunk_size, overlap = 0)[+ "text"+ ]+ assert "".join("".join(rows).split()) == "".join(text.split())+++ "html, chunk_size, overlap, expected",+ [+ ("<p>aaa.</p><p>bbb</p>", 4, 0, ["aaa.", "bbb"]),+ ("<p>a b</p>", 1, 0, ["a", "b"]),+ ("<p>a</p><p>b</p>", 1, 0, ["a", "b"]),+ ("<p>a b</p>", 2, 1, ["a", "b"]),+ ],+)+def test_whitespace_windows_do_not_abort_books(+ ebooks, byte_tokenizer, html, chunk_size, overlap, expected+):+ text = ebooks._html_text(html.encode())+ chunks = list(ebooks._chunk_text(text, byte_tokenizer, chunk_size, overlap))+ assert chunks == expected+ assert all(+ len(byte_tokenizer.encode(chunk, add_special_tokens = False)) <= chunk_size+ for chunk in chunks+ )+++def test_tiny_budget_still_rejects_an_incomplete_unicode_character(ebooks, byte_tokenizer):+ with pytest.raises(ValueError, match = "complete character"):+ list(ebooks._chunk_text("🦥", byte_tokenizer, 1, 0))diff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex 016ea4bd124..14aa836b937 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -372,6 +372,12 @@ if _IS_MLX:_raw_text_spec.loader.exec_module(_raw_text)RawTextDataLoader = _raw_text.RawTextDataLoaderTextPreprocessor = _raw_text.TextPreprocessor+ _ebook_path = _Path(__file__).resolve().parent / "dataprep" / "ebooks.py"+ _ebook_spec = importlib.util.spec_from_file_location("unsloth._mlx_ebooks", _ebook_path)+ _ebook = importlib.util.module_from_spec(_ebook_spec)+ _ebook_spec.loader.exec_module(_ebook)+ load_ebook_dataset = _ebook.load_ebook_dataset+ del _ebook, _ebook_spec, _ebook_pathdel _raw_text, _raw_text_spec, _raw_text_path, _Pathdef _mlx_from_pretrained(*args, **kwargs):diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 8bdccf49aaf..3879fc6316c 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -806,6 +806,9 @@ from .trainer import *# Export dataprep utilities for CLI and downstream usersfrom .dataprep.raw_text import RawTextDataLoader, TextPreprocessor+from .dataprep import ebooks as _ebooks++load_ebook_dataset = _ebooks.load_ebook_datasetfrom unsloth_zoo.rl_environments import (check_python_modules,create_locked_down_function,diff --git a/unsloth/dataprep/__init__.py b/unsloth/dataprep/__init__.pyindex 2c0ab5c22b4..8427ffab3f7 100644--- a/unsloth/dataprep/__init__.py+++ b/unsloth/dataprep/__init__.py@@ -11,3 +11,4 @@from .synthetic import *from .raw_text import *+from .ebooks import *diff --git a/unsloth/dataprep/ebooks.py b/unsloth/dataprep/ebooks.pynew file mode 100644index 00000000000..3e506cdcad4--- /dev/null+++ b/unsloth/dataprep/ebooks.py@@ -0,0 +1,426 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Load EPUB books and, with Calibre installed, MOBI books on CPU.++Pass a Hugging Face fast tokenizer, not model weights. Token counts exclude+special tokens added later by a trainer; reserve space for those in chunk_size.+"""++from bisect import bisect_right+from html.parser import HTMLParser+from pathlib import Path+import posixpath+import re+import shutil+import subprocess+import tempfile+import unicodedata+from urllib.parse import unquote, urlsplit+from xml.etree import ElementTree as ET+from zipfile import BadZipFile, ZipFile++__all__ = ["load_ebook_dataset"]++_MAX_MEMBER_BYTES = 32 * 1024 * 1024+_MAX_BOOK_BYTES = 256 * 1024 * 1024+_BLOCKS = {+ "p",+ "div",+ "section",+ "article",+ "blockquote",+ "li",+ "tr",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+ "pre",+ "ul",+ "ol",+ "dl",+ "dt",+ "dd",+ "table",+ "caption",+ "figure",+ "figcaption",+ "header",+ "footer",+ "main",+ "aside",+ "address",+ "details",+ "summary",+}+_VOID = {+ "area",+ "base",+ "br",+ "col",+ "embed",+ "hr",+ "img",+ "input",+ "link",+ "meta",+ "param",+ "source",+ "track",+ "wbr",+}+_IGNORE = {"head", "script", "style", "nav", "svg"}+++class _BookTextParser(HTMLParser):+ def __init__(self):+ super().__init__(convert_charrefs = True)+ self.parts = []+ self.stack = []+ self.encoding = None+ # <pre> text is kept verbatim and spliced back after whitespace cleanup.+ self.preformatted = []++ def handle_starttag(self, tag, attrs):+ if tag == "body":+ self.handle_endtag("head")+ attrs = dict(attrs)+ if tag == "meta" and self.encoding is None:+ self.encoding = attrs.get("charset")+ if not self.encoding and attrs.get("http-equiv", "").lower() == "content-type":+ charset = re.search(r"charset\s*=\s*[\"']?([\w-]+)", attrs.get("content", ""), re.I)+ if charset:+ self.encoding = charset[1]+ hidden = (+ bool(self.stack and self.stack[-1][1])+ or tag in _IGNORE+ or "hidden" in attrs+ or attrs.get("aria-hidden", "").lower() == "true"+ )+ if not hidden:+ if tag in _BLOCKS or tag == "hr":+ self.parts.append("\n\n")+ elif tag == "br":+ self.parts.append("\n")+ elif tag in {"td", "th"}:+ self.parts.append(" ")+ if tag not in _VOID:+ self.stack.append((tag, hidden))++ def handle_startendtag(self, tag, attrs):+ self.handle_starttag(tag, attrs)+ if tag not in _VOID:+ self.handle_endtag(tag)++ def handle_endtag(self, tag):+ hidden = bool(self.stack and self.stack[-1][1])+ for i in range(len(self.stack) - 1, -1, -1):+ if self.stack[i][0] == tag:+ del self.stack[i:]+ break+ if not hidden and tag in _BLOCKS:+ self.parts.append("\n\n")++ def handle_data(self, data):+ if not self.stack or not self.stack[-1][1]:+ if any(tag == "pre" for tag, _ in self.stack):+ self.parts.append(f"\ue000{len(self.preformatted)}\ue001")+ self.preformatted.append(data)+ else:+ # Source newlines inside an HTML paragraph render as spaces.+ self.parts.append(re.sub(r"\s+", " ", data))++ def unknown_decl(self, data):+ if data.startswith("CDATA["):+ self.handle_data(data[6:])+++def _html_text(data):+ if data.startswith((b"\xff\xfe", b"\xfe\xff")):+ encoding = "utf-16"+ elif data.startswith(b"\x00<"):+ encoding = "utf-16-be"+ elif data.startswith(b"<\x00"):+ encoding = "utf-16-le"+ else:+ declaration = re.match(+ rb'(?:\xef\xbb\xbf)?\s*<\?xml\b[^?]*\bencoding\s*=\s*["\']([\w-]+)', data[:1024], re.I+ )+ if declaration:+ encoding = declaration[1].decode("ascii")+ else:+ header = _BookTextParser()+ header.feed(data[:1024].decode("latin-1"))+ encoding = header.encoding or "utf-8-sig"+ parser = _BookTextParser()+ # A declared UTF-8 encoding keeps a leading BOM, which str.strip() leaves in place.+ parser.feed(data.decode(encoding).lstrip("\ufeff"))+ parser.close()+ text = unicodedata.normalize("NFC", "".join(parser.parts))+ text = re.sub(r"[^\S\n]+", " ", text)+ text = re.sub(r" *\n *", "\n", text)+ text = re.sub(r"\n{3,}", "\n\n", text).strip()+ return re.sub(+ "\ue000(\\d+)\ue001",+ lambda m: unicodedata.normalize("NFC", parser.preformatted[int(m[1])]),+ text,+ )+++def _archive_path(base, href):+ url = urlsplit(href)+ if url.scheme or url.netloc or url.path.startswith("/") or "\\" in url.path:+ raise ValueError(f"Invalid EPUB resource path: {href!r}")+ name = posixpath.normpath(posixpath.join(base, unquote(url.path)))+ if name == ".." or name.startswith("../") or name.startswith("/"):+ raise ValueError(f"Invalid EPUB resource path: {href!r}")+ return name+++def _read_epub(path, chapter_markers):+ with ZipFile(path) as archive:+ total = 0++ def read(name):+ nonlocal total+ info = archive.getinfo(name)+ total += info.file_size+ if info.file_size > _MAX_MEMBER_BYTES or total > _MAX_BOOK_BYTES:+ raise ValueError("EPUB exceeds the uncompressed size limit")+ return archive.read(info)++ container = ET.fromstring(read("META-INF/container.xml"))+ rootfile = container.find(".//{*}rootfile")+ if rootfile is None or not rootfile.get("full-path"):+ raise ValueError("EPUB container has no package document")+ package_path = _archive_path("", rootfile.get("full-path"))+ package = ET.fromstring(read(package_path))+ base = posixpath.dirname(package_path)+ manifest = package.find("{*}manifest")+ spine = package.find("{*}spine")+ if manifest is None or spine is None:+ raise ValueError("EPUB package is missing its manifest or reading order")+ items = {item.get("id"): item for item in manifest}+ encrypted = set()+ if "META-INF/encryption.xml" in archive.namelist():+ encryption = ET.fromstring(read("META-INF/encryption.xml"))+ encrypted = {+ _archive_path("", ref.get("URI", ""))+ for ref in encryption.iter("{http://www.w3.org/2001/04/xmlenc#}CipherReference")+ }+ chapters = []+ for ref in spine:+ if ref.get("linear", "yes") == "no":+ continue+ item = items.get(ref.get("idref"))+ if item is None:+ raise ValueError("EPUB reading order references a missing manifest item")+ if "nav" in item.get("properties", "").split():+ continue+ if item.get("media-type") not in {"application/xhtml+xml", "text/html"}:+ continue+ name = _archive_path(base, item.get("href", ""))+ if name in encrypted:+ raise ValueError("Encrypted/DRM-protected EPUB text is not supported")+ text = _html_text(read(name))+ if text:+ if chapter_markers:+ text = "<|chapter_start|>\n" + text + "\n<|chapter_end|>"+ chapters.append(text)+ metadata = package.find("{*}metadata")+ fields = {"title": "title", "author": "creator", "genre": "subject"}+ values = {}+ for key, field in fields.items():+ elements = (+ []+ if metadata is None+ else metadata.findall("{http://purl.org/dc/elements/1.1/}" + field)+ )+ values[key] = "; ".join(+ " ".join("".join(element.itertext()).split()) for element in elements+ )+ return "\n\n".join(chapters), values+++def _read_mobi(path, chapter_markers):+ converter = shutil.which("ebook-convert")+ if converter is None:+ raise ImportError(+ "MOBI support requires Calibre's ebook-convert on PATH. Install Calibre or use EPUB files."+ )+ with tempfile.TemporaryDirectory(prefix = "unsloth-ebook-") as directory:+ output = Path(directory) / "book.epub"+ subprocess.run(+ [converter, str(path), str(output)], check = True, capture_output = True, timeout = 120+ )+ return _read_epub(output, chapter_markers)+++# Add formats here. Readers return (clean_text, {"title": str, "author": str, "genre": str}).+_READERS = {".epub": _read_epub, ".mobi": _read_mobi}+++def _read_book(path, chapter_markers):+ try:+ return _READERS[path.suffix.lower()](path, chapter_markers)+ except (+ BadZipFile,+ ET.ParseError,+ LookupError,+ OSError,+ ValueError,+ subprocess.SubprocessError,+ ) as exc:+ raise ValueError(f"Could not read ebook '{path}': {exc}") from exc+++def _chunk_text(text, tokenizer, chunk_size, overlap):+ encoded = tokenizer(+ text,+ add_special_tokens = False,+ return_offsets_mapping = True,+ return_attention_mask = False,+ truncation = False,+ verbose = False,+ )+ offsets = [(a, b) for a, b in encoded["offset_mapping"] if b > a]+ if not offsets:+ raise ValueError("The tokenizer produced no tokens for the ebook text")+ ends = [b for _, b in offsets]+ starts = [a for a, _ in offsets]+ boundaries = [+ m.end() for m in re.finditer(r'(?<=[.!?。!?])["\u201d\u2019\)\]]*\s+|\n\s*\n', text)+ ]+ start = 0+ char_start = 0+ while start < len(offsets):+ while char_start < len(text) and text[char_start].isspace():+ char_start += 1+ start = max(start, bisect_right(ends, char_start))+ if start == len(offsets):+ break+ end = min(start + chunk_size, len(offsets))+ char_end = len(text) if end == len(offsets) else ends[end - 1]+ if end < len(offsets):+ boundary_idx = bisect_right(boundaries, char_end) - 1+ if boundary_idx >= 0:+ boundary = boundaries[boundary_idx]+ boundary_token = bisect_right(ends, boundary)+ if (+ boundary_token - start >= max(overlap + 1, chunk_size // 2)+ and text[char_start:boundary].strip()+ ):+ end, char_end = boundary_token, boundary+ # BPE offsets may share a Unicode character; re-encoding also checks the new left boundary.+ while end > start:+ chunk = text[char_start:char_end].strip()+ if chunk and len(tokenizer.encode(chunk, add_special_tokens = False)) <= chunk_size:+ break+ end -= 1+ char_end = ends[end - 1] if end > start else char_start+ if end == start:+ raise ValueError(+ "chunk_size is too small to represent a complete character with this tokenizer"+ )+ yield chunk+ # Consume every byte token of the final character, even when it spans the token window.+ consumed = bisect_right(ends, char_end)+ if consumed == len(offsets):+ break+ if overlap:+ start = max(consumed - overlap, bisect_right(starts, char_start))+ char_start = starts[start]+ else:+ start, char_start = consumed, char_end+++def load_ebook_dataset(+ path,+ file_types = None,+ chunk_size = 4096,+ overlap = 256,+ *,+ tokenizer = None,+ include_metadata = False,+ chapter_markers = False,+):+ """Return a Hugging Face Dataset with a ``text`` column from a book or library.++ ``tokenizer`` must be a Hugging Face fast tokenizer (``use_fast=True``).+ ``chunk_size`` and ``overlap`` count its tokens, excluding trainer-added+ special tokens. Chunks prefer sentence/paragraph boundaries in the latter+ half of a window; overlap uses source-token offsets and may expand to keep+ Unicode characters intact. No model is loaded and nothing is downloaded.++ Directories are searched recursively in sorted path order. Each book is+ chunked separately and any unreadable book raises an error naming its path.+ ``file_types=None`` selects all formats in the internal ``_READERS`` mapping.+ EPUB uses the package's primary reading order, skipping navigation and+ non-linear content. MOBI requires Calibre's ``ebook-convert`` on PATH.+ DRM-protected text is unsupported. Images, CSS and scripts are discarded;+ paragraphs and chapter breaks remain. ``chapter_markers=True`` inserts+ ``<|chapter_start|>`` and ``<|chapter_end|>`` around each content document.++ ``include_metadata=True`` adds ``source`` (absolute original filename),+ ``chunk_id`` (zero-based within each book), ``title``, ``author`` and+ ``genre``. Missing metadata is an empty string; multiple values use ``; ``.++ Example::++ from unsloth import load_ebook_dataset+ dataset = load_ebook_dataset(+ "./my_library/", tokenizer=tokenizer,+ chunk_size=4096, overlap=256, include_metadata=True,+ )++ Reserve room for EOS/BOS or chat-template tokens in ``chunk_size`` when+ passing the text dataset to a trainer.+ """+ if not isinstance(chunk_size, int) or isinstance(chunk_size, bool) or chunk_size <= 0:+ raise ValueError("chunk_size must be a positive integer")+ if not isinstance(overlap, int) or isinstance(overlap, bool) or not 0 <= overlap < chunk_size:+ raise ValueError("overlap must be a non-negative integer smaller than chunk_size")+ if tokenizer is None or not getattr(tokenizer, "is_fast", False):+ raise ValueError(+ "Pass a Hugging Face fast tokenizer (use_fast=True) to measure token counts without loading a model"+ )+ if isinstance(file_types, str):+ file_types = [file_types]+ extensions = (+ set(_READERS)+ if file_types is None+ else {"." + extension.lower().lstrip(".") for extension in file_types}+ )+ if not extensions or not extensions <= _READERS.keys():+ raise ValueError(+ f"file_types must select supported ebook formats: {', '.join(sorted(_READERS))}"+ )+ path = Path(path).expanduser().resolve()+ if not path.exists():+ raise FileNotFoundError(path)+ if path.is_dir():+ files = sorted(p for p in path.rglob("*") if p.is_file() and p.suffix.lower() in extensions)+ if not files:+ raise ValueError(f"No matching ebook files found in '{path}'")+ else:+ if path.suffix.lower() not in extensions:+ raise ValueError(f"Unsupported ebook file type: '{path.suffix}'")+ files = [path]+ columns = {"text": []}+ if include_metadata:+ columns.update({key: [] for key in ("source", "chunk_id", "title", "author", "genre")})+ for book in files:+ text, metadata = _read_book(book, chapter_markers)+ if not text:+ raise ValueError(f"Ebook '{book}' contains no readable text")+ for index, chunk in enumerate(_chunk_text(text, tokenizer, chunk_size, overlap)):+ columns["text"].append(chunk)+ if include_metadata:+ for key, value in {"source": str(book), "chunk_id": index, **metadata}.items():+ columns[key].append(value)+ from datasets import Dataset++ return Dataset.from_dict(columns)