Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): read EPUB chapters in spine order (#3977)

Lukasc9723b1198bf
diff --git a/docreader/parser/epub_parser.py b/docreader/parser/epub_parser.py
index 7e651ab77..41961b7e1 100644
--- a/docreader/parser/epub_parser.py
+++ b/docreader/parser/epub_parser.py
@@ -161,17 +161,6 @@ class EPUBParser(BaseParser):
images: Dict[str, str] = {}
image_aliases: Dict[str, str] = {}
- try:
- toc = book.get_table_of_contents()
- except Exception as e:
- logger.debug("Failed to get TOC: %s, processing all HTML items", e)
- toc = []
-
- html_items = {}
- for item in book.get_items():
- if item.get_type() == ebooklib.ITEM_DOCUMENT:
- html_items[item.get_name()] = item
-
if self.extract_images:
for item in book.get_items():
if item.get_type() == ebooklib.ITEM_IMAGE:
@@ -181,31 +170,39 @@ class EPUBParser(BaseParser):
images[img_path] = base64.b64encode(img_data).decode("utf-8")
self._add_image_aliases(image_aliases, item.get_name(), img_path)
- if toc:
- for item in toc:
- entries = item if isinstance(item, tuple) else (item,)
- for sub in entries:
- if hasattr(sub, "get_name") and sub.get_name() in html_items:
- markdown_parts.append(
- self._process_chapter(
- html_items[sub.get_name()],
- toc_index=len(markdown_parts),
- image_aliases=image_aliases,
- )
- )
-
- if not markdown_parts:
- for _name, item in html_items.items():
- markdown_parts.append(
- self._process_chapter(
- item,
- toc_index=len(markdown_parts),
- image_aliases=image_aliases,
- )
+ for item in self._documents_in_reading_order(book):
+ markdown_parts.append(
+ self._process_chapter(
+ item,
+ toc_index=len(markdown_parts),
+ image_aliases=image_aliases,
)
+ )
return "\n\n".join(part for part in markdown_parts if part.strip()), images
+ @staticmethod
+ def _documents_in_reading_order(book) -> list:
+ """Return the book's documents in spine order.
+
+ The spine is the reading order. ``get_items()`` follows the manifest,
+ which can list files in any order: Standard Ebooks puts the title page
+ after the last chapter there. Documents the spine leaves out follow, in
+ manifest order.
+ """
+ documents = [
+ item
+ for item in book.get_items()
+ if item.get_type() == ebooklib.ITEM_DOCUMENT
+ ]
+ ordered = []
+ for idref, _linear in book.spine:
+ item = book.get_item_with_id(idref)
+ if item in documents and item not in ordered:
+ ordered.append(item)
+ ordered.extend(item for item in documents if item not in ordered)
+ return ordered
+
def _process_chapter(
self,
html_item,
diff --git a/docreader/tests/test_epub_parser.py b/docreader/tests/test_epub_parser.py
index 1b5a665a1..4154f263c 100644
--- a/docreader/tests/test_epub_parser.py
+++ b/docreader/tests/test_epub_parser.py
@@ -87,6 +87,93 @@ class EPUBParserTest(unittest.TestCase):
def test_registry_resolves_epub(self):
self.assertIs(registry.get_parser_class("", "epub"), EPUBParser)
+ def test_chapters_follow_the_spine_not_the_manifest(self):
+ document = EPUBParser(
+ file_name="ordered.epub", file_type="epub"
+ ).parse_into_text(
+ _ordered_epub_bytes(
+ manifest=["three", "title", "one", "two"],
+ spine=["title", "one", "two", "three"],
+ svg_cover_in_spine=True,
+ )
+ )
+
+ positions = [
+ document.content.index(f"Body of {name}.")
+ for name in ("title", "one", "two", "three")
+ ]
+ self.assertEqual(positions, sorted(positions))
+ # An SVG in the spine is not a document and stays out, as before.
+ self.assertNotIn("COVER ART", document.content)
+
+ def test_documents_outside_the_spine_follow_it(self):
+ document = EPUBParser(
+ file_name="ordered.epub", file_type="epub"
+ ).parse_into_text(
+ _ordered_epub_bytes(
+ manifest=["appendix", "two", "one"],
+ spine=["one", "two"],
+ )
+ )
+
+ positions = [
+ document.content.index(f"Body of {name}.")
+ for name in ("one", "two", "appendix")
+ ]
+ self.assertEqual(positions, sorted(positions))
+
+ def test_a_document_listed_twice_in_the_spine_is_read_once(self):
+ document = EPUBParser(
+ file_name="ordered.epub", file_type="epub"
+ ).parse_into_text(
+ _ordered_epub_bytes(manifest=["one", "two"], spine=["one", "two", "one"])
+ )
+
+ self.assertEqual(document.content.count("Body of one."), 1)
+
+
+def _ordered_epub_bytes(
+ manifest: list[str], spine: list[str], svg_cover_in_spine: bool = False
+) -> bytes:
+ """Write an EPUB whose manifest lists the documents in ``manifest`` order
+ and whose spine reads them in ``spine`` order."""
+ book = epub.EpubBook()
+ book.set_identifier("ordered-epub")
+ book.set_title("Ordered EPUB")
+ book.set_language("en")
+
+ chapters = {}
+ for name in manifest:
+ chapter = epub.EpubHtml(title=name, file_name=f"text/{name}.xhtml", lang="en")
+ chapter.content = f"<html><body><p>Body of {name}.</p></body></html>"
+ book.add_item(chapter)
+ chapters[name] = chapter
+ book.add_item(epub.EpubNcx())
+ book.add_item(epub.EpubNav())
+ book.spine = ["nav"] + [chapters[name] for name in spine]
+ if svg_cover_in_spine:
+ cover = epub.EpubItem(
+ uid="cover-svg",
+ file_name="images/cover.svg",
+ media_type="image/svg+xml",
+ content=(
+ b'<svg xmlns="http://www.w3.org/2000/svg">'
+ b"<text>COVER ART</text></svg>"
+ ),
+ )
+ book.add_item(cover)
+ book.spine.insert(1, cover)
+
+ with tempfile.NamedTemporaryFile(suffix=".epub", delete=False) as handle:
+ path = handle.name
+ try:
+ epub.write_epub(path, book)
+ with open(path, "rb") as handle:
+ return handle.read()
+ finally:
+ if os.path.exists(path):
+ os.unlink(path)
+
if __name__ == "__main__":
unittest.main()