Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): read EPUB chapters in spine order (#3977)
Lukasc9723b1198bf
diff --git a/docreader/parser/epub_parser.py b/docreader/parser/epub_parser.pyindex 7e651ab77..41961b7e1 100644--- a/docreader/parser/epub_parser.py+++ b/docreader/parser/epub_parser.py@@ -161,17 +161,6 @@ class EPUBParser(BaseParser):images: Dict[str, str] = {}image_aliases: Dict[str, str] = {}- try:- toc = book.get_table_of_contents()- except Exception as e:- logger.debug("Failed to get TOC: %s, processing all HTML items", e)- toc = []-- html_items = {}- for item in book.get_items():- if item.get_type() == ebooklib.ITEM_DOCUMENT:- html_items[item.get_name()] = item-if self.extract_images:for item in book.get_items():if item.get_type() == ebooklib.ITEM_IMAGE:@@ -181,31 +170,39 @@ class EPUBParser(BaseParser):images[img_path] = base64.b64encode(img_data).decode("utf-8")self._add_image_aliases(image_aliases, item.get_name(), img_path)- if toc:- for item in toc:- entries = item if isinstance(item, tuple) else (item,)- for sub in entries:- if hasattr(sub, "get_name") and sub.get_name() in html_items:- markdown_parts.append(- self._process_chapter(- html_items[sub.get_name()],- toc_index=len(markdown_parts),- image_aliases=image_aliases,- )- )-- if not markdown_parts:- for _name, item in html_items.items():- markdown_parts.append(- self._process_chapter(- item,- toc_index=len(markdown_parts),- image_aliases=image_aliases,- )+ for item in self._documents_in_reading_order(book):+ markdown_parts.append(+ self._process_chapter(+ item,+ toc_index=len(markdown_parts),+ image_aliases=image_aliases,)+ )return "\n\n".join(part for part in markdown_parts if part.strip()), images+ @staticmethod+ def _documents_in_reading_order(book) -> list:+ """Return the book's documents in spine order.++ The spine is the reading order. ``get_items()`` follows the manifest,+ which can list files in any order: Standard Ebooks puts the title page+ after the last chapter there. Documents the spine leaves out follow, in+ manifest order.+ """+ documents = [+ item+ for item in book.get_items()+ if item.get_type() == ebooklib.ITEM_DOCUMENT+ ]+ ordered = []+ for idref, _linear in book.spine:+ item = book.get_item_with_id(idref)+ if item in documents and item not in ordered:+ ordered.append(item)+ ordered.extend(item for item in documents if item not in ordered)+ return ordered+def _process_chapter(self,html_item,diff --git a/docreader/tests/test_epub_parser.py b/docreader/tests/test_epub_parser.pyindex 1b5a665a1..4154f263c 100644--- a/docreader/tests/test_epub_parser.py+++ b/docreader/tests/test_epub_parser.py@@ -87,6 +87,93 @@ class EPUBParserTest(unittest.TestCase):def test_registry_resolves_epub(self):self.assertIs(registry.get_parser_class("", "epub"), EPUBParser)+ def test_chapters_follow_the_spine_not_the_manifest(self):+ document = EPUBParser(+ file_name="ordered.epub", file_type="epub"+ ).parse_into_text(+ _ordered_epub_bytes(+ manifest=["three", "title", "one", "two"],+ spine=["title", "one", "two", "three"],+ svg_cover_in_spine=True,+ )+ )++ positions = [+ document.content.index(f"Body of {name}.")+ for name in ("title", "one", "two", "three")+ ]+ self.assertEqual(positions, sorted(positions))+ # An SVG in the spine is not a document and stays out, as before.+ self.assertNotIn("COVER ART", document.content)++ def test_documents_outside_the_spine_follow_it(self):+ document = EPUBParser(+ file_name="ordered.epub", file_type="epub"+ ).parse_into_text(+ _ordered_epub_bytes(+ manifest=["appendix", "two", "one"],+ spine=["one", "two"],+ )+ )++ positions = [+ document.content.index(f"Body of {name}.")+ for name in ("one", "two", "appendix")+ ]+ self.assertEqual(positions, sorted(positions))++ def test_a_document_listed_twice_in_the_spine_is_read_once(self):+ document = EPUBParser(+ file_name="ordered.epub", file_type="epub"+ ).parse_into_text(+ _ordered_epub_bytes(manifest=["one", "two"], spine=["one", "two", "one"])+ )++ self.assertEqual(document.content.count("Body of one."), 1)+++def _ordered_epub_bytes(+ manifest: list[str], spine: list[str], svg_cover_in_spine: bool = False+) -> bytes:+ """Write an EPUB whose manifest lists the documents in ``manifest`` order+ and whose spine reads them in ``spine`` order."""+ book = epub.EpubBook()+ book.set_identifier("ordered-epub")+ book.set_title("Ordered EPUB")+ book.set_language("en")++ chapters = {}+ for name in manifest:+ chapter = epub.EpubHtml(title=name, file_name=f"text/{name}.xhtml", lang="en")+ chapter.content = f"<html><body><p>Body of {name}.</p></body></html>"+ book.add_item(chapter)+ chapters[name] = chapter+ book.add_item(epub.EpubNcx())+ book.add_item(epub.EpubNav())+ book.spine = ["nav"] + [chapters[name] for name in spine]+ if svg_cover_in_spine:+ cover = epub.EpubItem(+ uid="cover-svg",+ file_name="images/cover.svg",+ media_type="image/svg+xml",+ content=(+ b'<svg xmlns="http://www.w3.org/2000/svg">'+ b"<text>COVER ART</text></svg>"+ ),+ )+ book.add_item(cover)+ book.spine.insert(1, cover)++ with tempfile.NamedTemporaryFile(suffix=".epub", delete=False) as handle:+ path = handle.name+ try:+ epub.write_epub(path, book)+ with open(path, "rb") as handle:+ return handle.read()+ finally:+ if os.path.exists(path):+ os.unlink(path)+if __name__ == "__main__":unittest.main()