Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): honor explicit MHTML root parts (#4636)
Captaindfa3e5c52bfe
diff --git a/docreader/parser/mhtml_parser.py b/docreader/parser/mhtml_parser.pyindex 113493be0..4fdd143fd 100644--- a/docreader/parser/mhtml_parser.py+++ b/docreader/parser/mhtml_parser.py@@ -9,6 +9,7 @@ import htmlimport loggingimport osfrom email.header import decode_header+from email.message import Messagefrom urllib.parse import unquote, urljoin, urlparseimport uuidfrom typing import Dict@@ -105,6 +106,7 @@ class MHTMLParser(BaseParser):"Parsing MHTML file: %s, size: %d bytes", self.file_name, len(content))msg = email.message_from_bytes(content)+ explicit_root = self._explicit_html_root(msg)html_parts = []images: Dict[str, str] = {}@@ -117,8 +119,9 @@ class MHTMLParser(BaseParser):if content_type == "text/html":payload = part.get_payload(decode=True)- if not payload:+ if not payload and part is not explicit_root:continue+ payload = payload or b""charset = part.get_content_charset() or "utf-8"try:html_text = payload.decode(charset, errors="ignore")@@ -129,6 +132,7 @@ class MHTMLParser(BaseParser):"content": html_text,"location": location,"size": len(html_text),+ "is_root": part is explicit_root,})elif content_type.startswith("image/") and self.extract_images:@@ -162,9 +166,12 @@ class MHTMLParser(BaseParser):return Document(content=markdown_text, images=images, metadata=metadata)def _select_main_html(self, html_parts) -> dict:- """Pick the largest non-ad HTML part as the main document body."""+ """Honor an explicit root before the largest non-ad HTML fallback."""if not html_parts:return {}+ for part in html_parts:+ if part.get("is_root"):+ return partdef is_ad(location: str) -> bool:loc = _header_str(location)@@ -188,6 +195,38 @@ class MHTMLParser(BaseParser):)return largest+ @staticmethod+ def _explicit_html_root(msg: Message) -> Message | None:+ """Resolve a multipart/related start Content-ID to its HTML root."""+ start = _header_str(msg.get_param("start", "")).strip().strip("<>")+ if (+ not start+ or msg.get_content_type() != "multipart/related"+ or not msg.is_multipart()+ ):+ return None+ for part in msg.get_payload():+ content_id = _header_str(part.get("Content-ID", "")).strip().strip("<>")+ if content_id != start:+ continue+ if part.get_content_type() == "text/html":+ return part+ if (+ part.get_content_type() == "multipart/alternative"+ and part.is_multipart()+ ):+ # MIME alternatives are ordered from least to most faithful.+ return next(+ (+ alternative+ for alternative in reversed(part.get_payload())+ if alternative.get_content_type() == "text/html"+ ),+ None,+ )+ return None+ return None+@staticmethoddef _add_image_aliases(image_aliases: Dict[str, str], part, image_path: strdiff --git a/docreader/tests/test_mhtml_root.py b/docreader/tests/test_mhtml_root.pynew file mode 100644index 000000000..5aed4797b--- /dev/null+++ b/docreader/tests/test_mhtml_root.py@@ -0,0 +1,110 @@+import unittest+from email.message import EmailMessage++from docreader.parser.mhtml_parser import MHTMLParser+from docreader.parser.parser import Parser+++def _archive(*, start="<root@local>", root_first=False, alternative=False):+ archive = EmailMessage()+ archive.make_related()+ archive.set_param("type", "text/html")+ if start is not None:+ archive.set_param("start", start)++ root = EmailMessage()+ root.set_content(+ '<h1>Selected article</h1><img src="figure.png" alt="figure">',+ subtype="html",+ )+ root["Content-Location"] = "https://example.com/article/index.html"+ if alternative:+ container = EmailMessage()+ container.set_content("Plain text alternative")+ container.make_alternative()+ container.attach(root)+ root = container+ archive.set_param("type", "multipart/alternative")+ root["Content-ID"] = "<root@local>"++ frame = EmailMessage()+ frame.set_content("<h1>Unrelated frame</h1>" + "frame body " * 100, subtype="html")+ frame["Content-ID"] = "<frame@local>"+ frame["Content-Location"] = "https://example.com/frame.html"+ for part in [root, frame] if root_first else [frame, root]:+ archive.attach(part)++ image = EmailMessage()+ image.set_content(b"image fixture", maintype="image", subtype="png")+ image["Content-ID"] = "<image@local>"+ image["Content-Location"] = "https://example.com/article/figure.png"+ archive.attach(image)+ return archive+++def _parse(archive):+ return MHTMLParser(file_name="saved.mhtml", file_type="mhtml").parse(+ archive.as_bytes()+ )+++class MHTMLRootTest(unittest.TestCase):+ def assert_article(self, document):+ self.assertIn("Selected article", document.content)+ self.assertNotIn("Unrelated frame", document.content)+ self.assertIn("images/figure.png", document.content)+ self.assertEqual(document.metadata["image_count"], 1)++ def test_start_selects_shorter_html_root_in_either_position(self):+ for root_first in (False, True):+ with self.subTest(root_first=root_first):+ self.assert_article(_parse(_archive(root_first=root_first)))++ def test_start_selects_html_inside_alternative_root(self):+ self.assert_article(_parse(_archive(alternative=True)))++ def test_alternative_root_uses_last_supported_html_part(self):+ archive = _archive(alternative=True)+ container = archive.get_payload()[1]+ less_faithful = EmailMessage()+ less_faithful.set_content("<p>Older rendering</p>" * 100, subtype="html")+ container.get_payload().insert(1, less_faithful)+ document = _parse(archive)+ self.assert_article(document)+ self.assertNotIn("Older rendering", document.content)++ def test_start_is_used_by_file_parser_facade(self):+ archive = _archive()+ document = Parser().parse_file("saved.mhtml", "mhtml", archive.as_bytes())+ self.assert_article(document)++ def test_explicit_root_takes_precedence_over_ad_location_heuristic(self):+ archive = _archive()+ root = archive.get_payload()[1]+ root.replace_header("Content-Location", "https://example.com/analytics-report")+ document = _parse(archive)+ self.assertIn("Selected article", document.content)+ self.assertNotIn("Unrelated frame", document.content)++ def test_missing_or_unknown_start_keeps_existing_selection(self):+ for start in (None, "<missing@local>"):+ with self.subTest(start=start):+ document = _parse(_archive(start=start))+ self.assertIn("Unrelated frame", document.content)+ self.assertNotIn("Selected article", document.content)++ def test_non_html_start_keeps_existing_selection(self):+ document = _parse(_archive(start="<image@local>"))+ self.assertIn("Unrelated frame", document.content)++ def test_non_related_container_keeps_existing_selection(self):+ archive = _archive()+ archive.set_type("multipart/mixed")+ document = _parse(archive)+ self.assertIn("Unrelated frame", document.content)++ def test_empty_explicit_html_root_does_not_select_an_unrelated_frame(self):+ archive = _archive()+ archive.get_payload()[1].set_payload("")+ document = _parse(archive)+ self.assertNotIn("Unrelated frame", document.content)