Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): honor explicit MHTML root parts (#4636)

Captaindfa3e5c52bfe
diff --git a/docreader/parser/mhtml_parser.py b/docreader/parser/mhtml_parser.py
index 113493be0..4fdd143fd 100644
--- a/docreader/parser/mhtml_parser.py
+++ b/docreader/parser/mhtml_parser.py
@@ -9,6 +9,7 @@ import html
import logging
import os
from email.header import decode_header
+from email.message import Message
from urllib.parse import unquote, urljoin, urlparse
import uuid
from typing import Dict
@@ -105,6 +106,7 @@ class MHTMLParser(BaseParser):
"Parsing MHTML file: %s, size: %d bytes", self.file_name, len(content)
)
msg = email.message_from_bytes(content)
+ explicit_root = self._explicit_html_root(msg)
html_parts = []
images: Dict[str, str] = {}
@@ -117,8 +119,9 @@ class MHTMLParser(BaseParser):
if content_type == "text/html":
payload = part.get_payload(decode=True)
- if not payload:
+ if not payload and part is not explicit_root:
continue
+ payload = payload or b""
charset = part.get_content_charset() or "utf-8"
try:
html_text = payload.decode(charset, errors="ignore")
@@ -129,6 +132,7 @@ class MHTMLParser(BaseParser):
"content": html_text,
"location": location,
"size": len(html_text),
+ "is_root": part is explicit_root,
}
)
elif content_type.startswith("image/") and self.extract_images:
@@ -162,9 +166,12 @@ class MHTMLParser(BaseParser):
return Document(content=markdown_text, images=images, metadata=metadata)
def _select_main_html(self, html_parts) -> dict:
- """Pick the largest non-ad HTML part as the main document body."""
+ """Honor an explicit root before the largest non-ad HTML fallback."""
if not html_parts:
return {}
+ for part in html_parts:
+ if part.get("is_root"):
+ return part
def is_ad(location: str) -> bool:
loc = _header_str(location)
@@ -188,6 +195,38 @@ class MHTMLParser(BaseParser):
)
return largest
+ @staticmethod
+ def _explicit_html_root(msg: Message) -> Message | None:
+ """Resolve a multipart/related start Content-ID to its HTML root."""
+ start = _header_str(msg.get_param("start", "")).strip().strip("<>")
+ if (
+ not start
+ or msg.get_content_type() != "multipart/related"
+ or not msg.is_multipart()
+ ):
+ return None
+ for part in msg.get_payload():
+ content_id = _header_str(part.get("Content-ID", "")).strip().strip("<>")
+ if content_id != start:
+ continue
+ if part.get_content_type() == "text/html":
+ return part
+ if (
+ part.get_content_type() == "multipart/alternative"
+ and part.is_multipart()
+ ):
+ # MIME alternatives are ordered from least to most faithful.
+ return next(
+ (
+ alternative
+ for alternative in reversed(part.get_payload())
+ if alternative.get_content_type() == "text/html"
+ ),
+ None,
+ )
+ return None
+ return None
+
@staticmethod
def _add_image_aliases(
image_aliases: Dict[str, str], part, image_path: str
diff --git a/docreader/tests/test_mhtml_root.py b/docreader/tests/test_mhtml_root.py
new file mode 100644
index 000000000..5aed4797b
--- /dev/null
+++ b/docreader/tests/test_mhtml_root.py
@@ -0,0 +1,110 @@
+import unittest
+from email.message import EmailMessage
+
+from docreader.parser.mhtml_parser import MHTMLParser
+from docreader.parser.parser import Parser
+
+
+def _archive(*, start="<root@local>", root_first=False, alternative=False):
+ archive = EmailMessage()
+ archive.make_related()
+ archive.set_param("type", "text/html")
+ if start is not None:
+ archive.set_param("start", start)
+
+ root = EmailMessage()
+ root.set_content(
+ '<h1>Selected article</h1><img src="figure.png" alt="figure">',
+ subtype="html",
+ )
+ root["Content-Location"] = "https://example.com/article/index.html"
+ if alternative:
+ container = EmailMessage()
+ container.set_content("Plain text alternative")
+ container.make_alternative()
+ container.attach(root)
+ root = container
+ archive.set_param("type", "multipart/alternative")
+ root["Content-ID"] = "<root@local>"
+
+ frame = EmailMessage()
+ frame.set_content("<h1>Unrelated frame</h1>" + "frame body " * 100, subtype="html")
+ frame["Content-ID"] = "<frame@local>"
+ frame["Content-Location"] = "https://example.com/frame.html"
+ for part in [root, frame] if root_first else [frame, root]:
+ archive.attach(part)
+
+ image = EmailMessage()
+ image.set_content(b"image fixture", maintype="image", subtype="png")
+ image["Content-ID"] = "<image@local>"
+ image["Content-Location"] = "https://example.com/article/figure.png"
+ archive.attach(image)
+ return archive
+
+
+def _parse(archive):
+ return MHTMLParser(file_name="saved.mhtml", file_type="mhtml").parse(
+ archive.as_bytes()
+ )
+
+
+class MHTMLRootTest(unittest.TestCase):
+ def assert_article(self, document):
+ self.assertIn("Selected article", document.content)
+ self.assertNotIn("Unrelated frame", document.content)
+ self.assertIn("images/figure.png", document.content)
+ self.assertEqual(document.metadata["image_count"], 1)
+
+ def test_start_selects_shorter_html_root_in_either_position(self):
+ for root_first in (False, True):
+ with self.subTest(root_first=root_first):
+ self.assert_article(_parse(_archive(root_first=root_first)))
+
+ def test_start_selects_html_inside_alternative_root(self):
+ self.assert_article(_parse(_archive(alternative=True)))
+
+ def test_alternative_root_uses_last_supported_html_part(self):
+ archive = _archive(alternative=True)
+ container = archive.get_payload()[1]
+ less_faithful = EmailMessage()
+ less_faithful.set_content("<p>Older rendering</p>" * 100, subtype="html")
+ container.get_payload().insert(1, less_faithful)
+ document = _parse(archive)
+ self.assert_article(document)
+ self.assertNotIn("Older rendering", document.content)
+
+ def test_start_is_used_by_file_parser_facade(self):
+ archive = _archive()
+ document = Parser().parse_file("saved.mhtml", "mhtml", archive.as_bytes())
+ self.assert_article(document)
+
+ def test_explicit_root_takes_precedence_over_ad_location_heuristic(self):
+ archive = _archive()
+ root = archive.get_payload()[1]
+ root.replace_header("Content-Location", "https://example.com/analytics-report")
+ document = _parse(archive)
+ self.assertIn("Selected article", document.content)
+ self.assertNotIn("Unrelated frame", document.content)
+
+ def test_missing_or_unknown_start_keeps_existing_selection(self):
+ for start in (None, "<missing@local>"):
+ with self.subTest(start=start):
+ document = _parse(_archive(start=start))
+ self.assertIn("Unrelated frame", document.content)
+ self.assertNotIn("Selected article", document.content)
+
+ def test_non_html_start_keeps_existing_selection(self):
+ document = _parse(_archive(start="<image@local>"))
+ self.assertIn("Unrelated frame", document.content)
+
+ def test_non_related_container_keeps_existing_selection(self):
+ archive = _archive()
+ archive.set_type("multipart/mixed")
+ document = _parse(archive)
+ self.assertIn("Unrelated frame", document.content)
+
+ def test_empty_explicit_html_root_does_not_select_an_unrelated_frame(self):
+ archive = _archive()
+ archive.get_payload()[1].set_payload("")
+ document = _parse(archive)
+ self.assertNotIn("Unrelated frame", document.content)