Python tool for converting files and office documents to Markdown.
fix(rss): preserve Atom XHTML content (#2297)
will wangee6fe25e3e30
diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.pyindex 49aef4a3..5374933f 100644--- a/packages/markitdown/src/markitdown/converters/_rss_converter.py+++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py@@ -1,6 +1,7 @@import warningsfrom defusedxml import minidom+from xml.dom import Nodefrom xml.dom.minidom import Document, Elementfrom typing import BinaryIO, Any, Unionfrom bs4 import BeautifulSoup@@ -27,6 +28,8 @@ CANDIDATE_FILE_EXTENSIONS = [".xml",]+XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"+class RssConverter(DocumentConverter):"""Convert RSS / Atom type to markdown"""@@ -119,9 +122,9 @@ class RssConverter(DocumentConverter):md_text += f"{subtitle}\n"for entry in entries:entry_title = self._get_data_by_tag_name(entry, "title")- entry_summary = self._get_data_by_tag_name(entry, "summary")+ entry_summary = self._get_atom_content(entry, "summary")entry_updated = self._get_data_by_tag_name(entry, "updated")- entry_content = self._get_data_by_tag_name(entry, "content")+ entry_content = self._get_atom_content(entry, "content")if entry_title:md_text += f"\n## {entry_title}\n"@@ -137,6 +140,37 @@ class RssConverter(DocumentConverter):title=title,)+ def _get_atom_content(self, entry: Element, tag_name: str) -> Union[str, None]:+ nodes = entry.getElementsByTagName(tag_name)+ if not nodes:+ return None++ node = nodes[0]+ if node.getAttribute("type").lower() != "xhtml":+ return self._get_data_by_tag_name(entry, tag_name)++ return "".join(+ self._localize_xhtml_names(child.cloneNode(True)).toxml()+ for child in node.childNodes+ if child.nodeType == Node.ELEMENT_NODE+ )++ def _localize_xhtml_names(self, node: Node) -> Node:+ """Rewrite prefixed XHTML element names to their local HTML names.++ Atom permits XHTML content to be namespace-prefixed (e.g. ``x:strong``).+ The downstream HTML converter dispatches on HTML tag names, so the+ prefix has to be dropped or the element is treated as an unknown tag+ and its formatting is lost.+ """+ if node.nodeType == Node.ELEMENT_NODE:+ if node.prefix and node.namespaceURI == XHTML_NAMESPACE:+ node.tagName = node.nodeName = node.localName+ node.prefix = None+ for child in node.childNodes:+ self._localize_xhtml_names(child)+ return node+def _parse_rss_type(self, doc: Document, *, strict: bool = False) -> DocumentConverterResult:diff --git a/packages/markitdown/tests/test_rss_converter.py b/packages/markitdown/tests/test_rss_converter.pynew file mode 100644index 00000000..d443e59e--- /dev/null+++ b/packages/markitdown/tests/test_rss_converter.py@@ -0,0 +1,74 @@+import io++from markitdown import StreamInfo+from markitdown.converters import RssConverter+++def test_atom_xhtml_content_is_preserved() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="xhtml">+ <div xmlns="http://www.w3.org/1999/xhtml">+ <p>Read the <strong>important details</strong>.</p>+ </div>+ </content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "Read the **important details**." in result.markdown+++def test_atom_xhtml_summary_is_preserved() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <summary type="xhtml">+ <div xmlns="http://www.w3.org/1999/xhtml">+ <p>A <em>structured</em> summary.</p>+ </div>+ </summary>+ <content type="text">Plain text content.</content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "A *structured* summary." in result.markdown+ assert "Plain text content." in result.markdown+++def test_atom_prefixed_xhtml_content_is_preserved() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom" xmlns:x="http://www.w3.org/1999/xhtml">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="xhtml">+ <x:div>+ <x:p>Hello <x:strong>bold</x:strong> and <x:em>italic</x:em>.</x:p>+ <x:a href="https://example.com">link</x:a>+ </x:div>+ </content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "Hello **bold** and *italic*." in result.markdown+ assert "[link](https://example.com)" in result.markdown