Skip to content

Python tool for converting files and office documents to Markdown.

fix(rss): preserve Atom XHTML content (#2297)

will wangee6fe25e3e30
diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.py
index 49aef4a3..5374933f 100644
--- a/packages/markitdown/src/markitdown/converters/_rss_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py
@@ -1,6 +1,7 @@
import warnings
from defusedxml import minidom
+from xml.dom import Node
from xml.dom.minidom import Document, Element
from typing import BinaryIO, Any, Union
from bs4 import BeautifulSoup
@@ -27,6 +28,8 @@ CANDIDATE_FILE_EXTENSIONS = [
".xml",
]
+XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
+
class RssConverter(DocumentConverter):
"""Convert RSS / Atom type to markdown"""
@@ -119,9 +122,9 @@ class RssConverter(DocumentConverter):
md_text += f"{subtitle}\n"
for entry in entries:
entry_title = self._get_data_by_tag_name(entry, "title")
- entry_summary = self._get_data_by_tag_name(entry, "summary")
+ entry_summary = self._get_atom_content(entry, "summary")
entry_updated = self._get_data_by_tag_name(entry, "updated")
- entry_content = self._get_data_by_tag_name(entry, "content")
+ entry_content = self._get_atom_content(entry, "content")
if entry_title:
md_text += f"\n## {entry_title}\n"
@@ -137,6 +140,37 @@ class RssConverter(DocumentConverter):
title=title,
)
+ def _get_atom_content(self, entry: Element, tag_name: str) -> Union[str, None]:
+ nodes = entry.getElementsByTagName(tag_name)
+ if not nodes:
+ return None
+
+ node = nodes[0]
+ if node.getAttribute("type").lower() != "xhtml":
+ return self._get_data_by_tag_name(entry, tag_name)
+
+ return "".join(
+ self._localize_xhtml_names(child.cloneNode(True)).toxml()
+ for child in node.childNodes
+ if child.nodeType == Node.ELEMENT_NODE
+ )
+
+ def _localize_xhtml_names(self, node: Node) -> Node:
+ """Rewrite prefixed XHTML element names to their local HTML names.
+
+ Atom permits XHTML content to be namespace-prefixed (e.g. ``x:strong``).
+ The downstream HTML converter dispatches on HTML tag names, so the
+ prefix has to be dropped or the element is treated as an unknown tag
+ and its formatting is lost.
+ """
+ if node.nodeType == Node.ELEMENT_NODE:
+ if node.prefix and node.namespaceURI == XHTML_NAMESPACE:
+ node.tagName = node.nodeName = node.localName
+ node.prefix = None
+ for child in node.childNodes:
+ self._localize_xhtml_names(child)
+ return node
+
def _parse_rss_type(
self, doc: Document, *, strict: bool = False
) -> DocumentConverterResult:
diff --git a/packages/markitdown/tests/test_rss_converter.py b/packages/markitdown/tests/test_rss_converter.py
new file mode 100644
index 00000000..d443e59e
--- /dev/null
+++ b/packages/markitdown/tests/test_rss_converter.py
@@ -0,0 +1,74 @@
+import io
+
+from markitdown import StreamInfo
+from markitdown.converters import RssConverter
+
+
+def test_atom_xhtml_content_is_preserved() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="xhtml">
+ <div xmlns="http://www.w3.org/1999/xhtml">
+ <p>Read the <strong>important details</strong>.</p>
+ </div>
+ </content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "Read the **important details**." in result.markdown
+
+
+def test_atom_xhtml_summary_is_preserved() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <summary type="xhtml">
+ <div xmlns="http://www.w3.org/1999/xhtml">
+ <p>A <em>structured</em> summary.</p>
+ </div>
+ </summary>
+ <content type="text">Plain text content.</content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "A *structured* summary." in result.markdown
+ assert "Plain text content." in result.markdown
+
+
+def test_atom_prefixed_xhtml_content_is_preserved() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom" xmlns:x="http://www.w3.org/1999/xhtml">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="xhtml">
+ <x:div>
+ <x:p>Hello <x:strong>bold</x:strong> and <x:em>italic</x:em>.</x:p>
+ <x:a href="https://example.com">link</x:a>
+ </x:div>
+ </content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "Hello **bold** and *italic*." in result.markdown
+ assert "[link](https://example.com)" in result.markdown