Skip to content

Python tool for converting files and office documents to Markdown.

fix(rss): treat Atom text content and summary as plain text (#2374)

Lazizbek Ergashev697a2a0e0b95
diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.py
index e0cc9a4e..6b7b1201 100644
--- a/packages/markitdown/src/markitdown/converters/_rss_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py
@@ -1,3 +1,4 @@
+import textwrap
import warnings
from defusedxml import minidom
@@ -31,6 +32,35 @@ CANDIDATE_FILE_EXTENSIONS = [
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
+def _atom_content_kind(content_type: str, *, allow_media_types: bool) -> str:
+ """Classify an Atom ``type`` attribute as text, html, xhtml or binary.
+
+ Text constructs (title, summary, rights) admit only the ``text``/``html``/
+ ``xhtml`` keywords, while atom:content additionally admits a MIME media
+ type -- ``text/html`` is markup, any other ``text/*`` is plain text, inline
+ XML is markup, and everything else is base64-encoded binary.
+ See RFC 4287 sections 3.1 and 4.1.3.1.
+ """
+ content_type = content_type.strip().lower()
+ if content_type in ("", "text"):
+ return "text"
+ if content_type in ("html", "xhtml"):
+ return content_type
+ if not allow_media_types:
+ # An out-of-spec type on a text construct; the safe reading is text.
+ return "text"
+
+ media_type = content_type.split(";", 1)[0].strip()
+ if media_type == "text/html":
+ return "html"
+ if media_type.endswith(("+xml", "/xml")):
+ # Inline XML, carried as child elements just like xhtml content.
+ return "xhtml"
+ if media_type.startswith("text/"):
+ return "text"
+ return "binary"
+
+
class RssConverter(DocumentConverter):
"""Convert RSS / Atom type to markdown"""
@@ -122,38 +152,80 @@ class RssConverter(DocumentConverter):
md_text += f"{subtitle}\n"
for entry in entries:
entry_title = self._get_data_by_tag_name(entry, "title")
- entry_summary = self._get_atom_content(entry, "summary")
+ entry_summary, summary_is_markup = self._get_atom_content(entry, "summary")
entry_updated = self._get_data_by_tag_name(entry, "updated")
- entry_content = self._get_atom_content(entry, "content")
+ entry_content, content_is_markup = self._get_atom_content(entry, "content")
if entry_title:
md_text += f"\n## {entry_title}\n"
if entry_updated:
md_text += f"Updated on: {entry_updated}\n"
if entry_summary:
- md_text += self._parse_content(entry_summary, strict=strict)
+ md_text += self._render_atom_content(
+ entry_summary, is_markup=summary_is_markup, strict=strict
+ )
if entry_content:
- md_text += self._parse_content(entry_content, strict=strict)
+ md_text += self._render_atom_content(
+ entry_content, is_markup=content_is_markup, strict=strict
+ )
return DocumentConverterResult(
markdown=md_text,
title=title,
)
- def _get_atom_content(self, entry: Element, tag_name: str) -> Union[str, None]:
+ def _get_atom_content(
+ self, entry: Element, tag_name: str
+ ) -> tuple[Union[str, None], bool]:
+ """Return an Atom summary or content value, and whether it is markup.
+
+ Values flagged as markup are converted by ``_parse_content``; plain text
+ is returned verbatim for the caller to emit as-is.
+ """
nodes = entry.getElementsByTagName(tag_name)
if not nodes:
- return None
+ return None, True
node = nodes[0]
- if node.getAttribute("type").lower() != "xhtml":
- return self._get_data_by_tag_name(entry, tag_name)
-
- return "".join(
- self._localize_xhtml_names(child.cloneNode(True)).toxml()
- for child in node.childNodes
- if child.nodeType == Node.ELEMENT_NODE
+ kind = _atom_content_kind(
+ node.getAttribute("type"), allow_media_types=tag_name == "content"
)
+ if kind == "xhtml":
+ return (
+ "".join(
+ self._localize_xhtml_names(child.cloneNode(True)).toxml()
+ for child in node.childNodes
+ if child.nodeType == Node.ELEMENT_NODE
+ ),
+ True,
+ )
+ if kind == "binary":
+ # A base64-encoded payload; there is no text to render.
+ return None, True
+
+ text = self._get_data_by_tag_name(entry, tag_name)
+ if text is None or kind == "html":
+ return text, True
+
+ # Plain text carries no markup. Drop the whitespace the feed used to
+ # lay the element out -- indentation carried into the output would read
+ # as a Markdown code block -- but leave the text itself alone. The
+ # first line is excluded from the dedent because it may start on the
+ # element's own line, where it carries no indentation to share.
+ lines = text.splitlines()
+ if len(lines) > 1:
+ text = lines[0] + "\n" + textwrap.dedent("\n".join(lines[1:]))
+ return text.strip(), False
+
+ def _render_atom_content(
+ self, value: str, *, is_markup: bool, strict: bool = False
+ ) -> str:
+ """Render one Atom summary or content value as markdown."""
+ if not is_markup:
+ # Plain text is returned verbatim: routing it through the HTML
+ # parser drops tag-shaped text such as ``<job_id>`` entirely.
+ return value
+ return self._parse_content(value, strict=strict)
def _localize_xhtml_names(self, node: Node) -> Node:
"""Rewrite prefixed XHTML element names to their local HTML names.
diff --git a/packages/markitdown/tests/test_rss_converter.py b/packages/markitdown/tests/test_rss_converter.py
index d443e59e..19669cc7 100644
--- a/packages/markitdown/tests/test_rss_converter.py
+++ b/packages/markitdown/tests/test_rss_converter.py
@@ -72,3 +72,179 @@ def test_atom_prefixed_xhtml_content_is_preserved() -> None:
assert "Hello **bold** and *italic*." in result.markdown
assert "[link](https://example.com)" in result.markdown
+
+
+def test_atom_plain_text_content_is_not_parsed_as_html() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="text">Run &lt;job_id&gt; with &amp;lt;literal&amp;gt;.</content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "Run <job_id> with &lt;literal&gt;." in result.markdown
+
+
+def test_atom_untyped_summary_is_not_parsed_as_html() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <summary>A &lt;placeholder&gt; summary.</summary>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "A <placeholder> summary." in result.markdown
+
+
+def test_atom_html_content_is_still_converted() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="html">&lt;p&gt;Read the &lt;strong&gt;important details&lt;/strong&gt;.&lt;/p&gt;</content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "Read the **important details**." in result.markdown
+
+
+def test_atom_text_media_type_content_is_not_parsed_as_html() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="text/plain">Run &lt;job_id&gt; to start.</content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "Run <job_id> to start." in result.markdown
+
+
+def test_atom_html_media_type_content_is_still_converted() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="text/html; charset=utf-8">&lt;p&gt;Read the &lt;strong&gt;important details&lt;/strong&gt;.&lt;/p&gt;</content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "Read the **important details**." in result.markdown
+
+
+def test_atom_xml_media_type_content_is_preserved() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="application/xhtml+xml">
+ <div xmlns="http://www.w3.org/1999/xhtml">
+ <p>Read the <strong>important details</strong>.</p>
+ </div>
+ </content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "Read the **important details**." in result.markdown
+
+
+def test_atom_binary_content_is_skipped() -> None:
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="application/octet-stream">iVBORw0KGgoAAAANSUhEUg==</content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "iVBORw0KGgo" not in result.markdown
+
+
+def test_atom_summary_media_type_is_treated_as_text() -> None:
+ """Atom text constructs do not admit media types; treat one as plain text."""
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <summary type="text/plain">A &lt;placeholder&gt; summary.</summary>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert "A <placeholder> summary." in result.markdown
+
+
+def test_atom_plain_text_layout_whitespace_is_removed() -> None:
+ """Feed indentation must not survive as a Markdown code block."""
+ feed = b"""<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example feed</title>
+ <entry>
+ <title>Example entry</title>
+ <content type="text">
+ Run the job with &lt;job_id&gt;.
+
+ Then check status.
+ </content>
+ </entry>
+</feed>
+"""
+
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")
+ )
+
+ assert result.markdown.splitlines()[-3:] == [
+ "Run the job with <job_id>.",
+ "",
+ "Then check status.",
+ ]