Python tool for converting files and office documents to Markdown.
fix(rss): treat Atom text content and summary as plain text (#2374)
Lazizbek Ergashev697a2a0e0b95
diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.pyindex e0cc9a4e..6b7b1201 100644--- a/packages/markitdown/src/markitdown/converters/_rss_converter.py+++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py@@ -1,3 +1,4 @@+import textwrapimport warningsfrom defusedxml import minidom@@ -31,6 +32,35 @@ CANDIDATE_FILE_EXTENSIONS = [XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"+def _atom_content_kind(content_type: str, *, allow_media_types: bool) -> str:+ """Classify an Atom ``type`` attribute as text, html, xhtml or binary.++ Text constructs (title, summary, rights) admit only the ``text``/``html``/+ ``xhtml`` keywords, while atom:content additionally admits a MIME media+ type -- ``text/html`` is markup, any other ``text/*`` is plain text, inline+ XML is markup, and everything else is base64-encoded binary.+ See RFC 4287 sections 3.1 and 4.1.3.1.+ """+ content_type = content_type.strip().lower()+ if content_type in ("", "text"):+ return "text"+ if content_type in ("html", "xhtml"):+ return content_type+ if not allow_media_types:+ # An out-of-spec type on a text construct; the safe reading is text.+ return "text"++ media_type = content_type.split(";", 1)[0].strip()+ if media_type == "text/html":+ return "html"+ if media_type.endswith(("+xml", "/xml")):+ # Inline XML, carried as child elements just like xhtml content.+ return "xhtml"+ if media_type.startswith("text/"):+ return "text"+ return "binary"++class RssConverter(DocumentConverter):"""Convert RSS / Atom type to markdown"""@@ -122,38 +152,80 @@ class RssConverter(DocumentConverter):md_text += f"{subtitle}\n"for entry in entries:entry_title = self._get_data_by_tag_name(entry, "title")- entry_summary = self._get_atom_content(entry, "summary")+ entry_summary, summary_is_markup = self._get_atom_content(entry, "summary")entry_updated = self._get_data_by_tag_name(entry, "updated")- entry_content = self._get_atom_content(entry, "content")+ entry_content, content_is_markup = self._get_atom_content(entry, "content")if entry_title:md_text += f"\n## {entry_title}\n"if entry_updated:md_text += f"Updated on: {entry_updated}\n"if entry_summary:- md_text += self._parse_content(entry_summary, strict=strict)+ md_text += self._render_atom_content(+ entry_summary, is_markup=summary_is_markup, strict=strict+ )if entry_content:- md_text += self._parse_content(entry_content, strict=strict)+ md_text += self._render_atom_content(+ entry_content, is_markup=content_is_markup, strict=strict+ )return DocumentConverterResult(markdown=md_text,title=title,)- def _get_atom_content(self, entry: Element, tag_name: str) -> Union[str, None]:+ def _get_atom_content(+ self, entry: Element, tag_name: str+ ) -> tuple[Union[str, None], bool]:+ """Return an Atom summary or content value, and whether it is markup.++ Values flagged as markup are converted by ``_parse_content``; plain text+ is returned verbatim for the caller to emit as-is.+ """nodes = entry.getElementsByTagName(tag_name)if not nodes:- return None+ return None, Truenode = nodes[0]- if node.getAttribute("type").lower() != "xhtml":- return self._get_data_by_tag_name(entry, tag_name)-- return "".join(- self._localize_xhtml_names(child.cloneNode(True)).toxml()- for child in node.childNodes- if child.nodeType == Node.ELEMENT_NODE+ kind = _atom_content_kind(+ node.getAttribute("type"), allow_media_types=tag_name == "content")+ if kind == "xhtml":+ return (+ "".join(+ self._localize_xhtml_names(child.cloneNode(True)).toxml()+ for child in node.childNodes+ if child.nodeType == Node.ELEMENT_NODE+ ),+ True,+ )+ if kind == "binary":+ # A base64-encoded payload; there is no text to render.+ return None, True++ text = self._get_data_by_tag_name(entry, tag_name)+ if text is None or kind == "html":+ return text, True++ # Plain text carries no markup. Drop the whitespace the feed used to+ # lay the element out -- indentation carried into the output would read+ # as a Markdown code block -- but leave the text itself alone. The+ # first line is excluded from the dedent because it may start on the+ # element's own line, where it carries no indentation to share.+ lines = text.splitlines()+ if len(lines) > 1:+ text = lines[0] + "\n" + textwrap.dedent("\n".join(lines[1:]))+ return text.strip(), False++ def _render_atom_content(+ self, value: str, *, is_markup: bool, strict: bool = False+ ) -> str:+ """Render one Atom summary or content value as markdown."""+ if not is_markup:+ # Plain text is returned verbatim: routing it through the HTML+ # parser drops tag-shaped text such as ``<job_id>`` entirely.+ return value+ return self._parse_content(value, strict=strict)def _localize_xhtml_names(self, node: Node) -> Node:"""Rewrite prefixed XHTML element names to their local HTML names.diff --git a/packages/markitdown/tests/test_rss_converter.py b/packages/markitdown/tests/test_rss_converter.pyindex d443e59e..19669cc7 100644--- a/packages/markitdown/tests/test_rss_converter.py+++ b/packages/markitdown/tests/test_rss_converter.py@@ -72,3 +72,179 @@ def test_atom_prefixed_xhtml_content_is_preserved() -> None:assert "Hello **bold** and *italic*." in result.markdownassert "[link](https://example.com)" in result.markdown+++def test_atom_plain_text_content_is_not_parsed_as_html() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="text">Run <job_id> with &lt;literal&gt;.</content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "Run <job_id> with <literal>." in result.markdown+++def test_atom_untyped_summary_is_not_parsed_as_html() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <summary>A <placeholder> summary.</summary>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "A <placeholder> summary." in result.markdown+++def test_atom_html_content_is_still_converted() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="html"><p>Read the <strong>important details</strong>.</p></content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "Read the **important details**." in result.markdown+++def test_atom_text_media_type_content_is_not_parsed_as_html() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="text/plain">Run <job_id> to start.</content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "Run <job_id> to start." in result.markdown+++def test_atom_html_media_type_content_is_still_converted() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="text/html; charset=utf-8"><p>Read the <strong>important details</strong>.</p></content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "Read the **important details**." in result.markdown+++def test_atom_xml_media_type_content_is_preserved() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="application/xhtml+xml">+ <div xmlns="http://www.w3.org/1999/xhtml">+ <p>Read the <strong>important details</strong>.</p>+ </div>+ </content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "Read the **important details**." in result.markdown+++def test_atom_binary_content_is_skipped() -> None:+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="application/octet-stream">iVBORw0KGgoAAAANSUhEUg==</content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "iVBORw0KGgo" not in result.markdown+++def test_atom_summary_media_type_is_treated_as_text() -> None:+ """Atom text constructs do not admit media types; treat one as plain text."""+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <summary type="text/plain">A <placeholder> summary.</summary>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert "A <placeholder> summary." in result.markdown+++def test_atom_plain_text_layout_whitespace_is_removed() -> None:+ """Feed indentation must not survive as a Markdown code block."""+ feed = b"""<?xml version="1.0" encoding="utf-8"?>+<feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example feed</title>+ <entry>+ <title>Example entry</title>+ <content type="text">+ Run the job with <job_id>.++ Then check status.+ </content>+ </entry>+</feed>+"""++ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(mimetype="application/atom+xml")+ )++ assert result.markdown.splitlines()[-3:] == [+ "Run the job with <job_id>.",+ "",+ "Then check status.",+ ]