Python tool for converting files and office documents to Markdown.
fix: fall back to plain text when RSS item content triggers RecursionError (#2333)
Jeremy Schoemaker4752813fa5f7
diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.pyindex bec42484..49aef4a3 100644--- a/packages/markitdown/src/markitdown/converters/_rss_converter.py+++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py@@ -1,3 +1,5 @@+import warnings+from defusedxml import minidomfrom xml.dom.minidom import Document, Elementfrom typing import BinaryIO, Any, Union@@ -87,18 +89,23 @@ class RssConverter(DocumentConverter):stream_info: StreamInfo,**kwargs: Any, # Options to pass to the converter) -> DocumentConverterResult:+ # Pop our own keyword before forwarding the rest to markdownify.+ # strict=True raises RecursionError instead of falling back to plain text.+ strict: bool = kwargs.pop("strict", False)self._kwargs = kwargsdoc = minidom.parse(file_stream)feed_type = self._feed_type(doc)if feed_type == "rss":- return self._parse_rss_type(doc)+ return self._parse_rss_type(doc, strict=strict)elif feed_type == "atom":- return self._parse_atom_type(doc)+ return self._parse_atom_type(doc, strict=strict)else:raise ValueError("Unknown feed type")- def _parse_atom_type(self, doc: Document) -> DocumentConverterResult:+ def _parse_atom_type(+ self, doc: Document, *, strict: bool = False+ ) -> DocumentConverterResult:"""Parse the type of an Atom feed.Returns None if the feed type is not recognized or something goes wrong.@@ -121,16 +128,18 @@ class RssConverter(DocumentConverter):if entry_updated:md_text += f"Updated on: {entry_updated}\n"if entry_summary:- md_text += self._parse_content(entry_summary)+ md_text += self._parse_content(entry_summary, strict=strict)if entry_content:- md_text += self._parse_content(entry_content)+ md_text += self._parse_content(entry_content, strict=strict)return DocumentConverterResult(markdown=md_text,title=title,)- def _parse_rss_type(self, doc: Document) -> DocumentConverterResult:+ def _parse_rss_type(+ self, doc: Document, *, strict: bool = False+ ) -> DocumentConverterResult:"""Parse the type of an RSS feed.Returns None if the feed type is not recognized or something goes wrong.@@ -158,21 +167,34 @@ class RssConverter(DocumentConverter):if pubDate:md_text += f"Published on: {pubDate}\n"if description:- md_text += self._parse_content(description)+ md_text += self._parse_content(description, strict=strict)if content:- md_text += self._parse_content(content)+ md_text += self._parse_content(content, strict=strict)return DocumentConverterResult(markdown=md_text,title=channel_title,)- def _parse_content(self, content: str) -> str:+ def _parse_content(self, content: str, *, strict: bool = False) -> str:"""Parse the content of an RSS feed item"""try:# using bs4 because many RSS feeds have HTML-styled contentsoup = BeautifulSoup(content, "html.parser")return _CustomMarkdownify(**self._kwargs).convert_soup(soup)+ except RecursionError:+ if strict:+ raise+ # Deeply nested item content can exceed Python's recursion limit+ # during markdownify's recursive DOM traversal. Fall back to+ # BeautifulSoup's iterative get_text() so the caller still gets+ # usable plain-text content instead of raw HTML.+ warnings.warn(+ "RSS item content is too deeply nested for markdown conversion "+ "(RecursionError). Falling back to plain-text extraction.",+ stacklevel=2,+ )+ return BeautifulSoup(content, "html.parser").get_text("\n", strip=True)except BaseException as _:return contentdiff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex dd04e142..a263801d 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -9,6 +9,7 @@ from unittest.mock import MagicMockfrom markitdown._uri_utils import parse_data_uri, file_uri_to_pathfrom markitdown._markitdown import _get_content_disposition_filename+from markitdown.converters import RssConverterfrom markitdown import (MarkItDown,@@ -434,6 +435,7 @@ def test_deeply_nested_html_fallback() -> None:# Should have emitted a warning about the fallbackrecursion_warnings = [x for x in w if "deeply nested" in str(x.message)]assert len(recursion_warnings) > 0+finally:sys.setrecursionlimit(original_limit)@@ -444,6 +446,96 @@ def test_deeply_nested_html_fallback() -> None:assert "<p>" not in result.markdown+def test_deeply_nested_rss_item_fallback() -> None:+ """Deeply nested HTML inside an RSS item should fall back to plain-text+ extraction instead of silently embedding raw unconverted HTML in the+ markdown output (same failure class as the HTML converter fix in #1644).++ Note: This test uses sys.setrecursionlimit to guarantee a RecursionError+ regardless of the host environment's default limit, making it deterministic+ across different platforms and CI configurations.+ """+ import sys+ import warnings++ markitdown = MarkItDown()++ # Use a small recursion limit so the test is environment-independent.+ # We restore the original limit in a finally block to avoid side-effects.+ original_limit = sys.getrecursionlimit()+ low_limit = 200 # well below markdownify's traversal depth for depth=500++ # Build an RSS item whose content is deeply nested HTML+ depth = 500+ item_html = ""+ for _ in range(depth):+ item_html += '<div style="margin-left:10px">'+ item_html += "<p>Deep feed content with <b>bold text</b></p>"+ for _ in range(depth):+ item_html += "</div>"++ rss = (+ '<?xml version="1.0" encoding="UTF-8"?>'+ '<rss version="2.0" '+ 'xmlns:content="http://purl.org/rss/1.0/modules/content/">'+ "<channel>"+ "<title>Test Feed</title>"+ "<description>A test feed</description>"+ "<item>"+ "<title>Deep Item</title>"+ f"<content:encoded><![CDATA[{item_html}]]></content:encoded>"+ "</item>"+ "</channel>"+ "</rss>"+ )+ atom = (+ '<?xml version="1.0" encoding="UTF-8"?>'+ '<feed xmlns="http://www.w3.org/2005/Atom">'+ "<title>Test Feed</title>"+ "<entry>"+ "<title>Deep Entry</title>"+ f'<content type="html"><![CDATA[{item_html}]]></content>'+ "</entry>"+ "</feed>"+ )++ try:+ sys.setrecursionlimit(low_limit)+ with warnings.catch_warnings(record=True) as w:+ warnings.simplefilter("always")+ result = markitdown.convert_stream(+ io.BytesIO(rss.encode("utf-8")),+ file_extension=".rss",+ )++ # Should have emitted a warning about the fallback+ recursion_warnings = [x for x in w if "deeply nested" in str(x.message)]+ assert len(recursion_warnings) > 0++ # strict=True should expose the conversion failure rather than applying+ # the plain-text fallback.+ with pytest.raises(RecursionError):+ RssConverter().convert(+ io.BytesIO(rss.encode("utf-8")),+ StreamInfo(extension=".rss"),+ strict=True,+ )+ with pytest.raises(RecursionError):+ RssConverter().convert(+ io.BytesIO(atom.encode("utf-8")),+ StreamInfo(extension=".atom"),+ strict=True,+ )+ finally:+ sys.setrecursionlimit(original_limit)++ # The output should contain the text content, not raw HTML+ assert "Deep feed content" in result.markdown+ assert "bold text" in result.markdown+ assert "<div" not in result.markdown+ assert "<p>" not in result.markdown++def test_doc_rlink() -> None:# Test for: CVE-2025-11849markitdown = MarkItDown()@@ -644,7 +736,7 @@ def test_json_with_late_non_ascii_character(tmp_path) -> None:"title": "Example record","abstract": "This is sample test. " * 500,},- "notes": "non-ASCII character: è",+ "notes": "non-ASCII character: è",}json_path = tmp_path / "input.json"json_path.write_text(@@ -653,7 +745,7 @@ def test_json_with_late_non_ascii_character(tmp_path) -> None:result = MarkItDown().convert(str(json_path))- assert "non-ASCII character: è" in result.text_content+ assert "non-ASCII character: è" in result.text_contentif __name__ == "__main__":