Python tool for converting files and office documents to Markdown.
fix(rss): preserve complete feed content and resolve relative link (#2432)
afourney75cc34a364c1
diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.pyindex d24eea4a..ce768b84 100644--- a/packages/markitdown/src/markitdown/converters/_rss_converter.py+++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py@@ -1,11 +1,12 @@import textwrapimport warnings+from html import escape+from urllib.parse import urljoinfrom defusedxml import minidom-from xml.dom import Node-from xml.dom.minidom import Document, Element+from xml.dom.minidom import Document, Element, Nodefrom typing import BinaryIO, Any, Union-from bs4 import BeautifulSoup+from bs4 import BeautifulSoup, Tagfrom ._markdownify import _CustomMarkdownifyfrom .._stream_info import StreamInfo@@ -31,6 +32,24 @@ CANDIDATE_FILE_EXTENSIONS = [ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"+CONTENT_NAMESPACE = "http://purl.org/rss/1.0/modules/content/"+XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace"++# Boundaries to retain when reducing markup to readable heading/plain text.+# Inline elements must not introduce spaces (e.g. co<b>op</b>erate).+_TEXT_BREAK_ELEMENTS = frozenset(+ "address article aside blockquote br dd div dl dt figcaption figure footer "+ "h1 h2 h3 h4 h5 h6 header hr li main nav ol p pre section table td th tr ul".split()+)+++def _resolve_url(base_url: str, reference: str) -> str:+ """Resolve a reference without letting a malformed URI drop the body."""+ try:+ return urljoin(base_url, reference)+ except ValueError:+ # Leave malformed links to the Markdown converter's existing handling.+ return referencedef _atom_content_kind(content_type: str, *, allow_media_types: bool) -> str:@@ -107,12 +126,14 @@ class RssConverter(DocumentConverter):file_stream.seek(cur_pos)return False- def _feed_type(self, doc: Any) -> str | None:- if doc.getElementsByTagName("rss"):- return "rss"+ def _feed_type(self, doc: Document) -> str | None:root = doc.documentElement+ if root is None:+ return None+ if root.tagName == "rss":+ return "rss"if root.localName == "feed" and root.namespaceURI in (None, ATOM_NAMESPACE):- if root.getElementsByTagNameNS(root.namespaceURI, "entry"):+ if self._get_children(root, "entry"):# An Atom feed must have a root element of <feed> and at least one <entry>return "atom"return None@@ -128,6 +149,7 @@ class RssConverter(DocumentConverter):strict: bool = kwargs.pop("strict", False)self._kwargs = kwargsdoc = minidom.parse(file_stream)+ doc.documentURI = stream_info.url or kwargs.get("url")feed_type = self._feed_type(doc)if feed_type == "rss":@@ -145,15 +167,16 @@ class RssConverter(DocumentConverter):Returns None if the feed type is not recognized or something goes wrong."""root = doc.documentElement- title = self._get_flattened_text(root, "title")- subtitle = self._get_flattened_text(root, "subtitle")- entries = root.getElementsByTagNameNS(root.namespaceURI, "entry")+ assert root is not None+ title = self._get_flattened_text(root, "title", atom_text=True)+ subtitle = self._get_flattened_text(root, "subtitle", atom_text=True)+ entries = self._get_children(root, "entry")md_text = f"# {title}\n" if title else ""if subtitle:md_text += f"{subtitle}\n"for entry in entries:- entry_title = self._get_flattened_text(entry, "title")+ entry_title = self._get_flattened_text(entry, "title", atom_text=True)entry_summary, summary_is_markup = self._get_atom_content(entry, "summary")entry_updated = self._get_flattened_text(entry, "updated")entry_content, content_is_markup = self._get_atom_content(entry, "content")@@ -162,14 +185,23 @@ class RssConverter(DocumentConverter):md_text += f"\n## {entry_title}\n"if entry_updated:md_text += f"Updated on: {entry_updated}\n"- if entry_summary:- md_text += self._render_atom_content(- entry_summary, is_markup=summary_is_markup, strict=strict+ body_parts = (+ self._render_atom_content(+ value,+ is_markup=is_markup,+ base_url=self._get_field_base_url(entry, tag_name),+ strict=strict,)- if entry_content:- md_text += self._render_atom_content(- entry_content, is_markup=content_is_markup, strict=strict+ for value, is_markup, tag_name in (+ (entry_summary, summary_is_markup, "summary"),+ (entry_content, content_is_markup, "content"),)+ if value+ )+ body = "\n\n".join(part for part in body_parts if part)+ if body and md_text and not md_text.endswith("\n"):+ md_text += "\n\n"+ md_text += bodyreturn DocumentConverterResult(markdown=md_text,@@ -179,35 +211,25 @@ class RssConverter(DocumentConverter):def _get_atom_content(self, entry: Element, tag_name: str) -> tuple[Union[str, None], bool]:- """Return an Atom summary or content value, and whether it is markup.+ """Return an Atom text construct or content, and whether it is markup.Values flagged as markup are converted by ``_parse_content``; plain textis returned verbatim for the caller to emit as-is."""- nodes = entry.getElementsByTagNameNS(entry.namespaceURI, tag_name)- if not nodes:+ node = self._get_child(entry, tag_name)+ if node is None:return None, True- node = nodes[0]kind = _atom_content_kind(node.getAttribute("type"), allow_media_types=tag_name == "content")- if kind == "xhtml":- return (- "".join(- self._localize_xhtml_names(child.cloneNode(True)).toxml()- for child in node.childNodes- if child.nodeType == Node.ELEMENT_NODE- ),- True,- )if kind == "binary":# A base64-encoded payload; there is no text to render.return None, True- text = self._get_data_by_tag_name(entry, tag_name)- if text is None or kind == "html":- return text, True+ text = self._read_content(node, kind=kind)+ if kind != "text":+ return text or None, True# Plain text carries no markup. Drop the whitespace the feed used to# lay the element out -- indentation carried into the output would read@@ -220,32 +242,18 @@ class RssConverter(DocumentConverter):return text.strip(), Falsedef _render_atom_content(- self, value: str, *, is_markup: bool, strict: bool = False+ self, value: str, *, is_markup: bool, base_url: str = "", strict: bool = False) -> str:"""Render one Atom summary or content value as markdown."""if not is_markup:# Plain text is returned verbatim: routing it through the HTML# parser drops tag-shaped text such as ``<job_id>`` entirely.return value- return self._parse_content(value, strict=strict)+ return self._parse_content(value, base_url=base_url, strict=strict)- def _localize_xhtml_names(self, node: Node) -> Node:- """Rewrite prefixed XHTML element names to their local HTML names.-- Atom permits XHTML content to be namespace-prefixed (e.g. ``x:strong``).- The downstream HTML converter dispatches on HTML tag names, so the- prefix has to be dropped or the element is treated as an unknown tag- and its formatting is lost.- """- if node.nodeType == Node.ELEMENT_NODE:- if node.prefix and node.namespaceURI == XHTML_NAMESPACE:- node.tagName = node.nodeName = node.localName- node.prefix = None- for child in node.childNodes:- self._localize_xhtml_names(child)- return node-- def _get_flattened_text(self, element: Element, tag_name: str) -> Union[str, None]:+ def _get_flattened_text(+ self, element: Element, tag_name: str, *, atom_text: bool = False+ ) -> Union[str, None]:"""Get a value that is rendered as a heading or a metadata line.Feeds are routinely pretty-printed, so a value written as@@ -255,10 +263,23 @@ class RssConverter(DocumentConverter):text, so the layout whitespace is dropped here. A value that isnothing but whitespace is reported as absent."""- value = self._get_data_by_tag_name(element, tag_name)+ if atom_text:+ value, is_markup = self._get_atom_content(element, tag_name)+ if value and is_markup:+ soup = BeautifulSoup(value, "html.parser")+ for block in soup.find_all(_TEXT_BREAK_ELEMENTS):+ block.insert_before("\n")+ block.append("\n")+ value = soup.get_text()+ else:+ value = self._get_data_by_tag_name(element, tag_name)if value is None:return None- return " ".join(part.strip() for part in value.splitlines()).strip() or None+ # Block boundaries become line breaks, and adjacent blocks leave blank+ # lines between them; drop the empty parts so the flattened value is+ # separated by single spaces rather than by runs of them.+ parts = (part.strip() for part in value.splitlines())+ return " ".join(part for part in parts if part) or Nonedef _parse_rss_type(self, doc: Document, *, strict: bool = False@@ -267,14 +288,16 @@ class RssConverter(DocumentConverter):Returns None if the feed type is not recognized or something goes wrong."""- root = doc.getElementsByTagName("rss")[0]- channel_list = root.getElementsByTagName("channel")- if not channel_list:+ root = doc.documentElement+ assert root is not None+ channel = self._get_child(root, "channel")+ if channel is None:raise ValueError("No channel found in RSS feed")- channel = channel_list[0]channel_title = self._get_flattened_text(channel, "title")+ # A channel description is a text field; item descriptions carry HTML.+ # Preserve its existing whitespace while collecting the complete value.channel_description = self._get_data_by_tag_name(channel, "description")- items = channel.getElementsByTagName("item")+ items = self._get_children(channel, "item")md_text = ""if channel_title:md_text += f"# {channel_title}\n"@@ -282,29 +305,44 @@ class RssConverter(DocumentConverter):md_text += f"{channel_description}\n"for item in items:title = self._get_flattened_text(item, "title")- description = self._get_data_by_tag_name(item, "description")+ description = self._get_data_by_tag_name(item, "description", kind="html")pubDate = self._get_flattened_text(item, "pubDate")- content = self._get_data_by_tag_name(item, "content:encoded")+ content = self._get_data_by_tag_name(item, "content:encoded", kind="html")if title:md_text += f"\n## {title}\n"if pubDate:md_text += f"Published on: {pubDate}\n"- if description:- md_text += self._parse_content(description, strict=strict)- if content:- md_text += self._parse_content(content, strict=strict)+ body_parts = (+ self._parse_content(+ value,+ base_url=self._get_field_base_url(item, tag_name),+ strict=strict,+ )+ for value, tag_name in (+ (description, "description"),+ (content, "content:encoded"),+ )+ if value+ )+ body = "\n\n".join(part for part in body_parts if part)+ if body and md_text and not md_text.endswith("\n"):+ md_text += "\n\n"+ md_text += bodyreturn DocumentConverterResult(markdown=md_text,title=channel_title,)- def _parse_content(self, content: str, *, strict: bool = False) -> str:+ def _parse_content(+ self, content: str, *, base_url: str = "", strict: bool = False+ ) -> str:"""Parse the content of an RSS feed item"""try:# using bs4 because many RSS feeds have HTML-styled contentsoup = BeautifulSoup(content, "html.parser")+ self._resolve_content_links(soup, base_url)return _CustomMarkdownify(**self._kwargs).convert_soup(soup)except RecursionError:if strict:@@ -322,20 +360,129 @@ class RssConverter(DocumentConverter):except BaseException as _:return content- def _get_data_by_tag_name(- self, element: Element, tag_name: str- ) -> Union[str, None]:- """Get data from first child element with the given tag name.- Returns None when no such element is found.+ def _get_field_base_url(self, element: Element, tag_name: str) -> str:+ """Apply xml:base from the document root through the selected field.++ Relative overrides resolve against their parent's base. An empty+ xml:base inherits that base; it does not reset it to the document URL.+ The document URI is local to this conversion, even when a converter is+ reused. The source stream's URL also supplies a base for RSS HTML."""- if element.namespaceURI == ATOM_NAMESPACE:- nodes = element.getElementsByTagNameNS(ATOM_NAMESPACE, tag_name)+ bases: list[str] = []+ node: Node | None = self._get_child(element, tag_name)+ while node is not None:+ if isinstance(node, Element) and node.hasAttributeNS(XML_NAMESPACE, "base"):+ bases.append(node.getAttributeNS(XML_NAMESPACE, "base"))+ node = node.parentNode+ document = element.ownerDocument+ base_url = (document.documentURI or "") if document is not None else ""+ for reference in reversed(bases):+ base_url = _resolve_url(base_url, reference)+ return base_url++ def _resolve_content_links(self, soup: BeautifulSoup, base_url: str) -> None:+ """Resolve rendered links/images, retaining nested XML Base scopes.++ Inline XHTML keeps its xml:base attributes during serialization; HTML+ carried in text/CDATA inherits the enclosing field's base. Traverse+ iteratively so deeply nested content can still reach the fallback.+ """+ stack: list[tuple[Tag, str]] = [(soup, base_url)]+ while stack:+ node, inherited_base = stack.pop()+ override = node.get("xml:base")+ current_base = (+ _resolve_url(inherited_base, override)+ if isinstance(override, str)+ else inherited_base+ )+ attributes: tuple[str, ...] = ()+ if node.name == "a":+ attributes = ("href",)+ elif node.name == "img":+ attributes = ("src", "data-src")+ for attribute in attributes:+ reference = node.get(attribute)+ # Empty src must still allow markdownify's data-src fallback.+ if isinstance(reference, str) and (reference or attribute == "href"):+ node[attribute] = _resolve_url(current_base, reference)+ stack.extend(+ (child, current_base)+ for child in node.children+ if isinstance(child, Tag)+ )++ def _get_children(self, element: Element, tag_name: str) -> list[Element]:+ """Select fields on their owner, without borrowing descendant metadata."""+ namespace: str | None+ if tag_name == "content:encoded":+ namespace, local_name = CONTENT_NAMESPACE, "encoded"else:- nodes = element.getElementsByTagName(tag_name)- if not nodes:+ namespace, local_name = element.namespaceURI, tag_name+ return [+ child+ for child in element.childNodes+ if isinstance(child, Element)+ and child.namespaceURI == namespace+ and child.localName == local_name+ ]++ def _get_child(self, element: Element, tag_name: str) -> Element | None:+ return next(iter(self._get_children(element, tag_name)), None)++ def _get_data_by_tag_name(+ self, element: Element, tag_name: str, *, kind: str = "text"+ ) -> Union[str, None]:+ """Read the complete value of the first matching direct child field."""+ node = self._get_child(element, tag_name)+ if node is None:return None- fc = nodes[0].firstChild- if fc:- if hasattr(fc, "data"):- return fc.data- return None+ return self._read_content(node, kind=kind) or None++ def _read_content(self, element: Element, *, kind: str) -> str:+ """Collect plain text, encoded HTML, or inline XML in document order.++ Encoded HTML's direct text/CDATA children are parts of one HTML string.+ Text inside actual XML elements is already literal and must be escaped+ when serializing those elements for the HTML parser. In XHTML, all text+ is literal, including CDATA. Comments and processing instructions carry+ no display content.++ Use an explicit stack: minidom's recursive cloning/serialization can+ fail on deep XML before the Markdown renderer's fallback is reached.+ """+ parts: list[str] = []+ stack: list[Node | str] = list(reversed(element.childNodes))+ while stack:+ child = stack.pop()+ if isinstance(child, str):+ parts.append(child)+ elif child.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):+ text = child.nodeValue or ""+ if kind == "text" or (kind == "html" and child.parentNode is element):+ parts.append(text)+ else:+ parts.append(escape(text, quote=False))+ elif isinstance(child, Element):+ if kind == "text":+ if child.localName.lower() in _TEXT_BREAK_ELEMENTS:+ parts.append("\n")+ stack.append("\n")+ else:+ # markdownify recognizes local HTML names, not x:strong.+ name = (+ child.localName+ if child.namespaceURI == XHTML_NAMESPACE+ else child.tagName+ )+ attrs = "".join(+ f' {attr.name}="{escape(attr.value, quote=True)}"'+ for attr in child.attributes.values()+ )+ if child.childNodes:+ parts.append(f"<{name}{attrs}>")+ stack.append(f"</{name}>")+ else:+ parts.append(f"<{name}{attrs}/>")+ stack.extend(reversed(child.childNodes))+ return "".join(parts)diff --git a/packages/markitdown/tests/test_rss_converter.py b/packages/markitdown/tests/test_rss_converter.pyindex ddc690e4..6aaec65a 100644--- a/packages/markitdown/tests/test_rss_converter.py+++ b/packages/markitdown/tests/test_rss_converter.py@@ -1,4 +1,5 @@import io+import sysimport pytest@@ -338,3 +339,352 @@ def test_atom_plain_text_layout_whitespace_is_removed() -> None:"","Then check status.",]+++def _feed_with_body(field: str, payload: str, *, atom_type: str = "html") -> bytes:+ if field.startswith("rss-"):+ tag = "description" if field == "rss-description" else "content:encoded"+ return (+ '<rss version="2.0" '+ 'xmlns:content="http://purl.org/rss/1.0/modules/content/">'+ "<channel><title>Feed</title><description>About the feed.</description>"+ f"<item><title>Entry</title><{tag}>{payload}</{tag}></item>"+ "</channel></rss>"+ ).encode()+ tag = field.removeprefix("atom-")+ return (+ '<feed xmlns="http://www.w3.org/2005/Atom">'+ "<title>Feed</title><entry><title>Entry</title>"+ f'<{tag} type="{atom_type}">{payload}</{tag}>'+ "</entry></feed>"+ ).encode()+++ "field", ["rss-description", "rss-content", "atom-summary", "atom-content"]+)+ "payload, expected",+ [+ ("Plain body.", "Plain body."),+ ("<![CDATA[<p>Only CDATA.</p>]]>", "Only CDATA."),+ (+ "\n <![CDATA[<p>The <strong>body</strong>.</p>]]>\n ",+ "The **body**.",+ ),+ (+ "Before <![CDATA[<str]]><![CDATA[ong>middle</strong>]]> after.",+ "Before **middle** after.",+ ),+ ("<b>nested</b> trailing", "**nested** trailing"),+ ("leading <b>nested</b> trailing", "leading **nested** trailing"),+ ("leading <b>nested</b>", "leading **nested**"),+ ("First<br/>Second", "First \nSecond"),+ ("<pre>line 1\n line 2</pre>", "```\nline 1\n line 2\n```"),+ (+ "\n <!--ignore--><?instruction ignore?><b>nested<!--ignore--></b> trailing",+ "**nested** trailing",+ ),+ (+ '<p>Read <a href="https://example.com/?a=1&b=2">this</a>.</p>'+ "<p>Then <em>continue</em>.</p>",+ "Read [this](https://example.com/?a=1&b=2).\n\nThen *continue*.",+ ),+ (+ "<p>Use <slot> and &lt;literal&gt;.</p>",+ "Use <slot> and <literal>.",+ ),+ ],+)+def test_feed_body_preserves_complete_content(+ field: str, payload: str, expected: str+) -> None:+ feed = _feed_with_body(field, payload)+ stream_info = StreamInfo(extension=".rss" if field.startswith("rss-") else ".atom")+ result = RssConverter().convert(io.BytesIO(feed), stream_info)++ assert result.markdown.split("## Entry\n", 1)[1] == expected++[email protected]("field", ["atom-summary", "atom-content"])[email protected]("atom_type", ["text", "text/plain"])+def test_atom_plain_mixed_content_stays_literal(field: str, atom_type: str) -> None:+ feed = _feed_with_body(+ field,+ "Run <![CDATA[<job_id>]]> with <b>&lt;literal&gt;</b>."+ "<!--ignore--><?instruction ignore?>",+ atom_type=atom_type,+ )+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".atom"))++ assert result.markdown.split("## Entry\n", 1)[1] == (+ "Run <job_id> with <literal>."+ )++[email protected]("atom_type", ["xhtml", "application/xhtml+xml"])+def test_atom_xhtml_preserves_literal_text_and_surrounding_content(+ atom_type: str,+) -> None:+ feed = _feed_with_body(+ "atom-content",+ 'Before <slot> <x:div xmlns:x="http://www.w3.org/1999/xhtml">'+ "<x:p>Use <![CDATA[<slot> <literal>]]> and <x:b>bold</x:b>.</x:p>"+ "<!--ignore--><?instruction ignore?>"+ "<x:pre>line 1\n line 2</x:pre></x:div> after.",+ atom_type=atom_type,+ )+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".atom"))++ body = result.markdown.split("## Entry\n", 1)[1]+ assert body.startswith("Before <slot>")+ assert "Use <slot> <literal> and **bold**." in body+ assert "```\nline 1\n line 2\n```" in body+ assert body.endswith("after.")+ assert "ignore" not in body++[email protected]("extension", [".rss", ".atom"])+def test_feed_body_fields_are_separated(extension: str) -> None:+ if extension == ".rss":+ feed = _feed_with_body("rss-description", "Summary.").replace(+ b"</item>", b"<content:encoded>Body.</content:encoded></item>"+ )+ else:+ feed = _feed_with_body("atom-summary", "Summary.", atom_type="text").replace(+ b"</entry>", b'<content type="text">Body.</content></entry>'+ )+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=extension))++ assert result.markdown.split("## Entry\n", 1)[1] == "Summary.\n\nBody."++[email protected]("extension", [".rss", ".atom"])+ "summary, content, expected",+ [("<b/>", "Body.", "Body."), ("Summary.", "<b/>", "Summary."), ("<b/>", "", "")],+)+def test_empty_markup_fields_do_not_add_body_separators(+ extension: str, summary: str, content: str, expected: str+) -> None:+ if extension == ".rss":+ feed = _feed_with_body("rss-description", summary).replace(+ b"</item>", f"<content:encoded>{content}</content:encoded></item>".encode()+ )+ else:+ feed = _feed_with_body("atom-summary", summary).replace(+ b"</entry>", f'<content type="html">{content}</content></entry>'.encode()+ )+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=extension))++ assert result.markdown.split("## Entry\n", 1)[1] == expected++[email protected]("field", ["rss-description", "atom-content"])+def test_complete_feed_content_through_public_api(field: str) -> None:+ feed = _feed_with_body(field, "Before <b>nested</b> <![CDATA[after.]]>")+ result = MarkItDown().convert_stream(+ io.BytesIO(feed), stream_info=StreamInfo(extension=".xml")+ )++ assert result.title == "Feed"+ assert result.markdown.endswith("## Entry\nBefore **nested** after.")++[email protected]("field", ["rss-description", "atom-content"])+def test_deep_xml_body_uses_rendering_fallback(field: str) -> None:+ payload = "<div>" * 500 + "Deep <b>body</b>." + "</div>" * 500+ feed = _feed_with_body(field, payload, atom_type="xhtml")+ stream_info = StreamInfo(extension=".rss" if field.startswith("rss-") else ".atom")+ original_limit = sys.getrecursionlimit()+ try:+ sys.setrecursionlimit(200)+ with pytest.warns(UserWarning, match="too deeply nested"):+ result = RssConverter().convert(io.BytesIO(feed), stream_info)+ with pytest.raises(RecursionError):+ RssConverter().convert(io.BytesIO(feed), stream_info, strict=True)+ finally:+ sys.setrecursionlimit(original_limit)++ assert "Deep" in result.markdown+ assert "body" in result.markdown+ assert "<div>" not in result.markdown+++def test_rss_content_namespace_alias_and_field_ownership() -> None:+ feed = b"""<rss xmlns:c="http://purl.org/rss/1.0/modules/content/">+ <channel><title>Feed</title><item><title>Entry</title>+ <extension><description>Wrong body.</description></extension>+ <c:encoded>Before <b>nested</b> <![CDATA[after.]]></c:encoded>+ </item></channel>+ </rss>"""+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss"))++ assert result.markdown == "# Feed\n\n## Entry\nBefore **nested** after."+++ "field", ["rss-description", "rss-content", "atom-summary", "atom-content"]+)[email protected]("cdata", [False, True])+def test_feed_relative_links_use_document_url(field: str, cdata: bool) -> None:+ payload = '<a href="../stories/café%20notes?x=1&y=2#part">Read</a>'+ if cdata:+ payload = f"<![CDATA[{payload}]]>"+ feed = _feed_with_body(field, payload)+ result = MarkItDown().convert_stream(+ io.BytesIO(feed),+ stream_info=StreamInfo(+ extension=".xml", url="https://example.com/news/feed.xml"+ ),+ )++ assert result.markdown.endswith(+ "[Read](https://example.com/stories/caf%C3%A9%20notes?x=1&y=2#part)"+ )+++def test_atom_inherits_xml_base_and_scopes_nested_overrides() -> None:+ feed = b"""<a:feed xmlns:a="http://www.w3.org/2005/Atom"+ xmlns:x="http://www.w3.org/1999/xhtml" xml:base="https://example.com/root/">+ <a:title>Feed</a:title>+ <a:entry xml:base="entries/">+ <a:title>Entry</a:title>+ <a:summary type="html" xml:base="../summaries/">+ <![CDATA[<a href="summary">Summary</a>]]>+ </a:summary>+ <a:content type="xhtml" xml:base="../articles/">+ <x:div xml:base="chapter/">+ <x:p xml:base="../appendix/">+ <x:a xml:base="../../assets/" href="story">Story</x:a>+ <x:a href="sibling">Sibling</x:a>+ </x:p>+ <x:a href="after">After</x:a>+ <x:img src="diagram.png" alt="Diagram"/>+ <x:img src="" data-src="../lazy.png" alt="Lazy"/>+ </x:div>+ </a:content>+ </a:entry>+ <a:entry><a:title>Next</a:title>+ <a:content type="html"><a href="next">Next link</a></a:content>+ </a:entry>+ </a:feed>"""+ result = RssConverter().convert(+ io.BytesIO(feed),+ StreamInfo(extension=".atom", url="https://other.example/feed.xml"),+ )++ assert "[Summary](https://example.com/root/summaries/summary)" in result.markdown+ assert "[Story](https://example.com/root/assets/story)" in result.markdown+ assert (+ "[Sibling](https://example.com/root/articles/appendix/sibling)"+ in result.markdown+ )+ assert "[After](https://example.com/root/articles/chapter/after)" in result.markdown+ assert (+ ""+ in result.markdown+ )+ assert "" in result.markdown+ assert "[Next link](https://example.com/root/next)" in result.markdown+++ "document_url, feed_base, entry_base, content_base, expected",+ [+ (+ "https://example.com/news/feed.xml",+ "../stories/",+ "2026/",+ "details/",+ "https://example.com/stories/2026/details/story",+ ),+ (+ "https://example.com/news/feed.xml",+ "https://other.example/base/",+ "section/",+ "",+ "https://other.example/base/section/story",+ ),+ (None, "https://example.com/", "", "", "https://example.com/story"),+ (None, "", "", "", "story"),+ ],+)+def test_atom_relative_and_empty_xml_base(+ document_url: str | None,+ feed_base: str,+ entry_base: str,+ content_base: str,+ expected: str,+) -> None:+ xml = (+ f'<feed xmlns="http://www.w3.org/2005/Atom" xml:base="{feed_base}">'+ f'<entry xml:base="{entry_base}"><content type="html" xml:base="{content_base}">'+ '<![CDATA[<a href="story">Read</a>]]></content></entry></feed>'+ )+ result = RssConverter().convert(+ io.BytesIO(xml.encode()), StreamInfo(extension=".atom", url=document_url)+ )++ assert result.markdown == f"[Read]({expected})"+++ "href, expected",+ [+ ("#section", "https://example.com/news/feed.xml?lang=en#section"),+ ("?page=2", "https://example.com/news/feed.xml?page=2"),+ ("/story", "https://example.com/story"),+ ("//cdn.example.com/story", "https://cdn.example.com/story"),+ ("https://other.example/story", "https://other.example/story"),+ ],+)+def test_feed_link_reference_forms(href: str, expected: str) -> None:+ feed = _feed_with_body("atom-content", f'<a href="{href}">Read</a>')+ result = RssConverter().convert(+ io.BytesIO(feed),+ StreamInfo(extension=".atom", url="https://example.com/news/feed.xml?lang=en"),+ )++ assert result.markdown.endswith(f"[Read]({expected})")+++def test_link_resolution_preserves_plain_atom_text() -> None:+ feed = _feed_with_body(+ "atom-content",+ '<![CDATA[<a href="story">literal</a>]]>',+ atom_type="text",+ )+ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(extension=".atom", url="https://example.com/feed")+ )++ assert result.markdown.endswith('<a href="story">literal</a>')+++def test_link_resolution_retains_existing_invalid_link_handling() -> None:+ feed = _feed_with_body(+ "atom-content",+ '<a href="http://[invalid">Malformed</a> '+ '<a href="javascript:alert(1)">Script</a> '+ '<a href="story">Read</a>',+ )+ result = RssConverter().convert(+ io.BytesIO(feed), StreamInfo(extension=".atom", url="https://example.com/feed")+ )++ assert result.markdown.endswith(+ "Malformed Script [Read](https://example.com/story)"+ )+++def test_document_base_does_not_leak_between_conversions() -> None:+ feed = _feed_with_body("atom-content", '<a href="story">Read</a>')+ converter = RssConverter()+ first = converter.convert(+ io.BytesIO(feed), StreamInfo(extension=".atom"), url="https://example.com/feed"+ )+ second = converter.convert(io.BytesIO(feed), StreamInfo(extension=".atom"))++ assert first.markdown.endswith("[Read](https://example.com/story)")+ assert second.markdown.endswith("[Read](story)")diff --git a/packages/markitdown/tests/test_rss_titles.py b/packages/markitdown/tests/test_rss_titles.pyindex 7302d88a..ce21b3a6 100644--- a/packages/markitdown/tests/test_rss_titles.py+++ b/packages/markitdown/tests/test_rss_titles.py@@ -2,6 +2,9 @@"""Feed and entry titles must survive a pretty-printed feed."""import io+import sys++import pytestfrom markitdown import StreamInfofrom markitdown.converters import RssConverter@@ -146,3 +149,157 @@ def test_rss_single_line_titles_are_unchanged() -> None:"Published on: Mon, 01 Jan 2024 00:00:00 GMT\n""Body text.")++[email protected]("extension", [".rss", ".atom"])+ "payload, expected",+ [+ ("Quarterly <![CDATA[R&D]]> report", "Quarterly R&D report"),+ ("<![CDATA[Quarterly ]]><![CDATA[R&D]]> report", "Quarterly R&D report"),+ ("\n <![CDATA[Example]]>\n", "Example"),+ ("<b>nested</b> trailing", "nested trailing"),+ ("leading <b>nested</b> trailing", "leading nested trailing"),+ ("leading <b>nested</b>", "leading nested"),+ ("<b><i>nested</i></b>", "nested"),+ (+ " \n<!--ignore--><?instruction ignore?><b>nested</b> trailing",+ "nested trailing",+ ),+ ("co<b>op</b>erate", "cooperate"),+ ("<p>First</p><p>Second</p>", "First Second"),+ (+ "Use <slot> and &lt;literal&gt;",+ "Use <slot> and <literal>",+ ),+ ("", None),+ (" \n", None),+ ("<b/>", None),+ ("<!--ignore--><?instruction ignore?>", None),+ ],+)+def test_complete_feed_and_entry_titles(+ extension: str, payload: str, expected: str | None+) -> None:+ if extension == ".rss":+ xml = (+ f"<rss><channel><title>{payload}</title>"+ f"<item><title>{payload}</title><description>Body.</description></item>"+ "</channel></rss>"+ )+ else:+ xml = (+ f'<feed xmlns="http://www.w3.org/2005/Atom"><title>{payload}</title>'+ f"<entry><title>{payload}</title><content>Body.</content></entry></feed>"+ )+ result = RssConverter().convert(+ io.BytesIO(xml.encode()), StreamInfo(extension=extension)+ )++ assert result.title == expected+ assert result.markdown == (+ f"# {expected}\n\n## {expected}\nBody." if expected else "Body."+ )++[email protected]("prefix", ["", "a:"])+ "content_type, payload, expected",+ [+ ("html", "<b>nested</b> trailing", "nested trailing"),+ ("html", "<![CDATA[<b>nes]]><![CDATA[ted</b>]]> trailing", "nested trailing"),+ ("html", "<b>nested</b> trailing", "nested trailing"),+ ("html", "<p>First</p><p>Second</p>", "First Second"),+ ("html", "Use &lt;slot&gt;", "Use <slot>"),+ (+ "xhtml",+ '<x:div xmlns:x="http://www.w3.org/1999/xhtml">co<x:b>op</x:b>erate'+ " <![CDATA[<slot> <literal>]]></x:div>",+ "cooperate <slot> <literal>",+ ),+ ("text/plain", "Use <slot>", "Use <slot>"),+ ],+)+def test_atom_typed_titles_and_subtitles(+ prefix: str, content_type: str, payload: str, expected: str+) -> None:+ xml = (+ f'<{prefix}feed xmlns="http://www.w3.org/2005/Atom" '+ 'xmlns:a="http://www.w3.org/2005/Atom">'+ f'<{prefix}title type="{content_type}">{payload}</{prefix}title>'+ f'<{prefix}subtitle type="{content_type}">{payload}</{prefix}subtitle>'+ f'<{prefix}entry><{prefix}title type="{content_type}">{payload}</{prefix}title>'+ f"<{prefix}content>Body.</{prefix}content></{prefix}entry></{prefix}feed>"+ )+ result = RssConverter().convert(+ io.BytesIO(xml.encode()), StreamInfo(extension=".atom")+ )++ assert result.title == expected+ assert result.markdown == f"# {expected}\n{expected}\n\n## {expected}\nBody."++[email protected]("extension", [".rss", ".atom"])+def test_feed_does_not_borrow_nested_metadata(extension: str) -> None:+ if extension == ".rss":+ xml = (+ "<rss><channel><image><title>Wrong feed title</title></image>"+ "<item><title>Entry</title><description>Body.</description></item>"+ "<item><extension><title>Wrong entry title</title></extension>"+ "<description>Second body.</description></item></channel></rss>"+ )+ else:+ xml = (+ '<feed xmlns="http://www.w3.org/2005/Atom">'+ "<entry><title>Entry</title><content>Body.</content></entry>"+ "<entry><source><title>Wrong entry title</title><summary>Wrong body.</summary>"+ "</source><content>Second body.</content></entry></feed>"+ )+ result = RssConverter().convert(+ io.BytesIO(xml.encode()), StreamInfo(extension=extension)+ )++ assert result.title is None+ assert result.markdown.count("## Entry") == 1+ assert "Wrong" not in result.markdown+ assert not result.markdown.startswith("# ")+ assert "Body.\n\nSecond body." in result.markdown++[email protected]("content_type", ["rss", "text", "html", "xhtml"])+def test_deeply_nested_title_is_read_without_recursion(content_type: str) -> None:+ payload = "<b>" * 500 + "Deep title" + "</b>" * 500+ if content_type == "rss":+ xml = f"<rss><channel><title>{payload}</title></channel></rss>".encode()+ extension = ".rss"+ else:+ if content_type == "html":+ payload = f"<![CDATA[{payload}]]>"+ elif content_type == "xhtml":+ payload = f'<div xmlns="http://www.w3.org/1999/xhtml">{payload}</div>'+ xml = (+ f'<feed xmlns="http://www.w3.org/2005/Atom">'+ f'<title type="{content_type}">{payload}</title><entry/></feed>'+ ).encode()+ extension = ".atom"+ original_limit = sys.getrecursionlimit()+ try:+ sys.setrecursionlimit(200)+ result = RssConverter().convert(+ io.BytesIO(xml), StreamInfo(extension=extension)+ )+ finally:+ sys.setrecursionlimit(original_limit)++ assert result.title == "Deep title"+++def test_rss_channel_description_and_date_preserve_complete_text() -> None:+ xml = b"""<rss><channel><title>Feed</title>+ <description>About <![CDATA[R&D]]> and <b>research</b>.</description>+ <item><title>Entry</title><pubDate>Mon, <![CDATA[01 Jan 2024]]> 00:00:00 GMT</pubDate>+ </item></channel></rss>"""+ result = RssConverter().convert(io.BytesIO(xml), StreamInfo(extension=".rss"))++ assert "About R&D and research." in result.markdown+ assert "Published on: Mon, 01 Jan 2024 00:00:00 GMT" in result.markdown