Skip to content

Python tool for converting files and office documents to Markdown.

fix(rss): preserve complete feed content and resolve relative link (#2432)

afourney75cc34a364c1
diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.py
index d24eea4a..ce768b84 100644
--- a/packages/markitdown/src/markitdown/converters/_rss_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py
@@ -1,11 +1,12 @@
import textwrap
import warnings
+from html import escape
+from urllib.parse import urljoin
from defusedxml import minidom
-from xml.dom import Node
-from xml.dom.minidom import Document, Element
+from xml.dom.minidom import Document, Element, Node
from typing import BinaryIO, Any, Union
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, Tag
from ._markdownify import _CustomMarkdownify
from .._stream_info import StreamInfo
@@ -31,6 +32,24 @@ CANDIDATE_FILE_EXTENSIONS = [
ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
+CONTENT_NAMESPACE = "http://purl.org/rss/1.0/modules/content/"
+XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace"
+
+# Boundaries to retain when reducing markup to readable heading/plain text.
+# Inline elements must not introduce spaces (e.g. co<b>op</b>erate).
+_TEXT_BREAK_ELEMENTS = frozenset(
+ "address article aside blockquote br dd div dl dt figcaption figure footer "
+ "h1 h2 h3 h4 h5 h6 header hr li main nav ol p pre section table td th tr ul".split()
+)
+
+
+def _resolve_url(base_url: str, reference: str) -> str:
+ """Resolve a reference without letting a malformed URI drop the body."""
+ try:
+ return urljoin(base_url, reference)
+ except ValueError:
+ # Leave malformed links to the Markdown converter's existing handling.
+ return reference
def _atom_content_kind(content_type: str, *, allow_media_types: bool) -> str:
@@ -107,12 +126,14 @@ class RssConverter(DocumentConverter):
file_stream.seek(cur_pos)
return False
- def _feed_type(self, doc: Any) -> str | None:
- if doc.getElementsByTagName("rss"):
- return "rss"
+ def _feed_type(self, doc: Document) -> str | None:
root = doc.documentElement
+ if root is None:
+ return None
+ if root.tagName == "rss":
+ return "rss"
if root.localName == "feed" and root.namespaceURI in (None, ATOM_NAMESPACE):
- if root.getElementsByTagNameNS(root.namespaceURI, "entry"):
+ if self._get_children(root, "entry"):
# An Atom feed must have a root element of <feed> and at least one <entry>
return "atom"
return None
@@ -128,6 +149,7 @@ class RssConverter(DocumentConverter):
strict: bool = kwargs.pop("strict", False)
self._kwargs = kwargs
doc = minidom.parse(file_stream)
+ doc.documentURI = stream_info.url or kwargs.get("url")
feed_type = self._feed_type(doc)
if feed_type == "rss":
@@ -145,15 +167,16 @@ class RssConverter(DocumentConverter):
Returns None if the feed type is not recognized or something goes wrong.
"""
root = doc.documentElement
- title = self._get_flattened_text(root, "title")
- subtitle = self._get_flattened_text(root, "subtitle")
- entries = root.getElementsByTagNameNS(root.namespaceURI, "entry")
+ assert root is not None
+ title = self._get_flattened_text(root, "title", atom_text=True)
+ subtitle = self._get_flattened_text(root, "subtitle", atom_text=True)
+ entries = self._get_children(root, "entry")
md_text = f"# {title}\n" if title else ""
if subtitle:
md_text += f"{subtitle}\n"
for entry in entries:
- entry_title = self._get_flattened_text(entry, "title")
+ entry_title = self._get_flattened_text(entry, "title", atom_text=True)
entry_summary, summary_is_markup = self._get_atom_content(entry, "summary")
entry_updated = self._get_flattened_text(entry, "updated")
entry_content, content_is_markup = self._get_atom_content(entry, "content")
@@ -162,14 +185,23 @@ class RssConverter(DocumentConverter):
md_text += f"\n## {entry_title}\n"
if entry_updated:
md_text += f"Updated on: {entry_updated}\n"
- if entry_summary:
- md_text += self._render_atom_content(
- entry_summary, is_markup=summary_is_markup, strict=strict
+ body_parts = (
+ self._render_atom_content(
+ value,
+ is_markup=is_markup,
+ base_url=self._get_field_base_url(entry, tag_name),
+ strict=strict,
)
- if entry_content:
- md_text += self._render_atom_content(
- entry_content, is_markup=content_is_markup, strict=strict
+ for value, is_markup, tag_name in (
+ (entry_summary, summary_is_markup, "summary"),
+ (entry_content, content_is_markup, "content"),
)
+ if value
+ )
+ body = "\n\n".join(part for part in body_parts if part)
+ if body and md_text and not md_text.endswith("\n"):
+ md_text += "\n\n"
+ md_text += body
return DocumentConverterResult(
markdown=md_text,
@@ -179,35 +211,25 @@ class RssConverter(DocumentConverter):
def _get_atom_content(
self, entry: Element, tag_name: str
) -> tuple[Union[str, None], bool]:
- """Return an Atom summary or content value, and whether it is markup.
+ """Return an Atom text construct or content, and whether it is markup.
Values flagged as markup are converted by ``_parse_content``; plain text
is returned verbatim for the caller to emit as-is.
"""
- nodes = entry.getElementsByTagNameNS(entry.namespaceURI, tag_name)
- if not nodes:
+ node = self._get_child(entry, tag_name)
+ if node is None:
return None, True
- node = nodes[0]
kind = _atom_content_kind(
node.getAttribute("type"), allow_media_types=tag_name == "content"
)
- if kind == "xhtml":
- return (
- "".join(
- self._localize_xhtml_names(child.cloneNode(True)).toxml()
- for child in node.childNodes
- if child.nodeType == Node.ELEMENT_NODE
- ),
- True,
- )
if kind == "binary":
# A base64-encoded payload; there is no text to render.
return None, True
- text = self._get_data_by_tag_name(entry, tag_name)
- if text is None or kind == "html":
- return text, True
+ text = self._read_content(node, kind=kind)
+ if kind != "text":
+ return text or None, True
# Plain text carries no markup. Drop the whitespace the feed used to
# lay the element out -- indentation carried into the output would read
@@ -220,32 +242,18 @@ class RssConverter(DocumentConverter):
return text.strip(), False
def _render_atom_content(
- self, value: str, *, is_markup: bool, strict: bool = False
+ self, value: str, *, is_markup: bool, base_url: str = "", strict: bool = False
) -> str:
"""Render one Atom summary or content value as markdown."""
if not is_markup:
# Plain text is returned verbatim: routing it through the HTML
# parser drops tag-shaped text such as ``<job_id>`` entirely.
return value
- return self._parse_content(value, strict=strict)
+ return self._parse_content(value, base_url=base_url, strict=strict)
- def _localize_xhtml_names(self, node: Node) -> Node:
- """Rewrite prefixed XHTML element names to their local HTML names.
-
- Atom permits XHTML content to be namespace-prefixed (e.g. ``x:strong``).
- The downstream HTML converter dispatches on HTML tag names, so the
- prefix has to be dropped or the element is treated as an unknown tag
- and its formatting is lost.
- """
- if node.nodeType == Node.ELEMENT_NODE:
- if node.prefix and node.namespaceURI == XHTML_NAMESPACE:
- node.tagName = node.nodeName = node.localName
- node.prefix = None
- for child in node.childNodes:
- self._localize_xhtml_names(child)
- return node
-
- def _get_flattened_text(self, element: Element, tag_name: str) -> Union[str, None]:
+ def _get_flattened_text(
+ self, element: Element, tag_name: str, *, atom_text: bool = False
+ ) -> Union[str, None]:
"""Get a value that is rendered as a heading or a metadata line.
Feeds are routinely pretty-printed, so a value written as
@@ -255,10 +263,23 @@ class RssConverter(DocumentConverter):
text, so the layout whitespace is dropped here. A value that is
nothing but whitespace is reported as absent.
"""
- value = self._get_data_by_tag_name(element, tag_name)
+ if atom_text:
+ value, is_markup = self._get_atom_content(element, tag_name)
+ if value and is_markup:
+ soup = BeautifulSoup(value, "html.parser")
+ for block in soup.find_all(_TEXT_BREAK_ELEMENTS):
+ block.insert_before("\n")
+ block.append("\n")
+ value = soup.get_text()
+ else:
+ value = self._get_data_by_tag_name(element, tag_name)
if value is None:
return None
- return " ".join(part.strip() for part in value.splitlines()).strip() or None
+ # Block boundaries become line breaks, and adjacent blocks leave blank
+ # lines between them; drop the empty parts so the flattened value is
+ # separated by single spaces rather than by runs of them.
+ parts = (part.strip() for part in value.splitlines())
+ return " ".join(part for part in parts if part) or None
def _parse_rss_type(
self, doc: Document, *, strict: bool = False
@@ -267,14 +288,16 @@ class RssConverter(DocumentConverter):
Returns None if the feed type is not recognized or something goes wrong.
"""
- root = doc.getElementsByTagName("rss")[0]
- channel_list = root.getElementsByTagName("channel")
- if not channel_list:
+ root = doc.documentElement
+ assert root is not None
+ channel = self._get_child(root, "channel")
+ if channel is None:
raise ValueError("No channel found in RSS feed")
- channel = channel_list[0]
channel_title = self._get_flattened_text(channel, "title")
+ # A channel description is a text field; item descriptions carry HTML.
+ # Preserve its existing whitespace while collecting the complete value.
channel_description = self._get_data_by_tag_name(channel, "description")
- items = channel.getElementsByTagName("item")
+ items = self._get_children(channel, "item")
md_text = ""
if channel_title:
md_text += f"# {channel_title}\n"
@@ -282,29 +305,44 @@ class RssConverter(DocumentConverter):
md_text += f"{channel_description}\n"
for item in items:
title = self._get_flattened_text(item, "title")
- description = self._get_data_by_tag_name(item, "description")
+ description = self._get_data_by_tag_name(item, "description", kind="html")
pubDate = self._get_flattened_text(item, "pubDate")
- content = self._get_data_by_tag_name(item, "content:encoded")
+ content = self._get_data_by_tag_name(item, "content:encoded", kind="html")
if title:
md_text += f"\n## {title}\n"
if pubDate:
md_text += f"Published on: {pubDate}\n"
- if description:
- md_text += self._parse_content(description, strict=strict)
- if content:
- md_text += self._parse_content(content, strict=strict)
+ body_parts = (
+ self._parse_content(
+ value,
+ base_url=self._get_field_base_url(item, tag_name),
+ strict=strict,
+ )
+ for value, tag_name in (
+ (description, "description"),
+ (content, "content:encoded"),
+ )
+ if value
+ )
+ body = "\n\n".join(part for part in body_parts if part)
+ if body and md_text and not md_text.endswith("\n"):
+ md_text += "\n\n"
+ md_text += body
return DocumentConverterResult(
markdown=md_text,
title=channel_title,
)
- def _parse_content(self, content: str, *, strict: bool = False) -> str:
+ def _parse_content(
+ self, content: str, *, base_url: str = "", strict: bool = False
+ ) -> str:
"""Parse the content of an RSS feed item"""
try:
# using bs4 because many RSS feeds have HTML-styled content
soup = BeautifulSoup(content, "html.parser")
+ self._resolve_content_links(soup, base_url)
return _CustomMarkdownify(**self._kwargs).convert_soup(soup)
except RecursionError:
if strict:
@@ -322,20 +360,129 @@ class RssConverter(DocumentConverter):
except BaseException as _:
return content
- def _get_data_by_tag_name(
- self, element: Element, tag_name: str
- ) -> Union[str, None]:
- """Get data from first child element with the given tag name.
- Returns None when no such element is found.
+ def _get_field_base_url(self, element: Element, tag_name: str) -> str:
+ """Apply xml:base from the document root through the selected field.
+
+ Relative overrides resolve against their parent's base. An empty
+ xml:base inherits that base; it does not reset it to the document URL.
+ The document URI is local to this conversion, even when a converter is
+ reused. The source stream's URL also supplies a base for RSS HTML.
"""
- if element.namespaceURI == ATOM_NAMESPACE:
- nodes = element.getElementsByTagNameNS(ATOM_NAMESPACE, tag_name)
+ bases: list[str] = []
+ node: Node | None = self._get_child(element, tag_name)
+ while node is not None:
+ if isinstance(node, Element) and node.hasAttributeNS(XML_NAMESPACE, "base"):
+ bases.append(node.getAttributeNS(XML_NAMESPACE, "base"))
+ node = node.parentNode
+ document = element.ownerDocument
+ base_url = (document.documentURI or "") if document is not None else ""
+ for reference in reversed(bases):
+ base_url = _resolve_url(base_url, reference)
+ return base_url
+
+ def _resolve_content_links(self, soup: BeautifulSoup, base_url: str) -> None:
+ """Resolve rendered links/images, retaining nested XML Base scopes.
+
+ Inline XHTML keeps its xml:base attributes during serialization; HTML
+ carried in text/CDATA inherits the enclosing field's base. Traverse
+ iteratively so deeply nested content can still reach the fallback.
+ """
+ stack: list[tuple[Tag, str]] = [(soup, base_url)]
+ while stack:
+ node, inherited_base = stack.pop()
+ override = node.get("xml:base")
+ current_base = (
+ _resolve_url(inherited_base, override)
+ if isinstance(override, str)
+ else inherited_base
+ )
+ attributes: tuple[str, ...] = ()
+ if node.name == "a":
+ attributes = ("href",)
+ elif node.name == "img":
+ attributes = ("src", "data-src")
+ for attribute in attributes:
+ reference = node.get(attribute)
+ # Empty src must still allow markdownify's data-src fallback.
+ if isinstance(reference, str) and (reference or attribute == "href"):
+ node[attribute] = _resolve_url(current_base, reference)
+ stack.extend(
+ (child, current_base)
+ for child in node.children
+ if isinstance(child, Tag)
+ )
+
+ def _get_children(self, element: Element, tag_name: str) -> list[Element]:
+ """Select fields on their owner, without borrowing descendant metadata."""
+ namespace: str | None
+ if tag_name == "content:encoded":
+ namespace, local_name = CONTENT_NAMESPACE, "encoded"
else:
- nodes = element.getElementsByTagName(tag_name)
- if not nodes:
+ namespace, local_name = element.namespaceURI, tag_name
+ return [
+ child
+ for child in element.childNodes
+ if isinstance(child, Element)
+ and child.namespaceURI == namespace
+ and child.localName == local_name
+ ]
+
+ def _get_child(self, element: Element, tag_name: str) -> Element | None:
+ return next(iter(self._get_children(element, tag_name)), None)
+
+ def _get_data_by_tag_name(
+ self, element: Element, tag_name: str, *, kind: str = "text"
+ ) -> Union[str, None]:
+ """Read the complete value of the first matching direct child field."""
+ node = self._get_child(element, tag_name)
+ if node is None:
return None
- fc = nodes[0].firstChild
- if fc:
- if hasattr(fc, "data"):
- return fc.data
- return None
+ return self._read_content(node, kind=kind) or None
+
+ def _read_content(self, element: Element, *, kind: str) -> str:
+ """Collect plain text, encoded HTML, or inline XML in document order.
+
+ Encoded HTML's direct text/CDATA children are parts of one HTML string.
+ Text inside actual XML elements is already literal and must be escaped
+ when serializing those elements for the HTML parser. In XHTML, all text
+ is literal, including CDATA. Comments and processing instructions carry
+ no display content.
+
+ Use an explicit stack: minidom's recursive cloning/serialization can
+ fail on deep XML before the Markdown renderer's fallback is reached.
+ """
+ parts: list[str] = []
+ stack: list[Node | str] = list(reversed(element.childNodes))
+ while stack:
+ child = stack.pop()
+ if isinstance(child, str):
+ parts.append(child)
+ elif child.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
+ text = child.nodeValue or ""
+ if kind == "text" or (kind == "html" and child.parentNode is element):
+ parts.append(text)
+ else:
+ parts.append(escape(text, quote=False))
+ elif isinstance(child, Element):
+ if kind == "text":
+ if child.localName.lower() in _TEXT_BREAK_ELEMENTS:
+ parts.append("\n")
+ stack.append("\n")
+ else:
+ # markdownify recognizes local HTML names, not x:strong.
+ name = (
+ child.localName
+ if child.namespaceURI == XHTML_NAMESPACE
+ else child.tagName
+ )
+ attrs = "".join(
+ f' {attr.name}="{escape(attr.value, quote=True)}"'
+ for attr in child.attributes.values()
+ )
+ if child.childNodes:
+ parts.append(f"<{name}{attrs}>")
+ stack.append(f"</{name}>")
+ else:
+ parts.append(f"<{name}{attrs}/>")
+ stack.extend(reversed(child.childNodes))
+ return "".join(parts)
diff --git a/packages/markitdown/tests/test_rss_converter.py b/packages/markitdown/tests/test_rss_converter.py
index ddc690e4..6aaec65a 100644
--- a/packages/markitdown/tests/test_rss_converter.py
+++ b/packages/markitdown/tests/test_rss_converter.py
@@ -1,4 +1,5 @@
import io
+import sys
import pytest
@@ -338,3 +339,352 @@ def test_atom_plain_text_layout_whitespace_is_removed() -> None:
"",
"Then check status.",
]
+
+
+def _feed_with_body(field: str, payload: str, *, atom_type: str = "html") -> bytes:
+ if field.startswith("rss-"):
+ tag = "description" if field == "rss-description" else "content:encoded"
+ return (
+ '<rss version="2.0" '
+ 'xmlns:content="http://purl.org/rss/1.0/modules/content/">'
+ "<channel><title>Feed</title><description>About the feed.</description>"
+ f"<item><title>Entry</title><{tag}>{payload}</{tag}></item>"
+ "</channel></rss>"
+ ).encode()
+ tag = field.removeprefix("atom-")
+ return (
+ '<feed xmlns="http://www.w3.org/2005/Atom">'
+ "<title>Feed</title><entry><title>Entry</title>"
+ f'<{tag} type="{atom_type}">{payload}</{tag}>'
+ "</entry></feed>"
+ ).encode()
+
+
+ "field", ["rss-description", "rss-content", "atom-summary", "atom-content"]
+)
+ "payload, expected",
+ [
+ ("Plain body.", "Plain body."),
+ ("<![CDATA[<p>Only CDATA.</p>]]>", "Only CDATA."),
+ (
+ "\n <![CDATA[<p>The <strong>body</strong>.</p>]]>\n ",
+ "The **body**.",
+ ),
+ (
+ "Before <![CDATA[<str]]><![CDATA[ong>middle</strong>]]> after.",
+ "Before **middle** after.",
+ ),
+ ("<b>nested</b> trailing", "**nested** trailing"),
+ ("leading <b>nested</b> trailing", "leading **nested** trailing"),
+ ("leading <b>nested</b>", "leading **nested**"),
+ ("First<br/>Second", "First \nSecond"),
+ ("<pre>line 1\n line 2</pre>", "```\nline 1\n line 2\n```"),
+ (
+ "\n <!--ignore--><?instruction ignore?><b>nested<!--ignore--></b> trailing",
+ "**nested** trailing",
+ ),
+ (
+ '<p>Read <a href="https://example.com/?a=1&amp;b=2">this</a>.</p>'
+ "<p>Then <em>continue</em>.</p>",
+ "Read [this](https://example.com/?a=1&b=2).\n\nThen *continue*.",
+ ),
+ (
+ "<p>Use &lt;slot&gt; and &amp;lt;literal&amp;gt;.</p>",
+ "Use <slot> and &lt;literal&gt;.",
+ ),
+ ],
+)
+def test_feed_body_preserves_complete_content(
+ field: str, payload: str, expected: str
+) -> None:
+ feed = _feed_with_body(field, payload)
+ stream_info = StreamInfo(extension=".rss" if field.startswith("rss-") else ".atom")
+ result = RssConverter().convert(io.BytesIO(feed), stream_info)
+
+ assert result.markdown.split("## Entry\n", 1)[1] == expected
+
+
[email protected]("field", ["atom-summary", "atom-content"])
[email protected]("atom_type", ["text", "text/plain"])
+def test_atom_plain_mixed_content_stays_literal(field: str, atom_type: str) -> None:
+ feed = _feed_with_body(
+ field,
+ "Run <![CDATA[<job_id>]]> with <b>&amp;lt;literal&amp;gt;</b>."
+ "<!--ignore--><?instruction ignore?>",
+ atom_type=atom_type,
+ )
+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".atom"))
+
+ assert result.markdown.split("## Entry\n", 1)[1] == (
+ "Run <job_id> with &lt;literal&gt;."
+ )
+
+
[email protected]("atom_type", ["xhtml", "application/xhtml+xml"])
+def test_atom_xhtml_preserves_literal_text_and_surrounding_content(
+ atom_type: str,
+) -> None:
+ feed = _feed_with_body(
+ "atom-content",
+ 'Before &lt;slot&gt; <x:div xmlns:x="http://www.w3.org/1999/xhtml">'
+ "<x:p>Use <![CDATA[<slot> &lt;literal&gt;]]> and <x:b>bold</x:b>.</x:p>"
+ "<!--ignore--><?instruction ignore?>"
+ "<x:pre>line 1\n line 2</x:pre></x:div> after.",
+ atom_type=atom_type,
+ )
+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".atom"))
+
+ body = result.markdown.split("## Entry\n", 1)[1]
+ assert body.startswith("Before <slot>")
+ assert "Use <slot> &lt;literal&gt; and **bold**." in body
+ assert "```\nline 1\n line 2\n```" in body
+ assert body.endswith("after.")
+ assert "ignore" not in body
+
+
[email protected]("extension", [".rss", ".atom"])
+def test_feed_body_fields_are_separated(extension: str) -> None:
+ if extension == ".rss":
+ feed = _feed_with_body("rss-description", "Summary.").replace(
+ b"</item>", b"<content:encoded>Body.</content:encoded></item>"
+ )
+ else:
+ feed = _feed_with_body("atom-summary", "Summary.", atom_type="text").replace(
+ b"</entry>", b'<content type="text">Body.</content></entry>'
+ )
+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=extension))
+
+ assert result.markdown.split("## Entry\n", 1)[1] == "Summary.\n\nBody."
+
+
[email protected]("extension", [".rss", ".atom"])
+ "summary, content, expected",
+ [("<b/>", "Body.", "Body."), ("Summary.", "<b/>", "Summary."), ("<b/>", "", "")],
+)
+def test_empty_markup_fields_do_not_add_body_separators(
+ extension: str, summary: str, content: str, expected: str
+) -> None:
+ if extension == ".rss":
+ feed = _feed_with_body("rss-description", summary).replace(
+ b"</item>", f"<content:encoded>{content}</content:encoded></item>".encode()
+ )
+ else:
+ feed = _feed_with_body("atom-summary", summary).replace(
+ b"</entry>", f'<content type="html">{content}</content></entry>'.encode()
+ )
+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=extension))
+
+ assert result.markdown.split("## Entry\n", 1)[1] == expected
+
+
[email protected]("field", ["rss-description", "atom-content"])
+def test_complete_feed_content_through_public_api(field: str) -> None:
+ feed = _feed_with_body(field, "Before <b>nested</b> <![CDATA[after.]]>")
+ result = MarkItDown().convert_stream(
+ io.BytesIO(feed), stream_info=StreamInfo(extension=".xml")
+ )
+
+ assert result.title == "Feed"
+ assert result.markdown.endswith("## Entry\nBefore **nested** after.")
+
+
[email protected]("field", ["rss-description", "atom-content"])
+def test_deep_xml_body_uses_rendering_fallback(field: str) -> None:
+ payload = "<div>" * 500 + "Deep <b>body</b>." + "</div>" * 500
+ feed = _feed_with_body(field, payload, atom_type="xhtml")
+ stream_info = StreamInfo(extension=".rss" if field.startswith("rss-") else ".atom")
+ original_limit = sys.getrecursionlimit()
+ try:
+ sys.setrecursionlimit(200)
+ with pytest.warns(UserWarning, match="too deeply nested"):
+ result = RssConverter().convert(io.BytesIO(feed), stream_info)
+ with pytest.raises(RecursionError):
+ RssConverter().convert(io.BytesIO(feed), stream_info, strict=True)
+ finally:
+ sys.setrecursionlimit(original_limit)
+
+ assert "Deep" in result.markdown
+ assert "body" in result.markdown
+ assert "<div>" not in result.markdown
+
+
+def test_rss_content_namespace_alias_and_field_ownership() -> None:
+ feed = b"""<rss xmlns:c="http://purl.org/rss/1.0/modules/content/">
+ <channel><title>Feed</title><item><title>Entry</title>
+ <extension><description>Wrong body.</description></extension>
+ <c:encoded>Before <b>nested</b> <![CDATA[after.]]></c:encoded>
+ </item></channel>
+ </rss>"""
+ result = RssConverter().convert(io.BytesIO(feed), StreamInfo(extension=".rss"))
+
+ assert result.markdown == "# Feed\n\n## Entry\nBefore **nested** after."
+
+
+ "field", ["rss-description", "rss-content", "atom-summary", "atom-content"]
+)
[email protected]("cdata", [False, True])
+def test_feed_relative_links_use_document_url(field: str, cdata: bool) -> None:
+ payload = '<a href="../stories/café%20notes?x=1&amp;y=2#part">Read</a>'
+ if cdata:
+ payload = f"<![CDATA[{payload}]]>"
+ feed = _feed_with_body(field, payload)
+ result = MarkItDown().convert_stream(
+ io.BytesIO(feed),
+ stream_info=StreamInfo(
+ extension=".xml", url="https://example.com/news/feed.xml"
+ ),
+ )
+
+ assert result.markdown.endswith(
+ "[Read](https://example.com/stories/caf%C3%A9%20notes?x=1&y=2#part)"
+ )
+
+
+def test_atom_inherits_xml_base_and_scopes_nested_overrides() -> None:
+ feed = b"""<a:feed xmlns:a="http://www.w3.org/2005/Atom"
+ xmlns:x="http://www.w3.org/1999/xhtml" xml:base="https://example.com/root/">
+ <a:title>Feed</a:title>
+ <a:entry xml:base="entries/">
+ <a:title>Entry</a:title>
+ <a:summary type="html" xml:base="../summaries/">
+ <![CDATA[<a href="summary">Summary</a>]]>
+ </a:summary>
+ <a:content type="xhtml" xml:base="../articles/">
+ <x:div xml:base="chapter/">
+ <x:p xml:base="../appendix/">
+ <x:a xml:base="../../assets/" href="story">Story</x:a>
+ <x:a href="sibling">Sibling</x:a>
+ </x:p>
+ <x:a href="after">After</x:a>
+ <x:img src="diagram.png" alt="Diagram"/>
+ <x:img src="" data-src="../lazy.png" alt="Lazy"/>
+ </x:div>
+ </a:content>
+ </a:entry>
+ <a:entry><a:title>Next</a:title>
+ <a:content type="html">&lt;a href="next"&gt;Next link&lt;/a&gt;</a:content>
+ </a:entry>
+ </a:feed>"""
+ result = RssConverter().convert(
+ io.BytesIO(feed),
+ StreamInfo(extension=".atom", url="https://other.example/feed.xml"),
+ )
+
+ assert "[Summary](https://example.com/root/summaries/summary)" in result.markdown
+ assert "[Story](https://example.com/root/assets/story)" in result.markdown
+ assert (
+ "[Sibling](https://example.com/root/articles/appendix/sibling)"
+ in result.markdown
+ )
+ assert "[After](https://example.com/root/articles/chapter/after)" in result.markdown
+ assert (
+ "![Diagram](https://example.com/root/articles/chapter/diagram.png)"
+ in result.markdown
+ )
+ assert "![Lazy](https://example.com/root/articles/lazy.png)" in result.markdown
+ assert "[Next link](https://example.com/root/next)" in result.markdown
+
+
+ "document_url, feed_base, entry_base, content_base, expected",
+ [
+ (
+ "https://example.com/news/feed.xml",
+ "../stories/",
+ "2026/",
+ "details/",
+ "https://example.com/stories/2026/details/story",
+ ),
+ (
+ "https://example.com/news/feed.xml",
+ "https://other.example/base/",
+ "section/",
+ "",
+ "https://other.example/base/section/story",
+ ),
+ (None, "https://example.com/", "", "", "https://example.com/story"),
+ (None, "", "", "", "story"),
+ ],
+)
+def test_atom_relative_and_empty_xml_base(
+ document_url: str | None,
+ feed_base: str,
+ entry_base: str,
+ content_base: str,
+ expected: str,
+) -> None:
+ xml = (
+ f'<feed xmlns="http://www.w3.org/2005/Atom" xml:base="{feed_base}">'
+ f'<entry xml:base="{entry_base}"><content type="html" xml:base="{content_base}">'
+ '<![CDATA[<a href="story">Read</a>]]></content></entry></feed>'
+ )
+ result = RssConverter().convert(
+ io.BytesIO(xml.encode()), StreamInfo(extension=".atom", url=document_url)
+ )
+
+ assert result.markdown == f"[Read]({expected})"
+
+
+ "href, expected",
+ [
+ ("#section", "https://example.com/news/feed.xml?lang=en#section"),
+ ("?page=2", "https://example.com/news/feed.xml?page=2"),
+ ("/story", "https://example.com/story"),
+ ("//cdn.example.com/story", "https://cdn.example.com/story"),
+ ("https://other.example/story", "https://other.example/story"),
+ ],
+)
+def test_feed_link_reference_forms(href: str, expected: str) -> None:
+ feed = _feed_with_body("atom-content", f'<a href="{href}">Read</a>')
+ result = RssConverter().convert(
+ io.BytesIO(feed),
+ StreamInfo(extension=".atom", url="https://example.com/news/feed.xml?lang=en"),
+ )
+
+ assert result.markdown.endswith(f"[Read]({expected})")
+
+
+def test_link_resolution_preserves_plain_atom_text() -> None:
+ feed = _feed_with_body(
+ "atom-content",
+ '<![CDATA[<a href="story">literal</a>]]>',
+ atom_type="text",
+ )
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(extension=".atom", url="https://example.com/feed")
+ )
+
+ assert result.markdown.endswith('<a href="story">literal</a>')
+
+
+def test_link_resolution_retains_existing_invalid_link_handling() -> None:
+ feed = _feed_with_body(
+ "atom-content",
+ '<a href="http://[invalid">Malformed</a> '
+ '<a href="javascript:alert(1)">Script</a> '
+ '<a href="story">Read</a>',
+ )
+ result = RssConverter().convert(
+ io.BytesIO(feed), StreamInfo(extension=".atom", url="https://example.com/feed")
+ )
+
+ assert result.markdown.endswith(
+ "Malformed Script [Read](https://example.com/story)"
+ )
+
+
+def test_document_base_does_not_leak_between_conversions() -> None:
+ feed = _feed_with_body("atom-content", '<a href="story">Read</a>')
+ converter = RssConverter()
+ first = converter.convert(
+ io.BytesIO(feed), StreamInfo(extension=".atom"), url="https://example.com/feed"
+ )
+ second = converter.convert(io.BytesIO(feed), StreamInfo(extension=".atom"))
+
+ assert first.markdown.endswith("[Read](https://example.com/story)")
+ assert second.markdown.endswith("[Read](story)")
diff --git a/packages/markitdown/tests/test_rss_titles.py b/packages/markitdown/tests/test_rss_titles.py
index 7302d88a..ce21b3a6 100644
--- a/packages/markitdown/tests/test_rss_titles.py
+++ b/packages/markitdown/tests/test_rss_titles.py
@@ -2,6 +2,9 @@
"""Feed and entry titles must survive a pretty-printed feed."""
import io
+import sys
+
+import pytest
from markitdown import StreamInfo
from markitdown.converters import RssConverter
@@ -146,3 +149,157 @@ def test_rss_single_line_titles_are_unchanged() -> None:
"Published on: Mon, 01 Jan 2024 00:00:00 GMT\n"
"Body text."
)
+
+
[email protected]("extension", [".rss", ".atom"])
+ "payload, expected",
+ [
+ ("Quarterly <![CDATA[R&D]]> report", "Quarterly R&D report"),
+ ("<![CDATA[Quarterly ]]><![CDATA[R&D]]> report", "Quarterly R&D report"),
+ ("\n <![CDATA[Example]]>\n", "Example"),
+ ("<b>nested</b> trailing", "nested trailing"),
+ ("leading <b>nested</b> trailing", "leading nested trailing"),
+ ("leading <b>nested</b>", "leading nested"),
+ ("<b><i>nested</i></b>", "nested"),
+ (
+ " \n<!--ignore--><?instruction ignore?><b>nested</b> trailing",
+ "nested trailing",
+ ),
+ ("co<b>op</b>erate", "cooperate"),
+ ("<p>First</p><p>Second</p>", "First Second"),
+ (
+ "Use &lt;slot&gt; and &amp;lt;literal&amp;gt;",
+ "Use <slot> and &lt;literal&gt;",
+ ),
+ ("", None),
+ (" \n", None),
+ ("<b/>", None),
+ ("<!--ignore--><?instruction ignore?>", None),
+ ],
+)
+def test_complete_feed_and_entry_titles(
+ extension: str, payload: str, expected: str | None
+) -> None:
+ if extension == ".rss":
+ xml = (
+ f"<rss><channel><title>{payload}</title>"
+ f"<item><title>{payload}</title><description>Body.</description></item>"
+ "</channel></rss>"
+ )
+ else:
+ xml = (
+ f'<feed xmlns="http://www.w3.org/2005/Atom"><title>{payload}</title>'
+ f"<entry><title>{payload}</title><content>Body.</content></entry></feed>"
+ )
+ result = RssConverter().convert(
+ io.BytesIO(xml.encode()), StreamInfo(extension=extension)
+ )
+
+ assert result.title == expected
+ assert result.markdown == (
+ f"# {expected}\n\n## {expected}\nBody." if expected else "Body."
+ )
+
+
[email protected]("prefix", ["", "a:"])
+ "content_type, payload, expected",
+ [
+ ("html", "&lt;b&gt;nested&lt;/b&gt; trailing", "nested trailing"),
+ ("html", "<![CDATA[<b>nes]]><![CDATA[ted</b>]]> trailing", "nested trailing"),
+ ("html", "<b>nested</b> trailing", "nested trailing"),
+ ("html", "&lt;p&gt;First&lt;/p&gt;&lt;p&gt;Second&lt;/p&gt;", "First Second"),
+ ("html", "Use &amp;lt;slot&amp;gt;", "Use <slot>"),
+ (
+ "xhtml",
+ '<x:div xmlns:x="http://www.w3.org/1999/xhtml">co<x:b>op</x:b>erate'
+ " <![CDATA[<slot> &lt;literal&gt;]]></x:div>",
+ "cooperate <slot> &lt;literal&gt;",
+ ),
+ ("text/plain", "Use &lt;slot&gt;", "Use <slot>"),
+ ],
+)
+def test_atom_typed_titles_and_subtitles(
+ prefix: str, content_type: str, payload: str, expected: str
+) -> None:
+ xml = (
+ f'<{prefix}feed xmlns="http://www.w3.org/2005/Atom" '
+ 'xmlns:a="http://www.w3.org/2005/Atom">'
+ f'<{prefix}title type="{content_type}">{payload}</{prefix}title>'
+ f'<{prefix}subtitle type="{content_type}">{payload}</{prefix}subtitle>'
+ f'<{prefix}entry><{prefix}title type="{content_type}">{payload}</{prefix}title>'
+ f"<{prefix}content>Body.</{prefix}content></{prefix}entry></{prefix}feed>"
+ )
+ result = RssConverter().convert(
+ io.BytesIO(xml.encode()), StreamInfo(extension=".atom")
+ )
+
+ assert result.title == expected
+ assert result.markdown == f"# {expected}\n{expected}\n\n## {expected}\nBody."
+
+
[email protected]("extension", [".rss", ".atom"])
+def test_feed_does_not_borrow_nested_metadata(extension: str) -> None:
+ if extension == ".rss":
+ xml = (
+ "<rss><channel><image><title>Wrong feed title</title></image>"
+ "<item><title>Entry</title><description>Body.</description></item>"
+ "<item><extension><title>Wrong entry title</title></extension>"
+ "<description>Second body.</description></item></channel></rss>"
+ )
+ else:
+ xml = (
+ '<feed xmlns="http://www.w3.org/2005/Atom">'
+ "<entry><title>Entry</title><content>Body.</content></entry>"
+ "<entry><source><title>Wrong entry title</title><summary>Wrong body.</summary>"
+ "</source><content>Second body.</content></entry></feed>"
+ )
+ result = RssConverter().convert(
+ io.BytesIO(xml.encode()), StreamInfo(extension=extension)
+ )
+
+ assert result.title is None
+ assert result.markdown.count("## Entry") == 1
+ assert "Wrong" not in result.markdown
+ assert not result.markdown.startswith("# ")
+ assert "Body.\n\nSecond body." in result.markdown
+
+
[email protected]("content_type", ["rss", "text", "html", "xhtml"])
+def test_deeply_nested_title_is_read_without_recursion(content_type: str) -> None:
+ payload = "<b>" * 500 + "Deep title" + "</b>" * 500
+ if content_type == "rss":
+ xml = f"<rss><channel><title>{payload}</title></channel></rss>".encode()
+ extension = ".rss"
+ else:
+ if content_type == "html":
+ payload = f"<![CDATA[{payload}]]>"
+ elif content_type == "xhtml":
+ payload = f'<div xmlns="http://www.w3.org/1999/xhtml">{payload}</div>'
+ xml = (
+ f'<feed xmlns="http://www.w3.org/2005/Atom">'
+ f'<title type="{content_type}">{payload}</title><entry/></feed>'
+ ).encode()
+ extension = ".atom"
+ original_limit = sys.getrecursionlimit()
+ try:
+ sys.setrecursionlimit(200)
+ result = RssConverter().convert(
+ io.BytesIO(xml), StreamInfo(extension=extension)
+ )
+ finally:
+ sys.setrecursionlimit(original_limit)
+
+ assert result.title == "Deep title"
+
+
+def test_rss_channel_description_and_date_preserve_complete_text() -> None:
+ xml = b"""<rss><channel><title>Feed</title>
+ <description>About <![CDATA[R&D]]> and <b>research</b>.</description>
+ <item><title>Entry</title><pubDate>Mon, <![CDATA[01 Jan 2024]]> 00:00:00 GMT</pubDate>
+ </item></channel></rss>"""
+ result = RssConverter().convert(io.BytesIO(xml), StreamInfo(extension=".rss"))
+
+ assert "About R&D and research." in result.markdown
+ assert "Published on: Mon, 01 Jan 2024 00:00:00 GMT" in result.markdown