Skip to content

Python tool for converting files and office documents to Markdown.

fix(wikipedia): drop the site suffix from italicised titles (#2458)

Machen John1f9530a6d353
diff --git a/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py b/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py
index 82825046..f9bc1aed 100644
--- a/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py
@@ -64,7 +64,12 @@ class WikipediaConverter(DocumentConverter):
# Print only the main content
body_elm = soup.find("div", {"id": "mw-content-text"})
- title_elm = soup.find("span", {"class": "mw-page-title-main"})
+ # Wikipedia only wraps the title in mw-page-title-main when it is plain
+ # text. Titles that carry markup -- italicised species, film, album and
+ # journal names -- are written straight into the first heading instead.
+ title_elm = soup.find("span", {"class": "mw-page-title-main"}) or soup.find(
+ "h1", {"id": "firstHeading"}
+ )
webpage_text = ""
main_title = None if soup.title is None else soup.title.string
@@ -72,7 +77,10 @@ class WikipediaConverter(DocumentConverter):
if body_elm:
# What's the title
if title_elm and isinstance(title_elm, bs4.Tag):
- main_title = title_elm.string
+ # .string is None as soon as the element holds more than one
+ # child, which is what a title mixing markup with plain text
+ # looks like, so gather the descendant text instead.
+ main_title = title_elm.get_text()
# Treat whitespace-only titles as if they were absent
if main_title:
diff --git a/packages/markitdown/tests/test_wikipedia_titles.py b/packages/markitdown/tests/test_wikipedia_titles.py
new file mode 100644
index 00000000..47708e46
--- /dev/null
+++ b/packages/markitdown/tests/test_wikipedia_titles.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3 -m pytest
+"""Article titles that carry markup must survive conversion."""
+
+import io
+
+from markitdown import StreamInfo
+from markitdown.converters import WikipediaConverter
+
+STREAM_INFO = StreamInfo(
+ url="https://en.wikipedia.org/wiki/Example",
+ mimetype="text/html",
+ extension=".html",
+)
+
+
+def _page(heading: str, document_title: str) -> io.BytesIO:
+ """Build a page shaped like the article HTML Wikipedia actually serves."""
+ return io.BytesIO(
+ f"""<html><head><title>{document_title} - Wikipedia</title></head><body>
+<h1 id="firstHeading" class="firstHeading mw-first-heading">{heading}</h1>
+<div id="mw-content-text"><p>Body text.</p></div>
+</body></html>""".encode()
+ )
+
+
+def test_plain_title_is_read_from_the_title_span() -> None:
+ """A plain title keeps being read from mw-page-title-main."""
+ page = _page(
+ '<span lang="en" dir="ltr"><span class="mw-page-title-main">Paris</span></span>',
+ "Paris",
+ )
+
+ result = WikipediaConverter().convert(page, STREAM_INFO)
+
+ assert result.title == "Paris"
+ assert result.markdown.lstrip().startswith("# Paris")
+
+
+def test_fully_italicised_title_keeps_its_text() -> None:
+ """Wikipedia drops the title span for italicised titles, e.g. species names."""
+ page = _page("<i>Escherichia coli</i>", "Escherichia coli")
+
+ result = WikipediaConverter().convert(page, STREAM_INFO)
+
+ # Without the first-heading fallback this became "Escherichia coli - Wikipedia".
+ assert result.title == "Escherichia coli"
+ assert result.markdown.lstrip().startswith("# Escherichia coli")
+
+
+def test_title_mixing_markup_and_plain_text_keeps_both_parts() -> None:
+ """A disambiguated italic title spans several children, so .string is None."""
+ page = _page("<i>Titanic</i> (1997 film)", "Titanic (1997 film)")
+
+ result = WikipediaConverter().convert(page, STREAM_INFO)
+
+ assert result.title == "Titanic (1997 film)"
+ assert result.markdown.lstrip().startswith("# Titanic (1997 film)")