Python tool for converting files and office documents to Markdown.
fix(wikipedia): drop the site suffix from italicised titles (#2458)
Machen John1f9530a6d353
diff --git a/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py b/packages/markitdown/src/markitdown/converters/_wikipedia_converter.pyindex 82825046..f9bc1aed 100644--- a/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py+++ b/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py@@ -64,7 +64,12 @@ class WikipediaConverter(DocumentConverter):# Print only the main contentbody_elm = soup.find("div", {"id": "mw-content-text"})- title_elm = soup.find("span", {"class": "mw-page-title-main"})+ # Wikipedia only wraps the title in mw-page-title-main when it is plain+ # text. Titles that carry markup -- italicised species, film, album and+ # journal names -- are written straight into the first heading instead.+ title_elm = soup.find("span", {"class": "mw-page-title-main"}) or soup.find(+ "h1", {"id": "firstHeading"}+ )webpage_text = ""main_title = None if soup.title is None else soup.title.string@@ -72,7 +77,10 @@ class WikipediaConverter(DocumentConverter):if body_elm:# What's the titleif title_elm and isinstance(title_elm, bs4.Tag):- main_title = title_elm.string+ # .string is None as soon as the element holds more than one+ # child, which is what a title mixing markup with plain text+ # looks like, so gather the descendant text instead.+ main_title = title_elm.get_text()# Treat whitespace-only titles as if they were absentif main_title:diff --git a/packages/markitdown/tests/test_wikipedia_titles.py b/packages/markitdown/tests/test_wikipedia_titles.pynew file mode 100644index 00000000..47708e46--- /dev/null+++ b/packages/markitdown/tests/test_wikipedia_titles.py@@ -0,0 +1,57 @@+#!/usr/bin/env python3 -m pytest+"""Article titles that carry markup must survive conversion."""++import io++from markitdown import StreamInfo+from markitdown.converters import WikipediaConverter++STREAM_INFO = StreamInfo(+ url="https://en.wikipedia.org/wiki/Example",+ mimetype="text/html",+ extension=".html",+)+++def _page(heading: str, document_title: str) -> io.BytesIO:+ """Build a page shaped like the article HTML Wikipedia actually serves."""+ return io.BytesIO(+ f"""<html><head><title>{document_title} - Wikipedia</title></head><body>+<h1 id="firstHeading" class="firstHeading mw-first-heading">{heading}</h1>+<div id="mw-content-text"><p>Body text.</p></div>+</body></html>""".encode()+ )+++def test_plain_title_is_read_from_the_title_span() -> None:+ """A plain title keeps being read from mw-page-title-main."""+ page = _page(+ '<span lang="en" dir="ltr"><span class="mw-page-title-main">Paris</span></span>',+ "Paris",+ )++ result = WikipediaConverter().convert(page, STREAM_INFO)++ assert result.title == "Paris"+ assert result.markdown.lstrip().startswith("# Paris")+++def test_fully_italicised_title_keeps_its_text() -> None:+ """Wikipedia drops the title span for italicised titles, e.g. species names."""+ page = _page("<i>Escherichia coli</i>", "Escherichia coli")++ result = WikipediaConverter().convert(page, STREAM_INFO)++ # Without the first-heading fallback this became "Escherichia coli - Wikipedia".+ assert result.title == "Escherichia coli"+ assert result.markdown.lstrip().startswith("# Escherichia coli")+++def test_title_mixing_markup_and_plain_text_keeps_both_parts() -> None:+ """A disambiguated italic title spans several children, so .string is None."""+ page = _page("<i>Titanic</i> (1997 film)", "Titanic (1997 film)")++ result = WikipediaConverter().convert(page, STREAM_INFO)++ assert result.title == "Titanic (1997 film)"+ assert result.markdown.lstrip().startswith("# Titanic (1997 film)")