Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
lib: Switch convert_html_to_markdown to use markdownify.
Dhruv Shetty7aebdf4c4616
diff --git a/pyproject.toml b/pyproject.tomlindex 20f7ba56f4d..71e864881d4 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -52,6 +52,7 @@ prod = [# Needed for the email mirror"html2text",+ "markdownify","talon-core",# Needed for inlining the CSS in emailsdiff --git a/uv.lock b/uv.lockindex ab37d3fcb86..ad687f7c10d 100644--- a/uv.lock+++ b/uv.lock@@ -2586,6 +2586,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" },]+[[package]]+name = "markdownify"+version = "1.2.2"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "beautifulsoup4" },+ { name = "six" },+]+sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" },+]+[[package]]name = "markupsafe"version = "3.0.3"@@ -6364,6 +6377,7 @@ dev = [{ name = "lxml-stubs" },{ name = "markdown" },{ name = "markdown-it-py" },+ { name = "markdownify" },{ name = "moto", extra = ["s3"] },{ name = "mypy", extra = ["faster-cache"] },{ name = "mypy-boto3-s3" },@@ -6498,6 +6512,7 @@ prod = [{ name = "jsx-lexer" },{ name = "lxml" },{ name = "markdown" },+ { name = "markdownify" },{ name = "mypy-boto3-s3" },{ name = "mypy-boto3-ses" },{ name = "mypy-boto3-sns" },@@ -6603,6 +6618,7 @@ dev = [{ name = "lxml-stubs" },{ name = "markdown" },{ name = "markdown-it-py", specifier = "<4" },+ { name = "markdownify" },{ name = "moto", extras = ["s3"] },{ name = "mypy", extras = ["faster-cache"] },{ name = "mypy-boto3-s3" },@@ -6738,6 +6754,7 @@ prod = [{ name = "jsx-lexer" },{ name = "lxml" },{ name = "markdown" },+ { name = "markdownify" },{ name = "mypy-boto3-s3" },{ name = "mypy-boto3-ses" },{ name = "mypy-boto3-sns" },diff --git a/version.py b/version.pyindex 8c1545efe8f..9361dc8d3af 100644--- a/version.py+++ b/version.py@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 509# historical commits sharing the same major version, in which case a# minor version bump suffices.-PROVISION_VERSION = (386, 0) # bumped 2026-07-28 to upgrade JavaScript dependencies+PROVISION_VERSION = (386, 1) # bumped 2026-08-13 to add markdownify dependencydiff --git a/zerver/lib/markdown/from_html.py b/zerver/lib/markdown/from_html.pyindex 6a505c7650d..6f71b73d593 100644--- a/zerver/lib/markdown/from_html.py+++ b/zerver/lib/markdown/from_html.py@@ -1,18 +1,72 @@-import os+import posixpathimport re-import subprocessimport sys+import warnings+from urllib.parse import urlsplit++import markdownify+from bs4 import MarkupResemblesLocatorWarning, Tag++from zerver.lib.markdown import get_markdown_link_for_url, sanitize_url+++class ZulipMarkdownConverter(markdownify.MarkdownConverter):+ """HTML-to-Markdown converter adapted to Zulip's Markdown.++ Zulip doesn't inline-render external images, so images become+ `[label](src)` links instead of broken ``.+ """++ def convert_img(self, el: Tag, text: str, parent_tags: set[str]) -> str:+ src = el.get("src", "")+ alt = el.get("alt", "")+ # BeautifulSoup returns a list only for multi-valued attributes+ # (e.g. class), never for src or alt.+ assert isinstance(src, str)+ assert isinstance(alt, str)+ if not src:+ return alt+ url = sanitize_url(src)+ if not url:+ # Unlinkable src (e.g. data: URIs, unsafe schemes): emit just the+ # alt text rather than a bogus link.+ return alt+ label = alt or posixpath.basename(urlsplit(url).path)+ if not label:+ # No alt text, and no filename in the path to fall back on, as in+ # `<img src="https://example.com/?token=abc">`.+ return url+ if "a" in parent_tags:+ # Markdown has no nested links, so hand the label up for the+ # enclosing <a> to use as its text, with brackets stripped.+ return re.sub(r"\[|\]", "", label)+ return get_markdown_link_for_url(label, url)++ def convert_title(self, el: Tag, text: str, parent_tags: set[str]) -> str:+ return ""def convert_html_to_markdown(html: str) -> str:- # html2text is GPL licensed, so run it as a subprocess.- markdown = subprocess.check_output(- [os.path.join(sys.prefix, "bin", "html2text"), "--unicode-snob"], input=html, text=True- ).strip()-- # We want images to get linked and inline previewed, but html2text will turn- # them into links of the form ``, which is- # ugly. Run a regex over the resulting description, turning links of the- # form `` into- # `[image.png](http://foo.com/image.png)`.- return re.sub(r"!\[\]\((\S*)/(\S*)\?(\S*)\)", "[\\2](\\1/\\2)", markdown)+ # A bare URL is expected input here (the text/html part of an incoming+ # email), but BeautifulSoup warns about it ("looks like a URL, not markup").+ # Suppress the warning so that expected input doesn't spam our logs.+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", MarkupResemblesLocatorWarning)+ converter = ZulipMarkdownConverter(+ # Zulip's Markdown supports only ATX headings, not Setext headings.+ heading_style="ATX",+ # Zulip's Markdown has no backslash escaping; it renders "\" as a+ # literal character.+ escape_asterisks=False,+ escape_underscores=False,+ # HTML treats a newline as insignificant whitespace, but Zulip+ # renders it as a line break. Collapse source newlines to spaces.+ # <br>, <pre>, and block-level elements still produce line breaks.+ wrap=True,+ # An effectively infinite width disables column wrapping. Don't+ # use None, which also disables wrapping but skips the pass that+ # strips indentation from lines following a <br>.+ wrap_width=sys.maxsize,+ )+ markdown: str = converter.convert(html).strip()+ return markdowndiff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.pyindex 3414342ec44..a0723000f3c 100644--- a/zerver/tests/test_markdown.py+++ b/zerver/tests/test_markdown.py@@ -1,5 +1,6 @@import osimport re+import warningsfrom dataclasses import dataclassfrom html import escapefrom textwrap import dedent@@ -9,7 +10,7 @@ from unittest import mockimport orjsonimport requestsimport responses-from bs4 import BeautifulSoup+from bs4 import BeautifulSoup, MarkupResemblesLocatorWarningfrom django.conf import settingsfrom django.test import override_settingsfrom markdown import Markdown@@ -3958,42 +3959,69 @@ class TestHtmlToMarkdown(ZulipTestCase):def test_unordered_lists(self) -> None:html = "<ul><li>foo</li><li>bar</li></ul>"- self.assertEqual(convert_html_to_markdown(html), "* foo\n * bar")+ self.assertEqual(convert_html_to_markdown(html), "* foo\n* bar")++ def test_title_is_stripped(self) -> None:+ html = "<html><head><title>Some title</title></head><body><p>Hello</p></body></html>"+ self.assertEqual(convert_html_to_markdown(html), "Hello")def test_atx_style_headings(self) -> None:self.assertEqual(convert_html_to_markdown("<h1>Title</h1>"), "# Title")self.assertEqual(convert_html_to_markdown("<h2>Sub</h2>"), "## Sub")def test_external_img_with_alt(self) -> None:+ # Zulip doesn't inline-render external images, so an external <img>+ # must become a [label](url) link to render at all.html = '<img src="http://example.com/img.png" alt="photo">'- self.assertEqual(convert_html_to_markdown(html), "")+ self.assertEqual(convert_html_to_markdown(html), "[photo](http://example.com/img.png)")def test_external_img_with_query_string(self) -> None:+ # The query string is stripped from the filename label but kept in+ # the URL, where it can be important (e.g. signed URLs).html = '<img src="http://foo.com/image.png?12345">'- self.assertEqual(convert_html_to_markdown(html), "[image.png](http://foo.com/image.png)")+ self.assertEqual(+ convert_html_to_markdown(html), "[image.png](http://foo.com/image.png?12345)"+ )def test_external_img_without_alt_or_filename(self) -> None:html = '<img src="https://example.com/?token=abc">'- self.assertEqual(convert_html_to_markdown(html), "[](https://example.com/)")+ self.assertEqual(convert_html_to_markdown(html), "https://example.com/?token=abc")def test_external_img_alt_with_brackets(self) -> None:+ # Brackets in link text break Zulip's escaping-free Markdown.html = '<img src="http://x.com/a.png" alt="see [details]">'- self.assertEqual(- convert_html_to_markdown(html), "![see \\[details\\]](http://x.com/a.png)"- )+ self.assertEqual(convert_html_to_markdown(html), "[see details](http://x.com/a.png)")++ def test_img_without_src(self) -> None:+ self.assertEqual(convert_html_to_markdown('<img alt="logo">'), "logo")def test_data_uri_img(self) -> None:+ # Inline base64 data: images (e.g. email/canvas placeholders) aren't+ # linkable and Zulip can't render them; emit the alt text, or nothing,+ # rather than a bogus link or a dumped base64 blob.data_uri = "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=="- self.assertEqual(- convert_html_to_markdown(f'<img src="{data_uri}" alt="logo">'), f""- )- self.assertEqual(convert_html_to_markdown(f'<img src="{data_uri}">'), f"")+ self.assertEqual(convert_html_to_markdown(f'<img src="{data_uri}" alt="logo">'), "logo")+ self.assertEqual(convert_html_to_markdown(f'<img src="{data_uri}">'), "")def test_linked_external_img(self) -> None:+ # Inside an <a>, only the image's label is emitted, so that the+ # anchor doesn't end up with a nested link inside it.html = '<a href="https://example.com"><img src="https://foo.com/a.png" alt="logo"></a>'+ self.assertEqual(convert_html_to_markdown(html), "[logo](https://example.com)")++ def test_linked_external_img_without_alt_or_filename(self) -> None:+ # With no alt text or filename to fall back on, the image's bare+ # src URL is emitted as the anchor's label.+ html = '<a href="https://example.com"><img src="https://foo.com/?token=abc"></a>'self.assertEqual(- convert_html_to_markdown(html), "[](https://example.com)"+ convert_html_to_markdown(html), "[https://foo.com/?token=abc](https://example.com)"+ )++ def test_linked_external_img_alt_with_brackets(self) -> None:+ html = (+ '<a href="https://example.com"><img src="http://x.com/a.png" alt="see [details]"></a>')+ self.assertEqual(convert_html_to_markdown(html), "[see details](https://example.com)")def test_anchor_tag(self) -> None:html = '<a href="http://example.com">click here</a>'@@ -4001,7 +4029,7 @@ class TestHtmlToMarkdown(ZulipTestCase):def test_bold_and_italic(self) -> None:self.assertEqual(convert_html_to_markdown("<strong>bold</strong>"), "**bold**")- self.assertEqual(convert_html_to_markdown("<em>italic</em>"), "_italic_")+ self.assertEqual(convert_html_to_markdown("<em>italic</em>"), "*italic*")def test_special_characters_are_not_escaped(self) -> None:html = "<p>snake_case_var and 2*3 stars</p>"@@ -4009,17 +4037,39 @@ class TestHtmlToMarkdown(ZulipTestCase):def test_bare_url_content(self) -> None:# Message content (e.g. from the email mirror) is often a bare URL.+ # convert_html_to_markdown suppresses BeautifulSoup's warning about+ # such input; promote that warning to an error here so this test fails+ # if the suppression is ever removed. The warning would also be fatal+ # under the PYTHONWARNINGS=error policy CI runs with.url = "https://www.youtube.com/watch?v=MRmGDhlMhNA"- self.assertEqual(convert_html_to_markdown(url), url)+ with warnings.catch_warnings():+ warnings.simplefilter("error", MarkupResemblesLocatorWarning)+ self.assertEqual(convert_html_to_markdown(url), url)- def test_paragraph_wrapping(self) -> None:+ def test_no_unnecessary_paragraph_wrapping(self) -> None:sentence = ("The quick brown fox jumps over the lazy dog"" again and again until the sentence is long. ")self.assertEqual(- convert_html_to_markdown("<p>" + sentence * 2 + "</p>"),- "The quick brown fox jumps over the lazy dog again and again until the sentence\n"- "is long. The quick brown fox jumps over the lazy dog again and again until the\n"- "sentence is long.",+ convert_html_to_markdown("<p>" + sentence * 2 + "</p>"), (sentence * 2).strip()+ )++ def test_new_line_in_source_html(self) -> None:+ html = (+ "<p>Thanks for signing up for Acme. To get started, click the button\n"+ "below and confirm your email address.</p>"+ )+ self.assertEqual(+ convert_html_to_markdown(html),+ "Thanks for signing up for Acme. To get started, click the button below and"+ " confirm your email address.",+ )++ def test_explicit_line_break_is_preserved(self) -> None:+ self.assertEqual(+ convert_html_to_markdown("<p>line one<br>line two</p>"), "line one \nline two"+ )+ self.assertEqual(+ convert_html_to_markdown("<p>line one<br>\nline two</p>"), "line one \nline two")diff --git a/zerver/webhooks/freshdesk/tests.py b/zerver/webhooks/freshdesk/tests.pyindex 4e5702dff94..e0dd88b67e6 100644--- a/zerver/webhooks/freshdesk/tests.py+++ b/zerver/webhooks/freshdesk/tests.py@@ -117,7 +117,7 @@ Requester Bob <[email protected]> added a {} note to \"""expected_topic_name = "#12: Not enough ☃ guinea pigs"expected_message = """-Requester \u2603 Bob <[email protected]> created [ticket #12](http://test1234zzz.freshdesk.com/helpdesk/tickets/12):\n\n``` quote\nThere are too many cat pictures on the internet \u2603. We need more guinea pigs.\nExhibit 1:\n\n \n\n[guinea_pig.png](http://cdn.freshdesk.com/data/helpdesk/attachments/production/12744808/original/guinea_pig.png)\n```\n\n* **Type**: Problem\n* **Priority**: Urgent\n* **Status**: Open+Requester \u2603 Bob <[email protected]> created [ticket #12](http://test1234zzz.freshdesk.com/helpdesk/tickets/12):\n\n``` quote\nThere are too many cat pictures on the internet \u2603. We need more guinea pigs. Exhibit 1:\n\n[guinea_pig.png](http://cdn.freshdesk.com/data/helpdesk/attachments/production/12744808/original/guinea_pig.png?1383958880)\n```\n\n* **Type**: Problem\n* **Priority**: Urgent\n* **Status**: Open""".strip()self.api_channel_message(self.test_user,