Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

lib: Switch convert_html_to_markdown to use markdownify.

Dhruv Shetty7aebdf4c4616
diff --git a/pyproject.toml b/pyproject.toml
index 20f7ba56f4d..71e864881d4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,6 +52,7 @@ prod = [
# Needed for the email mirror
"html2text",
+ "markdownify",
"talon-core",
# Needed for inlining the CSS in emails
diff --git a/uv.lock b/uv.lock
index ab37d3fcb86..ad687f7c10d 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2586,6 +2586,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" },
]
+[[package]]
+name = "markdownify"
+version = "1.2.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "beautifulsoup4" },
+ { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" },
+]
+
[[package]]
name = "markupsafe"
version = "3.0.3"
@@ -6364,6 +6377,7 @@ dev = [
{ name = "lxml-stubs" },
{ name = "markdown" },
{ name = "markdown-it-py" },
+ { name = "markdownify" },
{ name = "moto", extra = ["s3"] },
{ name = "mypy", extra = ["faster-cache"] },
{ name = "mypy-boto3-s3" },
@@ -6498,6 +6512,7 @@ prod = [
{ name = "jsx-lexer" },
{ name = "lxml" },
{ name = "markdown" },
+ { name = "markdownify" },
{ name = "mypy-boto3-s3" },
{ name = "mypy-boto3-ses" },
{ name = "mypy-boto3-sns" },
@@ -6603,6 +6618,7 @@ dev = [
{ name = "lxml-stubs" },
{ name = "markdown" },
{ name = "markdown-it-py", specifier = "<4" },
+ { name = "markdownify" },
{ name = "moto", extras = ["s3"] },
{ name = "mypy", extras = ["faster-cache"] },
{ name = "mypy-boto3-s3" },
@@ -6738,6 +6754,7 @@ prod = [
{ name = "jsx-lexer" },
{ name = "lxml" },
{ name = "markdown" },
+ { name = "markdownify" },
{ name = "mypy-boto3-s3" },
{ name = "mypy-boto3-ses" },
{ name = "mypy-boto3-sns" },
diff --git a/version.py b/version.py
index 8c1545efe8f..9361dc8d3af 100644
--- a/version.py
+++ b/version.py
@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 509
# historical commits sharing the same major version, in which case a
# minor version bump suffices.
-PROVISION_VERSION = (386, 0) # bumped 2026-07-28 to upgrade JavaScript dependencies
+PROVISION_VERSION = (386, 1) # bumped 2026-08-13 to add markdownify dependency
diff --git a/zerver/lib/markdown/from_html.py b/zerver/lib/markdown/from_html.py
index 6a505c7650d..6f71b73d593 100644
--- a/zerver/lib/markdown/from_html.py
+++ b/zerver/lib/markdown/from_html.py
@@ -1,18 +1,72 @@
-import os
+import posixpath
import re
-import subprocess
import sys
+import warnings
+from urllib.parse import urlsplit
+
+import markdownify
+from bs4 import MarkupResemblesLocatorWarning, Tag
+
+from zerver.lib.markdown import get_markdown_link_for_url, sanitize_url
+
+
+class ZulipMarkdownConverter(markdownify.MarkdownConverter):
+ """HTML-to-Markdown converter adapted to Zulip's Markdown.
+
+ Zulip doesn't inline-render external images, so images become
+ `[label](src)` links instead of broken `![alt](src)`.
+ """
+
+ def convert_img(self, el: Tag, text: str, parent_tags: set[str]) -> str:
+ src = el.get("src", "")
+ alt = el.get("alt", "")
+ # BeautifulSoup returns a list only for multi-valued attributes
+ # (e.g. class), never for src or alt.
+ assert isinstance(src, str)
+ assert isinstance(alt, str)
+ if not src:
+ return alt
+ url = sanitize_url(src)
+ if not url:
+ # Unlinkable src (e.g. data: URIs, unsafe schemes): emit just the
+ # alt text rather than a bogus link.
+ return alt
+ label = alt or posixpath.basename(urlsplit(url).path)
+ if not label:
+ # No alt text, and no filename in the path to fall back on, as in
+ # `<img src="https://example.com/?token=abc">`.
+ return url
+ if "a" in parent_tags:
+ # Markdown has no nested links, so hand the label up for the
+ # enclosing <a> to use as its text, with brackets stripped.
+ return re.sub(r"\[|\]", "", label)
+ return get_markdown_link_for_url(label, url)
+
+ def convert_title(self, el: Tag, text: str, parent_tags: set[str]) -> str:
+ return ""
def convert_html_to_markdown(html: str) -> str:
- # html2text is GPL licensed, so run it as a subprocess.
- markdown = subprocess.check_output(
- [os.path.join(sys.prefix, "bin", "html2text"), "--unicode-snob"], input=html, text=True
- ).strip()
-
- # We want images to get linked and inline previewed, but html2text will turn
- # them into links of the form `![](http://foo.com/image.png)`, which is
- # ugly. Run a regex over the resulting description, turning links of the
- # form `![](http://foo.com/image.png?12345)` into
- # `[image.png](http://foo.com/image.png)`.
- return re.sub(r"!\[\]\((\S*)/(\S*)\?(\S*)\)", "[\\2](\\1/\\2)", markdown)
+ # A bare URL is expected input here (the text/html part of an incoming
+ # email), but BeautifulSoup warns about it ("looks like a URL, not markup").
+ # Suppress the warning so that expected input doesn't spam our logs.
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", MarkupResemblesLocatorWarning)
+ converter = ZulipMarkdownConverter(
+ # Zulip's Markdown supports only ATX headings, not Setext headings.
+ heading_style="ATX",
+ # Zulip's Markdown has no backslash escaping; it renders "\" as a
+ # literal character.
+ escape_asterisks=False,
+ escape_underscores=False,
+ # HTML treats a newline as insignificant whitespace, but Zulip
+ # renders it as a line break. Collapse source newlines to spaces.
+ # <br>, <pre>, and block-level elements still produce line breaks.
+ wrap=True,
+ # An effectively infinite width disables column wrapping. Don't
+ # use None, which also disables wrapping but skips the pass that
+ # strips indentation from lines following a <br>.
+ wrap_width=sys.maxsize,
+ )
+ markdown: str = converter.convert(html).strip()
+ return markdown
diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py
index 3414342ec44..a0723000f3c 100644
--- a/zerver/tests/test_markdown.py
+++ b/zerver/tests/test_markdown.py
@@ -1,5 +1,6 @@
import os
import re
+import warnings
from dataclasses import dataclass
from html import escape
from textwrap import dedent
@@ -9,7 +10,7 @@ from unittest import mock
import orjson
import requests
import responses
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
from django.conf import settings
from django.test import override_settings
from markdown import Markdown
@@ -3958,42 +3959,69 @@ class TestHtmlToMarkdown(ZulipTestCase):
def test_unordered_lists(self) -> None:
html = "<ul><li>foo</li><li>bar</li></ul>"
- self.assertEqual(convert_html_to_markdown(html), "* foo\n * bar")
+ self.assertEqual(convert_html_to_markdown(html), "* foo\n* bar")
+
+ def test_title_is_stripped(self) -> None:
+ html = "<html><head><title>Some title</title></head><body><p>Hello</p></body></html>"
+ self.assertEqual(convert_html_to_markdown(html), "Hello")
def test_atx_style_headings(self) -> None:
self.assertEqual(convert_html_to_markdown("<h1>Title</h1>"), "# Title")
self.assertEqual(convert_html_to_markdown("<h2>Sub</h2>"), "## Sub")
def test_external_img_with_alt(self) -> None:
+ # Zulip doesn't inline-render external images, so an external <img>
+ # must become a [label](url) link to render at all.
html = '<img src="http://example.com/img.png" alt="photo">'
- self.assertEqual(convert_html_to_markdown(html), "![photo](http://example.com/img.png)")
+ self.assertEqual(convert_html_to_markdown(html), "[photo](http://example.com/img.png)")
def test_external_img_with_query_string(self) -> None:
+ # The query string is stripped from the filename label but kept in
+ # the URL, where it can be important (e.g. signed URLs).
html = '<img src="http://foo.com/image.png?12345">'
- self.assertEqual(convert_html_to_markdown(html), "[image.png](http://foo.com/image.png)")
+ self.assertEqual(
+ convert_html_to_markdown(html), "[image.png](http://foo.com/image.png?12345)"
+ )
def test_external_img_without_alt_or_filename(self) -> None:
html = '<img src="https://example.com/?token=abc">'
- self.assertEqual(convert_html_to_markdown(html), "[](https://example.com/)")
+ self.assertEqual(convert_html_to_markdown(html), "https://example.com/?token=abc")
def test_external_img_alt_with_brackets(self) -> None:
+ # Brackets in link text break Zulip's escaping-free Markdown.
html = '<img src="http://x.com/a.png" alt="see [details]">'
- self.assertEqual(
- convert_html_to_markdown(html), "![see \\[details\\]](http://x.com/a.png)"
- )
+ self.assertEqual(convert_html_to_markdown(html), "[see details](http://x.com/a.png)")
+
+ def test_img_without_src(self) -> None:
+ self.assertEqual(convert_html_to_markdown('<img alt="logo">'), "logo")
def test_data_uri_img(self) -> None:
+ # Inline base64 data: images (e.g. email/canvas placeholders) aren't
+ # linkable and Zulip can't render them; emit the alt text, or nothing,
+ # rather than a bogus link or a dumped base64 blob.
data_uri = "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=="
- self.assertEqual(
- convert_html_to_markdown(f'<img src="{data_uri}" alt="logo">'), f"![logo]({data_uri})"
- )
- self.assertEqual(convert_html_to_markdown(f'<img src="{data_uri}">'), f"![]({data_uri})")
+ self.assertEqual(convert_html_to_markdown(f'<img src="{data_uri}" alt="logo">'), "logo")
+ self.assertEqual(convert_html_to_markdown(f'<img src="{data_uri}">'), "")
def test_linked_external_img(self) -> None:
+ # Inside an <a>, only the image's label is emitted, so that the
+ # anchor doesn't end up with a nested link inside it.
html = '<a href="https://example.com"><img src="https://foo.com/a.png" alt="logo"></a>'
+ self.assertEqual(convert_html_to_markdown(html), "[logo](https://example.com)")
+
+ def test_linked_external_img_without_alt_or_filename(self) -> None:
+ # With no alt text or filename to fall back on, the image's bare
+ # src URL is emitted as the anchor's label.
+ html = '<a href="https://example.com"><img src="https://foo.com/?token=abc"></a>'
self.assertEqual(
- convert_html_to_markdown(html), "[![logo](https://foo.com/a.png)](https://example.com)"
+ convert_html_to_markdown(html), "[https://foo.com/?token=abc](https://example.com)"
+ )
+
+ def test_linked_external_img_alt_with_brackets(self) -> None:
+ html = (
+ '<a href="https://example.com"><img src="http://x.com/a.png" alt="see [details]"></a>'
)
+ self.assertEqual(convert_html_to_markdown(html), "[see details](https://example.com)")
def test_anchor_tag(self) -> None:
html = '<a href="http://example.com">click here</a>'
@@ -4001,7 +4029,7 @@ class TestHtmlToMarkdown(ZulipTestCase):
def test_bold_and_italic(self) -> None:
self.assertEqual(convert_html_to_markdown("<strong>bold</strong>"), "**bold**")
- self.assertEqual(convert_html_to_markdown("<em>italic</em>"), "_italic_")
+ self.assertEqual(convert_html_to_markdown("<em>italic</em>"), "*italic*")
def test_special_characters_are_not_escaped(self) -> None:
html = "<p>snake_case_var and 2*3 stars</p>"
@@ -4009,17 +4037,39 @@ class TestHtmlToMarkdown(ZulipTestCase):
def test_bare_url_content(self) -> None:
# Message content (e.g. from the email mirror) is often a bare URL.
+ # convert_html_to_markdown suppresses BeautifulSoup's warning about
+ # such input; promote that warning to an error here so this test fails
+ # if the suppression is ever removed. The warning would also be fatal
+ # under the PYTHONWARNINGS=error policy CI runs with.
url = "https://www.youtube.com/watch?v=MRmGDhlMhNA"
- self.assertEqual(convert_html_to_markdown(url), url)
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", MarkupResemblesLocatorWarning)
+ self.assertEqual(convert_html_to_markdown(url), url)
- def test_paragraph_wrapping(self) -> None:
+ def test_no_unnecessary_paragraph_wrapping(self) -> None:
sentence = (
"The quick brown fox jumps over the lazy dog"
" again and again until the sentence is long. "
)
self.assertEqual(
- convert_html_to_markdown("<p>" + sentence * 2 + "</p>"),
- "The quick brown fox jumps over the lazy dog again and again until the sentence\n"
- "is long. The quick brown fox jumps over the lazy dog again and again until the\n"
- "sentence is long.",
+ convert_html_to_markdown("<p>" + sentence * 2 + "</p>"), (sentence * 2).strip()
+ )
+
+ def test_new_line_in_source_html(self) -> None:
+ html = (
+ "<p>Thanks for signing up for Acme. To get started, click the button\n"
+ "below and confirm your email address.</p>"
+ )
+ self.assertEqual(
+ convert_html_to_markdown(html),
+ "Thanks for signing up for Acme. To get started, click the button below and"
+ " confirm your email address.",
+ )
+
+ def test_explicit_line_break_is_preserved(self) -> None:
+ self.assertEqual(
+ convert_html_to_markdown("<p>line one<br>line two</p>"), "line one \nline two"
+ )
+ self.assertEqual(
+ convert_html_to_markdown("<p>line one<br>\nline two</p>"), "line one \nline two"
)
diff --git a/zerver/webhooks/freshdesk/tests.py b/zerver/webhooks/freshdesk/tests.py
index 4e5702dff94..e0dd88b67e6 100644
--- a/zerver/webhooks/freshdesk/tests.py
+++ b/zerver/webhooks/freshdesk/tests.py
@@ -117,7 +117,7 @@ Requester Bob <[email protected]> added a {} note to \
"""
expected_topic_name = "#12: Not enough ☃ guinea pigs"
expected_message = """
-Requester \u2603 Bob <[email protected]> created [ticket #12](http://test1234zzz.freshdesk.com/helpdesk/tickets/12):\n\n``` quote\nThere are too many cat pictures on the internet \u2603. We need more guinea pigs.\nExhibit 1:\n\n \n\n[guinea_pig.png](http://cdn.freshdesk.com/data/helpdesk/attachments/production/12744808/original/guinea_pig.png)\n```\n\n* **Type**: Problem\n* **Priority**: Urgent\n* **Status**: Open
+Requester \u2603 Bob <[email protected]> created [ticket #12](http://test1234zzz.freshdesk.com/helpdesk/tickets/12):\n\n``` quote\nThere are too many cat pictures on the internet \u2603. We need more guinea pigs. Exhibit 1:\n\n[guinea_pig.png](http://cdn.freshdesk.com/data/helpdesk/attachments/production/12744808/original/guinea_pig.png?1383958880)\n```\n\n* **Type**: Problem\n* **Priority**: Urgent\n* **Status**: Open
""".strip()
self.api_channel_message(
self.test_user,