Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

microsoft_teams: Convert message HTML with convert_html_to_markdown.

Dhruv Shetty8a795d205729
diff --git a/zerver/data_import/import_util.py b/zerver/data_import/import_util.py
index 2ef94fbf7cb..4b61d69c5b0 100644
--- a/zerver/data_import/import_util.py
+++ b/zerver/data_import/import_util.py
@@ -3,7 +3,6 @@ import os
import random
import secrets
import shutil
-import subprocess
from collections import defaultdict
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping
from collections.abc import Set as AbstractSet
@@ -931,11 +930,6 @@ def validate_user_emails_for_import(user_emails: list[str]) -> None:
raise ValidationError(error_log)
-def convert_html_to_text(content: str) -> str:
- # html2text is GPL licensed, so run it as a subprocess.
- return subprocess.check_output(["html2text", "--unicode-snob"], input=content, text=True)
-
-
def get_data_file(path: str) -> Any:
with open(path, "rb") as fp:
data = orjson.loads(fp.read())
diff --git a/zerver/data_import/microsoft_teams.py b/zerver/data_import/microsoft_teams.py
index 722c575dfc9..829a940d66a 100644
--- a/zerver/data_import/microsoft_teams.py
+++ b/zerver/data_import/microsoft_teams.py
@@ -29,7 +29,6 @@ from zerver.data_import.import_util import (
build_user_profile,
build_usermessages,
build_zerver_realm,
- convert_html_to_text,
create_converted_data_files,
get_attachment_path_and_content,
get_data_file,
@@ -39,6 +38,10 @@ from zerver.data_import.import_util import (
validate_user_emails_for_import,
write_response_file_stream_to_path,
)
+from zerver.data_import.microsoft_teams_message_conversion import (
+ HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX,
+ convert_microsoft_teams_html_to_markdown,
+)
from zerver.data_import.sequencer import NEXT_ID
from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE, do_common_export_processes
from zerver.lib.markdown import get_markdown_image_for_url
@@ -211,16 +214,6 @@ class ODataQueryParameter:
MICROSOFT_GRAPH_API_URL = "https://graph.microsoft.com/v1.0{endpoint}"
-# https://learn.microsoft.com/en-us/graph/api/chatmessagehostedcontent-get
-HOSTED_CONTENT_GRAPH_API_URL_REGEX = r"https://graph\.microsoft\.com/v1\.0/teams/[^/]+/channels/[^/]+/messages/[^/]+/hostedContents/[^/]+/\$value"
-HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX = rf"""
- !\[
- (?P<file_name>[^\]]+)
- \]\(
- (?P<api_url>{HOSTED_CONTENT_GRAPH_API_URL_REGEX})
- \)
- """
-
def get_microsoft_graph_api_data(
api_url: str,
@@ -507,8 +500,8 @@ def process_hosted_content_attachments(
* code snippets
Since custom emoji and code snippet are formatted as custom HTML blocks,
- convert_html_to_text will turn them into empty strings, so this currently
- only supports processing image hosted contents.
+ convert_microsoft_teams_html_to_markdown will turn them into empty strings,
+ so this currently only supports processing image hosted contents.
It's worth checking the API documentation for the hosted content once in a
while to verify what file types "hosted content" includes, since the code path
@@ -592,7 +585,7 @@ def process_messages(
message_content_type = message["Body"]["ContentType"]
if message_content_type == "html":
try:
- content = convert_html_to_text(message["Body"]["Content"])
+ content = convert_microsoft_teams_html_to_markdown(message["Body"]["Content"])
except Exception: # nocoverage
logging.warning(
"Error converting HTML to text for message: '%s'; continuing", content
diff --git a/zerver/data_import/microsoft_teams_message_conversion.py b/zerver/data_import/microsoft_teams_message_conversion.py
new file mode 100644
index 00000000000..395a24db58e
--- /dev/null
+++ b/zerver/data_import/microsoft_teams_message_conversion.py
@@ -0,0 +1,41 @@
+import re
+
+from bs4 import Tag
+from typing_extensions import override
+
+from zerver.lib.markdown import get_markdown_image_for_url
+from zerver.lib.markdown.from_html import ZulipMarkdownConverter, convert_html_to_markdown
+
+# https://learn.microsoft.com/en-us/graph/api/chatmessagehostedcontent-get
+HOSTED_CONTENT_GRAPH_API_URL_REGEX = r"https://graph\.microsoft\.com/v1\.0/teams/[^/]+/channels/[^/]+/messages/[^/]+/hostedContents/[^/]+/\$value"
+HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX = rf"""
+ !\[
+ (?P<file_name>[^\]]+)
+ \]\(
+ (?P<api_url>{HOSTED_CONTENT_GRAPH_API_URL_REGEX})
+ \)
+ """
+
+
+class MicrosoftTeamsToZulipMarkdownConverter(ZulipMarkdownConverter):
+ """HTML-to-Markdown converter for Microsoft Teams exports.
+
+ Hosted-content images are kept in inline `![alt](src)` form, which
+ process_hosted_content_attachments matches to download each image
+ and rewrite it into a Zulip upload link. Other images get the base
+ class's external-link treatment.
+ """
+
+ @override
+ def convert_img(self, el: Tag, text: str, parent_tags: set[str]) -> str:
+ src = el.get("src", "")
+ alt = el.get("alt", "")
+ assert isinstance(src, str)
+ assert isinstance(alt, str)
+ if re.fullmatch(HOSTED_CONTENT_GRAPH_API_URL_REGEX, src):
+ return get_markdown_image_for_url(alt, src)
+ return super().convert_img(el, text, parent_tags)
+
+
+def convert_microsoft_teams_html_to_markdown(html: str) -> str:
+ return convert_html_to_markdown(html, converter_class=MicrosoftTeamsToZulipMarkdownConverter)
diff --git a/zerver/lib/markdown/from_html.py b/zerver/lib/markdown/from_html.py
index 6f71b73d593..febd46514de 100644
--- a/zerver/lib/markdown/from_html.py
+++ b/zerver/lib/markdown/from_html.py
@@ -46,13 +46,15 @@ class ZulipMarkdownConverter(markdownify.MarkdownConverter):
return ""
-def convert_html_to_markdown(html: str) -> str:
+def convert_html_to_markdown(
+ html: str, *, converter_class: type[ZulipMarkdownConverter] = ZulipMarkdownConverter
+) -> str:
# A bare URL is expected input here (the text/html part of an incoming
# email), but BeautifulSoup warns about it ("looks like a URL, not markup").
# Suppress the warning so that expected input doesn't spam our logs.
with warnings.catch_warnings():
warnings.simplefilter("ignore", MarkupResemblesLocatorWarning)
- converter = ZulipMarkdownConverter(
+ converter = converter_class(
# Zulip's Markdown supports only ATX headings, not Setext headings.
heading_style="ATX",
# Zulip's Markdown has no backslash escaping; it renders "\" as a
diff --git a/zerver/tests/test_microsoft_teams_importer.py b/zerver/tests/test_microsoft_teams_importer.py
index 289ca27e756..c169372c4d3 100644
--- a/zerver/tests/test_microsoft_teams_importer.py
+++ b/zerver/tests/test_microsoft_teams_importer.py
@@ -17,9 +17,8 @@ from django.utils.timezone import now as timezone_now
from requests import PreparedRequest
from typing_extensions import ParamSpec
-from zerver.data_import.import_util import AttachmentRecordData, convert_html_to_text, get_data_file
+from zerver.data_import.import_util import AttachmentRecordData, get_data_file
from zerver.data_import.microsoft_teams import (
- HOSTED_CONTENT_GRAPH_API_URL_REGEX,
MICROSOFT_GRAPH_API_URL,
ChannelMetadata,
MicrosoftTeamsFieldsT,
@@ -37,6 +36,10 @@ from zerver.data_import.microsoft_teams import (
is_microsoft_teams_event_message,
process_hosted_content_attachments,
)
+from zerver.data_import.microsoft_teams_message_conversion import (
+ HOSTED_CONTENT_GRAPH_API_URL_REGEX,
+ convert_microsoft_teams_html_to_markdown,
+)
from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE
from zerver.lib.import_realm import do_import_realm
from zerver.lib.partial import partial
@@ -620,6 +623,36 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):
str(e.exception),
)
+ def test_convert_microsoft_teams_html_to_markdown_list_conversion(self) -> None:
+ """
+ convert_html_to_text used to mangle plain-text bulleted lists by
+ escaping the leading dashes and collapsing the line breaks;
+ convert_microsoft_teams_html_to_markdown leaves them intact.
+ See #39351.
+ """
+ html = "<p>- bullet 1<br>- bullet 2<br>- bullet 3<br>- bullet 4</p>"
+ content = convert_microsoft_teams_html_to_markdown(html)
+
+ self.assertEqual(content, "- bullet 1 \n- bullet 2 \n- bullet 3 \n- bullet 4")
+
+ def test_convert_microsoft_teams_html_to_markdown_image_conversion(self) -> None:
+ hosted_content_url = "https://graph.microsoft.com/v1.0/teams/1d513e46-d8cd-41db-b84f-381fe5730794/channels/19:[email protected]/messages/1755002994809/hostedContents/aWQ9eF8wLXd1cy1kNi0zZDZkYmNiODA0OGFhODhmMWMxY2Q1N2ZhYWIzYzRhZA==/$value"
+ self.assertEqual(
+ convert_microsoft_teams_html_to_markdown(
+ f'<img src="{hosted_content_url}" alt="image">'
+ ),
+ f"![image]({hosted_content_url})",
+ )
+
+ external_url = (
+ "https://raw.githubusercontent.com/zulip/zulip/main/static/images/logo/"
+ "zulip-icon-128x128.png"
+ )
+ self.assertEqual(
+ convert_microsoft_teams_html_to_markdown(f'<img src="{external_url}" alt="logo">'),
+ f"[logo]({external_url})",
+ )
+
@responses.activate
def test_failed_get_microsoft_graph_api_data(self) -> None:
responses.add(
@@ -759,7 +792,7 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):
# This is actual decoded content from a message in messages_1d513e46-d8cd-41db-b84f-381fe5730794.json
# The original message was just some text followed by an image.
MessageFixture(
- content=convert_html_to_text(
+ content=convert_microsoft_teams_html_to_markdown(
"""
<ul>
<li>dashes are ok</li>
@@ -776,7 +809,7 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):
alt="image"
itemid="0-wus-d6-3d6dbcb8048aa88f1c1cd57faab3c4ad">
</p>
- """
+ """,
),
hosted_content_count=1,
is_direct_message_type=False,
@@ -800,11 +833,12 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):
test_name="multiple images",
),
# This is actual decoded content from a message in messages_002145f2-eaba-4962-997d-6d841a9f50af.json
- # The original message was just two custom emojis, nothing else. convert_html_to_text doesn't know
- # how to process MS Teams-specific tags such as <customemoji>, so this will be converted to empty
- # string.
+ # The original message was just two custom emojis, nothing else.
+ # convert_microsoft_teams_html_to_markdown doesn't know how to process MS
+ # Teams-specific tags such as <customemoji>, so this will be converted to
+ # empty string.
MessageFixture(
- content=convert_html_to_text(
+ content=convert_microsoft_teams_html_to_markdown(
"""
<p>
<span>
@@ -822,7 +856,7 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):
</customemoji>
</span>
</p>
- """
+ """,
),
hosted_content_count=0,
is_direct_message_type=False,