Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
microsoft_teams: Convert message HTML with convert_html_to_markdown.
Dhruv Shetty8a795d205729
diff --git a/zerver/data_import/import_util.py b/zerver/data_import/import_util.pyindex 2ef94fbf7cb..4b61d69c5b0 100644--- a/zerver/data_import/import_util.py+++ b/zerver/data_import/import_util.py@@ -3,7 +3,6 @@ import osimport randomimport secretsimport shutil-import subprocessfrom collections import defaultdictfrom collections.abc import Callable, Hashable, Iterable, Iterator, Mappingfrom collections.abc import Set as AbstractSet@@ -931,11 +930,6 @@ def validate_user_emails_for_import(user_emails: list[str]) -> None:raise ValidationError(error_log)-def convert_html_to_text(content: str) -> str:- # html2text is GPL licensed, so run it as a subprocess.- return subprocess.check_output(["html2text", "--unicode-snob"], input=content, text=True)--def get_data_file(path: str) -> Any:with open(path, "rb") as fp:data = orjson.loads(fp.read())diff --git a/zerver/data_import/microsoft_teams.py b/zerver/data_import/microsoft_teams.pyindex 722c575dfc9..829a940d66a 100644--- a/zerver/data_import/microsoft_teams.py+++ b/zerver/data_import/microsoft_teams.py@@ -29,7 +29,6 @@ from zerver.data_import.import_util import (build_user_profile,build_usermessages,build_zerver_realm,- convert_html_to_text,create_converted_data_files,get_attachment_path_and_content,get_data_file,@@ -39,6 +38,10 @@ from zerver.data_import.import_util import (validate_user_emails_for_import,write_response_file_stream_to_path,)+from zerver.data_import.microsoft_teams_message_conversion import (+ HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX,+ convert_microsoft_teams_html_to_markdown,+)from zerver.data_import.sequencer import NEXT_IDfrom zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE, do_common_export_processesfrom zerver.lib.markdown import get_markdown_image_for_url@@ -211,16 +214,6 @@ class ODataQueryParameter:MICROSOFT_GRAPH_API_URL = "https://graph.microsoft.com/v1.0{endpoint}"-# https://learn.microsoft.com/en-us/graph/api/chatmessagehostedcontent-get-HOSTED_CONTENT_GRAPH_API_URL_REGEX = r"https://graph\.microsoft\.com/v1\.0/teams/[^/]+/channels/[^/]+/messages/[^/]+/hostedContents/[^/]+/\$value"-HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX = rf"""- !\[- (?P<file_name>[^\]]+)- \]\(- (?P<api_url>{HOSTED_CONTENT_GRAPH_API_URL_REGEX})- \)- """-def get_microsoft_graph_api_data(api_url: str,@@ -507,8 +500,8 @@ def process_hosted_content_attachments(* code snippetsSince custom emoji and code snippet are formatted as custom HTML blocks,- convert_html_to_text will turn them into empty strings, so this currently- only supports processing image hosted contents.+ convert_microsoft_teams_html_to_markdown will turn them into empty strings,+ so this currently only supports processing image hosted contents.It's worth checking the API documentation for the hosted content once in awhile to verify what file types "hosted content" includes, since the code path@@ -592,7 +585,7 @@ def process_messages(message_content_type = message["Body"]["ContentType"]if message_content_type == "html":try:- content = convert_html_to_text(message["Body"]["Content"])+ content = convert_microsoft_teams_html_to_markdown(message["Body"]["Content"])except Exception: # nocoveragelogging.warning("Error converting HTML to text for message: '%s'; continuing", contentdiff --git a/zerver/data_import/microsoft_teams_message_conversion.py b/zerver/data_import/microsoft_teams_message_conversion.pynew file mode 100644index 00000000000..395a24db58e--- /dev/null+++ b/zerver/data_import/microsoft_teams_message_conversion.py@@ -0,0 +1,41 @@+import re++from bs4 import Tag+from typing_extensions import override++from zerver.lib.markdown import get_markdown_image_for_url+from zerver.lib.markdown.from_html import ZulipMarkdownConverter, convert_html_to_markdown++# https://learn.microsoft.com/en-us/graph/api/chatmessagehostedcontent-get+HOSTED_CONTENT_GRAPH_API_URL_REGEX = r"https://graph\.microsoft\.com/v1\.0/teams/[^/]+/channels/[^/]+/messages/[^/]+/hostedContents/[^/]+/\$value"+HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX = rf"""+ !\[+ (?P<file_name>[^\]]+)+ \]\(+ (?P<api_url>{HOSTED_CONTENT_GRAPH_API_URL_REGEX})+ \)+ """+++class MicrosoftTeamsToZulipMarkdownConverter(ZulipMarkdownConverter):+ """HTML-to-Markdown converter for Microsoft Teams exports.++ Hosted-content images are kept in inline `` form, which+ process_hosted_content_attachments matches to download each image+ and rewrite it into a Zulip upload link. Other images get the base+ class's external-link treatment.+ """++ @override+ def convert_img(self, el: Tag, text: str, parent_tags: set[str]) -> str:+ src = el.get("src", "")+ alt = el.get("alt", "")+ assert isinstance(src, str)+ assert isinstance(alt, str)+ if re.fullmatch(HOSTED_CONTENT_GRAPH_API_URL_REGEX, src):+ return get_markdown_image_for_url(alt, src)+ return super().convert_img(el, text, parent_tags)+++def convert_microsoft_teams_html_to_markdown(html: str) -> str:+ return convert_html_to_markdown(html, converter_class=MicrosoftTeamsToZulipMarkdownConverter)diff --git a/zerver/lib/markdown/from_html.py b/zerver/lib/markdown/from_html.pyindex 6f71b73d593..febd46514de 100644--- a/zerver/lib/markdown/from_html.py+++ b/zerver/lib/markdown/from_html.py@@ -46,13 +46,15 @@ class ZulipMarkdownConverter(markdownify.MarkdownConverter):return ""-def convert_html_to_markdown(html: str) -> str:+def convert_html_to_markdown(+ html: str, *, converter_class: type[ZulipMarkdownConverter] = ZulipMarkdownConverter+) -> str:# A bare URL is expected input here (the text/html part of an incoming# email), but BeautifulSoup warns about it ("looks like a URL, not markup").# Suppress the warning so that expected input doesn't spam our logs.with warnings.catch_warnings():warnings.simplefilter("ignore", MarkupResemblesLocatorWarning)- converter = ZulipMarkdownConverter(+ converter = converter_class(# Zulip's Markdown supports only ATX headings, not Setext headings.heading_style="ATX",# Zulip's Markdown has no backslash escaping; it renders "\" as adiff --git a/zerver/tests/test_microsoft_teams_importer.py b/zerver/tests/test_microsoft_teams_importer.pyindex 289ca27e756..c169372c4d3 100644--- a/zerver/tests/test_microsoft_teams_importer.py+++ b/zerver/tests/test_microsoft_teams_importer.py@@ -17,9 +17,8 @@ from django.utils.timezone import now as timezone_nowfrom requests import PreparedRequestfrom typing_extensions import ParamSpec-from zerver.data_import.import_util import AttachmentRecordData, convert_html_to_text, get_data_file+from zerver.data_import.import_util import AttachmentRecordData, get_data_filefrom zerver.data_import.microsoft_teams import (- HOSTED_CONTENT_GRAPH_API_URL_REGEX,MICROSOFT_GRAPH_API_URL,ChannelMetadata,MicrosoftTeamsFieldsT,@@ -37,6 +36,10 @@ from zerver.data_import.microsoft_teams import (is_microsoft_teams_event_message,process_hosted_content_attachments,)+from zerver.data_import.microsoft_teams_message_conversion import (+ HOSTED_CONTENT_GRAPH_API_URL_REGEX,+ convert_microsoft_teams_html_to_markdown,+)from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZEfrom zerver.lib.import_realm import do_import_realmfrom zerver.lib.partial import partial@@ -620,6 +623,36 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):str(e.exception),)+ def test_convert_microsoft_teams_html_to_markdown_list_conversion(self) -> None:+ """+ convert_html_to_text used to mangle plain-text bulleted lists by+ escaping the leading dashes and collapsing the line breaks;+ convert_microsoft_teams_html_to_markdown leaves them intact.+ See #39351.+ """+ html = "<p>- bullet 1<br>- bullet 2<br>- bullet 3<br>- bullet 4</p>"+ content = convert_microsoft_teams_html_to_markdown(html)++ self.assertEqual(content, "- bullet 1 \n- bullet 2 \n- bullet 3 \n- bullet 4")++ def test_convert_microsoft_teams_html_to_markdown_image_conversion(self) -> None:+ hosted_content_url = "https://graph.microsoft.com/v1.0/teams/1d513e46-d8cd-41db-b84f-381fe5730794/channels/19:[email protected]/messages/1755002994809/hostedContents/aWQ9eF8wLXd1cy1kNi0zZDZkYmNiODA0OGFhODhmMWMxY2Q1N2ZhYWIzYzRhZA==/$value"+ self.assertEqual(+ convert_microsoft_teams_html_to_markdown(+ f'<img src="{hosted_content_url}" alt="image">'+ ),+ f"",+ )++ external_url = (+ "https://raw.githubusercontent.com/zulip/zulip/main/static/images/logo/"+ "zulip-icon-128x128.png"+ )+ self.assertEqual(+ convert_microsoft_teams_html_to_markdown(f'<img src="{external_url}" alt="logo">'),+ f"[logo]({external_url})",+ )+@responses.activatedef test_failed_get_microsoft_graph_api_data(self) -> None:responses.add(@@ -759,7 +792,7 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):# This is actual decoded content from a message in messages_1d513e46-d8cd-41db-b84f-381fe5730794.json# The original message was just some text followed by an image.MessageFixture(- content=convert_html_to_text(+ content=convert_microsoft_teams_html_to_markdown("""<ul><li>dashes are ok</li>@@ -776,7 +809,7 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):alt="image"itemid="0-wus-d6-3d6dbcb8048aa88f1c1cd57faab3c4ad"></p>- """+ """,),hosted_content_count=1,is_direct_message_type=False,@@ -800,11 +833,12 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):test_name="multiple images",),# This is actual decoded content from a message in messages_002145f2-eaba-4962-997d-6d841a9f50af.json- # The original message was just two custom emojis, nothing else. convert_html_to_text doesn't know- # how to process MS Teams-specific tags such as <customemoji>, so this will be converted to empty- # string.+ # The original message was just two custom emojis, nothing else.+ # convert_microsoft_teams_html_to_markdown doesn't know how to process MS+ # Teams-specific tags such as <customemoji>, so this will be converted to+ # empty string.MessageFixture(- content=convert_html_to_text(+ content=convert_microsoft_teams_html_to_markdown("""<p><span>@@ -822,7 +856,7 @@ class MicrosoftTeamsImporterUnitTest(MicrosoftTeamsImportTestCase):</customemoji></span></p>- """+ """,),hosted_content_count=0,is_direct_message_type=False,