Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
message_send: Build message event dicts in bulk.
Aman Agrawal649aca709d2c
diff --git a/zerver/actions/message_send.py b/zerver/actions/message_send.pyindex 516fadc34cd..7cbd5d88b35 100644--- a/zerver/actions/message_send.py+++ b/zerver/actions/message_send.py@@ -1039,6 +1039,26 @@ def get_message_url_and_link(return url, link+def get_wide_message_dicts(+ send_message_requests: list[SendMessageRequest],+) -> dict[int, dict[str, Any]]:+ messages_by_realm_id: dict[int, list[Message]] = defaultdict(list)+ for send_request in send_message_requests:+ messages_by_realm_id[send_request.message.realm_id].append(send_request.message)++ recipient_dm_user_ids = {+ send_request.message.recipient_id: send_request.dm_involved_user_ids+ for send_request in send_message_requests+ if send_request.dm_involved_user_ids is not None+ }+ wide_message_dicts: dict[int, dict[str, Any]] = {}+ for realm_id, messages in messages_by_realm_id.items():+ wide_message_dicts.update(+ MessageDict.wide_dicts_for_new_messages(messages, realm_id, recipient_dm_user_ids)+ )+ return wide_message_dicts++@transaction.atomic(savepoint=False)def do_send_messages(send_message_requests_maybe_none: Sequence[SendMessageRequest | None],@@ -1136,6 +1156,8 @@ def do_send_messages(for send_request in send_message_requests:do_widget_post_save_actions(send_request)+ wide_message_dicts = get_wide_message_dicts(send_message_requests)+# This next loop is responsible for notifying other parts of the# Zulip system about the messages we just committed to the database:# * Sender automatically follows or unmutes the topic depending on 'automatically_follow_topics_policy'@@ -1230,11 +1252,7 @@ def do_send_messages(# Deliver events to the real-time push system, as well as# enqueuing any additional processing triggered by the message.- wide_message_dict = MessageDict.wide_dict(- send_request.message,- realm_id,- dm_involved_user_ids=send_request.dm_involved_user_ids,- )+ wide_message_dict = wide_message_dicts[send_request.message.id]send_request.message_url, send_request.message_link = get_message_url_and_link(send_request, wide_message_dictdiff --git a/zerver/lib/message_cache.py b/zerver/lib/message_cache.pyindex 65a74a2456d..aefaa43b8ca 100644--- a/zerver/lib/message_cache.py+++ b/zerver/lib/message_cache.py@@ -112,14 +112,15 @@ class MessageDict:"""MessageDict is the core class responsible for marshalling Messageobjects obtained from the database into a format that can be sentto clients via the Zulip API, whether via `GET /messages`,- outgoing webhooks, or other code paths. There are two core flows through+ outgoing webhooks, or other code paths. There are two core flows throughwhich this class is used:- * For just-sent messages, we construct a single `wide_dict` object- containing all the data for the message and the related- UserProfile models (sender_info and recipient_info); this object- can be stored in queues, caches, etc., and then later turned- into an API-format JSONable dictionary via finalize_payload.+ * For just-sent messages, wide_dicts_for_new_messages constructs a+ wide dict for each message, containing all the data for the+ message and the related UserProfile models (sender_info and+ recipient_info); these dicts can be stored in queues, caches,+ etc., and then later turned into an API-format JSONable+ dictionary via finalize_payload.* When fetching messages from the database, we fetch their data inbulk using messages_for_ids, which makes use of caching, bulk@@ -145,12 +146,6 @@ class MessageDict:encoded_object_bytes = message_to_encoded_cache(message, realm_id)obj = extract_message_dict(encoded_object_bytes)- """- The steps below are similar to what we do in- post_process_dicts(), except we don't call finalize_payload(),- since that step happens later in the queue- processor.- """MessageDict.bulk_hydrate_sender_info([obj])MessageDict.bulk_hydrate_recipient_info([obj],@@ -159,6 +154,25 @@ class MessageDict:return obj+ @staticmethod+ def wide_dicts_for_new_messages(+ messages: list[Message],+ realm_id: int,+ recipient_dm_user_ids: Mapping[int, set[int]],+ ) -> dict[int, dict[str, Any]]:+ encoded_messages = update_message_cache(messages, realm_id)+ objs = [+ extract_message_dict(encoded_message) for encoded_message in encoded_messages.values()+ ]++ # The steps below are similar to what we do in+ # post_process_dicts(), except we don't call finalize_payload(),+ # since that step happens later in the queue processor.+ MessageDict.bulk_hydrate_sender_info(objs)+ MessageDict.bulk_hydrate_recipient_info(objs, recipient_dm_user_ids)++ return {obj["id"]: obj for obj in objs}+@staticmethoddef post_process_dicts(objs: list[dict[str, Any]],diff --git a/zerver/tests/test_channel_fetch.py b/zerver/tests/test_channel_fetch.pyindex ae76e6a9f10..e2cfca9c56c 100644--- a/zerver/tests/test_channel_fetch.py+++ b/zerver/tests/test_channel_fetch.py@@ -882,7 +882,7 @@ class GetSubscribersTest(ZulipTestCase):polonius.id,]- with self.assert_database_query_count(67):+ with self.assert_database_query_count(59):self.subscribe_via_post(self.user_profile,stream_names,diff --git a/zerver/tests/test_message_move_stream.py b/zerver/tests/test_message_move_stream.pyindex 74f6a1d36ed..104ce878ffb 100644--- a/zerver/tests/test_message_move_stream.py+++ b/zerver/tests/test_message_move_stream.py@@ -1622,7 +1622,7 @@ class MessageMoveStreamTest(ZulipTestCase):"iago", "test move stream", "new stream", "test")- with self.assert_database_query_count(55), self.assert_memcached_count(19):+ with self.assert_database_query_count(55), self.assert_memcached_count(17):result = self.client_patch(f"/json/messages/{msg_id}",{diff --git a/zerver/tests/test_message_send.py b/zerver/tests/test_message_send.pyindex 27909482825..f46ab4ddac3 100644--- a/zerver/tests/test_message_send.py+++ b/zerver/tests/test_message_send.py@@ -2893,8 +2893,8 @@ class DoSendMessagesTest(ZulipTestCase):internal_prep_stream_message(notification_bot, lear_stream, "#1234", "Hello, Lear!"),]with (- self.assert_database_query_count(37),- self.assert_memcached_count(20),+ self.assert_database_query_count(24),+ self.assert_memcached_count(11),self.capture_send_event_calls(expected_num_events=6) as events,):do_send_messages(send_requests)diff --git a/zerver/tests/test_signup.py b/zerver/tests/test_signup.pyindex 1e3eca46228..cf19e5194a2 100644--- a/zerver/tests/test_signup.py+++ b/zerver/tests/test_signup.py@@ -1069,7 +1069,7 @@ class LoginTest(ZulipTestCase):# the alert words for a realm, etc.with (self.assert_database_query_count(99),- self.assert_memcached_count(19),+ self.assert_memcached_count(18),self.captureOnCommitCallbacks(execute=True),):self.register(self.nonreg_email("test"), "test")diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.pyindex 6159a29ae33..29ceb1a06c4 100644--- a/zerver/tests/test_subs.py+++ b/zerver/tests/test_subs.py@@ -5232,7 +5232,7 @@ class SubscriptionAPITest(ZulipTestCase):new_stream_announcements_stream = get_stream(self.streams[0], self.test_realm)self.test_realm.new_stream_announcements_stream_id = new_stream_announcements_stream.idself.test_realm.save()- with self.assert_database_query_count(57):+ with self.assert_database_query_count(53):self.subscribe_via_post(self.test_user,[new_streams[2]],@@ -5426,7 +5426,7 @@ class SubscriptionAPITest(ZulipTestCase):# the bot should send DMs to all the newly subscribed users, but# no announcement message.now = timezone_now()- with self.assert_database_query_count(90):+ with self.assert_database_query_count(74):response = self.subscribe_via_post(desdemona,[test_channel.name],@@ -5456,7 +5456,7 @@ class SubscriptionAPITest(ZulipTestCase):# we expect an announcement message and new channel message,# but no DM notifications.now = timezone_now()- with self.assert_database_query_count(57):+ with self.assert_database_query_count(53):response = self.subscribe_via_post(desdemona,["test B"],@@ -5545,7 +5545,7 @@ class SubscriptionAPITest(ZulipTestCase):now = timezone_now()with (self.settings(MAX_BULK_NEW_SUBSCRIPTION_MESSAGES=5),- self.assert_database_query_count(57),+ self.assert_database_query_count(53),):response = self.subscribe_via_post(desdemona,diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.pyindex 5b9e944e648..f0a94cc240f 100644--- a/zerver/tests/test_users.py+++ b/zerver/tests/test_users.py@@ -1155,7 +1155,7 @@ class QueryCountTest(ZulipTestCase):with (self.assert_database_query_count(95),- self.assert_memcached_count(24),+ self.assert_memcached_count(22),self.capture_send_event_calls(expected_num_events=11) as events,):fred = do_create_user(