Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

message_send: Build message event dicts in bulk.

Aman Agrawal649aca709d2c
diff --git a/zerver/actions/message_send.py b/zerver/actions/message_send.py
index 516fadc34cd..7cbd5d88b35 100644
--- a/zerver/actions/message_send.py
+++ b/zerver/actions/message_send.py
@@ -1039,6 +1039,26 @@ def get_message_url_and_link(
return url, link
+def get_wide_message_dicts(
+ send_message_requests: list[SendMessageRequest],
+) -> dict[int, dict[str, Any]]:
+ messages_by_realm_id: dict[int, list[Message]] = defaultdict(list)
+ for send_request in send_message_requests:
+ messages_by_realm_id[send_request.message.realm_id].append(send_request.message)
+
+ recipient_dm_user_ids = {
+ send_request.message.recipient_id: send_request.dm_involved_user_ids
+ for send_request in send_message_requests
+ if send_request.dm_involved_user_ids is not None
+ }
+ wide_message_dicts: dict[int, dict[str, Any]] = {}
+ for realm_id, messages in messages_by_realm_id.items():
+ wide_message_dicts.update(
+ MessageDict.wide_dicts_for_new_messages(messages, realm_id, recipient_dm_user_ids)
+ )
+ return wide_message_dicts
+
+
@transaction.atomic(savepoint=False)
def do_send_messages(
send_message_requests_maybe_none: Sequence[SendMessageRequest | None],
@@ -1136,6 +1156,8 @@ def do_send_messages(
for send_request in send_message_requests:
do_widget_post_save_actions(send_request)
+ wide_message_dicts = get_wide_message_dicts(send_message_requests)
+
# This next loop is responsible for notifying other parts of the
# Zulip system about the messages we just committed to the database:
# * Sender automatically follows or unmutes the topic depending on 'automatically_follow_topics_policy'
@@ -1230,11 +1252,7 @@ def do_send_messages(
# Deliver events to the real-time push system, as well as
# enqueuing any additional processing triggered by the message.
- wide_message_dict = MessageDict.wide_dict(
- send_request.message,
- realm_id,
- dm_involved_user_ids=send_request.dm_involved_user_ids,
- )
+ wide_message_dict = wide_message_dicts[send_request.message.id]
send_request.message_url, send_request.message_link = get_message_url_and_link(
send_request, wide_message_dict
diff --git a/zerver/lib/message_cache.py b/zerver/lib/message_cache.py
index 65a74a2456d..aefaa43b8ca 100644
--- a/zerver/lib/message_cache.py
+++ b/zerver/lib/message_cache.py
@@ -112,14 +112,15 @@ class MessageDict:
"""MessageDict is the core class responsible for marshalling Message
objects obtained from the database into a format that can be sent
to clients via the Zulip API, whether via `GET /messages`,
- outgoing webhooks, or other code paths. There are two core flows through
+ outgoing webhooks, or other code paths. There are two core flows through
which this class is used:
- * For just-sent messages, we construct a single `wide_dict` object
- containing all the data for the message and the related
- UserProfile models (sender_info and recipient_info); this object
- can be stored in queues, caches, etc., and then later turned
- into an API-format JSONable dictionary via finalize_payload.
+ * For just-sent messages, wide_dicts_for_new_messages constructs a
+ wide dict for each message, containing all the data for the
+ message and the related UserProfile models (sender_info and
+ recipient_info); these dicts can be stored in queues, caches,
+ etc., and then later turned into an API-format JSONable
+ dictionary via finalize_payload.
* When fetching messages from the database, we fetch their data in
bulk using messages_for_ids, which makes use of caching, bulk
@@ -145,12 +146,6 @@ class MessageDict:
encoded_object_bytes = message_to_encoded_cache(message, realm_id)
obj = extract_message_dict(encoded_object_bytes)
- """
- The steps below are similar to what we do in
- post_process_dicts(), except we don't call finalize_payload(),
- since that step happens later in the queue
- processor.
- """
MessageDict.bulk_hydrate_sender_info([obj])
MessageDict.bulk_hydrate_recipient_info(
[obj],
@@ -159,6 +154,25 @@ class MessageDict:
return obj
+ @staticmethod
+ def wide_dicts_for_new_messages(
+ messages: list[Message],
+ realm_id: int,
+ recipient_dm_user_ids: Mapping[int, set[int]],
+ ) -> dict[int, dict[str, Any]]:
+ encoded_messages = update_message_cache(messages, realm_id)
+ objs = [
+ extract_message_dict(encoded_message) for encoded_message in encoded_messages.values()
+ ]
+
+ # The steps below are similar to what we do in
+ # post_process_dicts(), except we don't call finalize_payload(),
+ # since that step happens later in the queue processor.
+ MessageDict.bulk_hydrate_sender_info(objs)
+ MessageDict.bulk_hydrate_recipient_info(objs, recipient_dm_user_ids)
+
+ return {obj["id"]: obj for obj in objs}
+
@staticmethod
def post_process_dicts(
objs: list[dict[str, Any]],
diff --git a/zerver/tests/test_channel_fetch.py b/zerver/tests/test_channel_fetch.py
index ae76e6a9f10..e2cfca9c56c 100644
--- a/zerver/tests/test_channel_fetch.py
+++ b/zerver/tests/test_channel_fetch.py
@@ -882,7 +882,7 @@ class GetSubscribersTest(ZulipTestCase):
polonius.id,
]
- with self.assert_database_query_count(67):
+ with self.assert_database_query_count(59):
self.subscribe_via_post(
self.user_profile,
stream_names,
diff --git a/zerver/tests/test_message_move_stream.py b/zerver/tests/test_message_move_stream.py
index 74f6a1d36ed..104ce878ffb 100644
--- a/zerver/tests/test_message_move_stream.py
+++ b/zerver/tests/test_message_move_stream.py
@@ -1622,7 +1622,7 @@ class MessageMoveStreamTest(ZulipTestCase):
"iago", "test move stream", "new stream", "test"
)
- with self.assert_database_query_count(55), self.assert_memcached_count(19):
+ with self.assert_database_query_count(55), self.assert_memcached_count(17):
result = self.client_patch(
f"/json/messages/{msg_id}",
{
diff --git a/zerver/tests/test_message_send.py b/zerver/tests/test_message_send.py
index 27909482825..f46ab4ddac3 100644
--- a/zerver/tests/test_message_send.py
+++ b/zerver/tests/test_message_send.py
@@ -2893,8 +2893,8 @@ class DoSendMessagesTest(ZulipTestCase):
internal_prep_stream_message(notification_bot, lear_stream, "#1234", "Hello, Lear!"),
]
with (
- self.assert_database_query_count(37),
- self.assert_memcached_count(20),
+ self.assert_database_query_count(24),
+ self.assert_memcached_count(11),
self.capture_send_event_calls(expected_num_events=6) as events,
):
do_send_messages(send_requests)
diff --git a/zerver/tests/test_signup.py b/zerver/tests/test_signup.py
index 1e3eca46228..cf19e5194a2 100644
--- a/zerver/tests/test_signup.py
+++ b/zerver/tests/test_signup.py
@@ -1069,7 +1069,7 @@ class LoginTest(ZulipTestCase):
# the alert words for a realm, etc.
with (
self.assert_database_query_count(99),
- self.assert_memcached_count(19),
+ self.assert_memcached_count(18),
self.captureOnCommitCallbacks(execute=True),
):
self.register(self.nonreg_email("test"), "test")
diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.py
index 6159a29ae33..29ceb1a06c4 100644
--- a/zerver/tests/test_subs.py
+++ b/zerver/tests/test_subs.py
@@ -5232,7 +5232,7 @@ class SubscriptionAPITest(ZulipTestCase):
new_stream_announcements_stream = get_stream(self.streams[0], self.test_realm)
self.test_realm.new_stream_announcements_stream_id = new_stream_announcements_stream.id
self.test_realm.save()
- with self.assert_database_query_count(57):
+ with self.assert_database_query_count(53):
self.subscribe_via_post(
self.test_user,
[new_streams[2]],
@@ -5426,7 +5426,7 @@ class SubscriptionAPITest(ZulipTestCase):
# the bot should send DMs to all the newly subscribed users, but
# no announcement message.
now = timezone_now()
- with self.assert_database_query_count(90):
+ with self.assert_database_query_count(74):
response = self.subscribe_via_post(
desdemona,
[test_channel.name],
@@ -5456,7 +5456,7 @@ class SubscriptionAPITest(ZulipTestCase):
# we expect an announcement message and new channel message,
# but no DM notifications.
now = timezone_now()
- with self.assert_database_query_count(57):
+ with self.assert_database_query_count(53):
response = self.subscribe_via_post(
desdemona,
["test B"],
@@ -5545,7 +5545,7 @@ class SubscriptionAPITest(ZulipTestCase):
now = timezone_now()
with (
self.settings(MAX_BULK_NEW_SUBSCRIPTION_MESSAGES=5),
- self.assert_database_query_count(57),
+ self.assert_database_query_count(53),
):
response = self.subscribe_via_post(
desdemona,
diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.py
index 5b9e944e648..f0a94cc240f 100644
--- a/zerver/tests/test_users.py
+++ b/zerver/tests/test_users.py
@@ -1155,7 +1155,7 @@ class QueryCountTest(ZulipTestCase):
with (
self.assert_database_query_count(95),
- self.assert_memcached_count(24),
+ self.assert_memcached_count(22),
self.capture_send_event_calls(expected_num_events=11) as events,
):
fred = do_create_user(