Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
user_topics: Match muted topics by joining, not a clause per topic.
Aman Agrawalbe4917ba8d23
diff --git a/zerver/lib/user_topics.py b/zerver/lib/user_topics.pyindex 8c6a2707d4c..007aaf1c9f8 100644--- a/zerver/lib/user_topics.py+++ b/zerver/lib/user_topics.py@@ -2,7 +2,6 @@ import loggingfrom collections import defaultdictfrom collections.abc import Callablefrom datetime import datetime-from typing import TypedDictfrom django.db import connection, transactionfrom django.db.models import Q, QuerySet@@ -10,7 +9,7 @@ from django.utils.timezone import now as timezone_nowfrom psycopg2.sql import SQL, Literalfrom zerver.lib.timestamp import datetime_to_timestamp-from zerver.lib.topic import topic_match_q+from zerver.lib.topic import matching_user_topic_exists_qfrom zerver.lib.types import UserTopicDictfrom zerver.models import Recipient, Subscription, UserProfile, UserTopicfrom zerver.models.streams import get_stream@@ -230,7 +229,7 @@ def exclude_stream_and_topic_mutes(user_profile: UserProfile, stream_id: int | N# Note: Unlike get_topic_mutes, here we always want to# consider topics in deactivated streams, so they are# never filtered from the query in this method.- query = UserTopic.objects.filter(+ muted_topics = UserTopic.objects.filter(user_profile=user_profile,visibility_policy=UserTopic.VisibilityPolicy.MUTED,)@@ -238,33 +237,12 @@ def exclude_stream_and_topic_mutes(user_profile: UserProfile, stream_id: int | Nif stream_id is not None:# If we are narrowed to a stream, we can optimize the query# by not considering topic mutes outside the stream.- query = query.filter(stream_id=stream_id)-- excluded_topic_rows = list(- query.values(- "recipient_id",- "topic_name",- )- )+ muted_topics = muted_topics.filter(stream_id=stream_id)conditions = ~Q(pk__in=[]) # Always true.- class RecipientTopicDict(TypedDict):- recipient_id: int- topic_name: str-- def topic_cond(row: RecipientTopicDict) -> Q:- recipient_id = row["recipient_id"]- topic_name = row["topic_name"]- assert isinstance(topic_name, str)- return Q(recipient_id=recipient_id) & topic_match_q(topic_name)-- # Add this query later to reduce the number of messages it has to run on.- if excluded_topic_rows:- muted_topics_q = topic_cond(excluded_topic_rows[0])- for row in excluded_topic_rows[1:]:- muted_topics_q |= topic_cond(row)- conditions &= ~muted_topics_q+ if muted_topics.exists():+ conditions &= ~matching_user_topic_exists_q(muted_topics)# Channel-level muting only applies when looking at views that# include multiple channels, since we do want users to be able to@@ -281,37 +259,21 @@ def exclude_stream_and_topic_mutes(user_profile: UserProfile, stream_id: int | Nif len(muted_recipient_ids) == 0:return conditions- # Add entries with visibility_policy FOLLOWED or UNMUTED in muted_recipient_ids- query = UserTopic.objects.filter(+ followed_or_unmuted_topics = UserTopic.objects.filter(user_profile=user_profile,- recipient_id__in=muted_recipient_ids,visibility_policy__in=[UserTopic.VisibilityPolicy.FOLLOWED,UserTopic.VisibilityPolicy.UNMUTED,],)+ # Exclude messages in muted_recipient_ids, unless they match+ # followed_or_unmuted_topics.+ hidden_by_muted_channel = Q(recipient_id__in=muted_recipient_ids)- included_topic_rows = query.values(- "recipient_id",- "topic_name",- )-- # Exclude muted_recipient_ids unless they match include_followed_or_unmuted_topics_condition- muted_stream_condition = Q(recipient_id__in=muted_recipient_ids)-- if included_topic_rows:- include_followed_or_unmuted_topics_condition = topic_cond(included_topic_rows[0])- for row in included_topic_rows[1:]: # nocoverage- include_followed_or_unmuted_topics_condition |= topic_cond(row)-- exclude_muted_streams_condition = ~(- muted_stream_condition & ~include_followed_or_unmuted_topics_condition- )- else:- # If no included topics, exclude all muted streams- exclude_muted_streams_condition = ~muted_stream_condition+ if followed_or_unmuted_topics.filter(recipient_id__in=muted_recipient_ids).exists():+ hidden_by_muted_channel &= ~matching_user_topic_exists_q(followed_or_unmuted_topics)- conditions &= exclude_muted_streams_condition+ conditions &= ~hidden_by_muted_channelreturn conditionsdiff --git a/zerver/tests/test_message_fetch.py b/zerver/tests/test_message_fetch.pyindex 1a16763b82a..b12a964167e 100644--- a/zerver/tests/test_message_fetch.py+++ b/zerver/tests/test_message_fetch.py@@ -4981,13 +4981,23 @@ class GetOldMessagesTest(ZulipTestCase):# Do some tests on the main query, to verify the muting logic# runs on this code path.capture_anchor.assert_called_once()- sql = get_django_sql(capture_anchor.call_args.args[0])+ anchor_query = capture_anchor.call_args.args[0].query+ sql, params = anchor_query.sql_with_params()++ cond = """\+AND NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" = %s AND U0."stream_id" = %s \+AND U0."recipient_id" = ("zerver_message"."recipient_id") \+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1))\+"""+ self.assertIn(cond, sql)channel = get_stream("Scotland", realm)- assert channel.recipient is not None- recipient_id = channel.recipient.id- cond = f"""AND NOT ("zerver_message"."recipient_id" = {recipient_id} AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER('golf'))"""- self.assertIn(cond, sql)+ placeholders_before_cond = sql[: sql.index(cond)].count("%s")+ self.assertEqual(+ params[placeholders_before_cond : placeholders_before_cond + 4],+ (1, user_profile.id, UserTopic.VisibilityPolicy.MUTED, channel.id),+ )# Next, verify the use_first_unread_anchor setting invokes# the `message_id = LARGER_THAN_MAX_MESSAGE_ID` hack.@@ -5120,7 +5130,10 @@ class GetOldMessagesTest(ZulipTestCase):expected_query = """\SELECT "zerver_message"."id" AS "id" \FROM "zerver_message" \-WHERE NOT ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s))\+WHERE NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" = %s AND U0."stream_id" = %s \+AND U0."recipient_id" = ("zerver_message"."recipient_id") \+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1))\"""sql, params = query.sql_with_params()@@ -5129,8 +5142,10 @@ WHERE NOT ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channelself.assertEqual(params,(- get_recipient_id_for_channel_name(realm, "Scotland"),- "golf",+ 1,+ user_profile.id,+ UserTopic.VisibilityPolicy.MUTED,+ get_stream("Scotland", realm).id,),)@@ -5149,8 +5164,10 @@ WHERE NOT ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channelexpected_query = """\SELECT "zerver_message"."id" AS "id" \FROM "zerver_message" \-WHERE (NOT (("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s)) \-OR ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s))) \+WHERE (NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" = %s \+AND U0."recipient_id" = ("zerver_message"."recipient_id") \+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1)) \AND NOT ("zerver_message"."recipient_id" IN (%s)))\"""sql, params = query.sql_with_params()@@ -5158,10 +5175,9 @@ AND NOT ("zerver_message"."recipient_id" IN (%s)))\self.assertEqual(params,(- get_recipient_id_for_channel_name(realm, "Scotland"),- "golf",- get_recipient_id_for_channel_name(realm, "web stuff"),- "css",+ 1,+ user_profile.id,+ UserTopic.VisibilityPolicy.MUTED,channel_verona_id,),)@@ -5180,26 +5196,45 @@ AND NOT ("zerver_message"."recipient_id" IN (%s)))\expected_query = """\SELECT "zerver_message"."id" AS "id" \FROM "zerver_message" \-WHERE (NOT (("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s)) \-OR ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s))) \+WHERE (NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" = %s \+AND U0."recipient_id" = ("zerver_message"."recipient_id") \+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1)) \AND NOT ("zerver_message"."recipient_id" IN (%s) \-AND NOT ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s))))\+AND NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" IN (%s, %s) \+AND U0."recipient_id" = ("zerver_message"."recipient_id") \+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1))))\"""sql, params = query.sql_with_params()self.assertEqual(sql, expected_query)self.assertEqual(params,(- get_recipient_id_for_channel_name(realm, "Scotland"),- "golf",- get_recipient_id_for_channel_name(realm, "web stuff"),- "css",- channel_verona_id,+ 1,+ user_profile.id,+ UserTopic.VisibilityPolicy.MUTED,channel_verona_id,- "Hi",+ 1,+ user_profile.id,+ UserTopic.VisibilityPolicy.FOLLOWED,+ UserTopic.VisibilityPolicy.UNMUTED,),)+ # The query must not grow with the number of muted topics; the+ # inlined form it replaced added a condition per muted topic,+ # which for a user with thousands of them produced SQL that was+ # extremely expensive for PostgreSQL to plan.+ set_topic_visibility_policy(+ user_profile,+ [["Scotland", f"topic {i}"] for i in range(100)],+ UserTopic.VisibilityPolicy.MUTED,+ )+ muting_conditions = exclude_muting_conditions(user_profile, narrow)+ query = Message.objects.all().filter(muting_conditions).values("id").query+ self.assertEqual(query.sql_with_params(), (sql, params))+def test_get_messages_queries(self) -> None:query_ids = self.get_query_ids()