Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

user_topics: Match muted topics by joining, not a clause per topic.

Aman Agrawalbe4917ba8d23
diff --git a/zerver/lib/user_topics.py b/zerver/lib/user_topics.py
index 8c6a2707d4c..007aaf1c9f8 100644
--- a/zerver/lib/user_topics.py
+++ b/zerver/lib/user_topics.py
@@ -2,7 +2,6 @@ import logging
from collections import defaultdict
from collections.abc import Callable
from datetime import datetime
-from typing import TypedDict
from django.db import connection, transaction
from django.db.models import Q, QuerySet
@@ -10,7 +9,7 @@ from django.utils.timezone import now as timezone_now
from psycopg2.sql import SQL, Literal
from zerver.lib.timestamp import datetime_to_timestamp
-from zerver.lib.topic import topic_match_q
+from zerver.lib.topic import matching_user_topic_exists_q
from zerver.lib.types import UserTopicDict
from zerver.models import Recipient, Subscription, UserProfile, UserTopic
from zerver.models.streams import get_stream
@@ -230,7 +229,7 @@ def exclude_stream_and_topic_mutes(user_profile: UserProfile, stream_id: int | N
# Note: Unlike get_topic_mutes, here we always want to
# consider topics in deactivated streams, so they are
# never filtered from the query in this method.
- query = UserTopic.objects.filter(
+ muted_topics = UserTopic.objects.filter(
user_profile=user_profile,
visibility_policy=UserTopic.VisibilityPolicy.MUTED,
)
@@ -238,33 +237,12 @@ def exclude_stream_and_topic_mutes(user_profile: UserProfile, stream_id: int | N
if stream_id is not None:
# If we are narrowed to a stream, we can optimize the query
# by not considering topic mutes outside the stream.
- query = query.filter(stream_id=stream_id)
-
- excluded_topic_rows = list(
- query.values(
- "recipient_id",
- "topic_name",
- )
- )
+ muted_topics = muted_topics.filter(stream_id=stream_id)
conditions = ~Q(pk__in=[]) # Always true.
- class RecipientTopicDict(TypedDict):
- recipient_id: int
- topic_name: str
-
- def topic_cond(row: RecipientTopicDict) -> Q:
- recipient_id = row["recipient_id"]
- topic_name = row["topic_name"]
- assert isinstance(topic_name, str)
- return Q(recipient_id=recipient_id) & topic_match_q(topic_name)
-
- # Add this query later to reduce the number of messages it has to run on.
- if excluded_topic_rows:
- muted_topics_q = topic_cond(excluded_topic_rows[0])
- for row in excluded_topic_rows[1:]:
- muted_topics_q |= topic_cond(row)
- conditions &= ~muted_topics_q
+ if muted_topics.exists():
+ conditions &= ~matching_user_topic_exists_q(muted_topics)
# Channel-level muting only applies when looking at views that
# include multiple channels, since we do want users to be able to
@@ -281,37 +259,21 @@ def exclude_stream_and_topic_mutes(user_profile: UserProfile, stream_id: int | N
if len(muted_recipient_ids) == 0:
return conditions
- # Add entries with visibility_policy FOLLOWED or UNMUTED in muted_recipient_ids
- query = UserTopic.objects.filter(
+ followed_or_unmuted_topics = UserTopic.objects.filter(
user_profile=user_profile,
- recipient_id__in=muted_recipient_ids,
visibility_policy__in=[
UserTopic.VisibilityPolicy.FOLLOWED,
UserTopic.VisibilityPolicy.UNMUTED,
],
)
+ # Exclude messages in muted_recipient_ids, unless they match
+ # followed_or_unmuted_topics.
+ hidden_by_muted_channel = Q(recipient_id__in=muted_recipient_ids)
- included_topic_rows = query.values(
- "recipient_id",
- "topic_name",
- )
-
- # Exclude muted_recipient_ids unless they match include_followed_or_unmuted_topics_condition
- muted_stream_condition = Q(recipient_id__in=muted_recipient_ids)
-
- if included_topic_rows:
- include_followed_or_unmuted_topics_condition = topic_cond(included_topic_rows[0])
- for row in included_topic_rows[1:]: # nocoverage
- include_followed_or_unmuted_topics_condition |= topic_cond(row)
-
- exclude_muted_streams_condition = ~(
- muted_stream_condition & ~include_followed_or_unmuted_topics_condition
- )
- else:
- # If no included topics, exclude all muted streams
- exclude_muted_streams_condition = ~muted_stream_condition
+ if followed_or_unmuted_topics.filter(recipient_id__in=muted_recipient_ids).exists():
+ hidden_by_muted_channel &= ~matching_user_topic_exists_q(followed_or_unmuted_topics)
- conditions &= exclude_muted_streams_condition
+ conditions &= ~hidden_by_muted_channel
return conditions
diff --git a/zerver/tests/test_message_fetch.py b/zerver/tests/test_message_fetch.py
index 1a16763b82a..b12a964167e 100644
--- a/zerver/tests/test_message_fetch.py
+++ b/zerver/tests/test_message_fetch.py
@@ -4981,13 +4981,23 @@ class GetOldMessagesTest(ZulipTestCase):
# Do some tests on the main query, to verify the muting logic
# runs on this code path.
capture_anchor.assert_called_once()
- sql = get_django_sql(capture_anchor.call_args.args[0])
+ anchor_query = capture_anchor.call_args.args[0].query
+ sql, params = anchor_query.sql_with_params()
+
+ cond = """\
+AND NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \
+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" = %s AND U0."stream_id" = %s \
+AND U0."recipient_id" = ("zerver_message"."recipient_id") \
+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1))\
+"""
+ self.assertIn(cond, sql)
channel = get_stream("Scotland", realm)
- assert channel.recipient is not None
- recipient_id = channel.recipient.id
- cond = f"""AND NOT ("zerver_message"."recipient_id" = {recipient_id} AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER('golf'))"""
- self.assertIn(cond, sql)
+ placeholders_before_cond = sql[: sql.index(cond)].count("%s")
+ self.assertEqual(
+ params[placeholders_before_cond : placeholders_before_cond + 4],
+ (1, user_profile.id, UserTopic.VisibilityPolicy.MUTED, channel.id),
+ )
# Next, verify the use_first_unread_anchor setting invokes
# the `message_id = LARGER_THAN_MAX_MESSAGE_ID` hack.
@@ -5120,7 +5130,10 @@ class GetOldMessagesTest(ZulipTestCase):
expected_query = """\
SELECT "zerver_message"."id" AS "id" \
FROM "zerver_message" \
-WHERE NOT ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s))\
+WHERE NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \
+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" = %s AND U0."stream_id" = %s \
+AND U0."recipient_id" = ("zerver_message"."recipient_id") \
+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1))\
"""
sql, params = query.sql_with_params()
@@ -5129,8 +5142,10 @@ WHERE NOT ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel
self.assertEqual(
params,
(
- get_recipient_id_for_channel_name(realm, "Scotland"),
- "golf",
+ 1,
+ user_profile.id,
+ UserTopic.VisibilityPolicy.MUTED,
+ get_stream("Scotland", realm).id,
),
)
@@ -5149,8 +5164,10 @@ WHERE NOT ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel
expected_query = """\
SELECT "zerver_message"."id" AS "id" \
FROM "zerver_message" \
-WHERE (NOT (("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s)) \
-OR ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s))) \
+WHERE (NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \
+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" = %s \
+AND U0."recipient_id" = ("zerver_message"."recipient_id") \
+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1)) \
AND NOT ("zerver_message"."recipient_id" IN (%s)))\
"""
sql, params = query.sql_with_params()
@@ -5158,10 +5175,9 @@ AND NOT ("zerver_message"."recipient_id" IN (%s)))\
self.assertEqual(
params,
(
- get_recipient_id_for_channel_name(realm, "Scotland"),
- "golf",
- get_recipient_id_for_channel_name(realm, "web stuff"),
- "css",
+ 1,
+ user_profile.id,
+ UserTopic.VisibilityPolicy.MUTED,
channel_verona_id,
),
)
@@ -5180,26 +5196,45 @@ AND NOT ("zerver_message"."recipient_id" IN (%s)))\
expected_query = """\
SELECT "zerver_message"."id" AS "id" \
FROM "zerver_message" \
-WHERE (NOT (("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s)) \
-OR ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s))) \
+WHERE (NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \
+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" = %s \
+AND U0."recipient_id" = ("zerver_message"."recipient_id") \
+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1)) \
AND NOT ("zerver_message"."recipient_id" IN (%s) \
-AND NOT ("zerver_message"."recipient_id" = %s AND "zerver_message"."is_channel_message" AND UPPER("zerver_message"."subject"::text) = UPPER(%s))))\
+AND NOT (EXISTS(SELECT %s AS "a" FROM "zerver_usertopic" U0 \
+WHERE (U0."user_profile_id" = %s AND U0."visibility_policy" IN (%s, %s) \
+AND U0."recipient_id" = ("zerver_message"."recipient_id") \
+AND UPPER(U0."topic_name"::text) = UPPER(("zerver_message"."subject"))) LIMIT 1))))\
"""
sql, params = query.sql_with_params()
self.assertEqual(sql, expected_query)
self.assertEqual(
params,
(
- get_recipient_id_for_channel_name(realm, "Scotland"),
- "golf",
- get_recipient_id_for_channel_name(realm, "web stuff"),
- "css",
- channel_verona_id,
+ 1,
+ user_profile.id,
+ UserTopic.VisibilityPolicy.MUTED,
channel_verona_id,
- "Hi",
+ 1,
+ user_profile.id,
+ UserTopic.VisibilityPolicy.FOLLOWED,
+ UserTopic.VisibilityPolicy.UNMUTED,
),
)
+ # The query must not grow with the number of muted topics; the
+ # inlined form it replaced added a condition per muted topic,
+ # which for a user with thousands of them produced SQL that was
+ # extremely expensive for PostgreSQL to plan.
+ set_topic_visibility_policy(
+ user_profile,
+ [["Scotland", f"topic {i}"] for i in range(100)],
+ UserTopic.VisibilityPolicy.MUTED,
+ )
+ muting_conditions = exclude_muting_conditions(user_profile, narrow)
+ query = Message.objects.all().filter(muting_conditions).values("id").query
+ self.assertEqual(query.sql_with_params(), (sql, params))
+
def test_get_messages_queries(self) -> None:
query_ids = self.get_query_ids()