Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
event_queue: Don't send new message events to unsubscribed guests.
Sahil Batrac03db8916456
diff --git a/zerver/actions/message_send.py b/zerver/actions/message_send.pyindex 83b7c0f75cd..3cd4439b874 100644--- a/zerver/actions/message_send.py+++ b/zerver/actions/message_send.py@@ -127,7 +127,12 @@ from zerver.models.streams import (get_realm_stream,get_stream_by_id_in_realm,)-from zerver.models.users import get_system_bot, get_user_by_delivery_email, is_cross_realm_bot_email+from zerver.models.users import (+ active_guest_user_ids,+ get_system_bot,+ get_user_by_delivery_email,+ is_cross_realm_bot_email,+)from zerver.tornado.django_api import send_event_on_commit@@ -1255,8 +1260,16 @@ def do_send_messages(if send_request.stream.is_public():event["realm_id"] = send_request.stream.realm_idevent["stream_name"] = send_request.stream.name+ # Tornado uses these to exclude guest users' clients+ # registered for all public channels from receiving the+ # events for unsubscribed non web-public channel+ # messages, since guests can only access such channels+ # when subscribed.+ event["realm_guest_user_ids"] = active_guest_user_ids(send_request.stream.realm_id)if send_request.stream.invite_only:event["invite_only"] = True+ if send_request.stream.is_web_public:+ event["is_web_public"] = Trueif send_request.stream.first_message_id is None:send_request.stream.first_message_id = send_request.message.idstream_update_fields.append("first_message_id")diff --git a/zerver/lib/cache.py b/zerver/lib/cache.pyindex 6d2ad85f207..43bb79056fc 100644--- a/zerver/lib/cache.py+++ b/zerver/lib/cache.py@@ -564,6 +564,10 @@ def active_non_guest_user_ids_cache_key(realm_id: int) -> str:return f"active_non_guest_user_ids:{realm_id}"+def active_guest_user_ids_cache_key(realm_id: int) -> str:+ return f"active_guest_user_ids:{realm_id}"++def get_realm_system_groups_cache_key(realm_id: int) -> str:return f"realm_system_groups:{realm_id}"@@ -645,9 +649,11 @@ def bulk_flush_users(if changed(update_fields, ["is_active"]):cache_keys_to_delete.add(active_user_ids_cache_key(realm.id))cache_keys_to_delete.add(active_non_guest_user_ids_cache_key(realm.id))+ cache_keys_to_delete.add(active_guest_user_ids_cache_key(realm.id))if changed(update_fields, ["role"]):cache_keys_to_delete.add(active_non_guest_user_ids_cache_key(realm.id))+ cache_keys_to_delete.add(active_guest_user_ids_cache_key(realm.id))# Invalidate our bots_in_realm info dict if any bot has# changed the fields in the dict or become (in)active@@ -706,6 +712,7 @@ def flush_realm(cache_delete(realm_alert_words_cache_key(realm.id))cache_delete(realm_alert_words_automaton_cache_key(realm.id))cache_delete(active_non_guest_user_ids_cache_key(realm.id))+ cache_delete(active_guest_user_ids_cache_key(realm.id))cache_delete(realm_rendered_description_cache_key(realm))cache_delete(realm_text_description_cache_key(realm))elif changed(update_fields, ["description"]):diff --git a/zerver/models/users.py b/zerver/models/users.pyindex ef2a5978121..c0329af941f 100644--- a/zerver/models/users.py+++ b/zerver/models/users.py@@ -14,6 +14,7 @@ from django.utils.translation import gettext_lazyfrom typing_extensions import overridefrom zerver.lib.cache import (+ active_guest_user_ids_cache_key,active_non_guest_user_ids_cache_key,active_user_ids_cache_key,bot_dict_fields,@@ -1221,6 +1222,16 @@ def active_non_guest_user_ids(realm_id: int) -> list[int]:return list(query)+@cache_with_key(active_guest_user_ids_cache_key, timeout=3600 * 24 * 7)+def active_guest_user_ids(realm_id: int) -> list[int]:+ query = UserProfile.objects.filter(+ realm_id=realm_id,+ is_active=True,+ role=UserProfile.ROLE_GUEST,+ ).values_list("id", flat=True)+ return list(query)++def bot_owner_user_ids(user_profile: UserProfile) -> set[int]:is_private_bot = (user_profile.default_sending_stream and user_profile.default_sending_stream.invite_onlydiff --git a/zerver/tests/test_email_mirror.py b/zerver/tests/test_email_mirror.pyindex efc0f3f35aa..8a59302b0f8 100644--- a/zerver/tests/test_email_mirror.py+++ b/zerver/tests/test_email_mirror.py@@ -1411,7 +1411,7 @@ class TestMissedMessageEmailMessages(ZulipTestCase):incoming_valid_message["To"] = mm_addressincoming_valid_message["Reply-to"] = user_profile.delivery_email- with self.assert_database_query_count(18):+ with self.assert_database_query_count(19):process_message(incoming_valid_message)# confirm that Hamlet got the messagediff --git a/zerver/tests/test_event_system.py b/zerver/tests/test_event_system.pyindex 8e4732440f9..806708b999d 100644--- a/zerver/tests/test_event_system.py+++ b/zerver/tests/test_event_system.py@@ -17,6 +17,7 @@ from zerver.actions.message_send import check_send_messagefrom zerver.actions.presence import do_update_user_presencefrom zerver.actions.streams import do_change_stream_folderfrom zerver.actions.user_settings import do_change_avatar_fields, do_change_user_setting+from zerver.actions.users import do_change_user_rolefrom zerver.lib.event_schema import check_web_reload_client_eventfrom zerver.lib.events import fetch_initial_state_data, post_process_statefrom zerver.lib.exceptions import AccessDeniedError@@ -608,6 +609,166 @@ class GetEventsTest(ZulipTestCase):self.assertEqual(message["content"], "<p><strong>hello</strong></p>")self.assertEqual(message["avatar_url"], None)+ def test_get_events_guest_narrow_unsubscribed_stream(self) -> None:+ guest = self.example_user("polonius")+ hamlet = self.example_user("hamlet")+ self.login_user(guest)++ stream_name = "public_events_stream"+ self.subscribe(hamlet, stream_name)++ result = self.tornado_call(+ get_events,+ guest,+ dict(+ apply_markdown=orjson.dumps(True).decode(),+ client_gravatar=orjson.dumps(True).decode(),+ event_types=orjson.dumps(["message"]).decode(),+ narrow=orjson.dumps([["stream", stream_name]]).decode(),+ user_client="website",+ dont_block=orjson.dumps(True).decode(),+ ),+ )+ self.assert_json_success(result)+ queue_id = orjson.loads(result.content)["queue_id"]++ def get_events_for_queue() -> list[dict[str, Any]]:+ result = self.tornado_call(+ get_events,+ guest,+ {+ "queue_id": queue_id,+ "user_client": "website",+ "last_event_id": -1,+ "dont_block": orjson.dumps(True).decode(),+ },+ )+ self.assert_json_success(result)+ return orjson.loads(result.content)["events"]++ # A guest does not receive events for messages in a public+ # channel they are not subscribed to, even with a narrow+ # matching the message.+ self.send_stream_message(hamlet, stream_name, "secret message")+ self.assert_length(get_events_for_queue(), 0)++ # Once subscribed, the guest receives message events.+ self.subscribe(guest, stream_name)+ self.send_stream_message(hamlet, stream_name, "public message")+ events = get_events_for_queue()+ self.assert_length(events, 1)+ self.assertEqual(events[0]["type"], "message")+ self.assertEqual(events[0]["message"]["content"], "<p>public message</p>")++ def test_get_events_guest_narrow_unsubscribed_web_public_stream(self) -> None:+ guest = self.example_user("polonius")+ hamlet = self.example_user("hamlet")+ self.login_user(guest)++ stream_name = "web_public_events_stream"+ self.make_stream(stream_name, guest.realm, is_web_public=True)+ self.subscribe(hamlet, stream_name)++ result = self.tornado_call(+ get_events,+ guest,+ dict(+ apply_markdown=orjson.dumps(True).decode(),+ client_gravatar=orjson.dumps(True).decode(),+ event_types=orjson.dumps(["message"]).decode(),+ narrow=orjson.dumps([["stream", stream_name]]).decode(),+ user_client="website",+ dont_block=orjson.dumps(True).decode(),+ ),+ )+ self.assert_json_success(result)+ queue_id = orjson.loads(result.content)["queue_id"]++ # A guest can read web-public channels without subscribing.+ self.send_stream_message(hamlet, stream_name, "web-public message")+ result = self.tornado_call(+ get_events,+ guest,+ {+ "queue_id": queue_id,+ "user_client": "website",+ "last_event_id": -1,+ "dont_block": orjson.dumps(True).decode(),+ },+ )+ self.assert_json_success(result)+ events = orjson.loads(result.content)["events"]+ self.assert_length(events, 1)+ self.assertEqual(events[0]["type"], "message")+ self.assertEqual(events[0]["message"]["content"], "<p>web-public message</p>")++ def test_get_events_all_public_streams_after_guest_demotion(self) -> None:+ hamlet = self.example_user("hamlet")+ iago = self.example_user("iago")+ self.assertFalse(hamlet.is_guest)+ self.login_user(hamlet)++ stream_name = "public_events_stream"+ self.subscribe(iago, stream_name)++ result = self.tornado_call(+ get_events,+ hamlet,+ dict(+ apply_markdown=orjson.dumps(True).decode(),+ client_gravatar=orjson.dumps(True).decode(),+ event_types=orjson.dumps(["message"]).decode(),+ all_public_streams=orjson.dumps(True).decode(),+ user_client="website",+ dont_block=orjson.dumps(True).decode(),+ ),+ )+ self.assert_json_success(result)+ queue_id = orjson.loads(result.content)["queue_id"]++ last_event_id = -1++ def get_new_events() -> list[dict[str, Any]]:+ nonlocal last_event_id+ result = self.tornado_call(+ get_events,+ hamlet,+ {+ "queue_id": queue_id,+ "user_client": "website",+ "last_event_id": last_event_id,+ "dont_block": orjson.dumps(True).decode(),+ },+ )+ self.assert_json_success(result)+ events = orjson.loads(result.content)["events"]+ if events:+ last_event_id = events[-1]["id"]+ return events++ # As a member with an `all_public_streams` queue, hamlet+ # receives messages for public channels he is not subscribed+ # to.+ self.send_stream_message(iago, stream_name, "member message")+ events = get_new_events()+ self.assert_length(events, 1)+ self.assertEqual(events[0]["message"]["content"], "<p>member message</p>")++ # After hamlet's role is changed to guest, the same queue no+ # longer receives messages for public channels he is not+ # subscribed to.+ do_change_user_role(hamlet, UserProfile.ROLE_GUEST, acting_user=None, notify=False)+ self.send_stream_message(iago, stream_name, "secret message")+ self.assert_length(get_new_events(), 0)++ # Once subscribed, the guest receives message events again.+ self.subscribe(hamlet, stream_name)+ self.send_stream_message(iago, stream_name, "subscribed message")+ events = get_new_events()+ self.assert_length(events, 1)+ self.assertEqual(events[0]["type"], "message")+ self.assertEqual(events[0]["message"]["content"], "<p>subscribed message</p>")+def test_bogus_queue_id(self) -> None:user = self.example_user("hamlet")@@ -974,6 +1135,127 @@ class ClientDescriptorsTest(ZulipTestCase):dct = client_info[client.event_queue.id]self.assertEqual(dct["is_sender"], True)+ def test_get_client_info_for_narrow_guest(self) -> None:+ polonius = self.example_user("polonius")+ realm = polonius.realm++ queue_data = dict(+ all_public_streams=False,+ apply_markdown=True,+ client_gravatar=True,+ client_type_name="website",+ event_types=["message"],+ last_connection_time=time.time(),+ queue_timeout=0,+ realm_id=realm.id,+ user_profile_id=polonius.id,+ narrow=[["stream", "whatever"]],+ )++ client = allocate_client_descriptor(queue_data)++ # A guest's client in realm_clients_all_streams is skipped,+ # identified via `realm_guest_user_ids`.+ message_event = dict(+ realm_id=realm.id,+ stream_name="whatever",+ realm_guest_user_ids=[polonius.id],+ )+ client_info = get_client_info_for_message_event(+ message_event,+ users=[],+ )+ self.assert_length(client_info, 0)++ # As a subscriber, the guest user instead receives the event+ # via `users`, with their user-specific flags.+ client_info = get_client_info_for_message_event(+ message_event,+ users=[dict(id=polonius.id, flags=["mentioned"])],+ )+ self.assert_length(client_info, 1)+ dct = client_info[client.event_queue.id]+ self.assertEqual(dct["client"].user_profile_id, polonius.id)+ self.assertEqual(dct["flags"], ["mentioned"])++ def test_get_client_info_for_all_public_streams_guest(self) -> None:+ polonius = self.example_user("polonius")+ realm = polonius.realm++ queue_data = dict(+ all_public_streams=True,+ apply_markdown=True,+ client_gravatar=True,+ client_type_name="website",+ event_types=["message"],+ last_connection_time=time.time(),+ queue_timeout=0,+ realm_id=realm.id,+ user_profile_id=polonius.id,+ )++ client = allocate_client_descriptor(queue_data)++ # A guest's client in realm_clients_all_streams is skipped,+ # identified via `realm_guest_user_ids`.+ message_event = dict(+ realm_id=realm.id,+ stream_name="whatever",+ realm_guest_user_ids=[polonius.id],+ )+ client_info = get_client_info_for_message_event(+ message_event,+ users=[],+ )+ self.assert_length(client_info, 0)++ # As a subscriber, the guest user instead receives the event+ # via `users`, with their user-specific flags.+ client_info = get_client_info_for_message_event(+ message_event,+ users=[dict(id=polonius.id, flags=["mentioned"])],+ )+ self.assert_length(client_info, 1)+ dct = client_info[client.event_queue.id]+ self.assertEqual(dct["client"].user_profile_id, polonius.id)+ self.assertEqual(dct["flags"], ["mentioned"])++ def test_get_client_info_for_all_public_streams_guest_web_public(self) -> None:+ polonius = self.example_user("polonius")+ realm = polonius.realm++ queue_data = dict(+ all_public_streams=True,+ apply_markdown=True,+ client_gravatar=True,+ client_type_name="website",+ event_types=["message"],+ last_connection_time=time.time(),+ queue_timeout=0,+ realm_id=realm.id,+ user_profile_id=polonius.id,+ )++ client = allocate_client_descriptor(queue_data)++ # For a web-public channel, a guest's client in+ # realm_clients_all_streams is not skipped, since guests can+ # read web-public channels without subscribing.+ message_event = dict(+ realm_id=realm.id,+ stream_name="whatever",+ is_web_public=True,+ realm_guest_user_ids=[polonius.id],+ )+ client_info = get_client_info_for_message_event(+ message_event,+ users=[],+ )+ self.assert_length(client_info, 1)+ dct = client_info[client.event_queue.id]+ self.assertEqual(dct["client"].user_profile_id, polonius.id)+ self.assertEqual(dct["flags"], [])+def test_get_client_info_for_normal_users(self) -> None:hamlet = self.example_user("hamlet")cordelia = self.example_user("cordelia")diff --git a/zerver/tests/test_message_move_stream.py b/zerver/tests/test_message_move_stream.pyindex 032f14ca080..e4e59af98ea 100644--- a/zerver/tests/test_message_move_stream.py+++ b/zerver/tests/test_message_move_stream.py@@ -1592,7 +1592,7 @@ class MessageMoveStreamTest(ZulipTestCase):"iago", "test move stream", "new stream", "test")- with self.assert_database_query_count(56), self.assert_memcached_count(17):+ with self.assert_database_query_count(57), self.assert_memcached_count(19):result = self.client_patch(f"/json/messages/{msg_id}",{diff --git a/zerver/tests/test_message_send.py b/zerver/tests/test_message_send.pyindex fc614870037..eecc5b24970 100644--- a/zerver/tests/test_message_send.py+++ b/zerver/tests/test_message_send.py@@ -1785,7 +1785,7 @@ class StreamMessagesTest(ZulipTestCase):setting_value=UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_NEVER,acting_user=None,)- with self.assert_database_query_count(14):+ with self.assert_database_query_count(15):check_send_stream_message(sender=sender,client=sending_client,@@ -1805,7 +1805,7 @@ class StreamMessagesTest(ZulipTestCase):# 5 queries: 1 to check if it is the first message in the topic +# 1 to check if the topic is already followed + 3 to follow the topic.flush_per_request_caches()- with self.assert_database_query_count(19):+ with self.assert_database_query_count(20):check_send_stream_message(sender=sender,client=sending_client,@@ -1825,7 +1825,7 @@ class StreamMessagesTest(ZulipTestCase):# a message to a topic with visibility policy other than FOLLOWED.# 1 to check if the topic is already followed + 3 queries to follow the topic.flush_per_request_caches()- with self.assert_database_query_count(18):+ with self.assert_database_query_count(19):check_send_stream_message(sender=sender,client=sending_client,@@ -1836,7 +1836,7 @@ class StreamMessagesTest(ZulipTestCase):# If the topic is already FOLLOWED, there will be an increase in the query# count of 1 to check if the topic is already followed.flush_per_request_caches()- with self.assert_database_query_count(15):+ with self.assert_database_query_count(16):check_send_stream_message(sender=sender,client=sending_client,@@ -1861,7 +1861,7 @@ class StreamMessagesTest(ZulipTestCase):# 1 to get the user_id of the mentioned user + 1 to check if the topic# is already followed + 3 queries to follow the topic.flush_per_request_caches()- with self.assert_database_query_count(23):+ with self.assert_database_query_count(24):check_send_stream_message(sender=sender,client=sending_client,@@ -1874,7 +1874,7 @@ class StreamMessagesTest(ZulipTestCase):# 1 to get the user_id of the mentioned user + 1 to check if the topic is# already followed.flush_per_request_caches()- with self.assert_database_query_count(20):+ with self.assert_database_query_count(21):check_send_stream_message(sender=sender,client=sending_client,@@ -1884,7 +1884,7 @@ class StreamMessagesTest(ZulipTestCase):)flush_per_request_caches()- with self.assert_database_query_count(17):+ with self.assert_database_query_count(18):check_send_stream_message(sender=sender,client=sending_client,@@ -1907,7 +1907,7 @@ class StreamMessagesTest(ZulipTestCase):)flush_per_request_caches()- with self.assert_database_query_count(18):+ with self.assert_database_query_count(19):check_send_stream_message(sender=sender,client=sending_client,@@ -1990,6 +1990,47 @@ class StreamMessagesTest(ZulipTestCase):message = most_recent_message(user_profile)self.assertFalse(message.is_channel_message)+ def test_realm_guest_user_ids_in_message_event(self) -> None:+ hamlet = self.example_user("hamlet")+ polonius = self.example_user("polonius")++ # Events for public channel messages include all active guest+ # users in the realm, whether or not they are subscribed.+ stream_name = "Test stream"+ self.subscribe(hamlet, stream_name)+ with self.capture_send_event_calls(expected_num_events=1) as events:+ self.send_stream_message(+ hamlet,+ stream_name,+ content="test",+ skip_capture_on_commit_callbacks=True,+ )+ self.assertEqual(events[0]["event"]["realm_guest_user_ids"], [polonius.id])++ # The field is not attached to events for private channel+ # messages or direct messages as we do not send events to+ # clients in realm_clients_all_streams for these messages.+ private_stream_name = "Private test stream"+ self.make_stream(private_stream_name, invite_only=True)+ self.subscribe(hamlet, private_stream_name)+ with self.capture_send_event_calls(expected_num_events=1) as events:+ self.send_stream_message(+ hamlet,+ private_stream_name,+ content="test",+ skip_capture_on_commit_callbacks=True,+ )+ self.assertNotIn("realm_guest_user_ids", events[0]["event"])++ with self.capture_send_event_calls(expected_num_events=1) as events:+ self.send_personal_message(+ hamlet,+ polonius,+ content="test",+ skip_capture_on_commit_callbacks=True,+ )+ self.assertNotIn("realm_guest_user_ids", events[0]["event"])+def _send_stream_message(self, user: UserProfile, stream_name: str, content: str) -> set[int]:with self.capture_send_event_calls(expected_num_events=1) as events:self.send_stream_message(diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.pyindex ddac8ce8432..a7c2d399222 100644--- a/zerver/tests/test_subs.py+++ b/zerver/tests/test_subs.py@@ -4347,7 +4347,7 @@ class SubscriptionAPITest(ZulipTestCase):streams_to_sub = ["multi_user_stream"]with (self.capture_send_event_calls(expected_num_events=5) as events,- self.assert_database_query_count(44),+ self.assert_database_query_count(45),):self.subscribe_via_post(self.test_user,@@ -5256,7 +5256,7 @@ class SubscriptionAPITest(ZulipTestCase):]# Test creating a public stream when realm does not have a notification stream.- with self.assert_database_query_count(44):+ with self.assert_database_query_count(45):self.subscribe_via_post(self.test_user,[new_streams[0]],@@ -5276,7 +5276,7 @@ class SubscriptionAPITest(ZulipTestCase):new_stream_announcements_stream = get_stream(self.streams[0], self.test_realm)self.test_realm.new_stream_announcements_stream_id = new_stream_announcements_stream.idself.test_realm.save()- with self.assert_database_query_count(56):+ with self.assert_database_query_count(57):self.subscribe_via_post(self.test_user,[new_streams[2]],diff --git a/zerver/tornado/event_queue.py b/zerver/tornado/event_queue.pyindex 6324bab60c0..f4430da8e99 100644--- a/zerver/tornado/event_queue.py+++ b/zerver/tornado/event_queue.py@@ -1109,7 +1109,13 @@ def get_client_info_for_message_event(# bots) that are registered to get events for ALL streams.if "stream_name" in event_template and not event_template.get("invite_only"):realm_id = event_template["realm_id"]+ is_web_public = event_template.get("is_web_public", False)+ realm_guest_user_ids = set(event_template.get("realm_guest_user_ids", []))for client in get_client_descriptors_for_realm_all_streams(realm_id):+ # Guest users cannot access non-subscribed non web-public+ # channels.+ if not is_web_public and client.user_profile_id in realm_guest_user_ids:+ continuesend_to_clients[client.event_queue.id] = dict(client=client,flags=[],