Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
message_send: Remove mirroring; gate forging DMs on can_forge_sender.
Sahil Batrad1f9dab801d9
diff --git a/analytics/management/commands/populate_analytics_db.py b/analytics/management/commands/populate_analytics_db.pyindex da5420857f0..b5d690f7928 100644--- a/analytics/management/commands/populate_analytics_db.py+++ b/analytics/management/commands/populate_analytics_db.py@@ -286,14 +286,14 @@ class Command(ZulipBaseCommand):react_native, _created = Client.objects.get_or_create(name="ZulipMobile")flutter, _created = Client.objects.get_or_create(name="ZulipFlutter")API, _created = Client.objects.get_or_create(name="API: Python")- irc_mirror, _created = Client.objects.get_or_create(name="irc_mirror")+ terminal, _created = Client.objects.get_or_create(name="ZulipTerminal")unused, _created = Client.objects.get_or_create(name="unused")long_webhook, _created = Client.objects.get_or_create(name="ZulipLooooooooooongNameWebhook")stat = COUNT_STATS["messages_sent:client:day"]user_data = {website.id: self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8),- irc_mirror.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),+ terminal.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),}insert_fixture_data(stat, user_data, UserCount)realm_data = {@@ -304,7 +304,7 @@ class Command(ZulipBaseCommand):react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),- irc_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),+ terminal.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),}@@ -317,7 +317,7 @@ class Command(ZulipBaseCommand):flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),- irc_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),+ terminal.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),}diff --git a/starlight_help/src/content/docs/bots-overview.mdx b/starlight_help/src/content/docs/bots-overview.mdxindex 61b1b69dc8f..2578b3e9bae 100644--- a/starlight_help/src/content/docs/bots-overview.mdx+++ b/starlight_help/src/content/docs/bots-overview.mdx@@ -86,8 +86,7 @@ support](mailto:[email protected]) or your server administrator to run the`manage.py change_user_role can_forge_sender` command to give a botpermission to send messages as users in your organization. Bots with the`can_forge_sender` permission can also see the names of all channels,-including private channels. This is important for implementing integrations-like the Jabber and IRC mirrors.+including private channels.## Related articlesdiff --git a/zerver/actions/message_send.py b/zerver/actions/message_send.pyindex 55dde1be0b5..113b2dbf3a1 100644--- a/zerver/actions/message_send.py+++ b/zerver/actions/message_send.py@@ -4,13 +4,12 @@ from collections.abc import Callable, Collection, Sequencefrom collections.abc import Set as AbstractSetfrom dataclasses import dataclassfrom datetime import timedelta-from email.headerregistry import Addressfrom typing import Any, Literal, TypedDict, castimport orjsonfrom django.conf import settingsfrom django.core.exceptions import ValidationError-from django.db import IntegrityError, transaction+from django.db import transactionfrom django.db.models import Exists, F, OuterRef, QuerySetfrom django.utils.html import escapefrom django.utils.timezone import now as timezone_now@@ -25,8 +24,6 @@ from zerver.actions.user_topics import ()from zerver.lib.addressee import Addresseefrom zerver.lib.alert_words import get_alert_word_automaton-from zerver.lib.cache import cache_with_key, user_profile_delivery_email_cache_key-from zerver.lib.create_user import create_userfrom zerver.lib.exceptions import (DirectMessageInitiationError,DirectMessagePermissionError,@@ -130,53 +127,10 @@ from zerver.models.streams import (get_realm_stream,get_stream_by_id_in_realm,)-from zerver.models.users import (- active_guest_user_ids,- get_system_bot,- get_user_by_delivery_email,- is_cross_realm_bot_email,-)+from zerver.models.users import active_guest_user_ids, get_system_bot, is_cross_realm_bot_emailfrom zerver.tornado.django_api import send_event_on_commit-def compute_irc_user_fullname(email: str) -> str:- return Address(addr_spec=email).username + " (IRC)"---def compute_jabber_user_fullname(email: str) -> str:- return Address(addr_spec=email).username + " (XMPP)"---def get_user_profile_delivery_email_cache_key(- realm: Realm, email: str, email_to_fullname: Callable[[str], str]-) -> str:- return user_profile_delivery_email_cache_key(email, realm.id)---@cache_with_key(- get_user_profile_delivery_email_cache_key,- timeout=3600 * 24 * 7,-)-def create_mirror_user_if_needed(- realm: Realm, email: str, email_to_fullname: Callable[[str], str]-) -> UserProfile:- try:- return get_user_by_delivery_email(email, realm)- except UserProfile.DoesNotExist:- try:- # Forge a user for this person- return create_user(- email=email,- password=None,- realm=realm,- full_name=email_to_fullname(email),- active=False,- is_mirror_dummy=True,- )- except IntegrityError:- return get_user_by_delivery_email(email, realm)--def render_incoming_message(message: Message,content: str,@@ -1942,11 +1896,6 @@ def check_message(elif addressee.is_private():recipient_users = addressee.user_profiles()- mirror_message = client.name in [- "irc_mirror",- "jabber_mirror",- "JabberMirror",- ]dm_involved_user_ids = {user.id for user in recipient_users} | {sender.id}@@ -1956,13 +1905,9 @@ def check_message(realm, sender, recipient_users)- # API super-users who set the `forged` flag are allowed to- # forge messages sent by any user, so we disable the- # `forwarded_mirror_message` security check in that case.- forwarded_mirror_message = mirror_message and not forgedtry:recipient = recipient_for_user_profiles(- recipient_users, forwarded_mirror_message, forwarder_user_profile, sender+ recipient_users, forwarder_user_profile, sender, forged=forged)except ValidationError as e:assert isinstance(e.messages[0], str)diff --git a/zerver/lib/drafts.py b/zerver/lib/drafts.pyindex fedc5db0a82..ea92bb9a94a 100644--- a/zerver/lib/drafts.py+++ b/zerver/lib/drafts.py@@ -86,7 +86,7 @@ def further_validated_draft_dict(# check access before calling it.check_sender_can_access_recipients(user_profile, to_users)try:- recipient_id = recipient_for_user_profiles(to_users, False, None, user_profile).id+ recipient_id = recipient_for_user_profiles(to_users, None, user_profile).idexcept ValidationError as e: # nocoverageraise JsonableError(e.messages[0])diff --git a/zerver/lib/narrow.py b/zerver/lib/narrow.pyindex 1f0d5e9588a..07807ccd491 100644--- a/zerver/lib/narrow.py+++ b/zerver/lib/narrow.py@@ -541,7 +541,6 @@ class NarrowBuilder:recipient = recipient_for_user_profiles(user_profiles=user_profiles,- forwarded_mirror_message=False,forwarder_user_profile=None,sender=self.user_profile,allow_deactivated=True,diff --git a/zerver/lib/recipient_users.py b/zerver/lib/recipient_users.pyindex 6b20c5c733e..5fa9f85eafb 100644--- a/zerver/lib/recipient_users.py+++ b/zerver/lib/recipient_users.py@@ -15,26 +15,24 @@ from zerver.models.users import is_cross_realm_bot_emaildef get_recipient_from_user_profiles(recipient_profiles: Sequence[UserProfile],- forwarded_mirror_message: bool,forwarder_user_profile: UserProfile | None,sender: UserProfile,create: bool = True,+ forged: bool = False,) -> Recipient:# Avoid mutating the passed in list of recipient_profiles.recipient_profiles_map = {user_profile.id: user_profile for user_profile in recipient_profiles}- if forwarded_mirror_message:- # In our mirroring integrations with some third-party- # protocols, bots subscribed to the third-party protocol- # forward to Zulip messages that they received in the- # third-party service. The permissions model for that- # forwarding is that users can only submit to Zulip private- # messages they personally received, and here we do the check- # for whether forwarder_user_profile is among the private- # message recipients of the message.- assert forwarder_user_profile is not None- if forwarder_user_profile.id not in recipient_profiles_map:- raise ValidationError(_("User not authorized for this query"))+ forger = forwarder_user_profile+ impersonating = forger is not None and forger.id != sender.id+ if impersonating and not (+ forged or (forger is not None and forger.id in recipient_profiles_map)+ ):+ # Impersonating another user as the sender requires the+ # can_forge_sender permission, and is only allowed within a+ # conversation the forger belongs to, or when the `forged` flag+ # is set.+ raise ValidationError(_("User not authorized for this query"))# Make sure the sender is included in the group direct messages.recipient_profiles_map[sender.id] = sender@@ -88,19 +86,19 @@ def validate_recipient_user_profiles(def recipient_for_user_profiles(user_profiles: Sequence[UserProfile],- forwarded_mirror_message: bool,forwarder_user_profile: UserProfile | None,sender: UserProfile,*,allow_deactivated: bool = False,create: bool = True,+ forged: bool = False,) -> Recipient:recipient_profiles = validate_recipient_user_profiles(user_profiles, sender, allow_deactivated=allow_deactivated)return get_recipient_from_user_profiles(- recipient_profiles, forwarded_mirror_message, forwarder_user_profile, sender, create=create+ recipient_profiles, forwarder_user_profile, sender, create=create, forged=forged)diff --git a/zerver/models/users.py b/zerver/models/users.pyindex c0329af941f..004cdc32097 100644--- a/zerver/models/users.py+++ b/zerver/models/users.py@@ -615,17 +615,16 @@ class UserProfile(AbstractBaseUser, PermissionsMixin, UserBaseSettings):last_active_message_id = models.IntegerField(null=True)# Mirror dummies are fake (!is_active) users used to provide- # message senders in cross-protocol mirroring integrations, so- # that we can display mirrored content like native Zulip messages- # (with a name + avatar, etc.). We also abuse this for data- # imports and deleted users.+ # message senders for content imported from other products and for+ # deleted users, so that we can display their messages like native+ # Zulip messages (with a name + avatar, etc.).is_mirror_dummy = models.BooleanField(default=False)# Flag used for imported users who have not activated their account.is_imported_stub = models.BooleanField(default=False)# Users with this flag set are allowed to forge messages as sent by another- # user and to send to private streams; also used for Jabber mirroring.+ # user and to send to private streams.can_forge_sender = models.BooleanField(default=False, db_index=True)# Users with this flag set can create other users via API.can_create_users = models.BooleanField(default=False, db_index=True)diff --git a/zerver/tests/test_message_send.py b/zerver/tests/test_message_send.pyindex 29f8a72541b..c28e7c0ae63 100644--- a/zerver/tests/test_message_send.py+++ b/zerver/tests/test_message_send.py@@ -80,13 +80,7 @@ from zerver.models.groups import SystemGroupsfrom zerver.models.realms import RealmTopicsPolicyEnum, get_realmfrom zerver.models.recipients import get_or_create_direct_message_groupfrom zerver.models.streams import StreamTopicsPolicyEnum, get_stream-from zerver.models.users import (- get_system_bot,- get_user,- get_user_by_delivery_email,- is_cross_realm_bot_email,-)-from zerver.views.message_send import InvalidMirrorInputError+from zerver.models.users import get_system_bot, get_user_by_delivery_email, is_cross_realm_bot_emailclass MessagePOSTTest(ZulipTestCase):@@ -1349,80 +1343,129 @@ class MessagePOSTTest(ZulipTestCase):)self.assert_json_error(result, "Message must have recipients")- def test_mirrored_direct_message_group(self) -> None:+ def test_forged_direct_message(self) -> None:"""- Sending a mirrored group direct message works+ A user with can_forge_sender can send a direct message as another+ user in their organization."""+ cordelia = self.example_user("cordelia")+ hamlet = self.example_user("hamlet")+ othello = self.example_user("othello")+ do_change_can_forge_sender(cordelia, True)+result = self.api_post(- self.mit_user("starnine"),+ cordelia,"/api/v1/messages",{"type": "direct",- "sender": self.mit_email("sipbtest"),+ "sender": hamlet.email,"content": "Test message",- "client": "irc_mirror",- "to": orjson.dumps(- [self.mit_email("starnine"), self.mit_email("espuser")]- ).decode(),+ "to": orjson.dumps([cordelia.id, othello.id]).decode(),},- subdomain="zephyr",)self.assert_json_success(result)- def test_mirrored_personal(self) -> None:- """- Sending a mirrored personal message works- """+ message = self.get_last_message()+ self.assertEqual(message.sender_id, hamlet.id)+ self.assertEqual(message.content, "Test message")++ def test_forged_direct_message_without_permission(self) -> None:+ cordelia = self.example_user("cordelia")+ hamlet = self.example_user("hamlet")+result = self.api_post(- self.mit_user("starnine"),+ cordelia,"/api/v1/messages",{"type": "direct",- "sender": self.mit_email("sipbtest"),+ "sender": hamlet.email,"content": "Test message",- "client": "irc_mirror",- "to": orjson.dumps([self.mit_email("starnine")]).decode(),+ "to": orjson.dumps([cordelia.id]).decode(),},- subdomain="zephyr",)- self.assert_json_success(result)+ self.assert_json_error(result, "User not authorized for this query")- def test_mirrored_personal_browser(self) -> None:+ def test_forged_direct_message_to_conversation_one_is_not_in(self) -> None:"""- Sending a mirrored personal message via the browser should not work.+ Forging a direct message is limited to conversations that the+ forging user is part of themselves."""- user = self.mit_user("starnine")- self.login_user(user)- result = self.client_post(- "/json/messages",+ cordelia = self.example_user("cordelia")+ hamlet = self.example_user("hamlet")+ othello = self.example_user("othello")+ do_change_can_forge_sender(cordelia, True)++ result = self.api_post(+ cordelia,+ "/api/v1/messages",{"type": "direct",- "sender": self.mit_email("sipbtest"),+ "sender": hamlet.email,"content": "Test message",- "client": "irc_mirror",- "to": orjson.dumps([self.mit_email("starnine")]).decode(),+ "to": orjson.dumps([othello.id]).decode(),},- subdomain="zephyr",)- self.assert_json_error(result, "Invalid mirrored message")+ self.assert_json_error(result, "User not authorized for this query")- def test_mirrored_personal_to_someone_else(self) -> None:+ def test_forged_direct_message_with_forged_flag(self) -> None:"""- Sending a mirrored personal message to someone else is not allowed.+ Setting the `forged` flag lifts the restriction to conversations+ that the forging user is part of themselves."""+ cordelia = self.example_user("cordelia")+ hamlet = self.example_user("hamlet")+ othello = self.example_user("othello")+ do_change_can_forge_sender(cordelia, True)+result = self.api_post(- self.mit_user("starnine"),+ cordelia,+ "/api/v1/messages",+ {+ "type": "direct",+ "sender": hamlet.email,+ "forged": "true",+ "content": "Test message",+ "to": orjson.dumps([othello.id]).decode(),+ },+ )+ self.assert_json_success(result)++ direct_message_group = get_or_create_direct_message_group([hamlet.id, othello.id])+ message = self.get_last_message()+ self.assertEqual(message.sender_id, hamlet.id)+ self.assertEqual(message.recipient_id, direct_message_group.recipient_id)++ def test_forged_sender_must_be_an_existing_user(self) -> None:+ cordelia = self.example_user("cordelia")+ do_change_can_forge_sender(cordelia, True)++ result = self.api_post(+ cordelia,+ "/api/v1/messages",+ {+ "type": "direct",+ "sender": "[email protected]",+ "content": "Test message",+ "to": orjson.dumps([cordelia.id]).decode(),+ },+ )+ self.assert_json_error(result, "No such user")++ def test_forged_sender_cannot_be_in_another_organization(self) -> None:+ cordelia = self.example_user("cordelia")+ do_change_can_forge_sender(cordelia, True)++ result = self.api_post(+ cordelia,"/api/v1/messages",{"type": "direct","sender": self.mit_email("sipbtest"),"content": "Test message",- "client": "irc_mirror",- "to": orjson.dumps([self.mit_email("espuser")]).decode(),+ "to": orjson.dumps([cordelia.id]).decode(),},- subdomain="zephyr",)- self.assert_json_error(result, "User not authorized for this query")+ self.assert_json_error(result, "No such user")def test_message_with_null_bytes(self) -> None:"""@@ -1524,137 +1567,36 @@ class MessagePOSTTest(ZulipTestCase):)self.assert_json_error(result, "User not authorized for this query")- def test_send_message_when_sender_is_not_set(self) -> None:- result = self.api_post(- self.mit_user("starnine"),- "/api/v1/messages",- {- "type": "direct",- "content": "Test message",- "client": "irc_mirror",- "to": orjson.dumps([self.mit_email("starnine")]).decode(),- },- subdomain="zephyr",- )- self.assert_json_error(result, "Missing sender")-- def test_send_message_as_not_superuser_when_type_is_not_private(self) -> None:- result = self.api_post(- self.mit_user("starnine"),- "/api/v1/messages",- {- "type": "channel",- "sender": self.mit_email("sipbtest"),- "content": "Test message",- "client": "irc_mirror",- "to": orjson.dumps([self.mit_email("starnine")]).decode(),- },- subdomain="zephyr",- )- self.assert_json_error(result, "User not authorized for this query")-- @mock.patch("zerver.views.message_send.create_mirrored_message_users")- def test_send_message_create_mirrored_message_user_returns_invalid_input(- self, create_mirrored_message_users_mock: Any- ) -> None:- create_mirrored_message_users_mock.side_effect = InvalidMirrorInputError()- result = self.api_post(- self.mit_user("starnine"),- "/api/v1/messages",- {- "type": "direct",- "sender": self.mit_email("sipbtest"),- "content": "Test message",- "client": "irc_mirror",- "to": orjson.dumps([self.mit_email("starnine")]).decode(),- },- subdomain="zephyr",- )- self.assert_json_error(result, "Invalid mirrored message")-- @mock.patch("zerver.views.message_send.create_mirrored_message_users")- def test_send_message_when_client_is_mirror_but_recipient_is_user_id(- self, create_mirrored_message_users_mock: Any- ) -> None:- create_mirrored_message_users_mock.return_value = mock.Mock()- user = self.mit_user("starnine")- self.login_user(user)- result = self.api_post(- user,- "/api/v1/messages",- {- "type": "direct",- "sender": self.mit_email("sipbtest"),- "content": "Test message",- "client": "irc_mirror",- "to": orjson.dumps([user.id]).decode(),- },- subdomain="zephyr",- )- self.assert_json_error(result, "Mirroring not allowed with recipient user IDs")-- def test_send_message_irc_mirror(self) -> None:- reset_email_visibility_to_everyone_in_zulip_realm()- self.login("hamlet")- bot_info = {- "full_name": "IRC bot",- "short_name": "irc",- }- result = self.client_post("/json/bots", bot_info)- self.assert_json_success(result)-- email = "[email protected]"- user = get_user(email, get_realm("zulip"))- user.can_forge_sender = True- user.save()- user = get_user(email, get_realm("zulip"))- self.subscribe(user, "IRCland")-- # Simulate a mirrored message with a slightly old timestamp.- fake_date_sent = timezone_now() - timedelta(minutes=37)- fake_timestamp = datetime_to_timestamp(fake_date_sent)-- result = self.api_post(- user,- "/api/v1/messages",- {- "type": "channel",- "forged": "true",- "time": fake_timestamp,- "sender": "[email protected]",- "content": "Test message",- "client": "irc_mirror",- "topic": "from irc",- "to": orjson.dumps("IRCLand").decode(),- },- )- self.assert_json_success(result)-- msg = self.get_last_message()- self.assertEqual(int(datetime_to_timestamp(msg.date_sent)), int(fake_timestamp))+ def test_forged_channel_message(self) -> None:+ hamlet = self.example_user("hamlet")+ bot = self.create_test_bot(short_name="forger", user_profile=self.example_user("cordelia"))+ do_change_can_forge_sender(bot, True)+ self.subscribe(bot, "Verona")- # Now test again using forged=yes- fake_date_sent = timezone_now() - timedelta(minutes=22)- fake_timestamp = datetime_to_timestamp(fake_date_sent)+ # A forged message can carry the timestamp of the message that+ # it was copied from, in both the modern and legacy spellings.+ for forged_value in ["true", "yes"]:+ fake_date_sent = timezone_now() - timedelta(minutes=37)+ fake_timestamp = datetime_to_timestamp(fake_date_sent)- result = self.api_post(- user,- "/api/v1/messages",- {- "type": "channel",- "forged": "yes",- "time": fake_timestamp,- "sender": "[email protected]",- "content": "Test message",- "client": "irc_mirror",- "topic": "from irc",- "to": orjson.dumps("IRCLand").decode(),- },- )- self.assert_json_success(result)+ result = self.api_post(+ bot,+ "/api/v1/messages",+ {+ "type": "channel",+ "forged": forged_value,+ "time": fake_timestamp,+ "sender": hamlet.email,+ "content": "Test message",+ "topic": "forged",+ "to": orjson.dumps("Verona").decode(),+ },+ )+ self.assert_json_success(result)- msg = self.get_last_message()- self.assertEqual(int(datetime_to_timestamp(msg.date_sent)), int(fake_timestamp))+ message = self.get_last_message()+ self.assertEqual(message.sender_id, hamlet.id)+ self.assertEqual(int(datetime_to_timestamp(message.date_sent)), int(fake_timestamp))def test_send_message_in_archived_stream(self) -> None:self.login("hamlet")@@ -1693,9 +1635,9 @@ class MessagePOSTTest(ZulipTestCase):self.unsubscribe(cordelia, stream_name)- # As long as Cordelia cam_forge_sender, she can send messages- # to ANY stream, even one she is not unsubscribed to, and- # she can do it for herself or on behalf of a mirrored user.+ # As long as Cordelia can_forge_sender, she can send messages+ # to ANY stream, even one she is not subscribed to, and she+ # can do it for herself or on behalf of another user.def test_with(sender_email: str, client: str, forged: bool) -> None:payload = dict(@@ -1707,7 +1649,7 @@ class MessagePOSTTest(ZulipTestCase):forged=orjson.dumps(forged).decode(),)- # Only pass the 'sender' property when doing mirroring behavior.+ # Only pass the 'sender' property when forging a sender.if forged:payload["sender"] = sender_email@@ -1730,8 +1672,8 @@ class MessagePOSTTest(ZulipTestCase):)test_with(- sender_email="[email protected]",- client="irc_mirror",+ sender_email=self.example_user("hamlet").email,+ client="test suite",forged=True,)@@ -2770,7 +2712,7 @@ class StreamMessagesTest(ZulipTestCase):result = self.api_get(cordelia, "/api/v1/messages/" + str(msg_id))self.assert_json_success(result)- def test_stream_message_mirroring(self) -> None:+ def test_stream_message_forging(self) -> None:user = self.mit_user("starnine")self.subscribe(user, "Verona")@@ -2782,7 +2724,6 @@ class StreamMessagesTest(ZulipTestCase):"type": "channel","to": orjson.dumps("Verona").decode(),"sender": self.mit_email("sipbtest"),- "client": "irc_mirror","topic": "announcement","content": "Everyone knows Iago rules","forged": "true",@@ -2799,7 +2740,6 @@ class StreamMessagesTest(ZulipTestCase):"type": "channel","to": orjson.dumps("Verona").decode(),"sender": self.mit_email("sipbtest"),- "client": "irc_mirror","topic": "announcement","content": "Everyone knows Iago rules","forged": "true",diff --git a/zerver/tests/test_mirror_users.py b/zerver/tests/test_mirror_users.pydeleted file mode 100644index d0453a65c95..00000000000--- a/zerver/tests/test_mirror_users.py+++ /dev/null@@ -1,184 +0,0 @@-from typing import Any-from unittest import mock--from django.db import IntegrityError-from django.utils.timezone import now as timezone_now--from zerver.actions.message_send import create_mirror_user_if_needed-from zerver.lib.create_user import create_user_profile-from zerver.lib.test_classes import ZulipTestCase-from zerver.lib.test_helpers import reset_email_visibility_to_everyone_in_zulip_realm-from zerver.models import UserProfile-from zerver.models.clients import get_client-from zerver.models.realms import get_realm-from zerver.models.users import get_user-from zerver.views.message_send import InvalidMirrorInputError, create_mirrored_message_users---class MirroredMessageUsersTest(ZulipTestCase):- def test_invalid_client(self) -> None:- user = self.example_user("hamlet")- sender = user-- recipients: list[str] = []-- recipient_type_name = "private"- client = get_client("banned_mirror")-- with self.assertRaises(InvalidMirrorInputError):- create_mirrored_message_users(- client, user, recipients, sender.email, recipient_type_name- )-- def test_invalid_email(self) -> None:- invalid_email = "alice AT example.com"- recipients = [invalid_email]-- # We use an MIT user here to maximize code coverage- user = self.mit_user("starnine")- sender = user-- recipient_type_name = "private"-- for client_name in ["irc_mirror", "jabber_mirror"]:- client = get_client(client_name)-- with self.assertRaises(InvalidMirrorInputError):- create_mirrored_message_users(- client, user, recipients, sender.email, recipient_type_name- )-- def test_mirror_new_recipient(self) -> None:- """Test mirror dummy user creation for direct message recipients"""- user = self.mit_user("starnine")- sender = self.mit_user("sipbtest")- new_user_email = "[email protected]"- new_user_realm = get_realm("zephyr")-- recipients = [user.email, new_user_email]-- recipient_type_name = "private"- client = get_client("irc_mirror")-- mirror_sender = create_mirrored_message_users(- client, user, recipients, sender.email, recipient_type_name- )-- self.assertEqual(mirror_sender, sender)-- realm_users = UserProfile.objects.filter(realm=sender.realm)- realm_emails = {user.email for user in realm_users}- self.assertIn(user.email, realm_emails)- self.assertIn(new_user_email, realm_emails)-- bob = get_user(new_user_email, new_user_realm)- self.assertTrue(bob.is_mirror_dummy)-- def test_mirror_new_sender(self) -> None:- """Test mirror dummy user creation for sender when sending to stream"""- user = self.mit_user("starnine")- sender_email = "[email protected]"-- recipients = ["stream_name"]-- recipient_type_name = "stream"- client = get_client("irc_mirror")-- mirror_sender = create_mirrored_message_users(- client, user, recipients, sender_email, recipient_type_name- )-- assert mirror_sender is not None- self.assertEqual(mirror_sender.email, sender_email)- self.assertTrue(mirror_sender.is_mirror_dummy)-- def test_irc_mirror(self) -> None:- reset_email_visibility_to_everyone_in_zulip_realm()-- user = self.example_user("hamlet")- sender = user-- recipients = [- self.nonreg_email("alice"),- "[email protected]",- self.nonreg_email("cordelia"),- ]-- recipient_type_name = "private"- client = get_client("irc_mirror")-- mirror_sender = create_mirrored_message_users(- client, user, recipients, sender.email, recipient_type_name- )-- self.assertEqual(mirror_sender, sender)-- realm_users = UserProfile.objects.filter(realm=sender.realm)- realm_emails = {user.email for user in realm_users}- self.assertIn(self.nonreg_email("alice"), realm_emails)- self.assertIn("[email protected]", realm_emails)-- bob = get_user("[email protected]", sender.realm)- self.assertTrue(bob.is_mirror_dummy)-- def test_jabber_mirror(self) -> None:- reset_email_visibility_to_everyone_in_zulip_realm()-- user = self.example_user("hamlet")- sender = user-- recipients = [- self.nonreg_email("alice"),- self.nonreg_email("bob"),- self.nonreg_email("cordelia"),- ]-- recipient_type_name = "private"- client = get_client("jabber_mirror")-- mirror_sender = create_mirrored_message_users(- client, user, recipients, sender.email, recipient_type_name- )-- self.assertEqual(mirror_sender, sender)-- realm_users = UserProfile.objects.filter(realm=sender.realm)- realm_emails = {user.email for user in realm_users}- self.assertIn(self.nonreg_email("alice"), realm_emails)- self.assertIn(self.nonreg_email("bob"), realm_emails)-- bob = get_user(self.nonreg_email("bob"), sender.realm)- self.assertTrue(bob.is_mirror_dummy)-- def test_create_mirror_user_despite_race(self) -> None:- realm = get_realm("zulip")-- email = "[email protected]"-- email_to_full_name = lambda email: "fred"-- def create_user(**kwargs: Any) -> UserProfile:- self.assertEqual(kwargs["full_name"], "fred")- self.assertEqual(kwargs["email"], email)- self.assertEqual(kwargs["active"], False)- self.assertEqual(kwargs["is_mirror_dummy"], True)- # We create an actual user here to simulate a race.- # We use the minimal, un-mocked function.- kwargs["bot_type"] = None- kwargs["bot_owner"] = None- kwargs["tos_version"] = None- kwargs["timezone"] = timezone_now()- kwargs["default_language"] = "en"- kwargs["email_address_visibility"] = UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE- create_user_profile(**kwargs).save()- raise IntegrityError-- with mock.patch("zerver.actions.message_send.create_user", side_effect=create_user) as m:- mirror_fred_user = create_mirror_user_if_needed(- realm,- email,- email_to_full_name,- )-- self.assertEqual(mirror_fred_user.delivery_email, email)- m.assert_called()diff --git a/zerver/views/message_send.py b/zerver/views/message_send.pyindex e15c3043420..81b698cdd4a 100644--- a/zerver/views/message_send.py+++ b/zerver/views/message_send.py@@ -1,20 +1,14 @@-from collections.abc import Iterable, Sequence+from collections.abc import Sequencefrom dataclasses import asdict-from email.headerregistry import Address-from typing import Annotated, Literal, TypedDict, cast+from typing import Annotated, Literal, TypedDictfrom django.conf import settings-from django.core import validators-from django.core.exceptions import ValidationErrorfrom django.http import HttpRequest, HttpResponsefrom django.utils.translation import gettext as _from pydantic import Json, StringConstraintsfrom zerver.actions.message_send import (check_send_message,- compute_irc_user_fullname,- compute_jabber_user_fullname,- create_mirror_user_if_needed,extract_private_recipients,extract_stream_indicator,)@@ -29,7 +23,7 @@ from zerver.lib.typed_endpoint import (typed_endpoint,)from zerver.lib.zcommand import process_zcommands-from zerver.models import Client, Message, RealmDomain, UserProfile+from zerver.models import Message, UserProfilefrom zerver.models.users import get_user_including_cross_realm@@ -40,76 +34,6 @@ class SendMessageResponseData(TypedDict, total=False):automatic_new_visibility_policy: int-class InvalidMirrorInputError(Exception):- pass---def create_mirrored_message_users(- client: Client,- user_profile: UserProfile,- recipients: Iterable[str],- sender: str,- recipient_type_name: str,-) -> UserProfile:- sender_email = sender.strip().lower()- referenced_users = {sender_email}- if recipient_type_name == "private":- referenced_users.update(email.lower() for email in recipients)-- if client.name == "irc_mirror":- user_check = same_realm_irc_user- fullname_function = compute_irc_user_fullname- elif client.name in ("jabber_mirror", "JabberMirror"):- user_check = same_realm_jabber_user- fullname_function = compute_jabber_user_fullname- else:- raise InvalidMirrorInputError("Unrecognized mirroring client")-- for email in referenced_users:- # Check that all referenced users are in our realm:- if not user_check(user_profile, email):- raise InvalidMirrorInputError("At least one user cannot be mirrored")-- # Create users for the referenced users, if needed.- for email in referenced_users:- create_mirror_user_if_needed(user_profile.realm, email, fullname_function)-- sender_user_profile = get_user_including_cross_realm(sender_email, user_profile.realm)- return sender_user_profile---def same_realm_irc_user(user_profile: UserProfile, email: str) -> bool:- # Check whether the target email address is an IRC user in the- # same realm as user_profile, i.e. if the domain were example.com,- # the IRC user would need to be [email protected]- try:- validators.validate_email(email)- except ValidationError:- return False-- domain = Address(addr_spec=email).domain.lower()- domain = domain.removeprefix("irc.")-- # Assumes allow_subdomains=False for all RealmDomain's corresponding to- # these realms.- return RealmDomain.objects.filter(realm=user_profile.realm, domain=domain).exists()---def same_realm_jabber_user(user_profile: UserProfile, email: str) -> bool:- try:- validators.validate_email(email)- except ValidationError:- return False-- # If your Jabber users have a different email domain than the- # Zulip users, this is where you would do any translation.- domain = Address(addr_spec=email).domain.lower()-- # Assumes allow_subdomains=False for all RealmDomain's corresponding to- # these realms.- return RealmDomain.objects.filter(realm=user_profile.realm, domain=domain).exists()--@typed_endpointdef send_message_backend(request: HttpRequest,@@ -180,46 +104,22 @@ def send_message_backend(realm = user_profile.realm- if client.name in ["irc_mirror", "jabber_mirror", "JabberMirror"]:- # Here's how security works for mirroring:- #- # For direct messages, the message must be (1) both sent and- # received exclusively by users in your realm, and (2)- # received by the forwarding user.- #- # For stream messages, the message must be (1) being forwarded- # by an API superuser for your realm and (2) being sent to a- # mirrored stream.- #- # The most important security checks are in- # `create_mirrored_message_users` below, which checks the- # same-realm constraint.- if req_sender is None:- raise JsonableError(_("Missing sender"))- if recipient_type_name != "private" and not can_forge_sender:+ if req_sender is not None:+ # Forging the sender of a message is gated on the+ # `can_forge_sender` permission, which a server administrator+ # grants with `manage.py change_user_role`. The forged sender+ # must be an existing user in the forging user's own+ # organization; the further access checks for the forged+ # message live in `access_stream_for_send_message` and+ # `get_recipient_from_user_profiles`.+ if not can_forge_sender:raise JsonableError(_("User not authorized for this query"))- # For now, mirroring only works with recipient emails, not for- # recipient user IDs.- if not all(isinstance(to_item, str) for to_item in message_to):- raise JsonableError(_("Mirroring not allowed with recipient user IDs"))-- # We need this manual cast so that mypy doesn't complain about- # create_mirrored_message_users not being able to accept a Sequence[int]- # type parameter.- message_to = cast(Sequence[str], message_to)-try:- mirror_sender = create_mirrored_message_users(- client, user_profile, message_to, req_sender, recipient_type_name- )- except InvalidMirrorInputError:- raise JsonableError(_("Invalid mirrored message"))-- sender = mirror_sender+ sender = get_user_including_cross_realm(req_sender.strip().lower(), realm)+ except UserProfile.DoesNotExist:+ raise JsonableError(_("No such user"))else:- if req_sender is not None:- raise JsonableError(_("Invalid mirrored message"))sender = user_profileif read_by_sender is None: