Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
push_notifications: Retry APNs sends on a connection closed by APNs.
Prakhar Pratyushce18dedb2e9e
diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.pyindex c821d7bf246..7ea0abea628 100644--- a/zerver/lib/push_notifications.py+++ b/zerver/lib/push_notifications.py@@ -172,6 +172,38 @@ class APNsContext:apns: "aioapns.APNs"loop: asyncio.AbstractEventLoop+ async def send_notification(self, request: "aioapns.NotificationRequest") -> NotificationResult:+ # TODO: Remove once aioapns stops using closed connections:+ # https://github.com/Fatal1ty/aioapns/issues/77+ from h2.exceptions import ProtocolError++ for _attempt in range(APNS_MAX_RETRIES - 1):+ try:+ return await self.apns.send_notification(request)+ except ProtocolError as exc:+ # If aioapns tried to use a connection that APNs had already closed,+ # nothing reached APNs. aioapns keeps closed connections in its pool+ # until their TLS shutdown completes, so discard them before retrying.+ if (+ str(exc)+ != "Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED"+ ):+ raise+ self.discard_closed_connections()+ return await self.apns.send_notification(request)++ def discard_closed_connections(self) -> None:+ from h2.connection import ConnectionState++ pool = self.apns.pool+ for connection in list(pool.connections):+ if connection.conn.state_machine.state is ConnectionState.CLOSED:+ # The transport still calls connection_lost once its TLS shutdown+ # completes; that would try to discard the connection a second time,+ # raising ValueError. Set on_connection_lost to None to prevent that.+ connection.on_connection_lost = None+ pool.discard_connection(connection)+def has_apns_credentials() -> bool:return settings.APNS_TOKEN_KEY_FILE is not None or settings.APNS_CERT_FILE is not None@@ -352,7 +384,7 @@ def send_apple_push_notification()try:results[device] = apns_context.loop.run_until_complete(- apns_context.apns.send_notification(request)+ apns_context.send_notification(request))except BaseException as e:results[device] = ediff --git a/zerver/tests/test_e2ee_push_notifications.py b/zerver/tests/test_e2ee_push_notifications.pyindex b57fe194c12..a2e85299a73 100644--- a/zerver/tests/test_e2ee_push_notifications.py+++ b/zerver/tests/test_e2ee_push_notifications.py@@ -8,6 +8,7 @@ from django.test import override_settingsfrom django.utils.timezone import nowfrom firebase_admin.exceptions import InternalErrorfrom firebase_admin.messaging import UnregisteredError+from h2.exceptions import ProtocolErrorfrom analytics.models import RealmCountfrom corporate.lib.stripe import BillingUserCounts@@ -17,6 +18,7 @@ from zerver.lib.avatar import absolute_avatar_urlfrom zerver.lib.devices import b64encode_token_id_intfrom zerver.lib.exceptions import MissingRemoteRealmErrorfrom zerver.lib.push_notifications import (+ APNsContext,PushNotificationsDisallowedByBouncerError,handle_push_notification,handle_remove_push_notification,@@ -271,6 +273,54 @@ class SendPushNotificationTest(E2EEPushNotificationTestCase):self.assertIsNotNone(registered_device_android.expired_time)self.assertEqual(Device.objects.filter(push_token_id__isnull=False).count(), 0)+ def test_retry_after_connection_closed_by_apns(self) -> None:+ aaron = self.example_user("aaron")+ hamlet = self.example_user("hamlet")++ registered_device_apple, _registered_device_android = (+ self.register_push_devices_for_notification()+ )++ message_id = self.send_personal_message(+ from_user=aaron, to_user=hamlet, skip_capture_on_commit_callbacks=True+ )+ missed_message = {+ "message_id": message_id,+ "trigger": NotificationTriggers.DIRECT_MESSAGE,+ }++ with (+ self.mock_fcm() as mock_fcm_messaging,+ self.mock_apns() as send_notification,+ mock.patch.object(APNsContext, "discard_closed_connections") as discard,+ self.assertLogs("zerver.lib.push_notifications", level="INFO") as zerver_logger,+ self.assertLogs("zilencer.lib.push_notifications", level="INFO"),+ mock.patch("time.perf_counter", side_effect=[10.0, 15.0]),+ ):+ mock_fcm_messaging.send_each.return_value = self.make_fcm_success_response()+ send_notification.side_effect = [+ ProtocolError(+ "Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED"+ ),+ send_notification.return_value,+ ]+ send_notification.return_value.is_successful = True++ handle_push_notification(hamlet.id, missed_message)++ self.assertEqual(send_notification.await_count, 2)+ discard.assert_called_once()+ self.assertEqual(+ "INFO:zerver.lib.push_notifications:"+ f"APNs: Success sending to (realm={hamlet.realm.uuid}, device={registered_device_apple.token})",+ zerver_logger.output[2],+ )+ self.assertEqual(+ "INFO:zerver.lib.push_notifications:"+ f"Sent E2EE mobile push notifications for user {hamlet.id}: 1 via FCM, 1 via APNs in 5.000s",+ zerver_logger.output[3],+ )+def test_fcm_apns_error(self) -> None:hamlet = self.example_user("hamlet")aaron = self.example_user("aaron")diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.pyindex 268be51fadc..7426caf6854 100644--- a/zerver/tests/test_push_notifications.py+++ b/zerver/tests/test_push_notifications.py@@ -11,6 +11,7 @@ import orjsonimport requestsimport responsesimport time_machine+from aioapns.connection import APNsBaseConnectionPool, APNsProductionClientProtocolfrom django.conf import settingsfrom django.db.models import F, Qfrom django.test import override_settings@@ -18,6 +19,7 @@ from django.utils.crypto import get_random_stringfrom django.utils.timezone import nowfrom dns.resolver import NoAnswer as DNSNoAnswerfrom firebase_admin import exceptions as firebase_exceptions+from h2.exceptions import ProtocolErrorfrom requests.exceptions import ConnectionErrorfrom typing_extensions import override@@ -29,6 +31,7 @@ from zerver.actions.user_settings import do_regenerate_api_keyfrom zerver.lib.avatar import absolute_avatar_url, get_avatar_for_inaccessible_userfrom zerver.lib.exceptions import JsonableErrorfrom zerver.lib.push_notifications import (+ APNS_MAX_RETRIES,DeviceToken,InvalidRemotePushDeviceTokenError,UserPushIdentityCompat,@@ -1376,6 +1379,89 @@ class TestAPNs(PushNotificationTestCase):logger.output,)+ def test_retry_after_connection_closed_by_apns(self) -> None:+ self.setup_apns_tokens()+ with self.mock_apns() as (apns_context, send_notification):++ async def make_pool() -> APNsBaseConnectionPool:+ return APNsBaseConnectionPool(topic="org.zulip.Zulip")++ pool = apns_context.loop.run_until_complete(make_pool())+ apns_context.apns.pool = pool+ closed_connection = APNsProductionClientProtocol(+ "org.zulip.Zulip",+ loop=apns_context.loop,+ on_connection_lost=mock.Mock(side_effect=pool.discard_connection),+ )+ open_connection = APNsProductionClientProtocol(+ "org.zulip.Zulip", loop=apns_context.loop+ )+ pool.connections = [closed_connection, open_connection]++ # Get the exact error h2 raises when sending on a connection+ # that APNs has closed with GOAWAY.+ closed_connection.conn.close_connection()+ with self.assertRaises(ProtocolError) as protocol_error:+ closed_connection.conn.send_headers(1, [])++ send_notification.side_effect = [+ protocol_error.exception,+ send_notification.return_value,+ ]+ send_notification.return_value.is_successful = True+ with (+ self.assertLogs("aioapns", level="INFO") as aioapns_logger,+ self.assertLogs("zerver.lib.push_notifications", level="INFO") as logger,+ ):+ self.send(devices=self.devices()[0:1])++ self.assertEqual(send_notification.await_count, 2)+ self.assertEqual(pool.connections, [open_connection])+ self.assertEqual(aioapns_logger.output, ["INFO:aioapns:Connection released (total: 1)"])+ self.assertEqual(+ f"INFO:zerver.lib.push_notifications:APNs: Success sending for user <id:{self.user_profile.id}> to device {self.devices()[0].token}",+ logger.output[1],+ )++ # When the transport finishes closing later, aioapns must not+ # try to discard the connection from the pool again.+ closed_connection.connection_lost(None)+ self.assertEqual(pool.connections, [open_connection])++ def test_closed_connection_retry_limit(self) -> None:+ self.setup_apns_tokens()+ with (+ self.mock_apns() as (apns_context, send_notification),+ mock.patch.object(apns_context, "discard_closed_connections") as discard,+ self.assertLogs("zerver.lib.push_notifications", level="INFO") as logger,+ ):+ send_notification.side_effect = ProtocolError(+ "Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED"+ )+ self.send(devices=self.devices()[0:1])+ self.assertEqual(send_notification.await_count, APNS_MAX_RETRIES)+ self.assertEqual(discard.call_count, APNS_MAX_RETRIES - 1)+ self.assertIn(+ f"ERROR:zerver.lib.push_notifications:APNs: Error sending for user <id:{self.user_profile.id}> to device {self.devices()[0].token}",+ logger.output[1],+ )++ def test_other_protocol_error_not_retried(self) -> None:+ self.setup_apns_tokens()+ with (+ self.mock_apns() as (apns_context, send_notification),+ mock.patch.object(apns_context, "discard_closed_connections") as discard,+ self.assertLogs("zerver.lib.push_notifications", level="INFO") as logger,+ ):+ send_notification.side_effect = ProtocolError("Invalid header")+ self.send(devices=self.devices()[0:1])+ send_notification.assert_awaited_once()+ discard.assert_not_called()+ self.assertIn(+ f"ERROR:zerver.lib.push_notifications:APNs: Error sending for user <id:{self.user_profile.id}> to device {self.devices()[0].token}",+ logger.output[1],+ )+def test_other_exception(self) -> None:self.setup_apns_tokens()with (diff --git a/zilencer/lib/push_notifications.py b/zilencer/lib/push_notifications.pyindex d244df2333f..76421c2621d 100644--- a/zilencer/lib/push_notifications.py+++ b/zilencer/lib/push_notifications.py@@ -55,7 +55,7 @@ def send_e2ee_push_notification_apple(tuple[RemotePushDevice, aioapns.common.NotificationResult | BaseException]]:results = await asyncio.gather(- *(apns_context.apns.send_notification(request) for request in apns_requests),+ *(apns_context.send_notification(request) for request in apns_requests),return_exceptions=True,)return zip(apns_remote_push_devices, results, strict=False)