Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

push_notifications: Retry APNs sends on a connection closed by APNs.

Prakhar Pratyushce18dedb2e9e
diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py
index c821d7bf246..7ea0abea628 100644
--- a/zerver/lib/push_notifications.py
+++ b/zerver/lib/push_notifications.py
@@ -172,6 +172,38 @@ class APNsContext:
apns: "aioapns.APNs"
loop: asyncio.AbstractEventLoop
+ async def send_notification(self, request: "aioapns.NotificationRequest") -> NotificationResult:
+ # TODO: Remove once aioapns stops using closed connections:
+ # https://github.com/Fatal1ty/aioapns/issues/77
+ from h2.exceptions import ProtocolError
+
+ for _attempt in range(APNS_MAX_RETRIES - 1):
+ try:
+ return await self.apns.send_notification(request)
+ except ProtocolError as exc:
+ # If aioapns tried to use a connection that APNs had already closed,
+ # nothing reached APNs. aioapns keeps closed connections in its pool
+ # until their TLS shutdown completes, so discard them before retrying.
+ if (
+ str(exc)
+ != "Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED"
+ ):
+ raise
+ self.discard_closed_connections()
+ return await self.apns.send_notification(request)
+
+ def discard_closed_connections(self) -> None:
+ from h2.connection import ConnectionState
+
+ pool = self.apns.pool
+ for connection in list(pool.connections):
+ if connection.conn.state_machine.state is ConnectionState.CLOSED:
+ # The transport still calls connection_lost once its TLS shutdown
+ # completes; that would try to discard the connection a second time,
+ # raising ValueError. Set on_connection_lost to None to prevent that.
+ connection.on_connection_lost = None
+ pool.discard_connection(connection)
+
def has_apns_credentials() -> bool:
return settings.APNS_TOKEN_KEY_FILE is not None or settings.APNS_CERT_FILE is not None
@@ -352,7 +384,7 @@ def send_apple_push_notification(
)
try:
results[device] = apns_context.loop.run_until_complete(
- apns_context.apns.send_notification(request)
+ apns_context.send_notification(request)
)
except BaseException as e:
results[device] = e
diff --git a/zerver/tests/test_e2ee_push_notifications.py b/zerver/tests/test_e2ee_push_notifications.py
index b57fe194c12..a2e85299a73 100644
--- a/zerver/tests/test_e2ee_push_notifications.py
+++ b/zerver/tests/test_e2ee_push_notifications.py
@@ -8,6 +8,7 @@ from django.test import override_settings
from django.utils.timezone import now
from firebase_admin.exceptions import InternalError
from firebase_admin.messaging import UnregisteredError
+from h2.exceptions import ProtocolError
from analytics.models import RealmCount
from corporate.lib.stripe import BillingUserCounts
@@ -17,6 +18,7 @@ from zerver.lib.avatar import absolute_avatar_url
from zerver.lib.devices import b64encode_token_id_int
from zerver.lib.exceptions import MissingRemoteRealmError
from zerver.lib.push_notifications import (
+ APNsContext,
PushNotificationsDisallowedByBouncerError,
handle_push_notification,
handle_remove_push_notification,
@@ -271,6 +273,54 @@ class SendPushNotificationTest(E2EEPushNotificationTestCase):
self.assertIsNotNone(registered_device_android.expired_time)
self.assertEqual(Device.objects.filter(push_token_id__isnull=False).count(), 0)
+ def test_retry_after_connection_closed_by_apns(self) -> None:
+ aaron = self.example_user("aaron")
+ hamlet = self.example_user("hamlet")
+
+ registered_device_apple, _registered_device_android = (
+ self.register_push_devices_for_notification()
+ )
+
+ message_id = self.send_personal_message(
+ from_user=aaron, to_user=hamlet, skip_capture_on_commit_callbacks=True
+ )
+ missed_message = {
+ "message_id": message_id,
+ "trigger": NotificationTriggers.DIRECT_MESSAGE,
+ }
+
+ with (
+ self.mock_fcm() as mock_fcm_messaging,
+ self.mock_apns() as send_notification,
+ mock.patch.object(APNsContext, "discard_closed_connections") as discard,
+ self.assertLogs("zerver.lib.push_notifications", level="INFO") as zerver_logger,
+ self.assertLogs("zilencer.lib.push_notifications", level="INFO"),
+ mock.patch("time.perf_counter", side_effect=[10.0, 15.0]),
+ ):
+ mock_fcm_messaging.send_each.return_value = self.make_fcm_success_response()
+ send_notification.side_effect = [
+ ProtocolError(
+ "Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED"
+ ),
+ send_notification.return_value,
+ ]
+ send_notification.return_value.is_successful = True
+
+ handle_push_notification(hamlet.id, missed_message)
+
+ self.assertEqual(send_notification.await_count, 2)
+ discard.assert_called_once()
+ self.assertEqual(
+ "INFO:zerver.lib.push_notifications:"
+ f"APNs: Success sending to (realm={hamlet.realm.uuid}, device={registered_device_apple.token})",
+ zerver_logger.output[2],
+ )
+ self.assertEqual(
+ "INFO:zerver.lib.push_notifications:"
+ f"Sent E2EE mobile push notifications for user {hamlet.id}: 1 via FCM, 1 via APNs in 5.000s",
+ zerver_logger.output[3],
+ )
+
def test_fcm_apns_error(self) -> None:
hamlet = self.example_user("hamlet")
aaron = self.example_user("aaron")
diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py
index 268be51fadc..7426caf6854 100644
--- a/zerver/tests/test_push_notifications.py
+++ b/zerver/tests/test_push_notifications.py
@@ -11,6 +11,7 @@ import orjson
import requests
import responses
import time_machine
+from aioapns.connection import APNsBaseConnectionPool, APNsProductionClientProtocol
from django.conf import settings
from django.db.models import F, Q
from django.test import override_settings
@@ -18,6 +19,7 @@ from django.utils.crypto import get_random_string
from django.utils.timezone import now
from dns.resolver import NoAnswer as DNSNoAnswer
from firebase_admin import exceptions as firebase_exceptions
+from h2.exceptions import ProtocolError
from requests.exceptions import ConnectionError
from typing_extensions import override
@@ -29,6 +31,7 @@ from zerver.actions.user_settings import do_regenerate_api_key
from zerver.lib.avatar import absolute_avatar_url, get_avatar_for_inaccessible_user
from zerver.lib.exceptions import JsonableError
from zerver.lib.push_notifications import (
+ APNS_MAX_RETRIES,
DeviceToken,
InvalidRemotePushDeviceTokenError,
UserPushIdentityCompat,
@@ -1376,6 +1379,89 @@ class TestAPNs(PushNotificationTestCase):
logger.output,
)
+ def test_retry_after_connection_closed_by_apns(self) -> None:
+ self.setup_apns_tokens()
+ with self.mock_apns() as (apns_context, send_notification):
+
+ async def make_pool() -> APNsBaseConnectionPool:
+ return APNsBaseConnectionPool(topic="org.zulip.Zulip")
+
+ pool = apns_context.loop.run_until_complete(make_pool())
+ apns_context.apns.pool = pool
+ closed_connection = APNsProductionClientProtocol(
+ "org.zulip.Zulip",
+ loop=apns_context.loop,
+ on_connection_lost=mock.Mock(side_effect=pool.discard_connection),
+ )
+ open_connection = APNsProductionClientProtocol(
+ "org.zulip.Zulip", loop=apns_context.loop
+ )
+ pool.connections = [closed_connection, open_connection]
+
+ # Get the exact error h2 raises when sending on a connection
+ # that APNs has closed with GOAWAY.
+ closed_connection.conn.close_connection()
+ with self.assertRaises(ProtocolError) as protocol_error:
+ closed_connection.conn.send_headers(1, [])
+
+ send_notification.side_effect = [
+ protocol_error.exception,
+ send_notification.return_value,
+ ]
+ send_notification.return_value.is_successful = True
+ with (
+ self.assertLogs("aioapns", level="INFO") as aioapns_logger,
+ self.assertLogs("zerver.lib.push_notifications", level="INFO") as logger,
+ ):
+ self.send(devices=self.devices()[0:1])
+
+ self.assertEqual(send_notification.await_count, 2)
+ self.assertEqual(pool.connections, [open_connection])
+ self.assertEqual(aioapns_logger.output, ["INFO:aioapns:Connection released (total: 1)"])
+ self.assertEqual(
+ f"INFO:zerver.lib.push_notifications:APNs: Success sending for user <id:{self.user_profile.id}> to device {self.devices()[0].token}",
+ logger.output[1],
+ )
+
+ # When the transport finishes closing later, aioapns must not
+ # try to discard the connection from the pool again.
+ closed_connection.connection_lost(None)
+ self.assertEqual(pool.connections, [open_connection])
+
+ def test_closed_connection_retry_limit(self) -> None:
+ self.setup_apns_tokens()
+ with (
+ self.mock_apns() as (apns_context, send_notification),
+ mock.patch.object(apns_context, "discard_closed_connections") as discard,
+ self.assertLogs("zerver.lib.push_notifications", level="INFO") as logger,
+ ):
+ send_notification.side_effect = ProtocolError(
+ "Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED"
+ )
+ self.send(devices=self.devices()[0:1])
+ self.assertEqual(send_notification.await_count, APNS_MAX_RETRIES)
+ self.assertEqual(discard.call_count, APNS_MAX_RETRIES - 1)
+ self.assertIn(
+ f"ERROR:zerver.lib.push_notifications:APNs: Error sending for user <id:{self.user_profile.id}> to device {self.devices()[0].token}",
+ logger.output[1],
+ )
+
+ def test_other_protocol_error_not_retried(self) -> None:
+ self.setup_apns_tokens()
+ with (
+ self.mock_apns() as (apns_context, send_notification),
+ mock.patch.object(apns_context, "discard_closed_connections") as discard,
+ self.assertLogs("zerver.lib.push_notifications", level="INFO") as logger,
+ ):
+ send_notification.side_effect = ProtocolError("Invalid header")
+ self.send(devices=self.devices()[0:1])
+ send_notification.assert_awaited_once()
+ discard.assert_not_called()
+ self.assertIn(
+ f"ERROR:zerver.lib.push_notifications:APNs: Error sending for user <id:{self.user_profile.id}> to device {self.devices()[0].token}",
+ logger.output[1],
+ )
+
def test_other_exception(self) -> None:
self.setup_apns_tokens()
with (
diff --git a/zilencer/lib/push_notifications.py b/zilencer/lib/push_notifications.py
index d244df2333f..76421c2621d 100644
--- a/zilencer/lib/push_notifications.py
+++ b/zilencer/lib/push_notifications.py
@@ -55,7 +55,7 @@ def send_e2ee_push_notification_apple(
tuple[RemotePushDevice, aioapns.common.NotificationResult | BaseException]
]:
results = await asyncio.gather(
- *(apns_context.apns.send_notification(request) for request in apns_requests),
+ *(apns_context.send_notification(request) for request in apns_requests),
return_exceptions=True,
)
return zip(apns_remote_push_devices, results, strict=False)