Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(notification): remove device registrations for disabled push endpoints (#7900)

teoab50dff36491
diff --git a/crates/notification/src/domain/models/mobile.rs b/crates/notification/src/domain/models/mobile.rs
index b4a0b6830b..aaca23a900 100644
--- a/crates/notification/src/domain/models/mobile.rs
+++ b/crates/notification/src/domain/models/mobile.rs
@@ -80,3 +80,11 @@ pub enum DeviceEndpoint {
/// iOS VoIP device endpoint (APNS_VOIP / PushKit).
IosVoip(String),
}
+
+/// Report context for a push rejected because its endpoint is disabled or no longer exists.
+///
+/// Retrying cannot succeed until the device registers again, so the device registration
+/// for that endpoint should be removed.
+#[derive(Debug, thiserror::Error)]
+#[error("push endpoint is disabled or no longer exists")]
+pub struct PushEndpointUnavailable;
diff --git a/crates/notification/src/domain/models/push_notification_event.rs b/crates/notification/src/domain/models/push_notification_event.rs
index 0fa6c76e93..cb45651ff0 100644
--- a/crates/notification/src/domain/models/push_notification_event.rs
+++ b/crates/notification/src/domain/models/push_notification_event.rs
@@ -24,8 +24,8 @@ pub struct SnsPushNotificationEvent {
pub endpoint_arn: String,
/// The type of event that occurred.
pub event_type: EventType,
- /// the SNS message unique identifier
- pub message_id: MessageId,
+ /// The SNS message that failed. Only `DeliveryFailure` events carry one.
+ pub message_id: Option<MessageId>,
}
/// A raw push notification event message received from the queue.
diff --git a/crates/notification/src/domain/service/egress.rs b/crates/notification/src/domain/service/egress.rs
index 561ac2ec44..1178380280 100644
--- a/crates/notification/src/domain/service/egress.rs
+++ b/crates/notification/src/domain/service/egress.rs
@@ -11,7 +11,7 @@ use crate::domain::models::email_notification_digest::ports::{
use crate::domain::models::email_notification_digest::{
BulkDigestEgressStateMachine, ResumeMachineBRequest,
};
-use crate::domain::models::mobile::{MessageAttributes, PushType};
+use crate::domain::models::mobile::{MessageAttributes, PushEndpointUnavailable, PushType};
use crate::domain::models::queue_message::{
APNSTargets, ConnGatewayNotification, DeliveryFailure, DeliverySuccess, EmailCreateBundle,
EmailNotification, NotificationChannel, QueueMessage,
@@ -231,8 +231,11 @@ where
})
}))
.collect();
- if let Some(ref entry) = user_apns.digest_state {
- let platforms: Vec<_> = checkers.iter().map(|checker| checker.notif).collect();
+ let targets: Vec<_> = checkers
+ .iter()
+ .map(|checker| (checker.endpoint_arn, checker.notif))
+ .collect();
+ let results = if let Some(ref entry) = user_apns.digest_state {
let req = ResumeMachineBRequest {
notification_enabled: entry.inner().clone(),
send_notifs: checkers,
@@ -241,25 +244,45 @@ where
if let Either::Right(Err(ref batch_err)) = batch_decision {
tracing::error!(error=?batch_err, "failed to queue digest batch after all pushes failed");
}
- for (result, platform) in results.into_iter().zip(platforms) {
- out.push(result.map(|_| platform.success()));
- }
+ results
} else {
+ let mut results = Vec::with_capacity(checkers.len());
for checker in checkers {
- let platform = checker.notif;
- out.push(
- checker
- .send_notification()
- .await
- .map(|_| platform.success()),
- );
+ results.push(checker.send_notification().await);
+ }
+ results
+ };
+ for (result, (endpoint_arn, platform)) in results.into_iter().zip(targets) {
+ if let Err(e) = &result
+ && e.downcast_current_context::<PushEndpointUnavailable>()
+ .is_some()
+ {
+ self.remove_unavailable_endpoint(endpoint_arn).await;
}
+ out.push(result.map(|_| platform.success()));
}
}
out
}
+ /// Remove the device registration for an endpoint SNS will no longer deliver to, so
+ /// later notifications stop targeting it. The device gets a fresh registration the next
+ /// time the app registers its push token.
+ async fn remove_unavailable_endpoint(&self, endpoint_arn: &str) {
+ tracing::info!(
+ endpoint_arn,
+ "removing device registration for unavailable push endpoint"
+ );
+ let _ = self
+ .repository
+ .delete_device_by_endpoint(endpoint_arn)
+ .await
+ .inspect_err(|e| {
+ tracing::warn!(error=?e, endpoint_arn, "failed to remove device registration for unavailable push endpoint");
+ });
+ }
+
/// Deliver via email.
async fn deliver_email(
&self,
diff --git a/crates/notification/src/domain/service/push_notification_event.rs b/crates/notification/src/domain/service/push_notification_event.rs
index 27acc97cae..652023beb9 100644
--- a/crates/notification/src/domain/service/push_notification_event.rs
+++ b/crates/notification/src/domain/service/push_notification_event.rs
@@ -78,13 +78,15 @@ where
.delete_endpoint(&event.endpoint_arn)
.await?;
- self.digest_failure_sm
- .mark_message_as_failed(event.message_id.clone())
- .await
- .inspect_err(|e| {
- tracing::error!(error=?e, "failed to record delivery failure in digest state machine");
- })
- .ok();
+ if let Some(message_id) = &event.message_id {
+ self.digest_failure_sm
+ .mark_message_as_failed(message_id.clone())
+ .await
+ .inspect_err(|e| {
+ tracing::error!(error=?e, "failed to record delivery failure in digest state machine");
+ })
+ .ok();
+ }
}
EventType::EndpointDeleted => {}
}
diff --git a/crates/notification/src/domain/service/push_notification_event/test.rs b/crates/notification/src/domain/service/push_notification_event/test.rs
index 4f0e2347bb..7e956144f6 100644
--- a/crates/notification/src/domain/service/push_notification_event/test.rs
+++ b/crates/notification/src/domain/service/push_notification_event/test.rs
@@ -358,7 +358,7 @@ async fn test_delivery_failure_deletes_device_and_sns_endpoint() {
let event = SnsPushNotificationEvent {
endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),
event_type: EventType::DeliveryFailure,
- message_id: MessageId(String::new()),
+ message_id: Some(MessageId(String::new())),
};
service.handle_event(&event).await.unwrap();
@@ -380,7 +380,7 @@ async fn test_endpoint_deleted_only_deletes_device() {
let event = SnsPushNotificationEvent {
endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),
event_type: EventType::EndpointDeleted,
- message_id: MessageId(String::new()),
+ message_id: None,
};
service.handle_event(&event).await.unwrap();
@@ -402,7 +402,7 @@ async fn test_device_deletion_failure_propagates_error() {
let event = SnsPushNotificationEvent {
endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),
event_type: EventType::DeliveryFailure,
- message_id: MessageId(String::new()),
+ message_id: Some(MessageId(String::new())),
};
let result = service.handle_event(&event).await;
@@ -425,7 +425,7 @@ async fn test_sns_deletion_failure_propagates_error() {
let event = SnsPushNotificationEvent {
endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),
event_type: EventType::DeliveryFailure,
- message_id: MessageId(String::new()),
+ message_id: Some(MessageId(String::new())),
};
let result = service.handle_event(&event).await;
@@ -445,7 +445,7 @@ async fn test_delivery_failure_calls_digest_state_machine() {
let event = SnsPushNotificationEvent {
endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),
event_type: EventType::DeliveryFailure,
- message_id: MessageId("msg-123".to_string()),
+ message_id: Some(MessageId("msg-123".to_string())),
};
service.handle_event(&event).await.unwrap();
@@ -453,6 +453,29 @@ async fn test_delivery_failure_calls_digest_state_machine() {
assert_eq!(service.digest_failure_sm.get_calls(), vec!["msg-123"]);
}
+#[tokio::test]
+async fn test_delivery_failure_without_message_id_skips_digest_state_machine() {
+ let repository = MockNotifRepo::new();
+ let sns_manager = MockSnsManager::new();
+ let digest_sm = MockDigestFailureStateMachine::new();
+ let service = PushNotificationEventService::new(repository, sns_manager, digest_sm);
+
+ let event = SnsPushNotificationEvent {
+ endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),
+ event_type: EventType::DeliveryFailure,
+ message_id: None,
+ };
+
+ service.handle_event(&event).await.unwrap();
+
+ assert!(service.digest_failure_sm.get_calls().is_empty());
+ assert_eq!(
+ service.repository.get_deleted(),
+ vec![event.endpoint_arn.clone()]
+ );
+ assert_eq!(service.sns_manager.get_deleted(), vec![event.endpoint_arn]);
+}
+
#[tokio::test]
async fn test_endpoint_deleted_does_not_call_digest_state_machine() {
let repository = MockNotifRepo::new();
@@ -463,7 +486,7 @@ async fn test_endpoint_deleted_does_not_call_digest_state_machine() {
let event = SnsPushNotificationEvent {
endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),
event_type: EventType::EndpointDeleted,
- message_id: MessageId("msg-456".to_string()),
+ message_id: Some(MessageId("msg-456".to_string())),
};
service.handle_event(&event).await.unwrap();
@@ -484,7 +507,7 @@ async fn test_digest_state_machine_failure_does_not_propagate() {
let event = SnsPushNotificationEvent {
endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),
event_type: EventType::DeliveryFailure,
- message_id: MessageId("msg-789".to_string()),
+ message_id: Some(MessageId("msg-789".to_string())),
};
let result = service.handle_event(&event).await;
diff --git a/crates/notification/src/domain/service/test.rs b/crates/notification/src/domain/service/test.rs
index c030bc29d6..836dd3c1f1 100644
--- a/crates/notification/src/domain/service/test.rs
+++ b/crates/notification/src/domain/service/test.rs
@@ -122,6 +122,7 @@ struct MockRepository {
basic_notification_calls: Mutex<Vec<Vec<Uuid>>>,
mark_seen_calls: Mutex<Vec<(String, Vec<Uuid>)>>,
mark_done_calls: Mutex<Vec<(String, Vec<Uuid>, bool)>>,
+ deleted_device_endpoints: Mutex<Vec<String>>,
}
impl MockRepository {
@@ -143,6 +144,7 @@ impl MockRepository {
basic_notification_calls: Mutex::new(Vec::new()),
mark_seen_calls: Mutex::new(Vec::new()),
mark_done_calls: Mutex::new(Vec::new()),
+ deleted_device_endpoints: Mutex::new(Vec::new()),
}
}
@@ -521,7 +523,11 @@ impl NotificationRepository for MockRepository {
Ok(Vec::new())
}
- async fn delete_device_by_endpoint(&self, _endpoint_arn: &str) -> Result<(), Report> {
+ async fn delete_device_by_endpoint(&self, endpoint_arn: &str) -> Result<(), Report> {
+ self.deleted_device_endpoints
+ .lock()
+ .unwrap()
+ .push(endpoint_arn.to_string());
Ok(())
}
@@ -2175,6 +2181,8 @@ struct TrackingMobileSender {
attempted_endpoints: Mutex<Vec<String>>,
/// Endpoints that should fail when attempted.
failing_endpoints: HashSet<String>,
+ /// Endpoints that should fail as disabled or deleted.
+ unavailable_endpoints: HashSet<String>,
}
impl TrackingMobileSender {
@@ -2182,9 +2190,15 @@ impl TrackingMobileSender {
Self {
attempted_endpoints: Mutex::new(Vec::new()),
failing_endpoints,
+ unavailable_endpoints: HashSet::new(),
}
}
+ fn with_unavailable_endpoints(mut self, endpoints: HashSet<String>) -> Self {
+ self.unavailable_endpoints = endpoints;
+ self
+ }
+
fn get_attempted_endpoints(&self) -> Vec<String> {
self.attempted_endpoints.lock().unwrap().clone()
}
@@ -2207,6 +2221,11 @@ impl NotificationSender for TrackingMobileSender {
if self.failing_endpoints.contains(endpoint_arn) {
rootcause::bail!("Simulated APNS failure for endpoint: {}", endpoint_arn);
}
+ if self.unavailable_endpoints.contains(endpoint_arn) {
+ return Err(report!("Simulated disabled endpoint: {}", endpoint_arn)
+ .context(crate::domain::models::mobile::PushEndpointUnavailable)
+ .into_dynamic());
+ }
Ok(format!("msg-id-{endpoint_arn}"))
}
@@ -2318,6 +2337,75 @@ async fn test_egress_ios_attempts_all_endpoints_even_if_some_fail() {
assert_eq!(failures, 2, "Should have 2 failed deliveries");
}
+#[tokio::test]
+async fn test_egress_removes_unavailable_push_endpoints() {
+ use crate::domain::models::apple::{APNSPushNotification, Aps};
+ use crate::domain::models::mobile::{MessageAttributes, PushType};
+ use crate::domain::models::queue_message::{APNSTargets, UserApnsEndpoints};
+
+ let healthy = "arn:aws:sns:us-east-1:111:endpoint/APNS/app/healthy";
+ let disabled = "arn:aws:sns:us-east-1:111:endpoint/APNS/app/disabled";
+ let flaky = "arn:aws:sns:us-east-1:111:endpoint/APNS/app/flaky";
+
+ let mobile_sender = std::sync::Arc::new(
+ TrackingMobileSender::new([flaky.to_string()].into())
+ .with_unavailable_endpoints([disabled.to_string()].into()),
+ );
+ let repository = Arc::new(MockRepository::new());
+ let service = NotificationEgressService {
+ queue: MockQueue::new(),
+ repository: repository.clone(),
+ realtime: MockRealtimeSender,
+ mobile: mobile_sender.clone(),
+ email: MockEmailSender,
+ rate_limiter: allowing_rate_limiter(),
+ state_machine: MockEgressStateMachine,
+ digest_batcher: MockDigestBatcher,
+ };
+
+ let message = QueueMessage::new_test(
+ "test_notification".to_string(),
+ NotificationChannel::Ios(Box::new(APNSTargets {
+ notif: APNSPushNotification {
+ aps: Aps::default(),
+ push_notification_data: json!({"message": "Hello"}),
+ },
+ attributes: MessageAttributes {
+ push_type: PushType::Alert,
+ collapse_key: "test_collapse".to_string(),
+ },
+ ios_device_endpoints: HashMap::from([
+ (
+ test_user_id("[email protected]"),
+ UserApnsEndpoints {
+ android_endpoints: Vec::new(),
+ endpoints: vec![healthy.to_string(), disabled.to_string()],
+ digest_state: None,
+ },
+ ),
+ (
+ test_user_id("[email protected]"),
+ UserApnsEndpoints {
+ android_endpoints: Vec::new(),
+ endpoints: vec![flaky.to_string()],
+ digest_state: None,
+ },
+ ),
+ ]),
+ })),
+ );
+
+ let results = service.deliver_notification(message).await;
+
+ assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
+ assert_eq!(mobile_sender.get_attempted_endpoints().len(), 3);
+ assert_eq!(
+ *repository.deleted_device_endpoints.lock().unwrap(),
+ vec![disabled.to_string()],
+ "only the disabled endpoint's registration should be removed"
+ );
+}
+
// --- poll_email_digests tests ---
struct ReadyDigestBatcher {
diff --git a/crates/notification/src/inbound/push_notification_event_worker/test.rs b/crates/notification/src/inbound/push_notification_event_worker/test.rs
index 2c916a4508..59e9e36a9a 100644
--- a/crates/notification/src/inbound/push_notification_event_worker/test.rs
+++ b/crates/notification/src/inbound/push_notification_event_worker/test.rs
@@ -78,6 +78,28 @@ async fn test_deletes_queue_message_on_success() {
);
}
+#[tokio::test]
+async fn test_handles_endpoint_deleted_event_without_message_id() {
+ let handler = MockHandler { should_fail: false };
+ let queue = MockQueue::new();
+ let worker = PushNotificationEventWorker::new(handler, queue);
+
+ let body = serde_json::json!({
+ "EndpointArn": "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1",
+ "EventType": "EndpointDeleted",
+ "Resource": "arn:aws:sns:us-east-1:123:app/APNS/app",
+ "Service": "SNS",
+ "Time": "2026-10-10T00:00:00.000Z"
+ });
+ let messages = vec![RawPushNotificationEventMessage {
+ body: Some(body.to_string()),
+ receipt_handle: Some("receipt-1".to_string()),
+ }];
+ worker.process_messages(&messages).await;
+
+ assert_eq!(worker.queue.get_deleted_receipts(), vec!["receipt-1"]);
+}
+
#[tokio::test]
async fn test_does_not_delete_queue_message_on_handler_error() {
let handler = MockHandler { should_fail: true };
diff --git a/crates/notification/src/outbound/mobile.rs b/crates/notification/src/outbound/mobile.rs
index 57db11f8b9..98818bf3ba 100644
--- a/crates/notification/src/outbound/mobile.rs
+++ b/crates/notification/src/outbound/mobile.rs
@@ -1,5 +1,6 @@
//! Mobile push notification adapter.
+use aws_sdk_sns::operation::publish::PublishError;
use aws_sdk_sns::types::MessageAttributeValue;
use rootcause::Report;
use serde::Serialize;
@@ -7,12 +8,15 @@ use std::collections::HashMap;
use crate::domain::models::android::FCMMessage;
use crate::domain::models::apple::{APNSPushNotification, VoipPushPayload};
-use crate::domain::models::mobile::{MessageAttributes, PushType};
+use crate::domain::models::mobile::{MessageAttributes, PushEndpointUnavailable, PushType};
use crate::domain::ports::NotificationSender;
mod payload;
pub use payload::SnsTarget;
+#[cfg(test)]
+mod test;
+
/// Mobile push notification adapter.
///
/// This adapter sends push notifications to mobile devices via APNS (iOS)
@@ -58,16 +62,33 @@ impl MobilePushOps for aws_sdk_sns::Client {
.send()
.await
.map_err(|e| {
- rootcause::report!(
+ let report = rootcause::report!(
"SNS publish to {endpoint_arn} failed: {}",
aws_sdk_sns::error::DisplayErrorContext(&e)
- )
+ );
+ if e.as_service_error().is_some_and(is_unavailable_endpoint) {
+ report.context(PushEndpointUnavailable).into_dynamic()
+ } else {
+ report
+ }
})?;
Ok(output.message_id.unwrap_or_default())
}
}
+/// Whether SNS rejected the publish because the target endpoint can no longer receive pushes,
+/// as opposed to a problem with the message or the platform application.
+fn is_unavailable_endpoint(err: &PublishError) -> bool {
+ match err {
+ PublishError::EndpointDisabledException(_) | PublishError::NotFoundException(_) => true,
+ PublishError::InvalidParameterException(e) => e
+ .message()
+ .is_some_and(|message| message.contains("No endpoint found")),
+ _ => false,
+ }
+}
+
impl<P: MobilePushOps + Send + Sync + 'static> NotificationSender for MobilePushAdapter<P> {
async fn send_ios_push_notification<T: Serialize + Send + Sync>(
&self,
diff --git a/crates/notification/src/outbound/mobile/test.rs b/crates/notification/src/outbound/mobile/test.rs
new file mode 100644
index 0000000000..de29de8fb1
--- /dev/null
+++ b/crates/notification/src/outbound/mobile/test.rs
@@ -0,0 +1,49 @@
+use super::*;
+use aws_sdk_sns::types::error::{
+ EndpointDisabledException, InvalidParameterException, NotFoundException,
+ PlatformApplicationDisabledException,
+};
+
+fn invalid_parameter(message: &str) -> PublishError {
+ PublishError::InvalidParameterException(
+ InvalidParameterException::builder()
+ .message(message)
+ .build(),
+ )
+}
+
+#[test]
+fn disabled_endpoint_is_unavailable() {
+ let err = PublishError::EndpointDisabledException(
+ EndpointDisabledException::builder()
+ .message("Endpoint is disabled")
+ .build(),
+ );
+ assert!(is_unavailable_endpoint(&err));
+}
+
+#[test]
+fn deleted_endpoint_is_unavailable() {
+ assert!(is_unavailable_endpoint(&invalid_parameter(
+ "Invalid parameter: TargetArn Reason: No endpoint found for the target arn specified",
+ )));
+ assert!(is_unavailable_endpoint(&PublishError::NotFoundException(
+ NotFoundException::builder()
+ .message("Endpoint does not exist")
+ .build(),
+ )));
+}
+
+#[test]
+fn message_and_platform_errors_are_not_unavailable() {
+ assert!(!is_unavailable_endpoint(&invalid_parameter(
+ "Invalid parameter: Message Reason: Invalid notification for protocol APNS: Notification is too long",
+ )));
+ assert!(!is_unavailable_endpoint(
+ &PublishError::PlatformApplicationDisabledException(
+ PlatformApplicationDisabledException::builder()
+ .message("Platform application is disabled")
+ .build(),
+ )
+ ));
+}