Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(notification): remove device registrations for disabled push endpoints (#7900)
teoab50dff36491
diff --git a/crates/notification/src/domain/models/mobile.rs b/crates/notification/src/domain/models/mobile.rsindex b4a0b6830b..aaca23a900 100644--- a/crates/notification/src/domain/models/mobile.rs+++ b/crates/notification/src/domain/models/mobile.rs@@ -80,3 +80,11 @@ pub enum DeviceEndpoint {/// iOS VoIP device endpoint (APNS_VOIP / PushKit).IosVoip(String),}++/// Report context for a push rejected because its endpoint is disabled or no longer exists.+///+/// Retrying cannot succeed until the device registers again, so the device registration+/// for that endpoint should be removed.+#[derive(Debug, thiserror::Error)]+#[error("push endpoint is disabled or no longer exists")]+pub struct PushEndpointUnavailable;diff --git a/crates/notification/src/domain/models/push_notification_event.rs b/crates/notification/src/domain/models/push_notification_event.rsindex 0fa6c76e93..cb45651ff0 100644--- a/crates/notification/src/domain/models/push_notification_event.rs+++ b/crates/notification/src/domain/models/push_notification_event.rs@@ -24,8 +24,8 @@ pub struct SnsPushNotificationEvent {pub endpoint_arn: String,/// The type of event that occurred.pub event_type: EventType,- /// the SNS message unique identifier- pub message_id: MessageId,+ /// The SNS message that failed. Only `DeliveryFailure` events carry one.+ pub message_id: Option<MessageId>,}/// A raw push notification event message received from the queue.diff --git a/crates/notification/src/domain/service/egress.rs b/crates/notification/src/domain/service/egress.rsindex 561ac2ec44..1178380280 100644--- a/crates/notification/src/domain/service/egress.rs+++ b/crates/notification/src/domain/service/egress.rs@@ -11,7 +11,7 @@ use crate::domain::models::email_notification_digest::ports::{use crate::domain::models::email_notification_digest::{BulkDigestEgressStateMachine, ResumeMachineBRequest,};-use crate::domain::models::mobile::{MessageAttributes, PushType};+use crate::domain::models::mobile::{MessageAttributes, PushEndpointUnavailable, PushType};use crate::domain::models::queue_message::{APNSTargets, ConnGatewayNotification, DeliveryFailure, DeliverySuccess, EmailCreateBundle,EmailNotification, NotificationChannel, QueueMessage,@@ -231,8 +231,11 @@ where})})).collect();- if let Some(ref entry) = user_apns.digest_state {- let platforms: Vec<_> = checkers.iter().map(|checker| checker.notif).collect();+ let targets: Vec<_> = checkers+ .iter()+ .map(|checker| (checker.endpoint_arn, checker.notif))+ .collect();+ let results = if let Some(ref entry) = user_apns.digest_state {let req = ResumeMachineBRequest {notification_enabled: entry.inner().clone(),send_notifs: checkers,@@ -241,25 +244,45 @@ whereif let Either::Right(Err(ref batch_err)) = batch_decision {tracing::error!(error=?batch_err, "failed to queue digest batch after all pushes failed");}- for (result, platform) in results.into_iter().zip(platforms) {- out.push(result.map(|_| platform.success()));- }+ results} else {+ let mut results = Vec::with_capacity(checkers.len());for checker in checkers {- let platform = checker.notif;- out.push(- checker- .send_notification()- .await- .map(|_| platform.success()),- );+ results.push(checker.send_notification().await);+ }+ results+ };+ for (result, (endpoint_arn, platform)) in results.into_iter().zip(targets) {+ if let Err(e) = &result+ && e.downcast_current_context::<PushEndpointUnavailable>()+ .is_some()+ {+ self.remove_unavailable_endpoint(endpoint_arn).await;}+ out.push(result.map(|_| platform.success()));}}out}+ /// Remove the device registration for an endpoint SNS will no longer deliver to, so+ /// later notifications stop targeting it. The device gets a fresh registration the next+ /// time the app registers its push token.+ async fn remove_unavailable_endpoint(&self, endpoint_arn: &str) {+ tracing::info!(+ endpoint_arn,+ "removing device registration for unavailable push endpoint"+ );+ let _ = self+ .repository+ .delete_device_by_endpoint(endpoint_arn)+ .await+ .inspect_err(|e| {+ tracing::warn!(error=?e, endpoint_arn, "failed to remove device registration for unavailable push endpoint");+ });+ }+/// Deliver via email.async fn deliver_email(&self,diff --git a/crates/notification/src/domain/service/push_notification_event.rs b/crates/notification/src/domain/service/push_notification_event.rsindex 27acc97cae..652023beb9 100644--- a/crates/notification/src/domain/service/push_notification_event.rs+++ b/crates/notification/src/domain/service/push_notification_event.rs@@ -78,13 +78,15 @@ where.delete_endpoint(&event.endpoint_arn).await?;- self.digest_failure_sm- .mark_message_as_failed(event.message_id.clone())- .await- .inspect_err(|e| {- tracing::error!(error=?e, "failed to record delivery failure in digest state machine");- })- .ok();+ if let Some(message_id) = &event.message_id {+ self.digest_failure_sm+ .mark_message_as_failed(message_id.clone())+ .await+ .inspect_err(|e| {+ tracing::error!(error=?e, "failed to record delivery failure in digest state machine");+ })+ .ok();+ }}EventType::EndpointDeleted => {}}diff --git a/crates/notification/src/domain/service/push_notification_event/test.rs b/crates/notification/src/domain/service/push_notification_event/test.rsindex 4f0e2347bb..7e956144f6 100644--- a/crates/notification/src/domain/service/push_notification_event/test.rs+++ b/crates/notification/src/domain/service/push_notification_event/test.rs@@ -358,7 +358,7 @@ async fn test_delivery_failure_deletes_device_and_sns_endpoint() {let event = SnsPushNotificationEvent {endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),event_type: EventType::DeliveryFailure,- message_id: MessageId(String::new()),+ message_id: Some(MessageId(String::new())),};service.handle_event(&event).await.unwrap();@@ -380,7 +380,7 @@ async fn test_endpoint_deleted_only_deletes_device() {let event = SnsPushNotificationEvent {endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),event_type: EventType::EndpointDeleted,- message_id: MessageId(String::new()),+ message_id: None,};service.handle_event(&event).await.unwrap();@@ -402,7 +402,7 @@ async fn test_device_deletion_failure_propagates_error() {let event = SnsPushNotificationEvent {endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),event_type: EventType::DeliveryFailure,- message_id: MessageId(String::new()),+ message_id: Some(MessageId(String::new())),};let result = service.handle_event(&event).await;@@ -425,7 +425,7 @@ async fn test_sns_deletion_failure_propagates_error() {let event = SnsPushNotificationEvent {endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),event_type: EventType::DeliveryFailure,- message_id: MessageId(String::new()),+ message_id: Some(MessageId(String::new())),};let result = service.handle_event(&event).await;@@ -445,7 +445,7 @@ async fn test_delivery_failure_calls_digest_state_machine() {let event = SnsPushNotificationEvent {endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),event_type: EventType::DeliveryFailure,- message_id: MessageId("msg-123".to_string()),+ message_id: Some(MessageId("msg-123".to_string())),};service.handle_event(&event).await.unwrap();@@ -453,6 +453,29 @@ async fn test_delivery_failure_calls_digest_state_machine() {assert_eq!(service.digest_failure_sm.get_calls(), vec!["msg-123"]);}+#[tokio::test]+async fn test_delivery_failure_without_message_id_skips_digest_state_machine() {+ let repository = MockNotifRepo::new();+ let sns_manager = MockSnsManager::new();+ let digest_sm = MockDigestFailureStateMachine::new();+ let service = PushNotificationEventService::new(repository, sns_manager, digest_sm);++ let event = SnsPushNotificationEvent {+ endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),+ event_type: EventType::DeliveryFailure,+ message_id: None,+ };++ service.handle_event(&event).await.unwrap();++ assert!(service.digest_failure_sm.get_calls().is_empty());+ assert_eq!(+ service.repository.get_deleted(),+ vec![event.endpoint_arn.clone()]+ );+ assert_eq!(service.sns_manager.get_deleted(), vec![event.endpoint_arn]);+}+#[tokio::test]async fn test_endpoint_deleted_does_not_call_digest_state_machine() {let repository = MockNotifRepo::new();@@ -463,7 +486,7 @@ async fn test_endpoint_deleted_does_not_call_digest_state_machine() {let event = SnsPushNotificationEvent {endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),event_type: EventType::EndpointDeleted,- message_id: MessageId("msg-456".to_string()),+ message_id: Some(MessageId("msg-456".to_string())),};service.handle_event(&event).await.unwrap();@@ -484,7 +507,7 @@ async fn test_digest_state_machine_failure_does_not_propagate() {let event = SnsPushNotificationEvent {endpoint_arn: "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1".to_string(),event_type: EventType::DeliveryFailure,- message_id: MessageId("msg-789".to_string()),+ message_id: Some(MessageId("msg-789".to_string())),};let result = service.handle_event(&event).await;diff --git a/crates/notification/src/domain/service/test.rs b/crates/notification/src/domain/service/test.rsindex c030bc29d6..836dd3c1f1 100644--- a/crates/notification/src/domain/service/test.rs+++ b/crates/notification/src/domain/service/test.rs@@ -122,6 +122,7 @@ struct MockRepository {basic_notification_calls: Mutex<Vec<Vec<Uuid>>>,mark_seen_calls: Mutex<Vec<(String, Vec<Uuid>)>>,mark_done_calls: Mutex<Vec<(String, Vec<Uuid>, bool)>>,+ deleted_device_endpoints: Mutex<Vec<String>>,}impl MockRepository {@@ -143,6 +144,7 @@ impl MockRepository {basic_notification_calls: Mutex::new(Vec::new()),mark_seen_calls: Mutex::new(Vec::new()),mark_done_calls: Mutex::new(Vec::new()),+ deleted_device_endpoints: Mutex::new(Vec::new()),}}@@ -521,7 +523,11 @@ impl NotificationRepository for MockRepository {Ok(Vec::new())}- async fn delete_device_by_endpoint(&self, _endpoint_arn: &str) -> Result<(), Report> {+ async fn delete_device_by_endpoint(&self, endpoint_arn: &str) -> Result<(), Report> {+ self.deleted_device_endpoints+ .lock()+ .unwrap()+ .push(endpoint_arn.to_string());Ok(())}@@ -2175,6 +2181,8 @@ struct TrackingMobileSender {attempted_endpoints: Mutex<Vec<String>>,/// Endpoints that should fail when attempted.failing_endpoints: HashSet<String>,+ /// Endpoints that should fail as disabled or deleted.+ unavailable_endpoints: HashSet<String>,}impl TrackingMobileSender {@@ -2182,9 +2190,15 @@ impl TrackingMobileSender {Self {attempted_endpoints: Mutex::new(Vec::new()),failing_endpoints,+ unavailable_endpoints: HashSet::new(),}}+ fn with_unavailable_endpoints(mut self, endpoints: HashSet<String>) -> Self {+ self.unavailable_endpoints = endpoints;+ self+ }+fn get_attempted_endpoints(&self) -> Vec<String> {self.attempted_endpoints.lock().unwrap().clone()}@@ -2207,6 +2221,11 @@ impl NotificationSender for TrackingMobileSender {if self.failing_endpoints.contains(endpoint_arn) {rootcause::bail!("Simulated APNS failure for endpoint: {}", endpoint_arn);}+ if self.unavailable_endpoints.contains(endpoint_arn) {+ return Err(report!("Simulated disabled endpoint: {}", endpoint_arn)+ .context(crate::domain::models::mobile::PushEndpointUnavailable)+ .into_dynamic());+ }Ok(format!("msg-id-{endpoint_arn}"))}@@ -2318,6 +2337,75 @@ async fn test_egress_ios_attempts_all_endpoints_even_if_some_fail() {assert_eq!(failures, 2, "Should have 2 failed deliveries");}+#[tokio::test]+async fn test_egress_removes_unavailable_push_endpoints() {+ use crate::domain::models::apple::{APNSPushNotification, Aps};+ use crate::domain::models::mobile::{MessageAttributes, PushType};+ use crate::domain::models::queue_message::{APNSTargets, UserApnsEndpoints};++ let healthy = "arn:aws:sns:us-east-1:111:endpoint/APNS/app/healthy";+ let disabled = "arn:aws:sns:us-east-1:111:endpoint/APNS/app/disabled";+ let flaky = "arn:aws:sns:us-east-1:111:endpoint/APNS/app/flaky";++ let mobile_sender = std::sync::Arc::new(+ TrackingMobileSender::new([flaky.to_string()].into())+ .with_unavailable_endpoints([disabled.to_string()].into()),+ );+ let repository = Arc::new(MockRepository::new());+ let service = NotificationEgressService {+ queue: MockQueue::new(),+ repository: repository.clone(),+ realtime: MockRealtimeSender,+ mobile: mobile_sender.clone(),+ email: MockEmailSender,+ rate_limiter: allowing_rate_limiter(),+ state_machine: MockEgressStateMachine,+ digest_batcher: MockDigestBatcher,+ };++ let message = QueueMessage::new_test(+ "test_notification".to_string(),+ NotificationChannel::Ios(Box::new(APNSTargets {+ notif: APNSPushNotification {+ aps: Aps::default(),+ push_notification_data: json!({"message": "Hello"}),+ },+ attributes: MessageAttributes {+ push_type: PushType::Alert,+ collapse_key: "test_collapse".to_string(),+ },+ ios_device_endpoints: HashMap::from([+ (+ test_user_id("[email protected]"),+ UserApnsEndpoints {+ android_endpoints: Vec::new(),+ endpoints: vec![healthy.to_string(), disabled.to_string()],+ digest_state: None,+ },+ ),+ (+ test_user_id("[email protected]"),+ UserApnsEndpoints {+ android_endpoints: Vec::new(),+ endpoints: vec![flaky.to_string()],+ digest_state: None,+ },+ ),+ ]),+ })),+ );++ let results = service.deliver_notification(message).await;++ assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);+ assert_eq!(mobile_sender.get_attempted_endpoints().len(), 3);+ assert_eq!(+ *repository.deleted_device_endpoints.lock().unwrap(),+ vec![disabled.to_string()],+ "only the disabled endpoint's registration should be removed"+ );+}+// --- poll_email_digests tests ---struct ReadyDigestBatcher {diff --git a/crates/notification/src/inbound/push_notification_event_worker/test.rs b/crates/notification/src/inbound/push_notification_event_worker/test.rsindex 2c916a4508..59e9e36a9a 100644--- a/crates/notification/src/inbound/push_notification_event_worker/test.rs+++ b/crates/notification/src/inbound/push_notification_event_worker/test.rs@@ -78,6 +78,28 @@ async fn test_deletes_queue_message_on_success() {);}+#[tokio::test]+async fn test_handles_endpoint_deleted_event_without_message_id() {+ let handler = MockHandler { should_fail: false };+ let queue = MockQueue::new();+ let worker = PushNotificationEventWorker::new(handler, queue);++ let body = serde_json::json!({+ "EndpointArn": "arn:aws:sns:us-east-1:123:endpoint/APNS/app/device1",+ "EventType": "EndpointDeleted",+ "Resource": "arn:aws:sns:us-east-1:123:app/APNS/app",+ "Service": "SNS",+ "Time": "2026-10-10T00:00:00.000Z"+ });+ let messages = vec![RawPushNotificationEventMessage {+ body: Some(body.to_string()),+ receipt_handle: Some("receipt-1".to_string()),+ }];+ worker.process_messages(&messages).await;++ assert_eq!(worker.queue.get_deleted_receipts(), vec!["receipt-1"]);+}+#[tokio::test]async fn test_does_not_delete_queue_message_on_handler_error() {let handler = MockHandler { should_fail: true };diff --git a/crates/notification/src/outbound/mobile.rs b/crates/notification/src/outbound/mobile.rsindex 57db11f8b9..98818bf3ba 100644--- a/crates/notification/src/outbound/mobile.rs+++ b/crates/notification/src/outbound/mobile.rs@@ -1,5 +1,6 @@//! Mobile push notification adapter.+use aws_sdk_sns::operation::publish::PublishError;use aws_sdk_sns::types::MessageAttributeValue;use rootcause::Report;use serde::Serialize;@@ -7,12 +8,15 @@ use std::collections::HashMap;use crate::domain::models::android::FCMMessage;use crate::domain::models::apple::{APNSPushNotification, VoipPushPayload};-use crate::domain::models::mobile::{MessageAttributes, PushType};+use crate::domain::models::mobile::{MessageAttributes, PushEndpointUnavailable, PushType};use crate::domain::ports::NotificationSender;mod payload;pub use payload::SnsTarget;+#[cfg(test)]+mod test;+/// Mobile push notification adapter.////// This adapter sends push notifications to mobile devices via APNS (iOS)@@ -58,16 +62,33 @@ impl MobilePushOps for aws_sdk_sns::Client {.send().await.map_err(|e| {- rootcause::report!(+ let report = rootcause::report!("SNS publish to {endpoint_arn} failed: {}",aws_sdk_sns::error::DisplayErrorContext(&e)- )+ );+ if e.as_service_error().is_some_and(is_unavailable_endpoint) {+ report.context(PushEndpointUnavailable).into_dynamic()+ } else {+ report+ }})?;Ok(output.message_id.unwrap_or_default())}}+/// Whether SNS rejected the publish because the target endpoint can no longer receive pushes,+/// as opposed to a problem with the message or the platform application.+fn is_unavailable_endpoint(err: &PublishError) -> bool {+ match err {+ PublishError::EndpointDisabledException(_) | PublishError::NotFoundException(_) => true,+ PublishError::InvalidParameterException(e) => e+ .message()+ .is_some_and(|message| message.contains("No endpoint found")),+ _ => false,+ }+}+impl<P: MobilePushOps + Send + Sync + 'static> NotificationSender for MobilePushAdapter<P> {async fn send_ios_push_notification<T: Serialize + Send + Sync>(&self,diff --git a/crates/notification/src/outbound/mobile/test.rs b/crates/notification/src/outbound/mobile/test.rsnew file mode 100644index 0000000000..de29de8fb1--- /dev/null+++ b/crates/notification/src/outbound/mobile/test.rs@@ -0,0 +1,49 @@+use super::*;+use aws_sdk_sns::types::error::{+ EndpointDisabledException, InvalidParameterException, NotFoundException,+ PlatformApplicationDisabledException,+};++fn invalid_parameter(message: &str) -> PublishError {+ PublishError::InvalidParameterException(+ InvalidParameterException::builder()+ .message(message)+ .build(),+ )+}++#[test]+fn disabled_endpoint_is_unavailable() {+ let err = PublishError::EndpointDisabledException(+ EndpointDisabledException::builder()+ .message("Endpoint is disabled")+ .build(),+ );+ assert!(is_unavailable_endpoint(&err));+}++#[test]+fn deleted_endpoint_is_unavailable() {+ assert!(is_unavailable_endpoint(&invalid_parameter(+ "Invalid parameter: TargetArn Reason: No endpoint found for the target arn specified",+ )));+ assert!(is_unavailable_endpoint(&PublishError::NotFoundException(+ NotFoundException::builder()+ .message("Endpoint does not exist")+ .build(),+ )));+}++#[test]+fn message_and_platform_errors_are_not_unavailable() {+ assert!(!is_unavailable_endpoint(&invalid_parameter(+ "Invalid parameter: Message Reason: Invalid notification for protocol APNS: Notification is too long",+ )));+ assert!(!is_unavailable_endpoint(+ &PublishError::PlatformApplicationDisabledException(+ PlatformApplicationDisabledException::builder()+ .message("Platform application is disabled")+ .build(),+ )+ ));+}