Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(notification): drop rate-limited queue messages (#5201)
Sean Aye22c7035184c2
diff --git a/crates/ai_tools/src/tool_context.rs b/crates/ai_tools/src/tool_context.rsindex 78f47e85cc..143467c9e7 100644--- a/crates/ai_tools/src/tool_context.rs+++ b/crates/ai_tools/src/tool_context.rs@@ -562,24 +562,6 @@ impl notification::domain::ports::NotificationQueue for ToolNotificationQueue {ToolNotificationQueue::NoOp => Ok(()),}}-- async fn delay_message(- &self,- receipt_handle: &str,- delay: std::time::Duration,- ) -> Result<(), rootcause::Report> {- match self {- ToolNotificationQueue::Sqs(queue) => {- notification::domain::ports::NotificationQueue::delay_message(- queue,- receipt_handle,- delay,- )- .await- }- ToolNotificationQueue::NoOp => Ok(()),- }- }}/// Type alias for the entity access management service implementation used by AI toolsdiff --git a/crates/notification/src/domain/ports.rs b/crates/notification/src/domain/ports.rsindex 20615e782f..6f6d014b89 100644--- a/crates/notification/src/domain/ports.rs+++ b/crates/notification/src/domain/ports.rs@@ -315,21 +315,11 @@ pub trait NotificationQueue: Send + Sync + 'static {fn receive_messages(&self)-> impl Future<Output = Result<Vec<RawQueueMessage>, Report>> + Send;- /// Delete a message from the queue (after successful delivery).+ /// Delete a message from the queue after terminal processing.fn delete_message(&self,receipt_handle: &str,) -> impl Future<Output = Result<(), Report>> + Send;-- /// Delay a message by changing its visibility timeout.- ///- /// The message will not be redelivered to consumers until the duration elapses.- /// Uses SQS `ChangeMessageVisibility` under the hood.- fn delay_message(- &self,- receipt_handle: &str,- delay: std::time::Duration,- ) -> impl Future<Output = Result<(), Report>> + Send;}/// Port for delivering notifications from the queue.@@ -340,7 +330,8 @@ pub trait NotificationEgress: Send + Sync + 'static {/// Poll the queue and attempt to deliver notifications.////// Returns results for each delivery attempt across all messages received.- /// Messages are automatically deleted from the queue after successful delivery.+ /// Messages are deleted from the queue after terminal outcomes, including+ /// successful delivery and rate-limit rejection.fn poll_and_deliver(&self)-> impl Future<Output = Vec<Result<DeliverySuccess, Report>>> + Send;diff --git a/crates/notification/src/domain/service/device/test.rs b/crates/notification/src/domain/service/device/test.rsindex 5eeaf78cb6..cfc5ba2d97 100644--- a/crates/notification/src/domain/service/device/test.rs+++ b/crates/notification/src/domain/service/device/test.rs@@ -357,14 +357,6 @@ impl NotificationQueue for MockQueue {async fn delete_message(&self, _: &str) -> Result<(), Report> {Ok(())}-- async fn delay_message(- &self,- _receipt_handle: &str,- _delay: std::time::Duration,- ) -> Result<(), Report> {- Ok(())- }}fn test_config() -> PlatformArnConfig {diff --git a/crates/notification/src/domain/service/egress.rs b/crates/notification/src/domain/service/egress.rsindex 80f0d986be..bf2c126d35 100644--- a/crates/notification/src/domain/service/egress.rs+++ b/crates/notification/src/domain/service/egress.rs@@ -302,31 +302,20 @@ wherelet all_ios_failed = delivery_results.iter().all(|e| matches!(e, Err(e) if matches!(e.current_context(), DeliveryFailure::Ios )),);- let rate_limited = delivery_results.iter().find_map(|e| {- match e.as_ref().map_err(Report::current_context) {- Err(DeliveryFailure::RateLimit(rate_limit)) => Some(rate_limit),- Ok(_)- | Err(- DeliveryFailure::Ios | DeliveryFailure::Other | DeliveryFailure::Timeout,- ) => None,- }+ let rate_limited = delivery_results.iter().any(|e| {+ matches!(+ e.as_ref().map_err(Report::current_context),+ Err(DeliveryFailure::RateLimit(_))+ )});- // Delete from queue if any deliveries succeeded- // or all the failed notifs were ios- if (any_succeeded || all_ios_failed)+ // Delete from the queue if any delivery succeeded, all failures were+ // iOS failures, or delivery was rejected by the rate limit. Rate-limit+ // rejection is terminal; retrying the same notification would only+ // deliver stale notifications after the limit window expires.+ if (any_succeeded || all_ios_failed || rate_limited)&& let Err(e) = self.queue.delete_message(&receipt_handle).await{- // push the failed delete to errors- results.push(Err(e))- } else if let Some(rate_limited) = rate_limited- && let Err(e) = self- .queue- // if we got rate limited, delay this message by the rate limit expiry time- .delay_message(&receipt_handle, rate_limited.retry_after)- .await- {- // push the failed delay to errorsresults.push(Err(e))}diff --git a/crates/notification/src/domain/service/test.rs b/crates/notification/src/domain/service/test.rsindex 0698e4fe10..2846a0790b 100644--- a/crates/notification/src/domain/service/test.rs+++ b/crates/notification/src/domain/service/test.rs@@ -693,14 +693,6 @@ impl NotificationQueue for MockQueue {async fn delete_message(&self, _receipt_handle: &str) -> Result<(), Report> {Ok(())}-- async fn delay_message(- &self,- _receipt_handle: &str,- _delay: std::time::Duration,- ) -> Result<(), Report> {- Ok(())- }}impl NotificationQueue for std::sync::Arc<MockQueue> {@@ -715,16 +707,8 @@ impl NotificationQueue for std::sync::Arc<MockQueue> {(**self).receive_messages().await}- async fn delete_message(&self, _receipt_handle: &str) -> Result<(), Report> {- (**self).delete_message(_receipt_handle).await- }-- async fn delay_message(- &self,- receipt_handle: &str,- delay: std::time::Duration,- ) -> Result<(), Report> {- (**self).delay_message(receipt_handle, delay).await+ async fn delete_message(&self, receipt_handle: &str) -> Result<(), Report> {+ (**self).delete_message(receipt_handle).await}}@@ -2129,14 +2113,6 @@ impl NotificationQueue for EgressTestQueue {.push(receipt_handle.to_string());Ok(())}-- async fn delay_message(- &self,- _receipt_handle: &str,- _delay: std::time::Duration,- ) -> Result<(), Report> {- Ok(())- }}/// WebSocket sender that hangs indefinitely (simulates a stuck connection).@@ -2177,6 +2153,44 @@ impl NotificationSender for FailingMobileSender {}}+#[tokio::test]+async fn test_poll_and_deliver_deletes_rate_limited_message() {+ let recipient = test_user_id("[email protected]");+ let email = EmailCreateBundle::new(&TestNotification {+ message: "Hello".to_string(),+ })+ .with_recipient(recipient);+ let message = QueueMessage::new_test(+ "test_notification".to_string(),+ NotificationChannel::Email(email),+ );++ let queue = EgressTestQueue::new(vec![RawQueueMessage {+ body: message,+ receipt_handle: "receipt-rate-limited".to_string(),+ }]);+ let service = NotificationEgressService {+ queue,+ repository: MockRepository::new(),+ websocket: MockWebSocketSender,+ mobile: MockMobileSender,+ email: MockEmailSender,+ rate_limiter: exceeding_rate_limiter(),+ state_machine: MockEgressStateMachine,+ digest_batcher: MockDigestBatcher,+ };++ let results = service.poll_and_deliver().await;++ assert_eq!(results.len(), 1);+ assert!(results[0].is_err());+ assert_eq!(+ service.queue.deleted_handles(),+ vec!["receipt-rate-limited"],+ "rate-limited messages must be deleted instead of retried"+ );+}+#[tokio::test]async fn test_poll_and_deliver_times_out_slow_delivery() {tokio::time::pause();diff --git a/crates/notification/src/outbound/queue.rs b/crates/notification/src/outbound/queue.rsindex 5646433433..31de749ec0 100644--- a/crates/notification/src/outbound/queue.rs+++ b/crates/notification/src/outbound/queue.rs@@ -92,21 +92,6 @@ impl NotificationQueue for SqsQueue {async fn delete_message(&self, receipt_handle: &str) -> Result<(), Report> {self.delete(receipt_handle).await}-- async fn delay_message(- &self,- receipt_handle: &str,- delay: std::time::Duration,- ) -> Result<(), Report> {- self.client- .change_message_visibility()- .queue_url(&self.queue_url)- .receipt_handle(receipt_handle)- .visibility_timeout(delay.as_secs() as i32)- .send()- .await?;- Ok(())- }}impl NotificationIngressQueue for SqsQueue {@@ -232,13 +217,4 @@ impl NotificationQueue for FileQueue {}Ok(())}-- async fn delay_message(- &self,- _receipt_handle: &str,- _delay: std::time::Duration,- ) -> Result<(), Report> {- // No-op for file-based queue.- Ok(())- }}diff --git a/tooling/notification_sandbox/src/adapters/mpsc_queue.rs b/tooling/notification_sandbox/src/adapters/mpsc_queue.rsindex 3a61b0bd0d..81b94f69f8 100644--- a/tooling/notification_sandbox/src/adapters/mpsc_queue.rs+++ b/tooling/notification_sandbox/src/adapters/mpsc_queue.rs@@ -72,13 +72,4 @@ impl NotificationQueue for MpscQueue {// No-op: messages are consumed from the channel on receive.Ok(())}-- async fn delay_message(- &self,- _receipt_handle: &str,- _delay: std::time::Duration,- ) -> Result<(), Report> {- // No-op: in-process queue doesn't support visibility changes.- Ok(())- }}