Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(notification): drop rate-limited queue messages (#5201)

Sean Aye22c7035184c2
diff --git a/crates/ai_tools/src/tool_context.rs b/crates/ai_tools/src/tool_context.rs
index 78f47e85cc..143467c9e7 100644
--- a/crates/ai_tools/src/tool_context.rs
+++ b/crates/ai_tools/src/tool_context.rs
@@ -562,24 +562,6 @@ impl notification::domain::ports::NotificationQueue for ToolNotificationQueue {
ToolNotificationQueue::NoOp => Ok(()),
}
}
-
- async fn delay_message(
- &self,
- receipt_handle: &str,
- delay: std::time::Duration,
- ) -> Result<(), rootcause::Report> {
- match self {
- ToolNotificationQueue::Sqs(queue) => {
- notification::domain::ports::NotificationQueue::delay_message(
- queue,
- receipt_handle,
- delay,
- )
- .await
- }
- ToolNotificationQueue::NoOp => Ok(()),
- }
- }
}
/// Type alias for the entity access management service implementation used by AI tools
diff --git a/crates/notification/src/domain/ports.rs b/crates/notification/src/domain/ports.rs
index 20615e782f..6f6d014b89 100644
--- a/crates/notification/src/domain/ports.rs
+++ b/crates/notification/src/domain/ports.rs
@@ -315,21 +315,11 @@ pub trait NotificationQueue: Send + Sync + 'static {
fn receive_messages(&self)
-> impl Future<Output = Result<Vec<RawQueueMessage>, Report>> + Send;
- /// Delete a message from the queue (after successful delivery).
+ /// Delete a message from the queue after terminal processing.
fn delete_message(
&self,
receipt_handle: &str,
) -> impl Future<Output = Result<(), Report>> + Send;
-
- /// Delay a message by changing its visibility timeout.
- ///
- /// The message will not be redelivered to consumers until the duration elapses.
- /// Uses SQS `ChangeMessageVisibility` under the hood.
- fn delay_message(
- &self,
- receipt_handle: &str,
- delay: std::time::Duration,
- ) -> impl Future<Output = Result<(), Report>> + Send;
}
/// Port for delivering notifications from the queue.
@@ -340,7 +330,8 @@ pub trait NotificationEgress: Send + Sync + 'static {
/// Poll the queue and attempt to deliver notifications.
///
/// Returns results for each delivery attempt across all messages received.
- /// Messages are automatically deleted from the queue after successful delivery.
+ /// Messages are deleted from the queue after terminal outcomes, including
+ /// successful delivery and rate-limit rejection.
fn poll_and_deliver(&self)
-> impl Future<Output = Vec<Result<DeliverySuccess, Report>>> + Send;
diff --git a/crates/notification/src/domain/service/device/test.rs b/crates/notification/src/domain/service/device/test.rs
index 5eeaf78cb6..cfc5ba2d97 100644
--- a/crates/notification/src/domain/service/device/test.rs
+++ b/crates/notification/src/domain/service/device/test.rs
@@ -357,14 +357,6 @@ impl NotificationQueue for MockQueue {
async fn delete_message(&self, _: &str) -> Result<(), Report> {
Ok(())
}
-
- async fn delay_message(
- &self,
- _receipt_handle: &str,
- _delay: std::time::Duration,
- ) -> Result<(), Report> {
- Ok(())
- }
}
fn test_config() -> PlatformArnConfig {
diff --git a/crates/notification/src/domain/service/egress.rs b/crates/notification/src/domain/service/egress.rs
index 80f0d986be..bf2c126d35 100644
--- a/crates/notification/src/domain/service/egress.rs
+++ b/crates/notification/src/domain/service/egress.rs
@@ -302,31 +302,20 @@ where
let all_ios_failed = delivery_results.iter().all(
|e| matches!(e, Err(e) if matches!(e.current_context(), DeliveryFailure::Ios )),
);
- let rate_limited = delivery_results.iter().find_map(|e| {
- match e.as_ref().map_err(Report::current_context) {
- Err(DeliveryFailure::RateLimit(rate_limit)) => Some(rate_limit),
- Ok(_)
- | Err(
- DeliveryFailure::Ios | DeliveryFailure::Other | DeliveryFailure::Timeout,
- ) => None,
- }
+ let rate_limited = delivery_results.iter().any(|e| {
+ matches!(
+ e.as_ref().map_err(Report::current_context),
+ Err(DeliveryFailure::RateLimit(_))
+ )
});
- // Delete from queue if any deliveries succeeded
- // or all the failed notifs were ios
- if (any_succeeded || all_ios_failed)
+ // Delete from the queue if any delivery succeeded, all failures were
+ // iOS failures, or delivery was rejected by the rate limit. Rate-limit
+ // rejection is terminal; retrying the same notification would only
+ // deliver stale notifications after the limit window expires.
+ if (any_succeeded || all_ios_failed || rate_limited)
&& let Err(e) = self.queue.delete_message(&receipt_handle).await
{
- // push the failed delete to errors
- results.push(Err(e))
- } else if let Some(rate_limited) = rate_limited
- && let Err(e) = self
- .queue
- // if we got rate limited, delay this message by the rate limit expiry time
- .delay_message(&receipt_handle, rate_limited.retry_after)
- .await
- {
- // push the failed delay to errors
results.push(Err(e))
}
diff --git a/crates/notification/src/domain/service/test.rs b/crates/notification/src/domain/service/test.rs
index 0698e4fe10..2846a0790b 100644
--- a/crates/notification/src/domain/service/test.rs
+++ b/crates/notification/src/domain/service/test.rs
@@ -693,14 +693,6 @@ impl NotificationQueue for MockQueue {
async fn delete_message(&self, _receipt_handle: &str) -> Result<(), Report> {
Ok(())
}
-
- async fn delay_message(
- &self,
- _receipt_handle: &str,
- _delay: std::time::Duration,
- ) -> Result<(), Report> {
- Ok(())
- }
}
impl NotificationQueue for std::sync::Arc<MockQueue> {
@@ -715,16 +707,8 @@ impl NotificationQueue for std::sync::Arc<MockQueue> {
(**self).receive_messages().await
}
- async fn delete_message(&self, _receipt_handle: &str) -> Result<(), Report> {
- (**self).delete_message(_receipt_handle).await
- }
-
- async fn delay_message(
- &self,
- receipt_handle: &str,
- delay: std::time::Duration,
- ) -> Result<(), Report> {
- (**self).delay_message(receipt_handle, delay).await
+ async fn delete_message(&self, receipt_handle: &str) -> Result<(), Report> {
+ (**self).delete_message(receipt_handle).await
}
}
@@ -2129,14 +2113,6 @@ impl NotificationQueue for EgressTestQueue {
.push(receipt_handle.to_string());
Ok(())
}
-
- async fn delay_message(
- &self,
- _receipt_handle: &str,
- _delay: std::time::Duration,
- ) -> Result<(), Report> {
- Ok(())
- }
}
/// WebSocket sender that hangs indefinitely (simulates a stuck connection).
@@ -2177,6 +2153,44 @@ impl NotificationSender for FailingMobileSender {
}
}
+#[tokio::test]
+async fn test_poll_and_deliver_deletes_rate_limited_message() {
+ let recipient = test_user_id("[email protected]");
+ let email = EmailCreateBundle::new(&TestNotification {
+ message: "Hello".to_string(),
+ })
+ .with_recipient(recipient);
+ let message = QueueMessage::new_test(
+ "test_notification".to_string(),
+ NotificationChannel::Email(email),
+ );
+
+ let queue = EgressTestQueue::new(vec![RawQueueMessage {
+ body: message,
+ receipt_handle: "receipt-rate-limited".to_string(),
+ }]);
+ let service = NotificationEgressService {
+ queue,
+ repository: MockRepository::new(),
+ websocket: MockWebSocketSender,
+ mobile: MockMobileSender,
+ email: MockEmailSender,
+ rate_limiter: exceeding_rate_limiter(),
+ state_machine: MockEgressStateMachine,
+ digest_batcher: MockDigestBatcher,
+ };
+
+ let results = service.poll_and_deliver().await;
+
+ assert_eq!(results.len(), 1);
+ assert!(results[0].is_err());
+ assert_eq!(
+ service.queue.deleted_handles(),
+ vec!["receipt-rate-limited"],
+ "rate-limited messages must be deleted instead of retried"
+ );
+}
+
#[tokio::test]
async fn test_poll_and_deliver_times_out_slow_delivery() {
tokio::time::pause();
diff --git a/crates/notification/src/outbound/queue.rs b/crates/notification/src/outbound/queue.rs
index 5646433433..31de749ec0 100644
--- a/crates/notification/src/outbound/queue.rs
+++ b/crates/notification/src/outbound/queue.rs
@@ -92,21 +92,6 @@ impl NotificationQueue for SqsQueue {
async fn delete_message(&self, receipt_handle: &str) -> Result<(), Report> {
self.delete(receipt_handle).await
}
-
- async fn delay_message(
- &self,
- receipt_handle: &str,
- delay: std::time::Duration,
- ) -> Result<(), Report> {
- self.client
- .change_message_visibility()
- .queue_url(&self.queue_url)
- .receipt_handle(receipt_handle)
- .visibility_timeout(delay.as_secs() as i32)
- .send()
- .await?;
- Ok(())
- }
}
impl NotificationIngressQueue for SqsQueue {
@@ -232,13 +217,4 @@ impl NotificationQueue for FileQueue {
}
Ok(())
}
-
- async fn delay_message(
- &self,
- _receipt_handle: &str,
- _delay: std::time::Duration,
- ) -> Result<(), Report> {
- // No-op for file-based queue.
- Ok(())
- }
}
diff --git a/tooling/notification_sandbox/src/adapters/mpsc_queue.rs b/tooling/notification_sandbox/src/adapters/mpsc_queue.rs
index 3a61b0bd0d..81b94f69f8 100644
--- a/tooling/notification_sandbox/src/adapters/mpsc_queue.rs
+++ b/tooling/notification_sandbox/src/adapters/mpsc_queue.rs
@@ -72,13 +72,4 @@ impl NotificationQueue for MpscQueue {
// No-op: messages are consumed from the channel on receive.
Ok(())
}
-
- async fn delay_message(
- &self,
- _receipt_handle: &str,
- _delay: std::time::Duration,
- ) -> Result<(), Report> {
- // No-op: in-process queue doesn't support visibility changes.
- Ok(())
- }
}