Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(notification): ingress drop invalid messages (#6929)

Will Hutchinsoncc2c5433e0ea
diff --git a/Cargo.lock b/Cargo.lock
index 4464c8b490..3fa84b6fd5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11799,6 +11799,7 @@ dependencies = [
"url",
"utoipa",
"uuid",
+ "wiremock",
"workspace-hack",
]
diff --git a/crates/notification/Cargo.toml b/crates/notification/Cargo.toml
index 7e0f171c40..77481d5d01 100644
--- a/crates/notification/Cargo.toml
+++ b/crates/notification/Cargo.toml
@@ -91,3 +91,4 @@ macro_db_migrator = { path = "../macro_db_migrator" }
rate_limit = { path = "../rate_limit", features = ["testing"] }
tokio = { workspace = true, features = ["macros", "rt", "test-util"] }
tower = { workspace = true }
+wiremock = { workspace = true }
diff --git a/crates/notification/src/outbound/queue.rs b/crates/notification/src/outbound/queue.rs
index b6074efb31..b614f93998 100644
--- a/crates/notification/src/outbound/queue.rs
+++ b/crates/notification/src/outbound/queue.rs
@@ -9,6 +9,9 @@ use crate::domain::models::queue_message::{
};
use crate::domain::ports::{NotificationIngressQueue, NotificationQueue};
+#[cfg(test)]
+mod test;
+
/// SQS-backed implementation of the notification queue ports.
///
/// A single type implements both [`NotificationQueue`] (delivery/egress queue)
@@ -109,22 +112,35 @@ impl NotificationIngressQueue for SqsQueue {
.send()
.await?;
- let messages = result
- .messages
- .unwrap_or_default()
- .into_iter()
- .filter_map(|msg| {
- let body_str = msg.body?;
- let body = serde_json::from_str(&body_str)
- .inspect_err(|e| tracing::error!(error=?e, payload_length=body_str.len(), "failed to deserialize ingress queue message"))
- .ok()?;
- let receipt_handle = msg.receipt_handle?;
- Some(RawIngressQueueMessage {
+ let mut messages = Vec::new();
+ for msg in result.messages.unwrap_or_default() {
+ let (Some(body_str), Some(receipt_handle)) = (msg.body, msg.receipt_handle) else {
+ continue;
+ };
+ match serde_json::from_str(&body_str) {
+ Ok(body) => messages.push(RawIngressQueueMessage {
body,
receipt_handle,
- })
- })
- .collect();
+ }),
+ Err(error) => {
+ tracing::warn!(
+ error = ?error,
+ message_id = ?msg.message_id,
+ payload_length = body_str.len(),
+ "failed to deserialize ingress queue message; discarding message"
+ );
+ // Acknowledge malformed payloads rather than retrying them into the DLQ.
+ // A failed delete must not prevent valid messages in this batch from processing.
+ let _ = self.delete(&receipt_handle).await.inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ message_id = ?msg.message_id,
+ "failed to delete malformed ingress queue message"
+ );
+ });
+ }
+ }
+ }
Ok(messages)
}
diff --git a/crates/notification/src/outbound/queue/test.rs b/crates/notification/src/outbound/queue/test.rs
new file mode 100644
index 0000000000..e345e54802
--- /dev/null
+++ b/crates/notification/src/outbound/queue/test.rs
@@ -0,0 +1,124 @@
+use super::*;
+use aws_sdk_sqs::config::{BehaviorVersion, Credentials, Region, retry::RetryConfig};
+use serde_json::json;
+use wiremock::matchers::{body_json, header, method};
+use wiremock::{Mock, MockServer, ResponseTemplate};
+
+use crate::domain::models::{Notification, SendNotificationRequestBuilder};
+
+#[derive(serde::Serialize, serde::Deserialize)]
+struct TestNotification;
+
+impl Notification for TestNotification {
+ const TYPE_NAME: &'static str = "test_notification";
+}
+
+fn valid_body() -> String {
+ let request = SendNotificationRequestBuilder {
+ notification_entity: model_entity::EntityType::Document.with_entity_str("doc_1"),
+ secondary_notification_entity: None,
+ notification: TestNotification,
+ sender_id: None,
+ recipient_ids: Default::default(),
+ }
+ .into_request()
+ .with_conn_gateway();
+ serde_json::to_string(&IngressQueueMessage::from_request(&request).unwrap()).unwrap()
+}
+
+async fn mock_queue(bodies: &[&str]) -> (MockServer, SqsQueue) {
+ let server = MockServer::start().await;
+ let config = aws_sdk_sqs::Config::builder()
+ .behavior_version(BehaviorVersion::latest())
+ .region(Region::new("us-east-1"))
+ .credentials_provider(Credentials::new("test", "test", None, None, "test"))
+ .endpoint_url(server.uri())
+ .retry_config(RetryConfig::disabled())
+ .build();
+ let queue = SqsQueue::new(SqsClient::from_conf(config), server.uri());
+ let messages: Vec<_> = bodies
+ .iter()
+ .enumerate()
+ .map(|(index, body)| {
+ json!({
+ "MessageId": format!("message-{index}"),
+ "ReceiptHandle": format!("receipt-{index}"),
+ "Body": body,
+ })
+ })
+ .collect();
+ Mock::given(method("POST"))
+ .and(header("x-amz-target", "AmazonSQS.ReceiveMessage"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "Messages": messages })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ (server, queue)
+}
+
+async fn expect_delete(server: &MockServer, index: usize, status: u16) {
+ Mock::given(method("POST"))
+ .and(header("x-amz-target", "AmazonSQS.DeleteMessage"))
+ .and(body_json(json!({
+ "QueueUrl": server.uri(),
+ "ReceiptHandle": format!("receipt-{index}"),
+ })))
+ .respond_with(ResponseTemplate::new(status).set_body_json(json!({})))
+ .expect(1)
+ .mount(server)
+ .await;
+}
+
+#[tokio::test]
+async fn ingress_acknowledges_malformed_messages_and_returns_ok() {
+ let missing_request = json!({
+ "message_type": "clear_push_notification",
+ "content": { "Ios": {} },
+ })
+ .to_string();
+ let (server, queue) = mock_queue(&[&missing_request, "not json"]).await;
+ expect_delete(&server, 0, 200).await;
+ expect_delete(&server, 1, 200).await;
+
+ let messages = NotificationIngressQueue::receive_messages(&queue)
+ .await
+ .unwrap();
+
+ assert!(messages.is_empty());
+ assert_eq!(server.received_requests().await.unwrap().len(), 3);
+ server.verify().await;
+}
+
+#[tokio::test]
+async fn ingress_keeps_valid_messages_in_a_mixed_batch_unacknowledged() {
+ let valid = valid_body();
+ let (server, queue) = mock_queue(&[&valid, "{}", &valid]).await;
+ expect_delete(&server, 1, 200).await;
+
+ let messages = NotificationIngressQueue::receive_messages(&queue)
+ .await
+ .unwrap();
+
+ assert_eq!(messages.len(), 2);
+ assert_eq!(messages[0].receipt_handle, "receipt-0");
+ assert_eq!(messages[1].receipt_handle, "receipt-2");
+ assert!(messages[0].body.request.send_conn_gateway);
+ assert_eq!(server.received_requests().await.unwrap().len(), 2);
+ server.verify().await;
+}
+
+#[tokio::test]
+async fn ingress_delete_failure_does_not_block_valid_messages() {
+ let valid = valid_body();
+ let (server, queue) = mock_queue(&["{}", &valid]).await;
+ expect_delete(&server, 0, 500).await;
+
+ let messages = NotificationIngressQueue::receive_messages(&queue)
+ .await
+ .unwrap();
+
+ assert_eq!(messages.len(), 1);
+ assert_eq!(messages[0].receipt_handle, "receipt-1");
+ assert_eq!(server.received_requests().await.unwrap().len(), 2);
+ server.verify().await;
+}
diff --git a/infra/stacks/notification-service/index.ts b/infra/stacks/notification-service/index.ts
index 33ddda6285..85642eb31e 100644
--- a/infra/stacks/notification-service/index.ts
+++ b/infra/stacks/notification-service/index.ts
@@ -81,6 +81,7 @@ const notificationQueue = new Queue('notification', {
const notificationIngressQueue = new Queue('notification-ingress', {
tags,
+ maxReceiveCount: 2,
});
const pushNotificationEventHandler = new PushNotificationEventHandler(