Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(notification): ingress drop invalid messages (#6929)
Will Hutchinsoncc2c5433e0ea
diff --git a/Cargo.lock b/Cargo.lockindex 4464c8b490..3fa84b6fd5 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -11799,6 +11799,7 @@ dependencies = ["url","utoipa","uuid",+ "wiremock","workspace-hack",]diff --git a/crates/notification/Cargo.toml b/crates/notification/Cargo.tomlindex 7e0f171c40..77481d5d01 100644--- a/crates/notification/Cargo.toml+++ b/crates/notification/Cargo.toml@@ -91,3 +91,4 @@ macro_db_migrator = { path = "../macro_db_migrator" }rate_limit = { path = "../rate_limit", features = ["testing"] }tokio = { workspace = true, features = ["macros", "rt", "test-util"] }tower = { workspace = true }+wiremock = { workspace = true }diff --git a/crates/notification/src/outbound/queue.rs b/crates/notification/src/outbound/queue.rsindex b6074efb31..b614f93998 100644--- a/crates/notification/src/outbound/queue.rs+++ b/crates/notification/src/outbound/queue.rs@@ -9,6 +9,9 @@ use crate::domain::models::queue_message::{};use crate::domain::ports::{NotificationIngressQueue, NotificationQueue};+#[cfg(test)]+mod test;+/// SQS-backed implementation of the notification queue ports.////// A single type implements both [`NotificationQueue`] (delivery/egress queue)@@ -109,22 +112,35 @@ impl NotificationIngressQueue for SqsQueue {.send().await?;- let messages = result- .messages- .unwrap_or_default()- .into_iter()- .filter_map(|msg| {- let body_str = msg.body?;- let body = serde_json::from_str(&body_str)- .inspect_err(|e| tracing::error!(error=?e, payload_length=body_str.len(), "failed to deserialize ingress queue message"))- .ok()?;- let receipt_handle = msg.receipt_handle?;- Some(RawIngressQueueMessage {+ let mut messages = Vec::new();+ for msg in result.messages.unwrap_or_default() {+ let (Some(body_str), Some(receipt_handle)) = (msg.body, msg.receipt_handle) else {+ continue;+ };+ match serde_json::from_str(&body_str) {+ Ok(body) => messages.push(RawIngressQueueMessage {body,receipt_handle,- })- })- .collect();+ }),+ Err(error) => {+ tracing::warn!(+ error = ?error,+ message_id = ?msg.message_id,+ payload_length = body_str.len(),+ "failed to deserialize ingress queue message; discarding message"+ );+ // Acknowledge malformed payloads rather than retrying them into the DLQ.+ // A failed delete must not prevent valid messages in this batch from processing.+ let _ = self.delete(&receipt_handle).await.inspect_err(|error| {+ tracing::error!(+ error = ?error,+ message_id = ?msg.message_id,+ "failed to delete malformed ingress queue message"+ );+ });+ }+ }+ }Ok(messages)}diff --git a/crates/notification/src/outbound/queue/test.rs b/crates/notification/src/outbound/queue/test.rsnew file mode 100644index 0000000000..e345e54802--- /dev/null+++ b/crates/notification/src/outbound/queue/test.rs@@ -0,0 +1,124 @@+use super::*;+use aws_sdk_sqs::config::{BehaviorVersion, Credentials, Region, retry::RetryConfig};+use serde_json::json;+use wiremock::matchers::{body_json, header, method};+use wiremock::{Mock, MockServer, ResponseTemplate};++use crate::domain::models::{Notification, SendNotificationRequestBuilder};++#[derive(serde::Serialize, serde::Deserialize)]+struct TestNotification;++impl Notification for TestNotification {+ const TYPE_NAME: &'static str = "test_notification";+}++fn valid_body() -> String {+ let request = SendNotificationRequestBuilder {+ notification_entity: model_entity::EntityType::Document.with_entity_str("doc_1"),+ secondary_notification_entity: None,+ notification: TestNotification,+ sender_id: None,+ recipient_ids: Default::default(),+ }+ .into_request()+ .with_conn_gateway();+ serde_json::to_string(&IngressQueueMessage::from_request(&request).unwrap()).unwrap()+}++async fn mock_queue(bodies: &[&str]) -> (MockServer, SqsQueue) {+ let server = MockServer::start().await;+ let config = aws_sdk_sqs::Config::builder()+ .behavior_version(BehaviorVersion::latest())+ .region(Region::new("us-east-1"))+ .credentials_provider(Credentials::new("test", "test", None, None, "test"))+ .endpoint_url(server.uri())+ .retry_config(RetryConfig::disabled())+ .build();+ let queue = SqsQueue::new(SqsClient::from_conf(config), server.uri());+ let messages: Vec<_> = bodies+ .iter()+ .enumerate()+ .map(|(index, body)| {+ json!({+ "MessageId": format!("message-{index}"),+ "ReceiptHandle": format!("receipt-{index}"),+ "Body": body,+ })+ })+ .collect();+ Mock::given(method("POST"))+ .and(header("x-amz-target", "AmazonSQS.ReceiveMessage"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "Messages": messages })))+ .expect(1)+ .mount(&server)+ .await;+ (server, queue)+}++async fn expect_delete(server: &MockServer, index: usize, status: u16) {+ Mock::given(method("POST"))+ .and(header("x-amz-target", "AmazonSQS.DeleteMessage"))+ .and(body_json(json!({+ "QueueUrl": server.uri(),+ "ReceiptHandle": format!("receipt-{index}"),+ })))+ .respond_with(ResponseTemplate::new(status).set_body_json(json!({})))+ .expect(1)+ .mount(server)+ .await;+}++#[tokio::test]+async fn ingress_acknowledges_malformed_messages_and_returns_ok() {+ let missing_request = json!({+ "message_type": "clear_push_notification",+ "content": { "Ios": {} },+ })+ .to_string();+ let (server, queue) = mock_queue(&[&missing_request, "not json"]).await;+ expect_delete(&server, 0, 200).await;+ expect_delete(&server, 1, 200).await;++ let messages = NotificationIngressQueue::receive_messages(&queue)+ .await+ .unwrap();++ assert!(messages.is_empty());+ assert_eq!(server.received_requests().await.unwrap().len(), 3);+ server.verify().await;+}++#[tokio::test]+async fn ingress_keeps_valid_messages_in_a_mixed_batch_unacknowledged() {+ let valid = valid_body();+ let (server, queue) = mock_queue(&[&valid, "{}", &valid]).await;+ expect_delete(&server, 1, 200).await;++ let messages = NotificationIngressQueue::receive_messages(&queue)+ .await+ .unwrap();++ assert_eq!(messages.len(), 2);+ assert_eq!(messages[0].receipt_handle, "receipt-0");+ assert_eq!(messages[1].receipt_handle, "receipt-2");+ assert!(messages[0].body.request.send_conn_gateway);+ assert_eq!(server.received_requests().await.unwrap().len(), 2);+ server.verify().await;+}++#[tokio::test]+async fn ingress_delete_failure_does_not_block_valid_messages() {+ let valid = valid_body();+ let (server, queue) = mock_queue(&["{}", &valid]).await;+ expect_delete(&server, 0, 500).await;++ let messages = NotificationIngressQueue::receive_messages(&queue)+ .await+ .unwrap();++ assert_eq!(messages.len(), 1);+ assert_eq!(messages[0].receipt_handle, "receipt-1");+ assert_eq!(server.received_requests().await.unwrap().len(), 2);+ server.verify().await;+}diff --git a/infra/stacks/notification-service/index.ts b/infra/stacks/notification-service/index.tsindex 33ddda6285..85642eb31e 100644--- a/infra/stacks/notification-service/index.ts+++ b/infra/stacks/notification-service/index.ts@@ -81,6 +81,7 @@ const notificationQueue = new Queue('notification', {const notificationIngressQueue = new Queue('notification-ingress', {tags,+ maxReceiveCount: 2,});const pushNotificationEventHandler = new PushNotificationEventHandler(