Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sns): pass messageDeduplicationId from FIFO topics to SQS FIFO queues (#171)
Benedikt Schneppe452982355716
diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.javaindex b71c4c8f36..64ca829fa0 100644--- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java@@ -274,12 +274,12 @@ public class SnsService {}boolean isFifo = "true".equals(topic.getAttributes().get("FifoTopic"));+ String dedupId = messageDeduplicationId;if (isFifo) {if (messageGroupId == null || messageGroupId.isBlank()) {throw new AwsException("InvalidParameter","MessageGroupId is required for FIFO topics.", 400);}- String dedupId = messageDeduplicationId;if (dedupId == null && "true".equals(topic.getAttributes().get("ContentBasedDeduplication"))) {dedupId = sha256(message);}@@ -298,7 +298,7 @@ public class SnsService {if (!matchesFilterPolicy(sub, messageAttributes)) {continue;}- deliverMessage(sub, message, subject, messageAttributes, messageId, effectiveArn, messageGroupId);+ deliverMessage(sub, message, subject, messageAttributes, messageId, effectiveArn, messageGroupId, dedupId);}LOG.infov("Published message {0} to topic {1}", messageId, effectiveArn);return messageId;@@ -331,7 +331,8 @@ public class SnsService {subscriptionStore.put(key, sub);}- public record BatchPublishResult(List<String[]> successful, List<String[]> failed) {}+ public record BatchPublishResult(List<String[]> successful, List<String[]> failed) {+ }public BatchPublishResult publishBatch(String topicArn, List<Map<String, Object>> entries, String region) {String topicStoreKey = topicKey(region, topicArn);@@ -356,6 +357,10 @@ public class SnsService {"MessageGroupId is required for FIFO topics.", "true"});continue;}+ // Derive deduplication ID if ContentBasedDeduplication is enabled and not provided+ if (isFifo && messageDeduplicationId == null && "true".equals(topic.getAttributes().get("ContentBasedDeduplication"))) {+ messageDeduplicationId = sha256(message);+ }if (isFifo && messageDeduplicationId != null && isDuplicate(topicArn, messageDeduplicationId)) {successful.add(new String[]{id, UUID.randomUUID().toString()});continue;@@ -367,7 +372,7 @@ public class SnsService {for (Subscription sub : subscriptionsByTopic(topicArn, region)) {if ("true".equals(sub.getAttributes().get("PendingConfirmation"))) continue;if (!matchesFilterPolicy(sub, attrs)) continue;- deliverMessage(sub, message, subject, attrs, messageId, topicArn, messageGroupId);+ deliverMessage(sub, message, subject, attrs, messageId, topicArn, messageGroupId, messageDeduplicationId);}LOG.debugv("Batch published message {0} (id={1}) to {2}", messageId, id, topicArn);successful.add(new String[]{id, messageId});@@ -563,7 +568,7 @@ public class SnsService {private void deliverMessage(Subscription sub, String message, String subject,Map<String, MessageAttributeValue> messageAttributes, String messageId,- String topicArn, String messageGroupId) {+ String topicArn, String messageGroupId, String messageDeduplicationId) {try {switch (sub.getProtocol()) {case "sqs" -> {@@ -579,7 +584,7 @@ public class SnsService {Map<String, MessageAttributeValue> sqsAttributes = rawDelivery? toSqsMessageAttributes(messageAttributes): Collections.emptyMap();- sqsService.sendMessage(queueUrl, body, 0, messageGroupId, null, sqsAttributes, region);+ sqsService.sendMessage(queueUrl, body, 0, messageGroupId, messageDeduplicationId, sqsAttributes, region);LOG.debugv("Delivered SNS message to SQS: {0} ({1}) raw={2}", sub.getEndpoint(), queueUrl, rawDelivery);}case "lambda" -> {@@ -590,11 +595,9 @@ public class SnsService {lambdaService.invoke(region, fnName, eventJson.getBytes(), InvocationType.Event);LOG.debugv("Delivered SNS message to Lambda: {0}", sub.getEndpoint());}- case "email", "email-json" ->- LOG.infov("SNS email delivery (stub): to={0}, subject={1}, message={2}",- sub.getEndpoint(), subject, message);- case "sms" ->- LOG.infov("SNS SMS delivery (stub): to={0}, message={1}", sub.getEndpoint(), message);+ case "email", "email-json" -> LOG.infov("SNS email delivery (stub): to={0}, subject={1}, message={2}",+ sub.getEndpoint(), subject, message);+ case "sms" -> LOG.infov("SNS SMS delivery (stub): to={0}, message={1}", sub.getEndpoint(), message);default -> LOG.debugv("Protocol {0} delivery not implemented, skipping: {1}",sub.getProtocol(), sub.getEndpoint());}diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.javanew file mode 100644index 0000000000..eb7fb675b0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java@@ -0,0 +1,167 @@+package io.github.hectorvent.floci.services.sns;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.SqsServiceFactory;+import io.github.hectorvent.floci.services.sqs.model.Message;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.*;++/**+ * Tests that SNS FIFO topic delivery correctly passes messageDeduplicationId+ * through to subscribed SQS FIFO queues.+ */+class SnsSqsFanoutFifoDeliveryTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";+ private static final String BASE_URL = "http://localhost:4566";++ private SnsService snsService;+ private SqsService sqsService;++ @BeforeEach+ void setUp() {+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);+ sqsService = SqsServiceFactory.createInMemory(BASE_URL, regionResolver);+ snsService = new SnsService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ regionResolver, sqsService, null);+ }++ @Test+ void publish_withExplicitDedupId_deliversToFifoSqsQueue() {+ // Arrange+ sqsService.createQueue("fifo-queue.fifo", Map.of("FifoQueue", "true"), REGION);+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-queue.fifo";++ snsService.createTopic("fifo-topic.fifo", Map.of("FifoTopic", "true"), null, REGION);+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-topic.fifo";+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);++ // Act+ String messageId = snsService.publish(topicArn, null, null, "hello fifo",+ null, null, "group-1", "my-dedup-id", REGION);++ // Assert+ assertNotNull(messageId);+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-queue.fifo";+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);+ assertEquals(1, messages.size());+ assertTrue(messages.get(0).getBody().contains("hello fifo"));+ }++ @Test+ void publish_withTopicContentBasedDedup_deliversToFifoSqsQueue() {+ // Arrange — topic has ContentBasedDeduplication, queue does NOT+ sqsService.createQueue("fifo-cbd-queue.fifo", Map.of("FifoQueue", "true"), REGION);+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-cbd-queue.fifo";++ snsService.createTopic("fifo-cbd-topic.fifo",+ Map.of("FifoTopic", "true", "ContentBasedDeduplication", "true"), null, REGION);+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-cbd-topic.fifo";+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);++ // Act — no explicit dedup ID; topic derives one from message content+ String messageId = snsService.publish(topicArn, null, null, "cbd message",+ null, null, "group-1", null, REGION);++ // Assert+ assertNotNull(messageId);+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-cbd-queue.fifo";+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);+ assertEquals(1, messages.size());+ assertTrue(messages.get(0).getBody().contains("cbd message"));+ }++ @Test+ void publishBatch_withExplicitDedupIds_deliversToFifoSqsQueue() {+ // Arrange+ sqsService.createQueue("fifo-batch-queue.fifo", Map.of("FifoQueue", "true"), REGION);+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-batch-queue.fifo";++ snsService.createTopic("fifo-batch-topic.fifo", Map.of("FifoTopic", "true"), null, REGION);+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-batch-topic.fifo";+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);++ // Act+ var entries = List.<Map<String, Object>>of(+ Map.of("Id", "e1", "Message", "batch-msg-1",+ "MessageGroupId", "group-a", "MessageDeduplicationId", "dedup-1"),+ Map.of("Id", "e2", "Message", "batch-msg-2",+ "MessageGroupId", "group-b", "MessageDeduplicationId", "dedup-2")+ );+ var result = snsService.publishBatch(topicArn, entries, REGION);++ // Assert+ assertEquals(2, result.successful().size());+ assertEquals(0, result.failed().size());++ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-batch-queue.fifo";+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);+ assertEquals(2, messages.size());++ List<String> bodies = messages.stream().map(Message::getBody).toList();+ assertTrue(bodies.stream().anyMatch(b -> b.contains("batch-msg-1")));+ assertTrue(bodies.stream().anyMatch(b -> b.contains("batch-msg-2")));+ }++ @Test+ void publishBatch_withTopicContentBasedDedup_deliversToFifoSqsQueue() {+ // Arrange — topic has ContentBasedDeduplication, queue does NOT+ sqsService.createQueue("fifo-batch-cbd-queue.fifo", Map.of("FifoQueue", "true"), REGION);+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-batch-cbd-queue.fifo";++ snsService.createTopic("fifo-batch-cbd-topic.fifo",+ Map.of("FifoTopic", "true", "ContentBasedDeduplication", "true"), null, REGION);+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-batch-cbd-topic.fifo";+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);++ // Act — no explicit dedup IDs; topic derives them from message content+ var entries = List.<Map<String, Object>>of(+ Map.of("Id", "e1", "Message", "cbd-batch-msg-1", "MessageGroupId", "group-a"),+ Map.of("Id", "e2", "Message", "cbd-batch-msg-2", "MessageGroupId", "group-b")+ );+ var result = snsService.publishBatch(topicArn, entries, REGION);++ // Assert+ assertEquals(2, result.successful().size());+ assertEquals(0, result.failed().size());++ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-batch-cbd-queue.fifo";+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);+ assertEquals(2, messages.size());++ List<String> bodies = messages.stream().map(Message::getBody).toList();+ assertTrue(bodies.stream().anyMatch(b -> b.contains("cbd-batch-msg-1")));+ assertTrue(bodies.stream().anyMatch(b -> b.contains("cbd-batch-msg-2")));+ }++ @Test+ void publish_duplicateMessage_isDeduplicatedAtTopicLevel() {+ // Arrange+ sqsService.createQueue("fifo-dedup-queue.fifo", Map.of("FifoQueue", "true"), REGION);+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-dedup-queue.fifo";++ snsService.createTopic("fifo-dedup-topic.fifo", Map.of("FifoTopic", "true"), null, REGION);+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-dedup-topic.fifo";+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);++ // Act — publish same dedup ID twice+ snsService.publish(topicArn, null, null, "first",+ null, null, "group-1", "same-dedup", REGION);+ snsService.publish(topicArn, null, null, "second",+ null, null, "group-1", "same-dedup", REGION);++ // Assert — only first message should be delivered+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-dedup-queue.fifo";+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);+ assertEquals(1, messages.size());+ assertTrue(messages.get(0).getBody().contains("first"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.javanew file mode 100644index 0000000000..b46dec64fa--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java@@ -0,0 +1,15 @@+package io.github.hectorvent.floci.services.sqs;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;++/**+ * Test helper to create SqsService instances+ */+public class SqsServiceFactory {++ public static SqsService createInMemory(String baseUrl, RegionResolver regionResolver) {+ return new SqsService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),+ 30, 262144, baseUrl, regionResolver);+ }+}