Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sns): pass messageDeduplicationId from FIFO topics to SQS FIFO queues (#171)

Benedikt Schneppe452982355716
diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
index b71c4c8f36..64ca829fa0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
@@ -274,12 +274,12 @@ public class SnsService {
}
boolean isFifo = "true".equals(topic.getAttributes().get("FifoTopic"));
+ String dedupId = messageDeduplicationId;
if (isFifo) {
if (messageGroupId == null || messageGroupId.isBlank()) {
throw new AwsException("InvalidParameter",
"MessageGroupId is required for FIFO topics.", 400);
}
- String dedupId = messageDeduplicationId;
if (dedupId == null && "true".equals(topic.getAttributes().get("ContentBasedDeduplication"))) {
dedupId = sha256(message);
}
@@ -298,7 +298,7 @@ public class SnsService {
if (!matchesFilterPolicy(sub, messageAttributes)) {
continue;
}
- deliverMessage(sub, message, subject, messageAttributes, messageId, effectiveArn, messageGroupId);
+ deliverMessage(sub, message, subject, messageAttributes, messageId, effectiveArn, messageGroupId, dedupId);
}
LOG.infov("Published message {0} to topic {1}", messageId, effectiveArn);
return messageId;
@@ -331,7 +331,8 @@ public class SnsService {
subscriptionStore.put(key, sub);
}
- public record BatchPublishResult(List<String[]> successful, List<String[]> failed) {}
+ public record BatchPublishResult(List<String[]> successful, List<String[]> failed) {
+ }
public BatchPublishResult publishBatch(String topicArn, List<Map<String, Object>> entries, String region) {
String topicStoreKey = topicKey(region, topicArn);
@@ -356,6 +357,10 @@ public class SnsService {
"MessageGroupId is required for FIFO topics.", "true"});
continue;
}
+ // Derive deduplication ID if ContentBasedDeduplication is enabled and not provided
+ if (isFifo && messageDeduplicationId == null && "true".equals(topic.getAttributes().get("ContentBasedDeduplication"))) {
+ messageDeduplicationId = sha256(message);
+ }
if (isFifo && messageDeduplicationId != null && isDuplicate(topicArn, messageDeduplicationId)) {
successful.add(new String[]{id, UUID.randomUUID().toString()});
continue;
@@ -367,7 +372,7 @@ public class SnsService {
for (Subscription sub : subscriptionsByTopic(topicArn, region)) {
if ("true".equals(sub.getAttributes().get("PendingConfirmation"))) continue;
if (!matchesFilterPolicy(sub, attrs)) continue;
- deliverMessage(sub, message, subject, attrs, messageId, topicArn, messageGroupId);
+ deliverMessage(sub, message, subject, attrs, messageId, topicArn, messageGroupId, messageDeduplicationId);
}
LOG.debugv("Batch published message {0} (id={1}) to {2}", messageId, id, topicArn);
successful.add(new String[]{id, messageId});
@@ -563,7 +568,7 @@ public class SnsService {
private void deliverMessage(Subscription sub, String message, String subject,
Map<String, MessageAttributeValue> messageAttributes, String messageId,
- String topicArn, String messageGroupId) {
+ String topicArn, String messageGroupId, String messageDeduplicationId) {
try {
switch (sub.getProtocol()) {
case "sqs" -> {
@@ -579,7 +584,7 @@ public class SnsService {
Map<String, MessageAttributeValue> sqsAttributes = rawDelivery
? toSqsMessageAttributes(messageAttributes)
: Collections.emptyMap();
- sqsService.sendMessage(queueUrl, body, 0, messageGroupId, null, sqsAttributes, region);
+ sqsService.sendMessage(queueUrl, body, 0, messageGroupId, messageDeduplicationId, sqsAttributes, region);
LOG.debugv("Delivered SNS message to SQS: {0} ({1}) raw={2}", sub.getEndpoint(), queueUrl, rawDelivery);
}
case "lambda" -> {
@@ -590,11 +595,9 @@ public class SnsService {
lambdaService.invoke(region, fnName, eventJson.getBytes(), InvocationType.Event);
LOG.debugv("Delivered SNS message to Lambda: {0}", sub.getEndpoint());
}
- case "email", "email-json" ->
- LOG.infov("SNS email delivery (stub): to={0}, subject={1}, message={2}",
- sub.getEndpoint(), subject, message);
- case "sms" ->
- LOG.infov("SNS SMS delivery (stub): to={0}, message={1}", sub.getEndpoint(), message);
+ case "email", "email-json" -> LOG.infov("SNS email delivery (stub): to={0}, subject={1}, message={2}",
+ sub.getEndpoint(), subject, message);
+ case "sms" -> LOG.infov("SNS SMS delivery (stub): to={0}, message={1}", sub.getEndpoint(), message);
default -> LOG.debugv("Protocol {0} delivery not implemented, skipping: {1}",
sub.getProtocol(), sub.getEndpoint());
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java
new file mode 100644
index 0000000000..eb7fb675b0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java
@@ -0,0 +1,167 @@
+package io.github.hectorvent.floci.services.sns;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.services.sqs.SqsServiceFactory;
+import io.github.hectorvent.floci.services.sqs.model.Message;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Tests that SNS FIFO topic delivery correctly passes messageDeduplicationId
+ * through to subscribed SQS FIFO queues.
+ */
+class SnsSqsFanoutFifoDeliveryTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "000000000000";
+ private static final String BASE_URL = "http://localhost:4566";
+
+ private SnsService snsService;
+ private SqsService sqsService;
+
+ @BeforeEach
+ void setUp() {
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);
+ sqsService = SqsServiceFactory.createInMemory(BASE_URL, regionResolver);
+ snsService = new SnsService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ regionResolver, sqsService, null);
+ }
+
+ @Test
+ void publish_withExplicitDedupId_deliversToFifoSqsQueue() {
+ // Arrange
+ sqsService.createQueue("fifo-queue.fifo", Map.of("FifoQueue", "true"), REGION);
+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-queue.fifo";
+
+ snsService.createTopic("fifo-topic.fifo", Map.of("FifoTopic", "true"), null, REGION);
+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-topic.fifo";
+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);
+
+ // Act
+ String messageId = snsService.publish(topicArn, null, null, "hello fifo",
+ null, null, "group-1", "my-dedup-id", REGION);
+
+ // Assert
+ assertNotNull(messageId);
+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-queue.fifo";
+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);
+ assertEquals(1, messages.size());
+ assertTrue(messages.get(0).getBody().contains("hello fifo"));
+ }
+
+ @Test
+ void publish_withTopicContentBasedDedup_deliversToFifoSqsQueue() {
+ // Arrange — topic has ContentBasedDeduplication, queue does NOT
+ sqsService.createQueue("fifo-cbd-queue.fifo", Map.of("FifoQueue", "true"), REGION);
+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-cbd-queue.fifo";
+
+ snsService.createTopic("fifo-cbd-topic.fifo",
+ Map.of("FifoTopic", "true", "ContentBasedDeduplication", "true"), null, REGION);
+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-cbd-topic.fifo";
+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);
+
+ // Act — no explicit dedup ID; topic derives one from message content
+ String messageId = snsService.publish(topicArn, null, null, "cbd message",
+ null, null, "group-1", null, REGION);
+
+ // Assert
+ assertNotNull(messageId);
+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-cbd-queue.fifo";
+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);
+ assertEquals(1, messages.size());
+ assertTrue(messages.get(0).getBody().contains("cbd message"));
+ }
+
+ @Test
+ void publishBatch_withExplicitDedupIds_deliversToFifoSqsQueue() {
+ // Arrange
+ sqsService.createQueue("fifo-batch-queue.fifo", Map.of("FifoQueue", "true"), REGION);
+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-batch-queue.fifo";
+
+ snsService.createTopic("fifo-batch-topic.fifo", Map.of("FifoTopic", "true"), null, REGION);
+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-batch-topic.fifo";
+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);
+
+ // Act
+ var entries = List.<Map<String, Object>>of(
+ Map.of("Id", "e1", "Message", "batch-msg-1",
+ "MessageGroupId", "group-a", "MessageDeduplicationId", "dedup-1"),
+ Map.of("Id", "e2", "Message", "batch-msg-2",
+ "MessageGroupId", "group-b", "MessageDeduplicationId", "dedup-2")
+ );
+ var result = snsService.publishBatch(topicArn, entries, REGION);
+
+ // Assert
+ assertEquals(2, result.successful().size());
+ assertEquals(0, result.failed().size());
+
+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-batch-queue.fifo";
+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);
+ assertEquals(2, messages.size());
+
+ List<String> bodies = messages.stream().map(Message::getBody).toList();
+ assertTrue(bodies.stream().anyMatch(b -> b.contains("batch-msg-1")));
+ assertTrue(bodies.stream().anyMatch(b -> b.contains("batch-msg-2")));
+ }
+
+ @Test
+ void publishBatch_withTopicContentBasedDedup_deliversToFifoSqsQueue() {
+ // Arrange — topic has ContentBasedDeduplication, queue does NOT
+ sqsService.createQueue("fifo-batch-cbd-queue.fifo", Map.of("FifoQueue", "true"), REGION);
+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-batch-cbd-queue.fifo";
+
+ snsService.createTopic("fifo-batch-cbd-topic.fifo",
+ Map.of("FifoTopic", "true", "ContentBasedDeduplication", "true"), null, REGION);
+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-batch-cbd-topic.fifo";
+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);
+
+ // Act — no explicit dedup IDs; topic derives them from message content
+ var entries = List.<Map<String, Object>>of(
+ Map.of("Id", "e1", "Message", "cbd-batch-msg-1", "MessageGroupId", "group-a"),
+ Map.of("Id", "e2", "Message", "cbd-batch-msg-2", "MessageGroupId", "group-b")
+ );
+ var result = snsService.publishBatch(topicArn, entries, REGION);
+
+ // Assert
+ assertEquals(2, result.successful().size());
+ assertEquals(0, result.failed().size());
+
+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-batch-cbd-queue.fifo";
+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);
+ assertEquals(2, messages.size());
+
+ List<String> bodies = messages.stream().map(Message::getBody).toList();
+ assertTrue(bodies.stream().anyMatch(b -> b.contains("cbd-batch-msg-1")));
+ assertTrue(bodies.stream().anyMatch(b -> b.contains("cbd-batch-msg-2")));
+ }
+
+ @Test
+ void publish_duplicateMessage_isDeduplicatedAtTopicLevel() {
+ // Arrange
+ sqsService.createQueue("fifo-dedup-queue.fifo", Map.of("FifoQueue", "true"), REGION);
+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":fifo-dedup-queue.fifo";
+
+ snsService.createTopic("fifo-dedup-topic.fifo", Map.of("FifoTopic", "true"), null, REGION);
+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":fifo-dedup-topic.fifo";
+ snsService.subscribe(topicArn, "sqs", queueArn, REGION);
+
+ // Act — publish same dedup ID twice
+ snsService.publish(topicArn, null, null, "first",
+ null, null, "group-1", "same-dedup", REGION);
+ snsService.publish(topicArn, null, null, "second",
+ null, null, "group-1", "same-dedup", REGION);
+
+ // Assert — only first message should be delivered
+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/fifo-dedup-queue.fifo";
+ List<Message> messages = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);
+ assertEquals(1, messages.size());
+ assertTrue(messages.get(0).getBody().contains("first"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java
new file mode 100644
index 0000000000..b46dec64fa
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.sqs;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+
+/**
+ * Test helper to create SqsService instances
+ */
+public class SqsServiceFactory {
+
+ public static SqsService createInMemory(String baseUrl, RegionResolver regionResolver) {
+ return new SqsService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 30, 262144, baseUrl, regionResolver);
+ }
+}