Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sns): retry failed sqs subscription deliveries (#4468)
vamsiakf3dde75c0b17
diff --git a/docs/services/sns.md b/docs/services/sns.mdindex 01739a3195..88a5a20ef7 100644--- a/docs/services/sns.md+++ b/docs/services/sns.md@@ -93,6 +93,12 @@ Supported subscription protocols:For `FilterPolicyScope=MessageBody`, nested policy objects descend into JSON objects and arrays.An object inside an array matches when one array element satisfies the complete nested policy.+For SQS subscriptions, Floci retries a failed delivery up to three times total, with two immediate+retries. If all attempts fail and the subscription has a `RedrivePolicy` containing a valid+`deadLetterTargetArn` for an SQS queue, Floci sends the original notification to that queue. SNS+`Publish` still returns its message ID after downstream delivery fails. This is a bounded local+policy; it does not reproduce AWS delivery retry timing or scale.+## HTTP and HTTPS endpoint addressesFloci posts the `SubscriptionConfirmation` and every notification to a subscribed `http`/`https`diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.javaindex e56ab7adaf..0af76f92f1 100644--- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java@@ -68,6 +68,9 @@ public class SnsService implements Resettable, ResourceProvider {private static final Logger LOG = Logger.getLogger(SnsService.class);private static final Duration FIFO_DEDUP_WINDOW = Duration.ofMinutes(5);+ /** Includes the original send and two immediate retries for local SQS fan-out. */+ private static final int SQS_SUBSCRIPTION_DELIVERY_ATTEMPTS = 3;+ private static final String SUBSCRIPTION_REDRIVE_POLICY = "RedrivePolicy";/** Default value of the {@code MaximumMessageSize} topic attribute, and the ceiling below* which a topic carries no subscription restrictions. AWS raised the maximum to 1 MiB in* September 2026 but left the default at 256 KiB, so an unconfigured topic is unchanged. */@@ -1730,8 +1733,8 @@ public class SnsService implements Resettable, ResourceProvider {Map<String, MessageAttributeValue> sqsAttributes = rawDelivery? toSqsMessageAttributes(messageAttributes): Collections.emptyMap();- sqsService.sendMessage(queueUrl, body, null, messageGroupId, messageDeduplicationId, sqsAttributes, region);- LOG.debugv("Delivered SNS message to SQS: {0} ({1}) raw={2}", sub.getEndpoint(), queueUrl, rawDelivery);+ deliverToSqsSubscription(sub, queueUrl, body, messageId, messageGroupId,+ messageDeduplicationId, sqsAttributes, region, rawDelivery);}case "lambda" -> {String region = extractRegionFromArn(sub.getEndpoint());@@ -1846,6 +1849,73 @@ public class SnsService implements Resettable, ResourceProvider {}}+ private void deliverToSqsSubscription(Subscription sub, String queueUrl, String body, String messageId,+ String messageGroupId, String messageDeduplicationId,+ Map<String, MessageAttributeValue> sqsAttributes,+ String region, boolean rawDelivery) {+ RuntimeException deliveryFailure = null;+ for (int attempt = 1; attempt <= SQS_SUBSCRIPTION_DELIVERY_ATTEMPTS; attempt++) {+ try {+ sqsService.sendMessage(queueUrl, body, null, messageGroupId,+ messageDeduplicationId, sqsAttributes, region);+ LOG.debugv("Delivered SNS message to SQS: {0} ({1}) raw={2}",+ sub.getEndpoint(), queueUrl, rawDelivery);+ return;+ } catch (RuntimeException e) {+ deliveryFailure = e;+ }+ }++ String deadLetterTargetArn = subscriptionDeadLetterTargetArn(sub);+ if (deadLetterTargetArn != null) {+ try {+ String deadLetterRegion = extractRegionFromArn(deadLetterTargetArn);+ if (deadLetterRegion == null) {+ deadLetterRegion = region;+ }+ boolean fifoDeadLetterQueue = AwsArnUtils.parse(deadLetterTargetArn)+ .resource().endsWith(".fifo");+ String deadLetterGroupId = messageGroupId;+ String deadLetterDeduplicationId = messageDeduplicationId;+ if (fifoDeadLetterQueue) {+ if (deadLetterGroupId == null || deadLetterGroupId.isBlank()) {+ deadLetterGroupId = messageId;+ }+ // Each failed subscription is a distinct DLQ delivery, even when+ // several subscriptions share this queue for the same publish.+ deadLetterDeduplicationId = sha256(messageId + "\0" + sub.getSubscriptionArn());+ }+ sqsService.sendMessage(sqsArnToUrl(deadLetterTargetArn), body, null,+ deadLetterGroupId, deadLetterDeduplicationId, sqsAttributes, deadLetterRegion);+ LOG.warnv("SNS delivery to {0} failed after {1} attempts; sent notification to subscription DLQ {2}",+ sub.getEndpoint(), SQS_SUBSCRIPTION_DELIVERY_ATTEMPTS, deadLetterTargetArn);+ return;+ } catch (RuntimeException deadLetterFailure) {+ LOG.warnv("SNS delivery to {0} and subscription DLQ {1} failed: {2}",+ sub.getEndpoint(), deadLetterTargetArn, deadLetterFailure.getMessage());+ }+ }+ LOG.warnv("SNS delivery to {0} failed after {1} attempts: {2}",+ sub.getEndpoint(), SQS_SUBSCRIPTION_DELIVERY_ATTEMPTS,+ deliveryFailure == null ? "unknown failure" : deliveryFailure.getMessage());+ }++ private String subscriptionDeadLetterTargetArn(Subscription sub) {+ String redrivePolicy = sub.getAttributes().get(SUBSCRIPTION_REDRIVE_POLICY);+ if (redrivePolicy == null || redrivePolicy.isBlank()) {+ return null;+ }+ try {+ JsonNode policy = objectMapper.readTree(redrivePolicy);+ String targetArn = policy.path("deadLetterTargetArn").asText(null);+ return AwsArnUtils.isArnFor(targetArn, "sqs") ? targetArn : null;+ } catch (Exception e) {+ LOG.warnv("Ignoring invalid SNS subscription RedrivePolicy for {0}: {1}",+ sub.getSubscriptionArn(), e.getMessage());+ return null;+ }+ }+private String buildSnsLambdaEvent(String topicArn, String messageId, String message,String subject, Map<String, MessageAttributeValue> messageAttributes,String subscriptionArn) {diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.javaindex 2f71271d71..0968a53604 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java@@ -54,8 +54,14 @@ class GuardedMessageQueue {void addMessage(Message message) {try (Guard _ = hold()) {+ int mark = messages.size();messages.add(message);- persist();+ try {+ persist();+ } catch (RuntimeException e) {+ messages.remove(mark);+ throw e;+ }}}diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.javaindex 11c3f88a51..6434eb8a45 100644--- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java@@ -11,6 +11,8 @@ import io.github.hectorvent.floci.services.lambda.LambdaService;import io.github.hectorvent.floci.services.lambda.model.InvocationType;import io.github.hectorvent.floci.services.sns.model.Subscription;import io.github.hectorvent.floci.services.sns.model.Topic;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.model.Message;import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;@@ -24,9 +26,13 @@ import java.util.Map;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyMap;+import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.ArgumentMatchers.isNull;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;class SnsServiceTest {@@ -80,6 +86,55 @@ class SnsServiceTest {assertThrows(AwsException.class, () -> snsService.createTopic("", null, null, REGION));}+ @Test+ void publish_retriesTransientSqsSubscriptionDeliveryFailure() {+ SqsService sqs = mock(SqsService.class);+ SnsService service = new SnsService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver(REGION, ACCOUNT), sqs, null);+ Topic topic = service.createTopic("retry-topic", null, null, REGION);+ String queueArn = "arn:aws:sqs:us-east-1:000000000000:retry-queue";+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/retry-queue";+ service.subscribe(topic.getTopicArn(), "sqs", queueArn, REGION, Map.of());+ doThrow(new AwsException("ServiceUnavailable", "temporarily unavailable", 503))+ .doReturn(new Message("delivered"))+ .when(sqs).sendMessage(eq(queueUrl), anyString(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));++ String messageId = service.publish(topic.getTopicArn(), null, "payload", null, REGION);++ assertNotNull(messageId);+ verify(sqs, times(2)).sendMessage(eq(queueUrl), anyString(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));+ }++ @Test+ void publish_sendsExhaustedSqsSubscriptionDeliveryToConfiguredDlq() throws Exception {+ SqsService sqs = mock(SqsService.class);+ SnsService service = new SnsService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver(REGION, ACCOUNT), sqs, null);+ Topic topic = service.createTopic("dlq-topic", null, null, REGION);+ String queueArn = "arn:aws:sqs:us-east-1:000000000000:unavailable-queue";+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/unavailable-queue";+ String dlqArn = "arn:aws:sqs:us-east-1:000000000000:subscription-dlq";+ String dlqUrl = BASE_URL + "/" + ACCOUNT + "/subscription-dlq";+ service.subscribe(topic.getTopicArn(), "sqs", queueArn, REGION,+ Map.of("RedrivePolicy", "{\"deadLetterTargetArn\":\"" + dlqArn + "\"}"));+ doThrow(new AwsException("AWS.SimpleQueueService.NonExistentQueue", "unavailable", 400))+ .when(sqs).sendMessage(eq(queueUrl), anyString(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));++ String messageId = service.publish(topic.getTopicArn(), null,+ "payload", "subject", REGION);++ assertNotNull(messageId, "SNS Publish acceptance is independent of downstream SQS delivery");+ verify(sqs, times(3)).sendMessage(eq(queueUrl), anyString(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));+ ArgumentCaptor<String> deadLetterBody = ArgumentCaptor.forClass(String.class);+ verify(sqs).sendMessage(eq(dlqUrl), deadLetterBody.capture(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));+ JsonNode envelope = new ObjectMapper().readTree(deadLetterBody.getValue());+ assertEquals("Notification", envelope.path("Type").asText());+ assertEquals(topic.getTopicArn(), envelope.path("TopicArn").asText());+ assertEquals("payload", envelope.path("Message").asText());+ assertEquals("subject", envelope.path("Subject").asText());+ assertTrue(envelope.hasNonNull("MessageId"));+ }+@Testvoid listTopics_returnsCreatedTopics() {snsService.createTopic("topic-a", null, null, REGION);diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsSubscriptionDlqIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsSubscriptionDlqIntegrationTest.javanew file mode 100644index 0000000000..3490b1fe51--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsSubscriptionDlqIntegrationTest.java@@ -0,0 +1,123 @@+package io.github.hectorvent.floci.services.sns;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.sns.model.Subscription;+import io.github.hectorvent.floci.services.sns.model.Topic;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.SqsServiceFactory;+import io.github.hectorvent.floci.services.sqs.model.Message;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++class SnsSqsSubscriptionDlqIntegrationTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";+ private static final String BASE_URL = "http://localhost:4566";++ private SnsService snsService;+ private SqsService sqsService;++ @BeforeEach+ void setUp() {+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);+ sqsService = SqsServiceFactory.createInMemory(BASE_URL, regionResolver);+ snsService = new SnsService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ regionResolver, sqsService, null);+ }++ @Test+ void publish_routesExhaustedDeliveryToSubscriptionDlq() throws Exception {+ String dlqName = "sns-subscription-dlq";+ String dlqArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":" + dlqName;+ String dlqUrl = BASE_URL + "/" + ACCOUNT + "/" + dlqName;+ sqsService.createQueue(dlqName, null, REGION);++ Topic topic = snsService.createTopic("sns-dlq-topic", null, null, REGION);+ String unavailableQueueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":unavailable-queue";+ Subscription subscription = snsService.subscribe(+ topic.getTopicArn(), "sqs", unavailableQueueArn, REGION, Map.of());+ snsService.setSubscriptionAttribute(subscription.getSubscriptionArn(), "RedrivePolicy",+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\"}", REGION);++ String publishMessageId = snsService.publish(topic.getTopicArn(), null,+ "synthetic-payload", "synthetic-subject", REGION);++ assertNotNull(publishMessageId);+ List<Message> deadLetterMessages = sqsService.receiveMessage(dlqUrl, 1, 30, 0, REGION);+ assertEquals(1, deadLetterMessages.size());+ JsonNode envelope = new ObjectMapper().readTree(deadLetterMessages.getFirst().getBody());+ assertEquals("Notification", envelope.path("Type").asText());+ assertEquals(publishMessageId, envelope.path("MessageId").asText());+ assertEquals(topic.getTopicArn(), envelope.path("TopicArn").asText());+ assertEquals("synthetic-payload", envelope.path("Message").asText());+ assertEquals("synthetic-subject", envelope.path("Subject").asText());+ assertTrue(sqsService.receiveMessage(dlqUrl, 1, 30, 0, REGION).isEmpty(),+ "The notification should appear only once in the subscription DLQ");+ }++ @Test+ void publish_routesStandardTopicNotificationToFifoSubscriptionDlq() throws Exception {+ String dlqName = "sns-subscription-dlq.fifo";+ String dlqArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":" + dlqName;+ String dlqUrl = BASE_URL + "/" + ACCOUNT + "/" + dlqName;+ sqsService.createQueue(dlqName, Map.of("FifoQueue", "true"), REGION);++ Topic topic = snsService.createTopic("sns-standard-topic", null, null, REGION);+ Subscription subscription = snsService.subscribe(topic.getTopicArn(), "sqs",+ "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":unavailable-fifo-source.fifo", REGION, Map.of());+ snsService.setSubscriptionAttribute(subscription.getSubscriptionArn(), "RedrivePolicy",+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\"}", REGION);++ String publishMessageId = snsService.publish(topic.getTopicArn(), null,+ "synthetic-payload", null, REGION);++ List<Message> deadLetterMessages = sqsService.receiveMessage(dlqUrl, 1, 30, 0, REGION);+ assertEquals(1, deadLetterMessages.size());+ Message deadLetter = deadLetterMessages.getFirst();+ JsonNode envelope = new ObjectMapper().readTree(deadLetter.getBody());+ assertEquals(publishMessageId, deadLetter.getMessageGroupId());+ assertNotNull(deadLetter.getMessageDeduplicationId());+ assertEquals(publishMessageId, envelope.path("MessageId").asText());+ }++ @Test+ void publish_keepsBothFailedSubscriptionsInSharedFifoDlq() throws Exception {+ String dlqName = "shared-subscription-dlq.fifo";+ String dlqArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":" + dlqName;+ String dlqUrl = BASE_URL + "/" + ACCOUNT + "/" + dlqName;+ sqsService.createQueue(dlqName, Map.of("FifoQueue", "true"), REGION);++ Topic topic = snsService.createTopic("shared-dlq-topic", null, null, REGION);+ for (String endpoint : List.of("missing-subscription-a", "missing-subscription-b")) {+ Subscription subscription = snsService.subscribe(topic.getTopicArn(), "sqs",+ "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":" + endpoint, REGION, Map.of());+ snsService.setSubscriptionAttribute(subscription.getSubscriptionArn(), "RedrivePolicy",+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\"}", REGION);+ }++ String publishMessageId = snsService.publish(topic.getTopicArn(), null, "payload", null, REGION);++ List<Message> deadLetters = sqsService.receiveMessage(dlqUrl, 10, 30, 0, REGION);+ assertEquals(2, deadLetters.size());+ assertEquals(publishMessageId, deadLetters.get(0).getMessageGroupId());+ assertEquals(publishMessageId, deadLetters.get(1).getMessageGroupId());+ assertNotEquals(deadLetters.get(0).getMessageDeduplicationId(),+ deadLetters.get(1).getMessageDeduplicationId());+ ObjectMapper mapper = new ObjectMapper();+ for (Message deadLetter : deadLetters) {+ assertEquals(publishMessageId, mapper.readTree(deadLetter.getBody()).path("MessageId").asText());+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.javaindex 29ef09b06c..9660b34e7a 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java@@ -175,6 +175,28 @@ class GuardedMessageQueueTest {assertEquals(1, target.messageCounts().visible());}+ @Test+ void addMessageRollsBackTheInMemoryAddWhenPersistFails() {+ AtomicBoolean storeDown = new AtomicBoolean();+ InMemoryStorage<String, List<Message>> store = new InMemoryStorage<String, List<Message>>() {+ @Override+ public void put(String key, List<Message> value) {+ if (storeDown.get()) {+ throw new IllegalStateException("store unavailable");+ }+ super.put(key, value);+ }+ };+ GuardedMessageQueue target = new GuardedMessageQueue(store, "us-east-1::/000000000000/target");+ target.addMessage(new Message("already-there"));+ storeDown.set(true);++ assertThrows(IllegalStateException.class, () -> target.addMessage(new Message("incoming")));++ assertEquals(List.of("already-there"), target.peekAll().stream().map(Message::getBody).toList());+ assertEquals(1, target.messageCounts().visible());+ }+@Testvoid messageCountsReturnsVisibleInFlightAndDelayed() {queue.addMessage(new Message("msg1"));