Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sns): retry failed sqs subscription deliveries (#4468)

vamsiakf3dde75c0b17
diff --git a/docs/services/sns.md b/docs/services/sns.md
index 01739a3195..88a5a20ef7 100644
--- a/docs/services/sns.md
+++ b/docs/services/sns.md
@@ -93,6 +93,12 @@ Supported subscription protocols:
For `FilterPolicyScope=MessageBody`, nested policy objects descend into JSON objects and arrays.
An object inside an array matches when one array element satisfies the complete nested policy.
+For SQS subscriptions, Floci retries a failed delivery up to three times total, with two immediate
+retries. If all attempts fail and the subscription has a `RedrivePolicy` containing a valid
+`deadLetterTargetArn` for an SQS queue, Floci sends the original notification to that queue. SNS
+`Publish` still returns its message ID after downstream delivery fails. This is a bounded local
+policy; it does not reproduce AWS delivery retry timing or scale.
+
## HTTP and HTTPS endpoint addresses
Floci posts the `SubscriptionConfirmation` and every notification to a subscribed `http`/`https`
diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
index e56ab7adaf..0af76f92f1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
@@ -68,6 +68,9 @@ public class SnsService implements Resettable, ResourceProvider {
private static final Logger LOG = Logger.getLogger(SnsService.class);
private static final Duration FIFO_DEDUP_WINDOW = Duration.ofMinutes(5);
+ /** Includes the original send and two immediate retries for local SQS fan-out. */
+ private static final int SQS_SUBSCRIPTION_DELIVERY_ATTEMPTS = 3;
+ private static final String SUBSCRIPTION_REDRIVE_POLICY = "RedrivePolicy";
/** Default value of the {@code MaximumMessageSize} topic attribute, and the ceiling below
* which a topic carries no subscription restrictions. AWS raised the maximum to 1 MiB in
* September 2026 but left the default at 256 KiB, so an unconfigured topic is unchanged. */
@@ -1730,8 +1733,8 @@ public class SnsService implements Resettable, ResourceProvider {
Map<String, MessageAttributeValue> sqsAttributes = rawDelivery
? toSqsMessageAttributes(messageAttributes)
: Collections.emptyMap();
- sqsService.sendMessage(queueUrl, body, null, messageGroupId, messageDeduplicationId, sqsAttributes, region);
- LOG.debugv("Delivered SNS message to SQS: {0} ({1}) raw={2}", sub.getEndpoint(), queueUrl, rawDelivery);
+ deliverToSqsSubscription(sub, queueUrl, body, messageId, messageGroupId,
+ messageDeduplicationId, sqsAttributes, region, rawDelivery);
}
case "lambda" -> {
String region = extractRegionFromArn(sub.getEndpoint());
@@ -1846,6 +1849,73 @@ public class SnsService implements Resettable, ResourceProvider {
}
}
+ private void deliverToSqsSubscription(Subscription sub, String queueUrl, String body, String messageId,
+ String messageGroupId, String messageDeduplicationId,
+ Map<String, MessageAttributeValue> sqsAttributes,
+ String region, boolean rawDelivery) {
+ RuntimeException deliveryFailure = null;
+ for (int attempt = 1; attempt <= SQS_SUBSCRIPTION_DELIVERY_ATTEMPTS; attempt++) {
+ try {
+ sqsService.sendMessage(queueUrl, body, null, messageGroupId,
+ messageDeduplicationId, sqsAttributes, region);
+ LOG.debugv("Delivered SNS message to SQS: {0} ({1}) raw={2}",
+ sub.getEndpoint(), queueUrl, rawDelivery);
+ return;
+ } catch (RuntimeException e) {
+ deliveryFailure = e;
+ }
+ }
+
+ String deadLetterTargetArn = subscriptionDeadLetterTargetArn(sub);
+ if (deadLetterTargetArn != null) {
+ try {
+ String deadLetterRegion = extractRegionFromArn(deadLetterTargetArn);
+ if (deadLetterRegion == null) {
+ deadLetterRegion = region;
+ }
+ boolean fifoDeadLetterQueue = AwsArnUtils.parse(deadLetterTargetArn)
+ .resource().endsWith(".fifo");
+ String deadLetterGroupId = messageGroupId;
+ String deadLetterDeduplicationId = messageDeduplicationId;
+ if (fifoDeadLetterQueue) {
+ if (deadLetterGroupId == null || deadLetterGroupId.isBlank()) {
+ deadLetterGroupId = messageId;
+ }
+ // Each failed subscription is a distinct DLQ delivery, even when
+ // several subscriptions share this queue for the same publish.
+ deadLetterDeduplicationId = sha256(messageId + "\0" + sub.getSubscriptionArn());
+ }
+ sqsService.sendMessage(sqsArnToUrl(deadLetterTargetArn), body, null,
+ deadLetterGroupId, deadLetterDeduplicationId, sqsAttributes, deadLetterRegion);
+ LOG.warnv("SNS delivery to {0} failed after {1} attempts; sent notification to subscription DLQ {2}",
+ sub.getEndpoint(), SQS_SUBSCRIPTION_DELIVERY_ATTEMPTS, deadLetterTargetArn);
+ return;
+ } catch (RuntimeException deadLetterFailure) {
+ LOG.warnv("SNS delivery to {0} and subscription DLQ {1} failed: {2}",
+ sub.getEndpoint(), deadLetterTargetArn, deadLetterFailure.getMessage());
+ }
+ }
+ LOG.warnv("SNS delivery to {0} failed after {1} attempts: {2}",
+ sub.getEndpoint(), SQS_SUBSCRIPTION_DELIVERY_ATTEMPTS,
+ deliveryFailure == null ? "unknown failure" : deliveryFailure.getMessage());
+ }
+
+ private String subscriptionDeadLetterTargetArn(Subscription sub) {
+ String redrivePolicy = sub.getAttributes().get(SUBSCRIPTION_REDRIVE_POLICY);
+ if (redrivePolicy == null || redrivePolicy.isBlank()) {
+ return null;
+ }
+ try {
+ JsonNode policy = objectMapper.readTree(redrivePolicy);
+ String targetArn = policy.path("deadLetterTargetArn").asText(null);
+ return AwsArnUtils.isArnFor(targetArn, "sqs") ? targetArn : null;
+ } catch (Exception e) {
+ LOG.warnv("Ignoring invalid SNS subscription RedrivePolicy for {0}: {1}",
+ sub.getSubscriptionArn(), e.getMessage());
+ return null;
+ }
+ }
+
private String buildSnsLambdaEvent(String topicArn, String messageId, String message,
String subject, Map<String, MessageAttributeValue> messageAttributes,
String subscriptionArn) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
index 2f71271d71..0968a53604 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
@@ -54,8 +54,14 @@ class GuardedMessageQueue {
void addMessage(Message message) {
try (Guard _ = hold()) {
+ int mark = messages.size();
messages.add(message);
- persist();
+ try {
+ persist();
+ } catch (RuntimeException e) {
+ messages.remove(mark);
+ throw e;
+ }
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java
index 11c3f88a51..6434eb8a45 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java
@@ -11,6 +11,8 @@ import io.github.hectorvent.floci.services.lambda.LambdaService;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
import io.github.hectorvent.floci.services.sns.model.Subscription;
import io.github.hectorvent.floci.services.sns.model.Topic;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.services.sqs.model.Message;
import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -24,9 +26,13 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
class SnsServiceTest {
@@ -80,6 +86,55 @@ class SnsServiceTest {
assertThrows(AwsException.class, () -> snsService.createTopic("", null, null, REGION));
}
+ @Test
+ void publish_retriesTransientSqsSubscriptionDeliveryFailure() {
+ SqsService sqs = mock(SqsService.class);
+ SnsService service = new SnsService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver(REGION, ACCOUNT), sqs, null);
+ Topic topic = service.createTopic("retry-topic", null, null, REGION);
+ String queueArn = "arn:aws:sqs:us-east-1:000000000000:retry-queue";
+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/retry-queue";
+ service.subscribe(topic.getTopicArn(), "sqs", queueArn, REGION, Map.of());
+ doThrow(new AwsException("ServiceUnavailable", "temporarily unavailable", 503))
+ .doReturn(new Message("delivered"))
+ .when(sqs).sendMessage(eq(queueUrl), anyString(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));
+
+ String messageId = service.publish(topic.getTopicArn(), null, "payload", null, REGION);
+
+ assertNotNull(messageId);
+ verify(sqs, times(2)).sendMessage(eq(queueUrl), anyString(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));
+ }
+
+ @Test
+ void publish_sendsExhaustedSqsSubscriptionDeliveryToConfiguredDlq() throws Exception {
+ SqsService sqs = mock(SqsService.class);
+ SnsService service = new SnsService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver(REGION, ACCOUNT), sqs, null);
+ Topic topic = service.createTopic("dlq-topic", null, null, REGION);
+ String queueArn = "arn:aws:sqs:us-east-1:000000000000:unavailable-queue";
+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/unavailable-queue";
+ String dlqArn = "arn:aws:sqs:us-east-1:000000000000:subscription-dlq";
+ String dlqUrl = BASE_URL + "/" + ACCOUNT + "/subscription-dlq";
+ service.subscribe(topic.getTopicArn(), "sqs", queueArn, REGION,
+ Map.of("RedrivePolicy", "{\"deadLetterTargetArn\":\"" + dlqArn + "\"}"));
+ doThrow(new AwsException("AWS.SimpleQueueService.NonExistentQueue", "unavailable", 400))
+ .when(sqs).sendMessage(eq(queueUrl), anyString(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));
+
+ String messageId = service.publish(topic.getTopicArn(), null,
+ "payload", "subject", REGION);
+
+ assertNotNull(messageId, "SNS Publish acceptance is independent of downstream SQS delivery");
+ verify(sqs, times(3)).sendMessage(eq(queueUrl), anyString(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));
+ ArgumentCaptor<String> deadLetterBody = ArgumentCaptor.forClass(String.class);
+ verify(sqs).sendMessage(eq(dlqUrl), deadLetterBody.capture(), isNull(), isNull(), isNull(), anyMap(), eq(REGION));
+ JsonNode envelope = new ObjectMapper().readTree(deadLetterBody.getValue());
+ assertEquals("Notification", envelope.path("Type").asText());
+ assertEquals(topic.getTopicArn(), envelope.path("TopicArn").asText());
+ assertEquals("payload", envelope.path("Message").asText());
+ assertEquals("subject", envelope.path("Subject").asText());
+ assertTrue(envelope.hasNonNull("MessageId"));
+ }
+
@Test
void listTopics_returnsCreatedTopics() {
snsService.createTopic("topic-a", null, null, REGION);
diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsSubscriptionDlqIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsSubscriptionDlqIntegrationTest.java
new file mode 100644
index 0000000000..3490b1fe51
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsSubscriptionDlqIntegrationTest.java
@@ -0,0 +1,123 @@
+package io.github.hectorvent.floci.services.sns;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.sns.model.Subscription;
+import io.github.hectorvent.floci.services.sns.model.Topic;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.services.sqs.SqsServiceFactory;
+import io.github.hectorvent.floci.services.sqs.model.Message;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class SnsSqsSubscriptionDlqIntegrationTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "000000000000";
+ private static final String BASE_URL = "http://localhost:4566";
+
+ private SnsService snsService;
+ private SqsService sqsService;
+
+ @BeforeEach
+ void setUp() {
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);
+ sqsService = SqsServiceFactory.createInMemory(BASE_URL, regionResolver);
+ snsService = new SnsService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ regionResolver, sqsService, null);
+ }
+
+ @Test
+ void publish_routesExhaustedDeliveryToSubscriptionDlq() throws Exception {
+ String dlqName = "sns-subscription-dlq";
+ String dlqArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":" + dlqName;
+ String dlqUrl = BASE_URL + "/" + ACCOUNT + "/" + dlqName;
+ sqsService.createQueue(dlqName, null, REGION);
+
+ Topic topic = snsService.createTopic("sns-dlq-topic", null, null, REGION);
+ String unavailableQueueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":unavailable-queue";
+ Subscription subscription = snsService.subscribe(
+ topic.getTopicArn(), "sqs", unavailableQueueArn, REGION, Map.of());
+ snsService.setSubscriptionAttribute(subscription.getSubscriptionArn(), "RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\"}", REGION);
+
+ String publishMessageId = snsService.publish(topic.getTopicArn(), null,
+ "synthetic-payload", "synthetic-subject", REGION);
+
+ assertNotNull(publishMessageId);
+ List<Message> deadLetterMessages = sqsService.receiveMessage(dlqUrl, 1, 30, 0, REGION);
+ assertEquals(1, deadLetterMessages.size());
+ JsonNode envelope = new ObjectMapper().readTree(deadLetterMessages.getFirst().getBody());
+ assertEquals("Notification", envelope.path("Type").asText());
+ assertEquals(publishMessageId, envelope.path("MessageId").asText());
+ assertEquals(topic.getTopicArn(), envelope.path("TopicArn").asText());
+ assertEquals("synthetic-payload", envelope.path("Message").asText());
+ assertEquals("synthetic-subject", envelope.path("Subject").asText());
+ assertTrue(sqsService.receiveMessage(dlqUrl, 1, 30, 0, REGION).isEmpty(),
+ "The notification should appear only once in the subscription DLQ");
+ }
+
+ @Test
+ void publish_routesStandardTopicNotificationToFifoSubscriptionDlq() throws Exception {
+ String dlqName = "sns-subscription-dlq.fifo";
+ String dlqArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":" + dlqName;
+ String dlqUrl = BASE_URL + "/" + ACCOUNT + "/" + dlqName;
+ sqsService.createQueue(dlqName, Map.of("FifoQueue", "true"), REGION);
+
+ Topic topic = snsService.createTopic("sns-standard-topic", null, null, REGION);
+ Subscription subscription = snsService.subscribe(topic.getTopicArn(), "sqs",
+ "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":unavailable-fifo-source.fifo", REGION, Map.of());
+ snsService.setSubscriptionAttribute(subscription.getSubscriptionArn(), "RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\"}", REGION);
+
+ String publishMessageId = snsService.publish(topic.getTopicArn(), null,
+ "synthetic-payload", null, REGION);
+
+ List<Message> deadLetterMessages = sqsService.receiveMessage(dlqUrl, 1, 30, 0, REGION);
+ assertEquals(1, deadLetterMessages.size());
+ Message deadLetter = deadLetterMessages.getFirst();
+ JsonNode envelope = new ObjectMapper().readTree(deadLetter.getBody());
+ assertEquals(publishMessageId, deadLetter.getMessageGroupId());
+ assertNotNull(deadLetter.getMessageDeduplicationId());
+ assertEquals(publishMessageId, envelope.path("MessageId").asText());
+ }
+
+ @Test
+ void publish_keepsBothFailedSubscriptionsInSharedFifoDlq() throws Exception {
+ String dlqName = "shared-subscription-dlq.fifo";
+ String dlqArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":" + dlqName;
+ String dlqUrl = BASE_URL + "/" + ACCOUNT + "/" + dlqName;
+ sqsService.createQueue(dlqName, Map.of("FifoQueue", "true"), REGION);
+
+ Topic topic = snsService.createTopic("shared-dlq-topic", null, null, REGION);
+ for (String endpoint : List.of("missing-subscription-a", "missing-subscription-b")) {
+ Subscription subscription = snsService.subscribe(topic.getTopicArn(), "sqs",
+ "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":" + endpoint, REGION, Map.of());
+ snsService.setSubscriptionAttribute(subscription.getSubscriptionArn(), "RedrivePolicy",
+ "{\"deadLetterTargetArn\":\"" + dlqArn + "\"}", REGION);
+ }
+
+ String publishMessageId = snsService.publish(topic.getTopicArn(), null, "payload", null, REGION);
+
+ List<Message> deadLetters = sqsService.receiveMessage(dlqUrl, 10, 30, 0, REGION);
+ assertEquals(2, deadLetters.size());
+ assertEquals(publishMessageId, deadLetters.get(0).getMessageGroupId());
+ assertEquals(publishMessageId, deadLetters.get(1).getMessageGroupId());
+ assertNotEquals(deadLetters.get(0).getMessageDeduplicationId(),
+ deadLetters.get(1).getMessageDeduplicationId());
+ ObjectMapper mapper = new ObjectMapper();
+ for (Message deadLetter : deadLetters) {
+ assertEquals(publishMessageId, mapper.readTree(deadLetter.getBody()).path("MessageId").asText());
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
index 29ef09b06c..9660b34e7a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
@@ -175,6 +175,28 @@ class GuardedMessageQueueTest {
assertEquals(1, target.messageCounts().visible());
}
+ @Test
+ void addMessageRollsBackTheInMemoryAddWhenPersistFails() {
+ AtomicBoolean storeDown = new AtomicBoolean();
+ InMemoryStorage<String, List<Message>> store = new InMemoryStorage<String, List<Message>>() {
+ @Override
+ public void put(String key, List<Message> value) {
+ if (storeDown.get()) {
+ throw new IllegalStateException("store unavailable");
+ }
+ super.put(key, value);
+ }
+ };
+ GuardedMessageQueue target = new GuardedMessageQueue(store, "us-east-1::/000000000000/target");
+ target.addMessage(new Message("already-there"));
+ storeDown.set(true);
+
+ assertThrows(IllegalStateException.class, () -> target.addMessage(new Message("incoming")));
+
+ assertEquals(List.of("already-there"), target.peekAll().stream().map(Message::getBody).toList());
+ assertEquals(1, target.messageCounts().visible());
+ }
+
@Test
void messageCountsReturnsVisibleInFlightAndDelayed() {
queue.addMessage(new Message("msg1"));