Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sqs): clear SNS FIFO dedup cache on PurgeQueue (#594)

Raman Belakapytski5ad4c06d7470
diff --git a/docs/configuration/application-yml.md b/docs/configuration/application-yml.md
index dc0410b5ab..99bc585883 100644
--- a/docs/configuration/application-yml.md
+++ b/docs/configuration/application-yml.md
@@ -84,7 +84,7 @@ floci:
enabled: true
default-visibility-timeout: 30 # Seconds
max-message-size: 262144 # Bytes (256 KB)
- clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue also clears the FIFO deduplication cache
+ clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue clears SQS FIFO dedup and SNS FIFO topic dedup for topics subscribed to that queue
s3:
enabled: true
@@ -229,7 +229,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v
| `FLOCI_SERVICES_SSM_MAX_PARAMETER_HISTORY` | `5` | Max parameter versions kept |
| `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default visibility timeout (seconds) |
| `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `262144` | Max message size (bytes) |
-| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` also clears the FIFO 5-minute deduplication cache for the target queue |
+| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` clears the FIFO 5-minute deduplication cache for the target queue and matching SNS FIFO topic dedup entries |
| `FLOCI_SERVICES_S3_DEFAULT_PRESIGN_EXPIRY_SECONDS` | `3600` | Pre-signed URL expiry |
| `FLOCI_SERVICES_DOCKER_NETWORK` | *(unset)* | Shared Docker network for Lambda, RDS, ElastiCache containers |
| `FLOCI_SERVICES_ECS_MOCK` | `false` | Skip Docker; tasks go straight to RUNNING (useful for CI) |
diff --git a/docs/services/sqs.md b/docs/services/sqs.md
index 2ada12e83b..1d1dbf3d10 100644
--- a/docs/services/sqs.md
+++ b/docs/services/sqs.md
@@ -37,7 +37,7 @@ floci:
enabled: true
default-visibility-timeout: 30 # Seconds
max-message-size: 262144 # 256 KB
- clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue also clears the FIFO 5-minute deduplication cache
+ clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue clears the FIFO deduplication cache for the queue and for any SNS FIFO topics that subscribe to that queue (SNS in-memory dedup)
```
## Examples
diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
index f7eb818f3b..8872cd8c96 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
@@ -567,6 +567,54 @@ public class SnsService {
return false;
}
+ /**
+ * Removes all FIFO deduplication cache entries for SNS topics that have an SQS subscription
+ * whose endpoint resolves to the same queue path as {@code queueUrl} (used when purging SQS
+ * with {@code clearFifoDeduplicationCacheOnPurge}).
+ */
+ public void clearFifoDeduplicationCacheForSqsQueueSubscriptions(String queueUrl, String region) {
+ String queuePath = extractQueuePathFromUrl(queueUrl);
+ if (queuePath.isEmpty()) {
+ return;
+ }
+ String subPrefix = "sub::" + region + "::";
+ subscriptionStore.keys().stream()
+ .filter(key -> key.startsWith(subPrefix))
+ .map(key -> subscriptionStore.get(key).orElse(null))
+ .filter(Objects::nonNull)
+ .filter(sub -> "sqs".equals(sub.getProtocol()))
+ .filter(sub -> sqsSubscriptionEndpointMatchesQueuePath(sub.getEndpoint(), queuePath))
+ .map(Subscription::getTopicArn)
+ .forEach(topicArn ->
+ fifoDeduplicationCache.keySet().removeIf(cacheKey -> cacheKey.startsWith(topicArn + ":")));
+ }
+
+ private boolean sqsSubscriptionEndpointMatchesQueuePath(String endpoint, String queuePath) {
+ if (endpoint == null) {
+ return false;
+ }
+ String asUrl = sqsArnToUrl(endpoint);
+ return extractQueuePathFromUrl(asUrl).equals(queuePath);
+ }
+
+ /**
+ * Same path extraction as {@code SqsService} queue URL normalization ({@code /accountId/queueName}).
+ */
+ private static String extractQueuePathFromUrl(String url) {
+ if (url == null) {
+ return "";
+ }
+ int schemeEnd = url.indexOf("://");
+ if (schemeEnd < 0) {
+ return url;
+ }
+ int pathStart = url.indexOf('/', schemeEnd + 3);
+ if (pathStart < 0) {
+ return url;
+ }
+ return url.substring(pathStart);
+ }
+
private List<Subscription> subscriptionsByTopic(String topicArn, String region) {
List<Subscription> result = new ArrayList<>();
String prefix = "sub::" + region + "::";
@@ -739,4 +787,4 @@ public class SnsService {
private static String subKey(String region, String subscriptionArn) {
return "sub::" + region + "::" + subscriptionArn;
}
-}
\ No newline at end of file
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
index 9b9ac5d625..e8cf21d055 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.sns.SnsService;
import io.github.hectorvent.floci.services.sqs.model.Message;
import io.github.hectorvent.floci.services.sqs.model.Queue;
import com.fasterxml.jackson.core.type.TypeReference;
@@ -41,9 +42,11 @@ public class SqsService {
private final String baseUrl;
private final RegionResolver regionResolver;
private final boolean clearFifoDeduplicationCacheOnPurge;
+ private final SnsService snsService;
@Inject
- public SqsService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver) {
+ public SqsService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver,
+ SnsService snsService) {
this(
storageFactory.create("sqs", "sqs-queues.json",
new TypeReference<Map<String, Queue>>() {
@@ -58,7 +61,8 @@ public class SqsService {
config.services().sqs().maxMessageSize(),
config.effectiveBaseUrl(),
regionResolver,
- config.services().sqs().clearFifoDeduplicationCacheOnPurge()
+ config.services().sqs().clearFifoDeduplicationCacheOnPurge(),
+ snsService
);
}
@@ -68,7 +72,7 @@ public class SqsService {
SqsService(StorageBackend<String, Queue> queueStore,
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl) {
this(queueStore, null, null, defaultVisibilityTimeout, maxMessageSize, baseUrl,
- new RegionResolver("us-east-1", "000000000000"), false);
+ new RegionResolver("us-east-1", "000000000000"), false, null);
}
SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,
@@ -76,13 +80,14 @@ public class SqsService {
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
RegionResolver regionResolver) {
this(queueStore, messageStore, dedupStore, defaultVisibilityTimeout, maxMessageSize, baseUrl,
- regionResolver, false);
+ regionResolver, false, null);
}
SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,
StorageBackend<String, Map<String, Long>> dedupStore,
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
- RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge) {
+ RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,
+ SnsService snsService) {
this.queueStore = queueStore;
this.messageStore = messageStore;
this.dedupStore = dedupStore;
@@ -91,6 +96,7 @@ public class SqsService {
this.baseUrl = baseUrl;
this.regionResolver = regionResolver;
this.clearFifoDeduplicationCacheOnPurge = clearFifoDeduplicationCacheOnPurge;
+ this.snsService = snsService;
loadPersistedMessages();
loadPersistedDedup();
}
@@ -618,6 +624,9 @@ public class SqsService {
if (dedupStore != null) {
dedupStore.delete(storageKey);
}
+ if (snsService != null) {
+ snsService.clearFifoDeduplicationCacheForSqsQueueSubscriptions(queueUrl, region);
+ }
}
LOG.infov("Purged queue{0}: {1}",
clearFifoDeduplicationCacheOnPurge ? " (dedup cache cleared)" : "", queueUrl);
diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java
index ae4dc63c04..70302984fa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java
@@ -164,4 +164,34 @@ class SnsSqsFanoutFifoDeliveryTest {
assertEquals(1, messages.size());
assertTrue(messages.get(0).getBody().contains("first"));
}
+
+ @Test
+ void clearFifoDedupForSubscribedQueue_thenRepublishWithSameDedupId_deliversAgain() {
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);
+ SqsService purgeSqsService = SqsServiceFactory.createInMemoryWithFifoDedupPurgeAndSns(
+ BASE_URL, regionResolver, snsService);
+ sqsService.createQueue("manual-sns-dedup-queue.fifo", Map.of("FifoQueue", "true"), REGION);
+ purgeSqsService.createQueue("manual-sns-dedup-queue.fifo", Map.of("FifoQueue", "true"), REGION);
+ String queueArn = "arn:aws:sqs:" + REGION + ":" + ACCOUNT + ":manual-sns-dedup-queue.fifo";
+
+ snsService.createTopic("manual-sns-dedup-topic.fifo", Map.of("FifoTopic", "true"), null, REGION);
+ String topicArn = "arn:aws:sns:" + REGION + ":" + ACCOUNT + ":manual-sns-dedup-topic.fifo";
+ snsService.subscribe(topicArn, "sqs", queueArn, REGION, Map.of());
+
+ String queueUrl = BASE_URL + "/" + ACCOUNT + "/manual-sns-dedup-queue.fifo";
+
+ snsService.publish(topicArn, null, null, "before-clear", null, null, "group-1", "shared-dedup", REGION);
+ List<Message> first = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);
+ assertEquals(1, first.size());
+ for (Message m : first) {
+ sqsService.deleteMessage(queueUrl, m.getReceiptHandle(), REGION);
+ }
+
+ purgeSqsService.purgeQueue(queueUrl, REGION);
+
+ snsService.publish(topicArn, null, null, "after-clear", null, null, "group-1", "shared-dedup", REGION);
+ List<Message> after = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);
+ assertEquals(1, after.size());
+ assertTrue(after.getFirst().getBody().contains("after-clear"));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java
index b46dec64fa..92fae4a28b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceFactory.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.sqs;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.sns.SnsService;
/**
* Test helper to create SqsService instances
@@ -12,4 +13,10 @@ public class SqsServiceFactory {
return new SqsService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),
30, 262144, baseUrl, regionResolver);
}
+
+ public static SqsService createInMemoryWithFifoDedupPurgeAndSns(String baseUrl, RegionResolver regionResolver,
+ SnsService snsService) {
+ return new SqsService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 30, 262144, baseUrl, regionResolver, true, snsService);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index 7000a9b785..bb3a2ec1c9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.sqs;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.sns.SnsService;
import io.github.hectorvent.floci.services.sqs.model.Message;
import io.github.hectorvent.floci.services.sqs.model.Queue;
import org.junit.jupiter.api.BeforeEach;
@@ -12,6 +13,8 @@ import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
class SqsServiceTest {
@@ -353,7 +356,7 @@ class SqsServiceTest {
void purgeQueueClearsFifoDeduplicationCacheWhenEnabled() {
final var service = new SqsService(
new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),
- 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true);
+ 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true, null);
final var queue = service.createQueue("dedup-clear.fifo", Map.of("ContentBasedDeduplication", "true"));
@@ -406,7 +409,7 @@ class SqsServiceTest {
final var dedupStore = new InMemoryStorage<String, Map<String, Long>>();
final var service = new SqsService(
new InMemoryStorage<>(), new InMemoryStorage<>(), dedupStore,
- 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true);
+ 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true, null);
final var queue = service.createQueue("dedup-store-clear.fifo",
Map.of("ContentBasedDeduplication", "true"));
@@ -421,4 +424,16 @@ class SqsServiceTest {
assertTrue(dedupStore.keys().isEmpty(),
"Dedup store must be empty after purge with clearFifoDeduplicationCacheOnPurge=true");
}
+
+ @Test
+ void purgeQueueWithClearFifoDelegatesToSnsForFifoDedupOnSubscribedTopics() {
+ final var sns = mock(SnsService.class);
+ final var service = new SqsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true, sns);
+ final var queue = service.createQueue("sns-dedup-delegate.fifo", Map.of("FifoQueue", "true"));
+ service.purgeQueue(queue.getQueueUrl());
+ verify(sns).clearFifoDeduplicationCacheForSqsQueueSubscriptions(
+ queue.getQueueUrl(), "us-east-1");
+ }
}