Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sqs): add option to clear deduplication cache on PurgeQueue (#561)
Raman Belakapytski102cb997f355
diff --git a/docs/configuration/application-yml.md b/docs/configuration/application-yml.mdindex e9cf4816b3..dc0410b5ab 100644--- a/docs/configuration/application-yml.md+++ b/docs/configuration/application-yml.md@@ -84,6 +84,7 @@ floci:enabled: truedefault-visibility-timeout: 30 # Secondsmax-message-size: 262144 # Bytes (256 KB)+ clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue also clears the FIFO deduplication caches3:enabled: true@@ -228,6 +229,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v| `FLOCI_SERVICES_SSM_MAX_PARAMETER_HISTORY` | `5` | Max parameter versions kept || `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default visibility timeout (seconds) || `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `262144` | Max message size (bytes) |+| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` also clears the FIFO 5-minute deduplication cache for the target queue || `FLOCI_SERVICES_S3_DEFAULT_PRESIGN_EXPIRY_SECONDS` | `3600` | Pre-signed URL expiry || `FLOCI_SERVICES_DOCKER_NETWORK` | *(unset)* | Shared Docker network for Lambda, RDS, ElastiCache containers || `FLOCI_SERVICES_ECS_MOCK` | `false` | Skip Docker; tasks go straight to RUNNING (useful for CI) |diff --git a/docs/services/sqs.md b/docs/services/sqs.mdindex 4a546fd860..2ada12e83b 100644--- a/docs/services/sqs.md+++ b/docs/services/sqs.md@@ -37,6 +37,7 @@ floci:enabled: truedefault-visibility-timeout: 30 # Secondsmax-message-size: 262144 # 256 KB+ clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue also clears the FIFO 5-minute deduplication cache```## Examplesdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 7bb6370a62..ea0a8d5a0e 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -260,6 +260,9 @@ public interface EmulatorConfig {@WithDefault("262144")int maxMessageSize();++ @WithDefault("false")+ boolean clearFifoDeduplicationCacheOnPurge();}interface S3ServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.javaindex 1f490af709..9b9ac5d625 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java@@ -40,6 +40,7 @@ public class SqsService {private final int maxMessageSize;private final String baseUrl;private final RegionResolver regionResolver;+ private final boolean clearFifoDeduplicationCacheOnPurge;@Injectpublic SqsService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver) {@@ -56,7 +57,8 @@ public class SqsService {config.services().sqs().defaultVisibilityTimeout(),config.services().sqs().maxMessageSize(),config.effectiveBaseUrl(),- regionResolver+ regionResolver,+ config.services().sqs().clearFifoDeduplicationCacheOnPurge());}@@ -66,13 +68,21 @@ public class SqsService {SqsService(StorageBackend<String, Queue> queueStore,int defaultVisibilityTimeout, int maxMessageSize, String baseUrl) {this(queueStore, null, null, defaultVisibilityTimeout, maxMessageSize, baseUrl,- new RegionResolver("us-east-1", "000000000000"));+ new RegionResolver("us-east-1", "000000000000"), false);}SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,StorageBackend<String, Map<String, Long>> dedupStore,int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,RegionResolver regionResolver) {+ this(queueStore, messageStore, dedupStore, defaultVisibilityTimeout, maxMessageSize, baseUrl,+ regionResolver, false);+ }++ SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,+ StorageBackend<String, Map<String, Long>> dedupStore,+ int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,+ RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge) {this.queueStore = queueStore;this.messageStore = messageStore;this.dedupStore = dedupStore;@@ -80,6 +90,7 @@ public class SqsService {this.maxMessageSize = maxMessageSize;this.baseUrl = baseUrl;this.regionResolver = regionResolver;+ this.clearFifoDeduplicationCacheOnPurge = clearFifoDeduplicationCacheOnPurge;loadPersistedMessages();loadPersistedDedup();}@@ -602,7 +613,14 @@ public class SqsService {String storageKey = regionKey(region, queueUrl);ensureQueueExists(storageKey);getOrCreateQueue(storageKey).purge();- LOG.infov("Purged queue: {0}", queueUrl);+ if (clearFifoDeduplicationCacheOnPurge) {+ deduplicationCache.remove(storageKey);+ if (dedupStore != null) {+ dedupStore.delete(storageKey);+ }+ }+ LOG.infov("Purged queue{0}: {1}",+ clearFifoDeduplicationCacheOnPurge ? " (dedup cache cleared)" : "", queueUrl);}public void setQueueAttributes(String queueUrl, Map<String, String> attributes, String region) {diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 92bdf2563a..9cbfa721ff 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -80,6 +80,7 @@ floci:enabled: truedefault-visibility-timeout: 30max-message-size: 262144+ clear-fifo-deduplication-cache-on-purge: falses3:enabled: truedefault-presign-expiry-seconds: 3600diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.javaindex f6715ecf25..7000a9b785 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.sqs;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.sqs.model.Message;import io.github.hectorvent.floci.services.sqs.model.Queue;@@ -345,4 +346,79 @@ class SqsServiceTest {assertEquals(1, immediate.size(),"FIFO queues must ignore per-message DelaySeconds");}++ // --- clearFifoDeduplicationCacheOnPurge tests ---++ @Test+ void purgeQueueClearsFifoDeduplicationCacheWhenEnabled() {+ final var service = new SqsService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),+ 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true);++ final var queue = service.createQueue("dedup-clear.fifo", Map.of("ContentBasedDeduplication", "true"));++ // First send — message M1 added, dedup cache populated with "dedup-1"+ final var m1 = service.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");+ assertNotNull(m1.getMessageId());++ // Purge clears both messages and the dedup cache+ service.purgeQueue(queue.getQueueUrl());+ assertTrue(service.receiveMessage(queue.getQueueUrl(), 10, 0, 0).isEmpty(),+ "Queue must be empty after purge");++ // Re-send with the same dedup ID — cache was cleared so this is treated as a fresh send+ final var m2 = service.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");+ assertNotNull(m2.getMessageId());++ final var received = service.receiveMessage(queue.getQueueUrl(), 10, 30, 0);+ assertEquals(1, received.size(), "One message must be in the queue after re-send");++ // Third send with same dedup ID — fresh cache entry from m2 deduplicates correctly+ final var m3 = service.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");+ assertEquals(m2.getMessageId(), m3.getMessageId(),+ "Dedup must work with the fresh cache entry after purge");+ }++ @Test+ void purgeQueuePreservesFifoDeduplicationCacheByDefault() {+ // Default service has clearFifoDeduplicationCacheOnPurge=false+ final var queue = sqsService.createQueue("dedup-preserve.fifo",+ Map.of("ContentBasedDeduplication", "true"));++ // Send and then purge — messages are gone but dedup cache is intact+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");+ sqsService.purgeQueue(queue.getQueueUrl());++ assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 10, 0, 0).isEmpty(),+ "Queue must be empty after purge");++ // Re-send with same dedup ID — dedup cache fires but finds no message (purged),+ // so it falls through and creates a new message+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");++ final var received = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);+ assertEquals(1, received.size(),+ "Re-send after purge must produce exactly one message in the queue");+ }++ @Test+ void purgeQueueClearsDedupStoreWhenEnabled() {+ final var dedupStore = new InMemoryStorage<String, Map<String, Long>>();+ final var service = new SqsService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), dedupStore,+ 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true);++ final var queue = service.createQueue("dedup-store-clear.fifo",+ Map.of("ContentBasedDeduplication", "true"));++ // Send a message — dedup entry must be persisted to the store+ service.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");+ assertFalse(dedupStore.keys().isEmpty(),+ "Dedup store must have an entry after sending a FIFO message");++ // Purge with flag enabled — dedupStore entry for the queue must be removed+ service.purgeQueue(queue.getQueueUrl());+ assertTrue(dedupStore.keys().isEmpty(),+ "Dedup store must be empty after purge with clearFifoDeduplicationCacheOnPurge=true");+ }}