Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sqs): add option to clear deduplication cache on PurgeQueue (#561)

Raman Belakapytski102cb997f355
diff --git a/docs/configuration/application-yml.md b/docs/configuration/application-yml.md
index e9cf4816b3..dc0410b5ab 100644
--- a/docs/configuration/application-yml.md
+++ b/docs/configuration/application-yml.md
@@ -84,6 +84,7 @@ floci:
enabled: true
default-visibility-timeout: 30 # Seconds
max-message-size: 262144 # Bytes (256 KB)
+ clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue also clears the FIFO deduplication cache
s3:
enabled: true
@@ -228,6 +229,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v
| `FLOCI_SERVICES_SSM_MAX_PARAMETER_HISTORY` | `5` | Max parameter versions kept |
| `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default visibility timeout (seconds) |
| `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `262144` | Max message size (bytes) |
+| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` also clears the FIFO 5-minute deduplication cache for the target queue |
| `FLOCI_SERVICES_S3_DEFAULT_PRESIGN_EXPIRY_SECONDS` | `3600` | Pre-signed URL expiry |
| `FLOCI_SERVICES_DOCKER_NETWORK` | *(unset)* | Shared Docker network for Lambda, RDS, ElastiCache containers |
| `FLOCI_SERVICES_ECS_MOCK` | `false` | Skip Docker; tasks go straight to RUNNING (useful for CI) |
diff --git a/docs/services/sqs.md b/docs/services/sqs.md
index 4a546fd860..2ada12e83b 100644
--- a/docs/services/sqs.md
+++ b/docs/services/sqs.md
@@ -37,6 +37,7 @@ floci:
enabled: true
default-visibility-timeout: 30 # Seconds
max-message-size: 262144 # 256 KB
+ clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue also clears the FIFO 5-minute deduplication cache
```
## Examples
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 7bb6370a62..ea0a8d5a0e 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -260,6 +260,9 @@ public interface EmulatorConfig {
@WithDefault("262144")
int maxMessageSize();
+
+ @WithDefault("false")
+ boolean clearFifoDeduplicationCacheOnPurge();
}
interface S3ServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
index 1f490af709..9b9ac5d625 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
@@ -40,6 +40,7 @@ public class SqsService {
private final int maxMessageSize;
private final String baseUrl;
private final RegionResolver regionResolver;
+ private final boolean clearFifoDeduplicationCacheOnPurge;
@Inject
public SqsService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver) {
@@ -56,7 +57,8 @@ public class SqsService {
config.services().sqs().defaultVisibilityTimeout(),
config.services().sqs().maxMessageSize(),
config.effectiveBaseUrl(),
- regionResolver
+ regionResolver,
+ config.services().sqs().clearFifoDeduplicationCacheOnPurge()
);
}
@@ -66,13 +68,21 @@ public class SqsService {
SqsService(StorageBackend<String, Queue> queueStore,
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl) {
this(queueStore, null, null, defaultVisibilityTimeout, maxMessageSize, baseUrl,
- new RegionResolver("us-east-1", "000000000000"));
+ new RegionResolver("us-east-1", "000000000000"), false);
}
SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,
StorageBackend<String, Map<String, Long>> dedupStore,
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
RegionResolver regionResolver) {
+ this(queueStore, messageStore, dedupStore, defaultVisibilityTimeout, maxMessageSize, baseUrl,
+ regionResolver, false);
+ }
+
+ SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,
+ StorageBackend<String, Map<String, Long>> dedupStore,
+ int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
+ RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge) {
this.queueStore = queueStore;
this.messageStore = messageStore;
this.dedupStore = dedupStore;
@@ -80,6 +90,7 @@ public class SqsService {
this.maxMessageSize = maxMessageSize;
this.baseUrl = baseUrl;
this.regionResolver = regionResolver;
+ this.clearFifoDeduplicationCacheOnPurge = clearFifoDeduplicationCacheOnPurge;
loadPersistedMessages();
loadPersistedDedup();
}
@@ -602,7 +613,14 @@ public class SqsService {
String storageKey = regionKey(region, queueUrl);
ensureQueueExists(storageKey);
getOrCreateQueue(storageKey).purge();
- LOG.infov("Purged queue: {0}", queueUrl);
+ if (clearFifoDeduplicationCacheOnPurge) {
+ deduplicationCache.remove(storageKey);
+ if (dedupStore != null) {
+ dedupStore.delete(storageKey);
+ }
+ }
+ LOG.infov("Purged queue{0}: {1}",
+ clearFifoDeduplicationCacheOnPurge ? " (dedup cache cleared)" : "", queueUrl);
}
public void setQueueAttributes(String queueUrl, Map<String, String> attributes, String region) {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 92bdf2563a..9cbfa721ff 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -80,6 +80,7 @@ floci:
enabled: true
default-visibility-timeout: 30
max-message-size: 262144
+ clear-fifo-deduplication-cache-on-purge: false
s3:
enabled: true
default-presign-expiry-seconds: 3600
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index f6715ecf25..7000a9b785 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.sqs;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.sqs.model.Message;
import io.github.hectorvent.floci.services.sqs.model.Queue;
@@ -345,4 +346,79 @@ class SqsServiceTest {
assertEquals(1, immediate.size(),
"FIFO queues must ignore per-message DelaySeconds");
}
+
+ // --- clearFifoDeduplicationCacheOnPurge tests ---
+
+ @Test
+ void purgeQueueClearsFifoDeduplicationCacheWhenEnabled() {
+ final var service = new SqsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true);
+
+ final var queue = service.createQueue("dedup-clear.fifo", Map.of("ContentBasedDeduplication", "true"));
+
+ // First send — message M1 added, dedup cache populated with "dedup-1"
+ final var m1 = service.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");
+ assertNotNull(m1.getMessageId());
+
+ // Purge clears both messages and the dedup cache
+ service.purgeQueue(queue.getQueueUrl());
+ assertTrue(service.receiveMessage(queue.getQueueUrl(), 10, 0, 0).isEmpty(),
+ "Queue must be empty after purge");
+
+ // Re-send with the same dedup ID — cache was cleared so this is treated as a fresh send
+ final var m2 = service.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");
+ assertNotNull(m2.getMessageId());
+
+ final var received = service.receiveMessage(queue.getQueueUrl(), 10, 30, 0);
+ assertEquals(1, received.size(), "One message must be in the queue after re-send");
+
+ // Third send with same dedup ID — fresh cache entry from m2 deduplicates correctly
+ final var m3 = service.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");
+ assertEquals(m2.getMessageId(), m3.getMessageId(),
+ "Dedup must work with the fresh cache entry after purge");
+ }
+
+ @Test
+ void purgeQueuePreservesFifoDeduplicationCacheByDefault() {
+ // Default service has clearFifoDeduplicationCacheOnPurge=false
+ final var queue = sqsService.createQueue("dedup-preserve.fifo",
+ Map.of("ContentBasedDeduplication", "true"));
+
+ // Send and then purge — messages are gone but dedup cache is intact
+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");
+ sqsService.purgeQueue(queue.getQueueUrl());
+
+ assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 10, 0, 0).isEmpty(),
+ "Queue must be empty after purge");
+
+ // Re-send with same dedup ID — dedup cache fires but finds no message (purged),
+ // so it falls through and creates a new message
+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");
+
+ final var received = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);
+ assertEquals(1, received.size(),
+ "Re-send after purge must produce exactly one message in the queue");
+ }
+
+ @Test
+ void purgeQueueClearsDedupStoreWhenEnabled() {
+ final var dedupStore = new InMemoryStorage<String, Map<String, Long>>();
+ final var service = new SqsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), dedupStore,
+ 30, 262144, BASE_URL, new RegionResolver("us-east-1", "000000000000"), true);
+
+ final var queue = service.createQueue("dedup-store-clear.fifo",
+ Map.of("ContentBasedDeduplication", "true"));
+
+ // Send a message — dedup entry must be persisted to the store
+ service.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1");
+ assertFalse(dedupStore.keys().isEmpty(),
+ "Dedup store must have an entry after sending a FIFO message");
+
+ // Purge with flag enabled — dedupStore entry for the queue must be removed
+ service.purgeQueue(queue.getQueueUrl());
+ assertTrue(dedupStore.keys().isEmpty(),
+ "Dedup store must be empty after purge with clearFifoDeduplicationCacheOnPurge=true");
+ }
}