Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sqs): retain fifo deduplication after message deletion (#4460)

vamsiakbbb1e10fc84f
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
index 015ccba6ad..6e06e0ac2a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
@@ -57,6 +57,7 @@ public class SqsService implements Resettable, ResourceProvider {
private static final Logger LOG = Logger.getLogger(SqsService.class);
private static final int DEDUP_WINDOW_SECONDS = 300; // 5 minutes
+ private static final int DEDUP_LOCK_STRIPES = 256;
private static final int MAX_RECEIVE_WAIT_TIME_SECONDS = 20;
private static final int MAX_TERMINAL_MOVE_TASKS = 10;
private static final Duration TERMINAL_MOVE_TASK_TTL = Duration.ofHours(1);
@@ -68,10 +69,14 @@ public class SqsService implements Resettable, ResourceProvider {
private final StorageBackend<String, Queue> queueStore;
private final StorageBackend<String, List<Message>> messageStore;
private final StorageBackend<String, Map<String, Long>> dedupStore;
+ private final StorageBackend<String, Map<String, Map<String, String>>> dedupIdentityStore;
private final ConcurrentHashMap<String, GuardedMessageQueue> messagesByQueue = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Object> queueLocks = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, RedrivePolicy> redrivePolicyCache = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, ConcurrentHashMap<String, Instant>> deduplicationCache = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, ConcurrentHashMap<String, DeduplicationIdentity>> deduplicationIdentityCache =
+ new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, Object[]> deduplicationLocks = new ConcurrentHashMap<>();
/** Move tasks keyed by opaque task handle. */
private final MoveTaskStore moveTasksByHandle;
/** Per-task cancellation signal: the move worker waits on it between moves, so a cancel wakes
@@ -92,6 +97,9 @@ public class SqsService implements Resettable, ResourceProvider {
private record RedrivePolicy(int maxReceiveCount, String deadLetterTargetArn) {
}
+ private record DeduplicationIdentity(String messageId, long sequenceNumber) {
+ }
+
public record MoveTask(String taskHandle, String sourceArn, String destinationArn,
int maxNumberOfMessagesPerSecond, String status,
long approximateNumberOfMessagesMoved,
@@ -278,6 +286,9 @@ public class SqsService implements Resettable, ResourceProvider {
storageFactory.create("sqs", "sqs-dedup.json",
new TypeReference<Map<String, Map<String, Long>>>() {
}),
+ storageFactory.create("sqs", "sqs-dedup-identities.json",
+ new TypeReference<Map<String, Map<String, Map<String, String>>>>() {
+ }),
config.services().sqs().defaultVisibilityTimeout(),
config.services().sqs().maxMessageSize(),
config.effectiveBaseUrl(),
@@ -325,9 +336,20 @@ public class SqsService implements Resettable, ResourceProvider {
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,
SnsService snsService, Clock clock) {
+ this(queueStore, messageStore, dedupStore, null, defaultVisibilityTimeout, maxMessageSize,
+ baseUrl, regionResolver, clearFifoDeduplicationCacheOnPurge, snsService, clock);
+ }
+
+ SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,
+ StorageBackend<String, Map<String, Long>> dedupStore,
+ StorageBackend<String, Map<String, Map<String, String>>> dedupIdentityStore,
+ int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
+ RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,
+ SnsService snsService, Clock clock) {
this.queueStore = queueStore;
this.messageStore = messageStore;
this.dedupStore = dedupStore;
+ this.dedupIdentityStore = dedupIdentityStore;
this.defaultVisibilityTimeout = defaultVisibilityTimeout;
this.maxMessageSize = maxMessageSize;
this.maxAllowedMessageSize = Math.max(AWS_MAXIMUM_MESSAGE_SIZE, maxMessageSize);
@@ -339,6 +361,7 @@ public class SqsService implements Resettable, ResourceProvider {
this.moveTasksByHandle = new MoveTaskStore(clock);
loadPersistedMessages();
loadPersistedDedup();
+ loadPersistedDedupIdentities();
}
@PreDestroy
@@ -352,6 +375,8 @@ public class SqsService implements Resettable, ResourceProvider {
queueLocks.clear();
redrivePolicyCache.clear();
deduplicationCache.clear();
+ deduplicationIdentityCache.clear();
+ deduplicationLocks.clear();
moveTaskCancellation.values().forEach(MoveTaskCancellation::request);
moveTaskCancellation.clear();
moveTasksByHandle.clear();
@@ -376,7 +401,7 @@ public class SqsService implements Resettable, ResourceProvider {
if (dedupStore == null) {
return;
}
- Instant now = Instant.now();
+ Instant now = clock.instant();
if (dedupStore instanceof AccountAwareStorageBackend<Map<String, Long>> aware) {
aware.scanAllAccountsAsMap().forEach((key, entries) -> loadDedupEntries(key, entries, now));
} else {
@@ -399,6 +424,44 @@ public class SqsService implements Resettable, ResourceProvider {
}
}
+ private void loadPersistedDedupIdentities() {
+ if (dedupIdentityStore == null) {
+ return;
+ }
+ if (dedupIdentityStore instanceof AccountAwareStorageBackend<Map<String, Map<String, String>>> aware) {
+ aware.scanAllAccountsAsMap().forEach(this::loadDedupIdentities);
+ } else {
+ for (String key : dedupIdentityStore.keys()) {
+ dedupIdentityStore.get(key).ifPresent(identities -> loadDedupIdentities(key, identities));
+ }
+ }
+ }
+
+ private void loadDedupIdentities(String key, Map<String, Map<String, String>> entries) {
+ ConcurrentHashMap<String, Instant> activeDedupEntries = deduplicationCache.get(key);
+ if (activeDedupEntries == null) {
+ return;
+ }
+ ConcurrentHashMap<String, DeduplicationIdentity> activeIdentities = new ConcurrentHashMap<>();
+ entries.forEach((dedupKey, identityFields) -> {
+ if (!activeDedupEntries.containsKey(dedupKey)) {
+ return;
+ }
+ try {
+ String messageId = identityFields.get("messageId");
+ long sequenceNumber = Long.parseLong(identityFields.get("sequenceNumber"));
+ if (messageId != null && !messageId.isBlank()) {
+ activeIdentities.put(dedupKey, new DeduplicationIdentity(messageId, sequenceNumber));
+ }
+ } catch (RuntimeException e) {
+ LOG.warnv("Ignoring invalid FIFO deduplication identity for queue {0}, key {1}", key, dedupKey);
+ }
+ });
+ if (!activeIdentities.isEmpty()) {
+ deduplicationIdentityCache.put(key, activeIdentities);
+ }
+ }
+
private GuardedMessageQueue getOrCreateQueue(String storageKey) {
return messagesByQueue.computeIfAbsent(storageKey,
k -> new GuardedMessageQueue(messageStore, k));
@@ -409,15 +472,56 @@ public class SqsService implements Resettable, ResourceProvider {
return;
}
ConcurrentHashMap<String, Instant> dedupMap = deduplicationCache.get(storageKey);
- if (dedupMap != null && !dedupMap.isEmpty()) {
- Map<String, Long> serializable = new HashMap<>();
- dedupMap.forEach((id, expiry) -> serializable.put(id, expiry.toEpochMilli()));
- dedupStore.put(storageKey, serializable);
- } else {
+ if (dedupMap == null) {
dedupStore.delete(storageKey);
+ return;
+ }
+ // Serialize snapshots, not the entire send. Otherwise a slow message write
+ // also blocks sends for unrelated deduplication IDs.
+ synchronized (dedupMap) {
+ if (dedupMap.isEmpty()) {
+ dedupStore.delete(storageKey);
+ } else {
+ Map<String, Long> serializable = new HashMap<>();
+ dedupMap.forEach((id, expiry) -> serializable.put(id, expiry.toEpochMilli()));
+ dedupStore.put(storageKey, serializable);
+ }
}
}
+ private void persistDedupIdentities(String storageKey) {
+ if (dedupIdentityStore == null) {
+ return;
+ }
+ ConcurrentHashMap<String, DeduplicationIdentity> identityMap = deduplicationIdentityCache.get(storageKey);
+ if (identityMap == null) {
+ dedupIdentityStore.delete(storageKey);
+ return;
+ }
+ synchronized (identityMap) {
+ if (identityMap.isEmpty()) {
+ dedupIdentityStore.delete(storageKey);
+ } else {
+ Map<String, Map<String, String>> serializable = new HashMap<>();
+ identityMap.forEach((dedupKey, identity) -> serializable.put(dedupKey, Map.of(
+ "messageId", identity.messageId(),
+ "sequenceNumber", Long.toString(identity.sequenceNumber()))));
+ dedupIdentityStore.put(storageKey, serializable);
+ }
+ }
+ }
+
+ private Object deduplicationLock(String storageKey, String dedupKey) {
+ Object[] locks = deduplicationLocks.computeIfAbsent(storageKey, key -> {
+ Object[] stripes = new Object[DEDUP_LOCK_STRIPES];
+ for (int i = 0; i < stripes.length; i++) {
+ stripes[i] = new Object();
+ }
+ return stripes;
+ });
+ return locks[dedupKey.hashCode() & (DEDUP_LOCK_STRIPES - 1)];
+ }
+
@Override
public List<ExplorerResource> getResources() {
List<ExplorerResource> resources = new ArrayList<>();
@@ -532,12 +636,17 @@ public class SqsService implements Resettable, ResourceProvider {
removed.close();
}
deduplicationCache.remove(storageKey);
+ deduplicationIdentityCache.remove(storageKey);
+ deduplicationLocks.remove(storageKey);
if (messageStore != null) {
messageStore.delete(storageKey);
}
if (dedupStore != null) {
dedupStore.delete(storageKey);
}
+ if (dedupIdentityStore != null) {
+ dedupIdentityStore.delete(storageKey);
+ }
// Wake parked ReceiveMessage long polls so they observe the deletion
// and finish, instead of staying registered against this queue URL.
// Left alone they would survive a delete + recreate under the same URL
@@ -700,52 +809,86 @@ public class SqsService implements Resettable, ResourceProvider {
// Use NUL as the delimiter — it is outside the SQS-allowed character set for
// MessageGroupId/MessageDeduplicationId, so the composite key is unambiguous.
String dedupCacheKey = groupScoped ? messageGroupId + "\0" + dedupId : dedupId;
- cleanupDeduplicationCache(storageKey);
ConcurrentHashMap<String, Instant> dedupMap = deduplicationCache.computeIfAbsent(storageKey, k -> new ConcurrentHashMap<>());
- Instant expiry = Instant.now().plusSeconds(DEDUP_WINDOW_SECONDS);
- Instant previous = dedupMap.putIfAbsent(dedupCacheKey, expiry);
- persistDedup(storageKey);
- if (previous != null && Instant.now().isBefore(previous)) {
- // Duplicate within window — keep the original messageId and
- // sequenceNumber but compute response MD5s from this request's
- // body and attributes, otherwise SDK clients (which validate
- // MD5 against what they sent) reject the response.
- Message existing = getOrCreateQueue(storageKey).findByDeduplicationId(
- dedupId, groupScoped ? messageGroupId : null);
- if (existing != null) {
+ cleanupDeduplicationCache(storageKey);
+ synchronized (deduplicationLock(storageKey, dedupCacheKey)) {
+ Instant now = clock.instant();
+ Instant expiry = now.plusSeconds(DEDUP_WINDOW_SECONDS);
+ Instant previous = dedupMap.putIfAbsent(dedupCacheKey, expiry);
+ if (previous != null && !now.isBefore(previous)) {
+ dedupMap.replace(dedupCacheKey, previous, expiry);
+ ConcurrentHashMap<String, DeduplicationIdentity> identities =
+ deduplicationIdentityCache.get(storageKey);
+ if (identities != null) {
+ identities.remove(dedupCacheKey);
+ }
+ previous = null;
+ }
+ if (previous != null && now.isBefore(previous)) {
+ // Duplicate within window — keep the original messageId and
+ // sequenceNumber but compute response MD5s from this request's
+ // body and attributes, otherwise SDK clients (which validate
+ // MD5 against what they sent) reject the response.
+ ConcurrentHashMap<String, DeduplicationIdentity> identities =
+ deduplicationIdentityCache.get(storageKey);
+ DeduplicationIdentity identity = identities == null ? null : identities.get(dedupCacheKey);
+ if (identity == null) {
+ Message existing = getOrCreateQueue(storageKey).findByDeduplicationId(
+ dedupId, groupScoped ? messageGroupId : null);
+ if (existing != null) {
+ identity = new DeduplicationIdentity(
+ existing.getMessageId(), existing.getSequenceNumber());
+ deduplicationIdentityCache.computeIfAbsent(storageKey, k -> new ConcurrentHashMap<>())
+ .putIfAbsent(dedupCacheKey, identity);
+ persistDedupIdentities(storageKey);
+ }
+ }
+ // The original message may have been received and deleted, but SQS
+ // continues tracking the deduplication ID for the full interval.
+ // Return the original identity without re-enqueueing a message.
Message response = new Message(body);
- response.setMessageId(existing.getMessageId());
+ if (identity != null) {
+ response.setMessageId(identity.messageId());
+ response.setSequenceNumber(identity.sequenceNumber());
+ } else {
+ response.setSequenceNumber(sequenceCounter.incrementAndGet());
+ }
response.setMessageGroupId(messageGroupId);
response.setMessageDeduplicationId(dedupId);
- response.setSequenceNumber(existing.getSequenceNumber());
if (messageAttributes != null && !messageAttributes.isEmpty()) {
response.getMessageAttributes().putAll(messageAttributes);
response.updateMd5OfMessageAttributes();
}
return response;
}
- }
- Message message = new Message(body);
- message.setMessageGroupId(messageGroupId);
- message.setMessageDeduplicationId(dedupId);
- message.setSequenceNumber(sequenceCounter.incrementAndGet());
- message.setAwsTraceHeader(awsTraceHeader);
- if (effectiveDelaySeconds > 0) {
- message.setVisibleAt(Instant.now().plusSeconds(effectiveDelaySeconds));
- }
- if (messageAttributes != null && !messageAttributes.isEmpty()) {
- message.getMessageAttributes().putAll(messageAttributes);
- message.updateMd5OfMessageAttributes();
- }
+ persistDedup(storageKey);
+
+ Message message = new Message(body);
+ message.setMessageGroupId(messageGroupId);
+ message.setMessageDeduplicationId(dedupId);
+ message.setSequenceNumber(sequenceCounter.incrementAndGet());
+ message.setAwsTraceHeader(awsTraceHeader);
+ if (effectiveDelaySeconds > 0) {
+ message.setVisibleAt(Instant.now().plusSeconds(effectiveDelaySeconds));
+ }
+ if (messageAttributes != null && !messageAttributes.isEmpty()) {
+ message.getMessageAttributes().putAll(messageAttributes);
+ message.updateMd5OfMessageAttributes();
+ }
- getOrCreateQueue(storageKey).addMessage(message);
- notifyReceivers(storageKey);
- LOG.debugv("Sent FIFO message {0} to queue {1}, group={2}, seq={3}",
- message.getMessageId(), queueUrl, messageGroupId, message.getSequenceNumber());
- LOG.tracev("Sent message {0} to queue {1} body={2} attributes={3}",
- message.getMessageId(), queueUrl, body, message.getMessageAttributes());
- return message;
+ getOrCreateQueue(storageKey).addMessage(message);
+ deduplicationIdentityCache.computeIfAbsent(storageKey, k -> new ConcurrentHashMap<>())
+ .putIfAbsent(dedupCacheKey,
+ new DeduplicationIdentity(message.getMessageId(), message.getSequenceNumber()));
+ persistDedupIdentities(storageKey);
+ notifyReceivers(storageKey);
+ LOG.debugv("Sent FIFO message {0} to queue {1}, group={2}, seq={3}",
+ message.getMessageId(), queueUrl, messageGroupId, message.getSequenceNumber());
+ LOG.tracev("Sent message {0} to queue {1} body={2} attributes={3}",
+ message.getMessageId(), queueUrl, body, message.getMessageAttributes());
+ return message;
+ }
}
// Standard queue. MessageGroupId is retained for ReceiveMessage to
@@ -882,8 +1025,28 @@ public class SqsService implements Resettable, ResourceProvider {
private void cleanupDeduplicationCache(String queueUrl) {
ConcurrentHashMap<String, Instant> dedupMap = deduplicationCache.get(queueUrl);
if (dedupMap != null) {
- Instant now = Instant.now();
- dedupMap.entrySet().removeIf(e -> now.isAfter(e.getValue()));
+ Instant now = clock.instant();
+ Set<String> expiredKeys = dedupMap.entrySet().stream()
+ .filter(e -> !now.isBefore(e.getValue()))
+ .map(Map.Entry::getKey)
+ .collect(Collectors.toSet());
+ boolean removed = false;
+ for (String key : expiredKeys) {
+ synchronized (deduplicationLock(queueUrl, key)) {
+ Instant expiry = dedupMap.get(key);
+ if (expiry != null && !now.isBefore(expiry) && dedupMap.remove(key, expiry)) {
+ ConcurrentHashMap<String, DeduplicationIdentity> identityMap =
+ deduplicationIdentityCache.get(queueUrl);
+ if (identityMap != null) {
+ identityMap.remove(key);
+ }
+ removed = true;
+ }
+ }
+ }
+ if (removed) {
+ persistDedupIdentities(queueUrl);
+ }
}
}
@@ -1085,9 +1248,14 @@ public class SqsService implements Resettable, ResourceProvider {
getOrCreateQueue(storageKey).purge();
if (clearFifoDeduplicationCacheOnPurge) {
deduplicationCache.remove(storageKey);
+ deduplicationIdentityCache.remove(storageKey);
+ deduplicationLocks.remove(storageKey);
if (dedupStore != null) {
dedupStore.delete(storageKey);
}
+ if (dedupIdentityStore != null) {
+ dedupIdentityStore.delete(storageKey);
+ }
if (snsService != null) {
snsService.clearFifoDeduplicationCacheForSqsQueueSubscriptions(queueUrl, region);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java
index 3802624b8b..f0ecb77f6c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java
@@ -304,7 +304,7 @@ class SnsSqsFanoutFifoDeliveryTest {
}
@Test
- void clearFifoDedupForSubscribedQueue_thenRepublishWithSameDedupId_deliversAgain() {
+ void clearingTopicDedupOnSeparatePurgeServiceDoesNotClearSubscribedQueueDedup() {
RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);
SqsService purgeSqsService = SqsServiceFactory.createInMemoryWithFifoDedupPurgeAndSns(
BASE_URL, regionResolver, snsService);
@@ -329,7 +329,6 @@ class SnsSqsFanoutFifoDeliveryTest {
snsService.publish(topicArn, null, null, "after-clear", null, null, "group-1", "shared-dedup", REGION);
List<Message> after = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);
- assertEquals(1, after.size());
- assertTrue(after.getFirst().getBody().contains("after-clear"));
+ assertTrue(after.isEmpty(), "The delivery queue must still suppress the duplicate within five minutes");
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index 24273c3558..84b18c55f4 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -24,6 +24,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -564,6 +567,206 @@ class SqsServiceTest {
assertEquals(1, received.size());
}
+ @Test
+ void fifoDeduplicationStillSuppressesMessageAfterOriginalIsDeleted() {
+ String region = "eu-west-1";
+ Queue queue = sqsService.createQueue("deleted-original.fifo", null, region);
+ Message original = sqsService.sendMessage(
+ queue.getQueueUrl(), "original", 0, "group-a", "dedup-a", region);
+
+ List<Message> received = sqsService.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region);
+ assertEquals(1, received.size());
+ sqsService.deleteMessage(queue.getQueueUrl(), received.getFirst().getReceiptHandle(), region);
+
+ Message duplicateResponse = sqsService.sendMessage(
+ queue.getQueueUrl(), "duplicate", 0, "group-a", "dedup-a", region);
+
+ assertNotNull(duplicateResponse.getMessageId());
+ assertEquals(original.getMessageId(), duplicateResponse.getMessageId());
+ assertEquals(original.getSequenceNumber(), duplicateResponse.getSequenceNumber());
+ assertEquals("dedup-a", duplicateResponse.getMessageDeduplicationId());
+ assertTrue(sqsService.peekMessages(queue.getQueueUrl(), region).isEmpty(),
+ "A duplicate must remain suppressed after its original message is deleted");
+ assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region).isEmpty());
+ }
+
+ @Test
+ void fifoDeduplicationIdentitySurvivesRestartAfterOriginalIsDeleted() {
+ String region = "us-east-1";
+ InMemoryStorage<String, Queue> queueStore = new InMemoryStorage<>();
+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>();
+ InMemoryStorage<String, Map<String, Long>> dedupStore = new InMemoryStorage<>();
+ InMemoryStorage<String, Map<String, Map<String, String>>> identityStore = new InMemoryStorage<>();
+ RegionResolver regionResolver = new RegionResolver(region, "000000000000");
+ SqsService initialService = new SqsService(queueStore, messageStore, dedupStore, identityStore,
+ 30, 1048576, BASE_URL, regionResolver, false, null, clock);
+ Queue queue = initialService.createQueue("deleted-before-restart.fifo", null, region);
+ Message original = initialService.sendMessage(
+ queue.getQueueUrl(), "original", 0, "group-a", "dedup-a", region);
+ List<Message> received = initialService.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region);
+ initialService.deleteMessage(queue.getQueueUrl(), received.getFirst().getReceiptHandle(), region);
+
+ SqsService restartedService = new SqsService(queueStore, messageStore, dedupStore, identityStore,
+ 30, 1048576, BASE_URL, regionResolver, false, null, clock);
+ Message duplicateResponse = restartedService.sendMessage(
+ queue.getQueueUrl(), "duplicate", 0, "group-a", "dedup-a", region);
+
+ assertEquals(original.getMessageId(), duplicateResponse.getMessageId());
+ assertEquals(original.getSequenceNumber(), duplicateResponse.getSequenceNumber());
+ assertTrue(restartedService.peekMessages(queue.getQueueUrl(), region).isEmpty());
+ }
+
+ @Test
+ void fifoDeduplicationExpiresAtTheFiveMinuteBoundary() {
+ String region = "eu-west-1";
+ Queue queue = sqsService.createQueue("expired-dedup.fifo", null, region);
+ sqsService.sendMessage(queue.getQueueUrl(), "original", 0, "group-a", "dedup-a", region);
+
+ clock.advance(Duration.ofMinutes(5));
+ sqsService.sendMessage(queue.getQueueUrl(), "after-window", 0, "group-a", "dedup-a", region);
+
+ List<Message> received = sqsService.receiveMessage(queue.getQueueUrl(), 2, 30, 0, region);
+ assertEquals(2, received.size());
+ assertTrue(received.stream().anyMatch(message -> "after-window".equals(message.getBody())));
+ }
+
+ @Test
+ void concurrentFifoSendsAfterExpiryKeepTheNewDeduplicationEntry() throws Exception {
+ String region = "eu-west-1";
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ for (int round = 0; round < 25; round++) {
+ Queue queue = sqsService.createQueue("expired-concurrent-" + round + ".fifo", null, region);
+ String queueUrl = queue.getQueueUrl();
+ sqsService.sendMessage(queueUrl, "original", 0, "group-a", "dedup-a", region);
+ Message original = sqsService.receiveMessage(queueUrl, 1, 30, 0, region).getFirst();
+ sqsService.deleteMessage(queueUrl, original.getReceiptHandle(), region);
+ clock.advance(Duration.ofMinutes(5));
+
+ CountDownLatch ready = new CountDownLatch(2);
+ CountDownLatch start = new CountDownLatch(1);
+ Future<Message> first = executor.submit(() -> {
+ ready.countDown();
+ start.await();
+ return sqsService.sendMessage(queueUrl, "renewed", 0, "group-a", "dedup-a", region);
+ });
+ Future<Message> second = executor.submit(() -> {
+ ready.countDown();
+ start.await();
+ return sqsService.sendMessage(queueUrl, "renewed", 0, "group-a", "dedup-a", region);
+ });
+ assertTrue(ready.await(5, TimeUnit.SECONDS));
+ start.countDown();
+
+ Message firstResponse = first.get(5, TimeUnit.SECONDS);
+ Message secondResponse = second.get(5, TimeUnit.SECONDS);
+ assertEquals(firstResponse.getMessageId(), secondResponse.getMessageId());
+ assertEquals(firstResponse.getSequenceNumber(), secondResponse.getSequenceNumber());
+ assertEquals(1, sqsService.peekMessages(queueUrl, region).size());
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void duplicateInAnotherGroupDoesNotWaitForBlockedMessageWrite() throws Exception {
+ String region = "us-east-1";
+ CountDownLatch writing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>() {
+ @Override
+ public void put(String key, List<Message> messages) {
+ if (messages.stream().anyMatch(message -> "slow".equals(message.getBody()))) {
+ writing.countDown();
+ try {
+ if (!release.await(5, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("Timed out waiting to release message write");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+ super.put(key, messages);
+ }
+ };
+ SqsService service = new SqsService(new InMemoryStorage<>(), messageStore,
+ new InMemoryStorage<>(), new InMemoryStorage<>(), 30, 1048576, BASE_URL,
+ new RegionResolver(region, "000000000000"), false, null, clock);
+ Queue queue = service.createQueue("unrelated-send.fifo", null, region);
+ Message original = service.sendMessage(queue.getQueueUrl(), "original", 0,
+ "group-a", "dedup-a", region);
+
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ Future<Message> slow = executor.submit(() -> service.sendMessage(queue.getQueueUrl(),
+ "slow", 0, "group-b", "dedup-b", region));
+ assertTrue(writing.await(5, TimeUnit.SECONDS));
+ Future<Message> duplicate = executor.submit(() -> service.sendMessage(queue.getQueueUrl(),
+ "duplicate", 0, "group-a", "dedup-a", region));
+ assertEquals(original.getMessageId(), duplicate.get(2, TimeUnit.SECONDS).getMessageId());
+ release.countDown();
+ slow.get(5, TimeUnit.SECONDS);
+ } finally {
+ release.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void concurrentFifoSendsRetainAllDeduplicationEntriesAfterRestart() throws Exception {
+ String region = "us-east-1";
+ InMemoryStorage<String, Queue> queueStore = new InMemoryStorage<>();
+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>();
+ InMemoryStorage<String, Map<String, Long>> dedupStore = new InMemoryStorage<>();
+ InMemoryStorage<String, Map<String, Map<String, String>>> identityStore = new InMemoryStorage<>();
+ RegionResolver regionResolver = new RegionResolver(region, "000000000000");
+ SqsService service = new SqsService(queueStore, messageStore, dedupStore, identityStore,
+ 30, 1048576, BASE_URL, regionResolver, false, null, clock);
+ Queue queue = service.createQueue("concurrent-persist.fifo", null, region);
+ ExecutorService executor = Executors.newFixedThreadPool(8);
+ try {
+ List<Future<Message>> sent = new ArrayList<>();
+ for (int i = 0; i < 32; i++) {
+ int id = i;
+ sent.add(executor.submit(() -> service.sendMessage(queue.getQueueUrl(),
+ "message-" + id, 0, "group-" + id, "dedup-" + id, region)));
+ }
+ List<Message> originals = new ArrayList<>();
+ for (Future<Message> send : sent) {
+ originals.add(send.get(5, TimeUnit.SECONDS));
+ }
+ SqsService restarted = new SqsService(queueStore, messageStore, dedupStore, identityStore,
+ 30, 1048576, BASE_URL, regionResolver, false, null, clock);
+ for (int i = 0; i < originals.size(); i++) {
+ Message duplicate = restarted.sendMessage(queue.getQueueUrl(),
+ "duplicate-" + i, 0, "group-" + i, "dedup-" + i, region);
+ assertEquals(originals.get(i).getMessageId(), duplicate.getMessageId());
+ }
+ assertEquals(32, restarted.peekMessages(queue.getQueueUrl(), region).size());
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void contentBasedFifoDeduplicationStillSuppressesMessageAfterOriginalIsDeleted() {
+ String region = "eu-west-1";
+ Queue queue = sqsService.createQueue("content-deleted-original.fifo",
+ Map.of("ContentBasedDeduplication", "true"), region);
+ sqsService.sendMessage(queue.getQueueUrl(), "same-body", 0, "group-a", null, region);
+
+ List<Message> received = sqsService.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region);
+ assertEquals(1, received.size());
+ sqsService.deleteMessage(queue.getQueueUrl(), received.getFirst().getReceiptHandle(), region);
+
+ sqsService.sendMessage(queue.getQueueUrl(), "same-body", 0, "group-a", null, region);
+
+ assertTrue(sqsService.peekMessages(queue.getQueueUrl(), region).isEmpty(),
+ "Content-based duplicate IDs must remain suppressed after deletion");
+ }
+
@Test
void fifoQueueReceiveReturnsMultipleMessagesPerGroupInOrder() {
// AWS FIFO: a single ReceiveMessage call may return multiple messages
@@ -761,13 +964,13 @@ class SqsServiceTest {
assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 10, 0, 0, region).isEmpty(),
"Queue must be empty after purge");
- // Re-send with same dedup ID — dedup cache fires but finds no message (purged),
- // so it falls through and creates a new message
+ // Re-send with the same dedup ID. Purging removed the original message,
+ // but does not end its five-minute deduplication interval.
sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1", region);
List<Message> received = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0, region);
- assertEquals(1, received.size(),
- "Re-send after purge must produce exactly one message in the queue");
+ assertTrue(received.isEmpty(),
+ "A duplicate must remain suppressed after purge during the deduplication interval");
}
@Test