Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sqs): retain fifo deduplication after message deletion (#4460)
vamsiakbbb1e10fc84f
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.javaindex 015ccba6ad..6e06e0ac2a 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java@@ -57,6 +57,7 @@ public class SqsService implements Resettable, ResourceProvider {private static final Logger LOG = Logger.getLogger(SqsService.class);private static final int DEDUP_WINDOW_SECONDS = 300; // 5 minutes+ private static final int DEDUP_LOCK_STRIPES = 256;private static final int MAX_RECEIVE_WAIT_TIME_SECONDS = 20;private static final int MAX_TERMINAL_MOVE_TASKS = 10;private static final Duration TERMINAL_MOVE_TASK_TTL = Duration.ofHours(1);@@ -68,10 +69,14 @@ public class SqsService implements Resettable, ResourceProvider {private final StorageBackend<String, Queue> queueStore;private final StorageBackend<String, List<Message>> messageStore;private final StorageBackend<String, Map<String, Long>> dedupStore;+ private final StorageBackend<String, Map<String, Map<String, String>>> dedupIdentityStore;private final ConcurrentHashMap<String, GuardedMessageQueue> messagesByQueue = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, Object> queueLocks = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, RedrivePolicy> redrivePolicyCache = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, ConcurrentHashMap<String, Instant>> deduplicationCache = new ConcurrentHashMap<>();+ private final ConcurrentHashMap<String, ConcurrentHashMap<String, DeduplicationIdentity>> deduplicationIdentityCache =+ new ConcurrentHashMap<>();+ private final ConcurrentHashMap<String, Object[]> deduplicationLocks = new ConcurrentHashMap<>();/** Move tasks keyed by opaque task handle. */private final MoveTaskStore moveTasksByHandle;/** Per-task cancellation signal: the move worker waits on it between moves, so a cancel wakes@@ -92,6 +97,9 @@ public class SqsService implements Resettable, ResourceProvider {private record RedrivePolicy(int maxReceiveCount, String deadLetterTargetArn) {}+ private record DeduplicationIdentity(String messageId, long sequenceNumber) {+ }+public record MoveTask(String taskHandle, String sourceArn, String destinationArn,int maxNumberOfMessagesPerSecond, String status,long approximateNumberOfMessagesMoved,@@ -278,6 +286,9 @@ public class SqsService implements Resettable, ResourceProvider {storageFactory.create("sqs", "sqs-dedup.json",new TypeReference<Map<String, Map<String, Long>>>() {}),+ storageFactory.create("sqs", "sqs-dedup-identities.json",+ new TypeReference<Map<String, Map<String, Map<String, String>>>>() {+ }),config.services().sqs().defaultVisibilityTimeout(),config.services().sqs().maxMessageSize(),config.effectiveBaseUrl(),@@ -325,9 +336,20 @@ public class SqsService implements Resettable, ResourceProvider {int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,SnsService snsService, Clock clock) {+ this(queueStore, messageStore, dedupStore, null, defaultVisibilityTimeout, maxMessageSize,+ baseUrl, regionResolver, clearFifoDeduplicationCacheOnPurge, snsService, clock);+ }++ SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,+ StorageBackend<String, Map<String, Long>> dedupStore,+ StorageBackend<String, Map<String, Map<String, String>>> dedupIdentityStore,+ int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,+ RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,+ SnsService snsService, Clock clock) {this.queueStore = queueStore;this.messageStore = messageStore;this.dedupStore = dedupStore;+ this.dedupIdentityStore = dedupIdentityStore;this.defaultVisibilityTimeout = defaultVisibilityTimeout;this.maxMessageSize = maxMessageSize;this.maxAllowedMessageSize = Math.max(AWS_MAXIMUM_MESSAGE_SIZE, maxMessageSize);@@ -339,6 +361,7 @@ public class SqsService implements Resettable, ResourceProvider {this.moveTasksByHandle = new MoveTaskStore(clock);loadPersistedMessages();loadPersistedDedup();+ loadPersistedDedupIdentities();}@PreDestroy@@ -352,6 +375,8 @@ public class SqsService implements Resettable, ResourceProvider {queueLocks.clear();redrivePolicyCache.clear();deduplicationCache.clear();+ deduplicationIdentityCache.clear();+ deduplicationLocks.clear();moveTaskCancellation.values().forEach(MoveTaskCancellation::request);moveTaskCancellation.clear();moveTasksByHandle.clear();@@ -376,7 +401,7 @@ public class SqsService implements Resettable, ResourceProvider {if (dedupStore == null) {return;}- Instant now = Instant.now();+ Instant now = clock.instant();if (dedupStore instanceof AccountAwareStorageBackend<Map<String, Long>> aware) {aware.scanAllAccountsAsMap().forEach((key, entries) -> loadDedupEntries(key, entries, now));} else {@@ -399,6 +424,44 @@ public class SqsService implements Resettable, ResourceProvider {}}+ private void loadPersistedDedupIdentities() {+ if (dedupIdentityStore == null) {+ return;+ }+ if (dedupIdentityStore instanceof AccountAwareStorageBackend<Map<String, Map<String, String>>> aware) {+ aware.scanAllAccountsAsMap().forEach(this::loadDedupIdentities);+ } else {+ for (String key : dedupIdentityStore.keys()) {+ dedupIdentityStore.get(key).ifPresent(identities -> loadDedupIdentities(key, identities));+ }+ }+ }++ private void loadDedupIdentities(String key, Map<String, Map<String, String>> entries) {+ ConcurrentHashMap<String, Instant> activeDedupEntries = deduplicationCache.get(key);+ if (activeDedupEntries == null) {+ return;+ }+ ConcurrentHashMap<String, DeduplicationIdentity> activeIdentities = new ConcurrentHashMap<>();+ entries.forEach((dedupKey, identityFields) -> {+ if (!activeDedupEntries.containsKey(dedupKey)) {+ return;+ }+ try {+ String messageId = identityFields.get("messageId");+ long sequenceNumber = Long.parseLong(identityFields.get("sequenceNumber"));+ if (messageId != null && !messageId.isBlank()) {+ activeIdentities.put(dedupKey, new DeduplicationIdentity(messageId, sequenceNumber));+ }+ } catch (RuntimeException e) {+ LOG.warnv("Ignoring invalid FIFO deduplication identity for queue {0}, key {1}", key, dedupKey);+ }+ });+ if (!activeIdentities.isEmpty()) {+ deduplicationIdentityCache.put(key, activeIdentities);+ }+ }+private GuardedMessageQueue getOrCreateQueue(String storageKey) {return messagesByQueue.computeIfAbsent(storageKey,k -> new GuardedMessageQueue(messageStore, k));@@ -409,15 +472,56 @@ public class SqsService implements Resettable, ResourceProvider {return;}ConcurrentHashMap<String, Instant> dedupMap = deduplicationCache.get(storageKey);- if (dedupMap != null && !dedupMap.isEmpty()) {- Map<String, Long> serializable = new HashMap<>();- dedupMap.forEach((id, expiry) -> serializable.put(id, expiry.toEpochMilli()));- dedupStore.put(storageKey, serializable);- } else {+ if (dedupMap == null) {dedupStore.delete(storageKey);+ return;+ }+ // Serialize snapshots, not the entire send. Otherwise a slow message write+ // also blocks sends for unrelated deduplication IDs.+ synchronized (dedupMap) {+ if (dedupMap.isEmpty()) {+ dedupStore.delete(storageKey);+ } else {+ Map<String, Long> serializable = new HashMap<>();+ dedupMap.forEach((id, expiry) -> serializable.put(id, expiry.toEpochMilli()));+ dedupStore.put(storageKey, serializable);+ }}}+ private void persistDedupIdentities(String storageKey) {+ if (dedupIdentityStore == null) {+ return;+ }+ ConcurrentHashMap<String, DeduplicationIdentity> identityMap = deduplicationIdentityCache.get(storageKey);+ if (identityMap == null) {+ dedupIdentityStore.delete(storageKey);+ return;+ }+ synchronized (identityMap) {+ if (identityMap.isEmpty()) {+ dedupIdentityStore.delete(storageKey);+ } else {+ Map<String, Map<String, String>> serializable = new HashMap<>();+ identityMap.forEach((dedupKey, identity) -> serializable.put(dedupKey, Map.of(+ "messageId", identity.messageId(),+ "sequenceNumber", Long.toString(identity.sequenceNumber()))));+ dedupIdentityStore.put(storageKey, serializable);+ }+ }+ }++ private Object deduplicationLock(String storageKey, String dedupKey) {+ Object[] locks = deduplicationLocks.computeIfAbsent(storageKey, key -> {+ Object[] stripes = new Object[DEDUP_LOCK_STRIPES];+ for (int i = 0; i < stripes.length; i++) {+ stripes[i] = new Object();+ }+ return stripes;+ });+ return locks[dedupKey.hashCode() & (DEDUP_LOCK_STRIPES - 1)];+ }+@Overridepublic List<ExplorerResource> getResources() {List<ExplorerResource> resources = new ArrayList<>();@@ -532,12 +636,17 @@ public class SqsService implements Resettable, ResourceProvider {removed.close();}deduplicationCache.remove(storageKey);+ deduplicationIdentityCache.remove(storageKey);+ deduplicationLocks.remove(storageKey);if (messageStore != null) {messageStore.delete(storageKey);}if (dedupStore != null) {dedupStore.delete(storageKey);}+ if (dedupIdentityStore != null) {+ dedupIdentityStore.delete(storageKey);+ }// Wake parked ReceiveMessage long polls so they observe the deletion// and finish, instead of staying registered against this queue URL.// Left alone they would survive a delete + recreate under the same URL@@ -700,52 +809,86 @@ public class SqsService implements Resettable, ResourceProvider {// Use NUL as the delimiter — it is outside the SQS-allowed character set for// MessageGroupId/MessageDeduplicationId, so the composite key is unambiguous.String dedupCacheKey = groupScoped ? messageGroupId + "\0" + dedupId : dedupId;- cleanupDeduplicationCache(storageKey);ConcurrentHashMap<String, Instant> dedupMap = deduplicationCache.computeIfAbsent(storageKey, k -> new ConcurrentHashMap<>());- Instant expiry = Instant.now().plusSeconds(DEDUP_WINDOW_SECONDS);- Instant previous = dedupMap.putIfAbsent(dedupCacheKey, expiry);- persistDedup(storageKey);- if (previous != null && Instant.now().isBefore(previous)) {- // Duplicate within window — keep the original messageId and- // sequenceNumber but compute response MD5s from this request's- // body and attributes, otherwise SDK clients (which validate- // MD5 against what they sent) reject the response.- Message existing = getOrCreateQueue(storageKey).findByDeduplicationId(- dedupId, groupScoped ? messageGroupId : null);- if (existing != null) {+ cleanupDeduplicationCache(storageKey);+ synchronized (deduplicationLock(storageKey, dedupCacheKey)) {+ Instant now = clock.instant();+ Instant expiry = now.plusSeconds(DEDUP_WINDOW_SECONDS);+ Instant previous = dedupMap.putIfAbsent(dedupCacheKey, expiry);+ if (previous != null && !now.isBefore(previous)) {+ dedupMap.replace(dedupCacheKey, previous, expiry);+ ConcurrentHashMap<String, DeduplicationIdentity> identities =+ deduplicationIdentityCache.get(storageKey);+ if (identities != null) {+ identities.remove(dedupCacheKey);+ }+ previous = null;+ }+ if (previous != null && now.isBefore(previous)) {+ // Duplicate within window — keep the original messageId and+ // sequenceNumber but compute response MD5s from this request's+ // body and attributes, otherwise SDK clients (which validate+ // MD5 against what they sent) reject the response.+ ConcurrentHashMap<String, DeduplicationIdentity> identities =+ deduplicationIdentityCache.get(storageKey);+ DeduplicationIdentity identity = identities == null ? null : identities.get(dedupCacheKey);+ if (identity == null) {+ Message existing = getOrCreateQueue(storageKey).findByDeduplicationId(+ dedupId, groupScoped ? messageGroupId : null);+ if (existing != null) {+ identity = new DeduplicationIdentity(+ existing.getMessageId(), existing.getSequenceNumber());+ deduplicationIdentityCache.computeIfAbsent(storageKey, k -> new ConcurrentHashMap<>())+ .putIfAbsent(dedupCacheKey, identity);+ persistDedupIdentities(storageKey);+ }+ }+ // The original message may have been received and deleted, but SQS+ // continues tracking the deduplication ID for the full interval.+ // Return the original identity without re-enqueueing a message.Message response = new Message(body);- response.setMessageId(existing.getMessageId());+ if (identity != null) {+ response.setMessageId(identity.messageId());+ response.setSequenceNumber(identity.sequenceNumber());+ } else {+ response.setSequenceNumber(sequenceCounter.incrementAndGet());+ }response.setMessageGroupId(messageGroupId);response.setMessageDeduplicationId(dedupId);- response.setSequenceNumber(existing.getSequenceNumber());if (messageAttributes != null && !messageAttributes.isEmpty()) {response.getMessageAttributes().putAll(messageAttributes);response.updateMd5OfMessageAttributes();}return response;}- }- Message message = new Message(body);- message.setMessageGroupId(messageGroupId);- message.setMessageDeduplicationId(dedupId);- message.setSequenceNumber(sequenceCounter.incrementAndGet());- message.setAwsTraceHeader(awsTraceHeader);- if (effectiveDelaySeconds > 0) {- message.setVisibleAt(Instant.now().plusSeconds(effectiveDelaySeconds));- }- if (messageAttributes != null && !messageAttributes.isEmpty()) {- message.getMessageAttributes().putAll(messageAttributes);- message.updateMd5OfMessageAttributes();- }+ persistDedup(storageKey);++ Message message = new Message(body);+ message.setMessageGroupId(messageGroupId);+ message.setMessageDeduplicationId(dedupId);+ message.setSequenceNumber(sequenceCounter.incrementAndGet());+ message.setAwsTraceHeader(awsTraceHeader);+ if (effectiveDelaySeconds > 0) {+ message.setVisibleAt(Instant.now().plusSeconds(effectiveDelaySeconds));+ }+ if (messageAttributes != null && !messageAttributes.isEmpty()) {+ message.getMessageAttributes().putAll(messageAttributes);+ message.updateMd5OfMessageAttributes();+ }- getOrCreateQueue(storageKey).addMessage(message);- notifyReceivers(storageKey);- LOG.debugv("Sent FIFO message {0} to queue {1}, group={2}, seq={3}",- message.getMessageId(), queueUrl, messageGroupId, message.getSequenceNumber());- LOG.tracev("Sent message {0} to queue {1} body={2} attributes={3}",- message.getMessageId(), queueUrl, body, message.getMessageAttributes());- return message;+ getOrCreateQueue(storageKey).addMessage(message);+ deduplicationIdentityCache.computeIfAbsent(storageKey, k -> new ConcurrentHashMap<>())+ .putIfAbsent(dedupCacheKey,+ new DeduplicationIdentity(message.getMessageId(), message.getSequenceNumber()));+ persistDedupIdentities(storageKey);+ notifyReceivers(storageKey);+ LOG.debugv("Sent FIFO message {0} to queue {1}, group={2}, seq={3}",+ message.getMessageId(), queueUrl, messageGroupId, message.getSequenceNumber());+ LOG.tracev("Sent message {0} to queue {1} body={2} attributes={3}",+ message.getMessageId(), queueUrl, body, message.getMessageAttributes());+ return message;+ }}// Standard queue. MessageGroupId is retained for ReceiveMessage to@@ -882,8 +1025,28 @@ public class SqsService implements Resettable, ResourceProvider {private void cleanupDeduplicationCache(String queueUrl) {ConcurrentHashMap<String, Instant> dedupMap = deduplicationCache.get(queueUrl);if (dedupMap != null) {- Instant now = Instant.now();- dedupMap.entrySet().removeIf(e -> now.isAfter(e.getValue()));+ Instant now = clock.instant();+ Set<String> expiredKeys = dedupMap.entrySet().stream()+ .filter(e -> !now.isBefore(e.getValue()))+ .map(Map.Entry::getKey)+ .collect(Collectors.toSet());+ boolean removed = false;+ for (String key : expiredKeys) {+ synchronized (deduplicationLock(queueUrl, key)) {+ Instant expiry = dedupMap.get(key);+ if (expiry != null && !now.isBefore(expiry) && dedupMap.remove(key, expiry)) {+ ConcurrentHashMap<String, DeduplicationIdentity> identityMap =+ deduplicationIdentityCache.get(queueUrl);+ if (identityMap != null) {+ identityMap.remove(key);+ }+ removed = true;+ }+ }+ }+ if (removed) {+ persistDedupIdentities(queueUrl);+ }}}@@ -1085,9 +1248,14 @@ public class SqsService implements Resettable, ResourceProvider {getOrCreateQueue(storageKey).purge();if (clearFifoDeduplicationCacheOnPurge) {deduplicationCache.remove(storageKey);+ deduplicationIdentityCache.remove(storageKey);+ deduplicationLocks.remove(storageKey);if (dedupStore != null) {dedupStore.delete(storageKey);}+ if (dedupIdentityStore != null) {+ dedupIdentityStore.delete(storageKey);+ }if (snsService != null) {snsService.clearFifoDeduplicationCacheForSqsQueueSubscriptions(queueUrl, region);}diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.javaindex 3802624b8b..f0ecb77f6c 100644--- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsSqsFanoutFifoDeliveryTest.java@@ -304,7 +304,7 @@ class SnsSqsFanoutFifoDeliveryTest {}@Test- void clearFifoDedupForSubscribedQueue_thenRepublishWithSameDedupId_deliversAgain() {+ void clearingTopicDedupOnSeparatePurgeServiceDoesNotClearSubscribedQueueDedup() {RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);SqsService purgeSqsService = SqsServiceFactory.createInMemoryWithFifoDedupPurgeAndSns(BASE_URL, regionResolver, snsService);@@ -329,7 +329,6 @@ class SnsSqsFanoutFifoDeliveryTest {snsService.publish(topicArn, null, null, "after-clear", null, null, "group-1", "shared-dedup", REGION);List<Message> after = sqsService.receiveMessage(queueUrl, 10, 30, 0, REGION);- assertEquals(1, after.size());- assertTrue(after.getFirst().getBody().contains("after-clear"));+ assertTrue(after.isEmpty(), "The delivery queue must still suppress the duplicate within five minutes");}}diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.javaindex 24273c3558..84b18c55f4 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java@@ -24,6 +24,9 @@ import java.util.ArrayList;import java.util.List;import java.util.Map;import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicReference;@@ -564,6 +567,206 @@ class SqsServiceTest {assertEquals(1, received.size());}+ @Test+ void fifoDeduplicationStillSuppressesMessageAfterOriginalIsDeleted() {+ String region = "eu-west-1";+ Queue queue = sqsService.createQueue("deleted-original.fifo", null, region);+ Message original = sqsService.sendMessage(+ queue.getQueueUrl(), "original", 0, "group-a", "dedup-a", region);++ List<Message> received = sqsService.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region);+ assertEquals(1, received.size());+ sqsService.deleteMessage(queue.getQueueUrl(), received.getFirst().getReceiptHandle(), region);++ Message duplicateResponse = sqsService.sendMessage(+ queue.getQueueUrl(), "duplicate", 0, "group-a", "dedup-a", region);++ assertNotNull(duplicateResponse.getMessageId());+ assertEquals(original.getMessageId(), duplicateResponse.getMessageId());+ assertEquals(original.getSequenceNumber(), duplicateResponse.getSequenceNumber());+ assertEquals("dedup-a", duplicateResponse.getMessageDeduplicationId());+ assertTrue(sqsService.peekMessages(queue.getQueueUrl(), region).isEmpty(),+ "A duplicate must remain suppressed after its original message is deleted");+ assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region).isEmpty());+ }++ @Test+ void fifoDeduplicationIdentitySurvivesRestartAfterOriginalIsDeleted() {+ String region = "us-east-1";+ InMemoryStorage<String, Queue> queueStore = new InMemoryStorage<>();+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>();+ InMemoryStorage<String, Map<String, Long>> dedupStore = new InMemoryStorage<>();+ InMemoryStorage<String, Map<String, Map<String, String>>> identityStore = new InMemoryStorage<>();+ RegionResolver regionResolver = new RegionResolver(region, "000000000000");+ SqsService initialService = new SqsService(queueStore, messageStore, dedupStore, identityStore,+ 30, 1048576, BASE_URL, regionResolver, false, null, clock);+ Queue queue = initialService.createQueue("deleted-before-restart.fifo", null, region);+ Message original = initialService.sendMessage(+ queue.getQueueUrl(), "original", 0, "group-a", "dedup-a", region);+ List<Message> received = initialService.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region);+ initialService.deleteMessage(queue.getQueueUrl(), received.getFirst().getReceiptHandle(), region);++ SqsService restartedService = new SqsService(queueStore, messageStore, dedupStore, identityStore,+ 30, 1048576, BASE_URL, regionResolver, false, null, clock);+ Message duplicateResponse = restartedService.sendMessage(+ queue.getQueueUrl(), "duplicate", 0, "group-a", "dedup-a", region);++ assertEquals(original.getMessageId(), duplicateResponse.getMessageId());+ assertEquals(original.getSequenceNumber(), duplicateResponse.getSequenceNumber());+ assertTrue(restartedService.peekMessages(queue.getQueueUrl(), region).isEmpty());+ }++ @Test+ void fifoDeduplicationExpiresAtTheFiveMinuteBoundary() {+ String region = "eu-west-1";+ Queue queue = sqsService.createQueue("expired-dedup.fifo", null, region);+ sqsService.sendMessage(queue.getQueueUrl(), "original", 0, "group-a", "dedup-a", region);++ clock.advance(Duration.ofMinutes(5));+ sqsService.sendMessage(queue.getQueueUrl(), "after-window", 0, "group-a", "dedup-a", region);++ List<Message> received = sqsService.receiveMessage(queue.getQueueUrl(), 2, 30, 0, region);+ assertEquals(2, received.size());+ assertTrue(received.stream().anyMatch(message -> "after-window".equals(message.getBody())));+ }++ @Test+ void concurrentFifoSendsAfterExpiryKeepTheNewDeduplicationEntry() throws Exception {+ String region = "eu-west-1";+ ExecutorService executor = Executors.newFixedThreadPool(2);+ try {+ for (int round = 0; round < 25; round++) {+ Queue queue = sqsService.createQueue("expired-concurrent-" + round + ".fifo", null, region);+ String queueUrl = queue.getQueueUrl();+ sqsService.sendMessage(queueUrl, "original", 0, "group-a", "dedup-a", region);+ Message original = sqsService.receiveMessage(queueUrl, 1, 30, 0, region).getFirst();+ sqsService.deleteMessage(queueUrl, original.getReceiptHandle(), region);+ clock.advance(Duration.ofMinutes(5));++ CountDownLatch ready = new CountDownLatch(2);+ CountDownLatch start = new CountDownLatch(1);+ Future<Message> first = executor.submit(() -> {+ ready.countDown();+ start.await();+ return sqsService.sendMessage(queueUrl, "renewed", 0, "group-a", "dedup-a", region);+ });+ Future<Message> second = executor.submit(() -> {+ ready.countDown();+ start.await();+ return sqsService.sendMessage(queueUrl, "renewed", 0, "group-a", "dedup-a", region);+ });+ assertTrue(ready.await(5, TimeUnit.SECONDS));+ start.countDown();++ Message firstResponse = first.get(5, TimeUnit.SECONDS);+ Message secondResponse = second.get(5, TimeUnit.SECONDS);+ assertEquals(firstResponse.getMessageId(), secondResponse.getMessageId());+ assertEquals(firstResponse.getSequenceNumber(), secondResponse.getSequenceNumber());+ assertEquals(1, sqsService.peekMessages(queueUrl, region).size());+ }+ } finally {+ executor.shutdownNow();+ }+ }++ @Test+ void duplicateInAnotherGroupDoesNotWaitForBlockedMessageWrite() throws Exception {+ String region = "us-east-1";+ CountDownLatch writing = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>() {+ @Override+ public void put(String key, List<Message> messages) {+ if (messages.stream().anyMatch(message -> "slow".equals(message.getBody()))) {+ writing.countDown();+ try {+ if (!release.await(5, TimeUnit.SECONDS)) {+ throw new IllegalStateException("Timed out waiting to release message write");+ }+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ }+ super.put(key, messages);+ }+ };+ SqsService service = new SqsService(new InMemoryStorage<>(), messageStore,+ new InMemoryStorage<>(), new InMemoryStorage<>(), 30, 1048576, BASE_URL,+ new RegionResolver(region, "000000000000"), false, null, clock);+ Queue queue = service.createQueue("unrelated-send.fifo", null, region);+ Message original = service.sendMessage(queue.getQueueUrl(), "original", 0,+ "group-a", "dedup-a", region);++ ExecutorService executor = Executors.newFixedThreadPool(2);+ try {+ Future<Message> slow = executor.submit(() -> service.sendMessage(queue.getQueueUrl(),+ "slow", 0, "group-b", "dedup-b", region));+ assertTrue(writing.await(5, TimeUnit.SECONDS));+ Future<Message> duplicate = executor.submit(() -> service.sendMessage(queue.getQueueUrl(),+ "duplicate", 0, "group-a", "dedup-a", region));+ assertEquals(original.getMessageId(), duplicate.get(2, TimeUnit.SECONDS).getMessageId());+ release.countDown();+ slow.get(5, TimeUnit.SECONDS);+ } finally {+ release.countDown();+ executor.shutdownNow();+ }+ }++ @Test+ void concurrentFifoSendsRetainAllDeduplicationEntriesAfterRestart() throws Exception {+ String region = "us-east-1";+ InMemoryStorage<String, Queue> queueStore = new InMemoryStorage<>();+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>();+ InMemoryStorage<String, Map<String, Long>> dedupStore = new InMemoryStorage<>();+ InMemoryStorage<String, Map<String, Map<String, String>>> identityStore = new InMemoryStorage<>();+ RegionResolver regionResolver = new RegionResolver(region, "000000000000");+ SqsService service = new SqsService(queueStore, messageStore, dedupStore, identityStore,+ 30, 1048576, BASE_URL, regionResolver, false, null, clock);+ Queue queue = service.createQueue("concurrent-persist.fifo", null, region);+ ExecutorService executor = Executors.newFixedThreadPool(8);+ try {+ List<Future<Message>> sent = new ArrayList<>();+ for (int i = 0; i < 32; i++) {+ int id = i;+ sent.add(executor.submit(() -> service.sendMessage(queue.getQueueUrl(),+ "message-" + id, 0, "group-" + id, "dedup-" + id, region)));+ }+ List<Message> originals = new ArrayList<>();+ for (Future<Message> send : sent) {+ originals.add(send.get(5, TimeUnit.SECONDS));+ }+ SqsService restarted = new SqsService(queueStore, messageStore, dedupStore, identityStore,+ 30, 1048576, BASE_URL, regionResolver, false, null, clock);+ for (int i = 0; i < originals.size(); i++) {+ Message duplicate = restarted.sendMessage(queue.getQueueUrl(),+ "duplicate-" + i, 0, "group-" + i, "dedup-" + i, region);+ assertEquals(originals.get(i).getMessageId(), duplicate.getMessageId());+ }+ assertEquals(32, restarted.peekMessages(queue.getQueueUrl(), region).size());+ } finally {+ executor.shutdownNow();+ }+ }++ @Test+ void contentBasedFifoDeduplicationStillSuppressesMessageAfterOriginalIsDeleted() {+ String region = "eu-west-1";+ Queue queue = sqsService.createQueue("content-deleted-original.fifo",+ Map.of("ContentBasedDeduplication", "true"), region);+ sqsService.sendMessage(queue.getQueueUrl(), "same-body", 0, "group-a", null, region);++ List<Message> received = sqsService.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region);+ assertEquals(1, received.size());+ sqsService.deleteMessage(queue.getQueueUrl(), received.getFirst().getReceiptHandle(), region);++ sqsService.sendMessage(queue.getQueueUrl(), "same-body", 0, "group-a", null, region);++ assertTrue(sqsService.peekMessages(queue.getQueueUrl(), region).isEmpty(),+ "Content-based duplicate IDs must remain suppressed after deletion");+ }+@Testvoid fifoQueueReceiveReturnsMultipleMessagesPerGroupInOrder() {// AWS FIFO: a single ReceiveMessage call may return multiple messages@@ -761,13 +964,13 @@ class SqsServiceTest {assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 10, 0, 0, region).isEmpty(),"Queue must be empty after purge");- // Re-send with same dedup ID — dedup cache fires but finds no message (purged),- // so it falls through and creates a new message+ // Re-send with the same dedup ID. Purging removed the original message,+ // but does not end its five-minute deduplication interval.sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "group1", "dedup-1", region);List<Message> received = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0, region);- assertEquals(1, received.size(),- "Re-send after purge must produce exactly one message in the queue");+ assertTrue(received.isEmpty(),+ "A duplicate must remain suppressed after purge during the deduplication interval");}@Test