Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sqs): FIFO ReceiveMessage returns multiple messages per group per call (#785)
Stuart Lang3bf16e3fb9c3
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.javaindex 8fc8fcd138..825d169449 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java@@ -343,6 +343,45 @@ class SqsTest {@Test@Order(20)+ void fifoReceiveReturnsMultipleMessagesFromSameGroupInOneCall() {+ // Regression test for https://github.com/floci-io/floci/issues/777+ // AWS FIFO: a single ReceiveMessage may return multiple messages from+ // the same MessageGroupId, up to MaxNumberOfMessages.+ String fifoQueueName = "sdk-test-fifo-multi-" + System.currentTimeMillis() + ".fifo";+ String fifoQueueUrl = sqs.createQueue(CreateQueueRequest.builder()+ .queueName(fifoQueueName)+ .attributes(Map.of(QueueAttributeName.FIFO_QUEUE, "true"))+ .build()).queueUrl();+ try {+ for (int i = 1; i <= 3; i++) {+ sqs.sendMessage(SendMessageRequest.builder()+ .queueUrl(fifoQueueUrl)+ .messageBody("msg" + i)+ .messageGroupId("g1")+ .messageDeduplicationId("d" + i)+ .build());+ }++ ReceiveMessageResponse rcv = sqs.receiveMessage(ReceiveMessageRequest.builder()+ .queueUrl(fifoQueueUrl)+ .maxNumberOfMessages(10)+ .build());++ assertThat(rcv.messages())+ .as("FIFO ReceiveMessage must return all 3 messages from group g1 in one call")+ .hasSize(3);+ assertThat(rcv.messages().get(0).body()).isEqualTo("msg1");+ assertThat(rcv.messages().get(1).body()).isEqualTo("msg2");+ assertThat(rcv.messages().get(2).body()).isEqualTo("msg3");+ } finally {+ try {+ sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(fifoQueueUrl).build());+ } catch (Exception ignored) {}+ }+ }++ @Test+ @Order(21)void deleteQueue() {sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(queueUrl).build());if (dlqUrl != null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.javaindex 59317fdc2c..9f316744a7 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java@@ -115,10 +115,12 @@ class GuardedMessageQueue {private void claimFifo(int maxMessages, int effectiveTimeout,int maxReceiveCount, String deadLetterTargetArn,List<Message> claimed, List<Message> dlqCandidates) {- Set<String> groupsWithInFlight;- Set<String> groupsDelivered = new HashSet<>();-- groupsWithInFlight =+ // Cross-call group locking: a group that already has an in-flight+ // message from a previous ReceiveMessage call is blocked until that+ // message is deleted or its visibility expires. Within a single call+ // we may return multiple messages from the same group (preserving+ // insertion order), up to MaxNumberOfMessages.+ Set<String> groupsWithInFlight =messages.stream().filter(msg -> !msg.isVisible() && msg.getMessageGroupId() != null).map(Message::getMessageGroupId).collect(Collectors.toSet());@@ -128,12 +130,8 @@ class GuardedMessageQueue {String groupId = msg.getMessageGroupId();if (groupId != null && groupsWithInFlight.contains(groupId)) continue;- if (groupId != null && groupsDelivered.contains(groupId)) continue;- if (tryClaim(msg, effectiveTimeout, maxReceiveCount, deadLetterTargetArn, claimed, dlqCandidates)- && groupId != null) {- groupsDelivered.add(groupId);- }+ tryClaim(msg, effectiveTimeout, maxReceiveCount, deadLetterTargetArn, claimed, dlqCandidates);}}diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.javaindex a089b6dcd3..f9930a1700 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java@@ -167,7 +167,10 @@ class GuardedMessageQueueTest {// --- FIFO ---@Test- void fifoClaimRespectsGroupOrdering() {+ void fifoClaimReturnsMultipleMessagesPerGroupWithinSingleCall() {+ // AWS FIFO: a single ReceiveMessage may return multiple messages from+ // the same MessageGroupId, preserving insertion order. The cross-call+ // group lock applies only against future calls.Message g1m1 = new Message("g1-msg1");g1m1.setMessageGroupId("group1");Message g1m2 = new Message("g1-msg2");@@ -179,13 +182,18 @@ class GuardedMessageQueueTest {queue.addMessage(g1m2);queue.addMessage(g2m1);- // Should get first from each groupvar first = queue.claimVisibleMessages(10, 30, true, -1, null);- assertEquals(2, first.claimed().size());- assertEquals("g1-msg1", first.claimed().get(0).getBody());- assertEquals("g2-msg1", first.claimed().get(1).getBody());-- // Both groups blocked+ assertEquals(3, first.claimed().size(),+ "Single FIFO ReceiveMessage should return all visible messages up to MaxNumberOfMessages");+ List<String> bodies = first.claimed().stream().map(Message::getBody).toList();+ // Inter-group ordering is not guaranteed by FIFO; only within-group order is.+ assertTrue(bodies.contains("g2-msg1"), "batch must contain group2 message");+ int g1m1Idx = bodies.indexOf("g1-msg1");+ int g1m2Idx = bodies.indexOf("g1-msg2");+ assertTrue(g1m1Idx >= 0 && g1m2Idx >= 0, "batch must contain both group1 messages");+ assertTrue(g1m1Idx < g1m2Idx, "group1 messages must be in insertion order");++ // All groups now have in-flight messages — second call returns emptyvar second = queue.claimVisibleMessages(10, 30, true, -1, null);assertTrue(second.claimed().isEmpty());}diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.javaindex cd390a32d4..684370463e 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java@@ -277,27 +277,29 @@ class SqsServiceTest {}@Test- void fifoQueueReceiveRespectsGroupOrdering() {+ void fifoQueueReceiveReturnsMultipleMessagesPerGroupInOrder() {+ // AWS FIFO: a single ReceiveMessage call may return multiple messages+ // from the same MessageGroupId (in order), up to MaxNumberOfMessages.+ // The group lock only blocks subsequent ReceiveMessage calls.Queue queue = sqsService.createQueue("test.fifo", null);sqsService.sendMessage(queue.getQueueUrl(), "g1-msg1", 0, "group1", "d1");sqsService.sendMessage(queue.getQueueUrl(), "g1-msg2", 0, "group1", "d2");sqsService.sendMessage(queue.getQueueUrl(), "g2-msg1", 0, "group2", "d3");- // First receive: should get one message per group (first from each)List<Message> first = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);- assertEquals(2, first.size());- assertEquals("g1-msg1", first.get(0).getBody());- assertEquals("g2-msg1", first.get(1).getBody());-- // Second receive: group1 and group2 both have in-flight messages, so nothing returned+ assertEquals(3, first.size(),+ "Single FIFO ReceiveMessage should drain all visible messages up to MaxNumberOfMessages");+ List<String> bodies = first.stream().map(Message::getBody).toList();+ // Inter-group ordering is not guaranteed by FIFO; only within-group order is.+ assertTrue(bodies.contains("g2-msg1"), "batch must contain group2 message");+ int g1m1Idx = bodies.indexOf("g1-msg1");+ int g1m2Idx = bodies.indexOf("g1-msg2");+ assertTrue(g1m1Idx >= 0 && g1m2Idx >= 0, "batch must contain both group1 messages");+ assertTrue(g1m1Idx < g1m2Idx, "group1 messages must be in insertion order");++ // Both groups are now in-flight; second call returns empty.List<Message> second = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);assertTrue(second.isEmpty());-- // Delete the group1 message, then receive again — should get g1-msg2- sqsService.deleteMessage(queue.getQueueUrl(), first.get(0).getReceiptHandle());- List<Message> third = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);- assertEquals(1, third.size());- assertEquals("g1-msg2", third.get(0).getBody());}@Test@@ -553,6 +555,47 @@ class SqsServiceTest {assertEquals("AWS.SimpleQueueService.NonExistentQueue", ex.getErrorCode());}+ @Test+ void fifoQueueGroupLockBlocksAcrossCallsButNotWithin() {+ Queue queue = sqsService.createQueue("group-lock.fifo", null);+ for (int i = 1; i <= 5; i++) {+ sqsService.sendMessage(queue.getQueueUrl(), "msg" + i, 0, "g1", "d" + i);+ }++ // First call drains all five from the single group.+ List<Message> first = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);+ assertEquals(5, first.size());++ // Second call returns nothing because g1 is in-flight.+ assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0).isEmpty());+ }++ @Test+ void fifoQueueGroupUnlocksAfterAllInFlightMessagesDeleted() {+ Queue queue = sqsService.createQueue("unlock-test.fifo", null);+ sqsService.sendMessage(queue.getQueueUrl(), "msg1", 0, "g1", "d1");+ sqsService.sendMessage(queue.getQueueUrl(), "msg2", 0, "g1", "d2");+ sqsService.sendMessage(queue.getQueueUrl(), "msg3", 0, "g1", "d3");++ // Partial drain: MaxNumberOfMessages=2 returns msg1 + msg2+ List<Message> first = sqsService.receiveMessage(queue.getQueueUrl(), 2, 30, 0);+ assertEquals(2, first.size());+ assertEquals("msg1", first.get(0).getBody());+ assertEquals("msg2", first.get(1).getBody());++ // Group still locked — msg3 is not returned+ assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0).isEmpty());++ // Delete both in-flight messages → group unlocks+ sqsService.deleteMessage(queue.getQueueUrl(), first.get(0).getReceiptHandle());+ sqsService.deleteMessage(queue.getQueueUrl(), first.get(1).getReceiptHandle());++ // Now msg3 should be available+ List<Message> third = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);+ assertEquals(1, third.size());+ assertEquals("msg3", third.get(0).getBody());+ }+@Testvoid purgeQueueWithClearFifoDelegatesToSnsForFifoDedupOnSubscribedTopics() {final var sns = mock(SnsService.class);