Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sqs): FIFO ReceiveMessage returns multiple messages per group per call (#785)

Stuart Lang3bf16e3fb9c3
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java
index 8fc8fcd138..825d169449 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SqsTest.java
@@ -343,6 +343,45 @@ class SqsTest {
@Test
@Order(20)
+ void fifoReceiveReturnsMultipleMessagesFromSameGroupInOneCall() {
+ // Regression test for https://github.com/floci-io/floci/issues/777
+ // AWS FIFO: a single ReceiveMessage may return multiple messages from
+ // the same MessageGroupId, up to MaxNumberOfMessages.
+ String fifoQueueName = "sdk-test-fifo-multi-" + System.currentTimeMillis() + ".fifo";
+ String fifoQueueUrl = sqs.createQueue(CreateQueueRequest.builder()
+ .queueName(fifoQueueName)
+ .attributes(Map.of(QueueAttributeName.FIFO_QUEUE, "true"))
+ .build()).queueUrl();
+ try {
+ for (int i = 1; i <= 3; i++) {
+ sqs.sendMessage(SendMessageRequest.builder()
+ .queueUrl(fifoQueueUrl)
+ .messageBody("msg" + i)
+ .messageGroupId("g1")
+ .messageDeduplicationId("d" + i)
+ .build());
+ }
+
+ ReceiveMessageResponse rcv = sqs.receiveMessage(ReceiveMessageRequest.builder()
+ .queueUrl(fifoQueueUrl)
+ .maxNumberOfMessages(10)
+ .build());
+
+ assertThat(rcv.messages())
+ .as("FIFO ReceiveMessage must return all 3 messages from group g1 in one call")
+ .hasSize(3);
+ assertThat(rcv.messages().get(0).body()).isEqualTo("msg1");
+ assertThat(rcv.messages().get(1).body()).isEqualTo("msg2");
+ assertThat(rcv.messages().get(2).body()).isEqualTo("msg3");
+ } finally {
+ try {
+ sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(fifoQueueUrl).build());
+ } catch (Exception ignored) {}
+ }
+ }
+
+ @Test
+ @Order(21)
void deleteQueue() {
sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(queueUrl).build());
if (dlqUrl != null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
index 59317fdc2c..9f316744a7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
@@ -115,10 +115,12 @@ class GuardedMessageQueue {
private void claimFifo(int maxMessages, int effectiveTimeout,
int maxReceiveCount, String deadLetterTargetArn,
List<Message> claimed, List<Message> dlqCandidates) {
- Set<String> groupsWithInFlight;
- Set<String> groupsDelivered = new HashSet<>();
-
- groupsWithInFlight =
+ // Cross-call group locking: a group that already has an in-flight
+ // message from a previous ReceiveMessage call is blocked until that
+ // message is deleted or its visibility expires. Within a single call
+ // we may return multiple messages from the same group (preserving
+ // insertion order), up to MaxNumberOfMessages.
+ Set<String> groupsWithInFlight =
messages.stream().filter(msg -> !msg.isVisible() && msg.getMessageGroupId() != null)
.map(Message::getMessageGroupId).collect(Collectors.toSet());
@@ -128,12 +130,8 @@ class GuardedMessageQueue {
String groupId = msg.getMessageGroupId();
if (groupId != null && groupsWithInFlight.contains(groupId)) continue;
- if (groupId != null && groupsDelivered.contains(groupId)) continue;
- if (tryClaim(msg, effectiveTimeout, maxReceiveCount, deadLetterTargetArn, claimed, dlqCandidates)
- && groupId != null) {
- groupsDelivered.add(groupId);
- }
+ tryClaim(msg, effectiveTimeout, maxReceiveCount, deadLetterTargetArn, claimed, dlqCandidates);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
index a089b6dcd3..f9930a1700 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
@@ -167,7 +167,10 @@ class GuardedMessageQueueTest {
// --- FIFO ---
@Test
- void fifoClaimRespectsGroupOrdering() {
+ void fifoClaimReturnsMultipleMessagesPerGroupWithinSingleCall() {
+ // AWS FIFO: a single ReceiveMessage may return multiple messages from
+ // the same MessageGroupId, preserving insertion order. The cross-call
+ // group lock applies only against future calls.
Message g1m1 = new Message("g1-msg1");
g1m1.setMessageGroupId("group1");
Message g1m2 = new Message("g1-msg2");
@@ -179,13 +182,18 @@ class GuardedMessageQueueTest {
queue.addMessage(g1m2);
queue.addMessage(g2m1);
- // Should get first from each group
var first = queue.claimVisibleMessages(10, 30, true, -1, null);
- assertEquals(2, first.claimed().size());
- assertEquals("g1-msg1", first.claimed().get(0).getBody());
- assertEquals("g2-msg1", first.claimed().get(1).getBody());
-
- // Both groups blocked
+ assertEquals(3, first.claimed().size(),
+ "Single FIFO ReceiveMessage should return all visible messages up to MaxNumberOfMessages");
+ List<String> bodies = first.claimed().stream().map(Message::getBody).toList();
+ // Inter-group ordering is not guaranteed by FIFO; only within-group order is.
+ assertTrue(bodies.contains("g2-msg1"), "batch must contain group2 message");
+ int g1m1Idx = bodies.indexOf("g1-msg1");
+ int g1m2Idx = bodies.indexOf("g1-msg2");
+ assertTrue(g1m1Idx >= 0 && g1m2Idx >= 0, "batch must contain both group1 messages");
+ assertTrue(g1m1Idx < g1m2Idx, "group1 messages must be in insertion order");
+
+ // All groups now have in-flight messages — second call returns empty
var second = queue.claimVisibleMessages(10, 30, true, -1, null);
assertTrue(second.claimed().isEmpty());
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index cd390a32d4..684370463e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -277,27 +277,29 @@ class SqsServiceTest {
}
@Test
- void fifoQueueReceiveRespectsGroupOrdering() {
+ void fifoQueueReceiveReturnsMultipleMessagesPerGroupInOrder() {
+ // AWS FIFO: a single ReceiveMessage call may return multiple messages
+ // from the same MessageGroupId (in order), up to MaxNumberOfMessages.
+ // The group lock only blocks subsequent ReceiveMessage calls.
Queue queue = sqsService.createQueue("test.fifo", null);
sqsService.sendMessage(queue.getQueueUrl(), "g1-msg1", 0, "group1", "d1");
sqsService.sendMessage(queue.getQueueUrl(), "g1-msg2", 0, "group1", "d2");
sqsService.sendMessage(queue.getQueueUrl(), "g2-msg1", 0, "group2", "d3");
- // First receive: should get one message per group (first from each)
List<Message> first = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);
- assertEquals(2, first.size());
- assertEquals("g1-msg1", first.get(0).getBody());
- assertEquals("g2-msg1", first.get(1).getBody());
-
- // Second receive: group1 and group2 both have in-flight messages, so nothing returned
+ assertEquals(3, first.size(),
+ "Single FIFO ReceiveMessage should drain all visible messages up to MaxNumberOfMessages");
+ List<String> bodies = first.stream().map(Message::getBody).toList();
+ // Inter-group ordering is not guaranteed by FIFO; only within-group order is.
+ assertTrue(bodies.contains("g2-msg1"), "batch must contain group2 message");
+ int g1m1Idx = bodies.indexOf("g1-msg1");
+ int g1m2Idx = bodies.indexOf("g1-msg2");
+ assertTrue(g1m1Idx >= 0 && g1m2Idx >= 0, "batch must contain both group1 messages");
+ assertTrue(g1m1Idx < g1m2Idx, "group1 messages must be in insertion order");
+
+ // Both groups are now in-flight; second call returns empty.
List<Message> second = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);
assertTrue(second.isEmpty());
-
- // Delete the group1 message, then receive again — should get g1-msg2
- sqsService.deleteMessage(queue.getQueueUrl(), first.get(0).getReceiptHandle());
- List<Message> third = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);
- assertEquals(1, third.size());
- assertEquals("g1-msg2", third.get(0).getBody());
}
@Test
@@ -553,6 +555,47 @@ class SqsServiceTest {
assertEquals("AWS.SimpleQueueService.NonExistentQueue", ex.getErrorCode());
}
+ @Test
+ void fifoQueueGroupLockBlocksAcrossCallsButNotWithin() {
+ Queue queue = sqsService.createQueue("group-lock.fifo", null);
+ for (int i = 1; i <= 5; i++) {
+ sqsService.sendMessage(queue.getQueueUrl(), "msg" + i, 0, "g1", "d" + i);
+ }
+
+ // First call drains all five from the single group.
+ List<Message> first = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);
+ assertEquals(5, first.size());
+
+ // Second call returns nothing because g1 is in-flight.
+ assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0).isEmpty());
+ }
+
+ @Test
+ void fifoQueueGroupUnlocksAfterAllInFlightMessagesDeleted() {
+ Queue queue = sqsService.createQueue("unlock-test.fifo", null);
+ sqsService.sendMessage(queue.getQueueUrl(), "msg1", 0, "g1", "d1");
+ sqsService.sendMessage(queue.getQueueUrl(), "msg2", 0, "g1", "d2");
+ sqsService.sendMessage(queue.getQueueUrl(), "msg3", 0, "g1", "d3");
+
+ // Partial drain: MaxNumberOfMessages=2 returns msg1 + msg2
+ List<Message> first = sqsService.receiveMessage(queue.getQueueUrl(), 2, 30, 0);
+ assertEquals(2, first.size());
+ assertEquals("msg1", first.get(0).getBody());
+ assertEquals("msg2", first.get(1).getBody());
+
+ // Group still locked — msg3 is not returned
+ assertTrue(sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0).isEmpty());
+
+ // Delete both in-flight messages → group unlocks
+ sqsService.deleteMessage(queue.getQueueUrl(), first.get(0).getReceiptHandle());
+ sqsService.deleteMessage(queue.getQueueUrl(), first.get(1).getReceiptHandle());
+
+ // Now msg3 should be available
+ List<Message> third = sqsService.receiveMessage(queue.getQueueUrl(), 10, 30, 0);
+ assertEquals(1, third.size());
+ assertEquals("msg3", third.get(0).getBody());
+ }
+
@Test
void purgeQueueWithClearFifoDelegatesToSnsForFifoDedupOnSubscribedTopics() {
final var sns = mock(SnsService.class);