Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sqs): keep DLQ messages when a move task cannot deliver them (#3470)
Niels Kootstraa9feb3ccaf76
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.javaindex 6495e399bc..15c1b5e866 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.services.sqs.model.Message;import java.time.Instant;import java.util.*;import java.util.concurrent.locks.ReentrantLock;+import java.util.function.Predicate;import java.util.stream.Collectors;/**@@ -53,10 +54,18 @@ class GuardedMessageQueue {}}+ /** If persisting fails the in-memory add is rolled back and the exception propagates, so a+ * caller that compensates on the source side does not leave a duplicate here. */void addAll(List<Message> toAdd) {try (var _ = hold()) {+ int mark = messages.size();messages.addAll(toAdd);- persist();+ try {+ persist();+ } catch (RuntimeException e) {+ messages.subList(mark, messages.size()).clear();+ throw e;+ }}}@@ -194,20 +203,29 @@ class GuardedMessageQueue {}}- /** Remove and return the first message in insertion order, or null if empty.- * Used by the message-move-task worker so the source queue stays observably- * populated for the duration of a rate-limited move. */- Message drainOne() {+ /** Remove and return the head message if {@code eligible} accepts it; otherwise leave the+ * queue untouched and return {@code null}. The check and the removal happen under one lock+ * hold so nothing can slip in between. Used by the message-move-task worker so the source+ * queue stays observably populated for the duration of a rate-limited move. */+ Message drainFirstIf(Predicate<Message> eligible) {try (var _ = hold()) {- if (messages.isEmpty()) {+ if (messages.isEmpty() || !eligible.test(messages.getFirst())) {return null;}- Message head = messages.remove(0);+ Message head = messages.removeFirst();persist();return head;}}+ /** Put a message that could not be delivered back at the head, preserving queue order. */+ void restoreFirst(Message message) {+ try (var _ = hold()) {+ messages.addFirst(message);+ persist();+ }+ }+record MessageCounts(long visible, long inFlight, long delayed) {}diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.javaindex caf1c14cf5..3c341ef6b7 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java@@ -1042,8 +1042,7 @@ public class SqsService implements Resettable, ResourceProvider {// remainder runs on the background worker at the requested rate, draining// one message at a time so the source queue stays observably populated.long initialMoved = 0;- Message firstMsg = srcQueueInitial.drainOne();- if (firstMsg != null && deliverMovedMessage(firstMsg, destUrl, region)) {+ if (moveOneMessage(srcQueueInitial, destUrl, region)) {initialMoved = 1;updateMoveTaskCounter(taskHandle, initialMoved);}@@ -1059,31 +1058,55 @@ public class SqsService implements Resettable, ResourceProvider {/** Place a single drained message in its destination according to the rules of* StartMessageMoveTask (explicit destination, or the per-message origin- * recorded when it was DLQ'd). Resets per-receive state so the message- * starts a fresh life. Returns true when the message was placed. */+ * recorded when it was DLQ'd). Delivers a copy whose per-receive state starts+ * a fresh life; {@code msg} itself is left untouched so that, if delivery+ * fails, the caller can restore it to the source exactly as it was.+ * Returns true when the message was placed. */private boolean deliverMovedMessage(Message msg, String destUrl, String region) {- msg.setReceiveCount(0);- msg.setFirstReceiveTimestamp(null);- msg.setReceiptHandle(null);- msg.setVisibleAt(null);+ Message moved = msg.copyForRedrive();if (destUrl != null) {String destKey = regionKey(region, destUrl);- getOrCreateQueue(destKey).addAll(List.of(msg));+ getOrCreateQueue(destKey).addAll(List.of(moved));return true;}if (msg.getOriginalSourceQueueUrl() != null) {- // No request scope on the background worker, so queueStore.get() resolves- // to the default account prefix and can't verify the destination. The- // in-memory messagesByQueue map is keyed by the full URL (which carries+ // canDeliverMovedMessage() already confirmed the original source queue exists.+ // The in-memory messagesByQueue map is keyed by the full URL (which carries// the account), so addAll on the right storage key lands the message in// the queue any future receive will see.String originKey = regionKey(region, msg.getOriginalSourceQueueUrl());- getOrCreateQueue(originKey).addAll(List.of(msg));+ getOrCreateQueue(originKey).addAll(List.of(moved));return true;}return false;}+ /** The move worker runs outside any request scope, so the queue lookup must take the+ * account from the target URL instead of the (absent) request context. */+ private boolean canDeliverMovedMessage(Message msg, String destUrl, String region) {+ String targetUrl = destUrl != null ? destUrl : msg.getOriginalSourceQueueUrl();+ return targetUrl != null && getQueueByUrl(regionKey(region, targetUrl), targetUrl).isPresent();+ }++ /** Move the head message of {@code sourceQueue}. Returns {@code false} when there is+ * nothing to move, the target queue is missing, or delivery failed; in the latter case+ * the message is put back at the head so the source keeps its order. */+ private boolean moveOneMessage(GuardedMessageQueue sourceQueue, String destUrl, String region) {+ Message message = sourceQueue.drainFirstIf(msg -> canDeliverMovedMessage(msg, destUrl, region));+ if (message == null) {+ return false;+ }+ try {+ if (deliverMovedMessage(message, destUrl, region)) {+ return true;+ }+ } catch (RuntimeException e) {+ LOG.warnv(e, "Failed to move a message to {0}", destUrl);+ }+ sourceQueue.restoreFirst(message);+ return false;+ }+/** Background body of a move task; drains one message at a time so a* positive MaxNumberOfMessagesPerSecond can throttle, CancelMessageMoveTask* has a window to stop the work before it finishes, and the source queue@@ -1110,14 +1133,11 @@ public class SqsService implements Resettable, ResourceProvider {break;}}- Message msg = srcQueue.drainOne();- if (msg == null) {+ if (!moveOneMessage(srcQueue, destUrl, region)) {break;}- if (deliverMovedMessage(msg, destUrl, region)) {- moved++;- updateMoveTaskCounter(taskHandle, moved);- }+ moved++;+ updateMoveTaskCounter(taskHandle, moved);}} finally {MoveTask cur = moveTasksByHandle.get(taskHandle);diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java b/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.javaindex 87307a01e6..dfb4b713f0 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java@@ -57,6 +57,26 @@ public class Message {this.md5OfBody = computeMd5(body);}+ /** A copy for delivery to another queue by StartMessageMoveTask: identity and content carry+ * over; per-receive state (receive count, first-receive timestamp, receipt handle, visibility)+ * starts over. The source instance is untouched so a failed delivery can put it back as it was. */+ public Message copyForRedrive() {+ Message copy = new Message();+ copy.messageId = messageId;+ copy.body = body;+ copy.messageAttributes = messageAttributes == null ? new HashMap<>() : new HashMap<>(messageAttributes);+ copy.sentTimestamp = sentTimestamp;+ copy.md5OfBody = md5OfBody;+ copy.md5OfMessageAttributes = md5OfMessageAttributes;+ copy.messageGroupId = messageGroupId;+ copy.messageDeduplicationId = messageDeduplicationId;+ copy.sequenceNumber = sequenceNumber;+ copy.originalSourceQueueUrl = originalSourceQueueUrl;+ copy.awsTraceHeader = awsTraceHeader;+ // receiveCount 0, firstReceiveTimestamp / receiptHandle / visibleAt null: a fresh life.+ return copy;+ }+public String getMessageId() { return messageId; }public void setMessageId(String messageId) { this.messageId = messageId; }diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.javaindex e386865d64..ad18f967f3 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.sqs;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.sqs.model.Message;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;@@ -15,11 +16,13 @@ import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicInteger;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;class GuardedMessageQueueTest {@@ -148,6 +151,28 @@ class GuardedMessageQueueTest {assertEquals(2, result.claimed().size());}+ @Test+ void addAllRollsBackTheInMemoryAddWhenPersistFails() {+ AtomicBoolean storeDown = new AtomicBoolean();+ var store = new InMemoryStorage<String, List<Message>>() {+ @Override+ public void put(String key, List<Message> value) {+ if (storeDown.get()) {+ throw new IllegalStateException("store unavailable");+ }+ super.put(key, value);+ }+ };+ var target = new GuardedMessageQueue(store, "us-east-1::/000000000000/target");+ target.addMessage(new Message("already-there"));+ storeDown.set(true);++ assertThrows(IllegalStateException.class, () -> target.addAll(List.of(new Message("incoming"))));++ assertEquals(List.of("already-there"), target.peekAll().stream().map(Message::getBody).toList());+ assertEquals(1, target.messageCounts().visible());+ }+@Testvoid messageCountsReturnsVisibleInFlightAndDelayed() {queue.addMessage(new Message("msg1"));diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.javaindex 1e880fb452..f2d30c791e 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java@@ -2,26 +2,33 @@ package io.github.hectorvent.floci.services.sqs;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.RequestContext;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.sns.SnsService;import io.github.hectorvent.floci.services.sqs.model.Message;import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;import io.github.hectorvent.floci.services.sqs.model.Queue;import io.github.hectorvent.floci.testing.MutableClock;+import jakarta.enterprise.context.ContextNotActiveException;+import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import java.time.Duration;+import java.time.Instant;import java.util.ArrayList;import java.util.List;import java.util.Map;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicReference;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;class SqsServiceTest {@@ -860,6 +867,175 @@ class SqsServiceTest {assertEquals("ResourceNotFoundException", ex.getErrorCode());}+ @Test+ void startMessageMoveTask_withoutDestinationKeepsMessageWithoutOriginalSource() throws Exception {+ Queue dlq = sqsService.createQueue("orphan-dlq", null, "us-east-1");+ String dlqArn = queueArn("orphan-dlq");+ sqsService.createQueue("orphan-source", Map.of("RedrivePolicy", redrivePolicy(dlqArn)), "us-east-1");+ sqsService.sendMessage(dlq.getQueueUrl(), "orphan", 0, null, null, "us-east-1");+ sqsService.sendMessage(dlq.getQueueUrl(), "next", 0, null, null, "us-east-1");++ String taskHandle = sqsService.startMessageMoveTask(dlqArn, null, 0, "us-east-1");++ awaitMoveTaskStatus(dlqArn, taskHandle, "COMPLETED");+ assertEquals(List.of("orphan", "next"), bodies(sqsService.peekMessages(dlq.getQueueUrl(), "us-east-1")));+ }++ @Test+ void startMessageMoveTask_withoutDestinationKeepsMessageWhoseOriginalSourceWasDeleted() throws Exception {+ Queue dlq = sqsService.createQueue("gone-dlq", null, "us-east-1");+ String dlqArn = queueArn("gone-dlq");+ Queue source = sqsService.createQueue("gone-source", Map.of("RedrivePolicy", redrivePolicy(dlqArn)), "us-east-1");+ // A sibling keeps the DLQ referenced by a redrive policy after the original source is gone.+ sqsService.createQueue("gone-sibling", Map.of("RedrivePolicy", redrivePolicy(dlqArn)), "us-east-1");+ sqsService.sendMessage(source.getQueueUrl(), "stranded", 0, null, null, "us-east-1");+ // Receiving past maxReceiveCount redrives the message into the DLQ with its original source recorded.+ assertEquals(1, sqsService.receiveMessage(source.getQueueUrl(), 1, 0, 0, "us-east-1").size());+ assertEquals(0, sqsService.receiveMessage(source.getQueueUrl(), 1, 0, 0, "us-east-1").size());+ assertEquals(List.of("stranded"), bodies(sqsService.peekMessages(dlq.getQueueUrl(), "us-east-1")));+ sqsService.deleteQueue(source.getQueueUrl(), "us-east-1");++ String taskHandle = sqsService.startMessageMoveTask(dlqArn, null, 0, "us-east-1");++ awaitMoveTaskStatus(dlqArn, taskHandle, "COMPLETED");+ assertEquals(List.of("stranded"), bodies(sqsService.peekMessages(dlq.getQueueUrl(), "us-east-1")));+ }++ @Test+ void startMessageMoveTask_movesEveryMessageOfANonDefaultAccountQueue() throws Exception {+ String account = "111111111111";+ RequestContext requestContext = new RequestContext();+ requestContext.setAccountId(account);+ Thread caller = Thread.currentThread();+ @SuppressWarnings("unchecked")+ Instance<RequestContext> requestContextInstance = mock(Instance.class);+ // Only the calling thread has a request scope; the move worker runs outside any request.+ when(requestContextInstance.get()).thenAnswer(invocation -> {+ if (Thread.currentThread() != caller) {+ throw new ContextNotActiveException();+ }+ return requestContext;+ });+ SqsService service = new SqsService(+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), requestContextInstance, "000000000000"),+ null, null, 30, 1048576, BASE_URL, new RegionResolver("us-east-1", account), false, null, clock);+ String dlqArn = "arn:aws:sqs:us-east-1:" + account + ":acct-dlq";+ Queue dlq = service.createQueue("acct-dlq", null, "us-east-1");+ service.createQueue("acct-source", Map.of("RedrivePolicy", redrivePolicy(dlqArn)), "us-east-1");+ Queue replay = service.createQueue("acct-replay", null, "us-east-1");+ for (String body : List.of("one", "two", "three")) {+ service.sendMessage(dlq.getQueueUrl(), body, 0, null, null, "us-east-1");+ }++ String taskHandle = service.startMessageMoveTask(+ dlqArn, "arn:aws:sqs:us-east-1:" + account + ":acct-replay", 0, "us-east-1");++ awaitMoveTaskStatus(service, dlqArn, taskHandle, "COMPLETED");+ assertEquals(List.of("one", "two", "three"), bodies(service.peekMessages(replay.getQueueUrl(), "us-east-1")));+ assertEquals(List.of(), bodies(service.peekMessages(dlq.getQueueUrl(), "us-east-1")));+ }++ @Test+ void startMessageMoveTask_failedDeliveryLeavesSourceQueueOrderIntact() throws Exception {+ AtomicBoolean destinationStoreDown = new AtomicBoolean();+ SqsService service = serviceWithMessageStoreFailingFor("fail-replay", destinationStoreDown);+ Queue dlq = service.createQueue("fail-dlq", null, "us-east-1");+ String dlqArn = queueArn("fail-dlq");+ service.createQueue("fail-source", Map.of("RedrivePolicy", redrivePolicy(dlqArn)), "us-east-1");+ service.createQueue("fail-replay", null, "us-east-1");+ for (String body : List.of("first", "second", "third")) {+ service.sendMessage(dlq.getQueueUrl(), body, 0, null, null, "us-east-1");+ }+ destinationStoreDown.set(true);++ String taskHandle = service.startMessageMoveTask(dlqArn, queueArn("fail-replay"), 0, "us-east-1");++ awaitMoveTaskStatus(service, dlqArn, taskHandle, "COMPLETED");+ assertEquals(List.of("first", "second", "third"), bodies(service.peekMessages(dlq.getQueueUrl(), "us-east-1")));+ }++ @Test+ void startMessageMoveTask_failedDeliveryRestoresTheSourceMessageUnchanged() throws Exception {+ AtomicBoolean destinationStoreDown = new AtomicBoolean();+ SqsService service = serviceWithMessageStoreFailingFor("held-replay", destinationStoreDown);+ Queue dlq = service.createQueue("held-dlq", null, "us-east-1");+ String dlqArn = queueArn("held-dlq");+ service.createQueue("held-source", Map.of("RedrivePolicy", redrivePolicy(dlqArn)), "us-east-1");+ service.createQueue("held-replay", null, "us-east-1");+ service.sendMessage(dlq.getQueueUrl(), "held", 0, null, null, "us-east-1");+ service.sendMessage(dlq.getQueueUrl(), "next", 0, null, null, "us-east-1");+ // A consumer holds the head message when the redrive starts.+ Message held = service.receiveMessage(dlq.getQueueUrl(), 1, 30, 0, "us-east-1").getFirst();+ String messageId = held.getMessageId();+ String receiptHandle = held.getReceiptHandle();+ Instant firstReceive = held.getFirstReceiveTimestamp();+ Instant visibleAt = held.getVisibleAt();+ assertEquals(1, held.getReceiveCount());+ destinationStoreDown.set(true);++ String taskHandle = service.startMessageMoveTask(dlqArn, queueArn("held-replay"), 0, "us-east-1");++ awaitMoveTaskStatus(service, dlqArn, taskHandle, "COMPLETED");+ List<Message> dlqMessages = service.peekMessages(dlq.getQueueUrl(), "us-east-1");+ assertEquals(List.of("held", "next"), bodies(dlqMessages));+ Message restored = dlqMessages.getFirst();+ assertEquals(messageId, restored.getMessageId());+ assertEquals(1, restored.getReceiveCount());+ assertEquals(firstReceive, restored.getFirstReceiveTimestamp());+ assertEquals(receiptHandle, restored.getReceiptHandle());+ assertEquals(visibleAt, restored.getVisibleAt());+ Map<String, String> attributes = service.getQueueAttributes(dlq.getQueueUrl(),+ List.of("ApproximateNumberOfMessages", "ApproximateNumberOfMessagesNotVisible"), "us-east-1");+ assertEquals("1", attributes.get("ApproximateNumberOfMessages"));+ assertEquals("1", attributes.get("ApproximateNumberOfMessagesNotVisible"));+ }++ @Test+ void startMessageMoveTask_failedDeliveryLeavesNoCopyInTheDestination() throws Exception {+ AtomicBoolean destinationStoreDown = new AtomicBoolean();+ SqsService service = serviceWithMessageStoreFailingFor("dup-replay", destinationStoreDown);+ Queue dlq = service.createQueue("dup-dlq", null, "us-east-1");+ String dlqArn = queueArn("dup-dlq");+ service.createQueue("dup-source", Map.of("RedrivePolicy", redrivePolicy(dlqArn)), "us-east-1");+ Queue replay = service.createQueue("dup-replay", null, "us-east-1");+ for (String body : List.of("first", "second", "third")) {+ service.sendMessage(dlq.getQueueUrl(), body, 0, null, null, "us-east-1");+ }+ destinationStoreDown.set(true);++ String taskHandle = service.startMessageMoveTask(dlqArn, queueArn("dup-replay"), 0, "us-east-1");++ awaitMoveTaskStatus(service, dlqArn, taskHandle, "COMPLETED");+ assertEquals(List.of(), bodies(service.peekMessages(replay.getQueueUrl(), "us-east-1")));+ assertEquals("0", service.getQueueAttributes(replay.getQueueUrl(),+ List.of("ApproximateNumberOfMessages"), "us-east-1").get("ApproximateNumberOfMessages"));+ assertTrue(service.receiveMessage(replay.getQueueUrl(), 10, 0, 0, "us-east-1").isEmpty());+ assertEquals(List.of("first", "second", "third"), bodies(service.peekMessages(dlq.getQueueUrl(), "us-east-1")));+ }++ /** A service whose message store rejects writes for {@code queueName} while {@code down} is set. */+ private SqsService serviceWithMessageStoreFailingFor(String queueName, AtomicBoolean down) {+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>() {+ @Override+ public void put(String key, List<Message> value) {+ if (down.get() && key.endsWith("/" + queueName)) {+ throw new IllegalStateException("destination store unavailable");+ }+ super.put(key, value);+ }+ };+ return new SqsService(new InMemoryStorage<>(), messageStore, null, 30, 1048576, BASE_URL,+ new RegionResolver("us-east-1", "000000000000"), false, null, clock);+ }++ private static String redrivePolicy(String dlqArn) {+ return "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"1\"}";+ }++ private static List<String> bodies(List<Message> messages) {+ return messages.stream().map(Message::getBody).toList();+ }+@Testvoid startMessageMoveTask_sourceIsNotDeadLetterQueue_throwsInvalidParameterValue() {sqsService.createQueue("just-a-queue", null, "us-east-1");@@ -1058,8 +1234,13 @@ class SqsServiceTest {}private void awaitMoveTaskStatus(String sourceArn, String taskHandle, String expectedStatus) throws Exception {+ awaitMoveTaskStatus(sqsService, sourceArn, taskHandle, expectedStatus);+ }++ private static void awaitMoveTaskStatus(SqsService service, String sourceArn, String taskHandle,+ String expectedStatus) throws Exception {for (int attempt = 0; attempt < 100; attempt++) {- boolean reachedStatus = sqsService.listMessageMoveTasks(sourceArn, "us-east-1").stream()+ boolean reachedStatus = service.listMessageMoveTasks(sourceArn, "us-east-1").stream().anyMatch(task -> task.taskHandle().equals(taskHandle)&& expectedStatus.equals(task.status()));if (reachedStatus) {