Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sqs): match AWS on stale, gone and invalid receipt handles (#4730)
TATSUNO “Taz” Yasuhirobe21e8605743
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.mdindex 9929b95bd9..46677f64b3 100644--- a/docs/configuration/advanced/application-yml.md+++ b/docs/configuration/advanced/application-yml.md@@ -134,6 +134,7 @@ floci:default-visibility-timeout: 30 # Secondsmax-message-size: 1048576 # Bytes (1 MiB, the AWS maximum)clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue clears SQS FIFO dedup and SNS FIFO topic dedup for topics subscribed to that queue+ receipt-handle-secret: local-emulator-secret # HMAC secret that signs receipt handless3:enabled: true@@ -328,6 +329,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v| `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default visibility timeout (seconds) || `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `1048576` | Max message size (bytes) || `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` clears the FIFO 5-minute deduplication cache for the target queue and matching SNS FIFO topic dedup entries |+| `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles, so a handle that was edited or built by hand is rejected || `FLOCI_SERVICES_S3_DEFAULT_PRESIGN_EXPIRY_SECONDS` | `3600` | Pre-signed URL expiry || `FLOCI_SERVICES_DOCKER_NETWORK` | *(unset)* | Shared Docker network for Lambda, RDS, ElastiCache containers || `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable the RDS Data API service |diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex 703bbad8c9..0f3543b1d5 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -205,6 +205,7 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s| `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default message visibility timeout in seconds || `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `1048576` | Maximum message body size in bytes (1 MiB) || `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | Reset the deduplication cache when a FIFO queue is purged |+| `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles |### SNSdiff --git a/docs/services/sqs.md b/docs/services/sqs.mdindex 1d64c78d39..1a3ee70cf8 100644--- a/docs/services/sqs.md+++ b/docs/services/sqs.md@@ -114,6 +114,7 @@ A message older than the queue's `MessageRetentionPeriod` is no longer returned| `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default message visibility timeout (seconds) || `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `1048576` | Maximum message size in bytes (1 MiB) || `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` also clears the FIFO deduplication cache for the queue and any SNS FIFO topics subscribed to it |+| `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles, so a handle that was edited or built by hand is rejected |## Examplesdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 104b8ad3d7..3872d59942 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1309,6 +1309,9 @@ public interface EmulatorConfig {@WithDefault("false")boolean clearFifoDeduplicationCacheOnPurge();++ @WithDefault("local-emulator-secret")+ String receiptHandleSecret();}interface S3ServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.javaindex e3daa35086..d1c599cf93 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java@@ -6,11 +6,8 @@ import io.github.hectorvent.floci.services.sqs.model.Message;import java.time.Instant;import java.util.ArrayList;-import java.util.Iterator;import java.util.List;-import java.util.Optional;import java.util.Set;-import java.util.UUID;import java.util.concurrent.locks.ReentrantLock;import java.util.function.Predicate;import java.util.stream.Collectors;@@ -26,6 +23,7 @@ class GuardedMessageQueue {private final List<Message> messages;private final StorageBackend<String, List<Message>> messageStore;private final String storageKey;+ private final String receiptHandleSecret;private volatile boolean closed;@FunctionalInterface@@ -40,13 +38,19 @@ class GuardedMessageQueue {}GuardedMessageQueue(StorageBackend<String, List<Message>> messageStore, String storageKey) {- this(new ArrayList<>(), messageStore, storageKey);+ this(new ArrayList<>(), messageStore, storageKey, ReceiptHandle.DEFAULT_SECRET);}GuardedMessageQueue(List<Message> initial, StorageBackend<String, List<Message>> messageStore, String storageKey) {+ this(initial, messageStore, storageKey, ReceiptHandle.DEFAULT_SECRET);+ }++ GuardedMessageQueue(List<Message> initial, StorageBackend<String, List<Message>> messageStore, String storageKey,+ String receiptHandleSecret) {this.messages = new ArrayList<>(initial);this.messageStore = messageStore;this.storageKey = storageKey;+ this.receiptHandleSecret = receiptHandleSecret;}record ClaimResult(List<Message> claimed, List<Message> dlqCandidates) {@@ -117,7 +121,7 @@ class GuardedMessageQueue {return false;}- msg.setReceiptHandle(UUID.randomUUID().toString());+ msg.setReceiptHandle(ReceiptHandle.issue(storageKey, msg.getMessageId(), receiptHandleSecret).encode());msg.setVisibleAt(Instant.now().plusSeconds(effectiveTimeout));claimed.add(msg);return true;@@ -160,35 +164,57 @@ class GuardedMessageQueue {}}- Optional<Message> removeByReceiptHandle(String receiptHandle) {+ enum HandleResult { APPLIED, MESSAGE_GONE, HANDLE_EXPIRED }++ record Removal(HandleResult result, Message message) {+ }++ Removal removeByReceiptHandle(ReceiptHandle handle, boolean fifo) {try (Guard _ = hold()) {- Message removed = null;- for (Iterator<Message> it = messages.iterator(); it.hasNext(); ) {- Message m = it.next();- if (receiptHandle.equals(m.getReceiptHandle())) {- removed = m;- it.remove();- break;- }+ Message msg = findByMessageId(handle.messageId());+ HandleResult result = checkHandle(msg, handle, fifo);+ if (result != HandleResult.APPLIED) {+ return new Removal(result, null);}- if (removed != null) {+ messages.remove(msg);+ persist();+ return new Removal(HandleResult.APPLIED, msg);+ }+ }++ HandleResult changeVisibility(ReceiptHandle handle, int visibilityTimeout, boolean fifo) {+ try (Guard _ = hold()) {+ Message msg = findByMessageId(handle.messageId());+ HandleResult result = checkHandle(msg, handle, fifo);+ if (result == HandleResult.APPLIED) {+ msg.setVisibleAt(Instant.now().plusSeconds(visibilityTimeout));persist();}- return Optional.ofNullable(removed);+ return result;}}- boolean changeVisibility(String receiptHandle, int visibilityTimeout) {- try (Guard _ = hold()) {- for (Message msg : messages) {- if (receiptHandle.equals(msg.getReceiptHandle())) {- msg.setVisibleAt(Instant.now().plusSeconds(visibilityTimeout));- persist();- return true;- }+ private Message findByMessageId(String messageId) {+ for (Message msg : messages) {+ if (messageId.equals(msg.getMessageId())) {+ return msg;}- return false;}+ return null;+ }++ /**+ * On a standard queue an older handle of the message still works. On a FIFO queue only the+ * handle of the receive still in flight does.+ */+ private static HandleResult checkHandle(Message msg, ReceiptHandle handle, boolean fifo) {+ if (msg == null) {+ return HandleResult.MESSAGE_GONE;+ }+ if (fifo && (!handle.encode().equals(msg.getReceiptHandle()) || msg.isVisible())) {+ return HandleResult.HANDLE_EXPIRED;+ }+ return HandleResult.APPLIED;}void removeMessages(List<Message> toRemove) {diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/ReceiptHandle.java b/src/main/java/io/github/hectorvent/floci/services/sqs/ReceiptHandle.javanew file mode 100644index 0000000000..577efb4561--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/ReceiptHandle.java@@ -0,0 +1,87 @@+package io.github.hectorvent.floci.services.sqs;++import java.nio.charset.StandardCharsets;+import java.security.GeneralSecurityException;+import java.util.HexFormat;+import java.util.UUID;+import javax.crypto.Mac;+import javax.crypto.spec.SecretKeySpec;++/**+ * Floci's receipt handle. The fields are readable, and the signature rejects a handle that was+ * edited or built by hand, as AWS does.+ */+record ReceiptHandle(String region, String accountId, String queueName, String messageId, String receipt,+ String signature) {++ static final String DEFAULT_SECRET = "local-emulator-secret";++ private static final String[] KEYS = {"region", "account", "queue", "messageId", "receipt", "signature"};++ static ReceiptHandle issue(String storageKey, String messageId, String secret) {+ String[] queue = queueParts(storageKey);+ String receipt = UUID.randomUUID().toString();+ String signature = sign(body(queue[0], queue[1], queue[2], messageId, receipt), secret);+ return new ReceiptHandle(queue[0], queue[1], queue[2], messageId, receipt, signature);+ }++ String encode() {+ return body(region, accountId, queueName, messageId, receipt) + ":signature=" + signature;+ }++ /** Returns null when {@code value} is not in this format. */+ static ReceiptHandle parse(String value) {+ String[] parts = value.split(":", -1);+ if (parts.length != KEYS.length) {+ return null;+ }+ String[] values = new String[KEYS.length];+ for (int i = 0; i < KEYS.length; i++) {+ String prefix = KEYS[i] + "=";+ if (!parts[i].startsWith(prefix)) {+ return null;+ }+ values[i] = parts[i].substring(prefix.length());+ }+ return new ReceiptHandle(values[0], values[1], values[2], values[3], values[4], values[5]);+ }++ boolean isSignedWith(String secret) {+ return signature.equals(sign(body(region, accountId, queueName, messageId, receipt), secret));+ }++ boolean belongsTo(String storageKey) {+ String[] queue = queueParts(storageKey);+ return region.equals(queue[0]) && accountId.equals(queue[1]) && queueName.equals(queue[2]);+ }++ /** Splits a storage key of the form {@code region::/accountId/queueName}. */+ private static String[] queueParts(String storageKey) {+ if (storageKey == null) {+ return new String[] {"", "", ""};+ }+ int separator = storageKey.indexOf("::");+ String region = separator < 0 ? "" : storageKey.substring(0, separator);+ String path = separator < 0 ? storageKey : storageKey.substring(separator + 2);+ String trimmed = path.startsWith("/") ? path.substring(1) : path;+ int slash = trimmed.indexOf('/');+ String accountId = slash < 0 ? "" : trimmed.substring(0, slash);+ return new String[] {region, accountId, trimmed.substring(slash + 1)};+ }++ private static String body(String region, String accountId, String queueName, String messageId,+ String receipt) {+ return "region=" + region + ":account=" + accountId + ":queue=" + queueName+ + ":messageId=" + messageId + ":receipt=" + receipt;+ }++ private static String sign(String value, String secret) {+ try {+ Mac mac = Mac.getInstance("HmacSHA256");+ mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256"));+ return HexFormat.of().formatHex(mac.doFinal(value.getBytes(StandardCharsets.UTF_8)));+ } catch (GeneralSecurityException e) {+ throw new IllegalStateException("HmacSHA256 is not available", e);+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.javaindex 7ee871a31c..7ad1e0dd1a 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java@@ -329,7 +329,7 @@ public class SqsJsonHandler {String id = entry.path("Id").asText();String receiptHandle = entry.path("ReceiptHandle").asText(null);try {- sqsService.deleteMessage(queueUrl, receiptHandle, region);+ sqsService.deleteMessageInBatch(queueUrl, receiptHandle, region);ObjectNode success = objectMapper.createObjectNode();success.put("Id", id);successful.add(success);diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.javaindex 3d068bcc44..5e9ba9c5c5 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java@@ -332,7 +332,7 @@ public class SqsQueryHandler {if (id == null) break;String receiptHandle = getParam(params, "DeleteMessageBatchRequestEntry." + i + ".ReceiptHandle");try {- sqsService.deleteMessage(queueUrl, receiptHandle, region);+ sqsService.deleteMessageInBatch(queueUrl, receiptHandle, region);xml.start("DeleteMessageBatchResultEntry").elem("Id", id).end("DeleteMessageBatchResultEntry");} catch (AwsException e) {xml.start("BatchResultErrorEntry")diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.javaindex 82f5b6b5a0..1503b3a6a5 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java@@ -270,6 +270,7 @@ public class SqsService implements Resettable, ResourceProvider {private final boolean clearFifoDeduplicationCacheOnPurge;private final SnsService snsService;private final Clock clock;+ private final String receiptHandleSecret;public SqsService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver,SnsService snsService) {@@ -298,7 +299,8 @@ public class SqsService implements Resettable, ResourceProvider {regionResolver,config.services().sqs().clearFifoDeduplicationCacheOnPurge(),snsService,- clock+ clock,+ config.services().sqs().receiptHandleSecret());}@@ -349,6 +351,17 @@ public class SqsService implements Resettable, ResourceProvider {int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,SnsService snsService, Clock clock) {+ this(queueStore, messageStore, dedupStore, dedupIdentityStore, defaultVisibilityTimeout, maxMessageSize,+ baseUrl, regionResolver, clearFifoDeduplicationCacheOnPurge, snsService, clock,+ ReceiptHandle.DEFAULT_SECRET);+ }++ SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,+ StorageBackend<String, Map<String, Long>> dedupStore,+ StorageBackend<String, Map<String, Map<String, String>>> dedupIdentityStore,+ int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,+ RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,+ SnsService snsService, Clock clock, String receiptHandleSecret) {this.queueStore = queueStore;this.messageStore = messageStore;this.dedupStore = dedupStore;@@ -361,6 +374,7 @@ public class SqsService implements Resettable, ResourceProvider {this.clearFifoDeduplicationCacheOnPurge = clearFifoDeduplicationCacheOnPurge;this.snsService = snsService;this.clock = clock;+ this.receiptHandleSecret = receiptHandleSecret;this.moveTasksByHandle = new MoveTaskStore(clock);loadPersistedMessages();loadPersistedDedup();@@ -391,11 +405,12 @@ public class SqsService implements Resettable, ResourceProvider {}if (messageStore instanceof AccountAwareStorageBackend<List<Message>> aware) {aware.scanAllAccountsAsMap().forEach((key, msgs) ->- messagesByQueue.put(key, new GuardedMessageQueue(msgs, messageStore, key)));+ messagesByQueue.put(key, new GuardedMessageQueue(msgs, messageStore, key, receiptHandleSecret)));} else {for (String key : messageStore.keys()) {messageStore.get(key).ifPresent(msgs ->- messagesByQueue.put(key, new GuardedMessageQueue(msgs, messageStore, key)));+ messagesByQueue.put(key,+ new GuardedMessageQueue(msgs, messageStore, key, receiptHandleSecret)));}}}@@ -467,7 +482,7 @@ public class SqsService implements Resettable, ResourceProvider {private GuardedMessageQueue getOrCreateQueue(String storageKey) {return messagesByQueue.computeIfAbsent(storageKey,- k -> new GuardedMessageQueue(messageStore, k));+ k -> new GuardedMessageQueue(new ArrayList<>(), messageStore, k, receiptHandleSecret));}private void persistDedup(String storageKey) {@@ -623,7 +638,8 @@ public class SqsService implements Resettable, ResourceProvider {}queueStore.put(storageKey, queue);- messagesByQueue.put(storageKey, new GuardedMessageQueue(messageStore, storageKey));+ messagesByQueue.put(storageKey,+ new GuardedMessageQueue(new ArrayList<>(), messageStore, storageKey, receiptHandleSecret));LOG.infov("Created {0} queue: {1} in region {2}", queue.isFifo() ? "FIFO" : "standard", queueName, region);return queue;}@@ -1244,35 +1260,91 @@ public class SqsService implements Resettable, ResourceProvider {}public void deleteMessage(String queueUrl, String receiptHandle, String region) {- String storageKey = regionKey(region, queueUrl);- if (getQueueByUrl(storageKey, queueUrl).isEmpty()) {- throw new AwsException("AWS.SimpleQueueService.NonExistentQueue",- "The specified queue does not exist.", 400);- }+ deleteMessage(queueUrl, receiptHandle, region, false);+ }- Optional<Message> removed = getOrCreateQueue(storageKey).removeByReceiptHandle(receiptHandle);+ public void deleteMessageInBatch(String queueUrl, String receiptHandle, String region) {+ deleteMessage(queueUrl, receiptHandle, region, true);+ }- if (removed.isEmpty()) {- throw new AwsException("ReceiptHandleIsInvalid",- "The input receipt handle is not a valid receipt handle.", 400);+ private void deleteMessage(String queueUrl, String receiptHandle, String region, boolean inBatch) {+ String storageKey = regionKey(region, queueUrl);+ Queue queue = getQueueByUrl(storageKey, queueUrl)+ .orElseThrow(() -> new AwsException("AWS.SimpleQueueService.NonExistentQueue",+ "The specified queue does not exist.", 400));+ ReceiptHandle handle = checkReceiptHandle(storageKey, receiptHandle, inBatch);++ GuardedMessageQueue.Removal removal =+ getOrCreateQueue(storageKey).removeByReceiptHandle(handle, queue.isFifo());+ rejectUnusableHandle(removal.result(), receiptHandle, inBatch);+ // AWS reports success for a message that is already gone.+ if (removal.result() == GuardedMessageQueue.HandleResult.MESSAGE_GONE) {+ LOG.debugv("No message for receipt handle {0}", receiptHandle);+ return;}LOG.debugv("Deleted message with receipt handle {0}", receiptHandle);if (LOG.isTraceEnabled()) {- Message m = removed.get();+ Message m = removal.message();LOG.tracev("Deleted message {0} from queue {1} body={2}",m.getMessageId(), queueUrl, m.getBody());}}public void changeMessageVisibility(String queueUrl, String receiptHandle, int visibilityTimeout, String region) {+ changeMessageVisibility(queueUrl, receiptHandle, visibilityTimeout, region, false);+ }++ private void changeMessageVisibility(String queueUrl, String receiptHandle, int visibilityTimeout,+ String region, boolean inBatch) {String storageKey = regionKey(region, queueUrl);- ensureQueueExists(storageKey);+ Queue queue = queueStore.get(storageKey)+ .orElseThrow(() -> new AwsException("AWS.SimpleQueueService.NonExistentQueue",+ "The specified queue does not exist.", 400));+ ReceiptHandle handle = checkReceiptHandle(storageKey, receiptHandle, inBatch);++ GuardedMessageQueue.HandleResult result =+ getOrCreateQueue(storageKey).changeVisibility(handle, visibilityTimeout, queue.isFifo());+ rejectUnusableHandle(result, receiptHandle, inBatch);+ if (result == GuardedMessageQueue.HandleResult.MESSAGE_GONE) {+ throw invalidReceiptHandle(receiptHandle,+ "Message does not exist or is not available for visibility timeout change", inBatch);+ }+ }++ private static void rejectUnusableHandle(GuardedMessageQueue.HandleResult result, String receiptHandle,+ boolean inBatch) {+ if (result == GuardedMessageQueue.HandleResult.HANDLE_EXPIRED) {+ throw invalidReceiptHandle(receiptHandle, "The receipt handle has expired", inBatch);+ }+ }++ private static AwsException invalidReceiptHandle(String receiptHandle, String reason, boolean inBatch) {+ if (inBatch) {+ return new AwsException("ReceiptHandleIsInvalid", reason, 400);+ }+ return new AwsException("InvalidParameterValue",+ "Value " + receiptHandle + " for parameter ReceiptHandle is invalid. Reason: " + reason + ".", 400);+ }- boolean found = getOrCreateQueue(storageKey).changeVisibility(receiptHandle, visibilityTimeout);- if (!found) {- throw new AwsException("ReceiptHandleIsInvalid",- "The input receipt handle is not a valid receipt handle.", 400);+ /** A batch entry reports every receipt handle error under the ReceiptHandleIsInvalid code. */+ private ReceiptHandle checkReceiptHandle(String storageKey, String receiptHandle, boolean inBatch) {+ if (receiptHandle == null || receiptHandle.isEmpty()) {+ throw new AwsException(inBatch ? "ReceiptHandleIsInvalid" : "MissingParameter",+ "The request must contain the parameter ReceiptHandle.", 400);}+ ReceiptHandle parsed = ReceiptHandle.parse(receiptHandle);+ if (parsed == null) {+ throw new AwsException("ReceiptHandleIsInvalid", "The input receipt handle is invalid.", 400);+ }+ if (!parsed.isSignedWith(receiptHandleSecret) || !parsed.belongsTo(storageKey)) {+ throw handleNotValidForQueue(receiptHandle);+ }+ return parsed;+ }++ private static AwsException handleNotValidForQueue(String receiptHandle) {+ return new AwsException("ReceiptHandleIsInvalid",+ "The receipt handle \"" + receiptHandle + "\" is not valid for this queue.", 400);}public void purgeQueue(String queueUrl, String region) {@@ -1597,7 +1669,7 @@ public class SqsService implements Resettable, ResourceProvider {List<BatchResultEntry> results = new ArrayList<>();for (ChangeVisibilityBatchEntry entry : entries) {try {- changeMessageVisibility(queueUrl, entry.receiptHandle(), entry.visibilityTimeout(), region);+ changeMessageVisibility(queueUrl, entry.receiptHandle(), entry.visibilityTimeout(), region, true);results.add(new BatchResultEntry(entry.id(), true, null, null));} catch (AwsException e) {results.add(new BatchResultEntry(entry.id(), false, e.getErrorCode(), e.getMessage()));diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java b/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.javaindex d83a71199f..f698e25d74 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java@@ -69,12 +69,13 @@ public class Message {this.md5OfBody = computeMd5(body);}- /** A copy for delivery to another queue by StartMessageMoveTask: identity and content carry- * over; per-receive state (receive count, first-receive timestamp, receipt handle, visibility)- * starts over. The source instance is untouched so a failed delivery can put it back as it was. */+ /** A copy for delivery to another queue by StartMessageMoveTask. The content carries over. AWS+ * gives it a new message id, and per-receive state (receive count, first-receive timestamp,+ * receipt handle, visibility) starts over. The source instance is untouched so a failed+ * delivery can put it back as it was. */public Message copyForRedrive() {Message copy = new Message();- copy.messageId = messageId;+ copy.messageId = UUID.randomUUID().toString();copy.body = body;copy.messageAttributes = messageAttributes == null ? new HashMap<>() : new HashMap<>(messageAttributes);copy.sentTimestamp = sentTimestamp;diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex c53a55372b..82adf036c7 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -328,6 +328,7 @@ floci:default-visibility-timeout: 30max-message-size: 1048576clear-fifo-deduplication-cache-on-purge: false+ receipt-handle-secret: local-emulator-secrets3:enabled: trueenforce-auth: falsediff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.javaindex 9660b34e7a..551714c731 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.sqs;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.sqs.GuardedMessageQueue.ClaimResult;+import io.github.hectorvent.floci.services.sqs.GuardedMessageQueue.HandleResult;import io.github.hectorvent.floci.services.sqs.GuardedMessageQueue.MessageCounts;import io.github.hectorvent.floci.services.sqs.model.Message;import org.junit.jupiter.api.BeforeEach;@@ -95,7 +96,7 @@ class GuardedMessageQueueTest {ClaimResult claimed = queue.claimVisibleMessages(1, 30, false, -1, null);String handle = claimed.claimed().get(0).getReceiptHandle();- assertTrue(queue.removeByReceiptHandle(handle).isPresent());+ assertEquals(HandleResult.APPLIED, queue.removeByReceiptHandle(ReceiptHandle.parse(handle), false).result());// Message should be gone even with visibility timeout 0ClaimResult result = queue.claimVisibleMessages(1, 0, false, -1, null);@@ -104,7 +105,7 @@ class GuardedMessageQueueTest {@Testvoid removeByReceiptHandleInvalidReturnsEmpty() {- assertFalse(queue.removeByReceiptHandle("nonexistent").isPresent());+ assertEquals(HandleResult.MESSAGE_GONE, queue.removeByReceiptHandle(ReceiptHandle.issue(null, "nonexistent", ReceiptHandle.DEFAULT_SECRET), false).result());}@Test@@ -115,7 +116,7 @@ class GuardedMessageQueueTest {String handle = claimed.claimed().get(0).getReceiptHandle();// Set visibility to 0 — message becomes visible immediately- assertTrue(queue.changeVisibility(handle, 0));+ assertEquals(HandleResult.APPLIED, queue.changeVisibility(ReceiptHandle.parse(handle), 0, false));ClaimResult reClaimed = queue.claimVisibleMessages(1, 30, false, -1, null);assertEquals(1, reClaimed.claimed().size());@@ -123,7 +124,7 @@ class GuardedMessageQueueTest {@Testvoid changeVisibilityInvalidReturnsFalse() {- assertFalse(queue.changeVisibility("nonexistent", 0));+ assertEquals(HandleResult.MESSAGE_GONE, queue.changeVisibility(ReceiptHandle.issue(null, "nonexistent", ReceiptHandle.DEFAULT_SECRET), 0, false));}@Test@@ -389,7 +390,7 @@ class GuardedMessageQueueTest {ClaimResult result = queue.claimVisibleMessages(messageCount, 30, false, -1, null);claimedCount.addAndGet(result.claimed().size());for (Message m : result.claimed()) {- queue.removeByReceiptHandle(m.getReceiptHandle());+ queue.removeByReceiptHandle(ReceiptHandle.parse(m.getReceiptHandle()), false);}} else {ClaimResult result = queue.claimVisibleMessages(messageCount, 30, false, -1, null);diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.javaindex cca7ff0f0a..61a0f6c8a1 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java@@ -474,6 +474,33 @@ class SqsJsonProtocolTest {.body("message", equalTo("the parameter 'value' may not be null"));}+ @Test+ @Order(7)+ void malformedReceiptHandleIsRejected() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AmazonSQS.DeleteMessage")+ .body("{\"QueueUrl\":\"" + queueUrl + "\",\"ReceiptHandle\":\"abc\"}")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ReceiptHandleIsInvalid"))+ .body("message", equalTo("The input receipt handle is invalid."));++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AmazonSQS.DeleteMessageBatch")+ .body("{\"QueueUrl\":\"" + queueUrl + "\",\"Entries\":[{\"Id\":\"e\",\"ReceiptHandle\":\"\"}]}")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Failed[0].Id", equalTo("e"))+ .body("Failed[0].Code", equalTo("ReceiptHandleIsInvalid"))+ .body("Failed[0].Message", equalTo("The request must contain the parameter ReceiptHandle."));+ }+@Test@Order(8)void deleteQueueViaQueueUrlPath() {diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.javaindex 33f6de57b0..aef61e9094 100644--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java@@ -2175,4 +2175,201 @@ class SqsServiceTest {assertTrue(messageStore.get(storageKey).orElseThrow().isEmpty());}++ @ParameterizedTest+ @CsvSource({+ "delete, empty", "delete, malformed", "delete, edited", "delete, otherQueue", "delete, otherSecret",+ "changeVisibility, empty", "changeVisibility, malformed", "changeVisibility, edited",+ "changeVisibility, otherQueue", "changeVisibility, otherSecret"})+ void unusableReceiptHandleIsRejected(String operation, String kind) {+ String region = "us-east-1";+ Queue queue = sqsService.createQueue("handle-queue", null, region);+ Queue other = sqsService.createQueue("handle-other", null, region);+ Message sent = sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, region);+ sqsService.sendMessage(other.getQueueUrl(), "msg", 0, region);+ String handle = switch (kind) {+ case "empty" -> "";+ case "malformed" -> "abc";+ case "edited" -> receivedHandle(queue.getQueueUrl(), 30, region).replace(":receipt=", ":receipt=0");+ case "otherQueue" -> receivedHandle(other.getQueueUrl(), 30, region);+ case "otherSecret" -> ReceiptHandle.issue(region + "::/000000000000/handle-queue", sent.getMessageId(),+ "another-secret").encode();+ default -> throw new IllegalArgumentException(kind);+ };+ String expectedMessage = switch (kind) {+ case "empty" -> "The request must contain the parameter ReceiptHandle.";+ case "malformed" -> "The input receipt handle is invalid.";+ default -> "The receipt handle \"" + handle + "\" is not valid for this queue.";+ };++ AwsException ex = assertThrows(AwsException.class,+ () -> useReceiptHandle(operation, queue.getQueueUrl(), handle, region));++ assertEquals("empty".equals(kind) ? "MissingParameter" : "ReceiptHandleIsInvalid", ex.getErrorCode());+ assertEquals(expectedMessage, ex.getMessage());+ assertEquals(1, sqsService.peekMessages(queue.getQueueUrl(), region).size());+ }++ @Test+ void batchEntryReportsAnEmptyReceiptHandleAsInvalid() {+ String region = "us-east-1";+ Queue queue = sqsService.createQueue("handle-batch-empty", null, region);++ AwsException ex = assertThrows(AwsException.class,+ () -> sqsService.deleteMessageInBatch(queue.getQueueUrl(), "", region));+ assertEquals("ReceiptHandleIsInvalid", ex.getErrorCode());+ assertEquals("The request must contain the parameter ReceiptHandle.", ex.getMessage());++ List<SqsService.BatchResultEntry> results = sqsService.changeMessageVisibilityBatch(queue.getQueueUrl(),+ List.of(new SqsService.ChangeVisibilityBatchEntry("e", "", 10)), region);+ assertEquals("ReceiptHandleIsInvalid", results.get(0).errorCode());+ assertEquals("The request must contain the parameter ReceiptHandle.", results.get(0).errorMessage());+ }++ @Test+ void deletingAMessageThatIsAlreadyGoneSucceeds() {+ String region = "us-east-1";+ Queue queue = sqsService.createQueue("handle-gone", null, region);+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, region);+ String handle = receivedHandle(queue.getQueueUrl(), 30, region);+ sqsService.deleteMessage(queue.getQueueUrl(), handle, region);++ assertDoesNotThrow(() -> sqsService.deleteMessage(queue.getQueueUrl(), handle, region));+ assertDoesNotThrow(() -> sqsService.deleteMessageInBatch(queue.getQueueUrl(), handle, region));+ }++ @Test+ void changingVisibilityOfAMessageThatIsGoneIsInvalid() {+ String region = "us-east-1";+ Queue queue = sqsService.createQueue("handle-gone-visibility", null, region);+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, region);+ String handle = receivedHandle(queue.getQueueUrl(), 30, region);+ sqsService.deleteMessage(queue.getQueueUrl(), handle, region);++ AwsException ex = assertThrows(AwsException.class,+ () -> sqsService.changeMessageVisibility(queue.getQueueUrl(), handle, 10, region));+ assertEquals("InvalidParameterValue", ex.getErrorCode());+ assertEquals("Value " + handle + " for parameter ReceiptHandle is invalid. Reason: Message does not exist "+ + "or is not available for visibility timeout change.", ex.getMessage());++ List<SqsService.BatchResultEntry> results = sqsService.changeMessageVisibilityBatch(queue.getQueueUrl(),+ List.of(new SqsService.ChangeVisibilityBatchEntry("g", handle, 10)), region);+ assertEquals("ReceiptHandleIsInvalid", results.get(0).errorCode());+ assertEquals("Message does not exist or is not available for visibility timeout change",+ results.get(0).errorMessage());+ }++ @Test+ void standardQueueAcceptsAnOlderReceiptHandleOfTheMessage() {+ String region = "us-east-1";+ Queue queue = sqsService.createQueue("handle-stale", null, region);+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, region);+ String older = receivedHandle(queue.getQueueUrl(), 0, region);+ receivedHandle(queue.getQueueUrl(), 30, region);++ sqsService.changeMessageVisibility(queue.getQueueUrl(), older, 30, region);+ sqsService.deleteMessage(queue.getQueueUrl(), older, region);++ assertTrue(sqsService.peekMessages(queue.getQueueUrl(), region).isEmpty());+ }++ @ParameterizedTest+ @ValueSource(strings = {"delete", "changeVisibility"})+ void fifoQueueRejectsAnOlderReceiptHandleAsExpired(String operation) {+ String region = "us-east-1";+ Queue queue = sqsService.createQueue("handle-stale.fifo", Map.of("FifoQueue", "true"), region);+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "g", "d", region);+ String older = receivedHandle(queue.getQueueUrl(), 0, region);+ receivedHandle(queue.getQueueUrl(), 30, region);++ AwsException ex = assertThrows(AwsException.class,+ () -> useReceiptHandle(operation, queue.getQueueUrl(), older, region));++ assertEquals("InvalidParameterValue", ex.getErrorCode());+ assertEquals("Value " + older + " for parameter ReceiptHandle is invalid. Reason: The receipt handle has expired.",+ ex.getMessage());+ assertEquals(1, sqsService.peekMessages(queue.getQueueUrl(), region).size());+ }++ @Test+ void fifoQueueRejectsTheHandleOfAReceiveWhoseVisibilityEnded() {+ String region = "us-east-1";+ Queue queue = sqsService.createQueue("handle-visible.fifo", Map.of("FifoQueue", "true"), region);+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "g", "d", region);+ String handle = receivedHandle(queue.getQueueUrl(), 0, region);++ AwsException ex = assertThrows(AwsException.class,+ () -> sqsService.deleteMessageInBatch(queue.getQueueUrl(), handle, region));++ assertEquals("ReceiptHandleIsInvalid", ex.getErrorCode());+ assertEquals("The receipt handle has expired", ex.getMessage());+ }++ @Test+ void fifoQueueDeletesWithTheCurrentReceiptHandle() {+ String region = "us-east-1";+ Queue queue = sqsService.createQueue("handle-current.fifo", Map.of("FifoQueue", "true"), region);+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "g", "d", region);+ String handle = receivedHandle(queue.getQueueUrl(), 30, region);++ sqsService.changeMessageVisibility(queue.getQueueUrl(), handle, 60, region);+ sqsService.deleteMessage(queue.getQueueUrl(), handle, region);++ assertTrue(sqsService.peekMessages(queue.getQueueUrl(), region).isEmpty());+ }++ @Test+ void receiptHandleFromBeforeAMoveTaskDoesNotReachTheMovedMessage() throws Exception {+ String region = "us-east-1";+ sqsService.createQueue("handle-moved-dlq", null, region);+ String dlqArn = queueArn("handle-moved-dlq");+ Queue source = sqsService.createQueue("handle-moved-src",+ Map.of("RedrivePolicy", "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"1\"}"),+ region);+ Message sent = sqsService.sendMessage(source.getQueueUrl(), "msg", 0, region);+ String handle = receivedHandle(source.getQueueUrl(), 0, region);+ sqsService.receiveMessage(source.getQueueUrl(), 1, 0, 0, region);+ String taskHandle = sqsService.startMessageMoveTask(dlqArn, null, region);+ awaitMoveTaskStatus(dlqArn, taskHandle, "COMPLETED");++ AwsException ex = assertThrows(AwsException.class,+ () -> sqsService.changeMessageVisibility(source.getQueueUrl(), handle, 10, region));+ assertEquals("InvalidParameterValue", ex.getErrorCode());+ sqsService.deleteMessage(source.getQueueUrl(), handle, region);++ List<Message> moved = sqsService.peekMessages(source.getQueueUrl(), region);+ assertEquals(1, moved.size());+ assertNotEquals(sent.getMessageId(), moved.get(0).getMessageId());+ }++ @Test+ void receiptHandleStillWorksAfterARestart() {+ String region = "us-east-1";+ InMemoryStorage<String, Queue> queueStore = new InMemoryStorage<>();+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>();+ RegionResolver regionResolver = new RegionResolver(region, "000000000000");+ SqsService before = new SqsService(queueStore, messageStore, new InMemoryStorage<>(),+ 30, 1048576, BASE_URL, regionResolver);+ Queue queue = before.createQueue("handle-restart", null, region);+ before.sendMessage(queue.getQueueUrl(), "msg", 0, region);+ String handle = before.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region).get(0).getReceiptHandle();++ SqsService after = new SqsService(queueStore, messageStore, new InMemoryStorage<>(),+ 30, 1048576, BASE_URL, regionResolver);+ after.deleteMessage(queue.getQueueUrl(), handle, region);++ assertTrue(after.peekMessages(queue.getQueueUrl(), region).isEmpty());+ }++ private String receivedHandle(String queueUrl, int visibilityTimeout, String region) {+ return sqsService.receiveMessage(queueUrl, 1, visibilityTimeout, 0, region).get(0).getReceiptHandle();+ }++ private void useReceiptHandle(String operation, String queueUrl, String receiptHandle, String region) {+ if ("delete".equals(operation)) {+ sqsService.deleteMessage(queueUrl, receiptHandle, region);+ } else {+ sqsService.changeMessageVisibility(queueUrl, receiptHandle, 10, region);+ }+ }}