Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sqs): match AWS on stale, gone and invalid receipt handles (#4730)

TATSUNO “Taz” Yasuhirobe21e8605743
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.md
index 9929b95bd9..46677f64b3 100644
--- a/docs/configuration/advanced/application-yml.md
+++ b/docs/configuration/advanced/application-yml.md
@@ -134,6 +134,7 @@ floci:
default-visibility-timeout: 30 # Seconds
max-message-size: 1048576 # Bytes (1 MiB, the AWS maximum)
clear-fifo-deduplication-cache-on-purge: false # When true, PurgeQueue clears SQS FIFO dedup and SNS FIFO topic dedup for topics subscribed to that queue
+ receipt-handle-secret: local-emulator-secret # HMAC secret that signs receipt handles
s3:
enabled: true
@@ -328,6 +329,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v
| `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default visibility timeout (seconds) |
| `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `1048576` | Max message size (bytes) |
| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` clears the FIFO 5-minute deduplication cache for the target queue and matching SNS FIFO topic dedup entries |
+| `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles, so a handle that was edited or built by hand is rejected |
| `FLOCI_SERVICES_S3_DEFAULT_PRESIGN_EXPIRY_SECONDS` | `3600` | Pre-signed URL expiry |
| `FLOCI_SERVICES_DOCKER_NETWORK` | *(unset)* | Shared Docker network for Lambda, RDS, ElastiCache containers |
| `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable the RDS Data API service |
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md
index 703bbad8c9..0f3543b1d5 100644
--- a/docs/configuration/environment-variables.md
+++ b/docs/configuration/environment-variables.md
@@ -205,6 +205,7 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s
| `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default message visibility timeout in seconds |
| `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `1048576` | Maximum message body size in bytes (1 MiB) |
| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | Reset the deduplication cache when a FIFO queue is purged |
+| `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles |
### SNS
diff --git a/docs/services/sqs.md b/docs/services/sqs.md
index 1d64c78d39..1a3ee70cf8 100644
--- a/docs/services/sqs.md
+++ b/docs/services/sqs.md
@@ -114,6 +114,7 @@ A message older than the queue's `MessageRetentionPeriod` is no longer returned
| `FLOCI_SERVICES_SQS_DEFAULT_VISIBILITY_TIMEOUT` | `30` | Default message visibility timeout (seconds) |
| `FLOCI_SERVICES_SQS_MAX_MESSAGE_SIZE` | `1048576` | Maximum message size in bytes (1 MiB) |
| `FLOCI_SERVICES_SQS_CLEAR_FIFO_DEDUPLICATION_CACHE_ON_PURGE` | `false` | When `true`, `PurgeQueue` also clears the FIFO deduplication cache for the queue and any SNS FIFO topics subscribed to it |
+| `FLOCI_SERVICES_SQS_RECEIPT_HANDLE_SECRET` | `local-emulator-secret` | HMAC secret that signs receipt handles, so a handle that was edited or built by hand is rejected |
## Examples
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 104b8ad3d7..3872d59942 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1309,6 +1309,9 @@ public interface EmulatorConfig {
@WithDefault("false")
boolean clearFifoDeduplicationCacheOnPurge();
+
+ @WithDefault("local-emulator-secret")
+ String receiptHandleSecret();
}
interface S3ServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
index e3daa35086..d1c599cf93 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueue.java
@@ -6,11 +6,8 @@ import io.github.hectorvent.floci.services.sqs.model.Message;
import java.time.Instant;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -26,6 +23,7 @@ class GuardedMessageQueue {
private final List<Message> messages;
private final StorageBackend<String, List<Message>> messageStore;
private final String storageKey;
+ private final String receiptHandleSecret;
private volatile boolean closed;
@FunctionalInterface
@@ -40,13 +38,19 @@ class GuardedMessageQueue {
}
GuardedMessageQueue(StorageBackend<String, List<Message>> messageStore, String storageKey) {
- this(new ArrayList<>(), messageStore, storageKey);
+ this(new ArrayList<>(), messageStore, storageKey, ReceiptHandle.DEFAULT_SECRET);
}
GuardedMessageQueue(List<Message> initial, StorageBackend<String, List<Message>> messageStore, String storageKey) {
+ this(initial, messageStore, storageKey, ReceiptHandle.DEFAULT_SECRET);
+ }
+
+ GuardedMessageQueue(List<Message> initial, StorageBackend<String, List<Message>> messageStore, String storageKey,
+ String receiptHandleSecret) {
this.messages = new ArrayList<>(initial);
this.messageStore = messageStore;
this.storageKey = storageKey;
+ this.receiptHandleSecret = receiptHandleSecret;
}
record ClaimResult(List<Message> claimed, List<Message> dlqCandidates) {
@@ -117,7 +121,7 @@ class GuardedMessageQueue {
return false;
}
- msg.setReceiptHandle(UUID.randomUUID().toString());
+ msg.setReceiptHandle(ReceiptHandle.issue(storageKey, msg.getMessageId(), receiptHandleSecret).encode());
msg.setVisibleAt(Instant.now().plusSeconds(effectiveTimeout));
claimed.add(msg);
return true;
@@ -160,35 +164,57 @@ class GuardedMessageQueue {
}
}
- Optional<Message> removeByReceiptHandle(String receiptHandle) {
+ enum HandleResult { APPLIED, MESSAGE_GONE, HANDLE_EXPIRED }
+
+ record Removal(HandleResult result, Message message) {
+ }
+
+ Removal removeByReceiptHandle(ReceiptHandle handle, boolean fifo) {
try (Guard _ = hold()) {
- Message removed = null;
- for (Iterator<Message> it = messages.iterator(); it.hasNext(); ) {
- Message m = it.next();
- if (receiptHandle.equals(m.getReceiptHandle())) {
- removed = m;
- it.remove();
- break;
- }
+ Message msg = findByMessageId(handle.messageId());
+ HandleResult result = checkHandle(msg, handle, fifo);
+ if (result != HandleResult.APPLIED) {
+ return new Removal(result, null);
}
- if (removed != null) {
+ messages.remove(msg);
+ persist();
+ return new Removal(HandleResult.APPLIED, msg);
+ }
+ }
+
+ HandleResult changeVisibility(ReceiptHandle handle, int visibilityTimeout, boolean fifo) {
+ try (Guard _ = hold()) {
+ Message msg = findByMessageId(handle.messageId());
+ HandleResult result = checkHandle(msg, handle, fifo);
+ if (result == HandleResult.APPLIED) {
+ msg.setVisibleAt(Instant.now().plusSeconds(visibilityTimeout));
persist();
}
- return Optional.ofNullable(removed);
+ return result;
}
}
- boolean changeVisibility(String receiptHandle, int visibilityTimeout) {
- try (Guard _ = hold()) {
- for (Message msg : messages) {
- if (receiptHandle.equals(msg.getReceiptHandle())) {
- msg.setVisibleAt(Instant.now().plusSeconds(visibilityTimeout));
- persist();
- return true;
- }
+ private Message findByMessageId(String messageId) {
+ for (Message msg : messages) {
+ if (messageId.equals(msg.getMessageId())) {
+ return msg;
}
- return false;
}
+ return null;
+ }
+
+ /**
+ * On a standard queue an older handle of the message still works. On a FIFO queue only the
+ * handle of the receive still in flight does.
+ */
+ private static HandleResult checkHandle(Message msg, ReceiptHandle handle, boolean fifo) {
+ if (msg == null) {
+ return HandleResult.MESSAGE_GONE;
+ }
+ if (fifo && (!handle.encode().equals(msg.getReceiptHandle()) || msg.isVisible())) {
+ return HandleResult.HANDLE_EXPIRED;
+ }
+ return HandleResult.APPLIED;
}
void removeMessages(List<Message> toRemove) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/ReceiptHandle.java b/src/main/java/io/github/hectorvent/floci/services/sqs/ReceiptHandle.java
new file mode 100644
index 0000000000..577efb4561
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/ReceiptHandle.java
@@ -0,0 +1,87 @@
+package io.github.hectorvent.floci.services.sqs;
+
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.HexFormat;
+import java.util.UUID;
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
+/**
+ * Floci's receipt handle. The fields are readable, and the signature rejects a handle that was
+ * edited or built by hand, as AWS does.
+ */
+record ReceiptHandle(String region, String accountId, String queueName, String messageId, String receipt,
+ String signature) {
+
+ static final String DEFAULT_SECRET = "local-emulator-secret";
+
+ private static final String[] KEYS = {"region", "account", "queue", "messageId", "receipt", "signature"};
+
+ static ReceiptHandle issue(String storageKey, String messageId, String secret) {
+ String[] queue = queueParts(storageKey);
+ String receipt = UUID.randomUUID().toString();
+ String signature = sign(body(queue[0], queue[1], queue[2], messageId, receipt), secret);
+ return new ReceiptHandle(queue[0], queue[1], queue[2], messageId, receipt, signature);
+ }
+
+ String encode() {
+ return body(region, accountId, queueName, messageId, receipt) + ":signature=" + signature;
+ }
+
+ /** Returns null when {@code value} is not in this format. */
+ static ReceiptHandle parse(String value) {
+ String[] parts = value.split(":", -1);
+ if (parts.length != KEYS.length) {
+ return null;
+ }
+ String[] values = new String[KEYS.length];
+ for (int i = 0; i < KEYS.length; i++) {
+ String prefix = KEYS[i] + "=";
+ if (!parts[i].startsWith(prefix)) {
+ return null;
+ }
+ values[i] = parts[i].substring(prefix.length());
+ }
+ return new ReceiptHandle(values[0], values[1], values[2], values[3], values[4], values[5]);
+ }
+
+ boolean isSignedWith(String secret) {
+ return signature.equals(sign(body(region, accountId, queueName, messageId, receipt), secret));
+ }
+
+ boolean belongsTo(String storageKey) {
+ String[] queue = queueParts(storageKey);
+ return region.equals(queue[0]) && accountId.equals(queue[1]) && queueName.equals(queue[2]);
+ }
+
+ /** Splits a storage key of the form {@code region::/accountId/queueName}. */
+ private static String[] queueParts(String storageKey) {
+ if (storageKey == null) {
+ return new String[] {"", "", ""};
+ }
+ int separator = storageKey.indexOf("::");
+ String region = separator < 0 ? "" : storageKey.substring(0, separator);
+ String path = separator < 0 ? storageKey : storageKey.substring(separator + 2);
+ String trimmed = path.startsWith("/") ? path.substring(1) : path;
+ int slash = trimmed.indexOf('/');
+ String accountId = slash < 0 ? "" : trimmed.substring(0, slash);
+ return new String[] {region, accountId, trimmed.substring(slash + 1)};
+ }
+
+ private static String body(String region, String accountId, String queueName, String messageId,
+ String receipt) {
+ return "region=" + region + ":account=" + accountId + ":queue=" + queueName
+ + ":messageId=" + messageId + ":receipt=" + receipt;
+ }
+
+ private static String sign(String value, String secret) {
+ try {
+ Mac mac = Mac.getInstance("HmacSHA256");
+ mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256"));
+ return HexFormat.of().formatHex(mac.doFinal(value.getBytes(StandardCharsets.UTF_8)));
+ } catch (GeneralSecurityException e) {
+ throw new IllegalStateException("HmacSHA256 is not available", e);
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java
index 7ee871a31c..7ad1e0dd1a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java
@@ -329,7 +329,7 @@ public class SqsJsonHandler {
String id = entry.path("Id").asText();
String receiptHandle = entry.path("ReceiptHandle").asText(null);
try {
- sqsService.deleteMessage(queueUrl, receiptHandle, region);
+ sqsService.deleteMessageInBatch(queueUrl, receiptHandle, region);
ObjectNode success = objectMapper.createObjectNode();
success.put("Id", id);
successful.add(success);
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java
index 3d068bcc44..5e9ba9c5c5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java
@@ -332,7 +332,7 @@ public class SqsQueryHandler {
if (id == null) break;
String receiptHandle = getParam(params, "DeleteMessageBatchRequestEntry." + i + ".ReceiptHandle");
try {
- sqsService.deleteMessage(queueUrl, receiptHandle, region);
+ sqsService.deleteMessageInBatch(queueUrl, receiptHandle, region);
xml.start("DeleteMessageBatchResultEntry").elem("Id", id).end("DeleteMessageBatchResultEntry");
} catch (AwsException e) {
xml.start("BatchResultErrorEntry")
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
index 82f5b6b5a0..1503b3a6a5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
@@ -270,6 +270,7 @@ public class SqsService implements Resettable, ResourceProvider {
private final boolean clearFifoDeduplicationCacheOnPurge;
private final SnsService snsService;
private final Clock clock;
+ private final String receiptHandleSecret;
public SqsService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver,
SnsService snsService) {
@@ -298,7 +299,8 @@ public class SqsService implements Resettable, ResourceProvider {
regionResolver,
config.services().sqs().clearFifoDeduplicationCacheOnPurge(),
snsService,
- clock
+ clock,
+ config.services().sqs().receiptHandleSecret()
);
}
@@ -349,6 +351,17 @@ public class SqsService implements Resettable, ResourceProvider {
int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,
SnsService snsService, Clock clock) {
+ this(queueStore, messageStore, dedupStore, dedupIdentityStore, defaultVisibilityTimeout, maxMessageSize,
+ baseUrl, regionResolver, clearFifoDeduplicationCacheOnPurge, snsService, clock,
+ ReceiptHandle.DEFAULT_SECRET);
+ }
+
+ SqsService(StorageBackend<String, Queue> queueStore, StorageBackend<String, List<Message>> messageStore,
+ StorageBackend<String, Map<String, Long>> dedupStore,
+ StorageBackend<String, Map<String, Map<String, String>>> dedupIdentityStore,
+ int defaultVisibilityTimeout, int maxMessageSize, String baseUrl,
+ RegionResolver regionResolver, boolean clearFifoDeduplicationCacheOnPurge,
+ SnsService snsService, Clock clock, String receiptHandleSecret) {
this.queueStore = queueStore;
this.messageStore = messageStore;
this.dedupStore = dedupStore;
@@ -361,6 +374,7 @@ public class SqsService implements Resettable, ResourceProvider {
this.clearFifoDeduplicationCacheOnPurge = clearFifoDeduplicationCacheOnPurge;
this.snsService = snsService;
this.clock = clock;
+ this.receiptHandleSecret = receiptHandleSecret;
this.moveTasksByHandle = new MoveTaskStore(clock);
loadPersistedMessages();
loadPersistedDedup();
@@ -391,11 +405,12 @@ public class SqsService implements Resettable, ResourceProvider {
}
if (messageStore instanceof AccountAwareStorageBackend<List<Message>> aware) {
aware.scanAllAccountsAsMap().forEach((key, msgs) ->
- messagesByQueue.put(key, new GuardedMessageQueue(msgs, messageStore, key)));
+ messagesByQueue.put(key, new GuardedMessageQueue(msgs, messageStore, key, receiptHandleSecret)));
} else {
for (String key : messageStore.keys()) {
messageStore.get(key).ifPresent(msgs ->
- messagesByQueue.put(key, new GuardedMessageQueue(msgs, messageStore, key)));
+ messagesByQueue.put(key,
+ new GuardedMessageQueue(msgs, messageStore, key, receiptHandleSecret)));
}
}
}
@@ -467,7 +482,7 @@ public class SqsService implements Resettable, ResourceProvider {
private GuardedMessageQueue getOrCreateQueue(String storageKey) {
return messagesByQueue.computeIfAbsent(storageKey,
- k -> new GuardedMessageQueue(messageStore, k));
+ k -> new GuardedMessageQueue(new ArrayList<>(), messageStore, k, receiptHandleSecret));
}
private void persistDedup(String storageKey) {
@@ -623,7 +638,8 @@ public class SqsService implements Resettable, ResourceProvider {
}
queueStore.put(storageKey, queue);
- messagesByQueue.put(storageKey, new GuardedMessageQueue(messageStore, storageKey));
+ messagesByQueue.put(storageKey,
+ new GuardedMessageQueue(new ArrayList<>(), messageStore, storageKey, receiptHandleSecret));
LOG.infov("Created {0} queue: {1} in region {2}", queue.isFifo() ? "FIFO" : "standard", queueName, region);
return queue;
}
@@ -1244,35 +1260,91 @@ public class SqsService implements Resettable, ResourceProvider {
}
public void deleteMessage(String queueUrl, String receiptHandle, String region) {
- String storageKey = regionKey(region, queueUrl);
- if (getQueueByUrl(storageKey, queueUrl).isEmpty()) {
- throw new AwsException("AWS.SimpleQueueService.NonExistentQueue",
- "The specified queue does not exist.", 400);
- }
+ deleteMessage(queueUrl, receiptHandle, region, false);
+ }
- Optional<Message> removed = getOrCreateQueue(storageKey).removeByReceiptHandle(receiptHandle);
+ public void deleteMessageInBatch(String queueUrl, String receiptHandle, String region) {
+ deleteMessage(queueUrl, receiptHandle, region, true);
+ }
- if (removed.isEmpty()) {
- throw new AwsException("ReceiptHandleIsInvalid",
- "The input receipt handle is not a valid receipt handle.", 400);
+ private void deleteMessage(String queueUrl, String receiptHandle, String region, boolean inBatch) {
+ String storageKey = regionKey(region, queueUrl);
+ Queue queue = getQueueByUrl(storageKey, queueUrl)
+ .orElseThrow(() -> new AwsException("AWS.SimpleQueueService.NonExistentQueue",
+ "The specified queue does not exist.", 400));
+ ReceiptHandle handle = checkReceiptHandle(storageKey, receiptHandle, inBatch);
+
+ GuardedMessageQueue.Removal removal =
+ getOrCreateQueue(storageKey).removeByReceiptHandle(handle, queue.isFifo());
+ rejectUnusableHandle(removal.result(), receiptHandle, inBatch);
+ // AWS reports success for a message that is already gone.
+ if (removal.result() == GuardedMessageQueue.HandleResult.MESSAGE_GONE) {
+ LOG.debugv("No message for receipt handle {0}", receiptHandle);
+ return;
}
LOG.debugv("Deleted message with receipt handle {0}", receiptHandle);
if (LOG.isTraceEnabled()) {
- Message m = removed.get();
+ Message m = removal.message();
LOG.tracev("Deleted message {0} from queue {1} body={2}",
m.getMessageId(), queueUrl, m.getBody());
}
}
public void changeMessageVisibility(String queueUrl, String receiptHandle, int visibilityTimeout, String region) {
+ changeMessageVisibility(queueUrl, receiptHandle, visibilityTimeout, region, false);
+ }
+
+ private void changeMessageVisibility(String queueUrl, String receiptHandle, int visibilityTimeout,
+ String region, boolean inBatch) {
String storageKey = regionKey(region, queueUrl);
- ensureQueueExists(storageKey);
+ Queue queue = queueStore.get(storageKey)
+ .orElseThrow(() -> new AwsException("AWS.SimpleQueueService.NonExistentQueue",
+ "The specified queue does not exist.", 400));
+ ReceiptHandle handle = checkReceiptHandle(storageKey, receiptHandle, inBatch);
+
+ GuardedMessageQueue.HandleResult result =
+ getOrCreateQueue(storageKey).changeVisibility(handle, visibilityTimeout, queue.isFifo());
+ rejectUnusableHandle(result, receiptHandle, inBatch);
+ if (result == GuardedMessageQueue.HandleResult.MESSAGE_GONE) {
+ throw invalidReceiptHandle(receiptHandle,
+ "Message does not exist or is not available for visibility timeout change", inBatch);
+ }
+ }
+
+ private static void rejectUnusableHandle(GuardedMessageQueue.HandleResult result, String receiptHandle,
+ boolean inBatch) {
+ if (result == GuardedMessageQueue.HandleResult.HANDLE_EXPIRED) {
+ throw invalidReceiptHandle(receiptHandle, "The receipt handle has expired", inBatch);
+ }
+ }
+
+ private static AwsException invalidReceiptHandle(String receiptHandle, String reason, boolean inBatch) {
+ if (inBatch) {
+ return new AwsException("ReceiptHandleIsInvalid", reason, 400);
+ }
+ return new AwsException("InvalidParameterValue",
+ "Value " + receiptHandle + " for parameter ReceiptHandle is invalid. Reason: " + reason + ".", 400);
+ }
- boolean found = getOrCreateQueue(storageKey).changeVisibility(receiptHandle, visibilityTimeout);
- if (!found) {
- throw new AwsException("ReceiptHandleIsInvalid",
- "The input receipt handle is not a valid receipt handle.", 400);
+ /** A batch entry reports every receipt handle error under the ReceiptHandleIsInvalid code. */
+ private ReceiptHandle checkReceiptHandle(String storageKey, String receiptHandle, boolean inBatch) {
+ if (receiptHandle == null || receiptHandle.isEmpty()) {
+ throw new AwsException(inBatch ? "ReceiptHandleIsInvalid" : "MissingParameter",
+ "The request must contain the parameter ReceiptHandle.", 400);
}
+ ReceiptHandle parsed = ReceiptHandle.parse(receiptHandle);
+ if (parsed == null) {
+ throw new AwsException("ReceiptHandleIsInvalid", "The input receipt handle is invalid.", 400);
+ }
+ if (!parsed.isSignedWith(receiptHandleSecret) || !parsed.belongsTo(storageKey)) {
+ throw handleNotValidForQueue(receiptHandle);
+ }
+ return parsed;
+ }
+
+ private static AwsException handleNotValidForQueue(String receiptHandle) {
+ return new AwsException("ReceiptHandleIsInvalid",
+ "The receipt handle \"" + receiptHandle + "\" is not valid for this queue.", 400);
}
public void purgeQueue(String queueUrl, String region) {
@@ -1597,7 +1669,7 @@ public class SqsService implements Resettable, ResourceProvider {
List<BatchResultEntry> results = new ArrayList<>();
for (ChangeVisibilityBatchEntry entry : entries) {
try {
- changeMessageVisibility(queueUrl, entry.receiptHandle(), entry.visibilityTimeout(), region);
+ changeMessageVisibility(queueUrl, entry.receiptHandle(), entry.visibilityTimeout(), region, true);
results.add(new BatchResultEntry(entry.id(), true, null, null));
} catch (AwsException e) {
results.add(new BatchResultEntry(entry.id(), false, e.getErrorCode(), e.getMessage()));
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java b/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java
index d83a71199f..f698e25d74 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/model/Message.java
@@ -69,12 +69,13 @@ public class Message {
this.md5OfBody = computeMd5(body);
}
- /** A copy for delivery to another queue by StartMessageMoveTask: identity and content carry
- * over; per-receive state (receive count, first-receive timestamp, receipt handle, visibility)
- * starts over. The source instance is untouched so a failed delivery can put it back as it was. */
+ /** A copy for delivery to another queue by StartMessageMoveTask. The content carries over. AWS
+ * gives it a new message id, and per-receive state (receive count, first-receive timestamp,
+ * receipt handle, visibility) starts over. The source instance is untouched so a failed
+ * delivery can put it back as it was. */
public Message copyForRedrive() {
Message copy = new Message();
- copy.messageId = messageId;
+ copy.messageId = UUID.randomUUID().toString();
copy.body = body;
copy.messageAttributes = messageAttributes == null ? new HashMap<>() : new HashMap<>(messageAttributes);
copy.sentTimestamp = sentTimestamp;
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index c53a55372b..82adf036c7 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -328,6 +328,7 @@ floci:
default-visibility-timeout: 30
max-message-size: 1048576
clear-fifo-deduplication-cache-on-purge: false
+ receipt-handle-secret: local-emulator-secret
s3:
enabled: true
enforce-auth: false
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
index 9660b34e7a..551714c731 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/GuardedMessageQueueTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.sqs;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.sqs.GuardedMessageQueue.ClaimResult;
+import io.github.hectorvent.floci.services.sqs.GuardedMessageQueue.HandleResult;
import io.github.hectorvent.floci.services.sqs.GuardedMessageQueue.MessageCounts;
import io.github.hectorvent.floci.services.sqs.model.Message;
import org.junit.jupiter.api.BeforeEach;
@@ -95,7 +96,7 @@ class GuardedMessageQueueTest {
ClaimResult claimed = queue.claimVisibleMessages(1, 30, false, -1, null);
String handle = claimed.claimed().get(0).getReceiptHandle();
- assertTrue(queue.removeByReceiptHandle(handle).isPresent());
+ assertEquals(HandleResult.APPLIED, queue.removeByReceiptHandle(ReceiptHandle.parse(handle), false).result());
// Message should be gone even with visibility timeout 0
ClaimResult result = queue.claimVisibleMessages(1, 0, false, -1, null);
@@ -104,7 +105,7 @@ class GuardedMessageQueueTest {
@Test
void removeByReceiptHandleInvalidReturnsEmpty() {
- assertFalse(queue.removeByReceiptHandle("nonexistent").isPresent());
+ assertEquals(HandleResult.MESSAGE_GONE, queue.removeByReceiptHandle(ReceiptHandle.issue(null, "nonexistent", ReceiptHandle.DEFAULT_SECRET), false).result());
}
@Test
@@ -115,7 +116,7 @@ class GuardedMessageQueueTest {
String handle = claimed.claimed().get(0).getReceiptHandle();
// Set visibility to 0 — message becomes visible immediately
- assertTrue(queue.changeVisibility(handle, 0));
+ assertEquals(HandleResult.APPLIED, queue.changeVisibility(ReceiptHandle.parse(handle), 0, false));
ClaimResult reClaimed = queue.claimVisibleMessages(1, 30, false, -1, null);
assertEquals(1, reClaimed.claimed().size());
@@ -123,7 +124,7 @@ class GuardedMessageQueueTest {
@Test
void changeVisibilityInvalidReturnsFalse() {
- assertFalse(queue.changeVisibility("nonexistent", 0));
+ assertEquals(HandleResult.MESSAGE_GONE, queue.changeVisibility(ReceiptHandle.issue(null, "nonexistent", ReceiptHandle.DEFAULT_SECRET), 0, false));
}
@Test
@@ -389,7 +390,7 @@ class GuardedMessageQueueTest {
ClaimResult result = queue.claimVisibleMessages(messageCount, 30, false, -1, null);
claimedCount.addAndGet(result.claimed().size());
for (Message m : result.claimed()) {
- queue.removeByReceiptHandle(m.getReceiptHandle());
+ queue.removeByReceiptHandle(ReceiptHandle.parse(m.getReceiptHandle()), false);
}
} else {
ClaimResult result = queue.claimVisibleMessages(messageCount, 30, false, -1, null);
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java
index cca7ff0f0a..61a0f6c8a1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java
@@ -474,6 +474,33 @@ class SqsJsonProtocolTest {
.body("message", equalTo("the parameter 'value' may not be null"));
}
+ @Test
+ @Order(7)
+ void malformedReceiptHandleIsRejected() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AmazonSQS.DeleteMessage")
+ .body("{\"QueueUrl\":\"" + queueUrl + "\",\"ReceiptHandle\":\"abc\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ReceiptHandleIsInvalid"))
+ .body("message", equalTo("The input receipt handle is invalid."));
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AmazonSQS.DeleteMessageBatch")
+ .body("{\"QueueUrl\":\"" + queueUrl + "\",\"Entries\":[{\"Id\":\"e\",\"ReceiptHandle\":\"\"}]}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("Failed[0].Id", equalTo("e"))
+ .body("Failed[0].Code", equalTo("ReceiptHandleIsInvalid"))
+ .body("Failed[0].Message", equalTo("The request must contain the parameter ReceiptHandle."));
+ }
+
@Test
@Order(8)
void deleteQueueViaQueueUrlPath() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index 33f6de57b0..aef61e9094 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -2175,4 +2175,201 @@ class SqsServiceTest {
assertTrue(messageStore.get(storageKey).orElseThrow().isEmpty());
}
+
+ @ParameterizedTest
+ @CsvSource({
+ "delete, empty", "delete, malformed", "delete, edited", "delete, otherQueue", "delete, otherSecret",
+ "changeVisibility, empty", "changeVisibility, malformed", "changeVisibility, edited",
+ "changeVisibility, otherQueue", "changeVisibility, otherSecret"})
+ void unusableReceiptHandleIsRejected(String operation, String kind) {
+ String region = "us-east-1";
+ Queue queue = sqsService.createQueue("handle-queue", null, region);
+ Queue other = sqsService.createQueue("handle-other", null, region);
+ Message sent = sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, region);
+ sqsService.sendMessage(other.getQueueUrl(), "msg", 0, region);
+ String handle = switch (kind) {
+ case "empty" -> "";
+ case "malformed" -> "abc";
+ case "edited" -> receivedHandle(queue.getQueueUrl(), 30, region).replace(":receipt=", ":receipt=0");
+ case "otherQueue" -> receivedHandle(other.getQueueUrl(), 30, region);
+ case "otherSecret" -> ReceiptHandle.issue(region + "::/000000000000/handle-queue", sent.getMessageId(),
+ "another-secret").encode();
+ default -> throw new IllegalArgumentException(kind);
+ };
+ String expectedMessage = switch (kind) {
+ case "empty" -> "The request must contain the parameter ReceiptHandle.";
+ case "malformed" -> "The input receipt handle is invalid.";
+ default -> "The receipt handle \"" + handle + "\" is not valid for this queue.";
+ };
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> useReceiptHandle(operation, queue.getQueueUrl(), handle, region));
+
+ assertEquals("empty".equals(kind) ? "MissingParameter" : "ReceiptHandleIsInvalid", ex.getErrorCode());
+ assertEquals(expectedMessage, ex.getMessage());
+ assertEquals(1, sqsService.peekMessages(queue.getQueueUrl(), region).size());
+ }
+
+ @Test
+ void batchEntryReportsAnEmptyReceiptHandleAsInvalid() {
+ String region = "us-east-1";
+ Queue queue = sqsService.createQueue("handle-batch-empty", null, region);
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> sqsService.deleteMessageInBatch(queue.getQueueUrl(), "", region));
+ assertEquals("ReceiptHandleIsInvalid", ex.getErrorCode());
+ assertEquals("The request must contain the parameter ReceiptHandle.", ex.getMessage());
+
+ List<SqsService.BatchResultEntry> results = sqsService.changeMessageVisibilityBatch(queue.getQueueUrl(),
+ List.of(new SqsService.ChangeVisibilityBatchEntry("e", "", 10)), region);
+ assertEquals("ReceiptHandleIsInvalid", results.get(0).errorCode());
+ assertEquals("The request must contain the parameter ReceiptHandle.", results.get(0).errorMessage());
+ }
+
+ @Test
+ void deletingAMessageThatIsAlreadyGoneSucceeds() {
+ String region = "us-east-1";
+ Queue queue = sqsService.createQueue("handle-gone", null, region);
+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, region);
+ String handle = receivedHandle(queue.getQueueUrl(), 30, region);
+ sqsService.deleteMessage(queue.getQueueUrl(), handle, region);
+
+ assertDoesNotThrow(() -> sqsService.deleteMessage(queue.getQueueUrl(), handle, region));
+ assertDoesNotThrow(() -> sqsService.deleteMessageInBatch(queue.getQueueUrl(), handle, region));
+ }
+
+ @Test
+ void changingVisibilityOfAMessageThatIsGoneIsInvalid() {
+ String region = "us-east-1";
+ Queue queue = sqsService.createQueue("handle-gone-visibility", null, region);
+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, region);
+ String handle = receivedHandle(queue.getQueueUrl(), 30, region);
+ sqsService.deleteMessage(queue.getQueueUrl(), handle, region);
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> sqsService.changeMessageVisibility(queue.getQueueUrl(), handle, 10, region));
+ assertEquals("InvalidParameterValue", ex.getErrorCode());
+ assertEquals("Value " + handle + " for parameter ReceiptHandle is invalid. Reason: Message does not exist "
+ + "or is not available for visibility timeout change.", ex.getMessage());
+
+ List<SqsService.BatchResultEntry> results = sqsService.changeMessageVisibilityBatch(queue.getQueueUrl(),
+ List.of(new SqsService.ChangeVisibilityBatchEntry("g", handle, 10)), region);
+ assertEquals("ReceiptHandleIsInvalid", results.get(0).errorCode());
+ assertEquals("Message does not exist or is not available for visibility timeout change",
+ results.get(0).errorMessage());
+ }
+
+ @Test
+ void standardQueueAcceptsAnOlderReceiptHandleOfTheMessage() {
+ String region = "us-east-1";
+ Queue queue = sqsService.createQueue("handle-stale", null, region);
+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, region);
+ String older = receivedHandle(queue.getQueueUrl(), 0, region);
+ receivedHandle(queue.getQueueUrl(), 30, region);
+
+ sqsService.changeMessageVisibility(queue.getQueueUrl(), older, 30, region);
+ sqsService.deleteMessage(queue.getQueueUrl(), older, region);
+
+ assertTrue(sqsService.peekMessages(queue.getQueueUrl(), region).isEmpty());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"delete", "changeVisibility"})
+ void fifoQueueRejectsAnOlderReceiptHandleAsExpired(String operation) {
+ String region = "us-east-1";
+ Queue queue = sqsService.createQueue("handle-stale.fifo", Map.of("FifoQueue", "true"), region);
+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "g", "d", region);
+ String older = receivedHandle(queue.getQueueUrl(), 0, region);
+ receivedHandle(queue.getQueueUrl(), 30, region);
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> useReceiptHandle(operation, queue.getQueueUrl(), older, region));
+
+ assertEquals("InvalidParameterValue", ex.getErrorCode());
+ assertEquals("Value " + older + " for parameter ReceiptHandle is invalid. Reason: The receipt handle has expired.",
+ ex.getMessage());
+ assertEquals(1, sqsService.peekMessages(queue.getQueueUrl(), region).size());
+ }
+
+ @Test
+ void fifoQueueRejectsTheHandleOfAReceiveWhoseVisibilityEnded() {
+ String region = "us-east-1";
+ Queue queue = sqsService.createQueue("handle-visible.fifo", Map.of("FifoQueue", "true"), region);
+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "g", "d", region);
+ String handle = receivedHandle(queue.getQueueUrl(), 0, region);
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> sqsService.deleteMessageInBatch(queue.getQueueUrl(), handle, region));
+
+ assertEquals("ReceiptHandleIsInvalid", ex.getErrorCode());
+ assertEquals("The receipt handle has expired", ex.getMessage());
+ }
+
+ @Test
+ void fifoQueueDeletesWithTheCurrentReceiptHandle() {
+ String region = "us-east-1";
+ Queue queue = sqsService.createQueue("handle-current.fifo", Map.of("FifoQueue", "true"), region);
+ sqsService.sendMessage(queue.getQueueUrl(), "msg", 0, "g", "d", region);
+ String handle = receivedHandle(queue.getQueueUrl(), 30, region);
+
+ sqsService.changeMessageVisibility(queue.getQueueUrl(), handle, 60, region);
+ sqsService.deleteMessage(queue.getQueueUrl(), handle, region);
+
+ assertTrue(sqsService.peekMessages(queue.getQueueUrl(), region).isEmpty());
+ }
+
+ @Test
+ void receiptHandleFromBeforeAMoveTaskDoesNotReachTheMovedMessage() throws Exception {
+ String region = "us-east-1";
+ sqsService.createQueue("handle-moved-dlq", null, region);
+ String dlqArn = queueArn("handle-moved-dlq");
+ Queue source = sqsService.createQueue("handle-moved-src",
+ Map.of("RedrivePolicy", "{\"deadLetterTargetArn\":\"" + dlqArn + "\",\"maxReceiveCount\":\"1\"}"),
+ region);
+ Message sent = sqsService.sendMessage(source.getQueueUrl(), "msg", 0, region);
+ String handle = receivedHandle(source.getQueueUrl(), 0, region);
+ sqsService.receiveMessage(source.getQueueUrl(), 1, 0, 0, region);
+ String taskHandle = sqsService.startMessageMoveTask(dlqArn, null, region);
+ awaitMoveTaskStatus(dlqArn, taskHandle, "COMPLETED");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> sqsService.changeMessageVisibility(source.getQueueUrl(), handle, 10, region));
+ assertEquals("InvalidParameterValue", ex.getErrorCode());
+ sqsService.deleteMessage(source.getQueueUrl(), handle, region);
+
+ List<Message> moved = sqsService.peekMessages(source.getQueueUrl(), region);
+ assertEquals(1, moved.size());
+ assertNotEquals(sent.getMessageId(), moved.get(0).getMessageId());
+ }
+
+ @Test
+ void receiptHandleStillWorksAfterARestart() {
+ String region = "us-east-1";
+ InMemoryStorage<String, Queue> queueStore = new InMemoryStorage<>();
+ InMemoryStorage<String, List<Message>> messageStore = new InMemoryStorage<>();
+ RegionResolver regionResolver = new RegionResolver(region, "000000000000");
+ SqsService before = new SqsService(queueStore, messageStore, new InMemoryStorage<>(),
+ 30, 1048576, BASE_URL, regionResolver);
+ Queue queue = before.createQueue("handle-restart", null, region);
+ before.sendMessage(queue.getQueueUrl(), "msg", 0, region);
+ String handle = before.receiveMessage(queue.getQueueUrl(), 1, 30, 0, region).get(0).getReceiptHandle();
+
+ SqsService after = new SqsService(queueStore, messageStore, new InMemoryStorage<>(),
+ 30, 1048576, BASE_URL, regionResolver);
+ after.deleteMessage(queue.getQueueUrl(), handle, region);
+
+ assertTrue(after.peekMessages(queue.getQueueUrl(), region).isEmpty());
+ }
+
+ private String receivedHandle(String queueUrl, int visibilityTimeout, String region) {
+ return sqsService.receiveMessage(queueUrl, 1, visibilityTimeout, 0, region).get(0).getReceiptHandle();
+ }
+
+ private void useReceiptHandle(String operation, String queueUrl, String receiptHandle, String region) {
+ if ("delete".equals(operation)) {
+ sqsService.deleteMessage(queueUrl, receiptHandle, region);
+ } else {
+ sqsService.changeMessageVisibility(queueUrl, receiptHandle, 10, region);
+ }
+ }
}