Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sqs): key queues by account and name (#6053)

lex00f757d29aa052
diff --git a/docs/services/sqs.md b/docs/services/sqs.md
index 0a3aeed630..ac04cf8696 100644
--- a/docs/services/sqs.md
+++ b/docs/services/sqs.md
@@ -180,4 +180,4 @@ https://sqs.<region>.amazonaws.com/000000000000/<queue-name>
The region is the one the request was signed for, and the host carries that region's partition DNS suffix, so a queue in `cn-north-1` is reported as `https://sqs.cn-north-1.amazonaws.com.cn/...`. Use it for tooling that parses or compares queue URLs against the AWS form. Most SDKs send requests to the endpoint you configured and take only the path from the URL. Clients that send requests to the host in the returned URL, such as the AWS SDK for Java 1.x, need the default `floci` setting or DNS that points the `sqs.<region>` host at Floci.
-Under either setting every form is accepted as input. That covers the AWS URL, the Floci URL, any other host with the same `/<account>/<queue-name>` path, and the bare queue name. Existing queues, persisted ones included, are reported in the form the current setting and base URL give.
+Under either setting every form is accepted as input. That covers the AWS URL, the Floci URL, any other host whose path ends in `/<account>/<queue-name>`, and the bare queue name. A queue is stored by account and name, so a base URL with a path of its own, such as `http://localhost:4566/sqs`, keeps the same queues under either setting. Existing queues, persisted ones included, are reported in the form the current setting and base URL give.
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
index ccc9fdd484..70331379ba 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
@@ -426,6 +426,11 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>
delegate.delete(accountId + "/" + key);
}
+ /** Deletes the entry stored under {@code rawKey} exactly, as {@link #scanAllAccountsWithRawKeys} reports it. */
+ public void deleteRaw(String rawKey) {
+ delegate.delete(rawKey);
+ }
+
public List<V> scanForAccount(String accountId, Predicate<String> keyFilter) {
String prefix = accountId + "/";
return delegate.scan(k -> k.startsWith(prefix) && keyFilter.test(k.substring(prefix.length())));
@@ -544,7 +549,8 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>
return prefix() + "/" + key;
}
- private static boolean hasAccountPrefix(String key) {
+ /** Whether {@code key} starts with a 12-digit account id and a slash, as account-scoped keys do. */
+ public static boolean hasAccountPrefix(String key) {
if (key.length() < 13 || key.charAt(12) != '/') {
return false;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
index 34b5f22ecb..35264d4c94 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.config.EmulatorConfig.SqsServiceConfig.EndpointStrategy;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
@@ -432,6 +433,10 @@ public class SqsService implements Resettable, ResourceProvider {
this.requestContextInstance = requestContextInstance;
this.iamService = iamService;
this.moveTasksByHandle = new MoveTaskStore(clock);
+ rekeyToAccountAndName(queueStore);
+ rekeyToAccountAndName(messageStore);
+ rekeyToAccountAndName(dedupStore);
+ rekeyToAccountAndName(dedupIdentityStore);
loadPersistedMessages();
loadPersistedDedup();
loadPersistedDedupIdentities();
@@ -455,6 +460,90 @@ public class SqsService implements Resettable, ResourceProvider {
moveTasksByHandle.clear();
}
+ /**
+ * Moves entries stored under a longer queue path, as a base URL with a path of its own gave
+ * them, to the {@code region::/<account>/<name>} key every lookup now uses. It runs in the
+ * constructor, before any request can reach the stores. An entry whose destination already
+ * holds a different value stays where it is, so no stored data is dropped.
+ */
+ private static <V> void rekeyToAccountAndName(StorageBackend<String, V> store) {
+ if (store == null) {
+ return;
+ }
+ if (store instanceof AccountAwareStorageBackend<V> aware) {
+ aware.scanAllAccountsWithRawKeys().forEach((rawKey, value) -> {
+ boolean prefixed = AccountAwareStorageBackend.hasAccountPrefix(rawKey);
+ String storedKey = prefixed ? rawKey.substring(13) : rawKey;
+ if (!hasLongerQueuePath(storedKey)) {
+ return;
+ }
+ String key = accountAndNameKey(storedKey);
+ String accountId = prefixed ? rawKey.substring(0, 12) : queuePathAccount(key, aware.accountId());
+ Optional<V> existing = aware.getForAccount(accountId, key);
+ if (existing.isEmpty()) {
+ aware.putForAccount(accountId, key, value);
+ } else if (!sameValue(existing.get(), value)) {
+ LOG.warnv("SQS entry {0} kept in place; {1} already holds a different value", rawKey, key);
+ return;
+ }
+ aware.deleteRaw(rawKey);
+ });
+ return;
+ }
+ for (String storedKey : new ArrayList<>(store.keys())) {
+ if (!hasLongerQueuePath(storedKey)) {
+ continue;
+ }
+ Optional<V> value = store.get(storedKey);
+ if (value.isEmpty()) {
+ continue;
+ }
+ String key = accountAndNameKey(storedKey);
+ Optional<V> existing = store.get(key);
+ if (existing.isEmpty()) {
+ store.put(key, value.get());
+ } else if (!sameValue(existing.get(), value.get())) {
+ LOG.warnv("SQS entry {0} kept in place; {1} already holds a different value", storedKey, key);
+ continue;
+ }
+ store.delete(storedKey);
+ }
+ }
+
+ private static final ObjectMapper MIGRATION_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
+
+ /** Whether two stored values hold the same data, compared as the storage layer serializes them. */
+ private static boolean sameValue(Object a, Object b) {
+ if (Objects.equals(a, b)) {
+ return true;
+ }
+ try {
+ return MIGRATION_MAPPER.valueToTree(a).equals(MIGRATION_MAPPER.valueToTree(b));
+ } catch (IllegalArgumentException e) {
+ LOG.debugv("SQS migration could not compare stored values: {0}", e.getMessage());
+ return false;
+ }
+ }
+
+ /** The account segment of a {@code region::/<account>/<name>} key, or {@code fallback} when it is not an account id. */
+ private static String queuePathAccount(String key, String fallback) {
+ String path = key.substring(key.indexOf("::") + 2);
+ String account = path.length() > 1 ? path.substring(1, Math.max(1, path.indexOf('/', 1))) : "";
+ return account.matches("\\d{12}") ? account : fallback;
+ }
+
+ private static boolean hasLongerQueuePath(String storageKey) {
+ return !accountAndNameKey(storageKey).equals(storageKey);
+ }
+
+ private static String accountAndNameKey(String storageKey) {
+ int sep = storageKey.indexOf("::");
+ if (sep < 0) {
+ return storageKey;
+ }
+ return storageKey.substring(0, sep + 2) + accountAndName(storageKey.substring(sep + 2));
+ }
+
private void loadPersistedMessages() {
if (messageStore == null) {
return;
@@ -2054,7 +2143,17 @@ public class SqsService implements Resettable, ResourceProvider {
if (pathStart < 0) {
return queueUrl;
}
- return queueUrl.substring(pathStart);
+ return accountAndName(queueUrl.substring(pathStart));
+ }
+
+ /**
+ * The last two segments of a queue path, {@code /<account>/<name>}, so a path a base URL
+ * prefixes, such as {@code /sqs/<account>/<name>}, keys the same queue as the AWS form.
+ */
+ private static String accountAndName(String path) {
+ int last = path.lastIndexOf('/');
+ int previous = last > 0 ? path.lastIndexOf('/', last - 1) : -1;
+ return previous > 0 ? path.substring(previous) : path;
}
private GuardedMessageQueue unexpiredMessages(String storageKey, Queue queue) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
index c0f711a267..1a7196207b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsServiceTest.java
@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.RequestContext;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.sns.SnsService;
import io.github.hectorvent.floci.services.sqs.model.Message;
@@ -392,7 +393,7 @@ class SqsServiceTest {
assertEquals(BASE_URL + "/000000000000/my-queue", url);
}
- private SqsService serviceWithStrategy(InMemoryStorage<String, Queue> store, RegionResolver resolver,
+ private SqsService serviceWithStrategy(StorageBackend<String, Queue> store, RegionResolver resolver,
EndpointStrategy strategy) {
return new SqsService(store, null, null, null, 30, 1048576, BASE_URL, resolver, false, null, clock,
ReceiptHandle.DEFAULT_SECRET, null, null, strategy);
@@ -520,6 +521,91 @@ class SqsServiceTest {
assertEquals(expectedUrl, service.getQueueUrl("legacy-queue", region));
}
+ @Test
+ void aBaseUrlPathKeepsTheSameQueueAcrossAStrategySwitch() {
+ String region = "eu-west-1";
+ String prefixedBase = "http://localhost:4566/sqs";
+ RegionResolver resolver = new RegionResolver("us-east-1", "000000000000");
+ InMemoryStorage<String, Queue> store = new InMemoryStorage<>();
+ SqsService floci = new SqsService(store, null, null, null, 30, 1048576, prefixedBase, resolver, false,
+ null, clock, ReceiptHandle.DEFAULT_SECRET, null, null, EndpointStrategy.FLOCI);
+ assertEquals(prefixedBase + "/000000000000/kept", floci.createQueue("kept", null, region).getQueueUrl());
+
+ SqsService aws = new SqsService(store, null, null, null, 30, 1048576, prefixedBase, resolver, false,
+ null, clock, ReceiptHandle.DEFAULT_SECRET, null, null, EndpointStrategy.AWS);
+ String awsUrl = "https://sqs.eu-west-1.amazonaws.com/000000000000/kept";
+ assertEquals(awsUrl, aws.getQueueUrl("kept", region));
+ assertEquals(awsUrl, aws.createQueue("kept", null, region).getQueueUrl());
+ assertEquals(1, aws.listQueues(null, region).size());
+ }
+
+ @Test
+ void aQueueStoredUnderABaseUrlPathIsRekeyedByAccountAndName() {
+ String region = "eu-west-1";
+ AccountAwareStorageBackend<Queue> store = AccountAwareStorageBackend.inMemory("000000000000");
+ store.putForAccount("000000000000", region + "::/sqs/000000000000/prefixed",
+ prefixedQueue("000000000000", "prefixed"));
+
+ SqsService service = serviceWithStrategy(store, new RegionResolver("us-east-1", "000000000000"),
+ EndpointStrategy.FLOCI);
+
+ assertEquals(List.of("000000000000/" + region + "::/000000000000/prefixed"),
+ List.copyOf(store.scanAllAccountsWithRawKeys().keySet()));
+ assertEquals(BASE_URL + "/000000000000/prefixed", service.getQueueUrl("prefixed", region));
+ }
+
+ @Test
+ void anUnprefixedLegacyEntryMovesToTheAccountInItsQueuePath() {
+ String region = "eu-west-1";
+ InMemoryStorage<String, Queue> raw = new InMemoryStorage<>();
+ Queue stored = prefixedQueue("111111111111", "legacy");
+ raw.put(region + "::/sqs/111111111111/legacy", stored);
+ AccountAwareStorageBackend<Queue> store = new AccountAwareStorageBackend<>(raw, null, "000000000000");
+
+ serviceWithStrategy(store, new RegionResolver("us-east-1", "000000000000"), EndpointStrategy.FLOCI);
+
+ assertEquals(List.of("111111111111/" + region + "::/111111111111/legacy"),
+ List.copyOf(store.scanAllAccountsWithRawKeys().keySet()));
+ assertEquals(Optional.of(stored), store.getForAccount("111111111111", region + "::/111111111111/legacy"));
+ }
+
+ @Test
+ void anEntryWhoseDestinationHoldsADifferentValueStaysInPlace() {
+ String region = "eu-west-1";
+ AccountAwareStorageBackend<Queue> store = AccountAwareStorageBackend.inMemory("000000000000");
+ Queue current = prefixedQueue("000000000000", "kept");
+ Queue older = prefixedQueue("000000000000", "kept");
+ older.setAttributes(Map.of("VisibilityTimeout", "99"));
+ store.putForAccount("000000000000", region + "::/000000000000/kept", current);
+ store.putForAccount("000000000000", region + "::/sqs/000000000000/kept", older);
+
+ serviceWithStrategy(store, new RegionResolver("us-east-1", "000000000000"), EndpointStrategy.FLOCI);
+
+ assertEquals(Optional.of(current), store.getForAccount("000000000000", region + "::/000000000000/kept"));
+ assertEquals(Optional.of(older), store.getForAccount("000000000000", region + "::/sqs/000000000000/kept"));
+ }
+
+ @Test
+ void anEntryWhoseDestinationHoldsTheSameValueIsRemoved() {
+ String region = "eu-west-1";
+ AccountAwareStorageBackend<Queue> store = AccountAwareStorageBackend.inMemory("000000000000");
+ store.putForAccount("000000000000", region + "::/000000000000/kept", prefixedQueue("000000000000", "kept"));
+ store.putForAccount("000000000000", region + "::/sqs/000000000000/kept", prefixedQueue("000000000000", "kept"));
+
+ serviceWithStrategy(store, new RegionResolver("us-east-1", "000000000000"), EndpointStrategy.FLOCI);
+
+ assertEquals(List.of("000000000000/" + region + "::/000000000000/kept"),
+ List.copyOf(store.scanAllAccountsWithRawKeys().keySet()));
+ }
+
+ private static Queue prefixedQueue(String accountId, String name) {
+ Queue queue = new Queue(name, "http://localhost:4566/sqs/" + accountId + "/" + name);
+ queue.setAccountId(accountId);
+ queue.setCreatedTimestamp(Instant.EPOCH);
+ queue.setLastModifiedTimestamp(Instant.EPOCH);
+ return queue;
+ }
+
@Test
void getQueueUrlNotFound() {
assertThrows(AwsException.class, () ->