Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(docdb): scope cluster and instance identifiers per region (#2440)

Michal Schreiber3887a6ca6547
diff --git a/src/main/java/io/github/hectorvent/floci/services/docdb/DocDbService.java b/src/main/java/io/github/hectorvent/floci/services/docdb/DocDbService.java
index 8400fe5fd4..9afe7c65e0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/docdb/DocDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/docdb/DocDbService.java
@@ -19,10 +19,13 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
@ApplicationScoped
public class DocDbService {
@@ -40,8 +43,110 @@ public class DocDbService {
* One monitor per stored record, taken by everything that writes it. A tag update is a
* read-modify-write, so without a lock shared with delete it can put a deleted cluster back.
*/
+ /**
+ * One monitor per record, taken by everything that writes it, and retired with the record.
+ *
+ * <p>Retiring one means a straggler can hold the old monitor while a later caller takes a new
+ * one for the same key, so a write must not depend on the monitor alone for its safety: the
+ * writes that can outlive a delete check the record under the key is still theirs.
+ */
private final ConcurrentHashMap<String, Object> writeLocks = new ConcurrentHashMap<>();
+ /**
+ * Storage key for a record: AWS scopes these identifiers per region, so the region is part of
+ * the key rather than something read back off the record. The shape matches
+ * {@code RdsService.dbResourceKey}, which solves the same problem for the same identifiers.
+ */
+ private String key(String id) {
+ return key(regionResolver.getRegion(), id);
+ }
+
+ private static String key(String region, String id) {
+ return region + "::" + id;
+ }
+
+ /**
+ * A cluster in one region, migrating a record written before regions were part of the key.
+ *
+ * <p>Such a record was created when every ARN came from the configured default region, so that
+ * is the region it belongs to — the same assumption the ARN backfill makes. It is moved under
+ * its regional key on first read, so the unscoped key is not consulted again.
+ */
+ private Optional<DocDbCluster> findCluster(String region, String id) {
+ Optional<DocDbCluster> scoped = clusters.get(key(region, id));
+ if (scoped.isPresent() || !region.equals(regionResolver.getDefaultRegion())) {
+ return scoped;
+ }
+ // Under the record's monitor, and read again inside it: the move is a read-modify-write,
+ // so a delete landing between the two would otherwise be undone by writing the record
+ // back under its new key. Monitors are reentrant, so callers already holding this one —
+ // delete among them — pass straight through.
+ synchronized (lockFor("cluster:" + key(region, id))) {
+ Optional<DocDbCluster> legacy = clusters.get(id);
+ legacy.ifPresent(cluster -> {
+ clusters.put(key(region, id), cluster);
+ clusters.delete(id);
+ LOG.debugv("Moved DocDB cluster {0} under its {1} key", id, region);
+ });
+ return legacy;
+ }
+ }
+
+ private Optional<DocDbInstance> findInstance(String region, String id) {
+ Optional<DocDbInstance> scoped = instances.get(key(region, id));
+ if (scoped.isPresent() || !region.equals(regionResolver.getDefaultRegion())) {
+ return scoped;
+ }
+ synchronized (lockFor("instance:" + key(region, id))) {
+ Optional<DocDbInstance> legacy = instances.get(id);
+ legacy.ifPresent(instance -> {
+ instances.put(key(region, id), instance);
+ instances.delete(id);
+ LOG.debugv("Moved DocDB instance {0} under its {1} key", id, region);
+ });
+ return legacy;
+ }
+ }
+
+ /**
+ * Writes a record filled in on read, unless it has been deleted meanwhile.
+ *
+ * <p>Filling a field in on read is still a write, and a read is not otherwise serialised
+ * against a delete — so without the record's monitor and a second look inside it, a reader
+ * that started before a delete puts the record back after it.
+ */
+ private <T> void persistIfStillPresent(StorageBackend<String, T> store, String monitor,
+ String key, T record) {
+ // Two things, because a field filled in on read is still a write. The monitor is the one
+ // every other writer of this record takes — naming it differently would be no mutual
+ // exclusion at all — and the record stored under the key has to still be this record, so
+ // that a delete, or a delete and a re-create under the same name, is not written over.
+ synchronized (lockFor(monitor)) {
+ if (store.get(key).orElse(null) == record) {
+ store.put(key, record);
+ }
+ }
+ }
+
+ /**
+ * Every record of one kind in one region, including any not yet moved under a regional key.
+ *
+ * <p>Two scans of a store a migration is moving records within can each miss what the other
+ * sees, so the unscoped keys are read first and the regional ones second: a record the first
+ * scan misses has been moved already, which the second scan therefore sees, and one the second
+ * misses had not been moved when the first ran. Anything seen twice is one record caught
+ * mid-move, so the list is reduced by identifier.
+ */
+ private <T> List<T> inRegion(StorageBackend<String, T> store, String region,
+ Function<T, String> identifier) {
+ Map<String, T> found = new LinkedHashMap<>();
+ if (region.equals(regionResolver.getDefaultRegion())) {
+ store.scan(k -> !k.contains("::")).forEach(r -> found.put(identifier.apply(r), r));
+ }
+ store.scan(k -> k.startsWith(region + "::")).forEach(r -> found.put(identifier.apply(r), r));
+ return List.copyOf(found.values());
+ }
+
@Inject
public DocDbService(EmulatorConfig config,
RegionResolver regionResolver,
@@ -61,17 +166,13 @@ public class DocDbService {
public DocDbCluster createDbCluster(String id, String engineVersion,
String masterUsername, String masterPassword,
boolean iamEnabled) {
- synchronized (lockFor("cluster:" + id)) {
- if (clusters.get(id).isPresent()) {
+ String region = regionResolver.getRegion();
+ synchronized (lockFor("cluster:" + key(region, id))) {
+ if (findCluster(region, id).isPresent()) {
throw new AwsException("DBClusterAlreadyExistsFault",
"DocDB cluster " + id + " already exists.", 400);
}
- // The caller's region, not the configured default: the ARN this create answers with is
- // the one the caller tags by, and a tag call is checked against the region it is made
- // from — an ARN naming somewhere else is one its own creator cannot use.
- String region = regionResolver.getRegion();
-
DocDbCluster cluster = new DocDbCluster();
cluster.setDbClusterIdentifier(id);
cluster.setStatus("available");
@@ -113,7 +214,7 @@ public class DocDbService {
}
}
- clusters.put(id, cluster);
+ clusters.put(key(region, id), cluster);
LOG.infov("DocDB cluster {0} created, endpoint={1}:{2}",
id, cluster.getEndpoint(), String.valueOf(cluster.getPort()));
return cluster;
@@ -121,12 +222,13 @@ public class DocDbService {
}
public DocDbCluster getDbCluster(String id) {
- DocDbCluster cluster = clusters.get(id).orElseThrow(() ->
+ String region = regionResolver.getRegion();
+ DocDbCluster cluster = findCluster(region, id).orElseThrow(() ->
new AwsException("DBClusterNotFoundFault",
"DocDB cluster " + id + " not found.", 404));
if (cluster.getDbClusterArn() == null || cluster.getDbClusterArn().isBlank()) {
- cluster.setDbClusterArn(legacyArn("cluster:" + id));
- clusters.put(id, cluster);
+ cluster.setDbClusterArn(legacyArn(region, "cluster:" + id));
+ persistIfStillPresent(clusters, "cluster:" + key(region, id), key(region, id), cluster);
}
return cluster;
}
@@ -134,14 +236,14 @@ public class DocDbService {
/**
* The ARN a record written before ARNs were stored should have had.
*
- * <p>The default region rather than the caller's: such a record was created when every ARN was
- * built from the default, so that is the region it is in — and giving it the region of whoever
- * happens to read it first would let a caller elsewhere claim it. Without an ARN a record
- * cannot be told apart from another region's of the same identifier, and cannot be tagged
- * through an ARN at all.
+ * <p>The region it is stored under, not the caller's: a record reaches a region's key either
+ * by having been created there or, for one written before regions were part of the key, by
+ * belonging to the default region. Taking the region of whoever happens to read it first would
+ * let a caller elsewhere claim it, and a record with no ARN cannot be told apart from another
+ * region's of the same identifier or tagged through an ARN at all.
*/
- private String legacyArn(String resource) {
- return regionResolver.buildArn("rds", regionResolver.getDefaultRegion(), resource);
+ private String legacyArn(String region, String resource) {
+ return regionResolver.buildArn("rds", region, resource);
}
public boolean hasCluster(String id) {
@@ -149,18 +251,14 @@ public class DocDbService {
}
/**
- * Whether DocumentDB holds this cluster <em>in this region</em>.
- *
- * <p>The records are keyed by identifier alone, so the region has to come from the one place
- * that carries it — the stored ARN. Without that, an RDS request naming a cluster DocumentDB
- * holds somewhere else is answered from here, and a name RDS is free to reuse in another
- * region stops being usable.
+ * Whether DocumentDB holds this cluster <em>in this region</em>. An identifier belongs to one
+ * region, so this is a lookup rather than a filter over every record of that name.
*/
public boolean hasCluster(String id, String region) {
if (id == null || id.isBlank()) {
return false;
}
- return clusters.get(id).filter(c -> regionOf(c.getDbClusterArn()).equals(region)).isPresent();
+ return findCluster(region, id).isPresent();
}
/** Refuses a record whose own ARN is not the one that was asked for. */
@@ -204,27 +302,29 @@ public class DocDbService {
if (id == null || id.isBlank()) {
return false;
}
- return instances.get(id).filter(i -> regionOf(i.getDbInstanceArn()).equals(region)).isPresent();
+ return findInstance(region, id).isPresent();
}
public Collection<DocDbCluster> listDbClusters(String filterId) {
+ String region = regionResolver.getRegion();
if (filterId != null && !filterId.isBlank()) {
// The db-cluster-id filter accepts ARNs as well as identifiers. Match the
// full ARN against each cluster's stored ARN rather than reducing it to
// the bare identifier, so a cross-account or cross-region ARN does not
// resolve a same-named local cluster.
if (filterId.startsWith("arn:")) {
- return clusters.scan(k -> true).stream()
+ return inRegion(clusters, region, DocDbCluster::getDbClusterIdentifier).stream()
.filter(c -> filterId.equalsIgnoreCase(c.getDbClusterArn()))
.toList();
}
- return clusters.scan(k -> k.equalsIgnoreCase(filterId));
+ return findCluster(region, filterId).map(List::of).orElseGet(List::of);
}
- return clusters.scan(k -> true);
+ return inRegion(clusters, region, DocDbCluster::getDbClusterIdentifier);
}
public DocDbCluster modifyDbCluster(String id, String engineVersion, Boolean iamEnabled) {
- synchronized (lockFor("cluster:" + id)) {
+ String region = regionResolver.getRegion();
+ synchronized (lockFor("cluster:" + key(region, id))) {
DocDbCluster cluster = getDbCluster(id);
if (engineVersion != null && !engineVersion.isBlank()) {
cluster.setEngineVersion(engineVersion);
@@ -232,15 +332,16 @@ public class DocDbService {
if (iamEnabled != null) {
cluster.setIamDatabaseAuthenticationEnabled(iamEnabled);
}
- clusters.put(id, cluster);
+ clusters.put(key(region, id), cluster);
LOG.infov("DocDB cluster {0} modified", id);
return cluster;
}
}
public void deleteDbCluster(String id) {
- synchronized (lockFor("cluster:" + id)) {
- DocDbCluster cluster = clusters.get(id).orElseThrow(() ->
+ String region = regionResolver.getRegion();
+ synchronized (lockFor("cluster:" + key(region, id))) {
+ DocDbCluster cluster = findCluster(region, id).orElseThrow(() ->
new AwsException("DBClusterNotFoundFault",
"DocDB cluster " + id + " not found.", 404));
@@ -250,7 +351,7 @@ public class DocDbService {
}
cluster.setStatus("deleting");
- clusters.put(id, cluster);
+ clusters.put(key(region, id), cluster);
if (cluster.getContainerId() != null) {
containerManager.stop(new DocDbContainerHandle(
@@ -258,8 +359,8 @@ public class DocDbService {
cluster.getContainerHost(), cluster.getContainerPort()));
}
- clusters.delete(id);
- writeLocks.remove("cluster:" + id);
+ clusters.delete(key(region, id));
+ writeLocks.remove("cluster:" + key(region, id));
LOG.infov("DocDB cluster {0} deleted", id);
}
}
@@ -269,16 +370,16 @@ public class DocDbService {
public DocDbInstance createDbInstance(String id, String dbClusterIdentifier,
String dbInstanceClass, String engineVersion,
boolean iamEnabled) {
+ String region = regionResolver.getRegion();
// Instance monitor before cluster monitor, the one order every path that holds both uses.
- synchronized (lockFor("instance:" + id)) {
- synchronized (lockFor("cluster:" + dbClusterIdentifier)) {
- if (instances.get(id).isPresent()) {
+ synchronized (lockFor("instance:" + key(region, id))) {
+ synchronized (lockFor("cluster:" + key(region, dbClusterIdentifier))) {
+ if (findInstance(region, id).isPresent()) {
throw new AwsException("DBInstanceAlreadyExists",
"DocDB instance " + id + " already exists.", 400);
}
DocDbCluster cluster = getDbCluster(dbClusterIdentifier);
- String region = regionResolver.getRegion();
DocDbInstance instance = new DocDbInstance();
instance.setDbInstanceIdentifier(id);
@@ -295,9 +396,9 @@ public class DocDbService {
instance.setCreatedAt(Instant.now());
cluster.getDbClusterMembers().add(id);
- clusters.put(dbClusterIdentifier, cluster);
+ clusters.put(key(region, dbClusterIdentifier), cluster);
- instances.put(id, instance);
+ instances.put(key(region, id), instance);
LOG.infov("DocDB instance {0} created in cluster {1}", id, dbClusterIdentifier);
return instance;
}
@@ -305,32 +406,35 @@ public class DocDbService {
}
public DocDbInstance getDbInstance(String id) {
- DocDbInstance instance = instances.get(id).orElseThrow(() ->
+ String region = regionResolver.getRegion();
+ DocDbInstance instance = findInstance(region, id).orElseThrow(() ->
new AwsException("DBInstanceNotFound",
"DocDB instance " + id + " not found.", 404));
if (instance.getDbInstanceArn() == null || instance.getDbInstanceArn().isBlank()) {
- instance.setDbInstanceArn(legacyArn("db:" + id));
- instances.put(id, instance);
+ instance.setDbInstanceArn(legacyArn(region, "db:" + id));
+ persistIfStillPresent(instances, "instance:" + key(region, id), key(region, id), instance);
}
return instance;
}
public Collection<DocDbInstance> listDbInstances(String filterId) {
+ String region = regionResolver.getRegion();
if (filterId != null && !filterId.isBlank()) {
// The db-instance-id filter accepts ARNs as well as identifiers; see
// listDbClusters for why the match is against the stored ARN.
if (filterId.startsWith("arn:")) {
- return instances.scan(k -> true).stream()
+ return inRegion(instances, region, DocDbInstance::getDbInstanceIdentifier).stream()
.filter(i -> filterId.equalsIgnoreCase(i.getDbInstanceArn()))
.toList();
}
- return instances.scan(k -> k.equalsIgnoreCase(filterId));
+ return findInstance(region, filterId).map(List::of).orElseGet(List::of);
}
- return instances.scan(k -> true);
+ return inRegion(instances, region, DocDbInstance::getDbInstanceIdentifier);
}
public DocDbInstance modifyDbInstance(String id, String dbInstanceClass, Boolean iamEnabled) {
- synchronized (lockFor("instance:" + id)) {
+ String region = regionResolver.getRegion();
+ synchronized (lockFor("instance:" + key(region, id))) {
DocDbInstance instance = getDbInstance(id);
if (dbInstanceClass != null && !dbInstanceClass.isBlank()) {
instance.setDbInstanceClass(dbInstanceClass);
@@ -338,29 +442,30 @@ public class DocDbService {
if (iamEnabled != null) {
instance.setIamDatabaseAuthenticationEnabled(iamEnabled);
}
- instances.put(id, instance);
+ instances.put(key(region, id), instance);
LOG.infov("DocDB instance {0} modified", id);
return instance;
}
}
public void deleteDbInstance(String id) {
- synchronized (lockFor("instance:" + id)) {
- DocDbInstance instance = instances.get(id).orElseThrow(() ->
+ String region = regionResolver.getRegion();
+ synchronized (lockFor("instance:" + key(region, id))) {
+ DocDbInstance instance = findInstance(region, id).orElseThrow(() ->
new AwsException("DBInstanceNotFound",
"DocDB instance " + id + " not found.", 404));
String clusterId = instance.getDbClusterIdentifier();
- synchronized (lockFor("cluster:" + clusterId)) {
- DocDbCluster cluster = clusters.get(clusterId).orElse(null);
+ synchronized (lockFor("cluster:" + key(region, clusterId))) {
+ DocDbCluster cluster = findCluster(region, clusterId).orElse(null);
if (cluster != null) {
cluster.getDbClusterMembers().remove(id);
- clusters.put(clusterId, cluster);
+ clusters.put(key(region, clusterId), cluster);
}
}
- instances.delete(id);
- writeLocks.remove("instance:" + id);
+ instances.delete(key(region, id));
+ writeLocks.remove("instance:" + key(region, id));
LOG.infov("DocDB instance {0} deleted", id);
}
}
@@ -451,18 +556,18 @@ public class DocDbService {
DocDbCluster cluster = getDbCluster(id);
requireArnNamesRecord(resourceName, cluster.getDbClusterArn(),
"DBClusterNotFoundFault", "DocDB cluster " + id + " not found.");
- yield new TagTarget("cluster:" + id, cluster.getTags(), updated -> {
+ yield new TagTarget("cluster:" + key(id), cluster.getTags(), updated -> {
cluster.setTags(updated);
- clusters.put(id, cluster);
+ clusters.put(key(id), cluster);
});
}
case "db" -> {
DocDbInstance instance = getDbInstance(id);
requireArnNamesRecord(resourceName, instance.getDbInstanceArn(),
"DBInstanceNotFound", "DocDB instance " + id + " not found.");
- yield new TagTarget("instance:" + id, instance.getTags(), updated -> {
+ yield new TagTarget("instance:" + key(id), instance.getTags(), updated -> {
instance.setTags(updated);
- instances.put(id, instance);
+ instances.put(key(id), instance);
});
}
default -> throw new AwsException("InvalidParameterValue",
diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbLegacyMigrationRaceTest.java b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbLegacyMigrationRaceTest.java
new file mode 100644
index 0000000000..2720f224e5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbLegacyMigrationRaceTest.java
@@ -0,0 +1,385 @@
+package io.github.hectorvent.floci.services.docdb;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;
+import io.github.hectorvent.floci.services.docdb.model.DocDbCluster;
+import io.github.hectorvent.floci.services.docdb.model.DocDbInstance;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+/**
+ * Moving a record under its regional key must not undo a delete.
+ *
+ * <p>The move is a read-modify-write like a tag update: read the record from the unscoped key,
+ * write it under the regional one. A delete landing between the two would be undone by that write,
+ * leaving a cluster no describe created and no delete can remove — so the move takes the record's
+ * monitor, which is the one the delete already holds.
+ */
+class DocDbLegacyMigrationRaceTest {
+
+ private StorageBackend<String, DocDbCluster> clusterStore;
+ private DocDbService service;
+
+ private void serviceOver(PausingStorageBackend<DocDbCluster> pausing) {
+ clusterStore = new AccountAwareStorageBackend<>(pausing, null, "000000000000");
+ StorageBackend<String, DocDbInstance> instanceStore =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);
+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->
+ "docdb-clusters.json".equals(inv.getArgument(1)) ? clusterStore : instanceStore);
+
+ EmulatorConfig config = Mockito.mock(EmulatorConfig.class);
+ var servicesConfig = Mockito.mock(EmulatorConfig.ServicesConfig.class);
+ var docdbConfig = Mockito.mock(EmulatorConfig.DocDbServiceConfig.class);
+ when(config.services()).thenReturn(servicesConfig);
+ when(servicesConfig.docdb()).thenReturn(docdbConfig);
+ when(docdbConfig.mock()).thenReturn(true);
+ service = new DocDbService(config, new RegionResolver("us-east-1", "000000000000"),
+ Mockito.mock(DocDbContainerManager.class), storageFactory);
+ }
+
+ private void freshService() {
+ clusterStore = AccountAwareStorageBackend.inMemory("000000000000");
+ StorageBackend<String, DocDbInstance> instanceStore =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);
+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->
+ "docdb-clusters.json".equals(inv.getArgument(1)) ? clusterStore : instanceStore);
+
+ EmulatorConfig config = Mockito.mock(EmulatorConfig.class);
+ var servicesConfig = Mockito.mock(EmulatorConfig.ServicesConfig.class);
+ var docdbConfig = Mockito.mock(EmulatorConfig.DocDbServiceConfig.class);
+ when(config.services()).thenReturn(servicesConfig);
+ when(servicesConfig.docdb()).thenReturn(docdbConfig);
+ when(docdbConfig.mock()).thenReturn(true);
+
+ service = new DocDbService(config, new RegionResolver("us-east-1", "000000000000"),
+ Mockito.mock(DocDbContainerManager.class), storageFactory);
+ }
+
+ @Test
+ void readingALegacyRecordWhileItIsDeletedDoesNotBringItBack() throws Exception {
+ for (int attempt = 0; attempt < 25; attempt++) {
+ freshService();
+ String id = "legacy-race-" + attempt;
+
+ // A record as an earlier Floci wrote it: under the bare identifier.
+ DocDbCluster legacy = new DocDbCluster();
+ legacy.setDbClusterIdentifier(id);
+ legacy.setStatus("available");
+ clusterStore.put(id, legacy);
+
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+
+ Thread reader = new Thread(() -> {
+ await(start);
+ try {
+ service.getDbCluster(id);
+ } catch (AwsException expected) {
+ if (!"DBClusterNotFoundFault".equals(expected.getErrorCode())) {
+ unexpected.set(expected);
+ }
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ Thread deleter = new Thread(() -> {
+ await(start);
+ try {
+ service.deleteDbCluster(id);
+ } catch (AwsException expected) {
+ if (!"DBClusterNotFoundFault".equals(expected.getErrorCode())) {
+ unexpected.set(expected);
+ }
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+
+ reader.start();
+ deleter.start();
+ start.countDown();
+ reader.join(TimeUnit.SECONDS.toMillis(10));
+ deleter.join(TimeUnit.SECONDS.toMillis(10));
+ assertFalse(reader.isAlive() || deleter.isAlive(), "a thread never finished");
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+
+ // The status is reported because it names the writer: a record put back as "deleting"
+ // came from the delete's own object, one filled in on read from the reader's.
+ assertFalse(clusterStore.get("us-east-1::" + id).isPresent(),
+ () -> id + " came back under its regional key with status "
+ + clusterStore.get("us-east-1::" + id).map(DocDbCluster::getStatus).orElse("?"));
+ assertFalse(clusterStore.get(id).isPresent(),
+ id + " was left behind under the bare key (attempt " + attempt + ")");
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Test
+ void listingStagedAcrossTheMoveSeesTheRecordExactlyOnce() throws Exception {
+ // Staged rather than raced: the listing is held at its first scan while the move runs to
+ // completion, which is the interleaving that loses or doubles the record.
+ PausingStorageBackend<DocDbCluster> pausing =
+ new PausingStorageBackend<>(new io.github.hectorvent.floci.core.storage.InMemoryStorage<>());
+ serviceOver(pausing);
+ String id = "listed-across-the-move";
+ DocDbCluster legacy = new DocDbCluster();
+ legacy.setDbClusterIdentifier(id);
+ legacy.setStatus("available");
+ clusterStore.put(id, legacy);
+
+ AtomicReference<java.util.List<String>> listed = new AtomicReference<>();
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+ // The second scan: the first runs before the move, so holding the second is what puts the
+ // move between them — the interleaving in which a record can be lost or counted twice.
+ pausing.pauseOn(PausingStorageBackend.Call.SCAN, null, 1);
+
+ Thread lister = new Thread(() -> {
+ try {
+ listed.set(service.listDbClusters(null).stream()
+ .map(DocDbCluster::getDbClusterIdentifier).toList());
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ lister.start();
+ pausing.awaitReached();
+
+ // The move happens entirely inside the listing's first scan.
+ service.getDbCluster(id);
+ pausing.release();
+ lister.join(TimeUnit.SECONDS.toMillis(10));
+
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+ assertEquals(java.util.List.of(id), listed.get(),
+ "the cluster should be listed exactly once across the move");
+ }
+
+ @Test
+ void removingTheLastInstanceCannotWriteOverAClusterDelete() throws Exception {
+ // Staged: the instance delete is held as it writes the cluster back without its member,
+ // and the cluster delete runs to completion first. Sharing one monitor is what stops the
+ // held write from landing afterwards.
+ PausingStorageBackend<DocDbCluster> pausing =
+ new PausingStorageBackend<>(new io.github.hectorvent.floci.core.storage.InMemoryStorage<>());
+ serviceOver(pausing);
+ String clusterId = "co-delete-staged";
+ String instanceId = "co-delete-staged-instance";
+ service.createDbCluster(clusterId, "5.0.0", "admin", "secret99password", false);
+ service.createDbInstance(instanceId, clusterId, "db.r5.large", "5.0.0", false);
+
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+ pausing.pauseOn(PausingStorageBackend.Call.PUT, "us-east-1::" + clusterId);
+
+ Thread instanceDeleter = new Thread(() -> {
+ try {
+ service.deleteDbInstance(instanceId);
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ instanceDeleter.start();
+ pausing.awaitReached();
+
+ // Held mid-write, the instance is already out of the store, so the cluster deletes.
+ Thread clusterDeleter = new Thread(() -> {
+ try {
+ service.deleteDbCluster(clusterId);
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ clusterDeleter.start();
+ clusterDeleter.join(TimeUnit.SECONDS.toMillis(2));
+
+ pausing.release();
+ instanceDeleter.join(TimeUnit.SECONDS.toMillis(10));
+ clusterDeleter.join(TimeUnit.SECONDS.toMillis(10));
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+
+ assertFalse(clusterStore.get("us-east-1::" + clusterId).isPresent(),
+ "the member-list write landed after the cluster was deleted");
+ }
+
+
+ @Test
+ void listingWhileALegacyRecordIsMovedSeesItExactlyOnce() throws Exception {
+ // The listing reads two sets of keys while a migration is moving a record between them.
+ // Whichever way the two scans straddle the move, the record has to appear once: missing it
+ // hides a resource that exists, and seeing it twice reports one resource as two.
+ for (int attempt = 0; attempt < 25; attempt++) {
+ freshService();
+ String id = "listed-during-move-" + attempt;
+ DocDbCluster legacy = new DocDbCluster();
+ legacy.setDbClusterIdentifier(id);
+ legacy.setStatus("available");
+ clusterStore.put(id, legacy);
+
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+ AtomicReference<java.util.List<String>> listed = new AtomicReference<>();
+
+ Thread mover = new Thread(() -> {
+ await(start);
+ try {
+ service.getDbCluster(id);
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ Thread lister = new Thread(() -> {
+ await(start);
+ try {
+ listed.set(service.listDbClusters(null).stream()
+ .map(DocDbCluster::getDbClusterIdentifier).toList());
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+
+ mover.start();
+ lister.start();
+ start.countDown();
+ mover.join(TimeUnit.SECONDS.toMillis(10));
+ lister.join(TimeUnit.SECONDS.toMillis(10));
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+ assertEquals(java.util.List.of(id), listed.get(),
+ "the cluster should be listed exactly once, attempt " + attempt);
+ }
+ }
+
+ @Test
+ void deletingTheLastInstanceWhileItsClusterIsDeletedLeavesNothingBehind() throws Exception {
+ // Both paths write the cluster record — one removes the instance from its member list,
+ // the other deletes it — so they have to take the same monitor. Naming one of them after
+ // the bare identifier is a second monitor, and the member update then lands after the
+ // delete, leaving cluster metadata no delete can remove.
+ for (int attempt = 0; attempt < 25; attempt++) {
+ freshService();
+ String clusterId = "co-delete-" + attempt;
+ String instanceId = "co-delete-instance-" + attempt;
+ service.createDbCluster(clusterId, "5.0.0", "admin", "secret99password", false);
+ service.createDbInstance(instanceId, clusterId, "db.r5.large", "5.0.0", false);
+
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+
+ Thread instanceDeleter = new Thread(() -> {
+ await(start);
+ try {
+ service.deleteDbInstance(instanceId);
+ } catch (AwsException expected) {
+ // losing the race is fine; answering wrongly is not
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ Thread clusterDeleter = new Thread(() -> {
+ await(start);
+ try {
+ service.deleteDbCluster(clusterId);
+ } catch (AwsException expected) {
+ // the cluster still has a member until the other thread finishes
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+
+ instanceDeleter.start();
+ clusterDeleter.start();
+ start.countDown();
+ instanceDeleter.join(TimeUnit.SECONDS.toMillis(10));
+ clusterDeleter.join(TimeUnit.SECONDS.toMillis(10));
+ assertFalse(instanceDeleter.isAlive() || clusterDeleter.isAlive(), "a thread never finished");
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+
+ // Whoever won, the instance is gone and the cluster is either gone or still deletable.
+ assertFalse(clusterStore.get("us-east-1::" + clusterId)
+ .map(c -> c.getDbClusterMembers().contains(instanceId)).orElse(false),
+ "the deleted instance is still a member of " + clusterId);
+ if (clusterStore.get("us-east-1::" + clusterId).isPresent()) {
+ service.deleteDbCluster(clusterId);
+ }
+ assertFalse(clusterStore.get("us-east-1::" + clusterId).isPresent(),
+ clusterId + " could not be removed after the race (attempt " + attempt + ")");
+ }
+ }
+
+ @Test
+ void aDeleteCannotSlipBetweenTheMovesReadAndItsWrite() throws Exception {
+ // Staged, because the racing loop above does not reach this one: the move reads the record
+ // from the unscoped key and writes it under the regional one, and a delete landing between
+ // those two would be undone by that write. Holding the move at its write and letting a
+ // delete run is that interleaving exactly.
+ PausingStorageBackend<DocDbCluster> pausing =
+ new PausingStorageBackend<>(new io.github.hectorvent.floci.core.storage.InMemoryStorage<>());
+ serviceOver(pausing);
+ String id = "moved-while-deleted";
+ DocDbCluster legacy = new DocDbCluster();
+ legacy.setDbClusterIdentifier(id);
+ legacy.setStatus("available");
+ clusterStore.put(id, legacy);
+
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+ pausing.pauseOn(PausingStorageBackend.Call.PUT, "us-east-1::" + id);
+
+ Thread mover = new Thread(() -> {
+ try {
+ service.getDbCluster(id);
+ } catch (AwsException expected) {
+ // losing to the delete is a legitimate outcome
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ mover.start();
+ pausing.awaitReached();
+
+ Thread deleter = new Thread(() -> {
+ try {
+ service.deleteDbCluster(id);
+ } catch (AwsException expected) {
+ // so is finding it already gone
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ deleter.start();
+ // Sharing the monitor is what stops the delete finishing here: it should still be waiting.
+ deleter.join(TimeUnit.SECONDS.toMillis(1));
+
+ pausing.release();
+ mover.join(TimeUnit.SECONDS.toMillis(10));
+ deleter.join(TimeUnit.SECONDS.toMillis(10));
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+
+ assertFalse(clusterStore.get("us-east-1::" + id).isPresent(),
+ "the move wrote the record back after it was deleted");
+ assertFalse(clusterStore.get(id).isPresent(), "the unscoped key was left behind");
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbRegionScopedIdentifierIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbRegionScopedIdentifierIntegrationTest.java
new file mode 100644
index 0000000000..90d829b399
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbRegionScopedIdentifierIntegrationTest.java
@@ -0,0 +1,222 @@
+package io.github.hectorvent.floci.services.docdb;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static io.restassured.RestAssured.given;
+import static io.restassured.http.ContentType.URLENC;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * One identifier per region, as AWS scopes them.
+ *
+ * <p>Checked against a live account: the same name creates in two regions, each with its own ARN,
+ * and tagging one leaves the other untouched. Floci refused the second create, because its records
+ * were keyed by identifier alone — so the name a caller used in one region was spent everywhere.
+ */
+@QuarkusTest
+@TestProfile(DocDbRegionScopedIdentifierIntegrationTest.NoContainersProfile.class)
+class DocDbRegionScopedIdentifierIntegrationTest {
+
+ public static class NoContainersProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.services.docdb.mock", "true");
+ }
+ }
+
+ @Inject
+ DocDbService service;
+
+ private static final String ID = "shared-across-regions";
+
+ private static io.restassured.specification.RequestSpecification docdb(String region, String action) {
+ return given().header("Authorization",
+ "AWS4-HMAC-SHA256 Credential=test/20260615/" + region + "/docdb/aws4_request, "
+ + "SignedHeaders=content-type;host, Signature=test")
+ .contentType(URLENC)
+ .formParam("Action", action)
+ .formParam("Version", "2014-10-31");
+ }
+
+ private static void createCluster(String region, String tagValue) {
+ docdb(region, "CreateDBCluster")
+ .formParam("DBClusterIdentifier", ID)
+ .formParam("Engine", "docdb")
+ .formParam("MasterUsername", "docdbadmin")
+ .formParam("MasterUserPassword", "secret99password")
+ .formParam("Tags.Tag.1.Key", "where")
+ .formParam("Tags.Tag.1.Value", tagValue)
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("arn:aws:rds:" + region + ":000000000000:cluster:" + ID));
+ }
+
+ @Test
+ void oneNameCreatesInEveryRegionAndEachCarriesItsOwnState() {
+ createCluster("us-east-1", "east");
+ createCluster("eu-west-1", "west");
+
+ // Each region describes its own record and not the other's.
+ for (String region : new String[]{"us-east-1", "eu-west-1"}) {
+ docdb(region, "DescribeDBClusters")
+ .formParam("DBClusterIdentifier", ID)
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("arn:aws:rds:" + region + ":000000000000:cluster:" + ID));
+ }
+
+ // Tags belong to the record, not to the name.
+ docdb("us-east-1", "ListTagsForResource")
+ .formParam("ResourceName", "arn:aws:rds:us-east-1:000000000000:cluster:" + ID)
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<Value>east</Value>"))
+ .body(not(containsString("<Value>west</Value>")));
+
+ // A third region has neither.
+ docdb("ap-south-1", "DescribeDBClusters")
+ .formParam("DBClusterIdentifier", ID)
+ .when().post("/")
+ .then().statusCode(404);
+
+ // Deleting one leaves the other alone.
+ docdb("us-east-1", "DeleteDBCluster")
+ .formParam("DBClusterIdentifier", ID)
+ .formParam("SkipFinalSnapshot", "true")
+ .when().post("/").then().statusCode(200);
+
+ docdb("us-east-1", "DescribeDBClusters")
+ .formParam("DBClusterIdentifier", ID)
+ .when().post("/").then().statusCode(404);
+
+ docdb("eu-west-1", "DescribeDBClusters")
+ .formParam("DBClusterIdentifier", ID)
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("arn:aws:rds:eu-west-1:000000000000:cluster:" + ID));
+
+ // Its tags survived the other region's delete too.
+ docdb("eu-west-1", "ListTagsForResource")
+ .formParam("ResourceName", "arn:aws:rds:eu-west-1:000000000000:cluster:" + ID)
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<Value>west</Value>"));
+
+ docdb("eu-west-1", "DeleteDBCluster")
+ .formParam("DBClusterIdentifier", ID)
+ .formParam("SkipFinalSnapshot", "true")
+ .when().post("/").then().statusCode(200);
+ }
+
+ @Test
+ void instancesAreScopedTheSameWay() {
+ createCluster("us-east-1", "east");
+ createCluster("eu-west-1", "west");
+ String instanceId = "shared-instance";
+
+ for (String region : new String[]{"us-east-1", "eu-west-1"}) {
+ docdb(region, "CreateDBInstance")
+ .formParam("DBInstanceIdentifier", instanceId)
+ .formParam("DBClusterIdentifier", ID)
+ .formParam("Engine", "docdb")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("arn:aws:rds:" + region + ":000000000000:db:" + instanceId));
+ }
+
+ // Each cluster lists only the instance created against it.
+ for (String region : new String[]{"us-east-1", "eu-west-1"}) {
+ docdb(region, "DescribeDBInstances")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("arn:aws:rds:" + region + ":000000000000:db:" + instanceId));
+
+ docdb(region, "DeleteDBInstance")
+ .formParam("DBInstanceIdentifier", instanceId)
+ .when().post("/").then().statusCode(200);
+
+ docdb(region, "DeleteDBCluster")
+ .formParam("DBClusterIdentifier", ID)
+ .formParam("SkipFinalSnapshot", "true")
+ .when().post("/").then().statusCode(200);
+ }
+ }
+
+ @Test
+ void aRecordWrittenBeforeRegionsWereInTheKeyBelongsToTheDefaultRegion() {
+ // Written the way an earlier Floci wrote it: the service call runs outside a request, so
+ // it lands under the default region — which is where such a record belongs.
+ String legacyId = "pre-rekey-cluster";
+ service.createDbCluster(legacyId, "5.0.0", "admin", "secret99password", false);
+
+ docdb("us-east-1", "DescribeDBClusters")
+ .formParam("DBClusterIdentifier", legacyId)
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString(legacyId));
+
+ // And it is not visible from anywhere else, nor does it block that name there.
+ docdb("eu-west-1", "DescribeDBClusters")
+ .formParam("DBClusterIdentifier", legacyId)
+ .when().post("/").then().statusCode(404);
+
+ docdb("eu-west-1", "CreateDBCluster")
+ .formParam("DBClusterIdentifier", legacyId)
+ .formParam("Engine", "docdb")
+ .formParam("MasterUsername", "docdbadmin")
+ .formParam("MasterUserPassword", "secret99password")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("arn:aws:rds:eu-west-1:000000000000:cluster:" + legacyId));
+
+ assertEquals("arn:aws:rds:us-east-1:000000000000:cluster:" + legacyId,
+ service.getDbCluster(legacyId).getDbClusterArn());
+ assertTrue(service.hasCluster(legacyId, "us-east-1"));
+ assertTrue(service.hasCluster(legacyId, "eu-west-1"));
+
+ docdb("eu-west-1", "DeleteDBCluster")
+ .formParam("DBClusterIdentifier", legacyId)
+ .formParam("SkipFinalSnapshot", "true")
+ .when().post("/").then().statusCode(200);
+ service.deleteDbCluster(legacyId);
+ }
+
+ @Test
+ void anInstanceCannotAttachToAClusterInAnotherRegion() {
+ // The reference has to resolve in the region the create is made from — a cluster that
+ // exists only elsewhere is not there to attach to.
+ createCluster("eu-west-1", "west");
+
+ docdb("us-east-1", "CreateDBInstance")
+ .formParam("DBInstanceIdentifier", "orphan-instance")
+ .formParam("DBClusterIdentifier", ID)
+ .formParam("Engine", "docdb")
+ .when().post("/")
+ .then()
+ .statusCode(404)
+ .body(containsString("DBClusterNotFoundFault"));
+
+ docdb("eu-west-1", "DeleteDBCluster")
+ .formParam("DBClusterIdentifier", ID)
+ .formParam("SkipFinalSnapshot", "true")
+ .when().post("/").then().statusCode(200);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbServiceTest.java
index c1122bd3c0..96ec80201f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbServiceTest.java
@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;
import io.github.hectorvent.floci.services.docdb.model.DocDbCluster;
@@ -13,10 +14,12 @@ import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
@@ -183,4 +186,47 @@ class DocDbServiceTest {
() -> docDbService.listTagsForResource("scoped-cluster"));
assertTrue(notAnArn.getMessage().contains("Invalid resource name"));
}
+
+ @Test
+ void aRecordStoredUnderABareIdentifierIsFoundAndMovedUnderItsRegionalKey() {
+ // The upgrade path: records written before regions were part of the key sit under the
+ // bare identifier. They belong to the default region — that is the region they were
+ // created under — and are moved there on the first read, so the bare key is consulted
+ // once rather than for ever.
+ StorageBackend<String, DocDbCluster> clusterStore =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ StorageBackend<String, DocDbInstance> instanceStore =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);
+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->
+ "docdb-clusters.json".equals(inv.getArgument(1)) ? clusterStore : instanceStore);
+
+ EmulatorConfig config = Mockito.mock(EmulatorConfig.class);
+ var servicesConfig = Mockito.mock(EmulatorConfig.ServicesConfig.class);
+ var docdbConfig = Mockito.mock(EmulatorConfig.DocDbServiceConfig.class);
+ when(config.services()).thenReturn(servicesConfig);
+ when(servicesConfig.docdb()).thenReturn(docdbConfig);
+ when(docdbConfig.mock()).thenReturn(true);
+ DocDbService service = new DocDbService(config,
+ new RegionResolver("us-east-1", "000000000000"),
+ Mockito.mock(DocDbContainerManager.class), storageFactory);
+
+ DocDbCluster legacy = new DocDbCluster();
+ legacy.setDbClusterIdentifier("bare-key-cluster");
+ legacy.setStatus("available");
+ clusterStore.put("bare-key-cluster", legacy);
+
+ assertEquals("available", service.getDbCluster("bare-key-cluster").getStatus());
+ assertTrue(clusterStore.get("us-east-1::bare-key-cluster").isPresent(),
+ "the record should have been moved under its regional key");
+ assertTrue(clusterStore.get("bare-key-cluster").isEmpty(),
+ "the bare key should not be left behind");
+
+ // It is the default region's, and only that region's.
+ assertTrue(service.hasCluster("bare-key-cluster", "us-east-1"));
+ assertFalse(service.hasCluster("bare-key-cluster", "eu-west-1"));
+
+ // And it lists there, once rather than twice.
+ assertEquals(1, service.listDbClusters(null).size());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/PausingStorageBackend.java b/src/test/java/io/github/hectorvent/floci/services/docdb/PausingStorageBackend.java
new file mode 100644
index 0000000000..26fad437e1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/PausingStorageBackend.java
@@ -0,0 +1,123 @@
+package io.github.hectorvent.floci.services.docdb;
+
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+/**
+ * A store that can be stopped in the middle of an operation, so a race can be staged rather than
+ * hoped for.
+ *
+ * <p>A concurrency test that loops and waits for an unlucky interleaving proves nothing when the
+ * window is a few instructions wide: it passes with the fix and without it. Holding one thread at a
+ * chosen call while another runs to completion makes the interleaving the test claims to cover the
+ * one it actually runs.
+ */
+final class PausingStorageBackend<V> implements StorageBackend<String, V> {
+
+ /** Which call to hold, and on which key. */
+ enum Call { PUT, DELETE, SCAN }
+
+ private final StorageBackend<String, V> delegate;
+ private volatile Call pauseOn;
+ private volatile String pauseKey;
+ private final java.util.concurrent.atomic.AtomicInteger skip =
+ new java.util.concurrent.atomic.AtomicInteger();
+ private final CountDownLatch reached = new CountDownLatch(1);
+ private final CountDownLatch release = new CountDownLatch(1);
+
+ PausingStorageBackend(StorageBackend<String, V> delegate) {
+ this.delegate = delegate;
+ }
+
+ /** Hold the next matching call until {@link #release()}; {@code key} may be null for SCAN. */
+ void pauseOn(Call call, String key) {
+ pauseOn(call, key, 0);
+ }
+
+ /** As above, after letting {@code skipFirst} matching calls through untouched. */
+ void pauseOn(Call call, String key, int skipFirst) {
+ this.pauseOn = call;
+ this.pauseKey = key;
+ this.skip.set(skipFirst);
+ }
+
+ /** Waits until the paused call has been reached. */
+ void awaitReached() throws InterruptedException {
+ if (!reached.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("the paused call was never reached");
+ }
+ }
+
+ void release() {
+ release.countDown();
+ }
+
+ private void pauseIfMatching(Call call, String key) {
+ // Keys arrive account-prefixed here, since this sits underneath the account-aware layer.
+ if (pauseOn != call || (pauseKey != null && (key == null || !key.endsWith(pauseKey)))) {
+ return;
+ }
+ if (skip.getAndDecrement() > 0) {
+ return;
+ }
+ pauseOn = null;
+ reached.countDown();
+ try {
+ if (!release.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("the paused call was never released");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public void put(String key, V value) {
+ pauseIfMatching(Call.PUT, key);
+ delegate.put(key, value);
+ }
+
+ @Override
+ public Optional<V> get(String key) {
+ return delegate.get(key);
+ }
+
+ @Override
+ public void delete(String key) {
+ pauseIfMatching(Call.DELETE, key);
+ delegate.delete(key);
+ }
+
+ @Override
+ public List<V> scan(Predicate<String> keyFilter) {
+ pauseIfMatching(Call.SCAN, null);
+ return delegate.scan(keyFilter);
+ }
+
+ @Override
+ public Set<String> keys() {
+ return delegate.keys();
+ }
+
+ @Override
+ public void flush() {
+ delegate.flush();
+ }
+
+ @Override
+ public void load() {
+ delegate.load();
+ }
+
+ @Override
+ public void clear() {
+ delegate.clear();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/SharedClusterIdentifierIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/docdb/SharedClusterIdentifierIntegrationTest.java
index 49de8de0c6..1a049f1ac8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/docdb/SharedClusterIdentifierIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/SharedClusterIdentifierIntegrationTest.java
@@ -187,7 +187,12 @@ class SharedClusterIdentifierIntegrationTest {
assertTrue(rdsService.listTagsForResource(
"arn:aws:rds:us-east-1:000000000000:cluster:" + id, "us-east-1").isEmpty());
- docDbService.deleteDbCluster(id);
+ // Deleted over the endpoint: the record lives under its own region's key, and a direct
+ // call from the test thread carries no request to take a region from.
+ queryIn("eu-west-1", "DeleteDBCluster")
+ .formParam("DBClusterIdentifier", id)
+ .formParam("SkipFinalSnapshot", "true")
+ .when().post("/").then().statusCode(200);
rdsService.deleteDbCluster(id, "us-east-1");
}