Light, fluffy, and always free - The AWS Local Emulator alternative
fix(docdb): scope cluster and instance identifiers per region (#2440)
Michal Schreiber3887a6ca6547
diff --git a/src/main/java/io/github/hectorvent/floci/services/docdb/DocDbService.java b/src/main/java/io/github/hectorvent/floci/services/docdb/DocDbService.javaindex 8400fe5fd4..9afe7c65e0 100644--- a/src/main/java/io/github/hectorvent/floci/services/docdb/DocDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/docdb/DocDbService.java@@ -19,10 +19,13 @@ import java.time.Instant;import java.util.ArrayList;import java.util.Collection;import java.util.LinkedHashMap;+import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.Objects;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;+import java.util.function.Function;@ApplicationScopedpublic class DocDbService {@@ -40,8 +43,110 @@ public class DocDbService {* One monitor per stored record, taken by everything that writes it. A tag update is a* read-modify-write, so without a lock shared with delete it can put a deleted cluster back.*/+ /**+ * One monitor per record, taken by everything that writes it, and retired with the record.+ *+ * <p>Retiring one means a straggler can hold the old monitor while a later caller takes a new+ * one for the same key, so a write must not depend on the monitor alone for its safety: the+ * writes that can outlive a delete check the record under the key is still theirs.+ */private final ConcurrentHashMap<String, Object> writeLocks = new ConcurrentHashMap<>();+ /**+ * Storage key for a record: AWS scopes these identifiers per region, so the region is part of+ * the key rather than something read back off the record. The shape matches+ * {@code RdsService.dbResourceKey}, which solves the same problem for the same identifiers.+ */+ private String key(String id) {+ return key(regionResolver.getRegion(), id);+ }++ private static String key(String region, String id) {+ return region + "::" + id;+ }++ /**+ * A cluster in one region, migrating a record written before regions were part of the key.+ *+ * <p>Such a record was created when every ARN came from the configured default region, so that+ * is the region it belongs to — the same assumption the ARN backfill makes. It is moved under+ * its regional key on first read, so the unscoped key is not consulted again.+ */+ private Optional<DocDbCluster> findCluster(String region, String id) {+ Optional<DocDbCluster> scoped = clusters.get(key(region, id));+ if (scoped.isPresent() || !region.equals(regionResolver.getDefaultRegion())) {+ return scoped;+ }+ // Under the record's monitor, and read again inside it: the move is a read-modify-write,+ // so a delete landing between the two would otherwise be undone by writing the record+ // back under its new key. Monitors are reentrant, so callers already holding this one —+ // delete among them — pass straight through.+ synchronized (lockFor("cluster:" + key(region, id))) {+ Optional<DocDbCluster> legacy = clusters.get(id);+ legacy.ifPresent(cluster -> {+ clusters.put(key(region, id), cluster);+ clusters.delete(id);+ LOG.debugv("Moved DocDB cluster {0} under its {1} key", id, region);+ });+ return legacy;+ }+ }++ private Optional<DocDbInstance> findInstance(String region, String id) {+ Optional<DocDbInstance> scoped = instances.get(key(region, id));+ if (scoped.isPresent() || !region.equals(regionResolver.getDefaultRegion())) {+ return scoped;+ }+ synchronized (lockFor("instance:" + key(region, id))) {+ Optional<DocDbInstance> legacy = instances.get(id);+ legacy.ifPresent(instance -> {+ instances.put(key(region, id), instance);+ instances.delete(id);+ LOG.debugv("Moved DocDB instance {0} under its {1} key", id, region);+ });+ return legacy;+ }+ }++ /**+ * Writes a record filled in on read, unless it has been deleted meanwhile.+ *+ * <p>Filling a field in on read is still a write, and a read is not otherwise serialised+ * against a delete — so without the record's monitor and a second look inside it, a reader+ * that started before a delete puts the record back after it.+ */+ private <T> void persistIfStillPresent(StorageBackend<String, T> store, String monitor,+ String key, T record) {+ // Two things, because a field filled in on read is still a write. The monitor is the one+ // every other writer of this record takes — naming it differently would be no mutual+ // exclusion at all — and the record stored under the key has to still be this record, so+ // that a delete, or a delete and a re-create under the same name, is not written over.+ synchronized (lockFor(monitor)) {+ if (store.get(key).orElse(null) == record) {+ store.put(key, record);+ }+ }+ }++ /**+ * Every record of one kind in one region, including any not yet moved under a regional key.+ *+ * <p>Two scans of a store a migration is moving records within can each miss what the other+ * sees, so the unscoped keys are read first and the regional ones second: a record the first+ * scan misses has been moved already, which the second scan therefore sees, and one the second+ * misses had not been moved when the first ran. Anything seen twice is one record caught+ * mid-move, so the list is reduced by identifier.+ */+ private <T> List<T> inRegion(StorageBackend<String, T> store, String region,+ Function<T, String> identifier) {+ Map<String, T> found = new LinkedHashMap<>();+ if (region.equals(regionResolver.getDefaultRegion())) {+ store.scan(k -> !k.contains("::")).forEach(r -> found.put(identifier.apply(r), r));+ }+ store.scan(k -> k.startsWith(region + "::")).forEach(r -> found.put(identifier.apply(r), r));+ return List.copyOf(found.values());+ }+@Injectpublic DocDbService(EmulatorConfig config,RegionResolver regionResolver,@@ -61,17 +166,13 @@ public class DocDbService {public DocDbCluster createDbCluster(String id, String engineVersion,String masterUsername, String masterPassword,boolean iamEnabled) {- synchronized (lockFor("cluster:" + id)) {- if (clusters.get(id).isPresent()) {+ String region = regionResolver.getRegion();+ synchronized (lockFor("cluster:" + key(region, id))) {+ if (findCluster(region, id).isPresent()) {throw new AwsException("DBClusterAlreadyExistsFault","DocDB cluster " + id + " already exists.", 400);}- // The caller's region, not the configured default: the ARN this create answers with is- // the one the caller tags by, and a tag call is checked against the region it is made- // from — an ARN naming somewhere else is one its own creator cannot use.- String region = regionResolver.getRegion();-DocDbCluster cluster = new DocDbCluster();cluster.setDbClusterIdentifier(id);cluster.setStatus("available");@@ -113,7 +214,7 @@ public class DocDbService {}}- clusters.put(id, cluster);+ clusters.put(key(region, id), cluster);LOG.infov("DocDB cluster {0} created, endpoint={1}:{2}",id, cluster.getEndpoint(), String.valueOf(cluster.getPort()));return cluster;@@ -121,12 +222,13 @@ public class DocDbService {}public DocDbCluster getDbCluster(String id) {- DocDbCluster cluster = clusters.get(id).orElseThrow(() ->+ String region = regionResolver.getRegion();+ DocDbCluster cluster = findCluster(region, id).orElseThrow(() ->new AwsException("DBClusterNotFoundFault","DocDB cluster " + id + " not found.", 404));if (cluster.getDbClusterArn() == null || cluster.getDbClusterArn().isBlank()) {- cluster.setDbClusterArn(legacyArn("cluster:" + id));- clusters.put(id, cluster);+ cluster.setDbClusterArn(legacyArn(region, "cluster:" + id));+ persistIfStillPresent(clusters, "cluster:" + key(region, id), key(region, id), cluster);}return cluster;}@@ -134,14 +236,14 @@ public class DocDbService {/*** The ARN a record written before ARNs were stored should have had.*- * <p>The default region rather than the caller's: such a record was created when every ARN was- * built from the default, so that is the region it is in — and giving it the region of whoever- * happens to read it first would let a caller elsewhere claim it. Without an ARN a record- * cannot be told apart from another region's of the same identifier, and cannot be tagged- * through an ARN at all.+ * <p>The region it is stored under, not the caller's: a record reaches a region's key either+ * by having been created there or, for one written before regions were part of the key, by+ * belonging to the default region. Taking the region of whoever happens to read it first would+ * let a caller elsewhere claim it, and a record with no ARN cannot be told apart from another+ * region's of the same identifier or tagged through an ARN at all.*/- private String legacyArn(String resource) {- return regionResolver.buildArn("rds", regionResolver.getDefaultRegion(), resource);+ private String legacyArn(String region, String resource) {+ return regionResolver.buildArn("rds", region, resource);}public boolean hasCluster(String id) {@@ -149,18 +251,14 @@ public class DocDbService {}/**- * Whether DocumentDB holds this cluster <em>in this region</em>.- *- * <p>The records are keyed by identifier alone, so the region has to come from the one place- * that carries it — the stored ARN. Without that, an RDS request naming a cluster DocumentDB- * holds somewhere else is answered from here, and a name RDS is free to reuse in another- * region stops being usable.+ * Whether DocumentDB holds this cluster <em>in this region</em>. An identifier belongs to one+ * region, so this is a lookup rather than a filter over every record of that name.*/public boolean hasCluster(String id, String region) {if (id == null || id.isBlank()) {return false;}- return clusters.get(id).filter(c -> regionOf(c.getDbClusterArn()).equals(region)).isPresent();+ return findCluster(region, id).isPresent();}/** Refuses a record whose own ARN is not the one that was asked for. */@@ -204,27 +302,29 @@ public class DocDbService {if (id == null || id.isBlank()) {return false;}- return instances.get(id).filter(i -> regionOf(i.getDbInstanceArn()).equals(region)).isPresent();+ return findInstance(region, id).isPresent();}public Collection<DocDbCluster> listDbClusters(String filterId) {+ String region = regionResolver.getRegion();if (filterId != null && !filterId.isBlank()) {// The db-cluster-id filter accepts ARNs as well as identifiers. Match the// full ARN against each cluster's stored ARN rather than reducing it to// the bare identifier, so a cross-account or cross-region ARN does not// resolve a same-named local cluster.if (filterId.startsWith("arn:")) {- return clusters.scan(k -> true).stream()+ return inRegion(clusters, region, DocDbCluster::getDbClusterIdentifier).stream().filter(c -> filterId.equalsIgnoreCase(c.getDbClusterArn())).toList();}- return clusters.scan(k -> k.equalsIgnoreCase(filterId));+ return findCluster(region, filterId).map(List::of).orElseGet(List::of);}- return clusters.scan(k -> true);+ return inRegion(clusters, region, DocDbCluster::getDbClusterIdentifier);}public DocDbCluster modifyDbCluster(String id, String engineVersion, Boolean iamEnabled) {- synchronized (lockFor("cluster:" + id)) {+ String region = regionResolver.getRegion();+ synchronized (lockFor("cluster:" + key(region, id))) {DocDbCluster cluster = getDbCluster(id);if (engineVersion != null && !engineVersion.isBlank()) {cluster.setEngineVersion(engineVersion);@@ -232,15 +332,16 @@ public class DocDbService {if (iamEnabled != null) {cluster.setIamDatabaseAuthenticationEnabled(iamEnabled);}- clusters.put(id, cluster);+ clusters.put(key(region, id), cluster);LOG.infov("DocDB cluster {0} modified", id);return cluster;}}public void deleteDbCluster(String id) {- synchronized (lockFor("cluster:" + id)) {- DocDbCluster cluster = clusters.get(id).orElseThrow(() ->+ String region = regionResolver.getRegion();+ synchronized (lockFor("cluster:" + key(region, id))) {+ DocDbCluster cluster = findCluster(region, id).orElseThrow(() ->new AwsException("DBClusterNotFoundFault","DocDB cluster " + id + " not found.", 404));@@ -250,7 +351,7 @@ public class DocDbService {}cluster.setStatus("deleting");- clusters.put(id, cluster);+ clusters.put(key(region, id), cluster);if (cluster.getContainerId() != null) {containerManager.stop(new DocDbContainerHandle(@@ -258,8 +359,8 @@ public class DocDbService {cluster.getContainerHost(), cluster.getContainerPort()));}- clusters.delete(id);- writeLocks.remove("cluster:" + id);+ clusters.delete(key(region, id));+ writeLocks.remove("cluster:" + key(region, id));LOG.infov("DocDB cluster {0} deleted", id);}}@@ -269,16 +370,16 @@ public class DocDbService {public DocDbInstance createDbInstance(String id, String dbClusterIdentifier,String dbInstanceClass, String engineVersion,boolean iamEnabled) {+ String region = regionResolver.getRegion();// Instance monitor before cluster monitor, the one order every path that holds both uses.- synchronized (lockFor("instance:" + id)) {- synchronized (lockFor("cluster:" + dbClusterIdentifier)) {- if (instances.get(id).isPresent()) {+ synchronized (lockFor("instance:" + key(region, id))) {+ synchronized (lockFor("cluster:" + key(region, dbClusterIdentifier))) {+ if (findInstance(region, id).isPresent()) {throw new AwsException("DBInstanceAlreadyExists","DocDB instance " + id + " already exists.", 400);}DocDbCluster cluster = getDbCluster(dbClusterIdentifier);- String region = regionResolver.getRegion();DocDbInstance instance = new DocDbInstance();instance.setDbInstanceIdentifier(id);@@ -295,9 +396,9 @@ public class DocDbService {instance.setCreatedAt(Instant.now());cluster.getDbClusterMembers().add(id);- clusters.put(dbClusterIdentifier, cluster);+ clusters.put(key(region, dbClusterIdentifier), cluster);- instances.put(id, instance);+ instances.put(key(region, id), instance);LOG.infov("DocDB instance {0} created in cluster {1}", id, dbClusterIdentifier);return instance;}@@ -305,32 +406,35 @@ public class DocDbService {}public DocDbInstance getDbInstance(String id) {- DocDbInstance instance = instances.get(id).orElseThrow(() ->+ String region = regionResolver.getRegion();+ DocDbInstance instance = findInstance(region, id).orElseThrow(() ->new AwsException("DBInstanceNotFound","DocDB instance " + id + " not found.", 404));if (instance.getDbInstanceArn() == null || instance.getDbInstanceArn().isBlank()) {- instance.setDbInstanceArn(legacyArn("db:" + id));- instances.put(id, instance);+ instance.setDbInstanceArn(legacyArn(region, "db:" + id));+ persistIfStillPresent(instances, "instance:" + key(region, id), key(region, id), instance);}return instance;}public Collection<DocDbInstance> listDbInstances(String filterId) {+ String region = regionResolver.getRegion();if (filterId != null && !filterId.isBlank()) {// The db-instance-id filter accepts ARNs as well as identifiers; see// listDbClusters for why the match is against the stored ARN.if (filterId.startsWith("arn:")) {- return instances.scan(k -> true).stream()+ return inRegion(instances, region, DocDbInstance::getDbInstanceIdentifier).stream().filter(i -> filterId.equalsIgnoreCase(i.getDbInstanceArn())).toList();}- return instances.scan(k -> k.equalsIgnoreCase(filterId));+ return findInstance(region, filterId).map(List::of).orElseGet(List::of);}- return instances.scan(k -> true);+ return inRegion(instances, region, DocDbInstance::getDbInstanceIdentifier);}public DocDbInstance modifyDbInstance(String id, String dbInstanceClass, Boolean iamEnabled) {- synchronized (lockFor("instance:" + id)) {+ String region = regionResolver.getRegion();+ synchronized (lockFor("instance:" + key(region, id))) {DocDbInstance instance = getDbInstance(id);if (dbInstanceClass != null && !dbInstanceClass.isBlank()) {instance.setDbInstanceClass(dbInstanceClass);@@ -338,29 +442,30 @@ public class DocDbService {if (iamEnabled != null) {instance.setIamDatabaseAuthenticationEnabled(iamEnabled);}- instances.put(id, instance);+ instances.put(key(region, id), instance);LOG.infov("DocDB instance {0} modified", id);return instance;}}public void deleteDbInstance(String id) {- synchronized (lockFor("instance:" + id)) {- DocDbInstance instance = instances.get(id).orElseThrow(() ->+ String region = regionResolver.getRegion();+ synchronized (lockFor("instance:" + key(region, id))) {+ DocDbInstance instance = findInstance(region, id).orElseThrow(() ->new AwsException("DBInstanceNotFound","DocDB instance " + id + " not found.", 404));String clusterId = instance.getDbClusterIdentifier();- synchronized (lockFor("cluster:" + clusterId)) {- DocDbCluster cluster = clusters.get(clusterId).orElse(null);+ synchronized (lockFor("cluster:" + key(region, clusterId))) {+ DocDbCluster cluster = findCluster(region, clusterId).orElse(null);if (cluster != null) {cluster.getDbClusterMembers().remove(id);- clusters.put(clusterId, cluster);+ clusters.put(key(region, clusterId), cluster);}}- instances.delete(id);- writeLocks.remove("instance:" + id);+ instances.delete(key(region, id));+ writeLocks.remove("instance:" + key(region, id));LOG.infov("DocDB instance {0} deleted", id);}}@@ -451,18 +556,18 @@ public class DocDbService {DocDbCluster cluster = getDbCluster(id);requireArnNamesRecord(resourceName, cluster.getDbClusterArn(),"DBClusterNotFoundFault", "DocDB cluster " + id + " not found.");- yield new TagTarget("cluster:" + id, cluster.getTags(), updated -> {+ yield new TagTarget("cluster:" + key(id), cluster.getTags(), updated -> {cluster.setTags(updated);- clusters.put(id, cluster);+ clusters.put(key(id), cluster);});}case "db" -> {DocDbInstance instance = getDbInstance(id);requireArnNamesRecord(resourceName, instance.getDbInstanceArn(),"DBInstanceNotFound", "DocDB instance " + id + " not found.");- yield new TagTarget("instance:" + id, instance.getTags(), updated -> {+ yield new TagTarget("instance:" + key(id), instance.getTags(), updated -> {instance.setTags(updated);- instances.put(id, instance);+ instances.put(key(id), instance);});}default -> throw new AwsException("InvalidParameterValue",diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbLegacyMigrationRaceTest.java b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbLegacyMigrationRaceTest.javanew file mode 100644index 0000000000..2720f224e5--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbLegacyMigrationRaceTest.java@@ -0,0 +1,385 @@+package io.github.hectorvent.floci.services.docdb;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;+import io.github.hectorvent.floci.services.docdb.model.DocDbCluster;+import io.github.hectorvent.floci.services.docdb.model.DocDbInstance;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.when;++/**+ * Moving a record under its regional key must not undo a delete.+ *+ * <p>The move is a read-modify-write like a tag update: read the record from the unscoped key,+ * write it under the regional one. A delete landing between the two would be undone by that write,+ * leaving a cluster no describe created and no delete can remove — so the move takes the record's+ * monitor, which is the one the delete already holds.+ */+class DocDbLegacyMigrationRaceTest {++ private StorageBackend<String, DocDbCluster> clusterStore;+ private DocDbService service;++ private void serviceOver(PausingStorageBackend<DocDbCluster> pausing) {+ clusterStore = new AccountAwareStorageBackend<>(pausing, null, "000000000000");+ StorageBackend<String, DocDbInstance> instanceStore =+ AccountAwareStorageBackend.inMemory("000000000000");+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->+ "docdb-clusters.json".equals(inv.getArgument(1)) ? clusterStore : instanceStore);++ EmulatorConfig config = Mockito.mock(EmulatorConfig.class);+ var servicesConfig = Mockito.mock(EmulatorConfig.ServicesConfig.class);+ var docdbConfig = Mockito.mock(EmulatorConfig.DocDbServiceConfig.class);+ when(config.services()).thenReturn(servicesConfig);+ when(servicesConfig.docdb()).thenReturn(docdbConfig);+ when(docdbConfig.mock()).thenReturn(true);+ service = new DocDbService(config, new RegionResolver("us-east-1", "000000000000"),+ Mockito.mock(DocDbContainerManager.class), storageFactory);+ }++ private void freshService() {+ clusterStore = AccountAwareStorageBackend.inMemory("000000000000");+ StorageBackend<String, DocDbInstance> instanceStore =+ AccountAwareStorageBackend.inMemory("000000000000");+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->+ "docdb-clusters.json".equals(inv.getArgument(1)) ? clusterStore : instanceStore);++ EmulatorConfig config = Mockito.mock(EmulatorConfig.class);+ var servicesConfig = Mockito.mock(EmulatorConfig.ServicesConfig.class);+ var docdbConfig = Mockito.mock(EmulatorConfig.DocDbServiceConfig.class);+ when(config.services()).thenReturn(servicesConfig);+ when(servicesConfig.docdb()).thenReturn(docdbConfig);+ when(docdbConfig.mock()).thenReturn(true);++ service = new DocDbService(config, new RegionResolver("us-east-1", "000000000000"),+ Mockito.mock(DocDbContainerManager.class), storageFactory);+ }++ @Test+ void readingALegacyRecordWhileItIsDeletedDoesNotBringItBack() throws Exception {+ for (int attempt = 0; attempt < 25; attempt++) {+ freshService();+ String id = "legacy-race-" + attempt;++ // A record as an earlier Floci wrote it: under the bare identifier.+ DocDbCluster legacy = new DocDbCluster();+ legacy.setDbClusterIdentifier(id);+ legacy.setStatus("available");+ clusterStore.put(id, legacy);++ CountDownLatch start = new CountDownLatch(1);+ AtomicReference<Throwable> unexpected = new AtomicReference<>();++ Thread reader = new Thread(() -> {+ await(start);+ try {+ service.getDbCluster(id);+ } catch (AwsException expected) {+ if (!"DBClusterNotFoundFault".equals(expected.getErrorCode())) {+ unexpected.set(expected);+ }+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ Thread deleter = new Thread(() -> {+ await(start);+ try {+ service.deleteDbCluster(id);+ } catch (AwsException expected) {+ if (!"DBClusterNotFoundFault".equals(expected.getErrorCode())) {+ unexpected.set(expected);+ }+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });++ reader.start();+ deleter.start();+ start.countDown();+ reader.join(TimeUnit.SECONDS.toMillis(10));+ deleter.join(TimeUnit.SECONDS.toMillis(10));+ assertFalse(reader.isAlive() || deleter.isAlive(), "a thread never finished");+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());++ // The status is reported because it names the writer: a record put back as "deleting"+ // came from the delete's own object, one filled in on read from the reader's.+ assertFalse(clusterStore.get("us-east-1::" + id).isPresent(),+ () -> id + " came back under its regional key with status "+ + clusterStore.get("us-east-1::" + id).map(DocDbCluster::getStatus).orElse("?"));+ assertFalse(clusterStore.get(id).isPresent(),+ id + " was left behind under the bare key (attempt " + attempt + ")");+ }+ }++ private static void await(CountDownLatch latch) {+ try {+ latch.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ }++ @Test+ void listingStagedAcrossTheMoveSeesTheRecordExactlyOnce() throws Exception {+ // Staged rather than raced: the listing is held at its first scan while the move runs to+ // completion, which is the interleaving that loses or doubles the record.+ PausingStorageBackend<DocDbCluster> pausing =+ new PausingStorageBackend<>(new io.github.hectorvent.floci.core.storage.InMemoryStorage<>());+ serviceOver(pausing);+ String id = "listed-across-the-move";+ DocDbCluster legacy = new DocDbCluster();+ legacy.setDbClusterIdentifier(id);+ legacy.setStatus("available");+ clusterStore.put(id, legacy);++ AtomicReference<java.util.List<String>> listed = new AtomicReference<>();+ AtomicReference<Throwable> unexpected = new AtomicReference<>();+ // The second scan: the first runs before the move, so holding the second is what puts the+ // move between them — the interleaving in which a record can be lost or counted twice.+ pausing.pauseOn(PausingStorageBackend.Call.SCAN, null, 1);++ Thread lister = new Thread(() -> {+ try {+ listed.set(service.listDbClusters(null).stream()+ .map(DocDbCluster::getDbClusterIdentifier).toList());+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ lister.start();+ pausing.awaitReached();++ // The move happens entirely inside the listing's first scan.+ service.getDbCluster(id);+ pausing.release();+ lister.join(TimeUnit.SECONDS.toMillis(10));++ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());+ assertEquals(java.util.List.of(id), listed.get(),+ "the cluster should be listed exactly once across the move");+ }++ @Test+ void removingTheLastInstanceCannotWriteOverAClusterDelete() throws Exception {+ // Staged: the instance delete is held as it writes the cluster back without its member,+ // and the cluster delete runs to completion first. Sharing one monitor is what stops the+ // held write from landing afterwards.+ PausingStorageBackend<DocDbCluster> pausing =+ new PausingStorageBackend<>(new io.github.hectorvent.floci.core.storage.InMemoryStorage<>());+ serviceOver(pausing);+ String clusterId = "co-delete-staged";+ String instanceId = "co-delete-staged-instance";+ service.createDbCluster(clusterId, "5.0.0", "admin", "secret99password", false);+ service.createDbInstance(instanceId, clusterId, "db.r5.large", "5.0.0", false);++ AtomicReference<Throwable> unexpected = new AtomicReference<>();+ pausing.pauseOn(PausingStorageBackend.Call.PUT, "us-east-1::" + clusterId);++ Thread instanceDeleter = new Thread(() -> {+ try {+ service.deleteDbInstance(instanceId);+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ instanceDeleter.start();+ pausing.awaitReached();++ // Held mid-write, the instance is already out of the store, so the cluster deletes.+ Thread clusterDeleter = new Thread(() -> {+ try {+ service.deleteDbCluster(clusterId);+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ clusterDeleter.start();+ clusterDeleter.join(TimeUnit.SECONDS.toMillis(2));++ pausing.release();+ instanceDeleter.join(TimeUnit.SECONDS.toMillis(10));+ clusterDeleter.join(TimeUnit.SECONDS.toMillis(10));+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());++ assertFalse(clusterStore.get("us-east-1::" + clusterId).isPresent(),+ "the member-list write landed after the cluster was deleted");+ }+++ @Test+ void listingWhileALegacyRecordIsMovedSeesItExactlyOnce() throws Exception {+ // The listing reads two sets of keys while a migration is moving a record between them.+ // Whichever way the two scans straddle the move, the record has to appear once: missing it+ // hides a resource that exists, and seeing it twice reports one resource as two.+ for (int attempt = 0; attempt < 25; attempt++) {+ freshService();+ String id = "listed-during-move-" + attempt;+ DocDbCluster legacy = new DocDbCluster();+ legacy.setDbClusterIdentifier(id);+ legacy.setStatus("available");+ clusterStore.put(id, legacy);++ CountDownLatch start = new CountDownLatch(1);+ AtomicReference<Throwable> unexpected = new AtomicReference<>();+ AtomicReference<java.util.List<String>> listed = new AtomicReference<>();++ Thread mover = new Thread(() -> {+ await(start);+ try {+ service.getDbCluster(id);+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ Thread lister = new Thread(() -> {+ await(start);+ try {+ listed.set(service.listDbClusters(null).stream()+ .map(DocDbCluster::getDbClusterIdentifier).toList());+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });++ mover.start();+ lister.start();+ start.countDown();+ mover.join(TimeUnit.SECONDS.toMillis(10));+ lister.join(TimeUnit.SECONDS.toMillis(10));+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());+ assertEquals(java.util.List.of(id), listed.get(),+ "the cluster should be listed exactly once, attempt " + attempt);+ }+ }++ @Test+ void deletingTheLastInstanceWhileItsClusterIsDeletedLeavesNothingBehind() throws Exception {+ // Both paths write the cluster record — one removes the instance from its member list,+ // the other deletes it — so they have to take the same monitor. Naming one of them after+ // the bare identifier is a second monitor, and the member update then lands after the+ // delete, leaving cluster metadata no delete can remove.+ for (int attempt = 0; attempt < 25; attempt++) {+ freshService();+ String clusterId = "co-delete-" + attempt;+ String instanceId = "co-delete-instance-" + attempt;+ service.createDbCluster(clusterId, "5.0.0", "admin", "secret99password", false);+ service.createDbInstance(instanceId, clusterId, "db.r5.large", "5.0.0", false);++ CountDownLatch start = new CountDownLatch(1);+ AtomicReference<Throwable> unexpected = new AtomicReference<>();++ Thread instanceDeleter = new Thread(() -> {+ await(start);+ try {+ service.deleteDbInstance(instanceId);+ } catch (AwsException expected) {+ // losing the race is fine; answering wrongly is not+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ Thread clusterDeleter = new Thread(() -> {+ await(start);+ try {+ service.deleteDbCluster(clusterId);+ } catch (AwsException expected) {+ // the cluster still has a member until the other thread finishes+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });++ instanceDeleter.start();+ clusterDeleter.start();+ start.countDown();+ instanceDeleter.join(TimeUnit.SECONDS.toMillis(10));+ clusterDeleter.join(TimeUnit.SECONDS.toMillis(10));+ assertFalse(instanceDeleter.isAlive() || clusterDeleter.isAlive(), "a thread never finished");+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());++ // Whoever won, the instance is gone and the cluster is either gone or still deletable.+ assertFalse(clusterStore.get("us-east-1::" + clusterId)+ .map(c -> c.getDbClusterMembers().contains(instanceId)).orElse(false),+ "the deleted instance is still a member of " + clusterId);+ if (clusterStore.get("us-east-1::" + clusterId).isPresent()) {+ service.deleteDbCluster(clusterId);+ }+ assertFalse(clusterStore.get("us-east-1::" + clusterId).isPresent(),+ clusterId + " could not be removed after the race (attempt " + attempt + ")");+ }+ }++ @Test+ void aDeleteCannotSlipBetweenTheMovesReadAndItsWrite() throws Exception {+ // Staged, because the racing loop above does not reach this one: the move reads the record+ // from the unscoped key and writes it under the regional one, and a delete landing between+ // those two would be undone by that write. Holding the move at its write and letting a+ // delete run is that interleaving exactly.+ PausingStorageBackend<DocDbCluster> pausing =+ new PausingStorageBackend<>(new io.github.hectorvent.floci.core.storage.InMemoryStorage<>());+ serviceOver(pausing);+ String id = "moved-while-deleted";+ DocDbCluster legacy = new DocDbCluster();+ legacy.setDbClusterIdentifier(id);+ legacy.setStatus("available");+ clusterStore.put(id, legacy);++ AtomicReference<Throwable> unexpected = new AtomicReference<>();+ pausing.pauseOn(PausingStorageBackend.Call.PUT, "us-east-1::" + id);++ Thread mover = new Thread(() -> {+ try {+ service.getDbCluster(id);+ } catch (AwsException expected) {+ // losing to the delete is a legitimate outcome+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ mover.start();+ pausing.awaitReached();++ Thread deleter = new Thread(() -> {+ try {+ service.deleteDbCluster(id);+ } catch (AwsException expected) {+ // so is finding it already gone+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ deleter.start();+ // Sharing the monitor is what stops the delete finishing here: it should still be waiting.+ deleter.join(TimeUnit.SECONDS.toMillis(1));++ pausing.release();+ mover.join(TimeUnit.SECONDS.toMillis(10));+ deleter.join(TimeUnit.SECONDS.toMillis(10));+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());++ assertFalse(clusterStore.get("us-east-1::" + id).isPresent(),+ "the move wrote the record back after it was deleted");+ assertFalse(clusterStore.get(id).isPresent(), "the unscoped key was left behind");+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbRegionScopedIdentifierIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbRegionScopedIdentifierIntegrationTest.javanew file mode 100644index 0000000000..90d829b399--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbRegionScopedIdentifierIntegrationTest.java@@ -0,0 +1,222 @@+package io.github.hectorvent.floci.services.docdb;++import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;++import java.util.Map;++import static io.restassured.RestAssured.given;+import static io.restassured.http.ContentType.URLENC;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.not;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * One identifier per region, as AWS scopes them.+ *+ * <p>Checked against a live account: the same name creates in two regions, each with its own ARN,+ * and tagging one leaves the other untouched. Floci refused the second create, because its records+ * were keyed by identifier alone — so the name a caller used in one region was spent everywhere.+ */+@QuarkusTest+@TestProfile(DocDbRegionScopedIdentifierIntegrationTest.NoContainersProfile.class)+class DocDbRegionScopedIdentifierIntegrationTest {++ public static class NoContainersProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("floci.services.docdb.mock", "true");+ }+ }++ @Inject+ DocDbService service;++ private static final String ID = "shared-across-regions";++ private static io.restassured.specification.RequestSpecification docdb(String region, String action) {+ return given().header("Authorization",+ "AWS4-HMAC-SHA256 Credential=test/20260615/" + region + "/docdb/aws4_request, "+ + "SignedHeaders=content-type;host, Signature=test")+ .contentType(URLENC)+ .formParam("Action", action)+ .formParam("Version", "2014-10-31");+ }++ private static void createCluster(String region, String tagValue) {+ docdb(region, "CreateDBCluster")+ .formParam("DBClusterIdentifier", ID)+ .formParam("Engine", "docdb")+ .formParam("MasterUsername", "docdbadmin")+ .formParam("MasterUserPassword", "secret99password")+ .formParam("Tags.Tag.1.Key", "where")+ .formParam("Tags.Tag.1.Value", tagValue)+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString("arn:aws:rds:" + region + ":000000000000:cluster:" + ID));+ }++ @Test+ void oneNameCreatesInEveryRegionAndEachCarriesItsOwnState() {+ createCluster("us-east-1", "east");+ createCluster("eu-west-1", "west");++ // Each region describes its own record and not the other's.+ for (String region : new String[]{"us-east-1", "eu-west-1"}) {+ docdb(region, "DescribeDBClusters")+ .formParam("DBClusterIdentifier", ID)+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString("arn:aws:rds:" + region + ":000000000000:cluster:" + ID));+ }++ // Tags belong to the record, not to the name.+ docdb("us-east-1", "ListTagsForResource")+ .formParam("ResourceName", "arn:aws:rds:us-east-1:000000000000:cluster:" + ID)+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString("<Value>east</Value>"))+ .body(not(containsString("<Value>west</Value>")));++ // A third region has neither.+ docdb("ap-south-1", "DescribeDBClusters")+ .formParam("DBClusterIdentifier", ID)+ .when().post("/")+ .then().statusCode(404);++ // Deleting one leaves the other alone.+ docdb("us-east-1", "DeleteDBCluster")+ .formParam("DBClusterIdentifier", ID)+ .formParam("SkipFinalSnapshot", "true")+ .when().post("/").then().statusCode(200);++ docdb("us-east-1", "DescribeDBClusters")+ .formParam("DBClusterIdentifier", ID)+ .when().post("/").then().statusCode(404);++ docdb("eu-west-1", "DescribeDBClusters")+ .formParam("DBClusterIdentifier", ID)+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString("arn:aws:rds:eu-west-1:000000000000:cluster:" + ID));++ // Its tags survived the other region's delete too.+ docdb("eu-west-1", "ListTagsForResource")+ .formParam("ResourceName", "arn:aws:rds:eu-west-1:000000000000:cluster:" + ID)+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString("<Value>west</Value>"));++ docdb("eu-west-1", "DeleteDBCluster")+ .formParam("DBClusterIdentifier", ID)+ .formParam("SkipFinalSnapshot", "true")+ .when().post("/").then().statusCode(200);+ }++ @Test+ void instancesAreScopedTheSameWay() {+ createCluster("us-east-1", "east");+ createCluster("eu-west-1", "west");+ String instanceId = "shared-instance";++ for (String region : new String[]{"us-east-1", "eu-west-1"}) {+ docdb(region, "CreateDBInstance")+ .formParam("DBInstanceIdentifier", instanceId)+ .formParam("DBClusterIdentifier", ID)+ .formParam("Engine", "docdb")+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString("arn:aws:rds:" + region + ":000000000000:db:" + instanceId));+ }++ // Each cluster lists only the instance created against it.+ for (String region : new String[]{"us-east-1", "eu-west-1"}) {+ docdb(region, "DescribeDBInstances")+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString("arn:aws:rds:" + region + ":000000000000:db:" + instanceId));++ docdb(region, "DeleteDBInstance")+ .formParam("DBInstanceIdentifier", instanceId)+ .when().post("/").then().statusCode(200);++ docdb(region, "DeleteDBCluster")+ .formParam("DBClusterIdentifier", ID)+ .formParam("SkipFinalSnapshot", "true")+ .when().post("/").then().statusCode(200);+ }+ }++ @Test+ void aRecordWrittenBeforeRegionsWereInTheKeyBelongsToTheDefaultRegion() {+ // Written the way an earlier Floci wrote it: the service call runs outside a request, so+ // it lands under the default region — which is where such a record belongs.+ String legacyId = "pre-rekey-cluster";+ service.createDbCluster(legacyId, "5.0.0", "admin", "secret99password", false);++ docdb("us-east-1", "DescribeDBClusters")+ .formParam("DBClusterIdentifier", legacyId)+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString(legacyId));++ // And it is not visible from anywhere else, nor does it block that name there.+ docdb("eu-west-1", "DescribeDBClusters")+ .formParam("DBClusterIdentifier", legacyId)+ .when().post("/").then().statusCode(404);++ docdb("eu-west-1", "CreateDBCluster")+ .formParam("DBClusterIdentifier", legacyId)+ .formParam("Engine", "docdb")+ .formParam("MasterUsername", "docdbadmin")+ .formParam("MasterUserPassword", "secret99password")+ .when().post("/")+ .then()+ .statusCode(200)+ .body(containsString("arn:aws:rds:eu-west-1:000000000000:cluster:" + legacyId));++ assertEquals("arn:aws:rds:us-east-1:000000000000:cluster:" + legacyId,+ service.getDbCluster(legacyId).getDbClusterArn());+ assertTrue(service.hasCluster(legacyId, "us-east-1"));+ assertTrue(service.hasCluster(legacyId, "eu-west-1"));++ docdb("eu-west-1", "DeleteDBCluster")+ .formParam("DBClusterIdentifier", legacyId)+ .formParam("SkipFinalSnapshot", "true")+ .when().post("/").then().statusCode(200);+ service.deleteDbCluster(legacyId);+ }++ @Test+ void anInstanceCannotAttachToAClusterInAnotherRegion() {+ // The reference has to resolve in the region the create is made from — a cluster that+ // exists only elsewhere is not there to attach to.+ createCluster("eu-west-1", "west");++ docdb("us-east-1", "CreateDBInstance")+ .formParam("DBInstanceIdentifier", "orphan-instance")+ .formParam("DBClusterIdentifier", ID)+ .formParam("Engine", "docdb")+ .when().post("/")+ .then()+ .statusCode(404)+ .body(containsString("DBClusterNotFoundFault"));++ docdb("eu-west-1", "DeleteDBCluster")+ .formParam("DBClusterIdentifier", ID)+ .formParam("SkipFinalSnapshot", "true")+ .when().post("/").then().statusCode(200);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbServiceTest.javaindex c1122bd3c0..96ec80201f 100644--- a/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/DocDbServiceTest.java@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;import io.github.hectorvent.floci.services.docdb.model.DocDbCluster;@@ -13,10 +14,12 @@ import org.junit.jupiter.api.Test;import org.mockito.Mockito;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;@@ -183,4 +186,47 @@ class DocDbServiceTest {() -> docDbService.listTagsForResource("scoped-cluster"));assertTrue(notAnArn.getMessage().contains("Invalid resource name"));}++ @Test+ void aRecordStoredUnderABareIdentifierIsFoundAndMovedUnderItsRegionalKey() {+ // The upgrade path: records written before regions were part of the key sit under the+ // bare identifier. They belong to the default region — that is the region they were+ // created under — and are moved there on the first read, so the bare key is consulted+ // once rather than for ever.+ StorageBackend<String, DocDbCluster> clusterStore =+ AccountAwareStorageBackend.inMemory("000000000000");+ StorageBackend<String, DocDbInstance> instanceStore =+ AccountAwareStorageBackend.inMemory("000000000000");+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->+ "docdb-clusters.json".equals(inv.getArgument(1)) ? clusterStore : instanceStore);++ EmulatorConfig config = Mockito.mock(EmulatorConfig.class);+ var servicesConfig = Mockito.mock(EmulatorConfig.ServicesConfig.class);+ var docdbConfig = Mockito.mock(EmulatorConfig.DocDbServiceConfig.class);+ when(config.services()).thenReturn(servicesConfig);+ when(servicesConfig.docdb()).thenReturn(docdbConfig);+ when(docdbConfig.mock()).thenReturn(true);+ DocDbService service = new DocDbService(config,+ new RegionResolver("us-east-1", "000000000000"),+ Mockito.mock(DocDbContainerManager.class), storageFactory);++ DocDbCluster legacy = new DocDbCluster();+ legacy.setDbClusterIdentifier("bare-key-cluster");+ legacy.setStatus("available");+ clusterStore.put("bare-key-cluster", legacy);++ assertEquals("available", service.getDbCluster("bare-key-cluster").getStatus());+ assertTrue(clusterStore.get("us-east-1::bare-key-cluster").isPresent(),+ "the record should have been moved under its regional key");+ assertTrue(clusterStore.get("bare-key-cluster").isEmpty(),+ "the bare key should not be left behind");++ // It is the default region's, and only that region's.+ assertTrue(service.hasCluster("bare-key-cluster", "us-east-1"));+ assertFalse(service.hasCluster("bare-key-cluster", "eu-west-1"));++ // And it lists there, once rather than twice.+ assertEquals(1, service.listDbClusters(null).size());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/PausingStorageBackend.java b/src/test/java/io/github/hectorvent/floci/services/docdb/PausingStorageBackend.javanew file mode 100644index 0000000000..26fad437e1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/PausingStorageBackend.java@@ -0,0 +1,123 @@+package io.github.hectorvent.floci.services.docdb;++import io.github.hectorvent.floci.core.storage.StorageBackend;++import java.util.List;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.function.Predicate;++/**+ * A store that can be stopped in the middle of an operation, so a race can be staged rather than+ * hoped for.+ *+ * <p>A concurrency test that loops and waits for an unlucky interleaving proves nothing when the+ * window is a few instructions wide: it passes with the fix and without it. Holding one thread at a+ * chosen call while another runs to completion makes the interleaving the test claims to cover the+ * one it actually runs.+ */+final class PausingStorageBackend<V> implements StorageBackend<String, V> {++ /** Which call to hold, and on which key. */+ enum Call { PUT, DELETE, SCAN }++ private final StorageBackend<String, V> delegate;+ private volatile Call pauseOn;+ private volatile String pauseKey;+ private final java.util.concurrent.atomic.AtomicInteger skip =+ new java.util.concurrent.atomic.AtomicInteger();+ private final CountDownLatch reached = new CountDownLatch(1);+ private final CountDownLatch release = new CountDownLatch(1);++ PausingStorageBackend(StorageBackend<String, V> delegate) {+ this.delegate = delegate;+ }++ /** Hold the next matching call until {@link #release()}; {@code key} may be null for SCAN. */+ void pauseOn(Call call, String key) {+ pauseOn(call, key, 0);+ }++ /** As above, after letting {@code skipFirst} matching calls through untouched. */+ void pauseOn(Call call, String key, int skipFirst) {+ this.pauseOn = call;+ this.pauseKey = key;+ this.skip.set(skipFirst);+ }++ /** Waits until the paused call has been reached. */+ void awaitReached() throws InterruptedException {+ if (!reached.await(10, TimeUnit.SECONDS)) {+ throw new IllegalStateException("the paused call was never reached");+ }+ }++ void release() {+ release.countDown();+ }++ private void pauseIfMatching(Call call, String key) {+ // Keys arrive account-prefixed here, since this sits underneath the account-aware layer.+ if (pauseOn != call || (pauseKey != null && (key == null || !key.endsWith(pauseKey)))) {+ return;+ }+ if (skip.getAndDecrement() > 0) {+ return;+ }+ pauseOn = null;+ reached.countDown();+ try {+ if (!release.await(10, TimeUnit.SECONDS)) {+ throw new IllegalStateException("the paused call was never released");+ }+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ }++ @Override+ public void put(String key, V value) {+ pauseIfMatching(Call.PUT, key);+ delegate.put(key, value);+ }++ @Override+ public Optional<V> get(String key) {+ return delegate.get(key);+ }++ @Override+ public void delete(String key) {+ pauseIfMatching(Call.DELETE, key);+ delegate.delete(key);+ }++ @Override+ public List<V> scan(Predicate<String> keyFilter) {+ pauseIfMatching(Call.SCAN, null);+ return delegate.scan(keyFilter);+ }++ @Override+ public Set<String> keys() {+ return delegate.keys();+ }++ @Override+ public void flush() {+ delegate.flush();+ }++ @Override+ public void load() {+ delegate.load();+ }++ @Override+ public void clear() {+ delegate.clear();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/docdb/SharedClusterIdentifierIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/docdb/SharedClusterIdentifierIntegrationTest.javaindex 49de8de0c6..1a049f1ac8 100644--- a/src/test/java/io/github/hectorvent/floci/services/docdb/SharedClusterIdentifierIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/docdb/SharedClusterIdentifierIntegrationTest.java@@ -187,7 +187,12 @@ class SharedClusterIdentifierIntegrationTest {assertTrue(rdsService.listTagsForResource("arn:aws:rds:us-east-1:000000000000:cluster:" + id, "us-east-1").isEmpty());- docDbService.deleteDbCluster(id);+ // Deleted over the endpoint: the record lives under its own region's key, and a direct+ // call from the test thread carries no request to take a region from.+ queryIn("eu-west-1", "DeleteDBCluster")+ .formParam("DBClusterIdentifier", id)+ .formParam("SkipFinalSnapshot", "true")+ .when().post("/").then().statusCode(200);rdsService.deleteDbCluster(id, "us-east-1");}