Light, fluffy, and always free - The AWS Local Emulator alternative
fix(elasticache): implement parameter group reservation to prevent deletion during provisioning (#3023)
Matheus Andréd39837664228
diff --git a/docs/services/elasticache.md b/docs/services/elasticache.mdindex 5241582307..c3785584bd 100644--- a/docs/services/elasticache.md+++ b/docs/services/elasticache.md@@ -116,7 +116,9 @@ AWS accepts. `DescribeCacheParameters` returns those parameters; a request for `A replication group that names a parameter group is refused with `CacheParameterGroupNotFound` whenno such group exists, and a parameter group still referenced by a replication group cannot bedeleted: `DeleteCacheParameterGroup` answers `InvalidCacheParameterGroupState` until that-replication group is gone.+replication group is gone. The reference counts from the moment `CreateReplicationGroup` accepts+the name, not from when the group is stored, so a delete that lands while that create is still+provisioning its container is refused too.## Configurationdiff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.javaindex 06eedf158f..36d214471a 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.resource.ResourceProvider;import io.github.hectorvent.floci.core.resource.SupportedResourceType;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;@@ -57,7 +58,7 @@ public class ElastiCacheService implements ResourceProvider {private final StorageBackend<String, ReplicationGroup> groups;private final StorageBackend<String, ElastiCacheUser> users;- private final StorageBackend<String, CacheParameterGroup> parameterGroups;+ private final AccountAwareStorageBackend<CacheParameterGroup> parameterGroups;private final StorageBackend<String, CacheSubnetGroup> subnetGroups;private final ElastiCacheContainerManager containerManager;private final ElastiCacheProxyManager proxyManager;@@ -69,6 +70,14 @@ public class ElastiCacheService implements ResourceProvider {private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();private final Set<String> provisioningGroupIds = ConcurrentHashMap.newKeySet();private final ConcurrentHashMap<String, Object> parameterGroupLocks = new ConcurrentHashMap<>();+ /**+ * Parameter groups claimed by in-flight creates, keyed by account and name (see+ * {@link #parameterGroupReservationKey}) with the number of creates holding each. A+ * replication group is only persisted once provisioning finishes, so the stored-groups scan+ * alone would let a delete slip in during that window and leave the new group pointing at a+ * parameter group that no longer exists.+ */+ private final ConcurrentHashMap<String, Integer> reservedParameterGroups = new ConcurrentHashMap<>();@Injectpublic ElastiCacheService(ElastiCacheContainerManager containerManager,@@ -144,28 +153,68 @@ public class ElastiCacheService implements ResourceProvider {// resolved with the other validations, before a port is taken or a container startedReplicationGroupSettings resolvedSettings =settings.withKmsKeyId(resolveKmsKeyArn(settings.kmsKeyId(), request.region()));- if (request.cacheParameterGroupName() != null && !request.cacheParameterGroupName().isBlank()) {- requireParameterGroup(request.cacheParameterGroupName());+ // Held until the group is persisted (or the attempt fails) so a concurrent+ // DeleteCacheParameterGroup sees the dependency before the stored-groups scan can.+ String parameterGroupReservation = reserveParameterGroup(request.cacheParameterGroupName());+ try {+ if (groups.get(groupId).isPresent()) {+ throw new AwsException("ReplicationGroupAlreadyExistsFault",+ "Replication group " + groupId + " already exists.", 400);+ }+ // Claim the id for the whole provisioning attempt so a concurrent create can't race+ // ahead and be stopped by this request's handle-less rollback fallback.+ if (!provisioningGroupIds.add(groupId)) {+ throw new AwsException("ReplicationGroupAlreadyExistsFault",+ "Replication group " + groupId + " is already being created.", 400);+ }++ try {+ if (resolveClusterEnabled(request)) {+ return provisionClusterModeGroup(request, resolvedSettings);+ }+ return provisionSingleNodeGroup(request, resolvedSettings);+ } finally {+ provisioningGroupIds.remove(groupId);+ }+ } finally {+ releaseParameterGroup(parameterGroupReservation);}- if (groups.get(groupId).isPresent()) {- throw new AwsException("ReplicationGroupAlreadyExistsFault",- "Replication group " + groupId + " already exists.", 400);+ }++ /**+ * Checks the parameter group exists and marks it in use for the calling create, under the+ * same monitor {@link #deleteCacheParameterGroup} takes, so the existence check and the claim+ * can't straddle a delete. Returns the reservation key, or {@code null} when the request+ * names no parameter group, for the matching {@link #releaseParameterGroup} call.+ */+ private String reserveParameterGroup(String requestedName) {+ if (requestedName == null || requestedName.isBlank()) {+ return null;}- // Claim the id for the whole provisioning attempt so a concurrent create can't race- // ahead and be stopped by this request's handle-less rollback fallback.- if (!provisioningGroupIds.add(groupId)) {- throw new AwsException("ReplicationGroupAlreadyExistsFault",- "Replication group " + groupId + " is already being created.", 400);+ synchronized (lockFor(requestedName)) {+ requireParameterGroup(requestedName);+ String reservation = parameterGroupReservationKey(requestedName);+ reservedParameterGroups.merge(reservation, 1, Integer::sum);+ return reservation;}+ }- try {- if (resolveClusterEnabled(request)) {- return provisionClusterModeGroup(request, resolvedSettings);- }- return provisionSingleNodeGroup(request, resolvedSettings);- } finally {- provisioningGroupIds.remove(groupId);+ private void releaseParameterGroup(String reservation) {+ if (reservation == null) {+ return;}+ reservedParameterGroups.computeIfPresent(reservation,+ (key, holders) -> holders > 1 ? holders - 1 : null);+ }++ /**+ * Parameter groups are stored per account, so a reservation is scoped the same way: the+ * account the store is currently prefixing keys with, then the name. Without that, one+ * account's in-flight create would block another account's delete of its own group of the+ * same name.+ */+ private String parameterGroupReservationKey(String name) {+ return parameterGroups.accountId() + "/" + name;}private ReplicationGroup provisionSingleNodeGroup(CreateReplicationGroupRequest request,@@ -1156,10 +1205,14 @@ public class ElastiCacheService implements ResourceProvider {LOG.infov("Deleted cache parameter group {0}", name);}- /** Whether a stored replication group still references the parameter group by that name. */+ /**+ * Whether a stored replication group still references the parameter group by that name, or a+ * create that named it is still provisioning and about to store one.+ */private boolean isParameterGroupInUse(String name) {- return groups.scan(key -> true).stream()- .anyMatch(group -> name.equals(group.getCacheParameterGroupName()));+ return reservedParameterGroups.containsKey(parameterGroupReservationKey(name))+ || groups.scan(key -> true).stream()+ .anyMatch(group -> name.equals(group.getCacheParameterGroupName()));}@Overridediff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.javaindex aaabb9df04..9ad4ec460d 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.RequestContext;import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.kms.KmsService;@@ -29,6 +30,8 @@ import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;+import jakarta.enterprise.inject.Instance;+import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNull;@@ -714,4 +717,117 @@ class ElastiCacheServiceTest {service.deleteCacheParameterGroup("custom-pg");assertTrue(service.findParameterGroup("custom-pg").isEmpty());}++ @Test+ void aParameterGroupNamedByAProvisioningCreateCannotBeDeleted() throws InterruptedException {+ service.createCacheParameterGroup("custom-pg", "redis7", "in use", Map.of());+ CountDownLatch startedLatch = new CountDownLatch(1);+ CountDownLatch releaseLatch = new CountDownLatch(1);+ when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {+ startedLatch.countDown();+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");+ return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);+ });++ Thread create = new Thread(() ->+ service.createReplicationGroup(requestWithParameterGroup("grp", "custom-pg")));+ create.start();+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "create never reached container start");++ // The replication group is not stored yet: only the reservation can refuse this.+ AwsException ex = assertThrows(AwsException.class,+ () -> service.deleteCacheParameterGroup("custom-pg"));+ assertEquals("InvalidCacheParameterGroupState", ex.getErrorCode());+ assertTrue(service.findParameterGroup("custom-pg").isPresent());++ releaseLatch.countDown();+ create.join(5000);++ assertEquals("custom-pg", service.getReplicationGroup("grp").getCacheParameterGroupName());+ assertEquals("InvalidCacheParameterGroupState",+ assertThrows(AwsException.class, () -> service.deleteCacheParameterGroup("custom-pg"))+ .getErrorCode());++ service.deleteReplicationGroup("grp");+ service.deleteCacheParameterGroup("custom-pg");+ assertTrue(service.findParameterGroup("custom-pg").isEmpty());+ }++ @Test+ void aClaimHeldByOneAccountDoesNotBlockAnotherAccountsDeleteOfItsOwnSameNamedGroup()+ throws InterruptedException {+ // Storage prefixes keys with the account of the calling thread; the default account+ // applies to a thread that never registered one.+ ConcurrentHashMap<Thread, String> accountByThread = new ConcurrentHashMap<>();+ RequestContext requestContext = mock(RequestContext.class);+ when(requestContext.getAccountId()).thenAnswer(inv -> accountByThread.get(Thread.currentThread()));+ @SuppressWarnings("unchecked")+ Instance<RequestContext> requestContextInstance = mock(Instance.class);+ when(requestContextInstance.get()).thenReturn(requestContext);+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.create(anyString(), anyString(), any())).thenAnswer(inv ->+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), requestContextInstance, "000000000000"));+ ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, clusterFormation,+ factory, config, mock(Ec2Service.class),+ new RegionResolver("us-east-1", "000000000000"), kmsService);++ CountDownLatch startedLatch = new CountDownLatch(1);+ CountDownLatch releaseLatch = new CountDownLatch(1);+ when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {+ startedLatch.countDown();+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");+ return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);+ });++ // Account A: owns shared-pg and is provisioning a replication group that names it.+ Thread accountA = new Thread(() -> {+ accountByThread.put(Thread.currentThread(), "111111111111");+ svc.createCacheParameterGroup("shared-pg", "redis7", "account A", Map.of());+ svc.createReplicationGroup(requestWithParameterGroup("grp", "shared-pg"));+ });+ accountA.start();+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "create never reached container start");++ // Account B: owns an unrelated shared-pg of its own; account A's claim must not hold it.+ accountByThread.put(Thread.currentThread(), "222222222222");+ svc.createCacheParameterGroup("shared-pg", "redis7", "account B", Map.of());+ svc.deleteCacheParameterGroup("shared-pg");+ assertTrue(svc.findParameterGroup("shared-pg").isEmpty());++ // Account A's own group is still held by its in-flight create.+ accountByThread.put(Thread.currentThread(), "111111111111");+ assertEquals("InvalidCacheParameterGroupState",+ assertThrows(AwsException.class, () -> svc.deleteCacheParameterGroup("shared-pg"))+ .getErrorCode());++ releaseLatch.countDown();+ accountA.join(5000);+ assertEquals("shared-pg", svc.getReplicationGroup("grp").getCacheParameterGroupName());+ }++ @Test+ void aFailedCreateReleasesItsClaimOnTheParameterGroup() {+ service.createCacheParameterGroup("custom-pg", "redis7", "in use", Map.of());+ when(containerManager.start(anyString(), anyString()))+ .thenThrow(new RuntimeException("docker is down"));++ assertThrows(RuntimeException.class,+ () -> service.createReplicationGroup(requestWithParameterGroup("grp", "custom-pg")));++ service.deleteCacheParameterGroup("custom-pg");+ assertTrue(service.findParameterGroup("custom-pg").isEmpty());+ }++ @Test+ void aCreateThatFailsBeforeProvisioningStillReleasesItsClaimOnTheParameterGroup() {+ service.createCacheParameterGroup("custom-pg", "redis7", "in use", Map.of());+ service.createReplicationGroup("grp", "test", AuthMode.NO_AUTH, null, "us-east-1");++ AwsException ex = assertThrows(AwsException.class,+ () -> service.createReplicationGroup(requestWithParameterGroup("grp", "custom-pg")));+ assertEquals("ReplicationGroupAlreadyExistsFault", ex.jsonType());++ service.deleteCacheParameterGroup("custom-pg");+ assertTrue(service.findParameterGroup("custom-pg").isEmpty());+ }}