Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(elasticache): implement parameter group reservation to prevent deletion during provisioning (#3023)

Matheus Andréd39837664228
diff --git a/docs/services/elasticache.md b/docs/services/elasticache.md
index 5241582307..c3785584bd 100644
--- a/docs/services/elasticache.md
+++ b/docs/services/elasticache.md
@@ -116,7 +116,9 @@ AWS accepts. `DescribeCacheParameters` returns those parameters; a request for `
A replication group that names a parameter group is refused with `CacheParameterGroupNotFound` when
no such group exists, and a parameter group still referenced by a replication group cannot be
deleted: `DeleteCacheParameterGroup` answers `InvalidCacheParameterGroupState` until that
-replication group is gone.
+replication group is gone. The reference counts from the moment `CreateReplicationGroup` accepts
+the name, not from when the group is stored, so a delete that lands while that create is still
+provisioning its container is refused too.
## Configuration
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
index 06eedf158f..36d214471a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.resource.ResourceProvider;
import io.github.hectorvent.floci.core.resource.SupportedResourceType;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;
@@ -57,7 +58,7 @@ public class ElastiCacheService implements ResourceProvider {
private final StorageBackend<String, ReplicationGroup> groups;
private final StorageBackend<String, ElastiCacheUser> users;
- private final StorageBackend<String, CacheParameterGroup> parameterGroups;
+ private final AccountAwareStorageBackend<CacheParameterGroup> parameterGroups;
private final StorageBackend<String, CacheSubnetGroup> subnetGroups;
private final ElastiCacheContainerManager containerManager;
private final ElastiCacheProxyManager proxyManager;
@@ -69,6 +70,14 @@ public class ElastiCacheService implements ResourceProvider {
private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();
private final Set<String> provisioningGroupIds = ConcurrentHashMap.newKeySet();
private final ConcurrentHashMap<String, Object> parameterGroupLocks = new ConcurrentHashMap<>();
+ /**
+ * Parameter groups claimed by in-flight creates, keyed by account and name (see
+ * {@link #parameterGroupReservationKey}) with the number of creates holding each. A
+ * replication group is only persisted once provisioning finishes, so the stored-groups scan
+ * alone would let a delete slip in during that window and leave the new group pointing at a
+ * parameter group that no longer exists.
+ */
+ private final ConcurrentHashMap<String, Integer> reservedParameterGroups = new ConcurrentHashMap<>();
@Inject
public ElastiCacheService(ElastiCacheContainerManager containerManager,
@@ -144,28 +153,68 @@ public class ElastiCacheService implements ResourceProvider {
// resolved with the other validations, before a port is taken or a container started
ReplicationGroupSettings resolvedSettings =
settings.withKmsKeyId(resolveKmsKeyArn(settings.kmsKeyId(), request.region()));
- if (request.cacheParameterGroupName() != null && !request.cacheParameterGroupName().isBlank()) {
- requireParameterGroup(request.cacheParameterGroupName());
+ // Held until the group is persisted (or the attempt fails) so a concurrent
+ // DeleteCacheParameterGroup sees the dependency before the stored-groups scan can.
+ String parameterGroupReservation = reserveParameterGroup(request.cacheParameterGroupName());
+ try {
+ if (groups.get(groupId).isPresent()) {
+ throw new AwsException("ReplicationGroupAlreadyExistsFault",
+ "Replication group " + groupId + " already exists.", 400);
+ }
+ // Claim the id for the whole provisioning attempt so a concurrent create can't race
+ // ahead and be stopped by this request's handle-less rollback fallback.
+ if (!provisioningGroupIds.add(groupId)) {
+ throw new AwsException("ReplicationGroupAlreadyExistsFault",
+ "Replication group " + groupId + " is already being created.", 400);
+ }
+
+ try {
+ if (resolveClusterEnabled(request)) {
+ return provisionClusterModeGroup(request, resolvedSettings);
+ }
+ return provisionSingleNodeGroup(request, resolvedSettings);
+ } finally {
+ provisioningGroupIds.remove(groupId);
+ }
+ } finally {
+ releaseParameterGroup(parameterGroupReservation);
}
- if (groups.get(groupId).isPresent()) {
- throw new AwsException("ReplicationGroupAlreadyExistsFault",
- "Replication group " + groupId + " already exists.", 400);
+ }
+
+ /**
+ * Checks the parameter group exists and marks it in use for the calling create, under the
+ * same monitor {@link #deleteCacheParameterGroup} takes, so the existence check and the claim
+ * can't straddle a delete. Returns the reservation key, or {@code null} when the request
+ * names no parameter group, for the matching {@link #releaseParameterGroup} call.
+ */
+ private String reserveParameterGroup(String requestedName) {
+ if (requestedName == null || requestedName.isBlank()) {
+ return null;
}
- // Claim the id for the whole provisioning attempt so a concurrent create can't race
- // ahead and be stopped by this request's handle-less rollback fallback.
- if (!provisioningGroupIds.add(groupId)) {
- throw new AwsException("ReplicationGroupAlreadyExistsFault",
- "Replication group " + groupId + " is already being created.", 400);
+ synchronized (lockFor(requestedName)) {
+ requireParameterGroup(requestedName);
+ String reservation = parameterGroupReservationKey(requestedName);
+ reservedParameterGroups.merge(reservation, 1, Integer::sum);
+ return reservation;
}
+ }
- try {
- if (resolveClusterEnabled(request)) {
- return provisionClusterModeGroup(request, resolvedSettings);
- }
- return provisionSingleNodeGroup(request, resolvedSettings);
- } finally {
- provisioningGroupIds.remove(groupId);
+ private void releaseParameterGroup(String reservation) {
+ if (reservation == null) {
+ return;
}
+ reservedParameterGroups.computeIfPresent(reservation,
+ (key, holders) -> holders > 1 ? holders - 1 : null);
+ }
+
+ /**
+ * Parameter groups are stored per account, so a reservation is scoped the same way: the
+ * account the store is currently prefixing keys with, then the name. Without that, one
+ * account's in-flight create would block another account's delete of its own group of the
+ * same name.
+ */
+ private String parameterGroupReservationKey(String name) {
+ return parameterGroups.accountId() + "/" + name;
}
private ReplicationGroup provisionSingleNodeGroup(CreateReplicationGroupRequest request,
@@ -1156,10 +1205,14 @@ public class ElastiCacheService implements ResourceProvider {
LOG.infov("Deleted cache parameter group {0}", name);
}
- /** Whether a stored replication group still references the parameter group by that name. */
+ /**
+ * Whether a stored replication group still references the parameter group by that name, or a
+ * create that named it is still provisioning and about to store one.
+ */
private boolean isParameterGroupInUse(String name) {
- return groups.scan(key -> true).stream()
- .anyMatch(group -> name.equals(group.getCacheParameterGroupName()));
+ return reservedParameterGroups.containsKey(parameterGroupReservationKey(name))
+ || groups.scan(key -> true).stream()
+ .anyMatch(group -> name.equals(group.getCacheParameterGroupName()));
}
@Override
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
index aaabb9df04..9ad4ec460d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.RequestContext;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.kms.KmsService;
@@ -29,6 +30,8 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import jakarta.enterprise.inject.Instance;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -714,4 +717,117 @@ class ElastiCacheServiceTest {
service.deleteCacheParameterGroup("custom-pg");
assertTrue(service.findParameterGroup("custom-pg").isEmpty());
}
+
+ @Test
+ void aParameterGroupNamedByAProvisioningCreateCannotBeDeleted() throws InterruptedException {
+ service.createCacheParameterGroup("custom-pg", "redis7", "in use", Map.of());
+ CountDownLatch startedLatch = new CountDownLatch(1);
+ CountDownLatch releaseLatch = new CountDownLatch(1);
+ when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {
+ startedLatch.countDown();
+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");
+ return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);
+ });
+
+ Thread create = new Thread(() ->
+ service.createReplicationGroup(requestWithParameterGroup("grp", "custom-pg")));
+ create.start();
+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "create never reached container start");
+
+ // The replication group is not stored yet: only the reservation can refuse this.
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.deleteCacheParameterGroup("custom-pg"));
+ assertEquals("InvalidCacheParameterGroupState", ex.getErrorCode());
+ assertTrue(service.findParameterGroup("custom-pg").isPresent());
+
+ releaseLatch.countDown();
+ create.join(5000);
+
+ assertEquals("custom-pg", service.getReplicationGroup("grp").getCacheParameterGroupName());
+ assertEquals("InvalidCacheParameterGroupState",
+ assertThrows(AwsException.class, () -> service.deleteCacheParameterGroup("custom-pg"))
+ .getErrorCode());
+
+ service.deleteReplicationGroup("grp");
+ service.deleteCacheParameterGroup("custom-pg");
+ assertTrue(service.findParameterGroup("custom-pg").isEmpty());
+ }
+
+ @Test
+ void aClaimHeldByOneAccountDoesNotBlockAnotherAccountsDeleteOfItsOwnSameNamedGroup()
+ throws InterruptedException {
+ // Storage prefixes keys with the account of the calling thread; the default account
+ // applies to a thread that never registered one.
+ ConcurrentHashMap<Thread, String> accountByThread = new ConcurrentHashMap<>();
+ RequestContext requestContext = mock(RequestContext.class);
+ when(requestContext.getAccountId()).thenAnswer(inv -> accountByThread.get(Thread.currentThread()));
+ @SuppressWarnings("unchecked")
+ Instance<RequestContext> requestContextInstance = mock(Instance.class);
+ when(requestContextInstance.get()).thenReturn(requestContext);
+ StorageFactory factory = mock(StorageFactory.class);
+ when(factory.create(anyString(), anyString(), any())).thenAnswer(inv ->
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), requestContextInstance, "000000000000"));
+ ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, clusterFormation,
+ factory, config, mock(Ec2Service.class),
+ new RegionResolver("us-east-1", "000000000000"), kmsService);
+
+ CountDownLatch startedLatch = new CountDownLatch(1);
+ CountDownLatch releaseLatch = new CountDownLatch(1);
+ when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {
+ startedLatch.countDown();
+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");
+ return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);
+ });
+
+ // Account A: owns shared-pg and is provisioning a replication group that names it.
+ Thread accountA = new Thread(() -> {
+ accountByThread.put(Thread.currentThread(), "111111111111");
+ svc.createCacheParameterGroup("shared-pg", "redis7", "account A", Map.of());
+ svc.createReplicationGroup(requestWithParameterGroup("grp", "shared-pg"));
+ });
+ accountA.start();
+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "create never reached container start");
+
+ // Account B: owns an unrelated shared-pg of its own; account A's claim must not hold it.
+ accountByThread.put(Thread.currentThread(), "222222222222");
+ svc.createCacheParameterGroup("shared-pg", "redis7", "account B", Map.of());
+ svc.deleteCacheParameterGroup("shared-pg");
+ assertTrue(svc.findParameterGroup("shared-pg").isEmpty());
+
+ // Account A's own group is still held by its in-flight create.
+ accountByThread.put(Thread.currentThread(), "111111111111");
+ assertEquals("InvalidCacheParameterGroupState",
+ assertThrows(AwsException.class, () -> svc.deleteCacheParameterGroup("shared-pg"))
+ .getErrorCode());
+
+ releaseLatch.countDown();
+ accountA.join(5000);
+ assertEquals("shared-pg", svc.getReplicationGroup("grp").getCacheParameterGroupName());
+ }
+
+ @Test
+ void aFailedCreateReleasesItsClaimOnTheParameterGroup() {
+ service.createCacheParameterGroup("custom-pg", "redis7", "in use", Map.of());
+ when(containerManager.start(anyString(), anyString()))
+ .thenThrow(new RuntimeException("docker is down"));
+
+ assertThrows(RuntimeException.class,
+ () -> service.createReplicationGroup(requestWithParameterGroup("grp", "custom-pg")));
+
+ service.deleteCacheParameterGroup("custom-pg");
+ assertTrue(service.findParameterGroup("custom-pg").isEmpty());
+ }
+
+ @Test
+ void aCreateThatFailsBeforeProvisioningStillReleasesItsClaimOnTheParameterGroup() {
+ service.createCacheParameterGroup("custom-pg", "redis7", "in use", Map.of());
+ service.createReplicationGroup("grp", "test", AuthMode.NO_AUTH, null, "us-east-1");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.createReplicationGroup(requestWithParameterGroup("grp", "custom-pg")));
+ assertEquals("ReplicationGroupAlreadyExistsFault", ex.jsonType());
+
+ service.deleteCacheParameterGroup("custom-pg");
+ assertTrue(service.findParameterGroup("custom-pg").isEmpty());
+ }
}