Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(elasticache): clean up container by id on readiness-timeout rollback (#2090)

Ahmed khaled91b6a40bc77c
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
index eee8d55ea4..62a16e76d1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
@@ -39,6 +39,7 @@ public class ElastiCacheService {
private final ElastiCacheProxyManager proxyManager;
private final EmulatorConfig config;
private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();
+ private final Set<String> provisioningGroupIds = ConcurrentHashMap.newKeySet();
@Inject
public ElastiCacheService(ElastiCacheContainerManager containerManager,
@@ -60,38 +61,48 @@ public class ElastiCacheService {
throw new AwsException("ReplicationGroupAlreadyExistsFault",
"Replication group " + groupId + " already exists.", 400);
}
+ // Claim the id for the whole provisioning attempt so a concurrent create can't race
+ // ahead and be stopped by this request's handle-less rollback fallback.
+ if (!provisioningGroupIds.add(groupId)) {
+ throw new AwsException("ReplicationGroupAlreadyExistsFault",
+ "Replication group " + groupId + " is already being created.", 400);
+ }
- int proxyPort = allocateProxyPort();
- String image = config.services().elasticache().defaultImage();
-
- LOG.infov("Creating replication group {0} with authMode={1} on proxy port {2}",
- groupId, authMode, String.valueOf(proxyPort));
-
- ElastiCacheContainerHandle handle = null;
try {
- handle = containerManager.start(groupId, image);
-
- String endpointHost = resolveEndpointHost();
- Endpoint endpoint = new Endpoint(endpointHost, proxyPort);
- ReplicationGroup group = new ReplicationGroup(
- groupId, description, ReplicationGroupStatus.AVAILABLE,
- authMode, endpoint, Instant.now(), proxyPort);
- group.setContainerId(handle.getContainerId());
- group.setContainerHost(handle.getHost());
- group.setContainerPort(handle.getPort());
- group.setAuthToken(authToken);
-
- proxyManager.startProxy(groupId, authMode, proxyPort,
- handle.getHost(), handle.getPort(),
- (username, password) -> validatePassword(groupId, username, password));
-
- groups.put(groupId, group);
- LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));
- return group;
- } catch (RuntimeException e) {
- LOG.warnv("Replication group {0} provisioning failed, rolling back: {1}", groupId, e.getMessage());
- rollbackReplicationGroup(groupId, handle, proxyPort);
- throw e;
+ int proxyPort = allocateProxyPort();
+ String image = config.services().elasticache().defaultImage();
+
+ LOG.infov("Creating replication group {0} with authMode={1} on proxy port {2}",
+ groupId, authMode, String.valueOf(proxyPort));
+
+ ElastiCacheContainerHandle handle = null;
+ try {
+ handle = containerManager.start(groupId, image);
+
+ String endpointHost = resolveEndpointHost();
+ Endpoint endpoint = new Endpoint(endpointHost, proxyPort);
+ ReplicationGroup group = new ReplicationGroup(
+ groupId, description, ReplicationGroupStatus.AVAILABLE,
+ authMode, endpoint, Instant.now(), proxyPort);
+ group.setContainerId(handle.getContainerId());
+ group.setContainerHost(handle.getHost());
+ group.setContainerPort(handle.getPort());
+ group.setAuthToken(authToken);
+
+ proxyManager.startProxy(groupId, authMode, proxyPort,
+ handle.getHost(), handle.getPort(),
+ (username, password) -> validatePassword(groupId, username, password));
+
+ groups.put(groupId, group);
+ LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));
+ return group;
+ } catch (RuntimeException e) {
+ LOG.warnv("Replication group {0} provisioning failed, rolling back: {1}", groupId, e.getMessage());
+ rollbackReplicationGroup(groupId, handle, proxyPort);
+ throw e;
+ }
+ } finally {
+ provisioningGroupIds.remove(groupId);
}
}
@@ -106,6 +117,9 @@ public class ElastiCacheService {
try {
if (handle != null) {
containerManager.stop(handle);
+ } else {
+ // No handle: a readiness timeout throws before start() can return one.
+ containerManager.stopByGroupId(groupId);
}
} catch (RuntimeException e) {
LOG.warnv("Error stopping container for replication group {0}: {1}", groupId, e.getMessage());
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java
index 128ec9df60..32805bfcd6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java
@@ -72,7 +72,7 @@ public class ElastiCacheContainerManager {
public ElastiCacheContainerHandle start(String groupId, String image) {
LOG.infov("Starting ElastiCache backend container for group: {0}", groupId);
- String containerName = ContainerStorageHelper.resourceName(config, "valkey", null, groupId);
+ String containerName = containerName(groupId);
// Remove any stale container with the same name
lifecycleManager.removeIfExists(containerName);
@@ -184,6 +184,28 @@ public class ElastiCacheContainerManager {
lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());
}
+ /**
+ * Stops and removes the backend container for a group by id, if one exists.
+ * Used by the service's provisioning rollback: {@link #start} registers the container in
+ * {@code activeContainers} <em>before</em> {@link #waitForBackendReady}, so a readiness
+ * timeout throws without ever returning the handle to the caller. In that case rollback
+ * can't go through {@link #stop} (it has no handle), so it cleans up by id instead. Falls
+ * back to the deterministic container name to catch a container that failed before it was
+ * registered. Idempotent — a no-op when nothing is running for the id.
+ */
+ public void stopByGroupId(String groupId) {
+ ElastiCacheContainerHandle handle = activeContainers.get(groupId);
+ if (handle != null) {
+ stop(handle);
+ return;
+ }
+ lifecycleManager.removeIfExists(containerName(groupId));
+ }
+
+ private String containerName(String groupId) {
+ return ContainerStorageHelper.resourceName(config, "valkey", null, groupId);
+ }
+
public void stopAll() {
List<ElastiCacheContainerHandle> handles = new ArrayList<>(activeContainers.values());
if (!handles.isEmpty()) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
index 3e40fd552f..14e396523b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.elasticache;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;
@@ -12,6 +13,8 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -119,9 +122,10 @@ class ElastiCacheServiceTest {
assertThrows(RuntimeException.class,
() -> service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null));
- // Rollback stopped the proxy and the already-started container.
+ // Rollback stops by the exact handle, not a fresh by-id lookup.
verify(proxyManager).stopProxy("grp");
verify(containerManager).stop(handle);
+ verify(containerManager, never()).stopByGroupId(anyString());
// The reserved proxy port was released: a subsequent successful create reuses the base port
// instead of skipping to the next one (which is what a leak would cause).
@@ -134,18 +138,16 @@ class ElastiCacheServiceTest {
}
@Test
- void failedContainerStartReleasesPortWithoutTouchingProxyOrContainer() {
- // Container start blows up before any handle exists.
- doThrow(new RuntimeException("container boom"))
+ void failedContainerStartupCleansUpContainerByIdAndReleasesPort() {
+ // Models a readiness timeout: start() throws without ever returning a handle.
+ doThrow(new RuntimeException("readiness boom"))
.when(containerManager).start(eq("grp"), anyString());
- // The original failure must propagate to the caller (we clean up, then rethrow).
assertThrows(RuntimeException.class,
() -> service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null));
- // No container started and no proxy started, so rollback must not call either stop.
verify(proxyManager, never()).stopProxy(anyString());
- verify(containerManager, never()).stop(any());
+ verify(containerManager).stopByGroupId("grp");
// The reserved proxy port was still released: a subsequent successful create reuses the base port.
when(containerManager.start(anyString(), anyString()))
@@ -155,4 +157,31 @@ class ElastiCacheServiceTest {
assertEquals(16379, recovered.getProxyPort(),
"Port from the failed create must be released so the next group reuses it");
}
+
+ @Test
+ void concurrentCreateForSameGroupIdIsRejectedWhileFirstIsProvisioning() throws InterruptedException {
+ CountDownLatch startedLatch = new CountDownLatch(1);
+ CountDownLatch releaseLatch = new CountDownLatch(1);
+ when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {
+ startedLatch.countDown();
+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");
+ return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);
+ });
+
+ Thread firstRequest = new Thread(() ->
+ service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null));
+ firstRequest.start();
+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "first request never reached container start");
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null));
+ assertEquals("ReplicationGroupAlreadyExistsFault", ex.jsonType());
+ verify(containerManager, never()).stop(any());
+ verify(containerManager, never()).stopByGroupId(anyString());
+
+ releaseLatch.countDown();
+ firstRequest.join(5000);
+
+ assertEquals("grp", service.getReplicationGroup("grp").getReplicationGroupId());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.java
new file mode 100644
index 0000000000..bee213c232
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.java
@@ -0,0 +1,29 @@
+package io.github.hectorvent.floci.services.elasticache.container;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import org.junit.jupiter.api.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+class ElastiCacheContainerManagerTest {
+
+ @Test
+ void stopByGroupIdRemovesByDeterministicNameWhenNothingRegistered() {
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ ElastiCacheContainerManager manager = new ElastiCacheContainerManager(
+ mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),
+ mock(ContainerDetector.class), mock(EmulatorConfig.class), mock(RegionResolver.class));
+
+ // No container was ever registered for this id (e.g. it failed before registration).
+ // Rollback must still fall back to the deterministic name so nothing is orphaned.
+ manager.stopByGroupId("my-group");
+
+ verify(lifecycleManager).removeIfExists("floci-valkey-my-group");
+ }
+}