Light, fluffy, and always free - The AWS Local Emulator alternative
fix(elasticache): clean up container by id on readiness-timeout rollback (#2090)
Ahmed khaled91b6a40bc77c
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.javaindex eee8d55ea4..62a16e76d1 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java@@ -39,6 +39,7 @@ public class ElastiCacheService {private final ElastiCacheProxyManager proxyManager;private final EmulatorConfig config;private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();+ private final Set<String> provisioningGroupIds = ConcurrentHashMap.newKeySet();@Injectpublic ElastiCacheService(ElastiCacheContainerManager containerManager,@@ -60,38 +61,48 @@ public class ElastiCacheService {throw new AwsException("ReplicationGroupAlreadyExistsFault","Replication group " + groupId + " already exists.", 400);}+ // Claim the id for the whole provisioning attempt so a concurrent create can't race+ // ahead and be stopped by this request's handle-less rollback fallback.+ if (!provisioningGroupIds.add(groupId)) {+ throw new AwsException("ReplicationGroupAlreadyExistsFault",+ "Replication group " + groupId + " is already being created.", 400);+ }- int proxyPort = allocateProxyPort();- String image = config.services().elasticache().defaultImage();-- LOG.infov("Creating replication group {0} with authMode={1} on proxy port {2}",- groupId, authMode, String.valueOf(proxyPort));-- ElastiCacheContainerHandle handle = null;try {- handle = containerManager.start(groupId, image);-- String endpointHost = resolveEndpointHost();- Endpoint endpoint = new Endpoint(endpointHost, proxyPort);- ReplicationGroup group = new ReplicationGroup(- groupId, description, ReplicationGroupStatus.AVAILABLE,- authMode, endpoint, Instant.now(), proxyPort);- group.setContainerId(handle.getContainerId());- group.setContainerHost(handle.getHost());- group.setContainerPort(handle.getPort());- group.setAuthToken(authToken);-- proxyManager.startProxy(groupId, authMode, proxyPort,- handle.getHost(), handle.getPort(),- (username, password) -> validatePassword(groupId, username, password));-- groups.put(groupId, group);- LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));- return group;- } catch (RuntimeException e) {- LOG.warnv("Replication group {0} provisioning failed, rolling back: {1}", groupId, e.getMessage());- rollbackReplicationGroup(groupId, handle, proxyPort);- throw e;+ int proxyPort = allocateProxyPort();+ String image = config.services().elasticache().defaultImage();++ LOG.infov("Creating replication group {0} with authMode={1} on proxy port {2}",+ groupId, authMode, String.valueOf(proxyPort));++ ElastiCacheContainerHandle handle = null;+ try {+ handle = containerManager.start(groupId, image);++ String endpointHost = resolveEndpointHost();+ Endpoint endpoint = new Endpoint(endpointHost, proxyPort);+ ReplicationGroup group = new ReplicationGroup(+ groupId, description, ReplicationGroupStatus.AVAILABLE,+ authMode, endpoint, Instant.now(), proxyPort);+ group.setContainerId(handle.getContainerId());+ group.setContainerHost(handle.getHost());+ group.setContainerPort(handle.getPort());+ group.setAuthToken(authToken);++ proxyManager.startProxy(groupId, authMode, proxyPort,+ handle.getHost(), handle.getPort(),+ (username, password) -> validatePassword(groupId, username, password));++ groups.put(groupId, group);+ LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));+ return group;+ } catch (RuntimeException e) {+ LOG.warnv("Replication group {0} provisioning failed, rolling back: {1}", groupId, e.getMessage());+ rollbackReplicationGroup(groupId, handle, proxyPort);+ throw e;+ }+ } finally {+ provisioningGroupIds.remove(groupId);}}@@ -106,6 +117,9 @@ public class ElastiCacheService {try {if (handle != null) {containerManager.stop(handle);+ } else {+ // No handle: a readiness timeout throws before start() can return one.+ containerManager.stopByGroupId(groupId);}} catch (RuntimeException e) {LOG.warnv("Error stopping container for replication group {0}: {1}", groupId, e.getMessage());diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.javaindex 128ec9df60..32805bfcd6 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java@@ -72,7 +72,7 @@ public class ElastiCacheContainerManager {public ElastiCacheContainerHandle start(String groupId, String image) {LOG.infov("Starting ElastiCache backend container for group: {0}", groupId);- String containerName = ContainerStorageHelper.resourceName(config, "valkey", null, groupId);+ String containerName = containerName(groupId);// Remove any stale container with the same namelifecycleManager.removeIfExists(containerName);@@ -184,6 +184,28 @@ public class ElastiCacheContainerManager {lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());}+ /**+ * Stops and removes the backend container for a group by id, if one exists.+ * Used by the service's provisioning rollback: {@link #start} registers the container in+ * {@code activeContainers} <em>before</em> {@link #waitForBackendReady}, so a readiness+ * timeout throws without ever returning the handle to the caller. In that case rollback+ * can't go through {@link #stop} (it has no handle), so it cleans up by id instead. Falls+ * back to the deterministic container name to catch a container that failed before it was+ * registered. Idempotent — a no-op when nothing is running for the id.+ */+ public void stopByGroupId(String groupId) {+ ElastiCacheContainerHandle handle = activeContainers.get(groupId);+ if (handle != null) {+ stop(handle);+ return;+ }+ lifecycleManager.removeIfExists(containerName(groupId));+ }++ private String containerName(String groupId) {+ return ContainerStorageHelper.resourceName(config, "valkey", null, groupId);+ }+public void stopAll() {List<ElastiCacheContainerHandle> handles = new ArrayList<>(activeContainers.values());if (!handles.isEmpty()) {diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.javaindex 3e40fd552f..14e396523b 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.elasticache;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;@@ -12,6 +13,8 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import java.util.List;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -119,9 +122,10 @@ class ElastiCacheServiceTest {assertThrows(RuntimeException.class,() -> service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null));- // Rollback stopped the proxy and the already-started container.+ // Rollback stops by the exact handle, not a fresh by-id lookup.verify(proxyManager).stopProxy("grp");verify(containerManager).stop(handle);+ verify(containerManager, never()).stopByGroupId(anyString());// The reserved proxy port was released: a subsequent successful create reuses the base port// instead of skipping to the next one (which is what a leak would cause).@@ -134,18 +138,16 @@ class ElastiCacheServiceTest {}@Test- void failedContainerStartReleasesPortWithoutTouchingProxyOrContainer() {- // Container start blows up before any handle exists.- doThrow(new RuntimeException("container boom"))+ void failedContainerStartupCleansUpContainerByIdAndReleasesPort() {+ // Models a readiness timeout: start() throws without ever returning a handle.+ doThrow(new RuntimeException("readiness boom")).when(containerManager).start(eq("grp"), anyString());- // The original failure must propagate to the caller (we clean up, then rethrow).assertThrows(RuntimeException.class,() -> service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null));- // No container started and no proxy started, so rollback must not call either stop.verify(proxyManager, never()).stopProxy(anyString());- verify(containerManager, never()).stop(any());+ verify(containerManager).stopByGroupId("grp");// The reserved proxy port was still released: a subsequent successful create reuses the base port.when(containerManager.start(anyString(), anyString()))@@ -155,4 +157,31 @@ class ElastiCacheServiceTest {assertEquals(16379, recovered.getProxyPort(),"Port from the failed create must be released so the next group reuses it");}++ @Test+ void concurrentCreateForSameGroupIdIsRejectedWhileFirstIsProvisioning() throws InterruptedException {+ CountDownLatch startedLatch = new CountDownLatch(1);+ CountDownLatch releaseLatch = new CountDownLatch(1);+ when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {+ startedLatch.countDown();+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");+ return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);+ });++ Thread firstRequest = new Thread(() ->+ service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null));+ firstRequest.start();+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "first request never reached container start");++ AwsException ex = assertThrows(AwsException.class,+ () -> service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null));+ assertEquals("ReplicationGroupAlreadyExistsFault", ex.jsonType());+ verify(containerManager, never()).stop(any());+ verify(containerManager, never()).stopByGroupId(anyString());++ releaseLatch.countDown();+ firstRequest.join(5000);++ assertEquals("grp", service.getReplicationGroup("grp").getReplicationGroupId());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.javanew file mode 100644index 0000000000..bee213c232--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.java@@ -0,0 +1,29 @@+package io.github.hectorvent.floci.services.elasticache.container;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import org.junit.jupiter.api.Test;++import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;++class ElastiCacheContainerManagerTest {++ @Test+ void stopByGroupIdRemovesByDeterministicNameWhenNothingRegistered() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ ElastiCacheContainerManager manager = new ElastiCacheContainerManager(+ mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),+ mock(ContainerDetector.class), mock(EmulatorConfig.class), mock(RegionResolver.class));++ // No container was ever registered for this id (e.g. it failed before registration).+ // Rollback must still fall back to the deterministic name so nothing is orphaned.+ manager.stopByGroupId("my-group");++ verify(lifecycleManager).removeIfExists("floci-valkey-my-group");+ }+}