Light, fluffy, and always free - The AWS Local Emulator alternative
fix(memorydb): clean up container by id on readiness-timeout rollback (#2092)
Ahmed khaled11d527547af8
diff --git a/src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbService.java b/src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbService.javaindex f1101640da..08e782c776 100644--- a/src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbService.java@@ -65,6 +65,7 @@ public class MemoryDbService {private final EmulatorConfig config;private final RegionResolver regionResolver;private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();+ private final Set<String> provisioningClusterNames = ConcurrentHashMap.newKeySet();@Injectpublic MemoryDbService(MemoryDbContainerManager containerManager,@@ -97,40 +98,50 @@ public class MemoryDbService {throw new AwsException("ClusterAlreadyExistsFault","Cluster with specified name already exists.", 400);}-- String aclName = spec.getAclName();- if (aclName == null || aclName.isBlank()) {- throw new AwsException("InvalidParameterValueException", "ACLName is required.", 400);- }- requireAclExists(aclName);- boolean authRequired = isAuthRequired(aclName);-- Cluster cluster = new Cluster();- cluster.setName(name);- cluster.setDescription(spec.getDescription());- cluster.setStatus(ClusterStatus.AVAILABLE);- cluster.setNodeType(spec.getNodeType() != null ? spec.getNodeType() : "db.t4g.small");- cluster.setNumberOfShards(spec.getNumberOfShards() > 0 ? spec.getNumberOfShards() : 1);- cluster.setEngine(spec.getEngine() != null ? spec.getEngine() : DEFAULT_ENGINE);- cluster.setEngineVersion(spec.getEngineVersion() != null ? spec.getEngineVersion() : DEFAULT_ENGINE_VERSION);- cluster.setAclName(aclName);- cluster.setTlsEnabled(spec.isTlsEnabled());- cluster.setArn(buildArn(region, "cluster", name));- cluster.setCreatedAt(Instant.now());- cluster.setTags(spec.getTags());-- if (config.services().memorydb().mock()) {- LOG.infov("Creating MemoryDB cluster {0} in mock mode (no container)", name);- cluster.setClusterEndpoint(new Endpoint(resolveEndpointHost(), REDIS_PORT));- } else {- startBackend(cluster, authRequired);+ // Claim the name for the whole provisioning attempt so a concurrent create can't race+ // ahead and be stopped by this request's handle-less rollback fallback.+ if (!provisioningClusterNames.add(name)) {+ throw new AwsException("ClusterAlreadyExistsFault",+ "Cluster " + name + " is already being created.", 400);}- clusters.put(name, cluster);- LOG.infov("MemoryDB cluster {0} created (acl={1}, authRequired={2}), endpoint={3}:{4}",- name, aclName, String.valueOf(authRequired), cluster.getClusterEndpoint().address(),- String.valueOf(cluster.getClusterEndpoint().port()));- return cluster;+ try {+ String aclName = spec.getAclName();+ if (aclName == null || aclName.isBlank()) {+ throw new AwsException("InvalidParameterValueException", "ACLName is required.", 400);+ }+ requireAclExists(aclName);+ boolean authRequired = isAuthRequired(aclName);++ Cluster cluster = new Cluster();+ cluster.setName(name);+ cluster.setDescription(spec.getDescription());+ cluster.setStatus(ClusterStatus.AVAILABLE);+ cluster.setNodeType(spec.getNodeType() != null ? spec.getNodeType() : "db.t4g.small");+ cluster.setNumberOfShards(spec.getNumberOfShards() > 0 ? spec.getNumberOfShards() : 1);+ cluster.setEngine(spec.getEngine() != null ? spec.getEngine() : DEFAULT_ENGINE);+ cluster.setEngineVersion(spec.getEngineVersion() != null ? spec.getEngineVersion() : DEFAULT_ENGINE_VERSION);+ cluster.setAclName(aclName);+ cluster.setTlsEnabled(spec.isTlsEnabled());+ cluster.setArn(buildArn(region, "cluster", name));+ cluster.setCreatedAt(Instant.now());+ cluster.setTags(spec.getTags());++ if (config.services().memorydb().mock()) {+ LOG.infov("Creating MemoryDB cluster {0} in mock mode (no container)", name);+ cluster.setClusterEndpoint(new Endpoint(resolveEndpointHost(), REDIS_PORT));+ } else {+ startBackend(cluster, authRequired);+ }++ clusters.put(name, cluster);+ LOG.infov("MemoryDB cluster {0} created (acl={1}, authRequired={2}), endpoint={3}:{4}",+ name, aclName, String.valueOf(authRequired), cluster.getClusterEndpoint().address(),+ String.valueOf(cluster.getClusterEndpoint().port()));+ return cluster;+ } finally {+ provisioningClusterNames.remove(name);+ }}public Cluster getCluster(String name) {@@ -496,12 +507,33 @@ public class MemoryDbService {private void rollbackBackend(String name, MemoryDbContainerHandle handle, int proxyPort) {try {- proxyManager.stopProxy(name);- if (handle != null) {- containerManager.stop(handle);+ try {+ // The proxy only starts after the container is ready, so a null handle means it+ // never started — nothing to stop.+ if (handle != null) {+ proxyManager.stopProxy(name);+ }+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping proxy for MemoryDB cluster {0}: {1}", name, e.getMessage());+ }+ try {+ if (handle != null) {+ // We have the exact handle from this request's start() call, so stop by it+ // directly. Falling back to stopByClusterName here instead would look up+ // whatever is currently registered for name, which could be a different+ // container if an overlapping create for the same name raced ahead of this+ // rollback.+ containerManager.stop(handle);+ } else {+ // No handle: a readiness timeout in containerManager.start() throws after the+ // container was created and registered but before the handle is returned, so+ // cleaning up by handle here isn't possible. stopByClusterName is idempotent,+ // so it's safe when the container never started.+ containerManager.stopByClusterName(name);+ }+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container for MemoryDB cluster {0}: {1}", name, e.getMessage());}- } catch (RuntimeException e) {- LOG.warnv("Error rolling back MemoryDB cluster {0}: {1}", name, e.getMessage());} finally {releaseProxyPort(proxyPort);}diff --git a/src/main/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManager.javaindex 0e7be069dc..60d207b26e 100644--- a/src/main/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManager.java@@ -68,7 +68,7 @@ public class MemoryDbContainerManager {public MemoryDbContainerHandle start(String clusterName, String image) {LOG.infov("Starting MemoryDB backend container for cluster: {0}", clusterName);- String containerName = ContainerStorageHelper.resourceName(config, "memorydb", null, clusterName);+ String containerName = containerName(clusterName);lifecycleManager.removeIfExists(containerName);@@ -173,6 +173,28 @@ public class MemoryDbContainerManager {lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());}+ /**+ * Stops and removes the backend container for a cluster by name, if one exists.+ * Used by the service's provisioning rollback: {@link #start} registers the container in+ * {@code activeContainers} <em>before</em> {@link #waitForBackendReady}, so a readiness+ * timeout throws without ever returning the handle to the caller. In that case rollback+ * can't go through {@link #stop} (it has no handle), so it cleans up by name instead. Falls+ * back to the deterministic container name to catch a container that failed before it was+ * registered. Idempotent — a no-op when nothing is running for the name.+ */+ public void stopByClusterName(String clusterName) {+ MemoryDbContainerHandle handle = activeContainers.get(clusterName);+ if (handle != null) {+ stop(handle);+ return;+ }+ lifecycleManager.removeIfExists(containerName(clusterName));+ }++ private String containerName(String clusterName) {+ return ContainerStorageHelper.resourceName(config, "memorydb", null, clusterName);+ }+public void stopAll() {List<MemoryDbContainerHandle> handles = new ArrayList<>(activeContainers.values());if (!handles.isEmpty()) {diff --git a/src/test/java/io/github/hectorvent/floci/services/memorydb/MemoryDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/memorydb/MemoryDbServiceTest.javaindex 560532e74a..ebbc24e6a1 100644--- a/src/test/java/io/github/hectorvent/floci/services/memorydb/MemoryDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/memorydb/MemoryDbServiceTest.java@@ -21,6 +21,8 @@ import org.junit.jupiter.api.Test;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -30,8 +32,12 @@ import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyBoolean;import static org.mockito.ArgumentMatchers.anyInt;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.doNothing;+import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;@@ -39,13 +45,14 @@ class MemoryDbServiceTest {private MemoryDbService service;private MemoryDbContainerManager containerManager;+ private MemoryDbProxyManager proxyManager;private SigV4Validator sigV4Validator;private EmulatorConfig.MemoryDbServiceConfig mdbConfig;@BeforeEachvoid setUp() {containerManager = mock(MemoryDbContainerManager.class);- MemoryDbProxyManager proxyManager = mock(MemoryDbProxyManager.class);+ proxyManager = mock(MemoryDbProxyManager.class);sigV4Validator = mock(SigV4Validator.class);StorageFactory storageFactory = mock(StorageFactory.class);EmulatorConfig config = mock(EmulatorConfig.class);@@ -342,6 +349,74 @@ class MemoryDbServiceTest {assertEquals(ClusterStatus.AVAILABLE, created.getStatus());}+ @Test+ void failedProvisioningRollsBackContainerAndReleasesProxyPort() {+ MemoryDbContainerHandle handle = new MemoryDbContainerHandle("cid", "c1", "localhost", 6379);+ when(containerManager.start(anyString(), anyString())).thenReturn(handle);++ // Proxy startup blows up after the port is reserved and the container is started.+ doThrow(new RuntimeException("proxy boom"))+ .when(proxyManager).startProxy(eq("c1"), anyBoolean(), anyInt(), anyString(), anyInt(), any());++ Cluster spec = new Cluster();+ spec.setName("c1");+ spec.setAclName("open-access");++ // The original failure must propagate to the caller (we clean up, then rethrow).+ assertThrows(RuntimeException.class, () -> service.createCluster(spec, "us-east-1"));++ // Rollback stopped the proxy and the already-started container. Stopped by the exact+ // handle from this request, not by name: a concurrent create for the same cluster name+ // could have raced ahead and overwritten the registered container, and looking it up by+ // name here would risk stopping that other request's container instead of this one's.+ verify(proxyManager).stopProxy("c1");+ verify(containerManager).stop(handle);+ verify(containerManager, never()).stopByClusterName(anyString());++ // The reserved proxy port was released: a subsequent successful create reuses the base port+ // instead of skipping to the next one (which is what a leak would cause).+ doNothing().when(proxyManager)+ .startProxy(anyString(), anyBoolean(), anyInt(), anyString(), anyInt(), any());+ Cluster second = new Cluster();+ second.setName("c2");+ second.setAclName("open-access");+ Cluster recovered = service.createCluster(second, "us-east-1");+ assertEquals(16400, recovered.getProxyPort(),+ "Port from the failed create must be released so the next cluster reuses it");+ }++ @Test+ void failedContainerStartupCleansUpContainerByNameAndReleasesPort() {+ // containerManager.start(...) throws — this models both a container that never started+ // and (crucially) a readiness timeout, where start() created + registered the container+ // before throwing, so no handle ever reaches the service.+ doThrow(new RuntimeException("readiness boom"))+ .when(containerManager).start(eq("c1"), anyString());++ Cluster spec = new Cluster();+ spec.setName("c1");+ spec.setAclName("open-access");++ // The original failure must propagate to the caller (we clean up, then rethrow).+ assertThrows(RuntimeException.class, () -> service.createCluster(spec, "us-east-1"));++ // No handle returned, so the proxy never started and must not be stopped...+ verify(proxyManager, never()).stopProxy(anyString());+ // ...but the container must still be cleaned up by name, since start() may have created+ // and registered it before failing. Cleaning up by handle here would orphan it.+ verify(containerManager).stopByClusterName("c1");++ // The reserved proxy port was still released: a subsequent successful create reuses the base port.+ when(containerManager.start(anyString(), anyString()))+ .thenReturn(new MemoryDbContainerHandle("cid", "c2", "localhost", 6379));+ Cluster second = new Cluster();+ second.setName("c2");+ second.setAclName("open-access");+ Cluster recovered = service.createCluster(second, "us-east-1");+ assertEquals(16400, recovered.getProxyPort(),+ "Port from the failed create must be released so the next cluster reuses it");+ }+@Testvoid mockModeSkipsContainerAndReportsStandardPort() {when(mdbConfig.mock()).thenReturn(true);@@ -356,4 +431,36 @@ class MemoryDbServiceTest {assertEquals(6379, created.getClusterEndpoint().port());verifyNoInteractions(containerManager);}++ @Test+ void concurrentCreateForSameNameIsRejectedWhileFirstIsProvisioning() throws InterruptedException {+ CountDownLatch startedLatch = new CountDownLatch(1);+ CountDownLatch releaseLatch = new CountDownLatch(1);+ when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {+ startedLatch.countDown();+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");+ return new MemoryDbContainerHandle("cid", "c1", "localhost", 6379);+ });++ Cluster spec = new Cluster();+ spec.setName("c1");+ spec.setAclName("open-access");++ Thread firstRequest = new Thread(() -> service.createCluster(spec, "us-east-1"));+ firstRequest.start();+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "first request never reached container start");++ Cluster spec2 = new Cluster();+ spec2.setName("c1");+ spec2.setAclName("open-access");+ AwsException ex = assertThrows(AwsException.class, () -> service.createCluster(spec2, "us-east-1"));+ assertEquals("ClusterAlreadyExistsFault", ex.jsonType());+ verify(containerManager, never()).stop(any());+ verify(containerManager, never()).stopByClusterName(anyString());++ releaseLatch.countDown();+ firstRequest.join(5000);++ assertEquals(ClusterStatus.AVAILABLE, service.getCluster("c1").getStatus());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManagerTest.javanew file mode 100644index 0000000000..bed9d4ab6b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManagerTest.java@@ -0,0 +1,29 @@+package io.github.hectorvent.floci.services.memorydb.container;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import org.junit.jupiter.api.Test;++import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;++class MemoryDbContainerManagerTest {++ @Test+ void stopByClusterNameRemovesByDeterministicNameWhenNothingRegistered() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ MemoryDbContainerManager manager = new MemoryDbContainerManager(+ mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),+ mock(ContainerDetector.class), mock(EmulatorConfig.class), mock(RegionResolver.class));++ // No container was ever registered for this name (e.g. it failed before registration).+ // Rollback must still fall back to the deterministic name so nothing is orphaned.+ manager.stopByClusterName("my-cluster");++ verify(lifecycleManager).removeIfExists("floci-memorydb-my-cluster");+ }+}