Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(memorydb): clean up container by id on readiness-timeout rollback (#2092)

Ahmed khaled11d527547af8
diff --git a/src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbService.java b/src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbService.java
index f1101640da..08e782c776 100644
--- a/src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbService.java
@@ -65,6 +65,7 @@ public class MemoryDbService {
private final EmulatorConfig config;
private final RegionResolver regionResolver;
private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();
+ private final Set<String> provisioningClusterNames = ConcurrentHashMap.newKeySet();
@Inject
public MemoryDbService(MemoryDbContainerManager containerManager,
@@ -97,40 +98,50 @@ public class MemoryDbService {
throw new AwsException("ClusterAlreadyExistsFault",
"Cluster with specified name already exists.", 400);
}
-
- String aclName = spec.getAclName();
- if (aclName == null || aclName.isBlank()) {
- throw new AwsException("InvalidParameterValueException", "ACLName is required.", 400);
- }
- requireAclExists(aclName);
- boolean authRequired = isAuthRequired(aclName);
-
- Cluster cluster = new Cluster();
- cluster.setName(name);
- cluster.setDescription(spec.getDescription());
- cluster.setStatus(ClusterStatus.AVAILABLE);
- cluster.setNodeType(spec.getNodeType() != null ? spec.getNodeType() : "db.t4g.small");
- cluster.setNumberOfShards(spec.getNumberOfShards() > 0 ? spec.getNumberOfShards() : 1);
- cluster.setEngine(spec.getEngine() != null ? spec.getEngine() : DEFAULT_ENGINE);
- cluster.setEngineVersion(spec.getEngineVersion() != null ? spec.getEngineVersion() : DEFAULT_ENGINE_VERSION);
- cluster.setAclName(aclName);
- cluster.setTlsEnabled(spec.isTlsEnabled());
- cluster.setArn(buildArn(region, "cluster", name));
- cluster.setCreatedAt(Instant.now());
- cluster.setTags(spec.getTags());
-
- if (config.services().memorydb().mock()) {
- LOG.infov("Creating MemoryDB cluster {0} in mock mode (no container)", name);
- cluster.setClusterEndpoint(new Endpoint(resolveEndpointHost(), REDIS_PORT));
- } else {
- startBackend(cluster, authRequired);
+ // Claim the name for the whole provisioning attempt so a concurrent create can't race
+ // ahead and be stopped by this request's handle-less rollback fallback.
+ if (!provisioningClusterNames.add(name)) {
+ throw new AwsException("ClusterAlreadyExistsFault",
+ "Cluster " + name + " is already being created.", 400);
}
- clusters.put(name, cluster);
- LOG.infov("MemoryDB cluster {0} created (acl={1}, authRequired={2}), endpoint={3}:{4}",
- name, aclName, String.valueOf(authRequired), cluster.getClusterEndpoint().address(),
- String.valueOf(cluster.getClusterEndpoint().port()));
- return cluster;
+ try {
+ String aclName = spec.getAclName();
+ if (aclName == null || aclName.isBlank()) {
+ throw new AwsException("InvalidParameterValueException", "ACLName is required.", 400);
+ }
+ requireAclExists(aclName);
+ boolean authRequired = isAuthRequired(aclName);
+
+ Cluster cluster = new Cluster();
+ cluster.setName(name);
+ cluster.setDescription(spec.getDescription());
+ cluster.setStatus(ClusterStatus.AVAILABLE);
+ cluster.setNodeType(spec.getNodeType() != null ? spec.getNodeType() : "db.t4g.small");
+ cluster.setNumberOfShards(spec.getNumberOfShards() > 0 ? spec.getNumberOfShards() : 1);
+ cluster.setEngine(spec.getEngine() != null ? spec.getEngine() : DEFAULT_ENGINE);
+ cluster.setEngineVersion(spec.getEngineVersion() != null ? spec.getEngineVersion() : DEFAULT_ENGINE_VERSION);
+ cluster.setAclName(aclName);
+ cluster.setTlsEnabled(spec.isTlsEnabled());
+ cluster.setArn(buildArn(region, "cluster", name));
+ cluster.setCreatedAt(Instant.now());
+ cluster.setTags(spec.getTags());
+
+ if (config.services().memorydb().mock()) {
+ LOG.infov("Creating MemoryDB cluster {0} in mock mode (no container)", name);
+ cluster.setClusterEndpoint(new Endpoint(resolveEndpointHost(), REDIS_PORT));
+ } else {
+ startBackend(cluster, authRequired);
+ }
+
+ clusters.put(name, cluster);
+ LOG.infov("MemoryDB cluster {0} created (acl={1}, authRequired={2}), endpoint={3}:{4}",
+ name, aclName, String.valueOf(authRequired), cluster.getClusterEndpoint().address(),
+ String.valueOf(cluster.getClusterEndpoint().port()));
+ return cluster;
+ } finally {
+ provisioningClusterNames.remove(name);
+ }
}
public Cluster getCluster(String name) {
@@ -496,12 +507,33 @@ public class MemoryDbService {
private void rollbackBackend(String name, MemoryDbContainerHandle handle, int proxyPort) {
try {
- proxyManager.stopProxy(name);
- if (handle != null) {
- containerManager.stop(handle);
+ try {
+ // The proxy only starts after the container is ready, so a null handle means it
+ // never started — nothing to stop.
+ if (handle != null) {
+ proxyManager.stopProxy(name);
+ }
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping proxy for MemoryDB cluster {0}: {1}", name, e.getMessage());
+ }
+ try {
+ if (handle != null) {
+ // We have the exact handle from this request's start() call, so stop by it
+ // directly. Falling back to stopByClusterName here instead would look up
+ // whatever is currently registered for name, which could be a different
+ // container if an overlapping create for the same name raced ahead of this
+ // rollback.
+ containerManager.stop(handle);
+ } else {
+ // No handle: a readiness timeout in containerManager.start() throws after the
+ // container was created and registered but before the handle is returned, so
+ // cleaning up by handle here isn't possible. stopByClusterName is idempotent,
+ // so it's safe when the container never started.
+ containerManager.stopByClusterName(name);
+ }
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping container for MemoryDB cluster {0}: {1}", name, e.getMessage());
}
- } catch (RuntimeException e) {
- LOG.warnv("Error rolling back MemoryDB cluster {0}: {1}", name, e.getMessage());
} finally {
releaseProxyPort(proxyPort);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManager.java
index 0e7be069dc..60d207b26e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManager.java
@@ -68,7 +68,7 @@ public class MemoryDbContainerManager {
public MemoryDbContainerHandle start(String clusterName, String image) {
LOG.infov("Starting MemoryDB backend container for cluster: {0}", clusterName);
- String containerName = ContainerStorageHelper.resourceName(config, "memorydb", null, clusterName);
+ String containerName = containerName(clusterName);
lifecycleManager.removeIfExists(containerName);
@@ -173,6 +173,28 @@ public class MemoryDbContainerManager {
lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());
}
+ /**
+ * Stops and removes the backend container for a cluster by name, if one exists.
+ * Used by the service's provisioning rollback: {@link #start} registers the container in
+ * {@code activeContainers} <em>before</em> {@link #waitForBackendReady}, so a readiness
+ * timeout throws without ever returning the handle to the caller. In that case rollback
+ * can't go through {@link #stop} (it has no handle), so it cleans up by name instead. Falls
+ * back to the deterministic container name to catch a container that failed before it was
+ * registered. Idempotent — a no-op when nothing is running for the name.
+ */
+ public void stopByClusterName(String clusterName) {
+ MemoryDbContainerHandle handle = activeContainers.get(clusterName);
+ if (handle != null) {
+ stop(handle);
+ return;
+ }
+ lifecycleManager.removeIfExists(containerName(clusterName));
+ }
+
+ private String containerName(String clusterName) {
+ return ContainerStorageHelper.resourceName(config, "memorydb", null, clusterName);
+ }
+
public void stopAll() {
List<MemoryDbContainerHandle> handles = new ArrayList<>(activeContainers.values());
if (!handles.isEmpty()) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/memorydb/MemoryDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/memorydb/MemoryDbServiceTest.java
index 560532e74a..ebbc24e6a1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/memorydb/MemoryDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/memorydb/MemoryDbServiceTest.java
@@ -21,6 +21,8 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -30,8 +32,12 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@@ -39,13 +45,14 @@ class MemoryDbServiceTest {
private MemoryDbService service;
private MemoryDbContainerManager containerManager;
+ private MemoryDbProxyManager proxyManager;
private SigV4Validator sigV4Validator;
private EmulatorConfig.MemoryDbServiceConfig mdbConfig;
@BeforeEach
void setUp() {
containerManager = mock(MemoryDbContainerManager.class);
- MemoryDbProxyManager proxyManager = mock(MemoryDbProxyManager.class);
+ proxyManager = mock(MemoryDbProxyManager.class);
sigV4Validator = mock(SigV4Validator.class);
StorageFactory storageFactory = mock(StorageFactory.class);
EmulatorConfig config = mock(EmulatorConfig.class);
@@ -342,6 +349,74 @@ class MemoryDbServiceTest {
assertEquals(ClusterStatus.AVAILABLE, created.getStatus());
}
+ @Test
+ void failedProvisioningRollsBackContainerAndReleasesProxyPort() {
+ MemoryDbContainerHandle handle = new MemoryDbContainerHandle("cid", "c1", "localhost", 6379);
+ when(containerManager.start(anyString(), anyString())).thenReturn(handle);
+
+ // Proxy startup blows up after the port is reserved and the container is started.
+ doThrow(new RuntimeException("proxy boom"))
+ .when(proxyManager).startProxy(eq("c1"), anyBoolean(), anyInt(), anyString(), anyInt(), any());
+
+ Cluster spec = new Cluster();
+ spec.setName("c1");
+ spec.setAclName("open-access");
+
+ // The original failure must propagate to the caller (we clean up, then rethrow).
+ assertThrows(RuntimeException.class, () -> service.createCluster(spec, "us-east-1"));
+
+ // Rollback stopped the proxy and the already-started container. Stopped by the exact
+ // handle from this request, not by name: a concurrent create for the same cluster name
+ // could have raced ahead and overwritten the registered container, and looking it up by
+ // name here would risk stopping that other request's container instead of this one's.
+ verify(proxyManager).stopProxy("c1");
+ verify(containerManager).stop(handle);
+ verify(containerManager, never()).stopByClusterName(anyString());
+
+ // The reserved proxy port was released: a subsequent successful create reuses the base port
+ // instead of skipping to the next one (which is what a leak would cause).
+ doNothing().when(proxyManager)
+ .startProxy(anyString(), anyBoolean(), anyInt(), anyString(), anyInt(), any());
+ Cluster second = new Cluster();
+ second.setName("c2");
+ second.setAclName("open-access");
+ Cluster recovered = service.createCluster(second, "us-east-1");
+ assertEquals(16400, recovered.getProxyPort(),
+ "Port from the failed create must be released so the next cluster reuses it");
+ }
+
+ @Test
+ void failedContainerStartupCleansUpContainerByNameAndReleasesPort() {
+ // containerManager.start(...) throws — this models both a container that never started
+ // and (crucially) a readiness timeout, where start() created + registered the container
+ // before throwing, so no handle ever reaches the service.
+ doThrow(new RuntimeException("readiness boom"))
+ .when(containerManager).start(eq("c1"), anyString());
+
+ Cluster spec = new Cluster();
+ spec.setName("c1");
+ spec.setAclName("open-access");
+
+ // The original failure must propagate to the caller (we clean up, then rethrow).
+ assertThrows(RuntimeException.class, () -> service.createCluster(spec, "us-east-1"));
+
+ // No handle returned, so the proxy never started and must not be stopped...
+ verify(proxyManager, never()).stopProxy(anyString());
+ // ...but the container must still be cleaned up by name, since start() may have created
+ // and registered it before failing. Cleaning up by handle here would orphan it.
+ verify(containerManager).stopByClusterName("c1");
+
+ // The reserved proxy port was still released: a subsequent successful create reuses the base port.
+ when(containerManager.start(anyString(), anyString()))
+ .thenReturn(new MemoryDbContainerHandle("cid", "c2", "localhost", 6379));
+ Cluster second = new Cluster();
+ second.setName("c2");
+ second.setAclName("open-access");
+ Cluster recovered = service.createCluster(second, "us-east-1");
+ assertEquals(16400, recovered.getProxyPort(),
+ "Port from the failed create must be released so the next cluster reuses it");
+ }
+
@Test
void mockModeSkipsContainerAndReportsStandardPort() {
when(mdbConfig.mock()).thenReturn(true);
@@ -356,4 +431,36 @@ class MemoryDbServiceTest {
assertEquals(6379, created.getClusterEndpoint().port());
verifyNoInteractions(containerManager);
}
+
+ @Test
+ void concurrentCreateForSameNameIsRejectedWhileFirstIsProvisioning() throws InterruptedException {
+ CountDownLatch startedLatch = new CountDownLatch(1);
+ CountDownLatch releaseLatch = new CountDownLatch(1);
+ when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {
+ startedLatch.countDown();
+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");
+ return new MemoryDbContainerHandle("cid", "c1", "localhost", 6379);
+ });
+
+ Cluster spec = new Cluster();
+ spec.setName("c1");
+ spec.setAclName("open-access");
+
+ Thread firstRequest = new Thread(() -> service.createCluster(spec, "us-east-1"));
+ firstRequest.start();
+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "first request never reached container start");
+
+ Cluster spec2 = new Cluster();
+ spec2.setName("c1");
+ spec2.setAclName("open-access");
+ AwsException ex = assertThrows(AwsException.class, () -> service.createCluster(spec2, "us-east-1"));
+ assertEquals("ClusterAlreadyExistsFault", ex.jsonType());
+ verify(containerManager, never()).stop(any());
+ verify(containerManager, never()).stopByClusterName(anyString());
+
+ releaseLatch.countDown();
+ firstRequest.join(5000);
+
+ assertEquals(ClusterStatus.AVAILABLE, service.getCluster("c1").getStatus());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManagerTest.java
new file mode 100644
index 0000000000..bed9d4ab6b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/memorydb/container/MemoryDbContainerManagerTest.java
@@ -0,0 +1,29 @@
+package io.github.hectorvent.floci.services.memorydb.container;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import org.junit.jupiter.api.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+class MemoryDbContainerManagerTest {
+
+ @Test
+ void stopByClusterNameRemovesByDeterministicNameWhenNothingRegistered() {
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ MemoryDbContainerManager manager = new MemoryDbContainerManager(
+ mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),
+ mock(ContainerDetector.class), mock(EmulatorConfig.class), mock(RegionResolver.class));
+
+ // No container was ever registered for this name (e.g. it failed before registration).
+ // Rollback must still fall back to the deterministic name so nothing is orphaned.
+ manager.stopByClusterName("my-cluster");
+
+ verify(lifecycleManager).removeIfExists("floci-memorydb-my-cluster");
+ }
+}