Light, fluffy, and always free - The AWS Local Emulator alternative
fix(neptune): roll back proxy port and container on failed provisioning (#1742)
Ahmed khaled27f0124e3687
diff --git a/src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneService.java b/src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneService.javaindex fd64687bd6..dc0f70d342 100644--- a/src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneService.java+++ b/src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneService.java@@ -62,50 +62,93 @@ public class NeptuneService {"Neptune cluster " + id + " already exists.", 400);}+ // Open the try immediately after reserving the port so config reads below can't leak it.int proxyPort = allocateProxyPort();- String configuredDbType = config.services().neptune().dbType();- NeptuneDbType dbType = NeptuneDbType.fromConfig(configuredDbType).orElseGet(() -> {- LOG.warnv("Unsupported Neptune db-type ''{0}'', falling back to {1}. Supported: gremlin, neo4j.",- configuredDbType, NeptuneDbType.GREMLIN);- return NeptuneDbType.GREMLIN;- });- String image = switch (dbType) {- case GREMLIN -> config.services().neptune().defaultImage();- case NEO4J -> config.services().neptune().defaultNeo4jImage();- };-- LOG.infov("Creating Neptune cluster {0} on proxy port {1}, dbType={2}, image={3}",- id, String.valueOf(proxyPort), dbType, image);-- NeptuneContainerHandle handle = containerManager.start(id, image, dbType);-- String region = regionResolver.getDefaultRegion();- String endpointHost = resolveEndpointHost();-- NeptuneCluster cluster = new NeptuneCluster();- cluster.setDbClusterIdentifier(id);- cluster.setStatus("available");- cluster.setEngineVersion(engineVersion != null ? engineVersion : ENGINE_VERSION_DEFAULT);- cluster.setEndpoint(endpointHost);- cluster.setReaderEndpoint(endpointHost);- cluster.setPort(proxyPort);- cluster.setIamDatabaseAuthenticationEnabled(iamEnabled);- cluster.setDbClusterArn(regionResolver.buildArn("neptune", region, "cluster:" + id));- cluster.setDbClusterResourceId("cluster-" + UUID.randomUUID().toString()- .replace("-", "").substring(0, 24).toUpperCase());- cluster.setCreatedAt(Instant.now());- cluster.setDbClusterMembers(new ArrayList<>());- cluster.setContainerId(handle.getContainerId());- cluster.setContainerHost(handle.getHost());- cluster.setContainerPort(handle.getPort());- cluster.setProxyPort(proxyPort);-- proxyManager.startProxy(id, proxyPort, handle.getHost(), handle.getPort());+ NeptuneContainerHandle handle = null;+ boolean provisioned = false;+ try {+ String configuredDbType = config.services().neptune().dbType();+ NeptuneDbType dbType = NeptuneDbType.fromConfig(configuredDbType).orElseGet(() -> {+ LOG.warnv("Unsupported Neptune db-type ''{0}'', falling back to {1}. Supported: gremlin, neo4j.",+ configuredDbType, NeptuneDbType.GREMLIN);+ return NeptuneDbType.GREMLIN;+ });+ String image = switch (dbType) {+ case GREMLIN -> config.services().neptune().defaultImage();+ case NEO4J -> config.services().neptune().defaultNeo4jImage();+ };++ LOG.infov("Creating Neptune cluster {0} on proxy port {1}, dbType={2}, image={3}",+ id, String.valueOf(proxyPort), dbType, image);++ handle = containerManager.start(id, image, dbType);++ String region = regionResolver.getDefaultRegion();+ String endpointHost = resolveEndpointHost();++ NeptuneCluster cluster = new NeptuneCluster();+ cluster.setDbClusterIdentifier(id);+ cluster.setStatus("available");+ cluster.setEngineVersion(engineVersion != null ? engineVersion : ENGINE_VERSION_DEFAULT);+ cluster.setEndpoint(endpointHost);+ cluster.setReaderEndpoint(endpointHost);+ cluster.setPort(proxyPort);+ cluster.setIamDatabaseAuthenticationEnabled(iamEnabled);+ cluster.setDbClusterArn(regionResolver.buildArn("neptune", region, "cluster:" + id));+ cluster.setDbClusterResourceId("cluster-" + UUID.randomUUID().toString()+ .replace("-", "").substring(0, 24).toUpperCase());+ cluster.setCreatedAt(Instant.now());+ cluster.setDbClusterMembers(new ArrayList<>());+ cluster.setContainerId(handle.getContainerId());+ cluster.setContainerHost(handle.getHost());+ cluster.setContainerPort(handle.getPort());+ cluster.setProxyPort(proxyPort);++ proxyManager.startProxy(id, proxyPort, handle.getHost(), handle.getPort());++ clusters.put(id, cluster);+ provisioned = true;+ LOG.infov("Neptune cluster {0} created ({1}), endpoint={2}:{3}",+ id, dbType, endpointHost, String.valueOf(proxyPort));+ return cluster;+ } catch (RuntimeException e) {+ LOG.warnv("Neptune cluster {0} provisioning failed, rolling back: {1}", id, e.getMessage());+ throw e;+ } finally {+ // Roll back on ANY non-success exit — including a JVM Error, which a+ // catch (RuntimeException) would miss — so a failed create never leaks the+ // reserved port or leaves a container behind. Idempotent and a no-op on success.+ if (!provisioned) {+ rollbackDbCluster(id, handle, proxyPort);+ }+ }+ }- clusters.put(id, cluster);- LOG.infov("Neptune cluster {0} created ({1}), endpoint={2}:{3}",- id, dbType, endpointHost, String.valueOf(proxyPort));- return cluster;+ private void rollbackDbCluster(String id, NeptuneContainerHandle handle, int proxyPort) {+ try {+ try {+ // The proxy only starts after the container is ready, so a null handle means it+ // never started — nothing to stop.+ if (handle != null) {+ proxyManager.stopProxy(id);+ }+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping proxy for Neptune cluster {0}: {1}", id, e.getMessage());+ }+ try {+ // Stop by id, not handle: a readiness timeout in containerManager.start() throws+ // after the container was created and registered but before the handle is returned,+ // so cleaning up by handle here would miss (and orphan) it. stopByClusterId is+ // idempotent, so it's safe when the container never started.+ containerManager.stopByClusterId(id);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container for Neptune cluster {0}: {1}", id, e.getMessage());+ }+ } finally {+ // Always release the port — even if a cleanup step throws a non-RuntimeException+ // (e.g. an Error) — since leaking the port is the exact failure this rollback prevents.+ releaseProxyPort(proxyPort);+ }}public NeptuneCluster getDbCluster(String id) {diff --git a/src/main/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManager.javaindex e69bd130e0..01ef2f9cc0 100644--- a/src/main/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManager.java@@ -66,7 +66,7 @@ public class NeptuneContainerManager {LOG.infov("Starting Neptune backend container ({0}) for cluster: {1}", dbType, clusterId);int backendPort = dbType.backendPort();- String containerName = ContainerStorageHelper.resourceName(config, "neptune", null, clusterId);+ String containerName = containerName(clusterId);lifecycleManager.removeIfExists(containerName);ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)@@ -128,6 +128,28 @@ public class NeptuneContainerManager {lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());}+ /**+ * Stops and removes the backend container for a cluster by id, if one exists.+ * Used by the service's provisioning rollback: {@link #start} registers the container in+ * {@code activeContainers} <em>before</em> {@link #waitForBackendReady}, so a readiness+ * timeout throws without ever returning the handle to the caller. In that case rollback+ * can't go through {@link #stop} (it has no handle), so it cleans up by id instead. Falls+ * back to the deterministic container name to catch a container that failed before it was+ * registered. Idempotent — a no-op when nothing is running for the id.+ */+ public void stopByClusterId(String clusterId) {+ NeptuneContainerHandle handle = activeContainers.get(clusterId);+ if (handle != null) {+ stop(handle);+ return;+ }+ lifecycleManager.removeIfExists(containerName(clusterId));+ }++ private String containerName(String clusterId) {+ return ContainerStorageHelper.resourceName(config, "neptune", null, clusterId);+ }+public void stopAll() {List<NeptuneContainerHandle> handles = new ArrayList<>(activeContainers.values());if (!handles.isEmpty()) {diff --git a/src/test/java/io/github/hectorvent/floci/services/neptune/NeptuneServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/neptune/NeptuneServiceTest.javanew file mode 100644index 0000000000..b0173c663b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/neptune/NeptuneServiceTest.java@@ -0,0 +1,143 @@+package io.github.hectorvent.floci.services.neptune;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.neptune.container.NeptuneContainerHandle;+import io.github.hectorvent.floci.services.neptune.container.NeptuneContainerManager;+import io.github.hectorvent.floci.services.neptune.model.NeptuneCluster;+import io.github.hectorvent.floci.services.neptune.model.NeptuneDbType;+import io.github.hectorvent.floci.services.neptune.proxy.NeptuneProxyManager;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.Optional;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doNothing;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class NeptuneServiceTest {++ private NeptuneService service;+ private NeptuneContainerManager containerManager;+ private NeptuneProxyManager proxyManager;++ @BeforeEach+ void setUp() {+ containerManager = mock(NeptuneContainerManager.class);+ proxyManager = mock(NeptuneProxyManager.class);+ StorageFactory storageFactory = mock(StorageFactory.class);+ EmulatorConfig config = mock(EmulatorConfig.class);+ RegionResolver regionResolver = mock(RegionResolver.class);++ EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.NeptuneServiceConfig neptuneConfig = mock(EmulatorConfig.NeptuneServiceConfig.class);+ when(config.services()).thenReturn(servicesConfig);+ when(servicesConfig.neptune()).thenReturn(neptuneConfig);+ when(neptuneConfig.proxyBasePort()).thenReturn(18182);+ when(neptuneConfig.proxyMaxPort()).thenReturn(18199);+ when(neptuneConfig.dbType()).thenReturn("gremlin");+ when(neptuneConfig.defaultImage()).thenReturn("tinkerpop/gremlin-server:3.7");+ when(neptuneConfig.defaultNeo4jImage()).thenReturn("neo4j:5");+ when(config.hostname()).thenReturn(Optional.of("localhost"));++ when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");+ when(regionResolver.buildArn(anyString(), anyString(), anyString()))+ .thenReturn("arn:aws:neptune:us-east-1:000000000000:cluster:c");++ when(storageFactory.create(anyString(), anyString(), any()))+ .thenAnswer(inv -> new InMemoryStorage<>());+ when(containerManager.start(anyString(), anyString(), any(NeptuneDbType.class)))+ .thenReturn(new NeptuneContainerHandle("cid", "c", "localhost", 8182));+ doNothing().when(proxyManager).startProxy(anyString(), anyInt(), anyString(), anyInt());++ service = new NeptuneService(config, regionResolver, containerManager, proxyManager, storageFactory);+ }++ @Test+ void failedProvisioningRollsBackContainerAndReleasesProxyPort() {+ NeptuneContainerHandle handle = new NeptuneContainerHandle("cid", "c", "localhost", 8182);+ when(containerManager.start(anyString(), anyString(), any(NeptuneDbType.class)))+ .thenReturn(handle);++ // Proxy startup blows up after the port is reserved and the container is started.+ doThrow(new RuntimeException("proxy boom"))+ .when(proxyManager).startProxy(eq("c"), anyInt(), anyString(), anyInt());++ // The original failure must propagate to the caller (we clean up, then rethrow).+ assertThrows(RuntimeException.class,+ () -> service.createDbCluster("c", "1.3.2.1", false));++ // Rollback stopped the proxy and the already-started container (by id).+ verify(proxyManager).stopProxy("c");+ verify(containerManager).stopByClusterId("c");++ // The reserved proxy port was released: a subsequent successful create reuses the base port+ // instead of skipping to the next one (which is what a leak would cause).+ doNothing().when(proxyManager).startProxy(anyString(), anyInt(), anyString(), anyInt());+ NeptuneCluster recovered = service.createDbCluster("c2", "1.3.2.1", false);+ assertEquals(18182, recovered.getProxyPort(),+ "Port from the failed create must be released so the next cluster reuses it");+ }++ @Test+ void jvmErrorDuringProvisioningStillRollsBack() {+ NeptuneContainerHandle handle = new NeptuneContainerHandle("cid", "c", "localhost", 8182);+ when(containerManager.start(anyString(), anyString(), any(NeptuneDbType.class)))+ .thenReturn(handle);++ // A JVM Error (not a RuntimeException) escapes provisioning — a catch (RuntimeException)+ // would miss it, so rollback must run from a finally instead.+ doThrow(new StackOverflowError("boom"))+ .when(proxyManager).startProxy(eq("c"), anyInt(), anyString(), anyInt());++ assertThrows(StackOverflowError.class,+ () -> service.createDbCluster("c", "1.3.2.1", false));++ // Rollback still fired despite the Error: proxy and container stopped, port released.+ verify(proxyManager).stopProxy("c");+ verify(containerManager).stopByClusterId("c");++ doNothing().when(proxyManager).startProxy(anyString(), anyInt(), anyString(), anyInt());+ NeptuneCluster recovered = service.createDbCluster("c2", "1.3.2.1", false);+ assertEquals(18182, recovered.getProxyPort(),+ "Port from the failed create must be released even when the failure is an Error");+ }++ @Test+ void failedContainerStartupCleansUpContainerByIdAndReleasesPort() {+ // containerManager.start(...) throws — this models both a container that never started+ // and (crucially) a readiness timeout, where start() created + registered the container+ // before throwing, so no handle ever reaches the service.+ doThrow(new RuntimeException("readiness boom"))+ .when(containerManager).start(eq("c"), anyString(), any(NeptuneDbType.class));++ // The original failure must propagate to the caller (we clean up, then rethrow).+ assertThrows(RuntimeException.class,+ () -> service.createDbCluster("c", "1.3.2.1", false));++ // No handle returned, so the proxy never started and must not be stopped...+ verify(proxyManager, never()).stopProxy(anyString());+ // ...but the container must still be cleaned up by id, since start() may have created and+ // registered it before failing. Cleaning up by handle here would orphan it.+ verify(containerManager).stopByClusterId("c");++ // The reserved proxy port was still released: a subsequent successful create reuses the base port.+ when(containerManager.start(anyString(), anyString(), any(NeptuneDbType.class)))+ .thenReturn(new NeptuneContainerHandle("cid", "c2", "localhost", 8182));+ NeptuneCluster recovered = service.createDbCluster("c2", "1.3.2.1", false);+ assertEquals(18182, recovered.getProxyPort(),+ "Port from the failed create must be released so the next cluster reuses it");+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManagerTest.javanew file mode 100644index 0000000000..a14eb186cf--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManagerTest.java@@ -0,0 +1,29 @@+package io.github.hectorvent.floci.services.neptune.container;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import org.junit.jupiter.api.Test;++import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;++class NeptuneContainerManagerTest {++ @Test+ void stopByClusterIdRemovesByDeterministicNameWhenNothingRegistered() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ NeptuneContainerManager manager = new NeptuneContainerManager(+ mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),+ mock(ContainerDetector.class), mock(EmulatorConfig.class), mock(RegionResolver.class));++ // No container was ever registered for this id (e.g. it failed before registration).+ // Rollback must still fall back to the deterministic name so nothing is orphaned.+ manager.stopByClusterId("my-cluster");++ verify(lifecycleManager).removeIfExists("floci-neptune-my-cluster");+ }+}