Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(neptune): roll back proxy port and container on failed provisioning (#1742)

Ahmed khaled27f0124e3687
diff --git a/src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneService.java b/src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneService.java
index fd64687bd6..dc0f70d342 100644
--- a/src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneService.java
@@ -62,50 +62,93 @@ public class NeptuneService {
"Neptune cluster " + id + " already exists.", 400);
}
+ // Open the try immediately after reserving the port so config reads below can't leak it.
int proxyPort = allocateProxyPort();
- String configuredDbType = config.services().neptune().dbType();
- NeptuneDbType dbType = NeptuneDbType.fromConfig(configuredDbType).orElseGet(() -> {
- LOG.warnv("Unsupported Neptune db-type ''{0}'', falling back to {1}. Supported: gremlin, neo4j.",
- configuredDbType, NeptuneDbType.GREMLIN);
- return NeptuneDbType.GREMLIN;
- });
- String image = switch (dbType) {
- case GREMLIN -> config.services().neptune().defaultImage();
- case NEO4J -> config.services().neptune().defaultNeo4jImage();
- };
-
- LOG.infov("Creating Neptune cluster {0} on proxy port {1}, dbType={2}, image={3}",
- id, String.valueOf(proxyPort), dbType, image);
-
- NeptuneContainerHandle handle = containerManager.start(id, image, dbType);
-
- String region = regionResolver.getDefaultRegion();
- String endpointHost = resolveEndpointHost();
-
- NeptuneCluster cluster = new NeptuneCluster();
- cluster.setDbClusterIdentifier(id);
- cluster.setStatus("available");
- cluster.setEngineVersion(engineVersion != null ? engineVersion : ENGINE_VERSION_DEFAULT);
- cluster.setEndpoint(endpointHost);
- cluster.setReaderEndpoint(endpointHost);
- cluster.setPort(proxyPort);
- cluster.setIamDatabaseAuthenticationEnabled(iamEnabled);
- cluster.setDbClusterArn(regionResolver.buildArn("neptune", region, "cluster:" + id));
- cluster.setDbClusterResourceId("cluster-" + UUID.randomUUID().toString()
- .replace("-", "").substring(0, 24).toUpperCase());
- cluster.setCreatedAt(Instant.now());
- cluster.setDbClusterMembers(new ArrayList<>());
- cluster.setContainerId(handle.getContainerId());
- cluster.setContainerHost(handle.getHost());
- cluster.setContainerPort(handle.getPort());
- cluster.setProxyPort(proxyPort);
-
- proxyManager.startProxy(id, proxyPort, handle.getHost(), handle.getPort());
+ NeptuneContainerHandle handle = null;
+ boolean provisioned = false;
+ try {
+ String configuredDbType = config.services().neptune().dbType();
+ NeptuneDbType dbType = NeptuneDbType.fromConfig(configuredDbType).orElseGet(() -> {
+ LOG.warnv("Unsupported Neptune db-type ''{0}'', falling back to {1}. Supported: gremlin, neo4j.",
+ configuredDbType, NeptuneDbType.GREMLIN);
+ return NeptuneDbType.GREMLIN;
+ });
+ String image = switch (dbType) {
+ case GREMLIN -> config.services().neptune().defaultImage();
+ case NEO4J -> config.services().neptune().defaultNeo4jImage();
+ };
+
+ LOG.infov("Creating Neptune cluster {0} on proxy port {1}, dbType={2}, image={3}",
+ id, String.valueOf(proxyPort), dbType, image);
+
+ handle = containerManager.start(id, image, dbType);
+
+ String region = regionResolver.getDefaultRegion();
+ String endpointHost = resolveEndpointHost();
+
+ NeptuneCluster cluster = new NeptuneCluster();
+ cluster.setDbClusterIdentifier(id);
+ cluster.setStatus("available");
+ cluster.setEngineVersion(engineVersion != null ? engineVersion : ENGINE_VERSION_DEFAULT);
+ cluster.setEndpoint(endpointHost);
+ cluster.setReaderEndpoint(endpointHost);
+ cluster.setPort(proxyPort);
+ cluster.setIamDatabaseAuthenticationEnabled(iamEnabled);
+ cluster.setDbClusterArn(regionResolver.buildArn("neptune", region, "cluster:" + id));
+ cluster.setDbClusterResourceId("cluster-" + UUID.randomUUID().toString()
+ .replace("-", "").substring(0, 24).toUpperCase());
+ cluster.setCreatedAt(Instant.now());
+ cluster.setDbClusterMembers(new ArrayList<>());
+ cluster.setContainerId(handle.getContainerId());
+ cluster.setContainerHost(handle.getHost());
+ cluster.setContainerPort(handle.getPort());
+ cluster.setProxyPort(proxyPort);
+
+ proxyManager.startProxy(id, proxyPort, handle.getHost(), handle.getPort());
+
+ clusters.put(id, cluster);
+ provisioned = true;
+ LOG.infov("Neptune cluster {0} created ({1}), endpoint={2}:{3}",
+ id, dbType, endpointHost, String.valueOf(proxyPort));
+ return cluster;
+ } catch (RuntimeException e) {
+ LOG.warnv("Neptune cluster {0} provisioning failed, rolling back: {1}", id, e.getMessage());
+ throw e;
+ } finally {
+ // Roll back on ANY non-success exit — including a JVM Error, which a
+ // catch (RuntimeException) would miss — so a failed create never leaks the
+ // reserved port or leaves a container behind. Idempotent and a no-op on success.
+ if (!provisioned) {
+ rollbackDbCluster(id, handle, proxyPort);
+ }
+ }
+ }
- clusters.put(id, cluster);
- LOG.infov("Neptune cluster {0} created ({1}), endpoint={2}:{3}",
- id, dbType, endpointHost, String.valueOf(proxyPort));
- return cluster;
+ private void rollbackDbCluster(String id, NeptuneContainerHandle handle, int proxyPort) {
+ try {
+ try {
+ // The proxy only starts after the container is ready, so a null handle means it
+ // never started — nothing to stop.
+ if (handle != null) {
+ proxyManager.stopProxy(id);
+ }
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping proxy for Neptune cluster {0}: {1}", id, e.getMessage());
+ }
+ try {
+ // Stop by id, not handle: a readiness timeout in containerManager.start() throws
+ // after the container was created and registered but before the handle is returned,
+ // so cleaning up by handle here would miss (and orphan) it. stopByClusterId is
+ // idempotent, so it's safe when the container never started.
+ containerManager.stopByClusterId(id);
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping container for Neptune cluster {0}: {1}", id, e.getMessage());
+ }
+ } finally {
+ // Always release the port — even if a cleanup step throws a non-RuntimeException
+ // (e.g. an Error) — since leaking the port is the exact failure this rollback prevents.
+ releaseProxyPort(proxyPort);
+ }
}
public NeptuneCluster getDbCluster(String id) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManager.java
index e69bd130e0..01ef2f9cc0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManager.java
@@ -66,7 +66,7 @@ public class NeptuneContainerManager {
LOG.infov("Starting Neptune backend container ({0}) for cluster: {1}", dbType, clusterId);
int backendPort = dbType.backendPort();
- String containerName = ContainerStorageHelper.resourceName(config, "neptune", null, clusterId);
+ String containerName = containerName(clusterId);
lifecycleManager.removeIfExists(containerName);
ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)
@@ -128,6 +128,28 @@ public class NeptuneContainerManager {
lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());
}
+ /**
+ * Stops and removes the backend container for a cluster by id, if one exists.
+ * Used by the service's provisioning rollback: {@link #start} registers the container in
+ * {@code activeContainers} <em>before</em> {@link #waitForBackendReady}, so a readiness
+ * timeout throws without ever returning the handle to the caller. In that case rollback
+ * can't go through {@link #stop} (it has no handle), so it cleans up by id instead. Falls
+ * back to the deterministic container name to catch a container that failed before it was
+ * registered. Idempotent — a no-op when nothing is running for the id.
+ */
+ public void stopByClusterId(String clusterId) {
+ NeptuneContainerHandle handle = activeContainers.get(clusterId);
+ if (handle != null) {
+ stop(handle);
+ return;
+ }
+ lifecycleManager.removeIfExists(containerName(clusterId));
+ }
+
+ private String containerName(String clusterId) {
+ return ContainerStorageHelper.resourceName(config, "neptune", null, clusterId);
+ }
+
public void stopAll() {
List<NeptuneContainerHandle> handles = new ArrayList<>(activeContainers.values());
if (!handles.isEmpty()) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/neptune/NeptuneServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/neptune/NeptuneServiceTest.java
new file mode 100644
index 0000000000..b0173c663b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/neptune/NeptuneServiceTest.java
@@ -0,0 +1,143 @@
+package io.github.hectorvent.floci.services.neptune;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.neptune.container.NeptuneContainerHandle;
+import io.github.hectorvent.floci.services.neptune.container.NeptuneContainerManager;
+import io.github.hectorvent.floci.services.neptune.model.NeptuneCluster;
+import io.github.hectorvent.floci.services.neptune.model.NeptuneDbType;
+import io.github.hectorvent.floci.services.neptune.proxy.NeptuneProxyManager;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class NeptuneServiceTest {
+
+ private NeptuneService service;
+ private NeptuneContainerManager containerManager;
+ private NeptuneProxyManager proxyManager;
+
+ @BeforeEach
+ void setUp() {
+ containerManager = mock(NeptuneContainerManager.class);
+ proxyManager = mock(NeptuneProxyManager.class);
+ StorageFactory storageFactory = mock(StorageFactory.class);
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ RegionResolver regionResolver = mock(RegionResolver.class);
+
+ EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);
+ EmulatorConfig.NeptuneServiceConfig neptuneConfig = mock(EmulatorConfig.NeptuneServiceConfig.class);
+ when(config.services()).thenReturn(servicesConfig);
+ when(servicesConfig.neptune()).thenReturn(neptuneConfig);
+ when(neptuneConfig.proxyBasePort()).thenReturn(18182);
+ when(neptuneConfig.proxyMaxPort()).thenReturn(18199);
+ when(neptuneConfig.dbType()).thenReturn("gremlin");
+ when(neptuneConfig.defaultImage()).thenReturn("tinkerpop/gremlin-server:3.7");
+ when(neptuneConfig.defaultNeo4jImage()).thenReturn("neo4j:5");
+ when(config.hostname()).thenReturn(Optional.of("localhost"));
+
+ when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");
+ when(regionResolver.buildArn(anyString(), anyString(), anyString()))
+ .thenReturn("arn:aws:neptune:us-east-1:000000000000:cluster:c");
+
+ when(storageFactory.create(anyString(), anyString(), any()))
+ .thenAnswer(inv -> new InMemoryStorage<>());
+ when(containerManager.start(anyString(), anyString(), any(NeptuneDbType.class)))
+ .thenReturn(new NeptuneContainerHandle("cid", "c", "localhost", 8182));
+ doNothing().when(proxyManager).startProxy(anyString(), anyInt(), anyString(), anyInt());
+
+ service = new NeptuneService(config, regionResolver, containerManager, proxyManager, storageFactory);
+ }
+
+ @Test
+ void failedProvisioningRollsBackContainerAndReleasesProxyPort() {
+ NeptuneContainerHandle handle = new NeptuneContainerHandle("cid", "c", "localhost", 8182);
+ when(containerManager.start(anyString(), anyString(), any(NeptuneDbType.class)))
+ .thenReturn(handle);
+
+ // Proxy startup blows up after the port is reserved and the container is started.
+ doThrow(new RuntimeException("proxy boom"))
+ .when(proxyManager).startProxy(eq("c"), anyInt(), anyString(), anyInt());
+
+ // The original failure must propagate to the caller (we clean up, then rethrow).
+ assertThrows(RuntimeException.class,
+ () -> service.createDbCluster("c", "1.3.2.1", false));
+
+ // Rollback stopped the proxy and the already-started container (by id).
+ verify(proxyManager).stopProxy("c");
+ verify(containerManager).stopByClusterId("c");
+
+ // The reserved proxy port was released: a subsequent successful create reuses the base port
+ // instead of skipping to the next one (which is what a leak would cause).
+ doNothing().when(proxyManager).startProxy(anyString(), anyInt(), anyString(), anyInt());
+ NeptuneCluster recovered = service.createDbCluster("c2", "1.3.2.1", false);
+ assertEquals(18182, recovered.getProxyPort(),
+ "Port from the failed create must be released so the next cluster reuses it");
+ }
+
+ @Test
+ void jvmErrorDuringProvisioningStillRollsBack() {
+ NeptuneContainerHandle handle = new NeptuneContainerHandle("cid", "c", "localhost", 8182);
+ when(containerManager.start(anyString(), anyString(), any(NeptuneDbType.class)))
+ .thenReturn(handle);
+
+ // A JVM Error (not a RuntimeException) escapes provisioning — a catch (RuntimeException)
+ // would miss it, so rollback must run from a finally instead.
+ doThrow(new StackOverflowError("boom"))
+ .when(proxyManager).startProxy(eq("c"), anyInt(), anyString(), anyInt());
+
+ assertThrows(StackOverflowError.class,
+ () -> service.createDbCluster("c", "1.3.2.1", false));
+
+ // Rollback still fired despite the Error: proxy and container stopped, port released.
+ verify(proxyManager).stopProxy("c");
+ verify(containerManager).stopByClusterId("c");
+
+ doNothing().when(proxyManager).startProxy(anyString(), anyInt(), anyString(), anyInt());
+ NeptuneCluster recovered = service.createDbCluster("c2", "1.3.2.1", false);
+ assertEquals(18182, recovered.getProxyPort(),
+ "Port from the failed create must be released even when the failure is an Error");
+ }
+
+ @Test
+ void failedContainerStartupCleansUpContainerByIdAndReleasesPort() {
+ // containerManager.start(...) throws — this models both a container that never started
+ // and (crucially) a readiness timeout, where start() created + registered the container
+ // before throwing, so no handle ever reaches the service.
+ doThrow(new RuntimeException("readiness boom"))
+ .when(containerManager).start(eq("c"), anyString(), any(NeptuneDbType.class));
+
+ // The original failure must propagate to the caller (we clean up, then rethrow).
+ assertThrows(RuntimeException.class,
+ () -> service.createDbCluster("c", "1.3.2.1", false));
+
+ // No handle returned, so the proxy never started and must not be stopped...
+ verify(proxyManager, never()).stopProxy(anyString());
+ // ...but the container must still be cleaned up by id, since start() may have created and
+ // registered it before failing. Cleaning up by handle here would orphan it.
+ verify(containerManager).stopByClusterId("c");
+
+ // The reserved proxy port was still released: a subsequent successful create reuses the base port.
+ when(containerManager.start(anyString(), anyString(), any(NeptuneDbType.class)))
+ .thenReturn(new NeptuneContainerHandle("cid", "c2", "localhost", 8182));
+ NeptuneCluster recovered = service.createDbCluster("c2", "1.3.2.1", false);
+ assertEquals(18182, recovered.getProxyPort(),
+ "Port from the failed create must be released so the next cluster reuses it");
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManagerTest.java
new file mode 100644
index 0000000000..a14eb186cf
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/neptune/container/NeptuneContainerManagerTest.java
@@ -0,0 +1,29 @@
+package io.github.hectorvent.floci.services.neptune.container;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import org.junit.jupiter.api.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+class NeptuneContainerManagerTest {
+
+ @Test
+ void stopByClusterIdRemovesByDeterministicNameWhenNothingRegistered() {
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ NeptuneContainerManager manager = new NeptuneContainerManager(
+ mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),
+ mock(ContainerDetector.class), mock(EmulatorConfig.class), mock(RegionResolver.class));
+
+ // No container was ever registered for this id (e.g. it failed before registration).
+ // Rollback must still fall back to the deterministic name so nothing is orphaned.
+ manager.stopByClusterId("my-cluster");
+
+ verify(lifecycleManager).removeIfExists("floci-neptune-my-cluster");
+ }
+}