Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(redshift): reclaim anonymous container volumes (#6382)

Andre Kuraita96921afade5
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index f4195ae2e2..f59ffb4f32 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -8,7 +8,7 @@ Floci emulates Amazon Redshift by managing a real [PostgreSQL](https://www.postg
> **Always read the host and port from `DescribeClusters`** rather than assuming a fixed port. PostgreSQL listens on `5432` *inside* the container; the port you connect to is dynamically assigned on the host and returned as `Clusters[0].Endpoint.Port`. Redshift's conventional port is `5439`, but the emulator does not bind it: use whatever `DescribeClusters` reports.
-The container has **no persistent volume**: if the physical container survives a Floci restart it is adopted and its data is kept, but if the container itself is gone (host reboot, `docker rm`, a pruned dev box) the cluster comes back empty. Use `CreateClusterSnapshot` / `RestoreFromClusterSnapshot` to preserve data explicitly.
+The PostgreSQL image uses an anonymous volume, which Floci removes with the container and does not reattach to a replacement: if the physical container survives a Floci restart it is adopted and its data is kept, but if the container itself is gone (host reboot, `docker rm`, a pruned dev box) the cluster comes back empty. Use `CreateClusterSnapshot` / `RestoreFromClusterSnapshot` to preserve data explicitly. Failed recovery preserves an adopted container and its volume. Once reboot teardown begins, a failed reboot retains its pre-reboot dump for manual recovery. A failed proxy stop preserves the original database container.
Snapshot-dump and catalog-seed uploads retry transient Docker I/O failures for up to six attempts with a fresh upload each time. The SQL commands themselves are not retried. An exhausted snapshot upload fails the restore; an exhausted catalog-seed upload retains the warning-only startup behavior.
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index fe0f1064c4..d2fcede6b3 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -7,6 +7,7 @@ import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.command.InspectVolumeResponse;
import com.github.dockerjava.api.command.ListVolumesResponse;
+import com.github.dockerjava.api.command.RemoveContainerCmd;
import com.github.dockerjava.api.exception.ConflictException;
import com.github.dockerjava.api.exception.DockerException;
import com.github.dockerjava.api.exception.NotFoundException;
@@ -127,16 +128,34 @@ public class ContainerLifecycleManager {
* @return information about the created container including resolved endpoints and published ports
*/
public ContainerInfo createAndStart(ContainerSpec spec) {
- String containerId = create(spec);
+ return createAndStart(spec, false);
+ }
+
+ /** Starts a disposable container, reclaiming its anonymous volumes if startup fails. */
+ public ContainerInfo createAndStartEphemeral(ContainerSpec spec) {
+ return createAndStart(spec, true);
+ }
+
+ private ContainerInfo createAndStart(ContainerSpec spec, boolean removeAnonymousVolumes) {
+ String resolvedImage = imageCacheService.ensureImageExists(spec.image());
+ String containerId = create(spec, resolvedImage, null, removeAnonymousVolumes);
try {
- ContainerInfo info = startCreated(containerId, spec);
- return withPublishedHostPorts(info, spec);
- } catch (Exception e) {
- // A failed start (e.g. host-port conflict) must not leak the created
- // container: retrying callers would accumulate Created containers and
- // fixed-name callers would hit name conflicts on the next attempt.
- removeIfExists(containerId);
- throw e;
+ return withPublishedHostPorts(startCreated(containerId, spec), spec);
+ } catch (RuntimeException failure) {
+ if (removeAnonymousVolumes) {
+ removeFailedEphemeralContainer(containerId, failure);
+ } else {
+ removeIfExists(containerId);
+ }
+ throw failure;
+ }
+ }
+
+ private void removeFailedEphemeralContainer(String containerId, RuntimeException failure) {
+ try {
+ removeIfExistsStrict(containerId, true);
+ } catch (RuntimeException cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
}
}
@@ -175,12 +194,20 @@ public class ContainerLifecycleManager {
}
private String create(ContainerSpec spec, String resolvedImage, String platform) {
- String containerId = createWithCaBundle(spec, resolvedImage, platform);
+ return create(spec, resolvedImage, platform, false);
+ }
+
+ private String create(ContainerSpec spec, String resolvedImage, String platform, boolean removeAnonymousVolumes) {
+ String containerId = createWithCaBundle(spec, resolvedImage, platform, removeAnonymousVolumes);
if (spec.hasNetworkConfiguration()) {
try {
attachNetworkBeforeStart(containerId, spec);
} catch (RuntimeException e) {
- removeIfExists(containerId);
+ if (removeAnonymousVolumes) {
+ removeFailedEphemeralContainer(containerId, e);
+ } else {
+ removeIfExists(containerId);
+ }
throw new IllegalStateException(
"Failed to attach pre-start network configuration for container " + containerId, e);
}
@@ -188,7 +215,8 @@ public class ContainerLifecycleManager {
return containerId;
}
- private String createWithCaBundle(ContainerSpec spec, String resolvedImage, String platform) {
+ private String createWithCaBundle(ContainerSpec spec, String resolvedImage, String platform,
+ boolean removeAnonymousVolumes) {
LOG.debugv("Creating container from spec: image={0}, name={1}", spec.image(), spec.name());
// Built once: a dynamic port binding allocates its host port here.
@@ -204,7 +232,11 @@ public class ContainerLifecycleManager {
}
// SSL_CERT_FILE and friends replace the image's trust store, so they must never name a
// file that is not there. Start over without them: the container keeps its own trust.
- dockerClient.removeContainerCmd(containerId).withForce(true).exec();
+ if (removeAnonymousVolumes) {
+ removeIfExistsStrict(containerId, true);
+ } else {
+ dockerClient.removeContainerCmd(containerId).withForce(true).exec();
+ }
return createContainer(spec, resolvedImage, platform, hostConfig, spec.env());
}
@@ -1046,8 +1078,17 @@ public class ContainerLifecycleManager {
* @param name the container name to remove
*/
public void removeIfExistsStrict(String name) {
+ removeIfExistsStrict(name, false);
+ }
+
+ /** Removes a container, optionally reclaiming anonymous volumes that are no longer in use. */
+ public void removeIfExistsStrict(String name, boolean removeAnonymousVolumes) {
try {
- dockerClient.removeContainerCmd(name).withForce(true).exec();
+ RemoveContainerCmd remove = dockerClient.removeContainerCmd(name).withForce(true);
+ if (removeAnonymousVolumes) {
+ remove.withRemoveVolumes(true);
+ }
+ remove.exec();
LOG.infov("Removed stale container {0}", name);
} catch (NotFoundException e) {
// Not found means the fixed name is available.
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
index d7116fccf2..351352730f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
@@ -203,7 +203,7 @@ public class RedshiftService {
LOG.warnv(ex, "Failed to stop proxy during recovery rollback for cluster {0}", cluster.getClusterIdentifier());
}
try {
- containerManager.stop(entry.accountId(), cluster.getClusterIdentifier());
+ containerManager.rollbackStartup(entry.accountId(), cluster.getClusterIdentifier());
} catch (Exception ex) {
LOG.warnv(ex, "Failed to stop container during recovery rollback for cluster {0}", cluster.getClusterIdentifier());
}
@@ -950,8 +950,8 @@ public class RedshiftService {
Cluster cluster = clusters.get(clusterIdentifier)
.orElseThrow(() -> new AwsException("ClusterNotFound", "Cluster " + clusterIdentifier + " not found", 404));
- // The container backing a cluster has no persistent volume (see RedshiftContainerManager),
- // so a plain stop+recreate would silently drop the cluster's data. Dump before stopping and
+ // Replacement containers do not reattach the old anonymous volume, so a plain stop+recreate
+ // would silently drop the cluster's data. Dump before stopping and
// restore immediately after starting, using a throwaway temp file, so no Snapshot resource is
// created or exposed to the caller.
Path tempDump;
@@ -964,16 +964,18 @@ public class RedshiftService {
// Hoisted so a failure after the proxy is (re)started still tears it down and
// returns the port, matching createCluster/restoreFromClusterSnapshot rollback.
int proxyPort = cluster.getProxyPort() > 0 ? cluster.getProxyPort() : -1;
- boolean originalTornDown = false; // original proxy + container already stopped
+ boolean teardownStarted = false;
+ boolean containerTeardownStarted = false;
boolean rebooted = false;
try {
String accountId = clusters.accountId();
String key = relayKey(accountId, clusterIdentifier);
containerManager.takeSnapshot(accountId, clusterIdentifier, cluster.getMasterUsername(), tempDump);
+ teardownStarted = true;
proxyManager.stopProxy(key);
+ containerTeardownStarted = true;
containerManager.stop(accountId, clusterIdentifier);
- originalTornDown = true;
String password = cluster.getMasterPassword() != null ? cluster.getMasterPassword() : "admin";
RedshiftContainerHandle handle = containerManager.start(
@@ -997,15 +999,15 @@ public class RedshiftService {
cluster.setClusterStatus("available");
rebooted = true;
} catch (AwsException e) {
- rollbackReboot(clusterIdentifier, originalTornDown);
- if (originalTornDown) {
+ rollbackReboot(clusterIdentifier, containerTeardownStarted);
+ if (teardownStarted) {
cluster.setClusterStatus("failed");
}
clusters.flush();
throw e;
} catch (Exception e) {
- rollbackReboot(clusterIdentifier, originalTornDown);
- if (originalTornDown) {
+ rollbackReboot(clusterIdentifier, containerTeardownStarted);
+ if (teardownStarted) {
cluster.setClusterStatus("failed");
}
clusters.flush();
@@ -1018,9 +1020,9 @@ public class RedshiftService {
LOG.warnv(ex, "Failed to clean up temporary dump file {0} after rebooting cluster {1}", tempDump, clusterIdentifier);
}
} else {
- if (originalTornDown) {
- // Once the original container is torn down it holds no volume, so this dump can
- // be the only surviving copy of the cluster's data, so keep it for manual recovery.
+ if (teardownStarted) {
+ // Teardown may have removed the original container before failing. Keep the
+ // dump for manual recovery even when removal or log closure did not finish.
LOG.warnv("Reboot of cluster {0} did not complete; retained pre-reboot data dump at {1}",
clusterIdentifier, tempDump);
} else {
@@ -2115,17 +2117,14 @@ public class RedshiftService {
}
/**
- * Undo a failed reboot. If {@code originalTornDown} is false the reboot failed before
- * the original proxy + container were stopped, so the original data-bearing container
- * is still running and nothing must be touched. Once it is true the original is gone:
- * tear down the (replacement's) proxy and return its port, and remove any container
- * running under the cluster's name: {@code containerManager.stop} works by name, so
- * this also cleans a replacement that {@code containerManager.start} created before
- * throwing (e.g. its readiness check timed out). The pre-reboot data dump is kept by
- * the caller.
+ * Undo a failed reboot once container teardown has started. Before that point the
+ * original data-bearing container must be preserved, even if proxy shutdown failed.
+ * Teardown may be partial: stop the proxy and remove the container tracked by the
+ * manager, including a replacement retained after a failed readiness check. The
+ * caller keeps the pre-reboot data dump.
*/
- private void rollbackReboot(String identifier, boolean originalTornDown) {
- if (!originalTornDown) {
+ private void rollbackReboot(String identifier, boolean containerTeardownStarted) {
+ if (!containerTeardownStarted) {
return;
}
try {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
index 340e2ecf06..d5c1a68630 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
@@ -43,6 +43,7 @@ public class RedshiftContainerManager {
private final EmulatorConfig config;
private final long retryBackoffMillis;
private final Map<String, RedshiftContainerHandle> containers = new ConcurrentHashMap<>();
+ private final Map<String, String> adoptedContainers = new ConcurrentHashMap<>();
@Inject
public RedshiftContainerManager(ContainerBuilder containerBuilder,
@@ -76,9 +77,8 @@ public class RedshiftContainerManager {
String masterPassword, String databaseName) {
String image = config.services().redshift().imageVersion();
String containerName = containerName(accountId, clusterIdentifier);
- // Only the current name: no volume backs this container, so removing a legacy-named survivor
- // would destroy cluster data that adoptOrStart does not look for.
- lifecycleManager.removeIfExists(containerName);
+ // Only the current name: adoptOrStart does not look for legacy-named survivors.
+ stop(accountId, clusterIdentifier);
List<String> envVars = List.of(
"POSTGRES_USER=" + masterUsername,
@@ -101,7 +101,7 @@ public class RedshiftContainerManager {
}
ContainerSpec spec = specBuilder.build();
- ContainerInfo info = lifecycleManager.createAndStart(spec);
+ ContainerInfo info = lifecycleManager.createAndStartEphemeral(spec);
return initializeAndRegisterHandle(info, accountId, clusterIdentifier, masterUsername, containerName, enginePort,
databaseName);
}
@@ -109,12 +109,12 @@ public class RedshiftContainerManager {
/**
* Starts a cluster's container the same way {@link #start}, except that a container
* already running under this cluster's name (e.g. Floci's own process restarted but
- * Docker did not) is adopted instead of destroyed and recreated. The container has no
- * persistent volume, so a blind recreate would silently discard the cluster's data;
+ * Docker did not) is adopted instead of destroyed and recreated. The image's anonymous
+ * volume is not reattached on recreation, so a blind recreate would discard the cluster's data;
* adopting preserves it whenever the physical container survived. If no such container
* exists (first start, or the container itself was removed), this falls back to
* {@link #start}, and in that case the previous data is unrecoverable, matching this
- * project's decision not to back Redshift containers with a Docker volume.
+ * project's decision not to provision reusable named volumes for Redshift containers.
*/
public RedshiftContainerHandle adoptOrStart(String accountId, String clusterIdentifier, String masterUsername, String masterPassword) {
return adoptOrStart(accountId, clusterIdentifier, masterUsername, masterPassword, DEFAULT_DATABASE);
@@ -127,10 +127,11 @@ public class RedshiftContainerManager {
Optional<Container> existing = lifecycleManager.findByName(containerName);
if (existing.isEmpty()) {
LOG.warnv("No surviving container for cluster {0}; starting a fresh empty one: the previous"
- + " contents are not recoverable (no Docker volume backs Redshift containers)", clusterIdentifier);
+ + " contents are not recoverable (no reusable named volume backs Redshift containers)", clusterIdentifier);
return start(accountId, clusterIdentifier, masterUsername, masterPassword, databaseName);
}
+ adoptedContainers.put(containerKey(accountId, clusterIdentifier), existing.get().getId());
LOG.infov("Adopting existing container {0} for cluster {1} to avoid discarding its data",
containerName, clusterIdentifier);
int enginePort = 5432;
@@ -164,16 +165,56 @@ public class RedshiftContainerManager {
LOG.warnv("Failed to stream logs for {0}", containerName);
}
- waitForReady(containerName, info.containerId(), masterUsername, databaseName);
- bootstrapCatalog(info.containerId(), masterUsername, databaseName);
-
+ try {
+ waitForReady(containerName, info.containerId(), masterUsername, databaseName);
+ bootstrapCatalog(info.containerId(), masterUsername, databaseName);
+ } catch (RuntimeException e) {
+ containers.put(containerKey(accountId, clusterIdentifier), handle);
+ throw e;
+ }
containers.put(containerKey(accountId, clusterIdentifier), handle);
return handle;
}
public void stop(String accountId, String clusterIdentifier) {
- containers.remove(containerKey(accountId, clusterIdentifier));
- lifecycleManager.removeIfExists(containerName(accountId, clusterIdentifier));
+ String key = containerKey(accountId, clusterIdentifier);
+ RedshiftContainerHandle handle = containers.get(key);
+ String adoptedId = adoptedContainers.get(key);
+ String containerId = handle == null
+ ? adoptedContainers.getOrDefault(key, containerName(accountId, clusterIdentifier)) : handle.getContainerId();
+ lifecycleManager.removeIfExistsStrict(containerId, true);
+ releaseHandle(key, handle);
+ if (containerId.equals(adoptedId)) {
+ adoptedContainers.remove(key, adoptedId);
+ }
+ }
+
+ /** Rolls back startup without destroying a container that survived an earlier process. */
+ public void rollbackStartup(String accountId, String clusterIdentifier) {
+ String key = containerKey(accountId, clusterIdentifier);
+ String adoptedId = adoptedContainers.get(key);
+ if (adoptedId == null) {
+ stop(accountId, clusterIdentifier);
+ return;
+ }
+ RedshiftContainerHandle handle = containers.get(key);
+ if (handle != null && adoptedId.equals(handle.getContainerId())) {
+ releaseHandle(key, handle);
+ }
+ }
+
+ private void releaseHandle(String key, RedshiftContainerHandle handle) {
+ if (handle != null) {
+ Closeable stream = handle.getLogStream();
+ if (stream != null) {
+ try {
+ stream.close();
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to close logs for Redshift container " + handle.getContainerId(), e);
+ }
+ }
+ containers.remove(key, handle);
+ }
}
public Optional<RedshiftContainerHandle> getContainer(String accountId, String clusterIdentifier) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
index 0384fa236d..78fa4003eb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
@@ -63,8 +63,8 @@ public class RedshiftServerlessRuntime {
/**
* Starts (or, when {@code adopt} is set, re-attaches to) the container and fronts it with the
- * auth proxy on {@code endpoint}. Anything started before a failure is torn down again, so a
- * failed call leaves neither a container nor a bound port behind.
+ * auth proxy on {@code endpoint}. Failed startup removes newly created containers and closes
+ * the proxy; adopted containers are preserved for another recovery attempt.
*/
public Backend start(String accountId, String region, String workgroupName, String masterUsername,
String masterPassword, String dbName, Endpoint endpoint, boolean adopt,
@@ -206,7 +206,7 @@ public class RedshiftServerlessRuntime {
LOG.warnv(e, "Could not stop the proxy while rolling back workgroup backend {0}", backendId);
}
try {
- containerManager.stop(accountId, backendId);
+ containerManager.rollbackStartup(accountId, backendId);
} catch (RuntimeException e) {
LOG.warnv(e, "Could not remove the container while rolling back workgroup backend {0}", backendId);
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java
index dce357a14c..777df9565d 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java
@@ -43,9 +43,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -226,8 +228,9 @@ class ContainerLifecycleManagerCaBundleTest {
verify(dockerClient, never()).copyArchiveToContainerCmd(anyString());
}
- @Test
- void aFailedCopyRecreatesTheContainerWithoutTheTrustVariables() throws Exception {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void aFailedCopyRecreatesTheContainerWithoutTheTrustVariables(boolean ephemeral) throws Exception {
// SSL_CERT_FILE and friends replace the image's trust store, so they must never name a
// file that is not there: an image without /etc keeps its own trust instead.
when(tlsConfig.enabled()).thenReturn(true);
@@ -245,7 +248,17 @@ class ContainerLifecycleManagerCaBundleTest {
RemoveContainerCmd removeCmd = mock(RemoveContainerCmd.class, RETURNS_SELF);
when(dockerClient.removeContainerCmd("container-id")).thenReturn(removeCmd);
- assertEquals("container-id-2", manager().create(specWithEnv(List.of("FOO=bar"))));
+ ContainerSpec spec = specWithEnv(List.of("FOO=bar"));
+ if (ephemeral) {
+ ContainerLifecycleManager lifecycle = spy(manager());
+ ContainerLifecycleManager.ContainerInfo info =
+ new ContainerLifecycleManager.ContainerInfo("container-id-2", Map.of());
+ doReturn(info).when(lifecycle).startCreated("container-id-2", spec);
+ assertEquals(info, lifecycle.createAndStartEphemeral(spec));
+ } else {
+ assertEquals("container-id-2", manager().create(spec));
+ }
+ verify(removeCmd, ephemeral ? times(1) : never()).withRemoveVolumes(true);
verify(removeCmd).withForce(true);
verify(removeCmd).exec();
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerEphemeralTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerEphemeralTest.java
new file mode 100644
index 0000000000..fd757af7f7
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerEphemeralTest.java
@@ -0,0 +1,98 @@
+package io.github.hectorvent.floci.core.common.docker;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateContainerCmd;
+import com.github.dockerjava.api.command.CreateContainerResponse;
+import com.github.dockerjava.api.command.RemoveContainerCmd;
+import com.github.dockerjava.api.exception.NotFoundException;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.*;
+
+class ContainerLifecycleManagerEphemeralTest {
+ private DockerClient docker;
+ private ContainerLifecycleManager manager;
+ private RemoveContainerCmd remove;
+ private CreateContainerCmd create;
+ private final ContainerSpec spec = new ContainerSpec("postgres:16-alpine");
+
+ @BeforeEach
+ void setUp() {
+ docker = mock(DockerClient.class);
+ ImageCacheService images = mock(ImageCacheService.class);
+ when(images.ensureImageExists(spec.image())).thenReturn("image-id");
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ manager = spy(new ContainerLifecycleManager(docker, images, mock(ContainerDetector.class), config));
+ create = mock(CreateContainerCmd.class, RETURNS_SELF);
+ CreateContainerResponse response = new CreateContainerResponse();
+ response.setId("owned-id");
+ when(create.exec()).thenReturn(response);
+ when(docker.createContainerCmd("image-id")).thenReturn(create);
+ remove = mock(RemoveContainerCmd.class, RETURNS_SELF);
+ when(docker.removeContainerCmd(anyString())).thenReturn(remove);
+ }
+
+ @Test
+ void failedEphemeralStartRemovesAnonymousVolumes() {
+ IllegalStateException failure = new IllegalStateException("start failed");
+ doThrow(failure).when(manager).startCreated("owned-id", spec);
+ assertSame(failure, assertThrows(IllegalStateException.class, () -> manager.createAndStartEphemeral(spec)));
+ verify(remove).withRemoveVolumes(true);
+ verify(remove).exec();
+ }
+
+ @Test
+ void failedRemovalPreservesStartFailureAndSuppressedCleanupCause() {
+ IllegalStateException failure = new IllegalStateException("start failed");
+ IllegalStateException cleanup = new IllegalStateException("daemon unavailable");
+ doThrow(failure).when(manager).startCreated("owned-id", spec);
+ when(remove.exec()).thenThrow(cleanup);
+ assertSame(failure, assertThrows(IllegalStateException.class, () -> manager.createAndStartEphemeral(spec)));
+ assertEquals(1, failure.getSuppressed().length);
+ assertSame(cleanup, failure.getSuppressed()[0].getCause());
+ }
+
+ @Test
+ void failedCreateDoesNotRemoveUnownedContainer() {
+ when(create.exec()).thenThrow(new IllegalStateException("create failed"));
+ assertThrows(IllegalStateException.class, () -> manager.createAndStartEphemeral(spec));
+ verify(docker, never()).removeContainerCmd(anyString());
+ }
+
+ @Test
+ void successfulEphemeralStartKeepsItsVolume() {
+ ContainerLifecycleManager.ContainerInfo info = new ContainerLifecycleManager.ContainerInfo("owned-id", Map.of());
+ doReturn(info).when(manager).startCreated("owned-id", spec);
+ assertSame(info, manager.createAndStartEphemeral(spec));
+ verify(docker, never()).removeContainerCmd(anyString());
+ }
+
+ @Test
+ void defaultStrictRemovalDoesNotOptIntoVolumeRemoval() {
+ manager.removeIfExistsStrict("owned-id");
+ verify(remove, never()).withRemoveVolumes(true);
+ verify(remove).exec();
+ }
+
+ @Test
+ void missingEphemeralContainerIsAlreadyClean() {
+ when(remove.exec()).thenThrow(new NotFoundException("gone"));
+ assertDoesNotThrow(() -> manager.removeIfExistsStrict("owned-id", true));
+ verify(remove).withRemoveVolumes(true);
+ }
+
+ @Test
+ void strictEphemeralRemovalExposesFailure() {
+ IllegalStateException failure = new IllegalStateException("daemon unavailable");
+ when(remove.exec()).thenThrow(failure);
+ assertSame(failure, assertThrows(IllegalStateException.class,
+ () -> manager.removeIfExistsStrict("owned-id", true)).getCause());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerNetworkTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerNetworkTest.java
index e28ba43e10..e0379d37fe 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerNetworkTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerNetworkTest.java
@@ -118,14 +118,21 @@ class ContainerLifecycleManagerNetworkTest {
verify(connectCmd).exec();
}
- @Test
- void createRemovesContainerWhenNetworkAttachFails() {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void createRemovesContainerWhenNetworkAttachFails(boolean ephemeral) {
RemoveContainerCmd removeCmd = mock(RemoveContainerCmd.class, RETURNS_SELF);
lenient().when(dockerClient.removeContainerCmd("container-id")).thenReturn(removeCmd);
doThrow(new IllegalStateException("network gone")).when(connectCmd).exec();
- assertThrows(IllegalStateException.class, () -> manager().create(specWithLinkLocalIp(Map.of())));
-
+ assertThrows(IllegalStateException.class, () -> {
+ if (ephemeral) {
+ manager().createAndStartEphemeral(specWithLinkLocalIp(Map.of()));
+ } else {
+ manager().create(specWithLinkLocalIp(Map.of()));
+ }
+ });
+ verify(removeCmd, ephemeral ? times(1) : never()).withRemoveVolumes(true);
verify(dockerClient).removeContainerCmd("container-id");
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java
index 4100295738..3ceda3ba93 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java
@@ -1,6 +1,8 @@
package io.github.hectorvent.floci.core.common.docker;
import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateContainerCmd;
+import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.command.RemoveContainerCmd;
import com.github.dockerjava.api.command.StartContainerCmd;
import com.github.dockerjava.api.exception.DockerException;
@@ -17,7 +19,9 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Answers.RETURNS_DEEP_STUBS;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -44,7 +48,7 @@ class ContainerLifecycleManagerStartFailureTest {
@Mock
private ContainerDetector containerDetector;
- @Mock
+ @Mock(answer = RETURNS_DEEP_STUBS)
private EmulatorConfig config;
private ContainerLifecycleManager manager;
@@ -60,7 +64,7 @@ class ContainerLifecycleManagerStartFailureTest {
void createAndStartRemovesCreatedContainerWhenStartFails() {
RuntimeException startFailure =
new RuntimeException("Bind for 0.0.0.0:80 failed: port is already allocated");
- doReturn("container-id").when(manager).create(spec);
+ stubCreate();
doThrow(startFailure).when(manager).startCreated("container-id", spec);
RemoveContainerCmd removeCmd = mock(RemoveContainerCmd.class);
@@ -78,7 +82,7 @@ class ContainerLifecycleManagerStartFailureTest {
void createAndStartDoesNotRemoveContainerOnSuccess() {
ContainerLifecycleManager.ContainerInfo info =
new ContainerLifecycleManager.ContainerInfo("container-id", Map.of());
- doReturn("container-id").when(manager).create(spec);
+ stubCreate();
doReturn(info).when(manager).startCreated("container-id", spec);
assertSame(info, manager.createAndStart(spec));
@@ -86,6 +90,15 @@ class ContainerLifecycleManagerStartFailureTest {
verify(dockerClient, never()).removeContainerCmd(any(String.class));
}
+ private void stubCreate() {
+ when(imageCacheService.ensureImageExists(spec.image())).thenReturn("image-id");
+ CreateContainerCmd create = mock(CreateContainerCmd.class, RETURNS_SELF);
+ CreateContainerResponse response = new CreateContainerResponse();
+ response.setId("container-id");
+ when(create.exec()).thenReturn(response);
+ when(dockerClient.createContainerCmd("image-id")).thenReturn(create);
+ }
+
@Test
void startCreatedTranslatesKeyringQuotaError() {
// github.com/floci-io/floci/issues/2243: crun/runc report the kernel session-keyring
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
index 6cf66c71e9..d354912097 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
@@ -67,6 +67,70 @@ import static org.mockito.Mockito.*;
class RedshiftServiceTest {
+ @Test
+ void recoveryProxyFailureDoesNotDestructivelyStopAnAdoptedContainer() {
+ Cluster cluster = new Cluster();
+ cluster.setClusterIdentifier("survivor");
+ cluster.setMasterUsername("admin");
+ cluster.setMasterPassword("Secret123");
+ cluster.setClusterStatus("available");
+ cluster.setProxyPort(7107);
+ when(clusterBackend.scanAllAccountEntries(any())).thenReturn(List.of(
+ new AccountAwareStorageBackend.AccountEntry<>("111111111111", "survivor", cluster)));
+ when(cm.adoptOrStart("111111111111", "survivor", "admin", "Secret123"))
+ .thenReturn(new RedshiftContainerHandle("surviving-container", "survivor", "localhost", 5432));
+ doThrow(new IllegalStateException("proxy bind failed")).when(proxyManager).startProxy(
+ anyString(), anyInt(), anyString(), anyInt(), anyString(), anyString(), anyString(), anyString(),
+ any(), anyList(), isNull());
+
+ service.onStart(null);
+
+ assertEquals("unavailable", cluster.getClusterStatus());
+ verify(cm, never()).stop("111111111111", "survivor");
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void rebootTeardownFailureKeepsDumpAndMarksClusterFailed(boolean proxyFailure) throws Exception {
+ Cluster cluster = new Cluster();
+ cluster.setClusterIdentifier("partial-stop");
+ cluster.setMasterUsername("admin");
+ cluster.setMasterPassword("Secret123");
+ cluster.setClusterStatus("available");
+ cluster.setProxyPort(7107);
+ when(clusterBackend.get("partial-stop")).thenReturn(Optional.of(cluster));
+ AtomicReference<Path> dump = new AtomicReference<>();
+ doAnswer(invocation -> {
+ Path path = invocation.getArgument(3, Path.class);
+ Files.writeString(path, "preserved database");
+ dump.set(path);
+ return null;
+ }).when(cm).takeSnapshot(eq("111111111111"), eq("partial-stop"), eq("admin"), any(Path.class));
+ if (proxyFailure) {
+ doThrow(new IllegalStateException("proxy stop failed"))
+ .when(proxyManager).stopProxy("111111111111:partial-stop");
+ } else {
+ doThrow(new IllegalStateException("log close failed after container removal"))
+ .when(cm).stop("111111111111", "partial-stop");
+ }
+ try {
+ assertThrows(AwsException.class, () -> service.rebootCluster("partial-stop"));
+ assertTrue(Files.exists(dump.get()), "The pre-reboot dump must survive partial teardown");
+ assertEquals("preserved database", Files.readString(dump.get()));
+ assertEquals("failed", cluster.getClusterStatus());
+ if (proxyFailure) {
+ verify(cm, never()).stop(anyString(), anyString());
+ } else {
+ verify(cm, atLeastOnce()).stop("111111111111", "partial-stop");
+ }
+ } finally {
+ if (dump.get() != null) {
+ Files.deleteIfExists(dump.get());
+ }
+ }
+ }
+
+
@ParameterizedTest
@ValueSource(booleans = {false, true})
void slowSnapshotDoesNotBlockOtherClusterMutation(boolean finalSnapshot) throws Exception {
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftArchiveRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftArchiveRetryTest.java
index 4996e65108..a9adcebe61 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftArchiveRetryTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftArchiveRetryTest.java
@@ -68,7 +68,7 @@ class RedshiftArchiveRetryTest {
when(builder.newContainer(anyString())).thenReturn(mock(ContainerBuilder.Builder.class, RETURNS_SELF));
ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
when(lifecycle.getDockerClient()).thenReturn(docker);
- when(lifecycle.createAndStart(any())).thenReturn(
+ when(lifecycle.createAndStartEphemeral(any())).thenReturn(
new ContainerInfo(CONTAINER, Map.of(5432, new EndpointInfo("localhost", 5432))));
EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
when(config.services().redshift().imageVersion()).thenReturn("postgres:16-alpine");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManagerTest.java
index ef9ae44083..908bc2fbaf 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManagerTest.java
@@ -23,11 +23,14 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
import io.github.hectorvent.floci.services.redshift.model.Cluster;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.InOrder;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
+import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -46,6 +49,7 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
@@ -55,6 +59,50 @@ import static org.mockito.Mockito.when;
class RedshiftContainerManagerTest {
+ @ParameterizedTest
+ @ValueSource(strings = {"adopt", "readiness", "proxy"})
+ void startupRollbackPreservesAdoptedDataAndExplicitStopStillRemovesIt(String failureStage) throws IOException {
+ String name = "floci-aws-redshift-" + ACCOUNT_ID + "-survivor";
+ Container existing = mock(Container.class);
+ when(existing.getId()).thenReturn("cont-survivor");
+ when(lifecycleManager.findByName(name)).thenReturn(Optional.of(existing));
+ Closeable stream = mock(Closeable.class);
+ if ("adopt".equals(failureStage)) {
+ when(lifecycleManager.adopt(eq("cont-survivor"), any(), any()))
+ .thenThrow(new IllegalStateException("adoption failed"));
+ assertThrows(IllegalStateException.class,
+ () -> manager.adoptOrStart(ACCOUNT_ID, "survivor", "admin", "pass"));
+ } else {
+ when(lifecycleManager.adopt(eq("cont-survivor"), any(), any())).thenReturn(
+ new ContainerInfo("cont-survivor", Map.of(5432, new EndpointInfo("localhost", 5432))));
+ when(logStreamer.attach(anyString(), anyString(), anyString(), anyString(), anyString())).thenReturn(stream);
+ if ("readiness".equals(failureStage)) {
+ when(dockerClient.execCreateCmd(anyString())).thenThrow(new IllegalStateException("not ready"));
+ Thread.currentThread().interrupt();
+ try {
+ assertThrows(IllegalStateException.class,
+ () -> manager.adoptOrStart(ACCOUNT_ID, "survivor", "admin", "pass"));
+ } finally {
+ Thread.interrupted();
+ }
+ } else {
+ manager.adoptOrStart(ACCOUNT_ID, "survivor", "admin", "pass");
+ }
+ }
+
+ manager.rollbackStartup(ACCOUNT_ID, "survivor");
+ manager.rollbackStartup(ACCOUNT_ID, "survivor");
+
+ verify(lifecycleManager, never()).removeIfExistsStrict(anyString(), eq(true));
+ assertTrue(manager.getContainer(ACCOUNT_ID, "survivor").isEmpty());
+ if (!"adopt".equals(failureStage)) {
+ verify(stream).close();
+ }
+ manager.stop(ACCOUNT_ID, "survivor");
+ verify(lifecycleManager).removeIfExistsStrict("cont-survivor", true);
+ }
+
+
private ContainerBuilder containerBuilder;
private ContainerLifecycleManager lifecycleManager;
private ContainerLogStreamer logStreamer;
@@ -152,7 +200,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
RedshiftContainerHandle handle = manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
assertNotNull(handle);
@@ -211,7 +259,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
@@ -249,7 +297,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
@@ -263,7 +311,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
@@ -308,7 +356,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
assertTrue(manager.getContainer(ACCOUNT_ID, "test-cluster").isPresent());
@@ -316,7 +364,7 @@ class RedshiftContainerManagerTest {
clearInvocations(lifecycleManager);
manager.stop(ACCOUNT_ID, "test-cluster");
assertTrue(manager.getContainer(ACCOUNT_ID, "test-cluster").isEmpty());
- verify(lifecycleManager).removeIfExists("floci-aws-redshift-" + ACCOUNT_ID + "-test-cluster");
+ verify(lifecycleManager).removeIfExistsStrict("cont-123", true);
}
@Test
@@ -324,16 +372,16 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
// A container left under the fixed name by an earlier run would make createAndStart fail with a name conflict
InOrder order = inOrder(lifecycleManager);
- order.verify(lifecycleManager).removeIfExists("floci-aws-redshift-" + ACCOUNT_ID + "-test-cluster");
- order.verify(lifecycleManager).createAndStart(any());
- // A legacy-named container may still hold a cluster's data, and no volume backs it
- verify(lifecycleManager, never()).removeIfExists("floci-redshift-" + ACCOUNT_ID + "-test-cluster");
+ order.verify(lifecycleManager).removeIfExistsStrict("floci-aws-redshift-" + ACCOUNT_ID + "-test-cluster", true);
+ order.verify(lifecycleManager).createAndStartEphemeral(any());
+ // A legacy-named container may still hold a cluster's data.
+ verify(lifecycleManager, never()).removeIfExistsStrict("floci-redshift-" + ACCOUNT_ID + "-test-cluster", true);
}
@Test
@@ -342,7 +390,7 @@ class RedshiftContainerManagerTest {
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo infoA = new ContainerInfo("cont-account-a", Map.of(5432, new EndpointInfo("localhost", 5432)));
ContainerInfo infoB = new ContainerInfo("cont-account-b", Map.of(5432, new EndpointInfo("localhost", 5433)));
- when(lifecycleManager.createAndStart(any())).thenReturn(infoA, infoB);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(infoA, infoB);
String otherAccountId = "222222222222";
manager.start(ACCOUNT_ID, "shared-id", "admin", "pass");
@@ -370,10 +418,10 @@ class RedshiftContainerManagerTest {
assertEquals("cont-existing", handle.getContainerId());
assertEquals(6000, handle.getPort());
assertTrue(manager.getContainer(ACCOUNT_ID, "test-cluster").isPresent());
- // Data lives only in the container's writable layer (no volume): recreating it
- // would silently discard it, so a container found by name must never be recreated.
- verify(lifecycleManager, never()).createAndStart(any());
- verify(lifecycleManager, never()).removeIfExists(anyString());
+ // The surviving container retains its anonymous PostgreSQL volume. A replacement
+ // would start with an empty volume, so adoption must preserve the container.
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
+ verify(lifecycleManager, never()).removeIfExistsStrict(anyString(), eq(true));
}
@Test
@@ -383,7 +431,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-fresh", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
RedshiftContainerHandle handle = manager.adoptOrStart(ACCOUNT_ID, "test-cluster", "admin", "pass");
@@ -396,7 +444,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
ExecCreateCmd createCmd = mock(ExecCreateCmd.class, org.mockito.Mockito.RETURNS_SELF);
@@ -438,7 +486,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
// Test SQL injection attempt with semicolon
@@ -459,7 +507,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
// Forbidden characters: ', ", \, / and @ (AWS ModifyCluster rejects the same set)
@@ -492,7 +540,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
ExecCreateCmd createCmd = mock(ExecCreateCmd.class, org.mockito.Mockito.RETURNS_SELF);
@@ -529,7 +577,7 @@ class RedshiftContainerManagerTest {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
ContainerInfo info = new ContainerInfo("cont-stream", Map.of(5432, new EndpointInfo("localhost", 5432)));
- when(lifecycleManager.createAndStart(any())).thenReturn(info);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
Closeable stream = mock(Closeable.class);
when(logStreamer.attach("cont-stream", "/floci/redshift", "test-cluster", "us-east-1", "redshift:test-cluster"))
.thenReturn(stream);
@@ -605,4 +653,79 @@ class RedshiftContainerManagerTest {
org.junit.jupiter.api.Assertions.assertDoesNotThrow(() ->
manager.bootstrapCatalog("cont-bootstrap-fail", "admin", "dev"));
}
+
+ @Test
+ void stopFailureRetainsHandleForRetry() {
+ ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(
+ new ContainerInfo("cont-owned", Map.of(5432, new EndpointInfo("localhost", 5432))));
+ RedshiftContainerHandle handle = manager.start(ACCOUNT_ID, "owned", "admin", "pass");
+ doThrow(new IllegalStateException("Docker removal failed"))
+ .when(lifecycleManager).removeIfExistsStrict("cont-owned", true);
+ assertThrows(IllegalStateException.class, () -> manager.stop(ACCOUNT_ID, "owned"));
+ assertEquals(Optional.of(handle), manager.getContainer(ACCOUNT_ID, "owned"));
+ doNothing().when(lifecycleManager).removeIfExistsStrict("cont-owned", true);
+ manager.stop(ACCOUNT_ID, "owned");
+ assertTrue(manager.getContainer(ACCOUNT_ID, "owned").isEmpty());
+ }
+
+ @Test
+ void successfulStopClosesOwnedLogStream() throws IOException {
+ ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(
+ new ContainerInfo("cont-owned", Map.of(5432, new EndpointInfo("localhost", 5432))));
+ RedshiftContainerHandle handle = manager.start(ACCOUNT_ID, "owned", "admin", "pass");
+ Closeable stream = mock(Closeable.class);
+ handle.setLogStream(stream);
+ manager.stop(ACCOUNT_ID, "owned");
+ verify(stream).close();
+ }
+
+ @Test
+ void staleRemovalFailurePreventsReplacementCreation() {
+ doThrow(new IllegalStateException("Docker unavailable")).when(lifecycleManager)
+ .removeIfExistsStrict("floci-aws-redshift-" + ACCOUNT_ID + "-owned", true);
+ assertThrows(IllegalStateException.class, () -> manager.start(ACCOUNT_ID, "owned", "admin", "pass"));
+ verify(lifecycleManager, never()).createAndStartEphemeral(any());
+ }
+
+ @Test
+ void readinessFailureKeepsLogAndContainerOwnershipUntilRollback() throws IOException {
+ ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(
+ new ContainerInfo("cont-unready", Map.of(5432, new EndpointInfo("localhost", 5432))));
+ Closeable stream = mock(Closeable.class);
+ when(logStreamer.attach(anyString(), anyString(), anyString(), anyString(), anyString())).thenReturn(stream);
+ when(dockerClient.execCreateCmd(anyString())).thenThrow(new IllegalStateException("not ready"));
+ Thread.currentThread().interrupt();
+ try {
+ assertThrows(IllegalStateException.class, () -> manager.start(ACCOUNT_ID, "unready", "admin", "pass"));
+ } finally {
+ Thread.interrupted();
+ }
+ assertTrue(manager.getContainer(ACCOUNT_ID, "unready").isPresent());
+ verify(stream, never()).close();
+ manager.rollbackStartup(ACCOUNT_ID, "unready");
+ verify(lifecycleManager).removeIfExistsStrict("cont-unready", true);
+ verify(stream).close();
+ }
+
+ @Test
+ void logCloseFailureRetainsHandleForRetryAfterContainerRemoval() throws IOException {
+ ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(
+ new ContainerInfo("cont-owned", Map.of(5432, new EndpointInfo("localhost", 5432))));
+ RedshiftContainerHandle handle = manager.start(ACCOUNT_ID, "owned", "admin", "pass");
+ Closeable stream = mock(Closeable.class);
+ handle.setLogStream(stream);
+ doThrow(new IllegalStateException("close failed")).doNothing().when(stream).close();
+ assertThrows(IllegalStateException.class, () -> manager.stop(ACCOUNT_ID, "owned"));
+ assertEquals(Optional.of(handle), manager.getContainer(ACCOUNT_ID, "owned"));
+ manager.stop(ACCOUNT_ID, "owned");
+ assertTrue(manager.getContainer(ACCOUNT_ID, "owned").isEmpty());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntimeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntimeTest.java
index d88c0d4658..80f6a8592f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntimeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntimeTest.java
@@ -108,6 +108,20 @@ class RedshiftServerlessRuntimeTest {
any(Path.class), eq(true));
}
+ @Test
+ void failedAdoptedProxyUsesRecoveryRollback() {
+ when(containers.adoptOrStart(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics")).thenReturn(handle());
+ doThrow(new RuntimeException("port in use")).when(proxies).startProxy(any(), anyInt(), any(), anyInt(),
+ any(), any(), any(), any(), any(PasswordValidator.class), any());
+
+ assertThrows(RuntimeException.class, () -> runtime.start(ACCOUNT, REGION, "my-wg", "root", "Secret123",
+ "analytics", ENDPOINT, true, Optional::empty));
+
+ verify(proxies).stopProxy(RELAY_KEY);
+ verify(containers).rollbackStartup(ACCOUNT, BACKEND_ID);
+ verify(containers, never()).stop(ACCOUNT, BACKEND_ID);
+ }
+
@Test
void adoptReattachesToASurvivingContainer() {
when(containers.adoptOrStart(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics")).thenReturn(handle());
@@ -130,7 +144,7 @@ class RedshiftServerlessRuntimeTest {
"analytics", ENDPOINT, false, Optional::empty));
verify(proxies).stopProxy(RELAY_KEY);
- verify(containers).stop(ACCOUNT, BACKEND_ID);
+ verify(containers).rollbackStartup(ACCOUNT, BACKEND_ID);
assertTrue(broker.resolve(ACCOUNT, BACKEND_ID, "IAM:alice").isEmpty());
}