Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(eks): fail the delete when Docker cannot remove the cluster container (#4960)

Niels Kootstrad5fd4d52f181
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 84ea66b552..de43846912 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -521,24 +521,18 @@ public class ContainerLifecycleManager {
*
* <p>Most emulator shutdown paths intentionally use best-effort cleanup through
* {@link #stopAndRemove(String, Closeable)}. Resource-deletion paths that must retain their
- * persisted record for a retry use this stricter variant instead.
+ * persisted record for a retry use this stricter variant instead. The log stream is closed
+ * only once the container is gone; when removal fails it is left open for the retry.
*/
public void stopAndRemoveStrict(String containerId, Closeable logStream) {
LOG.infov("Stopping container {0}", containerId);
- if (logStream != null) {
- try {
- logStream.close();
- } catch (Exception e) {
- LOG.debugv("Error closing log stream: {0}", e.getMessage());
- }
- }
-
Exception stopFailure = null;
try {
dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();
} catch (NotFoundException e) {
LOG.debugv("Container {0} not found (already removed)", containerId);
+ closeLogStreamIfPresent(logStream);
return;
} catch (Exception e) {
stopFailure = e;
@@ -556,8 +550,19 @@ public class ContainerLifecycleManager {
if (stopFailure != null) {
cleanupFailure.addSuppressed(stopFailure);
}
+ // The container still exists and may still be logging, so the stream stays open for
+ // the caller's retry.
throw cleanupFailure;
}
+ // Closed only once the container is stopped and gone, as in stopAndRemove, so the follower
+ // can drain the final tail.
+ closeLogStreamIfPresent(logStream);
+ }
+
+ private void closeLogStreamIfPresent(Closeable logStream) {
+ if (logStream != null) {
+ closeLogStreamAfterContainerStop(logStream);
+ }
}
/**
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
index 7d36f7f02e..ac4ea15efe 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
@@ -916,18 +916,32 @@ public class EksClusterManager
* cluster's workloads survive a Floci restart and are re-latched by {@link #restoreCluster}.
*/
public void stopCluster(Cluster cluster) {
- unregisterMetadataEndpoint(cluster);
- Closeable logStream = clusterLogHandles.remove(clusterResourceName(cluster));
+ String resourceName = clusterResourceName(cluster);
if (cluster.getContainerId() == null) {
- closeQuietly(logStream);
+ unregisterMetadataEndpoint(cluster);
+ closeQuietly(clusterLogHandles.remove(resourceName));
return;
}
- lifecycleManager.stopAndRemove(cluster.getContainerId(), logStream);
- if (cluster.getHostPort() > 0) {
- portAllocator.release(cluster.getHostPort());
- // Cleared so a delete retried after a failed backup cleanup cannot free a reused port.
+ Closeable logStream = clusterLogHandles.get(resourceName);
+ // Strict: a container Docker could not remove may still be running and publishing the
+ // port, so the delete fails and keeps the cluster record, its metadata endpoint, log
+ // handle and port reservation; a retried delete releases them once the container is gone.
+ lifecycleManager.stopAndRemoveStrict(cluster.getContainerId(), logStream);
+ unregisterMetadataEndpoint(cluster);
+ // Removed by value so an overlapping delete cannot drop a handle it does not own.
+ if (logStream != null) {
+ clusterLogHandles.remove(resourceName, logStream);
+ }
+ int hostPort;
+ // Read and cleared together so overlapping deletes, or a delete retried after a failed
+ // backup cleanup, cannot release a port that has since been reused.
+ synchronized (cluster) {
+ hostPort = cluster.getHostPort();
cluster.setHostPort(0);
}
+ if (hostPort > 0) {
+ portAllocator.release(hostPort);
+ }
if (cluster.getDockerName() != null) {
// A failed backup cleanup must not leave the live node running. Keep the cluster
// record so explicit deletion can be retried once Docker accepts the removal.
@@ -1084,9 +1098,10 @@ public class EksClusterManager
return;
}
String resourceName = clusterResourceName(cluster);
- if (clusterLogHandles.containsKey(resourceName)) {
- return;
- }
+ // Logs are attached only for a container that was just started or adopted, so a handle
+ // still registered under this name follows an earlier container: close it rather than
+ // leave the new cluster without control-plane logs.
+ closeQuietly(clusterLogHandles.remove(resourceName));
String logGroup = "/aws/eks/" + cluster.getName() + "/cluster";
String hash = containerId.length() >= 32 ? containerId.substring(0, 32) : containerId;
String region = clusterRegion(cluster);
@@ -1124,13 +1139,13 @@ public class EksClusterManager
}
if (handles.size() == 1) {
- clusterLogHandles.put(resourceName, handles.getFirst());
+ closeQuietly(clusterLogHandles.put(resourceName, handles.getFirst()));
} else if (handles.size() > 1) {
- clusterLogHandles.put(resourceName, () -> {
+ closeQuietly(clusterLogHandles.put(resourceName, () -> {
for (Closeable h : handles) {
closeQuietly(h);
}
- });
+ }));
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
index f23a5a8dbb..82c1954049 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
@@ -106,6 +106,8 @@ public class EksService implements TagHandler, ResourceProvider {
private final Map<String, UserDataClaim> appliedClusterUserData = new ConcurrentHashMap<>();
private final Map<String, Object> nodeGroupCapacityLocks = new ConcurrentHashMap<>();
private final Map<String, CompletableFuture<Boolean>> pendingFirstNodeGroups = new ConcurrentHashMap<>();
+ // Storage hands every caller the same live Cluster, and Cluster keeps identity equality.
+ private final Set<Cluster> clustersBeingDeleted = ConcurrentHashMap.newKeySet();
@Inject
public EksService(StorageFactory storageFactory, EmulatorConfig config,
@@ -650,22 +652,39 @@ public class EksService implements TagHandler, ResourceProvider {
.orElseThrow(() -> new AwsException("ResourceNotFoundException",
"No cluster found for name: " + name, 404));
- cluster.setStatus(ClusterStatus.DELETING);
- if (!config.services().eks().mock()) {
- clusterManager.stopCluster(cluster);
- }
- deleteClusterSecurityGroup(cluster);
- accessEntries.deleteClusterEntries(cluster);
- if (podIdentityAssociations != null) {
- podIdentityAssociations.deleteClusterAssociations(cluster);
+ // One delete at a time per cluster, so a delete that fails cannot restore the status or
+ // state of a cluster another delete is tearing down.
+ if (!clustersBeingDeleted.add(cluster)) {
+ throw new AwsException("ResourceInUseException",
+ "Cluster is already being deleted: " + name, 409);
}
- if (addons != null) {
- addons.deleteClusterAddons(cluster);
+ try {
+ ClusterStatus previousStatus = cluster.getStatus();
+ cluster.setStatus(ClusterStatus.DELETING);
+ if (!config.services().eks().mock()) {
+ try {
+ clusterManager.stopCluster(cluster);
+ } catch (RuntimeException e) {
+ // The cluster record is kept for a retry, so it must not stay stuck in DELETING.
+ cluster.setStatus(previousStatus);
+ throw e;
+ }
+ }
+ deleteClusterSecurityGroup(cluster);
+ accessEntries.deleteClusterEntries(cluster);
+ if (podIdentityAssociations != null) {
+ podIdentityAssociations.deleteClusterAssociations(cluster);
+ }
+ if (addons != null) {
+ addons.deleteClusterAddons(cluster);
+ }
+ storage.delete(name);
+ clearAppliedClusterUserData(cluster, name);
+ oidcService.deleteKey(name);
+ return cluster;
+ } finally {
+ clustersBeingDeleted.remove(cluster);
}
- storage.delete(name);
- clearAppliedClusterUserData(cluster, name);
- oidcService.deleteKey(name);
- return cluster;
}
public Nodegroup createNodeGroup(String clusterName, CreateNodeGroupRequest request) {
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
index b31719c847..ff971a2f5f 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -170,6 +171,70 @@ class ContainerLifecycleManagerVolumeTest {
assertFalse(manager.volumeExists("some-volume"));
}
+ @Test
+ void strictContainerCleanupStopsContainerBeforeClosingItsLogStream() {
+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);
+ List<String> operations = new ArrayList<>();
+ doAnswer(invocation -> {
+ operations.add("stop");
+ return null;
+ }).when(stop).exec();
+ doAnswer(invocation -> {
+ operations.add("remove");
+ return null;
+ }).when(remove).exec();
+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);
+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);
+
+ manager.stopAndRemoveStrict("container-id", () -> operations.add("logs"));
+
+ assertEquals(List.of("stop", "remove", "logs"), operations);
+ }
+
+ @Test
+ void strictContainerCleanupClosesTheLogStreamOfAnAlreadyMissingContainer() {
+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);
+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);
+ when(stop.exec()).thenThrow(new NotFoundException("gone"));
+ List<String> operations = new ArrayList<>();
+
+ manager.stopAndRemoveStrict("container-id", () -> operations.add("logs"));
+
+ assertEquals(List.of("logs"), operations);
+ }
+
+ @Test
+ void strictContainerCleanupLeavesTheLogStreamOpenWhenDockerCouldNotStopOrRemoveTheContainer() {
+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);
+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);
+ when(stop.exec()).thenThrow(new DockerException("stop failed", 500));
+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);
+ when(remove.exec()).thenThrow(new DockerException("remove failed", 500));
+ List<String> operations = new ArrayList<>();
+
+ assertThrows(IllegalStateException.class, () ->
+ manager.stopAndRemoveStrict("container-id", () -> operations.add("logs")));
+
+ assertEquals(List.of(), operations);
+ }
+
+ @Test
+ void strictContainerCleanupLeavesTheLogStreamOpenWhenDockerStoppedButCouldNotRemoveTheContainer() {
+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);
+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);
+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);
+ when(remove.exec()).thenThrow(new DockerException("remove failed", 500));
+ List<String> operations = new ArrayList<>();
+
+ assertThrows(IllegalStateException.class, () ->
+ manager.stopAndRemoveStrict("container-id", () -> operations.add("logs")));
+
+ assertEquals(List.of(), operations);
+ }
+
@Test
void strictContainerCleanupPropagatesRemovalFailure() {
StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);
@@ -197,6 +262,20 @@ class ContainerLifecycleManagerVolumeTest {
verify(remove).exec();
}
+ @Test
+ void strictContainerCleanupClosesTheLogStreamOnceForcedRemovalSucceedsAfterStopFailure() throws Exception {
+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);
+ Closeable logStream = mock(Closeable.class);
+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);
+ when(stop.exec()).thenThrow(new DockerException("stop failed", 500));
+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);
+
+ manager.stopAndRemoveStrict("container-id", logStream);
+
+ verify(logStream).close();
+ }
+
@Test
void strictStaleContainerCleanupPropagatesRemovalFailure() {
RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
index 5f925d5caa..1879bf674e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
@@ -532,6 +532,33 @@ class EksClusterManagerTest {
verify(portAllocator).release(6441);
}
+ @Test
+ void deletingAClusterWhoseContainerDockerCouldNotRemoveKeepsItsPortReserved() {
+ stubFreshStart("cid-1", 6441);
+ Mockito.doThrow(new IllegalStateException("Failed to remove container cid-1"))
+ .when(lifecycleManager).stopAndRemoveStrict("cid-1", null);
+ Cluster cluster = cluster();
+ manager.startCluster(cluster);
+
+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));
+
+ verify(portAllocator, never()).release(6441);
+ }
+
+ @Test
+ void retryingADeleteAfterContainerRemovalFailedHandsThePortBack() {
+ stubFreshStart("cid-1", 6441);
+ Mockito.doThrow(new IllegalStateException("Failed to remove container cid-1")).doNothing()
+ .when(lifecycleManager).stopAndRemoveStrict("cid-1", null);
+ Cluster cluster = cluster();
+ manager.startCluster(cluster);
+
+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));
+ manager.stopCluster(cluster);
+
+ verify(portAllocator, Mockito.times(1)).release(6441);
+ }
+
@Test
void retryingADeleteAfterBackupCleanupFailedHandsThePortBackOnlyOnce() {
Cluster cluster = cluster();
@@ -554,6 +581,7 @@ class EksClusterManagerTest {
Cluster cluster = cluster();
assertThrows(IllegalStateException.class, () -> manager.startCluster(cluster));
+ assertEquals(0, cluster.getHostPort());
manager.stopCluster(cluster);
verify(portAllocator, Mockito.times(1)).release(6441);
@@ -584,7 +612,7 @@ class EksClusterManagerTest {
verify(lifecycleManager, Mockito.times(3))
.removeIfExistsStrict("floci-aws-eks-capacity-backup.foo");
verify(lifecycleManager, never()).removeIfExistsStrict(other.getDockerName());
- verify(lifecycleManager, never()).stopAndRemove("cid-other", null);
+ verify(lifecycleManager, never()).stopAndRemoveStrict("cid-other", null);
}
@ParameterizedTest
@@ -604,7 +632,7 @@ class EksClusterManagerTest {
manager.stopCluster(cluster);
- verify(lifecycleManager).stopAndRemove("cid-numeric", null);
+ verify(lifecycleManager).stopAndRemoveStrict("cid-numeric", null);
verify(lifecycleManager).removeIfExistsStrict("floci-aws-eks-capacity-backup.999999999999");
verify(lifecycleManager, never()).removeIfExistsStrict(otherDockerName);
}
@@ -752,7 +780,7 @@ class EksClusterManagerTest {
cluster.setContainerId("cid-1");
manager.stopCluster(cluster);
- verify(lifecycleManager).stopAndRemove("cid-1", null);
+ verify(lifecycleManager).stopAndRemoveStrict("cid-1", null);
// The volume must survive so restoreCluster can bring the workloads back.
verify(lifecycleManager, never()).removeVolume(anyString());
}
@@ -767,7 +795,7 @@ class EksClusterManagerTest {
assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));
- verify(lifecycleManager).stopAndRemove("cid-new", null);
+ verify(lifecycleManager).stopAndRemoveStrict("cid-new", null);
verify(lifecycleManager, never()).removeVolume(anyString());
}
@@ -779,7 +807,7 @@ class EksClusterManagerTest {
cluster.setContainerId("cid-1");
manager.stopCluster(cluster);
- verify(lifecycleManager).stopAndRemove("cid-1", null);
+ verify(lifecycleManager).stopAndRemoveStrict("cid-1", null);
verify(lifecycleManager).removeVolume("floci-aws-eks-demo");
}
@@ -1612,6 +1640,40 @@ class EksClusterManagerTest {
verify(metadataServer).unregisterInstance(any());
assertNull(manager.getRegisteredClusterNodeInstance(cluster));
}
+
+ @Test
+ void aDeleteThatCouldNotRemoveTheContainerKeepsItsNodeOnTheMetadataServer() {
+ Cluster cluster = new Cluster();
+ cluster.setName("test-cluster");
+ manager.configureLinkLocalMetadataEndpoint(cluster, "container-42");
+ cluster.setContainerId("container-42");
+ Mockito.doThrow(new IllegalStateException("Failed to remove container container-42"))
+ .when(lifecycleManager).stopAndRemoveStrict("container-42", null);
+
+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));
+
+ verify(metadataServer, never()).unregisterInstance(any());
+ assertNotNull(manager.getRegisteredClusterNodeInstance(cluster));
+ }
+
+ @Test
+ void retryingADeleteAfterContainerRemovalFailedRemovesTheNodeFromTheMetadataServer() {
+ EmulatorConfig.StorageConfig storage = Mockito.mock(EmulatorConfig.StorageConfig.class);
+ when(config.storage()).thenReturn(storage);
+ when(storage.mode()).thenReturn("memory");
+ Cluster cluster = new Cluster();
+ cluster.setName("test-cluster");
+ manager.configureLinkLocalMetadataEndpoint(cluster, "container-42");
+ cluster.setContainerId("container-42");
+ Mockito.doThrow(new IllegalStateException("Failed to remove container container-42"))
+ .doNothing()
+ .when(lifecycleManager).stopAndRemoveStrict("container-42", null);
+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));
+
+ manager.stopCluster(cluster);
+
+ assertNull(manager.getRegisteredClusterNodeInstance(cluster));
+ }
}
@Nested
@@ -2779,6 +2841,72 @@ class EksClusterManagerTest {
assertEquals("container-id-123456789012345678901234567890", cluster.getContainerId());
}
+ @Test
+ void logHandleSurvivesADeleteThatCouldNotRemoveTheContainer() {
+ Cluster cluster = new Cluster();
+ cluster.setName("prod-cluster");
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));
+ manager.startCluster(cluster);
+ Mockito.doThrow(new IllegalStateException("Failed to remove container"))
+ .when(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockHandle);
+
+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));
+
+ assertEquals(mockHandle, manager.getLogHandle(cluster));
+ }
+
+ @Test
+ void aRecreatedClusterReplacesALogHandleLeftBehindUnderItsName() throws Exception {
+ Cluster stale = new Cluster();
+ stale.setName("prod-cluster");
+ stale.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));
+ manager.startCluster(stale);
+ Closeable freshHandle = Mockito.mock(Closeable.class);
+ when(logStreamer.attachForAccount(anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
+ .thenReturn(freshHandle);
+ Cluster recreated = new Cluster();
+ recreated.setName("prod-cluster");
+ recreated.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));
+
+ manager.startCluster(recreated);
+
+ verify(mockHandle).close();
+ assertEquals(freshHandle, manager.getLogHandle(recreated));
+ }
+
+ @Test
+ void retryingADeleteAfterContainerRemovalFailedReleasesTheLogHandle() {
+ Cluster cluster = new Cluster();
+ cluster.setName("prod-cluster");
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));
+ manager.startCluster(cluster);
+ Mockito.doThrow(new IllegalStateException("Failed to remove container"))
+ .doNothing()
+ .when(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockHandle);
+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));
+
+ manager.stopCluster(cluster);
+
+ assertNull(manager.getLogHandle(cluster));
+ }
+
+ @Test
+ void aFailedDeleteOverlappingASuccessfulOneLeavesNoLogHandleBehind() {
+ Cluster cluster = new Cluster();
+ cluster.setName("prod-cluster");
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));
+ manager.startCluster(cluster);
+ String containerId = "container-id-123456789012345678901234567890";
+ Mockito.doAnswer(failing -> {
+ manager.stopCluster(cluster);
+ throw new IllegalStateException("Failed to remove container");
+ }).doNothing().when(lifecycleManager).stopAndRemoveStrict(eq(containerId), any());
+
+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));
+
+ assertNull(manager.getLogHandle(cluster));
+ }
+
@Test
void logHandleIsReleasedOnClusterStop() {
Cluster cluster = new Cluster();
@@ -2790,7 +2918,7 @@ class EksClusterManagerTest {
manager.stopCluster(cluster);
- verify(lifecycleManager).stopAndRemove("container-id-123456789012345678901234567890", mockHandle);
+ verify(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockHandle);
assertNull(manager.getLogHandle(cluster));
}
@@ -2810,7 +2938,7 @@ class EksClusterManagerTest {
manager.startCluster(cluster);
manager.stopCluster(cluster);
- verify(lifecycleManager).stopAndRemove("container-id-123456789012345678901234567890", mockHandle);
+ verify(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockHandle);
assertNull(manager.getLogHandle(cluster));
}
@@ -2824,7 +2952,7 @@ class EksClusterManagerTest {
manager.detachCluster(cluster);
verify(mockHandle).close();
- verify(lifecycleManager, never()).stopAndRemove(anyString(), any());
+ verify(lifecycleManager, never()).stopAndRemoveStrict(anyString(), any());
assertNull(manager.getLogHandle(cluster));
}
@@ -2995,7 +3123,7 @@ class EksClusterManagerTest {
"us-east-1", "eks-audit:audit-cluster", false);
manager.stopCluster(cluster);
- verify(lifecycleManager).stopAndRemove("container-id-123456789012345678901234567890", mockAuditHandle);
+ verify(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockAuditHandle);
assertNull(manager.getLogHandle(cluster));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
index 983a3bfb49..3dc73be3be 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
@@ -62,6 +62,7 @@ import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -521,6 +522,50 @@ class EksServiceTest {
verify(clusterManager, never()).detachCluster(any(Cluster.class));
}
+ @Test
+ void aDeleteOverlappingOneStillInProgressIsRejectedAndCannotReviveTheCluster() {
+ Cluster cluster = activeCluster();
+ EksClusterManager clusterManager = mock(EksClusterManager.class);
+ EksService service = serviceWithCluster(cluster, clusterManager, false);
+ AwsException[] overlapping = new AwsException[1];
+ doAnswer(firstDelete -> {
+ overlapping[0] = assertThrows(AwsException.class, () -> service.deleteCluster("running-cluster"));
+ return null;
+ }).when(clusterManager).stopCluster(cluster);
+
+ service.deleteCluster("running-cluster");
+
+ assertEquals("ResourceInUseException", overlapping[0].getErrorCode());
+ assertThrows(AwsException.class, () -> service.describeCluster("running-cluster"));
+ }
+
+ @Test
+ void aClusterLeftDeletingCanBeDeletedAgainOnceTheEarlierDeleteFinished() {
+ Cluster cluster = activeCluster();
+ EksClusterManager clusterManager = mock(EksClusterManager.class);
+ EksService service = serviceWithCluster(cluster, clusterManager, false);
+ doThrow(new IllegalStateException("Failed to remove container"))
+ .doNothing()
+ .when(clusterManager).stopCluster(cluster);
+ assertThrows(IllegalStateException.class, () -> service.deleteCluster("running-cluster"));
+
+ service.deleteCluster("running-cluster");
+
+ assertThrows(AwsException.class, () -> service.describeCluster("running-cluster"));
+ }
+
+ @Test
+ void aDeleteThatCouldNotRemoveTheClusterContainerLeavesTheClusterActive() {
+ Cluster cluster = activeCluster();
+ EksClusterManager clusterManager = mock(EksClusterManager.class);
+ EksService service = serviceWithCluster(cluster, clusterManager, false);
+ doThrow(new IllegalStateException("Failed to remove container")).when(clusterManager).stopCluster(cluster);
+
+ assertThrows(IllegalStateException.class, () -> service.deleteCluster("running-cluster"));
+
+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("running-cluster").getStatus());
+ }
+
private static Cluster activeCluster() {
Cluster cluster = new Cluster();
cluster.setName("running-cluster");