Light, fluffy, and always free - The AWS Local Emulator alternative
fix(eks): fail the delete when Docker cannot remove the cluster container (#4960)
Niels Kootstrad5fd4d52f181
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 84ea66b552..de43846912 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -521,24 +521,18 @@ public class ContainerLifecycleManager {** <p>Most emulator shutdown paths intentionally use best-effort cleanup through* {@link #stopAndRemove(String, Closeable)}. Resource-deletion paths that must retain their- * persisted record for a retry use this stricter variant instead.+ * persisted record for a retry use this stricter variant instead. The log stream is closed+ * only once the container is gone; when removal fails it is left open for the retry.*/public void stopAndRemoveStrict(String containerId, Closeable logStream) {LOG.infov("Stopping container {0}", containerId);- if (logStream != null) {- try {- logStream.close();- } catch (Exception e) {- LOG.debugv("Error closing log stream: {0}", e.getMessage());- }- }-Exception stopFailure = null;try {dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();} catch (NotFoundException e) {LOG.debugv("Container {0} not found (already removed)", containerId);+ closeLogStreamIfPresent(logStream);return;} catch (Exception e) {stopFailure = e;@@ -556,8 +550,19 @@ public class ContainerLifecycleManager {if (stopFailure != null) {cleanupFailure.addSuppressed(stopFailure);}+ // The container still exists and may still be logging, so the stream stays open for+ // the caller's retry.throw cleanupFailure;}+ // Closed only once the container is stopped and gone, as in stopAndRemove, so the follower+ // can drain the final tail.+ closeLogStreamIfPresent(logStream);+ }++ private void closeLogStreamIfPresent(Closeable logStream) {+ if (logStream != null) {+ closeLogStreamAfterContainerStop(logStream);+ }}/**diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.javaindex 7d36f7f02e..ac4ea15efe 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java@@ -916,18 +916,32 @@ public class EksClusterManager* cluster's workloads survive a Floci restart and are re-latched by {@link #restoreCluster}.*/public void stopCluster(Cluster cluster) {- unregisterMetadataEndpoint(cluster);- Closeable logStream = clusterLogHandles.remove(clusterResourceName(cluster));+ String resourceName = clusterResourceName(cluster);if (cluster.getContainerId() == null) {- closeQuietly(logStream);+ unregisterMetadataEndpoint(cluster);+ closeQuietly(clusterLogHandles.remove(resourceName));return;}- lifecycleManager.stopAndRemove(cluster.getContainerId(), logStream);- if (cluster.getHostPort() > 0) {- portAllocator.release(cluster.getHostPort());- // Cleared so a delete retried after a failed backup cleanup cannot free a reused port.+ Closeable logStream = clusterLogHandles.get(resourceName);+ // Strict: a container Docker could not remove may still be running and publishing the+ // port, so the delete fails and keeps the cluster record, its metadata endpoint, log+ // handle and port reservation; a retried delete releases them once the container is gone.+ lifecycleManager.stopAndRemoveStrict(cluster.getContainerId(), logStream);+ unregisterMetadataEndpoint(cluster);+ // Removed by value so an overlapping delete cannot drop a handle it does not own.+ if (logStream != null) {+ clusterLogHandles.remove(resourceName, logStream);+ }+ int hostPort;+ // Read and cleared together so overlapping deletes, or a delete retried after a failed+ // backup cleanup, cannot release a port that has since been reused.+ synchronized (cluster) {+ hostPort = cluster.getHostPort();cluster.setHostPort(0);}+ if (hostPort > 0) {+ portAllocator.release(hostPort);+ }if (cluster.getDockerName() != null) {// A failed backup cleanup must not leave the live node running. Keep the cluster// record so explicit deletion can be retried once Docker accepts the removal.@@ -1084,9 +1098,10 @@ public class EksClusterManagerreturn;}String resourceName = clusterResourceName(cluster);- if (clusterLogHandles.containsKey(resourceName)) {- return;- }+ // Logs are attached only for a container that was just started or adopted, so a handle+ // still registered under this name follows an earlier container: close it rather than+ // leave the new cluster without control-plane logs.+ closeQuietly(clusterLogHandles.remove(resourceName));String logGroup = "/aws/eks/" + cluster.getName() + "/cluster";String hash = containerId.length() >= 32 ? containerId.substring(0, 32) : containerId;String region = clusterRegion(cluster);@@ -1124,13 +1139,13 @@ public class EksClusterManager}if (handles.size() == 1) {- clusterLogHandles.put(resourceName, handles.getFirst());+ closeQuietly(clusterLogHandles.put(resourceName, handles.getFirst()));} else if (handles.size() > 1) {- clusterLogHandles.put(resourceName, () -> {+ closeQuietly(clusterLogHandles.put(resourceName, () -> {for (Closeable h : handles) {closeQuietly(h);}- });+ }));}}diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.javaindex f23a5a8dbb..82c1954049 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java@@ -106,6 +106,8 @@ public class EksService implements TagHandler, ResourceProvider {private final Map<String, UserDataClaim> appliedClusterUserData = new ConcurrentHashMap<>();private final Map<String, Object> nodeGroupCapacityLocks = new ConcurrentHashMap<>();private final Map<String, CompletableFuture<Boolean>> pendingFirstNodeGroups = new ConcurrentHashMap<>();+ // Storage hands every caller the same live Cluster, and Cluster keeps identity equality.+ private final Set<Cluster> clustersBeingDeleted = ConcurrentHashMap.newKeySet();@Injectpublic EksService(StorageFactory storageFactory, EmulatorConfig config,@@ -650,22 +652,39 @@ public class EksService implements TagHandler, ResourceProvider {.orElseThrow(() -> new AwsException("ResourceNotFoundException","No cluster found for name: " + name, 404));- cluster.setStatus(ClusterStatus.DELETING);- if (!config.services().eks().mock()) {- clusterManager.stopCluster(cluster);- }- deleteClusterSecurityGroup(cluster);- accessEntries.deleteClusterEntries(cluster);- if (podIdentityAssociations != null) {- podIdentityAssociations.deleteClusterAssociations(cluster);+ // One delete at a time per cluster, so a delete that fails cannot restore the status or+ // state of a cluster another delete is tearing down.+ if (!clustersBeingDeleted.add(cluster)) {+ throw new AwsException("ResourceInUseException",+ "Cluster is already being deleted: " + name, 409);}- if (addons != null) {- addons.deleteClusterAddons(cluster);+ try {+ ClusterStatus previousStatus = cluster.getStatus();+ cluster.setStatus(ClusterStatus.DELETING);+ if (!config.services().eks().mock()) {+ try {+ clusterManager.stopCluster(cluster);+ } catch (RuntimeException e) {+ // The cluster record is kept for a retry, so it must not stay stuck in DELETING.+ cluster.setStatus(previousStatus);+ throw e;+ }+ }+ deleteClusterSecurityGroup(cluster);+ accessEntries.deleteClusterEntries(cluster);+ if (podIdentityAssociations != null) {+ podIdentityAssociations.deleteClusterAssociations(cluster);+ }+ if (addons != null) {+ addons.deleteClusterAddons(cluster);+ }+ storage.delete(name);+ clearAppliedClusterUserData(cluster, name);+ oidcService.deleteKey(name);+ return cluster;+ } finally {+ clustersBeingDeleted.remove(cluster);}- storage.delete(name);- clearAppliedClusterUserData(cluster, name);- oidcService.deleteKey(name);- return cluster;}public Nodegroup createNodeGroup(String clusterName, CreateNodeGroupRequest request) {diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.javaindex b31719c847..ff971a2f5f 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java@@ -25,6 +25,7 @@ import org.junit.jupiter.api.extension.ExtendWith;import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;+import java.io.Closeable;import java.io.IOException;import java.util.ArrayList;import java.util.List;@@ -170,6 +171,70 @@ class ContainerLifecycleManagerVolumeTest {assertFalse(manager.volumeExists("some-volume"));}+ @Test+ void strictContainerCleanupStopsContainerBeforeClosingItsLogStream() {+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);+ List<String> operations = new ArrayList<>();+ doAnswer(invocation -> {+ operations.add("stop");+ return null;+ }).when(stop).exec();+ doAnswer(invocation -> {+ operations.add("remove");+ return null;+ }).when(remove).exec();+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);++ manager.stopAndRemoveStrict("container-id", () -> operations.add("logs"));++ assertEquals(List.of("stop", "remove", "logs"), operations);+ }++ @Test+ void strictContainerCleanupClosesTheLogStreamOfAnAlreadyMissingContainer() {+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);+ when(stop.exec()).thenThrow(new NotFoundException("gone"));+ List<String> operations = new ArrayList<>();++ manager.stopAndRemoveStrict("container-id", () -> operations.add("logs"));++ assertEquals(List.of("logs"), operations);+ }++ @Test+ void strictContainerCleanupLeavesTheLogStreamOpenWhenDockerCouldNotStopOrRemoveTheContainer() {+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);+ when(stop.exec()).thenThrow(new DockerException("stop failed", 500));+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);+ when(remove.exec()).thenThrow(new DockerException("remove failed", 500));+ List<String> operations = new ArrayList<>();++ assertThrows(IllegalStateException.class, () ->+ manager.stopAndRemoveStrict("container-id", () -> operations.add("logs")));++ assertEquals(List.of(), operations);+ }++ @Test+ void strictContainerCleanupLeavesTheLogStreamOpenWhenDockerStoppedButCouldNotRemoveTheContainer() {+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);+ when(remove.exec()).thenThrow(new DockerException("remove failed", 500));+ List<String> operations = new ArrayList<>();++ assertThrows(IllegalStateException.class, () ->+ manager.stopAndRemoveStrict("container-id", () -> operations.add("logs")));++ assertEquals(List.of(), operations);+ }+@Testvoid strictContainerCleanupPropagatesRemovalFailure() {StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);@@ -197,6 +262,20 @@ class ContainerLifecycleManagerVolumeTest {verify(remove).exec();}+ @Test+ void strictContainerCleanupClosesTheLogStreamOnceForcedRemovalSucceedsAfterStopFailure() throws Exception {+ StopContainerCmd stop = mock(StopContainerCmd.class, RETURNS_SELF);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);+ Closeable logStream = mock(Closeable.class);+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);+ when(stop.exec()).thenThrow(new DockerException("stop failed", 500));+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);++ manager.stopAndRemoveStrict("container-id", logStream);++ verify(logStream).close();+ }+@Testvoid strictStaleContainerCleanupPropagatesRemovalFailure() {RemoveContainerCmd remove = mock(RemoveContainerCmd.class, RETURNS_SELF);diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.javaindex 5f925d5caa..1879bf674e 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java@@ -532,6 +532,33 @@ class EksClusterManagerTest {verify(portAllocator).release(6441);}+ @Test+ void deletingAClusterWhoseContainerDockerCouldNotRemoveKeepsItsPortReserved() {+ stubFreshStart("cid-1", 6441);+ Mockito.doThrow(new IllegalStateException("Failed to remove container cid-1"))+ .when(lifecycleManager).stopAndRemoveStrict("cid-1", null);+ Cluster cluster = cluster();+ manager.startCluster(cluster);++ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));++ verify(portAllocator, never()).release(6441);+ }++ @Test+ void retryingADeleteAfterContainerRemovalFailedHandsThePortBack() {+ stubFreshStart("cid-1", 6441);+ Mockito.doThrow(new IllegalStateException("Failed to remove container cid-1")).doNothing()+ .when(lifecycleManager).stopAndRemoveStrict("cid-1", null);+ Cluster cluster = cluster();+ manager.startCluster(cluster);++ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));+ manager.stopCluster(cluster);++ verify(portAllocator, Mockito.times(1)).release(6441);+ }+@Testvoid retryingADeleteAfterBackupCleanupFailedHandsThePortBackOnlyOnce() {Cluster cluster = cluster();@@ -554,6 +581,7 @@ class EksClusterManagerTest {Cluster cluster = cluster();assertThrows(IllegalStateException.class, () -> manager.startCluster(cluster));+ assertEquals(0, cluster.getHostPort());manager.stopCluster(cluster);verify(portAllocator, Mockito.times(1)).release(6441);@@ -584,7 +612,7 @@ class EksClusterManagerTest {verify(lifecycleManager, Mockito.times(3)).removeIfExistsStrict("floci-aws-eks-capacity-backup.foo");verify(lifecycleManager, never()).removeIfExistsStrict(other.getDockerName());- verify(lifecycleManager, never()).stopAndRemove("cid-other", null);+ verify(lifecycleManager, never()).stopAndRemoveStrict("cid-other", null);}@ParameterizedTest@@ -604,7 +632,7 @@ class EksClusterManagerTest {manager.stopCluster(cluster);- verify(lifecycleManager).stopAndRemove("cid-numeric", null);+ verify(lifecycleManager).stopAndRemoveStrict("cid-numeric", null);verify(lifecycleManager).removeIfExistsStrict("floci-aws-eks-capacity-backup.999999999999");verify(lifecycleManager, never()).removeIfExistsStrict(otherDockerName);}@@ -752,7 +780,7 @@ class EksClusterManagerTest {cluster.setContainerId("cid-1");manager.stopCluster(cluster);- verify(lifecycleManager).stopAndRemove("cid-1", null);+ verify(lifecycleManager).stopAndRemoveStrict("cid-1", null);// The volume must survive so restoreCluster can bring the workloads back.verify(lifecycleManager, never()).removeVolume(anyString());}@@ -767,7 +795,7 @@ class EksClusterManagerTest {assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));- verify(lifecycleManager).stopAndRemove("cid-new", null);+ verify(lifecycleManager).stopAndRemoveStrict("cid-new", null);verify(lifecycleManager, never()).removeVolume(anyString());}@@ -779,7 +807,7 @@ class EksClusterManagerTest {cluster.setContainerId("cid-1");manager.stopCluster(cluster);- verify(lifecycleManager).stopAndRemove("cid-1", null);+ verify(lifecycleManager).stopAndRemoveStrict("cid-1", null);verify(lifecycleManager).removeVolume("floci-aws-eks-demo");}@@ -1612,6 +1640,40 @@ class EksClusterManagerTest {verify(metadataServer).unregisterInstance(any());assertNull(manager.getRegisteredClusterNodeInstance(cluster));}++ @Test+ void aDeleteThatCouldNotRemoveTheContainerKeepsItsNodeOnTheMetadataServer() {+ Cluster cluster = new Cluster();+ cluster.setName("test-cluster");+ manager.configureLinkLocalMetadataEndpoint(cluster, "container-42");+ cluster.setContainerId("container-42");+ Mockito.doThrow(new IllegalStateException("Failed to remove container container-42"))+ .when(lifecycleManager).stopAndRemoveStrict("container-42", null);++ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));++ verify(metadataServer, never()).unregisterInstance(any());+ assertNotNull(manager.getRegisteredClusterNodeInstance(cluster));+ }++ @Test+ void retryingADeleteAfterContainerRemovalFailedRemovesTheNodeFromTheMetadataServer() {+ EmulatorConfig.StorageConfig storage = Mockito.mock(EmulatorConfig.StorageConfig.class);+ when(config.storage()).thenReturn(storage);+ when(storage.mode()).thenReturn("memory");+ Cluster cluster = new Cluster();+ cluster.setName("test-cluster");+ manager.configureLinkLocalMetadataEndpoint(cluster, "container-42");+ cluster.setContainerId("container-42");+ Mockito.doThrow(new IllegalStateException("Failed to remove container container-42"))+ .doNothing()+ .when(lifecycleManager).stopAndRemoveStrict("container-42", null);+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));++ manager.stopCluster(cluster);++ assertNull(manager.getRegisteredClusterNodeInstance(cluster));+ }}@Nested@@ -2779,6 +2841,72 @@ class EksClusterManagerTest {assertEquals("container-id-123456789012345678901234567890", cluster.getContainerId());}+ @Test+ void logHandleSurvivesADeleteThatCouldNotRemoveTheContainer() {+ Cluster cluster = new Cluster();+ cluster.setName("prod-cluster");+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));+ manager.startCluster(cluster);+ Mockito.doThrow(new IllegalStateException("Failed to remove container"))+ .when(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockHandle);++ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));++ assertEquals(mockHandle, manager.getLogHandle(cluster));+ }++ @Test+ void aRecreatedClusterReplacesALogHandleLeftBehindUnderItsName() throws Exception {+ Cluster stale = new Cluster();+ stale.setName("prod-cluster");+ stale.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));+ manager.startCluster(stale);+ Closeable freshHandle = Mockito.mock(Closeable.class);+ when(logStreamer.attachForAccount(anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))+ .thenReturn(freshHandle);+ Cluster recreated = new Cluster();+ recreated.setName("prod-cluster");+ recreated.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));++ manager.startCluster(recreated);++ verify(mockHandle).close();+ assertEquals(freshHandle, manager.getLogHandle(recreated));+ }++ @Test+ void retryingADeleteAfterContainerRemovalFailedReleasesTheLogHandle() {+ Cluster cluster = new Cluster();+ cluster.setName("prod-cluster");+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));+ manager.startCluster(cluster);+ Mockito.doThrow(new IllegalStateException("Failed to remove container"))+ .doNothing()+ .when(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockHandle);+ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));++ manager.stopCluster(cluster);++ assertNull(manager.getLogHandle(cluster));+ }++ @Test+ void aFailedDeleteOverlappingASuccessfulOneLeavesNoLogHandleBehind() {+ Cluster cluster = new Cluster();+ cluster.setName("prod-cluster");+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));+ manager.startCluster(cluster);+ String containerId = "container-id-123456789012345678901234567890";+ Mockito.doAnswer(failing -> {+ manager.stopCluster(cluster);+ throw new IllegalStateException("Failed to remove container");+ }).doNothing().when(lifecycleManager).stopAndRemoveStrict(eq(containerId), any());++ assertThrows(IllegalStateException.class, () -> manager.stopCluster(cluster));++ assertNull(manager.getLogHandle(cluster));+ }+@Testvoid logHandleIsReleasedOnClusterStop() {Cluster cluster = new Cluster();@@ -2790,7 +2918,7 @@ class EksClusterManagerTest {manager.stopCluster(cluster);- verify(lifecycleManager).stopAndRemove("container-id-123456789012345678901234567890", mockHandle);+ verify(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockHandle);assertNull(manager.getLogHandle(cluster));}@@ -2810,7 +2938,7 @@ class EksClusterManagerTest {manager.startCluster(cluster);manager.stopCluster(cluster);- verify(lifecycleManager).stopAndRemove("container-id-123456789012345678901234567890", mockHandle);+ verify(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockHandle);assertNull(manager.getLogHandle(cluster));}@@ -2824,7 +2952,7 @@ class EksClusterManagerTest {manager.detachCluster(cluster);verify(mockHandle).close();- verify(lifecycleManager, never()).stopAndRemove(anyString(), any());+ verify(lifecycleManager, never()).stopAndRemoveStrict(anyString(), any());assertNull(manager.getLogHandle(cluster));}@@ -2995,7 +3123,7 @@ class EksClusterManagerTest {"us-east-1", "eks-audit:audit-cluster", false);manager.stopCluster(cluster);- verify(lifecycleManager).stopAndRemove("container-id-123456789012345678901234567890", mockAuditHandle);+ verify(lifecycleManager).stopAndRemoveStrict("container-id-123456789012345678901234567890", mockAuditHandle);assertNull(manager.getLogHandle(cluster));}diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.javaindex 983a3bfb49..3dc73be3be 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java@@ -62,6 +62,7 @@ import java.util.stream.Collectors;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;@@ -521,6 +522,50 @@ class EksServiceTest {verify(clusterManager, never()).detachCluster(any(Cluster.class));}+ @Test+ void aDeleteOverlappingOneStillInProgressIsRejectedAndCannotReviveTheCluster() {+ Cluster cluster = activeCluster();+ EksClusterManager clusterManager = mock(EksClusterManager.class);+ EksService service = serviceWithCluster(cluster, clusterManager, false);+ AwsException[] overlapping = new AwsException[1];+ doAnswer(firstDelete -> {+ overlapping[0] = assertThrows(AwsException.class, () -> service.deleteCluster("running-cluster"));+ return null;+ }).when(clusterManager).stopCluster(cluster);++ service.deleteCluster("running-cluster");++ assertEquals("ResourceInUseException", overlapping[0].getErrorCode());+ assertThrows(AwsException.class, () -> service.describeCluster("running-cluster"));+ }++ @Test+ void aClusterLeftDeletingCanBeDeletedAgainOnceTheEarlierDeleteFinished() {+ Cluster cluster = activeCluster();+ EksClusterManager clusterManager = mock(EksClusterManager.class);+ EksService service = serviceWithCluster(cluster, clusterManager, false);+ doThrow(new IllegalStateException("Failed to remove container"))+ .doNothing()+ .when(clusterManager).stopCluster(cluster);+ assertThrows(IllegalStateException.class, () -> service.deleteCluster("running-cluster"));++ service.deleteCluster("running-cluster");++ assertThrows(AwsException.class, () -> service.describeCluster("running-cluster"));+ }++ @Test+ void aDeleteThatCouldNotRemoveTheClusterContainerLeavesTheClusterActive() {+ Cluster cluster = activeCluster();+ EksClusterManager clusterManager = mock(EksClusterManager.class);+ EksService service = serviceWithCluster(cluster, clusterManager, false);+ doThrow(new IllegalStateException("Failed to remove container")).when(clusterManager).stopCluster(cluster);++ assertThrows(IllegalStateException.class, () -> service.deleteCluster("running-cluster"));++ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("running-cluster").getStatus());+ }+private static Cluster activeCluster() {Cluster cluster = new Cluster();cluster.setName("running-cluster");