Light, fluffy, and always free - The AWS Local Emulator alternative
fix(eks): let a cluster delete wait for a running node capacity swap (#6434)
Robin Baeckman17c738403c45
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.javaindex 6feb748815..57b50e76be 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java@@ -397,6 +397,12 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun}void deleteClusterForAccount(String accountId, String clusterName) {+ synchronized (capacityLock(accountId, clusterName)) {+ deleteClusterForAccountHoldingCapacity(accountId, clusterName);+ }+ }++ private void deleteClusterForAccountHoldingCapacity(String accountId, String clusterName) {Cluster cluster = (storage instanceof AccountAwareStorageBackend<Cluster> aware)? aware.getForAccount(accountId, clusterName).orElse(null): storage.get(clusterName).orElse(null);@@ -853,15 +859,24 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun}public Cluster deleteCluster(String name) {- Cluster cluster = storage.get(name)- .orElseThrow(() -> new AwsException("ResourceNotFoundException",- "No cluster found for name: " + name, 404));- if (cluster.getAccountId() == null) {- cluster.setAccountId(regionResolver.getAccountId());+ // A first node group replacing the node holds this lock while the swap runs; deleting+ // meanwhile would find no container to stop and have its cluster written back. Read the+ // cluster only once the lock is held: a delete that waited may find it gone.+ synchronized (capacityLock(regionResolver.getAccountId(), name)) {+ Cluster cluster = storage.get(name)+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "No cluster found for name: " + name, 404));+ if (cluster.getAccountId() == null) {+ cluster.setAccountId(regionResolver.getAccountId());+ }+ return deleteClusterHoldingCapacity(name, cluster);}+ }+ private Cluster deleteClusterHoldingCapacity(String name, Cluster cluster) {// One delete at a time per cluster, so a delete that fails cannot restore the status or- // state of a cluster another delete is tearing down.+ // state of a cluster another delete is tearing down. Deletes from other threads already+ // wait on the capacity lock; this refuses one nested on the same thread.if (!clustersBeingDeleted.add(cluster)) {throw new AwsException("ResourceInUseException","Cluster is already being deleted: " + name, 409);@@ -983,8 +998,7 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaunnodeGroup.setTags(request.getTags() != null ? new HashMap<>(request.getTags()) : new HashMap<>());String capacityKey = accountId + "/" + clusterName;- Object capacityLock = nodeGroupCapacityLocks.computeIfAbsent(capacityKey,- ignored -> new Object());+ Object capacityLock = capacityLock(accountId, clusterName);boolean hasUserData = launchTemplateData != null && launchTemplateData.getUserData() != null&& !launchTemplateData.getUserData().isBlank();Cluster currentCluster;@@ -1172,9 +1186,13 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun.collect(Collectors.toList());}+ /** Serializes node group capacity changes and cluster deletion; taken before clustersBeingDeleted. */+ private Object capacityLock(String accountId, String clusterName) {+ return nodeGroupCapacityLocks.computeIfAbsent(accountId + "/" + clusterName, ignored -> new Object());+ }+public Nodegroup deleteNodeGroup(String clusterName, String nodegroupName) {- Object capacityLock = nodeGroupCapacityLocks.computeIfAbsent(- regionResolver.getAccountId() + "/" + clusterName, ignored -> new Object());+ Object capacityLock = capacityLock(regionResolver.getAccountId(), clusterName);synchronized (capacityLock) {Nodegroup nodeGroup = describeNodeGroup(clusterName, nodegroupName);nodeGroup.setStatus(NodegroupStatus.DELETING);diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.javaindex fdd7de83ed..dd722d9e80 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java@@ -42,6 +42,7 @@ import io.github.hectorvent.floci.services.eks.model.NodegroupStatus;import io.github.hectorvent.floci.services.eks.model.OidcIdentity;import io.github.hectorvent.floci.services.eks.model.Provider;import io.github.hectorvent.floci.services.eks.model.ResourcesVpcConfig;+import io.github.hectorvent.floci.testing.ThreadStates;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;@@ -50,15 +51,19 @@ import java.lang.reflect.InvocationHandler;import java.lang.reflect.Method;import java.lang.reflect.Proxy;import java.nio.file.Path;+import java.time.Duration;+import java.util.EnumSet;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.concurrent.CopyOnWriteArrayList;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;import java.util.stream.Collectors;import static org.junit.jupiter.api.Assertions.*;@@ -560,6 +565,9 @@ class EksServiceTest {EksService service = serviceWithCluster(cluster, clusterManager, false);AwsException[] overlapping = new AwsException[1];doAnswer(firstDelete -> {+ // Nested on the same thread, so the reentrant capacity lock lets it through and the+ // clustersBeingDeleted check refuses it. Another thread waits instead; see+ // aSecondDeleteWaitsAndFindsTheClusterGone.overlapping[0] = assertThrows(AwsException.class, () -> service.deleteCluster("running-cluster"));return null;}).when(clusterManager).stopCluster(cluster);@@ -1056,6 +1064,106 @@ class EksServiceTest {assertThrows(AwsException.class, () -> service.describeCluster("deleted-node"));}+ @Test+ void deleteIssuedDuringTheFirstNodeGroupSwapWaitsAndThenDeletesCleanly() throws Exception {+ EksClusterManager clusterManager = capacitySwapManager(true);+ EksService service = newService(clusterManager, false);+ activeClusterWithRunningNode(service, "deleted-during-swap");+ List<String> stoppedContainers = new CopyOnWriteArrayList<>();+ doAnswer(invocation -> {+ stoppedContainers.add(String.valueOf(invocation.<Cluster>getArgument(0).getContainerId()));+ return null;+ }).when(clusterManager).stopCluster(any());+ AtomicReference<Throwable> deleteError = new AtomicReference<>();+ AtomicReference<Thread.State> deleterDuringSwap = new AtomicReference<>();+ Thread deleter = deleteThread(service, "deleted-during-swap", deleteError);+ when(clusterManager.restartForNodeCapacity(any(), anyString())).thenAnswer(invocation -> {+ // As replaceClusterContainer does: no container while the replacement starts.+ Cluster swapping = invocation.getArgument(0);+ swapping.setContainerId(null);+ deleter.start();+ deleterDuringSwap.set(awaitBlockedOrTerminated(deleter, "the delete during the swap"));+ swapping.setContainerId("replacement-container");+ return true;+ });++ Nodegroup group = service.createNodeGroup("deleted-during-swap", nodeGroupRequest("first"));+ deleter.join(5000);++ assertEquals(Thread.State.BLOCKED, deleterDuringSwap.get());+ assertFalse(deleter.isAlive());+ assertNull(deleteError.get());+ assertEquals(NodegroupStatus.ACTIVE, group.getStatus());+ assertTrue(createdStorage("eks-clusters.json").get("deleted-during-swap").isEmpty());+ assertEquals(List.of("replacement-container"), stoppedContainers);+ }++ @Test+ void aSecondDeleteWaitsAndFindsTheClusterGone() throws Exception {+ EksClusterManager clusterManager = capacitySwapManager(true);+ EksService service = newService(clusterManager, false);+ activeClusterWithRunningNode(service, "deleted-twice");+ CountDownLatch firstStopping = new CountDownLatch(1);+ CountDownLatch releaseFirst = new CountDownLatch(1);+ doAnswer(invocation -> {+ firstStopping.countDown();+ assertTrue(releaseFirst.await(5, TimeUnit.SECONDS));+ return null;+ }).when(clusterManager).stopCluster(any());+ AtomicReference<Throwable> firstError = new AtomicReference<>();+ AtomicReference<Throwable> secondError = new AtomicReference<>();+ Thread first = deleteThread(service, "deleted-twice", firstError);+ Thread second = deleteThread(service, "deleted-twice", secondError);++ first.start();+ assertTrue(firstStopping.await(5, TimeUnit.SECONDS));+ second.start();+ Thread.State secondWhileFirstRuns = awaitBlockedOrTerminated(second, "the second delete");+ releaseFirst.countDown();+ first.join(5000);+ second.join(5000);++ assertEquals(Thread.State.BLOCKED, secondWhileFirstRuns);+ assertFalse(first.isAlive());+ assertFalse(second.isAlive());+ assertNull(firstError.get());+ assertTrue(secondError.get() instanceof AwsException e+ && "ResourceNotFoundException".equals(e.getErrorCode()), String.valueOf(secondError.get()));+ verify(clusterManager, times(1)).stopCluster(any());+ assertTrue(createdStorage("eks-clusters.json").get("deleted-twice").isEmpty());+ }++ /** An unstarted thread that deletes {@code name} and keeps what it threw in {@code error}. */+ private static Thread deleteThread(EksService service, String name, AtomicReference<Throwable> error) {+ return Thread.ofPlatform().unstarted(() -> {+ try {+ service.deleteCluster(name);+ } catch (Throwable e) {+ error.set(e);+ }+ });+ }++ // BLOCKED on any monitor counts; the assertions after it show the wait was on the capacity lock.+ private static Thread.State awaitBlockedOrTerminated(Thread thread, String what) {+ return ThreadStates.awaitState(thread, EnumSet.of(Thread.State.BLOCKED, Thread.State.TERMINATED),+ Duration.ofSeconds(5), what);+ }++ @Test+ void firstNodeGroupSwapOnALiveClusterStoresBoth() {+ EksClusterManager clusterManager = capacitySwapManager(true);+ EksService service = newService(clusterManager, false);+ activeClusterWithRunningNode(service, "kept-during-swap");++ Nodegroup group = service.createNodeGroup("kept-during-swap", nodeGroupRequest("first"));++ assertEquals(NodegroupStatus.ACTIVE, group.getStatus());+ Cluster stored = (Cluster) createdStorage("eks-clusters.json").get("kept-during-swap").orElseThrow();+ assertTrue(stored.isFinalizePending());+ assertTrue(createdStorage("eks-nodegroups.json").get("kept-during-swap/first").isPresent());+ }+private static Cluster activeClusterWithRunningNode(EksService service, String name) {CreateClusterRequest request = new CreateClusterRequest();request.setName(name);@@ -1316,12 +1424,21 @@ class EksServiceTest {assertEquals(404, delete.getHttpStatus());}+ private final Map<String, AccountAwareStorageBackend<?>> createdStorages = new HashMap<>();++ @SuppressWarnings("unchecked")+ private <V> AccountAwareStorageBackend<V> createdStorage(String fileName) {+ return (AccountAwareStorageBackend<V>) createdStorages.get(fileName);+ }+private EksService newService(EksClusterManager clusterManager, boolean mock) {StorageFactory storageFactory = new StorageFactory(null, null) {@Overridepublic <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,TypeReference<Map<String, V>> typeReference) {- return AccountAwareStorageBackend.inMemory("000000000000");+ AccountAwareStorageBackend<V> backend = AccountAwareStorageBackend.inMemory("000000000000");+ createdStorages.put(fileName, backend);+ return backend;}};return new EksService(storageFactory, testConfig(mock),