Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(eks): let a cluster delete wait for a running node capacity swap (#6434)

Robin Baeckman17c738403c45
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
index 6feb748815..57b50e76be 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
@@ -397,6 +397,12 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun
}
void deleteClusterForAccount(String accountId, String clusterName) {
+ synchronized (capacityLock(accountId, clusterName)) {
+ deleteClusterForAccountHoldingCapacity(accountId, clusterName);
+ }
+ }
+
+ private void deleteClusterForAccountHoldingCapacity(String accountId, String clusterName) {
Cluster cluster = (storage instanceof AccountAwareStorageBackend<Cluster> aware)
? aware.getForAccount(accountId, clusterName).orElse(null)
: storage.get(clusterName).orElse(null);
@@ -853,15 +859,24 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun
}
public Cluster deleteCluster(String name) {
- Cluster cluster = storage.get(name)
- .orElseThrow(() -> new AwsException("ResourceNotFoundException",
- "No cluster found for name: " + name, 404));
- if (cluster.getAccountId() == null) {
- cluster.setAccountId(regionResolver.getAccountId());
+ // A first node group replacing the node holds this lock while the swap runs; deleting
+ // meanwhile would find no container to stop and have its cluster written back. Read the
+ // cluster only once the lock is held: a delete that waited may find it gone.
+ synchronized (capacityLock(regionResolver.getAccountId(), name)) {
+ Cluster cluster = storage.get(name)
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",
+ "No cluster found for name: " + name, 404));
+ if (cluster.getAccountId() == null) {
+ cluster.setAccountId(regionResolver.getAccountId());
+ }
+ return deleteClusterHoldingCapacity(name, cluster);
}
+ }
+ private Cluster deleteClusterHoldingCapacity(String name, Cluster cluster) {
// One delete at a time per cluster, so a delete that fails cannot restore the status or
- // state of a cluster another delete is tearing down.
+ // state of a cluster another delete is tearing down. Deletes from other threads already
+ // wait on the capacity lock; this refuses one nested on the same thread.
if (!clustersBeingDeleted.add(cluster)) {
throw new AwsException("ResourceInUseException",
"Cluster is already being deleted: " + name, 409);
@@ -983,8 +998,7 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun
nodeGroup.setTags(request.getTags() != null ? new HashMap<>(request.getTags()) : new HashMap<>());
String capacityKey = accountId + "/" + clusterName;
- Object capacityLock = nodeGroupCapacityLocks.computeIfAbsent(capacityKey,
- ignored -> new Object());
+ Object capacityLock = capacityLock(accountId, clusterName);
boolean hasUserData = launchTemplateData != null && launchTemplateData.getUserData() != null
&& !launchTemplateData.getUserData().isBlank();
Cluster currentCluster;
@@ -1172,9 +1186,13 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun
.collect(Collectors.toList());
}
+ /** Serializes node group capacity changes and cluster deletion; taken before clustersBeingDeleted. */
+ private Object capacityLock(String accountId, String clusterName) {
+ return nodeGroupCapacityLocks.computeIfAbsent(accountId + "/" + clusterName, ignored -> new Object());
+ }
+
public Nodegroup deleteNodeGroup(String clusterName, String nodegroupName) {
- Object capacityLock = nodeGroupCapacityLocks.computeIfAbsent(
- regionResolver.getAccountId() + "/" + clusterName, ignored -> new Object());
+ Object capacityLock = capacityLock(regionResolver.getAccountId(), clusterName);
synchronized (capacityLock) {
Nodegroup nodeGroup = describeNodeGroup(clusterName, nodegroupName);
nodeGroup.setStatus(NodegroupStatus.DELETING);
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
index fdd7de83ed..dd722d9e80 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
@@ -42,6 +42,7 @@ import io.github.hectorvent.floci.services.eks.model.NodegroupStatus;
import io.github.hectorvent.floci.services.eks.model.OidcIdentity;
import io.github.hectorvent.floci.services.eks.model.Provider;
import io.github.hectorvent.floci.services.eks.model.ResourcesVpcConfig;
+import io.github.hectorvent.floci.testing.ThreadStates;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -50,15 +51,19 @@ import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.nio.file.Path;
+import java.time.Duration;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
@@ -560,6 +565,9 @@ class EksServiceTest {
EksService service = serviceWithCluster(cluster, clusterManager, false);
AwsException[] overlapping = new AwsException[1];
doAnswer(firstDelete -> {
+ // Nested on the same thread, so the reentrant capacity lock lets it through and the
+ // clustersBeingDeleted check refuses it. Another thread waits instead; see
+ // aSecondDeleteWaitsAndFindsTheClusterGone.
overlapping[0] = assertThrows(AwsException.class, () -> service.deleteCluster("running-cluster"));
return null;
}).when(clusterManager).stopCluster(cluster);
@@ -1056,6 +1064,106 @@ class EksServiceTest {
assertThrows(AwsException.class, () -> service.describeCluster("deleted-node"));
}
+ @Test
+ void deleteIssuedDuringTheFirstNodeGroupSwapWaitsAndThenDeletesCleanly() throws Exception {
+ EksClusterManager clusterManager = capacitySwapManager(true);
+ EksService service = newService(clusterManager, false);
+ activeClusterWithRunningNode(service, "deleted-during-swap");
+ List<String> stoppedContainers = new CopyOnWriteArrayList<>();
+ doAnswer(invocation -> {
+ stoppedContainers.add(String.valueOf(invocation.<Cluster>getArgument(0).getContainerId()));
+ return null;
+ }).when(clusterManager).stopCluster(any());
+ AtomicReference<Throwable> deleteError = new AtomicReference<>();
+ AtomicReference<Thread.State> deleterDuringSwap = new AtomicReference<>();
+ Thread deleter = deleteThread(service, "deleted-during-swap", deleteError);
+ when(clusterManager.restartForNodeCapacity(any(), anyString())).thenAnswer(invocation -> {
+ // As replaceClusterContainer does: no container while the replacement starts.
+ Cluster swapping = invocation.getArgument(0);
+ swapping.setContainerId(null);
+ deleter.start();
+ deleterDuringSwap.set(awaitBlockedOrTerminated(deleter, "the delete during the swap"));
+ swapping.setContainerId("replacement-container");
+ return true;
+ });
+
+ Nodegroup group = service.createNodeGroup("deleted-during-swap", nodeGroupRequest("first"));
+ deleter.join(5000);
+
+ assertEquals(Thread.State.BLOCKED, deleterDuringSwap.get());
+ assertFalse(deleter.isAlive());
+ assertNull(deleteError.get());
+ assertEquals(NodegroupStatus.ACTIVE, group.getStatus());
+ assertTrue(createdStorage("eks-clusters.json").get("deleted-during-swap").isEmpty());
+ assertEquals(List.of("replacement-container"), stoppedContainers);
+ }
+
+ @Test
+ void aSecondDeleteWaitsAndFindsTheClusterGone() throws Exception {
+ EksClusterManager clusterManager = capacitySwapManager(true);
+ EksService service = newService(clusterManager, false);
+ activeClusterWithRunningNode(service, "deleted-twice");
+ CountDownLatch firstStopping = new CountDownLatch(1);
+ CountDownLatch releaseFirst = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ firstStopping.countDown();
+ assertTrue(releaseFirst.await(5, TimeUnit.SECONDS));
+ return null;
+ }).when(clusterManager).stopCluster(any());
+ AtomicReference<Throwable> firstError = new AtomicReference<>();
+ AtomicReference<Throwable> secondError = new AtomicReference<>();
+ Thread first = deleteThread(service, "deleted-twice", firstError);
+ Thread second = deleteThread(service, "deleted-twice", secondError);
+
+ first.start();
+ assertTrue(firstStopping.await(5, TimeUnit.SECONDS));
+ second.start();
+ Thread.State secondWhileFirstRuns = awaitBlockedOrTerminated(second, "the second delete");
+ releaseFirst.countDown();
+ first.join(5000);
+ second.join(5000);
+
+ assertEquals(Thread.State.BLOCKED, secondWhileFirstRuns);
+ assertFalse(first.isAlive());
+ assertFalse(second.isAlive());
+ assertNull(firstError.get());
+ assertTrue(secondError.get() instanceof AwsException e
+ && "ResourceNotFoundException".equals(e.getErrorCode()), String.valueOf(secondError.get()));
+ verify(clusterManager, times(1)).stopCluster(any());
+ assertTrue(createdStorage("eks-clusters.json").get("deleted-twice").isEmpty());
+ }
+
+ /** An unstarted thread that deletes {@code name} and keeps what it threw in {@code error}. */
+ private static Thread deleteThread(EksService service, String name, AtomicReference<Throwable> error) {
+ return Thread.ofPlatform().unstarted(() -> {
+ try {
+ service.deleteCluster(name);
+ } catch (Throwable e) {
+ error.set(e);
+ }
+ });
+ }
+
+ // BLOCKED on any monitor counts; the assertions after it show the wait was on the capacity lock.
+ private static Thread.State awaitBlockedOrTerminated(Thread thread, String what) {
+ return ThreadStates.awaitState(thread, EnumSet.of(Thread.State.BLOCKED, Thread.State.TERMINATED),
+ Duration.ofSeconds(5), what);
+ }
+
+ @Test
+ void firstNodeGroupSwapOnALiveClusterStoresBoth() {
+ EksClusterManager clusterManager = capacitySwapManager(true);
+ EksService service = newService(clusterManager, false);
+ activeClusterWithRunningNode(service, "kept-during-swap");
+
+ Nodegroup group = service.createNodeGroup("kept-during-swap", nodeGroupRequest("first"));
+
+ assertEquals(NodegroupStatus.ACTIVE, group.getStatus());
+ Cluster stored = (Cluster) createdStorage("eks-clusters.json").get("kept-during-swap").orElseThrow();
+ assertTrue(stored.isFinalizePending());
+ assertTrue(createdStorage("eks-nodegroups.json").get("kept-during-swap/first").isPresent());
+ }
+
private static Cluster activeClusterWithRunningNode(EksService service, String name) {
CreateClusterRequest request = new CreateClusterRequest();
request.setName(name);
@@ -1316,12 +1424,21 @@ class EksServiceTest {
assertEquals(404, delete.getHttpStatus());
}
+ private final Map<String, AccountAwareStorageBackend<?>> createdStorages = new HashMap<>();
+
+ @SuppressWarnings("unchecked")
+ private <V> AccountAwareStorageBackend<V> createdStorage(String fileName) {
+ return (AccountAwareStorageBackend<V>) createdStorages.get(fileName);
+ }
+
private EksService newService(EksClusterManager clusterManager, boolean mock) {
StorageFactory storageFactory = new StorageFactory(null, null) {
@Override
public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
TypeReference<Map<String, V>> typeReference) {
- return AccountAwareStorageBackend.inMemory("000000000000");
+ AccountAwareStorageBackend<V> backend = AccountAwareStorageBackend.inMemory("000000000000");
+ createdStorages.put(fileName, backend);
+ return backend;
}
};
return new EksService(storageFactory, testConfig(mock),