Light, fluffy, and always free - The AWS Local Emulator alternative
fix(eks): keep cluster ACTIVE when creating first node group (#5521)
Prakhar Kapoor7c122718f9ec
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.javaindex 71484a0565..a16b1b6e9f 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java@@ -1052,13 +1052,16 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaunString previousType = EksClusterManager.nodeInstanceType(currentCluster);currentCluster.setNodeInstanceType(selectedType);if (currentCluster.getContainerId() != null && !selectedType.equals(previousType)) {- currentCluster.setStatus(ClusterStatus.CREATING);try {if (!clusterManager.restartForNodeCapacity(currentCluster, previousType)) {nodeGroup.setStatus(NodegroupStatus.CREATE_FAILED);nodeGroup.setHealth(failedNodeGroupHealth(nodegroupName,"Could not apply node capacity; the previous node was restored"));}+ // Set only now: before the swap the poller would verify the old container, still+ // answering while it stops. A failed swap restarts the previous node, so it is+ // verified too.+ currentCluster.setFinalizePending(true);} catch (RuntimeException e) {currentCluster.setNodeInstanceType(previousType);currentCluster.setStatus(ClusterStatus.FAILED);@@ -1603,26 +1606,41 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun}private void startReadinessPoller() {- poller.scheduleAtFixedRate(() -> {- try {- for (AccountAwareStorageBackend.AccountEntry<Cluster> entry : allClusterEntries()) {- Cluster cluster = entry.value();- if (cluster.getStatus() == ClusterStatus.CREATING) {- if (clusterManager.isReady(cluster)) {- LOG.infov("EKS cluster {0} is now ACTIVE", cluster.getName());- clusterManager.finalizeCluster(cluster);+ poller.scheduleAtFixedRate(this::pollClusterReadiness, 2, 3, TimeUnit.SECONDS);+ }++ void pollClusterReadiness() {+ try {+ for (AccountAwareStorageBackend.AccountEntry<Cluster> entry : allClusterEntries()) {+ Cluster cluster = entry.value();+ boolean replacedNode = cluster.getStatus() == ClusterStatus.ACTIVE && cluster.isFinalizePending();+ if (cluster.getStatus() == ClusterStatus.CREATING || replacedNode) {+ if (clusterManager.isReady(cluster)) {+ clusterManager.finalizeCluster(cluster);+ // A delete that ran while the API server was probed must not get its+ // cluster written back.+ synchronized (clustersBeingDeleted) {+ if (isClusterDeleting(entry.accountId(), cluster)) {+ continue;+ }+ cluster.setFinalizePending(false);cluster.setStatus(ClusterStatus.ACTIVE);putCluster(cluster);- restoreJoinedInstancesAsync(entry.accountId(), cluster);}- } else if (cluster.getStatus() == ClusterStatus.ACTIVE && hasPendingJoinedInstances(cluster)) {+ if (replacedNode) {+ LOG.infov("Replaced k3s node of EKS cluster {0} is ready", cluster.getName());+ } else {+ LOG.infov("EKS cluster {0} is now ACTIVE", cluster.getName());+ }restoreJoinedInstancesAsync(entry.accountId(), cluster);}+ } else if (cluster.getStatus() == ClusterStatus.ACTIVE && hasPendingJoinedInstances(cluster)) {+ restoreJoinedInstancesAsync(entry.accountId(), cluster);}- } catch (Exception e) {- LOG.error("Error in EKS readiness poller", e);}- }, 2, 3, TimeUnit.SECONDS);+ } catch (Exception e) {+ LOG.error("Error in EKS readiness poller", e);+ }}private void restoreJoinedInstancesAsync(String accountId, Cluster cluster) {diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java b/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.javaindex 3886fee9b0..d4eb70a856 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java@@ -79,6 +79,14 @@ public class Cluster {@JsonIgnoreprivate String podCidr;+ /**+ * In-memory marker that the k3s container was replaced while the cluster stayed ACTIVE, as a+ * first node group changing the node's capacity does, so the readiness poller still has to+ * verify the new API server and finalize the cluster against it.+ */+ @JsonIgnore+ private boolean finalizePending;+/*** Internal flag indicating whether the Kubernetes version was explicitly requested by* the caller (true) or defaulted (false). Persisted to storage so that clusters explicitly@@ -222,6 +230,9 @@ public class Cluster {public boolean isExplicitVersion() { return explicitVersion; }public void setExplicitVersion(boolean explicitVersion) { this.explicitVersion = explicitVersion; }+ public boolean isFinalizePending() { return finalizePending; }+ public void setFinalizePending(boolean finalizePending) { this.finalizePending = finalizePending; }+public Boolean getDefaultStorageClass() { return defaultStorageClass; }public void setDefaultStorageClass(Boolean defaultStorageClass) { this.defaultStorageClass = defaultStorageClass; }@@ -247,6 +258,7 @@ public class Cluster {c.internalEndpoint = this.internalEndpoint;c.hostPort = this.hostPort;c.podCidr = this.podCidr;+ c.finalizePending = this.finalizePending;c.explicitVersion = this.explicitVersion;c.dockerName = this.dockerName;c.nodeInstanceType = this.nodeInstanceType;diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.javaindex 93a9ad0313..fdd7de83ed 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java@@ -986,25 +986,94 @@ class EksServiceTest {@Testvoid firstNodeGroupSelectsAndRestartsTheSharedNodeOnce() {- EksClusterManager clusterManager = mock(EksClusterManager.class);- when(clusterManager.tryStartCluster(any())).thenReturn(true);- when(clusterManager.selectNodeInstanceType("t3.medium")).thenReturn("t3.medium");- when(clusterManager.restartForNodeCapacity(any(), anyString())).thenReturn(true);+ EksClusterManager clusterManager = capacitySwapManager(true);EksService service = newService(clusterManager, false);- CreateClusterRequest request = new CreateClusterRequest();- request.setName("capacity-cluster");- request.setRoleArn("arn:aws:iam::000000000000:role/eks-role");- Cluster cluster = service.createCluster(request);- cluster.setContainerId("existing-container");+ Cluster cluster = activeClusterWithRunningNode(service, "capacity-cluster");service.createNodeGroup("capacity-cluster", nodeGroupRequest("first"));assertEquals("t3.medium", cluster.getNodeInstanceType());+ assertEquals(ClusterStatus.ACTIVE, cluster.getStatus());+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("capacity-cluster").getStatus());verify(clusterManager).restartForNodeCapacity(cluster, "m5.large");service.createNodeGroup("capacity-cluster", nodeGroupRequest("second"));+ assertEquals(ClusterStatus.ACTIVE, cluster.getStatus());+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("capacity-cluster").getStatus());verify(clusterManager).restartForNodeCapacity(cluster, "m5.large");}+ @Test+ void readinessPollerFinalizesTheReplacedNodeOfAnActiveCluster() {+ EksClusterManager clusterManager = capacitySwapManager(true);+ EksService service = newService(clusterManager, false);+ Cluster cluster = activeClusterWithRunningNode(service, "replaced-node");++ service.createNodeGroup("replaced-node", nodeGroupRequest("first"));+ assertTrue(cluster.isFinalizePending());++ service.pollClusterReadiness();+ verify(clusterManager, never()).finalizeCluster(cluster);+ assertTrue(cluster.isFinalizePending());+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("replaced-node").getStatus());++ when(clusterManager.isReady(cluster)).thenReturn(true);+ service.pollClusterReadiness();+ verify(clusterManager).finalizeCluster(cluster);+ assertFalse(cluster.isFinalizePending());+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("replaced-node").getStatus());++ service.pollClusterReadiness();+ verify(clusterManager).finalizeCluster(cluster);+ }++ @Test+ void failedNodeReplacementStillHasThePreviousNodeVerified() {+ EksClusterManager clusterManager = capacitySwapManager(false);+ EksService service = newService(clusterManager, false);+ Cluster cluster = activeClusterWithRunningNode(service, "restored-node");++ Nodegroup nodeGroup = service.createNodeGroup("restored-node", nodeGroupRequest("first"));++ assertEquals(NodegroupStatus.CREATE_FAILED, nodeGroup.getStatus());+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("restored-node").getStatus());+ assertTrue(cluster.isFinalizePending());+ }++ @Test+ void readinessPollerDoesNotWriteBackAClusterDeletedWhileItsNodeIsProbed() {+ EksClusterManager clusterManager = capacitySwapManager(true);+ EksService service = newService(clusterManager, false);+ Cluster cluster = activeClusterWithRunningNode(service, "deleted-node");+ service.createNodeGroup("deleted-node", nodeGroupRequest("first"));+ when(clusterManager.isReady(cluster)).thenAnswer(invocation -> {+ service.deleteCluster("deleted-node");+ return true;+ });++ service.pollClusterReadiness();++ assertEquals(ClusterStatus.DELETING, cluster.getStatus());+ assertThrows(AwsException.class, () -> service.describeCluster("deleted-node"));+ }++ private static Cluster activeClusterWithRunningNode(EksService service, String name) {+ CreateClusterRequest request = new CreateClusterRequest();+ request.setName(name);+ request.setRoleArn("arn:aws:iam::000000000000:role/eks-role");+ Cluster cluster = service.createCluster(request);+ cluster.setContainerId("existing-container");+ cluster.setStatus(ClusterStatus.ACTIVE);+ return cluster;+ }++ private static EksClusterManager capacitySwapManager(boolean swapSucceeds) {+ EksClusterManager clusterManager = mock(EksClusterManager.class);+ when(clusterManager.tryStartCluster(any())).thenReturn(true);+ when(clusterManager.selectNodeInstanceType("t3.medium")).thenReturn("t3.medium");+ when(clusterManager.restartForNodeCapacity(any(), anyString())).thenReturn(swapSucceeds);+ return clusterManager;+ }+@Testvoid concurrentFirstNodeGroupsChooseOnlyOneCapacity() throws Exception {EksClusterManager clusterManager = mock(EksClusterManager.class);