Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(eks): keep cluster ACTIVE when creating first node group (#5521)

Prakhar Kapoor7c122718f9ec
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
index 71484a0565..a16b1b6e9f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
@@ -1052,13 +1052,16 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun
String previousType = EksClusterManager.nodeInstanceType(currentCluster);
currentCluster.setNodeInstanceType(selectedType);
if (currentCluster.getContainerId() != null && !selectedType.equals(previousType)) {
- currentCluster.setStatus(ClusterStatus.CREATING);
try {
if (!clusterManager.restartForNodeCapacity(currentCluster, previousType)) {
nodeGroup.setStatus(NodegroupStatus.CREATE_FAILED);
nodeGroup.setHealth(failedNodeGroupHealth(nodegroupName,
"Could not apply node capacity; the previous node was restored"));
}
+ // Set only now: before the swap the poller would verify the old container, still
+ // answering while it stops. A failed swap restarts the previous node, so it is
+ // verified too.
+ currentCluster.setFinalizePending(true);
} catch (RuntimeException e) {
currentCluster.setNodeInstanceType(previousType);
currentCluster.setStatus(ClusterStatus.FAILED);
@@ -1603,26 +1606,41 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun
}
private void startReadinessPoller() {
- poller.scheduleAtFixedRate(() -> {
- try {
- for (AccountAwareStorageBackend.AccountEntry<Cluster> entry : allClusterEntries()) {
- Cluster cluster = entry.value();
- if (cluster.getStatus() == ClusterStatus.CREATING) {
- if (clusterManager.isReady(cluster)) {
- LOG.infov("EKS cluster {0} is now ACTIVE", cluster.getName());
- clusterManager.finalizeCluster(cluster);
+ poller.scheduleAtFixedRate(this::pollClusterReadiness, 2, 3, TimeUnit.SECONDS);
+ }
+
+ void pollClusterReadiness() {
+ try {
+ for (AccountAwareStorageBackend.AccountEntry<Cluster> entry : allClusterEntries()) {
+ Cluster cluster = entry.value();
+ boolean replacedNode = cluster.getStatus() == ClusterStatus.ACTIVE && cluster.isFinalizePending();
+ if (cluster.getStatus() == ClusterStatus.CREATING || replacedNode) {
+ if (clusterManager.isReady(cluster)) {
+ clusterManager.finalizeCluster(cluster);
+ // A delete that ran while the API server was probed must not get its
+ // cluster written back.
+ synchronized (clustersBeingDeleted) {
+ if (isClusterDeleting(entry.accountId(), cluster)) {
+ continue;
+ }
+ cluster.setFinalizePending(false);
cluster.setStatus(ClusterStatus.ACTIVE);
putCluster(cluster);
- restoreJoinedInstancesAsync(entry.accountId(), cluster);
}
- } else if (cluster.getStatus() == ClusterStatus.ACTIVE && hasPendingJoinedInstances(cluster)) {
+ if (replacedNode) {
+ LOG.infov("Replaced k3s node of EKS cluster {0} is ready", cluster.getName());
+ } else {
+ LOG.infov("EKS cluster {0} is now ACTIVE", cluster.getName());
+ }
restoreJoinedInstancesAsync(entry.accountId(), cluster);
}
+ } else if (cluster.getStatus() == ClusterStatus.ACTIVE && hasPendingJoinedInstances(cluster)) {
+ restoreJoinedInstancesAsync(entry.accountId(), cluster);
}
- } catch (Exception e) {
- LOG.error("Error in EKS readiness poller", e);
}
- }, 2, 3, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ LOG.error("Error in EKS readiness poller", e);
+ }
}
private void restoreJoinedInstancesAsync(String accountId, Cluster cluster) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java b/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java
index 3886fee9b0..d4eb70a856 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java
@@ -79,6 +79,14 @@ public class Cluster {
@JsonIgnore
private String podCidr;
+ /**
+ * In-memory marker that the k3s container was replaced while the cluster stayed ACTIVE, as a
+ * first node group changing the node's capacity does, so the readiness poller still has to
+ * verify the new API server and finalize the cluster against it.
+ */
+ @JsonIgnore
+ private boolean finalizePending;
+
/**
* Internal flag indicating whether the Kubernetes version was explicitly requested by
* the caller (true) or defaulted (false). Persisted to storage so that clusters explicitly
@@ -222,6 +230,9 @@ public class Cluster {
public boolean isExplicitVersion() { return explicitVersion; }
public void setExplicitVersion(boolean explicitVersion) { this.explicitVersion = explicitVersion; }
+ public boolean isFinalizePending() { return finalizePending; }
+ public void setFinalizePending(boolean finalizePending) { this.finalizePending = finalizePending; }
+
public Boolean getDefaultStorageClass() { return defaultStorageClass; }
public void setDefaultStorageClass(Boolean defaultStorageClass) { this.defaultStorageClass = defaultStorageClass; }
@@ -247,6 +258,7 @@ public class Cluster {
c.internalEndpoint = this.internalEndpoint;
c.hostPort = this.hostPort;
c.podCidr = this.podCidr;
+ c.finalizePending = this.finalizePending;
c.explicitVersion = this.explicitVersion;
c.dockerName = this.dockerName;
c.nodeInstanceType = this.nodeInstanceType;
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
index 93a9ad0313..fdd7de83ed 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
@@ -986,25 +986,94 @@ class EksServiceTest {
@Test
void firstNodeGroupSelectsAndRestartsTheSharedNodeOnce() {
- EksClusterManager clusterManager = mock(EksClusterManager.class);
- when(clusterManager.tryStartCluster(any())).thenReturn(true);
- when(clusterManager.selectNodeInstanceType("t3.medium")).thenReturn("t3.medium");
- when(clusterManager.restartForNodeCapacity(any(), anyString())).thenReturn(true);
+ EksClusterManager clusterManager = capacitySwapManager(true);
EksService service = newService(clusterManager, false);
- CreateClusterRequest request = new CreateClusterRequest();
- request.setName("capacity-cluster");
- request.setRoleArn("arn:aws:iam::000000000000:role/eks-role");
- Cluster cluster = service.createCluster(request);
- cluster.setContainerId("existing-container");
+ Cluster cluster = activeClusterWithRunningNode(service, "capacity-cluster");
service.createNodeGroup("capacity-cluster", nodeGroupRequest("first"));
assertEquals("t3.medium", cluster.getNodeInstanceType());
+ assertEquals(ClusterStatus.ACTIVE, cluster.getStatus());
+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("capacity-cluster").getStatus());
verify(clusterManager).restartForNodeCapacity(cluster, "m5.large");
service.createNodeGroup("capacity-cluster", nodeGroupRequest("second"));
+ assertEquals(ClusterStatus.ACTIVE, cluster.getStatus());
+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("capacity-cluster").getStatus());
verify(clusterManager).restartForNodeCapacity(cluster, "m5.large");
}
+ @Test
+ void readinessPollerFinalizesTheReplacedNodeOfAnActiveCluster() {
+ EksClusterManager clusterManager = capacitySwapManager(true);
+ EksService service = newService(clusterManager, false);
+ Cluster cluster = activeClusterWithRunningNode(service, "replaced-node");
+
+ service.createNodeGroup("replaced-node", nodeGroupRequest("first"));
+ assertTrue(cluster.isFinalizePending());
+
+ service.pollClusterReadiness();
+ verify(clusterManager, never()).finalizeCluster(cluster);
+ assertTrue(cluster.isFinalizePending());
+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("replaced-node").getStatus());
+
+ when(clusterManager.isReady(cluster)).thenReturn(true);
+ service.pollClusterReadiness();
+ verify(clusterManager).finalizeCluster(cluster);
+ assertFalse(cluster.isFinalizePending());
+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("replaced-node").getStatus());
+
+ service.pollClusterReadiness();
+ verify(clusterManager).finalizeCluster(cluster);
+ }
+
+ @Test
+ void failedNodeReplacementStillHasThePreviousNodeVerified() {
+ EksClusterManager clusterManager = capacitySwapManager(false);
+ EksService service = newService(clusterManager, false);
+ Cluster cluster = activeClusterWithRunningNode(service, "restored-node");
+
+ Nodegroup nodeGroup = service.createNodeGroup("restored-node", nodeGroupRequest("first"));
+
+ assertEquals(NodegroupStatus.CREATE_FAILED, nodeGroup.getStatus());
+ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("restored-node").getStatus());
+ assertTrue(cluster.isFinalizePending());
+ }
+
+ @Test
+ void readinessPollerDoesNotWriteBackAClusterDeletedWhileItsNodeIsProbed() {
+ EksClusterManager clusterManager = capacitySwapManager(true);
+ EksService service = newService(clusterManager, false);
+ Cluster cluster = activeClusterWithRunningNode(service, "deleted-node");
+ service.createNodeGroup("deleted-node", nodeGroupRequest("first"));
+ when(clusterManager.isReady(cluster)).thenAnswer(invocation -> {
+ service.deleteCluster("deleted-node");
+ return true;
+ });
+
+ service.pollClusterReadiness();
+
+ assertEquals(ClusterStatus.DELETING, cluster.getStatus());
+ assertThrows(AwsException.class, () -> service.describeCluster("deleted-node"));
+ }
+
+ private static Cluster activeClusterWithRunningNode(EksService service, String name) {
+ CreateClusterRequest request = new CreateClusterRequest();
+ request.setName(name);
+ request.setRoleArn("arn:aws:iam::000000000000:role/eks-role");
+ Cluster cluster = service.createCluster(request);
+ cluster.setContainerId("existing-container");
+ cluster.setStatus(ClusterStatus.ACTIVE);
+ return cluster;
+ }
+
+ private static EksClusterManager capacitySwapManager(boolean swapSucceeds) {
+ EksClusterManager clusterManager = mock(EksClusterManager.class);
+ when(clusterManager.tryStartCluster(any())).thenReturn(true);
+ when(clusterManager.selectNodeInstanceType("t3.medium")).thenReturn("t3.medium");
+ when(clusterManager.restartForNodeCapacity(any(), anyString())).thenReturn(swapSucceeds);
+ return clusterManager;
+ }
+
@Test
void concurrentFirstNodeGroupsChooseOnlyOneCapacity() throws Exception {
EksClusterManager clusterManager = mock(EksClusterManager.class);