Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(eks): claim the cluster user data slot atomically (#4190)

Simone Primarosadc70f60a6618
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
index 9cd145a3fe..6b18f07385 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
@@ -58,6 +58,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -75,6 +76,8 @@ public class EksService implements TagHandler, ResourceProvider {
private static final Logger LOG = Logger.getLogger(EksService.class);
+ private record UserDataClaim(String userData, CompletableFuture<UserDataPipeline.ExecutionResult> future) {}
+
private static final List<String> ALL_LOG_TYPES = List.of(
"api", "audit", "authenticator", "controllerManager", "scheduler"
);
@@ -98,7 +101,7 @@ public class EksService implements TagHandler, ResourceProvider {
private final EksPodIdentityAssociationService podIdentityAssociations;
private final EksAddonService addons;
private final ScheduledExecutorService poller = Executors.newSingleThreadScheduledExecutor();
- private final Map<String, String> appliedClusterUserData = new ConcurrentHashMap<>();
+ private final Map<String, UserDataClaim> appliedClusterUserData = new ConcurrentHashMap<>();
@Inject
public EksService(StorageFactory storageFactory, EmulatorConfig config,
@@ -1136,11 +1139,38 @@ public class EksService implements TagHandler, ResourceProvider {
return health;
}
+ static volatile Runnable userDataClaimWaitingHook;
+
private void applyNodeGroupUserData(Cluster cluster, String nodegroupName, String userData, Nodegroup nodeGroup) {
String clusterKey = cluster.getArn() != null ? cluster.getArn() : cluster.getName();
- String previouslyApplied = appliedClusterUserData.get(clusterKey);
- if (previouslyApplied != null) {
- if (previouslyApplied.equals(userData)) {
+ CompletableFuture<UserDataPipeline.ExecutionResult> future = new CompletableFuture<>();
+ UserDataClaim claim = new UserDataClaim(userData, future);
+
+ // putIfAbsent claims the slot atomically. If another caller is already executing, we wait
+ // for their execution to complete so that failure in the winner propagates to all racing
+ // nodegroups rather than letting them proceed ACTIVE on a failed bootstrap.
+ UserDataClaim existing = appliedClusterUserData.putIfAbsent(clusterKey, claim);
+ if (existing != null) {
+ Runnable hook = userDataClaimWaitingHook;
+ if (hook != null) {
+ hook.run();
+ }
+
+ UserDataPipeline.ExecutionResult winnerResult;
+ try {
+ winnerResult = existing.future().join();
+ } catch (Throwable t) {
+ Throwable cause = t.getCause() != null ? t.getCause() : t;
+ winnerResult = UserDataPipeline.ExecutionResult.failed(
+ -1L, 1, 1, "UserData execution failed: " + cause.getMessage());
+ }
+
+ if (winnerResult != null && !winnerResult.isSuccess()) {
+ failNodeGroupUserData(nodeGroup, nodegroupName, cluster, winnerResult);
+ return;
+ }
+
+ if (existing.userData().equals(userData)) {
LOG.infov("Nodegroup {0} specifies identical launch template user data already applied to cluster {1}; skipping",
nodegroupName, cluster.getName());
} else {
@@ -1151,24 +1181,46 @@ public class EksService implements TagHandler, ResourceProvider {
}
if (clusterManager == null) {
- appliedClusterUserData.put(clusterKey, userData);
+ claim.future().complete(UserDataPipeline.ExecutionResult.skipped("No cluster manager"));
return;
}
- UserDataPipeline.ExecutionResult result = clusterManager.executeUserData(cluster, nodegroupName, userData);
+ UserDataPipeline.ExecutionResult result = null;
+ try {
+ result = clusterManager.executeUserData(cluster, nodegroupName, userData);
+ if (result == null) {
+ result = UserDataPipeline.ExecutionResult.skipped("UserData execution returned no result");
+ }
+ claim.future().complete(result);
+ } catch (Throwable t) {
+ claim.future().completeExceptionally(t);
+ appliedClusterUserData.remove(clusterKey, claim);
+ throw t;
+ } finally {
+ if (result != null && !result.isSuccess()) {
+ // Release the claim so a later node group can retry the bootstrap. The conditional
+ // remove only drops our own claim, not one a concurrent delete/recreate or a newer
+ // caller has since put in its place.
+ appliedClusterUserData.remove(clusterKey, claim);
+ }
+ }
+
if (result != null && !result.isSuccess()) {
- nodeGroup.setStatus(NodegroupStatus.CREATE_FAILED);
- String failureDetail = result.getFailureMessage() != null
- ? result.getFailureMessage()
- : "UserData execution failed for EKS cluster " + cluster.getName();
- nodeGroup.setHealth(failedNodeGroupHealth(nodegroupName, failureDetail));
- LOG.warnv("Nodegroup {0} failed to execute launch template user data: {1}",
- nodegroupName, failureDetail);
- } else {
- appliedClusterUserData.put(clusterKey, userData);
+ failNodeGroupUserData(nodeGroup, nodegroupName, cluster, result);
}
}
+ private void failNodeGroupUserData(Nodegroup nodeGroup, String nodegroupName, Cluster cluster,
+ UserDataPipeline.ExecutionResult result) {
+ nodeGroup.setStatus(NodegroupStatus.CREATE_FAILED);
+ String failureDetail = result != null && result.getFailureMessage() != null
+ ? result.getFailureMessage()
+ : "UserData execution failed for EKS cluster " + cluster.getName();
+ nodeGroup.setHealth(failedNodeGroupHealth(nodegroupName, failureDetail));
+ LOG.warnv("Nodegroup {0} failed to execute launch template user data: {1}",
+ nodegroupName, failureDetail);
+ }
+
private FargateProfile.Health defaultFargateProfileHealth() {
FargateProfile.Health health = new FargateProfile.Health();
health.setIssues(List.of());
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksLaunchTemplateUserDataTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksLaunchTemplateUserDataTest.java
index 9605e0ce10..8684b1ccc8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksLaunchTemplateUserDataTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksLaunchTemplateUserDataTest.java
@@ -30,6 +30,9 @@ import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -203,6 +206,183 @@ class EksLaunchTemplateUserDataTest {
assertEquals(List.of("ng-timeout"), issue.get("resourceIds"));
}
+ @Test
+ void concurrentNodeGroupCreationExecutesUserDataExactlyOnce() throws InterruptedException {
+ String script = "#!/bin/bash\necho 'concurrent bootstrap'\n";
+ String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_8));
+ String ltId = createLaunchTemplate("lt-concurrent", encoded);
+
+ CountDownLatch enteredExecution = new CountDownLatch(1);
+ CountDownLatch ng2WaitingOnClaim = new CountDownLatch(1);
+ CountDownLatch releaseExecution = new CountDownLatch(1);
+
+ EksService.userDataClaimWaitingHook = ng2WaitingOnClaim::countDown;
+
+ try {
+ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded)))
+ .thenAnswer(invocation -> {
+ enteredExecution.countDown();
+ assertTrue(releaseExecution.await(5, TimeUnit.SECONDS));
+ return UserDataPipeline.ExecutionResult.success(0L, "ok");
+ });
+
+ CreateNodeGroupRequest req1 = nodeGroupRequest("ng-1");
+ req1.setLaunchTemplate(Map.of("id", ltId));
+ AtomicReference<Nodegroup> ng1Ref = new AtomicReference<>();
+ Thread first = new Thread(() -> ng1Ref.set(eksService.createNodeGroup(CLUSTER_NAME, req1)));
+ first.start();
+
+ // Wait until the first caller has claimed the slot and is blocked inside the container exec
+ assertTrue(enteredExecution.await(5, TimeUnit.SECONDS));
+
+ AtomicReference<Nodegroup> ng2Ref = new AtomicReference<>();
+ CreateNodeGroupRequest req2 = nodeGroupRequest("ng-2");
+ req2.setLaunchTemplate(Map.of("id", ltId));
+ Thread second = new Thread(() -> ng2Ref.set(eksService.createNodeGroup(CLUSTER_NAME, req2)));
+ second.start();
+
+ // Wait until second caller observes the claim and begins waiting for winner's future
+ assertTrue(ng2WaitingOnClaim.await(5, TimeUnit.SECONDS));
+
+ releaseExecution.countDown();
+ first.join(5000);
+ second.join(5000);
+
+ Nodegroup ng1 = ng1Ref.get();
+ Nodegroup ng2 = ng2Ref.get();
+ assertNotNull(ng1);
+ assertNotNull(ng2);
+ assertEquals(NodegroupStatus.ACTIVE, ng1.getStatus());
+ assertEquals(NodegroupStatus.ACTIVE, ng2.getStatus());
+
+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), any(), any());
+ verify(clusterManager, never()).executeUserData(any(Cluster.class), eq("ng-2"), any());
+ } finally {
+ EksService.userDataClaimWaitingHook = null;
+ }
+ }
+
+ @Test
+ void concurrentNodeGroupCreationFailsBothWhenWinnerFails() throws InterruptedException {
+ String script = "#!/bin/bash\nexit 1\n";
+ String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_8));
+ String ltId = createLaunchTemplate("lt-concurrent-fail", encoded);
+
+ CountDownLatch enteredExecution = new CountDownLatch(1);
+ CountDownLatch ng2WaitingOnClaim = new CountDownLatch(1);
+ CountDownLatch releaseExecution = new CountDownLatch(1);
+
+ EksService.userDataClaimWaitingHook = ng2WaitingOnClaim::countDown;
+
+ try {
+ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded)))
+ .thenAnswer(invocation -> {
+ enteredExecution.countDown();
+ assertTrue(releaseExecution.await(5, TimeUnit.SECONDS));
+ return UserDataPipeline.ExecutionResult.failed(1L, 1, 1, "bootstrap failed");
+ });
+
+ CreateNodeGroupRequest req1 = nodeGroupRequest("ng-1");
+ req1.setLaunchTemplate(Map.of("id", ltId));
+ AtomicReference<Nodegroup> ng1Ref = new AtomicReference<>();
+ Thread first = new Thread(() -> ng1Ref.set(eksService.createNodeGroup(CLUSTER_NAME, req1)));
+ first.start();
+
+ // Wait until first caller has claimed the slot and is blocked inside executeUserData
+ assertTrue(enteredExecution.await(5, TimeUnit.SECONDS));
+
+ AtomicReference<Nodegroup> ng2Ref = new AtomicReference<>();
+ CreateNodeGroupRequest req2 = nodeGroupRequest("ng-2");
+ req2.setLaunchTemplate(Map.of("id", ltId));
+ Thread second = new Thread(() -> ng2Ref.set(eksService.createNodeGroup(CLUSTER_NAME, req2)));
+ second.start();
+
+ // Wait until second caller has observed the claim and is waiting on the future
+ assertTrue(ng2WaitingOnClaim.await(5, TimeUnit.SECONDS));
+
+ // Let execution finish with failure
+ releaseExecution.countDown();
+ first.join(5000);
+ second.join(5000);
+
+ Nodegroup ng1 = ng1Ref.get();
+ Nodegroup ng2 = ng2Ref.get();
+
+ assertNotNull(ng1);
+ assertNotNull(ng2);
+ assertEquals(NodegroupStatus.CREATE_FAILED, ng1.getStatus());
+ assertEquals(NodegroupStatus.CREATE_FAILED, ng2.getStatus());
+
+ Map<?, ?> health1 = (Map<?, ?>) ng1.getHealth();
+ assertNotNull(health1);
+ List<?> issues1 = (List<?>) health1.get("issues");
+ assertEquals(1, issues1.size());
+ assertTrue(((Map<?, ?>) issues1.getFirst()).get("message").toString().contains("bootstrap failed"));
+
+ Map<?, ?> health2 = (Map<?, ?>) ng2.getHealth();
+ assertNotNull(health2);
+ List<?> issues2 = (List<?>) health2.get("issues");
+ assertEquals(1, issues2.size());
+ assertTrue(((Map<?, ?>) issues2.getFirst()).get("message").toString().contains("bootstrap failed"));
+
+ // Slot should be released, so subsequent retry ng-3 can run
+ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-3"), eq(encoded)))
+ .thenReturn(UserDataPipeline.ExecutionResult.success(0L, "ok"));
+
+ CreateNodeGroupRequest req3 = nodeGroupRequest("ng-3");
+ req3.setLaunchTemplate(Map.of("id", ltId));
+ Nodegroup ng3 = eksService.createNodeGroup(CLUSTER_NAME, req3);
+
+ assertEquals(NodegroupStatus.ACTIVE, ng3.getStatus());
+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded));
+ verify(clusterManager, never()).executeUserData(any(Cluster.class), eq("ng-2"), eq(encoded));
+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), eq("ng-3"), eq(encoded));
+ } finally {
+ EksService.userDataClaimWaitingHook = null;
+ }
+ }
+
+ @Test
+ void unstubbedClusterManagerDoesNotThrowNpe() {
+ String script = "#!/bin/bash\necho 'unstubbed'\n";
+ String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_8));
+ String ltId = createLaunchTemplate("lt-unstubbed", encoded);
+
+ // clusterManager is a mock and executeUserData is not stubbed (returns null by default)
+ CreateNodeGroupRequest req = nodeGroupRequest("ng-unstubbed");
+ req.setLaunchTemplate(Map.of("id", ltId));
+
+ Nodegroup ng = eksService.createNodeGroup(CLUSTER_NAME, req);
+ assertNotNull(ng);
+ assertEquals(NodegroupStatus.ACTIVE, ng.getStatus());
+ }
+
+ @Test
+ void failedExecutionReleasesClaimForSubsequentRetry() {
+ String script = "#!/bin/bash\nexit 3\n";
+ String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_8));
+ String ltId = createLaunchTemplate("lt-retry", encoded);
+
+ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded)))
+ .thenReturn(UserDataPipeline.ExecutionResult.failed(3L, 1, 1, "boom"));
+
+ CreateNodeGroupRequest req1 = nodeGroupRequest("ng-1");
+ req1.setLaunchTemplate(Map.of("id", ltId));
+ Nodegroup ng1 = eksService.createNodeGroup(CLUSTER_NAME, req1);
+ assertEquals(NodegroupStatus.CREATE_FAILED, ng1.getStatus());
+
+ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-2"), eq(encoded)))
+ .thenReturn(UserDataPipeline.ExecutionResult.success(0L, "ok"));
+
+ CreateNodeGroupRequest req2 = nodeGroupRequest("ng-2");
+ req2.setLaunchTemplate(Map.of("id", ltId));
+ Nodegroup ng2 = eksService.createNodeGroup(CLUSTER_NAME, req2);
+
+ assertEquals(NodegroupStatus.ACTIVE, ng2.getStatus());
+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded));
+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), eq("ng-2"), eq(encoded));
+ }
+
@Test
void clusterDeletionClearsAppliedUserDataTracker() {
String script = "#!/bin/bash\necho 'recreate test'\n";