Light, fluffy, and always free - The AWS Local Emulator alternative
fix(eks): claim the cluster user data slot atomically (#4190)
Simone Primarosadc70f60a6618
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.javaindex 9cd145a3fe..6b18f07385 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java@@ -58,6 +58,7 @@ import java.util.Map;import java.util.Optional;import java.util.Set;import java.util.UUID;+import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Executors;import java.util.concurrent.ScheduledExecutorService;@@ -75,6 +76,8 @@ public class EksService implements TagHandler, ResourceProvider {private static final Logger LOG = Logger.getLogger(EksService.class);+ private record UserDataClaim(String userData, CompletableFuture<UserDataPipeline.ExecutionResult> future) {}+private static final List<String> ALL_LOG_TYPES = List.of("api", "audit", "authenticator", "controllerManager", "scheduler");@@ -98,7 +101,7 @@ public class EksService implements TagHandler, ResourceProvider {private final EksPodIdentityAssociationService podIdentityAssociations;private final EksAddonService addons;private final ScheduledExecutorService poller = Executors.newSingleThreadScheduledExecutor();- private final Map<String, String> appliedClusterUserData = new ConcurrentHashMap<>();+ private final Map<String, UserDataClaim> appliedClusterUserData = new ConcurrentHashMap<>();@Injectpublic EksService(StorageFactory storageFactory, EmulatorConfig config,@@ -1136,11 +1139,38 @@ public class EksService implements TagHandler, ResourceProvider {return health;}+ static volatile Runnable userDataClaimWaitingHook;+private void applyNodeGroupUserData(Cluster cluster, String nodegroupName, String userData, Nodegroup nodeGroup) {String clusterKey = cluster.getArn() != null ? cluster.getArn() : cluster.getName();- String previouslyApplied = appliedClusterUserData.get(clusterKey);- if (previouslyApplied != null) {- if (previouslyApplied.equals(userData)) {+ CompletableFuture<UserDataPipeline.ExecutionResult> future = new CompletableFuture<>();+ UserDataClaim claim = new UserDataClaim(userData, future);++ // putIfAbsent claims the slot atomically. If another caller is already executing, we wait+ // for their execution to complete so that failure in the winner propagates to all racing+ // nodegroups rather than letting them proceed ACTIVE on a failed bootstrap.+ UserDataClaim existing = appliedClusterUserData.putIfAbsent(clusterKey, claim);+ if (existing != null) {+ Runnable hook = userDataClaimWaitingHook;+ if (hook != null) {+ hook.run();+ }++ UserDataPipeline.ExecutionResult winnerResult;+ try {+ winnerResult = existing.future().join();+ } catch (Throwable t) {+ Throwable cause = t.getCause() != null ? t.getCause() : t;+ winnerResult = UserDataPipeline.ExecutionResult.failed(+ -1L, 1, 1, "UserData execution failed: " + cause.getMessage());+ }++ if (winnerResult != null && !winnerResult.isSuccess()) {+ failNodeGroupUserData(nodeGroup, nodegroupName, cluster, winnerResult);+ return;+ }++ if (existing.userData().equals(userData)) {LOG.infov("Nodegroup {0} specifies identical launch template user data already applied to cluster {1}; skipping",nodegroupName, cluster.getName());} else {@@ -1151,24 +1181,46 @@ public class EksService implements TagHandler, ResourceProvider {}if (clusterManager == null) {- appliedClusterUserData.put(clusterKey, userData);+ claim.future().complete(UserDataPipeline.ExecutionResult.skipped("No cluster manager"));return;}- UserDataPipeline.ExecutionResult result = clusterManager.executeUserData(cluster, nodegroupName, userData);+ UserDataPipeline.ExecutionResult result = null;+ try {+ result = clusterManager.executeUserData(cluster, nodegroupName, userData);+ if (result == null) {+ result = UserDataPipeline.ExecutionResult.skipped("UserData execution returned no result");+ }+ claim.future().complete(result);+ } catch (Throwable t) {+ claim.future().completeExceptionally(t);+ appliedClusterUserData.remove(clusterKey, claim);+ throw t;+ } finally {+ if (result != null && !result.isSuccess()) {+ // Release the claim so a later node group can retry the bootstrap. The conditional+ // remove only drops our own claim, not one a concurrent delete/recreate or a newer+ // caller has since put in its place.+ appliedClusterUserData.remove(clusterKey, claim);+ }+ }+if (result != null && !result.isSuccess()) {- nodeGroup.setStatus(NodegroupStatus.CREATE_FAILED);- String failureDetail = result.getFailureMessage() != null- ? result.getFailureMessage()- : "UserData execution failed for EKS cluster " + cluster.getName();- nodeGroup.setHealth(failedNodeGroupHealth(nodegroupName, failureDetail));- LOG.warnv("Nodegroup {0} failed to execute launch template user data: {1}",- nodegroupName, failureDetail);- } else {- appliedClusterUserData.put(clusterKey, userData);+ failNodeGroupUserData(nodeGroup, nodegroupName, cluster, result);}}+ private void failNodeGroupUserData(Nodegroup nodeGroup, String nodegroupName, Cluster cluster,+ UserDataPipeline.ExecutionResult result) {+ nodeGroup.setStatus(NodegroupStatus.CREATE_FAILED);+ String failureDetail = result != null && result.getFailureMessage() != null+ ? result.getFailureMessage()+ : "UserData execution failed for EKS cluster " + cluster.getName();+ nodeGroup.setHealth(failedNodeGroupHealth(nodegroupName, failureDetail));+ LOG.warnv("Nodegroup {0} failed to execute launch template user data: {1}",+ nodegroupName, failureDetail);+ }+private FargateProfile.Health defaultFargateProfileHealth() {FargateProfile.Health health = new FargateProfile.Health();health.setIssues(List.of());diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksLaunchTemplateUserDataTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksLaunchTemplateUserDataTest.javaindex 9605e0ce10..8684b1ccc8 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksLaunchTemplateUserDataTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksLaunchTemplateUserDataTest.java@@ -30,6 +30,9 @@ import java.nio.charset.StandardCharsets;import java.util.Base64;import java.util.List;import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -203,6 +206,183 @@ class EksLaunchTemplateUserDataTest {assertEquals(List.of("ng-timeout"), issue.get("resourceIds"));}+ @Test+ void concurrentNodeGroupCreationExecutesUserDataExactlyOnce() throws InterruptedException {+ String script = "#!/bin/bash\necho 'concurrent bootstrap'\n";+ String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_8));+ String ltId = createLaunchTemplate("lt-concurrent", encoded);++ CountDownLatch enteredExecution = new CountDownLatch(1);+ CountDownLatch ng2WaitingOnClaim = new CountDownLatch(1);+ CountDownLatch releaseExecution = new CountDownLatch(1);++ EksService.userDataClaimWaitingHook = ng2WaitingOnClaim::countDown;++ try {+ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded)))+ .thenAnswer(invocation -> {+ enteredExecution.countDown();+ assertTrue(releaseExecution.await(5, TimeUnit.SECONDS));+ return UserDataPipeline.ExecutionResult.success(0L, "ok");+ });++ CreateNodeGroupRequest req1 = nodeGroupRequest("ng-1");+ req1.setLaunchTemplate(Map.of("id", ltId));+ AtomicReference<Nodegroup> ng1Ref = new AtomicReference<>();+ Thread first = new Thread(() -> ng1Ref.set(eksService.createNodeGroup(CLUSTER_NAME, req1)));+ first.start();++ // Wait until the first caller has claimed the slot and is blocked inside the container exec+ assertTrue(enteredExecution.await(5, TimeUnit.SECONDS));++ AtomicReference<Nodegroup> ng2Ref = new AtomicReference<>();+ CreateNodeGroupRequest req2 = nodeGroupRequest("ng-2");+ req2.setLaunchTemplate(Map.of("id", ltId));+ Thread second = new Thread(() -> ng2Ref.set(eksService.createNodeGroup(CLUSTER_NAME, req2)));+ second.start();++ // Wait until second caller observes the claim and begins waiting for winner's future+ assertTrue(ng2WaitingOnClaim.await(5, TimeUnit.SECONDS));++ releaseExecution.countDown();+ first.join(5000);+ second.join(5000);++ Nodegroup ng1 = ng1Ref.get();+ Nodegroup ng2 = ng2Ref.get();+ assertNotNull(ng1);+ assertNotNull(ng2);+ assertEquals(NodegroupStatus.ACTIVE, ng1.getStatus());+ assertEquals(NodegroupStatus.ACTIVE, ng2.getStatus());++ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), any(), any());+ verify(clusterManager, never()).executeUserData(any(Cluster.class), eq("ng-2"), any());+ } finally {+ EksService.userDataClaimWaitingHook = null;+ }+ }++ @Test+ void concurrentNodeGroupCreationFailsBothWhenWinnerFails() throws InterruptedException {+ String script = "#!/bin/bash\nexit 1\n";+ String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_8));+ String ltId = createLaunchTemplate("lt-concurrent-fail", encoded);++ CountDownLatch enteredExecution = new CountDownLatch(1);+ CountDownLatch ng2WaitingOnClaim = new CountDownLatch(1);+ CountDownLatch releaseExecution = new CountDownLatch(1);++ EksService.userDataClaimWaitingHook = ng2WaitingOnClaim::countDown;++ try {+ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded)))+ .thenAnswer(invocation -> {+ enteredExecution.countDown();+ assertTrue(releaseExecution.await(5, TimeUnit.SECONDS));+ return UserDataPipeline.ExecutionResult.failed(1L, 1, 1, "bootstrap failed");+ });++ CreateNodeGroupRequest req1 = nodeGroupRequest("ng-1");+ req1.setLaunchTemplate(Map.of("id", ltId));+ AtomicReference<Nodegroup> ng1Ref = new AtomicReference<>();+ Thread first = new Thread(() -> ng1Ref.set(eksService.createNodeGroup(CLUSTER_NAME, req1)));+ first.start();++ // Wait until first caller has claimed the slot and is blocked inside executeUserData+ assertTrue(enteredExecution.await(5, TimeUnit.SECONDS));++ AtomicReference<Nodegroup> ng2Ref = new AtomicReference<>();+ CreateNodeGroupRequest req2 = nodeGroupRequest("ng-2");+ req2.setLaunchTemplate(Map.of("id", ltId));+ Thread second = new Thread(() -> ng2Ref.set(eksService.createNodeGroup(CLUSTER_NAME, req2)));+ second.start();++ // Wait until second caller has observed the claim and is waiting on the future+ assertTrue(ng2WaitingOnClaim.await(5, TimeUnit.SECONDS));++ // Let execution finish with failure+ releaseExecution.countDown();+ first.join(5000);+ second.join(5000);++ Nodegroup ng1 = ng1Ref.get();+ Nodegroup ng2 = ng2Ref.get();++ assertNotNull(ng1);+ assertNotNull(ng2);+ assertEquals(NodegroupStatus.CREATE_FAILED, ng1.getStatus());+ assertEquals(NodegroupStatus.CREATE_FAILED, ng2.getStatus());++ Map<?, ?> health1 = (Map<?, ?>) ng1.getHealth();+ assertNotNull(health1);+ List<?> issues1 = (List<?>) health1.get("issues");+ assertEquals(1, issues1.size());+ assertTrue(((Map<?, ?>) issues1.getFirst()).get("message").toString().contains("bootstrap failed"));++ Map<?, ?> health2 = (Map<?, ?>) ng2.getHealth();+ assertNotNull(health2);+ List<?> issues2 = (List<?>) health2.get("issues");+ assertEquals(1, issues2.size());+ assertTrue(((Map<?, ?>) issues2.getFirst()).get("message").toString().contains("bootstrap failed"));++ // Slot should be released, so subsequent retry ng-3 can run+ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-3"), eq(encoded)))+ .thenReturn(UserDataPipeline.ExecutionResult.success(0L, "ok"));++ CreateNodeGroupRequest req3 = nodeGroupRequest("ng-3");+ req3.setLaunchTemplate(Map.of("id", ltId));+ Nodegroup ng3 = eksService.createNodeGroup(CLUSTER_NAME, req3);++ assertEquals(NodegroupStatus.ACTIVE, ng3.getStatus());+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded));+ verify(clusterManager, never()).executeUserData(any(Cluster.class), eq("ng-2"), eq(encoded));+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), eq("ng-3"), eq(encoded));+ } finally {+ EksService.userDataClaimWaitingHook = null;+ }+ }++ @Test+ void unstubbedClusterManagerDoesNotThrowNpe() {+ String script = "#!/bin/bash\necho 'unstubbed'\n";+ String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_8));+ String ltId = createLaunchTemplate("lt-unstubbed", encoded);++ // clusterManager is a mock and executeUserData is not stubbed (returns null by default)+ CreateNodeGroupRequest req = nodeGroupRequest("ng-unstubbed");+ req.setLaunchTemplate(Map.of("id", ltId));++ Nodegroup ng = eksService.createNodeGroup(CLUSTER_NAME, req);+ assertNotNull(ng);+ assertEquals(NodegroupStatus.ACTIVE, ng.getStatus());+ }++ @Test+ void failedExecutionReleasesClaimForSubsequentRetry() {+ String script = "#!/bin/bash\nexit 3\n";+ String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_8));+ String ltId = createLaunchTemplate("lt-retry", encoded);++ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded)))+ .thenReturn(UserDataPipeline.ExecutionResult.failed(3L, 1, 1, "boom"));++ CreateNodeGroupRequest req1 = nodeGroupRequest("ng-1");+ req1.setLaunchTemplate(Map.of("id", ltId));+ Nodegroup ng1 = eksService.createNodeGroup(CLUSTER_NAME, req1);+ assertEquals(NodegroupStatus.CREATE_FAILED, ng1.getStatus());++ when(clusterManager.executeUserData(any(Cluster.class), eq("ng-2"), eq(encoded)))+ .thenReturn(UserDataPipeline.ExecutionResult.success(0L, "ok"));++ CreateNodeGroupRequest req2 = nodeGroupRequest("ng-2");+ req2.setLaunchTemplate(Map.of("id", ltId));+ Nodegroup ng2 = eksService.createNodeGroup(CLUSTER_NAME, req2);++ assertEquals(NodegroupStatus.ACTIVE, ng2.getStatus());+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), eq("ng-1"), eq(encoded));+ verify(clusterManager, times(1)).executeUserData(any(Cluster.class), eq("ng-2"), eq(encoded));+ }+@Testvoid clusterDeletionClearsAppliedUserDataTracker() {String script = "#!/bin/bash\necho 'recreate test'\n";