Light, fluffy, and always free - The AWS Local Emulator alternative
fix(eks): restore persisted clusters at startup (#6214)
Robin Baeckman204231d4b3b6
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.javaindex 8ee4df35e7..bcb9306aba 100644--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java@@ -16,6 +16,7 @@ import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.ecs.EcsService;import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;+import io.github.hectorvent.floci.services.eks.EksService;import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;@@ -56,6 +57,7 @@ import org.jboss.logging.Logger;import java.io.IOException;import java.util.Optional;+import java.util.concurrent.CompletableFuture;@ApplicationScopedpublic class EmulatorLifecycle {@@ -115,8 +117,11 @@ public class EmulatorLifecycle {private final DurableExecutionService durableExecutionService;private final Instance<ContainerTeardown> containerTeardowns;private final PersistentPathValidator persistentPathValidator;+ private final EksService eksService;private final DynamoDbRuntime dynamoDbRuntime;private final RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer;+ // Set on the boot thread, read by the async HttpServerStart observer.+ private volatile CompletableFuture<Void> metadataServerStarted = CompletableFuture.completedFuture(null);@Injectpublic EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceRegistry,@@ -157,7 +162,8 @@ public class EmulatorLifecycle {Instance<ContainerTeardown> containerTeardowns,PersistentPathValidator persistentPathValidator,DynamoDbRuntime dynamoDbRuntime,- RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer) {+ RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer,+ EksService eksService) {this.storageFactory = storageFactory;this.serviceRegistry = serviceRegistry;this.config = config;@@ -196,6 +202,7 @@ public class EmulatorLifecycle {this.durableExecutionService = durableExecutionService;this.containerTeardowns = containerTeardowns;this.persistentPathValidator = persistentPathValidator;+ this.eksService = eksService;this.dynamoDbRuntime = dynamoDbRuntime;this.redshiftZeroEtlConsumer = redshiftZeroEtlConsumer;}@@ -279,7 +286,7 @@ public class EmulatorLifecycle {}if (isMetadataServerNeeded()) {- ec2MetadataServer.start().exceptionally(ex -> {+ metadataServerStarted = ec2MetadataServer.start().exceptionally(ex -> {LOG.warnv("EC2 IMDS server failed to start: {0}", ex.getMessage());return null;});@@ -310,6 +317,7 @@ public class EmulatorLifecycle {boolean hasStart = initializationHooksRunner.hasHooks(InitializationHook.START);boolean hasReady = initializationHooksRunner.hasHooks(InitializationHook.READY);if (!hasStart && !hasReady) {+ restoreEksClusters();return;}try {@@ -322,6 +330,7 @@ public class EmulatorLifecycle {}initLifecycleState.markReadyCompleted();logReady();+ restoreEksClusters();} catch (InterruptedException e) {Thread.currentThread().interrupt();LOG.error("Startup hook execution interrupted — shutting down", e);@@ -332,6 +341,25 @@ public class EmulatorLifecycle {}}+ /**+ * Creating EksService restarts persisted clusters; without this call that waited for the first EKS request.+ * It runs once Floci's HTTP port and IMDS server listen, because a recreated cluster's Pod Identity and IMDS+ * relays probe them before they route pod traffic there, and after the start and ready hooks, which it+ * could otherwise hold up for as long as the clusters take to start.+ */+ private void restoreEksClusters() {+ if (!config.services().eks().enabled() || config.services().eks().mock()) {+ return;+ }+ try {+ metadataServerStarted.join();+ eksService.restorePersistedRuntime();+ } catch (RuntimeException e) {+ // EKS requests retry the bean's creation, as before; the rest of the emulator still starts.+ LOG.error("Failed to restore persisted EKS clusters at startup", e);+ }+ }+/*** LocalStack prints a line ending in "Ready." when its gateway is up, and* ecosystem tooling keys on it — Testcontainers' LocalStackContainer defaultdiff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.javaindex a16b1b6e9f..fea594f840 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java@@ -156,6 +156,14 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaunoidcService, accessEntries, null, null);}+ /** Called at boot so this lazily created bean runs {@link #init()}, which restarts persisted clusters. */+ public void restorePersistedRuntime() {+ int persistedClusters = allClusterEntries().size();+ if (persistedClusters > 0) {+ LOG.infov("Found {0} persisted EKS cluster record(s)", persistedClusters);+ }+ }+@PostConstructpublic void init() {backfillOidcIdentities();diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.javaindex a0fa7949ef..e160ca834c 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java@@ -14,6 +14,7 @@ import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.ecs.EcsService;import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;+import io.github.hectorvent.floci.services.eks.EksService;import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;@@ -104,6 +105,7 @@ class EmulatorLifecycleTest {@Mock private io.github.hectorvent.floci.services.elbv2.ElbV2Service elbV2Service;@Mock private io.github.hectorvent.floci.services.elb.ElbClassicService elbClassicService;@Mock private EcsService ecsService;+ @Mock private EksService eksService;@Mock private InitializationHooksRunner initializationHooksRunner;@Mock private SqsEventSourcePoller sqsPoller;@Mock private KinesisEventSourcePoller kinesisPoller;@@ -166,7 +168,7 @@ class EmulatorLifecycleTest {pipesService, ec2MetadataServer, ecsTaskRoleCredentialsServer, ecrRegistryManager, flociUiManager, initLifecycleState,schemaCreationWorker, stepFunctionsService, durableExecutionService, containerTeardowns,persistentPathValidator,- dynamoDbRuntime, redshiftZeroEtlConsumer);+ dynamoDbRuntime, redshiftZeroEtlConsumer, eksService);Mockito.lenient().when(containerTeardowns.iterator()).thenReturn(java.util.Collections.emptyIterator());}@@ -285,6 +287,100 @@ class EmulatorLifecycleTest {inOrder.verify(ecsService).restorePersistedRuntime();}+ @Test+ void restoresPersistedEksClustersOnceTheHttpPortListens() throws Exception {+ stubStorageConfig();+ when(eksServiceConfig.enabled()).thenReturn(true);+ when(eksServiceConfig.mock()).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);++ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));+ verify(eksService, never()).restorePersistedRuntime();++ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));+ verify(eksService).restorePersistedRuntime();+ }++ @Test+ void restoresPersistedEksClustersOnlyOnceTheImdsServerHasStarted() throws Exception {+ stubStorageConfig();+ when(eksServiceConfig.enabled()).thenReturn(true);+ when(eksServiceConfig.mock()).thenReturn(false);+ when(eksServiceConfig.imds()).thenReturn(true);+ CompletableFuture<Void> imdsStarted = new CompletableFuture<>();+ when(ec2MetadataServer.start()).thenReturn(imdsStarted);+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));++ Thread httpStart = Thread.ofVirtual().start(() ->+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566))));+ verify(eksService, Mockito.after(300).never()).restorePersistedRuntime();++ imdsStarted.complete(null);+ httpStart.join(5000);+ verify(eksService).restorePersistedRuntime();+ }++ @Test+ void restoresPersistedEksClustersAfterTheStartAndReadyHooks() throws Exception {+ when(eksServiceConfig.enabled()).thenReturn(true);+ when(eksServiceConfig.mock()).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(true);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(true);++ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));++ InOrder inOrder = Mockito.inOrder(initializationHooksRunner, initLifecycleState, eksService);+ inOrder.verify(initializationHooksRunner).run(InitializationHook.START);+ inOrder.verify(initializationHooksRunner).run(InitializationHook.READY);+ inOrder.verify(initLifecycleState).markReadyCompleted();+ inOrder.verify(eksService).restorePersistedRuntime();+ }++ @Test+ void aFailedEksRestoreLeavesTheEmulatorReady() throws Exception {+ when(eksServiceConfig.enabled()).thenReturn(true);+ when(eksServiceConfig.mock()).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(true);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);+ doThrow(new IllegalStateException("corrupt cluster record")).when(eksService).restorePersistedRuntime();++ try (MockedStatic<Quarkus> quarkus = Mockito.mockStatic(Quarkus.class,+ Mockito.withSettings().mockMaker(MockMakers.INLINE))) {+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));+ quarkus.verify(Quarkus::asyncExit, never());+ }+ verify(initLifecycleState).markReadyCompleted();+ }++ @Test+ void leavesEksUntouchedAtBootWhenEksIsDisabled() throws Exception {+ stubStorageConfig();+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);++ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));++ verify(eksService, Mockito.never()).restorePersistedRuntime();+ }++ @Test+ void leavesEksUntouchedAtBootInMockMode() throws Exception {+ stubStorageConfig();+ when(eksServiceConfig.enabled()).thenReturn(true);+ when(eksServiceConfig.mock()).thenReturn(true);+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);++ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));++ verify(eksService, Mockito.never()).restorePersistedRuntime();+ }+@Testvoid leavesEcsUntouchedAtBootWhenEcsIsDisabled() {stubStorageConfig();diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksBootRestoreIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksBootRestoreIntegrationTest.javanew file mode 100644index 0000000000..5160e9f2cd--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksBootRestoreIntegrationTest.java@@ -0,0 +1,63 @@+package io.github.hectorvent.floci.services.eks;++import io.github.hectorvent.floci.testutil.FreePorts;+import io.quarkus.arc.Arc;+import io.quarkus.arc.InjectableBean;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.enterprise.context.ApplicationScoped;+import org.junit.jupiter.api.Test;++import java.io.IOException;+import java.io.UncheckedIOException;+import java.time.Duration;+import java.util.Map;++import static org.awaitility.Awaitility.await;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * EksService is created lazily, and creating it is what restarts each persisted cluster's k3s+ * container. Boot has to create it, or persisted clusters stay down after a restart until the+ * first EKS request. The profile of its own gives this test a fresh application that no other+ * test has sent an EKS request to, with EKS out of mock mode as boot restores only real clusters.+ */+@QuarkusTest+@TestProfile(EksBootRestoreIntegrationTest.FreshBootProfile.class)+class EksBootRestoreIntegrationTest {++ public static final class FreshBootProfile implements QuarkusTestProfile {+ // Boot restores once quarkus.http.port listens; the suite's random port 0 never matches the bound one.+ private static final int TEST_PORT = freePort();++ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of(+ "floci.services.eks.mock", "false",+ "quarkus.http.test-port", String.valueOf(TEST_PORT),+ "quarkus.http.port", String.valueOf(TEST_PORT),+ "floci.port", String.valueOf(TEST_PORT),+ "floci.base-url", "http://localhost:" + TEST_PORT);+ }++ private static int freePort() {+ try {+ return FreePorts.anyInterfacePort();+ } catch (IOException e) {+ throw new UncheckedIOException("No free port for the EKS boot test application", e);+ }+ }+ }++ @Test+ void bootCreatesEksServiceWithoutAnEksRequest() {+ // Boot restores once the HTTP port listens, from an async observer, so this can run first.+ await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> {+ Map<InjectableBean<?>, Object> created = Arc.container().getActiveContext(ApplicationScoped.class)+ .getState().getContextualInstances();+ assertTrue(created.keySet().stream().anyMatch(bean -> bean.getBeanClass() == EksService.class),+ "EksService must be created at boot, not on the first EKS request");+ });+ }+}