Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(eks): restore persisted clusters at startup (#6214)

Robin Baeckman204231d4b3b6
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
index 8ee4df35e7..bcb9306aba 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
@@ -16,6 +16,7 @@ import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.ecs.EcsService;
import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;
+import io.github.hectorvent.floci.services.eks.EksService;
import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;
import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
@@ -56,6 +57,7 @@ import org.jboss.logging.Logger;
import java.io.IOException;
import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
@ApplicationScoped
public class EmulatorLifecycle {
@@ -115,8 +117,11 @@ public class EmulatorLifecycle {
private final DurableExecutionService durableExecutionService;
private final Instance<ContainerTeardown> containerTeardowns;
private final PersistentPathValidator persistentPathValidator;
+ private final EksService eksService;
private final DynamoDbRuntime dynamoDbRuntime;
private final RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer;
+ // Set on the boot thread, read by the async HttpServerStart observer.
+ private volatile CompletableFuture<Void> metadataServerStarted = CompletableFuture.completedFuture(null);
@Inject
public EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceRegistry,
@@ -157,7 +162,8 @@ public class EmulatorLifecycle {
Instance<ContainerTeardown> containerTeardowns,
PersistentPathValidator persistentPathValidator,
DynamoDbRuntime dynamoDbRuntime,
- RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer) {
+ RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer,
+ EksService eksService) {
this.storageFactory = storageFactory;
this.serviceRegistry = serviceRegistry;
this.config = config;
@@ -196,6 +202,7 @@ public class EmulatorLifecycle {
this.durableExecutionService = durableExecutionService;
this.containerTeardowns = containerTeardowns;
this.persistentPathValidator = persistentPathValidator;
+ this.eksService = eksService;
this.dynamoDbRuntime = dynamoDbRuntime;
this.redshiftZeroEtlConsumer = redshiftZeroEtlConsumer;
}
@@ -279,7 +286,7 @@ public class EmulatorLifecycle {
}
if (isMetadataServerNeeded()) {
- ec2MetadataServer.start().exceptionally(ex -> {
+ metadataServerStarted = ec2MetadataServer.start().exceptionally(ex -> {
LOG.warnv("EC2 IMDS server failed to start: {0}", ex.getMessage());
return null;
});
@@ -310,6 +317,7 @@ public class EmulatorLifecycle {
boolean hasStart = initializationHooksRunner.hasHooks(InitializationHook.START);
boolean hasReady = initializationHooksRunner.hasHooks(InitializationHook.READY);
if (!hasStart && !hasReady) {
+ restoreEksClusters();
return;
}
try {
@@ -322,6 +330,7 @@ public class EmulatorLifecycle {
}
initLifecycleState.markReadyCompleted();
logReady();
+ restoreEksClusters();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.error("Startup hook execution interrupted — shutting down", e);
@@ -332,6 +341,25 @@ public class EmulatorLifecycle {
}
}
+ /**
+ * Creating EksService restarts persisted clusters; without this call that waited for the first EKS request.
+ * It runs once Floci's HTTP port and IMDS server listen, because a recreated cluster's Pod Identity and IMDS
+ * relays probe them before they route pod traffic there, and after the start and ready hooks, which it
+ * could otherwise hold up for as long as the clusters take to start.
+ */
+ private void restoreEksClusters() {
+ if (!config.services().eks().enabled() || config.services().eks().mock()) {
+ return;
+ }
+ try {
+ metadataServerStarted.join();
+ eksService.restorePersistedRuntime();
+ } catch (RuntimeException e) {
+ // EKS requests retry the bean's creation, as before; the rest of the emulator still starts.
+ LOG.error("Failed to restore persisted EKS clusters at startup", e);
+ }
+ }
+
/**
* LocalStack prints a line ending in "Ready." when its gateway is up, and
* ecosystem tooling keys on it — Testcontainers' LocalStackContainer default
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
index a16b1b6e9f..fea594f840 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
@@ -156,6 +156,14 @@ public class EksService implements TagHandler, ResourceProvider, Ec2InstanceLaun
oidcService, accessEntries, null, null);
}
+ /** Called at boot so this lazily created bean runs {@link #init()}, which restarts persisted clusters. */
+ public void restorePersistedRuntime() {
+ int persistedClusters = allClusterEntries().size();
+ if (persistedClusters > 0) {
+ LOG.infov("Found {0} persisted EKS cluster record(s)", persistedClusters);
+ }
+ }
+
@PostConstruct
public void init() {
backfillOidcIdentities();
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
index a0fa7949ef..e160ca834c 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
@@ -14,6 +14,7 @@ import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.ecs.EcsService;
import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;
+import io.github.hectorvent.floci.services.eks.EksService;
import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;
import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
@@ -104,6 +105,7 @@ class EmulatorLifecycleTest {
@Mock private io.github.hectorvent.floci.services.elbv2.ElbV2Service elbV2Service;
@Mock private io.github.hectorvent.floci.services.elb.ElbClassicService elbClassicService;
@Mock private EcsService ecsService;
+ @Mock private EksService eksService;
@Mock private InitializationHooksRunner initializationHooksRunner;
@Mock private SqsEventSourcePoller sqsPoller;
@Mock private KinesisEventSourcePoller kinesisPoller;
@@ -166,7 +168,7 @@ class EmulatorLifecycleTest {
pipesService, ec2MetadataServer, ecsTaskRoleCredentialsServer, ecrRegistryManager, flociUiManager, initLifecycleState,
schemaCreationWorker, stepFunctionsService, durableExecutionService, containerTeardowns,
persistentPathValidator,
- dynamoDbRuntime, redshiftZeroEtlConsumer);
+ dynamoDbRuntime, redshiftZeroEtlConsumer, eksService);
Mockito.lenient().when(containerTeardowns.iterator())
.thenReturn(java.util.Collections.emptyIterator());
}
@@ -285,6 +287,100 @@ class EmulatorLifecycleTest {
inOrder.verify(ecsService).restorePersistedRuntime();
}
+ @Test
+ void restoresPersistedEksClustersOnceTheHttpPortListens() throws Exception {
+ stubStorageConfig();
+ when(eksServiceConfig.enabled()).thenReturn(true);
+ when(eksServiceConfig.mock()).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+ verify(eksService, never()).restorePersistedRuntime();
+
+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));
+ verify(eksService).restorePersistedRuntime();
+ }
+
+ @Test
+ void restoresPersistedEksClustersOnlyOnceTheImdsServerHasStarted() throws Exception {
+ stubStorageConfig();
+ when(eksServiceConfig.enabled()).thenReturn(true);
+ when(eksServiceConfig.mock()).thenReturn(false);
+ when(eksServiceConfig.imds()).thenReturn(true);
+ CompletableFuture<Void> imdsStarted = new CompletableFuture<>();
+ when(ec2MetadataServer.start()).thenReturn(imdsStarted);
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+
+ Thread httpStart = Thread.ofVirtual().start(() ->
+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566))));
+ verify(eksService, Mockito.after(300).never()).restorePersistedRuntime();
+
+ imdsStarted.complete(null);
+ httpStart.join(5000);
+ verify(eksService).restorePersistedRuntime();
+ }
+
+ @Test
+ void restoresPersistedEksClustersAfterTheStartAndReadyHooks() throws Exception {
+ when(eksServiceConfig.enabled()).thenReturn(true);
+ when(eksServiceConfig.mock()).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(true);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(true);
+
+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));
+
+ InOrder inOrder = Mockito.inOrder(initializationHooksRunner, initLifecycleState, eksService);
+ inOrder.verify(initializationHooksRunner).run(InitializationHook.START);
+ inOrder.verify(initializationHooksRunner).run(InitializationHook.READY);
+ inOrder.verify(initLifecycleState).markReadyCompleted();
+ inOrder.verify(eksService).restorePersistedRuntime();
+ }
+
+ @Test
+ void aFailedEksRestoreLeavesTheEmulatorReady() throws Exception {
+ when(eksServiceConfig.enabled()).thenReturn(true);
+ when(eksServiceConfig.mock()).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(true);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+ doThrow(new IllegalStateException("corrupt cluster record")).when(eksService).restorePersistedRuntime();
+
+ try (MockedStatic<Quarkus> quarkus = Mockito.mockStatic(Quarkus.class,
+ Mockito.withSettings().mockMaker(MockMakers.INLINE))) {
+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));
+ quarkus.verify(Quarkus::asyncExit, never());
+ }
+ verify(initLifecycleState).markReadyCompleted();
+ }
+
+ @Test
+ void leavesEksUntouchedAtBootWhenEksIsDisabled() throws Exception {
+ stubStorageConfig();
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));
+
+ verify(eksService, Mockito.never()).restorePersistedRuntime();
+ }
+
+ @Test
+ void leavesEksUntouchedAtBootInMockMode() throws Exception {
+ stubStorageConfig();
+ when(eksServiceConfig.enabled()).thenReturn(true);
+ when(eksServiceConfig.mock()).thenReturn(true);
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+ emulatorLifecycle.onHttpStart(new HttpServerStart(new HttpServerOptions().setPort(4566)));
+
+ verify(eksService, Mockito.never()).restorePersistedRuntime();
+ }
+
@Test
void leavesEcsUntouchedAtBootWhenEcsIsDisabled() {
stubStorageConfig();
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksBootRestoreIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksBootRestoreIntegrationTest.java
new file mode 100644
index 0000000000..5160e9f2cd
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksBootRestoreIntegrationTest.java
@@ -0,0 +1,63 @@
+package io.github.hectorvent.floci.services.eks;
+
+import io.github.hectorvent.floci.testutil.FreePorts;
+import io.quarkus.arc.Arc;
+import io.quarkus.arc.InjectableBean;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.enterprise.context.ApplicationScoped;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.time.Duration;
+import java.util.Map;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * EksService is created lazily, and creating it is what restarts each persisted cluster's k3s
+ * container. Boot has to create it, or persisted clusters stay down after a restart until the
+ * first EKS request. The profile of its own gives this test a fresh application that no other
+ * test has sent an EKS request to, with EKS out of mock mode as boot restores only real clusters.
+ */
+@QuarkusTest
+@TestProfile(EksBootRestoreIntegrationTest.FreshBootProfile.class)
+class EksBootRestoreIntegrationTest {
+
+ public static final class FreshBootProfile implements QuarkusTestProfile {
+ // Boot restores once quarkus.http.port listens; the suite's random port 0 never matches the bound one.
+ private static final int TEST_PORT = freePort();
+
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of(
+ "floci.services.eks.mock", "false",
+ "quarkus.http.test-port", String.valueOf(TEST_PORT),
+ "quarkus.http.port", String.valueOf(TEST_PORT),
+ "floci.port", String.valueOf(TEST_PORT),
+ "floci.base-url", "http://localhost:" + TEST_PORT);
+ }
+
+ private static int freePort() {
+ try {
+ return FreePorts.anyInterfacePort();
+ } catch (IOException e) {
+ throw new UncheckedIOException("No free port for the EKS boot test application", e);
+ }
+ }
+ }
+
+ @Test
+ void bootCreatesEksServiceWithoutAnEksRequest() {
+ // Boot restores once the HTTP port listens, from an async observer, so this can run first.
+ await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> {
+ Map<InjectableBean<?>, Object> created = Arc.container().getActiveContext(ApplicationScoped.class)
+ .getState().getContextualInstances();
+ assertTrue(created.keySet().stream().anyMatch(bean -> bean.getBeanClass() == EksService.class),
+ "EksService must be created at boot, not on the first EKS request");
+ });
+ }
+}