Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): reconcile persisted services at boot (#4501)
Abhigya52225e2053f6
diff --git a/docs/services/ecs.md b/docs/services/ecs.mdindex 4287c6eb62..037e7b408b 100644--- a/docs/services/ecs.md+++ b/docs/services/ecs.md@@ -392,6 +392,11 @@ it to a `docker exec` in the container. Deliberate limits:with `maxResults` and `nextToken`. Unlike every other ECS listing, which pages a hundred at atime, a request that names no `maxResults` gets ten ARNs and a `nextToken`.+With persistent storage, services survive a restart but their tasks do not: task state is+held in memory. The service scheduler starts at boot, so within a few seconds of startup every+persisted service is brought back to its `desiredCount`, re-registering load balancer targets+and Cloud Map instances as its tasks start, without waiting for an ECS request.+#### Service deploymentsAn `ACTIVE` service reports exactly one `PRIMARY` entry in `services[].deployments`,diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.javaindex 1d4b60363c..7d389b1c9f 100644--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner;import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;+import io.github.hectorvent.floci.services.ecs.EcsService;import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.floci.ui.FlociUiManager;@@ -93,6 +94,7 @@ public class EmulatorLifecycle {private final RdsService rdsService;private final TimestreamInfluxDbService timestreamInfluxDbService;private final ElbV2Service elbV2Service;+ private final EcsService ecsService;private final ElbClassicService elbClassicService;private final InitializationHooksRunner initializationHooksRunner;private final SqsEventSourcePoller sqsPoller;@@ -133,6 +135,7 @@ public class EmulatorLifecycle {TimestreamInfluxDbService timestreamInfluxDbService,ElbV2Service elbV2Service,ElbClassicService elbClassicService,+ EcsService ecsService,InitializationHooksRunner initializationHooksRunner,SqsEventSourcePoller sqsPoller,KinesisEventSourcePoller kinesisPoller,@@ -171,6 +174,7 @@ public class EmulatorLifecycle {this.timestreamInfluxDbService = timestreamInfluxDbService;this.elbV2Service = elbV2Service;this.elbClassicService = elbClassicService;+ this.ecsService = ecsService;this.initializationHooksRunner = initializationHooksRunner;this.sqsPoller = sqsPoller;this.kinesisPoller = kinesisPoller;@@ -259,6 +263,10 @@ public class EmulatorLifecycle {if (config.services().elb().enabled()) {elbClassicService.restorePersistedRuntime();}+ // After the load balancers: the reconciler registers the tasks it starts as their targets.+ if (config.services().ecs().enabled()) {+ ecsService.restorePersistedRuntime();+ }if (isMetadataServerNeeded()) {ec2MetadataServer.start().exceptionally(ex -> {diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex e2075d959b..5c8ee4e462 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -348,6 +348,22 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}}+ /**+ * Brings persisted services back to their desired count after a restart. Invoked from+ * {@code EmulatorLifecycle} once storage is loaded. The bean is created lazily, so without this+ * call neither the ECS stores nor the reconciler start until the first ECS request, and every+ * persisted service stays without tasks until then. Creating the bean loads the stores and+ * schedules the reconciler; this reports what that reconciler picks up.+ */+ public void restorePersistedRuntime() {+ int persistedServices = servicesStore == null+ ? services.size()+ : servicesStore.scanAllAccountEntries(key -> true).size();+ if (persistedServices > 0) {+ LOG.infov("Reconciling {0} persisted ECS service(s)", persistedServices);+ }+ }+boolean isReconcilerShutdown() {return reconciler.isShutdown();}diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.javaindex 61b0c6d738..e72d8475dd 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.services.appsync.graphql.SchemaCreationWorker;import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;import io.github.hectorvent.floci.services.dynamodb.DynamoDbRuntime;import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;+import io.github.hectorvent.floci.services.ecs.EcsService;import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;@@ -99,6 +100,7 @@ class EmulatorLifecycleTest {@Mock private EmulatorConfig.TimestreamInfluxDbServiceConfig timestreamInfluxDbServiceConfig;@Mock private io.github.hectorvent.floci.services.elbv2.ElbV2Service elbV2Service;@Mock private io.github.hectorvent.floci.services.elb.ElbClassicService elbClassicService;+ @Mock private EcsService ecsService;@Mock private InitializationHooksRunner initializationHooksRunner;@Mock private SqsEventSourcePoller sqsPoller;@Mock private KinesisEventSourcePoller kinesisPoller;@@ -127,6 +129,7 @@ class EmulatorLifecycleTest {Mockito.lenient().when(servicesConfig.ecs()).thenReturn(ecsServiceConfig);Mockito.lenient().when(ecsServiceConfig.taskRoleCredentials()).thenReturn(ecsTaskRoleCredentialsConfig);Mockito.lenient().when(ecsTaskRoleCredentialsConfig.enabled()).thenReturn(false);+ Mockito.lenient().when(ecsServiceConfig.enabled()).thenReturn(false);Mockito.lenient().when(ec2ServiceConfig.enabled()).thenReturn(false);Mockito.lenient().when(servicesConfig.elbv2()).thenReturn(elbv2ServiceConfig);Mockito.lenient().when(elbv2ServiceConfig.enabled()).thenReturn(false);@@ -152,7 +155,7 @@ class EmulatorLifecycleTest {memoryDbContainerManager, memoryDbProxyManager,docDbContainerManager, neptuneContainerManager, neptuneProxyManager,rabbitMqManager, flinkContainerManager, rdsService, timestreamInfluxDbService,- elbV2Service, elbClassicService,+ elbV2Service, elbClassicService, ecsService,initializationHooksRunner, sqsPoller, kinesisPoller, dynamodbStreamsPoller,pipesService, ec2MetadataServer, ecsTaskRoleCredentialsServer, ecrRegistryManager, flociUiManager, initLifecycleState,schemaCreationWorker, stepFunctionsService, containerTeardowns, persistentPathValidator,@@ -256,6 +259,36 @@ class EmulatorLifecycleTest {Mockito.verify(elbV2Service, Mockito.never()).restorePersistedRuntime();}+ @Test+ void restoresPersistedEcsServicesAtBootAfterStorageAndLoadBalancers() {+ // EcsService is created lazily, and creating it is what loads its stores and starts the+ // reconciler. Without this call a restart leaves every persisted service without tasks+ // until the first ECS request.+ stubStorageConfig();+ when(elbv2ServiceConfig.enabled()).thenReturn(true);+ when(ecsServiceConfig.enabled()).thenReturn(true);+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);++ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));++ InOrder inOrder = Mockito.inOrder(storageFactory, elbV2Service, ecsService);+ inOrder.verify(storageFactory).loadAll();+ inOrder.verify(elbV2Service).restorePersistedRuntime();+ inOrder.verify(ecsService).restorePersistedRuntime();+ }++ @Test+ void leavesEcsUntouchedAtBootWhenEcsIsDisabled() {+ stubStorageConfig();+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);++ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));++ verify(ecsService, never()).restorePersistedRuntime();+ }+@Test@DisplayName("Should restore ElastiCache persisted runtime after loading storage when elasticache is enabled")void shouldRestoreElastiCachePersistedRuntimeAfterStorageLoad() {diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsBootReconcilerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsBootReconcilerIntegrationTest.javanew file mode 100644index 0000000000..adb7986ee1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsBootReconcilerIntegrationTest.java@@ -0,0 +1,42 @@+package io.github.hectorvent.floci.services.ecs;++import io.quarkus.arc.Arc;+import io.quarkus.arc.InjectableBean;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.enterprise.context.ApplicationScoped;+import org.junit.jupiter.api.Test;++import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;++/**+ * EcsService is created lazily, and creating it is what loads the ECS stores and schedules the+ * reconciler. Boot has to create it, or persisted services stay without tasks after a restart+ * until the first ECS request. The profile of its own gives this test a fresh application that+ * no other test has sent an ECS request to.+ */+@QuarkusTest+@TestProfile(EcsBootReconcilerIntegrationTest.FreshBootProfile.class)+class EcsBootReconcilerIntegrationTest {++ public static final class FreshBootProfile implements QuarkusTestProfile {+ }++ @Test+ void bootStartsTheReconcilerWithoutAnEcsRequest() {+ Map<InjectableBean<?>, Object> created = Arc.container().getActiveContext(ApplicationScoped.class)+ .getState().getContextualInstances();+ EcsService ecsService = created.entrySet().stream()+ .filter(entry -> entry.getKey().getBeanClass() == EcsService.class)+ .map(entry -> (EcsService) entry.getValue())+ .findFirst()+ .orElse(null);++ assertNotNull(ecsService, "EcsService must be created at boot, not on the first ECS request");+ assertFalse(ecsService.isReconcilerShutdown());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServicePersistenceTest.javaindex 93a646880e..ee379e67c0 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServicePersistenceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServicePersistenceTest.java@@ -106,6 +106,28 @@ class EcsServicePersistenceTest {"deleted attribute must not reappear after restart");}+ @Test+ void persistedServiceGetsItsTasksBackAfterRestartWithoutAnEcsRequest() {+ SharedStorageFactory storage = new SharedStorageFactory();++ EcsService first = serviceWithStorage(storage);+ first.createCluster("app-cluster", Map.of(), REGION);+ TaskDefinition td = first.registerTaskDefinition("web", List.of(container("app", "nginx:latest")),+ NetworkMode.awsvpc, "256", "512", null, null, List.of("FARGATE"), null, REGION);+ first.createService("app-cluster", "web-svc", td.getTaskDefinitionArn(),+ 2, LaunchType.FARGATE, List.of(), awsvpcConfiguration(), Map.of(), REGION);++ // Task state is memory-only, so the restarted process starts with none. What EmulatorLifecycle+ // does at boot is restore the persisted runtime, then the reconciler ticks.+ EcsService restarted = serviceWithStorage(storage);+ restarted.restorePersistedRuntime();+ assertTrue(restarted.listTasks("app-cluster", null, null, "web-svc", REGION).isEmpty());++ restarted.reconcile();++ assertEquals(2, restarted.listTasks("app-cluster", null, null, "web-svc", REGION).size());+ }+private static ContainerDefinition container(String name, String image) {ContainerDefinition cd = new ContainerDefinition();cd.setName(name);