Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): reconcile persisted services at boot (#4501)

Abhigya52225e2053f6
diff --git a/docs/services/ecs.md b/docs/services/ecs.md
index 4287c6eb62..037e7b408b 100644
--- a/docs/services/ecs.md
+++ b/docs/services/ecs.md
@@ -392,6 +392,11 @@ it to a `docker exec` in the container. Deliberate limits:
with `maxResults` and `nextToken`. Unlike every other ECS listing, which pages a hundred at a
time, a request that names no `maxResults` gets ten ARNs and a `nextToken`.
+With persistent storage, services survive a restart but their tasks do not: task state is
+held in memory. The service scheduler starts at boot, so within a few seconds of startup every
+persisted service is brought back to its `desiredCount`, re-registering load balancer targets
+and Cloud Map instances as its tasks start, without waiting for an ECS request.
+
#### Service deployments
An `ACTIVE` service reports exactly one `PRIMARY` entry in `services[].deployments`,
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
index 1d4b60363c..7d389b1c9f 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner;
import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;
+import io.github.hectorvent.floci.services.ecs.EcsService;
import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.floci.ui.FlociUiManager;
@@ -93,6 +94,7 @@ public class EmulatorLifecycle {
private final RdsService rdsService;
private final TimestreamInfluxDbService timestreamInfluxDbService;
private final ElbV2Service elbV2Service;
+ private final EcsService ecsService;
private final ElbClassicService elbClassicService;
private final InitializationHooksRunner initializationHooksRunner;
private final SqsEventSourcePoller sqsPoller;
@@ -133,6 +135,7 @@ public class EmulatorLifecycle {
TimestreamInfluxDbService timestreamInfluxDbService,
ElbV2Service elbV2Service,
ElbClassicService elbClassicService,
+ EcsService ecsService,
InitializationHooksRunner initializationHooksRunner,
SqsEventSourcePoller sqsPoller,
KinesisEventSourcePoller kinesisPoller,
@@ -171,6 +174,7 @@ public class EmulatorLifecycle {
this.timestreamInfluxDbService = timestreamInfluxDbService;
this.elbV2Service = elbV2Service;
this.elbClassicService = elbClassicService;
+ this.ecsService = ecsService;
this.initializationHooksRunner = initializationHooksRunner;
this.sqsPoller = sqsPoller;
this.kinesisPoller = kinesisPoller;
@@ -259,6 +263,10 @@ public class EmulatorLifecycle {
if (config.services().elb().enabled()) {
elbClassicService.restorePersistedRuntime();
}
+ // After the load balancers: the reconciler registers the tasks it starts as their targets.
+ if (config.services().ecs().enabled()) {
+ ecsService.restorePersistedRuntime();
+ }
if (isMetadataServerNeeded()) {
ec2MetadataServer.start().exceptionally(ex -> {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index e2075d959b..5c8ee4e462 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -348,6 +348,22 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
}
+ /**
+ * Brings persisted services back to their desired count after a restart. Invoked from
+ * {@code EmulatorLifecycle} once storage is loaded. The bean is created lazily, so without this
+ * call neither the ECS stores nor the reconciler start until the first ECS request, and every
+ * persisted service stays without tasks until then. Creating the bean loads the stores and
+ * schedules the reconciler; this reports what that reconciler picks up.
+ */
+ public void restorePersistedRuntime() {
+ int persistedServices = servicesStore == null
+ ? services.size()
+ : servicesStore.scanAllAccountEntries(key -> true).size();
+ if (persistedServices > 0) {
+ LOG.infov("Reconciling {0} persisted ECS service(s)", persistedServices);
+ }
+ }
+
boolean isReconcilerShutdown() {
return reconciler.isShutdown();
}
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
index 61b0c6d738..e72d8475dd 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.services.appsync.graphql.SchemaCreationWorker;
import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbRuntime;
import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;
+import io.github.hectorvent.floci.services.ecs.EcsService;
import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;
@@ -99,6 +100,7 @@ class EmulatorLifecycleTest {
@Mock private EmulatorConfig.TimestreamInfluxDbServiceConfig timestreamInfluxDbServiceConfig;
@Mock private io.github.hectorvent.floci.services.elbv2.ElbV2Service elbV2Service;
@Mock private io.github.hectorvent.floci.services.elb.ElbClassicService elbClassicService;
+ @Mock private EcsService ecsService;
@Mock private InitializationHooksRunner initializationHooksRunner;
@Mock private SqsEventSourcePoller sqsPoller;
@Mock private KinesisEventSourcePoller kinesisPoller;
@@ -127,6 +129,7 @@ class EmulatorLifecycleTest {
Mockito.lenient().when(servicesConfig.ecs()).thenReturn(ecsServiceConfig);
Mockito.lenient().when(ecsServiceConfig.taskRoleCredentials()).thenReturn(ecsTaskRoleCredentialsConfig);
Mockito.lenient().when(ecsTaskRoleCredentialsConfig.enabled()).thenReturn(false);
+ Mockito.lenient().when(ecsServiceConfig.enabled()).thenReturn(false);
Mockito.lenient().when(ec2ServiceConfig.enabled()).thenReturn(false);
Mockito.lenient().when(servicesConfig.elbv2()).thenReturn(elbv2ServiceConfig);
Mockito.lenient().when(elbv2ServiceConfig.enabled()).thenReturn(false);
@@ -152,7 +155,7 @@ class EmulatorLifecycleTest {
memoryDbContainerManager, memoryDbProxyManager,
docDbContainerManager, neptuneContainerManager, neptuneProxyManager,
rabbitMqManager, flinkContainerManager, rdsService, timestreamInfluxDbService,
- elbV2Service, elbClassicService,
+ elbV2Service, elbClassicService, ecsService,
initializationHooksRunner, sqsPoller, kinesisPoller, dynamodbStreamsPoller,
pipesService, ec2MetadataServer, ecsTaskRoleCredentialsServer, ecrRegistryManager, flociUiManager, initLifecycleState,
schemaCreationWorker, stepFunctionsService, containerTeardowns, persistentPathValidator,
@@ -256,6 +259,36 @@ class EmulatorLifecycleTest {
Mockito.verify(elbV2Service, Mockito.never()).restorePersistedRuntime();
}
+ @Test
+ void restoresPersistedEcsServicesAtBootAfterStorageAndLoadBalancers() {
+ // EcsService is created lazily, and creating it is what loads its stores and starts the
+ // reconciler. Without this call a restart leaves every persisted service without tasks
+ // until the first ECS request.
+ stubStorageConfig();
+ when(elbv2ServiceConfig.enabled()).thenReturn(true);
+ when(ecsServiceConfig.enabled()).thenReturn(true);
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+
+ InOrder inOrder = Mockito.inOrder(storageFactory, elbV2Service, ecsService);
+ inOrder.verify(storageFactory).loadAll();
+ inOrder.verify(elbV2Service).restorePersistedRuntime();
+ inOrder.verify(ecsService).restorePersistedRuntime();
+ }
+
+ @Test
+ void leavesEcsUntouchedAtBootWhenEcsIsDisabled() {
+ stubStorageConfig();
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+
+ verify(ecsService, never()).restorePersistedRuntime();
+ }
+
@Test
@DisplayName("Should restore ElastiCache persisted runtime after loading storage when elasticache is enabled")
void shouldRestoreElastiCachePersistedRuntimeAfterStorageLoad() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsBootReconcilerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsBootReconcilerIntegrationTest.java
new file mode 100644
index 0000000000..adb7986ee1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsBootReconcilerIntegrationTest.java
@@ -0,0 +1,42 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import io.quarkus.arc.Arc;
+import io.quarkus.arc.InjectableBean;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.enterprise.context.ApplicationScoped;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+/**
+ * EcsService is created lazily, and creating it is what loads the ECS stores and schedules the
+ * reconciler. Boot has to create it, or persisted services stay without tasks after a restart
+ * until the first ECS request. The profile of its own gives this test a fresh application that
+ * no other test has sent an ECS request to.
+ */
+@QuarkusTest
+@TestProfile(EcsBootReconcilerIntegrationTest.FreshBootProfile.class)
+class EcsBootReconcilerIntegrationTest {
+
+ public static final class FreshBootProfile implements QuarkusTestProfile {
+ }
+
+ @Test
+ void bootStartsTheReconcilerWithoutAnEcsRequest() {
+ Map<InjectableBean<?>, Object> created = Arc.container().getActiveContext(ApplicationScoped.class)
+ .getState().getContextualInstances();
+ EcsService ecsService = created.entrySet().stream()
+ .filter(entry -> entry.getKey().getBeanClass() == EcsService.class)
+ .map(entry -> (EcsService) entry.getValue())
+ .findFirst()
+ .orElse(null);
+
+ assertNotNull(ecsService, "EcsService must be created at boot, not on the first ECS request");
+ assertFalse(ecsService.isReconcilerShutdown());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServicePersistenceTest.java
index 93a646880e..ee379e67c0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServicePersistenceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServicePersistenceTest.java
@@ -106,6 +106,28 @@ class EcsServicePersistenceTest {
"deleted attribute must not reappear after restart");
}
+ @Test
+ void persistedServiceGetsItsTasksBackAfterRestartWithoutAnEcsRequest() {
+ SharedStorageFactory storage = new SharedStorageFactory();
+
+ EcsService first = serviceWithStorage(storage);
+ first.createCluster("app-cluster", Map.of(), REGION);
+ TaskDefinition td = first.registerTaskDefinition("web", List.of(container("app", "nginx:latest")),
+ NetworkMode.awsvpc, "256", "512", null, null, List.of("FARGATE"), null, REGION);
+ first.createService("app-cluster", "web-svc", td.getTaskDefinitionArn(),
+ 2, LaunchType.FARGATE, List.of(), awsvpcConfiguration(), Map.of(), REGION);
+
+ // Task state is memory-only, so the restarted process starts with none. What EmulatorLifecycle
+ // does at boot is restore the persisted runtime, then the reconciler ticks.
+ EcsService restarted = serviceWithStorage(storage);
+ restarted.restorePersistedRuntime();
+ assertTrue(restarted.listTasks("app-cluster", null, null, "web-svc", REGION).isEmpty());
+
+ restarted.reconcile();
+
+ assertEquals(2, restarted.listTasks("app-cluster", null, null, "web-svc", REGION).size());
+ }
+
private static ContainerDefinition container(String name, String image) {
ContainerDefinition cd = new ContainerDefinition();
cd.setName(name);