Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): wait for a stable ECS service (#4880)

Mario Toffia0ec64add2d0f
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudFormationEcsServiceTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudFormationEcsServiceTest.java
new file mode 100644
index 0000000000..4554dd5260
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudFormationEcsServiceTest.java
@@ -0,0 +1,140 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.cloudformation.CloudFormationClient;
+import software.amazon.awssdk.services.cloudformation.model.Parameter;
+import software.amazon.awssdk.services.cloudformation.model.StackStatus;
+import software.amazon.awssdk.services.ec2.Ec2Client;
+import software.amazon.awssdk.services.ec2.model.Subnet;
+import software.amazon.awssdk.services.ecs.EcsClient;
+import software.amazon.awssdk.services.ecs.model.DeploymentRolloutState;
+import software.amazon.awssdk.services.ecs.model.Service;
+
+import java.time.Duration;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * CloudFormation reports an {@code AWS::ECS::Service} CREATE_COMPLETE only once the service has
+ * reached a steady state, so DescribeServices right after the stack completes shows the desired
+ * tasks running.
+ */
+@DisplayName("CloudFormation AWS::ECS::Service steady state")
+class CloudFormationEcsServiceTest {
+
+ private static final String TEMPLATE = """
+ {
+ "Parameters": {
+ "Name": {"Type": "String"},
+ "Subnet": {"Type": "String"}
+ },
+ "Resources": {
+ "Cluster": {
+ "Type": "AWS::ECS::Cluster",
+ "Properties": {"ClusterName": {"Ref": "Name"}}
+ },
+ "TaskDef": {
+ "Type": "AWS::ECS::TaskDefinition",
+ "Properties": {
+ "Family": {"Ref": "Name"},
+ "NetworkMode": "awsvpc",
+ "RequiresCompatibilities": ["FARGATE"],
+ "Cpu": "256",
+ "Memory": "512",
+ "ContainerDefinitions": [
+ {"Name": "web", "Image": "nginx:alpine", "Essential": true}
+ ]
+ }
+ },
+ "Service": {
+ "Type": "AWS::ECS::Service",
+ "Properties": {
+ "ServiceName": {"Ref": "Name"},
+ "Cluster": {"Ref": "Cluster"},
+ "TaskDefinition": {"Ref": "TaskDef"},
+ "DesiredCount": 1,
+ "LaunchType": "FARGATE",
+ "NetworkConfiguration": {
+ "AwsvpcConfiguration": {"Subnets": [{"Ref": "Subnet"}], "AssignPublicIp": "ENABLED"}
+ }
+ }
+ }
+ }
+ }
+ """;
+
+ private static CloudFormationClient cfn;
+ private static EcsClient ecs;
+ private static Ec2Client ec2;
+ private String stackName;
+ private boolean stackCreated;
+
+ @BeforeAll
+ static void clients() {
+ cfn = TestFixtures.cloudFormationClient();
+ ecs = TestFixtures.ecsClient();
+ ec2 = TestFixtures.ec2Client();
+ }
+
+ @BeforeEach
+ void setup() {
+ stackName = TestFixtures.uniqueName("compat-cfn-ecs-svc");
+ }
+
+ @AfterEach
+ void cleanup() {
+ if (stackCreated) {
+ cfn.deleteStack(r -> r.stackName(stackName));
+ cfn.waiter().waitUntilStackDeleteComplete(r -> r.stackName(stackName),
+ w -> w.waitTimeout(Duration.ofMinutes(5)));
+ }
+ }
+
+ @AfterAll
+ static void closeClients() {
+ cfn.close();
+ ecs.close();
+ ec2.close();
+ }
+
+ @Test
+ @DisplayName("DescribeServices right after CREATE_COMPLETE shows the desired task running")
+ void createCompleteMeansTheDesiredTaskIsRunning() {
+ cfn.createStack(r -> r.stackName(stackName).templateBody(TEMPLATE).parameters(
+ parameter("Name", stackName), parameter("Subnet", defaultSubnet())));
+ stackCreated = true;
+ StackStatus status = cfn.waiter().waitUntilStackCreateComplete(r -> r.stackName(stackName),
+ w -> w.waitTimeout(Duration.ofMinutes(10)))
+ .matched().response().orElseThrow().stacks().get(0).stackStatus();
+ assertThat(status).isEqualTo(StackStatus.CREATE_COMPLETE);
+
+ Service service = ecs.describeServices(r -> r.cluster(stackName).services(stackName))
+ .services().get(0);
+
+ assertThat(service.desiredCount()).isEqualTo(1);
+ assertThat(service.runningCount()).as("runningCount right after CREATE_COMPLETE").isEqualTo(1);
+ assertThat(service.deployments()).hasSize(1);
+ assertThat(service.deployments().get(0).rolloutState()).isEqualTo(DeploymentRolloutState.COMPLETED);
+ }
+
+ private static Parameter parameter(String key, String value) {
+ return Parameter.builder().parameterKey(key).parameterValue(value).build();
+ }
+
+ /** A subnet of the default VPC, which the task's network interface goes into. */
+ private static String defaultSubnet() {
+ List<Subnet> subnets = ec2.describeSubnets().subnets();
+ return subnets.stream()
+ .filter(subnet -> Boolean.TRUE.equals(subnet.defaultForAz()))
+ .map(Subnet::subnetId)
+ .sorted()
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException("no default-VPC subnet to place the task in"));
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java
index 86631a5721..9689af0235 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java
@@ -168,7 +168,9 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {
* create time is read back from the {@code Name} attribute rather than the prior id. An update
* that keeps the name and the cluster updates the service in place; a changed name or a
* changed cluster (both create-only) is a replacement: the new service is created and the
- * displaced one deleted once the stack update commits, through the replacement cleanup.
+ * displaced one deleted once the stack update commits, through the replacement cleanup. Either
+ * way the resource completes only once the service is stable, as CloudFormation's does; an
+ * in-place update that does not stabilize is put back on the prior configuration.
*/
private void provisionService(StackResource r, JsonNode props, ProvisionContext ctx) {
String clusterRef = ctx.resolveOptional(props, "Cluster");
@@ -190,8 +192,14 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {
}
EcsServiceModel svc;
- if (ctx.isUpdate() && serviceName.equals(priorName)
- && clusterName(clusterRef).equals(clusterOfServiceArn(ctx.priorPhysicalId()))) {
+ boolean inPlace = ctx.isUpdate() && serviceName.equals(priorName)
+ && clusterName(clusterRef).equals(clusterOfServiceArn(ctx.priorPhysicalId()));
+ // Read before updateService, which changes this same model in place.
+ EcsServiceModel prior = inPlace ? ecsService.serviceByArn(ctx.priorPhysicalId()) : null;
+ String priorTaskDefinition = prior != null ? prior.getTaskDefinition() : null;
+ Integer priorDesiredCount = prior != null ? prior.getDesiredCount() : null;
+ NetworkConfiguration priorNetworkConfiguration = prior != null ? prior.getNetworkConfiguration() : null;
+ if (inPlace) {
svc = ecsService.updateService(clusterRef, serviceName, taskDefinition,
desiredCount, networkConfiguration, ctx.region());
} else {
@@ -202,6 +210,73 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {
r.setPhysicalId(svc.getServiceArn());
r.getAttributes().put("Name", svc.getServiceName());
r.getAttributes().put("ServiceArn", svc.getServiceArn());
+ if (inPlace) {
+ try {
+ ecsService.awaitServiceStable(svc);
+ } catch (RuntimeException failure) {
+ restoreInPlace(r, failure, () -> ecsService.awaitServiceStable(ecsService.updateService(clusterRef,
+ svc.getServiceName(), priorTaskDefinition, priorDesiredCount, priorNetworkConfiguration,
+ ctx.region())));
+ throw failure;
+ }
+ } else {
+ awaitCreatedServiceStable(r, svc, ctx);
+ }
+ }
+
+ /**
+ * Puts a service updated in place back on the task definition, desired count and network
+ * configuration it had, when the update does not stabilize, and waits for the restored
+ * configuration to stabilize, as CloudFormation's rollback does. CloudFormationService restores
+ * the prior resource record and reports the rollback complete unless told otherwise, so without
+ * this the stack would claim a rollback while ECS still runs the failed configuration. A
+ * restore that fails or does not stabilize is attached to the original failure and recorded as
+ * a rollback failure.
+ */
+ private void restoreInPlace(StackResource r, RuntimeException failure, Runnable restore) {
+ try {
+ restore.run();
+ } catch (RuntimeException restoreFailure) {
+ String reason = "Could not restore " + r.getPhysicalId() + " after a failed update: "
+ + restoreFailure.getMessage();
+ LOG.warnv("{0}", reason);
+ r.getAttributes().put(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR, reason);
+ if (restoreFailure != failure) {
+ failure.addSuppressed(restoreFailure);
+ }
+ }
+ }
+
+ /**
+ * Waits for a service this provision created to stabilize, as CloudFormation does before it
+ * reports the resource complete. The resource is marked as owned meanwhile, so a service that
+ * never stabilizes is deleted by the create rollback. A replacement created by a stack update
+ * is removed here instead: the engine restores the prior resource and never learns the new ARN.
+ */
+ private void awaitCreatedServiceStable(StackResource r, EcsServiceModel svc, ProvisionContext ctx) {
+ r.getAttributes().put(CfnRollback.ROLLBACK_OWNED_ATTR, "true");
+ try {
+ ecsService.awaitServiceStable(svc);
+ } catch (RuntimeException failure) {
+ if (ctx.isUpdate()) {
+ unwindReplacement(r, ctx.region(), failure);
+ }
+ throw failure;
+ }
+ r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);
+ }
+
+ private void unwindReplacement(StackResource r, String region, RuntimeException failure) {
+ try {
+ deleteService(r.getPhysicalId(), region);
+ r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);
+ } catch (RuntimeException cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
+ String reason = "Could not remove " + r.getPhysicalId() + ", the replacement created for "
+ + r.getLogicalId() + " by a failed update: " + cleanupFailure.getMessage();
+ LOG.warnv("{0}", reason);
+ r.getAttributes().put(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR, reason);
+ }
}
/** The cluster name a template value addresses: a name, an ARN's last segment, or the default. */
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index ede10ae55d..1a6fd0efee 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -73,6 +73,7 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Base64;
@@ -91,6 +92,8 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
@@ -179,6 +182,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
public static final int MAX_TASKS_PER_RUN = 10;
/** A listing returns at most a hundred ARNs per page. */
private static final int MAX_LIST_RESULTS = 100;
+ /** How long a CloudFormation-provisioned service may take to reach a steady state. */
+ private static final Duration SERVICE_STABLE_TIMEOUT = Duration.ofMinutes(5);
+ private static final long SERVICE_STABLE_POLL_MILLIS = 50;
+ /**
+ * Reconcile passes a steady-state wait runs at once instead of waiting for ticks. A pass
+ * reports the running count it found before it acted, so a deployment that both starts tasks
+ * and drains the previous deployment's reads as settled after the third.
+ */
+ private static final int SERVICE_STABLE_PASSES = 3;
/** ListServices answers with ten ARNs per page when the request names no maxResults. */
private static final int DEFAULT_SERVICE_PAGE_SIZE = 10;
public static final String STATUS_ACTIVE = "ACTIVE";
@@ -210,6 +222,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
private Map<String, String> accountSettings = new ConcurrentHashMap<>();
// deploymentIds for which SERVICE_DEPLOYMENT_IN_PROGRESS has already been emitted this process.
private final Set<String> inProgressEmitted = java.util.concurrent.ConcurrentHashMap.newKeySet();
+ // serviceArn → why the reconciler's last attempt to launch one of its tasks failed, and for which
+ // deployment. Memory-only.
+ private final Map<String, LaunchFailure> launchFailures = new ConcurrentHashMap<>();
@Inject
public EcsService(RegionResolver regionResolver, EcsContainerManager containerManager,
@@ -385,7 +400,8 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
@Override
public void clear() {
- // Nothing to wipe: the stores hold the ECS state and afterReset() restarts the reconciler.
+ // The stores hold the ECS state and afterReset() restarts the reconciler.
+ launchFailures.clear();
}
/**
@@ -426,6 +442,150 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
return reconciler.isShutdown();
}
+ // ── Steady state ─────────────────────────────────────────────────────────
+
+ /**
+ * Waits until a service reaches a steady state, which is what CloudFormation waits for before
+ * it reports an {@code AWS::ECS::Service} CREATE_COMPLETE or UPDATE_COMPLETE. CloudFormation
+ * reads it off {@code DescribeServices}, so the service must report {@code runningCount}
+ * equal to {@code desiredCount}, its current deployment must run that many tasks, and no task
+ * of an earlier deployment may still be running. A {@code desiredCount} of 0 with nothing
+ * running is stable at once.
+ *
+ * <p>Rather than sit idle until the next tick, the wait runs reconcile passes at once on the
+ * reconciler's own single thread, so a pass never overlaps a scheduled tick.
+ *
+ * @throws AwsException when the reconciler could not launch a task for the service, when a task
+ * launched for it during the wait failed to start, or when the service does not stabilize
+ * within the timeout
+ */
+ public void awaitServiceStable(EcsServiceModel svc) {
+ awaitServiceStable(svc, SERVICE_STABLE_TIMEOUT);
+ }
+
+ void awaitServiceStable(EcsServiceModel svc, Duration timeout) {
+ String serviceArn = svc.getServiceArn();
+ recordLaunchFailure(svc, null, null);
+ Instant waitStarted = Instant.now();
+ long deadline = System.nanoTime() + timeout.toNanos();
+ Future<?> pass = null;
+ int passes = 0;
+ while (true) {
+ // Checked before the service is read, so a finished pass's writes are visible to the read.
+ boolean passRunning = pass != null && !pass.isDone();
+ EcsServiceModel current = serviceByArn(serviceArn);
+ if (current == null || !STATUS_ACTIVE.equals(current.getStatus())) {
+ throw notStabilized(serviceArn, " The service is no longer ACTIVE.");
+ }
+ if (!passRunning) {
+ failIfTasksCannotStart(current, waitStarted);
+ }
+ if (isSteady(current)) {
+ return;
+ }
+ if (!passRunning && passes < SERVICE_STABLE_PASSES) {
+ pass = reconcileNow();
+ passes++;
+ }
+ if (System.nanoTime() - deadline >= 0) {
+ throw notStabilized(serviceArn, "");
+ }
+ try {
+ Thread.sleep(SERVICE_STABLE_POLL_MILLIS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AwsException("InternalError", "Interrupted while waiting for ECS service "
+ + serviceArn + " to stabilize", 500);
+ }
+ }
+ }
+
+ /**
+ * Runs one reconcile pass now on the reconciler's single thread. Null when the reconciler is
+ * shut down, which only happens on shutdown and for the length of a state reset.
+ */
+ private Future<?> reconcileNow() {
+ try {
+ return reconciler.submit(this::reconcile);
+ } catch (RejectedExecutionException e) {
+ LOG.debugv("ECS reconciler is shut down, waiting for the next scheduled tick: {0}", e.getMessage());
+ return null;
+ }
+ }
+
+ private boolean isSteady(EcsServiceModel svc) {
+ if (svc.getRunningCount() != svc.getDesiredCount()) {
+ return false;
+ }
+ EcsCluster cluster = resolveClusterByArn(svc.getClusterArn());
+ if (cluster == null) {
+ return svc.getDesiredCount() == 0;
+ }
+ String currentDeploymentId = deploymentId(svc);
+ String taskDefinitionArn = resolvedTaskDefinitionArn(svc, serviceRegion(svc));
+ long current = 0;
+ for (EcsTask task : tasks.values()) {
+ if (!ownedBy(task, svc, cluster) || !TaskStatus.RUNNING.name().equals(task.getLastStatus())) {
+ continue;
+ }
+ if (isStaleForDeployment(task, currentDeploymentId, taskDefinitionArn)) {
+ return false;
+ }
+ current++;
+ }
+ return current == svc.getDesiredCount();
+ }
+
+ /**
+ * Ends a wait that cannot succeed: the reconciler could not launch a task for the service's
+ * current deployment, or a task launched for it since {@code since} stopped before it ever ran.
+ */
+ private void failIfTasksCannotStart(EcsServiceModel svc, Instant since) {
+ String currentDeploymentId = deploymentId(svc);
+ LaunchFailure launchFailure = launchFailures.get(svc.getServiceArn());
+ if (launchFailure != null && currentDeploymentId.equals(launchFailure.deploymentId())) {
+ throw notStabilized(svc.getServiceArn(), " Task launch failed: " + launchFailure.message());
+ }
+ EcsCluster cluster = resolveClusterByArn(svc.getClusterArn());
+ if (cluster == null) {
+ return;
+ }
+ String taskDefinitionArn = resolvedTaskDefinitionArn(svc, serviceRegion(svc));
+ for (EcsTask task : tasks.values()) {
+ if (ownedBy(task, svc, cluster)
+ && TaskStatus.STOPPED.name().equals(task.getLastStatus())
+ && STOP_CODE_TASK_FAILED_TO_START.equals(task.getStopCode())
+ && task.getCreatedAt() != null && !task.getCreatedAt().isBefore(since)
+ && !isStaleForDeployment(task, currentDeploymentId, taskDefinitionArn)) {
+ throw notStabilized(svc.getServiceArn(),
+ " Task " + task.getTaskArn() + " stopped: " + task.getStoppedReason());
+ }
+ }
+ }
+
+ /** AWS's wording when a service does not reach a steady state, followed by any detail. */
+ private static AwsException notStabilized(String serviceArn, String detail) {
+ return new AwsException("NotStabilized", "Service " + serviceArn + " did not stabilize." + detail, 500);
+ }
+
+ private record LaunchFailure(String deploymentId, String message) {}
+
+ /** Records why the reconciler could not launch a task for {@code svc}'s {@code deploymentId}; null clears it. */
+ private void recordLaunchFailure(EcsServiceModel svc, String deploymentId, String failure) {
+ if (svc.getServiceArn() == null) {
+ return;
+ }
+ if (failure == null) {
+ launchFailures.remove(svc.getServiceArn());
+ } else {
+ launchFailures.put(svc.getServiceArn(), new LaunchFailure(deploymentId, failure));
+ }
+ }
+
+ boolean hasLaunchFailure(String serviceArn) {
+ return launchFailures.containsKey(serviceArn);
+ }
+
// ── Clusters ─────────────────────────────────────────────────────────────
public EcsCluster createCluster(String clusterName, String region) {
@@ -1480,13 +1640,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
*/
private EcsTask launchServiceTask(EcsCluster cluster, EcsServiceModel svc, LaunchType launchType,
String containerInstanceArn, String region) {
+ // Read before the task definition: UpdateService sets it first, so a launch it overtakes is stamped stale.
+ String deploymentId = deploymentId(svc);
TaskDefinition taskDef = resolveTaskDefinitionOrThrow(svc.getTaskDefinition(), region);
RunTaskRequest request = new RunTaskRequest();
request.setCount(1);
request.setLaunchType(launchType);
// AWS labels a service's tasks "service:<name>" and attributes them to the deployment.
request.setGroup("service:" + svc.getServiceName());
- request.setStartedBy(deploymentId(svc));
+ request.setStartedBy(deploymentId);
request.setNetworkConfiguration(svc.getNetworkConfiguration());
request.setPlatformVersion(svc.getPlatformVersion());
request.setEnableExecuteCommand(svc.isEnableExecuteCommand());
@@ -1503,7 +1665,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
EcsTask task = launchTasks(cluster, taskDef, request, containerInstanceArn,
svc.getServiceArn(), region).getFirst();
- task.setDeploymentId(deploymentId(svc));
+ task.setDeploymentId(deploymentId);
return task;
}
@@ -2775,6 +2937,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
if (serviceArn != null) {
serviceDeployments.values().removeIf(d -> serviceArn.equals(d.getServiceArn()));
serviceRevisions.values().removeIf(r -> serviceArn.equals(r.getServiceArn()));
+ launchFailures.remove(serviceArn);
}
return svc;
}
@@ -5084,12 +5247,14 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
for (int i = 0; i < toStart; i++) {
try {
EcsTask launched = launchServiceTask(cluster, svc, svc.getLaunchType(), null, region);
+ recordLaunchFailure(svc, null, null);
LOG.infov("Service reconciler started task {0} for service {1}",
launched.getTaskArn(), svc.getServiceName());
if (countLaunch(svc, deployment, launched)) {
break;
}
} catch (Exception e) {
+ recordLaunchFailure(svc, currentDeploymentId, String.valueOf(e.getMessage()));
LOG.warnv("Service reconciler failed to start task for {0}: {1}",
svc.getServiceName(), e.getMessage());
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEcsServiceStableIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEcsServiceStableIntegrationTest.java
new file mode 100644
index 0000000000..65edf29f32
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEcsServiceStableIntegrationTest.java
@@ -0,0 +1,208 @@
+package io.github.hectorvent.floci.services.cloudformation;
+
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * CloudFormation reports an {@code AWS::ECS::Service} complete only once the service is stable,
+ * so a caller that reads the service right after the stack operation finishes sees its tasks
+ * running. Floci used to report the resource complete as soon as CreateService or UpdateService
+ * returned, and the tasks only started on a later reconciler tick.
+ */
+@QuarkusTest
+class CloudFormationEcsServiceStableIntegrationTest {
+
+ private static final String ECS_TARGET = "AmazonEC2ContainerServiceV20141113.";
+ private static final String ECS_CT = "application/x-amz-json-1.1";
+
+ private static final String TEMPLATE = """
+ {
+ "Resources": {
+ "Cluster": {
+ "Type": "AWS::ECS::Cluster",
+ "Properties": { "ClusterName": "%1$s" }
+ },
+ "TaskDef": {
+ "Type": "AWS::ECS::TaskDefinition",
+ "Properties": {
+ "Family": "%1$s-td",
+ "ContainerDefinitions": [
+ { "Name": "app", "Image": "%2$s", "Essential": true, "Memory": 128 }
+ ]
+ }
+ },
+ "Service": {
+ "Type": "AWS::ECS::Service",
+ "Properties": {
+ "ServiceName": "%1$s-svc",
+ "Cluster": { "Ref": "Cluster" },
+ "TaskDefinition": { "Ref": "TaskDef" },
+ "DesiredCount": %3$d,
+ "LaunchType": "FARGATE"
+ }
+ }
+ }
+ }
+ """;
+
+ private static final String AWSVPC_TEMPLATE = """
+ {
+ "Resources": {
+ "Cluster": {
+ "Type": "AWS::ECS::Cluster",
+ "Properties": { "ClusterName": "%1$s" }
+ },
+ "TaskDef": {
+ "Type": "AWS::ECS::TaskDefinition",
+ "Properties": {
+ "Family": "%1$s-td",
+ "NetworkMode": "awsvpc",
+ "ContainerDefinitions": [
+ { "Name": "app", "Image": "app:v1", "Essential": true, "Memory": 128 }
+ ]
+ }
+ },
+ "Service": {
+ "Type": "AWS::ECS::Service",
+ "Properties": {
+ "ServiceName": "%1$s-svc",
+ "Cluster": { "Ref": "Cluster" },
+ "TaskDefinition": { "Ref": "TaskDef" },
+ "DesiredCount": %2$d,
+ "LaunchType": "FARGATE",
+ "NetworkConfiguration": {
+ "AwsvpcConfiguration": { "Subnets": ["%3$s"] }
+ }
+ }
+ }
+ }
+ }
+ """;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void createStack_ecsService_runsItsDesiredTasksWhenTheStackCompletes() {
+ String name = "cfn-stable-" + Long.toString(System.nanoTime(), 36);
+
+ stackCall("CreateStack", name, TEMPLATE.formatted(name, "app:v1", 1));
+
+ assertThat(stackStatus(name), equalTo("CREATE_COMPLETE"));
+ Response service = describeService(name);
+ assertThat(service.path("services[0].desiredCount"), equalTo(1));
+ assertThat(service.path("services[0].runningCount"), equalTo(1));
+ assertThat(service.path("services[0].deployments"), hasSize(1));
+ assertThat(service.path("services[0].deployments[0].runningCount"), equalTo(1));
+ assertThat(service.path("services[0].deployments[0].rolloutState"), equalTo("COMPLETED"));
+ assertThat(runningTaskDefinitions(name), hasSize(1));
+ }
+
+ @Test
+ void updateStack_ecsService_runsOnlyTheNewRevisionWhenTheUpdateCompletes() {
+ String name = "cfn-stable-upd-" + Long.toString(System.nanoTime(), 36);
+ stackCall("CreateStack", name, TEMPLATE.formatted(name, "app:v1", 1));
+
+ stackCall("UpdateStack", name, TEMPLATE.formatted(name, "app:v2", 2));
+
+ assertThat(stackStatus(name), equalTo("UPDATE_COMPLETE"));
+ Response service = describeService(name);
+ assertThat(service.path("services[0].desiredCount"), equalTo(2));
+ assertThat(service.path("services[0].runningCount"), equalTo(2));
+ assertThat(service.path("services[0].taskDefinition"), endsWith(":task-definition/" + name + "-td:2"));
+ List<String> running = runningTaskDefinitions(name);
+ assertThat(running, hasSize(2));
+ assertThat("the previous revision's task has stopped", running,
+ everyItem(endsWith(":task-definition/" + name + "-td:2")));
+ }
+
+ @Test
+ void updateStack_ecsServiceThatCannotStabilize_putsThePriorConfigurationBackOnRollback() {
+ String name = "cfn-stable-rb-" + Long.toString(System.nanoTime(), 36);
+ String subnet = Ec2Service.defaultSubnetId("us-east-1", "a");
+ stackCall("CreateStack", name, AWSVPC_TEMPLATE.formatted(name, 1, subnet));
+ assertThat(stackStatus(name), equalTo("CREATE_COMPLETE"));
+
+ stackCall("UpdateStack", name, AWSVPC_TEMPLATE.formatted(name, 2, "subnet-does-not-exist"));
+
+ assertThat(stackStatus(name), equalTo("UPDATE_ROLLBACK_COMPLETE"));
+ Response service = describeService(name);
+ assertThat("the rollback must not leave the failed update live",
+ service.path("services[0].desiredCount"), equalTo(1));
+ assertThat(service.path("services[0].networkConfiguration.awsvpcConfiguration.subnets"),
+ equalTo(List.of(subnet)));
+ }
+
+ private static void stackCall(String action, String stackName, String template) {
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", action)
+ .formParam("StackName", stackName)
+ .formParam("TemplateBody", template)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<StackId>"));
+ }
+
+ private static String stackStatus(String stackName) {
+ return given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStacks")
+ .formParam("StackName", stackName)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().xmlPath().getString("**.find { it.name() == 'StackStatus' }");
+ }
+
+ private static Response describeService(String name) {
+ return ecs("DescribeServices",
+ "{\"cluster\":\"" + name + "\",\"services\":[\"" + name + "-svc\"]}");
+ }
+
+ /** The task definition ARN of every RUNNING task of the stack's service. */
+ private static List<String> runningTaskDefinitions(String name) {
+ List<String> taskArns = ecs("ListTasks",
+ "{\"cluster\":\"" + name + "\",\"serviceName\":\"" + name + "-svc\"}").path("taskArns");
+ if (taskArns.isEmpty()) {
+ return List.of();
+ }
+ String quoted = String.join("\",\"", taskArns);
+ Response tasks = ecs("DescribeTasks", "{\"cluster\":\"" + name + "\",\"tasks\":[\"" + quoted + "\"]}");
+ List<String> statuses = tasks.path("tasks.lastStatus");
+ List<String> definitions = tasks.path("tasks.taskDefinitionArn");
+ assertThat(statuses, everyItem(equalTo("RUNNING")));
+ return definitions;
+ }
+
+ private static Response ecs(String action, String body) {
+ return given()
+ .header("X-Amz-Target", ECS_TARGET + action)
+ .contentType(ECS_CT)
+ .body(body)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().response();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java
index c229be4fb7..d456ea4198 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java
@@ -7438,6 +7438,11 @@ class CloudFormationIntegrationTest {
private static final String ECS_TARGET_PREFIX = "AmazonEC2ContainerServiceV20141113.";
private static final String ECS_CONTENT_TYPE = "application/x-amz-json-1.1";
+ // The service's tasks must be able to start, since CloudFormation waits for them: an awsvpc
+ // task gets its ENI in a subnet and security group that exist, here the default VPC's.
+ private static final String ECS_SUBNET_A = Ec2Service.defaultSubnetId("us-east-1", "a");
+ private static final String ECS_SUBNET_B = Ec2Service.defaultSubnetId("us-east-1", "b");
+ private static final String ECS_SECURITY_GROUP = Ec2Service.defaultSecurityGroupId("us-east-1");
private static String outputValue(String describeXml, String key) {
return describeXml.split("<OutputKey>" + key + "</OutputKey>")[1]
@@ -7485,8 +7490,8 @@ class CloudFormationIntegrationTest {
"LaunchType": "FARGATE",
"NetworkConfiguration": {
"AwsvpcConfiguration": {
- "Subnets": ["subnet-aaa", "subnet-bbb"],
- "SecurityGroups": ["sg-123"],
+ "Subnets": ["%1$s", "%2$s"],
+ "SecurityGroups": ["%3$s"],
"AssignPublicIp": "ENABLED"
}
}
@@ -7502,7 +7507,7 @@ class CloudFormationIntegrationTest {
"ServiceArn": { "Value": { "Fn::GetAtt": ["EcsService", "ServiceArn"] } }
}
}
- """;
+ """.formatted(ECS_SUBNET_A, ECS_SUBNET_B, ECS_SECURITY_GROUP);
String stackName = "cfn-ecs-stack";
@@ -7572,8 +7577,8 @@ class CloudFormationIntegrationTest {
.body("services[0].serviceName", equalTo("cfn-ecs-service"))
.body("services[0].desiredCount", equalTo(2))
.body("services[0].launchType", equalTo("FARGATE"))
- .body("services[0].networkConfiguration.awsvpcConfiguration.subnets", hasItem("subnet-aaa"))
- .body("services[0].networkConfiguration.awsvpcConfiguration.securityGroups", hasItem("sg-123"))
+ .body("services[0].networkConfiguration.awsvpcConfiguration.subnets", hasItem(ECS_SUBNET_A))
+ .body("services[0].networkConfiguration.awsvpcConfiguration.securityGroups", hasItem(ECS_SECURITY_GROUP))
.body("services[0].networkConfiguration.awsvpcConfiguration.assignPublicIp", equalTo("ENABLED"));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java
index 8d18640018..b06ae36251 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java
@@ -323,6 +323,180 @@ class EcsCfnProvisionerTest {
assertFalse(provisioner.hasReplacementUpdate(r), "an in-place update owes no cleanup");
}
+ @Test
+ void aCreatedServiceIsAwaitedUntilStableBeforeTheResourceCompletes() {
+ EcsServiceModel created = service("front");
+ when(ecs.createService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(), isNull(),
+ eq(REGION))).thenReturn(created);
+ StackResource r = resource("AWS::ECS::Service", "Service");
+
+ provisioner.provision(r, mapper.createObjectNode()
+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN), ctx());
+
+ InOrder order = inOrder(ecs);
+ order.verify(ecs).createService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(),
+ isNull(), eq(REGION));
+ order.verify(ecs).awaitServiceStable(created);
+ assertEquals(Set.of("Name", "ServiceArn"), r.getAttributes().keySet(),
+ "a stable service is no longer marked for the create rollback");
+ }
+
+ @Test
+ void anInPlaceUpdateIsAwaitedUntilStableBeforeTheResourceCompletes() {
+ EcsServiceModel updated = service("front");
+ when(ecs.updateService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(3), isNull(), eq(REGION)))
+ .thenReturn(updated);
+ StackResource r = resource("AWS::ECS::Service", "Service");
+ r.getAttributes().put("Name", "front");
+
+ provisioner.provision(r, mapper.createObjectNode()
+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN)
+ .put("DesiredCount", "3"), ctx(SERVICE_ARN));
+
+ InOrder order = inOrder(ecs);
+ order.verify(ecs).updateService("web", "front", TASK_DEF_ARN, 3, null, REGION);
+ order.verify(ecs).awaitServiceStable(updated);
+ }
+
+ @Test
+ void aCreatedServiceThatDoesNotStabilizeFailsAndStaysOwnedForTheCreateRollback() {
+ EcsServiceModel created = service("front");
+ when(ecs.createService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(), isNull(),
+ eq(REGION))).thenReturn(created);
+ doThrow(new AwsException("NotStabilized", "Service " + SERVICE_ARN + " did not stabilize.", 500))
+ .when(ecs).awaitServiceStable(created);
+ StackResource r = resource("AWS::ECS::Service", "Service");
+
+ AwsException e = assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()
+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN), ctx()));
+
+ assertEquals("Service " + SERVICE_ARN + " did not stabilize.", e.getMessage());
+ // The stack's create rollback deletes a CREATE_FAILED resource only when it is marked owned.
+ assertEquals(SERVICE_ARN, r.getPhysicalId());
+ assertEquals("true", r.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));
+ verify(ecs, never()).deleteService(anyString(), anyString(), eq(true), anyString());
+ }
+
+ @Test
+ void aReplacementThatDoesNotStabilizeIsDeletedBeforeTheUpdateFails() {
+ EcsServiceModel replacement = service("front-v2");
+ when(ecs.createService(eq("web"), eq("front-v2"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(), isNull(),
+ eq(REGION))).thenReturn(replacement);
+ doThrow(new AwsException("NotStabilized", "did not stabilize", 500))
+ .when(ecs).awaitServiceStable(replacement);
+ StackResource r = resource("AWS::ECS::Service", "Service");
+ r.getAttributes().put("Name", "front");
+
+ assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()
+ .put("Cluster", "web").put("ServiceName", "front-v2").put("TaskDefinition", TASK_DEF_ARN),
+ ctx(SERVICE_ARN)));
+
+ // The engine restores the prior resource on a failed update and never learns the new ARN,
+ // so the replacement is removed here; the prior service is left alone.
+ verify(ecs).deleteService("web", "front-v2", true, REGION);
+ verify(ecs, never()).deleteService("web", "front", true, REGION);
+ assertNull(r.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));
+ assertNull(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));
+ }
+
+ @Test
+ void aReplacementThatCannotBeDeletedEitherReportsTheRollbackFailure() {
+ EcsServiceModel replacement = service("front-v2");
+ when(ecs.createService(eq("web"), eq("front-v2"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(), isNull(),
+ eq(REGION))).thenReturn(replacement);
+ AwsException notStable = new AwsException("NotStabilized", "did not stabilize", 500);
+ doThrow(notStable).when(ecs).awaitServiceStable(replacement);
+ doThrow(new AwsException("ServerException", "busy", 500))
+ .when(ecs).deleteService("web", "front-v2", true, REGION);
+ StackResource r = resource("AWS::ECS::Service", "Service");
+ r.getAttributes().put("Name", "front");
+
+ AwsException e = assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()
+ .put("Cluster", "web").put("ServiceName", "front-v2").put("TaskDefinition", TASK_DEF_ARN),
+ ctx(SERVICE_ARN)));
+
+ assertEquals(notStable, e, "the stabilization failure is what fails the update");
+ assertEquals("busy", e.getSuppressed()[0].getMessage());
+ assertTrue(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR)
+ .contains(replacement.getServiceArn()));
+ }
+
+ @Test
+ void anInPlaceUpdateWhoseRestoreFailsReportsTheRollbackFailure() {
+ EcsServiceModel prior = service("front");
+ prior.setTaskDefinition(TASK_DEF_ARN);
+ prior.setDesiredCount(1);
+ when(ecs.serviceByArn(SERVICE_ARN)).thenReturn(prior);
+ EcsServiceModel updated = service("front");
+ when(ecs.updateService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(3), isNull(), eq(REGION)))
+ .thenReturn(updated);
+ AwsException notStable = new AwsException("NotStabilized", "did not stabilize", 500);
+ doThrow(notStable).when(ecs).awaitServiceStable(updated);
+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 1, null, REGION))
+ .thenThrow(new AwsException("ServerException", "busy", 500));
+ StackResource r = resource("AWS::ECS::Service", "Service");
+ r.getAttributes().put("Name", "front");
+
+ AwsException e = assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()
+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN)
+ .put("DesiredCount", "3"), ctx(SERVICE_ARN)));
+
+ assertEquals(notStable, e, "the stabilization failure is what fails the update");
+ assertEquals("busy", e.getSuppressed()[0].getMessage());
+ verify(ecs).updateService("web", "front", TASK_DEF_ARN, 1, null, REGION);
+ assertTrue(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR).contains(SERVICE_ARN));
+ }
+
+ @Test
+ void anInPlaceUpdateThatDoesNotStabilizeAwaitsTheRestoredConfiguration() {
+ EcsServiceModel prior = service("front");
+ prior.setTaskDefinition(TASK_DEF_ARN);
+ prior.setDesiredCount(1);
+ when(ecs.serviceByArn(SERVICE_ARN)).thenReturn(prior);
+ EcsServiceModel updated = service("front");
+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 3, null, REGION)).thenReturn(updated);
+ doThrow(new AwsException("NotStabilized", "did not stabilize", 500)).when(ecs).awaitServiceStable(updated);
+ EcsServiceModel restored = service("front");
+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 1, null, REGION)).thenReturn(restored);
+ StackResource r = resource("AWS::ECS::Service", "Service");
+ r.getAttributes().put("Name", "front");
+
+ assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()
+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN)
+ .put("DesiredCount", "3"), ctx(SERVICE_ARN)));
+
+ InOrder order = inOrder(ecs);
+ order.verify(ecs).updateService("web", "front", TASK_DEF_ARN, 1, null, REGION);
+ order.verify(ecs).awaitServiceStable(restored);
+ assertNull(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));
+ }
+
+ @Test
+ void anInPlaceUpdateWhoseRestoreDoesNotStabilizeReportsTheRollbackFailure() {
+ EcsServiceModel prior = service("front");
+ prior.setTaskDefinition(TASK_DEF_ARN);
+ prior.setDesiredCount(1);
+ when(ecs.serviceByArn(SERVICE_ARN)).thenReturn(prior);
+ EcsServiceModel updated = service("front");
+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 3, null, REGION)).thenReturn(updated);
+ AwsException notStable = new AwsException("NotStabilized", "did not stabilize", 500);
+ doThrow(notStable).when(ecs).awaitServiceStable(updated);
+ EcsServiceModel restored = service("front");
+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 1, null, REGION)).thenReturn(restored);
+ doThrow(new AwsException("NotStabilized", "restore did not stabilize", 500))
+ .when(ecs).awaitServiceStable(restored);
+ StackResource r = resource("AWS::ECS::Service", "Service");
+ r.getAttributes().put("Name", "front");
+
+ AwsException e = assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()
+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN)
+ .put("DesiredCount", "3"), ctx(SERVICE_ARN)));
+
+ assertEquals(notStable, e, "the stabilization failure is what fails the update");
+ assertEquals("restore did not stabilize", e.getSuppressed()[0].getMessage());
+ assertTrue(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR).contains(SERVICE_ARN));
+ }
+
@Test
void aTaskDefinitionUpdateDeregistersTheDisplacedRevisionAfterCommit() {
TaskDefinition next = new TaskDefinition();
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java
new file mode 100644
index 0000000000..2c45af5cb0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java
@@ -0,0 +1,374 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;
+import io.github.hectorvent.floci.services.ecs.model.AwsVpcConfiguration;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;
+import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.ecs.model.TaskStatus;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * The wait CloudFormation runs before it reports an {@code AWS::ECS::Service} complete. These
+ * services are built without CDI, so no scheduled tick ever runs: every task the tests see was
+ * started by a reconcile pass the wait itself asked for.
+ */
+class EcsServiceAwaitStableTest {
+
+ private static final String REGION = "us-east-1";
+ private static final Duration GENEROUS = Duration.ofSeconds(30);
+
+ private final List<EcsService> created = new ArrayList<>();
+
+ @AfterEach
+ void stopReconcilers() {
+ created.forEach(EcsService::stopManagedContainers);
+ }
+
+ @Test
+ void awaitServiceStable_newService_returnsOnceDescribeServicesReportsTheDesiredTasks() {
+ EcsService service = mockModeService();
+ service.createCluster("stable-cluster", REGION);
+ registerTaskDef(service, "stable-fam", "app:1");
+ EcsServiceModel svc = service.createService("stable-cluster", "stable-svc", "stable-fam", 2,
+ LaunchType.FARGATE, List.of(), null, REGION);
+
+ service.awaitServiceStable(svc, GENEROUS);
+
+ EcsServiceModel described = describe(service, "stable-cluster", "stable-svc");
+ assertEquals(2, described.getRunningCount(), "DescribeServices reports the tasks the wait waited for");
+ assertEquals(2, service.deploymentsFor(described).getFirst().getRunningCount());
+ assertEquals(2, runningTasks(service).size());
+ }
+
+ @Test
+ void awaitServiceStable_desiredCountZero_returnsAtOnceWithoutStartingATask() {
+ EcsService service = mockModeService();
+ service.createCluster("zero-cluster", REGION);
+ registerTaskDef(service, "zero-fam", "app:1");
+ EcsServiceModel svc = service.createService("zero-cluster", "zero-svc", "zero-fam", 0,
+ LaunchType.FARGATE, List.of(), null, REGION);
+
+ // A zero timeout leaves no room for a reconcile pass: only a service that is already
+ // stable when the wait starts gets through.
+ service.awaitServiceStable(svc, Duration.ZERO);
+
+ assertTrue(runningTasks(service).isEmpty());
+ }
+
+ @Test
+ void awaitServiceStable_rollingUpdate_returnsOnlyOnceThePreviousDeploymentsTasksHaveStopped() {
+ EcsService service = mockModeService();
+ service.createCluster("roll-cluster", REGION);
+ registerTaskDef(service, "roll-fam", "app:1");
+ EcsServiceModel svc = service.createService("roll-cluster", "roll-svc", "roll-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+ service.awaitServiceStable(svc, GENEROUS);
+ TaskDefinition rev2 = registerTaskDef(service, "roll-fam", "app:2");
+
+ EcsServiceModel updated = service.updateService("roll-cluster", "roll-svc",
+ rev2.getTaskDefinitionArn(), 2, null, REGION);
+ service.awaitServiceStable(updated, GENEROUS);
+
+ List<EcsTask> running = runningTasks(service);
+ assertEquals(2, running.size());
+ assertThat(running.stream().map(EcsTask::getTaskDefinitionArn).toList(),
+ everyItem(equalTo(rev2.getTaskDefinitionArn())));
+ assertEquals(2, describe(service, "roll-cluster", "roll-svc").getRunningCount());
+ }
+
+ @Test
+ void awaitServiceStable_reconcilesOnTheReconcilerThreadNotTheCallers() {
+ List<String> launchThreads = Collections.synchronizedList(new ArrayList<>());
+ EcsService service = dockerModeService(dockerContainerManager(
+ task -> launchThreads.add(Thread.currentThread().getName())));
+ service.createCluster("thread-cluster", REGION);
+ registerTaskDef(service, "thread-fam", "app:1");
+ EcsServiceModel svc = service.createService("thread-cluster", "thread-svc", "thread-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+
+ service.awaitServiceStable(svc, GENEROUS);
+
+ assertEquals(List.of("ecs-reconciler"), launchThreads,
+ "the pass runs on the single reconciler thread, so it never overlaps a scheduled tick");
+ }
+
+ @Test
+ void awaitServiceStable_serviceThatNeverStabilizes_throwsNotStabilizedOnTimeout() {
+ EcsContainerManager containerManager = dockerContainerManager(task -> { });
+ // Every container has exited by the time it is inspected: each pass finds the task gone and
+ // starts another, so the service never settles and nothing fails to launch.
+ when(containerManager.getExitCodeIfStopped(anyString())).thenReturn(1);
+ EcsService service = dockerModeService(containerManager);
+ service.createCluster("stuck-cluster", REGION);
+ registerTaskDef(service, "stuck-fam", "app:1");
+ EcsServiceModel svc = service.createService("stuck-cluster", "stuck-svc", "stuck-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.awaitServiceStable(svc, Duration.ofMillis(300)));
+
+ assertEquals("NotStabilized", error.getErrorCode());
+ assertEquals("Service " + svc.getServiceArn() + " did not stabilize.", error.getMessage());
+ }
+
+ @Test
+ void awaitServiceStable_taskCannotBeLaunched_failsWithTheLaunchFailureLongBeforeTheTimeout() {
+ EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ // What the container manager raises for an awsvpc task whose subnet EC2 does not know.
+ doAnswer(invocation -> {
+ EcsTask task = invocation.getArgument(0);
+ String subnet = task.getNetworkConfiguration().getAwsvpcConfiguration().getSubnets().getFirst();
+ throw new AwsException("InvalidParameterException", "The subnet ID '" + subnet + "' does not exist", 400);
+ }).when(containerManager).attachTaskNetwork(any(), any(), anyString());
+ EcsService service = mockModeService(containerManager);
+ service.createCluster("net-cluster", REGION);
+ ContainerDefinition container = new ContainerDefinition();
+ container.setName("app");
+ container.setImage("app:1");
+ service.registerTaskDefinition("net-fam", List.of(container), NetworkMode.awsvpc, null, null,
+ null, null, List.of(), REGION);
+ AwsVpcConfiguration awsvpc = new AwsVpcConfiguration();
+ awsvpc.setSubnets(List.of("subnet-does-not-exist"));
+ NetworkConfiguration network = new NetworkConfiguration();
+ network.setAwsvpcConfiguration(awsvpc);
+ EcsServiceModel svc = service.createService("net-cluster", "net-svc", "net-fam", 1,
+ LaunchType.FARGATE, List.of(), network, REGION);
+
+ long started = System.nanoTime();
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.awaitServiceStable(svc, Duration.ofMinutes(1)));
+ Duration waited = Duration.ofNanos(System.nanoTime() - started);
+
+ assertEquals("NotStabilized", error.getErrorCode());
+ assertEquals("Service " + svc.getServiceArn() + " did not stabilize. Task launch failed: "
+ + "The subnet ID 'subnet-does-not-exist' does not exist", error.getMessage());
+ assertTrue(waited.compareTo(Duration.ofSeconds(10)) < 0, "the wait took " + waited);
+ }
+
+ @Test
+ void awaitServiceStable_taskFailsToStart_failsWithTheStoppedReasonBeforeTheTimeout() {
+ EcsService service = dockerModeService(dockerContainerManager(task -> {
+ throw new IllegalStateException("CannotPullContainerError: pull access denied for no-such-image");
+ }));
+ service.createCluster("fail-cluster", REGION);
+ registerTaskDef(service, "fail-fam", "no-such-image");
+ EcsServiceModel svc = service.createService("fail-cluster", "fail-svc", "fail-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+
+ AwsException error = assertThrows(AwsException.class, () -> service.awaitServiceStable(svc, GENEROUS));
+
+ assertEquals("NotStabilized", error.getErrorCode());
+ assertThat(error.getMessage(), containsString("Service " + svc.getServiceArn() + " did not stabilize."));
+ assertThat(error.getMessage(), containsString("CannotPullContainerError: pull access denied for no-such-image"));
+ }
+
+ @Test
+ void awaitServiceStable_launchFailureOfThePreviousDeployment_doesNotEndTheWait() throws InterruptedException {
+ CountDownLatch staleLaunching = new CountDownLatch(1);
+ CountDownLatch releaseStale = new CountDownLatch(1);
+ EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ EcsService service = mockModeService(containerManager);
+ Thread stalePass = new Thread(service::reconcile);
+ stalePass.setDaemon(true);
+ // The pass that began before the update is still launching for the previous deployment.
+ doAnswer(invocation -> {
+ if (Thread.currentThread() == stalePass) {
+ staleLaunching.countDown();
+ releaseStale.await();
+ throw new AwsException("InvalidParameterException", "stale launch failure", 400);
+ }
+ return null;
+ }).when(containerManager).attachTaskNetwork(any(), any(), anyString());
+ // Its launch fails only once the wait has cleared the service's launch failure and started
+ // the new deployment's task: when the wait's second pass stops the previous deployment's.
+ doAnswer(invocation -> {
+ releaseStale.countDown();
+ stalePass.join();
+ return null;
+ }).when(containerManager).releaseTaskNetwork(any(), anyString());
+ service.createCluster("race-cluster", REGION);
+ registerTaskDef(service, "race-fam", "app:1");
+ EcsServiceModel svc = service.createService("race-cluster", "race-svc", "race-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+ service.awaitServiceStable(svc, GENEROUS);
+ service.updateService("race-cluster", "race-svc", null, 2, null, REGION);
+ stalePass.start();
+ assertTrue(staleLaunching.await(10, TimeUnit.SECONDS));
+ TaskDefinition rev2 = registerTaskDef(service, "race-fam", "app:2");
+ EcsServiceModel updated = service.updateService("race-cluster", "race-svc",
+ rev2.getTaskDefinitionArn(), 1, null, REGION);
+
+ service.awaitServiceStable(updated, GENEROUS);
+
+ assertEquals(List.of(rev2.getTaskDefinitionArn()),
+ runningTasks(service).stream().map(EcsTask::getTaskDefinitionArn).toList());
+ }
+
+ @Test
+ void awaitServiceStable_launchOfThePreviousDeploymentOvertakenByAnUpdate_isReplaced()
+ throws InterruptedException {
+ CountDownLatch staleLaunching = new CountDownLatch(1);
+ CountDownLatch releaseStale = new CountDownLatch(1);
+ EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ EcsService service = mockModeService(containerManager);
+ Thread stalePass = new Thread(service::reconcile);
+ stalePass.setDaemon(true);
+ // The pass launching the first revision's task is still attaching its network when the update lands.
+ doAnswer(invocation -> {
+ if (Thread.currentThread() == stalePass) {
+ staleLaunching.countDown();
+ releaseStale.await();
+ }
+ return null;
+ }).when(containerManager).attachTaskNetwork(any(), any(), anyString());
+ service.createCluster("overtaken-cluster", REGION);
+ TaskDefinition rev1 = registerTaskDef(service, "overtaken-fam", "app:1");
+ service.createService("overtaken-cluster", "overtaken-svc", rev1.getTaskDefinitionArn(), 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+ stalePass.start();
+ assertTrue(staleLaunching.await(10, TimeUnit.SECONDS));
+ TaskDefinition rev2 = registerTaskDef(service, "overtaken-fam", "app:2");
+ EcsServiceModel updated = service.updateService("overtaken-cluster", "overtaken-svc",
+ rev2.getTaskDefinitionArn(), 1, null, REGION);
+ releaseStale.countDown();
+ stalePass.join(TimeUnit.SECONDS.toMillis(10));
+
+ service.awaitServiceStable(updated, GENEROUS);
+
+ assertEquals(List.of(rev2.getTaskDefinitionArn()),
+ runningTasks(service).stream().map(EcsTask::getTaskDefinitionArn).toList());
+ }
+
+ @Test
+ void deleteService_forgetsTheServicesLaunchFailure() {
+ EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ doThrow(new AwsException("InvalidParameterException", "no such subnet", 400))
+ .when(containerManager).attachTaskNetwork(any(), any(), anyString());
+ EcsService service = mockModeService(containerManager);
+ service.createCluster("gone-cluster", REGION);
+ registerTaskDef(service, "gone-fam", "app:1");
+ EcsServiceModel svc = service.createService("gone-cluster", "gone-svc", "gone-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+ service.reconcile();
+ assertTrue(service.hasLaunchFailure(svc.getServiceArn()));
+
+ service.deleteService("gone-cluster", "gone-svc", true, REGION);
+
+ assertFalse(service.hasLaunchFailure(svc.getServiceArn()));
+ }
+
+ private EcsService mockModeService() {
+ return mockModeService(mock(EcsContainerManager.class));
+ }
+
+ private EcsService mockModeService(EcsContainerManager containerManager) {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(true);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ return track(new EcsService(new RegionResolver(REGION, "000000000000"), containerManager,
+ config, mock(EcsLoadBalancerRegistrar.class), new InMemoryStorageFactory(), null));
+ }
+
+ /** A container manager that runs {@code onStart} in {@code startTask}; its containers keep running. */
+ private static EcsContainerManager dockerContainerManager(Consumer<EcsTask> onStart) {
+ EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ // Mockito answers 0 for an Integer, which reads as "every container exited".
+ when(containerManager.getExitCodeIfStopped(anyString())).thenReturn(null);
+ when(containerManager.startTask(any(), any(), any(), anyString())).thenAnswer(invocation -> {
+ EcsTask task = invocation.getArgument(0);
+ onStart.accept(task);
+ return new EcsTaskHandle(task.getTaskArn(), Map.of("app", "docker-" + task.getTaskArn().hashCode()),
+ Map.of());
+ });
+ return containerManager;
+ }
+
+ private EcsService dockerModeService(EcsContainerManager containerManager) {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(false);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ return track(new EcsService(new RegionResolver(REGION, "000000000000"), containerManager,
+ config, mock(EcsLoadBalancerRegistrar.class), new InMemoryStorageFactory(), null));
+ }
+
+ private EcsService track(EcsService service) {
+ service.initializeStorage();
+ created.add(service);
+ return service;
+ }
+
+ private static TaskDefinition registerTaskDef(EcsService service, String family, String image) {
+ ContainerDefinition container = new ContainerDefinition();
+ container.setName("app");
+ container.setImage(image);
+ container.setEssential(true);
+ return service.registerTaskDefinition(family, List.of(container), null, null, null,
+ null, null, List.of(), REGION);
+ }
+
+ private static EcsServiceModel describe(EcsService service, String cluster, String name) {
+ return service.describeServices(cluster, List.of(name), REGION).getFirst();
+ }
+
+ private static List<EcsTask> runningTasks(EcsService service) {
+ return service.describeTasks(null, service.listTasks(null, null, null, null, REGION), REGION).stream()
+ .filter(t -> TaskStatus.RUNNING.name().equals(t.getLastStatus()))
+ .toList();
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+ ignored -> AccountAwareStorageBackend.inMemory("000000000000"));
+ }
+ }
+}