Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): wait for a stable ECS service (#4880)
Mario Toffia0ec64add2d0f
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudFormationEcsServiceTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudFormationEcsServiceTest.javanew file mode 100644index 0000000000..4554dd5260--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudFormationEcsServiceTest.java@@ -0,0 +1,140 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.cloudformation.CloudFormationClient;+import software.amazon.awssdk.services.cloudformation.model.Parameter;+import software.amazon.awssdk.services.cloudformation.model.StackStatus;+import software.amazon.awssdk.services.ec2.Ec2Client;+import software.amazon.awssdk.services.ec2.model.Subnet;+import software.amazon.awssdk.services.ecs.EcsClient;+import software.amazon.awssdk.services.ecs.model.DeploymentRolloutState;+import software.amazon.awssdk.services.ecs.model.Service;++import java.time.Duration;+import java.util.List;++import static org.assertj.core.api.Assertions.assertThat;++/**+ * CloudFormation reports an {@code AWS::ECS::Service} CREATE_COMPLETE only once the service has+ * reached a steady state, so DescribeServices right after the stack completes shows the desired+ * tasks running.+ */+@DisplayName("CloudFormation AWS::ECS::Service steady state")+class CloudFormationEcsServiceTest {++ private static final String TEMPLATE = """+ {+ "Parameters": {+ "Name": {"Type": "String"},+ "Subnet": {"Type": "String"}+ },+ "Resources": {+ "Cluster": {+ "Type": "AWS::ECS::Cluster",+ "Properties": {"ClusterName": {"Ref": "Name"}}+ },+ "TaskDef": {+ "Type": "AWS::ECS::TaskDefinition",+ "Properties": {+ "Family": {"Ref": "Name"},+ "NetworkMode": "awsvpc",+ "RequiresCompatibilities": ["FARGATE"],+ "Cpu": "256",+ "Memory": "512",+ "ContainerDefinitions": [+ {"Name": "web", "Image": "nginx:alpine", "Essential": true}+ ]+ }+ },+ "Service": {+ "Type": "AWS::ECS::Service",+ "Properties": {+ "ServiceName": {"Ref": "Name"},+ "Cluster": {"Ref": "Cluster"},+ "TaskDefinition": {"Ref": "TaskDef"},+ "DesiredCount": 1,+ "LaunchType": "FARGATE",+ "NetworkConfiguration": {+ "AwsvpcConfiguration": {"Subnets": [{"Ref": "Subnet"}], "AssignPublicIp": "ENABLED"}+ }+ }+ }+ }+ }+ """;++ private static CloudFormationClient cfn;+ private static EcsClient ecs;+ private static Ec2Client ec2;+ private String stackName;+ private boolean stackCreated;++ @BeforeAll+ static void clients() {+ cfn = TestFixtures.cloudFormationClient();+ ecs = TestFixtures.ecsClient();+ ec2 = TestFixtures.ec2Client();+ }++ @BeforeEach+ void setup() {+ stackName = TestFixtures.uniqueName("compat-cfn-ecs-svc");+ }++ @AfterEach+ void cleanup() {+ if (stackCreated) {+ cfn.deleteStack(r -> r.stackName(stackName));+ cfn.waiter().waitUntilStackDeleteComplete(r -> r.stackName(stackName),+ w -> w.waitTimeout(Duration.ofMinutes(5)));+ }+ }++ @AfterAll+ static void closeClients() {+ cfn.close();+ ecs.close();+ ec2.close();+ }++ @Test+ @DisplayName("DescribeServices right after CREATE_COMPLETE shows the desired task running")+ void createCompleteMeansTheDesiredTaskIsRunning() {+ cfn.createStack(r -> r.stackName(stackName).templateBody(TEMPLATE).parameters(+ parameter("Name", stackName), parameter("Subnet", defaultSubnet())));+ stackCreated = true;+ StackStatus status = cfn.waiter().waitUntilStackCreateComplete(r -> r.stackName(stackName),+ w -> w.waitTimeout(Duration.ofMinutes(10)))+ .matched().response().orElseThrow().stacks().get(0).stackStatus();+ assertThat(status).isEqualTo(StackStatus.CREATE_COMPLETE);++ Service service = ecs.describeServices(r -> r.cluster(stackName).services(stackName))+ .services().get(0);++ assertThat(service.desiredCount()).isEqualTo(1);+ assertThat(service.runningCount()).as("runningCount right after CREATE_COMPLETE").isEqualTo(1);+ assertThat(service.deployments()).hasSize(1);+ assertThat(service.deployments().get(0).rolloutState()).isEqualTo(DeploymentRolloutState.COMPLETED);+ }++ private static Parameter parameter(String key, String value) {+ return Parameter.builder().parameterKey(key).parameterValue(value).build();+ }++ /** A subnet of the default VPC, which the task's network interface goes into. */+ private static String defaultSubnet() {+ List<Subnet> subnets = ec2.describeSubnets().subnets();+ return subnets.stream()+ .filter(subnet -> Boolean.TRUE.equals(subnet.defaultForAz()))+ .map(Subnet::subnetId)+ .sorted()+ .findFirst()+ .orElseThrow(() -> new IllegalStateException("no default-VPC subnet to place the task in"));+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.javaindex 86631a5721..9689af0235 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java@@ -168,7 +168,9 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {* create time is read back from the {@code Name} attribute rather than the prior id. An update* that keeps the name and the cluster updates the service in place; a changed name or a* changed cluster (both create-only) is a replacement: the new service is created and the- * displaced one deleted once the stack update commits, through the replacement cleanup.+ * displaced one deleted once the stack update commits, through the replacement cleanup. Either+ * way the resource completes only once the service is stable, as CloudFormation's does; an+ * in-place update that does not stabilize is put back on the prior configuration.*/private void provisionService(StackResource r, JsonNode props, ProvisionContext ctx) {String clusterRef = ctx.resolveOptional(props, "Cluster");@@ -190,8 +192,14 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {}EcsServiceModel svc;- if (ctx.isUpdate() && serviceName.equals(priorName)- && clusterName(clusterRef).equals(clusterOfServiceArn(ctx.priorPhysicalId()))) {+ boolean inPlace = ctx.isUpdate() && serviceName.equals(priorName)+ && clusterName(clusterRef).equals(clusterOfServiceArn(ctx.priorPhysicalId()));+ // Read before updateService, which changes this same model in place.+ EcsServiceModel prior = inPlace ? ecsService.serviceByArn(ctx.priorPhysicalId()) : null;+ String priorTaskDefinition = prior != null ? prior.getTaskDefinition() : null;+ Integer priorDesiredCount = prior != null ? prior.getDesiredCount() : null;+ NetworkConfiguration priorNetworkConfiguration = prior != null ? prior.getNetworkConfiguration() : null;+ if (inPlace) {svc = ecsService.updateService(clusterRef, serviceName, taskDefinition,desiredCount, networkConfiguration, ctx.region());} else {@@ -202,6 +210,73 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {r.setPhysicalId(svc.getServiceArn());r.getAttributes().put("Name", svc.getServiceName());r.getAttributes().put("ServiceArn", svc.getServiceArn());+ if (inPlace) {+ try {+ ecsService.awaitServiceStable(svc);+ } catch (RuntimeException failure) {+ restoreInPlace(r, failure, () -> ecsService.awaitServiceStable(ecsService.updateService(clusterRef,+ svc.getServiceName(), priorTaskDefinition, priorDesiredCount, priorNetworkConfiguration,+ ctx.region())));+ throw failure;+ }+ } else {+ awaitCreatedServiceStable(r, svc, ctx);+ }+ }++ /**+ * Puts a service updated in place back on the task definition, desired count and network+ * configuration it had, when the update does not stabilize, and waits for the restored+ * configuration to stabilize, as CloudFormation's rollback does. CloudFormationService restores+ * the prior resource record and reports the rollback complete unless told otherwise, so without+ * this the stack would claim a rollback while ECS still runs the failed configuration. A+ * restore that fails or does not stabilize is attached to the original failure and recorded as+ * a rollback failure.+ */+ private void restoreInPlace(StackResource r, RuntimeException failure, Runnable restore) {+ try {+ restore.run();+ } catch (RuntimeException restoreFailure) {+ String reason = "Could not restore " + r.getPhysicalId() + " after a failed update: "+ + restoreFailure.getMessage();+ LOG.warnv("{0}", reason);+ r.getAttributes().put(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR, reason);+ if (restoreFailure != failure) {+ failure.addSuppressed(restoreFailure);+ }+ }+ }++ /**+ * Waits for a service this provision created to stabilize, as CloudFormation does before it+ * reports the resource complete. The resource is marked as owned meanwhile, so a service that+ * never stabilizes is deleted by the create rollback. A replacement created by a stack update+ * is removed here instead: the engine restores the prior resource and never learns the new ARN.+ */+ private void awaitCreatedServiceStable(StackResource r, EcsServiceModel svc, ProvisionContext ctx) {+ r.getAttributes().put(CfnRollback.ROLLBACK_OWNED_ATTR, "true");+ try {+ ecsService.awaitServiceStable(svc);+ } catch (RuntimeException failure) {+ if (ctx.isUpdate()) {+ unwindReplacement(r, ctx.region(), failure);+ }+ throw failure;+ }+ r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);+ }++ private void unwindReplacement(StackResource r, String region, RuntimeException failure) {+ try {+ deleteService(r.getPhysicalId(), region);+ r.getAttributes().remove(CfnRollback.ROLLBACK_OWNED_ATTR);+ } catch (RuntimeException cleanupFailure) {+ failure.addSuppressed(cleanupFailure);+ String reason = "Could not remove " + r.getPhysicalId() + ", the replacement created for "+ + r.getLogicalId() + " by a failed update: " + cleanupFailure.getMessage();+ LOG.warnv("{0}", reason);+ r.getAttributes().put(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR, reason);+ }}/** The cluster name a template value addresses: a name, an ARN's last segment, or the default. */diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex ede10ae55d..1a6fd0efee 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -73,6 +73,7 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.nio.charset.StandardCharsets;+import java.time.Duration;import java.time.Instant;import java.util.ArrayList;import java.util.Base64;@@ -91,6 +92,8 @@ import java.util.Set;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.RejectedExecutionException;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.TimeUnit;import java.util.function.Function;@@ -179,6 +182,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabpublic static final int MAX_TASKS_PER_RUN = 10;/** A listing returns at most a hundred ARNs per page. */private static final int MAX_LIST_RESULTS = 100;+ /** How long a CloudFormation-provisioned service may take to reach a steady state. */+ private static final Duration SERVICE_STABLE_TIMEOUT = Duration.ofMinutes(5);+ private static final long SERVICE_STABLE_POLL_MILLIS = 50;+ /**+ * Reconcile passes a steady-state wait runs at once instead of waiting for ticks. A pass+ * reports the running count it found before it acted, so a deployment that both starts tasks+ * and drains the previous deployment's reads as settled after the third.+ */+ private static final int SERVICE_STABLE_PASSES = 3;/** ListServices answers with ten ARNs per page when the request names no maxResults. */private static final int DEFAULT_SERVICE_PAGE_SIZE = 10;public static final String STATUS_ACTIVE = "ACTIVE";@@ -210,6 +222,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabprivate Map<String, String> accountSettings = new ConcurrentHashMap<>();// deploymentIds for which SERVICE_DEPLOYMENT_IN_PROGRESS has already been emitted this process.private final Set<String> inProgressEmitted = java.util.concurrent.ConcurrentHashMap.newKeySet();+ // serviceArn → why the reconciler's last attempt to launch one of its tasks failed, and for which+ // deployment. Memory-only.+ private final Map<String, LaunchFailure> launchFailures = new ConcurrentHashMap<>();@Injectpublic EcsService(RegionResolver regionResolver, EcsContainerManager containerManager,@@ -385,7 +400,8 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab@Overridepublic void clear() {- // Nothing to wipe: the stores hold the ECS state and afterReset() restarts the reconciler.+ // The stores hold the ECS state and afterReset() restarts the reconciler.+ launchFailures.clear();}/**@@ -426,6 +442,150 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabreturn reconciler.isShutdown();}+ // ── Steady state ─────────────────────────────────────────────────────────++ /**+ * Waits until a service reaches a steady state, which is what CloudFormation waits for before+ * it reports an {@code AWS::ECS::Service} CREATE_COMPLETE or UPDATE_COMPLETE. CloudFormation+ * reads it off {@code DescribeServices}, so the service must report {@code runningCount}+ * equal to {@code desiredCount}, its current deployment must run that many tasks, and no task+ * of an earlier deployment may still be running. A {@code desiredCount} of 0 with nothing+ * running is stable at once.+ *+ * <p>Rather than sit idle until the next tick, the wait runs reconcile passes at once on the+ * reconciler's own single thread, so a pass never overlaps a scheduled tick.+ *+ * @throws AwsException when the reconciler could not launch a task for the service, when a task+ * launched for it during the wait failed to start, or when the service does not stabilize+ * within the timeout+ */+ public void awaitServiceStable(EcsServiceModel svc) {+ awaitServiceStable(svc, SERVICE_STABLE_TIMEOUT);+ }++ void awaitServiceStable(EcsServiceModel svc, Duration timeout) {+ String serviceArn = svc.getServiceArn();+ recordLaunchFailure(svc, null, null);+ Instant waitStarted = Instant.now();+ long deadline = System.nanoTime() + timeout.toNanos();+ Future<?> pass = null;+ int passes = 0;+ while (true) {+ // Checked before the service is read, so a finished pass's writes are visible to the read.+ boolean passRunning = pass != null && !pass.isDone();+ EcsServiceModel current = serviceByArn(serviceArn);+ if (current == null || !STATUS_ACTIVE.equals(current.getStatus())) {+ throw notStabilized(serviceArn, " The service is no longer ACTIVE.");+ }+ if (!passRunning) {+ failIfTasksCannotStart(current, waitStarted);+ }+ if (isSteady(current)) {+ return;+ }+ if (!passRunning && passes < SERVICE_STABLE_PASSES) {+ pass = reconcileNow();+ passes++;+ }+ if (System.nanoTime() - deadline >= 0) {+ throw notStabilized(serviceArn, "");+ }+ try {+ Thread.sleep(SERVICE_STABLE_POLL_MILLIS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AwsException("InternalError", "Interrupted while waiting for ECS service "+ + serviceArn + " to stabilize", 500);+ }+ }+ }++ /**+ * Runs one reconcile pass now on the reconciler's single thread. Null when the reconciler is+ * shut down, which only happens on shutdown and for the length of a state reset.+ */+ private Future<?> reconcileNow() {+ try {+ return reconciler.submit(this::reconcile);+ } catch (RejectedExecutionException e) {+ LOG.debugv("ECS reconciler is shut down, waiting for the next scheduled tick: {0}", e.getMessage());+ return null;+ }+ }++ private boolean isSteady(EcsServiceModel svc) {+ if (svc.getRunningCount() != svc.getDesiredCount()) {+ return false;+ }+ EcsCluster cluster = resolveClusterByArn(svc.getClusterArn());+ if (cluster == null) {+ return svc.getDesiredCount() == 0;+ }+ String currentDeploymentId = deploymentId(svc);+ String taskDefinitionArn = resolvedTaskDefinitionArn(svc, serviceRegion(svc));+ long current = 0;+ for (EcsTask task : tasks.values()) {+ if (!ownedBy(task, svc, cluster) || !TaskStatus.RUNNING.name().equals(task.getLastStatus())) {+ continue;+ }+ if (isStaleForDeployment(task, currentDeploymentId, taskDefinitionArn)) {+ return false;+ }+ current++;+ }+ return current == svc.getDesiredCount();+ }++ /**+ * Ends a wait that cannot succeed: the reconciler could not launch a task for the service's+ * current deployment, or a task launched for it since {@code since} stopped before it ever ran.+ */+ private void failIfTasksCannotStart(EcsServiceModel svc, Instant since) {+ String currentDeploymentId = deploymentId(svc);+ LaunchFailure launchFailure = launchFailures.get(svc.getServiceArn());+ if (launchFailure != null && currentDeploymentId.equals(launchFailure.deploymentId())) {+ throw notStabilized(svc.getServiceArn(), " Task launch failed: " + launchFailure.message());+ }+ EcsCluster cluster = resolveClusterByArn(svc.getClusterArn());+ if (cluster == null) {+ return;+ }+ String taskDefinitionArn = resolvedTaskDefinitionArn(svc, serviceRegion(svc));+ for (EcsTask task : tasks.values()) {+ if (ownedBy(task, svc, cluster)+ && TaskStatus.STOPPED.name().equals(task.getLastStatus())+ && STOP_CODE_TASK_FAILED_TO_START.equals(task.getStopCode())+ && task.getCreatedAt() != null && !task.getCreatedAt().isBefore(since)+ && !isStaleForDeployment(task, currentDeploymentId, taskDefinitionArn)) {+ throw notStabilized(svc.getServiceArn(),+ " Task " + task.getTaskArn() + " stopped: " + task.getStoppedReason());+ }+ }+ }++ /** AWS's wording when a service does not reach a steady state, followed by any detail. */+ private static AwsException notStabilized(String serviceArn, String detail) {+ return new AwsException("NotStabilized", "Service " + serviceArn + " did not stabilize." + detail, 500);+ }++ private record LaunchFailure(String deploymentId, String message) {}++ /** Records why the reconciler could not launch a task for {@code svc}'s {@code deploymentId}; null clears it. */+ private void recordLaunchFailure(EcsServiceModel svc, String deploymentId, String failure) {+ if (svc.getServiceArn() == null) {+ return;+ }+ if (failure == null) {+ launchFailures.remove(svc.getServiceArn());+ } else {+ launchFailures.put(svc.getServiceArn(), new LaunchFailure(deploymentId, failure));+ }+ }++ boolean hasLaunchFailure(String serviceArn) {+ return launchFailures.containsKey(serviceArn);+ }+// ── Clusters ─────────────────────────────────────────────────────────────public EcsCluster createCluster(String clusterName, String region) {@@ -1480,13 +1640,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab*/private EcsTask launchServiceTask(EcsCluster cluster, EcsServiceModel svc, LaunchType launchType,String containerInstanceArn, String region) {+ // Read before the task definition: UpdateService sets it first, so a launch it overtakes is stamped stale.+ String deploymentId = deploymentId(svc);TaskDefinition taskDef = resolveTaskDefinitionOrThrow(svc.getTaskDefinition(), region);RunTaskRequest request = new RunTaskRequest();request.setCount(1);request.setLaunchType(launchType);// AWS labels a service's tasks "service:<name>" and attributes them to the deployment.request.setGroup("service:" + svc.getServiceName());- request.setStartedBy(deploymentId(svc));+ request.setStartedBy(deploymentId);request.setNetworkConfiguration(svc.getNetworkConfiguration());request.setPlatformVersion(svc.getPlatformVersion());request.setEnableExecuteCommand(svc.isEnableExecuteCommand());@@ -1503,7 +1665,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}EcsTask task = launchTasks(cluster, taskDef, request, containerInstanceArn,svc.getServiceArn(), region).getFirst();- task.setDeploymentId(deploymentId(svc));+ task.setDeploymentId(deploymentId);return task;}@@ -2775,6 +2937,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabif (serviceArn != null) {serviceDeployments.values().removeIf(d -> serviceArn.equals(d.getServiceArn()));serviceRevisions.values().removeIf(r -> serviceArn.equals(r.getServiceArn()));+ launchFailures.remove(serviceArn);}return svc;}@@ -5084,12 +5247,14 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabfor (int i = 0; i < toStart; i++) {try {EcsTask launched = launchServiceTask(cluster, svc, svc.getLaunchType(), null, region);+ recordLaunchFailure(svc, null, null);LOG.infov("Service reconciler started task {0} for service {1}",launched.getTaskArn(), svc.getServiceName());if (countLaunch(svc, deployment, launched)) {break;}} catch (Exception e) {+ recordLaunchFailure(svc, currentDeploymentId, String.valueOf(e.getMessage()));LOG.warnv("Service reconciler failed to start task for {0}: {1}",svc.getServiceName(), e.getMessage());}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEcsServiceStableIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEcsServiceStableIntegrationTest.javanew file mode 100644index 0000000000..65edf29f32--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationEcsServiceStableIntegrationTest.java@@ -0,0 +1,208 @@+package io.github.hectorvent.floci.services.cloudformation;++import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.List;++import static io.restassured.RestAssured.given;+import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.endsWith;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.everyItem;+import static org.hamcrest.Matchers.hasSize;++/**+ * CloudFormation reports an {@code AWS::ECS::Service} complete only once the service is stable,+ * so a caller that reads the service right after the stack operation finishes sees its tasks+ * running. Floci used to report the resource complete as soon as CreateService or UpdateService+ * returned, and the tasks only started on a later reconciler tick.+ */+@QuarkusTest+class CloudFormationEcsServiceStableIntegrationTest {++ private static final String ECS_TARGET = "AmazonEC2ContainerServiceV20141113.";+ private static final String ECS_CT = "application/x-amz-json-1.1";++ private static final String TEMPLATE = """+ {+ "Resources": {+ "Cluster": {+ "Type": "AWS::ECS::Cluster",+ "Properties": { "ClusterName": "%1$s" }+ },+ "TaskDef": {+ "Type": "AWS::ECS::TaskDefinition",+ "Properties": {+ "Family": "%1$s-td",+ "ContainerDefinitions": [+ { "Name": "app", "Image": "%2$s", "Essential": true, "Memory": 128 }+ ]+ }+ },+ "Service": {+ "Type": "AWS::ECS::Service",+ "Properties": {+ "ServiceName": "%1$s-svc",+ "Cluster": { "Ref": "Cluster" },+ "TaskDefinition": { "Ref": "TaskDef" },+ "DesiredCount": %3$d,+ "LaunchType": "FARGATE"+ }+ }+ }+ }+ """;++ private static final String AWSVPC_TEMPLATE = """+ {+ "Resources": {+ "Cluster": {+ "Type": "AWS::ECS::Cluster",+ "Properties": { "ClusterName": "%1$s" }+ },+ "TaskDef": {+ "Type": "AWS::ECS::TaskDefinition",+ "Properties": {+ "Family": "%1$s-td",+ "NetworkMode": "awsvpc",+ "ContainerDefinitions": [+ { "Name": "app", "Image": "app:v1", "Essential": true, "Memory": 128 }+ ]+ }+ },+ "Service": {+ "Type": "AWS::ECS::Service",+ "Properties": {+ "ServiceName": "%1$s-svc",+ "Cluster": { "Ref": "Cluster" },+ "TaskDefinition": { "Ref": "TaskDef" },+ "DesiredCount": %2$d,+ "LaunchType": "FARGATE",+ "NetworkConfiguration": {+ "AwsvpcConfiguration": { "Subnets": ["%3$s"] }+ }+ }+ }+ }+ }+ """;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void createStack_ecsService_runsItsDesiredTasksWhenTheStackCompletes() {+ String name = "cfn-stable-" + Long.toString(System.nanoTime(), 36);++ stackCall("CreateStack", name, TEMPLATE.formatted(name, "app:v1", 1));++ assertThat(stackStatus(name), equalTo("CREATE_COMPLETE"));+ Response service = describeService(name);+ assertThat(service.path("services[0].desiredCount"), equalTo(1));+ assertThat(service.path("services[0].runningCount"), equalTo(1));+ assertThat(service.path("services[0].deployments"), hasSize(1));+ assertThat(service.path("services[0].deployments[0].runningCount"), equalTo(1));+ assertThat(service.path("services[0].deployments[0].rolloutState"), equalTo("COMPLETED"));+ assertThat(runningTaskDefinitions(name), hasSize(1));+ }++ @Test+ void updateStack_ecsService_runsOnlyTheNewRevisionWhenTheUpdateCompletes() {+ String name = "cfn-stable-upd-" + Long.toString(System.nanoTime(), 36);+ stackCall("CreateStack", name, TEMPLATE.formatted(name, "app:v1", 1));++ stackCall("UpdateStack", name, TEMPLATE.formatted(name, "app:v2", 2));++ assertThat(stackStatus(name), equalTo("UPDATE_COMPLETE"));+ Response service = describeService(name);+ assertThat(service.path("services[0].desiredCount"), equalTo(2));+ assertThat(service.path("services[0].runningCount"), equalTo(2));+ assertThat(service.path("services[0].taskDefinition"), endsWith(":task-definition/" + name + "-td:2"));+ List<String> running = runningTaskDefinitions(name);+ assertThat(running, hasSize(2));+ assertThat("the previous revision's task has stopped", running,+ everyItem(endsWith(":task-definition/" + name + "-td:2")));+ }++ @Test+ void updateStack_ecsServiceThatCannotStabilize_putsThePriorConfigurationBackOnRollback() {+ String name = "cfn-stable-rb-" + Long.toString(System.nanoTime(), 36);+ String subnet = Ec2Service.defaultSubnetId("us-east-1", "a");+ stackCall("CreateStack", name, AWSVPC_TEMPLATE.formatted(name, 1, subnet));+ assertThat(stackStatus(name), equalTo("CREATE_COMPLETE"));++ stackCall("UpdateStack", name, AWSVPC_TEMPLATE.formatted(name, 2, "subnet-does-not-exist"));++ assertThat(stackStatus(name), equalTo("UPDATE_ROLLBACK_COMPLETE"));+ Response service = describeService(name);+ assertThat("the rollback must not leave the failed update live",+ service.path("services[0].desiredCount"), equalTo(1));+ assertThat(service.path("services[0].networkConfiguration.awsvpcConfiguration.subnets"),+ equalTo(List.of(subnet)));+ }++ private static void stackCall(String action, String stackName, String template) {+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", action)+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<StackId>"));+ }++ private static String stackStatus(String stackName) {+ return given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stackName)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("**.find { it.name() == 'StackStatus' }");+ }++ private static Response describeService(String name) {+ return ecs("DescribeServices",+ "{\"cluster\":\"" + name + "\",\"services\":[\"" + name + "-svc\"]}");+ }++ /** The task definition ARN of every RUNNING task of the stack's service. */+ private static List<String> runningTaskDefinitions(String name) {+ List<String> taskArns = ecs("ListTasks",+ "{\"cluster\":\"" + name + "\",\"serviceName\":\"" + name + "-svc\"}").path("taskArns");+ if (taskArns.isEmpty()) {+ return List.of();+ }+ String quoted = String.join("\",\"", taskArns);+ Response tasks = ecs("DescribeTasks", "{\"cluster\":\"" + name + "\",\"tasks\":[\"" + quoted + "\"]}");+ List<String> statuses = tasks.path("tasks.lastStatus");+ List<String> definitions = tasks.path("tasks.taskDefinitionArn");+ assertThat(statuses, everyItem(equalTo("RUNNING")));+ return definitions;+ }++ private static Response ecs(String action, String body) {+ return given()+ .header("X-Amz-Target", ECS_TARGET + action)+ .contentType(ECS_CT)+ .body(body)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().response();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.javaindex c229be4fb7..d456ea4198 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java@@ -7438,6 +7438,11 @@ class CloudFormationIntegrationTest {private static final String ECS_TARGET_PREFIX = "AmazonEC2ContainerServiceV20141113.";private static final String ECS_CONTENT_TYPE = "application/x-amz-json-1.1";+ // The service's tasks must be able to start, since CloudFormation waits for them: an awsvpc+ // task gets its ENI in a subnet and security group that exist, here the default VPC's.+ private static final String ECS_SUBNET_A = Ec2Service.defaultSubnetId("us-east-1", "a");+ private static final String ECS_SUBNET_B = Ec2Service.defaultSubnetId("us-east-1", "b");+ private static final String ECS_SECURITY_GROUP = Ec2Service.defaultSecurityGroupId("us-east-1");private static String outputValue(String describeXml, String key) {return describeXml.split("<OutputKey>" + key + "</OutputKey>")[1]@@ -7485,8 +7490,8 @@ class CloudFormationIntegrationTest {"LaunchType": "FARGATE","NetworkConfiguration": {"AwsvpcConfiguration": {- "Subnets": ["subnet-aaa", "subnet-bbb"],- "SecurityGroups": ["sg-123"],+ "Subnets": ["%1$s", "%2$s"],+ "SecurityGroups": ["%3$s"],"AssignPublicIp": "ENABLED"}}@@ -7502,7 +7507,7 @@ class CloudFormationIntegrationTest {"ServiceArn": { "Value": { "Fn::GetAtt": ["EcsService", "ServiceArn"] } }}}- """;+ """.formatted(ECS_SUBNET_A, ECS_SUBNET_B, ECS_SECURITY_GROUP);String stackName = "cfn-ecs-stack";@@ -7572,8 +7577,8 @@ class CloudFormationIntegrationTest {.body("services[0].serviceName", equalTo("cfn-ecs-service")).body("services[0].desiredCount", equalTo(2)).body("services[0].launchType", equalTo("FARGATE"))- .body("services[0].networkConfiguration.awsvpcConfiguration.subnets", hasItem("subnet-aaa"))- .body("services[0].networkConfiguration.awsvpcConfiguration.securityGroups", hasItem("sg-123"))+ .body("services[0].networkConfiguration.awsvpcConfiguration.subnets", hasItem(ECS_SUBNET_A))+ .body("services[0].networkConfiguration.awsvpcConfiguration.securityGroups", hasItem(ECS_SECURITY_GROUP)).body("services[0].networkConfiguration.awsvpcConfiguration.assignPublicIp", equalTo("ENABLED"));}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.javaindex 8d18640018..b06ae36251 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java@@ -323,6 +323,180 @@ class EcsCfnProvisionerTest {assertFalse(provisioner.hasReplacementUpdate(r), "an in-place update owes no cleanup");}+ @Test+ void aCreatedServiceIsAwaitedUntilStableBeforeTheResourceCompletes() {+ EcsServiceModel created = service("front");+ when(ecs.createService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(), isNull(),+ eq(REGION))).thenReturn(created);+ StackResource r = resource("AWS::ECS::Service", "Service");++ provisioner.provision(r, mapper.createObjectNode()+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN), ctx());++ InOrder order = inOrder(ecs);+ order.verify(ecs).createService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(),+ isNull(), eq(REGION));+ order.verify(ecs).awaitServiceStable(created);+ assertEquals(Set.of("Name", "ServiceArn"), r.getAttributes().keySet(),+ "a stable service is no longer marked for the create rollback");+ }++ @Test+ void anInPlaceUpdateIsAwaitedUntilStableBeforeTheResourceCompletes() {+ EcsServiceModel updated = service("front");+ when(ecs.updateService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(3), isNull(), eq(REGION)))+ .thenReturn(updated);+ StackResource r = resource("AWS::ECS::Service", "Service");+ r.getAttributes().put("Name", "front");++ provisioner.provision(r, mapper.createObjectNode()+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN)+ .put("DesiredCount", "3"), ctx(SERVICE_ARN));++ InOrder order = inOrder(ecs);+ order.verify(ecs).updateService("web", "front", TASK_DEF_ARN, 3, null, REGION);+ order.verify(ecs).awaitServiceStable(updated);+ }++ @Test+ void aCreatedServiceThatDoesNotStabilizeFailsAndStaysOwnedForTheCreateRollback() {+ EcsServiceModel created = service("front");+ when(ecs.createService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(), isNull(),+ eq(REGION))).thenReturn(created);+ doThrow(new AwsException("NotStabilized", "Service " + SERVICE_ARN + " did not stabilize.", 500))+ .when(ecs).awaitServiceStable(created);+ StackResource r = resource("AWS::ECS::Service", "Service");++ AwsException e = assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN), ctx()));++ assertEquals("Service " + SERVICE_ARN + " did not stabilize.", e.getMessage());+ // The stack's create rollback deletes a CREATE_FAILED resource only when it is marked owned.+ assertEquals(SERVICE_ARN, r.getPhysicalId());+ assertEquals("true", r.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));+ verify(ecs, never()).deleteService(anyString(), anyString(), eq(true), anyString());+ }++ @Test+ void aReplacementThatDoesNotStabilizeIsDeletedBeforeTheUpdateFails() {+ EcsServiceModel replacement = service("front-v2");+ when(ecs.createService(eq("web"), eq("front-v2"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(), isNull(),+ eq(REGION))).thenReturn(replacement);+ doThrow(new AwsException("NotStabilized", "did not stabilize", 500))+ .when(ecs).awaitServiceStable(replacement);+ StackResource r = resource("AWS::ECS::Service", "Service");+ r.getAttributes().put("Name", "front");++ assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()+ .put("Cluster", "web").put("ServiceName", "front-v2").put("TaskDefinition", TASK_DEF_ARN),+ ctx(SERVICE_ARN)));++ // The engine restores the prior resource on a failed update and never learns the new ARN,+ // so the replacement is removed here; the prior service is left alone.+ verify(ecs).deleteService("web", "front-v2", true, REGION);+ verify(ecs, never()).deleteService("web", "front", true, REGION);+ assertNull(r.getAttributes().get(CfnRollback.ROLLBACK_OWNED_ATTR));+ assertNull(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));+ }++ @Test+ void aReplacementThatCannotBeDeletedEitherReportsTheRollbackFailure() {+ EcsServiceModel replacement = service("front-v2");+ when(ecs.createService(eq("web"), eq("front-v2"), eq(TASK_DEF_ARN), eq(1), isNull(), anyList(), isNull(),+ eq(REGION))).thenReturn(replacement);+ AwsException notStable = new AwsException("NotStabilized", "did not stabilize", 500);+ doThrow(notStable).when(ecs).awaitServiceStable(replacement);+ doThrow(new AwsException("ServerException", "busy", 500))+ .when(ecs).deleteService("web", "front-v2", true, REGION);+ StackResource r = resource("AWS::ECS::Service", "Service");+ r.getAttributes().put("Name", "front");++ AwsException e = assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()+ .put("Cluster", "web").put("ServiceName", "front-v2").put("TaskDefinition", TASK_DEF_ARN),+ ctx(SERVICE_ARN)));++ assertEquals(notStable, e, "the stabilization failure is what fails the update");+ assertEquals("busy", e.getSuppressed()[0].getMessage());+ assertTrue(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR)+ .contains(replacement.getServiceArn()));+ }++ @Test+ void anInPlaceUpdateWhoseRestoreFailsReportsTheRollbackFailure() {+ EcsServiceModel prior = service("front");+ prior.setTaskDefinition(TASK_DEF_ARN);+ prior.setDesiredCount(1);+ when(ecs.serviceByArn(SERVICE_ARN)).thenReturn(prior);+ EcsServiceModel updated = service("front");+ when(ecs.updateService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(3), isNull(), eq(REGION)))+ .thenReturn(updated);+ AwsException notStable = new AwsException("NotStabilized", "did not stabilize", 500);+ doThrow(notStable).when(ecs).awaitServiceStable(updated);+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 1, null, REGION))+ .thenThrow(new AwsException("ServerException", "busy", 500));+ StackResource r = resource("AWS::ECS::Service", "Service");+ r.getAttributes().put("Name", "front");++ AwsException e = assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN)+ .put("DesiredCount", "3"), ctx(SERVICE_ARN)));++ assertEquals(notStable, e, "the stabilization failure is what fails the update");+ assertEquals("busy", e.getSuppressed()[0].getMessage());+ verify(ecs).updateService("web", "front", TASK_DEF_ARN, 1, null, REGION);+ assertTrue(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR).contains(SERVICE_ARN));+ }++ @Test+ void anInPlaceUpdateThatDoesNotStabilizeAwaitsTheRestoredConfiguration() {+ EcsServiceModel prior = service("front");+ prior.setTaskDefinition(TASK_DEF_ARN);+ prior.setDesiredCount(1);+ when(ecs.serviceByArn(SERVICE_ARN)).thenReturn(prior);+ EcsServiceModel updated = service("front");+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 3, null, REGION)).thenReturn(updated);+ doThrow(new AwsException("NotStabilized", "did not stabilize", 500)).when(ecs).awaitServiceStable(updated);+ EcsServiceModel restored = service("front");+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 1, null, REGION)).thenReturn(restored);+ StackResource r = resource("AWS::ECS::Service", "Service");+ r.getAttributes().put("Name", "front");++ assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN)+ .put("DesiredCount", "3"), ctx(SERVICE_ARN)));++ InOrder order = inOrder(ecs);+ order.verify(ecs).updateService("web", "front", TASK_DEF_ARN, 1, null, REGION);+ order.verify(ecs).awaitServiceStable(restored);+ assertNull(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));+ }++ @Test+ void anInPlaceUpdateWhoseRestoreDoesNotStabilizeReportsTheRollbackFailure() {+ EcsServiceModel prior = service("front");+ prior.setTaskDefinition(TASK_DEF_ARN);+ prior.setDesiredCount(1);+ when(ecs.serviceByArn(SERVICE_ARN)).thenReturn(prior);+ EcsServiceModel updated = service("front");+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 3, null, REGION)).thenReturn(updated);+ AwsException notStable = new AwsException("NotStabilized", "did not stabilize", 500);+ doThrow(notStable).when(ecs).awaitServiceStable(updated);+ EcsServiceModel restored = service("front");+ when(ecs.updateService("web", "front", TASK_DEF_ARN, 1, null, REGION)).thenReturn(restored);+ doThrow(new AwsException("NotStabilized", "restore did not stabilize", 500))+ .when(ecs).awaitServiceStable(restored);+ StackResource r = resource("AWS::ECS::Service", "Service");+ r.getAttributes().put("Name", "front");++ AwsException e = assertThrows(AwsException.class, () -> provisioner.provision(r, mapper.createObjectNode()+ .put("Cluster", "web").put("ServiceName", "front").put("TaskDefinition", TASK_DEF_ARN)+ .put("DesiredCount", "3"), ctx(SERVICE_ARN)));++ assertEquals(notStable, e, "the stabilization failure is what fails the update");+ assertEquals("restore did not stabilize", e.getSuppressed()[0].getMessage());+ assertTrue(r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR).contains(SERVICE_ARN));+ }+@Testvoid aTaskDefinitionUpdateDeregistersTheDisplacedRevisionAfterCommit() {TaskDefinition next = new TaskDefinition();diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.javanew file mode 100644index 0000000000..2c45af5cb0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java@@ -0,0 +1,374 @@+package io.github.hectorvent.floci.services.ecs;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;+import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;+import io.github.hectorvent.floci.services.ecs.model.AwsVpcConfiguration;+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;+import io.github.hectorvent.floci.services.ecs.model.EcsTask;+import io.github.hectorvent.floci.services.ecs.model.LaunchType;+import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;+import io.github.hectorvent.floci.services.ecs.model.NetworkMode;+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;+import io.github.hectorvent.floci.services.ecs.model.TaskStatus;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.util.ArrayList;+import java.util.Collections;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.function.Consumer;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.everyItem;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * The wait CloudFormation runs before it reports an {@code AWS::ECS::Service} complete. These+ * services are built without CDI, so no scheduled tick ever runs: every task the tests see was+ * started by a reconcile pass the wait itself asked for.+ */+class EcsServiceAwaitStableTest {++ private static final String REGION = "us-east-1";+ private static final Duration GENEROUS = Duration.ofSeconds(30);++ private final List<EcsService> created = new ArrayList<>();++ @AfterEach+ void stopReconcilers() {+ created.forEach(EcsService::stopManagedContainers);+ }++ @Test+ void awaitServiceStable_newService_returnsOnceDescribeServicesReportsTheDesiredTasks() {+ EcsService service = mockModeService();+ service.createCluster("stable-cluster", REGION);+ registerTaskDef(service, "stable-fam", "app:1");+ EcsServiceModel svc = service.createService("stable-cluster", "stable-svc", "stable-fam", 2,+ LaunchType.FARGATE, List.of(), null, REGION);++ service.awaitServiceStable(svc, GENEROUS);++ EcsServiceModel described = describe(service, "stable-cluster", "stable-svc");+ assertEquals(2, described.getRunningCount(), "DescribeServices reports the tasks the wait waited for");+ assertEquals(2, service.deploymentsFor(described).getFirst().getRunningCount());+ assertEquals(2, runningTasks(service).size());+ }++ @Test+ void awaitServiceStable_desiredCountZero_returnsAtOnceWithoutStartingATask() {+ EcsService service = mockModeService();+ service.createCluster("zero-cluster", REGION);+ registerTaskDef(service, "zero-fam", "app:1");+ EcsServiceModel svc = service.createService("zero-cluster", "zero-svc", "zero-fam", 0,+ LaunchType.FARGATE, List.of(), null, REGION);++ // A zero timeout leaves no room for a reconcile pass: only a service that is already+ // stable when the wait starts gets through.+ service.awaitServiceStable(svc, Duration.ZERO);++ assertTrue(runningTasks(service).isEmpty());+ }++ @Test+ void awaitServiceStable_rollingUpdate_returnsOnlyOnceThePreviousDeploymentsTasksHaveStopped() {+ EcsService service = mockModeService();+ service.createCluster("roll-cluster", REGION);+ registerTaskDef(service, "roll-fam", "app:1");+ EcsServiceModel svc = service.createService("roll-cluster", "roll-svc", "roll-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);+ service.awaitServiceStable(svc, GENEROUS);+ TaskDefinition rev2 = registerTaskDef(service, "roll-fam", "app:2");++ EcsServiceModel updated = service.updateService("roll-cluster", "roll-svc",+ rev2.getTaskDefinitionArn(), 2, null, REGION);+ service.awaitServiceStable(updated, GENEROUS);++ List<EcsTask> running = runningTasks(service);+ assertEquals(2, running.size());+ assertThat(running.stream().map(EcsTask::getTaskDefinitionArn).toList(),+ everyItem(equalTo(rev2.getTaskDefinitionArn())));+ assertEquals(2, describe(service, "roll-cluster", "roll-svc").getRunningCount());+ }++ @Test+ void awaitServiceStable_reconcilesOnTheReconcilerThreadNotTheCallers() {+ List<String> launchThreads = Collections.synchronizedList(new ArrayList<>());+ EcsService service = dockerModeService(dockerContainerManager(+ task -> launchThreads.add(Thread.currentThread().getName())));+ service.createCluster("thread-cluster", REGION);+ registerTaskDef(service, "thread-fam", "app:1");+ EcsServiceModel svc = service.createService("thread-cluster", "thread-svc", "thread-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);++ service.awaitServiceStable(svc, GENEROUS);++ assertEquals(List.of("ecs-reconciler"), launchThreads,+ "the pass runs on the single reconciler thread, so it never overlaps a scheduled tick");+ }++ @Test+ void awaitServiceStable_serviceThatNeverStabilizes_throwsNotStabilizedOnTimeout() {+ EcsContainerManager containerManager = dockerContainerManager(task -> { });+ // Every container has exited by the time it is inspected: each pass finds the task gone and+ // starts another, so the service never settles and nothing fails to launch.+ when(containerManager.getExitCodeIfStopped(anyString())).thenReturn(1);+ EcsService service = dockerModeService(containerManager);+ service.createCluster("stuck-cluster", REGION);+ registerTaskDef(service, "stuck-fam", "app:1");+ EcsServiceModel svc = service.createService("stuck-cluster", "stuck-svc", "stuck-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);++ AwsException error = assertThrows(AwsException.class,+ () -> service.awaitServiceStable(svc, Duration.ofMillis(300)));++ assertEquals("NotStabilized", error.getErrorCode());+ assertEquals("Service " + svc.getServiceArn() + " did not stabilize.", error.getMessage());+ }++ @Test+ void awaitServiceStable_taskCannotBeLaunched_failsWithTheLaunchFailureLongBeforeTheTimeout() {+ EcsContainerManager containerManager = mock(EcsContainerManager.class);+ // What the container manager raises for an awsvpc task whose subnet EC2 does not know.+ doAnswer(invocation -> {+ EcsTask task = invocation.getArgument(0);+ String subnet = task.getNetworkConfiguration().getAwsvpcConfiguration().getSubnets().getFirst();+ throw new AwsException("InvalidParameterException", "The subnet ID '" + subnet + "' does not exist", 400);+ }).when(containerManager).attachTaskNetwork(any(), any(), anyString());+ EcsService service = mockModeService(containerManager);+ service.createCluster("net-cluster", REGION);+ ContainerDefinition container = new ContainerDefinition();+ container.setName("app");+ container.setImage("app:1");+ service.registerTaskDefinition("net-fam", List.of(container), NetworkMode.awsvpc, null, null,+ null, null, List.of(), REGION);+ AwsVpcConfiguration awsvpc = new AwsVpcConfiguration();+ awsvpc.setSubnets(List.of("subnet-does-not-exist"));+ NetworkConfiguration network = new NetworkConfiguration();+ network.setAwsvpcConfiguration(awsvpc);+ EcsServiceModel svc = service.createService("net-cluster", "net-svc", "net-fam", 1,+ LaunchType.FARGATE, List.of(), network, REGION);++ long started = System.nanoTime();+ AwsException error = assertThrows(AwsException.class,+ () -> service.awaitServiceStable(svc, Duration.ofMinutes(1)));+ Duration waited = Duration.ofNanos(System.nanoTime() - started);++ assertEquals("NotStabilized", error.getErrorCode());+ assertEquals("Service " + svc.getServiceArn() + " did not stabilize. Task launch failed: "+ + "The subnet ID 'subnet-does-not-exist' does not exist", error.getMessage());+ assertTrue(waited.compareTo(Duration.ofSeconds(10)) < 0, "the wait took " + waited);+ }++ @Test+ void awaitServiceStable_taskFailsToStart_failsWithTheStoppedReasonBeforeTheTimeout() {+ EcsService service = dockerModeService(dockerContainerManager(task -> {+ throw new IllegalStateException("CannotPullContainerError: pull access denied for no-such-image");+ }));+ service.createCluster("fail-cluster", REGION);+ registerTaskDef(service, "fail-fam", "no-such-image");+ EcsServiceModel svc = service.createService("fail-cluster", "fail-svc", "fail-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);++ AwsException error = assertThrows(AwsException.class, () -> service.awaitServiceStable(svc, GENEROUS));++ assertEquals("NotStabilized", error.getErrorCode());+ assertThat(error.getMessage(), containsString("Service " + svc.getServiceArn() + " did not stabilize."));+ assertThat(error.getMessage(), containsString("CannotPullContainerError: pull access denied for no-such-image"));+ }++ @Test+ void awaitServiceStable_launchFailureOfThePreviousDeployment_doesNotEndTheWait() throws InterruptedException {+ CountDownLatch staleLaunching = new CountDownLatch(1);+ CountDownLatch releaseStale = new CountDownLatch(1);+ EcsContainerManager containerManager = mock(EcsContainerManager.class);+ EcsService service = mockModeService(containerManager);+ Thread stalePass = new Thread(service::reconcile);+ stalePass.setDaemon(true);+ // The pass that began before the update is still launching for the previous deployment.+ doAnswer(invocation -> {+ if (Thread.currentThread() == stalePass) {+ staleLaunching.countDown();+ releaseStale.await();+ throw new AwsException("InvalidParameterException", "stale launch failure", 400);+ }+ return null;+ }).when(containerManager).attachTaskNetwork(any(), any(), anyString());+ // Its launch fails only once the wait has cleared the service's launch failure and started+ // the new deployment's task: when the wait's second pass stops the previous deployment's.+ doAnswer(invocation -> {+ releaseStale.countDown();+ stalePass.join();+ return null;+ }).when(containerManager).releaseTaskNetwork(any(), anyString());+ service.createCluster("race-cluster", REGION);+ registerTaskDef(service, "race-fam", "app:1");+ EcsServiceModel svc = service.createService("race-cluster", "race-svc", "race-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);+ service.awaitServiceStable(svc, GENEROUS);+ service.updateService("race-cluster", "race-svc", null, 2, null, REGION);+ stalePass.start();+ assertTrue(staleLaunching.await(10, TimeUnit.SECONDS));+ TaskDefinition rev2 = registerTaskDef(service, "race-fam", "app:2");+ EcsServiceModel updated = service.updateService("race-cluster", "race-svc",+ rev2.getTaskDefinitionArn(), 1, null, REGION);++ service.awaitServiceStable(updated, GENEROUS);++ assertEquals(List.of(rev2.getTaskDefinitionArn()),+ runningTasks(service).stream().map(EcsTask::getTaskDefinitionArn).toList());+ }++ @Test+ void awaitServiceStable_launchOfThePreviousDeploymentOvertakenByAnUpdate_isReplaced()+ throws InterruptedException {+ CountDownLatch staleLaunching = new CountDownLatch(1);+ CountDownLatch releaseStale = new CountDownLatch(1);+ EcsContainerManager containerManager = mock(EcsContainerManager.class);+ EcsService service = mockModeService(containerManager);+ Thread stalePass = new Thread(service::reconcile);+ stalePass.setDaemon(true);+ // The pass launching the first revision's task is still attaching its network when the update lands.+ doAnswer(invocation -> {+ if (Thread.currentThread() == stalePass) {+ staleLaunching.countDown();+ releaseStale.await();+ }+ return null;+ }).when(containerManager).attachTaskNetwork(any(), any(), anyString());+ service.createCluster("overtaken-cluster", REGION);+ TaskDefinition rev1 = registerTaskDef(service, "overtaken-fam", "app:1");+ service.createService("overtaken-cluster", "overtaken-svc", rev1.getTaskDefinitionArn(), 1,+ LaunchType.FARGATE, List.of(), null, REGION);+ stalePass.start();+ assertTrue(staleLaunching.await(10, TimeUnit.SECONDS));+ TaskDefinition rev2 = registerTaskDef(service, "overtaken-fam", "app:2");+ EcsServiceModel updated = service.updateService("overtaken-cluster", "overtaken-svc",+ rev2.getTaskDefinitionArn(), 1, null, REGION);+ releaseStale.countDown();+ stalePass.join(TimeUnit.SECONDS.toMillis(10));++ service.awaitServiceStable(updated, GENEROUS);++ assertEquals(List.of(rev2.getTaskDefinitionArn()),+ runningTasks(service).stream().map(EcsTask::getTaskDefinitionArn).toList());+ }++ @Test+ void deleteService_forgetsTheServicesLaunchFailure() {+ EcsContainerManager containerManager = mock(EcsContainerManager.class);+ doThrow(new AwsException("InvalidParameterException", "no such subnet", 400))+ .when(containerManager).attachTaskNetwork(any(), any(), anyString());+ EcsService service = mockModeService(containerManager);+ service.createCluster("gone-cluster", REGION);+ registerTaskDef(service, "gone-fam", "app:1");+ EcsServiceModel svc = service.createService("gone-cluster", "gone-svc", "gone-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);+ service.reconcile();+ assertTrue(service.hasLaunchFailure(svc.getServiceArn()));++ service.deleteService("gone-cluster", "gone-svc", true, REGION);++ assertFalse(service.hasLaunchFailure(svc.getServiceArn()));+ }++ private EcsService mockModeService() {+ return mockModeService(mock(EcsContainerManager.class));+ }++ private EcsService mockModeService(EcsContainerManager containerManager) {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(true);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ return track(new EcsService(new RegionResolver(REGION, "000000000000"), containerManager,+ config, mock(EcsLoadBalancerRegistrar.class), new InMemoryStorageFactory(), null));+ }++ /** A container manager that runs {@code onStart} in {@code startTask}; its containers keep running. */+ private static EcsContainerManager dockerContainerManager(Consumer<EcsTask> onStart) {+ EcsContainerManager containerManager = mock(EcsContainerManager.class);+ // Mockito answers 0 for an Integer, which reads as "every container exited".+ when(containerManager.getExitCodeIfStopped(anyString())).thenReturn(null);+ when(containerManager.startTask(any(), any(), any(), anyString())).thenAnswer(invocation -> {+ EcsTask task = invocation.getArgument(0);+ onStart.accept(task);+ return new EcsTaskHandle(task.getTaskArn(), Map.of("app", "docker-" + task.getTaskArn().hashCode()),+ Map.of());+ });+ return containerManager;+ }++ private EcsService dockerModeService(EcsContainerManager containerManager) {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(false);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ return track(new EcsService(new RegionResolver(REGION, "000000000000"), containerManager,+ config, mock(EcsLoadBalancerRegistrar.class), new InMemoryStorageFactory(), null));+ }++ private EcsService track(EcsService service) {+ service.initializeStorage();+ created.add(service);+ return service;+ }++ private static TaskDefinition registerTaskDef(EcsService service, String family, String image) {+ ContainerDefinition container = new ContainerDefinition();+ container.setName("app");+ container.setImage(image);+ container.setEssential(true);+ return service.registerTaskDefinition(family, List.of(container), null, null, null,+ null, null, List.of(), REGION);+ }++ private static EcsServiceModel describe(EcsService service, String cluster, String name) {+ return service.describeServices(cluster, List.of(name), REGION).getFirst();+ }++ private static List<EcsTask> runningTasks(EcsService service) {+ return service.describeTasks(null, service.listTasks(null, null, null, null, REGION), REGION).stream()+ .filter(t -> TaskStatus.RUNNING.name().equals(t.getLastStatus()))+ .toList();+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();++ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ @SuppressWarnings("unchecked")+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,+ ignored -> AccountAwareStorageBackend.inMemory("000000000000"));+ }+ }+}