Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): a service reaches steady state, and a finished deployment stays finished (#4694)
Allen Sanbornc6b8a70ea8dd
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.javaindex 59c0bdebb0..148b8e689c 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java@@ -955,7 +955,11 @@ public class EcsResponseWriter {putInstant(n, "createdAt", d.getCreatedAt());putInstant(n, "startedAt", d.getStartedAt());putInstant(n, "finishedAt", d.getFinishedAt());+ putInstant(n, "stoppedAt", d.getStoppedAt());putInstant(n, "updatedAt", d.getUpdatedAt());+ if (d.getStatusReason() != null) {+ n.put("statusReason", d.getStatusReason());+ }if (d.getTargetServiceRevisionArn() != null) {n.set("targetServiceRevision",serviceRevisionSummaryNode(d.getTargetServiceRevisionArn(), d.getServiceArn()));diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex d35205d55b..5de43acdbb 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -159,6 +159,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabpublic static final String PROPAGATE_TAGS_SERVICE = "SERVICE";public static final String PROPAGATE_TAGS_TASK_DEFINITION = "TASK_DEFINITION";public static final String PROPAGATE_TAGS_NONE = "NONE";+ /** A service deployment whose target revision's tasks are not all up yet. */+ public static final String DEPLOYMENT_STATUS_IN_PROGRESS = "IN_PROGRESS";+ /** A service deployment whose target revision is running at its requested task count. */+ public static final String DEPLOYMENT_STATUS_SUCCESSFUL = "SUCCESSFUL";+ /** A service deployment that ended without completing. */+ public static final String DEPLOYMENT_STATUS_STOPPED = "STOPPED";/** RunTask places at most ten tasks in one call, and StartTask at most ten instances. */public static final int MAX_TASKS_PER_RUN = 10;/** A listing returns at most a hundred ARNs per page. */@@ -2473,6 +2479,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabthrow new AwsException("ServiceNotActiveException","Service " + serviceName + " is not active.", 400);}+ // Settle the outgoing deployment against the service as it stands, before this request+ // changes its desired count or task definition: one whose tasks are up has finished,+ // whether or not anything read it. recordServiceDeployment stops it if it has not.+ currentServiceDeployment(svc);if (request.getDesiredCount() != null) {if (request.getDesiredCount() < 0) {throw new AwsException("InvalidParameterException", "desiredCount cannot be a negative number.", 400);@@ -2718,7 +2728,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabthrow new AwsException("InvalidParameterException","The service cannot be stopped. Update the service to 0 tasks or use the force flag.", 400);}+ // Settle against the service as it stands, as UpdateService does: a deployment whose+ // tasks are up has finished, whether or not anything read it.+ currentServiceDeployment(svc);svc.setStatus("INACTIVE");+ // A deleted service's deployments can never finish, so stop them. This must come before+ // setDesiredCount(0): a concurrent read that saw desiredCount 0 on a still-ACTIVE service+ // would otherwise mark the deployment SUCCESSFUL. AWS deletes the records instead; that+ // is not done here yet.+ stopNonTerminalDeployments(svc.getServiceArn(), Instant.now(), "The service was deleted.");svc.setDesiredCount(0);cluster.setActiveServicesCount(Math.max(0, cluster.getActiveServicesCount() - 1));services.put(key, svc);@@ -3813,6 +3831,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabreturn deploymentArns.stream().map(arn -> serviceDeployments.get(arn)).filter(d -> d != null)+ .map(this::settleStatus).toList();}@@ -3829,6 +3848,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabreturn serviceDeployments.values().stream().filter(d -> d.getServiceArn().equals(svc.getServiceArn()))+ .map(this::settleStatus).filter(d -> statusFilter == null || statusFilter.isEmpty()|| statusFilter.contains(d.getStatus())).sorted((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()))@@ -3859,7 +3879,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}String deploymentId = deploymentId(svc);- boolean converged = svc.getRunningCount() >= svc.getDesiredCount();+ // This deployment's own tasks: counting the ones still draining from the previous+ // deployment would report a task-definition change COMPLETED as soon as it was made.+ long running = runningOnCurrentDeployment(svc);+ boolean converged = running >= svc.getDesiredCount();Deployment d = new Deployment();d.setId(deploymentId);@@ -3867,7 +3890,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabd.setTaskDefinition(svc.getTaskDefinition());d.setDesiredCount(svc.getDesiredCount());d.setPendingCount(svc.getPendingCount());- d.setRunningCount(svc.getRunningCount());+ d.setRunningCount((int) running);d.setFailedTasks(0);d.setRolloutState(converged ? "COMPLETED" : "IN_PROGRESS");d.setRolloutStateReason("ECS deployment " + deploymentId@@ -3906,7 +3929,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab*/public List<ServiceEvent> eventsFor(EcsServiceModel svc) {if (svc == null || !"ACTIVE".equals(svc.getStatus())- || svc.getRunningCount() < svc.getDesiredCount()) {+ || runningOnCurrentDeployment(svc) < svc.getDesiredCount()) {return List.of();}String deploymentId = deploymentId(svc);@@ -3943,22 +3966,62 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabreturn "ecs-svc/" + Long.toUnsignedString(hash);}+ /**+ * How many of the service's RUNNING tasks belong to the deployment it is currently on.+ * Not {@code svc.getRunningCount()}, which also counts tasks still draining from the+ * previous deployment.+ */+ private long runningOnCurrentDeployment(EcsServiceModel svc) {+ EcsCluster cluster = resolveClusterByArn(svc.getClusterArn());+ if (cluster == null) {+ return 0;+ }+ Stream<EcsTask> running = tasks.values().stream()+ .filter(t -> ownedBy(t, svc, cluster))+ .filter(t -> TaskStatus.RUNNING.name().equals(t.getLastStatus()));++ // reconcileDaemonService does not replace a DAEMON service's tasks when its task+ // definition changes, so the stale ones are the only ones there will ever be; counting+ // them keeps main's behaviour. Scope this like REPLICA once DAEMON services roll.+ if (SCHEDULING_DAEMON.equals(svc.getSchedulingStrategy())) {+ return running.count();+ }++ String currentDeploymentId = deploymentId(svc);+ String taskDefinitionArn = resolvedTaskDefinitionArn(svc, serviceRegion(svc));+ return running+ .filter(t -> !isStaleForDeployment(t, currentDeploymentId, taskDefinitionArn))+ .count();+ }++ /**+ * The bare identifier inside the service's {@code ecs-svc/<id>} deployment id: what AWS+ * calls the task set id, and what names the service revision that deployment targets.+ */+ private String taskSetId(EcsServiceModel svc) {+ String id = deploymentId(svc);+ int slash = id.indexOf('/');+ return slash < 0 ? id : id.substring(slash + 1);+ }+/*** Records the deployment and the revision it targets. The revision is the snapshot of the* service's configuration, so it is copied out of the service rather than read back from it* later; the deployment links to it, which is how a caller gets from* {@code DescribeServiceDeployments} to what was actually deployed.*- * <p>Floci applies a change in place instead of rolling it, so a deployment is finished the- * moment it is recorded: {@code startedAt} and {@code finishedAt} are both its creation time.+ * <p>The revision ARN ends in the deployment's task set id, the {@code <id>} of+ * {@code ecs-svc/<id>}, as AWS's does. The Terraform AWS provider's+ * {@code wait_for_steady_state} finds the deployment to poll by looking for that id in each+ * deployment's {@code targetServiceRevisionArn}. The ARN built here is+ * {@code service-revision/<id>}; AWS's is {@code service-revision/<cluster>/<service>/<id>}.*/private void recordServiceDeployment(EcsServiceModel svc, String taskDefinition, String region) {String deploymentId = UUID.randomUUID().toString().replace("-", "");String deploymentArn = regionResolver.buildArn("ecs", region,"service-deployment/" + deploymentId);- String revisionId = UUID.randomUUID().toString().replace("-", "");String revisionArn = regionResolver.buildArn("ecs", region,- "service-revision/" + revisionId);+ "service-revision/" + taskSetId(svc));Instant now = Instant.now();List<String> priorRevisions = serviceRevisions.values().stream()@@ -3974,13 +4037,14 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabdeployment.setServiceArn(svc.getServiceArn());deployment.setClusterArn(svc.getClusterArn());deployment.setTaskDefinition(taskDefinition);- deployment.setStatus("SUCCESSFUL");+ deployment.setStatus(DEPLOYMENT_STATUS_IN_PROGRESS);deployment.setCreatedAt(now);deployment.setStartedAt(now);- deployment.setFinishedAt(now);deployment.setUpdatedAt(now);deployment.setTargetServiceRevisionArn(revisionArn);deployment.setSourceServiceRevisionArns(priorRevisions);+ stopNonTerminalDeployments(svc.getServiceArn(), now,+ "Superseded by service deployment " + deploymentArn + ".");serviceDeployments.put(deploymentArn, deployment);ServiceRevision revision = new ServiceRevision();@@ -4001,6 +4065,75 @@ public class EcsService implements ContainerTeardown, ResourceProvider, ResettabserviceRevisions.put(revisionArn, revision);}+ /**+ * Marks the service's IN_PROGRESS deployments STOPPED, for a caller that has made them+ * unable to finish: a new deployment superseding them, or the service being deleted. Runs+ * before the new deployment is added to the map, so it never stops the current one.+ */+ private void stopNonTerminalDeployments(String serviceArn, Instant now, String reason) {+ // A service model restored from the store may have no ARN.+ if (serviceArn == null) {+ return;+ }+ for (ServiceDeployment prior : serviceDeployments.values()) {+ if (!serviceArn.equals(prior.getServiceArn())) {+ continue;+ }+ synchronized (prior) {+ if (DEPLOYMENT_STATUS_IN_PROGRESS.equals(prior.getStatus())) {+ prior.setStatus(DEPLOYMENT_STATUS_STOPPED);+ prior.setStatusReason(reason);+ prior.setFinishedAt(now);+ prior.setStoppedAt(now);+ prior.setUpdatedAt(now);+ }+ }+ }+ }++ /**+ * Marks the service's current deployment SUCCESSFUL once its own tasks are all running.+ * Only an IN_PROGRESS deployment moves, so a finished one never changes again.+ */+ private ServiceDeployment settleStatus(ServiceDeployment deployment) {+ EcsServiceModel svc = serviceByArn(deployment.getServiceArn());+ synchronized (deployment) {+ if (svc != null && deployment.getTargetServiceRevisionArn() != null+ && deployment.getTargetServiceRevisionArn().endsWith("/" + taskSetId(svc))+ && DEPLOYMENT_STATUS_IN_PROGRESS.equals(deployment.getStatus())+ && STATUS_ACTIVE.equals(svc.getStatus())+ && runningOnCurrentDeployment(svc) >= svc.getDesiredCount()) {+ Instant now = Instant.now();+ deployment.setStatus(DEPLOYMENT_STATUS_SUCCESSFUL);+ deployment.setFinishedAt(now);+ deployment.setUpdatedAt(now);+ }+ return copyOf(deployment);+ }+ }++ /**+ * A copy taken under the deployment's lock, so a response never sees a half-applied+ * transition.+ */+ private static ServiceDeployment copyOf(ServiceDeployment source) {+ ServiceDeployment copy = new ServiceDeployment();+ copy.setServiceDeploymentArn(source.getServiceDeploymentArn());+ copy.setServiceArn(source.getServiceArn());+ copy.setClusterArn(source.getClusterArn());+ copy.setTaskDefinition(source.getTaskDefinition());+ copy.setStatus(source.getStatus());+ copy.setCreatedAt(source.getCreatedAt());+ copy.setStartedAt(source.getStartedAt());+ copy.setFinishedAt(source.getFinishedAt());+ copy.setStoppedAt(source.getStoppedAt());+ copy.setStatusReason(source.getStatusReason());+ copy.setUpdatedAt(source.getUpdatedAt());+ copy.setTargetServiceRevisionArn(source.getTargetServiceRevisionArn());+ copy.setSourceServiceRevisionArns(source.getSourceServiceRevisionArns());+ return copy;+ }+/** The images the revision's task definition pins, one entry per container definition. */private List<ContainerImage> containerImagesOf(String taskDefinitionRef, String region) {TaskDefinition taskDef;@@ -4043,6 +4176,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab.filter(d -> svc.getServiceArn().equals(d.getServiceArn())).max(Comparator.comparing(ServiceDeployment::getCreatedAt,Comparator.nullsFirst(Comparator.naturalOrder())))+ .map(this::settleStatus).orElse(null);}@@ -4271,6 +4405,21 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}}+ /** The region a service lives in, read off its ARN, falling back to the default region. */+ private String serviceRegion(EcsServiceModel svc) {+ if (svc.getServiceArn() != null) {+ try {+ String region = AwsArnUtils.parse(svc.getServiceArn()).region();+ if (region != null && !region.isBlank()) {+ return region;+ }+ } catch (Exception e) {+ LOG.debugv("Could not parse region from service ARN {0}", svc.getServiceArn());+ }+ }+ return regionResolver.getDefaultRegion();+ }+/** The region a task lives in, read off its ARN, falling back to the default region. */private String taskRegion(EcsTask task) {if (task.getTaskArn() != null) {@@ -4438,15 +4587,29 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab* simply matches nothing, which is the pre-existing behaviour for an unresolvable service.*/private String pinnedTaskDefinitionArn(EcsServiceModel svc, String key, String region) {+ String ref = svc.getTaskDefinition();+ String arn = resolvedTaskDefinitionArn(svc, region);+ if (arn != null && !arn.equals(ref)) {+ svc.setTaskDefinition(arn);+ services.put(key, svc);+ }+ return arn;+ }++ /**+ * The service's task definition as an ARN, resolving a {@code family} or+ * {@code family:revision} reference without storing the result.+ *+ * <p>Shared with {@link #pinnedTaskDefinitionArn} so that the read path and the reconciler+ * decide staleness the same way.+ */+ private String resolvedTaskDefinitionArn(EcsServiceModel svc, String region) {String ref = svc.getTaskDefinition();if (ref == null || ref.startsWith("arn:")) {return ref;}try {- String arn = resolveTaskDefinitionOrThrow(ref, region).getTaskDefinitionArn();- svc.setTaskDefinition(arn);- services.put(key, svc);- return arn;+ return resolveTaskDefinitionOrThrow(ref, region).getTaskDefinitionArn();} catch (AwsException e) {return ref;}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java b/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.javaindex 92bd833162..d689d1cb3f 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java@@ -21,6 +21,10 @@ public class ServiceDeployment {private Instant createdAt;private Instant startedAt;private Instant finishedAt;+ /** When a deployment that ended without completing was stopped. */+ private Instant stoppedAt;+ /** Why the deployment is in its status; set when it is stopped. */+ private String statusReason;private Instant updatedAt;private String targetServiceRevisionArn;private List<String> sourceServiceRevisionArns;@@ -31,6 +35,12 @@ public class ServiceDeployment {public Instant getFinishedAt() { return finishedAt; }public void setFinishedAt(Instant finishedAt) { this.finishedAt = finishedAt; }+ public Instant getStoppedAt() { return stoppedAt; }+ public void setStoppedAt(Instant stoppedAt) { this.stoppedAt = stoppedAt; }++ public String getStatusReason() { return statusReason; }+ public void setStatusReason(String statusReason) { this.statusReason = statusReason; }+public String getTargetServiceRevisionArn() { return targetServiceRevisionArn; }public void setTargetServiceRevisionArn(String targetServiceRevisionArn) {this.targetServiceRevisionArn = targetServiceRevisionArn;diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.javanew file mode 100644index 0000000000..4eff9cdef0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java@@ -0,0 +1,594 @@+package io.github.hectorvent.floci.services.ecs;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;+import io.github.hectorvent.floci.services.ecs.model.EcsTask;+import io.github.hectorvent.floci.services.ecs.model.LaunchType;+import io.github.hectorvent.floci.services.ecs.model.ServiceDeployment;+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;+import org.junit.jupiter.api.Test;++import java.time.Instant;+import java.util.HashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * What a service deployment reports, driven off the reconciler tick by tick. Deployments are+ * selected by task set id rather than by position in a listing: several minted in one clock tick+ * share a createdAt, so their order in the listing is map iteration order.+ */+class EcsServiceDeploymentStatusTest {++ private static final String REGION = "us-east-1";++ @Test+ void theCurrentDeploymentIsInProgressUntilTheServicesTasksAreRunning() {+ EcsService service = newMockModeService();+ service.createCluster("dstat-cluster", REGION);+ registerTaskDef(service, "dstat-fam", "app:1");+ service.createService("dstat-cluster", "dstat-svc", "dstat-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);++ // Created, nothing launched yet: runningCount 0 of a requested 1.+ ServiceDeployment pending = onlyDeployment(service, "dstat-svc", "dstat-cluster");+ assertEquals("IN_PROGRESS", pending.getStatus());+ assertNull(pending.getFinishedAt(), "an unfinished deployment has no finishedAt");++ // One tick: the count is taken from the tasks at read time, not from svc.runningCount,+ // which lags a tick behind.+ service.reconcileServices();++ ServiceDeployment done = onlyDeployment(service, "dstat-svc", "dstat-cluster");+ assertEquals("SUCCESSFUL", done.getStatus());+ assertNotNull(done.getFinishedAt(), "a finished deployment reports when it finished");+ }++ /** A service already at its requested count has converged, so its deployment is finished. */+ @Test+ void aServiceAlreadyAtItsRequestedCountHasNothingToWaitFor() {+ EcsService service = newMockModeService();+ service.createCluster("dzero-cluster", REGION);+ registerTaskDef(service, "dzero-fam", "app:1");+ service.createService("dzero-cluster", "dzero-svc", "dzero-fam", 0,+ LaunchType.FARGATE, List.of(), null, REGION);++ ServiceDeployment deployment = onlyDeployment(service, "dzero-svc", "dzero-cluster");+ assertEquals("SUCCESSFUL", deployment.getStatus());+ assertNotNull(deployment.getFinishedAt());+ }++ @Test+ void everyDeploymentTargetsARevisionNamedByItsOwnTaskSetId() {+ EcsService service = newMockModeService();+ service.createCluster("drev-cluster", REGION);+ registerTaskDef(service, "drev-fam", "app:1");+ EcsServiceModel created = service.createService("drev-cluster", "drev-svc", "drev-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);+ // Read out now: createService and updateService hand back the same live model object.+ String firstDeploymentId = created.getDeploymentId();+ assertRevisionCarriesTaskSetId(service, "drev-svc", "drev-cluster", firstDeploymentId);++ service.reconcileServices();+ String secondDeploymentId = service.updateService("drev-cluster", "drev-svc", null, null,+ null, null, true, REGION).getDeploymentId();+ assertNotEquals(firstDeploymentId, secondDeploymentId,+ "forceNewDeployment mints a new deployment id");+ assertRevisionCarriesTaskSetId(service, "drev-svc", "drev-cluster", secondDeploymentId);++ // The superseded deployment keeps pointing at its own revision, so the two never collide.+ List<ServiceDeployment> all = service.listServiceDeploymentsDetailed("drev-svc",+ "drev-cluster", null, REGION);+ assertEquals(2, all.size());+ assertEquals(2, all.stream().map(ServiceDeployment::getTargetServiceRevisionArn)+ .distinct().count(), "each deployment targets its own revision");+ }++ /**+ * A task-definition change is judged on the new deployment's own tasks, not on the old+ * revision's task that is still running.+ */+ @Test+ void aTaskDefinitionChangeIsNotFinishedWhileOnlyTheOldTasksAreRunning() {+ EcsService service = newMockModeService();+ service.createCluster("dupd-cluster", REGION);+ registerTaskDef(service, "dupd-fam", "app:1");+ EcsServiceModel created = service.createService("dupd-cluster", "dupd-svc", "dupd-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);+ String firstDeploymentId = created.getDeploymentId();+ service.reconcileServices();+ assertEquals("SUCCESSFUL", deploymentOf(service, "dupd-svc", "dupd-cluster",+ firstDeploymentId).getStatus(),+ "precondition: the first deployment converged");++ TaskDefinition rev2 = registerTaskDef(service, "dupd-fam", "app:2");+ String rolled = service.updateService("dupd-cluster", "dupd-svc",+ "dupd-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();++ // No tick yet. The only RUNNING task belongs to the superseded deployment, so the new+ // deployment has nothing of its own up.+ ServiceDeployment rolling = deploymentOf(service, "dupd-svc", "dupd-cluster", rolled);+ assertEquals("IN_PROGRESS", rolling.getStatus(),+ "a deployment with none of its own tasks running is not finished");+ assertNull(rolling.getFinishedAt(), "and carries no finishedAt");++ // One tick launches the new revision's task; now the deployment really has converged,+ // even though the old task is still draining beside it.+ service.reconcileServices();+ ServiceDeployment settled = deploymentOf(service, "dupd-svc", "dupd-cluster", rolled);+ assertEquals("SUCCESSFUL", settled.getStatus(),+ "its own task is running, so it is finished even while the old one drains");+ assertNotNull(settled.getFinishedAt());+ }++ /** SUCCESSFUL is terminal: tasks dying afterwards do not un-finish the deployment. */+ @Test+ void aFinishedDeploymentStaysFinishedWhenItsTasksDie() {+ EcsService service = newMockModeService();+ service.createCluster("ddie-cluster", REGION);+ registerTaskDef(service, "ddie-fam", "app:1");+ String deploymentId = service.createService("ddie-cluster", "ddie-svc", "ddie-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();+ service.reconcileServices();+ ServiceDeployment finished = deploymentOf(service, "ddie-svc", "ddie-cluster", deploymentId);+ assertEquals("SUCCESSFUL", finished.getStatus());+ Instant finishedAt = finished.getFinishedAt();+ assertNotNull(finishedAt);++ // Read again while still converged: finishedAt is stamped once, not on every read.+ assertEquals(finishedAt,+ deploymentOf(service, "ddie-svc", "ddie-cluster", deploymentId).getFinishedAt(),+ "finishedAt is stamped once, not re-stamped on every read while converged");++ String taskArn = runningTasks(service).getFirst().getTaskArn();+ service.stopTask("ddie-cluster", taskArn, "test kill", REGION);+ assertEquals(0, runningTasks(service).size(), "precondition: nothing of it is running");++ ServiceDeployment after = deploymentOf(service, "ddie-svc", "ddie-cluster", deploymentId);+ assertEquals("SUCCESSFUL", after.getStatus(), "a completed deployment does not un-complete");+ assertEquals(finishedAt, after.getFinishedAt(), "and keeps the instant it finished at");+ }++ /**+ * Scaling to zero is converged while the task is still draining, since {@code >=} is used.+ * AWS's own waiter compares with {@code ==} and would wait for the task to stop.+ */+ @Test+ void scalingToZeroIsConvergedBeforeTheTaskHasActuallyStopped() {+ EcsService service = newMockModeService();+ service.createCluster("dscale-cluster", REGION);+ registerTaskDef(service, "dscale-fam", "app:1");+ String id = service.createService("dscale-cluster", "dscale-svc", "dscale-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();+ service.reconcileServices();++ service.updateService("dscale-cluster", "dscale-svc", null, 0, null, REGION);+ assertEquals(1, runningTasks(service).size(), "the task has not been drained yet");+ assertEquals("SUCCESSFUL", deploymentOf(service, "dscale-svc", "dscale-cluster", id).getStatus());++ service.reconcileServices();+ assertEquals(0, runningTasks(service).size(), "and now it is drained");+ assertEquals("SUCCESSFUL", deploymentOf(service, "dscale-svc", "dscale-cluster", id).getStatus());+ }++ /**+ * Two task-definition changes in a row with no read between them. The first deployment had+ * converged, so the update that supersedes it settles it SUCCESSFUL; the second never had a+ * task of its own, so the next update stops it.+ */+ @Test+ void twoUpdatesInSuccessionSettleOnlyTheNewestDeployment() {+ EcsService service = newMockModeService();+ service.createCluster("drapid-cluster", REGION);+ registerTaskDef(service, "drapid-fam", "app:1");+ // createService and updateService return the same live model, so read each id at once.+ String first = service.createService("drapid-cluster", "drapid-svc", "drapid-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();+ service.reconcileServices();++ TaskDefinition rev2 = registerTaskDef(service, "drapid-fam", "app:2");+ String second = service.updateService("drapid-cluster", "drapid-svc",+ "drapid-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();+ TaskDefinition rev3 = registerTaskDef(service, "drapid-fam", "app:3");+ String newest = service.updateService("drapid-cluster", "drapid-svc",+ "drapid-fam:" + rev3.getRevision(), null, null, REGION).getDeploymentId();++ assertEquals(3, service.listServiceDeploymentsDetailed("drapid-svc", "drapid-cluster",+ null, REGION).size(), "one per create/update");+ assertEquals("IN_PROGRESS",+ deploymentOf(service, "drapid-svc", "drapid-cluster", newest).getStatus(),+ "the newest has none of its own tasks running");+ assertEquals("SUCCESSFUL",+ deploymentOf(service, "drapid-svc", "drapid-cluster", first).getStatus(),+ "the first had converged before it was superseded, though nothing read it");+ assertEquals("STOPPED",+ deploymentOf(service, "drapid-svc", "drapid-cluster", second).getStatus(),+ "the second never had a task of its own");++ service.reconcileServices();+ assertEquals("SUCCESSFUL",+ deploymentOf(service, "drapid-svc", "drapid-cluster", newest).getStatus(),+ "and converges once its own task is up");+ }++ /**+ * Right after a task-definition change, {@code rolloutState} and the deployment record+ * describe the same moment and must agree. Later they may differ: the record is history,+ * rolloutState follows the live service.+ */+ @Test+ void theRolloutStateAndTheDeploymentRecordAgreeWhileARolloutIsInFlight() {+ EcsService service = newMockModeService();+ service.createCluster("dboth-cluster", REGION);+ registerTaskDef(service, "dboth-fam", "app:1");+ EcsServiceModel created = service.createService("dboth-cluster", "dboth-svc", "dboth-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION);+ // Two ticks: the first launches the task, the second counts it in svc.runningCount.+ // With only one, the service-wide count is 0 and the old miscount passes by accident.+ service.reconcileServices();+ service.reconcileServices();+ assertEquals(1, created.getRunningCount(),+ "precondition: a genuine steady state, not one tick short of it");++ TaskDefinition rev2 = registerTaskDef(service, "dboth-fam", "app:2");+ String rolled = service.updateService("dboth-cluster", "dboth-svc",+ "dboth-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();++ EcsServiceModel svc = service.serviceByArn(+ service.describeServices("dboth-cluster", List.of("dboth-svc"), REGION)+ .getFirst().getServiceArn());+ assertEquals("IN_PROGRESS", service.deploymentsFor(svc).getFirst().getRolloutState(),+ "the live rollout has not started: only the old revision's task is up");+ assertEquals("IN_PROGRESS",+ deploymentOf(service, "dboth-svc", "dboth-cluster", rolled).getStatus(),+ "and the record says the same about the same moment");+ assertTrue(service.eventsFor(svc).isEmpty(),+ "no steady-state event while the rollout is in flight");++ service.reconcileServices();+ assertEquals("COMPLETED", service.deploymentsFor(svc).getFirst().getRolloutState());+ assertEquals("SUCCESSFUL",+ deploymentOf(service, "dboth-svc", "dboth-cluster", rolled).getStatus());+ }++ /**+ * A DAEMON service still finishes its deployment after a task-definition change. DAEMON+ * services do not roll yet, so the old revision's task is still the one running; the last+ * assertions fail once they do, which is when the DAEMON case in the count should go.+ */+ @Test+ void aDaemonServiceStillFinishesItsDeploymentAfterATaskDefinitionChange() {+ EcsService service = newMockModeService();+ service.createCluster("ddmn-cluster", REGION);+ service.registerContainerInstance("ddmn-cluster", null, List.of(), REGION);+ TaskDefinition rev1 = registerTaskDef(service, "ddmn-fam", "app:1");+ service.createService("ddmn-cluster", "ddmn-svc", "ddmn-fam", 1, LaunchType.EC2,+ List.of(), null, null, "DAEMON", null, null, REGION);+ service.reconcileServices();+ service.reconcileServices();++ TaskDefinition rev2 = registerTaskDef(service, "ddmn-fam", "app:2");+ String rolled = service.updateService("ddmn-cluster", "ddmn-svc",+ "ddmn-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();+ service.reconcileServices();+ service.reconcileServices();++ ServiceDeployment record = deploymentOf(service, "ddmn-svc", "ddmn-cluster", rolled);+ assertEquals("SUCCESSFUL", record.getStatus(),+ "a DAEMON deployment must still finish, or a steady-state wait hangs for ever");+ assertNotNull(record.getFinishedAt());++ List<EcsTask> live = runningTasks(service);+ assertEquals(1, live.size(), "one task per container instance");+ assertEquals(rev1.getTaskDefinitionArn(), live.getFirst().getTaskDefinitionArn(),+ "DAEMON does not roll on a task-definition change; the count is honest about that");+ assertNotEquals(rev2.getTaskDefinitionArn(), live.getFirst().getTaskDefinitionArn());+ }++ /**+ * A deployment superseded before it converged is STOPPED, not left IN_PROGRESS. The second+ * service is never updated, so its deployment must stay in flight.+ */+ @Test+ void supersedingAnUnconvergedDeploymentStopsIt() {+ EcsService service = newMockModeService();+ service.createCluster("dsup-cluster", REGION);+ registerTaskDef(service, "dsup-fam", "app:1");+ String first = service.createService("dsup-cluster", "dsup-svc", "dsup-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();+ registerTaskDef(service, "dsup-other-fam", "other:1");+ String bystander = service.createService("dsup-cluster", "dsup-other", "dsup-other-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();++ // No tick: neither service has launched anything, so both deployments are in flight.+ assertEquals("IN_PROGRESS",+ deploymentOf(service, "dsup-svc", "dsup-cluster", first).getStatus(),+ "precondition: the first deployment never converged");++ TaskDefinition rev2 = registerTaskDef(service, "dsup-fam", "app:2");+ String second = service.updateService("dsup-cluster", "dsup-svc",+ "dsup-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();++ ServiceDeployment superseded = deploymentOf(service, "dsup-svc", "dsup-cluster", first);+ assertEquals("STOPPED", superseded.getStatus(),+ "a deployment another one took over from is finished, not still rolling out");+ assertNotNull(superseded.getFinishedAt(),+ "and reports when it ended, like every deployment that is no longer running");+ assertNotNull(superseded.getStoppedAt(), "and when it was stopped");+ assertTrue(superseded.getStatusReason() != null+ && superseded.getStatusReason().contains(deploymentArnOf(service,+ "dsup-svc", "dsup-cluster", second)),+ "the reason names the deployment that took over, which is what a reader needs "+ + "and the status alone does not say; was: " + superseded.getStatusReason());++ assertEquals("IN_PROGRESS",+ deploymentOf(service, "dsup-svc", "dsup-cluster", second).getStatus(),+ "while the deployment that took over is the one now in flight");+ assertEquals("IN_PROGRESS",+ deploymentOf(service, "dsup-other", "dsup-cluster", bystander).getStatus(),+ "another service's deployment is none of this update's business");+ }++ /** STOPPED is terminal: the replacement converging does not make the stopped one succeed. */+ @Test+ void aSupersededDeploymentDoesNotSucceedWhenItsReplacementDoes() {+ EcsService service = newMockModeService();+ service.createCluster("dsup2-cluster", REGION);+ registerTaskDef(service, "dsup2-fam", "app:1");+ String first = service.createService("dsup2-cluster", "dsup2-svc", "dsup2-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();++ TaskDefinition rev2 = registerTaskDef(service, "dsup2-fam", "app:2");+ String second = service.updateService("dsup2-cluster", "dsup2-svc",+ "dsup2-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();+ Instant stoppedAt = deploymentOf(service, "dsup2-svc", "dsup2-cluster", first).getFinishedAt();+ assertNotNull(stoppedAt, "precondition: the superseded deployment ended");++ service.reconcileServices();+ assertEquals("SUCCESSFUL",+ deploymentOf(service, "dsup2-svc", "dsup2-cluster", second).getStatus(),+ "precondition: the replacement converged");++ ServiceDeployment superseded = deploymentOf(service, "dsup2-svc", "dsup2-cluster", first);+ assertEquals("STOPPED", superseded.getStatus(),+ "the deployment that was taken over from did not succeed, its replacement did");+ assertEquals(stoppedAt, superseded.getFinishedAt(),+ "and still reports the instant it ended");+ }++ /** A deployment that finished, and was read, before it was superseded keeps its record. */+ @Test+ void aDeploymentThatFinishedBeforeItWasSupersededStaysSuccessful() {+ EcsService service = newMockModeService();+ service.createCluster("dsup3-cluster", REGION);+ registerTaskDef(service, "dsup3-fam", "app:1");+ String first = service.createService("dsup3-cluster", "dsup3-svc", "dsup3-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();+ service.reconcileServices();++ ServiceDeployment finished = deploymentOf(service, "dsup3-svc", "dsup3-cluster", first);+ assertEquals("SUCCESSFUL", finished.getStatus(), "precondition: it converged and was read");+ Instant finishedAt = finished.getFinishedAt();+ assertNotNull(finishedAt);++ TaskDefinition rev2 = registerTaskDef(service, "dsup3-fam", "app:2");+ service.updateService("dsup3-cluster", "dsup3-svc", "dsup3-fam:" + rev2.getRevision(),+ null, null, REGION);++ ServiceDeployment after = deploymentOf(service, "dsup3-svc", "dsup3-cluster", first);+ assertEquals("SUCCESSFUL", after.getStatus(),+ "a deployment that completed was not stopped by the one that followed it");+ assertEquals(finishedAt, after.getFinishedAt(),+ "and keeps the instant it finished at, not the instant it was superseded");+ assertNull(after.getStoppedAt(), "a deployment that succeeded was never stopped");+ assertNull(after.getStatusReason(), "and has no reason it failed to finish");+ }++ /**+ * The update settles the outgoing deployment itself, so whether it finished does not depend+ * on anything having read it. Changing the desired count in the same request must not count+ * against it either: it had reached the count it was asked for.+ */+ @Test+ void aConvergedDeploymentNobodyReadIsSuccessfulWhenSuperseded() {+ EcsService service = newMockModeService();+ service.createCluster("dunread-cluster", REGION);+ registerTaskDef(service, "dunread-fam", "app:1");+ String first = service.createService("dunread-cluster", "dunread-svc", "dunread-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();+ service.reconcileServices();++ TaskDefinition rev2 = registerTaskDef(service, "dunread-fam", "app:2");+ String second = service.updateService("dunread-cluster", "dunread-svc",+ "dunread-fam:" + rev2.getRevision(), 3, null, REGION).getDeploymentId();++ ServiceDeployment finished = deploymentOf(service, "dunread-svc", "dunread-cluster", first);+ assertEquals("SUCCESSFUL", finished.getStatus(),+ "its task was running when the update arrived");+ assertNotNull(finished.getFinishedAt());+ assertNull(finished.getStoppedAt());+ assertEquals(List.of(deploymentArnOf(service, "dunread-svc", "dunread-cluster", second)),+ service.listServiceDeployments("dunread-svc", "dunread-cluster",+ List.of("IN_PROGRESS"), REGION),+ "only the new deployment is in progress");+ }++ /**+ * Deleting a service stops a deployment it leaves unfinished, so a listing filtered on+ * IN_PROGRESS no longer returns it.+ */+ @Test+ void deletingAServiceStopsTheDeploymentItLeavesUnfinished() {+ EcsService service = newMockModeService();+ service.createCluster("ddel-cluster", REGION);+ registerTaskDef(service, "ddel-fam", "app:1");+ String only = service.createService("ddel-cluster", "ddel-svc", "ddel-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();++ // No tick, so nothing was ever launched and the deployment is still rolling out.+ assertEquals("IN_PROGRESS",+ deploymentOf(service, "ddel-svc", "ddel-cluster", only).getStatus(),+ "precondition: the deployment never converged");+ assertEquals(1, service.listServiceDeployments("ddel-svc", "ddel-cluster",+ List.of("IN_PROGRESS"), REGION).size(), "precondition: and reads as in flight");++ service.deleteService("ddel-cluster", "ddel-svc", true, REGION);++ ServiceDeployment stopped = deploymentOf(service, "ddel-svc", "ddel-cluster", only);+ assertEquals("STOPPED", stopped.getStatus(),+ "a deployment whose service is gone has ended, and nothing else can ever say so");+ assertNotNull(stopped.getFinishedAt(), "it reports when it ended");+ assertNotNull(stopped.getStoppedAt(), "and when it was stopped");+ assertEquals("The service was deleted.", stopped.getStatusReason(),+ "naming the reason it could not finish");++ assertTrue(service.listServiceDeployments("ddel-svc", "ddel-cluster",+ List.of("IN_PROGRESS"), REGION).isEmpty(),+ "and a client asking what is rolling out is no longer told this is");+ }++ /** Deleting a service leaves its finished deployments SUCCESSFUL. */+ @Test+ void deletingAServiceLeavesItsFinishedDeploymentsSuccessful() {+ EcsService service = newMockModeService();+ service.createCluster("ddel2-cluster", REGION);+ registerTaskDef(service, "ddel2-fam", "app:1");+ String only = service.createService("ddel2-cluster", "ddel2-svc", "ddel2-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();+ service.reconcileServices();++ ServiceDeployment finished = deploymentOf(service, "ddel2-svc", "ddel2-cluster", only);+ assertEquals("SUCCESSFUL", finished.getStatus(), "precondition: it converged and was read");+ Instant finishedAt = finished.getFinishedAt();+ assertNotNull(finishedAt);++ service.deleteService("ddel2-cluster", "ddel2-svc", true, REGION);++ ServiceDeployment after = deploymentOf(service, "ddel2-svc", "ddel2-cluster", only);+ assertEquals("SUCCESSFUL", after.getStatus(),+ "a rollout that completed is history, and deleting the service does not undo it");+ assertEquals(finishedAt, after.getFinishedAt(), "keeping the instant it finished at");+ assertNull(after.getStoppedAt(), "and carrying none of the stopped fields");+ assertNull(after.getStatusReason());+ }++ /**+ * As {@link #aConvergedDeploymentNobodyReadIsSuccessfulWhenSuperseded}, for a delete: the+ * delete settles the deployment against the service as it stood, so a rollout that had+ * finished is SUCCESSFUL whether or not anything read it first.+ */+ @Test+ void deletingAServiceSettlesAConvergedDeploymentNobodyRead() {+ EcsService service = newMockModeService();+ service.createCluster("ddel3-cluster", REGION);+ registerTaskDef(service, "ddel3-fam", "app:1");+ String only = service.createService("ddel3-cluster", "ddel3-svc", "ddel3-fam", 1,+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();+ service.reconcileServices();++ service.deleteService("ddel3-cluster", "ddel3-svc", true, REGION);++ ServiceDeployment after = deploymentOf(service, "ddel3-svc", "ddel3-cluster", only);+ assertEquals("SUCCESSFUL", after.getStatus(),+ "its task was running when the delete arrived");+ assertNotNull(after.getFinishedAt());+ assertNull(after.getStoppedAt(), "a deployment that succeeded was never stopped");+ assertNull(after.getStatusReason());+ }++ /** The one place the provider's join is asserted: the revision ARN carries the task set id. */+ private static void assertRevisionCarriesTaskSetId(EcsService service, String name,+ String cluster, String deploymentId) {+ String taskSetId = deploymentId.substring(deploymentId.indexOf('/') + 1);+ List<String> targets = service.listServiceDeploymentsDetailed(name, cluster, null, REGION)+ .stream().map(ServiceDeployment::getTargetServiceRevisionArn).toList();+ assertTrue(targets.stream().anyMatch(arn -> arn != null && arn.contains(taskSetId)),+ "some deployment's revision ARN must carry task set id " + taskSetId+ + ", targets were: " + targets);+ }++ /** The deployment carrying {@code deploymentId}, found by the task set id in its revision ARN. */+ private static ServiceDeployment deploymentOf(EcsService service, String name, String cluster,+ String deploymentId) {+ String taskSetId = deploymentId.substring(deploymentId.indexOf('/') + 1);+ return service.listServiceDeploymentsDetailed(name, cluster, null, REGION).stream()+ .filter(d -> d.getTargetServiceRevisionArn() != null+ && d.getTargetServiceRevisionArn().endsWith("/" + taskSetId))+ .findFirst()+ .orElseThrow(() -> new AssertionError("no deployment for " + deploymentId));+ }++ /** The ARN of the deployment carrying {@code deploymentId}, selected the same way. */+ private static String deploymentArnOf(EcsService service, String name, String cluster,+ String deploymentId) {+ return deploymentOf(service, name, cluster, deploymentId).getServiceDeploymentArn();+ }++ private static ServiceDeployment onlyDeployment(EcsService service, String name, String cluster) {+ List<ServiceDeployment> deployments =+ service.listServiceDeploymentsDetailed(name, cluster, null, REGION);+ assertEquals(1, deployments.size(), "one create, one deployment");+ return deployments.getFirst();+ }++ private static List<EcsTask> runningTasks(EcsService service) {+ return service.describeTasks(null, service.listTasks(null, null, null, null, REGION), REGION)+ .stream().filter(t -> "RUNNING".equals(t.getLastStatus())).toList();+ }++ private static TaskDefinition registerTaskDef(EcsService service, String family, String image) {+ ContainerDefinition cd = new ContainerDefinition();+ cd.setName("app");+ cd.setImage(image);+ return service.registerTaskDefinition(family, List.of(cd), null, null, null,+ null, null, List.of(), REGION);+ }++ private static EcsService newMockModeService() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(true);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ EcsService service = new EcsService(+ new RegionResolver(REGION, "000000000000"),+ mock(EcsContainerManager.class),+ config,+ mock(EcsLoadBalancerRegistrar.class),+ new InMemoryStorageFactory(),+ null);+ service.initializeStorage();+ return service;+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();++ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ @SuppressWarnings("unchecked")+ public <V> AccountAwareStorageBackend<V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,+ ignored -> AccountAwareStorageBackend.inMemory("000000000000"));+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.javanew file mode 100644index 0000000000..d4cff003bf--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java@@ -0,0 +1,206 @@+package io.github.hectorvent.floci.services.ecs;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.Map;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasSize;+import static org.hamcrest.Matchers.notNullValue;+import static org.hamcrest.Matchers.nullValue;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * The join a steady-state wait walks, end to end.+ *+ * <p>Since provider 6.x the Terraform AWS provider no longer decides {@code wait_for_steady_state}+ * from {@code DescribeServices} alone. For a service under the ECS deployment controller whose+ * PRIMARY deployment started after the operation, it:+ *+ * <ol>+ * <li>reads {@code services[0].deployments[0].id}, which is {@code ecs-svc/<taskSetId>};</li>+ * <li>calls {@code ListServiceDeployments} and takes the brief whose+ * {@code targetServiceRevisionArn} <em>contains that {@code taskSetId}</em>;</li>+ * <li>polls {@code DescribeServiceDeployments} on that ARN until its status is+ * {@code SUCCESSFUL}.</li>+ * </ol>+ *+ * <p>Step 2 is a join AWS makes work by naming the service revision with the very id the+ * deployment reports. Floci minted the two independently, so the join matched nothing, the+ * provider never reached step 3, and every {@code wait_for_steady_state} apply sat on+ * {@code tfPENDING} for its full twenty-minute timeout against a service that was running and+ * reporting itself stable. Six of the first sixteen {@code terraform-aws-ecs} Terratest suites+ * failed that way in the 2026-09-28 sweep.+ */+@QuarkusTest+class EcsServiceSteadyStateWaiterIntegrationTest {++ private static final String TARGET = "AmazonEC2ContainerServiceV20141113.";+ private static final String CT = "application/x-amz-json-1.1";+ private static final String CLUSTER = "waiter-cluster";+ private static final String NETWORK = "\"networkConfiguration\":{\"awsvpcConfiguration\":"+ + "{\"subnets\":[\"subnet-1\"],\"securityGroups\":[\"sg-1\"]}}";++ @BeforeAll+ static void configure() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ private static Response call(String action, String body) {+ return given().contentType(CT).header("X-Amz-Target", TARGET + action)+ .body(body)+ .when().post("/")+ .then().statusCode(200)+ .extract().response();+ }++ /**+ * As {@link #seedService}, but at a task count the service has not reached yet, so its+ * deployment is in flight. Whatever is done with it must not read the service back: see+ * {@link #aStoppedDeploymentReportsWhenItStoppedAndWhy}.+ */+ private static String seedUnconvergedService(String name) {+ call("CreateCluster", "{\"clusterName\":\"" + CLUSTER + "\"}");+ call("RegisterTaskDefinition", "{\"family\":\"" + name + "-td\",\"networkMode\":\"awsvpc\","+ + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\",\"memory\":128}]}");+ call("CreateService", "{\"cluster\":\"" + CLUSTER + "\",\"serviceName\":\"" + name + "\","+ + "\"taskDefinition\":\"" + name + "-td\",\"desiredCount\":1,"+ + "\"launchType\":\"FARGATE\"," + NETWORK + "}");+ return name;+ }++ /** Creates the cluster, a task definition and a service, and returns the service name. */+ private static String seedService(String name) {+ call("CreateCluster", "{\"clusterName\":\"" + CLUSTER + "\"}");+ call("RegisterTaskDefinition", "{\"family\":\"" + name + "-td\",\"networkMode\":\"awsvpc\","+ + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\",\"memory\":128}]}");+ // desiredCount 0 is already converged, so the deployment is SUCCESSFUL without waiting on+ // the reconciler: this test is about the join, not about convergence.+ call("CreateService", "{\"cluster\":\"" + CLUSTER + "\",\"serviceName\":\"" + name + "\","+ + "\"taskDefinition\":\"" + name + "-td\",\"desiredCount\":0,"+ + "\"launchType\":\"FARGATE\"," + NETWORK + "}");+ return name;+ }++ /** The {@code <taskSetId>} half of the service's PRIMARY {@code ecs-svc/<taskSetId>}. */+ private static String primaryTaskSetId(String service) {+ String id = call("DescribeServices", "{\"cluster\":\"" + CLUSTER+ + "\",\"services\":[\"" + service + "\"]}")+ .jsonPath().getString("services[0].deployments[0].id");+ assertNotNull(id, "an ACTIVE service reports a PRIMARY deployment id");+ assertTrue(id.startsWith("ecs-svc/"), "deployment id is ecs-svc/<taskSetId>, was: " + id);+ return id.substring(id.indexOf('/') + 1);+ }++ @Test+ void theRevisionADeploymentTargetsIsNamedByThePrimaryDeploymentsTaskSetId() {+ String service = seedService("waiter-join-svc");+ String taskSetId = primaryTaskSetId(service);++ String targetRevision = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER+ + "\",\"service\":\"" + service + "\"}")+ .jsonPath().getString("serviceDeployments[0].targetServiceRevisionArn");++ // Verbatim the provider's predicate: strings.Contains(targetServiceRevisionArn, taskSetID).+ assertTrue(targetRevision != null && targetRevision.contains(taskSetId),+ "targetServiceRevisionArn must carry the PRIMARY deployment's task set id "+ + taskSetId + ", so a steady-state wait can find the deployment to poll; was: "+ + targetRevision);+ }++ /**+ * Named for what it checks. It walks list -> join -> describe on an already-converged+ * service, so it never observes a deployment mid-flight: {@code desiredCount} is 0, which is+ * converged on arrival. Observing IN_PROGRESS over the wire would mean racing the 5-second+ * reconciler inside a {@code @QuarkusTest}, so that direction is asserted deterministically+ * in {@code EcsServiceDeploymentStatusTest} instead, and the writer echoes the field verbatim+ * ({@code EcsResponseWriter.serviceDeploymentNode}).+ */+ @Test+ void theJoinedDeploymentIsDescribable() {+ String service = seedService("waiter-walk-svc");+ String taskSetId = primaryTaskSetId(service);++ Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER+ + "\",\"service\":\"" + service + "\"}");+ listed.then().body("serviceDeployments", hasSize(1));++ String deploymentArn = null;+ for (Object brief : listed.jsonPath().getList("serviceDeployments")) {+ @SuppressWarnings("unchecked")+ Map<String, Object> map = (Map<String, Object>) brief;+ Object revision = map.get("targetServiceRevisionArn");+ if (revision != null && revision.toString().contains(taskSetId)) {+ deploymentArn = String.valueOf(map.get("serviceDeploymentArn"));+ }+ }+ assertNotNull(deploymentArn, "the listing must contain the PRIMARY deployment");++ // A service at its requested task count, desiredCount 0, is a finished deployment.+ call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")+ .then()+ .body("serviceDeployments", hasSize(1))+ .body("serviceDeployments[0].status", equalTo("SUCCESSFUL"))+ // A deployment that succeeded was never stopped, so AWS reports neither field and+ // nor do we. Absent, not null: a client reading stoppedAt to learn when a+ // deployment ended must not be handed a value for one that did not stop.+ .body("serviceDeployments[0].stoppedAt", nullValue())+ .body("serviceDeployments[0].statusReason", nullValue());+ }++ /**+ * The stopped fields over the wire. A deployment that ends without completing carries+ * {@code stoppedAt} and {@code statusReason}, both of which are members of AWS's+ * {@code ServiceDeployment} shape and neither of which floci used to write at all, so a+ * client asking when a stopped deployment ended, or why, got nothing.+ *+ * <p>The deployment has to still be in flight when the delete arrives, and a test cannot+ * promise that: the 5-second reconciler runs on its own thread, and a tick that lands between+ * CreateService and DeleteService starts the task, so the delete correctly settles the+ * deployment SUCCESSFUL instead. That is a lost race, not a failure, so the test tries again+ * on a fresh service, a few times; the window is milliseconds against a 5000ms period. It+ * fails only if no attempt produced a stopped deployment, or one did and its fields are wrong.+ * The behaviour itself is pinned without any timing in+ * {@code EcsServiceDeploymentStatusTest.deletingAServiceStopsTheDeploymentItLeavesUnfinished};+ * what only this test can show is that the two fields reach the wire.+ */+ @Test+ void aStoppedDeploymentReportsWhenItStoppedAndWhy() {+ for (int attempt = 1; attempt <= 5; attempt++) {+ String service = seedUnconvergedService("waiter-stop-svc-" + attempt);++ call("DeleteService", "{\"cluster\":\"" + CLUSTER + "\",\"service\":\"" + service+ + "\",\"force\":true}");++ // Identified after the delete: the service was created once and never updated, so it+ // has exactly one deployment, and a terminal record cannot change under a read.+ Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER+ + "\",\"service\":\"" + service + "\"}");+ listed.then().body("serviceDeployments", hasSize(1));+ String deploymentArn = listed.jsonPath()+ .getString("serviceDeployments[0].serviceDeploymentArn");+ assertNotNull(deploymentArn, "the deleted service's deployment is still describable");++ Response described = call("DescribeServiceDeployments",+ "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}");+ if ("SUCCESSFUL".equals(described.jsonPath().getString("serviceDeployments[0].status"))) {+ continue;+ }+ described.then()+ .body("serviceDeployments", hasSize(1))+ .body("serviceDeployments[0].status", equalTo("STOPPED"))+ .body("serviceDeployments[0].stoppedAt", notNullValue())+ .body("serviceDeployments[0].finishedAt", notNullValue())+ .body("serviceDeployments[0].statusReason", equalTo("The service was deleted."));+ return;+ }+ fail("the reconciler converged the service before every one of five deletes");+ }+}