Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): a service reaches steady state, and a finished deployment stays finished (#4694)

Allen Sanbornc6b8a70ea8dd
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java
index 59c0bdebb0..148b8e689c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java
@@ -955,7 +955,11 @@ public class EcsResponseWriter {
putInstant(n, "createdAt", d.getCreatedAt());
putInstant(n, "startedAt", d.getStartedAt());
putInstant(n, "finishedAt", d.getFinishedAt());
+ putInstant(n, "stoppedAt", d.getStoppedAt());
putInstant(n, "updatedAt", d.getUpdatedAt());
+ if (d.getStatusReason() != null) {
+ n.put("statusReason", d.getStatusReason());
+ }
if (d.getTargetServiceRevisionArn() != null) {
n.set("targetServiceRevision",
serviceRevisionSummaryNode(d.getTargetServiceRevisionArn(), d.getServiceArn()));
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index d35205d55b..5de43acdbb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -159,6 +159,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
public static final String PROPAGATE_TAGS_SERVICE = "SERVICE";
public static final String PROPAGATE_TAGS_TASK_DEFINITION = "TASK_DEFINITION";
public static final String PROPAGATE_TAGS_NONE = "NONE";
+ /** A service deployment whose target revision's tasks are not all up yet. */
+ public static final String DEPLOYMENT_STATUS_IN_PROGRESS = "IN_PROGRESS";
+ /** A service deployment whose target revision is running at its requested task count. */
+ public static final String DEPLOYMENT_STATUS_SUCCESSFUL = "SUCCESSFUL";
+ /** A service deployment that ended without completing. */
+ public static final String DEPLOYMENT_STATUS_STOPPED = "STOPPED";
/** RunTask places at most ten tasks in one call, and StartTask at most ten instances. */
public static final int MAX_TASKS_PER_RUN = 10;
/** A listing returns at most a hundred ARNs per page. */
@@ -2473,6 +2479,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
throw new AwsException("ServiceNotActiveException",
"Service " + serviceName + " is not active.", 400);
}
+ // Settle the outgoing deployment against the service as it stands, before this request
+ // changes its desired count or task definition: one whose tasks are up has finished,
+ // whether or not anything read it. recordServiceDeployment stops it if it has not.
+ currentServiceDeployment(svc);
if (request.getDesiredCount() != null) {
if (request.getDesiredCount() < 0) {
throw new AwsException("InvalidParameterException", "desiredCount cannot be a negative number.", 400);
@@ -2718,7 +2728,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
throw new AwsException("InvalidParameterException",
"The service cannot be stopped. Update the service to 0 tasks or use the force flag.", 400);
}
+ // Settle against the service as it stands, as UpdateService does: a deployment whose
+ // tasks are up has finished, whether or not anything read it.
+ currentServiceDeployment(svc);
svc.setStatus("INACTIVE");
+ // A deleted service's deployments can never finish, so stop them. This must come before
+ // setDesiredCount(0): a concurrent read that saw desiredCount 0 on a still-ACTIVE service
+ // would otherwise mark the deployment SUCCESSFUL. AWS deletes the records instead; that
+ // is not done here yet.
+ stopNonTerminalDeployments(svc.getServiceArn(), Instant.now(), "The service was deleted.");
svc.setDesiredCount(0);
cluster.setActiveServicesCount(Math.max(0, cluster.getActiveServicesCount() - 1));
services.put(key, svc);
@@ -3813,6 +3831,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
return deploymentArns.stream()
.map(arn -> serviceDeployments.get(arn))
.filter(d -> d != null)
+ .map(this::settleStatus)
.toList();
}
@@ -3829,6 +3848,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
return serviceDeployments.values().stream()
.filter(d -> d.getServiceArn().equals(svc.getServiceArn()))
+ .map(this::settleStatus)
.filter(d -> statusFilter == null || statusFilter.isEmpty()
|| statusFilter.contains(d.getStatus()))
.sorted((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()))
@@ -3859,7 +3879,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
String deploymentId = deploymentId(svc);
- boolean converged = svc.getRunningCount() >= svc.getDesiredCount();
+ // This deployment's own tasks: counting the ones still draining from the previous
+ // deployment would report a task-definition change COMPLETED as soon as it was made.
+ long running = runningOnCurrentDeployment(svc);
+ boolean converged = running >= svc.getDesiredCount();
Deployment d = new Deployment();
d.setId(deploymentId);
@@ -3867,7 +3890,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
d.setTaskDefinition(svc.getTaskDefinition());
d.setDesiredCount(svc.getDesiredCount());
d.setPendingCount(svc.getPendingCount());
- d.setRunningCount(svc.getRunningCount());
+ d.setRunningCount((int) running);
d.setFailedTasks(0);
d.setRolloutState(converged ? "COMPLETED" : "IN_PROGRESS");
d.setRolloutStateReason("ECS deployment " + deploymentId
@@ -3906,7 +3929,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
*/
public List<ServiceEvent> eventsFor(EcsServiceModel svc) {
if (svc == null || !"ACTIVE".equals(svc.getStatus())
- || svc.getRunningCount() < svc.getDesiredCount()) {
+ || runningOnCurrentDeployment(svc) < svc.getDesiredCount()) {
return List.of();
}
String deploymentId = deploymentId(svc);
@@ -3943,22 +3966,62 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
return "ecs-svc/" + Long.toUnsignedString(hash);
}
+ /**
+ * How many of the service's RUNNING tasks belong to the deployment it is currently on.
+ * Not {@code svc.getRunningCount()}, which also counts tasks still draining from the
+ * previous deployment.
+ */
+ private long runningOnCurrentDeployment(EcsServiceModel svc) {
+ EcsCluster cluster = resolveClusterByArn(svc.getClusterArn());
+ if (cluster == null) {
+ return 0;
+ }
+ Stream<EcsTask> running = tasks.values().stream()
+ .filter(t -> ownedBy(t, svc, cluster))
+ .filter(t -> TaskStatus.RUNNING.name().equals(t.getLastStatus()));
+
+ // reconcileDaemonService does not replace a DAEMON service's tasks when its task
+ // definition changes, so the stale ones are the only ones there will ever be; counting
+ // them keeps main's behaviour. Scope this like REPLICA once DAEMON services roll.
+ if (SCHEDULING_DAEMON.equals(svc.getSchedulingStrategy())) {
+ return running.count();
+ }
+
+ String currentDeploymentId = deploymentId(svc);
+ String taskDefinitionArn = resolvedTaskDefinitionArn(svc, serviceRegion(svc));
+ return running
+ .filter(t -> !isStaleForDeployment(t, currentDeploymentId, taskDefinitionArn))
+ .count();
+ }
+
+ /**
+ * The bare identifier inside the service's {@code ecs-svc/<id>} deployment id: what AWS
+ * calls the task set id, and what names the service revision that deployment targets.
+ */
+ private String taskSetId(EcsServiceModel svc) {
+ String id = deploymentId(svc);
+ int slash = id.indexOf('/');
+ return slash < 0 ? id : id.substring(slash + 1);
+ }
+
/**
* Records the deployment and the revision it targets. The revision is the snapshot of the
* service's configuration, so it is copied out of the service rather than read back from it
* later; the deployment links to it, which is how a caller gets from
* {@code DescribeServiceDeployments} to what was actually deployed.
*
- * <p>Floci applies a change in place instead of rolling it, so a deployment is finished the
- * moment it is recorded: {@code startedAt} and {@code finishedAt} are both its creation time.
+ * <p>The revision ARN ends in the deployment's task set id, the {@code <id>} of
+ * {@code ecs-svc/<id>}, as AWS's does. The Terraform AWS provider's
+ * {@code wait_for_steady_state} finds the deployment to poll by looking for that id in each
+ * deployment's {@code targetServiceRevisionArn}. The ARN built here is
+ * {@code service-revision/<id>}; AWS's is {@code service-revision/<cluster>/<service>/<id>}.
*/
private void recordServiceDeployment(EcsServiceModel svc, String taskDefinition, String region) {
String deploymentId = UUID.randomUUID().toString().replace("-", "");
String deploymentArn = regionResolver.buildArn("ecs", region,
"service-deployment/" + deploymentId);
- String revisionId = UUID.randomUUID().toString().replace("-", "");
String revisionArn = regionResolver.buildArn("ecs", region,
- "service-revision/" + revisionId);
+ "service-revision/" + taskSetId(svc));
Instant now = Instant.now();
List<String> priorRevisions = serviceRevisions.values().stream()
@@ -3974,13 +4037,14 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
deployment.setServiceArn(svc.getServiceArn());
deployment.setClusterArn(svc.getClusterArn());
deployment.setTaskDefinition(taskDefinition);
- deployment.setStatus("SUCCESSFUL");
+ deployment.setStatus(DEPLOYMENT_STATUS_IN_PROGRESS);
deployment.setCreatedAt(now);
deployment.setStartedAt(now);
- deployment.setFinishedAt(now);
deployment.setUpdatedAt(now);
deployment.setTargetServiceRevisionArn(revisionArn);
deployment.setSourceServiceRevisionArns(priorRevisions);
+ stopNonTerminalDeployments(svc.getServiceArn(), now,
+ "Superseded by service deployment " + deploymentArn + ".");
serviceDeployments.put(deploymentArn, deployment);
ServiceRevision revision = new ServiceRevision();
@@ -4001,6 +4065,75 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
serviceRevisions.put(revisionArn, revision);
}
+ /**
+ * Marks the service's IN_PROGRESS deployments STOPPED, for a caller that has made them
+ * unable to finish: a new deployment superseding them, or the service being deleted. Runs
+ * before the new deployment is added to the map, so it never stops the current one.
+ */
+ private void stopNonTerminalDeployments(String serviceArn, Instant now, String reason) {
+ // A service model restored from the store may have no ARN.
+ if (serviceArn == null) {
+ return;
+ }
+ for (ServiceDeployment prior : serviceDeployments.values()) {
+ if (!serviceArn.equals(prior.getServiceArn())) {
+ continue;
+ }
+ synchronized (prior) {
+ if (DEPLOYMENT_STATUS_IN_PROGRESS.equals(prior.getStatus())) {
+ prior.setStatus(DEPLOYMENT_STATUS_STOPPED);
+ prior.setStatusReason(reason);
+ prior.setFinishedAt(now);
+ prior.setStoppedAt(now);
+ prior.setUpdatedAt(now);
+ }
+ }
+ }
+ }
+
+ /**
+ * Marks the service's current deployment SUCCESSFUL once its own tasks are all running.
+ * Only an IN_PROGRESS deployment moves, so a finished one never changes again.
+ */
+ private ServiceDeployment settleStatus(ServiceDeployment deployment) {
+ EcsServiceModel svc = serviceByArn(deployment.getServiceArn());
+ synchronized (deployment) {
+ if (svc != null && deployment.getTargetServiceRevisionArn() != null
+ && deployment.getTargetServiceRevisionArn().endsWith("/" + taskSetId(svc))
+ && DEPLOYMENT_STATUS_IN_PROGRESS.equals(deployment.getStatus())
+ && STATUS_ACTIVE.equals(svc.getStatus())
+ && runningOnCurrentDeployment(svc) >= svc.getDesiredCount()) {
+ Instant now = Instant.now();
+ deployment.setStatus(DEPLOYMENT_STATUS_SUCCESSFUL);
+ deployment.setFinishedAt(now);
+ deployment.setUpdatedAt(now);
+ }
+ return copyOf(deployment);
+ }
+ }
+
+ /**
+ * A copy taken under the deployment's lock, so a response never sees a half-applied
+ * transition.
+ */
+ private static ServiceDeployment copyOf(ServiceDeployment source) {
+ ServiceDeployment copy = new ServiceDeployment();
+ copy.setServiceDeploymentArn(source.getServiceDeploymentArn());
+ copy.setServiceArn(source.getServiceArn());
+ copy.setClusterArn(source.getClusterArn());
+ copy.setTaskDefinition(source.getTaskDefinition());
+ copy.setStatus(source.getStatus());
+ copy.setCreatedAt(source.getCreatedAt());
+ copy.setStartedAt(source.getStartedAt());
+ copy.setFinishedAt(source.getFinishedAt());
+ copy.setStoppedAt(source.getStoppedAt());
+ copy.setStatusReason(source.getStatusReason());
+ copy.setUpdatedAt(source.getUpdatedAt());
+ copy.setTargetServiceRevisionArn(source.getTargetServiceRevisionArn());
+ copy.setSourceServiceRevisionArns(source.getSourceServiceRevisionArns());
+ return copy;
+ }
+
/** The images the revision's task definition pins, one entry per container definition. */
private List<ContainerImage> containerImagesOf(String taskDefinitionRef, String region) {
TaskDefinition taskDef;
@@ -4043,6 +4176,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
.filter(d -> svc.getServiceArn().equals(d.getServiceArn()))
.max(Comparator.comparing(ServiceDeployment::getCreatedAt,
Comparator.nullsFirst(Comparator.naturalOrder())))
+ .map(this::settleStatus)
.orElse(null);
}
@@ -4271,6 +4405,21 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
}
+ /** The region a service lives in, read off its ARN, falling back to the default region. */
+ private String serviceRegion(EcsServiceModel svc) {
+ if (svc.getServiceArn() != null) {
+ try {
+ String region = AwsArnUtils.parse(svc.getServiceArn()).region();
+ if (region != null && !region.isBlank()) {
+ return region;
+ }
+ } catch (Exception e) {
+ LOG.debugv("Could not parse region from service ARN {0}", svc.getServiceArn());
+ }
+ }
+ return regionResolver.getDefaultRegion();
+ }
+
/** The region a task lives in, read off its ARN, falling back to the default region. */
private String taskRegion(EcsTask task) {
if (task.getTaskArn() != null) {
@@ -4438,15 +4587,29 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
* simply matches nothing, which is the pre-existing behaviour for an unresolvable service.
*/
private String pinnedTaskDefinitionArn(EcsServiceModel svc, String key, String region) {
+ String ref = svc.getTaskDefinition();
+ String arn = resolvedTaskDefinitionArn(svc, region);
+ if (arn != null && !arn.equals(ref)) {
+ svc.setTaskDefinition(arn);
+ services.put(key, svc);
+ }
+ return arn;
+ }
+
+ /**
+ * The service's task definition as an ARN, resolving a {@code family} or
+ * {@code family:revision} reference without storing the result.
+ *
+ * <p>Shared with {@link #pinnedTaskDefinitionArn} so that the read path and the reconciler
+ * decide staleness the same way.
+ */
+ private String resolvedTaskDefinitionArn(EcsServiceModel svc, String region) {
String ref = svc.getTaskDefinition();
if (ref == null || ref.startsWith("arn:")) {
return ref;
}
try {
- String arn = resolveTaskDefinitionOrThrow(ref, region).getTaskDefinitionArn();
- svc.setTaskDefinition(arn);
- services.put(key, svc);
- return arn;
+ return resolveTaskDefinitionOrThrow(ref, region).getTaskDefinitionArn();
} catch (AwsException e) {
return ref;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java b/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java
index 92bd833162..d689d1cb3f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java
@@ -21,6 +21,10 @@ public class ServiceDeployment {
private Instant createdAt;
private Instant startedAt;
private Instant finishedAt;
+ /** When a deployment that ended without completing was stopped. */
+ private Instant stoppedAt;
+ /** Why the deployment is in its status; set when it is stopped. */
+ private String statusReason;
private Instant updatedAt;
private String targetServiceRevisionArn;
private List<String> sourceServiceRevisionArns;
@@ -31,6 +35,12 @@ public class ServiceDeployment {
public Instant getFinishedAt() { return finishedAt; }
public void setFinishedAt(Instant finishedAt) { this.finishedAt = finishedAt; }
+ public Instant getStoppedAt() { return stoppedAt; }
+ public void setStoppedAt(Instant stoppedAt) { this.stoppedAt = stoppedAt; }
+
+ public String getStatusReason() { return statusReason; }
+ public void setStatusReason(String statusReason) { this.statusReason = statusReason; }
+
public String getTargetServiceRevisionArn() { return targetServiceRevisionArn; }
public void setTargetServiceRevisionArn(String targetServiceRevisionArn) {
this.targetServiceRevisionArn = targetServiceRevisionArn;
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java
new file mode 100644
index 0000000000..4eff9cdef0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java
@@ -0,0 +1,594 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.ServiceDeployment;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * What a service deployment reports, driven off the reconciler tick by tick. Deployments are
+ * selected by task set id rather than by position in a listing: several minted in one clock tick
+ * share a createdAt, so their order in the listing is map iteration order.
+ */
+class EcsServiceDeploymentStatusTest {
+
+ private static final String REGION = "us-east-1";
+
+ @Test
+ void theCurrentDeploymentIsInProgressUntilTheServicesTasksAreRunning() {
+ EcsService service = newMockModeService();
+ service.createCluster("dstat-cluster", REGION);
+ registerTaskDef(service, "dstat-fam", "app:1");
+ service.createService("dstat-cluster", "dstat-svc", "dstat-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+
+ // Created, nothing launched yet: runningCount 0 of a requested 1.
+ ServiceDeployment pending = onlyDeployment(service, "dstat-svc", "dstat-cluster");
+ assertEquals("IN_PROGRESS", pending.getStatus());
+ assertNull(pending.getFinishedAt(), "an unfinished deployment has no finishedAt");
+
+ // One tick: the count is taken from the tasks at read time, not from svc.runningCount,
+ // which lags a tick behind.
+ service.reconcileServices();
+
+ ServiceDeployment done = onlyDeployment(service, "dstat-svc", "dstat-cluster");
+ assertEquals("SUCCESSFUL", done.getStatus());
+ assertNotNull(done.getFinishedAt(), "a finished deployment reports when it finished");
+ }
+
+ /** A service already at its requested count has converged, so its deployment is finished. */
+ @Test
+ void aServiceAlreadyAtItsRequestedCountHasNothingToWaitFor() {
+ EcsService service = newMockModeService();
+ service.createCluster("dzero-cluster", REGION);
+ registerTaskDef(service, "dzero-fam", "app:1");
+ service.createService("dzero-cluster", "dzero-svc", "dzero-fam", 0,
+ LaunchType.FARGATE, List.of(), null, REGION);
+
+ ServiceDeployment deployment = onlyDeployment(service, "dzero-svc", "dzero-cluster");
+ assertEquals("SUCCESSFUL", deployment.getStatus());
+ assertNotNull(deployment.getFinishedAt());
+ }
+
+ @Test
+ void everyDeploymentTargetsARevisionNamedByItsOwnTaskSetId() {
+ EcsService service = newMockModeService();
+ service.createCluster("drev-cluster", REGION);
+ registerTaskDef(service, "drev-fam", "app:1");
+ EcsServiceModel created = service.createService("drev-cluster", "drev-svc", "drev-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+ // Read out now: createService and updateService hand back the same live model object.
+ String firstDeploymentId = created.getDeploymentId();
+ assertRevisionCarriesTaskSetId(service, "drev-svc", "drev-cluster", firstDeploymentId);
+
+ service.reconcileServices();
+ String secondDeploymentId = service.updateService("drev-cluster", "drev-svc", null, null,
+ null, null, true, REGION).getDeploymentId();
+ assertNotEquals(firstDeploymentId, secondDeploymentId,
+ "forceNewDeployment mints a new deployment id");
+ assertRevisionCarriesTaskSetId(service, "drev-svc", "drev-cluster", secondDeploymentId);
+
+ // The superseded deployment keeps pointing at its own revision, so the two never collide.
+ List<ServiceDeployment> all = service.listServiceDeploymentsDetailed("drev-svc",
+ "drev-cluster", null, REGION);
+ assertEquals(2, all.size());
+ assertEquals(2, all.stream().map(ServiceDeployment::getTargetServiceRevisionArn)
+ .distinct().count(), "each deployment targets its own revision");
+ }
+
+ /**
+ * A task-definition change is judged on the new deployment's own tasks, not on the old
+ * revision's task that is still running.
+ */
+ @Test
+ void aTaskDefinitionChangeIsNotFinishedWhileOnlyTheOldTasksAreRunning() {
+ EcsService service = newMockModeService();
+ service.createCluster("dupd-cluster", REGION);
+ registerTaskDef(service, "dupd-fam", "app:1");
+ EcsServiceModel created = service.createService("dupd-cluster", "dupd-svc", "dupd-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+ String firstDeploymentId = created.getDeploymentId();
+ service.reconcileServices();
+ assertEquals("SUCCESSFUL", deploymentOf(service, "dupd-svc", "dupd-cluster",
+ firstDeploymentId).getStatus(),
+ "precondition: the first deployment converged");
+
+ TaskDefinition rev2 = registerTaskDef(service, "dupd-fam", "app:2");
+ String rolled = service.updateService("dupd-cluster", "dupd-svc",
+ "dupd-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();
+
+ // No tick yet. The only RUNNING task belongs to the superseded deployment, so the new
+ // deployment has nothing of its own up.
+ ServiceDeployment rolling = deploymentOf(service, "dupd-svc", "dupd-cluster", rolled);
+ assertEquals("IN_PROGRESS", rolling.getStatus(),
+ "a deployment with none of its own tasks running is not finished");
+ assertNull(rolling.getFinishedAt(), "and carries no finishedAt");
+
+ // One tick launches the new revision's task; now the deployment really has converged,
+ // even though the old task is still draining beside it.
+ service.reconcileServices();
+ ServiceDeployment settled = deploymentOf(service, "dupd-svc", "dupd-cluster", rolled);
+ assertEquals("SUCCESSFUL", settled.getStatus(),
+ "its own task is running, so it is finished even while the old one drains");
+ assertNotNull(settled.getFinishedAt());
+ }
+
+ /** SUCCESSFUL is terminal: tasks dying afterwards do not un-finish the deployment. */
+ @Test
+ void aFinishedDeploymentStaysFinishedWhenItsTasksDie() {
+ EcsService service = newMockModeService();
+ service.createCluster("ddie-cluster", REGION);
+ registerTaskDef(service, "ddie-fam", "app:1");
+ String deploymentId = service.createService("ddie-cluster", "ddie-svc", "ddie-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+ service.reconcileServices();
+ ServiceDeployment finished = deploymentOf(service, "ddie-svc", "ddie-cluster", deploymentId);
+ assertEquals("SUCCESSFUL", finished.getStatus());
+ Instant finishedAt = finished.getFinishedAt();
+ assertNotNull(finishedAt);
+
+ // Read again while still converged: finishedAt is stamped once, not on every read.
+ assertEquals(finishedAt,
+ deploymentOf(service, "ddie-svc", "ddie-cluster", deploymentId).getFinishedAt(),
+ "finishedAt is stamped once, not re-stamped on every read while converged");
+
+ String taskArn = runningTasks(service).getFirst().getTaskArn();
+ service.stopTask("ddie-cluster", taskArn, "test kill", REGION);
+ assertEquals(0, runningTasks(service).size(), "precondition: nothing of it is running");
+
+ ServiceDeployment after = deploymentOf(service, "ddie-svc", "ddie-cluster", deploymentId);
+ assertEquals("SUCCESSFUL", after.getStatus(), "a completed deployment does not un-complete");
+ assertEquals(finishedAt, after.getFinishedAt(), "and keeps the instant it finished at");
+ }
+
+ /**
+ * Scaling to zero is converged while the task is still draining, since {@code >=} is used.
+ * AWS's own waiter compares with {@code ==} and would wait for the task to stop.
+ */
+ @Test
+ void scalingToZeroIsConvergedBeforeTheTaskHasActuallyStopped() {
+ EcsService service = newMockModeService();
+ service.createCluster("dscale-cluster", REGION);
+ registerTaskDef(service, "dscale-fam", "app:1");
+ String id = service.createService("dscale-cluster", "dscale-svc", "dscale-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+ service.reconcileServices();
+
+ service.updateService("dscale-cluster", "dscale-svc", null, 0, null, REGION);
+ assertEquals(1, runningTasks(service).size(), "the task has not been drained yet");
+ assertEquals("SUCCESSFUL", deploymentOf(service, "dscale-svc", "dscale-cluster", id).getStatus());
+
+ service.reconcileServices();
+ assertEquals(0, runningTasks(service).size(), "and now it is drained");
+ assertEquals("SUCCESSFUL", deploymentOf(service, "dscale-svc", "dscale-cluster", id).getStatus());
+ }
+
+ /**
+ * Two task-definition changes in a row with no read between them. The first deployment had
+ * converged, so the update that supersedes it settles it SUCCESSFUL; the second never had a
+ * task of its own, so the next update stops it.
+ */
+ @Test
+ void twoUpdatesInSuccessionSettleOnlyTheNewestDeployment() {
+ EcsService service = newMockModeService();
+ service.createCluster("drapid-cluster", REGION);
+ registerTaskDef(service, "drapid-fam", "app:1");
+ // createService and updateService return the same live model, so read each id at once.
+ String first = service.createService("drapid-cluster", "drapid-svc", "drapid-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+ service.reconcileServices();
+
+ TaskDefinition rev2 = registerTaskDef(service, "drapid-fam", "app:2");
+ String second = service.updateService("drapid-cluster", "drapid-svc",
+ "drapid-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();
+ TaskDefinition rev3 = registerTaskDef(service, "drapid-fam", "app:3");
+ String newest = service.updateService("drapid-cluster", "drapid-svc",
+ "drapid-fam:" + rev3.getRevision(), null, null, REGION).getDeploymentId();
+
+ assertEquals(3, service.listServiceDeploymentsDetailed("drapid-svc", "drapid-cluster",
+ null, REGION).size(), "one per create/update");
+ assertEquals("IN_PROGRESS",
+ deploymentOf(service, "drapid-svc", "drapid-cluster", newest).getStatus(),
+ "the newest has none of its own tasks running");
+ assertEquals("SUCCESSFUL",
+ deploymentOf(service, "drapid-svc", "drapid-cluster", first).getStatus(),
+ "the first had converged before it was superseded, though nothing read it");
+ assertEquals("STOPPED",
+ deploymentOf(service, "drapid-svc", "drapid-cluster", second).getStatus(),
+ "the second never had a task of its own");
+
+ service.reconcileServices();
+ assertEquals("SUCCESSFUL",
+ deploymentOf(service, "drapid-svc", "drapid-cluster", newest).getStatus(),
+ "and converges once its own task is up");
+ }
+
+ /**
+ * Right after a task-definition change, {@code rolloutState} and the deployment record
+ * describe the same moment and must agree. Later they may differ: the record is history,
+ * rolloutState follows the live service.
+ */
+ @Test
+ void theRolloutStateAndTheDeploymentRecordAgreeWhileARolloutIsInFlight() {
+ EcsService service = newMockModeService();
+ service.createCluster("dboth-cluster", REGION);
+ registerTaskDef(service, "dboth-fam", "app:1");
+ EcsServiceModel created = service.createService("dboth-cluster", "dboth-svc", "dboth-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+ // Two ticks: the first launches the task, the second counts it in svc.runningCount.
+ // With only one, the service-wide count is 0 and the old miscount passes by accident.
+ service.reconcileServices();
+ service.reconcileServices();
+ assertEquals(1, created.getRunningCount(),
+ "precondition: a genuine steady state, not one tick short of it");
+
+ TaskDefinition rev2 = registerTaskDef(service, "dboth-fam", "app:2");
+ String rolled = service.updateService("dboth-cluster", "dboth-svc",
+ "dboth-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();
+
+ EcsServiceModel svc = service.serviceByArn(
+ service.describeServices("dboth-cluster", List.of("dboth-svc"), REGION)
+ .getFirst().getServiceArn());
+ assertEquals("IN_PROGRESS", service.deploymentsFor(svc).getFirst().getRolloutState(),
+ "the live rollout has not started: only the old revision's task is up");
+ assertEquals("IN_PROGRESS",
+ deploymentOf(service, "dboth-svc", "dboth-cluster", rolled).getStatus(),
+ "and the record says the same about the same moment");
+ assertTrue(service.eventsFor(svc).isEmpty(),
+ "no steady-state event while the rollout is in flight");
+
+ service.reconcileServices();
+ assertEquals("COMPLETED", service.deploymentsFor(svc).getFirst().getRolloutState());
+ assertEquals("SUCCESSFUL",
+ deploymentOf(service, "dboth-svc", "dboth-cluster", rolled).getStatus());
+ }
+
+ /**
+ * A DAEMON service still finishes its deployment after a task-definition change. DAEMON
+ * services do not roll yet, so the old revision's task is still the one running; the last
+ * assertions fail once they do, which is when the DAEMON case in the count should go.
+ */
+ @Test
+ void aDaemonServiceStillFinishesItsDeploymentAfterATaskDefinitionChange() {
+ EcsService service = newMockModeService();
+ service.createCluster("ddmn-cluster", REGION);
+ service.registerContainerInstance("ddmn-cluster", null, List.of(), REGION);
+ TaskDefinition rev1 = registerTaskDef(service, "ddmn-fam", "app:1");
+ service.createService("ddmn-cluster", "ddmn-svc", "ddmn-fam", 1, LaunchType.EC2,
+ List.of(), null, null, "DAEMON", null, null, REGION);
+ service.reconcileServices();
+ service.reconcileServices();
+
+ TaskDefinition rev2 = registerTaskDef(service, "ddmn-fam", "app:2");
+ String rolled = service.updateService("ddmn-cluster", "ddmn-svc",
+ "ddmn-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();
+ service.reconcileServices();
+ service.reconcileServices();
+
+ ServiceDeployment record = deploymentOf(service, "ddmn-svc", "ddmn-cluster", rolled);
+ assertEquals("SUCCESSFUL", record.getStatus(),
+ "a DAEMON deployment must still finish, or a steady-state wait hangs for ever");
+ assertNotNull(record.getFinishedAt());
+
+ List<EcsTask> live = runningTasks(service);
+ assertEquals(1, live.size(), "one task per container instance");
+ assertEquals(rev1.getTaskDefinitionArn(), live.getFirst().getTaskDefinitionArn(),
+ "DAEMON does not roll on a task-definition change; the count is honest about that");
+ assertNotEquals(rev2.getTaskDefinitionArn(), live.getFirst().getTaskDefinitionArn());
+ }
+
+ /**
+ * A deployment superseded before it converged is STOPPED, not left IN_PROGRESS. The second
+ * service is never updated, so its deployment must stay in flight.
+ */
+ @Test
+ void supersedingAnUnconvergedDeploymentStopsIt() {
+ EcsService service = newMockModeService();
+ service.createCluster("dsup-cluster", REGION);
+ registerTaskDef(service, "dsup-fam", "app:1");
+ String first = service.createService("dsup-cluster", "dsup-svc", "dsup-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+ registerTaskDef(service, "dsup-other-fam", "other:1");
+ String bystander = service.createService("dsup-cluster", "dsup-other", "dsup-other-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+
+ // No tick: neither service has launched anything, so both deployments are in flight.
+ assertEquals("IN_PROGRESS",
+ deploymentOf(service, "dsup-svc", "dsup-cluster", first).getStatus(),
+ "precondition: the first deployment never converged");
+
+ TaskDefinition rev2 = registerTaskDef(service, "dsup-fam", "app:2");
+ String second = service.updateService("dsup-cluster", "dsup-svc",
+ "dsup-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();
+
+ ServiceDeployment superseded = deploymentOf(service, "dsup-svc", "dsup-cluster", first);
+ assertEquals("STOPPED", superseded.getStatus(),
+ "a deployment another one took over from is finished, not still rolling out");
+ assertNotNull(superseded.getFinishedAt(),
+ "and reports when it ended, like every deployment that is no longer running");
+ assertNotNull(superseded.getStoppedAt(), "and when it was stopped");
+ assertTrue(superseded.getStatusReason() != null
+ && superseded.getStatusReason().contains(deploymentArnOf(service,
+ "dsup-svc", "dsup-cluster", second)),
+ "the reason names the deployment that took over, which is what a reader needs "
+ + "and the status alone does not say; was: " + superseded.getStatusReason());
+
+ assertEquals("IN_PROGRESS",
+ deploymentOf(service, "dsup-svc", "dsup-cluster", second).getStatus(),
+ "while the deployment that took over is the one now in flight");
+ assertEquals("IN_PROGRESS",
+ deploymentOf(service, "dsup-other", "dsup-cluster", bystander).getStatus(),
+ "another service's deployment is none of this update's business");
+ }
+
+ /** STOPPED is terminal: the replacement converging does not make the stopped one succeed. */
+ @Test
+ void aSupersededDeploymentDoesNotSucceedWhenItsReplacementDoes() {
+ EcsService service = newMockModeService();
+ service.createCluster("dsup2-cluster", REGION);
+ registerTaskDef(service, "dsup2-fam", "app:1");
+ String first = service.createService("dsup2-cluster", "dsup2-svc", "dsup2-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+
+ TaskDefinition rev2 = registerTaskDef(service, "dsup2-fam", "app:2");
+ String second = service.updateService("dsup2-cluster", "dsup2-svc",
+ "dsup2-fam:" + rev2.getRevision(), null, null, REGION).getDeploymentId();
+ Instant stoppedAt = deploymentOf(service, "dsup2-svc", "dsup2-cluster", first).getFinishedAt();
+ assertNotNull(stoppedAt, "precondition: the superseded deployment ended");
+
+ service.reconcileServices();
+ assertEquals("SUCCESSFUL",
+ deploymentOf(service, "dsup2-svc", "dsup2-cluster", second).getStatus(),
+ "precondition: the replacement converged");
+
+ ServiceDeployment superseded = deploymentOf(service, "dsup2-svc", "dsup2-cluster", first);
+ assertEquals("STOPPED", superseded.getStatus(),
+ "the deployment that was taken over from did not succeed, its replacement did");
+ assertEquals(stoppedAt, superseded.getFinishedAt(),
+ "and still reports the instant it ended");
+ }
+
+ /** A deployment that finished, and was read, before it was superseded keeps its record. */
+ @Test
+ void aDeploymentThatFinishedBeforeItWasSupersededStaysSuccessful() {
+ EcsService service = newMockModeService();
+ service.createCluster("dsup3-cluster", REGION);
+ registerTaskDef(service, "dsup3-fam", "app:1");
+ String first = service.createService("dsup3-cluster", "dsup3-svc", "dsup3-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+ service.reconcileServices();
+
+ ServiceDeployment finished = deploymentOf(service, "dsup3-svc", "dsup3-cluster", first);
+ assertEquals("SUCCESSFUL", finished.getStatus(), "precondition: it converged and was read");
+ Instant finishedAt = finished.getFinishedAt();
+ assertNotNull(finishedAt);
+
+ TaskDefinition rev2 = registerTaskDef(service, "dsup3-fam", "app:2");
+ service.updateService("dsup3-cluster", "dsup3-svc", "dsup3-fam:" + rev2.getRevision(),
+ null, null, REGION);
+
+ ServiceDeployment after = deploymentOf(service, "dsup3-svc", "dsup3-cluster", first);
+ assertEquals("SUCCESSFUL", after.getStatus(),
+ "a deployment that completed was not stopped by the one that followed it");
+ assertEquals(finishedAt, after.getFinishedAt(),
+ "and keeps the instant it finished at, not the instant it was superseded");
+ assertNull(after.getStoppedAt(), "a deployment that succeeded was never stopped");
+ assertNull(after.getStatusReason(), "and has no reason it failed to finish");
+ }
+
+ /**
+ * The update settles the outgoing deployment itself, so whether it finished does not depend
+ * on anything having read it. Changing the desired count in the same request must not count
+ * against it either: it had reached the count it was asked for.
+ */
+ @Test
+ void aConvergedDeploymentNobodyReadIsSuccessfulWhenSuperseded() {
+ EcsService service = newMockModeService();
+ service.createCluster("dunread-cluster", REGION);
+ registerTaskDef(service, "dunread-fam", "app:1");
+ String first = service.createService("dunread-cluster", "dunread-svc", "dunread-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+ service.reconcileServices();
+
+ TaskDefinition rev2 = registerTaskDef(service, "dunread-fam", "app:2");
+ String second = service.updateService("dunread-cluster", "dunread-svc",
+ "dunread-fam:" + rev2.getRevision(), 3, null, REGION).getDeploymentId();
+
+ ServiceDeployment finished = deploymentOf(service, "dunread-svc", "dunread-cluster", first);
+ assertEquals("SUCCESSFUL", finished.getStatus(),
+ "its task was running when the update arrived");
+ assertNotNull(finished.getFinishedAt());
+ assertNull(finished.getStoppedAt());
+ assertEquals(List.of(deploymentArnOf(service, "dunread-svc", "dunread-cluster", second)),
+ service.listServiceDeployments("dunread-svc", "dunread-cluster",
+ List.of("IN_PROGRESS"), REGION),
+ "only the new deployment is in progress");
+ }
+
+ /**
+ * Deleting a service stops a deployment it leaves unfinished, so a listing filtered on
+ * IN_PROGRESS no longer returns it.
+ */
+ @Test
+ void deletingAServiceStopsTheDeploymentItLeavesUnfinished() {
+ EcsService service = newMockModeService();
+ service.createCluster("ddel-cluster", REGION);
+ registerTaskDef(service, "ddel-fam", "app:1");
+ String only = service.createService("ddel-cluster", "ddel-svc", "ddel-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+
+ // No tick, so nothing was ever launched and the deployment is still rolling out.
+ assertEquals("IN_PROGRESS",
+ deploymentOf(service, "ddel-svc", "ddel-cluster", only).getStatus(),
+ "precondition: the deployment never converged");
+ assertEquals(1, service.listServiceDeployments("ddel-svc", "ddel-cluster",
+ List.of("IN_PROGRESS"), REGION).size(), "precondition: and reads as in flight");
+
+ service.deleteService("ddel-cluster", "ddel-svc", true, REGION);
+
+ ServiceDeployment stopped = deploymentOf(service, "ddel-svc", "ddel-cluster", only);
+ assertEquals("STOPPED", stopped.getStatus(),
+ "a deployment whose service is gone has ended, and nothing else can ever say so");
+ assertNotNull(stopped.getFinishedAt(), "it reports when it ended");
+ assertNotNull(stopped.getStoppedAt(), "and when it was stopped");
+ assertEquals("The service was deleted.", stopped.getStatusReason(),
+ "naming the reason it could not finish");
+
+ assertTrue(service.listServiceDeployments("ddel-svc", "ddel-cluster",
+ List.of("IN_PROGRESS"), REGION).isEmpty(),
+ "and a client asking what is rolling out is no longer told this is");
+ }
+
+ /** Deleting a service leaves its finished deployments SUCCESSFUL. */
+ @Test
+ void deletingAServiceLeavesItsFinishedDeploymentsSuccessful() {
+ EcsService service = newMockModeService();
+ service.createCluster("ddel2-cluster", REGION);
+ registerTaskDef(service, "ddel2-fam", "app:1");
+ String only = service.createService("ddel2-cluster", "ddel2-svc", "ddel2-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+ service.reconcileServices();
+
+ ServiceDeployment finished = deploymentOf(service, "ddel2-svc", "ddel2-cluster", only);
+ assertEquals("SUCCESSFUL", finished.getStatus(), "precondition: it converged and was read");
+ Instant finishedAt = finished.getFinishedAt();
+ assertNotNull(finishedAt);
+
+ service.deleteService("ddel2-cluster", "ddel2-svc", true, REGION);
+
+ ServiceDeployment after = deploymentOf(service, "ddel2-svc", "ddel2-cluster", only);
+ assertEquals("SUCCESSFUL", after.getStatus(),
+ "a rollout that completed is history, and deleting the service does not undo it");
+ assertEquals(finishedAt, after.getFinishedAt(), "keeping the instant it finished at");
+ assertNull(after.getStoppedAt(), "and carrying none of the stopped fields");
+ assertNull(after.getStatusReason());
+ }
+
+ /**
+ * As {@link #aConvergedDeploymentNobodyReadIsSuccessfulWhenSuperseded}, for a delete: the
+ * delete settles the deployment against the service as it stood, so a rollout that had
+ * finished is SUCCESSFUL whether or not anything read it first.
+ */
+ @Test
+ void deletingAServiceSettlesAConvergedDeploymentNobodyRead() {
+ EcsService service = newMockModeService();
+ service.createCluster("ddel3-cluster", REGION);
+ registerTaskDef(service, "ddel3-fam", "app:1");
+ String only = service.createService("ddel3-cluster", "ddel3-svc", "ddel3-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
+ service.reconcileServices();
+
+ service.deleteService("ddel3-cluster", "ddel3-svc", true, REGION);
+
+ ServiceDeployment after = deploymentOf(service, "ddel3-svc", "ddel3-cluster", only);
+ assertEquals("SUCCESSFUL", after.getStatus(),
+ "its task was running when the delete arrived");
+ assertNotNull(after.getFinishedAt());
+ assertNull(after.getStoppedAt(), "a deployment that succeeded was never stopped");
+ assertNull(after.getStatusReason());
+ }
+
+ /** The one place the provider's join is asserted: the revision ARN carries the task set id. */
+ private static void assertRevisionCarriesTaskSetId(EcsService service, String name,
+ String cluster, String deploymentId) {
+ String taskSetId = deploymentId.substring(deploymentId.indexOf('/') + 1);
+ List<String> targets = service.listServiceDeploymentsDetailed(name, cluster, null, REGION)
+ .stream().map(ServiceDeployment::getTargetServiceRevisionArn).toList();
+ assertTrue(targets.stream().anyMatch(arn -> arn != null && arn.contains(taskSetId)),
+ "some deployment's revision ARN must carry task set id " + taskSetId
+ + ", targets were: " + targets);
+ }
+
+ /** The deployment carrying {@code deploymentId}, found by the task set id in its revision ARN. */
+ private static ServiceDeployment deploymentOf(EcsService service, String name, String cluster,
+ String deploymentId) {
+ String taskSetId = deploymentId.substring(deploymentId.indexOf('/') + 1);
+ return service.listServiceDeploymentsDetailed(name, cluster, null, REGION).stream()
+ .filter(d -> d.getTargetServiceRevisionArn() != null
+ && d.getTargetServiceRevisionArn().endsWith("/" + taskSetId))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("no deployment for " + deploymentId));
+ }
+
+ /** The ARN of the deployment carrying {@code deploymentId}, selected the same way. */
+ private static String deploymentArnOf(EcsService service, String name, String cluster,
+ String deploymentId) {
+ return deploymentOf(service, name, cluster, deploymentId).getServiceDeploymentArn();
+ }
+
+ private static ServiceDeployment onlyDeployment(EcsService service, String name, String cluster) {
+ List<ServiceDeployment> deployments =
+ service.listServiceDeploymentsDetailed(name, cluster, null, REGION);
+ assertEquals(1, deployments.size(), "one create, one deployment");
+ return deployments.getFirst();
+ }
+
+ private static List<EcsTask> runningTasks(EcsService service) {
+ return service.describeTasks(null, service.listTasks(null, null, null, null, REGION), REGION)
+ .stream().filter(t -> "RUNNING".equals(t.getLastStatus())).toList();
+ }
+
+ private static TaskDefinition registerTaskDef(EcsService service, String family, String image) {
+ ContainerDefinition cd = new ContainerDefinition();
+ cd.setName("app");
+ cd.setImage(image);
+ return service.registerTaskDefinition(family, List.of(cd), null, null, null,
+ null, null, List.of(), REGION);
+ }
+
+ private static EcsService newMockModeService() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(true);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ EcsService service = new EcsService(
+ new RegionResolver(REGION, "000000000000"),
+ mock(EcsContainerManager.class),
+ config,
+ mock(EcsLoadBalancerRegistrar.class),
+ new InMemoryStorageFactory(),
+ null);
+ service.initializeStorage();
+ return service;
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+ ignored -> AccountAwareStorageBackend.inMemory("000000000000"));
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java
new file mode 100644
index 0000000000..d4cff003bf
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java
@@ -0,0 +1,206 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * The join a steady-state wait walks, end to end.
+ *
+ * <p>Since provider 6.x the Terraform AWS provider no longer decides {@code wait_for_steady_state}
+ * from {@code DescribeServices} alone. For a service under the ECS deployment controller whose
+ * PRIMARY deployment started after the operation, it:
+ *
+ * <ol>
+ * <li>reads {@code services[0].deployments[0].id}, which is {@code ecs-svc/<taskSetId>};</li>
+ * <li>calls {@code ListServiceDeployments} and takes the brief whose
+ * {@code targetServiceRevisionArn} <em>contains that {@code taskSetId}</em>;</li>
+ * <li>polls {@code DescribeServiceDeployments} on that ARN until its status is
+ * {@code SUCCESSFUL}.</li>
+ * </ol>
+ *
+ * <p>Step 2 is a join AWS makes work by naming the service revision with the very id the
+ * deployment reports. Floci minted the two independently, so the join matched nothing, the
+ * provider never reached step 3, and every {@code wait_for_steady_state} apply sat on
+ * {@code tfPENDING} for its full twenty-minute timeout against a service that was running and
+ * reporting itself stable. Six of the first sixteen {@code terraform-aws-ecs} Terratest suites
+ * failed that way in the 2026-09-28 sweep.
+ */
+@QuarkusTest
+class EcsServiceSteadyStateWaiterIntegrationTest {
+
+ private static final String TARGET = "AmazonEC2ContainerServiceV20141113.";
+ private static final String CT = "application/x-amz-json-1.1";
+ private static final String CLUSTER = "waiter-cluster";
+ private static final String NETWORK = "\"networkConfiguration\":{\"awsvpcConfiguration\":"
+ + "{\"subnets\":[\"subnet-1\"],\"securityGroups\":[\"sg-1\"]}}";
+
+ @BeforeAll
+ static void configure() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ private static Response call(String action, String body) {
+ return given().contentType(CT).header("X-Amz-Target", TARGET + action)
+ .body(body)
+ .when().post("/")
+ .then().statusCode(200)
+ .extract().response();
+ }
+
+ /**
+ * As {@link #seedService}, but at a task count the service has not reached yet, so its
+ * deployment is in flight. Whatever is done with it must not read the service back: see
+ * {@link #aStoppedDeploymentReportsWhenItStoppedAndWhy}.
+ */
+ private static String seedUnconvergedService(String name) {
+ call("CreateCluster", "{\"clusterName\":\"" + CLUSTER + "\"}");
+ call("RegisterTaskDefinition", "{\"family\":\"" + name + "-td\",\"networkMode\":\"awsvpc\","
+ + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\",\"memory\":128}]}");
+ call("CreateService", "{\"cluster\":\"" + CLUSTER + "\",\"serviceName\":\"" + name + "\","
+ + "\"taskDefinition\":\"" + name + "-td\",\"desiredCount\":1,"
+ + "\"launchType\":\"FARGATE\"," + NETWORK + "}");
+ return name;
+ }
+
+ /** Creates the cluster, a task definition and a service, and returns the service name. */
+ private static String seedService(String name) {
+ call("CreateCluster", "{\"clusterName\":\"" + CLUSTER + "\"}");
+ call("RegisterTaskDefinition", "{\"family\":\"" + name + "-td\",\"networkMode\":\"awsvpc\","
+ + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\",\"memory\":128}]}");
+ // desiredCount 0 is already converged, so the deployment is SUCCESSFUL without waiting on
+ // the reconciler: this test is about the join, not about convergence.
+ call("CreateService", "{\"cluster\":\"" + CLUSTER + "\",\"serviceName\":\"" + name + "\","
+ + "\"taskDefinition\":\"" + name + "-td\",\"desiredCount\":0,"
+ + "\"launchType\":\"FARGATE\"," + NETWORK + "}");
+ return name;
+ }
+
+ /** The {@code <taskSetId>} half of the service's PRIMARY {@code ecs-svc/<taskSetId>}. */
+ private static String primaryTaskSetId(String service) {
+ String id = call("DescribeServices", "{\"cluster\":\"" + CLUSTER
+ + "\",\"services\":[\"" + service + "\"]}")
+ .jsonPath().getString("services[0].deployments[0].id");
+ assertNotNull(id, "an ACTIVE service reports a PRIMARY deployment id");
+ assertTrue(id.startsWith("ecs-svc/"), "deployment id is ecs-svc/<taskSetId>, was: " + id);
+ return id.substring(id.indexOf('/') + 1);
+ }
+
+ @Test
+ void theRevisionADeploymentTargetsIsNamedByThePrimaryDeploymentsTaskSetId() {
+ String service = seedService("waiter-join-svc");
+ String taskSetId = primaryTaskSetId(service);
+
+ String targetRevision = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER
+ + "\",\"service\":\"" + service + "\"}")
+ .jsonPath().getString("serviceDeployments[0].targetServiceRevisionArn");
+
+ // Verbatim the provider's predicate: strings.Contains(targetServiceRevisionArn, taskSetID).
+ assertTrue(targetRevision != null && targetRevision.contains(taskSetId),
+ "targetServiceRevisionArn must carry the PRIMARY deployment's task set id "
+ + taskSetId + ", so a steady-state wait can find the deployment to poll; was: "
+ + targetRevision);
+ }
+
+ /**
+ * Named for what it checks. It walks list -> join -> describe on an already-converged
+ * service, so it never observes a deployment mid-flight: {@code desiredCount} is 0, which is
+ * converged on arrival. Observing IN_PROGRESS over the wire would mean racing the 5-second
+ * reconciler inside a {@code @QuarkusTest}, so that direction is asserted deterministically
+ * in {@code EcsServiceDeploymentStatusTest} instead, and the writer echoes the field verbatim
+ * ({@code EcsResponseWriter.serviceDeploymentNode}).
+ */
+ @Test
+ void theJoinedDeploymentIsDescribable() {
+ String service = seedService("waiter-walk-svc");
+ String taskSetId = primaryTaskSetId(service);
+
+ Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER
+ + "\",\"service\":\"" + service + "\"}");
+ listed.then().body("serviceDeployments", hasSize(1));
+
+ String deploymentArn = null;
+ for (Object brief : listed.jsonPath().getList("serviceDeployments")) {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> map = (Map<String, Object>) brief;
+ Object revision = map.get("targetServiceRevisionArn");
+ if (revision != null && revision.toString().contains(taskSetId)) {
+ deploymentArn = String.valueOf(map.get("serviceDeploymentArn"));
+ }
+ }
+ assertNotNull(deploymentArn, "the listing must contain the PRIMARY deployment");
+
+ // A service at its requested task count, desiredCount 0, is a finished deployment.
+ call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")
+ .then()
+ .body("serviceDeployments", hasSize(1))
+ .body("serviceDeployments[0].status", equalTo("SUCCESSFUL"))
+ // A deployment that succeeded was never stopped, so AWS reports neither field and
+ // nor do we. Absent, not null: a client reading stoppedAt to learn when a
+ // deployment ended must not be handed a value for one that did not stop.
+ .body("serviceDeployments[0].stoppedAt", nullValue())
+ .body("serviceDeployments[0].statusReason", nullValue());
+ }
+
+ /**
+ * The stopped fields over the wire. A deployment that ends without completing carries
+ * {@code stoppedAt} and {@code statusReason}, both of which are members of AWS's
+ * {@code ServiceDeployment} shape and neither of which floci used to write at all, so a
+ * client asking when a stopped deployment ended, or why, got nothing.
+ *
+ * <p>The deployment has to still be in flight when the delete arrives, and a test cannot
+ * promise that: the 5-second reconciler runs on its own thread, and a tick that lands between
+ * CreateService and DeleteService starts the task, so the delete correctly settles the
+ * deployment SUCCESSFUL instead. That is a lost race, not a failure, so the test tries again
+ * on a fresh service, a few times; the window is milliseconds against a 5000ms period. It
+ * fails only if no attempt produced a stopped deployment, or one did and its fields are wrong.
+ * The behaviour itself is pinned without any timing in
+ * {@code EcsServiceDeploymentStatusTest.deletingAServiceStopsTheDeploymentItLeavesUnfinished};
+ * what only this test can show is that the two fields reach the wire.
+ */
+ @Test
+ void aStoppedDeploymentReportsWhenItStoppedAndWhy() {
+ for (int attempt = 1; attempt <= 5; attempt++) {
+ String service = seedUnconvergedService("waiter-stop-svc-" + attempt);
+
+ call("DeleteService", "{\"cluster\":\"" + CLUSTER + "\",\"service\":\"" + service
+ + "\",\"force\":true}");
+
+ // Identified after the delete: the service was created once and never updated, so it
+ // has exactly one deployment, and a terminal record cannot change under a read.
+ Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER
+ + "\",\"service\":\"" + service + "\"}");
+ listed.then().body("serviceDeployments", hasSize(1));
+ String deploymentArn = listed.jsonPath()
+ .getString("serviceDeployments[0].serviceDeploymentArn");
+ assertNotNull(deploymentArn, "the deleted service's deployment is still describable");
+
+ Response described = call("DescribeServiceDeployments",
+ "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}");
+ if ("SUCCESSFUL".equals(described.jsonPath().getString("serviceDeployments[0].status"))) {
+ continue;
+ }
+ described.then()
+ .body("serviceDeployments", hasSize(1))
+ .body("serviceDeployments[0].status", equalTo("STOPPED"))
+ .body("serviceDeployments[0].stoppedAt", notNullValue())
+ .body("serviceDeployments[0].finishedAt", notNullValue())
+ .body("serviceDeployments[0].statusReason", equalTo("The service was deleted."));
+ return;
+ }
+ fail("the reconciler converged the service before every one of five deletes");
+ }
+}