Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): complete failed service deployment handling (#4874)
Vũ Thế Huy1d85e6bf6e19
diff --git a/docs/services/ecs.md b/docs/services/ecs.mdindex f0b6287c85..0c2909e70f 100644--- a/docs/services/ecs.md+++ b/docs/services/ecs.md@@ -436,13 +436,19 @@ Known differences from AWS:and rolls the running tasks: a replacement on the new deployment starts first, thenthe task from the previous deployment is drained one reconciler tick later. The`deployments` list still reports a single `PRIMARY` throughout.-- `updatedAt` equals `createdAt`. AWS advances it as a rollout progresses; Floci has no- intermediate rollout state to report.-- `deploymentConfiguration` (including the circuit breaker), `healthCheckGracePeriodSeconds`,- `serviceRegistries` and the placement constraints and strategies are stored and reported as- given, so a client that reads them back sees no drift, but the reconciler does not act on them:- it converges to `desiredCount` without a maximum or minimum percent, registers nothing in Cloud- Map, and places tasks without evaluating constraints.+- `updatedAt` starts at `createdAt` and advances when the deployment status changes.+- `deploymentConfiguration` is stored and reported as given. The deployment circuit breaker+ counts tasks that fail to start and moves a deployment through `STOP_REQUESTED` to `STOPPED`,+ or, with rollback enabled, through `ROLLBACK_IN_PROGRESS` to `ROLLBACK_SUCCESSFUL` or+ `ROLLBACK_FAILED`. A rollback restores the most recent successful service revision while+ retaining the service's current `desiredCount`. `DescribeServiceDeployments` reports the+ rollback target and reason under `rollback`, and configured and triggered alarm names under+ `alarms`. Enabled+ CloudWatch deployment alarms are also checked during reconciliation and fail the deployment+ when any configured alarm is in `ALARM`. Floci stops monitoring alarms when tasks converge;+ AWS continues monitoring through the deployment bake time. `healthCheckGracePeriodSeconds`, `serviceRegistries`+ and placement constraints and strategies are stored and reported but are not enforced; task+ placement does not evaluate constraints.- `StopServiceDeployment` is not implemented.#### ECS EventBridge events@@ -454,10 +460,10 @@ and mock mode.| `detail-type` | When | Key `detail` fields ||---|---|---|| `ECS Task State Change` | a task starts or stops | `lastStatus`, `desiredStatus`, `taskDefinitionArn`, `group`, `startedBy`, `stoppedReason`, `containers[].exitCode` |-| `ECS Deployment State Change` | a service deployment starts, is in progress, or reaches steady state | `eventType` (always `INFO`), `eventName`, `deploymentId` |+| `ECS Deployment State Change` | a service deployment starts, fails, rolls back, or reaches steady state | `eventType` (`ERROR` on failure, otherwise `INFO`), `eventName`, `deploymentId` |`eventName` is one of `SERVICE_DEPLOYMENT_STARTED`, `SERVICE_DEPLOYMENT_IN_PROGRESS`,-`SERVICE_DEPLOYMENT_COMPLETED`.+`SERVICE_DEPLOYMENT_COMPLETED`, `SERVICE_DEPLOYMENT_FAILED`.Known differences from AWS:@@ -466,7 +472,7 @@ Known differences from AWS:`PROVISIONING -> PENDING -> ACTIVATING -> RUNNING` and a stop emits`DEACTIVATING -> STOPPING -> DEPROVISIONING -> STOPPED`, one `ECS Task State Change`per phase, so rules that filter on `detail.lastStatus` behave as on AWS.-- `SERVICE_DEPLOYMENT_FAILED` and the deployment circuit breaker are not emitted.+- `SERVICE_DEPLOYMENT_FAILED` is not emitted.- `SubmitTaskStateChange` / `SubmitContainerStateChange` remain ACK-only; Floci drivesthe task lifecycle itself rather than via agent submissions.diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsEventPublisher.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsEventPublisher.javaindex 2d76b89431..afefadcde9 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsEventPublisher.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsEventPublisher.java@@ -102,11 +102,11 @@ public class EcsEventPublisher {}}- /** Emits one ECS Deployment State Change (eventType INFO). */+ /** Emits one ECS Deployment State Change. */public void emitDeploymentStateChange(EcsServiceModel svc, String eventName, String reason, String region) {try {ObjectNode detail = objectMapper.createObjectNode();- detail.put("eventType", "INFO");+ detail.put("eventType", "SERVICE_DEPLOYMENT_FAILED".equals(eventName) ? "ERROR" : "INFO");detail.put("eventName", eventName);detail.put("deploymentId", svc.getDeploymentId());detail.put("updatedAt", Instant.now().toString());diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.javaindex 148b8e689c..e7cc8d7b1f 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriter.java@@ -970,6 +970,22 @@ public class EcsResponseWriter {.forEach(arn -> sources.add(serviceRevisionSummaryNode(arn, d.getServiceArn())));}n.set("sourceServiceRevisions", sources);+ if (d.getRollbackTargetServiceRevisionArn() != null) {+ ObjectNode rollback = n.putObject("rollback");+ rollback.put("serviceRevisionArn", d.getRollbackTargetServiceRevisionArn());+ rollback.put("reason", d.getRollbackReason());+ putInstant(rollback, "startedAt", d.getRollbackStartedAt());+ }+ if (d.getAlarmNames() != null) {+ ObjectNode alarms = n.putObject("alarms");+ ArrayNode alarmNames = alarms.putArray("alarmNames");+ d.getAlarmNames().forEach(alarmNames::add);+ ArrayNode triggered = alarms.putArray("triggeredAlarmNames");+ if (d.getTriggeredAlarmNames() != null) {+ d.getTriggeredAlarmNames().forEach(triggered::add);+ }+ alarms.put("status", d.getAlarmStatus());+ }return n;}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex 5a4f36fd17..dc3814a405 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -11,6 +11,8 @@ import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackedMap;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;import io.github.hectorvent.floci.services.ecs.exec.EcsExecChannelHandler;@@ -87,6 +89,7 @@ import java.util.Comparator;import java.util.HashSet;import java.util.UUID;import java.util.function.Function;+import java.util.function.Predicate;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Executors;@@ -113,6 +116,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab// Null for a service assembled without CDI, which then registers nothing in Cloud Map:// what every such caller already expects.private final EcsServiceDiscoveryRegistrar discoveryRegistrar;+ private final CloudWatchMetricsService cloudWatchMetricsService;private final boolean dockerMode;private final boolean sweepLeftoverContainers;private final String baseUrl;@@ -164,10 +168,15 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabpublic static final String DEPLOYMENT_STATUS_IN_PROGRESS = "IN_PROGRESS";/** A service deployment whose target revision is running at its requested task count. */public static final String DEPLOYMENT_STATUS_SUCCESSFUL = "SUCCESSFUL";+ public static final String DEPLOYMENT_STATUS_STOP_REQUESTED = "STOP_REQUESTED";/** A service deployment that ended without completing. */public static final String DEPLOYMENT_STATUS_STOPPED = "STOPPED";+ public static final String DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS = "ROLLBACK_IN_PROGRESS";+ public static final String DEPLOYMENT_STATUS_ROLLBACK_SUCCESSFUL = "ROLLBACK_SUCCESSFUL";+ public static final String DEPLOYMENT_STATUS_ROLLBACK_FAILED = "ROLLBACK_FAILED";/** Why a deployment the circuit breaker ended stopped, and why its rollout FAILED. */public static final String CIRCUIT_BREAKER_REASON = "The deployment circuit breaker detected a failure.";+ public static final String ALARM_FAILURE_REASON = "A deployment alarm entered the ALARM state.";/** RunTask places at most ten tasks in one call, and StartTask at most ten instances. */public static final int MAX_TASKS_PER_RUN = 10;/** A listing returns at most a hundred ARNs per page. */@@ -209,7 +218,8 @@ public class EcsService implements ContainerTeardown, ResourceProvider, ResettabEmulatorConfig config, EcsLoadBalancerRegistrar lbRegistrar,StorageFactory storageFactory, EcsEventPublisher eventPublisher,EcsExecSessionRegistry execSessions,- EcsServiceDiscoveryRegistrar discoveryRegistrar) {+ EcsServiceDiscoveryRegistrar discoveryRegistrar,+ CloudWatchMetricsService cloudWatchMetricsService) {this.regionResolver = regionResolver;this.containerManager = containerManager;this.dockerMode = !config.services().ecs().mock();@@ -220,6 +230,16 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabthis.eventPublisher = eventPublisher;this.execSessions = execSessions;this.discoveryRegistrar = discoveryRegistrar;+ this.cloudWatchMetricsService = cloudWatchMetricsService;+ }++ public EcsService(RegionResolver regionResolver, EcsContainerManager containerManager,+ EmulatorConfig config, EcsLoadBalancerRegistrar lbRegistrar,+ StorageFactory storageFactory, EcsEventPublisher eventPublisher,+ EcsExecSessionRegistry execSessions,+ EcsServiceDiscoveryRegistrar discoveryRegistrar) {+ this(regionResolver, containerManager, config, lbRegistrar, storageFactory, eventPublisher,+ execSessions, discoveryRegistrar, null);}/** With an exec session registry of its own, for callers that assemble the service without CDI. */@@ -227,7 +247,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, ResettabEmulatorConfig config, EcsLoadBalancerRegistrar lbRegistrar,StorageFactory storageFactory, EcsEventPublisher eventPublisher) {this(regionResolver, containerManager, config, lbRegistrar, storageFactory, eventPublisher,- new EcsExecSessionRegistry(), null);+ new EcsExecSessionRegistry(), null, null);}@PostConstruct@@ -2480,106 +2500,108 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabif (svc == null) {throw new AwsException("ServiceNotFoundException", "Service " + serviceName + " not found.", 404);}- if ("INACTIVE".equals(svc.getStatus())) {- throw new AwsException("ServiceNotActiveException",- "Service " + serviceName + " is not active.", 400);- }- // Settle the outgoing deployment against the service as it stands, before this request- // changes its desired count or task definition: one whose tasks are up has finished,- // whether or not anything read it. recordServiceDeployment stops it if it has not.- currentServiceDeployment(svc);- if (request.getDesiredCount() != null) {- if (request.getDesiredCount() < 0) {- throw new AwsException("InvalidParameterException", "desiredCount cannot be a negative number.", 400);+ synchronized (svc) {+ if ("INACTIVE".equals(svc.getStatus())) {+ throw new AwsException("ServiceNotActiveException",+ "Service " + serviceName + " is not active.", 400);}- svc.setDesiredCount(request.getDesiredCount());- }- // The members whose change starts new tasks, as UpdateService documents them: the network- // configuration, the load balancers, the service registries, the Service Connect- // configuration, the task definition and the platform version. The rest (desired count,- // deployment configuration, the exec and managed-tag switches, placement, propagateTags,- // the capacity provider strategy) are applied without rolling anything.- boolean rollingChange = false;- if (request.getNetworkConfiguration() != null) {- rollingChange |= networkConfigurationChanged(svc.getNetworkConfiguration(),- request.getNetworkConfiguration());- svc.setNetworkConfiguration(request.getNetworkConfiguration());- }- if (request.getAvailabilityZoneRebalancing() != null) {- svc.setAvailabilityZoneRebalancing(request.getAvailabilityZoneRebalancing());- }- if (request.getCapacityProviderStrategy() != null) {- svc.setCapacityProviderStrategy(request.getCapacityProviderStrategy());- svc.setLaunchType(request.getCapacityProviderStrategy().isEmpty()- ? svc.getLaunchType()- : placementFor(request.getCapacityProviderStrategy().getFirst().capacityProvider()).launchType());- }- if (request.getEnableExecuteCommand() != null) {- svc.setEnableExecuteCommand(request.getEnableExecuteCommand());- }- if (request.getEnableECSManagedTags() != null) {- svc.setEnableECSManagedTags(request.getEnableECSManagedTags());- }- if (request.getPropagateTags() != null) {- svc.setPropagateTags(request.getPropagateTags());- }- if (request.getHealthCheckGracePeriodSeconds() != null) {- svc.setHealthCheckGracePeriodSeconds(request.getHealthCheckGracePeriodSeconds());- }- if (request.getDeploymentConfiguration() != null) {- svc.setDeploymentConfiguration(request.getDeploymentConfiguration());- }- if (request.getLoadBalancers() != null) {- rollingChange |= loadBalancersChanged(svc.getLoadBalancers(), request.getLoadBalancers());- svc.setLoadBalancers(request.getLoadBalancers());- }- if (request.getServiceRegistries() != null) {- boolean registriesChanged = !request.getServiceRegistries().equals(svc.getServiceRegistries());- rollingChange |= registriesChanged;- svc.setServiceRegistries(request.getServiceRegistries());- if (registriesChanged) {- reconcileServiceDiscoveryRegistries(svc, cluster, region);+ // Settle the outgoing deployment against the service as it stands, before this request+ // changes its desired count or task definition: one whose tasks are up has finished,+ // whether or not anything read it. recordServiceDeployment stops it if it has not.+ currentServiceDeployment(svc);+ if (request.getDesiredCount() != null) {+ if (request.getDesiredCount() < 0) {+ throw new AwsException("InvalidParameterException", "desiredCount cannot be a negative number.", 400);+ }+ svc.setDesiredCount(request.getDesiredCount());}- }- if (request.getUnparsed() != null) {- svc.setUnparsed(mergedUnparsed(svc.getUnparsed(), request.getUnparsed()));- }- // UpdateServiceRequest.serviceConnectConfiguration is documented as "This parameter- // triggers a new service deployment", so a real change rolls the deployment the way a- // task-definition change does. An omitted parameter is not a change and rolls nothing.- Map<String, Object> serviceConnectConfiguration = request.getServiceConnectConfiguration();- boolean serviceConnectChanged = serviceConnectConfiguration != null- && !serviceConnectConfiguration.equals(svc.getServiceConnectConfiguration());- if (serviceConnectConfiguration != null) {- svc.setServiceConnectConfiguration(serviceConnectConfiguration);- }- boolean taskDefChanged = false;- String platformVersionBefore = svc.getPlatformVersion();- if (request.getTaskDefinition() != null) {- TaskDefinition resolved = resolveLaunchableTaskDefinition(request.getTaskDefinition(), region);- taskDefChanged = !resolved.getTaskDefinitionArn().equals(svc.getTaskDefinition());- svc.setTaskDefinition(resolved.getTaskDefinitionArn());- applyServicePlatform(svc, resolved, request.getPlatformVersion() != null- ? request.getPlatformVersion() : svc.getPlatformVersion());- } else if (request.getPlatformVersion() != null) {- svc.setPlatformVersion(PLATFORM_VERSION_LATEST.equals(request.getPlatformVersion())- ? DEFAULT_PLATFORM_VERSION : request.getPlatformVersion());- }- // UpdateServiceRequest.platformVersion is documented as "This parameter triggers a new- // service deployment". Compared after resolution, so a request that asks for LATEST on a- // service already running the version LATEST resolves to is not a change.- rollingChange |= !Objects.equals(platformVersionBefore, svc.getPlatformVersion());- if (taskDefChanged || forceNewDeployment || serviceConnectChanged || rollingChange) {- svc.setDeploymentId(newDeploymentId());- svc.setLastDeploymentAt(Instant.now());- recordServiceDeployment(svc, svc.getTaskDefinition(), region);- if (eventPublisher != null) {- eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_STARTED",- "ECS deployment " + svc.getDeploymentId() + " started.", region);+ // The members whose change starts new tasks, as UpdateService documents them: the network+ // configuration, the load balancers, the service registries, the Service Connect+ // configuration, the task definition and the platform version. The rest (desired count,+ // deployment configuration, the exec and managed-tag switches, placement, propagateTags,+ // the capacity provider strategy) are applied without rolling anything.+ boolean rollingChange = false;+ if (request.getNetworkConfiguration() != null) {+ rollingChange |= networkConfigurationChanged(svc.getNetworkConfiguration(),+ request.getNetworkConfiguration());+ svc.setNetworkConfiguration(request.getNetworkConfiguration());+ }+ if (request.getAvailabilityZoneRebalancing() != null) {+ svc.setAvailabilityZoneRebalancing(request.getAvailabilityZoneRebalancing());+ }+ if (request.getCapacityProviderStrategy() != null) {+ svc.setCapacityProviderStrategy(request.getCapacityProviderStrategy());+ svc.setLaunchType(request.getCapacityProviderStrategy().isEmpty()+ ? svc.getLaunchType()+ : placementFor(request.getCapacityProviderStrategy().getFirst().capacityProvider()).launchType());+ }+ if (request.getEnableExecuteCommand() != null) {+ svc.setEnableExecuteCommand(request.getEnableExecuteCommand());+ }+ if (request.getEnableECSManagedTags() != null) {+ svc.setEnableECSManagedTags(request.getEnableECSManagedTags());+ }+ if (request.getPropagateTags() != null) {+ svc.setPropagateTags(request.getPropagateTags());+ }+ if (request.getHealthCheckGracePeriodSeconds() != null) {+ svc.setHealthCheckGracePeriodSeconds(request.getHealthCheckGracePeriodSeconds());+ }+ if (request.getDeploymentConfiguration() != null) {+ svc.setDeploymentConfiguration(request.getDeploymentConfiguration());+ }+ if (request.getLoadBalancers() != null) {+ rollingChange |= loadBalancersChanged(svc.getLoadBalancers(), request.getLoadBalancers());+ svc.setLoadBalancers(request.getLoadBalancers());+ }+ if (request.getServiceRegistries() != null) {+ boolean registriesChanged = !request.getServiceRegistries().equals(svc.getServiceRegistries());+ rollingChange |= registriesChanged;+ svc.setServiceRegistries(request.getServiceRegistries());+ if (registriesChanged) {+ reconcileServiceDiscoveryRegistries(svc, cluster, region);+ }+ }+ if (request.getUnparsed() != null) {+ svc.setUnparsed(mergedUnparsed(svc.getUnparsed(), request.getUnparsed()));}+ // UpdateServiceRequest.serviceConnectConfiguration is documented as "This parameter+ // triggers a new service deployment", so a real change rolls the deployment the way a+ // task-definition change does. An omitted parameter is not a change and rolls nothing.+ Map<String, Object> serviceConnectConfiguration = request.getServiceConnectConfiguration();+ boolean serviceConnectChanged = serviceConnectConfiguration != null+ && !serviceConnectConfiguration.equals(svc.getServiceConnectConfiguration());+ if (serviceConnectConfiguration != null) {+ svc.setServiceConnectConfiguration(serviceConnectConfiguration);+ }+ boolean taskDefChanged = false;+ String platformVersionBefore = svc.getPlatformVersion();+ if (request.getTaskDefinition() != null) {+ TaskDefinition resolved = resolveLaunchableTaskDefinition(request.getTaskDefinition(), region);+ taskDefChanged = !resolved.getTaskDefinitionArn().equals(svc.getTaskDefinition());+ svc.setTaskDefinition(resolved.getTaskDefinitionArn());+ applyServicePlatform(svc, resolved, request.getPlatformVersion() != null+ ? request.getPlatformVersion() : svc.getPlatformVersion());+ } else if (request.getPlatformVersion() != null) {+ svc.setPlatformVersion(PLATFORM_VERSION_LATEST.equals(request.getPlatformVersion())+ ? DEFAULT_PLATFORM_VERSION : request.getPlatformVersion());+ }+ // UpdateServiceRequest.platformVersion is documented as "This parameter triggers a new+ // service deployment". Compared after resolution, so a request that asks for LATEST on a+ // service already running the version LATEST resolves to is not a change.+ rollingChange |= !Objects.equals(platformVersionBefore, svc.getPlatformVersion());+ if (taskDefChanged || forceNewDeployment || serviceConnectChanged || rollingChange) {+ svc.setDeploymentId(newDeploymentId());+ svc.setLastDeploymentAt(Instant.now());+ recordServiceDeployment(svc, svc.getTaskDefinition(), region);+ if (eventPublisher != null) {+ eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_STARTED",+ "ECS deployment " + svc.getDeploymentId() + " started.", region);+ }+ }+ services.put(key, svc);+ return svc;}- services.put(key, svc);- return svc;}private void reconcileServiceDiscoveryRegistries(EcsServiceModel svc, EcsCluster cluster, String region) {@@ -3887,8 +3909,11 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab// deployment would report a task-definition change COMPLETED as soon as it was made.long running = runningOnCurrentDeployment(svc);boolean converged = running >= svc.getDesiredCount();+ boolean rollbackPending = latestServiceDeployment(svc.getServiceArn(), candidate ->+ DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS.equals(candidate.getStatus())) != null;ServiceDeployment record = lockedCopyOf(deploymentRecordOf(svc));boolean failed = deploymentId.equals(svc.getFailedDeploymentId());+ ServiceDeployment failedRecord = failed ? failedDeploymentOf(svc) : null;Deployment d = new Deployment();d.setId(deploymentId);@@ -3900,11 +3925,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabd.setFailedTasks(record != null ? record.getFailedTasks() : 0);if (failed) {d.setRolloutState("FAILED");- d.setRolloutStateReason(CIRCUIT_BREAKER_REASON);+ d.setRolloutStateReason(failedRecord != null && failedRecord.getStatusReason() != null+ ? failedRecord.getStatusReason() : CIRCUIT_BREAKER_REASON);} else {- d.setRolloutState(converged ? "COMPLETED" : "IN_PROGRESS");+ d.setRolloutState(converged && !rollbackPending ? "COMPLETED" : "IN_PROGRESS");d.setRolloutStateReason("ECS deployment " + deploymentId- + (converged ? " completed." : " in progress."));+ + (converged && !rollbackPending ? " completed." : " in progress."));}// A deployment reports the placement it runs under the same way the service does: a// capacity provider strategy when there is one, a launch type otherwise, never both.@@ -3928,6 +3954,22 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabreturn List.of(d);}+ private ServiceDeployment failedDeploymentOf(EcsServiceModel svc) {+ String failedId = svc.getFailedDeploymentId();+ if (failedId == null) {+ return null;+ }+ String failedTaskSetId = failedId.substring(failedId.lastIndexOf('/') + 1);+ String suffix = "/" + failedTaskSetId;+ return serviceDeployments.values().stream()+ .filter(deployment -> svc.getServiceArn().equals(deployment.getServiceArn()))+ .filter(deployment -> deployment.getTargetServiceRevisionArn() != null+ && deployment.getTargetServiceRevisionArn().endsWith(suffix))+ .findFirst()+ .map(EcsService::lockedCopyOf)+ .orElse(null);+ }+/*** The service's event log, derived from its current state for the same reason* {@link #deploymentsFor} derives its deployments: Floci applies a change in place instead of@@ -4001,7 +4043,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab* calls the task set id, and what names the service revision that deployment targets.*/private String taskSetId(EcsServiceModel svc) {- String id = deploymentId(svc);+ return taskSetId(deploymentId(svc));+ }++ private static String taskSetId(String id) {int slash = id.indexOf('/');return slash < 0 ? id : id.substring(slash + 1);}@@ -4019,6 +4064,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab* {@code service-revision/<id>}; AWS's is {@code service-revision/<cluster>/<service>/<id>}.*/private void recordServiceDeployment(EcsServiceModel svc, String taskDefinition, String region) {+ svc.setFailedDeploymentId(null);String deploymentId = UUID.randomUUID().toString().replace("-", "");String deploymentArn = regionResolver.buildArn("ecs", region,"service-deployment/" + deploymentId);@@ -4034,21 +4080,18 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab.map(ServiceRevision::getServiceRevisionArn).toList();- ServiceDeployment deployment = new ServiceDeployment();- deployment.setServiceDeploymentArn(deploymentArn);- deployment.setServiceArn(svc.getServiceArn());- deployment.setClusterArn(svc.getClusterArn());- deployment.setTaskDefinition(taskDefinition);- deployment.setStatus(DEPLOYMENT_STATUS_IN_PROGRESS);- deployment.setCreatedAt(now);- deployment.setStartedAt(now);- deployment.setUpdatedAt(now);- deployment.setTargetServiceRevisionArn(revisionArn);+ ServiceDeployment deployment = newServiceDeployment(svc, taskDefinition, deploymentArn, revisionArn, now);deployment.setSourceServiceRevisionArns(priorRevisions);stopNonTerminalDeployments(svc.getServiceArn(), now,"Superseded by service deployment " + deploymentArn + ".");serviceDeployments.put(deploymentArn, deployment);+ ServiceRevision revision = newServiceRevision(svc, taskDefinition, revisionArn, now, region);+ serviceRevisions.put(revisionArn, revision);+ }++ private ServiceRevision newServiceRevision(EcsServiceModel svc, String taskDefinition,+ String revisionArn, Instant now, String region) {ServiceRevision revision = new ServiceRevision();revision.setServiceRevisionArn(revisionArn);revision.setServiceArn(svc.getServiceArn());@@ -4064,7 +4107,23 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabrevision.setServiceConnectConfiguration(svc.getServiceConnectConfiguration());revision.setContainerImages(containerImagesOf(taskDefinition, region));revision.setCreatedAt(now);- serviceRevisions.put(revisionArn, revision);+ return revision;+ }++ private ServiceDeployment newServiceDeployment(EcsServiceModel svc, String taskDefinition,+ String deploymentArn, String revisionArn, Instant now) {+ ServiceDeployment deployment = new ServiceDeployment();+ deployment.setServiceDeploymentArn(deploymentArn);+ deployment.setServiceArn(svc.getServiceArn());+ deployment.setClusterArn(svc.getClusterArn());+ deployment.setTaskDefinition(taskDefinition);+ deployment.setStatus(DEPLOYMENT_STATUS_IN_PROGRESS);+ deployment.setCreatedAt(now);+ deployment.setStartedAt(now);+ deployment.setUpdatedAt(now);+ deployment.setTargetServiceRevisionArn(revisionArn);+ refreshDeploymentAlarms(deployment, svc);+ return deployment;}/**@@ -4082,7 +4141,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabcontinue;}synchronized (prior) {- if (DEPLOYMENT_STATUS_IN_PROGRESS.equals(prior.getStatus())) {+ if (DEPLOYMENT_STATUS_IN_PROGRESS.equals(prior.getStatus())+ || DEPLOYMENT_STATUS_STOP_REQUESTED.equals(prior.getStatus())+ || DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS.equals(prior.getStatus())) {prior.setStatus(DEPLOYMENT_STATUS_STOPPED);prior.setStatusReason(reason);prior.setFinishedAt(now);@@ -4099,6 +4160,8 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab*/private ServiceDeployment settleStatus(ServiceDeployment deployment) {EcsServiceModel svc = serviceByArn(deployment.getServiceArn());+ ServiceDeployment copy;+ boolean completedNow = false;synchronized (deployment) {if (svc != null && deployment.getTargetServiceRevisionArn() != null&& deployment.getTargetServiceRevisionArn().endsWith("/" + taskSetId(svc))@@ -4107,10 +4170,30 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab&& runningOnCurrentDeployment(svc) >= svc.getDesiredCount()) {Instant now = Instant.now();deployment.setStatus(DEPLOYMENT_STATUS_SUCCESSFUL);+ if (deployment.getAlarmNames() != null) {+ deployment.setAlarmStatus("MONITORING_COMPLETE");+ }deployment.setFinishedAt(now);deployment.setUpdatedAt(now);+ svc.setLastSettledDeploymentId(deploymentId(svc));+ ServiceRevision revision = serviceRevisions.get(deployment.getTargetServiceRevisionArn());+ if (revision != null) {+ svc.setLastSuccessfulServiceRevision(revision);+ }+ completedNow = true;}- return copyOf(deployment);+ copy = copyOf(deployment);+ }+ if (completedNow) {+ persistServiceModel(svc);+ }+ return copy;+ }++ private void persistServiceModel(EcsServiceModel svc) {+ EcsCluster cluster = resolveClusterByArn(svc.getClusterArn());+ if (cluster != null) {+ services.put(serviceKey(serviceRegion(svc), cluster.getClusterName(), svc.getServiceName()), svc);}}@@ -4137,33 +4220,183 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}Map<String, Object> breaker = circuitBreakerOf(svc);synchronized (deployment) {- if (!DEPLOYMENT_STATUS_IN_PROGRESS.equals(deployment.getStatus())) {- return false;- }if (TaskStatus.RUNNING.name().equals(task.getLastStatus())) {+ if (DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS.equals(deployment.getStatus())) {+ return false;+ }+ if (!DEPLOYMENT_STATUS_IN_PROGRESS.equals(deployment.getStatus())) {+ return false;+ }if (flag(breaker.get("resetOnHealthyTask"), true)) {deployment.setFailedTasks(0);}return false;}+ if (DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS.equals(deployment.getStatus())) {+ if (TaskStatus.STOPPED.name().equals(task.getLastStatus())+ && STOP_CODE_TASK_FAILED_TO_START.equals(task.getStopCode())) {+ finishRollback(deployment, false,+ "The previous service revision could not be started during rollback.");+ return true;+ }+ return false;+ }+ if (!DEPLOYMENT_STATUS_IN_PROGRESS.equals(deployment.getStatus())) {+ return false;+ }// Only a task that failed to start counts, not one a user stopped during its pull.if (!TaskStatus.STOPPED.name().equals(task.getLastStatus())|| !STOP_CODE_TASK_FAILED_TO_START.equals(task.getStopCode())) {return false;}deployment.setFailedTasks(deployment.getFailedTasks() + 1);- // With rollback on, AWS rolls back instead of stopping; that is not modelled.- if (!flag(breaker.get("enable"), false) || flag(breaker.get("rollback"), false)+ if (!flag(breaker.get("enable"), false)|| deployment.getFailedTasks() < circuitBreakerThreshold(breaker, svc.getDesiredCount())) {return false;}+ }+ return requestDeploymentFailure(svc, deployment, CIRCUIT_BREAKER_REASON,+ flag(breaker.get("rollback"), false));+ }++ private boolean requestDeploymentFailure(EcsServiceModel svc, ServiceDeployment deployment,+ String reason, boolean rollback) {+ synchronized (svc) {+ synchronized (deployment) {+ if (!DEPLOYMENT_STATUS_IN_PROGRESS.equals(deployment.getStatus())+ || !deployment.getTargetServiceRevisionArn().endsWith("/" + taskSetId(svc))) {+ return false;+ }+ String failedId = deploymentId(svc);+ Instant now = Instant.now();+ deployment.setStatus(rollback ? DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS+ : DEPLOYMENT_STATUS_STOP_REQUESTED);+ deployment.setStatusReason(reason);+ deployment.setUpdatedAt(now);+ svc.setFailedDeploymentId(failedId);+ if (eventPublisher != null) {+ eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_FAILED", reason,+ serviceRegion(svc));+ }+ if (rollback) {+ ServiceRevision target = previousSuccessfulRevision(svc, deployment);+ if (target != null) {+ deployment.setRollbackTargetServiceRevisionArn(target.getServiceRevisionArn());+ deployment.setRollbackStartedAt(now);+ deployment.setRollbackReason(reason);+ restoreServiceRevision(svc, target);+ String targetId = deploymentId(svc);+ inProgressEmitted.add(targetId);+ if (eventPublisher != null) {+ eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_IN_PROGRESS",+ "ECS deployment " + failedId + " is rolling back to deployment "+ + targetId + ".", serviceRegion(svc));+ }+ }+ }+ }+ persistServiceModel(svc);+ return true;+ }+ }++ private ServiceRevision previousSuccessfulRevision(EcsServiceModel svc, ServiceDeployment failedDeployment) {+ ServiceDeployment successful = latestServiceDeployment(svc.getServiceArn(), candidate ->+ candidate != failedDeployment && DEPLOYMENT_STATUS_SUCCESSFUL.equals(candidate.getStatus()));+ ServiceRevision previous = successful == null ? null+ : serviceRevisions.get(successful.getTargetServiceRevisionArn());+ if (previous != null) {+ return previous;+ }+ ServiceRevision retained = svc.getLastSuccessfulServiceRevision();+ return retained != null && svc.getServiceArn().equals(retained.getServiceArn())+ && !Objects.equals(failedDeployment.getTargetServiceRevisionArn(),+ retained.getServiceRevisionArn()) ? retained : null;+ }++ private void restoreServiceRevision(EcsServiceModel svc, ServiceRevision revision) {+ String revisionArn = revision.getServiceRevisionArn();+ String taskSetId = taskSetId(revisionArn);+ svc.setTaskDefinition(revision.getTaskDefinition());+ svc.setLaunchType(revision.getLaunchType());+ svc.setCapacityProviderStrategy(revision.getCapacityProviderStrategy());+ svc.setPlatformVersion(revision.getPlatformVersion());+ svc.setPlatformFamily(revision.getPlatformFamily());+ svc.setLoadBalancers(revision.getLoadBalancers());+ svc.setServiceRegistries(revision.getServiceRegistries());+ svc.setNetworkConfiguration(revision.getNetworkConfiguration());+ svc.setServiceConnectConfiguration(revision.getServiceConnectConfiguration());+ svc.setDeploymentId("ecs-svc/" + taskSetId);+ svc.setLastDeploymentAt(revision.getCreatedAt());+ svc.setLastCompletedDeploymentId(null);+ }++ private boolean progressPendingDeploymentFailure(String key, EcsServiceModel svc) {+ ServiceDeployment pending = latestServiceDeployment(svc.getServiceArn(), deployment ->+ DEPLOYMENT_STATUS_STOP_REQUESTED.equals(deployment.getStatus())+ || DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS.equals(deployment.getStatus()));+ if (pending == null) {+ return false;+ }+ if (DEPLOYMENT_STATUS_STOP_REQUESTED.equals(pending.getStatus())) {Instant now = Instant.now();- deployment.setStatus(DEPLOYMENT_STATUS_STOPPED);- deployment.setStatusReason(CIRCUIT_BREAKER_REASON);+ synchronized (pending) {+ if (DEPLOYMENT_STATUS_STOP_REQUESTED.equals(pending.getStatus())) {+ pending.setStatus(DEPLOYMENT_STATUS_STOPPED);+ pending.setFinishedAt(now);+ pending.setStoppedAt(now);+ pending.setUpdatedAt(now);+ }+ }+ services.put(key, svc);+ return true;+ }+ if (pending.getRollbackTargetServiceRevisionArn() == null) {+ finishRollback(pending, false,+ "Rollback failed because no successful service revision is available to restore.");+ services.put(key, svc);+ return true;+ }+ if (runningOnCurrentDeployment(svc) >= svc.getDesiredCount()+ && !hasRunningTasksFromFailedDeployment(svc, pending)) {+ finishRollback(pending, true, null);+ services.put(key, svc);+ }+ return false;+ }++ private boolean hasRunningTasksFromFailedDeployment(EcsServiceModel svc, ServiceDeployment deployment) {+ String targetRevisionArn = deployment.getTargetServiceRevisionArn();+ if (targetRevisionArn == null) {+ return false;+ }+ String failedTaskSetId = taskSetId(targetRevisionArn);+ String failedDeploymentId = "ecs-svc/" + failedTaskSetId;+ EcsCluster cluster = resolveClusterByArn(svc.getClusterArn());+ if (cluster == null) {+ return false;+ }+ return tasks.values().stream()+ .filter(task -> ownedBy(task, svc, cluster))+ .filter(task -> !TaskStatus.STOPPED.name().equals(task.getLastStatus()))+ .anyMatch(task -> failedDeploymentId.equals(task.getDeploymentId())+ || (task.getDeploymentId() == null+ && Objects.equals(deployment.getTaskDefinition(), task.getTaskDefinitionArn())));+ }++ private void finishRollback(ServiceDeployment deployment, boolean successful, String detail) {+ synchronized (deployment) {+ if (!DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS.equals(deployment.getStatus())) {+ return;+ }+ Instant now = Instant.now();+ deployment.setStatus(successful ? DEPLOYMENT_STATUS_ROLLBACK_SUCCESSFUL+ : DEPLOYMENT_STATUS_ROLLBACK_FAILED);+ if (detail != null) {+ deployment.setStatusReason(deployment.getStatusReason() + " " + detail);+ }deployment.setFinishedAt(now);- deployment.setStoppedAt(now);deployment.setUpdatedAt(now);- return true;}}@@ -4224,6 +4457,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabcopy.setUpdatedAt(source.getUpdatedAt());copy.setTargetServiceRevisionArn(source.getTargetServiceRevisionArn());copy.setSourceServiceRevisionArns(source.getSourceServiceRevisionArns());+ copy.setRollbackTargetServiceRevisionArn(source.getRollbackTargetServiceRevisionArn());+ copy.setRollbackStartedAt(source.getRollbackStartedAt());+ copy.setRollbackReason(source.getRollbackReason());+ copy.setAlarmNames(source.getAlarmNames());+ copy.setTriggeredAlarmNames(source.getTriggeredAlarmNames());+ copy.setAlarmStatus(source.getAlarmStatus());copy.setFailedTasks(source.getFailedTasks());return copy;}@@ -4775,6 +5014,27 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}String region = extractRegionFromServiceKey(key);+ if (progressPendingDeploymentFailure(key, svc)) {+ return;+ }+ ServiceDeployment deployment = deploymentRecordForReconciliation(svc, key, region);+ if (deployment != null && DEPLOYMENT_STATUS_IN_PROGRESS.equals(deployment.getStatus())) {+ synchronized (svc) {+ refreshDeploymentAlarms(deployment, svc);+ List<String> triggeredAlarms = triggeredDeploymentAlarms(svc, region);+ if (!triggeredAlarms.isEmpty()) {+ Map<String, Object> alarms = deploymentAlarmsOf(svc);+ synchronized (deployment) {+ deployment.setTriggeredAlarmNames(triggeredAlarms);+ deployment.setAlarmStatus("TRIGGERED");+ }+ requestDeploymentFailure(svc, deployment,+ ALARM_FAILURE_REASON + " Alarms: " + String.join(", ", triggeredAlarms),+ flag(alarms.get("rollback"), false));+ return;+ }+ }+ }String clusterName = extractClusterNameFromServiceKey(key);if (SCHEDULING_DAEMON.equals(svc.getSchedulingStrategy())) {@@ -4800,7 +5060,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabif (eventPublisher != null) {String deploymentId = currentDeploymentId;boolean converged = current >= svc.getDesiredCount();- if (converged) {+ ServiceDeployment pendingFailure = latestServiceDeployment(svc.getServiceArn(), candidate ->+ DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS.equals(candidate.getStatus()));+ if (converged && pendingFailure == null) {if (!deploymentId.equals(svc.getLastCompletedDeploymentId())) {svc.setLastCompletedDeploymentId(deploymentId);services.put(key, svc);@@ -4822,7 +5084,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabif (currentDeploymentId.equals(svc.getFailedDeploymentId())) {return;}- ServiceDeployment deployment = deploymentRecordOf(svc);+ deployment = deploymentRecordForCurrentWorkload(svc);int toStart = svc.getDesiredCount() - (int) current;for (int i = 0; i < toStart; i++) {try {@@ -4830,8 +5092,6 @@ public class EcsService implements ContainerTeardown, ResourceProvider, ResettabLOG.infov("Service reconciler started task {0} for service {1}",launched.getTaskArn(), svc.getServiceName());if (countLaunch(svc, deployment, launched)) {- svc.setFailedDeploymentId(currentDeploymentId);- services.put(key, svc);break;}} catch (Exception e) {@@ -4867,6 +5127,98 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}}+ private ServiceDeployment deploymentRecordForCurrentWorkload(EcsServiceModel svc) {+ String currentTaskSetId = taskSetId(svc);+ ServiceDeployment rollingBack = latestServiceDeployment(svc.getServiceArn(), deployment ->+ DEPLOYMENT_STATUS_ROLLBACK_IN_PROGRESS.equals(deployment.getStatus())+ && deployment.getRollbackTargetServiceRevisionArn() != null+ && deployment.getRollbackTargetServiceRevisionArn().endsWith("/" + currentTaskSetId));+ return rollingBack != null ? rollingBack : deploymentRecordOf(svc);+ }++ private ServiceDeployment latestServiceDeployment(String serviceArn,+ Predicate<ServiceDeployment> filter) {+ return serviceDeployments.values().stream()+ .filter(deployment -> Objects.equals(serviceArn, deployment.getServiceArn()))+ .filter(filter)+ .max(Comparator.comparing(ServiceDeployment::getCreatedAt,+ Comparator.nullsFirst(Comparator.naturalOrder())))+ .orElse(null);+ }++ private ServiceDeployment deploymentRecordForReconciliation(EcsServiceModel svc, String key, String region) {+ ServiceDeployment deployment = deploymentRecordOf(svc);+ String currentDeploymentId = deploymentId(svc);+ if (deployment != null || currentDeploymentId.equals(svc.getLastSettledDeploymentId())+ || currentDeploymentId.equals(svc.getLastCompletedDeploymentId())+ || currentDeploymentId.equals(svc.getFailedDeploymentId())) {+ return deployment;+ }++ Instant createdAt = svc.getLastDeploymentAt() != null ? svc.getLastDeploymentAt() : svc.getCreatedAt();+ String revisionArn = regionResolver.buildArn("ecs", region,+ "service-revision/" + taskSetId(svc));+ ServiceRevision revision = newServiceRevision(svc, pinnedTaskDefinitionArn(svc, key, region),+ revisionArn, createdAt, region);+ serviceRevisions.putIfAbsent(revisionArn, revision);++ ServiceDeployment recovered = newServiceDeployment(svc, revision.getTaskDefinition(),+ regionResolver.buildArn("ecs", region,+ "service-deployment/" + UUID.randomUUID().toString().replace("-", "")),+ revisionArn, createdAt);+ recovered.setSourceServiceRevisionArns(List.of());+ ServiceDeployment existing = serviceDeployments.putIfAbsent(+ recovered.getServiceDeploymentArn(), recovered);+ return existing == null ? recovered : existing;+ }++ @SuppressWarnings("unchecked")+ private static Map<String, Object> deploymentAlarmsOf(EcsServiceModel svc) {+ Map<String, Object> config = svc.getDeploymentConfiguration();+ Object alarms = config == null ? null : config.get("alarms");+ return alarms instanceof Map<?, ?> map ? (Map<String, Object>) map : Map.of();+ }++ private void refreshDeploymentAlarms(ServiceDeployment deployment, EcsServiceModel svc) {+ Map<String, Object> alarms = deploymentAlarmsOf(svc);+ synchronized (deployment) {+ if (!flag(alarms.get("enable"), false)+ || !(alarms.get("alarmNames") instanceof List<?> names)) {+ deployment.setAlarmNames(null);+ deployment.setTriggeredAlarmNames(null);+ deployment.setAlarmStatus(null);+ return;+ }+ List<String> currentNames = names.stream().filter(String.class::isInstance)+ .map(String.class::cast).toList();+ if (!currentNames.equals(deployment.getAlarmNames())) {+ deployment.setAlarmNames(currentNames);+ deployment.setTriggeredAlarmNames(List.of());+ deployment.setAlarmStatus("MONITORING");+ }+ }+ }++ private List<String> triggeredDeploymentAlarms(EcsServiceModel svc, String region) {+ Map<String, Object> alarms = deploymentAlarmsOf(svc);+ if (cloudWatchMetricsService == null || !flag(alarms.get("enable"), false)+ || !(alarms.get("alarmNames") instanceof List<?> rawNames)) {+ return List.of();+ }+ List<String> alarmNames = rawNames.stream()+ .filter(String.class::isInstance)+ .map(String.class::cast)+ .toList();+ if (alarmNames.isEmpty()) {+ return List.of();+ }+ List<String> alarmsInAlarm = cloudWatchMetricsService.describeAlarms(alarmNames, null, region).stream()+ .filter(alarm -> "ALARM".equals(alarm.getStateValue()))+ .map(MetricAlarm::getAlarmName)+ .toList();+ return alarmsInAlarm;+ }+// ── Resolution helpers ────────────────────────────────────────────────────private EcsCluster getOrCreateDefaultCluster(String region) {diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/model/EcsServiceModel.java b/src/main/java/io/github/hectorvent/floci/services/ecs/model/EcsServiceModel.javaindex 1032bddb20..135abc515e 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/model/EcsServiceModel.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/model/EcsServiceModel.java@@ -26,10 +26,14 @@ public class EcsServiceModel {private Instant lastDeploymentAt;/** Current deployment identifier ("ecs-svc/<hex>"). Rolls on a task-definition change or forceNewDeployment. */private String deploymentId;- /** The deploymentId last observed to reach steady state; guards against re-emitting COMPLETED. */+ /** The deployment whose completion event has been emitted. */private String lastCompletedDeploymentId;+ /** Last deployment to reach steady state, persisted to avoid resuming it after a restart. */+ private String lastSettledDeploymentId;/** The deploymentId the circuit breaker failed; persisted so a restart does not resume it. */private String failedDeploymentId;+ /** Last completed revision, retained so a rollback can survive a process restart. */+ private ServiceRevision lastSuccessfulServiceRevision;private String namespace;private String deploymentController;private String schedulingStrategy;@@ -101,6 +105,16 @@ public class EcsServiceModel {this.lastCompletedDeploymentId = lastCompletedDeploymentId;}+ public String getLastSettledDeploymentId() { return lastSettledDeploymentId; }+ public void setLastSettledDeploymentId(String lastSettledDeploymentId) {+ this.lastSettledDeploymentId = lastSettledDeploymentId;+ }++ public ServiceRevision getLastSuccessfulServiceRevision() { return lastSuccessfulServiceRevision; }+ public void setLastSuccessfulServiceRevision(ServiceRevision lastSuccessfulServiceRevision) {+ this.lastSuccessfulServiceRevision = lastSuccessfulServiceRevision;+ }+public String getFailedDeploymentId() { return failedDeploymentId; }public void setFailedDeploymentId(String failedDeploymentId) { this.failedDeploymentId = failedDeploymentId; }diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java b/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.javaindex 682aba2a0d..0541a3d760 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/model/ServiceDeployment.java@@ -28,6 +28,13 @@ public class ServiceDeployment {private Instant updatedAt;private String targetServiceRevisionArn;private List<String> sourceServiceRevisionArns;+ /** Internal revision restored if this deployment rolls back. */+ private String rollbackTargetServiceRevisionArn;+ private Instant rollbackStartedAt;+ private String rollbackReason;+ private List<String> alarmNames;+ private List<String> triggeredAlarmNames;+ private String alarmStatus;/** Tasks the circuit breaker has counted as failing to start. */private int failedTasks;@@ -53,6 +60,28 @@ public class ServiceDeployment {this.sourceServiceRevisionArns = sourceServiceRevisionArns;}+ public String getRollbackTargetServiceRevisionArn() { return rollbackTargetServiceRevisionArn; }+ public void setRollbackTargetServiceRevisionArn(String rollbackTargetServiceRevisionArn) {+ this.rollbackTargetServiceRevisionArn = rollbackTargetServiceRevisionArn;+ }++ public Instant getRollbackStartedAt() { return rollbackStartedAt; }+ public void setRollbackStartedAt(Instant rollbackStartedAt) { this.rollbackStartedAt = rollbackStartedAt; }++ public String getRollbackReason() { return rollbackReason; }+ public void setRollbackReason(String rollbackReason) { this.rollbackReason = rollbackReason; }++ public List<String> getAlarmNames() { return alarmNames; }+ public void setAlarmNames(List<String> alarmNames) { this.alarmNames = alarmNames; }++ public List<String> getTriggeredAlarmNames() { return triggeredAlarmNames; }+ public void setTriggeredAlarmNames(List<String> triggeredAlarmNames) {+ this.triggeredAlarmNames = triggeredAlarmNames;+ }++ public String getAlarmStatus() { return alarmStatus; }+ public void setAlarmStatus(String alarmStatus) { this.alarmStatus = alarmStatus; }+public int getFailedTasks() { return failedTasks; }public void setFailedTasks(int failedTasks) { this.failedTasks = failedTasks; }diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsEventPublisherTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsEventPublisherTest.javaindex 4fc9349822..95ecdcd99b 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsEventPublisherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsEventPublisherTest.java@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.services.ecs.model.Container;import io.github.hectorvent.floci.services.ecs.model.EcsTask;+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;import io.github.hectorvent.floci.services.ecs.model.TaskStatus;import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;import org.junit.jupiter.api.Test;@@ -98,4 +99,21 @@ class EcsEventPublisherTest {publisher.emitTaskLadder(runningTask(), TaskStatus.PENDING, TaskStatus.RUNNING, REGION);// no exception propagates}++ @Test+ void failedDeploymentEventUsesErrorType() throws Exception {+ EventBridgeService eb = mock(EventBridgeService.class);+ EcsEventPublisher publisher = new EcsEventPublisher(eb,+ new RegionResolver(REGION, "000000000000"), objectMapper);+ EcsServiceModel svc = new EcsServiceModel();+ svc.setServiceArn("arn:aws:ecs:us-east-1:000000000000:service/c/s");+ svc.setDeploymentId("ecs-svc/failed");++ publisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_FAILED", "Alarm triggered", REGION);++ JsonNode detail = objectMapper.readTree((String) capture(eb, 1).getFirst().get("Detail"));+ assertEquals("ERROR", detail.path("eventType").asText());+ assertEquals("SERVICE_DEPLOYMENT_FAILED", detail.path("eventName").asText());+ assertEquals("ecs-svc/failed", detail.path("deploymentId").asText());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriterDeploymentTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriterDeploymentTest.javanew file mode 100644index 0000000000..9ca454ed77--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsResponseWriterDeploymentTest.java@@ -0,0 +1,37 @@+package io.github.hectorvent.floci.services.ecs;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.ecs.model.ServiceDeployment;+import org.junit.jupiter.api.Test;++import java.time.Instant;+import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.mockito.Mockito.mock;++class EcsResponseWriterDeploymentTest {++ @Test+ void deploymentIncludesRollbackAndTriggeredAlarms() {+ ServiceDeployment deployment = new ServiceDeployment();+ deployment.setRollbackTargetServiceRevisionArn("revision/previous");+ deployment.setRollbackReason("Deployment alarm triggered");+ deployment.setRollbackStartedAt(Instant.parse("2026-10-01T00:00:00Z"));+ deployment.setAlarmNames(List.of("cpu-high", "memory-high"));+ deployment.setTriggeredAlarmNames(List.of("cpu-high"));+ deployment.setAlarmStatus("TRIGGERED");++ JsonNode node = new EcsResponseWriter(mock(EcsService.class), new ObjectMapper())+ .serviceDeploymentNode(deployment);++ assertEquals("revision/previous", node.path("rollback").path("serviceRevisionArn").asText());+ assertEquals("Deployment alarm triggered", node.path("rollback").path("reason").asText());+ assertFalse(node.path("rollback").path("startedAt").isMissingNode());+ assertEquals("cpu-high", node.path("alarms").path("triggeredAlarmNames").get(0).asText());+ assertEquals(2, node.path("alarms").path("alarmNames").size());+ assertEquals("TRIGGERED", node.path("alarms").path("status").asText());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentCircuitBreakerTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentCircuitBreakerTest.javaindex 64546e5aaf..132a814f53 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentCircuitBreakerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentCircuitBreakerTest.java@@ -6,8 +6,11 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;+import io.github.hectorvent.floci.services.ecs.exec.EcsExecSessionRegistry;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.CreateServiceRequest;import io.github.hectorvent.floci.services.ecs.model.Deployment;@@ -15,7 +18,9 @@ import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.LaunchType;import io.github.hectorvent.floci.services.ecs.model.ServiceDeployment;+import io.github.hectorvent.floci.services.ecs.model.ServiceRevision;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;+import io.github.hectorvent.floci.services.ecs.model.UpdateServiceRequest;import org.junit.jupiter.api.Test;import java.util.ArrayDeque;@@ -30,8 +35,12 @@ import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.argThat;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;/**@@ -64,8 +73,13 @@ class EcsServiceDeploymentCircuitBreakerTest {service.reconcileServices();ServiceDeployment failed = deploymentOf(service, "cb-fail", id);+ assertEquals("STOP_REQUESTED", failed.getStatus());+ assertNull(failed.getFinishedAt());++ service.reconcileServices();+ failed = deploymentOf(service, "cb-fail", id);assertEquals("STOPPED", failed.getStatus(),- "without rollback a failed deployment is stopped; there is no FAILED status");+ "without rollback a failed deployment transitions through STOP_REQUESTED");assertNotNull(failed.getFinishedAt());assertNotNull(failed.getStoppedAt());assertEquals("The deployment circuit breaker detected a failure.", failed.getStatusReason());@@ -93,7 +107,7 @@ class EcsServiceDeploymentCircuitBreakerTest {service.reconcileServices();- assertEquals("STOPPED", deploymentOf(service, "cb-ten", id).getStatus());+ assertEquals("STOP_REQUESTED", deploymentOf(service, "cb-ten", id).getStatus());assertEquals(5, stoppedTasks(service).size(),"the launch that reaches the threshold is the last one");assertEquals(5, liveDeployment(service, "cb-ten").getFailedTasks());@@ -107,7 +121,7 @@ class EcsServiceDeploymentCircuitBreakerTest {service.reconcileServices();- assertEquals("STOPPED", deploymentOf(service, "cb-ceil", id).getStatus());+ assertEquals("STOP_REQUESTED", deploymentOf(service, "cb-ceil", id).getStatus());assertEquals(4, liveDeployment(service, "cb-ceil").getFailedTasks());}@@ -119,7 +133,7 @@ class EcsServiceDeploymentCircuitBreakerTest {service.reconcileServices();- assertEquals("STOPPED", deploymentOf(service, "cb-cap", id).getStatus());+ assertEquals("STOP_REQUESTED", deploymentOf(service, "cb-cap", id).getStatus());assertEquals(200, liveDeployment(service, "cb-cap").getFailedTasks());}@@ -133,7 +147,7 @@ class EcsServiceDeploymentCircuitBreakerTest {service.reconcileServices();- assertEquals("STOPPED", deploymentOf(service, "cb-unb", id).getStatus());+ assertEquals("STOP_REQUESTED", deploymentOf(service, "cb-unb", id).getStatus());assertEquals(201, liveDeployment(service, "cb-unb").getFailedTasks());}@@ -150,7 +164,7 @@ class EcsServiceDeploymentCircuitBreakerTest {"four failures, and the bounded default for 2 would already have tripped at 3");service.reconcileServices();- assertEquals("STOPPED", deploymentOf(service, "cb-count", id).getStatus());+ assertEquals("STOP_REQUESTED", deploymentOf(service, "cb-count", id).getStatus());assertEquals(5, stoppedTasks(service).size());}@@ -185,23 +199,191 @@ class EcsServiceDeploymentCircuitBreakerTest {assertEquals("IN_PROGRESS", liveDeployment(service, "cb-none").getRolloutState());}- /**- * With rollback on, AWS rolls back rather than stopping (ROLLBACK_IN_PROGRESS, then- * ROLLBACK_SUCCESSFUL or ROLLBACK_FAILED). Rollback is not modelled, so the deployment must- * not claim the STOPPED outcome that only applies without it.- */@Test- void withRollbackOnTheDeploymentIsNotReportedStopped() {+ void anEnabledCloudWatchAlarmStopsTheDeploymentWithAReason() {+ CloudWatchMetricsService metricsService = mock(CloudWatchMetricsService.class);+ MetricAlarm alarm = new MetricAlarm();+ alarm.setAlarmName("deployment-failure");+ alarm.setStateValue("ALARM");+ when(metricsService.describeAlarms(List.of("deployment-failure"), null, REGION))+ .thenReturn(List.of(alarm));+ EcsService service = newService(new InMemoryStorageFactory(), metricsService);+ EcsServiceModel model = createServiceWithDeploymentConfiguration(service, "cb-alarm", 1,+ Map.of("alarms", Map.of(+ "enable", true,+ "rollback", false,+ "alarmNames", List.of("deployment-failure"))));+ String id = model.getDeploymentId();++ service.reconcileServices();++ ServiceDeployment deployment = deploymentOf(service, "cb-alarm", id);+ assertEquals("STOP_REQUESTED", deployment.getStatus());+ assertTrue(deployment.getStatusReason().contains("deployment-failure"));+ assertEquals(List.of("deployment-failure"), deployment.getAlarmNames());+ assertEquals(List.of("deployment-failure"), deployment.getTriggeredAlarmNames());+ assertEquals("TRIGGERED", deployment.getAlarmStatus());+ assertTrue(stoppedTasks(service).isEmpty(), "an active alarm is checked before starting another task");++ service.reconcileServices();+ assertEquals("STOPPED", deploymentOf(service, "cb-alarm", id).getStatus());+ }++ @Test+ void updatedAlarmNamesMatchTheNamesReportedOnTheDeployment() {+ CloudWatchMetricsService metricsService = mock(CloudWatchMetricsService.class);+ MetricAlarm alarm = new MetricAlarm();+ alarm.setAlarmName("new-alarm");+ alarm.setStateValue("ALARM");+ when(metricsService.describeAlarms(List.of("new-alarm"), null, REGION))+ .thenReturn(List.of(alarm));+ EcsService service = newService(new InMemoryStorageFactory(), metricsService);+ EcsServiceModel model = createServiceWithDeploymentConfiguration(service, "cb-alarm-update", 1,+ Map.of("alarms", Map.of("enable", true, "rollback", false,+ "alarmNames", List.of("old-alarm"))));+ UpdateServiceRequest request = new UpdateServiceRequest();+ request.setCluster("cb-alarm-update-cluster");+ request.setService("cb-alarm-update");+ request.setDeploymentConfiguration(Map.of("alarms", Map.of("enable", true, "rollback", false,+ "alarmNames", List.of("new-alarm"))));+ service.updateService(request, REGION);++ service.reconcileServices();++ ServiceDeployment deployment = deploymentOf(service, "cb-alarm-update", model.getDeploymentId());+ assertEquals("STOP_REQUESTED", deployment.getStatus());+ assertEquals(List.of("new-alarm"), deployment.getAlarmNames());+ assertEquals(List.of("new-alarm"), deployment.getTriggeredAlarmNames());+ }++ /** A failed initial deployment has no successful revision to roll back to. */+ @Test+ void withRollbackOnAnInitialDeploymentEndsAsRollbackFailed() {EcsService service = newService();String id = createService(service, "cb-rb", 1, breaker(true, true)).getDeploymentId();- for (int i = 0; i < 5; i++) {+ for (int i = 0; i < 3; i++) {service.reconcileServices();}ServiceDeployment deployment = deploymentOf(service, "cb-rb", id);- assertEquals("IN_PROGRESS", deployment.getStatus());+ assertEquals("ROLLBACK_IN_PROGRESS", deployment.getStatus());assertNull(deployment.getStoppedAt());++ service.reconcileServices();+ deployment = deploymentOf(service, "cb-rb", id);+ assertEquals("ROLLBACK_FAILED", deployment.getStatus());+ assertNotNull(deployment.getFinishedAt());+ assertTrue(deployment.getStatusReason().contains("Rollback"));+ }++ @Test+ void rollbackRestoresTheMostRecentSuccessfulRevision() {+ EcsEventPublisher publisher = mock(EcsEventPublisher.class);+ EcsService service = newService(new InMemoryStorageFactory(), null, publisher);+ healthy = true;+ String successfulDeploymentId = createService(service, "cb-rollback", 1, breaker(true, true))+ .getDeploymentId();+ service.reconcileServices();+ service.reconcileServices();+ EcsTask original = runningTasks(service).getFirst();+ String originalTaskDefinition = service.serviceByArn(original.getOwningServiceArn()).getTaskDefinition();++ healthy = false;+ TaskDefinition broken = registerTaskDef(service, "cb-rollback-fam", "app:broken");+ String failedDeploymentId = service.updateService("cb-rollback-cluster", "cb-rollback",+ "cb-rollback-fam:" + broken.getRevision(), null, null, REGION).getDeploymentId();+ for (int i = 0; i < 3; i++) {+ service.reconcileServices();+ }++ ServiceDeployment rollingBack = deploymentOf(service, "cb-rollback", failedDeploymentId);+ assertEquals("ROLLBACK_IN_PROGRESS", rollingBack.getStatus());+ assertNotNull(rollingBack.getRollbackStartedAt());+ assertNotNull(rollingBack.getRollbackReason());+ assertEquals(originalTaskDefinition,+ service.serviceByArn(rollingBack.getServiceArn()).getTaskDefinition());+ assertEquals("IN_PROGRESS", liveDeployment(service, "cb-rollback").getRolloutState());+ assertEquals(deploymentOf(service, "cb-rollback", successfulDeploymentId)+ .getTargetServiceRevisionArn(), rollingBack.getRollbackTargetServiceRevisionArn());+ verify(publisher).emitDeploymentStateChange(any(), eq("SERVICE_DEPLOYMENT_FAILED"),+ anyString(), eq(REGION));+ verify(publisher).emitDeploymentStateChange(any(), eq("SERVICE_DEPLOYMENT_IN_PROGRESS"),+ argThat(reason -> reason.contains("rolling back to deployment " + successfulDeploymentId)),+ eq(REGION));++ service.reconcileServices();+ assertEquals("ROLLBACK_SUCCESSFUL",+ deploymentOf(service, "cb-rollback", failedDeploymentId).getStatus());+ assertEquals(List.of(original.getTaskArn()), runningTasks(service).stream().map(EcsTask::getTaskArn).toList());+ assertEquals("SUCCESSFUL", deploymentOf(service, "cb-rollback", successfulDeploymentId).getStatus());+ assertEquals("COMPLETED", liveDeployment(service, "cb-rollback").getRolloutState());+ verify(publisher, times(2)).emitDeploymentStateChange(any(), eq("SERVICE_DEPLOYMENT_COMPLETED"),+ argThat(reason -> reason.contains(successfulDeploymentId)), eq(REGION));+ }++ @Test+ void rollbackKeepsTheCurrentDesiredCountAfterAScaleChange() {+ EcsService service = newService();+ healthy = true;+ createService(service, "cb-scale-rollback", 1, breaker(true, true));+ service.reconcileServices();+ service.updateService("cb-scale-rollback-cluster", "cb-scale-rollback", null, 2, null, REGION);+ service.reconcileServices();+ assertEquals(2, runningTasks(service).size());++ healthy = false;+ TaskDefinition broken = registerTaskDef(service, "cb-scale-rollback-fam", "app:broken");+ String failedId = service.updateService("cb-scale-rollback-cluster", "cb-scale-rollback",+ "cb-scale-rollback-fam:" + broken.getRevision(), null, null, REGION).getDeploymentId();+ for (int i = 0; i < 2; i++) {+ service.reconcileServices();+ }++ assertEquals("ROLLBACK_IN_PROGRESS", deploymentOf(service, "cb-scale-rollback", failedId).getStatus());+ assertEquals(2, service.serviceByArn(deploymentOf(service, "cb-scale-rollback", failedId)+ .getServiceArn()).getDesiredCount(),+ "rollback must retain the desired count set independently of the failed deployment");+ service.reconcileServices();+ assertEquals("ROLLBACK_SUCCESSFUL", deploymentOf(service, "cb-scale-rollback", failedId).getStatus());+ }++ @Test+ void rollbackWaitsUntilRunningTasksFromTheFailedRevisionAreDrained() {+ CloudWatchMetricsService metricsService = mock(CloudWatchMetricsService.class);+ MetricAlarm alarm = new MetricAlarm();+ alarm.setAlarmName("rollback-drain-alarm");+ alarm.setStateValue("OK");+ when(metricsService.describeAlarms(List.of("rollback-drain-alarm"), null, REGION))+ .thenReturn(List.of(alarm));+ EcsService service = newService(new InMemoryStorageFactory(), metricsService);+ healthy = true;+ EcsServiceModel model = createService(service, "cb-drain", 2, breaker(true, true));+ model.setDeploymentConfiguration(Map.of(+ "deploymentCircuitBreaker", breaker(true, true),+ "alarms", Map.of("enable", true, "rollback", true,+ "alarmNames", List.of("rollback-drain-alarm"))));+ service.reconcileServices();+ assertEquals(2, runningTasks(service).size());++ TaskDefinition next = registerTaskDef(service, "cb-drain-fam", "app:next");+ String failedId = service.updateService("cb-drain-cluster", "cb-drain",+ "cb-drain-fam:" + next.getRevision(), null, null, REGION).getDeploymentId();+ launches.addAll(List.of(true, false));+ service.reconcileServices();+ assertEquals(3, runningTasks(service).size(), "old tasks and one new-revision task are still running");++ alarm.setStateValue("ALARM");+ service.reconcileServices();+ assertEquals("ROLLBACK_IN_PROGRESS", deploymentOf(service, "cb-drain", failedId).getStatus());+ service.reconcileServices();+ assertEquals("ROLLBACK_IN_PROGRESS", deploymentOf(service, "cb-drain", failedId).getStatus(),+ "rollback remains active while a task from its failed revision is still draining");+ assertEquals(2, runningTasks(service).size());++ service.reconcile();+ service.reconcileServices();+ assertEquals("ROLLBACK_SUCCESSFUL", deploymentOf(service, "cb-drain", failedId).getStatus());}/** Only a task that failed to start counts; one a user stopped while it pulled did not fail. */@@ -273,7 +455,7 @@ class EcsServiceDeploymentCircuitBreakerTest {service.reconcileServices();service.reconcileServices();- assertEquals("STOPPED", deploymentOf(service, "cb-cumul", id).getStatus(),+ assertEquals("STOP_REQUESTED", deploymentOf(service, "cb-cumul", id).getStatus(),"the third failure trips it although a task started in between");}@@ -347,6 +529,8 @@ class EcsServiceDeploymentCircuitBreakerTest {service.reconcileServices();}+ assertEquals("STOP_REQUESTED", deploymentOf(service, "cb-upd", second).getStatus());+ service.reconcileServices();assertEquals("STOPPED", deploymentOf(service, "cb-upd", second).getStatus());assertEquals("SUCCESSFUL", deploymentOf(service, "cb-upd", first).getStatus());assertEquals(List.of(original.getTaskArn()),@@ -390,6 +574,95 @@ class EcsServiceDeploymentCircuitBreakerTest {assertEquals("FAILED", liveDeployment(after, "cb-restart").getRolloutState());}+ @Test+ void anAlarmStillFailsAnInProgressDeploymentAfterARestart() {+ InMemoryStorageFactory storage = new InMemoryStorageFactory();+ CloudWatchMetricsService metricsService = mock(CloudWatchMetricsService.class);+ MetricAlarm alarm = new MetricAlarm();+ alarm.setAlarmName("restart-alarm");+ alarm.setStateValue("ALARM");+ when(metricsService.describeAlarms(List.of("restart-alarm"), null, REGION))+ .thenReturn(List.of(alarm));+ EcsService before = newService(storage, metricsService);+ EcsServiceModel model = createServiceWithDeploymentConfiguration(before, "cb-alarm-restart", 1,+ Map.of("alarms", Map.of("enable", true, "rollback", false,+ "alarmNames", List.of("restart-alarm"))));++ EcsService after = newService(storage, metricsService);+ after.reconcileServices();++ ServiceDeployment deployment = deploymentOf(after, "cb-alarm-restart", model.getDeploymentId());+ assertEquals("STOP_REQUESTED", deployment.getStatus());+ assertTrue(deployment.getStatusReason().contains("restart-alarm"));+ assertTrue(stoppedTasks(after).isEmpty(), "the alarm is checked before a task starts after restart");+ after.reconcileServices();+ assertEquals("STOPPED", deploymentOf(after, "cb-alarm-restart", model.getDeploymentId()).getStatus());+ }++ @Test+ void aCompletedDeploymentIsNotFailedByAnAlarmAfterARestart() {+ InMemoryStorageFactory storage = new InMemoryStorageFactory();+ CloudWatchMetricsService metricsService = mock(CloudWatchMetricsService.class);+ MetricAlarm alarm = new MetricAlarm();+ alarm.setAlarmName("completed-restart-alarm");+ alarm.setStateValue("OK");+ when(metricsService.describeAlarms(List.of("completed-restart-alarm"), null, REGION))+ .thenReturn(List.of(alarm));+ EcsService before = newService(storage, metricsService);+ healthy = true;+ EcsServiceModel model = createServiceWithDeploymentConfiguration(before, "cb-completed-restart", 1,+ Map.of("alarms", Map.of("enable", true, "rollback", true,+ "alarmNames", List.of("completed-restart-alarm"))));+ before.reconcileServices();+ assertEquals(model.getDeploymentId(), model.getLastSettledDeploymentId());++ alarm.setStateValue("ALARM");+ EcsService after = newService(storage, metricsService);+ after.reconcileServices();++ EcsServiceModel restarted = after.serviceByArn(model.getServiceArn());+ assertEquals(model.getDeploymentId(), restarted.getLastSettledDeploymentId());+ assertNull(restarted.getFailedDeploymentId(), "a completed deployment is not reclassified by a later alarm");+ assertEquals("COMPLETED", liveDeployment(after, "cb-completed-restart").getRolloutState());+ }++ @Test+ void aRollbackAfterRestartUsesThePersistedSuccessfulRevision() {+ InMemoryStorageFactory storage = new InMemoryStorageFactory();+ CloudWatchMetricsService metricsService = mock(CloudWatchMetricsService.class);+ MetricAlarm alarm = new MetricAlarm();+ alarm.setAlarmName("restart-rollback-alarm");+ alarm.setStateValue("OK");+ when(metricsService.describeAlarms(List.of("restart-rollback-alarm"), null, REGION))+ .thenReturn(List.of(alarm));+ EcsService before = newService(storage, metricsService);+ healthy = true;+ EcsServiceModel model = createServiceWithDeploymentConfiguration(before, "cb-rollback-restart", 1,+ Map.of("alarms", Map.of("enable", true, "rollback", true,+ "alarmNames", List.of("restart-rollback-alarm"))));+ before.reconcileServices();+ String successfulTaskDefinition = model.getTaskDefinition();++ TaskDefinition next = registerTaskDef(before, "cb-rollback-restart-fam", "app:next");+ String failedId = before.updateService("cb-rollback-restart-cluster", "cb-rollback-restart",+ "cb-rollback-restart-fam:" + next.getRevision(), null, null, REGION).getDeploymentId();+ alarm.setStateValue("ALARM");+ EcsService after = newService(storage, metricsService);+ after.reconcileServices();++ ServiceDeployment failed = deploymentOf(after, "cb-rollback-restart", failedId);+ assertEquals("ROLLBACK_IN_PROGRESS", failed.getStatus());+ assertEquals(successfulTaskDefinition, after.serviceByArn(model.getServiceArn()).getTaskDefinition());+ ServiceRevision recoveredRevision = after.describeServiceRevisions(+ List.of(failed.getTargetServiceRevisionArn())).getFirst();+ assertEquals(1, recoveredRevision.getContainerImages().size(),+ "the recovered revision keeps its container image details");++ after.reconcileServices();+ after.reconcileServices();+ assertEquals("ROLLBACK_SUCCESSFUL", deploymentOf(after, "cb-rollback-restart", failedId).getStatus());+ }+// ── helpers ──────────────────────────────────────────────────────────────private static Map<String, Object> breaker(boolean enable, boolean rollback) {@@ -401,6 +674,14 @@ class EcsServiceDeploymentCircuitBreakerTest {private static EcsServiceModel createService(EcsService service, String name, int desiredCount,Map<String, Object> circuitBreaker) {+ Map<String, Object> deploymentConfiguration = circuitBreaker == null ? null+ : Map.of("deploymentCircuitBreaker", circuitBreaker);+ return createServiceWithDeploymentConfiguration(service, name, desiredCount, deploymentConfiguration);+ }++ private static EcsServiceModel createServiceWithDeploymentConfiguration(EcsService service, String name,+ int desiredCount,+ Map<String, Object> deploymentConfiguration) {service.createCluster(name + "-cluster", REGION);registerTaskDef(service, name + "-fam", "app:1");CreateServiceRequest request = new CreateServiceRequest();@@ -409,9 +690,7 @@ class EcsServiceDeploymentCircuitBreakerTest {request.setTaskDefinition(name + "-fam");request.setDesiredCount(desiredCount);request.setLaunchType(LaunchType.FARGATE);- if (circuitBreaker != null) {- request.setDeploymentConfiguration(Map.of("deploymentCircuitBreaker", circuitBreaker));- }+ request.setDeploymentConfiguration(deploymentConfiguration);return service.createService(request, REGION);}@@ -456,10 +735,24 @@ class EcsServiceDeploymentCircuitBreakerTest {/** A service over {@code storage}; a second one over the same storage is a restart. */private EcsService newService(InMemoryStorageFactory storage) {+ return newService(storage, null);+ }++ private EcsService newService(InMemoryStorageFactory storage, CloudWatchMetricsService metricsService) {+ return newService(storage, metricsService, null);+ }++ private EcsService newService(InMemoryStorageFactory storage, CloudWatchMetricsService metricsService,+ EcsEventPublisher eventPublisher) {EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);when(config.services().ecs().mock()).thenReturn(false);when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");EcsContainerManager containerManager = mock(EcsContainerManager.class);+ when(containerManager.stopTaskAndCollectExitCodes(any())).thenAnswer(invocation -> {+ EcsTaskHandle handle = invocation.getArgument(0);+ handle.getContainerIds().keySet().forEach(handle::recordContainerRemoved);+ return Map.of("app", 0);+ });when(containerManager.startTask(any(), any(), any(), anyString())).thenAnswer(invocation -> {Boolean scripted = launches.poll();if (scripted != null ? !scripted : !healthy) {@@ -478,7 +771,10 @@ class EcsServiceDeploymentCircuitBreakerTest {config,mock(EcsLoadBalancerRegistrar.class),storage,- null);+ eventPublisher,+ new EcsExecSessionRegistry(),+ null,+ metricsService);service.initializeStorage();return service;}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRolloutTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRolloutTest.javaindex 06a30cb1e3..c766724793 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRolloutTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRolloutTest.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.LaunchType;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;@@ -27,6 +28,7 @@ import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;import static org.mockito.Mockito.atLeastOnce;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -194,15 +196,18 @@ class EcsServiceRolloutTest {LaunchType.FARGATE, List.of(), null, REGION);verify(publisher).emitDeploymentStateChange(any(), eq("SERVICE_DEPLOYMENT_STARTED"), any(), eq(REGION));- service.reconcileServices(); // task starts, still converging or converged same tick- service.reconcileServices(); // converged- verify(publisher, atLeastOnce())+ service.reconcileServices(); // task starts and can become RUNNING in this tick+ verify(publisher, never())+ .emitDeploymentStateChange(any(), eq("SERVICE_DEPLOYMENT_COMPLETED"), any(), eq(REGION));+ service.reconcileServices(); // completion event is emitted on the next reconciliation+ verify(publisher, times(1)).emitDeploymentStateChange(any(), eq("SERVICE_DEPLOYMENT_COMPLETED"), any(), eq(REGION));// COMPLETED is not re-emitted on further steady-state ticks.service.reconcileServices();verify(publisher, times(1)).emitDeploymentStateChange(any(), eq("SERVICE_DEPLOYMENT_COMPLETED"), any(), eq(REGION));+}private static List<EcsTask> runningTasks(EcsService service) {diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.javaindex b11778a8cd..de2dcba144 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java@@ -1,8 +1,11 @@package io.github.hectorvent.floci.services.ecs;+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;import io.restassured.response.Response;+import jakarta.inject.Inject;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;@@ -43,6 +46,12 @@ import static org.junit.jupiter.api.Assertions.fail;@QuarkusTestclass EcsServiceSteadyStateWaiterIntegrationTest {+ @Inject+ EcsService ecsService;++ @Inject+ CloudWatchMetricsService cloudWatchMetricsService;+private static final String TARGET = "AmazonEC2ContainerServiceV20141113.";private static final String CT = "application/x-amz-json-1.1";private static final String CLUSTER = "waiter-cluster";@@ -203,4 +212,43 @@ class EcsServiceSteadyStateWaiterIntegrationTest {}fail("the reconciler converged the service before every one of five updates");}++ @Test+ void anAlarmFailureBecomesTerminalAndMarksTheServiceRolloutFailed() {+ String alarmName = "waiter-alarm-failure";+ MetricAlarm alarm = new MetricAlarm();+ alarm.setAlarmName(alarmName);+ alarm.setStateValue("ALARM");+ cloudWatchMetricsService.putMetricAlarm(alarm, "us-east-1");++ call("CreateCluster", "{\"clusterName\":\"" + CLUSTER + "\"}");+ call("RegisterTaskDefinition", "{\"family\":\"waiter-alarm-td\",\"networkMode\":\"awsvpc\","+ + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\",\"memory\":128}]}");+ call("CreateService", "{\"cluster\":\"" + CLUSTER + "\",\"serviceName\":\"waiter-alarm-svc\","+ + "\"taskDefinition\":\"waiter-alarm-td\",\"desiredCount\":1,"+ + "\"launchType\":\"FARGATE\",\"deploymentConfiguration\":{\"alarms\":{"+ + "\"alarmNames\":[\"" + alarmName + "\"],\"enable\":true,\"rollback\":false}},"+ + NETWORK + "}");++ ecsService.reconcileServices();+ Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER+ + "\",\"service\":\"waiter-alarm-svc\",\"status\":[\"STOP_REQUESTED\"]}");+ listed.then().body("serviceDeployments", hasSize(1));+ String deploymentArn = listed.jsonPath().getString("serviceDeployments[0].serviceDeploymentArn");+ call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")+ .then()+ .body("serviceDeployments[0].status", equalTo("STOP_REQUESTED"))+ .body("serviceDeployments[0].statusReason", startsWith(+ "A deployment alarm entered the ALARM state."));++ ecsService.reconcileServices();+ call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")+ .then()+ .body("serviceDeployments[0].status", equalTo("STOPPED"))+ .body("serviceDeployments[0].stoppedAt", notNullValue());+ call("DescribeServices", "{\"cluster\":\"" + CLUSTER+ + "\",\"services\":[\"waiter-alarm-svc\"]}")+ .then()+ .body("services[0].deployments[0].rolloutState", equalTo("FAILED"));+ }}