Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(ecs): publish Container Insights task counts (#6054)

Mario Toffia1b15299228a5
diff --git a/docs/services/ecs.md b/docs/services/ecs.md
index ee1e9545a6..2bffa4a3a7 100644
--- a/docs/services/ecs.md
+++ b/docs/services/ecs.md
@@ -537,6 +537,30 @@ Known differences from AWS:
- `SubmitTaskStateChange` / `SubmitContainerStateChange` remain ACK-only; Floci drives
the task lifecycle itself rather than via agent submissions.
+#### Container Insights metrics
+
+A service on a cluster whose own `containerInsights` setting is `enabled` or `enhanced`, set
+through `CreateCluster`, `UpdateCluster`, `UpdateClusterSettings` or CloudFormation
+`ClusterSettings`, publishes its task counts to CloudWatch, one sample a minute, in both docker
+and mock mode.
+
+| Namespace | Metrics | Dimensions | Unit |
+|---|---|---|---|
+| `ECS/ContainerInsights` | `RunningTaskCount`, `PendingTaskCount`, `DesiredTaskCount` | `ClusterName`, `ServiceName` | `Count` |
+
+As on AWS, nothing is published while the service has no `RUNNING` task, so an alarm on
+`RunningTaskCount` sees missing data rather than a zero: set `TreatMissingData` to `breaching`
+to alarm on a stopped service.
+
+Known differences from AWS:
+
+- Only these three service task counts are published. CPU, memory, network and storage metrics,
+ the cluster and task dimension sets, and the performance log events are not.
+- The account default set with `PutAccountSettingDefault` does not turn metrics on for a cluster;
+ only the cluster's own setting does.
+- A stack update that removes `ClusterSettings` leaves the cluster's settings in place: declare
+ `containerInsights` as `disabled` to stop the metrics.
+
#### Service discovery
A service that declares `serviceRegistries` has each of its running tasks registered as a
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
index 3bc5ce1802..2e740f3247 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java
@@ -265,6 +265,12 @@ public final class CfnRollback {
*/
public static final String ASG_UPDATE_SNAPSHOT_ATTR = "__FlociAsgUpdateSnapshot";
+ /**
+ * Holds the settings an ECS cluster carried before an in-place update changed its
+ * {@code ClusterSettings}, so a failed stack update can restore them.
+ */
+ public static final String ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR = "__FlociEcsClusterSettingsSnapshot";
+
/**
* Holds the version a launch template created during an in-place update and the prior version,
* so a failed stack update can roll it back.
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java
index 693791182e..3b514e2f9c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java
@@ -1,11 +1,16 @@
package io.github.hectorvent.floci.services.cloudformation.provisioners;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
import io.github.hectorvent.floci.services.ecs.EcsService;
import io.github.hectorvent.floci.services.ecs.model.AwsVpcConfiguration;
+import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsCluster;
import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;
@@ -43,6 +48,8 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {
private static final String CLUSTER = "AWS::ECS::Cluster";
private static final String TASK_DEFINITION = "AWS::ECS::TaskDefinition";
private static final String SERVICE = "AWS::ECS::Service";
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final TypeReference<List<ClusterSetting>> SETTINGS = new TypeReference<>() { };
private final EcsService ecsService;
@@ -101,6 +108,7 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {
@Override
public UpdateCleanupResult completeUpdate(StackResource resource) {
+ resource.getAttributes().remove(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);
return ReplacementCleanup.complete(resource, this::delete);
}
@@ -110,33 +118,127 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {
}
/**
- * A replacement is undone through the cleanup record. Without one, a cluster has nothing to put
- * back: its only property is the create-only name, so an update that kept it re-issued the
- * idempotent create and changed nothing. A service without a record was updated in place, and
- * putting that back needs a snapshot this provisioner does not keep, so the engine reports it as
- * not rolled back, as it did for the switch.
+ * When a settings update and its own restore both failed, the snapshot is left on the attempted
+ * resource only, and the engine puts back the resource it held before. Carried over, it lets
+ * ContinueUpdateRollback retry the restore through {@link #rollbackUpdate} instead of finding
+ * nothing to put back. A restore that succeeded removed it, so it is here only after a failed one.
+ */
+ @Override
+ public void mergeFailedUpdateResourceTracking(StackResource previous, StackResource attempted) {
+ String snapshot = attempted.getAttributes().get(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);
+ if (snapshot != null) {
+ previous.getAttributes().put(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR, snapshot);
+ }
+ }
+
+ /**
+ * A replacement is undone through the cleanup record. Without one, a cluster puts back the
+ * settings snapshot its update took; with no snapshot the update changed no settings, and the
+ * name is create-only, so the idempotent create changed nothing. A service without a record was
+ * updated in place, and putting that back needs a snapshot this provisioner does not keep, so the
+ * engine reports it as not rolled back, as it did for the switch.
*/
@Override
public boolean rollbackUpdate(StackResource resource) {
if (ReplacementCleanup.rollback(resource, this::delete)) {
return true;
}
+ restoreSettingsSnapshot(resource);
return CLUSTER.equals(resource.getResourceType());
}
+ private void restoreSettingsSnapshot(StackResource resource) {
+ String rawSnapshot = resource.getAttributes().get(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);
+ if (rawSnapshot == null) {
+ return;
+ }
+ JsonNode snapshot;
+ try {
+ snapshot = MAPPER.readTree(rawSnapshot);
+ } catch (JsonProcessingException e) {
+ throw new IllegalStateException("Unreadable ECS cluster settings snapshot for "
+ + resource.getLogicalId(), e);
+ }
+ ecsService.updateClusterSettings(resource.getPhysicalId(),
+ MAPPER.convertValue(snapshot.get("settings"), SETTINGS), snapshot.get("region").asText());
+ resource.getAttributes().remove(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);
+ }
+
/**
* The cluster name is create-only and the physical id, so an unnamed cluster keeps the name its
* first execution generated. createCluster is idempotent, which is what makes an unchanged
- * update a no-op here.
+ * update a no-op here. A new cluster is created with its declared {@code ClusterSettings}; an
+ * existing one has them applied when they differ, and an update snapshots the prior ones for
+ * its rollback.
*/
private void provisionCluster(StackResource r, JsonNode props, ProvisionContext ctx) {
+ r.getAttributes().remove(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);
String clusterName = ctx.stablePhysicalName(ctx.resolveOptional(props, "ClusterName"),
r.getLogicalId(), 255, false);
- EcsCluster cluster = ecsService.createCluster(clusterName, ctx.region());
+ List<ClusterSetting> settings = new ArrayList<>();
+ JsonNode declared = props != null ? props.get("ClusterSettings") : null;
+ JsonNode resolved = declared != null ? ctx.engine().resolveNode(declared) : null;
+ if (resolved != null && resolved.isArray()) {
+ for (JsonNode item : resolved) {
+ // An entry that resolved to AWS::NoValue is not an object.
+ if (item.isObject()) {
+ settings.add(new ClusterSetting(item.path("Name").asText(null), item.path("Value").asText(null)));
+ }
+ }
+ }
+ // A new cluster takes its settings at creation, so no later call can fail and leave it
+ // behind; an existing one (this resource's own on an update, or one the idempotent create
+ // returned) has them applied below.
+ EcsCluster cluster = ecsService.createCluster(clusterName, null, settings, ctx.region());
+ if (!settings.isEmpty() && !settings.equals(cluster.getSettings())) {
+ List<ClusterSetting> prior = cluster.getSettings() == null ? null : List.copyOf(cluster.getSettings());
+ if (ctx.reusesPriorEntity(clusterName)) {
+ ObjectNode snapshot = MAPPER.createObjectNode().put("region", ctx.region());
+ snapshot.set("settings", MAPPER.valueToTree(prior));
+ r.getAttributes().put(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR, snapshot.toString());
+ }
+ // Before the id is set: a cluster the idempotent create returned for a resource this
+ // update adds was not created by the stack, and a failed update deletes an added
+ // resource that carries an id. An update of this resource's own cluster keeps its id.
+ try {
+ ecsService.updateClusterSettings(clusterName, settings, ctx.region());
+ } catch (RuntimeException failure) {
+ unwindSettings(r, clusterName, prior, ctx.region(), failure);
+ throw failure;
+ }
+ }
r.setPhysicalId(cluster.getClusterName());
r.getAttributes().put("Arn", cluster.getClusterArn());
}
+ /**
+ * Puts back the settings an existing cluster had before a settings call that failed, which may
+ * have changed them before failing. For this resource's own cluster the engine restores the
+ * resource it held before the attempt, which never carried the snapshot, so the rollback walker
+ * would skip {@link #rollbackUpdate}; a cluster the stack found rather than created is not
+ * restored by anything else. A restore that fails too is recorded as a rollback failure, which
+ * ends an update in UPDATE_ROLLBACK_FAILED for the resource's own cluster, and is attached to
+ * the original failure. The restore repeats the call that just failed, so it can come back
+ * with that very exception, which cannot be suppressed under itself.
+ */
+ private void unwindSettings(StackResource r, String clusterName, List<ClusterSetting> prior, String region,
+ RuntimeException failure) {
+ try {
+ ecsService.updateClusterSettings(clusterName, prior, region);
+ r.getAttributes().remove(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);
+ } catch (RuntimeException unwindFailure) {
+ String reason = "Could not restore the settings of ECS cluster " + clusterName + ": "
+ + unwindFailure.getMessage();
+ // ponytail: a cluster the stack found has no id here, so the engine drops the resource
+ // and nothing retries this; carrying it would need engine-side tracking for added resources.
+ LOG.warnv("{0}", reason);
+ r.getAttributes().put(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR, reason);
+ if (unwindFailure != failure) {
+ failure.addSuppressed(unwindFailure);
+ }
+ }
+ }
+
/**
* Every property of a task definition is create-only, so an update registers a fresh revision
* under the same family, as CloudFormation does on AWS; the physical id is the new revision's
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index a4b1309ee2..1bbbfcff01 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -16,7 +16,9 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackedMap;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;
+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.Dimension;
import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;
+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricDatum;
import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
import io.github.hectorvent.floci.services.ecs.container.EcsLinuxParameters;
import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;
@@ -79,8 +81,10 @@ import org.jboss.logging.Logger;
import java.net.URI;
import java.nio.charset.StandardCharsets;
+import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collection;
@@ -125,6 +129,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
// what every such caller already expects.
private final EcsServiceDiscoveryRegistrar discoveryRegistrar;
private final CloudWatchMetricsService cloudWatchMetricsService;
+ private final Clock clock;
private final boolean dockerMode;
private final boolean sweepLeftoverContainers;
private final String baseUrl;
@@ -229,6 +234,8 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
private final Map<String, ServiceDeployment> serviceDeployments = new ConcurrentHashMap<>();
// serviceRevisionArn → ServiceRevision
private final Map<String, ServiceRevision> serviceRevisions = new ConcurrentHashMap<>();
+ // accountId::serviceKey → the Container Insights sample last written for that service
+ private final Map<String, InsightsSample> publishedInsights = new ConcurrentHashMap<>();
// targetArn → List<Attribute>
private Map<String, List<Attribute>> attributes = new ConcurrentHashMap<>();
// name → value (account-level settings)
@@ -248,6 +255,17 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
EcsExecSessionRegistry execSessions,
EcsServiceDiscoveryRegistrar discoveryRegistrar,
CloudWatchMetricsService cloudWatchMetricsService) {
+ this(regionResolver, containerManager, config, lbRegistrar, storageFactory, eventPublisher,
+ execSessions, discoveryRegistrar, cloudWatchMetricsService, Clock.systemUTC());
+ }
+
+ /** With the clock that dates Container Insights samples, which a test pins. */
+ EcsService(RegionResolver regionResolver, EcsContainerManager containerManager,
+ EmulatorConfig config, EcsLoadBalancerRegistrar lbRegistrar,
+ StorageFactory storageFactory, EcsEventPublisher eventPublisher,
+ EcsExecSessionRegistry execSessions,
+ EcsServiceDiscoveryRegistrar discoveryRegistrar,
+ CloudWatchMetricsService cloudWatchMetricsService, Clock clock) {
this.regionResolver = regionResolver;
this.containerManager = containerManager;
this.dockerMode = !config.services().ecs().mock();
@@ -260,6 +278,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
this.execSessions = execSessions;
this.discoveryRegistrar = discoveryRegistrar;
this.cloudWatchMetricsService = cloudWatchMetricsService;
+ this.clock = clock;
}
public EcsService(RegionResolver regionResolver, EcsContainerManager containerManager,
@@ -425,8 +444,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
@Override
public void clear() {
- // The stores hold the ECS state and afterReset() restarts the reconciler.
+ // The stores hold the ECS state and afterReset() restarts the reconciler. The reset also
+ // wipes CloudWatch, so the Container Insights samples it held must be written again.
launchFailures.clear();
+ publishedInsights.clear();
}
/**
@@ -5227,9 +5248,72 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
} catch (Exception e) {
LOG.debugv("Error reconciling ECS service {0}: {1}", entry.getKey(), e.getMessage());
}
+ try {
+ publishContainerInsights(entry.getKey(), entry.getValue());
+ } catch (Exception e) {
+ LOG.warnv(e, "Could not publish Container Insights task counts for ECS service {0}", entry.getKey());
+ }
}
}
+ /**
+ * Container Insights task counts for a service on a cluster whose own {@code containerInsights}
+ * setting is enabled or enhanced, one sample per minute, counted from the service's tasks once
+ * the tick has started and stopped them. As in AWS, nothing is published while the service has
+ * no RUNNING task, so an alarm sees missing data rather than a zero.
+ */
+ private void publishContainerInsights(String key, EcsServiceModel svc) {
+ if (cloudWatchMetricsService == null || !STATUS_ACTIVE.equals(svc.getStatus())) {
+ return;
+ }
+ String region = extractRegionFromServiceKey(key);
+ String clusterName = extractClusterNameFromServiceKey(key);
+ EcsCluster cluster = clusters.get(clusterKey(region, clusterName));
+ if (cluster == null || cluster.getSettings() == null || cluster.getSettings().stream().noneMatch(s ->
+ "containerInsights".equals(s.name())
+ && ("enabled".equals(s.value()) || "enhanced".equals(s.value())))) {
+ return;
+ }
+ List<String> statuses = tasks.values().stream()
+ .filter(t -> ownedBy(t, svc, cluster))
+ .map(EcsTask::getLastStatus)
+ .toList();
+ int running = Collections.frequency(statuses, TaskStatus.RUNNING.name());
+ if (running < 1) {
+ return;
+ }
+ long minute = clock.instant().truncatedTo(ChronoUnit.MINUTES).getEpochSecond();
+ Map<String, Integer> counts = Map.of("RunningTaskCount", running,
+ "PendingTaskCount", Collections.frequency(statuses, TaskStatus.PENDING.name()),
+ "DesiredTaskCount", svc.getDesiredCount());
+ String accountId = regionResolver.getAccountId();
+ String sampleKey = accountId + "::" + key;
+ InsightsSample sample = new InsightsSample(minute, counts);
+ // A persistent metric store rewrites its file on every put, and the reconciler ticks every
+ // five seconds, so the minute's sample is written again only when a count changes.
+ if (sample.equals(publishedInsights.get(sampleKey))) {
+ return;
+ }
+ // Forgotten before writing, so a write that fails partway is repeated on the next tick
+ // even when the counts are back to the ones last written in full.
+ publishedInsights.remove(sampleKey);
+ for (Map.Entry<String, Integer> count : counts.entrySet()) {
+ MetricDatum datum = new MetricDatum();
+ datum.setMetricName(count.getKey());
+ datum.setUnit("Count");
+ datum.setDimensions(List.of(new Dimension("ClusterName", clusterName),
+ new Dimension("ServiceName", svc.getServiceName())));
+ datum.setTimestamp(minute);
+ datum.setValue(count.getValue());
+ cloudWatchMetricsService.publishMetricForAccount(accountId, "ECS/ContainerInsights",
+ datum, region, "ecs-container-insights");
+ }
+ publishedInsights.put(sampleKey, sample);
+ }
+
+ private record InsightsSample(long minute, Map<String, Integer> counts) {
+ }
+
/**
* Whether a task is genuinely owned by {@code svc}. Both halves matter: {@code
* owningServiceArn} is stamped only by {@link #launchServiceTask} and so cannot be forged
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java
index fd38ab9e98..0eca2c1798 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;
import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsCluster;
import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;
@@ -17,8 +18,12 @@ import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;
import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.NullSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
+import org.mockito.invocation.InvocationOnMock;
import java.util.ArrayList;
import java.util.HashMap;
@@ -30,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -116,7 +122,7 @@ class EcsCfnProvisionerTest {
@Test
void clusterUsesTheDeclaredNameAndExposesArn() {
- when(ecs.createCluster("web", REGION)).thenReturn(cluster("web"));
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(cluster("web"));
StackResource r = resource("AWS::ECS::Cluster", "Cluster");
provisioner.provision(r, mapper.createObjectNode().put("ClusterName", "web"), ctx());
@@ -128,7 +134,7 @@ class EcsCfnProvisionerTest {
@Test
void anUnnamedClusterGeneratesANameOnCreateAndKeepsItOnUpdate() {
- when(ecs.createCluster(anyString(), eq(REGION))).thenAnswer(inv -> cluster(inv.getArgument(0)));
+ when(ecs.createCluster(anyString(), isNull(), anyList(), eq(REGION))).thenAnswer(inv -> cluster(inv.getArgument(0)));
StackResource created = resource("AWS::ECS::Cluster", "Cluster");
provisioner.provision(created, mapper.createObjectNode(), ctx());
String generated = created.getPhysicalId();
@@ -140,7 +146,7 @@ class EcsCfnProvisionerTest {
// createCluster is idempotent, so the update re-issues it under the same name instead of
// minting a second cluster.
assertEquals(generated, updated.getPhysicalId());
- verify(ecs, times(2)).createCluster(generated, REGION);
+ verify(ecs, times(2)).createCluster(eq(generated), isNull(), anyList(), eq(REGION));
}
@Test
@@ -696,7 +702,7 @@ class EcsCfnProvisionerTest {
@Test
void anUnchangedClusterHasNothingToRollBack() {
- when(ecs.createCluster("web", REGION)).thenReturn(cluster("web"));
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(cluster("web"));
StackResource r = resource("AWS::ECS::Cluster", "Cluster");
provisioner.provision(r, mapper.createObjectNode().put("ClusterName", "web"), ctx("web"));
@@ -706,6 +712,246 @@ class EcsCfnProvisionerTest {
verify(ecs, never()).deleteCluster(anyString(), anyString());
}
+ @Test
+ void clusterSettingsAreAppliedOnCreate() {
+ List<ClusterSetting> enabled = List.of(new ClusterSetting("containerInsights", "enabled"));
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenAnswer(inv -> created(inv));
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+
+ provisioner.provision(r, clusterWithInsights("enabled"), ctx());
+
+ // In the create itself, so no later call can fail and leave a cluster the rollback misses.
+ verify(ecs).createCluster("web", null, enabled, REGION);
+ verify(ecs, never()).updateClusterSettings(anyString(), any(), anyString());
+ assertEquals(Set.of("Arn"), r.getAttributes().keySet());
+ }
+
+ @Test
+ void aRenamedClusterTakesItsSettingsAtCreation() {
+ List<ClusterSetting> enabled = List.of(new ClusterSetting("containerInsights", "enabled"));
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenAnswer(inv -> created(inv));
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+
+ provisioner.provision(r, clusterWithInsights("enabled"), ctx("old"));
+
+ verify(ecs).createCluster("web", null, enabled, REGION);
+ verify(ecs, never()).updateClusterSettings(anyString(), any(), anyString());
+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void aClusterThatAlreadyExistedGetsItsSettingsBackAndIsNotDeleted() {
+ // The idempotent create hands back a cluster the stack did not create, without the settings.
+ List<ClusterSetting> priorSettings = List.of(new ClusterSetting("containerInsights", "disabled"));
+ EcsCluster existing = cluster("web");
+ existing.setSettings(priorSettings);
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(existing);
+ AwsException failure = new AwsException("ServerException", "write failed", 500);
+ when(ecs.updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enabled")), REGION))
+ .thenThrow(failure);
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+
+ AwsException thrown = assertThrows(AwsException.class,
+ () -> provisioner.provision(r, clusterWithInsights("enabled"), ctx()));
+
+ assertSame(failure, thrown);
+ // The failed call may have changed them, and nothing else puts back a cluster the stack found.
+ verify(ecs).updateClusterSettings("web", priorSettings, REGION);
+ // Without an id, neither a create rollback nor an update that added this resource deletes it.
+ assertNull(r.getPhysicalId());
+ verify(ecs, never()).deleteCluster(anyString(), anyString());
+ assertFalse(r.getAttributes().containsKey(CfnRollback.ROLLBACK_OWNED_ATTR));
+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));
+ }
+
+ private EcsCluster created(InvocationOnMock inv) {
+ EcsCluster c = cluster(inv.getArgument(0));
+ List<ClusterSetting> settings = inv.getArgument(2);
+ c.setSettings(settings.isEmpty() ? null : settings);
+ return c;
+ }
+
+ @ParameterizedTest
+ @NullSource
+ @ValueSource(strings = "disabled")
+ void aFailedStackUpdateRestoresThePriorClusterSettings(String priorValue) {
+ List<ClusterSetting> priorSettings = priorValue == null ? null : List.of(new ClusterSetting("containerInsights", priorValue));
+ EcsCluster prior = cluster("web");
+ prior.setSettings(priorSettings);
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+
+ provisioner.provision(r, clusterWithInsights("enhanced"), ctx("web"));
+ verify(ecs).updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enhanced")), REGION);
+
+ assertTrue(provisioner.rollbackUpdate(r));
+ verify(ecs).updateClusterSettings("web", priorSettings, REGION);
+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void aSettingsUpdateThatFailsPutsThePriorSettingsBackItself() {
+ List<ClusterSetting> priorSettings = List.of(new ClusterSetting("containerInsights", "disabled"));
+ EcsCluster prior = cluster("web");
+ prior.setSettings(priorSettings);
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);
+ AwsException failure = new AwsException("ServerException", "write failed", 500);
+ when(ecs.updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enhanced")), REGION))
+ .thenThrow(failure);
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+ // An update hands the provisioner the resource with its prior physical id.
+ r.setPhysicalId("web");
+
+ AwsException thrown = assertThrows(AwsException.class,
+ () -> provisioner.provision(r, clusterWithInsights("enhanced"), ctx("web")));
+
+ // The engine restores the resource it held before this attempt, which never carried the
+ // snapshot, so the rollback walker would not put these settings back.
+ assertSame(failure, thrown);
+ verify(ecs).updateClusterSettings("web", priorSettings, REGION);
+ assertFalse(r.getAttributes().containsKey(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));
+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void aSettingsRestoreThatAlsoFailsIsRecordedAsARollbackFailure() {
+ EcsCluster prior = cluster("web");
+ prior.setSettings(List.of(new ClusterSetting("containerInsights", "disabled")));
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);
+ AwsException failure = new AwsException("ServerException", "write failed", 500);
+ AwsException restoreFailure = new AwsException("ServerException", "still failing", 500);
+ when(ecs.updateClusterSettings(eq("web"), any(), eq(REGION))).thenThrow(failure, restoreFailure);
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+ // An update hands the provisioner the resource with its prior physical id.
+ r.setPhysicalId("web");
+
+ AwsException thrown = assertThrows(AwsException.class,
+ () -> provisioner.provision(r, clusterWithInsights("enhanced"), ctx("web")));
+
+ assertSame(failure, thrown);
+ assertEquals(List.of(restoreFailure), List.of(thrown.getSuppressed()));
+ assertEquals("Could not restore the settings of ECS cluster web: still failing",
+ r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));
+ }
+
+ @Test
+ void aRestoreThatFailedIsRetriedWhenTheRollbackIsContinued() {
+ List<ClusterSetting> priorSettings = List.of(new ClusterSetting("containerInsights", "disabled"));
+ EcsCluster prior = cluster("web");
+ prior.setSettings(priorSettings);
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);
+ when(ecs.updateClusterSettings(eq("web"), any(), eq(REGION)))
+ .thenThrow(new AwsException("ServerException", "write failed", 500))
+ .thenThrow(new AwsException("ServerException", "still failing", 500))
+ .thenReturn(prior);
+ StackResource previous = resource("AWS::ECS::Cluster", "Cluster");
+ previous.setPhysicalId("web");
+ StackResource attempted = resource("AWS::ECS::Cluster", "Cluster");
+ attempted.setPhysicalId("web");
+ assertThrows(AwsException.class,
+ () -> provisioner.provision(attempted, clusterWithInsights("enhanced"), ctx("web")));
+
+ // The engine puts the previous resource back after the failed provision, and the first
+ // rollback walk reports the recorded failure without calling rollbackUpdate.
+ provisioner.mergeFailedUpdateResourceTracking(previous, attempted);
+ // ContinueUpdateRollback then asks for the restore again.
+ assertTrue(provisioner.rollbackUpdate(previous));
+
+ verify(ecs, times(2)).updateClusterSettings("web", priorSettings, REGION);
+ assertFalse(previous.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void aRestoreThatSucceededLeavesNothingToCarryOver() {
+ EcsCluster prior = cluster("web");
+ prior.setSettings(List.of(new ClusterSetting("containerInsights", "disabled")));
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);
+ when(ecs.updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enhanced")), REGION))
+ .thenThrow(new AwsException("ServerException", "write failed", 500));
+ StackResource previous = resource("AWS::ECS::Cluster", "Cluster");
+ StackResource attempted = resource("AWS::ECS::Cluster", "Cluster");
+ assertThrows(AwsException.class,
+ () -> provisioner.provision(attempted, clusterWithInsights("enhanced"), ctx("web")));
+
+ provisioner.mergeFailedUpdateResourceTracking(previous, attempted);
+
+ assertFalse(previous.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void aCommittedSettingsUpdateDropsItsSnapshot() {
+ EcsCluster prior = cluster("web");
+ prior.setSettings(List.of(new ClusterSetting("containerInsights", "disabled")));
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+ provisioner.provision(r, clusterWithInsights("enabled"), ctx("web"));
+
+ assertEquals(UpdateCleanupResult.notApplicable(), provisioner.completeUpdate(r));
+
+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));
+ }
+
+ @Test
+ void anUnchangedSettingIsNotReappliedAndDropsAnEarlierSnapshot() {
+ EcsCluster current = cluster("web");
+ current.setSettings(List.of(new ClusterSetting("containerInsights", "enabled")));
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(current);
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+ r.getAttributes().put(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR,
+ "{\"region\":\"us-east-1\",\"settings\":null}");
+
+ provisioner.provision(r, clusterWithInsights("enabled"), ctx("web"));
+
+ assertTrue(provisioner.rollbackUpdate(r));
+ verify(ecs, never()).updateClusterSettings(anyString(), any(), anyString());
+ }
+
+ @Test
+ void conditionalClusterSettingsAreResolvedBeforeTheyAreRead() {
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(cluster("web"));
+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");
+ ObjectNode props = clusterWithInsights("enabled");
+ JsonNode resolved = props.get("ClusterSettings");
+ ObjectNode conditional = mapper.createObjectNode();
+ conditional.putArray("Fn::If").add("Insights").add(resolved).addObject().put("Ref", "AWS::NoValue");
+ props.set("ClusterSettings", conditional);
+ ProvisionContext ctx = ctx();
+ when(ctx.engine().resolveNode(conditional)).thenReturn(resolved);
+
+ provisioner.provision(r, props, ctx);
+
+ verify(ecs).updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enabled")), REGION);
+ }
+
+ @Test
+ void clusterSettingsThatFailToResolveCreateNoCluster() {
+ ObjectNode props = clusterWithInsights("enabled");
+ ProvisionContext ctx = ctx();
+ when(ctx.engine().resolveNode(props.get("ClusterSettings"))).thenThrow(new IllegalStateException("unresolvable"));
+
+ assertThrows(IllegalStateException.class,
+ () -> provisioner.provision(resource("AWS::ECS::Cluster", "Cluster"), props, ctx));
+
+ verify(ecs, never()).createCluster(anyString(), any(), any(), anyString());
+ }
+
+ @Test
+ void aClusterSettingResolvedToNoValueIsDropped() {
+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(cluster("web"));
+ ObjectNode props = clusterWithInsights("enabled");
+ ((ArrayNode) props.get("ClusterSettings")).add("");
+
+ provisioner.provision(resource("AWS::ECS::Cluster", "Cluster"), props, ctx());
+
+ verify(ecs).updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enabled")), REGION);
+ }
+
+ private ObjectNode clusterWithInsights(String value) {
+ ObjectNode props = mapper.createObjectNode().put("ClusterName", "web");
+ props.putArray("ClusterSettings").addObject().put("Name", "containerInsights").put("Value", value);
+ return props;
+ }
+
@Test
void anInPlaceUpdateIsNotRolledBackHere() {
when(ecs.updateService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(3), isNull(), eq(REGION)))
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsContainerInsightsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsContainerInsightsIntegrationTest.java
new file mode 100644
index 0000000000..9a236584f3
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsContainerInsightsIntegrationTest.java
@@ -0,0 +1,202 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.mockito.InjectSpy;
+import io.restassured.specification.RequestSpecification;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+
+import static io.restassured.RestAssured.given;
+import static org.awaitility.Awaitility.await;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doCallRealMethod;
+
+/**
+ * A cluster whose CloudFormation {@code ClusterSettings} enable Container Insights publishes its
+ * services' task counts to {@code ECS/ContainerInsights}, which is what an alarm on a stopped
+ * service watches. The test drives the reconciler itself, as the other ECS integration tests do,
+ * rather than waiting on its five-second ticks: one pass launches the task, the next counts it.
+ */
+@QuarkusTest
+class EcsContainerInsightsIntegrationTest {
+
+ @InjectSpy
+ EcsService ecsService;
+
+ private static final String CLOUDWATCH_AUTH =
+ "AWS4-HMAC-SHA256 Credential=AKID/20260908/us-east-1/monitoring/aws4_request";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void aClusterWithInsightsFromCloudFormationPublishesItsRunningTaskCount() {
+ String cluster = "insights-" + Long.toString(System.nanoTime(), 36);
+ String template = """
+ {"Resources": {
+ "Cluster": {"Type": "AWS::ECS::Cluster", "Properties": {"ClusterName": "%1$s",
+ "ClusterSettings": [{"Name": "containerInsights", "Value": "enabled"}]}},
+ "TaskDef": {"Type": "AWS::ECS::TaskDefinition", "Properties": {"Family": "%1$s",
+ "NetworkMode": "awsvpc", "RequiresCompatibilities": ["FARGATE"], "Cpu": "256", "Memory": "512",
+ "ContainerDefinitions": [{"Name": "app", "Image": "nginx:alpine", "Essential": true}]}},
+ "Service": {"Type": "AWS::ECS::Service", "Properties": {"ServiceName": "web",
+ "Cluster": {"Ref": "Cluster"}, "TaskDefinition": {"Ref": "TaskDef"},
+ "DesiredCount": 1, "LaunchType": "FARGATE", "NetworkConfiguration": {"AwsvpcConfiguration":
+ {"Subnets": ["subnet-default-us-east-1-a"]}}}}}}
+ """.formatted(cluster);
+
+ given().contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "CreateStack")
+ .formParam("StackName", cluster)
+ .formParam("TemplateBody", template)
+ .post("/").then().statusCode(200);
+ given().contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStacks")
+ .formParam("StackName", cluster)
+ .post("/").then().statusCode(200)
+ .body(containsString("<StackStatus>CREATE_COMPLETE</StackStatus>"));
+ given().contentType("application/x-amz-json-1.1")
+ .header("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.DescribeClusters")
+ .body("{\"clusters\": [\"" + cluster + "\"], \"include\": [\"SETTINGS\"]}")
+ .post("/").then().statusCode(200)
+ .body("clusters[0].settings[0].value", equalTo("enabled"));
+
+ ecsService.reconcileServices();
+ ecsService.reconcileServices();
+
+ given()
+ .contentType("application/x-amz-json-1.0")
+ .header("Authorization", CLOUDWATCH_AUTH)
+ .header("X-Amz-Target", "GraniteServiceVersion20100801.GetMetricStatistics")
+ .body("""
+ {"Namespace": "ECS/ContainerInsights", "MetricName": "RunningTaskCount",
+ "Dimensions": [{"Name": "ClusterName", "Value": "%s"}, {"Name": "ServiceName", "Value": "web"}],
+ "StartTime": %d, "EndTime": %d, "Period": 60, "Statistics": ["Maximum"]}
+ """.formatted(cluster, Instant.now().minusSeconds(300).getEpochSecond(),
+ Instant.now().plusSeconds(60).getEpochSecond()))
+ .post("/").then().statusCode(200)
+ .body("Datapoints.Maximum", hasItem(1.0f));
+ }
+
+ /**
+ * The update's settings land and the call still fails, and the provisioner's own restore fails
+ * too, so the stack ends in UPDATE_ROLLBACK_FAILED with the new setting live. The snapshot
+ * survives on the resource the stack keeps, so ContinueUpdateRollback puts the prior setting
+ * back rather than reporting a rollback with nothing left to do.
+ */
+ @Test
+ void continueUpdateRollbackRetriesASettingsRestoreThatFailed() {
+ String cluster = "insights-rb-" + Long.toString(System.nanoTime(), 36);
+ String template = """
+ {"Resources": {"Cluster": {"Type": "AWS::ECS::Cluster", "Properties": {"ClusterName": "%s",
+ "ClusterSettings": [{"Name": "containerInsights", "Value": "%s"}]}}}}
+ """;
+ cloudFormation("CreateStack", cluster, template.formatted(cluster, "disabled"));
+ awaitStackStatus(cluster, "CREATE_COMPLETE");
+
+ List<ClusterSetting> enabled = List.of(new ClusterSetting("containerInsights", "enabled"));
+ doAnswer(invocation -> {
+ if (enabled.equals(invocation.getArgument(1))) {
+ invocation.callRealMethod();
+ }
+ throw new IllegalStateException("simulated settings write failure");
+ }).when(ecsService).updateClusterSettings(anyString(), anyList(), anyString());
+ try {
+ cloudFormation("UpdateStack", cluster, template.formatted(cluster, "enabled"));
+ awaitStackStatus(cluster, "UPDATE_ROLLBACK_FAILED");
+ } finally {
+ doCallRealMethod().when(ecsService).updateClusterSettings(anyString(), anyList(), anyString());
+ }
+ assertInsightsSetting(cluster, "enabled");
+
+ cloudFormation("ContinueUpdateRollback", cluster, null);
+ awaitStackStatus(cluster, "UPDATE_ROLLBACK_COMPLETE");
+ assertInsightsSetting(cluster, "disabled");
+ }
+
+ /**
+ * An update that adds a cluster under the name of one created outside the stack gets that
+ * cluster back from the idempotent create. When its settings then land and the call still
+ * fails, the cluster gets its own settings back, and the update rollback, which deletes what
+ * the update added, leaves the cluster in place because the resource never claimed it.
+ */
+ @Test
+ void aFailedUpdateLeavesAClusterItFoundAsItWas() {
+ String name = "insights-pre-" + Long.toString(System.nanoTime(), 36);
+ given().contentType("application/x-amz-json-1.1")
+ .header("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.CreateCluster")
+ .body("{\"clusterName\": \"" + name + "\", "
+ + "\"settings\": [{\"name\": \"containerInsights\", \"value\": \"disabled\"}]}")
+ .post("/").then().statusCode(200);
+ String parameter = """
+ "Param": {"Type": "AWS::SSM::Parameter", "Properties": {"Type": "String", "Value": "v"}}""";
+ cloudFormation("CreateStack", name, "{\"Resources\": {" + parameter + "}}");
+ awaitStackStatus(name, "CREATE_COMPLETE");
+
+ List<ClusterSetting> enabled = List.of(new ClusterSetting("containerInsights", "enabled"));
+ doAnswer(invocation -> {
+ Object result = invocation.callRealMethod();
+ if (enabled.equals(invocation.getArgument(1))) {
+ throw new IllegalStateException("simulated settings write failure");
+ }
+ return result;
+ }).when(ecsService).updateClusterSettings(anyString(), anyList(), anyString());
+ try {
+ cloudFormation("UpdateStack", name, """
+ {"Resources": {%s, "Cluster": {"Type": "AWS::ECS::Cluster", "Properties": {"ClusterName": "%s",
+ "ClusterSettings": [{"Name": "containerInsights", "Value": "enabled"}]}}}}
+ """.formatted(parameter, name));
+ awaitStackStatus(name, "UPDATE_ROLLBACK_COMPLETE");
+ } finally {
+ doCallRealMethod().when(ecsService).updateClusterSettings(anyString(), anyList(), anyString());
+ }
+
+ given().contentType("application/x-amz-json-1.1")
+ .header("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.DescribeClusters")
+ .body("{\"clusters\": [\"" + name + "\"], \"include\": [\"SETTINGS\"]}")
+ .post("/").then().statusCode(200)
+ .body("clusters[0].status", equalTo("ACTIVE"))
+ .body("clusters[0].settings[0].value", equalTo("disabled"));
+ }
+
+ private static void cloudFormation(String action, String stack, String template) {
+ RequestSpecification request = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", action)
+ .formParam("StackName", stack);
+ if (template != null) {
+ request.formParam("TemplateBody", template);
+ }
+ request.post("/").then().statusCode(200);
+ }
+
+ private static void awaitStackStatus(String stack, String status) {
+ await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "DescribeStacks")
+ .formParam("StackName", stack)
+ .post("/").then().statusCode(200)
+ .body(containsString("<StackStatus>" + status + "</StackStatus>")));
+ }
+
+ private static void assertInsightsSetting(String cluster, String value) {
+ given().contentType("application/x-amz-json-1.1")
+ .header("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.DescribeClusters")
+ .body("{\"clusters\": [\"" + cluster + "\"], \"include\": [\"SETTINGS\"]}")
+ .post("/").then().statusCode(200)
+ .body("clusters[0].settings[0].value", equalTo(value));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.java
new file mode 100644
index 0000000000..f79cf28d5b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.java
@@ -0,0 +1,196 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;
+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService.Datapoint;
+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.Dimension;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.exec.EcsExecSessionRegistry;
+import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Container Insights task counts, published by the service reconciler into a real CloudWatch
+ * metric store: {@code ECS/ContainerInsights} on {@code ClusterName} and {@code ServiceName}, once
+ * per minute, only for a cluster whose containerInsights setting turns it on and only while the
+ * service has a RUNNING task.
+ */
+class EcsServiceContainerInsightsTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String NAMESPACE = "ECS/ContainerInsights";
+ private static final Instant MINUTE = Instant.parse("2026-10-20T12:00:00Z");
+
+ private CloudWatchMetricsService metrics;
+ private EcsService service;
+
+ @BeforeEach
+ void setUp() {
+ InMemoryStorageFactory storage = new InMemoryStorageFactory();
+ RegionResolver regionResolver = new RegionResolver(REGION, "000000000000");
+ metrics = spy(new CloudWatchMetricsService(storage, regionResolver));
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(true);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ service = new EcsService(regionResolver, mock(EcsContainerManager.class), config,
+ mock(EcsLoadBalancerRegistrar.class), storage, null, new EcsExecSessionRegistry(), null, metrics,
+ Clock.fixed(MINUTE.plusSeconds(42), ZoneOffset.UTC));
+ service.initializeStorage();
+ ContainerDefinition cd = new ContainerDefinition();
+ cd.setName("app");
+ cd.setImage("nginx:alpine");
+ service.registerTaskDefinition("ci-fam", List.of(cd), null, null, null, null, null, List.of(), REGION);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"enabled", "enhanced"})
+ void anInsightsClusterPublishesItsServiceTaskCounts(String mode) {
+ startService(List.of(new ClusterSetting("containerInsights", mode)), 1);
+
+ service.reconcile();
+
+ Datapoint running = only("RunningTaskCount");
+ assertEquals(MINUTE, running.timestamp());
+ assertEquals(1.0, running.maximum());
+ assertEquals("Count", running.unit());
+ assertEquals(0.0, only("PendingTaskCount").maximum());
+ assertEquals(1.0, only("DesiredTaskCount").maximum());
+ }
+
+ @Test
+ void repeatedTicksKeepOneSamplePerMinute() {
+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 1);
+
+ service.reconcile();
+ service.reconcile();
+
+ List<Datapoint> points = statistics("RunningTaskCount");
+ assertFalse(points.isEmpty());
+ assertTrue(points.stream().allMatch(p -> p.sampleCount() == 1.0), points.toString());
+ }
+
+ @Test
+ void anUnchangedSampleIsNotWrittenAgainWithinTheMinute() {
+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 1);
+
+ service.reconcile();
+ service.reconcile();
+
+ verify(metrics, times(3)).publishMetricForAccount(anyString(), eq(NAMESPACE), any(), eq(REGION), anyString());
+ }
+
+ @Test
+ void aWriteThatFailsPartwayIsRepeatedEvenWhenTheCountsReturn() {
+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 1);
+ service.reconcile();
+ service.updateService("insights", "web", null, 2, null, REGION);
+ // One of the three counts lands, the next write fails.
+ doCallRealMethod().doThrow(new IllegalStateException("disk full")).doCallRealMethod().when(metrics)
+ .publishMetricForAccount(anyString(), eq(NAMESPACE), any(), eq(REGION), anyString());
+ service.reconcile();
+
+ service.updateService("insights", "web", null, 1, null, REGION);
+ clearInvocations(metrics);
+ service.reconcile();
+
+ verify(metrics, times(3)).publishMetricForAccount(anyString(), eq(NAMESPACE), any(), eq(REGION), anyString());
+ }
+
+ @Test
+ void aScaleInIsCountedInTheTickThatStopsTheTask() {
+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 2);
+ service.reconcile();
+
+ service.updateService("insights", "web", null, 1, null, REGION);
+ service.reconcile();
+
+ assertEquals(1.0, statistics("RunningTaskCount").getLast().maximum());
+ assertEquals(1.0, statistics("DesiredTaskCount").getLast().maximum());
+ }
+
+ @Test
+ void aClusterWithoutInsightsPublishesNothing() {
+ startService(List.of(new ClusterSetting("containerInsights", "disabled")), 1);
+ service.createCluster("plain", REGION);
+ service.createService("plain", "web", "ci-fam", 1, LaunchType.FARGATE, List.of(), null, REGION);
+
+ service.reconcile();
+
+ assertTrue(metrics.listMetrics(NAMESPACE, null, null, REGION).isEmpty());
+ }
+
+ @Test
+ void aServiceWithoutARunningTaskPublishesNothing() {
+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 0);
+
+ service.reconcile();
+
+ assertTrue(metrics.listMetrics(NAMESPACE, null, null, REGION).isEmpty());
+ }
+
+ private void startService(List<ClusterSetting> settings, int desiredCount) {
+ service.createCluster("insights", null, settings, REGION);
+ service.createService("insights", "web", "ci-fam", desiredCount, LaunchType.FARGATE, List.of(), null, REGION);
+ }
+
+ private Datapoint only(String metricName) {
+ List<Datapoint> points = statistics(metricName);
+ assertEquals(1, points.size(), metricName + ": " + points);
+ return points.get(0);
+ }
+
+ private List<Datapoint> statistics(String metricName) {
+ return metrics.getMetricStatistics(NAMESPACE, metricName,
+ List.of(new Dimension("ServiceName", "web"), new Dimension("ClusterName", "insights")),
+ MINUTE.minusSeconds(600), MINUTE.plusSeconds(600), 60, List.of("Maximum"), null, REGION);
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+ ignored -> AccountAwareStorageBackend.inMemory("000000000000"));
+ }
+ }
+}