Light, fluffy, and always free - The AWS Local Emulator alternative
feat(ecs): publish Container Insights task counts (#6054)
Mario Toffia1b15299228a5
diff --git a/docs/services/ecs.md b/docs/services/ecs.mdindex ee1e9545a6..2bffa4a3a7 100644--- a/docs/services/ecs.md+++ b/docs/services/ecs.md@@ -537,6 +537,30 @@ Known differences from AWS:- `SubmitTaskStateChange` / `SubmitContainerStateChange` remain ACK-only; Floci drivesthe task lifecycle itself rather than via agent submissions.+#### Container Insights metrics++A service on a cluster whose own `containerInsights` setting is `enabled` or `enhanced`, set+through `CreateCluster`, `UpdateCluster`, `UpdateClusterSettings` or CloudFormation+`ClusterSettings`, publishes its task counts to CloudWatch, one sample a minute, in both docker+and mock mode.++| Namespace | Metrics | Dimensions | Unit |+|---|---|---|---|+| `ECS/ContainerInsights` | `RunningTaskCount`, `PendingTaskCount`, `DesiredTaskCount` | `ClusterName`, `ServiceName` | `Count` |++As on AWS, nothing is published while the service has no `RUNNING` task, so an alarm on+`RunningTaskCount` sees missing data rather than a zero: set `TreatMissingData` to `breaching`+to alarm on a stopped service.++Known differences from AWS:++- Only these three service task counts are published. CPU, memory, network and storage metrics,+ the cluster and task dimension sets, and the performance log events are not.+- The account default set with `PutAccountSettingDefault` does not turn metrics on for a cluster;+ only the cluster's own setting does.+- A stack update that removes `ClusterSettings` leaves the cluster's settings in place: declare+ `containerInsights` as `disabled` to stop the metrics.+#### Service discoveryA service that declares `serviceRegistries` has each of its running tasks registered as adiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.javaindex 3bc5ce1802..2e740f3247 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CfnRollback.java@@ -265,6 +265,12 @@ public final class CfnRollback {*/public static final String ASG_UPDATE_SNAPSHOT_ATTR = "__FlociAsgUpdateSnapshot";+ /**+ * Holds the settings an ECS cluster carried before an in-place update changed its+ * {@code ClusterSettings}, so a failed stack update can restore them.+ */+ public static final String ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR = "__FlociEcsClusterSettingsSnapshot";+/*** Holds the version a launch template created during an in-place update and the prior version,* so a failed stack update can roll it back.diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.javaindex 693791182e..3b514e2f9c 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisioner.java@@ -1,11 +1,16 @@package io.github.hectorvent.floci.services.cloudformation.provisioners;+import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.ecs.EcsService;import io.github.hectorvent.floci.services.ecs.model.AwsVpcConfiguration;+import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsCluster;import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;@@ -43,6 +48,8 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {private static final String CLUSTER = "AWS::ECS::Cluster";private static final String TASK_DEFINITION = "AWS::ECS::TaskDefinition";private static final String SERVICE = "AWS::ECS::Service";+ private static final ObjectMapper MAPPER = new ObjectMapper();+ private static final TypeReference<List<ClusterSetting>> SETTINGS = new TypeReference<>() { };private final EcsService ecsService;@@ -101,6 +108,7 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {@Overridepublic UpdateCleanupResult completeUpdate(StackResource resource) {+ resource.getAttributes().remove(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);return ReplacementCleanup.complete(resource, this::delete);}@@ -110,33 +118,127 @@ public class EcsCfnProvisioner implements CfnResourceProvisioner {}/**- * A replacement is undone through the cleanup record. Without one, a cluster has nothing to put- * back: its only property is the create-only name, so an update that kept it re-issued the- * idempotent create and changed nothing. A service without a record was updated in place, and- * putting that back needs a snapshot this provisioner does not keep, so the engine reports it as- * not rolled back, as it did for the switch.+ * When a settings update and its own restore both failed, the snapshot is left on the attempted+ * resource only, and the engine puts back the resource it held before. Carried over, it lets+ * ContinueUpdateRollback retry the restore through {@link #rollbackUpdate} instead of finding+ * nothing to put back. A restore that succeeded removed it, so it is here only after a failed one.+ */+ @Override+ public void mergeFailedUpdateResourceTracking(StackResource previous, StackResource attempted) {+ String snapshot = attempted.getAttributes().get(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);+ if (snapshot != null) {+ previous.getAttributes().put(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR, snapshot);+ }+ }++ /**+ * A replacement is undone through the cleanup record. Without one, a cluster puts back the+ * settings snapshot its update took; with no snapshot the update changed no settings, and the+ * name is create-only, so the idempotent create changed nothing. A service without a record was+ * updated in place, and putting that back needs a snapshot this provisioner does not keep, so the+ * engine reports it as not rolled back, as it did for the switch.*/@Overridepublic boolean rollbackUpdate(StackResource resource) {if (ReplacementCleanup.rollback(resource, this::delete)) {return true;}+ restoreSettingsSnapshot(resource);return CLUSTER.equals(resource.getResourceType());}+ private void restoreSettingsSnapshot(StackResource resource) {+ String rawSnapshot = resource.getAttributes().get(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);+ if (rawSnapshot == null) {+ return;+ }+ JsonNode snapshot;+ try {+ snapshot = MAPPER.readTree(rawSnapshot);+ } catch (JsonProcessingException e) {+ throw new IllegalStateException("Unreadable ECS cluster settings snapshot for "+ + resource.getLogicalId(), e);+ }+ ecsService.updateClusterSettings(resource.getPhysicalId(),+ MAPPER.convertValue(snapshot.get("settings"), SETTINGS), snapshot.get("region").asText());+ resource.getAttributes().remove(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);+ }+/*** The cluster name is create-only and the physical id, so an unnamed cluster keeps the name its* first execution generated. createCluster is idempotent, which is what makes an unchanged- * update a no-op here.+ * update a no-op here. A new cluster is created with its declared {@code ClusterSettings}; an+ * existing one has them applied when they differ, and an update snapshots the prior ones for+ * its rollback.*/private void provisionCluster(StackResource r, JsonNode props, ProvisionContext ctx) {+ r.getAttributes().remove(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);String clusterName = ctx.stablePhysicalName(ctx.resolveOptional(props, "ClusterName"),r.getLogicalId(), 255, false);- EcsCluster cluster = ecsService.createCluster(clusterName, ctx.region());+ List<ClusterSetting> settings = new ArrayList<>();+ JsonNode declared = props != null ? props.get("ClusterSettings") : null;+ JsonNode resolved = declared != null ? ctx.engine().resolveNode(declared) : null;+ if (resolved != null && resolved.isArray()) {+ for (JsonNode item : resolved) {+ // An entry that resolved to AWS::NoValue is not an object.+ if (item.isObject()) {+ settings.add(new ClusterSetting(item.path("Name").asText(null), item.path("Value").asText(null)));+ }+ }+ }+ // A new cluster takes its settings at creation, so no later call can fail and leave it+ // behind; an existing one (this resource's own on an update, or one the idempotent create+ // returned) has them applied below.+ EcsCluster cluster = ecsService.createCluster(clusterName, null, settings, ctx.region());+ if (!settings.isEmpty() && !settings.equals(cluster.getSettings())) {+ List<ClusterSetting> prior = cluster.getSettings() == null ? null : List.copyOf(cluster.getSettings());+ if (ctx.reusesPriorEntity(clusterName)) {+ ObjectNode snapshot = MAPPER.createObjectNode().put("region", ctx.region());+ snapshot.set("settings", MAPPER.valueToTree(prior));+ r.getAttributes().put(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR, snapshot.toString());+ }+ // Before the id is set: a cluster the idempotent create returned for a resource this+ // update adds was not created by the stack, and a failed update deletes an added+ // resource that carries an id. An update of this resource's own cluster keeps its id.+ try {+ ecsService.updateClusterSettings(clusterName, settings, ctx.region());+ } catch (RuntimeException failure) {+ unwindSettings(r, clusterName, prior, ctx.region(), failure);+ throw failure;+ }+ }r.setPhysicalId(cluster.getClusterName());r.getAttributes().put("Arn", cluster.getClusterArn());}+ /**+ * Puts back the settings an existing cluster had before a settings call that failed, which may+ * have changed them before failing. For this resource's own cluster the engine restores the+ * resource it held before the attempt, which never carried the snapshot, so the rollback walker+ * would skip {@link #rollbackUpdate}; a cluster the stack found rather than created is not+ * restored by anything else. A restore that fails too is recorded as a rollback failure, which+ * ends an update in UPDATE_ROLLBACK_FAILED for the resource's own cluster, and is attached to+ * the original failure. The restore repeats the call that just failed, so it can come back+ * with that very exception, which cannot be suppressed under itself.+ */+ private void unwindSettings(StackResource r, String clusterName, List<ClusterSetting> prior, String region,+ RuntimeException failure) {+ try {+ ecsService.updateClusterSettings(clusterName, prior, region);+ r.getAttributes().remove(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR);+ } catch (RuntimeException unwindFailure) {+ String reason = "Could not restore the settings of ECS cluster " + clusterName + ": "+ + unwindFailure.getMessage();+ // ponytail: a cluster the stack found has no id here, so the engine drops the resource+ // and nothing retries this; carrying it would need engine-side tracking for added resources.+ LOG.warnv("{0}", reason);+ r.getAttributes().put(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR, reason);+ if (unwindFailure != failure) {+ failure.addSuppressed(unwindFailure);+ }+ }+ }+/*** Every property of a task definition is create-only, so an update registers a fresh revision* under the same family, as CloudFormation does on AWS; the physical id is the new revision'sdiff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex a4b1309ee2..1bbbfcff01 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -16,7 +16,9 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackedMap;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.Dimension;import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricDatum;import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;import io.github.hectorvent.floci.services.ecs.container.EcsLinuxParameters;import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;@@ -79,8 +81,10 @@ import org.jboss.logging.Logger;import java.net.URI;import java.nio.charset.StandardCharsets;+import java.time.Clock;import java.time.Duration;import java.time.Instant;+import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.Base64;import java.util.Collection;@@ -125,6 +129,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab// what every such caller already expects.private final EcsServiceDiscoveryRegistrar discoveryRegistrar;private final CloudWatchMetricsService cloudWatchMetricsService;+ private final Clock clock;private final boolean dockerMode;private final boolean sweepLeftoverContainers;private final String baseUrl;@@ -229,6 +234,8 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabprivate final Map<String, ServiceDeployment> serviceDeployments = new ConcurrentHashMap<>();// serviceRevisionArn → ServiceRevisionprivate final Map<String, ServiceRevision> serviceRevisions = new ConcurrentHashMap<>();+ // accountId::serviceKey → the Container Insights sample last written for that service+ private final Map<String, InsightsSample> publishedInsights = new ConcurrentHashMap<>();// targetArn → List<Attribute>private Map<String, List<Attribute>> attributes = new ConcurrentHashMap<>();// name → value (account-level settings)@@ -248,6 +255,17 @@ public class EcsService implements ContainerTeardown, ResourceProvider, ResettabEcsExecSessionRegistry execSessions,EcsServiceDiscoveryRegistrar discoveryRegistrar,CloudWatchMetricsService cloudWatchMetricsService) {+ this(regionResolver, containerManager, config, lbRegistrar, storageFactory, eventPublisher,+ execSessions, discoveryRegistrar, cloudWatchMetricsService, Clock.systemUTC());+ }++ /** With the clock that dates Container Insights samples, which a test pins. */+ EcsService(RegionResolver regionResolver, EcsContainerManager containerManager,+ EmulatorConfig config, EcsLoadBalancerRegistrar lbRegistrar,+ StorageFactory storageFactory, EcsEventPublisher eventPublisher,+ EcsExecSessionRegistry execSessions,+ EcsServiceDiscoveryRegistrar discoveryRegistrar,+ CloudWatchMetricsService cloudWatchMetricsService, Clock clock) {this.regionResolver = regionResolver;this.containerManager = containerManager;this.dockerMode = !config.services().ecs().mock();@@ -260,6 +278,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabthis.execSessions = execSessions;this.discoveryRegistrar = discoveryRegistrar;this.cloudWatchMetricsService = cloudWatchMetricsService;+ this.clock = clock;}public EcsService(RegionResolver regionResolver, EcsContainerManager containerManager,@@ -425,8 +444,10 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab@Overridepublic void clear() {- // The stores hold the ECS state and afterReset() restarts the reconciler.+ // The stores hold the ECS state and afterReset() restarts the reconciler. The reset also+ // wipes CloudWatch, so the Container Insights samples it held must be written again.launchFailures.clear();+ publishedInsights.clear();}/**@@ -5227,9 +5248,72 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab} catch (Exception e) {LOG.debugv("Error reconciling ECS service {0}: {1}", entry.getKey(), e.getMessage());}+ try {+ publishContainerInsights(entry.getKey(), entry.getValue());+ } catch (Exception e) {+ LOG.warnv(e, "Could not publish Container Insights task counts for ECS service {0}", entry.getKey());+ }}}+ /**+ * Container Insights task counts for a service on a cluster whose own {@code containerInsights}+ * setting is enabled or enhanced, one sample per minute, counted from the service's tasks once+ * the tick has started and stopped them. As in AWS, nothing is published while the service has+ * no RUNNING task, so an alarm sees missing data rather than a zero.+ */+ private void publishContainerInsights(String key, EcsServiceModel svc) {+ if (cloudWatchMetricsService == null || !STATUS_ACTIVE.equals(svc.getStatus())) {+ return;+ }+ String region = extractRegionFromServiceKey(key);+ String clusterName = extractClusterNameFromServiceKey(key);+ EcsCluster cluster = clusters.get(clusterKey(region, clusterName));+ if (cluster == null || cluster.getSettings() == null || cluster.getSettings().stream().noneMatch(s ->+ "containerInsights".equals(s.name())+ && ("enabled".equals(s.value()) || "enhanced".equals(s.value())))) {+ return;+ }+ List<String> statuses = tasks.values().stream()+ .filter(t -> ownedBy(t, svc, cluster))+ .map(EcsTask::getLastStatus)+ .toList();+ int running = Collections.frequency(statuses, TaskStatus.RUNNING.name());+ if (running < 1) {+ return;+ }+ long minute = clock.instant().truncatedTo(ChronoUnit.MINUTES).getEpochSecond();+ Map<String, Integer> counts = Map.of("RunningTaskCount", running,+ "PendingTaskCount", Collections.frequency(statuses, TaskStatus.PENDING.name()),+ "DesiredTaskCount", svc.getDesiredCount());+ String accountId = regionResolver.getAccountId();+ String sampleKey = accountId + "::" + key;+ InsightsSample sample = new InsightsSample(minute, counts);+ // A persistent metric store rewrites its file on every put, and the reconciler ticks every+ // five seconds, so the minute's sample is written again only when a count changes.+ if (sample.equals(publishedInsights.get(sampleKey))) {+ return;+ }+ // Forgotten before writing, so a write that fails partway is repeated on the next tick+ // even when the counts are back to the ones last written in full.+ publishedInsights.remove(sampleKey);+ for (Map.Entry<String, Integer> count : counts.entrySet()) {+ MetricDatum datum = new MetricDatum();+ datum.setMetricName(count.getKey());+ datum.setUnit("Count");+ datum.setDimensions(List.of(new Dimension("ClusterName", clusterName),+ new Dimension("ServiceName", svc.getServiceName())));+ datum.setTimestamp(minute);+ datum.setValue(count.getValue());+ cloudWatchMetricsService.publishMetricForAccount(accountId, "ECS/ContainerInsights",+ datum, region, "ecs-container-insights");+ }+ publishedInsights.put(sampleKey, sample);+ }++ private record InsightsSample(long minute, Map<String, Integer> counts) {+ }+/*** Whether a task is genuinely owned by {@code svc}. Both halves matter: {@code* owningServiceArn} is stamped only by {@link #launchServiceTask} and so cannot be forgeddiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.javaindex fd38ab9e98..0eca2c1798 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/EcsCfnProvisionerTest.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.cloudformation.CloudFormationTemplateEngine;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.ecs.EcsService;+import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsCluster;import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;@@ -17,8 +18,12 @@ import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;import io.github.hectorvent.floci.services.ecs.model.NetworkMode;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.NullSource;+import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;import org.mockito.InOrder;+import org.mockito.invocation.InvocationOnMock;import java.util.ArrayList;import java.util.HashMap;@@ -30,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotEquals;import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;@@ -116,7 +122,7 @@ class EcsCfnProvisionerTest {@Testvoid clusterUsesTheDeclaredNameAndExposesArn() {- when(ecs.createCluster("web", REGION)).thenReturn(cluster("web"));+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(cluster("web"));StackResource r = resource("AWS::ECS::Cluster", "Cluster");provisioner.provision(r, mapper.createObjectNode().put("ClusterName", "web"), ctx());@@ -128,7 +134,7 @@ class EcsCfnProvisionerTest {@Testvoid anUnnamedClusterGeneratesANameOnCreateAndKeepsItOnUpdate() {- when(ecs.createCluster(anyString(), eq(REGION))).thenAnswer(inv -> cluster(inv.getArgument(0)));+ when(ecs.createCluster(anyString(), isNull(), anyList(), eq(REGION))).thenAnswer(inv -> cluster(inv.getArgument(0)));StackResource created = resource("AWS::ECS::Cluster", "Cluster");provisioner.provision(created, mapper.createObjectNode(), ctx());String generated = created.getPhysicalId();@@ -140,7 +146,7 @@ class EcsCfnProvisionerTest {// createCluster is idempotent, so the update re-issues it under the same name instead of// minting a second cluster.assertEquals(generated, updated.getPhysicalId());- verify(ecs, times(2)).createCluster(generated, REGION);+ verify(ecs, times(2)).createCluster(eq(generated), isNull(), anyList(), eq(REGION));}@Test@@ -696,7 +702,7 @@ class EcsCfnProvisionerTest {@Testvoid anUnchangedClusterHasNothingToRollBack() {- when(ecs.createCluster("web", REGION)).thenReturn(cluster("web"));+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(cluster("web"));StackResource r = resource("AWS::ECS::Cluster", "Cluster");provisioner.provision(r, mapper.createObjectNode().put("ClusterName", "web"), ctx("web"));@@ -706,6 +712,246 @@ class EcsCfnProvisionerTest {verify(ecs, never()).deleteCluster(anyString(), anyString());}+ @Test+ void clusterSettingsAreAppliedOnCreate() {+ List<ClusterSetting> enabled = List.of(new ClusterSetting("containerInsights", "enabled"));+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenAnswer(inv -> created(inv));+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");++ provisioner.provision(r, clusterWithInsights("enabled"), ctx());++ // In the create itself, so no later call can fail and leave a cluster the rollback misses.+ verify(ecs).createCluster("web", null, enabled, REGION);+ verify(ecs, never()).updateClusterSettings(anyString(), any(), anyString());+ assertEquals(Set.of("Arn"), r.getAttributes().keySet());+ }++ @Test+ void aRenamedClusterTakesItsSettingsAtCreation() {+ List<ClusterSetting> enabled = List.of(new ClusterSetting("containerInsights", "enabled"));+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenAnswer(inv -> created(inv));+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");++ provisioner.provision(r, clusterWithInsights("enabled"), ctx("old"));++ verify(ecs).createCluster("web", null, enabled, REGION);+ verify(ecs, never()).updateClusterSettings(anyString(), any(), anyString());+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));+ }++ @Test+ void aClusterThatAlreadyExistedGetsItsSettingsBackAndIsNotDeleted() {+ // The idempotent create hands back a cluster the stack did not create, without the settings.+ List<ClusterSetting> priorSettings = List.of(new ClusterSetting("containerInsights", "disabled"));+ EcsCluster existing = cluster("web");+ existing.setSettings(priorSettings);+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(existing);+ AwsException failure = new AwsException("ServerException", "write failed", 500);+ when(ecs.updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enabled")), REGION))+ .thenThrow(failure);+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");++ AwsException thrown = assertThrows(AwsException.class,+ () -> provisioner.provision(r, clusterWithInsights("enabled"), ctx()));++ assertSame(failure, thrown);+ // The failed call may have changed them, and nothing else puts back a cluster the stack found.+ verify(ecs).updateClusterSettings("web", priorSettings, REGION);+ // Without an id, neither a create rollback nor an update that added this resource deletes it.+ assertNull(r.getPhysicalId());+ verify(ecs, never()).deleteCluster(anyString(), anyString());+ assertFalse(r.getAttributes().containsKey(CfnRollback.ROLLBACK_OWNED_ATTR));+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));+ }++ private EcsCluster created(InvocationOnMock inv) {+ EcsCluster c = cluster(inv.getArgument(0));+ List<ClusterSetting> settings = inv.getArgument(2);+ c.setSettings(settings.isEmpty() ? null : settings);+ return c;+ }++ @ParameterizedTest+ @NullSource+ @ValueSource(strings = "disabled")+ void aFailedStackUpdateRestoresThePriorClusterSettings(String priorValue) {+ List<ClusterSetting> priorSettings = priorValue == null ? null : List.of(new ClusterSetting("containerInsights", priorValue));+ EcsCluster prior = cluster("web");+ prior.setSettings(priorSettings);+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");++ provisioner.provision(r, clusterWithInsights("enhanced"), ctx("web"));+ verify(ecs).updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enhanced")), REGION);++ assertTrue(provisioner.rollbackUpdate(r));+ verify(ecs).updateClusterSettings("web", priorSettings, REGION);+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));+ }++ @Test+ void aSettingsUpdateThatFailsPutsThePriorSettingsBackItself() {+ List<ClusterSetting> priorSettings = List.of(new ClusterSetting("containerInsights", "disabled"));+ EcsCluster prior = cluster("web");+ prior.setSettings(priorSettings);+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);+ AwsException failure = new AwsException("ServerException", "write failed", 500);+ when(ecs.updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enhanced")), REGION))+ .thenThrow(failure);+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");+ // An update hands the provisioner the resource with its prior physical id.+ r.setPhysicalId("web");++ AwsException thrown = assertThrows(AwsException.class,+ () -> provisioner.provision(r, clusterWithInsights("enhanced"), ctx("web")));++ // The engine restores the resource it held before this attempt, which never carried the+ // snapshot, so the rollback walker would not put these settings back.+ assertSame(failure, thrown);+ verify(ecs).updateClusterSettings("web", priorSettings, REGION);+ assertFalse(r.getAttributes().containsKey(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));+ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));+ }++ @Test+ void aSettingsRestoreThatAlsoFailsIsRecordedAsARollbackFailure() {+ EcsCluster prior = cluster("web");+ prior.setSettings(List.of(new ClusterSetting("containerInsights", "disabled")));+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);+ AwsException failure = new AwsException("ServerException", "write failed", 500);+ AwsException restoreFailure = new AwsException("ServerException", "still failing", 500);+ when(ecs.updateClusterSettings(eq("web"), any(), eq(REGION))).thenThrow(failure, restoreFailure);+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");+ // An update hands the provisioner the resource with its prior physical id.+ r.setPhysicalId("web");++ AwsException thrown = assertThrows(AwsException.class,+ () -> provisioner.provision(r, clusterWithInsights("enhanced"), ctx("web")));++ assertSame(failure, thrown);+ assertEquals(List.of(restoreFailure), List.of(thrown.getSuppressed()));+ assertEquals("Could not restore the settings of ECS cluster web: still failing",+ r.getAttributes().get(CfnRollback.UPDATE_ROLLBACK_FAILURE_ATTR));+ }++ @Test+ void aRestoreThatFailedIsRetriedWhenTheRollbackIsContinued() {+ List<ClusterSetting> priorSettings = List.of(new ClusterSetting("containerInsights", "disabled"));+ EcsCluster prior = cluster("web");+ prior.setSettings(priorSettings);+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);+ when(ecs.updateClusterSettings(eq("web"), any(), eq(REGION)))+ .thenThrow(new AwsException("ServerException", "write failed", 500))+ .thenThrow(new AwsException("ServerException", "still failing", 500))+ .thenReturn(prior);+ StackResource previous = resource("AWS::ECS::Cluster", "Cluster");+ previous.setPhysicalId("web");+ StackResource attempted = resource("AWS::ECS::Cluster", "Cluster");+ attempted.setPhysicalId("web");+ assertThrows(AwsException.class,+ () -> provisioner.provision(attempted, clusterWithInsights("enhanced"), ctx("web")));++ // The engine puts the previous resource back after the failed provision, and the first+ // rollback walk reports the recorded failure without calling rollbackUpdate.+ provisioner.mergeFailedUpdateResourceTracking(previous, attempted);+ // ContinueUpdateRollback then asks for the restore again.+ assertTrue(provisioner.rollbackUpdate(previous));++ verify(ecs, times(2)).updateClusterSettings("web", priorSettings, REGION);+ assertFalse(previous.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));+ }++ @Test+ void aRestoreThatSucceededLeavesNothingToCarryOver() {+ EcsCluster prior = cluster("web");+ prior.setSettings(List.of(new ClusterSetting("containerInsights", "disabled")));+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);+ when(ecs.updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enhanced")), REGION))+ .thenThrow(new AwsException("ServerException", "write failed", 500));+ StackResource previous = resource("AWS::ECS::Cluster", "Cluster");+ StackResource attempted = resource("AWS::ECS::Cluster", "Cluster");+ assertThrows(AwsException.class,+ () -> provisioner.provision(attempted, clusterWithInsights("enhanced"), ctx("web")));++ provisioner.mergeFailedUpdateResourceTracking(previous, attempted);++ assertFalse(previous.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));+ }++ @Test+ void aCommittedSettingsUpdateDropsItsSnapshot() {+ EcsCluster prior = cluster("web");+ prior.setSettings(List.of(new ClusterSetting("containerInsights", "disabled")));+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(prior);+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");+ provisioner.provision(r, clusterWithInsights("enabled"), ctx("web"));++ assertEquals(UpdateCleanupResult.notApplicable(), provisioner.completeUpdate(r));++ assertFalse(r.getAttributes().containsKey(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR));+ }++ @Test+ void anUnchangedSettingIsNotReappliedAndDropsAnEarlierSnapshot() {+ EcsCluster current = cluster("web");+ current.setSettings(List.of(new ClusterSetting("containerInsights", "enabled")));+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(current);+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");+ r.getAttributes().put(CfnRollback.ECS_CLUSTER_SETTINGS_SNAPSHOT_ATTR,+ "{\"region\":\"us-east-1\",\"settings\":null}");++ provisioner.provision(r, clusterWithInsights("enabled"), ctx("web"));++ assertTrue(provisioner.rollbackUpdate(r));+ verify(ecs, never()).updateClusterSettings(anyString(), any(), anyString());+ }++ @Test+ void conditionalClusterSettingsAreResolvedBeforeTheyAreRead() {+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(cluster("web"));+ StackResource r = resource("AWS::ECS::Cluster", "Cluster");+ ObjectNode props = clusterWithInsights("enabled");+ JsonNode resolved = props.get("ClusterSettings");+ ObjectNode conditional = mapper.createObjectNode();+ conditional.putArray("Fn::If").add("Insights").add(resolved).addObject().put("Ref", "AWS::NoValue");+ props.set("ClusterSettings", conditional);+ ProvisionContext ctx = ctx();+ when(ctx.engine().resolveNode(conditional)).thenReturn(resolved);++ provisioner.provision(r, props, ctx);++ verify(ecs).updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enabled")), REGION);+ }++ @Test+ void clusterSettingsThatFailToResolveCreateNoCluster() {+ ObjectNode props = clusterWithInsights("enabled");+ ProvisionContext ctx = ctx();+ when(ctx.engine().resolveNode(props.get("ClusterSettings"))).thenThrow(new IllegalStateException("unresolvable"));++ assertThrows(IllegalStateException.class,+ () -> provisioner.provision(resource("AWS::ECS::Cluster", "Cluster"), props, ctx));++ verify(ecs, never()).createCluster(anyString(), any(), any(), anyString());+ }++ @Test+ void aClusterSettingResolvedToNoValueIsDropped() {+ when(ecs.createCluster(eq("web"), isNull(), anyList(), eq(REGION))).thenReturn(cluster("web"));+ ObjectNode props = clusterWithInsights("enabled");+ ((ArrayNode) props.get("ClusterSettings")).add("");++ provisioner.provision(resource("AWS::ECS::Cluster", "Cluster"), props, ctx());++ verify(ecs).updateClusterSettings("web", List.of(new ClusterSetting("containerInsights", "enabled")), REGION);+ }++ private ObjectNode clusterWithInsights(String value) {+ ObjectNode props = mapper.createObjectNode().put("ClusterName", "web");+ props.putArray("ClusterSettings").addObject().put("Name", "containerInsights").put("Value", value);+ return props;+ }+@Testvoid anInPlaceUpdateIsNotRolledBackHere() {when(ecs.updateService(eq("web"), eq("front"), eq(TASK_DEF_ARN), eq(3), isNull(), eq(REGION)))diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsContainerInsightsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsContainerInsightsIntegrationTest.javanew file mode 100644index 0000000000..9a236584f3--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsContainerInsightsIntegrationTest.java@@ -0,0 +1,202 @@+package io.github.hectorvent.floci.services.ecs;++import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.mockito.InjectSpy;+import io.restassured.specification.RequestSpecification;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.time.Instant;+import java.util.List;++import static io.restassured.RestAssured.given;+import static org.awaitility.Awaitility.await;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasItem;+import static org.mockito.ArgumentMatchers.anyList;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.doCallRealMethod;++/**+ * A cluster whose CloudFormation {@code ClusterSettings} enable Container Insights publishes its+ * services' task counts to {@code ECS/ContainerInsights}, which is what an alarm on a stopped+ * service watches. The test drives the reconciler itself, as the other ECS integration tests do,+ * rather than waiting on its five-second ticks: one pass launches the task, the next counts it.+ */+@QuarkusTest+class EcsContainerInsightsIntegrationTest {++ @InjectSpy+ EcsService ecsService;++ private static final String CLOUDWATCH_AUTH =+ "AWS4-HMAC-SHA256 Credential=AKID/20260908/us-east-1/monitoring/aws4_request";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void aClusterWithInsightsFromCloudFormationPublishesItsRunningTaskCount() {+ String cluster = "insights-" + Long.toString(System.nanoTime(), 36);+ String template = """+ {"Resources": {+ "Cluster": {"Type": "AWS::ECS::Cluster", "Properties": {"ClusterName": "%1$s",+ "ClusterSettings": [{"Name": "containerInsights", "Value": "enabled"}]}},+ "TaskDef": {"Type": "AWS::ECS::TaskDefinition", "Properties": {"Family": "%1$s",+ "NetworkMode": "awsvpc", "RequiresCompatibilities": ["FARGATE"], "Cpu": "256", "Memory": "512",+ "ContainerDefinitions": [{"Name": "app", "Image": "nginx:alpine", "Essential": true}]}},+ "Service": {"Type": "AWS::ECS::Service", "Properties": {"ServiceName": "web",+ "Cluster": {"Ref": "Cluster"}, "TaskDefinition": {"Ref": "TaskDef"},+ "DesiredCount": 1, "LaunchType": "FARGATE", "NetworkConfiguration": {"AwsvpcConfiguration":+ {"Subnets": ["subnet-default-us-east-1-a"]}}}}}}+ """.formatted(cluster);++ given().contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateStack")+ .formParam("StackName", cluster)+ .formParam("TemplateBody", template)+ .post("/").then().statusCode(200);+ given().contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", cluster)+ .post("/").then().statusCode(200)+ .body(containsString("<StackStatus>CREATE_COMPLETE</StackStatus>"));+ given().contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.DescribeClusters")+ .body("{\"clusters\": [\"" + cluster + "\"], \"include\": [\"SETTINGS\"]}")+ .post("/").then().statusCode(200)+ .body("clusters[0].settings[0].value", equalTo("enabled"));++ ecsService.reconcileServices();+ ecsService.reconcileServices();++ given()+ .contentType("application/x-amz-json-1.0")+ .header("Authorization", CLOUDWATCH_AUTH)+ .header("X-Amz-Target", "GraniteServiceVersion20100801.GetMetricStatistics")+ .body("""+ {"Namespace": "ECS/ContainerInsights", "MetricName": "RunningTaskCount",+ "Dimensions": [{"Name": "ClusterName", "Value": "%s"}, {"Name": "ServiceName", "Value": "web"}],+ "StartTime": %d, "EndTime": %d, "Period": 60, "Statistics": ["Maximum"]}+ """.formatted(cluster, Instant.now().minusSeconds(300).getEpochSecond(),+ Instant.now().plusSeconds(60).getEpochSecond()))+ .post("/").then().statusCode(200)+ .body("Datapoints.Maximum", hasItem(1.0f));+ }++ /**+ * The update's settings land and the call still fails, and the provisioner's own restore fails+ * too, so the stack ends in UPDATE_ROLLBACK_FAILED with the new setting live. The snapshot+ * survives on the resource the stack keeps, so ContinueUpdateRollback puts the prior setting+ * back rather than reporting a rollback with nothing left to do.+ */+ @Test+ void continueUpdateRollbackRetriesASettingsRestoreThatFailed() {+ String cluster = "insights-rb-" + Long.toString(System.nanoTime(), 36);+ String template = """+ {"Resources": {"Cluster": {"Type": "AWS::ECS::Cluster", "Properties": {"ClusterName": "%s",+ "ClusterSettings": [{"Name": "containerInsights", "Value": "%s"}]}}}}+ """;+ cloudFormation("CreateStack", cluster, template.formatted(cluster, "disabled"));+ awaitStackStatus(cluster, "CREATE_COMPLETE");++ List<ClusterSetting> enabled = List.of(new ClusterSetting("containerInsights", "enabled"));+ doAnswer(invocation -> {+ if (enabled.equals(invocation.getArgument(1))) {+ invocation.callRealMethod();+ }+ throw new IllegalStateException("simulated settings write failure");+ }).when(ecsService).updateClusterSettings(anyString(), anyList(), anyString());+ try {+ cloudFormation("UpdateStack", cluster, template.formatted(cluster, "enabled"));+ awaitStackStatus(cluster, "UPDATE_ROLLBACK_FAILED");+ } finally {+ doCallRealMethod().when(ecsService).updateClusterSettings(anyString(), anyList(), anyString());+ }+ assertInsightsSetting(cluster, "enabled");++ cloudFormation("ContinueUpdateRollback", cluster, null);+ awaitStackStatus(cluster, "UPDATE_ROLLBACK_COMPLETE");+ assertInsightsSetting(cluster, "disabled");+ }++ /**+ * An update that adds a cluster under the name of one created outside the stack gets that+ * cluster back from the idempotent create. When its settings then land and the call still+ * fails, the cluster gets its own settings back, and the update rollback, which deletes what+ * the update added, leaves the cluster in place because the resource never claimed it.+ */+ @Test+ void aFailedUpdateLeavesAClusterItFoundAsItWas() {+ String name = "insights-pre-" + Long.toString(System.nanoTime(), 36);+ given().contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.CreateCluster")+ .body("{\"clusterName\": \"" + name + "\", "+ + "\"settings\": [{\"name\": \"containerInsights\", \"value\": \"disabled\"}]}")+ .post("/").then().statusCode(200);+ String parameter = """+ "Param": {"Type": "AWS::SSM::Parameter", "Properties": {"Type": "String", "Value": "v"}}""";+ cloudFormation("CreateStack", name, "{\"Resources\": {" + parameter + "}}");+ awaitStackStatus(name, "CREATE_COMPLETE");++ List<ClusterSetting> enabled = List.of(new ClusterSetting("containerInsights", "enabled"));+ doAnswer(invocation -> {+ Object result = invocation.callRealMethod();+ if (enabled.equals(invocation.getArgument(1))) {+ throw new IllegalStateException("simulated settings write failure");+ }+ return result;+ }).when(ecsService).updateClusterSettings(anyString(), anyList(), anyString());+ try {+ cloudFormation("UpdateStack", name, """+ {"Resources": {%s, "Cluster": {"Type": "AWS::ECS::Cluster", "Properties": {"ClusterName": "%s",+ "ClusterSettings": [{"Name": "containerInsights", "Value": "enabled"}]}}}}+ """.formatted(parameter, name));+ awaitStackStatus(name, "UPDATE_ROLLBACK_COMPLETE");+ } finally {+ doCallRealMethod().when(ecsService).updateClusterSettings(anyString(), anyList(), anyString());+ }++ given().contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.DescribeClusters")+ .body("{\"clusters\": [\"" + name + "\"], \"include\": [\"SETTINGS\"]}")+ .post("/").then().statusCode(200)+ .body("clusters[0].status", equalTo("ACTIVE"))+ .body("clusters[0].settings[0].value", equalTo("disabled"));+ }++ private static void cloudFormation(String action, String stack, String template) {+ RequestSpecification request = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", action)+ .formParam("StackName", stack);+ if (template != null) {+ request.formParam("TemplateBody", template);+ }+ request.post("/").then().statusCode(200);+ }++ private static void awaitStackStatus(String stack, String status) {+ await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DescribeStacks")+ .formParam("StackName", stack)+ .post("/").then().statusCode(200)+ .body(containsString("<StackStatus>" + status + "</StackStatus>")));+ }++ private static void assertInsightsSetting(String cluster, String value) {+ given().contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.DescribeClusters")+ .body("{\"clusters\": [\"" + cluster + "\"], \"include\": [\"SETTINGS\"]}")+ .post("/").then().statusCode(200)+ .body("clusters[0].settings[0].value", equalTo(value));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.javanew file mode 100644index 0000000000..f79cf28d5b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceContainerInsightsTest.java@@ -0,0 +1,196 @@+package io.github.hectorvent.floci.services.ecs;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;+import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService.Datapoint;+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.Dimension;+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;+import io.github.hectorvent.floci.services.ecs.exec.EcsExecSessionRegistry;+import io.github.hectorvent.floci.services.ecs.model.ClusterSetting;+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;+import io.github.hectorvent.floci.services.ecs.model.LaunchType;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;+import java.util.HashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.clearInvocations;+import static org.mockito.Mockito.doCallRealMethod;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * Container Insights task counts, published by the service reconciler into a real CloudWatch+ * metric store: {@code ECS/ContainerInsights} on {@code ClusterName} and {@code ServiceName}, once+ * per minute, only for a cluster whose containerInsights setting turns it on and only while the+ * service has a RUNNING task.+ */+class EcsServiceContainerInsightsTest {++ private static final String REGION = "us-east-1";+ private static final String NAMESPACE = "ECS/ContainerInsights";+ private static final Instant MINUTE = Instant.parse("2026-10-20T12:00:00Z");++ private CloudWatchMetricsService metrics;+ private EcsService service;++ @BeforeEach+ void setUp() {+ InMemoryStorageFactory storage = new InMemoryStorageFactory();+ RegionResolver regionResolver = new RegionResolver(REGION, "000000000000");+ metrics = spy(new CloudWatchMetricsService(storage, regionResolver));+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(true);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ service = new EcsService(regionResolver, mock(EcsContainerManager.class), config,+ mock(EcsLoadBalancerRegistrar.class), storage, null, new EcsExecSessionRegistry(), null, metrics,+ Clock.fixed(MINUTE.plusSeconds(42), ZoneOffset.UTC));+ service.initializeStorage();+ ContainerDefinition cd = new ContainerDefinition();+ cd.setName("app");+ cd.setImage("nginx:alpine");+ service.registerTaskDefinition("ci-fam", List.of(cd), null, null, null, null, null, List.of(), REGION);+ }++ @ParameterizedTest+ @ValueSource(strings = {"enabled", "enhanced"})+ void anInsightsClusterPublishesItsServiceTaskCounts(String mode) {+ startService(List.of(new ClusterSetting("containerInsights", mode)), 1);++ service.reconcile();++ Datapoint running = only("RunningTaskCount");+ assertEquals(MINUTE, running.timestamp());+ assertEquals(1.0, running.maximum());+ assertEquals("Count", running.unit());+ assertEquals(0.0, only("PendingTaskCount").maximum());+ assertEquals(1.0, only("DesiredTaskCount").maximum());+ }++ @Test+ void repeatedTicksKeepOneSamplePerMinute() {+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 1);++ service.reconcile();+ service.reconcile();++ List<Datapoint> points = statistics("RunningTaskCount");+ assertFalse(points.isEmpty());+ assertTrue(points.stream().allMatch(p -> p.sampleCount() == 1.0), points.toString());+ }++ @Test+ void anUnchangedSampleIsNotWrittenAgainWithinTheMinute() {+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 1);++ service.reconcile();+ service.reconcile();++ verify(metrics, times(3)).publishMetricForAccount(anyString(), eq(NAMESPACE), any(), eq(REGION), anyString());+ }++ @Test+ void aWriteThatFailsPartwayIsRepeatedEvenWhenTheCountsReturn() {+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 1);+ service.reconcile();+ service.updateService("insights", "web", null, 2, null, REGION);+ // One of the three counts lands, the next write fails.+ doCallRealMethod().doThrow(new IllegalStateException("disk full")).doCallRealMethod().when(metrics)+ .publishMetricForAccount(anyString(), eq(NAMESPACE), any(), eq(REGION), anyString());+ service.reconcile();++ service.updateService("insights", "web", null, 1, null, REGION);+ clearInvocations(metrics);+ service.reconcile();++ verify(metrics, times(3)).publishMetricForAccount(anyString(), eq(NAMESPACE), any(), eq(REGION), anyString());+ }++ @Test+ void aScaleInIsCountedInTheTickThatStopsTheTask() {+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 2);+ service.reconcile();++ service.updateService("insights", "web", null, 1, null, REGION);+ service.reconcile();++ assertEquals(1.0, statistics("RunningTaskCount").getLast().maximum());+ assertEquals(1.0, statistics("DesiredTaskCount").getLast().maximum());+ }++ @Test+ void aClusterWithoutInsightsPublishesNothing() {+ startService(List.of(new ClusterSetting("containerInsights", "disabled")), 1);+ service.createCluster("plain", REGION);+ service.createService("plain", "web", "ci-fam", 1, LaunchType.FARGATE, List.of(), null, REGION);++ service.reconcile();++ assertTrue(metrics.listMetrics(NAMESPACE, null, null, REGION).isEmpty());+ }++ @Test+ void aServiceWithoutARunningTaskPublishesNothing() {+ startService(List.of(new ClusterSetting("containerInsights", "enabled")), 0);++ service.reconcile();++ assertTrue(metrics.listMetrics(NAMESPACE, null, null, REGION).isEmpty());+ }++ private void startService(List<ClusterSetting> settings, int desiredCount) {+ service.createCluster("insights", null, settings, REGION);+ service.createService("insights", "web", "ci-fam", desiredCount, LaunchType.FARGATE, List.of(), null, REGION);+ }++ private Datapoint only(String metricName) {+ List<Datapoint> points = statistics(metricName);+ assertEquals(1, points.size(), metricName + ": " + points);+ return points.get(0);+ }++ private List<Datapoint> statistics(String metricName) {+ return metrics.getMetricStatistics(NAMESPACE, metricName,+ List.of(new Dimension("ServiceName", "web"), new Dimension("ClusterName", "insights")),+ MINUTE.minusSeconds(600), MINUTE.plusSeconds(600), 60, List.of("Maximum"), null, REGION);+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();++ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ @SuppressWarnings("unchecked")+ public <V> AccountAwareStorageBackend<V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,+ ignored -> AccountAwareStorageBackend.inMemory("000000000000"));+ }+ }+}