Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudwatch): evaluate alarms over CloudWatch's wider evaluation range (#2771)
Matheus Oliveiracfe8566511cf
diff --git a/docs/services/applicationautoscaling.md b/docs/services/applicationautoscaling.mdindex 2071fe2ead..09f765f97d 100644--- a/docs/services/applicationautoscaling.md+++ b/docs/services/applicationautoscaling.md@@ -138,15 +138,17 @@ arn:aws:iam::<account>:role/aws-service-role/<namespace>.application-autoscalingvalidated; the configuration block is not stored.- Pagination is not implemented — `DescribeScalableTargets` and `DescribeScalingPolicies`return all matching results and never emit a `NextToken`.-- **Missing-data evaluation uses a fixed window, not AWS's sliding one.** Real CloudWatch- queries a wider "evaluation range" than `Period × EvaluationPeriods` and, whenever enough- *real* datapoints exist within that wider range, evaluates using those and ignores- `TreatMissingData` entirely — falling back to it only when genuinely too few real- datapoints exist even with the extra lookback. Real CloudWatch also has "premature- transition" avoidance logic that can delay a move into `ALARM` when trailing data is- missing. Floci's evaluator queries exactly `EvaluationPeriods` period-aligned buckets and- applies `TreatMissingData` as soon as any of them is empty — correct for the common case- (continuous metric reporting) but less faithful once data has partial, intermittent gaps.+- **The evaluation range's exact width is an approximation.** Alarm evaluation follows+ CloudWatch's documented precedence: a wider "evaluation range" than+ `Period × EvaluationPeriods` is queried, and whenever enough *real* datapoints exist within+ it the alarm is evaluated on those and `TreatMissingData` is ignored entirely; the setting+ fills in only what real data cannot cover. The premature-transition rule is implemented too,+ so a breach that has aged past `DatapointsToAlarm` with only missing periods after it reaches+ `ALARM`, while a breach at the very end of the window does not. AWS does not publish how wide+ the range is (only that it varies with period length and metric resolution), so Floci uses+ `EvaluationPeriods + 2`, which reproduces the single worked example in AWS's documentation.+ Both published example tables are transcribed as tests in+ `AlarmEvaluatorMissingDataTablesTest`.- **Scale-out cooldown rarely blocks a repeated scale-out in practice.** Its "proceed iflarger than the last applied capacity" carve-out (see above) is almost always satisfiedwhen nothing external has changed capacity, because both capacity formulas growdiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluator.javaindex efcc1b63b2..cfe5c3e889 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluator.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluator.java@@ -12,6 +12,7 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.time.Instant;+import java.util.ArrayList;import java.util.List;import java.util.concurrent.Executors;import java.util.concurrent.ScheduledExecutorService;@@ -90,11 +91,20 @@ public class AlarmEvaluator {}/**- * Evaluates the last {@code evaluationPeriods} <em>complete</em> periods, not counting- * whichever period is still accumulating right now — matching real CloudWatch, which- * evaluates elapsed periods only. Counting the in-progress period would make it look- * "missing" on nearly every tick even under perfectly healthy, continuous reporting,- * since data for it typically hasn't arrived yet.+ * Queries the <em>evaluation range</em> — more periods than {@code evaluationPeriods} — and+ * resolves the state with CloudWatch's documented precedence ("How alarm state is evaluated+ * when data is missing"): real datapoints reaching further back are preferred, and {@code+ * TreatMissingData} only fills what real data cannot cover.+ *+ * <p>Once the range holds at least {@code evaluationPeriods} real datapoints, the most recent+ * of those decide the state and {@code TreatMissingData} is not consulted at all — AWS: "the+ * value you set for how to treat missing data is not needed and is ignored". Reaching further+ * back is precisely what makes that branch reachable, so widening the query narrows how often+ * the fallback applies without changing what any of its modes mean.</p>+ *+ * <p>Only <em>complete</em> periods are considered; the period still accumulating right now+ * is excluded, since data for it typically has not arrived yet and counting it would make it+ * look missing on nearly every tick even under perfectly healthy reporting.</p>*/void evaluate(MetricAlarm alarm) {if (!isEvaluable(alarm)) {@@ -103,50 +113,71 @@ public class AlarmEvaluator {String region = alarm.getRegion();int period = alarm.getPeriod();int evaluationPeriods = alarm.getEvaluationPeriods();+ int range = evaluationRange(evaluationPeriods);long nowBucket = (Instant.now().getEpochSecond() / period) * period;long lastCompleteBucket = nowBucket - period;- long oldestBucket = lastCompleteBucket - (long) period * (evaluationPeriods - 1);+ long oldestBucket = lastCompleteBucket - (long) period * (range - 1);Instant start = Instant.ofEpochSecond(oldestBucket);Instant end = Instant.ofEpochSecond(lastCompleteBucket + period - 1);- List<CloudWatchMetricsService.Datapoint> recent = metricsService.getMetricStatistics(+ List<CloudWatchMetricsService.Datapoint> retrieved = metricsService.getMetricStatistics(alarm.getNamespace(), alarm.getMetricName(), alarm.getDimensions(),start, end, period, List.of(alarm.getStatistic()), alarm.getUnit(), region);- int missing = evaluationPeriods - recent.size();- int breaching = 0;- double latestValue = Double.NaN;- for (CloudWatchMetricsService.Datapoint dp : recent) {- double value = CloudWatchMetricsService.resolveStatValue(dp, alarm.getStatistic());- if (breaches(value, alarm.getComparisonOperator(), alarm.getThreshold())) {- breaching++;- latestValue = value;+ Double[] buckets = new Double[range];+ for (CloudWatchMetricsService.Datapoint dp : retrieved) {+ long bucket = (dp.timestamp().getEpochSecond() / period) * period;+ int index = (int) ((bucket - oldestBucket) / period);+ if (index >= 0 && index < range) {+ buckets[index] = CloudWatchMetricsService.resolveStatValue(dp, alarm.getStatistic());+ }+ }++ List<Double> real = new ArrayList<>();+ for (Double value : buckets) {+ if (value != null) {+ real.add(value);}}String newState;String reason;- if (missing > 0) {+ double latestValue;+ if (real.size() >= evaluationPeriods) {+ List<Double> evaluated = real.subList(real.size() - evaluationPeriods, real.size());+ int breaching = countBreaches(evaluated, alarm);+ latestValue = latestBreachingValue(evaluated, alarm);+ newState = evaluateBreachCount(breaching, alarm, evaluationPeriods);+ reason = ("ALARM".equals(newState) ? "Threshold Crossed: " : "Threshold Not Crossed: ")+ + breaching + " datapoint(s) breaching the threshold.";+ } else {+ int fill = evaluationPeriods - real.size();+ int breaching = countBreaches(real, alarm);+ latestValue = latestBreachingValue(real, alarm);String treatMissingData = alarm.getTreatMissingData();if ("ignore".equalsIgnoreCase(treatMissingData)) {newState = alarm.getStateValue();reason = alarm.getStateReason();} else if ("breaching".equalsIgnoreCase(treatMissingData)) {- newState = evaluateBreachCount(breaching + missing, alarm, evaluationPeriods);+ newState = evaluateBreachCount(breaching + fill, alarm, evaluationPeriods);reason = "Threshold Crossed (missing datapoints treated as breaching): "- + (breaching + missing) + " datapoint(s) breaching the threshold.";+ + (breaching + fill) + " datapoint(s) breaching the threshold.";} else if ("notBreaching".equalsIgnoreCase(treatMissingData)) {newState = evaluateBreachCount(breaching, alarm, evaluationPeriods);reason = "Threshold evaluated with missing datapoints treated as not breaching: "+ breaching + " datapoint(s) breaching the threshold.";- } else {+ } else if (real.isEmpty()) {newState = "INSUFFICIENT_DATA";- reason = "Insufficient Data: " + recent.size() + " of " + evaluationPeriods + " datapoints available";+ reason = "Insufficient Data: 0 of " + evaluationPeriods + " datapoints available";+ } else if (settledBreach(buckets, alarm, evaluationPeriods)) {+ newState = "ALARM";+ reason = "Threshold Crossed: the oldest breaching datapoint is old enough to alarm "+ + "and every more recent datapoint is breaching or missing.";+ } else {+ newState = evaluateBreachCount(breaching, alarm, evaluationPeriods);+ reason = ("ALARM".equals(newState) ? "Threshold Crossed: " : "Threshold Not Crossed: ")+ + breaching + " real datapoint(s) breaching the threshold.";}- } else {- newState = evaluateBreachCount(breaching, alarm, evaluationPeriods);- reason = ("ALARM".equals(newState) ? "Threshold Crossed: " : "Threshold Not Crossed: ")- + breaching + " datapoint(s) breaching the threshold.";}if (!newState.equals(alarm.getStateValue())) {@@ -158,6 +189,68 @@ public class AlarmEvaluator {}}+ /**+ * How many periods to retrieve. AWS states only that it is "a higher number of data points+ * than the number specified as Evaluation Periods", varying with period length and metric+ * resolution, without publishing the formula; its one worked example pairs+ * {@code EvaluationPeriods = 3} with an evaluation range of 5, which is what this reproduces.+ */+ private static int evaluationRange(int evaluationPeriods) {+ return evaluationPeriods + 2;+ }++ /**+ * CloudWatch's premature-transition rule, which fires when {@code TreatMissingData} is left at+ * its {@code missing} default: an alarm goes to {@code ALARM} "when the oldest available+ * breaching datapoint during the Evaluation Periods number of data points is at least as old+ * as the value of Datapoints to Alarm" and every more recent datapoint is breaching or+ * missing. A lone breach at the very end of the window ({@code - - - - X}) deliberately does+ * not qualify — the next datapoint may be non-breaching — while one that has already aged past+ * {@code DatapointsToAlarm} ({@code - - X - -}) does, even with fewer real datapoints than M.+ *+ * <p>"All other" spans the whole evaluation range, not just the most recent {@code+ * EvaluationPeriods}: a single non-breaching reading anywhere in the range disqualifies the+ * shortcut. That is what separates {@code 0 - X - -} (documented {@code OK} at+ * {@code DatapointsToAlarm = 2}) from {@code - - X - -} (documented {@code ALARM}), since the+ * two are indistinguishable across their most recent three buckets alone.</p>+ */+ private static boolean settledBreach(Double[] buckets, MetricAlarm alarm, int evaluationPeriods) {+ int datapointsToAlarm = alarm.getDatapointsToAlarm() > 0+ ? alarm.getDatapointsToAlarm() : evaluationPeriods;+ int oldestBreachingAge = 0;+ for (int age = 1; age <= buckets.length; age++) {+ Double value = buckets[buckets.length - age];+ if (value == null) {+ continue;+ }+ if (!breaches(value, alarm.getComparisonOperator(), alarm.getThreshold())) {+ return false;+ }+ oldestBreachingAge = age;+ }+ return oldestBreachingAge >= datapointsToAlarm;+ }++ private static int countBreaches(List<Double> values, MetricAlarm alarm) {+ int breaching = 0;+ for (Double value : values) {+ if (breaches(value, alarm.getComparisonOperator(), alarm.getThreshold())) {+ breaching++;+ }+ }+ return breaching;+ }++ private static double latestBreachingValue(List<Double> values, MetricAlarm alarm) {+ double latest = Double.NaN;+ for (Double value : values) {+ if (breaches(value, alarm.getComparisonOperator(), alarm.getThreshold())) {+ latest = value;+ }+ }+ return latest;+ }+private static String evaluateBreachCount(int breaching, MetricAlarm alarm, int evaluationPeriods) {int datapointsToAlarm = alarm.getDatapointsToAlarm() > 0 ? alarm.getDatapointsToAlarm() : evaluationPeriods;return breaching >= datapointsToAlarm ? "ALARM" : "OK";diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorMissingDataTablesTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorMissingDataTablesTest.javanew file mode 100644index 0000000000..54d08100c5--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorMissingDataTablesTest.java@@ -0,0 +1,161 @@+package io.github.hectorvent.floci.services.cloudwatch.metrics;++import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;+import jakarta.enterprise.inject.Instance;+import org.junit.jupiter.api.Test;++import java.time.Instant;+import java.util.ArrayList;+import java.util.List;++import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * The two worked examples from AWS's "Configuring how CloudWatch alarms treat missing data",+ * transcribed as executable cases. Each test is one row of a published table and asserts the+ * documented state for all four {@code TreatMissingData} settings at once.+ *+ * <p>Patterns read oldest-to-newest, as in the AWS tables: {@code 0} is a non-breaching datapoint,+ * {@code X} a breaching one and {@code -} a missing period. Both tables use+ * {@code EvaluationPeriods = 3}, giving an evaluation range of 5.</p>+ */+class AlarmEvaluatorMissingDataTablesTest {++ private static final String REGION = "us-east-1";+ private static final double THRESHOLD = 50.0;+ private static final String RETAIN = "<retain>";++ @Test+ void enoughRealDatapointsIgnoresTheMissingDataSetting() {+ assertRow("0 - X - X", 3, "OK", "OK", "OK", "OK");+ }++ @Test+ void oneOldNonBreachingDatapointStaysOkUnderEverySetting() {+ assertRow("0 - - - -", 3, "OK", "OK", "OK", "OK");+ }++ @Test+ void anEntirelyEmptyRangeIsTheOnlyInsufficientDataCase() {+ assertRow("- - - - -", 3, "INSUFFICIENT_DATA", RETAIN, "ALARM", "OK");+ }++ @Test+ void threeRecentBreachesAlarmUnderEverySetting() {+ assertRow("0 X X - X", 3, "ALARM", "ALARM", "ALARM", "ALARM");+ }++ @Test+ void aBreachAgedPastDatapointsToAlarmAlarmsEvenWithTooFewRealDatapoints() {+ assertRow("- - X - -", 3, "ALARM", RETAIN, "ALARM", "OK");+ }++ @Test+ void twoOfThreeBreachesAlarmWhenEnoughRealDatapointsExist() {+ assertRow("0 - X - X", 2, "ALARM", "ALARM", "ALARM", "ALARM");+ }++ @Test+ void aFullRangeUsesOnlyTheMostRecentEvaluationPeriods() {+ assertRow("0 0 X 0 X", 2, "ALARM", "ALARM", "ALARM", "ALARM");+ }++ @Test+ void aNonBreachingReadingAnywhereInRangeBlocksThePrematureShortcut() {+ assertRow("0 - X - -", 2, "OK", "OK", "ALARM", "OK");+ }++ @Test+ void trailingGapsOnlyAlarmWhenMissingDataIsTreatedAsBreaching() {+ assertRow("- - - - 0", 2, "OK", "OK", "ALARM", "OK");+ }++ @Test+ void theShortcutAppliesToMOutOfNAlarmsToo() {+ assertRow("- - - X -", 2, "ALARM", RETAIN, "ALARM", "OK");+ }++ /**+ * @param expected states for {@code missing}, {@code ignore}, {@code breaching} and+ * {@code notBreaching}, in the column order of the AWS tables. {@link #RETAIN}+ * means the alarm must keep the state it already had.+ */+ private void assertRow(String pattern, int datapointsToAlarm, String... expected) {+ String[] treatments = {"missing", "ignore", "breaching", "notBreaching"};+ for (int i = 0; i < treatments.length; i++) {+ assertState(pattern, datapointsToAlarm, treatments[i], expected[i]);+ }+ }++ private void assertState(String pattern, int datapointsToAlarm, String treatment, String expected) {+ CloudWatchMetricsService service = mock(CloudWatchMetricsService.class);+ @SuppressWarnings("unchecked")+ Instance<AlarmActionHandler> noHandlers = mock(Instance.class);+ AlarmEvaluator subject = new AlarmEvaluator(service, noHandlers);++ when(service.getMetricStatistics(any(), any(), any(), any(), any(), anyInt(), any(), any(), anyString()))+ .thenAnswer(invocation -> datapointsFor(pattern, invocation.getArgument(3)));++ MetricAlarm alarm = alarm(datapointsToAlarm, treatment);+ String before = alarm.getStateValue();+ subject.evaluate(alarm);++ String context = pattern + " / DatapointsToAlarm=" + datapointsToAlarm + " / " + treatment;+ if (RETAIN.equals(expected)) {+ verify(service, never().description(context + " must retain " + before))+ .setAlarmState(anyString(), anyString(), anyString(), any(), anyString());+ } else if (expected.equals(before)) {+ verify(service, never().description(context + " must stay " + expected))+ .setAlarmState(anyString(), anyString(), anyString(), any(), anyString());+ } else {+ verify(service, times(1).description(context + " must become " + expected))+ .setAlarmState(anyString(), eq(expected), anyString(), any(), anyString());+ }+ }++ /**+ * Builds the pattern's datapoints against the window the evaluator actually asked for, so the+ * mapping from slot to period bucket cannot drift if the clock crosses a period boundary+ * between building the alarm and evaluating it.+ */+ private static List<CloudWatchMetricsService.Datapoint> datapointsFor(String pattern, Instant start) {+ List<CloudWatchMetricsService.Datapoint> points = new ArrayList<>();+ String[] slots = pattern.trim().split("\\s+");+ for (int i = 0; i < slots.length; i++) {+ if ("-".equals(slots[i])) {+ continue;+ }+ double value = "X".equals(slots[i]) ? 80.0 : 20.0;+ Instant timestamp = start.plusSeconds(60L * i);+ points.add(new CloudWatchMetricsService.Datapoint(+ timestamp, 1, value, value, value, value, "Percent"));+ }+ return points;+ }++ private static MetricAlarm alarm(int datapointsToAlarm, String treatMissingData) {+ MetricAlarm alarm = new MetricAlarm();+ alarm.setAlarmName("doc-table");+ alarm.setRegion(REGION);+ alarm.setNamespace("AWS/ECS");+ alarm.setMetricName("CPUUtilization");+ alarm.setStatistic("Average");+ alarm.setPeriod(60);+ alarm.setEvaluationPeriods(3);+ alarm.setDatapointsToAlarm(datapointsToAlarm);+ alarm.setThreshold(THRESHOLD);+ alarm.setComparisonOperator("GreaterThanThreshold");+ alarm.setTreatMissingData(treatMissingData);+ alarm.setActionsEnabled(false);+ alarm.setStateValue("OK");+ return alarm;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorTest.javaindex 988de3652e..64426d3a6a 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorTest.java@@ -93,9 +93,14 @@ class AlarmEvaluatorTest {verify(handler).handle(eq(POLICY_ARN), eq(alarm), eq(90.0), eq(REGION));}+ /**+ * AWS reaches {@code INSUFFICIENT_DATA} only when every period in the evaluation range is+ * empty; a range holding fewer real datapoints than {@code EvaluationPeriods} is still+ * evaluated on the real data it does have.+ */@Test- void insufficientDataWhenFewerDatapointsThanEvaluationPeriods() {- stubMetrics(datapoints(80));+ void insufficientDataOnlyWhenTheWholeEvaluationRangeIsEmpty() {+ stubMetrics(List.of());MetricAlarm alarm = alarm();alarm.setStateValue("OK");