Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudwatch): evaluate alarms over CloudWatch's wider evaluation range (#2771)

Matheus Oliveiracfe8566511cf
diff --git a/docs/services/applicationautoscaling.md b/docs/services/applicationautoscaling.md
index 2071fe2ead..09f765f97d 100644
--- a/docs/services/applicationautoscaling.md
+++ b/docs/services/applicationautoscaling.md
@@ -138,15 +138,17 @@ arn:aws:iam::<account>:role/aws-service-role/<namespace>.application-autoscaling
validated; the configuration block is not stored.
- Pagination is not implemented — `DescribeScalableTargets` and `DescribeScalingPolicies`
return all matching results and never emit a `NextToken`.
-- **Missing-data evaluation uses a fixed window, not AWS's sliding one.** Real CloudWatch
- queries a wider "evaluation range" than `Period × EvaluationPeriods` and, whenever enough
- *real* datapoints exist within that wider range, evaluates using those and ignores
- `TreatMissingData` entirely — falling back to it only when genuinely too few real
- datapoints exist even with the extra lookback. Real CloudWatch also has "premature
- transition" avoidance logic that can delay a move into `ALARM` when trailing data is
- missing. Floci's evaluator queries exactly `EvaluationPeriods` period-aligned buckets and
- applies `TreatMissingData` as soon as any of them is empty — correct for the common case
- (continuous metric reporting) but less faithful once data has partial, intermittent gaps.
+- **The evaluation range's exact width is an approximation.** Alarm evaluation follows
+ CloudWatch's documented precedence: a wider "evaluation range" than
+ `Period × EvaluationPeriods` is queried, and whenever enough *real* datapoints exist within
+ it the alarm is evaluated on those and `TreatMissingData` is ignored entirely; the setting
+ fills in only what real data cannot cover. The premature-transition rule is implemented too,
+ so a breach that has aged past `DatapointsToAlarm` with only missing periods after it reaches
+ `ALARM`, while a breach at the very end of the window does not. AWS does not publish how wide
+ the range is (only that it varies with period length and metric resolution), so Floci uses
+ `EvaluationPeriods + 2`, which reproduces the single worked example in AWS's documentation.
+ Both published example tables are transcribed as tests in
+ `AlarmEvaluatorMissingDataTablesTest`.
- **Scale-out cooldown rarely blocks a repeated scale-out in practice.** Its "proceed if
larger than the last applied capacity" carve-out (see above) is almost always satisfied
when nothing external has changed capacity, because both capacity formulas grow
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluator.java
index efcc1b63b2..cfe5c3e889 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluator.java
@@ -12,6 +12,7 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.time.Instant;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -90,11 +91,20 @@ public class AlarmEvaluator {
}
/**
- * Evaluates the last {@code evaluationPeriods} <em>complete</em> periods, not counting
- * whichever period is still accumulating right now — matching real CloudWatch, which
- * evaluates elapsed periods only. Counting the in-progress period would make it look
- * "missing" on nearly every tick even under perfectly healthy, continuous reporting,
- * since data for it typically hasn't arrived yet.
+ * Queries the <em>evaluation range</em> — more periods than {@code evaluationPeriods} — and
+ * resolves the state with CloudWatch's documented precedence ("How alarm state is evaluated
+ * when data is missing"): real datapoints reaching further back are preferred, and {@code
+ * TreatMissingData} only fills what real data cannot cover.
+ *
+ * <p>Once the range holds at least {@code evaluationPeriods} real datapoints, the most recent
+ * of those decide the state and {@code TreatMissingData} is not consulted at all — AWS: "the
+ * value you set for how to treat missing data is not needed and is ignored". Reaching further
+ * back is precisely what makes that branch reachable, so widening the query narrows how often
+ * the fallback applies without changing what any of its modes mean.</p>
+ *
+ * <p>Only <em>complete</em> periods are considered; the period still accumulating right now
+ * is excluded, since data for it typically has not arrived yet and counting it would make it
+ * look missing on nearly every tick even under perfectly healthy reporting.</p>
*/
void evaluate(MetricAlarm alarm) {
if (!isEvaluable(alarm)) {
@@ -103,50 +113,71 @@ public class AlarmEvaluator {
String region = alarm.getRegion();
int period = alarm.getPeriod();
int evaluationPeriods = alarm.getEvaluationPeriods();
+ int range = evaluationRange(evaluationPeriods);
long nowBucket = (Instant.now().getEpochSecond() / period) * period;
long lastCompleteBucket = nowBucket - period;
- long oldestBucket = lastCompleteBucket - (long) period * (evaluationPeriods - 1);
+ long oldestBucket = lastCompleteBucket - (long) period * (range - 1);
Instant start = Instant.ofEpochSecond(oldestBucket);
Instant end = Instant.ofEpochSecond(lastCompleteBucket + period - 1);
- List<CloudWatchMetricsService.Datapoint> recent = metricsService.getMetricStatistics(
+ List<CloudWatchMetricsService.Datapoint> retrieved = metricsService.getMetricStatistics(
alarm.getNamespace(), alarm.getMetricName(), alarm.getDimensions(),
start, end, period, List.of(alarm.getStatistic()), alarm.getUnit(), region);
- int missing = evaluationPeriods - recent.size();
- int breaching = 0;
- double latestValue = Double.NaN;
- for (CloudWatchMetricsService.Datapoint dp : recent) {
- double value = CloudWatchMetricsService.resolveStatValue(dp, alarm.getStatistic());
- if (breaches(value, alarm.getComparisonOperator(), alarm.getThreshold())) {
- breaching++;
- latestValue = value;
+ Double[] buckets = new Double[range];
+ for (CloudWatchMetricsService.Datapoint dp : retrieved) {
+ long bucket = (dp.timestamp().getEpochSecond() / period) * period;
+ int index = (int) ((bucket - oldestBucket) / period);
+ if (index >= 0 && index < range) {
+ buckets[index] = CloudWatchMetricsService.resolveStatValue(dp, alarm.getStatistic());
+ }
+ }
+
+ List<Double> real = new ArrayList<>();
+ for (Double value : buckets) {
+ if (value != null) {
+ real.add(value);
}
}
String newState;
String reason;
- if (missing > 0) {
+ double latestValue;
+ if (real.size() >= evaluationPeriods) {
+ List<Double> evaluated = real.subList(real.size() - evaluationPeriods, real.size());
+ int breaching = countBreaches(evaluated, alarm);
+ latestValue = latestBreachingValue(evaluated, alarm);
+ newState = evaluateBreachCount(breaching, alarm, evaluationPeriods);
+ reason = ("ALARM".equals(newState) ? "Threshold Crossed: " : "Threshold Not Crossed: ")
+ + breaching + " datapoint(s) breaching the threshold.";
+ } else {
+ int fill = evaluationPeriods - real.size();
+ int breaching = countBreaches(real, alarm);
+ latestValue = latestBreachingValue(real, alarm);
String treatMissingData = alarm.getTreatMissingData();
if ("ignore".equalsIgnoreCase(treatMissingData)) {
newState = alarm.getStateValue();
reason = alarm.getStateReason();
} else if ("breaching".equalsIgnoreCase(treatMissingData)) {
- newState = evaluateBreachCount(breaching + missing, alarm, evaluationPeriods);
+ newState = evaluateBreachCount(breaching + fill, alarm, evaluationPeriods);
reason = "Threshold Crossed (missing datapoints treated as breaching): "
- + (breaching + missing) + " datapoint(s) breaching the threshold.";
+ + (breaching + fill) + " datapoint(s) breaching the threshold.";
} else if ("notBreaching".equalsIgnoreCase(treatMissingData)) {
newState = evaluateBreachCount(breaching, alarm, evaluationPeriods);
reason = "Threshold evaluated with missing datapoints treated as not breaching: "
+ breaching + " datapoint(s) breaching the threshold.";
- } else {
+ } else if (real.isEmpty()) {
newState = "INSUFFICIENT_DATA";
- reason = "Insufficient Data: " + recent.size() + " of " + evaluationPeriods + " datapoints available";
+ reason = "Insufficient Data: 0 of " + evaluationPeriods + " datapoints available";
+ } else if (settledBreach(buckets, alarm, evaluationPeriods)) {
+ newState = "ALARM";
+ reason = "Threshold Crossed: the oldest breaching datapoint is old enough to alarm "
+ + "and every more recent datapoint is breaching or missing.";
+ } else {
+ newState = evaluateBreachCount(breaching, alarm, evaluationPeriods);
+ reason = ("ALARM".equals(newState) ? "Threshold Crossed: " : "Threshold Not Crossed: ")
+ + breaching + " real datapoint(s) breaching the threshold.";
}
- } else {
- newState = evaluateBreachCount(breaching, alarm, evaluationPeriods);
- reason = ("ALARM".equals(newState) ? "Threshold Crossed: " : "Threshold Not Crossed: ")
- + breaching + " datapoint(s) breaching the threshold.";
}
if (!newState.equals(alarm.getStateValue())) {
@@ -158,6 +189,68 @@ public class AlarmEvaluator {
}
}
+ /**
+ * How many periods to retrieve. AWS states only that it is "a higher number of data points
+ * than the number specified as Evaluation Periods", varying with period length and metric
+ * resolution, without publishing the formula; its one worked example pairs
+ * {@code EvaluationPeriods = 3} with an evaluation range of 5, which is what this reproduces.
+ */
+ private static int evaluationRange(int evaluationPeriods) {
+ return evaluationPeriods + 2;
+ }
+
+ /**
+ * CloudWatch's premature-transition rule, which fires when {@code TreatMissingData} is left at
+ * its {@code missing} default: an alarm goes to {@code ALARM} "when the oldest available
+ * breaching datapoint during the Evaluation Periods number of data points is at least as old
+ * as the value of Datapoints to Alarm" and every more recent datapoint is breaching or
+ * missing. A lone breach at the very end of the window ({@code - - - - X}) deliberately does
+ * not qualify — the next datapoint may be non-breaching — while one that has already aged past
+ * {@code DatapointsToAlarm} ({@code - - X - -}) does, even with fewer real datapoints than M.
+ *
+ * <p>"All other" spans the whole evaluation range, not just the most recent {@code
+ * EvaluationPeriods}: a single non-breaching reading anywhere in the range disqualifies the
+ * shortcut. That is what separates {@code 0 - X - -} (documented {@code OK} at
+ * {@code DatapointsToAlarm = 2}) from {@code - - X - -} (documented {@code ALARM}), since the
+ * two are indistinguishable across their most recent three buckets alone.</p>
+ */
+ private static boolean settledBreach(Double[] buckets, MetricAlarm alarm, int evaluationPeriods) {
+ int datapointsToAlarm = alarm.getDatapointsToAlarm() > 0
+ ? alarm.getDatapointsToAlarm() : evaluationPeriods;
+ int oldestBreachingAge = 0;
+ for (int age = 1; age <= buckets.length; age++) {
+ Double value = buckets[buckets.length - age];
+ if (value == null) {
+ continue;
+ }
+ if (!breaches(value, alarm.getComparisonOperator(), alarm.getThreshold())) {
+ return false;
+ }
+ oldestBreachingAge = age;
+ }
+ return oldestBreachingAge >= datapointsToAlarm;
+ }
+
+ private static int countBreaches(List<Double> values, MetricAlarm alarm) {
+ int breaching = 0;
+ for (Double value : values) {
+ if (breaches(value, alarm.getComparisonOperator(), alarm.getThreshold())) {
+ breaching++;
+ }
+ }
+ return breaching;
+ }
+
+ private static double latestBreachingValue(List<Double> values, MetricAlarm alarm) {
+ double latest = Double.NaN;
+ for (Double value : values) {
+ if (breaches(value, alarm.getComparisonOperator(), alarm.getThreshold())) {
+ latest = value;
+ }
+ }
+ return latest;
+ }
+
private static String evaluateBreachCount(int breaching, MetricAlarm alarm, int evaluationPeriods) {
int datapointsToAlarm = alarm.getDatapointsToAlarm() > 0 ? alarm.getDatapointsToAlarm() : evaluationPeriods;
return breaching >= datapointsToAlarm ? "ALARM" : "OK";
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorMissingDataTablesTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorMissingDataTablesTest.java
new file mode 100644
index 0000000000..54d08100c5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorMissingDataTablesTest.java
@@ -0,0 +1,161 @@
+package io.github.hectorvent.floci.services.cloudwatch.metrics;
+
+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;
+import jakarta.enterprise.inject.Instance;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * The two worked examples from AWS's "Configuring how CloudWatch alarms treat missing data",
+ * transcribed as executable cases. Each test is one row of a published table and asserts the
+ * documented state for all four {@code TreatMissingData} settings at once.
+ *
+ * <p>Patterns read oldest-to-newest, as in the AWS tables: {@code 0} is a non-breaching datapoint,
+ * {@code X} a breaching one and {@code -} a missing period. Both tables use
+ * {@code EvaluationPeriods = 3}, giving an evaluation range of 5.</p>
+ */
+class AlarmEvaluatorMissingDataTablesTest {
+
+ private static final String REGION = "us-east-1";
+ private static final double THRESHOLD = 50.0;
+ private static final String RETAIN = "<retain>";
+
+ @Test
+ void enoughRealDatapointsIgnoresTheMissingDataSetting() {
+ assertRow("0 - X - X", 3, "OK", "OK", "OK", "OK");
+ }
+
+ @Test
+ void oneOldNonBreachingDatapointStaysOkUnderEverySetting() {
+ assertRow("0 - - - -", 3, "OK", "OK", "OK", "OK");
+ }
+
+ @Test
+ void anEntirelyEmptyRangeIsTheOnlyInsufficientDataCase() {
+ assertRow("- - - - -", 3, "INSUFFICIENT_DATA", RETAIN, "ALARM", "OK");
+ }
+
+ @Test
+ void threeRecentBreachesAlarmUnderEverySetting() {
+ assertRow("0 X X - X", 3, "ALARM", "ALARM", "ALARM", "ALARM");
+ }
+
+ @Test
+ void aBreachAgedPastDatapointsToAlarmAlarmsEvenWithTooFewRealDatapoints() {
+ assertRow("- - X - -", 3, "ALARM", RETAIN, "ALARM", "OK");
+ }
+
+ @Test
+ void twoOfThreeBreachesAlarmWhenEnoughRealDatapointsExist() {
+ assertRow("0 - X - X", 2, "ALARM", "ALARM", "ALARM", "ALARM");
+ }
+
+ @Test
+ void aFullRangeUsesOnlyTheMostRecentEvaluationPeriods() {
+ assertRow("0 0 X 0 X", 2, "ALARM", "ALARM", "ALARM", "ALARM");
+ }
+
+ @Test
+ void aNonBreachingReadingAnywhereInRangeBlocksThePrematureShortcut() {
+ assertRow("0 - X - -", 2, "OK", "OK", "ALARM", "OK");
+ }
+
+ @Test
+ void trailingGapsOnlyAlarmWhenMissingDataIsTreatedAsBreaching() {
+ assertRow("- - - - 0", 2, "OK", "OK", "ALARM", "OK");
+ }
+
+ @Test
+ void theShortcutAppliesToMOutOfNAlarmsToo() {
+ assertRow("- - - X -", 2, "ALARM", RETAIN, "ALARM", "OK");
+ }
+
+ /**
+ * @param expected states for {@code missing}, {@code ignore}, {@code breaching} and
+ * {@code notBreaching}, in the column order of the AWS tables. {@link #RETAIN}
+ * means the alarm must keep the state it already had.
+ */
+ private void assertRow(String pattern, int datapointsToAlarm, String... expected) {
+ String[] treatments = {"missing", "ignore", "breaching", "notBreaching"};
+ for (int i = 0; i < treatments.length; i++) {
+ assertState(pattern, datapointsToAlarm, treatments[i], expected[i]);
+ }
+ }
+
+ private void assertState(String pattern, int datapointsToAlarm, String treatment, String expected) {
+ CloudWatchMetricsService service = mock(CloudWatchMetricsService.class);
+ @SuppressWarnings("unchecked")
+ Instance<AlarmActionHandler> noHandlers = mock(Instance.class);
+ AlarmEvaluator subject = new AlarmEvaluator(service, noHandlers);
+
+ when(service.getMetricStatistics(any(), any(), any(), any(), any(), anyInt(), any(), any(), anyString()))
+ .thenAnswer(invocation -> datapointsFor(pattern, invocation.getArgument(3)));
+
+ MetricAlarm alarm = alarm(datapointsToAlarm, treatment);
+ String before = alarm.getStateValue();
+ subject.evaluate(alarm);
+
+ String context = pattern + " / DatapointsToAlarm=" + datapointsToAlarm + " / " + treatment;
+ if (RETAIN.equals(expected)) {
+ verify(service, never().description(context + " must retain " + before))
+ .setAlarmState(anyString(), anyString(), anyString(), any(), anyString());
+ } else if (expected.equals(before)) {
+ verify(service, never().description(context + " must stay " + expected))
+ .setAlarmState(anyString(), anyString(), anyString(), any(), anyString());
+ } else {
+ verify(service, times(1).description(context + " must become " + expected))
+ .setAlarmState(anyString(), eq(expected), anyString(), any(), anyString());
+ }
+ }
+
+ /**
+ * Builds the pattern's datapoints against the window the evaluator actually asked for, so the
+ * mapping from slot to period bucket cannot drift if the clock crosses a period boundary
+ * between building the alarm and evaluating it.
+ */
+ private static List<CloudWatchMetricsService.Datapoint> datapointsFor(String pattern, Instant start) {
+ List<CloudWatchMetricsService.Datapoint> points = new ArrayList<>();
+ String[] slots = pattern.trim().split("\\s+");
+ for (int i = 0; i < slots.length; i++) {
+ if ("-".equals(slots[i])) {
+ continue;
+ }
+ double value = "X".equals(slots[i]) ? 80.0 : 20.0;
+ Instant timestamp = start.plusSeconds(60L * i);
+ points.add(new CloudWatchMetricsService.Datapoint(
+ timestamp, 1, value, value, value, value, "Percent"));
+ }
+ return points;
+ }
+
+ private static MetricAlarm alarm(int datapointsToAlarm, String treatMissingData) {
+ MetricAlarm alarm = new MetricAlarm();
+ alarm.setAlarmName("doc-table");
+ alarm.setRegion(REGION);
+ alarm.setNamespace("AWS/ECS");
+ alarm.setMetricName("CPUUtilization");
+ alarm.setStatistic("Average");
+ alarm.setPeriod(60);
+ alarm.setEvaluationPeriods(3);
+ alarm.setDatapointsToAlarm(datapointsToAlarm);
+ alarm.setThreshold(THRESHOLD);
+ alarm.setComparisonOperator("GreaterThanThreshold");
+ alarm.setTreatMissingData(treatMissingData);
+ alarm.setActionsEnabled(false);
+ alarm.setStateValue("OK");
+ return alarm;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorTest.java
index 988de3652e..64426d3a6a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/metrics/AlarmEvaluatorTest.java
@@ -93,9 +93,14 @@ class AlarmEvaluatorTest {
verify(handler).handle(eq(POLICY_ARN), eq(alarm), eq(90.0), eq(REGION));
}
+ /**
+ * AWS reaches {@code INSUFFICIENT_DATA} only when every period in the evaluation range is
+ * empty; a range holding fewer real datapoints than {@code EvaluationPeriods} is still
+ * evaluated on the real data it does have.
+ */
@Test
- void insufficientDataWhenFewerDatapointsThanEvaluationPeriods() {
- stubMetrics(datapoints(80));
+ void insufficientDataOnlyWhenTheWholeEvaluationRangeIsEmpty() {
+ stubMetrics(List.of());
MetricAlarm alarm = alarm();
alarm.setStateValue("OK");