Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(cloudwatch): evaluate composite alarm rules (#2260)

iot-rocket5e28822e164a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c3df1190..6e77b749 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
### Fixed
+- **CloudWatch — composite alarms follow their rule** — a composite alarm stayed in `INSUFFICIENT_DATA` whatever its children did and `DeleteAlarms` removed alarms that composite alarms depend on; a composite alarm now evaluates its `AlarmRule` (`ALARM`, `OK`, `INSUFFICIENT_DATA`, `TRUE`, `FALSE`, `AND`, `OR`, `NOT`, `AT_LEAST`) when it is created or updated and whenever a child changes state, with the `StateReason`, `StateReasonData` and history items AWS writes, `PutCompositeAlarm` refuses an unknown child or a malformed rule with `ValidationError`, `DeleteAlarms` refuses an alarm a composite alarm depends on and more than one composite alarm per call, and `DescribeAlarmHistory` returns composite alarm items only for `AlarmTypes=["CompositeAlarm"]`, as AWS does. Contributed by @iot-rocket.
- **IoT — unacknowledged QoS 1 messages survive a persistent session reconnect** — a QoS 1 message the subscriber had not acknowledged was dropped when its persistent session disconnected; the session now keeps it with its packet identifier, and the resumed connection sends it again once its retransmit interval has passed (at once when it passed while the client was offline, ahead of the messages queued meanwhile), continuing the session's packet identifiers. Contributed by @iot-rocket.
- **IoT — job documents nested too deep** — `CreateJob` and `CreateJobTemplate` accepted an inline document that is not JSON, and a document nested deeper than Python's JSON parser goes answered 500 (from `documentSource`, or inline once it headed a thing's queue, after which every later `CreateJob` for that thing answered 500 too); they now refuse a document that is not JSON or nests more than 1000 levels deep with AWS's `InvalidRequestException`, and a device gets a 1000-level document compact. Contributed by @iot-rocket.
- **IoT — job execution versions, `statusDetails`, execution numbers and timeouts** — `UpdateJobExecution` answered a stale `expectedVersion` with 409 `InvalidStateTransitionException` and now answers 400 `InvalidRequestException` (`VersionMismatch` over MQTT, as before); an execution has no `statusDetails` until a device reports some, and a reported empty map replaces the previous one; the MQTT `get` and `update` topics answer `ResourceNotFound` for an `executionNumber` other than the execution's; and `DescribeJobExecution`, `ListJobExecutionsForJob`, `ListJobExecutionsForThing` and `DescribeJob` time out an expired `IN_PROGRESS` execution as the device reads do. Contributed by @iot-rocket.
diff --git a/ministack/services/cloudwatch.py b/ministack/services/cloudwatch.py
index 25cd4581..a0100f56 100644
--- a/ministack/services/cloudwatch.py
+++ b/ministack/services/cloudwatch.py
@@ -14,6 +14,7 @@ Operations: PutMetricData, GetMetricStatistics, GetMetricData, ListMetrics,
import base64
import copy
+import functools
import json
import logging
import os
@@ -41,6 +42,8 @@ _metrics = AccountRegionScopedDict()
# dashboards/history have no ARN so legacy data lands in the default region.
_alarms = AccountRegionScopedDict()
_composite_alarms = AccountRegionScopedDict()
+# Composite alarm name -> names of its children that changed state since its last transition, newest first.
+_composite_triggers = AccountRegionScopedDict()
# Alarm state-change history, per-account+region. Stored under a single key
# "entries" so the standard list manipulation still applies to the caller's scope.
_alarm_history = AccountRegionScopedDict()
@@ -112,6 +115,7 @@ def get_state():
"metrics": copy.deepcopy(_metrics),
"alarms": copy.deepcopy(_alarms),
"composite_alarms": copy.deepcopy(_composite_alarms),
+ "composite_triggers": copy.deepcopy(_composite_triggers),
"alarm_history": copy.deepcopy(_alarm_history),
"dashboards": copy.deepcopy(_dashboards),
"resource_tags": copy.deepcopy(_resource_tags),
@@ -127,6 +131,7 @@ def _restore_state(data):
_metrics.update(data.get("metrics", {}))
_alarms.update(data.get("alarms", {}))
_composite_alarms.update(data.get("composite_alarms", {}))
+ _composite_triggers.update(data.get("composite_triggers", {}))
_alarm_history.update(data.get("alarm_history", {}))
_dashboards.update(data.get("dashboards", {}))
_resource_tags.update(data.get("resource_tags", {}))
@@ -336,6 +341,7 @@ def _evaluate_alarm(alarm):
alarm["StateUpdatedTimestamp"] = int(time.time())
_record_history(alarm["AlarmName"], old_state, new_state, reason)
_dispatch_alarm_actions(alarm, old_state, new_state, reason)
+ _evaluate_parent_alarms(alarm)
def _evaluate_all_alarms():
@@ -346,11 +352,14 @@ def _evaluate_all_alarms():
logger.debug("alarm eval error", exc_info=True)
-def _record_history(alarm_name, old_state, new_state, reason):
+def _record_history(alarm_name, old_state, new_state, reason, alarm_type="MetricAlarm", reason_data=None):
+ new = {"stateValue": new_state, "stateReason": reason}
+ if reason_data is not None:
+ new["stateReasonData"] = reason_data
_history_entries().append(
{
"AlarmName": alarm_name,
- "AlarmType": "MetricAlarm",
+ "AlarmType": alarm_type,
"Timestamp": _ts_iso(time.time()),
"HistoryItemType": "StateUpdate",
"HistorySummary": f"Alarm updated from {old_state} to {new_state}",
@@ -358,7 +367,7 @@ def _record_history(alarm_name, old_state, new_state, reason):
{
"version": "1.0",
"oldState": {"stateValue": old_state},
- "newState": {"stateValue": new_state, "stateReason": reason},
+ "newState": new,
}
),
}
@@ -1087,6 +1096,234 @@ def _put_metric_alarm(params, cbor_data, is_cbor, is_json=False):
return _xml(200, "PutMetricAlarmResponse", "")
+# ---------------------------------------------------------------------------
+# Composite alarm rules
+# ---------------------------------------------------------------------------
+
+_RULE_TOKEN = re.compile(r'[(),]|"(?:[^"\\]|\\.)*"|[^\s(),"]+|\S')
+_RULE_STATES = ("ALARM", "OK", "INSUFFICIENT_DATA")
+_RULE_OPERAND = "'(', 'NOT', AT_LEAST, TRUE or FALSE, ALARM, OK, or INSUFFICIENT_DATA"
+
+
[email protected]_cache(maxsize=512)
+def _parse_alarm_rule(rule):
+ """Parse an AlarmRule into a tree of tuples, raising ValueError with the AWS message."""
+ tokens = [(m.group(), m.start()) for m in _RULE_TOKEN.finditer(rule)] + [("<EOF>", len(rule))]
+ at = 0
+
+ def word():
+ return tokens[at][0]
+
+ def fail(expected=None):
+ text, pos = tokens[at]
+ expected = f", must be: {expected}" if expected else ""
+ raise ValueError(f"Error in AlarmRule [Unsupported token '{text}' at char {pos}{expected}]")
+
+ def take(expected):
+ nonlocal at
+ if word() != expected:
+ fail()
+ at += 1
+
+ def ref():
+ nonlocal at
+ text = tokens[at][0]
+ if text in ("(", ")", ",", '"', "<EOF>"):
+ fail()
+ at += 1
+ return re.sub(r"\\(.)", r"\1", text[1:-1]) if text.startswith('"') else text
+
+ def operand():
+ nonlocal at
+ w = word()
+ at += 1
+ if w == "NOT":
+ return ("not", operand())
+ if w == "(":
+ node = disjunction()
+ take(")")
+ return node
+ if w in ("TRUE", "FALSE"):
+ return ("const", w == "TRUE")
+ if w in _RULE_STATES:
+ take("(")
+ name = ref()
+ take(")")
+ return ("state", w, name)
+ if w == "AT_LEAST":
+ take("(")
+ count = re.fullmatch(r"(\d+)(%?)", tokens[at][0])
+ if not count:
+ fail()
+ at += 1
+ take(",")
+ negate = word() == "NOT"
+ at += negate
+ state = word()
+ if state not in _RULE_STATES:
+ fail()
+ at += 1
+ take(",")
+ take("(")
+ names = [ref()]
+ while word() == ",":
+ at += 1
+ names.append(ref())
+ take(")")
+ take(")")
+ return ("at_least", int(count.group(1)), bool(count.group(2)), negate, state, tuple(names))
+ at -= 1
+ fail(_RULE_OPERAND)
+
+ def conjunction():
+ nonlocal at
+ node = operand()
+ while word() == "AND":
+ at += 1
+ node = ("and", node, operand())
+ return node
+
+ def disjunction():
+ nonlocal at
+ node = conjunction()
+ while word() == "OR":
+ at += 1
+ node = ("or", node, conjunction())
+ return node
+
+ tree = disjunction()
+ if at != len(tokens) - 1:
+ fail("<EOF>, 'AND', 'OR'")
+ return tree
+
+
+def _rule_alarm_state(ref):
+ name = _rule_ref_name(ref)
+ alarm = (_alarms.get(name) or _composite_alarms.get(name)) if name else None
+ return alarm["StateValue"] if alarm else None
+
+
+def _rule_true(node):
+ """Truth of a parsed AlarmRule over the current alarm states."""
+ kind = node[0]
+ if kind == "const":
+ return node[1]
+ if kind == "state":
+ return _rule_alarm_state(node[2]) == node[1]
+ if kind == "not":
+ return not _rule_true(node[1])
+ if kind == "and":
+ return _rule_true(node[1]) and _rule_true(node[2])
+ if kind == "or":
+ return _rule_true(node[1]) or _rule_true(node[2])
+ _, count, percent, negate, state, refs = node
+ hits = sum((_rule_alarm_state(r) == state) != negate for r in refs)
+ return hits * 100 >= count * len(refs) if percent else hits >= count
+
+
+def _rule_matches(node, target):
+ """Alarm references whose own condition gives the rule the value target (NOT flips it), in rule order."""
+ kind = node[0]
+ if kind == "state":
+ return [node[2]] if (_rule_alarm_state(node[2]) == node[1]) == target else []
+ if kind == "at_least":
+ _, _, _, negate, state, refs = node
+ return [r for r in refs if ((_rule_alarm_state(r) == state) != negate) == target]
+ if kind == "not":
+ return _rule_matches(node[1], not target)
+ if kind == "const":
+ return []
+ return _rule_matches(node[1], target) + _rule_matches(node[2], target)
+
+
+def _rule_refs(node):
+ """Alarm references of a parsed AlarmRule, in rule order."""
+ if node[0] == "state":
+ return [node[2]]
+ if node[0] == "at_least":
+ return list(node[5])
+ if node[0] == "const":
+ return []
+ return [r for child in node[1:] for r in _rule_refs(child)]
+
+
+def _rule_ref_name(ref):
+ """Alarm name for a rule reference (a name or an alarm ARN of this account and region)."""
+ return _alarm_name_from_local_arn(ref) if ref.startswith("arn:") else ref
+
+
+def _composite_children(alarm):
+ """Names of the alarms a composite alarm's rule references, in rule order."""
+ try:
+ refs = _rule_refs(_parse_alarm_rule(alarm.get("AlarmRule", "")))
+ except ValueError:
+ return []
+ return list(dict.fromkeys(name for name in map(_rule_ref_name, refs) if name))
+
+
+def _composite_rule_state(alarm):
+ return "ALARM" if _rule_true(_parse_alarm_rule(alarm["AlarmRule"])) else "OK"
+
+
+def _matching_children(alarm, state):
+ """Children whose condition in the rule agrees with state, the most recently changed first."""
+ refs = _rule_matches(_parse_alarm_rule(alarm["AlarmRule"]), state == "ALARM")
+ names = {n for n in map(_rule_ref_name, refs) if n and n != alarm["AlarmName"]}
+ # Whole-second timestamps tie often: the alarm created later counts as the more recent one.
+ created = [n for n in list(_alarms.keys()) + list(_composite_alarms.keys()) if n in names]
+ return sorted(reversed(created), reverse=True,
+ key=lambda n: (_alarms.get(n) or _composite_alarms[n]).get("StateUpdatedTimestamp", 0))
+
+
+def _transition_composite(alarm, new_state, reason, triggers, seen=frozenset()):
+ """Move a composite alarm to new_state and re-evaluate the composite alarms above it."""
+ triggering = []
+ for name in triggers:
+ child = _alarms.get(name) or _composite_alarms.get(name)
+ if child:
+ at = datetime.fromtimestamp(child.get("StateUpdatedTimestamp", time.time()), tz=timezone.utc)
+ triggering.append({
+ "arn": child["AlarmArn"],
+ "state": {
+ "value": child["StateValue"],
+ "timestamp": at.strftime("%Y-%m-%dT%H:%M:%S.") + f"{at.microsecond // 1000:03d}+0000",
+ },
+ })
+ reason_data = {"triggeringAlarms": triggering}
+ old_state = alarm["StateValue"]
+ now = int(time.time())
+ alarm["StateValue"] = new_state
+ alarm["StateReason"] = reason
+ alarm["StateReasonData"] = json.dumps(reason_data, separators=(",", ":"))
+ alarm["StateUpdatedTimestamp"] = now
+ alarm["StateTransitionedTimestamp"] = now
+ _composite_triggers.pop(alarm["AlarmName"], None)
+ _record_history(alarm["AlarmName"], old_state, new_state, reason, "CompositeAlarm", reason_data)
+ _dispatch_alarm_actions(alarm, old_state, new_state, reason)
+ _evaluate_parent_alarms(alarm, seen)
+
+
+def _evaluate_parent_alarms(child, seen=frozenset()):
+ """Re-evaluate the composite alarms whose rule references child after its state changed."""
+ name = child["AlarmName"]
+ # A composite already on this path is part of a cycle: stop there.
+ seen = seen | {name}
+ for parent in list(_composite_alarms.values()):
+ parent_name = parent["AlarmName"]
+ if parent_name in seen or name not in _composite_children(parent):
+ continue
+ pending = [name] + [n for n in _composite_triggers.get(parent_name, []) if n != name]
+ _composite_triggers[parent_name] = pending
+ new_state = _composite_rule_state(parent)
+ if new_state != parent["StateValue"]:
+ at = datetime.fromtimestamp(child["StateUpdatedTimestamp"], tz=timezone.utc)
+ reason = (f"{child['AlarmArn']} transitioned to {child['StateValue']} at "
+ f"{at.strftime('%A %d %B, %Y %H:%M:%S')} UTC")
+ # Named: the children that changed since the last transition and agree with the new state.
+ agree = set(_matching_children(parent, new_state))
+ _transition_composite(parent, new_state, reason, [n for n in pending if n in agree], seen)
+
+
def _put_composite_alarm(params, cbor_data, is_cbor, is_json=False):
if is_cbor or is_json:
name = cbor_data.get("AlarmName", "")
@@ -1117,7 +1354,25 @@ def _put_composite_alarm(params, cbor_data, is_cbor, is_json=False):
insuff_actions.append(_p(params, f"InsufficientDataActions.member.{ii}"))
ii += 1
- _composite_alarms[name] = {
+ try:
+ refs = _rule_refs(_parse_alarm_rule(alarm_rule))
+ except ValueError as exc:
+ return _error("ValidationError", str(exc), 400, use_json=is_json, use_cbor=is_cbor)
+ missing = []
+ for ref in dict.fromkeys(refs):
+ child = _rule_ref_name(ref)
+ if not child or (child not in _alarms and child not in _composite_alarms):
+ missing.append(ref if ref.startswith("arn:") else
+ f"arn:aws:cloudwatch:{get_region()}:{get_account_id()}:alarm:{ref}")
+ if missing:
+ return _error(
+ "ValidationError",
+ f"Could not save the composite alarm as alarms [{', '.join(missing)}] in the alarm rule do not exist",
+ 400, use_json=is_json, use_cbor=is_cbor,
+ )
+
+ existing = _composite_alarms.get(name)
+ alarm = {
"AlarmName": name,
"AlarmArn": f"arn:aws:cloudwatch:{get_region()}:{get_account_id()}:alarm:{name}",
"AlarmDescription": desc,
@@ -1131,6 +1386,19 @@ def _put_composite_alarm(params, cbor_data, is_cbor, is_json=False):
"InsufficientDataActions": insuff_actions,
"AlarmConfigurationUpdatedTimestamp": int(time.time()),
}
+ # An update keeps the state; the new rule is evaluated at once and moves it only when the result differs.
+ if existing:
+ for key in ("StateValue", "StateReason", "StateReasonData", "StateUpdatedTimestamp",
+ "StateTransitionedTimestamp"):
+ if key in existing:
+ alarm[key] = existing[key]
+ _composite_alarms[name] = alarm
+ new_state = _composite_rule_state(alarm)
+ if not existing or new_state != alarm["StateValue"]:
+ verb = "was updated and its new alarm rule evaluates to" if existing else \
+ "was created and its alarm rule evaluates to"
+ _transition_composite(alarm, new_state, f"{alarm['AlarmArn']} {verb} {new_state}",
+ _matching_children(alarm, new_state))
if is_cbor:
return _cbor_ok({})
if is_json:
@@ -1327,6 +1595,7 @@ def _describe_alarm_history(params, cbor_data, is_cbor, is_json=False):
history_type = cbor_data.get("HistoryItemType")
start_date = _parse_ts(cbor_data.get("StartDate"))
end_date = _parse_ts(cbor_data.get("EndDate"))
+ alarm_types = cbor_data.get("AlarmTypes") or ["MetricAlarm"]
max_records = cbor_data.get("MaxRecords")
next_token = cbor_data.get("NextToken")
scan_by = cbor_data.get("ScanBy")
@@ -1335,11 +1604,16 @@ def _describe_alarm_history(params, cbor_data, is_cbor, is_json=False):
history_type = _p(params, "HistoryItemType")
start_date = _parse_ts(_p(params, "StartDate"))
end_date = _parse_ts(_p(params, "EndDate"))
+ alarm_types = [
+ _p(params, f"AlarmTypes.member.{i}")
+ for i in range(1, 11)
+ if _p(params, f"AlarmTypes.member.{i}")
+ ] or ["MetricAlarm"]
max_records = _p(params, "MaxRecords")
next_token = _p(params, "NextToken")
scan_by = _p(params, "ScanBy")
- items = list(enumerate(_history_entries()))
+ items = [(i, h) for i, h in enumerate(_history_entries()) if h.get("AlarmType", "MetricAlarm") in alarm_types]
if alarm_name:
items = [(i, h) for i, h in items if h["AlarmName"] == alarm_name]
if history_type:
@@ -1413,9 +1687,28 @@ def _delete_alarms(params, cbor_data, is_cbor, is_json=False):
names.append(_p(params, f"AlarmNames.member.{i}"))
i += 1
+ composites = [n for n in dict.fromkeys(names) if n in _composite_alarms]
+ if len(composites) > 1:
+ return _error(
+ "ValidationError",
+ f"Cannot delete [{len(composites)}] Composite Alarms as only one composite alarm delete is allowed "
+ f"in single request",
+ 400, use_json=is_json, use_cbor=is_cbor,
+ )
+ for n in names:
+ if (n in _alarms or n in _composite_alarms) and any(
+ n in _composite_children(parent) for parent in _composite_alarms.values()
+ ):
+ return _error(
+ "ValidationError",
+ f"Cannot delete {n} as there are composite alarm(s) depending on it.",
+ 400, use_json=is_json, use_cbor=is_cbor,
+ )
+
for n in names:
_alarms.pop(n, None)
_composite_alarms.pop(n, None)
+ _composite_triggers.pop(n, None)
_resource_tags.pop(f"arn:aws:cloudwatch:{get_region()}:{get_account_id()}:alarm:{n}", None)
if is_cbor:
@@ -1490,16 +1783,24 @@ def _set_alarm_state(params, cbor_data, is_cbor, is_json=False):
"ResourceNotFound", f"Alarm {name} not found", 404, use_json=is_json, use_cbor=is_cbor
)
+ is_composite = name not in _alarms
old_state = alarm["StateValue"]
alarm["StateValue"] = new_state
alarm["StateReason"] = reason
if reason_data:
alarm["StateReasonData"] = reason_data
+ elif is_composite:
+ alarm.pop("StateReasonData", None)
alarm["StateUpdatedTimestamp"] = int(time.time())
if old_state != new_state:
- _record_history(name, old_state, new_state, reason)
+ if is_composite:
+ # The rule is evaluated again only when one of its children changes state.
+ alarm["StateTransitionedTimestamp"] = alarm["StateUpdatedTimestamp"]
+ _composite_triggers.pop(name, None)
+ _record_history(name, old_state, new_state, reason, "CompositeAlarm" if is_composite else "MetricAlarm")
_dispatch_alarm_actions(alarm, old_state, new_state, reason)
+ _evaluate_parent_alarms(alarm)
if is_cbor:
return _cbor_ok({})
@@ -2007,6 +2308,7 @@ def cloudformation_delete_dashboard(name: str) -> None:
def reset():
_alarms.clear()
_composite_alarms.clear()
+ _composite_triggers.clear()
_alarm_history.clear()
_metrics.clear()
_resource_tags.clear()
diff --git a/tests/test_cloudwatch.py b/tests/test_cloudwatch.py
index 337a4dda..1e5025bf 100644
--- a/tests/test_cloudwatch.py
+++ b/tests/test_cloudwatch.py
@@ -1,4 +1,5 @@
import json
+import re
import time
import uuid as _uuid_mod
@@ -587,7 +588,245 @@ def test_cloudwatch_composite_alarm(cw):
)
resp = cw.describe_alarms(AlarmNames=[composite], AlarmTypes=["CompositeAlarm"])
assert any(a["AlarmName"] == composite for a in resp.get("CompositeAlarms", []))
- cw.delete_alarms(AlarmNames=[child, composite])
+ cw.delete_alarms(AlarmNames=[composite])
+ cw.delete_alarms(AlarmNames=[child])
+
+
+def _composite(cw, name):
+ return cw.describe_alarms(AlarmNames=[name], AlarmTypes=["CompositeAlarm"])["CompositeAlarms"][0]
+
+
+def _triggering(alarm):
+ return [(t["arn"].rsplit(":", 1)[1], t["state"]["value"])
+ for t in json.loads(alarm["StateReasonData"])["triggeringAlarms"]]
+
+
+def _delete_composites_then_children(cw, composites, children):
+ for name in composites:
+ cw.delete_alarms(AlarmNames=[name])
+ cw.delete_alarms(AlarmNames=children)
+
+
+def test_cloudwatch_composite_alarm_follows_child_states(cw):
+ p = f"cw-comp-{_uuid_mod.uuid4().hex[:8]}-"
+ a, b = f"{p}a", f"{p}b"
+ for name in (a, b):
+ _put_paging_alarm(cw, name)
+ a_arn = cw.describe_alarms(AlarmNames=[a])["MetricAlarms"][0]["AlarmArn"]
+ rules = {
+ "and": f'ALARM("{a}") AND ALARM("{b}")',
+ "or": f"ALARM({a_arn}) OR ALARM({b})",
+ "not": f'NOT ALARM("{a}")',
+ "ok": f'OK("{a}")',
+ "ins": f'INSUFFICIENT_DATA("{b}")',
+ "true": "TRUE",
+ "false": "FALSE",
+ "mix": f'(ALARM("{a}") OR OK("{b}")) AND NOT INSUFFICIENT_DATA("{b}")',
+ "least": f'AT_LEAST(2, ALARM, ("{a}", "{b}"))',
+ "pct": f"AT_LEAST(50%, NOT OK, ({a}, {b}))",
+ "prec": "TRUE OR TRUE AND FALSE",
+ "notprec": "NOT TRUE AND FALSE",
+ }
+ for key, rule in rules.items():
+ cw.put_composite_alarm(AlarmName=f"{p}{key}", AlarmRule=rule)
+ cw.put_composite_alarm(AlarmName=f"{p}nest", AlarmRule=f'ALARM("{p}or")')
+ names = [f"{p}{key}" for key in rules] + [f"{p}nest"]
+
+ def states():
+ resp = cw.describe_alarms(AlarmNames=names, AlarmTypes=["CompositeAlarm"])
+ return {x["AlarmName"][len(p):]: x["StateValue"] for x in resp["CompositeAlarms"]}
+
+ try:
+ assert states() == {"and": "OK", "or": "OK", "not": "ALARM", "ok": "OK", "ins": "ALARM", "true": "ALARM",
+ "false": "OK", "mix": "OK", "least": "OK", "pct": "ALARM", "prec": "ALARM",
+ "notprec": "OK", "nest": "OK"}
+ created = _composite(cw, f"{p}and")
+ assert created["StateReason"] == f"{created['AlarmArn']} was created and its alarm rule evaluates to OK"
+ assert created["StateUpdatedTimestamp"] == created["StateTransitionedTimestamp"]
+ assert sorted(_triggering(created)) == [(a, "INSUFFICIENT_DATA"), (b, "INSUFFICIENT_DATA")]
+ assert _triggering(_composite(cw, f"{p}true")) == []
+
+ cw.set_alarm_state(AlarmName=a, StateValue="ALARM", StateReason="test")
+ assert states() == {"and": "OK", "or": "ALARM", "not": "OK", "ok": "OK", "ins": "ALARM", "true": "ALARM",
+ "false": "OK", "mix": "OK", "least": "OK", "pct": "ALARM", "prec": "ALARM",
+ "notprec": "OK", "nest": "ALARM"}
+ nest = _composite(cw, f"{p}nest")
+ assert re.fullmatch(
+ rf"{re.escape(nest['AlarmArn'][:-4])}or transitioned to ALARM at "
+ r"[A-Z][a-z]+day \d\d [A-Z][a-z]+, \d{4} \d\d:\d\d:\d\d UTC", nest["StateReason"])
+ assert _triggering(nest) == [(f"{p}or", "ALARM")]
+ timestamp = json.loads(nest["StateReasonData"])["triggeringAlarms"][0]["state"]["timestamp"]
+ assert re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}\+0000", timestamp)
+
+ cw.set_alarm_state(AlarmName=b, StateValue="ALARM", StateReason="test")
+ assert states()["and"] == "ALARM" and states()["mix"] == "ALARM" and states()["ins"] == "OK"
+ # Both children changed since the last transition of "and".
+ assert sorted(_triggering(_composite(cw, f"{p}and"))) == [(a, "ALARM"), (b, "ALARM")]
+
+ cw.set_alarm_state(AlarmName=a, StateValue="OK", StateReason="test")
+ assert states() == {"and": "OK", "or": "ALARM", "not": "ALARM", "ok": "ALARM", "ins": "OK", "true": "ALARM",
+ "false": "OK", "mix": "OK", "least": "OK", "pct": "ALARM", "prec": "ALARM",
+ "notprec": "OK", "nest": "ALARM"}
+ # Only the child that changed since the last transition is named.
+ assert _triggering(_composite(cw, f"{p}mix")) == [(a, "OK")]
+
+ cw.set_alarm_state(AlarmName=b, StateValue="OK", StateReason="test")
+ assert states()["or"] == "OK" and states()["nest"] == "OK" and states()["pct"] == "OK"
+ assert sorted(_triggering(_composite(cw, f"{p}or"))) == [(a, "OK"), (b, "OK")]
+ # A composite whose state did not change keeps its reason.
+ assert _composite(cw, f"{p}true")["StateReason"].endswith("was created and its alarm rule evaluates to ALARM")
+ finally:
+ _delete_composites_then_children(cw, [f"{p}nest"] + names[:-1], [a, b])
+
+
+def test_cloudwatch_composite_alarm_update_and_set_state(cw):
+ p = f"cw-comp-upd-{_uuid_mod.uuid4().hex[:8]}-"
+ a, b = f"{p}a", f"{p}b"
+ for name in (a, b):
+ _put_paging_alarm(cw, name)
+ cw.set_alarm_state(AlarmName=a, StateValue="ALARM", StateReason="test")
+ comp = f"{p}comp"
+ cw.put_composite_alarm(AlarmName=comp, AlarmRule=f'ALARM("{a}") AND ALARM("{b}")')
+ try:
+ assert _composite(cw, comp)["StateValue"] == "OK"
+ created = _composite(cw, comp)
+
+ # An update that leaves the result alone keeps state and reason.
+ cw.put_composite_alarm(AlarmName=comp, AlarmRule=f'ALARM("{b}")')
+ same = _composite(cw, comp)
+ assert (same["StateValue"], same["StateReason"]) == ("OK", created["StateReason"])
+
+ # An update that changes the result moves the state at once, naming the children that agree with it.
+ cw.put_composite_alarm(AlarmName=comp, AlarmRule=f'ALARM("{a}") OR ALARM("{b}")')
+ updated = _composite(cw, comp)
+ assert updated["StateValue"] == "ALARM"
+ assert updated["StateReason"] == f"{updated['AlarmArn']} was updated and its new alarm rule evaluates to ALARM"
+ assert _triggering(updated) == [(a, "ALARM")]
+
+ # Of the children that changed since the last transition, only those that agree with the new state are named.
+ cw.set_alarm_state(AlarmName=a, StateValue="OK", StateReason="test")
+ cw.set_alarm_state(AlarmName=b, StateValue="OK", StateReason="test")
+ cw.set_alarm_state(AlarmName=a, StateValue="ALARM", StateReason="test")
+ assert _triggering(_composite(cw, comp)) == [(a, "ALARM")]
+
+ # SetAlarmState holds until a child changes state.
+ cw.set_alarm_state(AlarmName=comp, StateValue="OK", StateReason="manual")
+ manual = _composite(cw, comp)
+ assert (manual["StateValue"], manual["StateReason"]) == ("OK", "manual")
+ assert "StateReasonData" not in manual
+ cw.set_alarm_state(AlarmName=b, StateValue="ALARM", StateReason="test")
+ back = _composite(cw, comp)
+ assert back["StateValue"] == "ALARM"
+ assert back["StateReason"].startswith(f"{back['AlarmArn'][:-4]}b transitioned to ALARM at ")
+ assert _triggering(back) == [(b, "ALARM")]
+ finally:
+ _delete_composites_then_children(cw, [comp], [a, b])
+
+
+def test_cloudwatch_put_composite_alarm_validates_rule(cw):
+ p = f"cw-comp-val-{_uuid_mod.uuid4().hex[:8]}-"
+ a = f"{p}a"
+ _put_paging_alarm(cw, a)
+ try:
+ cases = {
+ f'ALARM("{p}missing")': (
+ "Could not save the composite alarm as alarms "
+ f"[arn:aws:cloudwatch:us-east-1:000000000000:alarm:{p}missing] in the alarm rule do not exist"),
+ f'ALARM("{a}") AND': (
+ f"Error in AlarmRule [Unsupported token '<EOF>' at char {len(a) + 13}, must be: '(', 'NOT', "
+ "AT_LEAST, TRUE or FALSE, ALARM, OK, or INSUFFICIENT_DATA]"),
+ f'ALARM("{a}"': f"Error in AlarmRule [Unsupported token '<EOF>' at char {len(a) + 8}]",
+ f'ALARM("{a}") XOR TRUE': (
+ f"Error in AlarmRule [Unsupported token 'XOR' at char {len(a) + 10}, must be: <EOF>, 'AND', 'OR']"),
+ "false": (
+ "Error in AlarmRule [Unsupported token 'false' at char 0, must be: '(', 'NOT', AT_LEAST, "
+ "TRUE or FALSE, ALARM, OK, or INSUFFICIENT_DATA]"),
+ }
+ for rule, message in cases.items():
+ with pytest.raises(ClientError) as exc:
+ cw.put_composite_alarm(AlarmName=f"{p}bad", AlarmRule=rule)
+ assert exc.value.response["Error"]["Code"] == "ValidationError"
+ assert exc.value.response["Error"]["Message"] == message
+ assert cw.describe_alarms(AlarmNames=[f"{p}bad"], AlarmTypes=["CompositeAlarm"])["CompositeAlarms"] == []
+ finally:
+ cw.delete_alarms(AlarmNames=[a])
+
+
+def test_cloudwatch_delete_alarms_refuses_children_and_several_composites(cw):
+ p = f"cw-comp-del-{_uuid_mod.uuid4().hex[:8]}-"
+ a = f"{p}a"
+ _put_paging_alarm(cw, a)
+ cw.put_composite_alarm(AlarmName=f"{p}c1", AlarmRule=f'ALARM("{a}")')
+ cw.put_composite_alarm(AlarmName=f"{p}c2", AlarmRule=f'ALARM("{p}c1")')
+ cw.put_composite_alarm(AlarmName=f"{p}c3", AlarmRule="TRUE")
+ refused = {
+ (a,): f"Cannot delete {a} as there are composite alarm(s) depending on it.",
+ (a, f"{p}c1"): f"Cannot delete {a} as there are composite alarm(s) depending on it.",
+ (f"{p}c1",): f"Cannot delete {p}c1 as there are composite alarm(s) depending on it.",
+ (f"{p}c2", f"{p}c3"): (
+ "Cannot delete [2] Composite Alarms as only one composite alarm delete is allowed in single request"),
+ }
+ try:
+ for names, message in refused.items():
+ with pytest.raises(ClientError) as exc:
+ cw.delete_alarms(AlarmNames=list(names))
+ assert exc.value.response["Error"]["Code"] == "ValidationError"
+ assert exc.value.response["Error"]["Message"] == message
+ assert len(cw.describe_alarms(AlarmNamePrefix=p, AlarmTypes=["CompositeAlarm"])["CompositeAlarms"]) == 3
+ assert cw.describe_alarms(AlarmNames=[a])["MetricAlarms"]
+ cw.delete_alarms(AlarmNames=[f"{p}missing", f"{p}c3"])
+ finally:
+ _delete_composites_then_children(cw, [f"{p}c2", f"{p}c1", f"{p}c3"], [a])
+ left = cw.describe_alarms(AlarmNamePrefix=p, AlarmTypes=["MetricAlarm", "CompositeAlarm"])
+ assert left["MetricAlarms"] == left["CompositeAlarms"] == []
+
+
+def test_cloudwatch_composite_alarm_history_needs_alarm_types(cw):
+ p = f"cw-comp-hist-{_uuid_mod.uuid4().hex[:8]}-"
+ a, comp = f"{p}a", f"{p}comp"
+ _put_paging_alarm(cw, a)
+ cw.put_composite_alarm(AlarmName=comp, AlarmRule=f'ALARM("{a}")')
+ cw.set_alarm_state(AlarmName=a, StateValue="ALARM", StateReason="test")
+ try:
+ assert cw.describe_alarm_history(AlarmName=comp)["AlarmHistoryItems"] == []
+ items = cw.describe_alarm_history(AlarmName=comp, AlarmTypes=["CompositeAlarm"],
+ ScanBy="TimestampAscending")["AlarmHistoryItems"]
+ assert [(i["AlarmType"], i["HistorySummary"]) for i in items] == [
+ ("CompositeAlarm", "Alarm updated from INSUFFICIENT_DATA to OK"),
+ ("CompositeAlarm", "Alarm updated from OK to ALARM"),
+ ]
+ new_state = json.loads(items[1]["HistoryData"])["newState"]
+ assert new_state["stateReasonData"]["triggeringAlarms"][0]["state"]["value"] == "ALARM"
+ assert [i["AlarmType"] for i in cw.describe_alarm_history(AlarmName=a)["AlarmHistoryItems"]] == [
+ "MetricAlarm", "MetricAlarm"]
+ finally:
+ _delete_composites_then_children(cw, [comp], [a])
+
+
+def test_cloudwatch_composite_alarm_triggers_survive_persistence():
+ from ministack.core.responses import set_request_account_id, set_request_region
+ from ministack.services import cloudwatch as service
+
+ set_request_account_id("111111111111")
+ set_request_region("us-east-1")
+ service.reset()
+ try:
+ for name in ("a", "b"):
+ service._put_metric_alarm({}, {"AlarmName": name, "MetricName": "M", "Namespace": "N", "Period": 60,
+ "EvaluationPeriods": 1, "Threshold": 1.0, "Statistic": "Sum",
+ "ComparisonOperator": "GreaterThanThreshold"}, False, True)
+ service._put_composite_alarm({}, {"AlarmName": "comp", "AlarmRule": "ALARM(a) AND ALARM(b)"}, False, True)
+ service._set_alarm_state({}, {"AlarmName": "b", "StateValue": "ALARM", "StateReason": "t"}, False, True)
+ state = service.get_state()
+ service.reset()
+ service.load_persisted_state(state)
+ service._set_alarm_state({}, {"AlarmName": "a", "StateValue": "ALARM", "StateReason": "t"}, False, True)
+ comp = service._composite_alarms["comp"]
+ assert comp["StateValue"] == "ALARM"
+ assert [t["arn"].rsplit(":", 1)[1] for t in json.loads(comp["StateReasonData"])["triggeringAlarms"]] == [
+ "a", "b"]
+ finally:
+ service.reset()
def test_cloudwatch_describe_alarms_for_metric(cw):
import uuid as _uuid