Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(autoscaling): create and delete the alarms of target tracking policies (#1926)

iot-rocket77af3a93cdc2
diff --git a/ministack/services/autoscaling.py b/ministack/services/autoscaling.py
index 9715f13c..85781a63 100644
--- a/ministack/services/autoscaling.py
+++ b/ministack/services/autoscaling.py
@@ -21,6 +21,8 @@ Supports:
import copy
import logging
import os
+import time
+from decimal import Decimal
from xml.sax.saxutils import escape
from ministack.core.responses import (
@@ -365,6 +367,7 @@ def _delete_asg(p):
keys_to_del = [k for k in _hooks if k.startswith(f"{name}/")]
for k in keys_to_del:
del _hooks[k]
+ _delete_group_policies(name)
return _xml(200, "DeleteAutoScalingGroupResponse", "<DeleteAutoScalingGroupResult/>")
@@ -619,9 +622,85 @@ def _put_scaling_policy(p):
record = _policy_record(asg_name, policy_name, arn, fields)
except ValueError as exc:
return _error("ValidationError", f"Invalid numeric value in the scaling policy: {escape(str(exc))}")
+ _delete_policy_alarms(_policies.get(key))
+ _put_tracking_alarms(record)
_policies[key] = record
return _xml(200, "PutScalingPolicyResponse",
- f"<PutScalingPolicyResult><PolicyARN>{arn}</PolicyARN></PutScalingPolicyResult>")
+ f"<PutScalingPolicyResult><PolicyARN>{arn}</PolicyARN>"
+ f"<Alarms>{_member_xml(record.get('Alarms', []))}</Alarms></PutScalingPolicyResult>")
+
+
+_TRACKING_METRICS = {
+ "ASGAverageCPUUtilization": ("CPUUtilization", None),
+ "ASGAverageNetworkIn": ("NetworkIn", "Bytes"),
+ "ASGAverageNetworkOut": ("NetworkOut", "Bytes"),
+}
+
+
+def _tracking_metric(tracking, asg_name):
+ """The alarm metric of a target tracking configuration, or None for types without modelled alarms."""
+ predefined = tracking.get("PredefinedMetricSpecification") or {}
+ if predefined.get("PredefinedMetricType") in _TRACKING_METRICS:
+ name, unit = _TRACKING_METRICS[predefined["PredefinedMetricType"]]
+ return {"Namespace": "AWS/EC2", "MetricName": name, "Statistic": "Average", "Unit": unit,
+ "Dimensions": [{"Name": "AutoScalingGroupName", "Value": asg_name}]}
+ custom = tracking.get("CustomizedMetricSpecification") or {}
+ if custom.get("MetricName"):
+ return {"Namespace": custom.get("Namespace"), "MetricName": custom["MetricName"],
+ "Statistic": custom.get("Statistic"), "Unit": custom.get("Unit"),
+ "Dimensions": custom.get("Dimensions", [])}
+ return None
+
+
+def _put_tracking_alarms(record):
+ """Create the AlarmHigh and, unless scale-in is disabled, AlarmLow alarms of a target tracking policy."""
+ tracking = record.get("TargetTrackingConfiguration")
+ metric = tracking and _tracking_metric(tracking, record["AutoScalingGroupName"])
+ if not metric:
+ return
+ from ministack.services import cloudwatch as _cw
+ target = Decimal(str(tracking.get("TargetValue", 0)))
+ levels = [("High", 3, target, "GreaterThanThreshold")]
+ if not tracking.get("DisableScaleIn"):
+ levels.append(("Low", 15, target * Decimal("0.9"), "LessThanThreshold"))
+ record["Alarms"] = []
+ for level, periods, threshold, operator in levels:
+ name = f"TargetTracking-{record['AutoScalingGroupName']}-Alarm{level}-{new_uuid()}"
+ now = int(time.time())
+ alarm = {k: v for k, v in {
+ "AlarmName": name,
+ "AlarmArn": f"arn:aws:cloudwatch:{get_region()}:{get_account_id()}:alarm:{name}",
+ "AlarmDescription": f"DO NOT EDIT OR DELETE. For TargetTrackingScaling policy {record['PolicyARN']}.",
+ "ActionsEnabled": True,
+ "OKActions": [],
+ "AlarmActions": [record["PolicyARN"]],
+ "InsufficientDataActions": [],
+ "StateValue": "INSUFFICIENT_DATA",
+ "StateReason": "Unchecked: Initial alarm creation",
+ "StateUpdatedTimestamp": now,
+ "AlarmConfigurationUpdatedTimestamp": now,
+ **metric,
+ "Period": 60,
+ "EvaluationPeriods": periods,
+ "Threshold": float(threshold),
+ "ComparisonOperator": operator,
+ }.items() if v is not None}
+ _cw.cloudformation_put_metric_alarm(alarm)
+ record["Alarms"].append({"AlarmName": name, "AlarmARN": alarm["AlarmArn"]})
+
+
+def _delete_policy_alarms(record):
+ """Delete the alarms a policy created."""
+ from ministack.services import cloudwatch as _cw
+ for alarm in (record or {}).get("Alarms", []):
+ _cw.cloudformation_delete_metric_alarm(alarm["AlarmName"])
+
+
+def _delete_group_policies(asg_name):
+ """Delete the policies of a group and their alarms."""
+ for k, policy in list(_policies.items()):
+ if policy.get("AutoScalingGroupName") == asg_name:
+ _delete_policy_alarms(_policies.pop(k, None))
def _member_xml(value):
@@ -656,7 +735,7 @@ def _delete_policy(p):
policy_name = _p(p, "PolicyName")
asg_name = _p(p, "AutoScalingGroupName")
key = f"{asg_name}/{policy_name}"
- _policies.pop(key, None)
+ _delete_policy_alarms(_policies.pop(key, None))
return _xml(200, "DeletePolicyResponse", "<DeletePolicyResult/>")
diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.py
index 7c639af8..7ccefac1 100644
--- a/ministack/services/cloudformation/provisioners.py
+++ b/ministack/services/cloudformation/provisioners.py
@@ -10356,6 +10356,7 @@ def _asg_update(physical_id, old_props, new_props, stack_name, logical_id=None):
def _asg_delete(physical_id, props):
_asg._asgs.pop(physical_id, None)
_asg._tags.pop(physical_id, None)
+ _asg._delete_group_policies(physical_id)
def _asg_lc_create(logical_id, props, stack_name):
@@ -10383,7 +10384,10 @@ def _asg_policy_create(logical_id, props, stack_name):
policy_name = props.get("PolicyName") or _physical_name(stack_name, logical_id, max_len=255)
arn = f"arn:aws:autoscaling:{get_region()}:{get_account_id()}:scalingPolicy:{new_uuid()}:autoScalingGroupName/{asg_name}:policyName/{policy_name}"
key = f"{asg_name}/{policy_name}"
- _asg._policies[key] = _asg._policy_record(asg_name, policy_name, arn, props)
+ record = _asg._policy_record(asg_name, policy_name, arn, props)
+ _asg._delete_policy_alarms(_asg._policies.get(key))
+ _asg._put_tracking_alarms(record)
+ _asg._policies[key] = record
return arn, {"Arn": arn, "PolicyName": policy_name}
@@ -10400,6 +10404,8 @@ def _asg_policy_update(physical_id, old_props, new_props, stack_name, logical_id
if replaced is not None:
return replaced
record = _asg._policy_record(asg_name, current["PolicyName"], physical_id, new_props)
+ _asg._delete_policy_alarms(current)
+ _asg._put_tracking_alarms(record)
current.clear()
current.update(record)
return physical_id, {"Arn": physical_id, "PolicyName": current["PolicyName"]}
@@ -10409,7 +10415,7 @@ def _asg_policy_delete(physical_id, props):
# physical_id is the ARN, find matching key
for k, v in list(_asg._policies.items()):
if v.get("PolicyARN") == physical_id:
- _asg._policies.pop(k, None)
+ _asg._delete_policy_alarms(_asg._policies.pop(k, None))
break
diff --git a/tests/test_autoscaling.py b/tests/test_autoscaling.py
index f4362607..9d9046b6 100644
--- a/tests/test_autoscaling.py
+++ b/tests/test_autoscaling.py
@@ -512,6 +512,125 @@ def test_put_scaling_policy_predictive_read_back(autoscaling):
autoscaling.delete_auto_scaling_group(AutoScalingGroupName=asg)
+_CPU_TRACKING = {"PredefinedMetricSpecification": {"PredefinedMetricType": "ASGAverageCPUUtilization"},
+ "TargetValue": 50.0}
+
+
+def _tracking_group(autoscaling):
+ asg = _uid("asg-ttalarm")
+ autoscaling.create_auto_scaling_group(
+ AutoScalingGroupName=asg, MinSize=0, MaxSize=2,
+ AvailabilityZones=["us-east-1a"], LaunchConfigurationName="dummy-lc",
+ )
+ return asg
+
+
+def _put_tracking(autoscaling, asg, name, tracking):
+ return autoscaling.put_scaling_policy(
+ AutoScalingGroupName=asg, PolicyName=name, PolicyType="TargetTrackingScaling",
+ TargetTrackingConfiguration=tracking)
+
+
+def test_target_tracking_policy_creates_alarms(autoscaling, cw):
+ """A target tracking policy creates an AlarmHigh and an AlarmLow alarm and lists them."""
+ asg = _tracking_group(autoscaling)
+ try:
+ put = _put_tracking(autoscaling, asg, "cpu", _CPU_TRACKING)
+ arn = put["PolicyARN"]
+ names = [a["AlarmName"] for a in put["Alarms"]]
+ assert [n.rsplit("-", 5)[0] for n in names] == [
+ f"TargetTracking-{asg}-AlarmHigh", f"TargetTracking-{asg}-AlarmLow"]
+ assert autoscaling.describe_policies(
+ AutoScalingGroupName=asg)["ScalingPolicies"][0]["Alarms"] == put["Alarms"]
+ alarms = {a["AlarmName"]: a for a in cw.describe_alarms(AlarmNames=names)["MetricAlarms"]}
+ dims = [{"Name": "AutoScalingGroupName", "Value": asg}]
+ assert [(a["AlarmArn"], a["Namespace"], a["MetricName"], a["Statistic"], a["Dimensions"],
+ a["Period"], a["EvaluationPeriods"], a["Threshold"], a["ComparisonOperator"],
+ a["AlarmActions"]) for a in (alarms[n] for n in names)] == [
+ (put["Alarms"][0]["AlarmARN"], "AWS/EC2", "CPUUtilization", "Average", dims,
+ 60, 3, 50.0, "GreaterThanThreshold", [arn]),
+ (put["Alarms"][1]["AlarmARN"], "AWS/EC2", "CPUUtilization", "Average", dims,
+ 60, 15, 45.0, "LessThanThreshold", [arn]),
+ ]
+ for alarm in alarms.values():
+ assert alarm["AlarmDescription"] == (
+ f"DO NOT EDIT OR DELETE. For TargetTrackingScaling policy {arn}.")
+ assert not {"Unit", "DatapointsToAlarm", "TreatMissingData"} & set(alarm)
+ finally:
+ autoscaling.delete_auto_scaling_group(AutoScalingGroupName=asg)
+
+
[email protected]("tracking,metric,thresholds", [
+ ({"PredefinedMetricSpecification": {"PredefinedMetricType": "ASGAverageNetworkIn"},
+ "TargetValue": 1000000.0},
+ ("AWS/EC2", "NetworkIn", "Average", "Bytes"), [1000000.0, 900000.0]),
+ ({"PredefinedMetricSpecification": {"PredefinedMetricType": "ASGAverageNetworkOut"},
+ "TargetValue": 1000000.0},
+ ("AWS/EC2", "NetworkOut", "Average", "Bytes"), [1000000.0, 900000.0]),
+ ({"CustomizedMetricSpecification": {"MetricName": "Depth", "Namespace": "Test/App",
+ "Dimensions": [{"Name": "Q", "Value": "q1"}],
+ "Statistic": "Sum", "Unit": "Count"},
+ "TargetValue": 0.1},
+ ("Test/App", "Depth", "Sum", "Count"), [0.1, 0.09]),
+ ({**_CPU_TRACKING, "DisableScaleIn": True},
+ ("AWS/EC2", "CPUUtilization", "Average", None), [50.0]),
+], ids=["network-in", "network-out", "customized", "disable-scale-in"])
+def test_target_tracking_alarm_metric_and_thresholds(autoscaling, cw, tracking, metric, thresholds):
+ """The alarms follow the tracked metric and target; DisableScaleIn leaves only AlarmHigh."""
+ asg = _tracking_group(autoscaling)
+ try:
+ names = [a["AlarmName"] for a in _put_tracking(autoscaling, asg, "p", tracking)["Alarms"]]
+ alarms = {a["AlarmName"]: a for a in cw.describe_alarms(AlarmNames=names)["MetricAlarms"]}
+ assert [(a["Namespace"], a["MetricName"], a["Statistic"], a.get("Unit"))
+ for a in (alarms[n] for n in names)] == [metric] * len(thresholds)
+ assert [alarms[n]["Threshold"] for n in names] == thresholds
+ finally:
+ autoscaling.delete_auto_scaling_group(AutoScalingGroupName=asg)
+
+
+def test_target_tracking_alarms_follow_policy_lifecycle(autoscaling, cw):
+ """Updating a policy replaces its alarms; deleting the policy or the group deletes them."""
+ asg = _tracking_group(autoscaling)
+
+ def existing(put):
+ names = [a["AlarmName"] for a in put["Alarms"]]
+ return [a["AlarmName"] for a in cw.describe_alarms(AlarmNames=names)["MetricAlarms"]]
+
+ try:
+ first = _put_tracking(autoscaling, asg, "cpu", _CPU_TRACKING)
+ second = _put_tracking(autoscaling, asg, "cpu", {**_CPU_TRACKING, "TargetValue": 60.0})
+ assert existing(first) == []
+ assert len(existing(second)) == 2
+ autoscaling.delete_policy(AutoScalingGroupName=asg, PolicyName="cpu")
+ assert existing(second) == []
+ kept = [_put_tracking(autoscaling, asg, name, _CPU_TRACKING) for name in ("a", "b")]
+ finally:
+ autoscaling.delete_auto_scaling_group(AutoScalingGroupName=asg)
+ assert [existing(put) for put in kept] == [[], []]
+ assert cw.describe_alarms(AlarmNamePrefix=f"TargetTracking-{asg}-")["MetricAlarms"] == []
+
+
+def test_group_delete_keeps_the_alarms_of_a_group_with_a_longer_name(autoscaling, cw):
+ """Deleting a group leaves the policies and alarms of a group whose name starts with it."""
+ asg = _tracking_group(autoscaling)
+ other = f"{asg}/blue"
+ autoscaling.create_auto_scaling_group(
+ AutoScalingGroupName=other, MinSize=0, MaxSize=2,
+ AvailabilityZones=["us-east-1a"], LaunchConfigurationName="dummy-lc",
+ )
+ try:
+ _put_tracking(autoscaling, asg, "cpu", _CPU_TRACKING)
+ kept = _put_tracking(autoscaling, other, "cpu", _CPU_TRACKING)
+ autoscaling.delete_auto_scaling_group(AutoScalingGroupName=asg)
+ names = [a["AlarmName"] for a in kept["Alarms"]]
+ assert len(cw.describe_alarms(AlarmNames=names)["MetricAlarms"]) == 2
+ assert [p["PolicyName"] for p in autoscaling.describe_policies(
+ AutoScalingGroupName=other)["ScalingPolicies"]] == ["cpu"]
+ finally:
+ autoscaling.delete_auto_scaling_group(AutoScalingGroupName=other)
+ assert cw.describe_alarms(AlarmNamePrefix=f"TargetTracking-{other}-")["MetricAlarms"] == []
+
+
@pytest.mark.parametrize("members", [
{"PolicyType": "TargetTrackingScaling",
"TargetTrackingConfiguration.PredefinedMetricSpecification.PredefinedMetricType":
diff --git a/tests/test_cfn.py b/tests/test_cfn.py
index 01e36ec2..b929c745 100644
--- a/tests/test_cfn.py
+++ b/tests/test_cfn.py
@@ -26927,6 +26927,61 @@ def test_cfn_asg_scaling_policy_keeps_a_template_policy_name(cfn, autoscaling):
_delete_cfn_test_stack(cfn, stack_name)
+def test_cfn_asg_delete_removes_target_tracking_alarms(cfn, autoscaling, cw):
+ """Deleting a template group deletes the policies put on it and their alarms."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ stack_name = f"cfn-asg-ttdel-{suffix}"
+ group = f"cfn-asg-ttdel-{suffix}"
+ cfn.create_stack(StackName=stack_name, TemplateBody=_cfn_asg_policy_template(group, {}))
+ try:
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")
+ put = autoscaling.put_scaling_policy(
+ AutoScalingGroupName=group, PolicyName="cpu", PolicyType="TargetTrackingScaling",
+ TargetTrackingConfiguration={"PredefinedMetricSpecification": {
+ "PredefinedMetricType": "ASGAverageCPUUtilization"}, "TargetValue": 50.0})
+ assert len(put["Alarms"]) == 2
+ finally:
+ _delete_cfn_test_stack(cfn, stack_name)
+ assert cw.describe_alarms(AlarmNamePrefix=f"TargetTracking-{group}-")["MetricAlarms"] == []
+ assert autoscaling.describe_policies(AutoScalingGroupName=group)["ScalingPolicies"] == []
+
+
+def test_cfn_asg_target_tracking_policy_creates_its_alarms(cfn, autoscaling, cw):
+ """A template target tracking policy creates its alarms, replaces them on update and deletes them with the stack."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ stack_name = f"cfn-asg-ttal-{suffix}"
+ group = f"cfn-asg-ttal-{suffix}"
+
+ def template(target):
+ return _cfn_asg_policy_template(group, {"Tt": {
+ "PolicyType": "TargetTrackingScaling",
+ "TargetTrackingConfiguration": {
+ "PredefinedMetricSpecification": {
+ "PredefinedMetricType": "ASGAverageCPUUtilization"},
+ "TargetValue": target}}})
+
+ def thresholds():
+ alarms = cw.describe_alarms(AlarmNamePrefix=f"TargetTracking-{group}-")["MetricAlarms"]
+ return sorted(a["Threshold"] for a in alarms)
+
+ cfn.create_stack(StackName=stack_name, TemplateBody=template(50))
+ try:
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")
+ arn = _cfn_output(cfn, stack_name, "Tt")
+ assert len(_cfn_asg_policies_by_arn(autoscaling, group)[arn]["Alarms"]) == 2
+ assert thresholds() == [45.0, 50.0]
+
+ cfn.update_stack(StackName=stack_name, TemplateBody=template(60))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")
+ assert thresholds() == [54.0, 60.0]
+ finally:
+ _delete_cfn_test_stack(cfn, stack_name)
+ assert thresholds() == []
+
+
def test_cfn_asg_scaling_policy_name_change_is_not_applied(cfn, autoscaling):
"""A PolicyName change or removal keeps the policy's name and ARN."""
suffix = _uuid_mod.uuid4().hex[:8]