Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(cloudformation): update EKS clusters and node groups in place (#1937)

iot-rocket095a129ce511
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 32d31bf9..3db5a7e7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
### Added
+- **CloudFormation — EKS cluster and node group updates** — `AWS::EKS::Cluster` applies `Version`, `Logging`, `ResourcesVpcConfig`, `AccessConfig.AuthenticationMode` and `Tags` in place and `AWS::EKS::Nodegroup` applies `ScalingConfig`, `Labels`, `Taints`, `UpdateConfig`, `LaunchTemplate`, `Version`, `ReleaseVersion` and `Tags`, where every such change used to report `UPDATE_COMPLETE` and was dropped.
- **SES — a sandboxed account sends only to verified or simulator recipients** — SES v2 `PutAccountDetails` with `ProductionAccessEnabled=false` puts the account (per region) in the sandbox, as on AWS, and `GetAccount` reports it with the submitted `Details`. A sandboxed send to a recipient that is neither a verified address or domain identity nor a `@simulator.amazonses.com` address fails with `MessageRejected` "Email address is not verified. The following identities failed the check in region …" for v1 `SendEmail`, `SendRawEmail` and `SendTemplatedEmail` and v2 `SendEmail`; bulk sends reject only the affected entries. Accounts stay in production by default. Reported by @skialpine.
### Fixed
diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.py
index 71092acb..76b2e2dd 100644
--- a/ministack/services/cloudformation/provisioners.py
+++ b/ministack/services/cloudformation/provisioners.py
@@ -733,6 +733,12 @@ _CUSTOM_NAME_REPLACEMENT = {
old.get("Scope", "REGIONAL") != new.get("Scope", "REGIONAL")
),
},
+ "AWS::EKS::Cluster": {
+ "name": "Name",
+ "requires_replacement": lambda old, new: (
+ _eks_cluster_create_only(old) != _eks_cluster_create_only(new)
+ ),
+ },
# "Update requires: Replacement" in the template reference, the name
# itself aside.
"AWS::ElastiCache::CacheCluster": {
@@ -3474,21 +3480,61 @@ def _scheduler_group_delete(physical_id, props):
# --- EKS Cluster ---
+_EKS_LOG_TYPES = ("api", "audit", "authenticator", "controllerManager", "scheduler")
+
+
+def _eks_cluster_create_only(props):
+ """The cluster's createOnlyProperties other than Name."""
+ network = props.get("KubernetesNetworkConfig") or {}
+ return (
+ props.get("RoleArn"), props.get("EncryptionConfig"), props.get("OutpostConfig"),
+ props.get("BootstrapSelfManagedAddons"), network.get("IpFamily"),
+ network.get("ServiceIpv4Cidr"),
+ (props.get("AccessConfig") or {}).get("BootstrapClusterCreatorAdminPermissions"),
+ )
+
+
+def _eks_cluster_fields(props):
+ """The cluster members UpdateClusterConfig and UpdateClusterVersion change."""
+ vpc = _pascal_to_camel(props.get("ResourcesVpcConfig") or {})
+ for key in ("endpointPublicAccess", "endpointPrivateAccess"):
+ if key in vpc:
+ vpc[key] = _cfn_bool(vpc[key])
+ logging = ((props.get("Logging") or {}).get("ClusterLogging") or {}).get("EnabledTypes") or []
+ enabled = [t.get("Type") for t in logging]
+ disabled = [t for t in _EKS_LOG_TYPES if t not in enabled]
+ mode = (props.get("AccessConfig") or {}).get("AuthenticationMode")
+ fields = {
+ "resourcesVpcConfig": vpc,
+ "logging": {"clusterLogging": [
+ {"types": types, "enabled": on}
+ for types, on in ((enabled, True), (disabled, False)) if types
+ ]},
+ "accessConfig": {"authenticationMode": mode} if mode else {},
+ }
+ if props.get("Version"):
+ fields["version"] = str(props["Version"])
+ return fields
+
+
def _eks_cluster_create(logical_id, props, stack_name):
import ministack.services.eks as _eks
name = props.get("Name") or _physical_name(stack_name, logical_id, max_len=100)
body = {
"name": name,
- "version": props.get("Version", "1.30"),
"roleArn": props.get("RoleArn", f"arn:aws:iam::{get_account_id()}:role/eks-role"),
- "resourcesVpcConfig": props.get("ResourcesVpcConfig", {}),
- "tags": {t["Key"]: t["Value"] for t in props.get("Tags", [])},
+ "tags": _tag_map(props.get("Tags")),
+ **_eks_cluster_fields(props),
}
- _eks._create_cluster(body)
- arn = _eks._cluster_arn(name)
- cluster = _eks._clusters.get(name, {})
- return name, {
- "Arn": arn,
+ resp = _eks._create_cluster(body)
+ if resp[0] >= 400:
+ raise ValueError(f"AWS::EKS::Cluster create failed: {resp[2]!r}")
+ return name, _eks_cluster_attrs(_eks._clusters[name])
+
+
+def _eks_cluster_attrs(cluster):
+ return {
+ "Arn": cluster["arn"],
"Endpoint": cluster.get("endpoint", ""),
"CertificateAuthorityData": cluster.get("certificateAuthority", {}).get("data", ""),
"ClusterSecurityGroupId": cluster.get("resourcesVpcConfig", {}).get("clusterSecurityGroupId", ""),
@@ -3496,33 +3542,118 @@ def _eks_cluster_create(logical_id, props, stack_name):
}
+def _eks_cluster_update(physical_id, old_props, new_props, stack_name, logical_id=None):
+ """Version, Logging, ResourcesVpcConfig, AuthenticationMode and Tags are in place."""
+ import ministack.services.eks as _eks
+ name = new_props.get("Name") or _physical_name(
+ stack_name, logical_id or physical_id, max_len=100)
+ cluster = _eks._clusters.get(physical_id)
+ replaced = _rename_replacement(
+ physical_id, old_props, new_props, stack_name, logical_id,
+ name, physical_id if cluster else None,
+ _eks_cluster_create, _eks_cluster_delete,
+ )
+ if replaced is not None:
+ return replaced
+ if _requires_replacement("AWS::EKS::Cluster", old_props, new_props):
+ raise ValueError(
+ f"AWS::EKS::Cluster {name} requires replacement, which MiniStack does not "
+ "perform under a generated name; set a Name to create the replacement."
+ )
+ fields = _eks_cluster_fields(new_props)
+ if "Logging" not in new_props:
+ # A rollback to a template without Logging leaves the logging as it is.
+ del fields["logging"]
+ # A template without the endpoint access members or AuthenticationMode
+ # leaves them as they are, as with Logging; dropped SecurityGroupIds go.
+ cluster["resourcesVpcConfig"].update({"securityGroupIds": [], **fields.pop("resourcesVpcConfig")})
+ cluster.setdefault("accessConfig", {}).update(fields.pop("accessConfig"))
+ cluster.update(fields)
+ for store in (cluster["tags"], _eks._tags.setdefault(cluster["arn"], {})):
+ _reconcile_tag_map(store, old_props, new_props)
+ return physical_id, _eks_cluster_attrs(cluster)
+
+
def _eks_cluster_delete(physical_id, props):
import ministack.services.eks as _eks
_eks._delete_cluster(physical_id)
+def _eks_nodegroup_fields(props):
+ """The node group members UpdateNodegroupConfig and UpdateNodegroupVersion change."""
+ fields = {
+ "labels": dict(props.get("Labels") or {}),
+ "taints": _pascal_to_camel(props.get("Taints") or []),
+ }
+ if props.get("ScalingConfig"):
+ fields["scalingConfig"] = {
+ key: int(value) for key, value in _pascal_to_camel(props["ScalingConfig"]).items()}
+ for prop in ("UpdateConfig", "LaunchTemplate", "Version", "ReleaseVersion"):
+ if props.get(prop):
+ fields[prop[:1].lower() + prop[1:]] = _pascal_to_camel(props[prop])
+ return fields
+
+
+def _eks_nodegroup_attrs(ng):
+ return {"ClusterName": ng["clusterName"], "NodegroupName": ng["nodegroupName"],
+ "Arn": ng["nodegroupArn"]}
+
+
def _eks_nodegroup_create(logical_id, props, stack_name):
import ministack.services.eks as _eks
cluster_name = props.get("ClusterName", "")
ng_name = props.get("NodegroupName") or _physical_name(stack_name, logical_id, max_len=63)
body = {
"nodegroupName": ng_name,
- "scalingConfig": props.get("ScalingConfig", {"minSize": 1, "maxSize": 2, "desiredSize": 1}),
"instanceTypes": props.get("InstanceTypes", ["t3.medium"]),
"subnets": props.get("Subnets", []),
"nodeRole": props.get("NodeRole", f"arn:aws:iam::{get_account_id()}:role/eks-node-role"),
"amiType": props.get("AmiType", "AL2_x86_64"),
"diskSize": props.get("DiskSize", 20),
- "labels": props.get("Labels", {}),
"tags": _tag_map(props.get("Tags")),
+ **_eks_nodegroup_fields(props),
}
- _eks._create_nodegroup(cluster_name, body)
- key = f"{cluster_name}/{ng_name}"
- ng = _eks._nodegroups.get(key, {})
- arn = ng.get("nodegroupArn", "")
- return ng_name, {"ClusterName": ng.get("clusterName", cluster_name),
- "NodegroupName": ng.get("nodegroupName", ng_name),
- "Arn": arn}
+ resp = _eks._create_nodegroup(cluster_name, body)
+ if resp[0] >= 400:
+ raise ValueError(f"AWS::EKS::Nodegroup create failed: {resp[2]!r}")
+ ng = _eks._nodegroups[f"{cluster_name}/{ng_name}"]
+ ng.update(_eks_nodegroup_fields(props))
+ return ng_name, _eks_nodegroup_attrs(ng)
+
+
+def _eks_nodegroup_update(physical_id, old_props, new_props, stack_name, logical_id=None):
+ """ScalingConfig, Labels, Taints, UpdateConfig, LaunchTemplate, versions and Tags are in place."""
+ import ministack.services.eks as _eks
+ cluster_name = new_props.get("ClusterName", "")
+ ng_name = new_props.get("NodegroupName") or _physical_name(
+ stack_name, logical_id or physical_id, max_len=63)
+ ng = _eks._nodegroups.get(f"{old_props.get('ClusterName', '')}/{physical_id}")
+ replaced = _rename_replacement(
+ physical_id, old_props, new_props, stack_name, logical_id,
+ f"{cluster_name}/{ng_name}", f"{ng['clusterName']}/{physical_id}" if ng else None,
+ _eks_nodegroup_create, _eks_nodegroup_delete, delete_when_id_unchanged=True,
+ )
+ if replaced is not None:
+ return replaced
+ if any(old_props.get(p) != new_props.get(p) for p in (
+ "NodeRole", "Subnets", "InstanceTypes", "AmiType", "CapacityType", "DiskSize",
+ "RemoteAccess")):
+ if not new_props.get("NodegroupName"):
+ raise ValueError(
+ f"AWS::EKS::Nodegroup {ng_name} requires replacement, which MiniStack does not "
+ "perform under a generated name; set a NodegroupName to create the replacement."
+ )
+ # The replacement's create meets the name still in use and fails, as on AWS.
+ return _eks_nodegroup_create(logical_id or physical_id, new_props, stack_name)
+ fields = _eks_nodegroup_fields(new_props)
+ for key in ("updateConfig", "launchTemplate"):
+ if key not in fields:
+ ng.pop(key, None)
+ ng.update(fields)
+ ng["modifiedAt"] = _eks._now()
+ for store in (ng["tags"], _eks._tags.setdefault(ng["nodegroupArn"], {})):
+ _reconcile_tag_map(store, old_props, new_props)
+ return physical_id, _eks_nodegroup_attrs(ng)
def _eks_nodegroup_delete(physical_id, props):
@@ -13265,8 +13396,18 @@ _RESOURCE_HANDLERS = {
"delete": _location_tracker_delete,
},
# EKS
- "AWS::EKS::Cluster": {"create": _eks_cluster_create, "delete": _eks_cluster_delete},
- "AWS::EKS::Nodegroup": {"create": _eks_nodegroup_create, "delete": _eks_nodegroup_delete},
+ "AWS::EKS::Cluster": {
+ "create": _eks_cluster_create,
+ "update": _eks_cluster_update,
+ "update_with_logical_id": True,
+ "delete": _eks_cluster_delete,
+ },
+ "AWS::EKS::Nodegroup": {
+ "create": _eks_nodegroup_create,
+ "update": _eks_nodegroup_update,
+ "update_with_logical_id": True,
+ "delete": _eks_nodegroup_delete,
+ },
# AWS Backup
"AWS::Backup::BackupVault": {
"create": _backup_vault_create,
diff --git a/ministack/services/eks.py b/ministack/services/eks.py
index 2162d579..08587694 100644
--- a/ministack/services/eks.py
+++ b/ministack/services/eks.py
@@ -889,7 +889,8 @@ def _create_nodegroup(cluster_name, body):
key = f"{cluster_name}/{ng_name}"
if key in _nodegroups:
- return _error(409, "ResourceInUseException", f"Nodegroup already exists with name: {ng_name}")
+ return _error(409, "ResourceInUseException",
+ f"NodeGroup already exists with name {ng_name} and cluster name {cluster_name}")
arn = _nodegroup_arn(cluster_name, ng_name)
now = _now()
diff --git a/tests/test_cfn.py b/tests/test_cfn.py
index 8f063bf4..6cee005d 100644
--- a/tests/test_cfn.py
+++ b/tests/test_cfn.py
@@ -9843,6 +9843,153 @@ def test_cfn_stack_tags_map_valued_tags_and_eks_nodegroup(cfn, sqs, eks):
_delete_cfn_test_stack(cfn, name)
+def _cfn_eks_template(name, cluster=None, nodes=None):
+ """A cluster and a node group, with property overrides for each."""
+ return json.dumps({"Resources": {
+ "Cluster": {"Type": "AWS::EKS::Cluster", "Properties": {
+ "Name": name, "RoleArn": "arn:aws:iam::000000000000:role/eks-role",
+ "ResourcesVpcConfig": {"SubnetIds": ["subnet-1", "subnet-2"]},
+ "Tags": [{"Key": "team", "Value": "a"}], **(cluster or {})}},
+ "Nodes": {"Type": "AWS::EKS::Nodegroup", "Properties": {
+ "ClusterName": {"Ref": "Cluster"}, "NodegroupName": f"{name}-ng",
+ "NodeRole": "arn:aws:iam::000000000000:role/eks-node-role",
+ "Subnets": ["subnet-1"], "InstanceTypes": ["t3.small"],
+ "ScalingConfig": {"MinSize": 0, "DesiredSize": 0, "MaxSize": 1},
+ "Labels": {"tier": "a"}, **(nodes or {})}},
+ }})
+
+
+def test_cfn_update_eks_cluster_in_place(cfn, eks):
+ """Logging, Version, endpoint access and Tags change on the same cluster; dropped SecurityGroupIds go."""
+ name = f"cfn-eks-upd-{_uuid_mod.uuid4().hex[:8]}"
+ api = {"Logging": {"ClusterLogging": {"EnabledTypes": [{"Type": "api"}]}}}
+ try:
+ cfn.create_stack(StackName=name, TemplateBody=_cfn_eks_template(name))
+ assert _wait_stack(cfn, name)["StackStatus"] == "CREATE_COMPLETE"
+ before = eks.describe_cluster(name=name)["cluster"]
+ assert before["logging"]["clusterLogging"] == [{"types": [
+ "api", "audit", "authenticator", "controllerManager", "scheduler"], "enabled": False}]
+ eks.tag_resource(resourceArn=before["arn"], tags={"outside": "api"})
+
+ steps = [
+ (api, lambda c: c["logging"]["clusterLogging"] == [
+ {"types": ["api"], "enabled": True},
+ {"types": ["audit", "authenticator", "controllerManager", "scheduler"],
+ "enabled": False}]),
+ ({**api, "Version": "1.31"}, lambda c: c["version"] == "1.31"),
+ ({**api, "Version": "1.31", "ResourcesVpcConfig": {
+ "SubnetIds": ["subnet-1", "subnet-2"], "EndpointPrivateAccess": True,
+ "PublicAccessCidrs": ["203.0.113.0/24"], "SecurityGroupIds": ["sg-1"]}},
+ lambda c: c["resourcesVpcConfig"]["endpointPrivateAccess"] is True
+ and c["resourcesVpcConfig"]["publicAccessCidrs"] == ["203.0.113.0/24"]
+ and c["resourcesVpcConfig"]["securityGroupIds"] == ["sg-1"]),
+ ({**api, "Version": "1.31", "ResourcesVpcConfig": {
+ "SubnetIds": ["subnet-1", "subnet-2"], "EndpointPrivateAccess": True,
+ "PublicAccessCidrs": ["203.0.113.0/24"]}, "Tags": [{"Key": "team", "Value": "b"}]},
+ lambda c: _template_tags(c["tags"]) == {"team": "b"}
+ and c["resourcesVpcConfig"]["securityGroupIds"] == []),
+ ]
+ for props, check in steps:
+ cfn.update_stack(StackName=name, TemplateBody=_cfn_eks_template(name, props))
+ stack = _wait_stack(cfn, name)
+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")
+ cluster = eks.describe_cluster(name=name)["cluster"]
+ assert check(cluster), cluster
+ assert (cluster["arn"], cluster["createdAt"]) == (before["arn"], before["createdAt"])
+ tags = eks.list_tags_for_resource(resourceArn=before["arn"])["tags"]
+ assert _template_tags(tags) == {"team": "b", "outside": "api"}
+ finally:
+ _delete_cfn_test_stack(cfn, name)
+
+
+def test_cfn_update_eks_nodegroup_in_place(cfn, eks):
+ """ScalingConfig, Labels, Taints, Tags, LaunchTemplate and UpdateConfig change on the same node group."""
+ name = f"cfn-eks-ng-{_uuid_mod.uuid4().hex[:8]}"
+ taint = {"Key": "dedicated", "Value": "batch", "Effect": "NO_SCHEDULE"}
+ lt = {"LaunchTemplate": {"Id": "lt-1", "Version": "1"}, "UpdateConfig": {"MaxUnavailable": 2}}
+ steps = [
+ ({**lt, "ScalingConfig": {"MinSize": 0, "DesiredSize": 0, "MaxSize": 2}},
+ lambda n: n["scalingConfig"] == {"minSize": 0, "desiredSize": 0, "maxSize": 2}),
+ ({**lt, "Labels": {"tier": "b"}}, lambda n: n["labels"] == {"tier": "b"}),
+ ({**lt, "Taints": [taint]},
+ lambda n: n["taints"] == [{"key": "dedicated", "value": "batch", "effect": "NO_SCHEDULE"}]),
+ ({"LaunchTemplate": {"Id": "lt-1", "Version": "2"}, "UpdateConfig": {"MaxUnavailable": 1}},
+ lambda n: (n["launchTemplate"], n["updateConfig"]) == ({"id": "lt-1", "version": "2"}, {"maxUnavailable": 1})),
+ ({"Tags": {"team": "b"}},
+ lambda n: _template_tags(n["tags"]) == {"team": "b"} and not {"launchTemplate", "updateConfig"} & n.keys()),
+ ]
+ try:
+ cfn.create_stack(StackName=name, TemplateBody=_cfn_eks_template(name, nodes=lt))
+ assert _wait_stack(cfn, name)["StackStatus"] == "CREATE_COMPLETE"
+ before = eks.describe_nodegroup(clusterName=name, nodegroupName=f"{name}-ng")["nodegroup"]
+ assert (before["launchTemplate"], before["updateConfig"]) == (
+ {"id": "lt-1", "version": "1"}, {"maxUnavailable": 2})
+ for props, check in steps:
+ cfn.update_stack(StackName=name, TemplateBody=_cfn_eks_template(name, nodes=props))
+ stack = _wait_stack(cfn, name)
+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")
+ ng = eks.describe_nodegroup(clusterName=name, nodegroupName=f"{name}-ng")["nodegroup"]
+ assert check(ng), ng
+ assert (ng["nodegroupArn"], ng["createdAt"]) == (before["nodegroupArn"], before["createdAt"])
+ tags = eks.list_tags_for_resource(resourceArn=before["nodegroupArn"])["tags"]
+ assert _template_tags(tags) == {"team": "b"}
+ finally:
+ _delete_cfn_test_stack(cfn, name)
+
+
[email protected]("created,enabled", [
+ ({"Logging": {"ClusterLogging": {"EnabledTypes": [{"Type": "api"}]}}}, [["api"]]),
+ (None, [["api", "audit"]]),
+], ids=["logging-changed", "logging-added"])
+def test_cfn_update_eks_change_is_rolled_back(cfn, eks, created, enabled):
+ """The rollback sends Logging and ScalingConfig back; a template without Logging keeps it."""
+ name = f"cfn-eks-rb-{_uuid_mod.uuid4().hex[:8]}"
+ api = {"Type": "api"}
+ try:
+ cfn.create_stack(StackName=name, TemplateBody=_cfn_eks_template(name, created))
+ assert _wait_stack(cfn, name)["StackStatus"] == "CREATE_COMPLETE"
+ changed = _cfn_eks_template(
+ name, {"Logging": {"ClusterLogging": {"EnabledTypes": [api, {"Type": "audit"}]}}},
+ {"ScalingConfig": {"MinSize": 0, "DesiredSize": 0, "MaxSize": 3}})
+ cfn.update_stack(StackName=name,
+ TemplateBody=_cfn_with_failing_resource(changed, ["Cluster", "Nodes"]))
+ stack = _wait_stack(cfn, name)
+ assert stack["StackStatus"] == "UPDATE_ROLLBACK_COMPLETE", stack.get("StackStatusReason")
+ cluster = eks.describe_cluster(name=name)["cluster"]
+ assert [g["types"] for g in cluster["logging"]["clusterLogging"] if g["enabled"]] == enabled
+ ng = eks.describe_nodegroup(clusterName=name, nodegroupName=f"{name}-ng")["nodegroup"]
+ assert ng["scalingConfig"]["maxSize"] == 1
+ finally:
+ _delete_cfn_test_stack(cfn, name)
+
+
[email protected]("cluster,nodes,reason", [
+ ({"RoleArn": "arn:aws:iam::000000000000:role/eks-role-2"}, None,
+ "CloudFormation cannot update a stack when a custom-named resource requires "
+ "replacing. Rename {name} and update the stack again."),
+ (None, {"InstanceTypes": ["t3.medium"]},
+ "NodeGroup already exists with name {name}-ng and cluster name {name}"),
+], ids=["cluster-role", "nodegroup-instance-types"])
+def test_cfn_update_eks_create_only_change_fails_under_a_custom_name(
+ cfn, eks, cluster, nodes, reason):
+ """A createOnly change under an explicit name fails the update, as on AWS."""
+ name = f"cfn-eks-co-{_uuid_mod.uuid4().hex[:8]}"
+ try:
+ cfn.create_stack(StackName=name, TemplateBody=_cfn_eks_template(name))
+ assert _wait_stack(cfn, name)["StackStatus"] == "CREATE_COMPLETE"
+ cfn.update_stack(StackName=name, TemplateBody=_cfn_eks_template(name, cluster, nodes))
+ stack = _wait_stack(cfn, name)
+ assert stack["StackStatus"] == "UPDATE_ROLLBACK_COMPLETE", stack.get("StackStatusReason")
+ reasons = [e.get("ResourceStatusReason", "") for e in
+ cfn.describe_stack_events(StackName=name)["StackEvents"]
+ if e["ResourceStatus"] == "UPDATE_FAILED"]
+ assert any(reason.format(name=name) in r for r in reasons), reasons
+ ng = eks.describe_nodegroup(clusterName=name, nodegroupName=f"{name}-ng")["nodegroup"]
+ assert ng["instanceTypes"] == ["t3.small"]
+ finally:
+ _delete_cfn_test_stack(cfn, name)
+
+
def test_cfn_change_set_tags_replace_the_previous_stack_tags(cfn, sqs):
"""An UPDATE change set that carries new stack tags: after execute the
queue carries the new tag and no longer the old one. The execution