Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(cloudformation): update EKS clusters and node groups in place (#1937)
iot-rocket095a129ce511
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 32d31bf9..3db5a7e7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Added+- **CloudFormation — EKS cluster and node group updates** — `AWS::EKS::Cluster` applies `Version`, `Logging`, `ResourcesVpcConfig`, `AccessConfig.AuthenticationMode` and `Tags` in place and `AWS::EKS::Nodegroup` applies `ScalingConfig`, `Labels`, `Taints`, `UpdateConfig`, `LaunchTemplate`, `Version`, `ReleaseVersion` and `Tags`, where every such change used to report `UPDATE_COMPLETE` and was dropped.- **SES — a sandboxed account sends only to verified or simulator recipients** — SES v2 `PutAccountDetails` with `ProductionAccessEnabled=false` puts the account (per region) in the sandbox, as on AWS, and `GetAccount` reports it with the submitted `Details`. A sandboxed send to a recipient that is neither a verified address or domain identity nor a `@simulator.amazonses.com` address fails with `MessageRejected` "Email address is not verified. The following identities failed the check in region …" for v1 `SendEmail`, `SendRawEmail` and `SendTemplatedEmail` and v2 `SendEmail`; bulk sends reject only the affected entries. Accounts stay in production by default. Reported by @skialpine.### Fixeddiff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.pyindex 71092acb..76b2e2dd 100644--- a/ministack/services/cloudformation/provisioners.py+++ b/ministack/services/cloudformation/provisioners.py@@ -733,6 +733,12 @@ _CUSTOM_NAME_REPLACEMENT = {old.get("Scope", "REGIONAL") != new.get("Scope", "REGIONAL")),},+ "AWS::EKS::Cluster": {+ "name": "Name",+ "requires_replacement": lambda old, new: (+ _eks_cluster_create_only(old) != _eks_cluster_create_only(new)+ ),+ },# "Update requires: Replacement" in the template reference, the name# itself aside."AWS::ElastiCache::CacheCluster": {@@ -3474,21 +3480,61 @@ def _scheduler_group_delete(physical_id, props):# --- EKS Cluster ---+_EKS_LOG_TYPES = ("api", "audit", "authenticator", "controllerManager", "scheduler")+++def _eks_cluster_create_only(props):+ """The cluster's createOnlyProperties other than Name."""+ network = props.get("KubernetesNetworkConfig") or {}+ return (+ props.get("RoleArn"), props.get("EncryptionConfig"), props.get("OutpostConfig"),+ props.get("BootstrapSelfManagedAddons"), network.get("IpFamily"),+ network.get("ServiceIpv4Cidr"),+ (props.get("AccessConfig") or {}).get("BootstrapClusterCreatorAdminPermissions"),+ )+++def _eks_cluster_fields(props):+ """The cluster members UpdateClusterConfig and UpdateClusterVersion change."""+ vpc = _pascal_to_camel(props.get("ResourcesVpcConfig") or {})+ for key in ("endpointPublicAccess", "endpointPrivateAccess"):+ if key in vpc:+ vpc[key] = _cfn_bool(vpc[key])+ logging = ((props.get("Logging") or {}).get("ClusterLogging") or {}).get("EnabledTypes") or []+ enabled = [t.get("Type") for t in logging]+ disabled = [t for t in _EKS_LOG_TYPES if t not in enabled]+ mode = (props.get("AccessConfig") or {}).get("AuthenticationMode")+ fields = {+ "resourcesVpcConfig": vpc,+ "logging": {"clusterLogging": [+ {"types": types, "enabled": on}+ for types, on in ((enabled, True), (disabled, False)) if types+ ]},+ "accessConfig": {"authenticationMode": mode} if mode else {},+ }+ if props.get("Version"):+ fields["version"] = str(props["Version"])+ return fields++def _eks_cluster_create(logical_id, props, stack_name):import ministack.services.eks as _eksname = props.get("Name") or _physical_name(stack_name, logical_id, max_len=100)body = {"name": name,- "version": props.get("Version", "1.30"),"roleArn": props.get("RoleArn", f"arn:aws:iam::{get_account_id()}:role/eks-role"),- "resourcesVpcConfig": props.get("ResourcesVpcConfig", {}),- "tags": {t["Key"]: t["Value"] for t in props.get("Tags", [])},+ "tags": _tag_map(props.get("Tags")),+ **_eks_cluster_fields(props),}- _eks._create_cluster(body)- arn = _eks._cluster_arn(name)- cluster = _eks._clusters.get(name, {})- return name, {- "Arn": arn,+ resp = _eks._create_cluster(body)+ if resp[0] >= 400:+ raise ValueError(f"AWS::EKS::Cluster create failed: {resp[2]!r}")+ return name, _eks_cluster_attrs(_eks._clusters[name])+++def _eks_cluster_attrs(cluster):+ return {+ "Arn": cluster["arn"],"Endpoint": cluster.get("endpoint", ""),"CertificateAuthorityData": cluster.get("certificateAuthority", {}).get("data", ""),"ClusterSecurityGroupId": cluster.get("resourcesVpcConfig", {}).get("clusterSecurityGroupId", ""),@@ -3496,33 +3542,118 @@ def _eks_cluster_create(logical_id, props, stack_name):}+def _eks_cluster_update(physical_id, old_props, new_props, stack_name, logical_id=None):+ """Version, Logging, ResourcesVpcConfig, AuthenticationMode and Tags are in place."""+ import ministack.services.eks as _eks+ name = new_props.get("Name") or _physical_name(+ stack_name, logical_id or physical_id, max_len=100)+ cluster = _eks._clusters.get(physical_id)+ replaced = _rename_replacement(+ physical_id, old_props, new_props, stack_name, logical_id,+ name, physical_id if cluster else None,+ _eks_cluster_create, _eks_cluster_delete,+ )+ if replaced is not None:+ return replaced+ if _requires_replacement("AWS::EKS::Cluster", old_props, new_props):+ raise ValueError(+ f"AWS::EKS::Cluster {name} requires replacement, which MiniStack does not "+ "perform under a generated name; set a Name to create the replacement."+ )+ fields = _eks_cluster_fields(new_props)+ if "Logging" not in new_props:+ # A rollback to a template without Logging leaves the logging as it is.+ del fields["logging"]+ # A template without the endpoint access members or AuthenticationMode+ # leaves them as they are, as with Logging; dropped SecurityGroupIds go.+ cluster["resourcesVpcConfig"].update({"securityGroupIds": [], **fields.pop("resourcesVpcConfig")})+ cluster.setdefault("accessConfig", {}).update(fields.pop("accessConfig"))+ cluster.update(fields)+ for store in (cluster["tags"], _eks._tags.setdefault(cluster["arn"], {})):+ _reconcile_tag_map(store, old_props, new_props)+ return physical_id, _eks_cluster_attrs(cluster)++def _eks_cluster_delete(physical_id, props):import ministack.services.eks as _eks_eks._delete_cluster(physical_id)+def _eks_nodegroup_fields(props):+ """The node group members UpdateNodegroupConfig and UpdateNodegroupVersion change."""+ fields = {+ "labels": dict(props.get("Labels") or {}),+ "taints": _pascal_to_camel(props.get("Taints") or []),+ }+ if props.get("ScalingConfig"):+ fields["scalingConfig"] = {+ key: int(value) for key, value in _pascal_to_camel(props["ScalingConfig"]).items()}+ for prop in ("UpdateConfig", "LaunchTemplate", "Version", "ReleaseVersion"):+ if props.get(prop):+ fields[prop[:1].lower() + prop[1:]] = _pascal_to_camel(props[prop])+ return fields+++def _eks_nodegroup_attrs(ng):+ return {"ClusterName": ng["clusterName"], "NodegroupName": ng["nodegroupName"],+ "Arn": ng["nodegroupArn"]}++def _eks_nodegroup_create(logical_id, props, stack_name):import ministack.services.eks as _ekscluster_name = props.get("ClusterName", "")ng_name = props.get("NodegroupName") or _physical_name(stack_name, logical_id, max_len=63)body = {"nodegroupName": ng_name,- "scalingConfig": props.get("ScalingConfig", {"minSize": 1, "maxSize": 2, "desiredSize": 1}),"instanceTypes": props.get("InstanceTypes", ["t3.medium"]),"subnets": props.get("Subnets", []),"nodeRole": props.get("NodeRole", f"arn:aws:iam::{get_account_id()}:role/eks-node-role"),"amiType": props.get("AmiType", "AL2_x86_64"),"diskSize": props.get("DiskSize", 20),- "labels": props.get("Labels", {}),"tags": _tag_map(props.get("Tags")),+ **_eks_nodegroup_fields(props),}- _eks._create_nodegroup(cluster_name, body)- key = f"{cluster_name}/{ng_name}"- ng = _eks._nodegroups.get(key, {})- arn = ng.get("nodegroupArn", "")- return ng_name, {"ClusterName": ng.get("clusterName", cluster_name),- "NodegroupName": ng.get("nodegroupName", ng_name),- "Arn": arn}+ resp = _eks._create_nodegroup(cluster_name, body)+ if resp[0] >= 400:+ raise ValueError(f"AWS::EKS::Nodegroup create failed: {resp[2]!r}")+ ng = _eks._nodegroups[f"{cluster_name}/{ng_name}"]+ ng.update(_eks_nodegroup_fields(props))+ return ng_name, _eks_nodegroup_attrs(ng)+++def _eks_nodegroup_update(physical_id, old_props, new_props, stack_name, logical_id=None):+ """ScalingConfig, Labels, Taints, UpdateConfig, LaunchTemplate, versions and Tags are in place."""+ import ministack.services.eks as _eks+ cluster_name = new_props.get("ClusterName", "")+ ng_name = new_props.get("NodegroupName") or _physical_name(+ stack_name, logical_id or physical_id, max_len=63)+ ng = _eks._nodegroups.get(f"{old_props.get('ClusterName', '')}/{physical_id}")+ replaced = _rename_replacement(+ physical_id, old_props, new_props, stack_name, logical_id,+ f"{cluster_name}/{ng_name}", f"{ng['clusterName']}/{physical_id}" if ng else None,+ _eks_nodegroup_create, _eks_nodegroup_delete, delete_when_id_unchanged=True,+ )+ if replaced is not None:+ return replaced+ if any(old_props.get(p) != new_props.get(p) for p in (+ "NodeRole", "Subnets", "InstanceTypes", "AmiType", "CapacityType", "DiskSize",+ "RemoteAccess")):+ if not new_props.get("NodegroupName"):+ raise ValueError(+ f"AWS::EKS::Nodegroup {ng_name} requires replacement, which MiniStack does not "+ "perform under a generated name; set a NodegroupName to create the replacement."+ )+ # The replacement's create meets the name still in use and fails, as on AWS.+ return _eks_nodegroup_create(logical_id or physical_id, new_props, stack_name)+ fields = _eks_nodegroup_fields(new_props)+ for key in ("updateConfig", "launchTemplate"):+ if key not in fields:+ ng.pop(key, None)+ ng.update(fields)+ ng["modifiedAt"] = _eks._now()+ for store in (ng["tags"], _eks._tags.setdefault(ng["nodegroupArn"], {})):+ _reconcile_tag_map(store, old_props, new_props)+ return physical_id, _eks_nodegroup_attrs(ng)def _eks_nodegroup_delete(physical_id, props):@@ -13265,8 +13396,18 @@ _RESOURCE_HANDLERS = {"delete": _location_tracker_delete,},# EKS- "AWS::EKS::Cluster": {"create": _eks_cluster_create, "delete": _eks_cluster_delete},- "AWS::EKS::Nodegroup": {"create": _eks_nodegroup_create, "delete": _eks_nodegroup_delete},+ "AWS::EKS::Cluster": {+ "create": _eks_cluster_create,+ "update": _eks_cluster_update,+ "update_with_logical_id": True,+ "delete": _eks_cluster_delete,+ },+ "AWS::EKS::Nodegroup": {+ "create": _eks_nodegroup_create,+ "update": _eks_nodegroup_update,+ "update_with_logical_id": True,+ "delete": _eks_nodegroup_delete,+ },# AWS Backup"AWS::Backup::BackupVault": {"create": _backup_vault_create,diff --git a/ministack/services/eks.py b/ministack/services/eks.pyindex 2162d579..08587694 100644--- a/ministack/services/eks.py+++ b/ministack/services/eks.py@@ -889,7 +889,8 @@ def _create_nodegroup(cluster_name, body):key = f"{cluster_name}/{ng_name}"if key in _nodegroups:- return _error(409, "ResourceInUseException", f"Nodegroup already exists with name: {ng_name}")+ return _error(409, "ResourceInUseException",+ f"NodeGroup already exists with name {ng_name} and cluster name {cluster_name}")arn = _nodegroup_arn(cluster_name, ng_name)now = _now()diff --git a/tests/test_cfn.py b/tests/test_cfn.pyindex 8f063bf4..6cee005d 100644--- a/tests/test_cfn.py+++ b/tests/test_cfn.py@@ -9843,6 +9843,153 @@ def test_cfn_stack_tags_map_valued_tags_and_eks_nodegroup(cfn, sqs, eks):_delete_cfn_test_stack(cfn, name)+def _cfn_eks_template(name, cluster=None, nodes=None):+ """A cluster and a node group, with property overrides for each."""+ return json.dumps({"Resources": {+ "Cluster": {"Type": "AWS::EKS::Cluster", "Properties": {+ "Name": name, "RoleArn": "arn:aws:iam::000000000000:role/eks-role",+ "ResourcesVpcConfig": {"SubnetIds": ["subnet-1", "subnet-2"]},+ "Tags": [{"Key": "team", "Value": "a"}], **(cluster or {})}},+ "Nodes": {"Type": "AWS::EKS::Nodegroup", "Properties": {+ "ClusterName": {"Ref": "Cluster"}, "NodegroupName": f"{name}-ng",+ "NodeRole": "arn:aws:iam::000000000000:role/eks-node-role",+ "Subnets": ["subnet-1"], "InstanceTypes": ["t3.small"],+ "ScalingConfig": {"MinSize": 0, "DesiredSize": 0, "MaxSize": 1},+ "Labels": {"tier": "a"}, **(nodes or {})}},+ }})+++def test_cfn_update_eks_cluster_in_place(cfn, eks):+ """Logging, Version, endpoint access and Tags change on the same cluster; dropped SecurityGroupIds go."""+ name = f"cfn-eks-upd-{_uuid_mod.uuid4().hex[:8]}"+ api = {"Logging": {"ClusterLogging": {"EnabledTypes": [{"Type": "api"}]}}}+ try:+ cfn.create_stack(StackName=name, TemplateBody=_cfn_eks_template(name))+ assert _wait_stack(cfn, name)["StackStatus"] == "CREATE_COMPLETE"+ before = eks.describe_cluster(name=name)["cluster"]+ assert before["logging"]["clusterLogging"] == [{"types": [+ "api", "audit", "authenticator", "controllerManager", "scheduler"], "enabled": False}]+ eks.tag_resource(resourceArn=before["arn"], tags={"outside": "api"})++ steps = [+ (api, lambda c: c["logging"]["clusterLogging"] == [+ {"types": ["api"], "enabled": True},+ {"types": ["audit", "authenticator", "controllerManager", "scheduler"],+ "enabled": False}]),+ ({**api, "Version": "1.31"}, lambda c: c["version"] == "1.31"),+ ({**api, "Version": "1.31", "ResourcesVpcConfig": {+ "SubnetIds": ["subnet-1", "subnet-2"], "EndpointPrivateAccess": True,+ "PublicAccessCidrs": ["203.0.113.0/24"], "SecurityGroupIds": ["sg-1"]}},+ lambda c: c["resourcesVpcConfig"]["endpointPrivateAccess"] is True+ and c["resourcesVpcConfig"]["publicAccessCidrs"] == ["203.0.113.0/24"]+ and c["resourcesVpcConfig"]["securityGroupIds"] == ["sg-1"]),+ ({**api, "Version": "1.31", "ResourcesVpcConfig": {+ "SubnetIds": ["subnet-1", "subnet-2"], "EndpointPrivateAccess": True,+ "PublicAccessCidrs": ["203.0.113.0/24"]}, "Tags": [{"Key": "team", "Value": "b"}]},+ lambda c: _template_tags(c["tags"]) == {"team": "b"}+ and c["resourcesVpcConfig"]["securityGroupIds"] == []),+ ]+ for props, check in steps:+ cfn.update_stack(StackName=name, TemplateBody=_cfn_eks_template(name, props))+ stack = _wait_stack(cfn, name)+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")+ cluster = eks.describe_cluster(name=name)["cluster"]+ assert check(cluster), cluster+ assert (cluster["arn"], cluster["createdAt"]) == (before["arn"], before["createdAt"])+ tags = eks.list_tags_for_resource(resourceArn=before["arn"])["tags"]+ assert _template_tags(tags) == {"team": "b", "outside": "api"}+ finally:+ _delete_cfn_test_stack(cfn, name)+++def test_cfn_update_eks_nodegroup_in_place(cfn, eks):+ """ScalingConfig, Labels, Taints, Tags, LaunchTemplate and UpdateConfig change on the same node group."""+ name = f"cfn-eks-ng-{_uuid_mod.uuid4().hex[:8]}"+ taint = {"Key": "dedicated", "Value": "batch", "Effect": "NO_SCHEDULE"}+ lt = {"LaunchTemplate": {"Id": "lt-1", "Version": "1"}, "UpdateConfig": {"MaxUnavailable": 2}}+ steps = [+ ({**lt, "ScalingConfig": {"MinSize": 0, "DesiredSize": 0, "MaxSize": 2}},+ lambda n: n["scalingConfig"] == {"minSize": 0, "desiredSize": 0, "maxSize": 2}),+ ({**lt, "Labels": {"tier": "b"}}, lambda n: n["labels"] == {"tier": "b"}),+ ({**lt, "Taints": [taint]},+ lambda n: n["taints"] == [{"key": "dedicated", "value": "batch", "effect": "NO_SCHEDULE"}]),+ ({"LaunchTemplate": {"Id": "lt-1", "Version": "2"}, "UpdateConfig": {"MaxUnavailable": 1}},+ lambda n: (n["launchTemplate"], n["updateConfig"]) == ({"id": "lt-1", "version": "2"}, {"maxUnavailable": 1})),+ ({"Tags": {"team": "b"}},+ lambda n: _template_tags(n["tags"]) == {"team": "b"} and not {"launchTemplate", "updateConfig"} & n.keys()),+ ]+ try:+ cfn.create_stack(StackName=name, TemplateBody=_cfn_eks_template(name, nodes=lt))+ assert _wait_stack(cfn, name)["StackStatus"] == "CREATE_COMPLETE"+ before = eks.describe_nodegroup(clusterName=name, nodegroupName=f"{name}-ng")["nodegroup"]+ assert (before["launchTemplate"], before["updateConfig"]) == (+ {"id": "lt-1", "version": "1"}, {"maxUnavailable": 2})+ for props, check in steps:+ cfn.update_stack(StackName=name, TemplateBody=_cfn_eks_template(name, nodes=props))+ stack = _wait_stack(cfn, name)+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")+ ng = eks.describe_nodegroup(clusterName=name, nodegroupName=f"{name}-ng")["nodegroup"]+ assert check(ng), ng+ assert (ng["nodegroupArn"], ng["createdAt"]) == (before["nodegroupArn"], before["createdAt"])+ tags = eks.list_tags_for_resource(resourceArn=before["nodegroupArn"])["tags"]+ assert _template_tags(tags) == {"team": "b"}+ finally:+ _delete_cfn_test_stack(cfn, name)++[email protected]("created,enabled", [+ ({"Logging": {"ClusterLogging": {"EnabledTypes": [{"Type": "api"}]}}}, [["api"]]),+ (None, [["api", "audit"]]),+], ids=["logging-changed", "logging-added"])+def test_cfn_update_eks_change_is_rolled_back(cfn, eks, created, enabled):+ """The rollback sends Logging and ScalingConfig back; a template without Logging keeps it."""+ name = f"cfn-eks-rb-{_uuid_mod.uuid4().hex[:8]}"+ api = {"Type": "api"}+ try:+ cfn.create_stack(StackName=name, TemplateBody=_cfn_eks_template(name, created))+ assert _wait_stack(cfn, name)["StackStatus"] == "CREATE_COMPLETE"+ changed = _cfn_eks_template(+ name, {"Logging": {"ClusterLogging": {"EnabledTypes": [api, {"Type": "audit"}]}}},+ {"ScalingConfig": {"MinSize": 0, "DesiredSize": 0, "MaxSize": 3}})+ cfn.update_stack(StackName=name,+ TemplateBody=_cfn_with_failing_resource(changed, ["Cluster", "Nodes"]))+ stack = _wait_stack(cfn, name)+ assert stack["StackStatus"] == "UPDATE_ROLLBACK_COMPLETE", stack.get("StackStatusReason")+ cluster = eks.describe_cluster(name=name)["cluster"]+ assert [g["types"] for g in cluster["logging"]["clusterLogging"] if g["enabled"]] == enabled+ ng = eks.describe_nodegroup(clusterName=name, nodegroupName=f"{name}-ng")["nodegroup"]+ assert ng["scalingConfig"]["maxSize"] == 1+ finally:+ _delete_cfn_test_stack(cfn, name)++[email protected]("cluster,nodes,reason", [+ ({"RoleArn": "arn:aws:iam::000000000000:role/eks-role-2"}, None,+ "CloudFormation cannot update a stack when a custom-named resource requires "+ "replacing. Rename {name} and update the stack again."),+ (None, {"InstanceTypes": ["t3.medium"]},+ "NodeGroup already exists with name {name}-ng and cluster name {name}"),+], ids=["cluster-role", "nodegroup-instance-types"])+def test_cfn_update_eks_create_only_change_fails_under_a_custom_name(+ cfn, eks, cluster, nodes, reason):+ """A createOnly change under an explicit name fails the update, as on AWS."""+ name = f"cfn-eks-co-{_uuid_mod.uuid4().hex[:8]}"+ try:+ cfn.create_stack(StackName=name, TemplateBody=_cfn_eks_template(name))+ assert _wait_stack(cfn, name)["StackStatus"] == "CREATE_COMPLETE"+ cfn.update_stack(StackName=name, TemplateBody=_cfn_eks_template(name, cluster, nodes))+ stack = _wait_stack(cfn, name)+ assert stack["StackStatus"] == "UPDATE_ROLLBACK_COMPLETE", stack.get("StackStatusReason")+ reasons = [e.get("ResourceStatusReason", "") for e in+ cfn.describe_stack_events(StackName=name)["StackEvents"]+ if e["ResourceStatus"] == "UPDATE_FAILED"]+ assert any(reason.format(name=name) in r for r in reasons), reasons+ ng = eks.describe_nodegroup(clusterName=name, nodegroupName=f"{name}-ng")["nodegroup"]+ assert ng["instanceTypes"] == ["t3.small"]+ finally:+ _delete_cfn_test_stack(cfn, name)++def test_cfn_change_set_tags_replace_the_previous_stack_tags(cfn, sqs):"""An UPDATE change set that carries new stack tags: after execute thequeue carries the new tag and no longer the old one. The execution