Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(cloudformation): apply every AWS::SQS::Queue property (#1859)
iot-rocket425042e53055
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 021ae158..a9d6bf02 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Fixed+- **CloudFormation — `AWS::SQS::Queue` applies every queue property** — `RedrivePolicy`, `RedriveAllowPolicy`, `KmsMasterKeyId`, `KmsDataKeyReusePeriodSeconds`, `SqsManagedSseEnabled`, `DeduplicationScope` and `FifoThroughputLimit` were dropped on create and update, so a dead-letter queue declared in a template never received messages, and a value SQS refuses now fails the resource instead of being stored. Contributed by @iot-rocket.- **Lambda — Docker executor honors timeouts above 300 seconds** — pass the configured `Timeout` to AWS RIE through `AWS_LAMBDA_FUNCTION_TIMEOUT`, preventing its default 300-second limit from ending longer invocations early. Timeout updates recycle warm containers so the RIE deadline follows the new configuration (#1844).- **KMS — `GenerateDataKey` returns `KeyMaterialId`** — requests without `Recipient` now include the identifier of the wrapping key material. The identifier remains stable across requests and persistence reloads, and changes when the wrapping material changes. Fixes #1853.- **CloudFormation — a nested stack's update deletes the resources its template drops** — a resource removed from the child template, or created by a failed child update that was rolled back, stayed in its service and in the nested stack's resource list. Contributed by @iot-rocket.diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.pyindex 6adb63c1..258341ac 100644--- a/ministack/services/cloudformation/provisioners.py+++ b/ministack/services/cloudformation/provisioners.py@@ -1350,27 +1350,47 @@ def _s3_delete(physical_id, props):# --- SQS Queue ---+_SQS_QUEUE_DEFAULTS = {+ "VisibilityTimeout": "30",+ "MaximumMessageSize": "262144",+ "MessageRetentionPeriod": "345600",+ "DelaySeconds": "0",+ "ReceiveMessageWaitTimeSeconds": "0",+}+_SQS_QUEUE_OPTIONAL = (+ "KmsMasterKeyId", "KmsDataKeyReusePeriodSeconds", "SqsManagedSseEnabled",+ "RedrivePolicy", "RedriveAllowPolicy",+)+_SQS_FIFO_OPTIONAL = ("ContentBasedDeduplication", "DeduplicationScope", "FifoThroughputLimit")+++def _sqs_queue_fields(props, is_fifo):+ """The queue attributes of the properties, validated as SetQueueAttributes does."""+ fields = {key: str(props.get(key, default)) for key, default in _SQS_QUEUE_DEFAULTS.items()}+ for key in _SQS_QUEUE_OPTIONAL + (_SQS_FIFO_OPTIONAL if is_fifo else ()):+ value = props.get(key)+ if isinstance(value, dict):+ fields[key] = json.dumps(value, separators=(",", ":"))+ elif isinstance(value, bool):+ fields[key] = str(value).lower()+ elif value is not None:+ fields[key] = str(value)+ if "RedrivePolicy" in fields:+ _sqs._validate_redrive_policy(fields["RedrivePolicy"])+ _sqs._validate_numeric_attrs(fields)+ return fields++def _sqs_create(logical_id, props, stack_name):name = props.get("QueueName") or _physical_name(stack_name, logical_id, max_len=80)is_fifo = name.endswith(".fifo")+ attributes = _sqs_queue_fields(props, is_fifo)url = f"http://{_sqs.DEFAULT_HOST}:{_sqs.DEFAULT_PORT}/{get_account_id()}/{name}"arn = f"arn:aws:sqs:{get_region()}:{get_account_id()}:{name}"now_ts = str(int(time.time()))-- attributes = {- "QueueArn": arn,- "CreatedTimestamp": now_ts,- "LastModifiedTimestamp": now_ts,- "VisibilityTimeout": str(props.get("VisibilityTimeout", "30")),- "MaximumMessageSize": str(props.get("MaximumMessageSize", "262144")),- "MessageRetentionPeriod": str(props.get("MessageRetentionPeriod", "345600")),- "DelaySeconds": str(props.get("DelaySeconds", "0")),- "ReceiveMessageWaitTimeSeconds": str(props.get("ReceiveMessageWaitTimeSeconds", "0")),- }+ attributes.update(QueueArn=arn, CreatedTimestamp=now_ts, LastModifiedTimestamp=now_ts)if is_fifo:attributes["FifoQueue"] = "true"- if props.get("ContentBasedDeduplication"):- attributes["ContentBasedDeduplication"] = str(props["ContentBasedDeduplication"]).lower()queue = {"name": name,@@ -1405,21 +1425,12 @@ def _sqs_update(physical_id, old_props, new_props, stack_name, logical_id=None):)if replaced is not None:return replaced+ fields = _sqs_queue_fields(new_props, queue["is_fifo"])attributes = queue["attributes"]- attributes["VisibilityTimeout"] = str(new_props.get("VisibilityTimeout", "30"))- attributes["MaximumMessageSize"] = str(new_props.get("MaximumMessageSize", "262144"))- attributes["MessageRetentionPeriod"] = str(new_props.get("MessageRetentionPeriod", "345600"))- attributes["DelaySeconds"] = str(new_props.get("DelaySeconds", "0"))- attributes["ReceiveMessageWaitTimeSeconds"] = str(new_props.get("ReceiveMessageWaitTimeSeconds", "0"))- if queue["is_fifo"]:- # Same omit-unless-truthy shape as create: an absent property must not- # materialize as ContentBasedDeduplication "false" on the record.- if new_props.get("ContentBasedDeduplication"):- attributes["ContentBasedDeduplication"] = str(- new_props["ContentBasedDeduplication"]- ).lower()- else:- attributes.pop("ContentBasedDeduplication", None)+ for key in _SQS_QUEUE_OPTIONAL + _SQS_FIFO_OPTIONAL:+ if key in old_props and key not in fields:+ attributes.pop(key, None)+ attributes.update(fields)_reconcile_tag_map(queue.setdefault("tags", {}), old_props, new_props)attributes["LastModifiedTimestamp"] = str(int(time.time()))arn = attributes["QueueArn"]diff --git a/tests/test_cfn.py b/tests/test_cfn.pyindex 5e6be157..3de8739b 100644--- a/tests/test_cfn.py+++ b/tests/test_cfn.py@@ -17886,6 +17886,97 @@ def test_cfn_update_sqs_queue_rename_replaces(cfn, sqs):_delete_cfn_test_stack(cfn, stack_name)+def _sqs_dlq_tpl(queue_props):+ """A dead-letter queue and a queue with the given properties."""+ return {"Resources": {+ "Dlq": {"Type": "AWS::SQS::Queue"},+ "Queue": {"Type": "AWS::SQS::Queue", "Properties": queue_props},+ }}+++def _sqs_redrive(max_receive_count):+ return {"deadLetterTargetArn": {"Fn::GetAtt": ["Dlq", "Arn"]},+ "maxReceiveCount": max_receive_count}+++def _sqs_stack_attrs(sqs, cfn, stack_name):+ """The Queue's attributes, with the Dlq's ARN replaced by DLQ_ARN."""+ dlq_arn = sqs.get_queue_attributes(QueueUrl=_stack_physical_id(cfn, stack_name, "Dlq"),+ AttributeNames=["QueueArn"])["Attributes"]["QueueArn"]+ attrs = sqs.get_queue_attributes(QueueUrl=_stack_physical_id(cfn, stack_name, "Queue"),+ AttributeNames=["All"])["Attributes"]+ return {k: v.replace(dlq_arn, "DLQ_ARN") for k, v in attrs.items()}++[email protected]("props,expected", [+ ({"RedrivePolicy": _sqs_redrive(3), "RedriveAllowPolicy": {"redrivePermission": "denyAll"}},+ {"RedrivePolicy": {"deadLetterTargetArn": "DLQ_ARN", "maxReceiveCount": 3},+ "RedriveAllowPolicy": {"redrivePermission": "denyAll"}}),+ ({"KmsMasterKeyId": "alias/aws/sqs", "KmsDataKeyReusePeriodSeconds": 600},+ {"KmsMasterKeyId": "alias/aws/sqs", "KmsDataKeyReusePeriodSeconds": "600"}),+ ({"SqsManagedSseEnabled": False}, {"SqsManagedSseEnabled": "false"}),+ ({"FifoQueue": True, "ContentBasedDeduplication": True,+ "DeduplicationScope": "messageGroup", "FifoThroughputLimit": "perMessageGroupId"},+ {"FifoQueue": "true", "ContentBasedDeduplication": "true",+ "DeduplicationScope": "messageGroup", "FifoThroughputLimit": "perMessageGroupId"}),+], ids=["redrive", "kms", "sse", "fifo"])+def test_cfn_sqs_queue_properties_reach_the_queue(cfn, sqs, props, expected):+ """Each queue property shows up in GetQueueAttributes, Json ones as compact JSON."""+ stack_name = f"cfn-sqs-attrs-{_uuid_mod.uuid4().hex[:8]}"+ name = f"{stack_name}-q" + (".fifo" if props.get("FifoQueue") else "")+ props = {**props, "QueueName": name}+ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(_sqs_dlq_tpl(props)))+ try:+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")+ attrs = _sqs_stack_attrs(sqs, cfn, stack_name)+ assert {k: attrs.get(k) for k in expected} == {+ k: json.dumps(v, separators=(",", ":")) if isinstance(v, dict) else v for k, v in expected.items()}+ finally:+ _delete_cfn_test_stack(cfn, stack_name)+++def test_cfn_update_sqs_queue_redrive_policy_in_place(cfn, sqs):+ """A RedrivePolicy is added, changed and removed by updates to the same queue."""+ stack_name = f"cfn-up-sqs-rp-{_uuid_mod.uuid4().hex[:8]}"+ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(_sqs_dlq_tpl({})))+ try:+ assert _wait_stack(cfn, stack_name)["StackStatus"] == "CREATE_COMPLETE"+ queue_url = _stack_physical_id(cfn, stack_name, "Queue")+ for count in (2, 5, None):+ props = {"RedrivePolicy": _sqs_redrive(count)} if count else {}+ cfn.update_stack(StackName=stack_name, TemplateBody=json.dumps(_sqs_dlq_tpl(props)))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")+ assert _stack_physical_id(cfn, stack_name, "Queue") == queue_url+ policy = _sqs_stack_attrs(sqs, cfn, stack_name).get("RedrivePolicy")+ assert (json.loads(policy) if policy else None) == (+ {"deadLetterTargetArn": "DLQ_ARN", "maxReceiveCount": count} if count else None)+ finally:+ _delete_cfn_test_stack(cfn, stack_name)++[email protected]("props,parameter", [+ ({"MessageRetentionPeriod": 10}, "MessageRetentionPeriod"),+ ({"RedrivePolicy": _sqs_redrive(0)}, "RedrivePolicy"),+])+def test_cfn_sqs_queue_invalid_attribute_fails_the_resource(cfn, sqs, props, parameter):+ """A value SQS refuses fails the queue with the SQS message and rolls the stack back."""+ stack_name = f"cfn-sqs-bad-{_uuid_mod.uuid4().hex[:8]}"+ props = {**props, "QueueName": f"{stack_name}-q"}+ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(_sqs_dlq_tpl(props)))+ try:+ assert _wait_stack(cfn, stack_name)["StackStatus"] == "ROLLBACK_COMPLETE"+ failed = [e for e in cfn.describe_stack_events(StackName=stack_name)["StackEvents"]+ if e["LogicalResourceId"] == "Queue" and e["ResourceStatus"] == "CREATE_FAILED"]+ assert failed and (f"Invalid value for the parameter {parameter}."+ in failed[0]["ResourceStatusReason"])+ with pytest.raises(ClientError):+ sqs.get_queue_url(QueueName=f"{stack_name}-q")+ finally:+ _delete_cfn_test_stack(cfn, stack_name)++def test_cfn_update_logs_log_group_in_place(cfn, logs):"""A log group's retention updates in place; its streams survive."""suffix = _uuid_mod.uuid4().hex[:8]