Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix: six fields accepted on write but never reported on read (#1558)
Jacob Schooley554b83806231
diff --git a/ministack/app.py b/ministack/app.pyindex fced9f34..fce27fd4 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -1194,7 +1194,11 @@ async def _handle_s3_control_request(path: str, method: str, body: bytes, query_if method == "GET":tags = _get_module("s3")._bucket_tags.get(bucket_name, {})- tag_members = "".join(f"<member><Key>{k}</Key><Value>{v}</Value></member>" for k, v in tags.items())+ # s3control's TagList declares locationName "Tag", so each entry is+ # <Tag>, not the <member> the query protocol uses elsewhere. boto3+ # tolerates <member>; aws-sdk-go-v2 — and therefore Terraform —+ # parses an empty list from it and reports the resource as untagged.+ tag_members = "".join(f"<Tag><Key>{k}</Key><Value>{v}</Value></Tag>" for k, v in tags.items())xml_body = ('<?xml version="1.0" encoding="UTF-8"?>''<ListTagsForResourceResult xmlns="https://awss3control.amazonaws.com/doc/2018-08-20/">'diff --git a/ministack/services/cognito.py b/ministack/services/cognito.pyindex 16fc74bc..5365dc58 100644--- a/ministack/services/cognito.py+++ b/ministack/services/cognito.py@@ -2207,6 +2207,8 @@ def _create_user_pool(data):pool["UserPoolAddOns"] = data["UserPoolAddOns"]if data.get("VerificationMessageTemplate"):pool["VerificationMessageTemplate"] = data["VerificationMessageTemplate"]+ if data.get("UserAttributeUpdateSettings"):+ pool["UserAttributeUpdateSettings"] = data["UserAttributeUpdateSettings"]_user_pools[pid] = poollogger.info("Cognito: CreateUserPool %s (%s)", name, pid)return json_response({"UserPool": _pool_out(pool)})@@ -2262,6 +2264,7 @@ def _update_user_pool(data):"EmailConfiguration", "SmsConfiguration", "UserPoolTags","AdminCreateUserConfig", "UserPoolAddOns", "VerificationMessageTemplate","AccountRecoverySetting", "LambdaConfig",+ "UserAttributeUpdateSettings",}for k in updatable:if k in data:@@ -4707,8 +4710,18 @@ def _get_user_pool_mfa_config(data):resp = {"MfaConfiguration": pool.get("MfaConfiguration", "OFF")}for key in ("SmsMfaConfiguration", "SoftwareTokenMfaConfiguration","EmailMfaConfiguration", "WebAuthnConfiguration"):- if pool.get(key):- resp[key] = pool[key]+ # `is not None`, not truthiness: a caller turning software-token MFA+ # off sends an empty object — `enabled = false` serialises with the+ # false field omitted — and that is a value it set, not a field it left+ # unset. Dropping it makes the caller re-send the change on every run.+ value = pool.get(key)+ if value is None:+ continue+ if key == "SoftwareTokenMfaConfiguration":+ value = {"Enabled": bool(value.get("Enabled", False))}+ elif not value:+ continue+ resp[key] = valuereturn json_response(resp)@@ -4784,8 +4797,18 @@ def _set_user_pool_mfa_config(data):resp = {"MfaConfiguration": pool.get("MfaConfiguration", "OFF")}for key in ("SmsMfaConfiguration", "SoftwareTokenMfaConfiguration","EmailMfaConfiguration", "WebAuthnConfiguration"):- if pool.get(key):- resp[key] = pool[key]+ # `is not None`, not truthiness: a caller turning software-token MFA+ # off sends an empty object — `enabled = false` serialises with the+ # false field omitted — and that is a value it set, not a field it left+ # unset. Dropping it makes the caller re-send the change on every run.+ value = pool.get(key)+ if value is None:+ continue+ if key == "SoftwareTokenMfaConfiguration":+ value = {"Enabled": bool(value.get("Enabled", False))}+ elif not value:+ continue+ resp[key] = valuereturn json_response(resp)diff --git a/ministack/services/kinesis.py b/ministack/services/kinesis.pyindex 1dbd3a98..ba367a3d 100644--- a/ministack/services/kinesis.py+++ b/ministack/services/kinesis.py@@ -499,6 +499,7 @@ def _describe_stream_summary(data):"StreamCreationTimestamp": stream["CreationTimestamp"],"EnhancedMonitoring": [{"ShardLevelMetrics": []}],"EncryptionType": stream.get("EncryptionType", "NONE"),+ **({"KeyId": stream["KeyId"]} if stream.get("KeyId") else {}),"OpenShardCount": len(stream["shards"]),"ConsumerCount": consumer_count,}})@@ -1218,6 +1219,10 @@ def _stream_desc(stream, shard_ids=None):"HasMoreShards": False,"EnhancedMonitoring": [{"ShardLevelMetrics": []}],"EncryptionType": stream.get("EncryptionType", "NONE"),+ # KeyId accompanies EncryptionType on AWS, and is omitted entirely when+ # the stream is not encrypted. Storing it without echoing it here leaves+ # kms_key_id reading empty on an encrypted stream.+ **({"KeyId": stream["KeyId"]} if stream.get("KeyId") else {}),}diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.pyindex d0412a13..75a715bb 100644--- a/ministack/services/lambda_svc.py+++ b/ministack/services/lambda_svc.py@@ -1078,6 +1078,13 @@ def _layer_arn(name: str) -> str:return f"arn:aws:lambda:{get_region()}:{get_account_id()}:layer:{name}"+def _esm_arn(esm_id: str) -> str:+ return (+ f"arn:aws:lambda:{get_region()}:{get_account_id()}"+ f":event-source-mapping:{esm_id}"+ )++def _now_iso() -> str:now = datetime.now(timezone.utc)ms = now.microsecond // 1000@@ -6011,8 +6018,18 @@ def _delete_provisioned_concurrency(func_name: str, qualifier: str):def _esm_response(esm: dict) -> dict:- """Return ESM dict without internal-only fields."""- return {k: v for k, v in esm.items() if k not in ("FunctionName", "Enabled")}+ """Return ESM dict without internal-only fields.++ EventSourceMappingArn is derived here rather than stored on the record so+ that mappings created before it existed — restored state included — report+ it too. Callers cannot construct this ARN themselves, and ListTags is keyed+ on it, so omitting it leaves tags unreadable.+ """+ out = {k: v for k, v in esm.items() if k not in ("FunctionName", "Enabled")}+ esm_id = esm.get("UUID")+ if esm_id:+ out["EventSourceMappingArn"] = _esm_arn(esm_id)+ return outdef _resolve_existing_esm_function(function_ref: str):diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex a553a087..66f976e9 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -3827,7 +3827,9 @@ def _create_db_instance_impl(p):"IAMDatabaseAuthenticationEnabled": _p(p, "EnableIAMDatabaseAuthentication") == "true","PerformanceInsightsEnabled": _p(p, "EnablePerformanceInsights") == "true","PerformanceInsightsKMSKeyId": "",- "PerformanceInsightsRetentionPeriod": 7,+ "PerformanceInsightsRetentionPeriod": int(+ _p(p, "PerformanceInsightsRetentionPeriod") or "7"+ ),"EnabledCloudwatchLogsExports": [],"ProcessorFeatures": [],"DeletionProtection": _p(p, "DeletionProtection") == "true",@@ -4511,6 +4513,22 @@ def _restore_from_snapshot(p):# DB Clusters# ---------------------------------------------------------------------------+def _serverlessv2_from_params(p):+ """Read ServerlessV2ScalingConfiguration.{Min,Max}Capacity from query params.++ Returned as a dict only when at least one bound was supplied — an Aurora+ cluster that is not Serverless v2 has no such configuration, and echoing an+ empty one would read as drift the other way.+ """+ prefix = "ServerlessV2ScalingConfiguration."+ out = {}+ for key, field in (("MinCapacity", "MinCapacity"), ("MaxCapacity", "MaxCapacity")):+ raw = _p(p, f"{prefix}{key}")+ if raw not in (None, ""):+ out[field] = float(raw)+ return out or None++def _create_db_cluster(p):cluster_id = _p(p, "DBClusterIdentifier")if not cluster_id:@@ -4682,6 +4700,7 @@ def _create_db_cluster(p):"Engine": engine,"EngineVersion": engine_version,"EngineMode": _p(p, "EngineMode") or "provisioned",+ "ServerlessV2ScalingConfiguration": _serverlessv2_from_params(p),"Status": "available","MasterUsername": master_user,"_MasterUserPassword": master_pass,@@ -5072,6 +5091,12 @@ def _modify_db_cluster(p):cluster["Port"] = int(_p(p, "Port"))if _p(p, "BackupRetentionPeriod"):cluster["BackupRetentionPeriod"] = int(_p(p, "BackupRetentionPeriod"))+ # Without this a cluster created before the field was stored can never be+ # brought into line: the plan proposes the block, the apply drops it, and+ # the next plan proposes it again.+ serverlessv2 = _serverlessv2_from_params(p)+ if serverlessv2:+ cluster["ServerlessV2ScalingConfiguration"] = serverlessv2if _p(p, "PreferredBackupWindow"):cluster["PreferredBackupWindow"] = _p(p, "PreferredBackupWindow")if _p(p, "PreferredMaintenanceWindow"):@@ -7035,6 +7060,7 @@ def _instance_xml(i):<DBInstanceArn>{i['DBInstanceArn']}</DBInstanceArn><IAMDatabaseAuthenticationEnabled>{str(i.get('IAMDatabaseAuthenticationEnabled',False)).lower()}</IAMDatabaseAuthenticationEnabled><PerformanceInsightsEnabled>{str(i.get('PerformanceInsightsEnabled',False)).lower()}</PerformanceInsightsEnabled>+ <PerformanceInsightsRetentionPeriod>{i.get('PerformanceInsightsRetentionPeriod',7)}</PerformanceInsightsRetentionPeriod><EnabledCloudwatchLogsExports/><ProcessorFeatures/><DeletionProtection>{str(i.get('DeletionProtection',False)).lower()}</DeletionProtection>@@ -7049,6 +7075,18 @@ def _instance_xml(i):<IsStorageConfigUpgradeAvailable>{str(i.get('IsStorageConfigUpgradeAvailable',False)).lower()}</IsStorageConfigUpgradeAvailable>"""+def _serverlessv2_xml(c):+ cfg = c.get("ServerlessV2ScalingConfiguration")+ if not cfg:+ return ""+ return (+ "<ServerlessV2ScalingConfiguration>"+ f"<MinCapacity>{cfg.get('MinCapacity', 0.5)}</MinCapacity>"+ f"<MaxCapacity>{cfg.get('MaxCapacity', 1.0)}</MaxCapacity>"+ "</ServerlessV2ScalingConfiguration>"+ )++def _cluster_xml(c):"""Render a cluster dict to XML fields."""vpc_sg_xml = ""@@ -7110,6 +7148,7 @@ def _cluster_xml(c):<Engine>{c['Engine']}</Engine><EngineVersion>{c['EngineVersion']}</EngineVersion><EngineMode>{c.get('EngineMode','provisioned')}</EngineMode>+ {_serverlessv2_xml(c)}<Status>{c['Status']}</Status><MasterUsername>{c.get('MasterUsername','admin')}</MasterUsername>{db_name_xml}diff --git a/tests/test_cognito.py b/tests/test_cognito.pyindex 037b063c..1cb6d91f 100644--- a/tests/test_cognito.py+++ b/tests/test_cognito.py@@ -7794,3 +7794,62 @@ def test_cognito_required_custom_attribute_is_refused(cognito_idp):CustomAttributes=[{"Name": "tenant", "AttributeDataType": "String","Required": True}])assert exc.value.response["Error"]["Code"] == "InvalidParameterException"+++def test_cognito_user_pool_keeps_mfa_and_attribute_update_settings(cognito_idp):+ """UserAttributeUpdateSettings and a false SoftwareTokenMfaConfiguration+ round-trip.++ Both are declared on aws_cognito_user_pool. The provider sends the attribute+ settings on Create and Update, and the MFA config through+ SetUserPoolMfaConfig as an empty object — `enabled = false` serialises with+ the false field omitted. Neither was reported back: the settings were never+ stored, and `{}` was treated as never-set. Terraform proposed the same two+ blocks on every plan, and applying did not settle it.+ """+ pool = cognito_idp.create_user_pool(+ PoolName="qa-pool-settings",+ MfaConfiguration="OPTIONAL",+ UserAttributeUpdateSettings={+ "AttributesRequireVerificationBeforeUpdate": ["email", "phone_number"]+ },+ )["UserPool"]+ pid = pool["Id"]++ assert pool["UserAttributeUpdateSettings"][+ "AttributesRequireVerificationBeforeUpdate"+ ] == ["email", "phone_number"]+ described = cognito_idp.describe_user_pool(UserPoolId=pid)["UserPool"]+ assert described["UserAttributeUpdateSettings"][+ "AttributesRequireVerificationBeforeUpdate"+ ] == ["email", "phone_number"]++ # An update must carry the setting through, not drop it.+ cognito_idp.update_user_pool(+ UserPoolId=pid,+ MfaConfiguration="OPTIONAL",+ UserAttributeUpdateSettings={+ "AttributesRequireVerificationBeforeUpdate": ["email"]+ },+ )+ described = cognito_idp.describe_user_pool(UserPoolId=pid)["UserPool"]+ assert described["UserAttributeUpdateSettings"][+ "AttributesRequireVerificationBeforeUpdate"+ ] == ["email"]++ # The empty object the provider sends for `enabled = false` is a value that+ # was set, not an unset field, so it has to be reported as {"Enabled": false}.+ cognito_idp.set_user_pool_mfa_config(+ UserPoolId=pid,+ SoftwareTokenMfaConfiguration={},+ MfaConfiguration="OPTIONAL",+ )+ mfa = cognito_idp.get_user_pool_mfa_config(UserPoolId=pid)+ assert mfa["SoftwareTokenMfaConfiguration"] == {"Enabled": False}++ # A pool that never configured software-token MFA still reports nothing,+ # which is what keeps an unset field from reading as drift.+ other = cognito_idp.create_user_pool(PoolName="qa-pool-no-mfa")["UserPool"]["Id"]+ assert "SoftwareTokenMfaConfiguration" not in cognito_idp.get_user_pool_mfa_config(+ UserPoolId=other+ )diff --git a/tests/test_kinesis.py b/tests/test_kinesis.pyindex 7c15fc5d..100a6f4d 100644--- a/tests/test_kinesis.py+++ b/tests/test_kinesis.py@@ -573,3 +573,38 @@ def test_kinesis_cbor_put_record(kin):)records = kin.get_records(ShardIterator=it["ShardIterator"])assert len(records["Records"]) == 1+++def test_kinesis_describe_echoes_the_encryption_key_id(kin):+ """StartStreamEncryption stores KeyId; the describe paths must return it.++ AWS reports KeyId alongside EncryptionType in both StreamDescription and+ StreamDescriptionSummary. Storing it without echoing it means the Terraform+ provider reads kms_key_id as empty on a stream that is in fact encrypted, so+ every plan proposes setting it again.+ """+ name = "qa-kinesis-keyid"+ kin.create_stream(StreamName=name, ShardCount=1)+ key = "arn:aws:kms:us-east-1:000000000000:alias/aws/kinesis"+ kin.start_stream_encryption(+ StreamName=name, EncryptionType="KMS", KeyId=key+ )++ desc = kin.describe_stream(StreamName=name)["StreamDescription"]+ assert desc["EncryptionType"] == "KMS"+ assert desc.get("KeyId") == key++ summary = kin.describe_stream_summary(StreamName=name)[+ "StreamDescriptionSummary"+ ]+ assert summary["EncryptionType"] == "KMS"+ assert summary.get("KeyId") == key++ # An unencrypted stream must not report a KeyId at all — AWS omits the field+ # rather than returning an empty one, and an empty string would read as drift.+ kin.stop_stream_encryption(+ StreamName=name, EncryptionType="KMS", KeyId=key+ )+ desc = kin.describe_stream(StreamName=name)["StreamDescription"]+ assert desc["EncryptionType"] == "NONE"+ assert "KeyId" not in descdiff --git a/tests/test_lambda.py b/tests/test_lambda.pyindex ea67c87b..9ff8696b 100644--- a/tests/test_lambda.py+++ b/tests/test_lambda.py@@ -10688,3 +10688,50 @@ def test_lambda_invoke_with_response_stream_missing_function_is_plain_error(lam)with pytest.raises(ClientError) as exc:lam.invoke_with_response_stream(FunctionName="stream-does-not-exist")assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"+++def test_lambda_event_source_mapping_response_carries_its_arn(lam, sqs):+ """CreateEventSourceMapping / Get / List must return EventSourceMappingArn.++ ListTags already works when given the ARN, but a caller has no way to build+ that ARN itself — the Terraform provider reads EventSourceMappingArn off the+ ESM and calls ListTags with it. With the field absent the provider reads no+ tags at all, so `tags_all` comes back empty and every plan shows a tag diff+ on every mapping, forever.+ """+ code = _zip_lambda("def handler(e,c): return 'ok'")+ fn = "qa-esm-arn-fn"+ lam.create_function(+ FunctionName=fn,+ Runtime="python3.12",+ Role="arn:aws:iam::000000000000:role/r",+ Handler="index.handler",+ Code={"ZipFile": code},+ )+ q = sqs.create_queue(QueueName="qa-esm-arn-queue")+ q_arn = sqs.get_queue_attributes(+ QueueUrl=q["QueueUrl"], AttributeNames=["QueueArn"]+ )["Attributes"]["QueueArn"]++ created = lam.create_event_source_mapping(+ FunctionName=fn, EventSourceArn=q_arn, Tags={"Team": "billing"}+ )+ uuid = created["UUID"]+ expected = f"arn:aws:lambda:us-east-1:000000000000:event-source-mapping:{uuid}"++ assert created.get("EventSourceMappingArn") == expected++ got = lam.get_event_source_mapping(UUID=uuid)+ assert got.get("EventSourceMappingArn") == expected++ listed = [+ m for m in lam.list_event_source_mappings(FunctionName=fn)["EventSourceMappings"]+ if m["UUID"] == uuid+ ]+ assert listed and listed[0].get("EventSourceMappingArn") == expected++ # The ARN the response hands back must be the one ListTags accepts, which is+ # the whole point of returning it.+ assert lam.list_tags(Resource=created["EventSourceMappingArn"])["Tags"] == {+ "Team": "billing"+ }diff --git a/tests/test_rds.py b/tests/test_rds.pyindex 61c781fb..a1a700b1 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -12918,3 +12918,77 @@ def test_rds_db_proxy_target_health_follows_the_backing_instance(rds.deregister_db_proxy_targets(DBProxyName="health-proxy", DBInstanceIdentifiers=["health-pg"],)+++def test_rds_cluster_echoes_serverlessv2_scaling_configuration(rds):+ """ServerlessV2ScalingConfiguration must round-trip on a cluster.++ Terraform's serverlessv2_scaling_configuration block reads back empty when+ the field is dropped, so every plan proposes adding it again.+ """+ cid = "qa-slv2-cluster"+ rds.create_db_cluster(+ DBClusterIdentifier=cid,+ Engine="aurora-postgresql",+ MasterUsername="admin",+ MasterUserPassword="password123",+ ServerlessV2ScalingConfiguration={"MinCapacity": 0.5, "MaxCapacity": 1.0},+ )+ c = rds.describe_db_clusters(DBClusterIdentifier=cid)["DBClusters"][0]+ slv2 = c.get("ServerlessV2ScalingConfiguration")+ assert slv2, "ServerlessV2ScalingConfiguration missing from DescribeDBClusters"+ assert slv2["MinCapacity"] == 0.5+ assert slv2["MaxCapacity"] == 1.0+++def test_rds_instance_echoes_performance_insights_retention(rds):+ """PerformanceInsightsRetentionPeriod must reflect what was requested.++ The value was hardcoded to 7 on the record and then never serialised, so the+ provider read 0 and planned the same change on every run.+ """+ iid = "qa-pi-retention"+ rds.create_db_instance(+ DBInstanceIdentifier=iid,+ DBInstanceClass="db.t3.medium",+ Engine="postgres",+ MasterUsername="admin",+ MasterUserPassword="password123",+ AllocatedStorage=20,+ EnablePerformanceInsights=True,+ PerformanceInsightsRetentionPeriod=31,+ )+ i = rds.describe_db_instances(DBInstanceIdentifier=iid)["DBInstances"][0]+ assert i["PerformanceInsightsEnabled"] is True+ assert i.get("PerformanceInsightsRetentionPeriod") == 31+++def test_rds_modify_cluster_sets_serverlessv2_scaling_configuration(rds):+ """ModifyDBCluster must accept ServerlessV2ScalingConfiguration too.++ A cluster created without it — or before it was stored at all — is brought+ into line by an update, not a create. If modify drops the field the plan+ proposes the block, the apply reports success, and the next plan proposes+ it again unchanged.+ """+ cid = "qa-slv2-modify"+ rds.create_db_cluster(+ DBClusterIdentifier=cid,+ Engine="aurora-postgresql",+ MasterUsername="admin",+ MasterUserPassword="password123",+ )+ assert not rds.describe_db_clusters(DBClusterIdentifier=cid)["DBClusters"][0].get(+ "ServerlessV2ScalingConfiguration"+ )++ rds.modify_db_cluster(+ DBClusterIdentifier=cid,+ ServerlessV2ScalingConfiguration={"MinCapacity": 2.0, "MaxCapacity": 8.0},+ ApplyImmediately=True,+ )+ slv2 = rds.describe_db_clusters(DBClusterIdentifier=cid)["DBClusters"][0][+ "ServerlessV2ScalingConfiguration"+ ]+ assert slv2["MinCapacity"] == 2.0+ assert slv2["MaxCapacity"] == 8.0diff --git a/tests/test_s3.py b/tests/test_s3.pyindex 3e47f466..4e769f1f 100644--- a/tests/test_s3.py+++ b/tests/test_s3.py@@ -6086,3 +6086,37 @@ def test_s3_replication_covers_copy_and_versioned_reads(s3):finally:_purge_versioned(s3, src)_purge_versioned(s3, dst)+++def test_s3_control_list_tags_uses_the_tag_element_aws_uses(s3):+ """ListTagsForResource must wrap each tag in <Tag>, not <member>.++ The two tests above assert through boto3, which is lenient about the element+ name and so passes either way. AWS's own model is explicit — s3control's+ TagList declares `locationName: "Tag"` — and aws-sdk-go-v2 honours it+ strictly: given <member> wrappers it parses an empty tag list and reports no+ tags at all. That is what the Terraform AWS provider uses, so a bucket's+ `tags_all` refreshes to empty and every plan shows the same tag diff, which+ survives being applied.++ #447 fixed the TagResource *write* path to read <Tag>; the read path was+ left emitting <member>. Asserting on the wire format is the only way to+ catch that from Python.+ """+ from urllib.parse import quote++ import requests++ bkt = "intg-s3control-tag-element"+ s3.create_bucket(Bucket=bkt)+ s3.put_bucket_tagging(+ Bucket=bkt, Tagging={"TagSet": [{"Key": "team", "Value": "platform"}]}+ )++ arn = quote(f"arn:aws:s3:::{bkt}", safe="")+ resp = requests.get(f"{ENDPOINT}/v20180820/tags/{arn}", timeout=10)+ assert resp.status_code == 200+ body = resp.text++ assert "<Tag><Key>team</Key><Value>platform</Value></Tag>" in body, body+ assert "<member>" not in body, body