Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
Fix RDS subnet and security group fidelity (#1289)
Ian Oberst2b04810b2445
diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex c4fb8aa9..0ec71776 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -3851,6 +3851,35 @@ def _describe_db_snapshots(p):# Subnet Groups# ---------------------------------------------------------------------------+_INVALID_SUBNET_MESSAGE = (+ "The requested subnet is invalid, or multiple subnets were requested that "+ "are not all in a common VPC."+)+++def _resolve_subnet_group_members(subnet_ids):+ from ministack.services import ec2++ ec2._ensure_defaults_initialized()+ subnets = []+ vpc_ids = set()+ for subnet_id in subnet_ids:+ subnet = ec2._subnets.get(subnet_id)+ if not subnet:+ return None+ vpc_ids.add(subnet["VpcId"])+ subnets.append({+ "SubnetIdentifier": subnet_id,+ "SubnetAvailabilityZone": {+ "Name": subnet.get("AvailabilityZone", f"{get_region()}a")+ },+ "SubnetOutpost": {},+ "SubnetStatus": "Active",+ })+ if len(vpc_ids) != 1:+ return None+ return subnets, vpc_ids.pop()+def _create_subnet_group(p):name = _p(p, "DBSubnetGroupName")if not name:@@ -3859,13 +3888,15 @@ def _create_subnet_group(p):subnet_ids = _parse_member_list(p, "SubnetIds")arn = f"arn:aws:rds:{get_region()}:{get_account_id()}:subgrp:{name}"- subnets = [{"SubnetIdentifier": sid, "SubnetAvailabilityZone": {"Name": f"{get_region()}a"},- "SubnetOutpost": {}, "SubnetStatus": "Active"} for sid in subnet_ids]+ resolved_subnets = _resolve_subnet_group_members(subnet_ids)+ if resolved_subnets is None:+ return _error("InvalidSubnet", _INVALID_SUBNET_MESSAGE, 400)+ subnets, vpc_id = resolved_subnets_subnet_groups[name] = {"DBSubnetGroupName": name,"DBSubnetGroupDescription": desc,- "VpcId": "vpc-00000000",+ "VpcId": vpc_id,"SubnetGroupStatus": "Complete","Subnets": subnets,"DBSubnetGroupArn": arn,@@ -4275,15 +4306,23 @@ def _modify_subnet_group(p):if not sg:return _error("DBSubnetGroupNotFoundFault", f"Subnet group {name} not found.", 404)- if _p(p, "DBSubnetGroupDescription"):- sg["DBSubnetGroupDescription"] = _p(p, "DBSubnetGroupDescription")-subnet_ids = _parse_member_list(p, "SubnetIds")+ resolved_subnets = Noneif subnet_ids:- sg["Subnets"] = [- {"SubnetIdentifier": sid, "SubnetAvailabilityZone": {"Name": f"{get_region()}a"},- "SubnetOutpost": {}, "SubnetStatus": "Active"} for sid in subnet_ids- ]+ resolved_subnets = _resolve_subnet_group_members(subnet_ids)+ if resolved_subnets is None:+ return _error("InvalidSubnet", _INVALID_SUBNET_MESSAGE, 400)+ stored_vpc_id = sg.get("VpcId")+ if (+ stored_vpc_id not in (None, "", "vpc-00000000")+ and resolved_subnets[1] != stored_vpc_id+ ):+ return _error("InvalidSubnet", _INVALID_SUBNET_MESSAGE, 400)++ if _p(p, "DBSubnetGroupDescription"):+ sg["DBSubnetGroupDescription"] = _p(p, "DBSubnetGroupDescription")+ if resolved_subnets is not None:+ sg["Subnets"], sg["VpcId"] = resolved_subnetsreturn _xml(200, "ModifyDBSubnetGroupResponse",f"<ModifyDBSubnetGroupResult><DBSubnetGroup>{_subnet_group_xml(sg)}</DBSubnetGroup></ModifyDBSubnetGroupResult>")diff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.pyindex 0da30942..f0e27348 100644--- a/ministack/services/stepfunctions.py+++ b/ministack/services/stepfunctions.py@@ -4632,6 +4632,15 @@ _AWS_ACRONYMS = frozenset({# Some global-cluster operations are AWS-shape exceptions: their members use# "DbClusterIdentifier", and sending "DBClusterIdentifier" is ignored._QUERY_PARAM_NAME_OVERRIDES = {+ # RDS spells this request member with mixed-case ``Vpc``. The shared+ # acronym converter expands it to ``VPCSecurityGroupIds``, which the RDS+ # handler correctly ignores as an unknown Query API parameter.+ ("rds", "CreateDBCluster"): {+ "VpcSecurityGroupIds": "VpcSecurityGroupIds",+ },+ ("rds", "ModifyDBCluster"): {+ "VpcSecurityGroupIds": "VpcSecurityGroupIds",+ },("rds", "RemoveFromGlobalCluster"): {"DbClusterIdentifier": "DbClusterIdentifier",},diff --git a/tests/test_rds.py b/tests/test_rds.pyindex b0ce194a..a955c806 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -429,19 +429,223 @@ def test_rds_start_stop_cluster(rds):resp2 = rds.describe_db_clusters(DBClusterIdentifier="ss-cl")assert resp2["DBClusters"][0]["Status"] == "available"-def test_rds_modify_subnet_group(rds):+def test_rds_subnet_group_reflects_ec2_vpc_and_azs(rds, ec2):+ vpc_id = ec2.create_vpc(CidrBlock="10.70.0.0/16")["Vpc"]["VpcId"]+ subnet_a = ec2.create_subnet(+ VpcId=vpc_id,+ CidrBlock="10.70.1.0/24",+ AvailabilityZone="us-east-1a",+ )["Subnet"]["SubnetId"]+ subnet_b = ec2.create_subnet(+ VpcId=vpc_id,+ CidrBlock="10.70.2.0/24",+ AvailabilityZone="us-east-1b",+ )["Subnet"]["SubnetId"]++ response = rds.create_db_subnet_group(+ DBSubnetGroupName="test-reflected-sg",+ DBSubnetGroupDescription="Test reflected SG",+ SubnetIds=[subnet_a, subnet_b],+ )["DBSubnetGroup"]++ assert response["VpcId"] == vpc_id+ assert {+ (subnet["SubnetIdentifier"], subnet["SubnetAvailabilityZone"]["Name"])+ for subnet in response["Subnets"]+ } == {(subnet_a, "us-east-1a"), (subnet_b, "us-east-1b")}+++def test_rds_subnet_group_initializes_default_subnets_in_request_scope():+ scoped_rds = _regional_rds("ap-southeast-5", "333333333333")++ group = scoped_rds.create_db_subnet_group(+ DBSubnetGroupName="test-scoped-default-sg",+ DBSubnetGroupDescription="Scoped default subnets",+ SubnetIds=["subnet-00000001", "subnet-00000002"],+ )["DBSubnetGroup"]++ assert group["VpcId"] == "vpc-00000001"+ assert {+ (subnet["SubnetIdentifier"], subnet["SubnetAvailabilityZone"]["Name"])+ for subnet in group["Subnets"]+ } == {+ ("subnet-00000001", "ap-southeast-5a"),+ ("subnet-00000002", "ap-southeast-5b"),+ }++[email protected]("failure", ["missing", "mixed-vpc"])+def test_rds_create_subnet_group_rejects_invalid_subnets(rds, ec2, failure):+ vpc_a = ec2.create_vpc(CidrBlock="10.71.0.0/16")["Vpc"]["VpcId"]+ subnet_a = ec2.create_subnet(+ VpcId=vpc_a,+ CidrBlock="10.71.1.0/24",+ AvailabilityZone="us-east-1a",+ )["Subnet"]["SubnetId"]+ subnet_ids = [subnet_a, "subnet-does-not-exist"]+ if failure == "mixed-vpc":+ vpc_b = ec2.create_vpc(CidrBlock="10.72.0.0/16")["Vpc"]["VpcId"]+ subnet_b = ec2.create_subnet(+ VpcId=vpc_b,+ CidrBlock="10.72.1.0/24",+ AvailabilityZone="us-east-1b",+ )["Subnet"]["SubnetId"]+ subnet_ids = [subnet_a, subnet_b]++ with pytest.raises(ClientError) as exc_info:+ rds.create_db_subnet_group(+ DBSubnetGroupName=f"test-invalid-{failure}",+ DBSubnetGroupDescription="Test invalid subnet group",+ SubnetIds=subnet_ids,+ )++ assert exc_info.value.response["Error"]["Code"] == "InvalidSubnet"+ assert exc_info.value.response["Error"]["Message"] == (+ "The requested subnet is invalid, or multiple subnets were requested that "+ "are not all in a common VPC."+ )+++def test_rds_modify_subnet_group(rds, ec2):+ vpc_id = ec2.create_vpc(CidrBlock="10.73.0.0/16")["Vpc"]["VpcId"]+ subnets = [+ ec2.create_subnet(+ VpcId=vpc_id,+ CidrBlock=f"10.73.{index}.0/24",+ AvailabilityZone=f"us-east-1{az}",+ )["Subnet"]["SubnetId"]+ for index, az in ((1, "a"), (2, "b"), (3, "c"))+ ]rds.create_db_subnet_group(DBSubnetGroupName="test-mod-sg",DBSubnetGroupDescription="Test SG",- SubnetIds=["subnet-111"],+ SubnetIds=subnets[:2],)rds.modify_db_subnet_group(DBSubnetGroupName="test-mod-sg",DBSubnetGroupDescription="Updated SG",- SubnetIds=["subnet-222", "subnet-333"],+ SubnetIds=subnets[1:],)resp = rds.describe_db_subnet_groups(DBSubnetGroupName="test-mod-sg")- assert resp["DBSubnetGroups"][0]["DBSubnetGroupDescription"] == "Updated SG"+ group = resp["DBSubnetGroups"][0]+ assert group["DBSubnetGroupDescription"] == "Updated SG"+ assert group["VpcId"] == vpc_id+ assert [subnet["SubnetIdentifier"] for subnet in group["Subnets"]] == subnets[1:]++ with pytest.raises(ClientError) as exc_info:+ rds.modify_db_subnet_group(+ DBSubnetGroupName="test-mod-sg",+ DBSubnetGroupDescription="Should not persist",+ SubnetIds=[subnets[0], "subnet-does-not-exist"],+ )+ assert exc_info.value.response["Error"]["Code"] == "InvalidSubnet"++ unchanged = rds.describe_db_subnet_groups(DBSubnetGroupName="test-mod-sg")+ assert unchanged["DBSubnetGroups"][0]["DBSubnetGroupDescription"] == "Updated SG"++ before_cross_vpc = unchanged["DBSubnetGroups"][0]+ other_vpc_id = ec2.create_vpc(CidrBlock="10.74.0.0/16")["Vpc"]["VpcId"]+ other_subnets = [+ ec2.create_subnet(+ VpcId=other_vpc_id,+ CidrBlock=f"10.74.{index}.0/24",+ AvailabilityZone=f"us-east-1{az}",+ )["Subnet"]["SubnetId"]+ for index, az in ((1, "a"), (2, "b"))+ ]+ with pytest.raises(ClientError) as exc_info:+ rds.modify_db_subnet_group(+ DBSubnetGroupName="test-mod-sg",+ DBSubnetGroupDescription="Cross-VPC description must not persist",+ SubnetIds=other_subnets,+ )+ assert exc_info.value.response["Error"]["Code"] == "InvalidSubnet"+ assert exc_info.value.response["Error"]["Message"] == (+ "The requested subnet is invalid, or multiple subnets were requested that "+ "are not all in a common VPC."+ )++ after_cross_vpc = rds.describe_db_subnet_groups(+ DBSubnetGroupName="test-mod-sg"+ )["DBSubnetGroups"][0]+ assert after_cross_vpc == before_cross_vpc+++def test_rds_modify_legacy_subnet_group_adopts_resolved_vpc():+ from copy import deepcopy++ from ministack.services import ec2 as ec2_service+ from ministack.services import rds as rds_service++ group_name = "test-legacy-mod-sg"+ vpc_id = "vpc-test-legacy"+ subnet_ids = [+ "subnet-test-legacy-a",+ "subnet-test-legacy-b",+ "subnet-test-legacy-c",+ ]+ other_vpc_id = "vpc-test-legacy-other"+ other_subnet_ids = [+ "subnet-test-legacy-other-a",+ "subnet-test-legacy-other-b",+ ]+ all_subnet_ids = subnet_ids + other_subnet_ids++ for index, subnet_id in enumerate(subnet_ids):+ ec2_service._subnets[subnet_id] = {+ "SubnetId": subnet_id,+ "VpcId": vpc_id,+ "AvailabilityZone": f"us-east-1{chr(ord('a') + index)}",+ }+ for index, subnet_id in enumerate(other_subnet_ids):+ ec2_service._subnets[subnet_id] = {+ "SubnetId": subnet_id,+ "VpcId": other_vpc_id,+ "AvailabilityZone": f"us-east-1{chr(ord('a') + index)}",+ }++ try:+ status, _, _ = rds_service._create_subnet_group({+ "DBSubnetGroupName": group_name,+ "DBSubnetGroupDescription": "Legacy SG",+ "SubnetIds.member.1": subnet_ids[0],+ "SubnetIds.member.2": subnet_ids[1],+ })+ assert status == 200++ persisted_groups = rds_service.get_state()["subnet_groups"]+ persisted_groups[group_name]["VpcId"] = "vpc-00000000"+ rds_service._subnet_groups.clear()+ rds_service.restore_state({"subnet_groups": persisted_groups})++ status, _, _ = rds_service._modify_subnet_group({+ "DBSubnetGroupName": group_name,+ "DBSubnetGroupDescription": "Upgraded SG",+ "SubnetIds.member.1": subnet_ids[1],+ "SubnetIds.member.2": subnet_ids[2],+ })+ assert status == 200+ upgraded = rds_service._subnet_groups[group_name]+ assert upgraded["VpcId"] == vpc_id+ assert [+ subnet["SubnetIdentifier"] for subnet in upgraded["Subnets"]+ ] == subnet_ids[1:]++ before_cross_vpc = deepcopy(upgraded)+ status, _, body = rds_service._modify_subnet_group({+ "DBSubnetGroupName": group_name,+ "DBSubnetGroupDescription": "Cross-VPC description must not persist",+ "SubnetIds.member.1": other_subnet_ids[0],+ "SubnetIds.member.2": other_subnet_ids[1],+ })+ assert status == 400+ assert b"<Code>InvalidSubnet</Code>" in body+ assert rds_service._subnet_groups[group_name] == before_cross_vpc+ finally:+ rds_service._subnet_groups.pop(group_name, None)+ for subnet_id in all_subnet_ids:+ ec2_service._subnets.pop(subnet_id, None)+def test_rds_snapshot_crud(rds):"""CreateDBSnapshot / DescribeDBSnapshots / DeleteDBSnapshot."""diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.pyindex 55ab6331..cdc3f44f 100644--- a/tests/test_stepfunctions.py+++ b/tests/test_stepfunctions.py@@ -4783,11 +4783,30 @@ def test_sfn_aws_sdk_json_pascal_case(sfn, sfn_sync, sm):sm.delete_secret(SecretId="sfn-pascal-json-secret", ForceDeleteWithoutRecovery=True)-def test_sfn_aws_sdk_query_acronym_param_mapping(sfn, sfn_sync, rds):+def test_sfn_aws_sdk_query_acronym_param_mapping(sfn, sfn_sync, rds, ec2):"""SFN aws-sdk query dispatch maps SDK-style param names to wire-format names."""import uuid as _uuid++ def normalize_vpc_security_groups(value):+ # The converter currently collapses singleton wrappers to a dict; keep+ # this assertion compatible with a future list-fidelity fix.+ if isinstance(value, list):+ return value+ return [value["VpcSecurityGroupMembership"]]+cluster_id = f"acronym-test-{_uuid.uuid4().hex[:8]}"sm_name = f"sdk-acronym-{_uuid.uuid4().hex[:8]}"+ vpc_id = ec2.create_vpc(CidrBlock="10.98.0.0/16")["Vpc"]["VpcId"]+ create_sg_id = ec2.create_security_group(+ GroupName=f"{sm_name}-create",+ Description="SFN RDS create cluster security group",+ VpcId=vpc_id,+ )["GroupId"]+ modify_sg_id = ec2.create_security_group(+ GroupName=f"{sm_name}-modify",+ Description="SFN RDS modify cluster security group",+ VpcId=vpc_id,+ )["GroupId"]definition = json.dumps({"StartAt": "CreateCluster",@@ -4800,8 +4819,19 @@ def test_sfn_aws_sdk_query_acronym_param_mapping(sfn, sfn_sync, rds):"Engine": "aurora-postgresql","MasterUsername": "admin","MasterUserPassword": "testpass123",+ "VpcSecurityGroupIds": [create_sg_id],},"ResultPath": "$.createResult",+ "Next": "ModifyCluster",+ },+ "ModifyCluster": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:rds:modifyDBCluster",+ "Parameters": {+ "DbClusterIdentifier": cluster_id,+ "VpcSecurityGroupIds": [modify_sg_id],+ },+ "ResultPath": "$.modifyResult","Next": "DescribeClusters",},"DescribeClusters": {@@ -4822,15 +4852,42 @@ def test_sfn_aws_sdk_query_acronym_param_mapping(sfn, sfn_sync, rds):roleArn="arn:aws:iam::000000000000:role/sfn-role",)["stateMachineArn"]- resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))- assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"- output = json.loads(resp["output"])-- create_cluster = output["createResult"]["DbCluster"]- assert create_cluster["DbClusterIdentifier"] == cluster_id- assert create_cluster["Engine"] == "aurora-postgresql"+ try:+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"+ output = json.loads(resp["output"])- sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+ create_cluster = output["createResult"]["DbCluster"]+ assert create_cluster["DbClusterIdentifier"] == cluster_id+ assert create_cluster["Engine"] == "aurora-postgresql"+ assert normalize_vpc_security_groups(create_cluster["VpcSecurityGroups"]) == [+ {+ "VpcSecurityGroupId": create_sg_id,+ "Status": "active",+ },+ ]+ assert normalize_vpc_security_groups(+ output["modifyResult"]["DbCluster"]["VpcSecurityGroups"],+ ) == [+ {+ "VpcSecurityGroupId": modify_sg_id,+ "Status": "active",+ },+ ]+ assert normalize_vpc_security_groups(+ output["describeResult"]["DbClusters"][0]["VpcSecurityGroups"],+ ) == [+ {+ "VpcSecurityGroupId": modify_sg_id,+ "Status": "active",+ },+ ]+ finally:+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+ rds.delete_db_cluster(DBClusterIdentifier=cluster_id, SkipFinalSnapshot=True)+ ec2.delete_security_group(GroupId=create_sg_id)+ ec2.delete_security_group(GroupId=modify_sg_id)+ ec2.delete_vpc(VpcId=vpc_id)def test_sfn_key_to_api_name_must_convert():