Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
Fix Step Functions aws-sdk integrations for Query-protocol services (#1227)
Maxd1cb89f20486
diff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.pyindex be9bc9f4..724ee6d3 100644--- a/ministack/services/stepfunctions.py+++ b/ministack/services/stepfunctions.py@@ -4411,22 +4411,76 @@ def _flatten_query_params(data, prefix=""):return params-_EC2_QUERY_LIST_NAME_OVERRIDES = {- "Filters": "Filter",- "Values": "Value",- "GroupIds": "GroupId",- "GroupNames": "GroupName",- "TagSpecifications": "TagSpecification",- "Tags": "Tag",+# Wire names EC2 numbers list parameters under. Mostly the singular of the SDK+# member name (VolumeIds -> VolumeId.N), but sometimes the plural+# (IpPermissions.N) and sometimes a different word (Resources -> ResourceId.N).+# A wrong name is dropped silently — the call still succeeds and the parameter+# never applies — so the exceptions are listed rather than guessed. Generated+# from the botocore EC2 model (queryName, else capitalized locationName) for the+# actions ec2.py serves; see check_sfn_ec2_list_params.py.+_EC2_QUERY_LIST_NAMES = {+ "ExecutableUsers": "ExecutableBy",+ "Groups": "SecurityGroupId",+ "IpPermissions": "IpPermissions",+ "IpRanges": "IpRanges",+ "Ipv6Ranges": "Ipv6Ranges",+ "LaunchTemplateConfigs": "LaunchTemplateConfigs",+ "Overrides": "Overrides",+ "OwnerIds": "Owner",+ "Resources": "ResourceId",+ "RestorableByUserIds": "RestorableBy",+ "TunnelOptions": "TunnelOptions",+ "UserIdGroupPairs": "Groups",+ "Versions": "LaunchTemplateVersion",}+# Names spelled both ways across those actions, so they cannot go above. The+# enclosing list settles most — PrefixListIds is plural inside an IpPermission+# and singular in DescribePrefixLists — the rest are top-level, where only the+# action can.+_EC2_QUERY_LIST_NAMES_BY_PARENT = {+ ("IpPermissions", "PrefixListIds"): "PrefixListIds",+ ("NetworkInterfaces", "Ipv6Addresses"): "Ipv6Addresses",+ ("NetworkInterfaces", "PrivateIpAddresses"): "PrivateIpAddresses",+}++_EC2_QUERY_LIST_NAMES_BY_ACTION = {+ ("CreateNetworkInterface", "Ipv6Addresses"): "Ipv6Addresses",+ ("CreateNetworkInterface", "PrivateIpAddresses"): "PrivateIpAddresses",+ ("DescribeVpcClassicLinkDnsSupport", "VpcIds"): "VpcIds",+ ("ModifySnapshotAttribute", "GroupNames"): "UserGroup",+}+++def _ec2_query_list_name(key, action=None, parent=""):+ """Wire name EC2 numbers a list parameter under: VolumeIds -> VolumeId.N."""+ # Most specific context first: enclosing list, then action, then the names+ # that are the same everywhere.+ if (parent, key) in _EC2_QUERY_LIST_NAMES_BY_PARENT:+ return _EC2_QUERY_LIST_NAMES_BY_PARENT[(parent, key)]+ if (action, key) in _EC2_QUERY_LIST_NAMES_BY_ACTION:+ return _EC2_QUERY_LIST_NAMES_BY_ACTION[(action, key)]+ if key in _EC2_QUERY_LIST_NAMES:+ return _EC2_QUERY_LIST_NAMES[key]+ # Everything else is the singular of an English plural: Entries -> Entry,+ # Ipv4Prefixes -> Ipv4Prefix, VolumeIds -> VolumeId.+ if len(key) < 2 or not key.endswith("s") or key.endswith("ss"):+ return key+ if key.endswith("ies") and len(key) > 3:+ return key[:-3] + "y"+ if key.endswith(("ses", "xes", "zes", "ches", "shes")):+ return key[:-2]+ return key[:-1]+-def _flatten_ec2_query_params(data, prefix=""):+def _flatten_ec2_query_params(data, prefix="", action=None, parent=""):"""Flatten EC2 query params using EC2's numbered-list convention.Most query services in MiniStack use ``member.N`` in the Step Functionsadapter. EC2's Query API expects bare numbered lists for the shapes used- here (e.g. ``Filter.1.Value.1`` and ``GroupId.1``).+ here (e.g. ``Filter.1.Value.1`` and ``GroupId.1``). ``action`` and+ ``parent`` — the list this one is nested in — select the names EC2 spells+ differently depending on where they appear."""params = {}if not isinstance(data, dict):@@ -4434,14 +4488,15 @@ def _flatten_ec2_query_params(data, prefix=""):for key, value in data.items():full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"if isinstance(value, dict):- params.update(_flatten_ec2_query_params(value, full_key))+ # A struct member does not rename its children; the enclosing list does.+ params.update(_flatten_ec2_query_params(value, full_key, action, parent))elif isinstance(value, list):- list_key = _EC2_QUERY_LIST_NAME_OVERRIDES.get(key, key)+ list_key = _ec2_query_list_name(key, action, parent)full_list_key = f"{prefix}{list_key}" if not prefix else f"{prefix}.{list_key}"for i, item in enumerate(value, 1):item_key = f"{full_list_key}.{i}"if isinstance(item, dict):- params.update(_flatten_ec2_query_params(item, item_key))+ params.update(_flatten_ec2_query_params(item, item_key, action, key))else:params[item_key] = str(item)elif isinstance(value, bool):@@ -4468,6 +4523,10 @@ _XML_LIST_WRAPPER_TAGS = frozenset({"EnabledCloudwatchLogsExports", "GlobalClusterMembers","DBParameterGroups", "DBInstances", "DBClusters", "Readers","SupportedNetworkTypes",+ "Roles", "Users", "Groups", "Policies", "AttachedPolicies", "PolicyNames",+ "InstanceProfiles", "ServerCertificateMetadataList", "AccessKeyMetadata",+ "Metrics", "Dimensions", "MetricAlarms", "CompositeAlarms",+ "MetricDataResults", "Datapoints", "QueueUrls",})_XML_BOOLEAN_FIELDS = frozenset({"MultiAZ", "Multiaz", "StorageEncrypted", "DeletionProtection",@@ -4476,6 +4535,7 @@ _XML_BOOLEAN_FIELDS = frozenset({"PerformanceInsightsEnabled", "HttpEndpointEnabled","CrossAccountClone", "CustomerOwnedIpEnabled","IsStorageConfigUpgradeAvailable", "IsWriter", "IsDataLossAllowed",+ "IsTruncated",})@@ -4722,7 +4782,21 @@ def _normalize_ec2_security_group(group):return group+def _normalize_sqs_attribute_map(result):+ """Repeated <Attribute><Name>/<Value> children as the SDK's Attributes map."""+ items = result if isinstance(result, list) else [result]+ attributes = {}+ for item in items:+ if isinstance(item, dict) and "Name" in item:+ attributes[item["Name"]] = item.get("Value", "")+ return {"Attributes": attributes}++def _normalize_query_response(service_key, action, result):+ # GetQueueAttributes is the one Query result that arrives as a list, so it is+ # handled before the struct guard below.+ if service_key == "sqs" and action == "GetQueueAttributes":+ return _normalize_sqs_attribute_map(result)if not isinstance(result, dict):return resultif service_key == "ec2" and action == "DescribeSecurityGroups":@@ -4764,10 +4838,11 @@ def _dispatch_aws_sdk_query(service_info, service_name, action, input_data):wire_data = _convert_params_to_api_names(input_data, name_overrides)form_params = {"Action": pascal_action}if service_key == "ec2":- form_params.update(_flatten_ec2_query_params(wire_data))+ form_params.update(_flatten_ec2_query_params(wire_data, action=pascal_action))else:form_params.update(_flatten_query_params(wire_data))- body = urlencode(form_params)+ # Query-protocol handlers follow HTTP-server contract and take body as bytes.+ body = urlencode(form_params).encode("utf-8")headers = {"content-type": "application/x-www-form-urlencoded",@@ -4826,9 +4901,9 @@ def _dispatch_aws_sdk_query(service_info, service_name, action, input_data):result_key = f"{pascal_action}Result"if result_key in result:result = result[result_key]- # Drop ResponseMetadata+ if isinstance(result, dict):result.pop("ResponseMetadata", None)- result = _normalize_query_response(service_key, pascal_action, result)+ result = _normalize_query_response(service_key, pascal_action, result)return _convert_keys_to_sfn_convention(result)except ET.ParseError:raise _ExecutionError("States.Runtime", f"Failed to parse {service_name} XML response")diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.pyindex f20285f2..15cacd8d 100644--- a/tests/test_stepfunctions.py+++ b/tests/test_stepfunctions.py@@ -701,6 +701,96 @@ def test_sfn_aws_sdk_secretsmanager_create_and_get(sfn, sfn_sync, sm):sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+def test_sfn_aws_sdk_query_services_accept_string_parameters(sfn_sync, sns, sqs, cw):+ """aws-sdk integrations for Query-protocol services dispatch succesfully."""+ import uuid as _uuid++ suffix = _uuid.uuid4().hex[:8]+ topic_arn = sns.create_topic(Name=f"sfn-sdk-query-{suffix}")["TopicArn"]+ queue_url = sqs.create_queue(QueueName=f"sfn-sdk-query-{suffix}")["QueueUrl"]++ definition = json.dumps({+ "StartAt": "GetCallerIdentity",+ "States": {+ "GetCallerIdentity": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:sts:getCallerIdentity",+ # AWS requires Parameters on every aws-sdk Task, even empty.+ "Parameters": {},+ "ResultPath": "$.caller",+ "Next": "Publish",+ },+ "Publish": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:sns:publish",+ "Parameters": {"TopicArn": topic_arn, "Message": "hello from sfn"},+ "ResultPath": "$.publish",+ "Next": "SendMessage",+ },+ "SendMessage": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:sqs:sendMessage",+ "Parameters": {"QueueUrl": queue_url, "MessageBody": "hello from sfn"},+ "ResultPath": "$.send",+ "Next": "GetQueueAttributes",+ },+ "GetQueueAttributes": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:sqs:getQueueAttributes",+ "Parameters": {"QueueUrl": queue_url, "AttributeNames": ["All"]},+ "ResultPath": "$.attributes",+ "Next": "ListRoles",+ },+ "ListRoles": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:iam:listRoles",+ "Parameters": {},+ "ResultPath": "$.roles",+ "Next": "PutMetricData",+ },+ "PutMetricData": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:cloudwatch:putMetricData",+ "Parameters": {+ "Namespace": f"sfn-sdk-query-{suffix}",+ "MetricData": [{"MetricName": "calls", "Value": 1}],+ },+ "ResultPath": "$.metric",+ "End": True,+ },+ },+ })++ sm_arn = sfn_sync.create_state_machine(+ name=f"sfn-sdk-query-{suffix}",+ definition=definition,+ roleArn="arn:aws:iam::000000000000:role/sfn-role",+ )["stateMachineArn"]+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"++ output = json.loads(resp["output"])+ assert output["caller"]["Account"] == "000000000000"+ assert output["publish"]["MessageId"]+ assert output["send"]["MessageId"]+ # Repeated result elements unwrap to a list; it must not break the dispatcher.+ # Repeated <Attribute> elements become the SDK's Attributes map, as in AWS.+ assert isinstance(output["attributes"]["Attributes"], dict)+ assert output["attributes"]["Attributes"]["QueueArn"].endswith(f"sfn-sdk-query-{suffix}")+ # Collections are JSON arrays and IsTruncated a bool, as the SDK returns them,+ # so an ASL definition can index or iterate the result.+ assert isinstance(output["roles"]["Roles"], list)+ assert output["roles"]["IsTruncated"] is False+ assert "metric" in output+ # The call reached the service, not just the adapter.+ metrics = cw.list_metrics(Namespace=f"sfn-sdk-query-{suffix}")["Metrics"]+ assert [m["MetricName"] for m in metrics] == ["calls"]++ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+ sqs.delete_queue(QueueUrl=queue_url)+ sns.delete_topic(TopicArn=topic_arn)++def test_sfn_jsonata_arguments_output_and_catch_output(sfn, sfn_sync, sm):"""JSONata Task states evaluate Arguments and Output for aws-sdk integrations."""import uuid as _uuid@@ -1655,6 +1745,155 @@ def test_sfn_aws_sdk_ec2_security_group_duplicate_error(sfn_sync, ec2):sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+def test_sfn_aws_sdk_ec2_list_params_reach_the_service(sfn_sync, ec2):+ """EC2 numbers list parameters in the singular: VolumeIds must arrive as VolumeId.N."""+ suffix = _uuid_mod.uuid4().hex[:8]+ az = ec2.describe_availability_zones()["AvailabilityZones"][0]["ZoneName"]+ wanted = ec2.create_volume(AvailabilityZone=az, Size=1, VolumeType="gp3")["VolumeId"]+ other = ec2.create_volume(AvailabilityZone=az, Size=1, VolumeType="gp3")["VolumeId"]+ sm_arn = None+ try:+ definition = json.dumps({+ "StartAt": "DescribeVolumes",+ "States": {"DescribeVolumes": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:ec2:describeVolumes",+ "Parameters": {"VolumeIds": [wanted]},+ "End": True,+ }},+ })+ sm_arn = sfn_sync.create_state_machine(+ name=f"sdk-ec2-list-{suffix}", definition=definition,+ roleArn="arn:aws:iam::000000000000:role/sfn-role",+ )["stateMachineArn"]+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"+ output = resp["output"]+ assert wanted in output+ assert other not in output+ finally:+ if sm_arn:+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+ for volume_id in (wanted, other):+ ec2.delete_volume(VolumeId=volume_id)+++def test_sfn_aws_sdk_ec2_list_names_match_the_botocore_model():+ """Every EC2 list parameter flattens to the name botocore puts on the wire."""+ from botocore.loaders import Loader+ from botocore.model import ServiceModel++ from ministack.services.ec2 import _ACTION_MAP+ from ministack.services.stepfunctions import _ec2_query_list_name++ model = ServiceModel(Loader().load_service_model("ec2", "service-2"))++ def wire_name(shape, member_name):+ """botocore's EC2Serializer._get_serialized_name."""+ serialization = shape.serialization+ if "queryName" in serialization:+ return serialization["queryName"]+ if "name" in serialization:+ name = serialization["name"]+ return name[0].upper() + name[1:]+ return member_name++ def walk(shape, action, parent, seen, checked, wrong):+ if (shape.name, parent) in seen:+ return+ seen.add((shape.name, parent))+ for name, member in getattr(shape, "members", {}).items():+ if member.type_name == "list":+ checked.append(name)+ expected = wire_name(member, name)+ got = _ec2_query_list_name(name, action, parent)+ if got != expected:+ wrong.append(f"{action}: {parent or '<top>'}.{name} -> {got}, want {expected}")+ walk(member.member, action, name, seen, checked, wrong)+ elif member.type_name == "structure":+ walk(member, action, parent, seen, checked, wrong)++ checked, wrong = [], []+ for action in sorted(set(_ACTION_MAP) & set(model.operation_names)):+ shape = model.operation_model(action).input_shape+ if shape is not None:+ walk(shape, action, "", set(), checked, wrong)++ assert not wrong, "\n".join(wrong)+ # Guard against the walk silently covering nothing after a model change.+ assert len(checked) > 300+++def test_sfn_aws_sdk_ec2_plural_list_params_reach_the_service(sfn_sync, ec2):+ """EC2 keeps some list parameters plural: IpPermissions must stay IpPermissions.N."""+ suffix = _uuid_mod.uuid4().hex[:8]+ group_name = f"sdk-ec2-perms-{suffix}"+ group_id = ec2.create_security_group(+ GroupName=group_name, Description="list param check", VpcId="vpc-00000001"+ )["GroupId"]+ sm_arn = None+ try:+ definition = json.dumps({+ "StartAt": "Authorize",+ "States": {"Authorize": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:ec2:authorizeSecurityGroupIngress",+ "Parameters": {"GroupId": group_id, "IpPermissions": [{+ "IpProtocol": "tcp", "FromPort": 443, "ToPort": 443,+ # Nested lists stay plural too: IpPermissions.1.IpRanges.1.CidrIp+ "IpRanges": [{"CidrIp": "10.0.0.0/8"}],+ }]},+ "End": True,+ }},+ })+ sm_arn = sfn_sync.create_state_machine(+ name=f"sdk-ec2-perms-{suffix}", definition=definition,+ roleArn="arn:aws:iam::000000000000:role/sfn-role",+ )["stateMachineArn"]+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"++ rules = ec2.describe_security_groups(GroupIds=[group_id])["SecurityGroups"][0]["IpPermissions"]+ assert [r["FromPort"] for r in rules] == [443], f"rule not applied: {rules}"+ assert rules[0]["IpRanges"] == [{"CidrIp": "10.0.0.0/8"}]+ finally:+ if sm_arn:+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+ ec2.delete_security_group(GroupId=group_id)+++def test_sfn_aws_sdk_ec2_irregular_list_params_reach_the_service(sfn_sync, ec2):+ """EC2 renames some list parameters: Resources arrives as ResourceId.N."""+ suffix = _uuid_mod.uuid4().hex[:8]+ az = ec2.describe_availability_zones()["AvailabilityZones"][0]["ZoneName"]+ volume_id = ec2.create_volume(AvailabilityZone=az, Size=1, VolumeType="gp3")["VolumeId"]+ sm_arn = None+ try:+ definition = json.dumps({+ "StartAt": "CreateTags",+ "States": {"CreateTags": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:ec2:createTags",+ "Parameters": {"Resources": [volume_id],+ "Tags": [{"Key": "checked-by", "Value": suffix}]},+ "End": True,+ }},+ })+ sm_arn = sfn_sync.create_state_machine(+ name=f"sdk-ec2-tags-{suffix}", definition=definition,+ roleArn="arn:aws:iam::000000000000:role/sfn-role",+ )["stateMachineArn"]+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"++ tags = ec2.describe_volumes(VolumeIds=[volume_id])["Volumes"][0].get("Tags", [])+ assert {"Key": "checked-by", "Value": suffix} in tags, f"tag not applied: {tags}"+ finally:+ if sm_arn:+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+ ec2.delete_volume(VolumeId=volume_id)++def test_sfn_aws_sdk_rds_create_and_describe_instance(sfn, sfn_sync):"""aws-sdk:rds CreateDBInstance + DescribeDBInstances via query-protocol dispatch."""import uuid as _uuid