Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

Fix Step Functions aws-sdk integrations for Query-protocol services (#1227)

Maxd1cb89f20486
diff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.py
index be9bc9f4..724ee6d3 100644
--- a/ministack/services/stepfunctions.py
+++ b/ministack/services/stepfunctions.py
@@ -4411,22 +4411,76 @@ def _flatten_query_params(data, prefix=""):
return params
-_EC2_QUERY_LIST_NAME_OVERRIDES = {
- "Filters": "Filter",
- "Values": "Value",
- "GroupIds": "GroupId",
- "GroupNames": "GroupName",
- "TagSpecifications": "TagSpecification",
- "Tags": "Tag",
+# Wire names EC2 numbers list parameters under. Mostly the singular of the SDK
+# member name (VolumeIds -> VolumeId.N), but sometimes the plural
+# (IpPermissions.N) and sometimes a different word (Resources -> ResourceId.N).
+# A wrong name is dropped silently — the call still succeeds and the parameter
+# never applies — so the exceptions are listed rather than guessed. Generated
+# from the botocore EC2 model (queryName, else capitalized locationName) for the
+# actions ec2.py serves; see check_sfn_ec2_list_params.py.
+_EC2_QUERY_LIST_NAMES = {
+ "ExecutableUsers": "ExecutableBy",
+ "Groups": "SecurityGroupId",
+ "IpPermissions": "IpPermissions",
+ "IpRanges": "IpRanges",
+ "Ipv6Ranges": "Ipv6Ranges",
+ "LaunchTemplateConfigs": "LaunchTemplateConfigs",
+ "Overrides": "Overrides",
+ "OwnerIds": "Owner",
+ "Resources": "ResourceId",
+ "RestorableByUserIds": "RestorableBy",
+ "TunnelOptions": "TunnelOptions",
+ "UserIdGroupPairs": "Groups",
+ "Versions": "LaunchTemplateVersion",
}
+# Names spelled both ways across those actions, so they cannot go above. The
+# enclosing list settles most — PrefixListIds is plural inside an IpPermission
+# and singular in DescribePrefixLists — the rest are top-level, where only the
+# action can.
+_EC2_QUERY_LIST_NAMES_BY_PARENT = {
+ ("IpPermissions", "PrefixListIds"): "PrefixListIds",
+ ("NetworkInterfaces", "Ipv6Addresses"): "Ipv6Addresses",
+ ("NetworkInterfaces", "PrivateIpAddresses"): "PrivateIpAddresses",
+}
+
+_EC2_QUERY_LIST_NAMES_BY_ACTION = {
+ ("CreateNetworkInterface", "Ipv6Addresses"): "Ipv6Addresses",
+ ("CreateNetworkInterface", "PrivateIpAddresses"): "PrivateIpAddresses",
+ ("DescribeVpcClassicLinkDnsSupport", "VpcIds"): "VpcIds",
+ ("ModifySnapshotAttribute", "GroupNames"): "UserGroup",
+}
+
+
+def _ec2_query_list_name(key, action=None, parent=""):
+ """Wire name EC2 numbers a list parameter under: VolumeIds -> VolumeId.N."""
+ # Most specific context first: enclosing list, then action, then the names
+ # that are the same everywhere.
+ if (parent, key) in _EC2_QUERY_LIST_NAMES_BY_PARENT:
+ return _EC2_QUERY_LIST_NAMES_BY_PARENT[(parent, key)]
+ if (action, key) in _EC2_QUERY_LIST_NAMES_BY_ACTION:
+ return _EC2_QUERY_LIST_NAMES_BY_ACTION[(action, key)]
+ if key in _EC2_QUERY_LIST_NAMES:
+ return _EC2_QUERY_LIST_NAMES[key]
+ # Everything else is the singular of an English plural: Entries -> Entry,
+ # Ipv4Prefixes -> Ipv4Prefix, VolumeIds -> VolumeId.
+ if len(key) < 2 or not key.endswith("s") or key.endswith("ss"):
+ return key
+ if key.endswith("ies") and len(key) > 3:
+ return key[:-3] + "y"
+ if key.endswith(("ses", "xes", "zes", "ches", "shes")):
+ return key[:-2]
+ return key[:-1]
+
-def _flatten_ec2_query_params(data, prefix=""):
+def _flatten_ec2_query_params(data, prefix="", action=None, parent=""):
"""Flatten EC2 query params using EC2's numbered-list convention.
Most query services in MiniStack use ``member.N`` in the Step Functions
adapter. EC2's Query API expects bare numbered lists for the shapes used
- here (e.g. ``Filter.1.Value.1`` and ``GroupId.1``).
+ here (e.g. ``Filter.1.Value.1`` and ``GroupId.1``). ``action`` and
+ ``parent`` — the list this one is nested in — select the names EC2 spells
+ differently depending on where they appear.
"""
params = {}
if not isinstance(data, dict):
@@ -4434,14 +4488,15 @@ def _flatten_ec2_query_params(data, prefix=""):
for key, value in data.items():
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
if isinstance(value, dict):
- params.update(_flatten_ec2_query_params(value, full_key))
+ # A struct member does not rename its children; the enclosing list does.
+ params.update(_flatten_ec2_query_params(value, full_key, action, parent))
elif isinstance(value, list):
- list_key = _EC2_QUERY_LIST_NAME_OVERRIDES.get(key, key)
+ list_key = _ec2_query_list_name(key, action, parent)
full_list_key = f"{prefix}{list_key}" if not prefix else f"{prefix}.{list_key}"
for i, item in enumerate(value, 1):
item_key = f"{full_list_key}.{i}"
if isinstance(item, dict):
- params.update(_flatten_ec2_query_params(item, item_key))
+ params.update(_flatten_ec2_query_params(item, item_key, action, key))
else:
params[item_key] = str(item)
elif isinstance(value, bool):
@@ -4468,6 +4523,10 @@ _XML_LIST_WRAPPER_TAGS = frozenset({
"EnabledCloudwatchLogsExports", "GlobalClusterMembers",
"DBParameterGroups", "DBInstances", "DBClusters", "Readers",
"SupportedNetworkTypes",
+ "Roles", "Users", "Groups", "Policies", "AttachedPolicies", "PolicyNames",
+ "InstanceProfiles", "ServerCertificateMetadataList", "AccessKeyMetadata",
+ "Metrics", "Dimensions", "MetricAlarms", "CompositeAlarms",
+ "MetricDataResults", "Datapoints", "QueueUrls",
})
_XML_BOOLEAN_FIELDS = frozenset({
"MultiAZ", "Multiaz", "StorageEncrypted", "DeletionProtection",
@@ -4476,6 +4535,7 @@ _XML_BOOLEAN_FIELDS = frozenset({
"PerformanceInsightsEnabled", "HttpEndpointEnabled",
"CrossAccountClone", "CustomerOwnedIpEnabled",
"IsStorageConfigUpgradeAvailable", "IsWriter", "IsDataLossAllowed",
+ "IsTruncated",
})
@@ -4722,7 +4782,21 @@ def _normalize_ec2_security_group(group):
return group
+def _normalize_sqs_attribute_map(result):
+ """Repeated <Attribute><Name>/<Value> children as the SDK's Attributes map."""
+ items = result if isinstance(result, list) else [result]
+ attributes = {}
+ for item in items:
+ if isinstance(item, dict) and "Name" in item:
+ attributes[item["Name"]] = item.get("Value", "")
+ return {"Attributes": attributes}
+
+
def _normalize_query_response(service_key, action, result):
+ # GetQueueAttributes is the one Query result that arrives as a list, so it is
+ # handled before the struct guard below.
+ if service_key == "sqs" and action == "GetQueueAttributes":
+ return _normalize_sqs_attribute_map(result)
if not isinstance(result, dict):
return result
if service_key == "ec2" and action == "DescribeSecurityGroups":
@@ -4764,10 +4838,11 @@ def _dispatch_aws_sdk_query(service_info, service_name, action, input_data):
wire_data = _convert_params_to_api_names(input_data, name_overrides)
form_params = {"Action": pascal_action}
if service_key == "ec2":
- form_params.update(_flatten_ec2_query_params(wire_data))
+ form_params.update(_flatten_ec2_query_params(wire_data, action=pascal_action))
else:
form_params.update(_flatten_query_params(wire_data))
- body = urlencode(form_params)
+ # Query-protocol handlers follow HTTP-server contract and take body as bytes.
+ body = urlencode(form_params).encode("utf-8")
headers = {
"content-type": "application/x-www-form-urlencoded",
@@ -4826,9 +4901,9 @@ def _dispatch_aws_sdk_query(service_info, service_name, action, input_data):
result_key = f"{pascal_action}Result"
if result_key in result:
result = result[result_key]
- # Drop ResponseMetadata
+ if isinstance(result, dict):
result.pop("ResponseMetadata", None)
- result = _normalize_query_response(service_key, pascal_action, result)
+ result = _normalize_query_response(service_key, pascal_action, result)
return _convert_keys_to_sfn_convention(result)
except ET.ParseError:
raise _ExecutionError("States.Runtime", f"Failed to parse {service_name} XML response")
diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.py
index f20285f2..15cacd8d 100644
--- a/tests/test_stepfunctions.py
+++ b/tests/test_stepfunctions.py
@@ -701,6 +701,96 @@ def test_sfn_aws_sdk_secretsmanager_create_and_get(sfn, sfn_sync, sm):
sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+def test_sfn_aws_sdk_query_services_accept_string_parameters(sfn_sync, sns, sqs, cw):
+ """aws-sdk integrations for Query-protocol services dispatch succesfully."""
+ import uuid as _uuid
+
+ suffix = _uuid.uuid4().hex[:8]
+ topic_arn = sns.create_topic(Name=f"sfn-sdk-query-{suffix}")["TopicArn"]
+ queue_url = sqs.create_queue(QueueName=f"sfn-sdk-query-{suffix}")["QueueUrl"]
+
+ definition = json.dumps({
+ "StartAt": "GetCallerIdentity",
+ "States": {
+ "GetCallerIdentity": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:sts:getCallerIdentity",
+ # AWS requires Parameters on every aws-sdk Task, even empty.
+ "Parameters": {},
+ "ResultPath": "$.caller",
+ "Next": "Publish",
+ },
+ "Publish": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:sns:publish",
+ "Parameters": {"TopicArn": topic_arn, "Message": "hello from sfn"},
+ "ResultPath": "$.publish",
+ "Next": "SendMessage",
+ },
+ "SendMessage": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:sqs:sendMessage",
+ "Parameters": {"QueueUrl": queue_url, "MessageBody": "hello from sfn"},
+ "ResultPath": "$.send",
+ "Next": "GetQueueAttributes",
+ },
+ "GetQueueAttributes": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:sqs:getQueueAttributes",
+ "Parameters": {"QueueUrl": queue_url, "AttributeNames": ["All"]},
+ "ResultPath": "$.attributes",
+ "Next": "ListRoles",
+ },
+ "ListRoles": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:iam:listRoles",
+ "Parameters": {},
+ "ResultPath": "$.roles",
+ "Next": "PutMetricData",
+ },
+ "PutMetricData": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:cloudwatch:putMetricData",
+ "Parameters": {
+ "Namespace": f"sfn-sdk-query-{suffix}",
+ "MetricData": [{"MetricName": "calls", "Value": 1}],
+ },
+ "ResultPath": "$.metric",
+ "End": True,
+ },
+ },
+ })
+
+ sm_arn = sfn_sync.create_state_machine(
+ name=f"sfn-sdk-query-{suffix}",
+ definition=definition,
+ roleArn="arn:aws:iam::000000000000:role/sfn-role",
+ )["stateMachineArn"]
+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))
+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"
+
+ output = json.loads(resp["output"])
+ assert output["caller"]["Account"] == "000000000000"
+ assert output["publish"]["MessageId"]
+ assert output["send"]["MessageId"]
+ # Repeated result elements unwrap to a list; it must not break the dispatcher.
+ # Repeated <Attribute> elements become the SDK's Attributes map, as in AWS.
+ assert isinstance(output["attributes"]["Attributes"], dict)
+ assert output["attributes"]["Attributes"]["QueueArn"].endswith(f"sfn-sdk-query-{suffix}")
+ # Collections are JSON arrays and IsTruncated a bool, as the SDK returns them,
+ # so an ASL definition can index or iterate the result.
+ assert isinstance(output["roles"]["Roles"], list)
+ assert output["roles"]["IsTruncated"] is False
+ assert "metric" in output
+ # The call reached the service, not just the adapter.
+ metrics = cw.list_metrics(Namespace=f"sfn-sdk-query-{suffix}")["Metrics"]
+ assert [m["MetricName"] for m in metrics] == ["calls"]
+
+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+ sqs.delete_queue(QueueUrl=queue_url)
+ sns.delete_topic(TopicArn=topic_arn)
+
+
def test_sfn_jsonata_arguments_output_and_catch_output(sfn, sfn_sync, sm):
"""JSONata Task states evaluate Arguments and Output for aws-sdk integrations."""
import uuid as _uuid
@@ -1655,6 +1745,155 @@ def test_sfn_aws_sdk_ec2_security_group_duplicate_error(sfn_sync, ec2):
sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+def test_sfn_aws_sdk_ec2_list_params_reach_the_service(sfn_sync, ec2):
+ """EC2 numbers list parameters in the singular: VolumeIds must arrive as VolumeId.N."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ az = ec2.describe_availability_zones()["AvailabilityZones"][0]["ZoneName"]
+ wanted = ec2.create_volume(AvailabilityZone=az, Size=1, VolumeType="gp3")["VolumeId"]
+ other = ec2.create_volume(AvailabilityZone=az, Size=1, VolumeType="gp3")["VolumeId"]
+ sm_arn = None
+ try:
+ definition = json.dumps({
+ "StartAt": "DescribeVolumes",
+ "States": {"DescribeVolumes": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:ec2:describeVolumes",
+ "Parameters": {"VolumeIds": [wanted]},
+ "End": True,
+ }},
+ })
+ sm_arn = sfn_sync.create_state_machine(
+ name=f"sdk-ec2-list-{suffix}", definition=definition,
+ roleArn="arn:aws:iam::000000000000:role/sfn-role",
+ )["stateMachineArn"]
+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))
+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"
+ output = resp["output"]
+ assert wanted in output
+ assert other not in output
+ finally:
+ if sm_arn:
+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+ for volume_id in (wanted, other):
+ ec2.delete_volume(VolumeId=volume_id)
+
+
+def test_sfn_aws_sdk_ec2_list_names_match_the_botocore_model():
+ """Every EC2 list parameter flattens to the name botocore puts on the wire."""
+ from botocore.loaders import Loader
+ from botocore.model import ServiceModel
+
+ from ministack.services.ec2 import _ACTION_MAP
+ from ministack.services.stepfunctions import _ec2_query_list_name
+
+ model = ServiceModel(Loader().load_service_model("ec2", "service-2"))
+
+ def wire_name(shape, member_name):
+ """botocore's EC2Serializer._get_serialized_name."""
+ serialization = shape.serialization
+ if "queryName" in serialization:
+ return serialization["queryName"]
+ if "name" in serialization:
+ name = serialization["name"]
+ return name[0].upper() + name[1:]
+ return member_name
+
+ def walk(shape, action, parent, seen, checked, wrong):
+ if (shape.name, parent) in seen:
+ return
+ seen.add((shape.name, parent))
+ for name, member in getattr(shape, "members", {}).items():
+ if member.type_name == "list":
+ checked.append(name)
+ expected = wire_name(member, name)
+ got = _ec2_query_list_name(name, action, parent)
+ if got != expected:
+ wrong.append(f"{action}: {parent or '<top>'}.{name} -> {got}, want {expected}")
+ walk(member.member, action, name, seen, checked, wrong)
+ elif member.type_name == "structure":
+ walk(member, action, parent, seen, checked, wrong)
+
+ checked, wrong = [], []
+ for action in sorted(set(_ACTION_MAP) & set(model.operation_names)):
+ shape = model.operation_model(action).input_shape
+ if shape is not None:
+ walk(shape, action, "", set(), checked, wrong)
+
+ assert not wrong, "\n".join(wrong)
+ # Guard against the walk silently covering nothing after a model change.
+ assert len(checked) > 300
+
+
+def test_sfn_aws_sdk_ec2_plural_list_params_reach_the_service(sfn_sync, ec2):
+ """EC2 keeps some list parameters plural: IpPermissions must stay IpPermissions.N."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ group_name = f"sdk-ec2-perms-{suffix}"
+ group_id = ec2.create_security_group(
+ GroupName=group_name, Description="list param check", VpcId="vpc-00000001"
+ )["GroupId"]
+ sm_arn = None
+ try:
+ definition = json.dumps({
+ "StartAt": "Authorize",
+ "States": {"Authorize": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:ec2:authorizeSecurityGroupIngress",
+ "Parameters": {"GroupId": group_id, "IpPermissions": [{
+ "IpProtocol": "tcp", "FromPort": 443, "ToPort": 443,
+ # Nested lists stay plural too: IpPermissions.1.IpRanges.1.CidrIp
+ "IpRanges": [{"CidrIp": "10.0.0.0/8"}],
+ }]},
+ "End": True,
+ }},
+ })
+ sm_arn = sfn_sync.create_state_machine(
+ name=f"sdk-ec2-perms-{suffix}", definition=definition,
+ roleArn="arn:aws:iam::000000000000:role/sfn-role",
+ )["stateMachineArn"]
+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))
+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"
+
+ rules = ec2.describe_security_groups(GroupIds=[group_id])["SecurityGroups"][0]["IpPermissions"]
+ assert [r["FromPort"] for r in rules] == [443], f"rule not applied: {rules}"
+ assert rules[0]["IpRanges"] == [{"CidrIp": "10.0.0.0/8"}]
+ finally:
+ if sm_arn:
+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+ ec2.delete_security_group(GroupId=group_id)
+
+
+def test_sfn_aws_sdk_ec2_irregular_list_params_reach_the_service(sfn_sync, ec2):
+ """EC2 renames some list parameters: Resources arrives as ResourceId.N."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ az = ec2.describe_availability_zones()["AvailabilityZones"][0]["ZoneName"]
+ volume_id = ec2.create_volume(AvailabilityZone=az, Size=1, VolumeType="gp3")["VolumeId"]
+ sm_arn = None
+ try:
+ definition = json.dumps({
+ "StartAt": "CreateTags",
+ "States": {"CreateTags": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:ec2:createTags",
+ "Parameters": {"Resources": [volume_id],
+ "Tags": [{"Key": "checked-by", "Value": suffix}]},
+ "End": True,
+ }},
+ })
+ sm_arn = sfn_sync.create_state_machine(
+ name=f"sdk-ec2-tags-{suffix}", definition=definition,
+ roleArn="arn:aws:iam::000000000000:role/sfn-role",
+ )["stateMachineArn"]
+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))
+ assert resp["status"] == "SUCCEEDED", f"Execution failed: {resp.get('error')} — {resp.get('cause')}"
+
+ tags = ec2.describe_volumes(VolumeIds=[volume_id])["Volumes"][0].get("Tags", [])
+ assert {"Key": "checked-by", "Value": suffix} in tags, f"tag not applied: {tags}"
+ finally:
+ if sm_arn:
+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+ ec2.delete_volume(VolumeId=volume_id)
+
+
def test_sfn_aws_sdk_rds_create_and_describe_instance(sfn, sfn_sync):
"""aws-sdk:rds CreateDBInstance + DescribeDBInstances via query-protocol dispatch."""
import uuid as _uuid