Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(stepfunctions): return S3 object Metadata from aws-sdk:s3:headObject (#1226)
Michael Denyer4283ef4dc15f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3abc52a7..366f83b3 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]### Fixed+- **Step Functions — `aws-sdk:s3:headObject` dropped the object's user `Metadata` map** — the S3 REST-XML SDK-integration spec maps response headers to output fields by exact name, which cannot express the N dynamic `x-amz-meta-*` headers, so the task result carried no `Metadata` key and an ASL reading `$.Metadata.<key>` (e.g. `States.StringToJson($.Metadata.shardcount)` to size a downstream fan-out) failed with `States.Runtime`. The dispatcher now collects `x-amz-meta-*` response headers into a `Metadata` map with the prefix stripped, attached after the SFN key-convention pass so the metadata keys stay verbatim (lowercase, as S3 stores them) instead of being title-cased like API member names. Contributed by @michael-denyer.- **Cognito — `CUSTOM_AUTH` now honors Cognito-owned SRP challenges and passes `ClientMetadata` to DefineAuth** — Amplify `CUSTOM_WITH_SRP` failed with `USER_ID_FOR_SRP was not found` because `CUSTOM_AUTH` always invoked CreateAuth and returned `CUSTOM_CHALLENGE`, even when DefineAuth returned `PASSWORD_VERIFIER` / `SRP_A`. InitiateAuth with `SRP_A` now records `SRP_A` success (AWS-like), returns built-in `PASSWORD_VERIFIER` challenge parameters, and continues the custom-auth session after RespondToAuthChallenge so MFA `CUSTOM_CHALLENGE` rounds still work. DefineAuth also receives `ClientMetadata` (previously hard-coded `{}`), matching Create/Verify. Contributed by @onelshina.- **EC2 - `ModifySecurityGroupRules`** - this action was unimplemented and returned `InvalidAction: Unknown EC2 action: ModifySecurityGroupRules`. Full support is now provided.diff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.pyindex cd55ffbc..be9bc9f4 100644--- a/ministack/services/stepfunctions.py+++ b/ministack/services/stepfunctions.py@@ -5077,6 +5077,10 @@ def _dispatch_aws_sdk_lambda_rest(service_info, service_name, action, input_data# not a wrapped list).# header_outputs:{HTTP-header-name: OutputFieldPascalCase} response headers# to fold into the result dict (e.g. ETag from PUT/COPY).+# metadata_output: collect the dynamic x-amz-meta-* response headers into a+# result "Metadata" map (prefix stripped). Needed because+# header_outputs maps exact header names and user metadata+# arrives as N dynamic headers.## Phase 1 covers non-Body operations (no GetObject/PutObject). Body shape for# aws-sdk:s3:getObject/putObject is convention-based (Java SDK V2 → base64) and@@ -5144,6 +5148,7 @@ _S3_OP_SPECS = {"last-modified": "LastModified","x-amz-version-id": "VersionId",},+ "metadata_output": True,},"CopyObject": {"method": "PUT", "path": "/{Bucket}/{Key+}",@@ -5378,7 +5383,20 @@ def _dispatch_aws_sdk_rest_xml(service_info, service_name, action, input_data):result = _s3_normalize_lists(result, spec.get("list_fields") or ())- return _convert_keys_to_sfn_convention(result)+ converted = _convert_keys_to_sfn_convention(result)++ # User metadata keys are returned verbatim by AWS (S3 lowercases them on+ # write), so they must be attached AFTER the SFN key conversion — running+ # them through _api_name_to_sfn_key would turn "shardcount" into+ # "Shardcount" and break $.Metadata.<key> paths.+ if spec.get("metadata_output"):+ converted["Metadata"] = {+ hname[len("x-amz-meta-"):]: value+ for hname, value in norm_resp_headers.items()+ if hname.startswith("x-amz-meta-")+ }++ return converteddef _invoke_aws_sdk_integration(resource, input_data):diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.pyindex 63bbea75..f20285f2 100644--- a/tests/test_stepfunctions.py+++ b/tests/test_stepfunctions.py@@ -4624,6 +4624,119 @@ def test_sfn_aws_sdk_s3_copy_object(sfn_sync, s3):sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+def test_sfn_aws_sdk_s3_head_object_metadata(sfn_sync, s3):+ """aws-sdk:s3:headObject returns the object's user Metadata map with keys verbatim."""+ import uuid as _uuid++ bucket = f"sfn-s3-head-{_uuid.uuid4().hex[:8]}"+ sm_name = f"sdk-s3-head-{_uuid.uuid4().hex[:8]}"+ sm_arn = None+ try:+ s3.create_bucket(Bucket=bucket)+ s3.put_object(+ Bucket=bucket,+ Key="work/shard_manifest.json",+ Body=b"{}",+ Metadata={"shardcount": "2"},+ )++ definition = json.dumps({+ "StartAt": "Head",+ "States": {+ "Head": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:s3:headObject",+ "Parameters": {"Bucket": bucket, "Key": "work/shard_manifest.json"},+ "End": True,+ },+ },+ })++ sm_arn = sfn_sync.create_state_machine(+ name=sm_name,+ definition=definition,+ roleArn="arn:aws:iam::000000000000:role/sfn-role",+ )["stateMachineArn"]++ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))+ assert resp["status"] == "SUCCEEDED", f"{resp.get('error')} — {resp.get('cause')}"+ output = json.loads(resp["output"])+ # Keys must NOT be run through the SFN key convention: "shardcount",+ # not "Shardcount". $.Metadata.shardcount is the path real ASLs use.+ assert output["Metadata"] == {"shardcount": "2"}+ assert output["ContentLength"] == 2+ finally:+ try:+ s3.delete_object(Bucket=bucket, Key="work/shard_manifest.json")+ except Exception:+ pass+ try:+ s3.delete_bucket(Bucket=bucket)+ except Exception:+ pass+ if sm_arn:+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)+++def test_sfn_aws_sdk_s3_head_object_metadata_result_selector(sfn_sync, s3):+ """States.StringToJson($.Metadata.<key>) works on headObject output.++ This is the real-world consumer shape: an ASL sizing a downstream fan-out+ from a manifest object's metadata. It catches both failure modes — a+ missing Metadata map and metadata keys mangled by the SFN key convention.+ """+ import uuid as _uuid++ bucket = f"sfn-s3-headrs-{_uuid.uuid4().hex[:8]}"+ sm_name = f"sdk-s3-headrs-{_uuid.uuid4().hex[:8]}"+ sm_arn = None+ try:+ s3.create_bucket(Bucket=bucket)+ s3.put_object(+ Bucket=bucket,+ Key="work/shard_manifest.json",+ Body=b"{}",+ Metadata={"shardcount": "2"},+ )++ definition = json.dumps({+ "StartAt": "Head",+ "States": {+ "Head": {+ "Type": "Task",+ "Resource": "arn:aws:states:::aws-sdk:s3:headObject",+ "Parameters": {"Bucket": bucket, "Key": "work/shard_manifest.json"},+ "ResultSelector": {+ "shard_count.$": "States.StringToJson($.Metadata.shardcount)",+ },+ "End": True,+ },+ },+ })++ sm_arn = sfn_sync.create_state_machine(+ name=sm_name,+ definition=definition,+ roleArn="arn:aws:iam::000000000000:role/sfn-role",+ )["stateMachineArn"]++ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))+ assert resp["status"] == "SUCCEEDED", f"{resp.get('error')} — {resp.get('cause')}"+ output = json.loads(resp["output"])+ assert output == {"shard_count": 2}+ finally:+ try:+ s3.delete_object(Bucket=bucket, Key="work/shard_manifest.json")+ except Exception:+ pass+ try:+ s3.delete_bucket(Bucket=bucket)+ except Exception:+ pass+ if sm_arn:+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)++def test_sfn_aws_sdk_s3_list_buckets(sfn_sync, s3):"""aws-sdk:s3:listBuckets returns the bucket list via REST-XML."""import uuid as _uuid