Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(stepfunctions): return S3 object Metadata from aws-sdk:s3:headObject (#1226)

Michael Denyer4283ef4dc15f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3abc52a7..366f83b3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
### Fixed
+- **Step Functions — `aws-sdk:s3:headObject` dropped the object's user `Metadata` map** — the S3 REST-XML SDK-integration spec maps response headers to output fields by exact name, which cannot express the N dynamic `x-amz-meta-*` headers, so the task result carried no `Metadata` key and an ASL reading `$.Metadata.<key>` (e.g. `States.StringToJson($.Metadata.shardcount)` to size a downstream fan-out) failed with `States.Runtime`. The dispatcher now collects `x-amz-meta-*` response headers into a `Metadata` map with the prefix stripped, attached after the SFN key-convention pass so the metadata keys stay verbatim (lowercase, as S3 stores them) instead of being title-cased like API member names. Contributed by @michael-denyer.
- **Cognito — `CUSTOM_AUTH` now honors Cognito-owned SRP challenges and passes `ClientMetadata` to DefineAuth** — Amplify `CUSTOM_WITH_SRP` failed with `USER_ID_FOR_SRP was not found` because `CUSTOM_AUTH` always invoked CreateAuth and returned `CUSTOM_CHALLENGE`, even when DefineAuth returned `PASSWORD_VERIFIER` / `SRP_A`. InitiateAuth with `SRP_A` now records `SRP_A` success (AWS-like), returns built-in `PASSWORD_VERIFIER` challenge parameters, and continues the custom-auth session after RespondToAuthChallenge so MFA `CUSTOM_CHALLENGE` rounds still work. DefineAuth also receives `ClientMetadata` (previously hard-coded `{}`), matching Create/Verify. Contributed by @onelshina.
- **EC2 - `ModifySecurityGroupRules`** - this action was unimplemented and returned `InvalidAction: Unknown EC2 action: ModifySecurityGroupRules`. Full support is now provided.
diff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.py
index cd55ffbc..be9bc9f4 100644
--- a/ministack/services/stepfunctions.py
+++ b/ministack/services/stepfunctions.py
@@ -5077,6 +5077,10 @@ def _dispatch_aws_sdk_lambda_rest(service_info, service_name, action, input_data
# not a wrapped list).
# header_outputs:{HTTP-header-name: OutputFieldPascalCase} response headers
# to fold into the result dict (e.g. ETag from PUT/COPY).
+# metadata_output: collect the dynamic x-amz-meta-* response headers into a
+# result "Metadata" map (prefix stripped). Needed because
+# header_outputs maps exact header names and user metadata
+# arrives as N dynamic headers.
#
# Phase 1 covers non-Body operations (no GetObject/PutObject). Body shape for
# aws-sdk:s3:getObject/putObject is convention-based (Java SDK V2 → base64) and
@@ -5144,6 +5148,7 @@ _S3_OP_SPECS = {
"last-modified": "LastModified",
"x-amz-version-id": "VersionId",
},
+ "metadata_output": True,
},
"CopyObject": {
"method": "PUT", "path": "/{Bucket}/{Key+}",
@@ -5378,7 +5383,20 @@ def _dispatch_aws_sdk_rest_xml(service_info, service_name, action, input_data):
result = _s3_normalize_lists(result, spec.get("list_fields") or ())
- return _convert_keys_to_sfn_convention(result)
+ converted = _convert_keys_to_sfn_convention(result)
+
+ # User metadata keys are returned verbatim by AWS (S3 lowercases them on
+ # write), so they must be attached AFTER the SFN key conversion — running
+ # them through _api_name_to_sfn_key would turn "shardcount" into
+ # "Shardcount" and break $.Metadata.<key> paths.
+ if spec.get("metadata_output"):
+ converted["Metadata"] = {
+ hname[len("x-amz-meta-"):]: value
+ for hname, value in norm_resp_headers.items()
+ if hname.startswith("x-amz-meta-")
+ }
+
+ return converted
def _invoke_aws_sdk_integration(resource, input_data):
diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.py
index 63bbea75..f20285f2 100644
--- a/tests/test_stepfunctions.py
+++ b/tests/test_stepfunctions.py
@@ -4624,6 +4624,119 @@ def test_sfn_aws_sdk_s3_copy_object(sfn_sync, s3):
sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+def test_sfn_aws_sdk_s3_head_object_metadata(sfn_sync, s3):
+ """aws-sdk:s3:headObject returns the object's user Metadata map with keys verbatim."""
+ import uuid as _uuid
+
+ bucket = f"sfn-s3-head-{_uuid.uuid4().hex[:8]}"
+ sm_name = f"sdk-s3-head-{_uuid.uuid4().hex[:8]}"
+ sm_arn = None
+ try:
+ s3.create_bucket(Bucket=bucket)
+ s3.put_object(
+ Bucket=bucket,
+ Key="work/shard_manifest.json",
+ Body=b"{}",
+ Metadata={"shardcount": "2"},
+ )
+
+ definition = json.dumps({
+ "StartAt": "Head",
+ "States": {
+ "Head": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:s3:headObject",
+ "Parameters": {"Bucket": bucket, "Key": "work/shard_manifest.json"},
+ "End": True,
+ },
+ },
+ })
+
+ sm_arn = sfn_sync.create_state_machine(
+ name=sm_name,
+ definition=definition,
+ roleArn="arn:aws:iam::000000000000:role/sfn-role",
+ )["stateMachineArn"]
+
+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))
+ assert resp["status"] == "SUCCEEDED", f"{resp.get('error')} — {resp.get('cause')}"
+ output = json.loads(resp["output"])
+ # Keys must NOT be run through the SFN key convention: "shardcount",
+ # not "Shardcount". $.Metadata.shardcount is the path real ASLs use.
+ assert output["Metadata"] == {"shardcount": "2"}
+ assert output["ContentLength"] == 2
+ finally:
+ try:
+ s3.delete_object(Bucket=bucket, Key="work/shard_manifest.json")
+ except Exception:
+ pass
+ try:
+ s3.delete_bucket(Bucket=bucket)
+ except Exception:
+ pass
+ if sm_arn:
+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+
+
+def test_sfn_aws_sdk_s3_head_object_metadata_result_selector(sfn_sync, s3):
+ """States.StringToJson($.Metadata.<key>) works on headObject output.
+
+ This is the real-world consumer shape: an ASL sizing a downstream fan-out
+ from a manifest object's metadata. It catches both failure modes — a
+ missing Metadata map and metadata keys mangled by the SFN key convention.
+ """
+ import uuid as _uuid
+
+ bucket = f"sfn-s3-headrs-{_uuid.uuid4().hex[:8]}"
+ sm_name = f"sdk-s3-headrs-{_uuid.uuid4().hex[:8]}"
+ sm_arn = None
+ try:
+ s3.create_bucket(Bucket=bucket)
+ s3.put_object(
+ Bucket=bucket,
+ Key="work/shard_manifest.json",
+ Body=b"{}",
+ Metadata={"shardcount": "2"},
+ )
+
+ definition = json.dumps({
+ "StartAt": "Head",
+ "States": {
+ "Head": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::aws-sdk:s3:headObject",
+ "Parameters": {"Bucket": bucket, "Key": "work/shard_manifest.json"},
+ "ResultSelector": {
+ "shard_count.$": "States.StringToJson($.Metadata.shardcount)",
+ },
+ "End": True,
+ },
+ },
+ })
+
+ sm_arn = sfn_sync.create_state_machine(
+ name=sm_name,
+ definition=definition,
+ roleArn="arn:aws:iam::000000000000:role/sfn-role",
+ )["stateMachineArn"]
+
+ resp = sfn_sync.start_sync_execution(stateMachineArn=sm_arn, input=json.dumps({}))
+ assert resp["status"] == "SUCCEEDED", f"{resp.get('error')} — {resp.get('cause')}"
+ output = json.loads(resp["output"])
+ assert output == {"shard_count": 2}
+ finally:
+ try:
+ s3.delete_object(Bucket=bucket, Key="work/shard_manifest.json")
+ except Exception:
+ pass
+ try:
+ s3.delete_bucket(Bucket=bucket)
+ except Exception:
+ pass
+ if sm_arn:
+ sfn_sync.delete_state_machine(stateMachineArn=sm_arn)
+
+
def test_sfn_aws_sdk_s3_list_buckets(sfn_sync, s3):
"""aws-sdk:s3:listBuckets returns the bucket list via REST-XML."""
import uuid as _uuid