Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(s3): apply the x-amz-grant-* headers (#2282)

iot-rocket9e100f4c64fb
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 06c1d7a1..c75824ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,8 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
### Fixed
+- **S3 — `x-amz-grant-*` headers** — `PutBucketAcl` and `PutObjectAcl` answered the grant headers with 400 `MissingSecurityHeader`, and `CreateBucket`, `PutObject` and `CopyObject` dropped them. On buckets with ACLs enabled all five now set an ACL of exactly the named grants, in the order S3 lists them, and refuse what S3 refuses: a malformed grant, an unknown group, an email grantee, and grant headers next to `x-amz-acl` or an ACL body. On a bucket with ACLs disabled (`BucketOwnerEnforced`) `PutBucketAcl` and `PutObjectAcl` refuse grant headers with 400 `AccessControlListNotSupported`. Contributed by @iot-rocket.
+- **S3 — Block Public Access settings and public bucket ACL grants** — any one of the four Block Public Access settings shut off every public grant on a bucket, none refused setting a public ACL, and a public bucket ACL opened every read and write in the bucket. `BlockPublicAcls` now refuses a call that sets a public ACL on a bucket with ACLs enabled (403 `AccessDenied`) and leaves the grants in place, `IgnorePublicAcls` drops public ACL grants and `RestrictPublicBuckets` the bucket policy. A bucket ACL grants what its permission covers: `READ` the listings, `WRITE` adding and removing objects, `READ_ACP` and `WRITE_ACP` the bucket ACL. Contributed by @iot-rocket.
- **IoT — `NaN` and `Infinity` in job documents and shadow updates** — `CreateJob` and `CreateJobTemplate` (inline and `documentSource`) and `UpdateThingShadow` (HTTP and the MQTT `update`, `get` and `delete` topics) accepted `NaN`, `Infinity` and `-Infinity`, and a shadow wrote them back as bare tokens, which is not JSON; they are now refused as invalid JSON with the messages AWS gives (`Payload contains invalid json` for shadows, also for any other invalid shadow payload), and a shadow number beyond a double's range, such as `1E400`, is stored as the string `"Infinity"` or `"-Infinity"` as on AWS. Contributed by @iot-rocket.
- **AppSync — GraphQL requests under `AUTH=true` follow the API's authentication type** — a GraphQL call to `/v1/apis/{apiId}/graphql` signed with an IAM key was authorized as `appsync:UpdateGraphqlApi` whatever the API's authentication type, and the API's own host served an `AWS_IAM` API to any signed call; an API key request now ignores its signature, and a SigV4 request to an `AWS_IAM` API is authorized on both endpoints as `appsync:GraphQL` on each top-level field, answering 401 `Permission denied` when no field is allowed, a field error for each denied one, and 400 `BadRequestException` for a wrong signature. Contributed by @iot-rocket.
- **Auth — a signed call to a REST API method without authorization is anonymous to the API policy** — under `AUTH=true` a SigV4-signed call to a method whose `authorizationType` is `NONE` was judged as its signer, so a user the API policy names, or one whose identity policy allows the invoke, got through; API Gateway evaluates every call to such a method as anonymous, signed or not, and now so does MiniStack, answering `ACCESS_DENIED` unless the policy allows everyone. Contributed by @iot-rocket.
diff --git a/ministack/services/s3.py b/ministack/services/s3.py
index b4316730..06778a7d 100644
--- a/ministack/services/s3.py
+++ b/ministack/services/s3.py
@@ -2645,6 +2645,12 @@ def _create_bucket(name: str, body: bytes, headers: dict = None, endpoint_region
canned_acl = headers.get("x-amz-acl")
if canned_acl and canned_acl not in _CANNED_BUCKET_ACLS:
return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)
+ current = _buckets.get(name)
+ enforced = (_bucket_owner_enforced(current) if current
+ else headers.get("x-amz-object-ownership", "BucketOwnerEnforced") == "BucketOwnerEnforced")
+ grant_acl, err = _grant_headers_acl(headers, enforced)
+ if err:
+ return err
if endpoint_region:
err = _location_constraint_error(name, _location_constraint(body), endpoint_region)
if err is not None:
@@ -2659,6 +2665,8 @@ def _create_bucket(name: str, body: bytes, headers: dict = None, endpoint_region
_bucket_acl.pop(name, None)
if canned_acl:
_bucket_acl[name] = _canned_acl_policy_xml(canned_acl, _canonical_owner_id())
+ elif grant_acl:
+ _bucket_acl[name] = grant_acl
return 200, {"Location": f"/{name}"}, b""
if _bucket_owner_account(name) is not None:
# "After creating a general purpose bucket in the shared global
@@ -2709,6 +2717,8 @@ def _create_bucket(name: str, body: bytes, headers: dict = None, endpoint_region
_bucket_tags[name] = tags
if canned_acl:
_bucket_acl[name] = _canned_acl_policy_xml(canned_acl, _canonical_owner_id())
+ elif grant_acl:
+ _bucket_acl[name] = grant_acl
if headers.get("x-amz-bucket-object-lock-enabled", "").lower() == "true":
_bucket_object_lock[name] = {"enabled": True, "default_retention": None}
@@ -3238,6 +3248,10 @@ def _put_bucket_acl(name: str, body: bytes, headers: dict | None = None):
refused = _public_acl_refused(name, headers, body)
if refused:
return refused
+ enforced = _bucket_owner_enforced(_buckets[name])
+ grant_acl, err = _grant_headers_acl(headers, enforced)
+ if err:
+ return err
# Canned ACL from x-amz-acl header takes precedence and is mutually
# exclusive with an XML body per the AWS API reference. SDK callers send
@@ -3251,9 +3265,14 @@ def _put_bucket_acl(name: str, body: bytes, headers: dict | None = None):
return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned}", 400)
_bucket_acl[name] = _canned_acl_policy_xml(canned, _canonical_owner_id())
return 200, {}, b""
+ if grant_acl:
+ if body:
+ return _error("UnexpectedContent", "This request does not support content", 400)
+ _bucket_acl[name] = grant_acl
+ return 200, {}, b""
if not body:
- return _error("MissingSecurityHeader", "Your request was missing a required header.", 400)
+ return _missing_acl_error(headers, enforced)
try:
# Validate XML well-formedness — real AWS rejects malformed bodies
# with MalformedACLError. We don't enforce grantee/permission
@@ -4357,6 +4376,9 @@ def _put_object(bucket_name: str, key: str, body: bytes, headers: dict):
canned_acl = headers.get("x-amz-acl")
if canned_acl and canned_acl not in _CANNED_OBJECT_ACLS:
return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)
+ grant_acl, err = _grant_headers_acl(headers, _bucket_owner_enforced(bucket))
+ if err:
+ return err
refused = _public_acl_refused(bucket_name, headers)
if refused:
return refused
@@ -4403,6 +4425,8 @@ def _put_object(bucket_name: str, key: str, body: bytes, headers: dict):
_object_acl[(bucket_name, key, obj.get("version_id"))] = _canned_acl_policy_xml(
canned_acl, _canonical_owner_id()
)
+ elif grant_acl:
+ _object_acl[(bucket_name, key, obj.get("version_id"))] = grant_acl
# Persist only after the versioning block: the .meta.json sidecar must
# carry the version_id assigned above (#1058), and the tags and ACL.
@@ -5910,6 +5934,9 @@ def _copy_object(bucket_name: str, dest_key: str, headers: dict):
canned_acl = headers.get("x-amz-acl")
if canned_acl and canned_acl not in _CANNED_OBJECT_ACLS:
return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)
+ grant_acl, err = _grant_headers_acl(headers, _bucket_owner_enforced(dest_bucket))
+ if err:
+ return err
refused = _public_acl_refused(bucket_name, headers)
if refused:
return refused
@@ -6113,6 +6140,8 @@ def _copy_object(bucket_name: str, dest_key: str, headers: dict):
_object_acl[(bucket_name, dest_key, dest_version_id)] = _canned_acl_policy_xml(
canned_acl, _canonical_owner_id()
)
+ elif grant_acl:
+ _object_acl[(bucket_name, dest_key, dest_version_id)] = grant_acl
else:
# The destination is a new object: it does not inherit whatever the
# key it replaced was permissioned with.
@@ -7011,6 +7040,59 @@ def _canned_acl_policy_xml(canned: str, owner_id: str) -> str:
)
+# The x-amz-grant-* headers in the order S3 lists their grants back.
+_GRANT_HEADER_PERMISSIONS = (
+ ("x-amz-grant-read", "READ"),
+ ("x-amz-grant-write", "WRITE"),
+ ("x-amz-grant-full-control", "FULL_CONTROL"),
+ ("x-amz-grant-read-acp", "READ_ACP"),
+ ("x-amz-grant-write-acp", "WRITE_ACP"),
+)
+
+
+def _grant_headers_acl(headers: dict, enforced: bool) -> tuple[str | None, tuple | None]:
+ """The ACL the request's ``x-amz-grant-*`` headers set (None without any or with ACLs disabled), or their error."""
+ present = [(headers[name], permission) for name, permission in _GRANT_HEADER_PERMISSIONS if headers.get(name)]
+ if not present or enforced:
+ return None, None
+ if headers.get("x-amz-acl"):
+ return None, _error("InvalidRequest", "Specifying both Canned ACLs and Header Grants is not allowed", 400)
+ grants = []
+ for value, permission in present:
+ for item in value.split(","):
+ kind, _, grantee = item.partition("=")
+ kind, grantee = kind.strip().lower(), grantee.strip().strip('"')
+ if kind == "id" and re.fullmatch(r"[0-9a-fA-F]{64}", grantee):
+ grantee_xml = f'xsi:type="CanonicalUser"><ID>{grantee}</ID>'
+ elif kind == "uri" and grantee in (_ACL_GROUP_ALL_USERS, _ACL_GROUP_AUTH_USERS, _ACL_GROUP_LOG_DELIVERY):
+ grantee_xml = f'xsi:type="Group"><URI>{grantee}</URI>'
+ elif kind in ("id", "uri"):
+ return None, _error("InvalidArgument", "Invalid id" if kind == "id" else "Invalid group uri", 400)
+ elif kind == "emailaddress":
+ return None, _error("UnsupportedArgument", "The request contained an unsupported argument.", 400)
+ else:
+ return None, _error("InvalidArgument", "Argument format not recognized", 400)
+ grants.append(
+ '<Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
+ f"{grantee_xml}</Grantee><Permission>{permission}</Permission></Grant>"
+ )
+ owner_id = _canonical_owner_id()
+ return (
+ XML_DECL.decode() + "\n"
+ f'<AccessControlPolicy xmlns="{S3_NS}">'
+ f"<Owner><ID>{owner_id}</ID><DisplayName>ministack</DisplayName></Owner>"
+ f'<AccessControlList>{"".join(grants)}</AccessControlList>'
+ "</AccessControlPolicy>"
+ ), None
+
+
+def _missing_acl_error(headers: dict, enforced: bool) -> tuple:
+ """The error for an ACL write with neither a canned ACL nor a body (grant headers with ACLs disabled included)."""
+ if enforced and any(headers.get(name) for name, _ in _GRANT_HEADER_PERMISSIONS):
+ return _error("AccessControlListNotSupported", "The bucket does not allow ACLs", 400)
+ return _error("MissingSecurityHeader", "Your request was missing a required header.", 400)
+
+
def _default_object_acl_xml() -> bytes:
"""Default ACL real AWS returns when no ACL has been set on an object:
a single Grant of FULL_CONTROL to the bucket owner (CanonicalUser).
@@ -7079,6 +7161,10 @@ def _put_object_acl(bucket_name: str, key: str, body: bytes, headers: dict, quer
refused = _public_acl_refused(bucket_name, headers, body)
if refused:
return refused
+ enforced = _bucket_owner_enforced(bucket)
+ grant_acl, err = _grant_headers_acl(headers, enforced)
+ if err:
+ return err
# Canned ACL from x-amz-acl header takes precedence and is mutually
# exclusive with an XML body per the AWS API reference. Either path
@@ -7091,9 +7177,15 @@ def _put_object_acl(bucket_name: str, key: str, body: bytes, headers: dict, quer
_object_acl[(bucket_name, key, version_id)] = _canned_acl_policy_xml(canned, _canonical_owner_id())
_persist_version_state(bucket_name, key, bucket)
return 200, {}, b""
+ if grant_acl:
+ if body:
+ return _error("UnexpectedContent", "This request does not support content", 400)
+ _object_acl[(bucket_name, key, version_id)] = grant_acl
+ _persist_version_state(bucket_name, key, bucket)
+ return 200, {}, b""
if not body:
- return _error("MissingSecurityHeader", "Your request was missing a required header.", 400)
+ return _missing_acl_error(headers, enforced)
try:
# Validate XML well-formedness — real AWS rejects malformed bodies
# with MalformedACLError. We don't enforce grantee/permission
diff --git a/tests/test_s3.py b/tests/test_s3.py
index a5b902cd..626cdc05 100644
--- a/tests/test_s3.py
+++ b/tests/test_s3.py
@@ -3285,6 +3285,62 @@ def test_s3_put_bucket_acl_xml_body(s3):
assert perms == ["FULL_CONTROL", "READ"]
s3.delete_bucket(Bucket=bucket)
+def _grant_pairs(acl):
+ """``(ID or URI, permission)`` for each grant of a GetBucketAcl/GetObjectAcl response, in order."""
+ return [(g["Grantee"].get("ID") or g["Grantee"].get("URI"), g["Permission"]) for g in acl["Grants"]]
+
+
+def test_s3_bucket_acl_grant_headers(s3):
+ """x-amz-grant-* headers on CreateBucket and PutBucketAcl set exactly the grants they name."""
+ bucket = f"acl-bkt-grant-{uuid.uuid4().hex[:8]}"
+ owner_id = s3.list_buckets()["Owner"]["ID"]
+ other_id = "c4c1ede66af53448b93c283ce9448c4ba468c9432aa01d700d3878632f77d2d0"
+ log_delivery = "http://acs.amazonaws.com/groups/s3/LogDelivery"
+ owner, other, logs = f'id="{owner_id}"', f'id="{other_id}"', f'uri="{log_delivery}"'
+ s3.create_bucket(Bucket=bucket, ObjectOwnership="BucketOwnerPreferred", GrantFullControl=owner, GrantRead=other)
+ try:
+ assert _grant_pairs(s3.get_bucket_acl(Bucket=bucket)) == [(other_id, "READ"), (owner_id, "FULL_CONTROL")]
+
+ s3.put_bucket_acl(Bucket=bucket, GrantFullControl=owner, GrantRead=f"{other}, {logs}", GrantWrite=logs,
+ GrantReadACP=other, GrantWriteACP=logs)
+ expected = [(other_id, "READ"), (log_delivery, "READ"), (log_delivery, "WRITE"),
+ (owner_id, "FULL_CONTROL"), (other_id, "READ_ACP"), (log_delivery, "WRITE_ACP")]
+ assert _grant_pairs(s3.get_bucket_acl(Bucket=bucket)) == expected
+
+ policy = {"Owner": {"ID": owner_id}, "Grants": [
+ {"Grantee": {"Type": "CanonicalUser", "ID": owner_id}, "Permission": "FULL_CONTROL"}]}
+ for kwargs, code in (
+ ({"GrantRead": other, "ACL": "private"}, "InvalidRequest"),
+ ({"GrantRead": other, "AccessControlPolicy": policy}, "UnexpectedContent"),
+ ({"GrantRead": "bogus"}, "InvalidArgument"),
+ ({"GrantRead": 'id="abc"'}, "InvalidArgument"),
+ ({"GrantRead": 'uri="http://acs.amazonaws.com/groups/global/Nobody"'}, "InvalidArgument"),
+ ({"GrantRead": 'emailAddress="[email protected]"'}, "UnsupportedArgument"),
+ ):
+ with pytest.raises(ClientError) as exc:
+ s3.put_bucket_acl(Bucket=bucket, **kwargs)
+ assert exc.value.response["Error"]["Code"] == code, kwargs
+ assert _grant_pairs(s3.get_bucket_acl(Bucket=bucket)) == expected
+
+ s3.put_bucket_acl(Bucket=bucket, GrantRead=f"ID={owner_id}")
+ assert _grant_pairs(s3.get_bucket_acl(Bucket=bucket)) == [(owner_id, "READ")]
+ with pytest.raises(ClientError) as exc:
+ s3.create_bucket(Bucket=f"{bucket}-x", ObjectOwnership="ObjectWriter", GrantRead="bogus")
+ assert exc.value.response["Error"]["Code"] == "InvalidArgument"
+
+ # BucketOwnerEnforced, the default, disables ACLs: grant headers on an ACL write are refused.
+ s3.create_bucket(Bucket=f"{bucket}-e")
+ s3.put_object(Bucket=f"{bucket}-e", Key="k", Body=b"x")
+ for call, kwargs in ((s3.put_bucket_acl, {}), (s3.put_object_acl, {"Key": "k"})):
+ with pytest.raises(ClientError) as exc:
+ call(Bucket=f"{bucket}-e", GrantRead=other, **kwargs)
+ assert exc.value.response["Error"]["Code"] == "AccessControlListNotSupported"
+ finally:
+ s3.delete_bucket(Bucket=bucket)
+ s3.delete_object(Bucket=f"{bucket}-e", Key="k")
+ s3.delete_bucket(Bucket=f"{bucket}-e")
+
+
def test_s3_range_suffix(s3):
"""Range: bytes=-N returns last N bytes."""
s3.create_bucket(Bucket="qa-s3-range-suffix")
@@ -5441,6 +5497,43 @@ def test_s3_object_acl_grant_to_another_account_follows_the_version(s3):
_purge_foreign(owner, bucket)
+def test_s3_object_grant_headers_set_the_object_acl(s3):
+ """x-amz-grant-* headers on PutObject and PutObjectAcl set the object ACL, which then answers reads."""
+ owner, reader = _s3_account("111111111111"), _s3_account("222222222222")
+ owner_id, reader_id = owner.list_buckets()["Owner"]["ID"], reader.list_buckets()["Owner"]["ID"]
+ bucket = f"acl-grant-hdr-{uuid.uuid4().hex[:8]}"
+ _acl_enabled_bucket(owner, bucket)
+ try:
+ owner.put_object(Bucket=bucket, Key="k", Body=b"one", GrantRead=f'id="{reader_id}"')
+ assert _grant_pairs(owner.get_object_acl(Bucket=bucket, Key="k")) == [(reader_id, "READ")]
+ assert reader.get_object(Bucket=bucket, Key="k")["Body"].read() == b"one"
+
+ owner.put_object_acl(Bucket=bucket, Key="k", GrantFullControl=f'id="{owner_id}"',
+ GrantReadACP=f'id="{reader_id}"')
+ assert _grant_pairs(owner.get_object_acl(Bucket=bucket, Key="k")) == [
+ (owner_id, "FULL_CONTROL"), (reader_id, "READ_ACP")]
+ assert _denied(reader.get_object, Bucket=bucket, Key="k")
+ assert _grant_pairs(reader.get_object_acl(Bucket=bucket, Key="k"))[1] == (reader_id, "READ_ACP")
+
+ policy = {"Owner": {"ID": owner_id}, "Grants": [
+ {"Grantee": {"Type": "CanonicalUser", "ID": owner_id}, "Permission": "FULL_CONTROL"}]}
+ grant = f'id="{reader_id}"'
+ for call, kwargs, code in (
+ (owner.put_object, {"Key": "n", "Body": b"x", "GrantRead": grant, "ACL": "private"}, "InvalidRequest"),
+ (owner.put_object, {"Key": "n", "Body": b"x", "GrantRead": "bogus"}, "InvalidArgument"),
+ (owner.put_object_acl, {"Key": "k", "GrantRead": grant, "ACL": "private"}, "InvalidRequest"),
+ (owner.put_object_acl, {"Key": "k", "GrantRead": grant, "AccessControlPolicy": policy},
+ "UnexpectedContent"),
+ ):
+ with pytest.raises(ClientError) as exc:
+ call(Bucket=bucket, **kwargs)
+ assert exc.value.response["Error"]["Code"] == code, kwargs
+ assert owner.list_objects_v2(Bucket=bucket)["KeyCount"] == 1
+ assert len(owner.get_object_acl(Bucket=bucket, Key="k")["Grants"]) == 2
+ finally:
+ _purge_foreign(owner, bucket)
+
+
def test_s3_public_read_object_acl_opens_reads_to_another_account(s3):
"""A public-read object ACL answers another account's read until IgnorePublicAcls is set."""
owner, reader = _s3_account("111111111111"), _s3_account("222222222222")
@@ -6974,6 +7067,26 @@ def test_s3_copy_object_applies_canned_acl(s3):
assert exc.value.response["Error"]["Code"] == "InvalidArgument"
+def test_s3_copy_object_applies_grant_headers(s3):
+ """x-amz-grant-* headers on a copy set the destination's ACL; together with x-amz-acl the copy is refused."""
+ bkt = f"s3-copy-grant-{uuid.uuid4().hex[:8]}"
+ _acl_enabled_bucket(s3, bkt)
+ owner_id = s3.list_buckets()["Owner"]["ID"]
+ other_id = "c4c1ede66af53448b93c283ce9448c4ba468c9432aa01d700d3878632f77d2d0"
+ s3.put_object(Bucket=bkt, Key="src.txt", Body=b"body")
+ source = {"Bucket": bkt, "Key": "src.txt"}
+
+ s3.copy_object(Bucket=bkt, Key="dst.txt", CopySource=source, GrantRead=f'id="{other_id}"',
+ GrantFullControl=f'id="{owner_id}"')
+ assert _grant_pairs(s3.get_object_acl(Bucket=bkt, Key="dst.txt")) == [
+ (other_id, "READ"), (owner_id, "FULL_CONTROL")]
+
+ with pytest.raises(ClientError) as exc:
+ s3.copy_object(Bucket=bkt, Key="both.txt", CopySource=source, GrantRead=f'id="{other_id}"', ACL="private")
+ assert exc.value.response["Error"]["Code"] == "InvalidRequest"
+ assert [o["Key"] for o in s3.list_objects_v2(Bucket=bkt)["Contents"]] == ["dst.txt", "src.txt"]
+
+
def test_s3_upload_part_copy_honours_source_preconditions(s3):
"""UploadPartCopy carries the same copy-source conditions CopyObject does."""
bkt = "s3-upc-precond"