Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(s3): apply the x-amz-grant-* headers (#2282)
iot-rocket9e100f4c64fb
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 06c1d7a1..c75824ba 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,8 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]### Fixed+- **S3 — `x-amz-grant-*` headers** — `PutBucketAcl` and `PutObjectAcl` answered the grant headers with 400 `MissingSecurityHeader`, and `CreateBucket`, `PutObject` and `CopyObject` dropped them. On buckets with ACLs enabled all five now set an ACL of exactly the named grants, in the order S3 lists them, and refuse what S3 refuses: a malformed grant, an unknown group, an email grantee, and grant headers next to `x-amz-acl` or an ACL body. On a bucket with ACLs disabled (`BucketOwnerEnforced`) `PutBucketAcl` and `PutObjectAcl` refuse grant headers with 400 `AccessControlListNotSupported`. Contributed by @iot-rocket.+- **S3 — Block Public Access settings and public bucket ACL grants** — any one of the four Block Public Access settings shut off every public grant on a bucket, none refused setting a public ACL, and a public bucket ACL opened every read and write in the bucket. `BlockPublicAcls` now refuses a call that sets a public ACL on a bucket with ACLs enabled (403 `AccessDenied`) and leaves the grants in place, `IgnorePublicAcls` drops public ACL grants and `RestrictPublicBuckets` the bucket policy. A bucket ACL grants what its permission covers: `READ` the listings, `WRITE` adding and removing objects, `READ_ACP` and `WRITE_ACP` the bucket ACL. Contributed by @iot-rocket.- **IoT — `NaN` and `Infinity` in job documents and shadow updates** — `CreateJob` and `CreateJobTemplate` (inline and `documentSource`) and `UpdateThingShadow` (HTTP and the MQTT `update`, `get` and `delete` topics) accepted `NaN`, `Infinity` and `-Infinity`, and a shadow wrote them back as bare tokens, which is not JSON; they are now refused as invalid JSON with the messages AWS gives (`Payload contains invalid json` for shadows, also for any other invalid shadow payload), and a shadow number beyond a double's range, such as `1E400`, is stored as the string `"Infinity"` or `"-Infinity"` as on AWS. Contributed by @iot-rocket.- **AppSync — GraphQL requests under `AUTH=true` follow the API's authentication type** — a GraphQL call to `/v1/apis/{apiId}/graphql` signed with an IAM key was authorized as `appsync:UpdateGraphqlApi` whatever the API's authentication type, and the API's own host served an `AWS_IAM` API to any signed call; an API key request now ignores its signature, and a SigV4 request to an `AWS_IAM` API is authorized on both endpoints as `appsync:GraphQL` on each top-level field, answering 401 `Permission denied` when no field is allowed, a field error for each denied one, and 400 `BadRequestException` for a wrong signature. Contributed by @iot-rocket.- **Auth — a signed call to a REST API method without authorization is anonymous to the API policy** — under `AUTH=true` a SigV4-signed call to a method whose `authorizationType` is `NONE` was judged as its signer, so a user the API policy names, or one whose identity policy allows the invoke, got through; API Gateway evaluates every call to such a method as anonymous, signed or not, and now so does MiniStack, answering `ACCESS_DENIED` unless the policy allows everyone. Contributed by @iot-rocket.diff --git a/ministack/services/s3.py b/ministack/services/s3.pyindex b4316730..06778a7d 100644--- a/ministack/services/s3.py+++ b/ministack/services/s3.py@@ -2645,6 +2645,12 @@ def _create_bucket(name: str, body: bytes, headers: dict = None, endpoint_regioncanned_acl = headers.get("x-amz-acl")if canned_acl and canned_acl not in _CANNED_BUCKET_ACLS:return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)+ current = _buckets.get(name)+ enforced = (_bucket_owner_enforced(current) if current+ else headers.get("x-amz-object-ownership", "BucketOwnerEnforced") == "BucketOwnerEnforced")+ grant_acl, err = _grant_headers_acl(headers, enforced)+ if err:+ return errif endpoint_region:err = _location_constraint_error(name, _location_constraint(body), endpoint_region)if err is not None:@@ -2659,6 +2665,8 @@ def _create_bucket(name: str, body: bytes, headers: dict = None, endpoint_region_bucket_acl.pop(name, None)if canned_acl:_bucket_acl[name] = _canned_acl_policy_xml(canned_acl, _canonical_owner_id())+ elif grant_acl:+ _bucket_acl[name] = grant_aclreturn 200, {"Location": f"/{name}"}, b""if _bucket_owner_account(name) is not None:# "After creating a general purpose bucket in the shared global@@ -2709,6 +2717,8 @@ def _create_bucket(name: str, body: bytes, headers: dict = None, endpoint_region_bucket_tags[name] = tagsif canned_acl:_bucket_acl[name] = _canned_acl_policy_xml(canned_acl, _canonical_owner_id())+ elif grant_acl:+ _bucket_acl[name] = grant_aclif headers.get("x-amz-bucket-object-lock-enabled", "").lower() == "true":_bucket_object_lock[name] = {"enabled": True, "default_retention": None}@@ -3238,6 +3248,10 @@ def _put_bucket_acl(name: str, body: bytes, headers: dict | None = None):refused = _public_acl_refused(name, headers, body)if refused:return refused+ enforced = _bucket_owner_enforced(_buckets[name])+ grant_acl, err = _grant_headers_acl(headers, enforced)+ if err:+ return err# Canned ACL from x-amz-acl header takes precedence and is mutually# exclusive with an XML body per the AWS API reference. SDK callers send@@ -3251,9 +3265,14 @@ def _put_bucket_acl(name: str, body: bytes, headers: dict | None = None):return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned}", 400)_bucket_acl[name] = _canned_acl_policy_xml(canned, _canonical_owner_id())return 200, {}, b""+ if grant_acl:+ if body:+ return _error("UnexpectedContent", "This request does not support content", 400)+ _bucket_acl[name] = grant_acl+ return 200, {}, b""if not body:- return _error("MissingSecurityHeader", "Your request was missing a required header.", 400)+ return _missing_acl_error(headers, enforced)try:# Validate XML well-formedness — real AWS rejects malformed bodies# with MalformedACLError. We don't enforce grantee/permission@@ -4357,6 +4376,9 @@ def _put_object(bucket_name: str, key: str, body: bytes, headers: dict):canned_acl = headers.get("x-amz-acl")if canned_acl and canned_acl not in _CANNED_OBJECT_ACLS:return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)+ grant_acl, err = _grant_headers_acl(headers, _bucket_owner_enforced(bucket))+ if err:+ return errrefused = _public_acl_refused(bucket_name, headers)if refused:return refused@@ -4403,6 +4425,8 @@ def _put_object(bucket_name: str, key: str, body: bytes, headers: dict):_object_acl[(bucket_name, key, obj.get("version_id"))] = _canned_acl_policy_xml(canned_acl, _canonical_owner_id())+ elif grant_acl:+ _object_acl[(bucket_name, key, obj.get("version_id"))] = grant_acl# Persist only after the versioning block: the .meta.json sidecar must# carry the version_id assigned above (#1058), and the tags and ACL.@@ -5910,6 +5934,9 @@ def _copy_object(bucket_name: str, dest_key: str, headers: dict):canned_acl = headers.get("x-amz-acl")if canned_acl and canned_acl not in _CANNED_OBJECT_ACLS:return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)+ grant_acl, err = _grant_headers_acl(headers, _bucket_owner_enforced(dest_bucket))+ if err:+ return errrefused = _public_acl_refused(bucket_name, headers)if refused:return refused@@ -6113,6 +6140,8 @@ def _copy_object(bucket_name: str, dest_key: str, headers: dict):_object_acl[(bucket_name, dest_key, dest_version_id)] = _canned_acl_policy_xml(canned_acl, _canonical_owner_id())+ elif grant_acl:+ _object_acl[(bucket_name, dest_key, dest_version_id)] = grant_aclelse:# The destination is a new object: it does not inherit whatever the# key it replaced was permissioned with.@@ -7011,6 +7040,59 @@ def _canned_acl_policy_xml(canned: str, owner_id: str) -> str:)+# The x-amz-grant-* headers in the order S3 lists their grants back.+_GRANT_HEADER_PERMISSIONS = (+ ("x-amz-grant-read", "READ"),+ ("x-amz-grant-write", "WRITE"),+ ("x-amz-grant-full-control", "FULL_CONTROL"),+ ("x-amz-grant-read-acp", "READ_ACP"),+ ("x-amz-grant-write-acp", "WRITE_ACP"),+)+++def _grant_headers_acl(headers: dict, enforced: bool) -> tuple[str | None, tuple | None]:+ """The ACL the request's ``x-amz-grant-*`` headers set (None without any or with ACLs disabled), or their error."""+ present = [(headers[name], permission) for name, permission in _GRANT_HEADER_PERMISSIONS if headers.get(name)]+ if not present or enforced:+ return None, None+ if headers.get("x-amz-acl"):+ return None, _error("InvalidRequest", "Specifying both Canned ACLs and Header Grants is not allowed", 400)+ grants = []+ for value, permission in present:+ for item in value.split(","):+ kind, _, grantee = item.partition("=")+ kind, grantee = kind.strip().lower(), grantee.strip().strip('"')+ if kind == "id" and re.fullmatch(r"[0-9a-fA-F]{64}", grantee):+ grantee_xml = f'xsi:type="CanonicalUser"><ID>{grantee}</ID>'+ elif kind == "uri" and grantee in (_ACL_GROUP_ALL_USERS, _ACL_GROUP_AUTH_USERS, _ACL_GROUP_LOG_DELIVERY):+ grantee_xml = f'xsi:type="Group"><URI>{grantee}</URI>'+ elif kind in ("id", "uri"):+ return None, _error("InvalidArgument", "Invalid id" if kind == "id" else "Invalid group uri", 400)+ elif kind == "emailaddress":+ return None, _error("UnsupportedArgument", "The request contained an unsupported argument.", 400)+ else:+ return None, _error("InvalidArgument", "Argument format not recognized", 400)+ grants.append(+ '<Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '+ f"{grantee_xml}</Grantee><Permission>{permission}</Permission></Grant>"+ )+ owner_id = _canonical_owner_id()+ return (+ XML_DECL.decode() + "\n"+ f'<AccessControlPolicy xmlns="{S3_NS}">'+ f"<Owner><ID>{owner_id}</ID><DisplayName>ministack</DisplayName></Owner>"+ f'<AccessControlList>{"".join(grants)}</AccessControlList>'+ "</AccessControlPolicy>"+ ), None+++def _missing_acl_error(headers: dict, enforced: bool) -> tuple:+ """The error for an ACL write with neither a canned ACL nor a body (grant headers with ACLs disabled included)."""+ if enforced and any(headers.get(name) for name, _ in _GRANT_HEADER_PERMISSIONS):+ return _error("AccessControlListNotSupported", "The bucket does not allow ACLs", 400)+ return _error("MissingSecurityHeader", "Your request was missing a required header.", 400)++def _default_object_acl_xml() -> bytes:"""Default ACL real AWS returns when no ACL has been set on an object:a single Grant of FULL_CONTROL to the bucket owner (CanonicalUser).@@ -7079,6 +7161,10 @@ def _put_object_acl(bucket_name: str, key: str, body: bytes, headers: dict, querrefused = _public_acl_refused(bucket_name, headers, body)if refused:return refused+ enforced = _bucket_owner_enforced(bucket)+ grant_acl, err = _grant_headers_acl(headers, enforced)+ if err:+ return err# Canned ACL from x-amz-acl header takes precedence and is mutually# exclusive with an XML body per the AWS API reference. Either path@@ -7091,9 +7177,15 @@ def _put_object_acl(bucket_name: str, key: str, body: bytes, headers: dict, quer_object_acl[(bucket_name, key, version_id)] = _canned_acl_policy_xml(canned, _canonical_owner_id())_persist_version_state(bucket_name, key, bucket)return 200, {}, b""+ if grant_acl:+ if body:+ return _error("UnexpectedContent", "This request does not support content", 400)+ _object_acl[(bucket_name, key, version_id)] = grant_acl+ _persist_version_state(bucket_name, key, bucket)+ return 200, {}, b""if not body:- return _error("MissingSecurityHeader", "Your request was missing a required header.", 400)+ return _missing_acl_error(headers, enforced)try:# Validate XML well-formedness — real AWS rejects malformed bodies# with MalformedACLError. We don't enforce grantee/permissiondiff --git a/tests/test_s3.py b/tests/test_s3.pyindex a5b902cd..626cdc05 100644--- a/tests/test_s3.py+++ b/tests/test_s3.py@@ -3285,6 +3285,62 @@ def test_s3_put_bucket_acl_xml_body(s3):assert perms == ["FULL_CONTROL", "READ"]s3.delete_bucket(Bucket=bucket)+def _grant_pairs(acl):+ """``(ID or URI, permission)`` for each grant of a GetBucketAcl/GetObjectAcl response, in order."""+ return [(g["Grantee"].get("ID") or g["Grantee"].get("URI"), g["Permission"]) for g in acl["Grants"]]+++def test_s3_bucket_acl_grant_headers(s3):+ """x-amz-grant-* headers on CreateBucket and PutBucketAcl set exactly the grants they name."""+ bucket = f"acl-bkt-grant-{uuid.uuid4().hex[:8]}"+ owner_id = s3.list_buckets()["Owner"]["ID"]+ other_id = "c4c1ede66af53448b93c283ce9448c4ba468c9432aa01d700d3878632f77d2d0"+ log_delivery = "http://acs.amazonaws.com/groups/s3/LogDelivery"+ owner, other, logs = f'id="{owner_id}"', f'id="{other_id}"', f'uri="{log_delivery}"'+ s3.create_bucket(Bucket=bucket, ObjectOwnership="BucketOwnerPreferred", GrantFullControl=owner, GrantRead=other)+ try:+ assert _grant_pairs(s3.get_bucket_acl(Bucket=bucket)) == [(other_id, "READ"), (owner_id, "FULL_CONTROL")]++ s3.put_bucket_acl(Bucket=bucket, GrantFullControl=owner, GrantRead=f"{other}, {logs}", GrantWrite=logs,+ GrantReadACP=other, GrantWriteACP=logs)+ expected = [(other_id, "READ"), (log_delivery, "READ"), (log_delivery, "WRITE"),+ (owner_id, "FULL_CONTROL"), (other_id, "READ_ACP"), (log_delivery, "WRITE_ACP")]+ assert _grant_pairs(s3.get_bucket_acl(Bucket=bucket)) == expected++ policy = {"Owner": {"ID": owner_id}, "Grants": [+ {"Grantee": {"Type": "CanonicalUser", "ID": owner_id}, "Permission": "FULL_CONTROL"}]}+ for kwargs, code in (+ ({"GrantRead": other, "ACL": "private"}, "InvalidRequest"),+ ({"GrantRead": other, "AccessControlPolicy": policy}, "UnexpectedContent"),+ ({"GrantRead": "bogus"}, "InvalidArgument"),+ ({"GrantRead": 'id="abc"'}, "InvalidArgument"),+ ({"GrantRead": 'uri="http://acs.amazonaws.com/groups/global/Nobody"'}, "InvalidArgument"),+ ({"GrantRead": 'emailAddress="[email protected]"'}, "UnsupportedArgument"),+ ):+ with pytest.raises(ClientError) as exc:+ s3.put_bucket_acl(Bucket=bucket, **kwargs)+ assert exc.value.response["Error"]["Code"] == code, kwargs+ assert _grant_pairs(s3.get_bucket_acl(Bucket=bucket)) == expected++ s3.put_bucket_acl(Bucket=bucket, GrantRead=f"ID={owner_id}")+ assert _grant_pairs(s3.get_bucket_acl(Bucket=bucket)) == [(owner_id, "READ")]+ with pytest.raises(ClientError) as exc:+ s3.create_bucket(Bucket=f"{bucket}-x", ObjectOwnership="ObjectWriter", GrantRead="bogus")+ assert exc.value.response["Error"]["Code"] == "InvalidArgument"++ # BucketOwnerEnforced, the default, disables ACLs: grant headers on an ACL write are refused.+ s3.create_bucket(Bucket=f"{bucket}-e")+ s3.put_object(Bucket=f"{bucket}-e", Key="k", Body=b"x")+ for call, kwargs in ((s3.put_bucket_acl, {}), (s3.put_object_acl, {"Key": "k"})):+ with pytest.raises(ClientError) as exc:+ call(Bucket=f"{bucket}-e", GrantRead=other, **kwargs)+ assert exc.value.response["Error"]["Code"] == "AccessControlListNotSupported"+ finally:+ s3.delete_bucket(Bucket=bucket)+ s3.delete_object(Bucket=f"{bucket}-e", Key="k")+ s3.delete_bucket(Bucket=f"{bucket}-e")++def test_s3_range_suffix(s3):"""Range: bytes=-N returns last N bytes."""s3.create_bucket(Bucket="qa-s3-range-suffix")@@ -5441,6 +5497,43 @@ def test_s3_object_acl_grant_to_another_account_follows_the_version(s3):_purge_foreign(owner, bucket)+def test_s3_object_grant_headers_set_the_object_acl(s3):+ """x-amz-grant-* headers on PutObject and PutObjectAcl set the object ACL, which then answers reads."""+ owner, reader = _s3_account("111111111111"), _s3_account("222222222222")+ owner_id, reader_id = owner.list_buckets()["Owner"]["ID"], reader.list_buckets()["Owner"]["ID"]+ bucket = f"acl-grant-hdr-{uuid.uuid4().hex[:8]}"+ _acl_enabled_bucket(owner, bucket)+ try:+ owner.put_object(Bucket=bucket, Key="k", Body=b"one", GrantRead=f'id="{reader_id}"')+ assert _grant_pairs(owner.get_object_acl(Bucket=bucket, Key="k")) == [(reader_id, "READ")]+ assert reader.get_object(Bucket=bucket, Key="k")["Body"].read() == b"one"++ owner.put_object_acl(Bucket=bucket, Key="k", GrantFullControl=f'id="{owner_id}"',+ GrantReadACP=f'id="{reader_id}"')+ assert _grant_pairs(owner.get_object_acl(Bucket=bucket, Key="k")) == [+ (owner_id, "FULL_CONTROL"), (reader_id, "READ_ACP")]+ assert _denied(reader.get_object, Bucket=bucket, Key="k")+ assert _grant_pairs(reader.get_object_acl(Bucket=bucket, Key="k"))[1] == (reader_id, "READ_ACP")++ policy = {"Owner": {"ID": owner_id}, "Grants": [+ {"Grantee": {"Type": "CanonicalUser", "ID": owner_id}, "Permission": "FULL_CONTROL"}]}+ grant = f'id="{reader_id}"'+ for call, kwargs, code in (+ (owner.put_object, {"Key": "n", "Body": b"x", "GrantRead": grant, "ACL": "private"}, "InvalidRequest"),+ (owner.put_object, {"Key": "n", "Body": b"x", "GrantRead": "bogus"}, "InvalidArgument"),+ (owner.put_object_acl, {"Key": "k", "GrantRead": grant, "ACL": "private"}, "InvalidRequest"),+ (owner.put_object_acl, {"Key": "k", "GrantRead": grant, "AccessControlPolicy": policy},+ "UnexpectedContent"),+ ):+ with pytest.raises(ClientError) as exc:+ call(Bucket=bucket, **kwargs)+ assert exc.value.response["Error"]["Code"] == code, kwargs+ assert owner.list_objects_v2(Bucket=bucket)["KeyCount"] == 1+ assert len(owner.get_object_acl(Bucket=bucket, Key="k")["Grants"]) == 2+ finally:+ _purge_foreign(owner, bucket)++def test_s3_public_read_object_acl_opens_reads_to_another_account(s3):"""A public-read object ACL answers another account's read until IgnorePublicAcls is set."""owner, reader = _s3_account("111111111111"), _s3_account("222222222222")@@ -6974,6 +7067,26 @@ def test_s3_copy_object_applies_canned_acl(s3):assert exc.value.response["Error"]["Code"] == "InvalidArgument"+def test_s3_copy_object_applies_grant_headers(s3):+ """x-amz-grant-* headers on a copy set the destination's ACL; together with x-amz-acl the copy is refused."""+ bkt = f"s3-copy-grant-{uuid.uuid4().hex[:8]}"+ _acl_enabled_bucket(s3, bkt)+ owner_id = s3.list_buckets()["Owner"]["ID"]+ other_id = "c4c1ede66af53448b93c283ce9448c4ba468c9432aa01d700d3878632f77d2d0"+ s3.put_object(Bucket=bkt, Key="src.txt", Body=b"body")+ source = {"Bucket": bkt, "Key": "src.txt"}++ s3.copy_object(Bucket=bkt, Key="dst.txt", CopySource=source, GrantRead=f'id="{other_id}"',+ GrantFullControl=f'id="{owner_id}"')+ assert _grant_pairs(s3.get_object_acl(Bucket=bkt, Key="dst.txt")) == [+ (other_id, "READ"), (owner_id, "FULL_CONTROL")]++ with pytest.raises(ClientError) as exc:+ s3.copy_object(Bucket=bkt, Key="both.txt", CopySource=source, GrantRead=f'id="{other_id}"', ACL="private")+ assert exc.value.response["Error"]["Code"] == "InvalidRequest"+ assert [o["Key"] for o in s3.list_objects_v2(Bucket=bkt)["Contents"]] == ["dst.txt", "src.txt"]++def test_s3_upload_part_copy_honours_source_preconditions(s3):"""UploadPartCopy carries the same copy-source conditions CopyObject does."""bkt = "s3-upc-precond"