Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(s3): apply each Block Public Access setting and bucket ACL grant (#2262)
iot-rocketfd8f6eb2722f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 969254cb..4acce07e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]### Fixed+- **S3 — Block Public Access settings and public bucket ACL grants** — any one of the four Block Public Access settings shut off every public grant on a bucket, none refused setting a public ACL, and a public bucket ACL opened every read and write in the bucket. `BlockPublicAcls` now refuses a call that sets a public ACL on a bucket with ACLs enabled (403 `AccessDenied`) and leaves the grants in place, `IgnorePublicAcls` drops public ACL grants and `RestrictPublicBuckets` the bucket policy. A bucket ACL grants what its permission covers: `READ` the listings, `WRITE` adding and removing objects, `READ_ACP` and `WRITE_ACP` the bucket ACL. Contributed by @iot-rocket.- **CloudTrail — `RecursiveLogging` on trails** — `CreateTrail` and `UpdateTrail` ignored `RecursiveLogging` and no trail response carried it; trails now store it (default `true`, kept by an `UpdateTrail` that leaves it out) and return it from `CreateTrail`, `UpdateTrail`, `GetTrail` and `DescribeTrails`. Contributed by @iot-rocket.- **CloudWatch — composite alarms follow their rule** — a composite alarm stayed in `INSUFFICIENT_DATA` whatever its children did and `DeleteAlarms` removed alarms that composite alarms depend on; a composite alarm now evaluates its `AlarmRule` (`ALARM`, `OK`, `INSUFFICIENT_DATA`, `TRUE`, `FALSE`, `AND`, `OR`, `NOT`, `AT_LEAST`) when it is created or updated and whenever a child changes state, with the `StateReason`, `StateReasonData` and history items AWS writes, `PutCompositeAlarm` refuses an unknown child or a malformed rule with `ValidationError`, `DeleteAlarms` refuses an alarm a composite alarm depends on and more than one composite alarm per call, and `DescribeAlarmHistory` returns composite alarm items only for `AlarmTypes=["CompositeAlarm"]`, as AWS does. Contributed by @iot-rocket.- **IoT — unacknowledged QoS 1 messages survive a persistent session reconnect** — a QoS 1 message the subscriber had not acknowledged was dropped when its persistent session disconnected; the session now keeps it with its packet identifier, and the resumed connection sends it again once its retransmit interval has passed (at once when it passed while the client was offline, ahead of the messages queued meanwhile), continuing the session's packet identifiers. Contributed by @iot-rocket.diff --git a/ministack/services/s3.py b/ministack/services/s3.pyindex e22442bb..a11e58b3 100644--- a/ministack/services/s3.py+++ b/ministack/services/s3.py@@ -985,9 +985,8 @@ def _get_object_data(bucket_name: str, key: str, version_id: str | None = None)_ACL_GROUP_AUTHENTICATED = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"-# The permissions that answer a read and a write, FULL_CONTROL covering both.+# The permissions that answer an object read, FULL_CONTROL included._ACL_READ_PERMS = ("READ", "FULL_CONTROL")-_ACL_WRITE_PERMS = ("WRITE", "FULL_CONTROL")_ACL_READ_ACP_PERMS = ("READ_ACP", "FULL_CONTROL")# Object reads an object ACL grants nothing for: READ covers the data, READ_ACP the ACL._ACL_UNGRANTED_OBJECT_READS = ("uploadId", "annotation", "tagging", "retention", "legal-hold", "attributes")@@ -1016,8 +1015,8 @@ def _acl_grants(stored_xml: str | None, grantee: str = "URI") -> list[tuple[str,return grants-def _public_access_blocked(owner_account: str, bucket_name: str) -> bool:- """Whether the bucket's Public Access Block shuts public grants off."""+def _public_access_blocked(owner_account: str, bucket_name: str, setting: str) -> bool:+ """Whether the bucket's Public Access Block turns *setting* on, e.g. ``IgnorePublicAcls``."""record = _buckets.get_scoped(owner_account, None, bucket_name) or {}stored = record.get("_public_access_block")if not stored:@@ -1027,10 +1026,8 @@ def _public_access_blocked(owner_account: str, bucket_name: str) -> bool:except Exception:return Falsefor child in root.iter():- if child.tag.endswith(("BlockPublicAcls", "RestrictPublicBuckets",- "BlockPublicPolicy", "IgnorePublicAcls")):- if (child.text or "").strip().lower() == "true":- return True+ if child.tag.rsplit("}", 1)[-1] == setting:+ return (child.text or "").strip().lower() == "true"return False@@ -1074,7 +1071,8 @@ def _foreign_bucket_allows(owner_account: str, bucket_name: str, key: str,The owner's grants decide it: a Group grant on the bucket or the objectACL, a grant to the caller's canonical ID on the object ACL, or an allowin the bucket policy. Nothing granted means the request is denied, which- is S3's default for a bucket nobody has opened up.+ is S3's default for a bucket nobody has opened up. IgnorePublicAcls drops+ the Group grants and RestrictPublicBuckets the policy."""object_acl, object_wants = _foreign_object_acl(owner_account, bucket_name, key, method, query_params)caller_id = _canonical_owner_id(caller) if caller else ""@@ -1082,20 +1080,19 @@ def _foreign_bucket_allows(owner_account: str, bucket_name: str, key: str,for grantee, permission in _acl_grants(object_acl, "ID")):# A grant to one account is not public, so Block Public Access leaves it alone.return True- if _public_access_blocked(owner_account, bucket_name):- return False- wants = _ACL_READ_PERMS if method in ("GET", "HEAD") else _ACL_WRITE_PERMS+ wants = _bucket_acl_wants(method, bucket_name, key, query_params)groups = [_ACL_GROUP_ALL_USERS] if not caller else [_ACL_GROUP_ALL_USERS, _ACL_GROUP_AUTHENTICATED]- for stored, granted in ((_bucket_acl.get_scoped(owner_account, None, bucket_name), wants),- (object_acl, object_wants)):- for uri, permission in _acl_grants(stored):- if uri in groups and permission in granted:- return True+ if not _public_access_blocked(owner_account, bucket_name, "IgnorePublicAcls"):+ for stored, granted in ((_bucket_acl.get_scoped(owner_account, None, bucket_name), wants),+ (object_acl, object_wants)):+ for uri, permission in _acl_grants(stored):+ if uri in groups and permission in granted:+ return Truepolicy = _bucket_policies.get_scoped(owner_account, None, bucket_name)- if policy:+ if policy and not _public_access_blocked(owner_account, bucket_name, "RestrictPublicBuckets"):from ministack.core.iam_evaluator import EvalContext, evaluate_resource_policyaction = "s3:GetObject" if method in ("GET", "HEAD") else "s3:PutObject"@@ -1127,6 +1124,23 @@ def _foreign_object_acl(owner_account: str, bucket_name: str, key: str, method:return _object_acl.get_scoped(owner_account, None, (bucket_name, key, version_id)), wants+# The bucket ACL permission each action needs: READ lists, WRITE adds and removes objects.+_BUCKET_ACL_PERMISSIONS = {+ "ListBucket": "READ", "ListBucketVersions": "READ", "ListBucketMultipartUploads": "READ",+ "ListMultipartUploadParts": "READ", "PutObject": "WRITE", "DeleteObject": "WRITE",+ "AbortMultipartUpload": "WRITE", "GetBucketAcl": "READ_ACP", "PutBucketAcl": "WRITE_ACP",+}+++def _bucket_acl_wants(method: str, bucket_name: str, key: str, query_params: dict) -> tuple[str, ...]:+ """The bucket ACL permissions that grant the request's action, none for an action a bucket ACL cannot grant."""+ from ministack.core.iam_actions import _s3_action++ action = _s3_action(method, f"/{bucket_name}/{key}" if key else f"/{bucket_name}", query_params)+ permission = _BUCKET_ACL_PERMISSIONS.get(action or "")+ return (permission, "FULL_CONTROL") if permission else ()++def _request_access_key(method: str, key: str, headers: dict, query_params: dict,body: bytes) -> str:"""The caller's access key, including the one a browser POST signs into its@@ -3207,6 +3221,9 @@ def _put_bucket_acl(name: str, body: bytes, headers: dict | None = None):headers = headers or {}if name not in _buckets:return _no_such_bucket(name)+ refused = _public_acl_refused(name, headers, body)+ if refused:+ return refused# Canned ACL from x-amz-acl header takes precedence and is mutually# exclusive with an XML body per the AWS API reference. SDK callers send@@ -4326,6 +4343,9 @@ def _put_object(bucket_name: str, key: str, body: bytes, headers: dict):canned_acl = headers.get("x-amz-acl")if canned_acl and canned_acl not in _CANNED_OBJECT_ACLS:return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)+ refused = _public_acl_refused(bucket_name, headers)+ if refused:+ return refusedlock_err = _object_lock_header_error(bucket_name, headers)if lock_err:@@ -4718,6 +4738,9 @@ def _post_object(bucket_name: str, body: bytes, headers: dict):canned_acl = synth.get("x-amz-acl")if canned_acl and canned_acl not in _CANNED_OBJECT_ACLS:return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)+ refused = _public_acl_refused(bucket_name, synth)+ if refused:+ return refused# The POST form names its SSE fields exactly like the request headers.sse_fields = {@@ -5873,6 +5896,9 @@ def _copy_object(bucket_name: str, dest_key: str, headers: dict):canned_acl = headers.get("x-amz-acl")if canned_acl and canned_acl not in _CANNED_OBJECT_ACLS:return _error("InvalidArgument", f"Invalid x-amz-acl value: {canned_acl}", 400)+ refused = _public_acl_refused(bucket_name, headers)+ if refused:+ return refused# A new checksum algorithm on the copy also applies to the annotations.annotation_directive = headers.get("x-amz-object-annotation-directive", "COPY").upper()@@ -6927,6 +6953,22 @@ _CANNED_ACL_GROUP_GRANTS = {}+def _public_acl_refused(bucket_name: str, headers: dict, acl_xml: str | bytes | None = None) -> tuple | None:+ """AccessDenied for a request that sets a public ACL on a bucket whose BlockPublicAcls is on."""+ public = (_ACL_GROUP_ALL_USERS, _ACL_GROUP_AUTHENTICATED)+ uris = [uri for uri, _ in _CANNED_ACL_GROUP_GRANTS.get(headers.get("x-amz-acl", ""), [])]+ uris += [uri for uri, _ in _acl_grants(acl_xml)]+ grants = " ".join(v for k, v in headers.items() if k.lower().startswith("x-amz-grant-"))+ if not any(uri in public for uri in uris) and not any(group in grants for group in public):+ return None+ bucket = _buckets.get(bucket_name)+ if (bucket is None or _bucket_owner_enforced(bucket)+ or not _public_access_blocked(get_account_id(), bucket_name, "BlockPublicAcls")):+ return None+ return _error("AccessDenied",+ "public ACLs are prevented by the BlockPublicAcls setting in S3 Block Public Access.", 403)++def _canned_acl_policy_xml(canned: str, owner_id: str) -> str:"""AccessControlPolicy XML for a canned ACL: the owner's FULL_CONTROL plus thegroup grants the canned value implies (e.g. public-read grants AllUsers READ),@@ -7020,6 +7062,9 @@ def _put_object_acl(bucket_name: str, key: str, body: bytes, headers: dict, querif gone is not None:return goneversion_id = _resolve_subresource_version(query_params, bucket, key)+ refused = _public_acl_refused(bucket_name, headers, body)+ if refused:+ return refused# Canned ACL from x-amz-acl header takes precedence and is mutually# exclusive with an XML body per the AWS API reference. Either path@@ -7585,6 +7630,9 @@ def _create_multipart_upload(bucket_name: str, key: str, headers: dict):lock_err = _object_lock_header_error(bucket_name, headers)if lock_err:return lock_err+ refused = _public_acl_refused(bucket_name, headers)+ if refused:+ return refusedstorage_class, sc_err = _resolve_storage_class(headers)if sc_err:diff --git a/tests/test_s3.py b/tests/test_s3.pyindex a2cc895c..ade8f333 100644--- a/tests/test_s3.py+++ b/tests/test_s3.py@@ -1395,6 +1395,53 @@ def test_s3_public_access_block(s3):s3.get_public_access_block(Bucket=bkt)assert exc.value.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration"++def test_s3_block_public_acls_refuses_setting_a_public_acl(s3):+ """BlockPublicAcls refuses every call that sets a public ACL; private grants and IgnorePublicAcls alone pass."""+ import requests++ bkt = f"bpa-acl-{uuid.uuid4().hex[:8]}"+ _acl_enabled_bucket(s3, bkt)+ s3.put_object(Bucket=bkt, Key="k", Body=b"x")+ owner_id = s3.list_buckets()["Owner"]["ID"]+ all_users = "http://acs.amazonaws.com/groups/global/AllUsers"+ public_policy = {"Owner": {"ID": owner_id}, "Grants": [+ {"Grantee": {"Type": "Group", "URI": all_users}, "Permission": "READ"}]}+ s3.put_public_access_block(Bucket=bkt, PublicAccessBlockConfiguration={"BlockPublicAcls": True})+ try:+ for call, kwargs in (+ (s3.put_object, {"Key": "n", "Body": b"n", "ACL": "public-read"}),+ (s3.put_object, {"Key": "n", "Body": b"n", "ACL": "authenticated-read"}),+ (s3.put_object, {"Key": "n", "Body": b"n", "GrantRead": f'uri="{all_users}"'}),+ (s3.put_object_acl, {"Key": "k", "ACL": "public-read"}),+ (s3.put_object_acl, {"Key": "k", "GrantRead": f'uri="{all_users}"'}),+ (s3.put_object_acl, {"Key": "k", "AccessControlPolicy": public_policy}),+ (s3.copy_object, {"Key": "c", "CopySource": {"Bucket": bkt, "Key": "k"}, "ACL": "public-read"}),+ (s3.create_multipart_upload, {"Key": "m", "ACL": "public-read"}),+ (s3.put_bucket_acl, {"ACL": "authenticated-read"}),+ (s3.put_bucket_acl, {"AccessControlPolicy": public_policy}),+ ):+ with pytest.raises(ClientError) as exc:+ call(Bucket=bkt, **kwargs)+ assert exc.value.response["Error"]["Code"] == "AccessDenied", kwargs+ assert "BlockPublicAcls" in exc.value.response["Error"]["Message"]+ post = s3.generate_presigned_post(+ Bucket=bkt, Key="p", Fields={"acl": "public-read"}, Conditions=[{"acl": "public-read"}])+ assert requests.post(post["url"], data=post["fields"], files={"file": ("p", b"x")}).status_code == 403+ assert s3.list_objects_v2(Bucket=bkt)["KeyCount"] == 1++ s3.put_object(Bucket=bkt, Key="n", Body=b"n", ACL="bucket-owner-full-control")+ s3.put_object(Bucket=bkt, Key="n", Body=b"n", GrantRead=f'id="{owner_id}"')+ s3.put_object_acl(Bucket=bkt, Key="k", ACL="private")+ s3.put_bucket_acl(Bucket=bkt, ACL="log-delivery-write")+ assert not any("URI" in g["Grantee"] and g["Grantee"]["URI"].endswith("/global/AllUsers")+ for g in s3.get_object_acl(Bucket=bkt, Key="k")["Grants"])++ s3.put_public_access_block(Bucket=bkt, PublicAccessBlockConfiguration={"IgnorePublicAcls": True})+ s3.put_object(Bucket=bkt, Key="n", Body=b"n", ACL="public-read")+ finally:+ _purge_foreign(s3, bkt)+def test_s3_ownership_controls(s3):bkt = "intg-s3-ownership"s3.create_bucket(Bucket=bkt)@@ -5368,6 +5415,59 @@ def test_s3_public_read_object_acl_opens_reads_to_another_account(s3):owner.delete_bucket(Bucket=bucket)+def test_s3_block_public_access_settings_each_cover_their_own_grants(s3):+ """Only IgnorePublicAcls drops a public ACL and only RestrictPublicBuckets a public policy."""+ owner, reader = _s3_account("111111111111"), _s3_account("222222222222")+ bucket = f"bpa-each-{uuid.uuid4().hex[:8]}"+ _acl_enabled_bucket(owner, bucket)+ settings = ("BlockPublicAcls", "IgnorePublicAcls", "BlockPublicPolicy", "RestrictPublicBuckets")+ try:+ owner.put_object(Bucket=bucket, Key="acl", Body=b"acl", ACL="public-read")+ owner.put_object(Bucket=bucket, Key="pol/k", Body=b"pol")+ owner.put_bucket_policy(Bucket=bucket, Policy=json.dumps({"Version": "2012-10-17", "Statement": [{+ "Effect": "Allow", "Principal": "*", "Action": "s3:GetObject",+ "Resource": f"arn:aws:s3:::{bucket}/pol/*"}]}))+ for setting in settings:+ owner.put_public_access_block(Bucket=bucket, PublicAccessBlockConfiguration={setting: True})+ for key, ignored_by in (("acl", "IgnorePublicAcls"), ("pol/k", "RestrictPublicBuckets")):+ if setting == ignored_by:+ assert _denied(reader.get_object, Bucket=bucket, Key=key), (setting, key)+ else:+ assert reader.get_object(Bucket=bucket, Key=key)["Body"].read(), (setting, key)+ finally:+ owner.delete_bucket_policy(Bucket=bucket)+ _purge_foreign(owner, bucket)+++def test_s3_public_bucket_acl_grants_only_what_the_bucket_permissions_cover(s3):+ """Bucket READ lists and WRITE adds and removes objects; neither opens objects, configuration or the ACL."""+ owner, reader = _s3_account("111111111111"), _s3_account("222222222222")+ bucket = f"bacl-pub-{uuid.uuid4().hex[:8]}"+ _acl_enabled_bucket(owner, bucket)+ try:+ owner.put_object(Bucket=bucket, Key="k", Body=b"k")+ owner.put_bucket_acl(Bucket=bucket, ACL="public-read")+ assert [o["Key"] for o in reader.list_objects_v2(Bucket=bucket)["Contents"]] == ["k"]+ reader.list_object_versions(Bucket=bucket)+ reader.list_multipart_uploads(Bucket=bucket)+ for call in (reader.get_object, reader.head_object, reader.get_object_acl):+ assert _denied(call, Bucket=bucket, Key="k"), call+ for call in (reader.get_bucket_acl, reader.get_bucket_versioning, reader.get_bucket_tagging):+ assert _denied(call, Bucket=bucket), call++ owner.put_bucket_acl(Bucket=bucket, ACL="public-read-write")+ reader.put_object(Bucket=bucket, Key="w", Body=b"w")+ reader.delete_object(Bucket=bucket, Key="w")+ assert _denied(reader.put_object_acl, Bucket=bucket, Key="k", ACL="public-read")+ assert _denied(reader.put_bucket_tagging, Bucket=bucket, Tagging={"TagSet": [{"Key": "a", "Value": "b"}]})+ assert _denied(reader.put_bucket_acl, Bucket=bucket, ACL="private")+ assert _denied(reader.delete_bucket, Bucket=bucket)+ grants = owner.get_bucket_acl(Bucket=bucket)["Grants"]+ assert sorted(g["Permission"] for g in grants if "URI" in g["Grantee"]) == ["READ", "WRITE"]+ finally:+ _purge_foreign(owner, bucket)++def test_s3_post_object_acl_field(s3):"""The POST form names its canned-ACL field `acl`; it applies to theuploaded object like x-amz-acl does on PutObject."""