Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(s3): answer CORS from the bucket's rules under AUTH=true (#2407)
iot-rocket83ef7f6c8d07
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 4b1fd5ce..b3839cac 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]### Fixed+- **S3 — CORS under `AUTH=true`** — every S3 preflight answered 200 with `Access-Control-Allow-Origin: *` and every response carried the same wildcard headers, whatever the bucket's CORS configuration; under `AUTH=true` a preflight to an existing bucket now has to match one of its CORS rules (403 `AccessForbidden` without a configuration or a matching rule, 400 `BadRequest` without `Origin`) and gets that rule's headers, and a request with an `Origin` carries CORS headers only when a rule allows its origin and method. Contributed by @iot-rocket.- **IoT — MQTT 5 No Local follows AWS IoT** — a subscription with No Local kept the client's own messages from it; AWS IoT Core ignores that option and delivers them, and so does the broker now. Contributed by @iot-rocket.- **CloudFront — a viewer-request function that leaves the query string alone no longer re-encodes it** — any function that returned the request it was given counted as a query-string rewrite, so the origin got the query rebuilt from the parsed parameters instead of as the viewer sent it: a bare key gained an `=` (`?flag` became `?flag=`). The query string is now rebuilt only when the function's returned `querystring` differs from the one in its event. Contributed by @jeffreyheller.- **API Gateway — an HTTP API's responses carry `apigw-requestid`, not a REST API's request ids** — a response from an HTTP API's route got `x-amzn-requestid` and `x-amz-request-id`; an HTTP API names its request id `apigw-requestid`, 16 base64 characters, which it now sends instead. REST APIs keep `x-amzn-requestid`. Contributed by @jeffreyheller.diff --git a/ministack/app.py b/ministack/app.pyindex b41f37f4..666f6ef4 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -859,10 +859,15 @@ async def _send_if_handled(send, response, receive=None) -> bool:# ----------------------------------------------------------------------------def _handle_options_request(method: str, request_id: str):- """Return the standard CORS preflight response when applicable."""+def _handle_options_request(method: str, path: str, headers: dict, query_params: dict, request_id: str):+ """Return the CORS preflight response when applicable."""if method != "OPTIONS":return None+ target = _s3_cors_target(headers.get("host", ""), path) if AUTH else None+ if target and detect_service(method, path, headers, query_params) == "s3":+ # Under AUTH a bucket's CORS configuration answers the preflight.+ status, resp_headers, body = _get_module("s3").cors_preflight(*target, headers)+ return status, {**resp_headers, "x-amzn-requestid": request_id, "x-amz-request-id": request_id}, bodyreturn (200,{@@ -1292,7 +1297,7 @@ async def _handle_pre_body_request(method: str, path: str, headers: dict, query_or _parse_cloudfront_dataplane_host(host) is not None)for response in (- None if owns_cors else _handle_options_request(method, request_id),+ None if owns_cors else _handle_options_request(method, path, headers, query_params, request_id),_handle_health_request(path, request_id),_handle_ready_request(path, request_id),_handle_unknown_localstack_request(path, request_id),@@ -2392,6 +2397,20 @@ async def _handle_s3_vhost_request(host: str, path: str, method: str, headers: d)+def _s3_cors_target(host: str, path: str, path_style: bool = True) -> tuple[str, str] | None:+ """``(bucket, key)`` of an existing bucket a request addresses by virtual host or path, for its CORS answer."""+ bucket = _extract_s3_vhost_bucket(host)+ if bucket and not _S3_VHOST_EXCLUDE_RE.search(host) and bucket not in _NON_S3_VHOST_NAMES:+ key = path.lstrip("/")+ elif path_style:+ bucket, _, key = path.lstrip("/").partition("/")+ else:+ return None+ if not bucket or _get_module("s3")._bucket_owner_account(bucket) is None:+ return None+ return bucket, key++def _with_data_plane_headers(response, request_id: str, include_s3_id: bool = False, wildcard_cors: bool = True):"""Attach common data-plane request-id headers to a response tuple.@@ -2628,7 +2647,9 @@ async def _handle_special_data_plane_request(if response := await _handle_lambda_url_request(host, path, method, headers, body, query_params):return _with_data_plane_headers(response, request_id, wildcard_cors=False)if response := await _handle_s3_vhost_request(host, path, method, headers, body, query_params):- return _with_data_plane_headers(response, request_id, include_s3_id=True)+ if AUTH and (target := _s3_cors_target(host, path, path_style=False)):+ response[1].update(_get_module("s3").cors_response_headers(target[0], method, headers))+ return _with_data_plane_headers(response, request_id, include_s3_id=True, wildcard_cors=not AUTH)if response := await _handle_alb_request(host, path, method, headers, body, query_params):return _with_data_plane_headers(response, request_id)return None@@ -3103,12 +3124,19 @@ async def _dispatch_service_request(_maybe_record_cloudtrail(service, method, path, headers, body, query_params, request_id, region)+ cors = {+ "Access-Control-Allow-Origin": "*",+ "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH",+ "Access-Control-Allow-Headers": "*",+ "Access-Control-Expose-Headers": "*",+ }+ if AUTH and service == "s3":+ # Under AUTH only the bucket's CORS configuration adds CORS headers.+ target = _s3_cors_target(headers.get("host", ""), path)+ cors = _get_module("s3").cors_response_headers(target[0], method, headers) if target else {}resp_headers.update({- "Access-Control-Allow-Origin": "*",- "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH",- "Access-Control-Allow-Headers": "*",- "Access-Control-Expose-Headers": "*",+ **cors,"x-amzn-requestid": request_id,"x-amz-request-id": request_id,"x-amz-id-2": base64.b64encode(os.urandom(48)).decode(),diff --git a/ministack/services/s3.py b/ministack/services/s3.pyindex 55e41199..25d78f80 100644--- a/ministack/services/s3.py+++ b/ministack/services/s3.py@@ -3498,6 +3498,100 @@ def _delete_bucket_cors(name: str):return 204, {}, b""+_CORS_VARY = "Origin, Access-Control-Request-Headers, Access-Control-Request-Method"+_CORS_NOT_ALLOWED = (+ "CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, "+ "request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted "+ "by the resource's CORS spec."+)+++def _cors_rules(bucket_name: str) -> list[dict] | None:+ """The bucket's CORS rules as ``{element: [values]}`` in its owner's account, or None without a configuration."""+ owner = _bucket_owner_account(bucket_name)+ stored = _bucket_cors.get_scoped(owner, None, bucket_name) if owner else None+ if not stored:+ return None+ try:+ root = fromstring(stored.encode() if isinstance(stored, str) else stored)+ except Exception:+ return None+ rules = []+ for rule in root.iter():+ if rule.tag.rsplit("}", 1)[-1] == "CORSRule":+ values = {}+ for child in rule:+ values.setdefault(child.tag.rsplit("}", 1)[-1], []).append((child.text or "").strip())+ rules.append(values)+ return rules+++def _cors_pattern_matches(pattern: str, value: str, flags: int = 0) -> bool:+ """Whether *value* matches an AllowedOrigin or AllowedHeader, where ``*`` stands for any characters."""+ return re.fullmatch(re.escape(pattern).replace(r"\*", ".*"), value, flags) is not None+++def _cors_answer(rules: list[dict], origin: str, method: str, request_headers: list[str]) -> dict | None:+ """The Access-Control-* headers of the first rule that allows the origin, method and headers, or None."""+ for rule in rules:+ origins = [o for o in rule.get("AllowedOrigin", []) if _cors_pattern_matches(o, origin)]+ if not origins or method not in rule.get("AllowedMethod", []):+ continue+ allowed_headers = rule.get("AllowedHeader", [])+ if not all(any(_cors_pattern_matches(allowed, wanted, re.IGNORECASE) for allowed in allowed_headers)+ for wanted in request_headers):+ continue+ answer = {"Access-Control-Allow-Origin": "*"} if origins == ["*"] else {+ "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Credentials": "true"}+ answer["Access-Control-Allow-Methods"] = ", ".join(rule["AllowedMethod"])+ if request_headers:+ answer["Access-Control-Allow-Headers"] = ", ".join(request_headers)+ if rule.get("ExposeHeader"):+ answer["Access-Control-Expose-Headers"] = ", ".join(rule["ExposeHeader"])+ if rule.get("MaxAgeSeconds"):+ answer["Access-Control-Max-Age"] = rule["MaxAgeSeconds"][0]+ answer["Vary"] = _CORS_VARY+ return answer+ return None+++def _cors_error(code: str, message: str, status: int, method: str = "", resource_type: str = "") -> tuple:+ """A CORS refusal, which names the method and resource type instead of a resource."""+ root = Element("Error")+ SubElement(root, "Code").text = code+ SubElement(root, "Message").text = message+ if method:+ SubElement(root, "Method").text = method+ SubElement(root, "ResourceType").text = resource_type+ SubElement(root, "RequestId").text = new_uuid()+ return status, {"Content-Type": "application/xml"}, _xml_body(root)+++def cors_preflight(bucket_name: str, key: str, headers: dict) -> tuple:+ """The answer to a browser preflight (OPTIONS) from the bucket's CORS configuration."""+ origin = headers.get("origin")+ if not origin:+ return _cors_error("BadRequest", "Insufficient information. Origin request header needed.", 400)+ method = headers.get("access-control-request-method", "")+ rules = _cors_rules(bucket_name)+ if rules is None:+ return _cors_error("AccessForbidden", "CORSResponse: CORS is not enabled for this bucket.", 403,+ method or "OPTIONS", "BUCKET")+ wanted = [h.strip() for h in headers.get("access-control-request-headers", "").split(",") if h.strip()]+ answer = _cors_answer(rules, origin, method, wanted) if method else None+ if answer is None:+ return _cors_error("AccessForbidden", _CORS_NOT_ALLOWED, 403, method or "OPTIONS",+ "OBJECT" if key else "BUCKET")+ return 200, {**answer, "Content-Length": "0"}, b""+++def cors_response_headers(bucket_name: str, method: str, headers: dict) -> dict:+ """The Access-Control-* headers for a request with an Origin that one of the bucket's CORS rules allows."""+ origin = headers.get("origin")+ rules = _cors_rules(bucket_name) if origin else None+ return (_cors_answer(rules, origin, method, []) if rules else None) or {}++def _get_bucket_acl(name: str):if name not in _buckets:return _no_such_bucket(name)diff --git a/tests/test_s3.py b/tests/test_s3.pyindex 5b57f17c..962cbe16 100644--- a/tests/test_s3.py+++ b/tests/test_s3.py@@ -3322,6 +3322,76 @@ def test_s3_bucket_cors(s3):s3.get_bucket_cors(Bucket="intg-s3-cors")assert exc.value.response["Error"]["Code"] == "NoSuchCORSConfiguration"+def _cors_headers(headers):+ return {k.lower(): v for k, v in headers.items() if k.lower().startswith("access-control-") or k.lower() == "vary"}++[email protected]_contract+def test_s3_cors_follows_the_bucket_configuration_contract(auth_enabled, s3):+ """Under AUTH preflights and the CORS headers of a request follow the bucket's CORS rules, else the wildcard."""+ bucket, plain = f"cors-rules-{uuid.uuid4().hex[:8]}", f"cors-none-{uuid.uuid4().hex[:8]}"+ for name in (bucket, plain):+ s3.create_bucket(Bucket=name)+ s3.put_object(Bucket=name, Key="k", Body=b"k")+ s3.put_bucket_cors(Bucket=bucket, CORSConfiguration={"CORSRules": [+ {"AllowedOrigins": ["https://a.example"], "AllowedMethods": ["GET", "PUT"],+ "AllowedHeaders": ["x-amz-meta-*"], "ExposeHeaders": ["ETag"], "MaxAgeSeconds": 300},+ {"AllowedOrigins": ["*"], "AllowedMethods": ["HEAD"]}]})+ vary = "Origin, Access-Control-Request-Headers, Access-Control-Request-Method"++ def preflight(name, vhost, origin="https://a.example", method="GET", **extra):+ headers = {"Origin": origin, "Access-Control-Request-Method": method, **extra}+ if vhost:+ headers["Host"] = f"{name}.{urlparse(ENDPOINT).netloc}"+ url = f"{ENDPOINT}/k" if vhost else f"{ENDPOINT}/{name}/k"+ return requests.options(url, headers={k: v for k, v in headers.items() if v})++ def get_from(origin, name, operation="get_object"):+ def add_origin(request, **_):+ request.headers["Origin"] = origin++ event = f"before-sign.s3.{''.join(p.title() for p in operation.split('_'))}"+ s3.meta.events.register(event, add_origin)+ try:+ return _cors_headers(getattr(s3, operation)(Bucket=name, Key="k")["ResponseMetadata"]["HTTPHeaders"])+ finally:+ s3.meta.events.unregister(event, add_origin)++ try:+ for vhost in (False, True):+ allowed = preflight(bucket, vhost, method="PUT", **{"Access-Control-Request-Headers": "x-amz-meta-a"})+ refused = [preflight(plain, vhost), preflight(bucket, vhost, method="DELETE"),+ preflight(bucket, vhost, origin="https://b.example"),+ preflight(bucket, vhost, **{"Access-Control-Request-Headers": "x-other"})]+ assert allowed.status_code == 200+ if not auth_enabled:+ assert allowed.headers["Access-Control-Allow-Origin"] == "*"+ assert [r.status_code for r in refused] == [200] * 4+ continue+ assert _cors_headers(allowed.headers) == {+ "access-control-allow-origin": "https://a.example", "access-control-allow-credentials": "true",+ "access-control-allow-methods": "GET, PUT", "access-control-allow-headers": "x-amz-meta-a",+ "access-control-expose-headers": "ETag", "access-control-max-age": "300", "vary": vary}+ assert [r.status_code for r in refused] == [403] * 4+ assert "CORSResponse: CORS is not enabled for this bucket." in refused[0].text+ assert all("<Code>AccessForbidden</Code>" in r.text and not _cors_headers(r.headers) for r in refused)+ no_origin = requests.options(f"{ENDPOINT}/{bucket}/k", headers={"Access-Control-Request-Method": "GET"})+ assert no_origin.status_code == (400 if auth_enabled else 200)++ if auth_enabled:+ assert get_from("https://a.example", bucket)["access-control-allow-origin"] == "https://a.example"+ assert get_from("https://b.example", bucket, "head_object") == {+ "access-control-allow-origin": "*", "access-control-allow-methods": "HEAD", "vary": vary}+ assert get_from("https://b.example", bucket) == {}+ assert get_from("https://a.example", plain) == {}+ else:+ assert get_from("https://b.example", plain)["access-control-allow-origin"] == "*"+ finally:+ for name in (bucket, plain):+ s3.delete_object(Bucket=name, Key="k")+ s3.delete_bucket(Bucket=name)++def test_s3_bucket_acl(s3):s3.create_bucket(Bucket="intg-s3-acl")resp = s3.get_bucket_acl(Bucket="intg-s3-acl")