Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(app): inflate gzip-compressed request bodies (#1513)
Lukas Döllerer5e73f3ef261a
diff --git a/ministack/app.py b/ministack/app.pyindex 0c13facc..1f89ade4 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -8,6 +8,7 @@ Compatible with AWS CLI, boto3, and any AWS SDK via --endpoint-url.import argparseimport asyncioimport base64+import gzipimport jsonimport loggingimport math@@ -555,6 +556,33 @@ def _decode_aws_chunked_body(body: bytes, headers: dict) -> bytes:return body+def _decompress_request_body(body: bytes, headers: dict) -> bytes:+ """Inflate a gzip-compressed request body and drop the gzip token.++ Smithy's ``@requestCompression`` trait makes AWS SDKs gzip a request body+ once it passes ``REQUEST_MIN_COMPRESSION_SIZE_BYTES`` (default 10240) and+ send ``Content-Encoding: gzip``. CloudWatch ``PutMetricData`` carries the+ trait today; any client may compress a body it is allowed to compress.+ Returns ``body`` unchanged when the header does not ask for gzip.+ """+ content_encoding = headers.get("content-encoding", "")+ if "gzip" not in content_encoding.lower():+ return body+ try:+ inflated = gzip.decompress(body)+ except (OSError, EOFError) as e:+ # A body flagged gzip that will not inflate is a real bug. Log it and+ # pass the raw bytes on, so the handler's own error surfaces.+ logger.warning("gzip request body failed to inflate: %s", e)+ return body+ encodings = [p.strip() for p in content_encoding.split(",") if p.strip().lower() != "gzip"]+ if encodings:+ headers["content-encoding"] = ", ".join(encodings)+ else:+ headers.pop("content-encoding", None)+ return inflated++async def _read_request_body(receive, method: str, headers: dict) -> bytes:"""Read and decode the request body only for methods or headers that can carry one."""body = b""@@ -1950,6 +1978,16 @@ async def _dispatch_service_request("""Dispatch a request through the generic service router."""routing_params = _routing_params(method, path, headers, body, query_params)service = detect_service(method, path, headers, routing_params)++ # S3 is the exception: there Content-Encoding is object metadata, and the+ # body must reach the handler exactly as sent. Everywhere else the header+ # means the request body itself is compressed.+ if service != "s3":+ inflated = _decompress_request_body(body, headers)+ if inflated is not body:+ body = inflated+ routing_params = _routing_params(method, path, headers, body, query_params)+region = extract_region(headers)logger.debug("%s %s -> service=%s region=%s", method, path, service, region)diff --git a/tests/test_request_compression.py b/tests/test_request_compression.pynew file mode 100644index 00000000..cfbc36cc--- /dev/null+++ b/tests/test_request_compression.py@@ -0,0 +1,131 @@+"""Gzip-compressed request bodies (smithy @requestCompression).++An operation that carries smithy's ``@requestCompression`` trait makes the SDK+gzip its request body once the body passes REQUEST_MIN_COMPRESSION_SIZE_BYTES+(default 10240) and send ``Content-Encoding: gzip``. CloudWatch PutMetricData+carries the trait in botocore's and aws-sdk-go-v2's bundled models, so boto3+compresses it with no client configuration at all.++The body must be inflated before the service handler parses it. S3 is the+exception: there Content-Encoding is object metadata, so the body must stay+exactly as sent.+"""++import gzip+import http.client+import json+import os+import uuid as _uuid_mod+from urllib.parse import urlparse++ENDPOINT = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566").rstrip("/")+++def _post_gzip(target: str, payload: dict, content_type: str):+ """POST a gzipped JSON body with Content-Encoding: gzip."""+ parsed = urlparse(ENDPOINT)+ conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 4566, timeout=10)+ body = gzip.compress(json.dumps(payload).encode())+ conn.request("POST", "/", body=body, headers={+ "Content-Type": content_type,+ "Content-Encoding": "gzip",+ "Content-Length": str(len(body)),+ "X-Amz-Target": target,+ "Authorization":+ "AWS4-HMAC-SHA256 Credential=test/20240101/us-east-1/logs/aws4_request,"+ " SignedHeaders=host, Signature=x",+ })+ resp = conn.getresponse()+ resp.read()+ conn.close()+ return resp.status+++def test_gzip_json_request_body_is_inflated(logs):+ """A gzipped JSON body reaches the handler as plain JSON. Without the+ inflate the handler read compressed bytes and failed on invalid JSON.+ The body is built by hand: this covers the wire behavior for the JSON+ protocol, whatever the client that compressed it."""+ name = f"/gzip-req-{_uuid_mod.uuid4().hex[:8]}"++ status = _post_gzip(+ "Logs_20140328.CreateLogGroup",+ {"logGroupName": name},+ "application/x-amz-json-1.1",+ )+ assert status == 200++ groups = logs.describe_log_groups(logGroupNamePrefix=name)["logGroups"]+ assert [g["logGroupName"] for g in groups] == [name]++ logs.delete_log_group(logGroupName=name)+++def test_gzip_form_request_body_is_inflated():+ """Same for the Query protocol, where a compressed body also hides the+ Action parameter the request routes on. Body built by hand."""+ parsed = urlparse(ENDPOINT)+ conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 4566, timeout=10)+ body = gzip.compress(b"Action=DescribeRegions&Version=2016-11-15")+ conn.request("POST", "/", body=body, headers={+ "Content-Type": "application/x-www-form-urlencoded",+ "Content-Encoding": "gzip",+ "Content-Length": str(len(body)),+ "Authorization":+ "AWS4-HMAC-SHA256 Credential=test/20240101/us-east-1/ec2/aws4_request,"+ " SignedHeaders=host, Signature=x",+ })+ resp = conn.getresponse()+ payload = resp.read().decode()+ conn.close()++ assert resp.status == 200+ assert "<DescribeRegionsResponse" in payload+++def test_gzip_put_metric_data_stores_every_datapoint(cw):+ """PutMetricData carries @requestCompression, so boto3 gzips the body on+ its own once it passes 10240 bytes. No client configuration is involved.++ This is the operation a real SDK compresses today, and the failure was+ silent: MiniStack answered 200 and stored none of the batch, so a caller+ lost every datapoint with no error to see."""+ namespace = f"GzipMetrics-{_uuid_mod.uuid4().hex[:8]}"+ sent = {}+ cw.meta.events.register(+ "before-send.cloudwatch.PutMetricData",+ lambda request, **kw: sent.update(+ encoding=request.headers.get("Content-Encoding")),+ )++ # ~600 datapoints serialize well past the compression threshold.+ cw.put_metric_data(Namespace=namespace, MetricData=[+ {+ "MetricName": f"metric-{i}",+ "Value": float(i),+ "Dimensions": [+ {"Name": "PaddingDimensionName", "Value": f"padding-value-{i:05d}"},+ ],+ }+ for i in range(600)+ ])++ # Guard: if botocore stops compressing, this test no longer covers the bug.+ assert sent.get("encoding") in (b"gzip", "gzip"), sent+ assert len(cw.list_metrics(Namespace=namespace)["Metrics"]) == 600+++def test_s3_put_object_keeps_gzip_body_and_metadata(s3):+ """S3 must not inflate. Content-Encoding is object metadata there, so the+ stored bytes stay compressed and the header comes back on GET."""+ bucket = "intg-gzip-passthrough"+ s3.create_bucket(Bucket=bucket)+ compressed = gzip.compress(b"stored-compressed" * 64)++ s3.put_object(+ Bucket=bucket, Key="payload.gz", Body=compressed, ContentEncoding="gzip",+ )++ resp = s3.get_object(Bucket=bucket, Key="payload.gz")+ assert resp["Body"].read() == compressed+ assert resp["ContentEncoding"] == "gzip"