Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(s3): verify the checksums a request supplies, however they arrive (#1481)
Andrew Gaul36f026225417
diff --git a/ministack/app.py b/ministack/app.pyindex 0e439ab7..d4cebe9e 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -502,6 +502,7 @@ def _decode_aws_chunked_body(body: bytes, headers: dict) -> bytes:decoded = b""remaining = body+ trailer = b""while remaining:crlf = remaining.find(b"\r\n")if crlf == -1:@@ -513,11 +514,28 @@ def _decode_aws_chunked_body(body: bytes, headers: dict) -> bytes:except ValueError:breakif chunk_size == 0:+ # Whatever follows the final chunk is the trailing header+ # section the request announced in x-amz-trailer.+ trailer = remaining[crlf + 2 :]breakdata_start = crlf + 2decoded += remaining[data_start : data_start + chunk_size]remaining = remaining[data_start + chunk_size + 2 :] # skip trailing \r\n+ # A checksum computed while the body streamed arrives here rather than+ # among the request headers, and the rest of the request has no way to+ # tell the two apart -- so lift it into the headers, where every+ # checksum reader already looks. An explicit header wins: the trailer+ # is the late copy of the same field, not an override.+ for line in trailer.split(b"\r\n"):+ name, sep, value = line.partition(b":")+ if not sep:+ continue+ name = name.decode("ascii", errors="replace").strip().lower()+ if name:+ headers.setdefault(+ name, value.decode("utf-8", errors="replace").strip())+body = decodedif "aws-chunked" in content_encoding:encodings = [p.strip() for p in content_encoding.split(",") if p.strip() != "aws-chunked"]diff --git a/ministack/services/s3.py b/ministack/services/s3.pyindex 40fcc095..a662682e 100644--- a/ministack/services/s3.py+++ b/ministack/services/s3.py@@ -1213,12 +1213,13 @@ def _resolve_object_checksums(body: bytes, headers: dict):"""Build the stored checksum dict and validate any client-supplied values.AWS PutObject contract:- - `x-amz-checksum-{alg}` headers carry a client-computed value.+ - `x-amz-checksum-{alg}` headers carry a client-computed value, which+ the server recomputes from the body and MUST reject with `BadDigest`+ (HTTP 400) when the two disagree. A value that is never checked is+ worse than none: it round-trips on Get as though the body had been+ verified.- `x-amz-sdk-checksum-algorithm: ALG` asks the server to compute thatalgorithm; the resulting value is returned alongside the response.- - When the SDK both names an algorithm AND supplies its value, the- server-computed value MUST match the supplied one or the request is- rejected with `BadDigest` (HTTP 400).Ministack-specific: CRC32C requires an optional native library that the"no new dependencies" rule forbids us from adding. Rather than silently@@ -1257,18 +1258,24 @@ def _resolve_object_checksums(body: bytes, headers: dict):400,)+ # Every supplied value is verified against the body, whether or not the+ # request also names an algorithm for the server to compute: the client+ # asked for the object to be checked, and an unverified value would be+ # stored and echoed as if it had been.checksums = dict(provided)+ for alg, supplied in provided.items():+ computed = _compute_s3_checksum(alg, body)+ if computed is not None and supplied != computed:+ return {}, _error(+ "BadDigest",+ f"The {alg} you specified did not match the calculated checksum.",+ 400,+ )+if sdk_alg_raw:sdk_key = sdk_alg_raw.upper().replace("_", "")computed = _compute_s3_checksum(sdk_alg_raw, body)if computed is not None:- existing = provided.get(sdk_key)- if existing and existing != computed:- return {}, _error(- "BadDigest",- f"The {sdk_key} you specified did not match the calculated checksum.",- 400,- )checksums[sdk_key] = computedreturn checksums, Nonediff --git a/tests/test_s3.py b/tests/test_s3.pyindex 49bf6ac8..d96c56c2 100644--- a/tests/test_s3.py+++ b/tests/test_s3.py@@ -5732,3 +5732,28 @@ def test_s3_restore_notifications_to_sqs(s3, sqs):red = completed["glacierEventData"]["restoreEventData"]assert red["lifecycleRestoreStorageClass"] == "DEEP_ARCHIVE"assert red["lifecycleRestorationExpiryTime"].endswith("Z")+++def test_s3_put_object_rejects_a_mismatched_checksum(s3):+ """A supplied checksum is verified against the body, not just stored.++ The client sends one to have the object checked; storing an unverified+ value would echo it on every later read as though it had been."""+ bkt = "s3-checksum-mismatch"+ s3.create_bucket(Bucket=bkt)++ with pytest.raises(ClientError) as exc:+ s3.put_object(Bucket=bkt, Key="bad.txt", Body=b"hello",+ ChecksumSHA256="q6MvHqxNQz8xzL6UcqhbSY9dAMYo5S3VZjjuQFDJq4c=")+ assert exc.value.response["Error"]["Code"] == "BadDigest"++ # The rejected put stored nothing.+ with pytest.raises(ClientError) as exc:+ s3.head_object(Bucket=bkt, Key="bad.txt")+ assert exc.value.response["ResponseMetadata"]["HTTPStatusCode"] == 404++ # The matching value is accepted and echoed back.+ good = "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="+ s3.put_object(Bucket=bkt, Key="good.txt", Body=b"hello", ChecksumSHA256=good)+ head = s3.head_object(Bucket=bkt, Key="good.txt", ChecksumMode="ENABLED")+ assert head["ChecksumSHA256"] == good