Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): validate Content-MD5 on PutObject and UploadPart (#4524)
TiMbe87f992203e
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 4cf17286dd..56405a4c55 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -53,12 +53,15 @@ import java.io.InputStream;import java.io.OutputStream;import java.net.URLDecoder;import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.security.NoSuchAlgorithmException;import java.time.Instant;import java.time.ZoneId;import java.time.format.DateTimeFormatter;import java.time.format.DateTimeParseException;import java.util.ArrayList;import java.util.Arrays;+import java.util.Base64;import java.util.HashMap;import java.util.Iterator;import java.util.LinkedHashMap;@@ -2359,7 +2362,6 @@ public class S3Controller {String algorithmHeader = getChecksumAlgorithm(httpHeaders, uriInfo);ChecksumAlgorithm algorithm = ChecksumAlgorithm.fromWireValue(algorithmHeader);validateChecksumHeaders(httpHeaders, uriInfo, payload, algorithmHeader);- validateContentMd5(httpHeaders, payload);ObjectAnnotation annotation = s3Service.putObjectAnnotation(bucket, key, annotationName,versionId, payload, httpHeaders.getHeaderString("x-amz-object-if-match"), algorithm);Response.ResponseBuilder response = Response.ok(putObjectAnnotationXml(annotation))@@ -2510,21 +2512,6 @@ public class S3Controller {}}- private void validateContentMd5(HttpHeaders httpHeaders, byte[] body) {- String contentMd5 = httpHeaders.getHeaderString("Content-MD5");- if (contentMd5 == null) {- return;- }- try {- byte[] digest = java.security.MessageDigest.getInstance("MD5").digest(body);- if (!contentMd5.equals(java.util.Base64.getEncoder().encodeToString(digest))) {- throw new AwsException("BadDigest", "The Content-MD5 you specified did not match the payload.", 400);- }- } catch (java.security.NoSuchAlgorithmException e) {- throw new IllegalStateException("MD5 algorithm is not available", e);- }- }-// --- Object Lock Configuration ---private Response handlePutObjectLockConfiguration(String bucket, byte[] body) {@@ -3057,6 +3044,8 @@ public class S3Controller {}private void validateChecksumHeaders(HttpHeaders httpHeaders, UriInfo uriInfo, byte[] data, String algorithm) {+ validateContentMd5(httpHeaders.getHeaderString("Content-MD5"), data);+String sha1 = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-sha1");if (sha1 != null && !sha1.equals(S3Checksum.sha1Base64(data))) {throw new AwsException("BadDigest", "The SHA1 checksum you specified did not match the payload.", 400);@@ -3083,6 +3072,32 @@ public class S3Controller {}}+ // S3 answers InvalidDigest for a Content-MD5 that is not the base64 of a 16-byte digest, and+ // BadDigest for a well-formed one that does not match the payload.+ private static void validateContentMd5(String contentMd5, byte[] data) {+ if (contentMd5 == null) {+ return;+ }+ byte[] expected;+ try {+ expected = Base64.getDecoder().decode(contentMd5.trim());+ } catch (IllegalArgumentException e) {+ expected = null;+ }+ if (expected == null || expected.length != 16) {+ throw new AwsException("InvalidDigest", "The Content-MD5 you specified is not valid.", 400);+ }+ byte[] actual;+ try {+ actual = MessageDigest.getInstance("MD5").digest(data);+ } catch (NoSuchAlgorithmException e) {+ throw new IllegalStateException("MD5 algorithm is not available", e);+ }+ if (!MessageDigest.isEqual(expected, actual)) {+ throw new AwsException("BadDigest", "The Content-MD5 you specified did not match what we received.", 400);+ }+ }+static boolean isWebsiteRequest(HttpHeaders httpHeaders, UriInfo uriInfo) {// HTTP/2 (RFC 9113) carries no Host header, so fall back to the request URI authority —// the same resolution S3VirtualHostFilter applies. Without this, a website bucket reached@@ -3593,7 +3608,7 @@ public class S3Controller {private void validatePolicyExpiration(String policyBase64) {try {- byte[] decoded = java.util.Base64.getDecoder().decode(policyBase64);+ byte[] decoded = Base64.getDecoder().decode(policyBase64);JsonNode policy = OBJECT_MAPPER.readTree(decoded);JsonNode expirationNode = policy.get("expiration");if (expirationNode == null || expirationNode.isNull()) {@@ -3616,7 +3631,7 @@ public class S3Controller {private void validatePolicyConditions(String policyBase64, String bucket,Map<String, String> fields, int contentLength) {try {- byte[] decoded = java.util.Base64.getDecoder().decode(policyBase64);+ byte[] decoded = Base64.getDecoder().decode(policyBase64);JsonNode policy = OBJECT_MAPPER.readTree(decoded);JsonNode conditions = policy.get("conditions");if (conditions == null || !conditions.isArray()) {diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ContentMd5IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ContentMd5IntegrationTest.javanew file mode 100644index 0000000000..49d5d5564f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ContentMd5IntegrationTest.java@@ -0,0 +1,190 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.Test;++import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.util.Base64;+import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.not;++@QuarkusTest+class S3ContentMd5IntegrationTest {++ private static final String INVALID_DIGEST_MESSAGE = "The Content-MD5 you specified is not valid.";+ private static final String BAD_DIGEST_MESSAGE = "The Content-MD5 you specified did not match what we received.";++ @Test+ void putObject_matchingContentMd5_isStored() throws Exception {+ String bucket = createBucket("md5-match");++ given()+ .header("Content-MD5", md5("body"))+ .body("body")+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(200);++ assertObjectBody(bucket, "object.txt", "body");+ }++ @Test+ void putObject_mismatchedContentMd5_isBadDigestAndNotStored() throws Exception {+ String bucket = createBucket("md5-mismatch");++ given()+ .header("Content-MD5", md5("other"))+ .body("body")+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(400)+ .body(containsString("<Code>BadDigest</Code>"))+ .body(containsString("<Message>" + BAD_DIGEST_MESSAGE + "</Message>"));++ assertObjectAbsent(bucket, "object.txt");+ }++ @Test+ void putObject_mismatchedContentMd5_doesNotReplaceExistingObject() throws Exception {+ String bucket = createBucket("md5-overwrite");+ given().body("original").when().put("/" + bucket + "/object.txt").then().statusCode(200);++ given()+ .header("Content-MD5", md5("other"))+ .body("replacement")+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(400)+ .body(containsString("<Code>BadDigest</Code>"))+ .body(containsString("<Message>" + BAD_DIGEST_MESSAGE + "</Message>"));++ assertObjectBody(bucket, "object.txt", "original");+ }++ @Test+ void putObject_malformedContentMd5_isInvalidDigest() {+ String bucket = createBucket("md5-malformed");++ // Not base64 at all, then base64 of the wrong length.+ for (String malformed : new String[] {"not-base64!!", Base64.getEncoder().encodeToString(new byte[8])}) {+ given()+ .header("Content-MD5", malformed)+ .body("body")+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(400)+ .body(containsString("<Code>InvalidDigest</Code>"))+ .body(containsString("<Message>" + INVALID_DIGEST_MESSAGE + "</Message>"));+ }++ assertObjectAbsent(bucket, "object.txt");+ }++ @Test+ void putObject_awsChunkedBody_isCheckedAgainstTheDecodedPayload() throws Exception {+ String bucket = createBucket("md5-chunked");+ String payload = "decoded-payload";+ String chunked = Integer.toHexString(payload.length()) + "\r\n" + payload + "\r\n0\r\n\r\n";++ given()+ .header("Content-Encoding", "aws-chunked")+ .header("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")+ .header("x-amz-decoded-content-length", String.valueOf(payload.length()))+ .header("Content-MD5", md5(payload))+ .body(chunked)+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(200);++ assertObjectBody(bucket, "object.txt", payload);+ }++ @Test+ void putObject_awsChunkedBody_withTheDigestOfTheFramedBody_isBadDigest() throws Exception {+ String bucket = createBucket("md5-chunked-framed");+ String payload = "decoded-payload";+ String chunked = Integer.toHexString(payload.length()) + "\r\n" + payload + "\r\n0\r\n\r\n";++ // The digest is of the payload S3 stores, not of the chunk framing on the wire.+ given()+ .header("Content-Encoding", "aws-chunked")+ .header("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")+ .header("x-amz-decoded-content-length", String.valueOf(payload.length()))+ .header("Content-MD5", md5(chunked))+ .body(chunked)+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(400)+ .body(containsString("<Code>BadDigest</Code>"))+ .body(containsString("<Message>" + BAD_DIGEST_MESSAGE + "</Message>"));++ assertObjectAbsent(bucket, "object.txt");+ }++ @Test+ void uploadPart_mismatchedContentMd5_isBadDigest() throws Exception {+ String bucket = createBucket("md5-part");+ String uploadId = given()+ .when()+ .post("/" + bucket + "/object.txt?uploads")+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");++ given()+ .header("Content-MD5", md5("other"))+ .body("part")+ .when()+ .put("/" + bucket + "/object.txt?partNumber=1&uploadId=" + uploadId)+ .then()+ .statusCode(400)+ .body(containsString("<Code>BadDigest</Code>"))+ .body(containsString("<Message>" + BAD_DIGEST_MESSAGE + "</Message>"));++ // The rejected part was not stored.+ given()+ .when()+ .get("/" + bucket + "/object.txt?uploadId=" + uploadId)+ .then()+ .statusCode(200)+ .body(not(containsString("<Part>")));++ given()+ .header("Content-MD5", md5("part"))+ .body("part")+ .when()+ .put("/" + bucket + "/object.txt?partNumber=1&uploadId=" + uploadId)+ .then()+ .statusCode(200);+ }++ private static String md5(String value) throws Exception {+ byte[] digest = MessageDigest.getInstance("MD5").digest(value.getBytes(StandardCharsets.UTF_8));+ return Base64.getEncoder().encodeToString(digest);+ }++ private static String createBucket(String prefix) {+ String bucket = prefix + "-" + UUID.randomUUID().toString().substring(0, 8);+ given().when().put("/" + bucket).then().statusCode(200);+ return bucket;+ }++ private static void assertObjectBody(String bucket, String key, String body) {+ given().when().get("/" + bucket + "/" + key).then().statusCode(200).body(equalTo(body));+ }++ private static void assertObjectAbsent(String bucket, String key) {+ given().when().get("/" + bucket + "/" + key).then().statusCode(404);+ }+}