Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): omit checksum headers from GetObject/HeadObject unless x-amz-checksum-mode: ENABLED (#1633)
Mihai Blidaru48f310b63fa5
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 2ee28a61b1..891cf2c8bc 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -638,6 +638,7 @@ public class S3Controller {@HeaderParam("If-Modified-Since") String ifModifiedSince,@HeaderParam("If-Unmodified-Since") String ifUnmodifiedSince,@HeaderParam("Range") String rangeHeader,+ @HeaderParam("x-amz-checksum-mode") String checksumMode,@Context UriInfo uriInfo,@Context HttpHeaders httpHeaders) {S3Service.RequestAuthorization authorization = S3Service.RequestAuthorization.unsigned();@@ -698,11 +699,12 @@ public class S3Controller {"Request specific response headers cannot be used for anonymous GET requests.", 400));}+ boolean includeChecksum = "ENABLED".equalsIgnoreCase(checksumMode);if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {- return handleRangeRequest(bucket, key, versionId, obj, rangeHeader, overrides);+ return handleRangeRequest(bucket, key, versionId, obj, rangeHeader, overrides, includeChecksum);}- return fullObjectResponse(bucket, key, versionId, obj, overrides);+ return fullObjectResponse(bucket, key, versionId, obj, overrides, includeChecksum);} catch (AwsException e) {if (isWebsiteErrorDocumentTrigger(e) && isWebsiteRequest(httpHeaders)) {try {@@ -722,7 +724,8 @@ public class S3Controller {}private Response fullObjectResponse(String bucket, String key, String versionId,- S3Object obj, ResponseHeaderOverrides overrides) {+ S3Object obj, ResponseHeaderOverrides overrides,+ boolean includeChecksum) {StreamingOutput stream = output -> {try (InputStream input = s3Service.openObjectStream(bucket, key, versionId)) {input.transferTo(output);@@ -737,13 +740,14 @@ public class S3Controller {if (obj.getVersionId() != null) {resp.header("x-amz-version-id", obj.getVersionId());}- appendObjectHeaders(resp, obj, overrides);+ appendObjectHeaders(resp, obj, overrides, includeChecksum);return resp.build();}private Response handleRangeRequest(String bucket, String key, String versionId,S3Object obj, String rangeHeader,- ResponseHeaderOverrides overrides) {+ ResponseHeaderOverrides overrides,+ boolean includeChecksum) {long totalSize = obj.getSize();String rangeSpec = rangeHeader.substring("bytes=".length()).trim();@@ -775,7 +779,7 @@ public class S3Controller {if (start < 0 || start >= totalSize || start > end) {if (totalSize == 0 && rangeSpec.startsWith("-")) {- return fullObjectResponse(bucket, key, versionId, obj, overrides);+ return fullObjectResponse(bucket, key, versionId, obj, overrides, includeChecksum);}return invalidRangeResponse(totalSize);}@@ -848,6 +852,7 @@ public class S3Controller {@HeaderParam("If-None-Match") String ifNoneMatch,@HeaderParam("If-Modified-Since") String ifModifiedSince,@HeaderParam("If-Unmodified-Since") String ifUnmodifiedSince,+ @HeaderParam("x-amz-checksum-mode") String checksumMode,@Context UriInfo uriInfo,@Context HttpHeaders httpHeaders) {try {@@ -882,7 +887,8 @@ public class S3Controller {if (obj.getVersionId() != null) {resp.header("x-amz-version-id", obj.getVersionId());}- appendObjectHeaders(resp, obj, overrides);+ boolean includeChecksum = "ENABLED".equalsIgnoreCase(checksumMode);+ appendObjectHeaders(resp, obj, overrides, includeChecksum);return resp.build();} catch (AwsException e) {return xmlErrorResponse(e);@@ -1696,8 +1702,6 @@ public class S3Controller {String cacheControl,String contentDisposition,String contentEncoding) {- static final ResponseHeaderOverrides NONE = new ResponseHeaderOverrides(null, null, null, null, null, null);-boolean hasAny() {return contentType != null || contentLanguage != null || expires != null|| cacheControl != null || contentDisposition != null || contentEncoding != null;@@ -1718,10 +1722,6 @@ public class S3Controller {}}- private void appendObjectHeaders(Response.ResponseBuilder resp, S3Object obj) {- appendObjectHeaders(resp, obj, ResponseHeaderOverrides.NONE);- }-// PutObject's response body is empty — body-describing headers and x-amz-meta-*// must not be emitted, or SDK clients will try to decompress and fail.private void appendPutObjectResponseHeaders(Response.ResponseBuilder resp, S3Object obj) {@@ -1736,10 +1736,6 @@ public class S3Controller {appendLockHeaders(resp, obj);}- private void appendObjectHeaders(Response.ResponseBuilder resp, S3Object obj, ResponseHeaderOverrides overrides) {- appendObjectHeaders(resp, obj, overrides, true);- }-// includeChecksum must be false for partial (206) responses: obj.getChecksum() is the// whole-object checksum, which does not match the range bytes returned. SDKs that// validate it against the received body fail. Real S3 omits it on ranged responses.diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.javaindex 2ccd494a19..7680c06311 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java@@ -87,7 +87,7 @@ class S3IntegrationTest {.header("Content-Length", notNullValue()).header("x-amz-meta-owner", equalTo("team-a")).header("x-amz-storage-class", equalTo("STANDARD_IA"))- .header("x-amz-checksum-crc64nvme", notNullValue())+ .header("x-amz-checksum-crc64nvme", nullValue()).body(equalTo("Hello World from S3!"));}@@ -118,7 +118,7 @@ class S3IntegrationTest {.header("Content-Length", notNullValue()).header("x-amz-meta-owner", equalTo("team-a")).header("x-amz-storage-class", equalTo("STANDARD_IA"))- .header("x-amz-checksum-crc64nvme", notNullValue());+ .header("x-amz-checksum-crc64nvme", nullValue());}@Test@@ -204,6 +204,7 @@ class S3IntegrationTest {// Verify the copygiven()+ .header("x-amz-checksum-mode", "ENABLED").when().get("/test-bucket/greeting-copy.txt").then()@@ -241,6 +242,7 @@ class S3IntegrationTest {// Verify the copy inherits the source checksum (CRC64NVME)given()+ .header("x-amz-checksum-mode", "ENABLED").when().get("/test-bucket/greeting-copy-no-override.txt").then()@@ -274,6 +276,7 @@ class S3IntegrationTest {// Verify the source object has a SHA256 checksumgiven()+ .header("x-amz-checksum-mode", "ENABLED").when().get("/test-bucket/sha256-source.txt").then()@@ -293,6 +296,7 @@ class S3IntegrationTest {// Verify the copy preserves the source's SHA256 checksumgiven()+ .header("x-amz-checksum-mode", "ENABLED").when().get("/test-bucket/sha256-copy.txt").then()@@ -532,6 +536,62 @@ class S3IntegrationTest {given().delete("/large-object-bucket");}+ @Test+ @Order(118)+ void getObjectWithChecksumModeReturnsChecksum() {+ given()+ .when()+ .put("/checksum-mode-bucket")+ .then()+ .statusCode(200);++ given()+ .body("Hello World from S3!")+ .when()+ .put("/checksum-mode-bucket/greeting.txt")+ .then()+ .statusCode(200);++ given()+ .header("x-amz-checksum-mode", "ENABLED")+ .when()+ .get("/checksum-mode-bucket/greeting.txt")+ .then()+ .statusCode(200)+ .header("x-amz-checksum-crc64nvme", notNullValue());++ given().delete("/checksum-mode-bucket/greeting.txt");+ given().delete("/checksum-mode-bucket");+ }++ @Test+ @Order(119)+ void headObjectWithChecksumModeReturnsChecksum() {+ given()+ .when()+ .put("/checksum-mode-head-bucket")+ .then()+ .statusCode(200);++ given()+ .body("Hello World from S3!")+ .when()+ .put("/checksum-mode-head-bucket/greeting.txt")+ .then()+ .statusCode(200);++ given()+ .header("x-amz-checksum-mode", "ENABLED")+ .when()+ .head("/checksum-mode-head-bucket/greeting.txt")+ .then()+ .statusCode(200)+ .header("x-amz-checksum-crc64nvme", notNullValue());++ given().delete("/checksum-mode-head-bucket/greeting.txt");+ given().delete("/checksum-mode-head-bucket");+ }+@Test@Order(27)void getObjectWithFullRange() {@@ -694,6 +754,7 @@ class S3IntegrationTest {.header("Content-Length", equalTo("0")).header("Accept-Ranges", equalTo("bytes")).header("x-amz-meta-kind", equalTo("empty"))+ .header("x-amz-checksum-crc64nvme", nullValue()).body(equalTo(""));given()@@ -703,6 +764,36 @@ class S3IntegrationTest {.statusCode(204);}+ @Test+ @Order(207)+ void getObjectWithSuffixRangeForEmptyObjectAndChecksumModeIncludesChecksum() {+ // The empty-object suffix-range path returns via fullObjectResponse (a full 200, not+ // a 206), so it must honor x-amz-checksum-mode like any other full-object response.+ given()+ .header("x-amz-meta-kind", "empty")+ .body(new byte[0])+ .when()+ .put("/test-bucket/empty-checksum.txt")+ .then()+ .statusCode(200);++ given()+ .header("Range", "bytes=-13")+ .header("x-amz-checksum-mode", "ENABLED")+ .when()+ .get("/test-bucket/empty-checksum.txt")+ .then()+ .statusCode(200)+ .header("Content-Length", equalTo("0"))+ .header("x-amz-checksum-crc64nvme", notNullValue());++ given()+ .when()+ .delete("/test-bucket/empty-checksum.txt")+ .then()+ .statusCode(204);+ }+@Test@Order(39)void getObjectRangeStreamsExactBytesFromLargeObject() {