Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): omit checksum headers from GetObject/HeadObject unless x-amz-checksum-mode: ENABLED (#1633)

Mihai Blidaru48f310b63fa5
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 2ee28a61b1..891cf2c8bc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -638,6 +638,7 @@ public class S3Controller {
@HeaderParam("If-Modified-Since") String ifModifiedSince,
@HeaderParam("If-Unmodified-Since") String ifUnmodifiedSince,
@HeaderParam("Range") String rangeHeader,
+ @HeaderParam("x-amz-checksum-mode") String checksumMode,
@Context UriInfo uriInfo,
@Context HttpHeaders httpHeaders) {
S3Service.RequestAuthorization authorization = S3Service.RequestAuthorization.unsigned();
@@ -698,11 +699,12 @@ public class S3Controller {
"Request specific response headers cannot be used for anonymous GET requests.", 400));
}
+ boolean includeChecksum = "ENABLED".equalsIgnoreCase(checksumMode);
if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {
- return handleRangeRequest(bucket, key, versionId, obj, rangeHeader, overrides);
+ return handleRangeRequest(bucket, key, versionId, obj, rangeHeader, overrides, includeChecksum);
}
- return fullObjectResponse(bucket, key, versionId, obj, overrides);
+ return fullObjectResponse(bucket, key, versionId, obj, overrides, includeChecksum);
} catch (AwsException e) {
if (isWebsiteErrorDocumentTrigger(e) && isWebsiteRequest(httpHeaders)) {
try {
@@ -722,7 +724,8 @@ public class S3Controller {
}
private Response fullObjectResponse(String bucket, String key, String versionId,
- S3Object obj, ResponseHeaderOverrides overrides) {
+ S3Object obj, ResponseHeaderOverrides overrides,
+ boolean includeChecksum) {
StreamingOutput stream = output -> {
try (InputStream input = s3Service.openObjectStream(bucket, key, versionId)) {
input.transferTo(output);
@@ -737,13 +740,14 @@ public class S3Controller {
if (obj.getVersionId() != null) {
resp.header("x-amz-version-id", obj.getVersionId());
}
- appendObjectHeaders(resp, obj, overrides);
+ appendObjectHeaders(resp, obj, overrides, includeChecksum);
return resp.build();
}
private Response handleRangeRequest(String bucket, String key, String versionId,
S3Object obj, String rangeHeader,
- ResponseHeaderOverrides overrides) {
+ ResponseHeaderOverrides overrides,
+ boolean includeChecksum) {
long totalSize = obj.getSize();
String rangeSpec = rangeHeader.substring("bytes=".length()).trim();
@@ -775,7 +779,7 @@ public class S3Controller {
if (start < 0 || start >= totalSize || start > end) {
if (totalSize == 0 && rangeSpec.startsWith("-")) {
- return fullObjectResponse(bucket, key, versionId, obj, overrides);
+ return fullObjectResponse(bucket, key, versionId, obj, overrides, includeChecksum);
}
return invalidRangeResponse(totalSize);
}
@@ -848,6 +852,7 @@ public class S3Controller {
@HeaderParam("If-None-Match") String ifNoneMatch,
@HeaderParam("If-Modified-Since") String ifModifiedSince,
@HeaderParam("If-Unmodified-Since") String ifUnmodifiedSince,
+ @HeaderParam("x-amz-checksum-mode") String checksumMode,
@Context UriInfo uriInfo,
@Context HttpHeaders httpHeaders) {
try {
@@ -882,7 +887,8 @@ public class S3Controller {
if (obj.getVersionId() != null) {
resp.header("x-amz-version-id", obj.getVersionId());
}
- appendObjectHeaders(resp, obj, overrides);
+ boolean includeChecksum = "ENABLED".equalsIgnoreCase(checksumMode);
+ appendObjectHeaders(resp, obj, overrides, includeChecksum);
return resp.build();
} catch (AwsException e) {
return xmlErrorResponse(e);
@@ -1696,8 +1702,6 @@ public class S3Controller {
String cacheControl,
String contentDisposition,
String contentEncoding) {
- static final ResponseHeaderOverrides NONE = new ResponseHeaderOverrides(null, null, null, null, null, null);
-
boolean hasAny() {
return contentType != null || contentLanguage != null || expires != null
|| cacheControl != null || contentDisposition != null || contentEncoding != null;
@@ -1718,10 +1722,6 @@ public class S3Controller {
}
}
- private void appendObjectHeaders(Response.ResponseBuilder resp, S3Object obj) {
- appendObjectHeaders(resp, obj, ResponseHeaderOverrides.NONE);
- }
-
// PutObject's response body is empty — body-describing headers and x-amz-meta-*
// must not be emitted, or SDK clients will try to decompress and fail.
private void appendPutObjectResponseHeaders(Response.ResponseBuilder resp, S3Object obj) {
@@ -1736,10 +1736,6 @@ public class S3Controller {
appendLockHeaders(resp, obj);
}
- private void appendObjectHeaders(Response.ResponseBuilder resp, S3Object obj, ResponseHeaderOverrides overrides) {
- appendObjectHeaders(resp, obj, overrides, true);
- }
-
// includeChecksum must be false for partial (206) responses: obj.getChecksum() is the
// whole-object checksum, which does not match the range bytes returned. SDKs that
// validate it against the received body fail. Real S3 omits it on ranged responses.
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
index 2ccd494a19..7680c06311 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
@@ -87,7 +87,7 @@ class S3IntegrationTest {
.header("Content-Length", notNullValue())
.header("x-amz-meta-owner", equalTo("team-a"))
.header("x-amz-storage-class", equalTo("STANDARD_IA"))
- .header("x-amz-checksum-crc64nvme", notNullValue())
+ .header("x-amz-checksum-crc64nvme", nullValue())
.body(equalTo("Hello World from S3!"));
}
@@ -118,7 +118,7 @@ class S3IntegrationTest {
.header("Content-Length", notNullValue())
.header("x-amz-meta-owner", equalTo("team-a"))
.header("x-amz-storage-class", equalTo("STANDARD_IA"))
- .header("x-amz-checksum-crc64nvme", notNullValue());
+ .header("x-amz-checksum-crc64nvme", nullValue());
}
@Test
@@ -204,6 +204,7 @@ class S3IntegrationTest {
// Verify the copy
given()
+ .header("x-amz-checksum-mode", "ENABLED")
.when()
.get("/test-bucket/greeting-copy.txt")
.then()
@@ -241,6 +242,7 @@ class S3IntegrationTest {
// Verify the copy inherits the source checksum (CRC64NVME)
given()
+ .header("x-amz-checksum-mode", "ENABLED")
.when()
.get("/test-bucket/greeting-copy-no-override.txt")
.then()
@@ -274,6 +276,7 @@ class S3IntegrationTest {
// Verify the source object has a SHA256 checksum
given()
+ .header("x-amz-checksum-mode", "ENABLED")
.when()
.get("/test-bucket/sha256-source.txt")
.then()
@@ -293,6 +296,7 @@ class S3IntegrationTest {
// Verify the copy preserves the source's SHA256 checksum
given()
+ .header("x-amz-checksum-mode", "ENABLED")
.when()
.get("/test-bucket/sha256-copy.txt")
.then()
@@ -532,6 +536,62 @@ class S3IntegrationTest {
given().delete("/large-object-bucket");
}
+ @Test
+ @Order(118)
+ void getObjectWithChecksumModeReturnsChecksum() {
+ given()
+ .when()
+ .put("/checksum-mode-bucket")
+ .then()
+ .statusCode(200);
+
+ given()
+ .body("Hello World from S3!")
+ .when()
+ .put("/checksum-mode-bucket/greeting.txt")
+ .then()
+ .statusCode(200);
+
+ given()
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .get("/checksum-mode-bucket/greeting.txt")
+ .then()
+ .statusCode(200)
+ .header("x-amz-checksum-crc64nvme", notNullValue());
+
+ given().delete("/checksum-mode-bucket/greeting.txt");
+ given().delete("/checksum-mode-bucket");
+ }
+
+ @Test
+ @Order(119)
+ void headObjectWithChecksumModeReturnsChecksum() {
+ given()
+ .when()
+ .put("/checksum-mode-head-bucket")
+ .then()
+ .statusCode(200);
+
+ given()
+ .body("Hello World from S3!")
+ .when()
+ .put("/checksum-mode-head-bucket/greeting.txt")
+ .then()
+ .statusCode(200);
+
+ given()
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .head("/checksum-mode-head-bucket/greeting.txt")
+ .then()
+ .statusCode(200)
+ .header("x-amz-checksum-crc64nvme", notNullValue());
+
+ given().delete("/checksum-mode-head-bucket/greeting.txt");
+ given().delete("/checksum-mode-head-bucket");
+ }
+
@Test
@Order(27)
void getObjectWithFullRange() {
@@ -694,6 +754,7 @@ class S3IntegrationTest {
.header("Content-Length", equalTo("0"))
.header("Accept-Ranges", equalTo("bytes"))
.header("x-amz-meta-kind", equalTo("empty"))
+ .header("x-amz-checksum-crc64nvme", nullValue())
.body(equalTo(""));
given()
@@ -703,6 +764,36 @@ class S3IntegrationTest {
.statusCode(204);
}
+ @Test
+ @Order(207)
+ void getObjectWithSuffixRangeForEmptyObjectAndChecksumModeIncludesChecksum() {
+ // The empty-object suffix-range path returns via fullObjectResponse (a full 200, not
+ // a 206), so it must honor x-amz-checksum-mode like any other full-object response.
+ given()
+ .header("x-amz-meta-kind", "empty")
+ .body(new byte[0])
+ .when()
+ .put("/test-bucket/empty-checksum.txt")
+ .then()
+ .statusCode(200);
+
+ given()
+ .header("Range", "bytes=-13")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .get("/test-bucket/empty-checksum.txt")
+ .then()
+ .statusCode(200)
+ .header("Content-Length", equalTo("0"))
+ .header("x-amz-checksum-crc64nvme", notNullValue());
+
+ given()
+ .when()
+ .delete("/test-bucket/empty-checksum.txt")
+ .then()
+ .statusCode(204);
+ }
+
@Test
@Order(39)
void getObjectRangeStreamsExactBytesFromLargeObject() {