Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): answer GetObject Range requests as S3 does (#5436)

Priamo Germosen09a2d5c005eb
diff --git a/docs/services/s3.md b/docs/services/s3.md
index bdc0f71fa2..9ee0ccc1b2 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -102,11 +102,11 @@ Redirect-all and advanced routing-rule configurations are not implemented.
## Range Reads
-`GetObject` supports standard `Range: bytes=` requests. Partial responses return `206`, `Content-Range`, `Content-Length`, `Accept-Ranges: bytes`, and object metadata. Whole-object checksum headers are omitted on ranged responses because the stored checksum covers the full object, not the returned byte slice. Full-object and ranged reads stream object bytes from storage instead of materializing the response body first.
+`GetObject` supports standard `Range: bytes=` requests. Partial responses return `206`, `Content-Range`, `Content-Length`, `Accept-Ranges: bytes`, and object metadata. Whole-object checksum headers are omitted on ranged responses because the stored checksum covers the full object, not the returned byte slice, except for a range covering the whole of an object stored in one request, or completed from one part with a full-object checksum: that object is its one part, and like a range aligned with a single part, the range gets its checksum. A composite checksum is not that of any one part, nor is a checksum of an object of several parts, so their ranges get none. A range that starts past the end of the object, or a suffix of zero bytes (`bytes=-0`), returns `416` `InvalidRange`, while a `Range` that is not one valid range, such as several ranges, a last byte before the first, or a malformed value, is ignored and the whole object is returned with `200`, as S3 does. Full-object and ranged reads stream object bytes from storage instead of materializing the response body first.
Suffix ranges against empty objects return an empty `200` response, matching AWS behavior used by clients that issue tail reads.
-`HeadObject` honors `Range` as the API Reference describes: a satisfiable range changes `Content-Length` to the length of the range, and one that cannot be satisfied returns `416`, while a `Range` that is not one valid range, such as several ranges or a last byte before the first, is ignored. Like a ranged `GetObject`, it leaves out the whole-object checksum headers.
+`HeadObject` honors `Range` as the API Reference describes: a satisfiable range changes `Content-Length` to the length of the range, and one that cannot be satisfied returns `416`, while a `Range` that is not one valid range, such as several ranges or a last byte before the first, is ignored. Like a ranged `GetObject`, it leaves out the whole-object checksum headers unless the range covers the whole of an object that is one part.
An `If-Range` header makes the range conditional, so a resumed download never mixes two versions of an object. The range is served while the validator still describes the object: a quoted ETag equal to the object's (a multipart ETag compared whole, `-N` suffix included), or an HTTP-date equal to its `Last-Modified`, an exact match as RFC 9110 section 13.1.5 requires, so a later date does not qualify. Otherwise, including a weak (`W/`) or malformed validator, `GetObject` returns the whole object with `200`. A presigned URL that signs `Range` must also sign an `If-Range` the request carries, or the request fails with `403 AccessDenied` naming it in `HeadersNotSigned`.
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java
index b67111045e..19dc95243e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java
@@ -618,7 +618,7 @@ public class CloudFrontServingController {
closeUnsent(read);
yield rangeNotSatisfiable(obj.getSize());
}
- case WHOLE, INVALID -> new OriginResponse(
+ case WHOLE -> new OriginResponse(
200, contentType(obj), null, read.body(), obj.getSize(), s3ObjectHeaders(obj));
};
}
@@ -634,7 +634,7 @@ public class CloudFrontServingController {
case PART -> new OriginResponse(
200, contentType(meta), null, resolution.range().length(), s3ObjectHeaders(meta));
case NOT_SATISFIABLE -> rangeNotSatisfiable(meta.getSize());
- case WHOLE, INVALID -> new OriginResponse(
+ case WHOLE -> new OriginResponse(
200, contentType(meta), null, meta.getSize(), s3ObjectHeaders(meta));
};
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/ByteRange.java b/src/main/java/io/github/hectorvent/floci/services/s3/ByteRange.java
index 1da7a0e8d4..92f9b2bdb8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/ByteRange.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/ByteRange.java
@@ -25,11 +25,12 @@ public record ByteRange(long first, long last) {
private static final Pattern SPEC = Pattern.compile("(\\d*)-(\\d*)");
/**
- * What a {@code Range} header asks of an object: a part of it, the whole of it (no header, a
- * unit other than bytes, or a suffix of an empty object), a range that starts past its end, or
- * something that is not one valid range, such as several ranges or a last byte before the first.
+ * What a {@code Range} header asks of an object: a part of it, the whole of it, or a range that
+ * starts past its end. The whole object is asked for by no header, a unit other than bytes, a
+ * suffix of an empty object, or anything that is not one valid range, such as several ranges or
+ * a last byte before the first, which S3 answers as a request without a {@code Range}.
*/
- public enum Outcome { PART, WHOLE, NOT_SATISFIABLE, INVALID }
+ public enum Outcome { PART, WHOLE, NOT_SATISFIABLE }
/** A resolved {@code Range} header; {@code range} is set only for {@link Outcome#PART}. */
public record Resolution(Outcome outcome, ByteRange range) {
@@ -102,7 +103,7 @@ public record ByteRange(long first, long last) {
}
Matcher matcher = SPEC.matcher(header.substring(UNIT.length()).trim());
if (!matcher.matches() || (matcher.group(1).isEmpty() && matcher.group(2).isEmpty())) {
- return new Resolution(Outcome.INVALID, null);
+ return new Resolution(Outcome.WHOLE, null);
}
long first;
long last;
@@ -120,7 +121,7 @@ public record ByteRange(long first, long last) {
first = position(matcher.group(1));
last = matcher.group(2).isEmpty() ? Long.MAX_VALUE : position(matcher.group(2));
if (last < first) {
- return new Resolution(Outcome.INVALID, null);
+ return new Resolution(Outcome.WHOLE, null);
}
last = Math.min(last, size - 1);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index fa451e8aba..8a27bcf8d7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -1126,7 +1126,7 @@ public class S3Controller {
emitCloudTrailEvent("GetObject", bucket, key, 0L, totalSize, null, null);
return fullObjectResponse(obj, body, overrides, includeChecksum);
}
- if (resolution.outcome() != ByteRange.Outcome.PART) {
+ if (resolution.outcome() == ByteRange.Outcome.NOT_SATISFIABLE) {
emitCloudTrailEvent("GetObject", bucket, key, 0L, 0L,
"InvalidRange", ByteRange.NOT_SATISFIABLE_MESSAGE);
return invalidRangeResponse(rangeHeader, totalSize);
@@ -1150,11 +1150,24 @@ public class S3Controller {
if (obj.getVersionId() != null) {
resp.header("x-amz-version-id", obj.getVersionId());
}
- // includeChecksum=false: the stored checksum covers the whole object, not this range.
- appendObjectHeaders(resp, obj, overrides, false);
+ appendObjectHeaders(resp, obj, overrides, includeChecksum && rangeCarriesObjectChecksum(range, obj));
return resp.build();
}
+ /**
+ * Whether the stored checksum is the checksum of {@code range}: S3 returns a part's checksum for a
+ * range aligned with a single part, and the whole of an object stored in one request, or completed
+ * from one part, is that part, whose checksum a full-object checksum of the object is. A composite
+ * checksum is not that of any one part, nor is a checksum of an object of several parts, so their
+ * ranges get none.
+ */
+ private static boolean rangeCarriesObjectChecksum(ByteRange range, S3Object obj) {
+ S3Checksum checksum = obj.getChecksum();
+ return range.first() == 0 && range.last() == obj.getSize() - 1
+ && (obj.getParts() == null || obj.getParts().size() <= 1)
+ && checksum != null && checksum.getChecksumType() != ChecksumType.COMPOSITE;
+ }
+
private Response websiteObjectResponse(S3Service.ObjectRead read, ResponseHeaderOverrides overrides) {
return fullObjectResponse(read.object(), read.body(), overrides, false);
}
@@ -1249,14 +1262,14 @@ public class S3Controller {
// "If the Range is satisfiable, only the ContentLength is affected in the response. If the
// Range is not satisfiable, S3 returns a 416." A Range that is not one valid range is ignored.
long contentLength = obj.getSize();
- boolean ranged = false;
+ boolean checksumDescribesRange = true;
String rangeHeader = httpHeaders.getHeaderString("Range");
if (ByteRange.inBytes(rangeHeader) && ByteRange.rangeApplies(
httpHeaders.getHeaderString("If-Range"), obj.getETag(), obj.getLastModified())) {
ByteRange.Resolution resolution = ByteRange.resolve(rangeHeader, obj.getSize());
if (resolution.outcome() == ByteRange.Outcome.PART) {
contentLength = resolution.range().length();
- ranged = true;
+ checksumDescribesRange = rangeCarriesObjectChecksum(resolution.range(), obj);
} else if (resolution.outcome() == ByteRange.Outcome.NOT_SATISFIABLE) {
emitCloudTrailEvent("HeadObject", bucket, key, 0L, 0L,
"InvalidRange", ByteRange.NOT_SATISFIABLE_MESSAGE);
@@ -1272,8 +1285,8 @@ public class S3Controller {
if (obj.getVersionId() != null) {
resp.header("x-amz-version-id", obj.getVersionId());
}
- // As on a ranged GetObject, the stored checksum covers the whole object, not the range.
- boolean includeChecksum = !ranged
+ // As on a ranged GetObject, the stored checksum is sent only for a range it describes.
+ boolean includeChecksum = checksumDescribesRange
&& "ENABLED".equalsIgnoreCase(resolveHeaderOrQueryParam(checksumMode, uriInfo, "x-amz-checksum-mode"));
appendObjectHeaders(resp, obj, overrides, includeChecksum);
emitCloudTrailEvent("HeadObject", bucket, key, 0L, obj.getSize(), null, null);
@@ -2720,9 +2733,9 @@ public class S3Controller {
appendLockHeaders(resp, obj);
}
- // includeChecksum must be false for partial (206) responses: obj.getChecksum() is the
- // whole-object checksum, which does not match the range bytes returned. SDKs that
- // validate it against the received body fail. Real S3 omits it on ranged responses.
+ // includeChecksum must be false for a range the stored checksum does not describe:
+ // obj.getChecksum() is the whole-object checksum, which does not match the range bytes
+ // returned. SDKs that validate it against the received body fail. Real S3 omits it there.
private void appendObjectHeaders(Response.ResponseBuilder resp, S3Object obj, ResponseHeaderOverrides overrides,
boolean includeChecksum) {
if (obj.getStorageClass() != null) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/ByteRangeTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/ByteRangeTest.java
index 8cfff2acdd..a0e5641fa4 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/ByteRangeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/ByteRangeTest.java
@@ -70,10 +70,10 @@ class ByteRangeTest {
}
@Test
- void anythingButOneValidRangeIsInvalid() {
+ void anythingButOneValidRangeIsTheWholeObject() {
for (String header : List.of("bytes=5-2", "bytes=0-1,4-5", "bytes=abc", "bytes=-", "bytes=1",
- "bytes=--5", "bytes=+1-2", "bytes=")) {
- assertOutcome(Outcome.INVALID, header, 100);
+ "bytes=--5", "bytes=+1-2", "bytes=", "bytes=-1-", "bytes=0--1")) {
+ assertOutcome(Outcome.WHOLE, header, 100);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
index 7b5b3ecea2..97d3ac76bb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
@@ -1112,6 +1112,16 @@ class S3IntegrationTest {
.statusCode(200)
.header("Content-Length", equalTo("20"))
.header("x-amz-checksum-crc64nvme", notNullValue());
+ given()
+ .header("Range", "bytes=0-")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .head("/head-range-bucket/greeting.txt")
+ .then()
+ .statusCode(200)
+ .header("Content-Length", equalTo("20"))
+ .header("x-amz-checksum-crc64nvme", notNullValue())
+ .header("x-amz-checksum-type", equalTo("FULL_OBJECT"));
given()
.header("Range", "bytes=50-")
.when()
@@ -1188,54 +1198,81 @@ class S3IntegrationTest {
.body(hasXPath("/Error/Message", equalTo("The requested range is not satisfiable")))
.body(hasXPath("/Error/RangeRequested", equalTo("bytes=50-100")))
.body(hasXPath("/Error/ActualObjectSize", equalTo("20")));
- }
- @Test
- @Order(31)
- void getObjectWithMalformedRangeNoDash() {
+ // A suffix of no bytes cannot be satisfied either.
given()
- .header("Range", "bytes=0")
+ .header("Range", "bytes=-0")
.when()
.get("/test-bucket/greeting.txt")
.then()
.statusCode(416)
- .body(containsString("InvalidRange"));
+ .header("Content-Range", equalTo("bytes */20"))
+ .body(hasXPath("/Error/Code", equalTo("InvalidRange")))
+ .body(hasXPath("/Error/RangeRequested", equalTo("bytes=-0")));
+ }
+
+ @Test
+ @Order(31)
+ void getObjectIgnoresARangeThatIsNotOneValidRange() {
+ // S3 answers each of these as a GetObject without a Range, with the whole object and 200.
+ for (String invalid : List.of("bytes=0", "bytes=-", "bytes=abc-def", "bytes=-1-4", "bytes=1-0",
+ "bytes=-1-", "bytes=0--1", "bytes=0-1,3-4,7-9", "0-1")) {
+ given()
+ .header("Range", invalid)
+ .when()
+ .get("/test-bucket/greeting.txt")
+ .then()
+ .statusCode(200)
+ .header("Content-Length", equalTo("20"))
+ .header("Content-Range", nullValue())
+ .header("Accept-Ranges", equalTo("bytes"))
+ .body(equalTo("Hello World from S3!"));
+ }
}
@Test
@Order(32)
- void getObjectWithMalformedRangeEmptySuffix() {
- given()
- .header("Range", "bytes=-")
+ void getObjectRangeCoveringTheWholeObjectCarriesItsChecksum() {
+ String wholeChecksum = given()
+ .header("x-amz-checksum-mode", "ENABLED")
.when()
.get("/test-bucket/greeting.txt")
.then()
- .statusCode(416)
- .body(containsString("InvalidRange"));
- }
+ .statusCode(200)
+ .extract().header("x-amz-checksum-crc64nvme");
+ assertNotNull(wholeChecksum);
- @Test
- @Order(33)
- void getObjectWithMalformedRangeNonNumeric() {
+ // The whole of an object stored in one request is its one part, so a range covering it gets
+ // the object's checksum, as a range aligned with a single part gets that part's.
+ for (String whole : List.of("bytes=0-19", "bytes=0-100", "bytes=-50", "bytes=0-")) {
+ given()
+ .header("Range", whole)
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .get("/test-bucket/greeting.txt")
+ .then()
+ .statusCode(206)
+ .header("Content-Range", equalTo("bytes 0-19/20"))
+ .header("x-amz-checksum-crc64nvme", equalTo(wholeChecksum))
+ .header("x-amz-checksum-type", equalTo("FULL_OBJECT"))
+ .body(equalTo("Hello World from S3!"));
+ }
given()
- .header("Range", "bytes=abc-def")
+ .header("Range", "bytes=0-19")
.when()
.get("/test-bucket/greeting.txt")
.then()
- .statusCode(416)
- .body(containsString("InvalidRange"));
- }
-
- @Test
- @Order(34)
- void getObjectWithMalformedRangeNegativeStart() {
+ .statusCode(206)
+ .header("x-amz-checksum-crc64nvme", nullValue());
given()
- .header("Range", "bytes=-1-4")
+ .header("Range", "bytes=0-18")
+ .header("x-amz-checksum-mode", "ENABLED")
.when()
.get("/test-bucket/greeting.txt")
.then()
- .statusCode(416)
- .body(containsString("InvalidRange"));
+ .statusCode(206)
+ .header("x-amz-checksum-crc64nvme", nullValue())
+ .header("x-amz-checksum-type", nullValue());
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
index cfbd344a7d..276d94e741 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
@@ -163,6 +163,27 @@ class S3MultipartIntegrationTest {
.statusCode(200)
.header("x-amz-checksum-sha256", equalTo(COMPOSITE_SHA256))
.header("x-amz-checksum-type", equalTo("COMPOSITE"));
+
+ // The composite checksum is not the checksum of any one part, so not even a range covering
+ // the whole object gets it.
+ given()
+ .header("Range", "bytes=0-")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .get("/" + BUCKET + "/" + KEY)
+ .then()
+ .statusCode(206)
+ .header("x-amz-checksum-sha256", nullValue())
+ .header("x-amz-checksum-type", nullValue())
+ .body(equalTo(PART_1 + PART_2));
+ given()
+ .header("Range", "bytes=0-")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .head("/" + BUCKET + "/" + KEY)
+ .then()
+ .statusCode(200)
+ .header("x-amz-checksum-sha256", nullValue());
}
@Test
@@ -1030,6 +1051,16 @@ class S3MultipartIntegrationTest {
.statusCode(200)
.header("x-amz-checksum-crc64nvme", equalTo(fullObjectCrc64))
.header("x-amz-checksum-type", equalTo("FULL_OBJECT"));
+
+ // A range covering an object of two parts is aligned with neither, so it gets no checksum.
+ given()
+ .header("Range", "bytes=0-")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .get("/" + BUCKET + "/" + key)
+ .then()
+ .statusCode(206)
+ .header("x-amz-checksum-crc64nvme", nullValue());
}
@Test
@@ -1410,6 +1441,56 @@ class S3MultipartIntegrationTest {
.statusCode(204);
}
+ @Test
+ @Order(39)
+ void aRangeCoveringAnUploadCompletedFromOnePartCarriesItsFullObjectChecksum() {
+ String key = "one-part-multipart.bin";
+ completeOnePartUpload(key, "CRC64NVME", "");
+
+ // The object is its one part, so a range covering it is aligned with that part, whose checksum
+ // the full-object checksum is.
+ String partCrc64 = S3Checksum.crc64NvmeBase64(PART_1.getBytes(StandardCharsets.UTF_8));
+ given()
+ .header("Range", "bytes=0-")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .get("/" + BUCKET + "/" + key)
+ .then()
+ .statusCode(206)
+ .header("x-amz-checksum-crc64nvme", equalTo(partCrc64))
+ .header("x-amz-checksum-type", equalTo("FULL_OBJECT"))
+ .body(equalTo(PART_1));
+ given()
+ .header("Range", "bytes=0-")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .head("/" + BUCKET + "/" + key)
+ .then()
+ .statusCode(200)
+ .header("x-amz-checksum-crc64nvme", equalTo(partCrc64));
+ given()
+ .header("Range", "bytes=1-")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .get("/" + BUCKET + "/" + key)
+ .then()
+ .statusCode(206)
+ .header("x-amz-checksum-crc64nvme", nullValue());
+
+ // A composite checksum of one part is not that part's checksum, so its range gets none.
+ completeOnePartUpload("one-part-composite.bin", "SHA256",
+ "<ChecksumSHA256>" + sha256Base64(PART_1) + "</ChecksumSHA256>");
+ given()
+ .header("Range", "bytes=0-")
+ .header("x-amz-checksum-mode", "ENABLED")
+ .when()
+ .get("/" + BUCKET + "/one-part-composite.bin")
+ .then()
+ .statusCode(206)
+ .header("x-amz-checksum-sha256", nullValue())
+ .header("x-amz-checksum-type", nullValue());
+ }
+
@Test
@Order(40)
void cleanUp() {
@@ -1428,6 +1509,8 @@ class S3MultipartIntegrationTest {
given().when().delete("/" + BUCKET + "/checksum-type-multipart.bin").then().statusCode(204);
given().when().delete("/" + BUCKET + "/completion-validation.bin").then().statusCode(204);
given().when().delete("/" + BUCKET + "/range-source.bin").then().statusCode(204);
+ given().when().delete("/" + BUCKET + "/one-part-multipart.bin").then().statusCode(204);
+ given().when().delete("/" + BUCKET + "/one-part-composite.bin").then().statusCode(204);
given().when().delete("/" + BUCKET).then().statusCode(204);
}
@@ -1442,6 +1525,26 @@ class S3MultipartIntegrationTest {
return S3Checksum.sha256Base64(data.getBytes(StandardCharsets.UTF_8));
}
+ private static void completeOnePartUpload(String key, String checksumAlgorithm, String checksumElement) {
+ String newUploadId = given()
+ .header("x-amz-checksum-algorithm", checksumAlgorithm)
+ .when()
+ .post("/" + BUCKET + "/" + key + "?uploads")
+ .then()
+ .statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+ String partETag = given().body(PART_1).when()
+ .put("/" + BUCKET + "/" + key + "?uploadId=" + newUploadId + "&partNumber=1")
+ .then().statusCode(200).extract().header("ETag");
+ given()
+ .contentType("application/xml")
+ .body(singlePartCompleteXml(partETag, checksumElement))
+ .when()
+ .post("/" + BUCKET + "/" + key + "?uploadId=" + newUploadId)
+ .then()
+ .statusCode(200);
+ }
+
private static String compositeSha256(String... parts) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");