Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): validate multipart upload object size (#5643)
Mohith4d6ecc1ac239
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartObjectSizeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartObjectSizeTest.javanew file mode 100644index 0000000000..525567b4c3--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartObjectSizeTest.java@@ -0,0 +1,65 @@+package com.floci.test;++import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import software.amazon.awssdk.core.sync.RequestBody;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;+import software.amazon.awssdk.services.s3.model.ChecksumType;+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;+import software.amazon.awssdk.services.s3.model.CompletedPart;+import software.amazon.awssdk.services.s3.model.S3Exception;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++@DisplayName("S3 multipart object size validation")+class S3MultipartObjectSizeTest {++ @ParameterizedTest+ @ValueSource(longs = {-1, Long.MIN_VALUE, 3, 5})+ void invalidObjectSizePreservesTheObjectAndUploadForRetry(long expectedSize) {+ String bucket = TestFixtures.uniqueName("sdk-multipart-size");+ String key = "object";+ try (S3Client s3 = TestFixtures.s3Client()) {+ s3.createBucket(request -> request.bucket(bucket));+ try {+ s3.putObject(request -> request.bucket(bucket).key(key), RequestBody.fromString("prior"));+ String uploadId = s3.createMultipartUpload(request -> request.bucket(bucket).key(key)+ .checksumAlgorithm(ChecksumAlgorithm.CRC32).checksumType(ChecksumType.FULL_OBJECT)).uploadId();+ String eTag = s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId)+ .partNumber(1), RequestBody.fromString("data")).eTag();+ s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId).partNumber(2),+ RequestBody.fromString("unused"));+ CompletedMultipartUpload parts = CompletedMultipartUpload.builder()+ .parts(CompletedPart.builder().partNumber(1).eTag(eTag).build()).build();+ assertThatThrownBy(() -> s3.completeMultipartUpload(request -> request.bucket(bucket).key(key)+ .uploadId(uploadId).multipartUpload(parts).mpuObjectSize(expectedSize)))+ .isInstanceOfSatisfying(S3Exception.class, error -> {+ assertThat(error.statusCode()).isEqualTo(400);+ assertThat(error.awsErrorDetails().errorCode()).isEqualTo("InvalidRequest");+ String expectedMessage = expectedSize < 0+ ? "Value for x-amz-mp-object-size header is less than zero: '" + expectedSize + "'"+ : "The provided 'x-amz-mp-object-size' header value " + expectedSize+ + " does not match what was computed: 4";+ assertThat(error.awsErrorDetails().errorMessage()).isEqualTo(expectedMessage);+ });+ assertThat(s3.getObjectAsBytes(request -> request.bucket(bucket).key(key)).asUtf8String())+ .isEqualTo("prior");+ assertThat(s3.listParts(request -> request.bucket(bucket).key(key).uploadId(uploadId)).parts())+ .hasSize(2);+ s3.completeMultipartUpload(request -> request.bucket(bucket).key(key).uploadId(uploadId)+ .multipartUpload(parts).mpuObjectSize(4L));+ assertThat(s3.getObjectAsBytes(request -> request.bucket(bucket).key(key)).asUtf8String())+ .isEqualTo("data");+ } finally {+ s3.listMultipartUploads(request -> request.bucket(bucket)).uploads().forEach(upload ->+ s3.abortMultipartUpload(request -> request.bucket(bucket).key(upload.key())+ .uploadId(upload.uploadId())));+ s3.deleteObject(request -> request.bucket(bucket).key(key));+ s3.deleteBucket(request -> request.bucket(bucket));+ }+ }+ }+}diff --git a/docs/services/s3.md b/docs/services/s3.mdindex e23993062e..e009d7487c 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -601,6 +601,7 @@ Floci now persists and returns the following object attribute state on S3 object- storage class from `x-amz-storage-class`- checksum metadata for object reads and `GetObjectAttributes`- multipart part manifests for `GetObjectAttributes(ObjectParts)`+- `CompleteMultipartUpload` validates `x-amz-mp-object-size` against the total size of the selected parts. A mismatch returns `InvalidRequest` without replacing the object or discarding the upload, so the request can be retried with the correct size. Blank, malformed, overflowing, or negative sizes also return `InvalidRequest` and preserve the object and upload- multipart checksums follow the AWS checksum type: `COMPOSITE` (`<checksum of the part checksums>-<part count>`) for SHA1, SHA256 and, by default, CRC32 and CRC32C; `FULL_OBJECT` for CRC64NVME or when `x-amz-checksum-type: FULL_OBJECT` was requested on `CreateMultipartUpload`. The `-N` suffix is returned by `CompleteMultipartUpload`, `HeadObject` and `GetObject`, and omitted by `GetObjectAttributes`, which lists part-level checksums only for `COMPOSITE` objects (a `FULL_OBJECT` multipart object reports its `PartsCount` alone, a single-part object has no `ObjectParts`), as on AWS. `UploadPart` echoes the part checksum (`x-amz-checksum-<algorithm>`) when the upload declared an algorithm, and completing a `COMPOSITE` upload requires the checksum of every part in the `CompleteMultipartUpload` body (`InvalidRequest` otherwise), while `FULL_OBJECT` uploads accept a body without them- canned object ACLs from `x-amz-acl` on `PutObject`, `CopyObject`, and multipart initiation- explicit object SSE headers from `x-amz-server-side-encryption` on `PutObject`, `CopyObject`, and multipart initiation, replayed on `GetObject` and `HeadObject`diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 5cb1068a12..ec39cb9d11 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -1645,7 +1645,7 @@ public class S3Controller {S3Checksum expectedChecksum = extractChecksum(httpHeaders, uriInfo);S3Object obj = s3Service.completeMultipartUpload(bucket, key, uploadId, partNumbers,completedPartETags(completedParts), completedPartChecksums(completedParts), checksumType,- expectedChecksum);+ expectedChecksum, multipartObjectSize(httpHeaders, uriInfo));String baseUrl = uriInfo.getBaseUri().toString();if (baseUrl.endsWith("/")) {baseUrl = baseUrl.substring(0, baseUrl.length() - 1);@@ -2298,6 +2298,35 @@ public class S3Controller {}}+ private static Long multipartObjectSize(HttpHeaders httpHeaders, UriInfo uriInfo) {+ String name = "x-amz-mp-object-size";+ String value = httpHeaders != null ? httpHeaders.getHeaderString(name) : null;+ if (value == null && uriInfo != null && uriInfo.getQueryParameters() != null) {+ for (Map.Entry<String, List<String>> entry : uriInfo.getQueryParameters().entrySet()) {+ if (entry.getKey().equalsIgnoreCase(name) && !entry.getValue().isEmpty()) {+ value = entry.getValue().getFirst();+ break;+ }+ }+ }+ if (value == null) {+ return null;+ }+ String normalized = value.trim();+ long size;+ try {+ size = Long.parseLong(normalized);+ } catch (NumberFormatException e) {+ throw new AwsException("InvalidRequest", "Value for x-amz-mp-object-size header is invalid: '"+ + normalized + "'", 400);+ }+ if (size < 0) {+ throw new AwsException("InvalidRequest", "Value for x-amz-mp-object-size header is less than zero: '"+ + normalized + "'", 400);+ }+ return size;+ }+private static String trailerHeader(HttpHeaders httpHeaders, UriInfo uriInfo) {return resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-trailer");}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 794a536206..80d2543c46 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -4197,6 +4197,13 @@ public class S3Service implements Resettable, ResourceProvider {public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List<Integer> partNumbers,Map<Integer, String> partETags, Map<Integer, S3Checksum> partChecksums,String checksumType, S3Checksum expectedChecksum) {+ return completeMultipartUpload(bucket, key, uploadId, partNumbers, partETags, partChecksums,+ checksumType, expectedChecksum, null);+ }++ public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List<Integer> partNumbers,+ Map<Integer, String> partETags, Map<Integer, S3Checksum> partChecksums,+ String checksumType, S3Checksum expectedChecksum, Long expectedObjectSize) {S3Object completed = withMultipartOperationLock(bucket, uploadId, () -> {MultipartUpload upload = getMultipartUpload(bucket, key, uploadId);@@ -4227,8 +4234,13 @@ public class S3Service implements Resettable, ResourceProvider {}validateCompleteChecksumType(algorithm, storedChecksumType, ChecksumType.fromWireValue(checksumType));+ long objectSize = parts.stream().mapToLong(Part::getSize).sum();+ if (expectedObjectSize != null && expectedObjectSize != objectSize) {+ throw new AwsException("InvalidRequest", "The provided 'x-amz-mp-object-size' header value "+ + expectedObjectSize + " does not match what was computed: " + objectSize, 400);+ }if (inMemory) {- requireFitsInMemory("Multipart upload " + uploadId, parts.stream().mapToLong(Part::getSize).sum());+ requireFitsInMemory("Multipart upload " + uploadId, objectSize);}// Concatenate parts in orderdiff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartObjectSizeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartObjectSizeIntegrationTest.javanew file mode 100644index 0000000000..0887bebb45--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartObjectSizeIntegrationTest.java@@ -0,0 +1,104 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.specification.RequestSpecification;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;++@QuarkusTest+class S3MultipartObjectSizeIntegrationTest {++ @ParameterizedTest+ @ValueSource(strings = {"", " ", "not-a-number", "1.5", "9223372036854775808", "-9223372036854775809"})+ void malformedSizeHeaderPreservesObjectAndUploadForRetry(String expectedSize) {+ assertRejectedSizePreservesObjectAndUpload(expectedSize, false, "invalid");+ }++ @ParameterizedTest+ @ValueSource(strings = {"", " ", "not-a-number", "9223372036854775808"})+ void malformedSizeQueryPreservesObjectAndUploadForRetry(String expectedSize) {+ assertRejectedSizePreservesObjectAndUpload(expectedSize, true, "invalid");+ }++ @ParameterizedTest+ @ValueSource(strings = {"-1", "-9223372036854775808"})+ void negativeSizeHeaderPreservesObjectAndUploadForRetry(String expectedSize) {+ assertRejectedSizePreservesObjectAndUpload(expectedSize, false, "less than zero");+ }++ private void assertRejectedSizePreservesObjectAndUpload(String expectedSize, boolean query, String reason) {+ String bucket = "multipart-size-" + UUID.randomUUID();+ String path = "/" + bucket + "/object";+ given().put("/" + bucket).then().statusCode(200);+ String uploadId = null;+ try {+ given().body("prior").put(path).then().statusCode(200);+ uploadId = given().post(path + "?uploads").then().statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");+ String eTag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 1).body("data")+ .put(path).then().statusCode(200).extract().header("ETag");+ String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"+ + eTag + "</ETag></Part></CompleteMultipartUpload>";+ RequestSpecification request = given().queryParam("uploadId", uploadId);+ if (query) {+ request.queryParam("x-amz-mp-object-size", expectedSize);+ } else {+ request.header("x-amz-mp-object-size", expectedSize).queryParam("x-amz-mp-object-size", 4);+ }+ request.body(completion).post(path).then().statusCode(400)+ .body("Error.Code", equalTo("InvalidRequest"))+ .body("Error.Message", equalTo("Value for x-amz-mp-object-size header is "+ + reason + ": '" + expectedSize.trim() + "'"));+ given().get(path).then().statusCode(200).body(equalTo("prior"));+ given().queryParam("uploadId", uploadId).get(path).then().statusCode(200)+ .body("ListPartsResult.Part.PartNumber", equalTo("1"))+ .body("ListPartsResult.Part.ETag", equalTo(eTag))+ .body("ListPartsResult.Part.Size", equalTo("4"));+ given().queryParam("uploadId", uploadId).header("x-amz-mp-object-size", 4)+ .body(completion).post(path).then().statusCode(200);+ given().get(path).then().statusCode(200).body(equalTo("data"));+ } finally {+ if (uploadId != null) {+ given().queryParam("uploadId", uploadId).delete(path);+ }+ given().delete(path);+ given().delete("/" + bucket);+ }+ }++ @ParameterizedTest+ @ValueSource(longs = {0, 3, 5, 4294967296L})+ void mismatchedSizeRejectsCompletionAndPreservesUploadForRetry(long expectedSize) {+ String bucket = "multipart-size-" + UUID.randomUUID();+ String path = "/" + bucket + "/object";+ given().put("/" + bucket).then().statusCode(200);+ try {+ String uploadId = given().post(path + "?uploads").then().statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");+ String eTag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 1).body("data")+ .put(path).then().statusCode(200).extract().header("ETag");+ String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"+ + eTag + "</ETag></Part></CompleteMultipartUpload>";+ given().queryParam("uploadId", uploadId).header("x-amz-mp-object-size", expectedSize)+ .body(completion).post(path).then().statusCode(400)+ .body("Error.Code", equalTo("InvalidRequest"))+ .body("Error.Message", equalTo("The provided 'x-amz-mp-object-size' header value "+ + expectedSize + " does not match what was computed: 4"));+ given().head(path).then().statusCode(404);+ given().queryParam("uploadId", uploadId).get(path).then().statusCode(200)+ .body(containsString("<PartNumber>1</PartNumber>"));+ given().queryParam("uploadId", uploadId).header("x-amz-mp-object-size", 4)+ .body(completion).post(path).then().statusCode(200);+ given().get(path).then().statusCode(200).body(equalTo("data"));+ } finally {+ given().delete(path);+ given().delete("/" + bucket);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.javaindex c6cb3d159c..27f6727648 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java@@ -13,6 +13,8 @@ import io.github.hectorvent.floci.services.s3.model.S3Object;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import java.nio.charset.StandardCharsets;import java.nio.file.Path;@@ -40,6 +42,31 @@ class S3MultipartServiceTest {s3Service.createBucket("test-bucket", "us-east-1");}+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void expectedObjectSizeUsesSelectedPartsAndPreservesExistingObjectOnMismatch(boolean memory) {+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),+ tempDir.resolve("size-check"), memory);+ service.createBucket("sized-bucket", "us-east-1");+ service.putObject("sized-bucket", "object", "prior".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ MultipartUpload upload = service.initiateMultipartUpload("sized-bucket", "object", null);+ service.uploadPart("sized-bucket", "object", upload.getUploadId(), 1,+ "data".getBytes(StandardCharsets.UTF_8));+ service.uploadPart("sized-bucket", "object", upload.getUploadId(), 2,+ "unused".getBytes(StandardCharsets.UTF_8));+ AwsException error = assertThrows(AwsException.class, () -> service.completeMultipartUpload(+ "sized-bucket", "object", upload.getUploadId(), List.of(1), Map.of(), Map.of(), null, null, 10L));+ assertEquals("InvalidRequest", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("prior", new String(service.getObject("sized-bucket", "object").getData(),+ StandardCharsets.UTF_8));+ S3Object completed = service.completeMultipartUpload("sized-bucket", "object", upload.getUploadId(),+ List.of(1), Map.of(), Map.of(), null, null, 4L);+ assertEquals(4, completed.getSize());+ assertEquals("data", new String(service.getObject("sized-bucket", "object").getData(),+ StandardCharsets.UTF_8));+ }+@Testvoid initiateMultipartUpload() {MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "large-file.bin", "application/octet-stream");