Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): validate multipart upload object size (#5643)

Mohith4d6ecc1ac239
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartObjectSizeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartObjectSizeTest.java
new file mode 100644
index 0000000000..525567b4c3
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartObjectSizeTest.java
@@ -0,0 +1,65 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
+import software.amazon.awssdk.services.s3.model.ChecksumType;
+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@DisplayName("S3 multipart object size validation")
+class S3MultipartObjectSizeTest {
+
+ @ParameterizedTest
+ @ValueSource(longs = {-1, Long.MIN_VALUE, 3, 5})
+ void invalidObjectSizePreservesTheObjectAndUploadForRetry(long expectedSize) {
+ String bucket = TestFixtures.uniqueName("sdk-multipart-size");
+ String key = "object";
+ try (S3Client s3 = TestFixtures.s3Client()) {
+ s3.createBucket(request -> request.bucket(bucket));
+ try {
+ s3.putObject(request -> request.bucket(bucket).key(key), RequestBody.fromString("prior"));
+ String uploadId = s3.createMultipartUpload(request -> request.bucket(bucket).key(key)
+ .checksumAlgorithm(ChecksumAlgorithm.CRC32).checksumType(ChecksumType.FULL_OBJECT)).uploadId();
+ String eTag = s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId)
+ .partNumber(1), RequestBody.fromString("data")).eTag();
+ s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId).partNumber(2),
+ RequestBody.fromString("unused"));
+ CompletedMultipartUpload parts = CompletedMultipartUpload.builder()
+ .parts(CompletedPart.builder().partNumber(1).eTag(eTag).build()).build();
+ assertThatThrownBy(() -> s3.completeMultipartUpload(request -> request.bucket(bucket).key(key)
+ .uploadId(uploadId).multipartUpload(parts).mpuObjectSize(expectedSize)))
+ .isInstanceOfSatisfying(S3Exception.class, error -> {
+ assertThat(error.statusCode()).isEqualTo(400);
+ assertThat(error.awsErrorDetails().errorCode()).isEqualTo("InvalidRequest");
+ String expectedMessage = expectedSize < 0
+ ? "Value for x-amz-mp-object-size header is less than zero: '" + expectedSize + "'"
+ : "The provided 'x-amz-mp-object-size' header value " + expectedSize
+ + " does not match what was computed: 4";
+ assertThat(error.awsErrorDetails().errorMessage()).isEqualTo(expectedMessage);
+ });
+ assertThat(s3.getObjectAsBytes(request -> request.bucket(bucket).key(key)).asUtf8String())
+ .isEqualTo("prior");
+ assertThat(s3.listParts(request -> request.bucket(bucket).key(key).uploadId(uploadId)).parts())
+ .hasSize(2);
+ s3.completeMultipartUpload(request -> request.bucket(bucket).key(key).uploadId(uploadId)
+ .multipartUpload(parts).mpuObjectSize(4L));
+ assertThat(s3.getObjectAsBytes(request -> request.bucket(bucket).key(key)).asUtf8String())
+ .isEqualTo("data");
+ } finally {
+ s3.listMultipartUploads(request -> request.bucket(bucket)).uploads().forEach(upload ->
+ s3.abortMultipartUpload(request -> request.bucket(bucket).key(upload.key())
+ .uploadId(upload.uploadId())));
+ s3.deleteObject(request -> request.bucket(bucket).key(key));
+ s3.deleteBucket(request -> request.bucket(bucket));
+ }
+ }
+ }
+}
diff --git a/docs/services/s3.md b/docs/services/s3.md
index e23993062e..e009d7487c 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -601,6 +601,7 @@ Floci now persists and returns the following object attribute state on S3 object
- storage class from `x-amz-storage-class`
- checksum metadata for object reads and `GetObjectAttributes`
- multipart part manifests for `GetObjectAttributes(ObjectParts)`
+- `CompleteMultipartUpload` validates `x-amz-mp-object-size` against the total size of the selected parts. A mismatch returns `InvalidRequest` without replacing the object or discarding the upload, so the request can be retried with the correct size. Blank, malformed, overflowing, or negative sizes also return `InvalidRequest` and preserve the object and upload
- multipart checksums follow the AWS checksum type: `COMPOSITE` (`<checksum of the part checksums>-<part count>`) for SHA1, SHA256 and, by default, CRC32 and CRC32C; `FULL_OBJECT` for CRC64NVME or when `x-amz-checksum-type: FULL_OBJECT` was requested on `CreateMultipartUpload`. The `-N` suffix is returned by `CompleteMultipartUpload`, `HeadObject` and `GetObject`, and omitted by `GetObjectAttributes`, which lists part-level checksums only for `COMPOSITE` objects (a `FULL_OBJECT` multipart object reports its `PartsCount` alone, a single-part object has no `ObjectParts`), as on AWS. `UploadPart` echoes the part checksum (`x-amz-checksum-<algorithm>`) when the upload declared an algorithm, and completing a `COMPOSITE` upload requires the checksum of every part in the `CompleteMultipartUpload` body (`InvalidRequest` otherwise), while `FULL_OBJECT` uploads accept a body without them
- canned object ACLs from `x-amz-acl` on `PutObject`, `CopyObject`, and multipart initiation
- explicit object SSE headers from `x-amz-server-side-encryption` on `PutObject`, `CopyObject`, and multipart initiation, replayed on `GetObject` and `HeadObject`
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 5cb1068a12..ec39cb9d11 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -1645,7 +1645,7 @@ public class S3Controller {
S3Checksum expectedChecksum = extractChecksum(httpHeaders, uriInfo);
S3Object obj = s3Service.completeMultipartUpload(bucket, key, uploadId, partNumbers,
completedPartETags(completedParts), completedPartChecksums(completedParts), checksumType,
- expectedChecksum);
+ expectedChecksum, multipartObjectSize(httpHeaders, uriInfo));
String baseUrl = uriInfo.getBaseUri().toString();
if (baseUrl.endsWith("/")) {
baseUrl = baseUrl.substring(0, baseUrl.length() - 1);
@@ -2298,6 +2298,35 @@ public class S3Controller {
}
}
+ private static Long multipartObjectSize(HttpHeaders httpHeaders, UriInfo uriInfo) {
+ String name = "x-amz-mp-object-size";
+ String value = httpHeaders != null ? httpHeaders.getHeaderString(name) : null;
+ if (value == null && uriInfo != null && uriInfo.getQueryParameters() != null) {
+ for (Map.Entry<String, List<String>> entry : uriInfo.getQueryParameters().entrySet()) {
+ if (entry.getKey().equalsIgnoreCase(name) && !entry.getValue().isEmpty()) {
+ value = entry.getValue().getFirst();
+ break;
+ }
+ }
+ }
+ if (value == null) {
+ return null;
+ }
+ String normalized = value.trim();
+ long size;
+ try {
+ size = Long.parseLong(normalized);
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidRequest", "Value for x-amz-mp-object-size header is invalid: '"
+ + normalized + "'", 400);
+ }
+ if (size < 0) {
+ throw new AwsException("InvalidRequest", "Value for x-amz-mp-object-size header is less than zero: '"
+ + normalized + "'", 400);
+ }
+ return size;
+ }
+
private static String trailerHeader(HttpHeaders httpHeaders, UriInfo uriInfo) {
return resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-trailer");
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 794a536206..80d2543c46 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -4197,6 +4197,13 @@ public class S3Service implements Resettable, ResourceProvider {
public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List<Integer> partNumbers,
Map<Integer, String> partETags, Map<Integer, S3Checksum> partChecksums,
String checksumType, S3Checksum expectedChecksum) {
+ return completeMultipartUpload(bucket, key, uploadId, partNumbers, partETags, partChecksums,
+ checksumType, expectedChecksum, null);
+ }
+
+ public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List<Integer> partNumbers,
+ Map<Integer, String> partETags, Map<Integer, S3Checksum> partChecksums,
+ String checksumType, S3Checksum expectedChecksum, Long expectedObjectSize) {
S3Object completed = withMultipartOperationLock(bucket, uploadId, () -> {
MultipartUpload upload = getMultipartUpload(bucket, key, uploadId);
@@ -4227,8 +4234,13 @@ public class S3Service implements Resettable, ResourceProvider {
}
validateCompleteChecksumType(algorithm, storedChecksumType, ChecksumType.fromWireValue(checksumType));
+ long objectSize = parts.stream().mapToLong(Part::getSize).sum();
+ if (expectedObjectSize != null && expectedObjectSize != objectSize) {
+ throw new AwsException("InvalidRequest", "The provided 'x-amz-mp-object-size' header value "
+ + expectedObjectSize + " does not match what was computed: " + objectSize, 400);
+ }
if (inMemory) {
- requireFitsInMemory("Multipart upload " + uploadId, parts.stream().mapToLong(Part::getSize).sum());
+ requireFitsInMemory("Multipart upload " + uploadId, objectSize);
}
// Concatenate parts in order
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartObjectSizeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartObjectSizeIntegrationTest.java
new file mode 100644
index 0000000000..0887bebb45
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartObjectSizeIntegrationTest.java
@@ -0,0 +1,104 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.specification.RequestSpecification;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+@QuarkusTest
+class S3MultipartObjectSizeIntegrationTest {
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", " ", "not-a-number", "1.5", "9223372036854775808", "-9223372036854775809"})
+ void malformedSizeHeaderPreservesObjectAndUploadForRetry(String expectedSize) {
+ assertRejectedSizePreservesObjectAndUpload(expectedSize, false, "invalid");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", " ", "not-a-number", "9223372036854775808"})
+ void malformedSizeQueryPreservesObjectAndUploadForRetry(String expectedSize) {
+ assertRejectedSizePreservesObjectAndUpload(expectedSize, true, "invalid");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"-1", "-9223372036854775808"})
+ void negativeSizeHeaderPreservesObjectAndUploadForRetry(String expectedSize) {
+ assertRejectedSizePreservesObjectAndUpload(expectedSize, false, "less than zero");
+ }
+
+ private void assertRejectedSizePreservesObjectAndUpload(String expectedSize, boolean query, String reason) {
+ String bucket = "multipart-size-" + UUID.randomUUID();
+ String path = "/" + bucket + "/object";
+ given().put("/" + bucket).then().statusCode(200);
+ String uploadId = null;
+ try {
+ given().body("prior").put(path).then().statusCode(200);
+ uploadId = given().post(path + "?uploads").then().statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+ String eTag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 1).body("data")
+ .put(path).then().statusCode(200).extract().header("ETag");
+ String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"
+ + eTag + "</ETag></Part></CompleteMultipartUpload>";
+ RequestSpecification request = given().queryParam("uploadId", uploadId);
+ if (query) {
+ request.queryParam("x-amz-mp-object-size", expectedSize);
+ } else {
+ request.header("x-amz-mp-object-size", expectedSize).queryParam("x-amz-mp-object-size", 4);
+ }
+ request.body(completion).post(path).then().statusCode(400)
+ .body("Error.Code", equalTo("InvalidRequest"))
+ .body("Error.Message", equalTo("Value for x-amz-mp-object-size header is "
+ + reason + ": '" + expectedSize.trim() + "'"));
+ given().get(path).then().statusCode(200).body(equalTo("prior"));
+ given().queryParam("uploadId", uploadId).get(path).then().statusCode(200)
+ .body("ListPartsResult.Part.PartNumber", equalTo("1"))
+ .body("ListPartsResult.Part.ETag", equalTo(eTag))
+ .body("ListPartsResult.Part.Size", equalTo("4"));
+ given().queryParam("uploadId", uploadId).header("x-amz-mp-object-size", 4)
+ .body(completion).post(path).then().statusCode(200);
+ given().get(path).then().statusCode(200).body(equalTo("data"));
+ } finally {
+ if (uploadId != null) {
+ given().queryParam("uploadId", uploadId).delete(path);
+ }
+ given().delete(path);
+ given().delete("/" + bucket);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(longs = {0, 3, 5, 4294967296L})
+ void mismatchedSizeRejectsCompletionAndPreservesUploadForRetry(long expectedSize) {
+ String bucket = "multipart-size-" + UUID.randomUUID();
+ String path = "/" + bucket + "/object";
+ given().put("/" + bucket).then().statusCode(200);
+ try {
+ String uploadId = given().post(path + "?uploads").then().statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+ String eTag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 1).body("data")
+ .put(path).then().statusCode(200).extract().header("ETag");
+ String completion = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"
+ + eTag + "</ETag></Part></CompleteMultipartUpload>";
+ given().queryParam("uploadId", uploadId).header("x-amz-mp-object-size", expectedSize)
+ .body(completion).post(path).then().statusCode(400)
+ .body("Error.Code", equalTo("InvalidRequest"))
+ .body("Error.Message", equalTo("The provided 'x-amz-mp-object-size' header value "
+ + expectedSize + " does not match what was computed: 4"));
+ given().head(path).then().statusCode(404);
+ given().queryParam("uploadId", uploadId).get(path).then().statusCode(200)
+ .body(containsString("<PartNumber>1</PartNumber>"));
+ given().queryParam("uploadId", uploadId).header("x-amz-mp-object-size", 4)
+ .body(completion).post(path).then().statusCode(200);
+ given().get(path).then().statusCode(200).body(equalTo("data"));
+ } finally {
+ given().delete(path);
+ given().delete("/" + bucket);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
index c6cb3d159c..27f6727648 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
@@ -13,6 +13,8 @@ import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
@@ -40,6 +42,31 @@ class S3MultipartServiceTest {
s3Service.createBucket("test-bucket", "us-east-1");
}
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void expectedObjectSizeUsesSelectedPartsAndPreservesExistingObjectOnMismatch(boolean memory) {
+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ tempDir.resolve("size-check"), memory);
+ service.createBucket("sized-bucket", "us-east-1");
+ service.putObject("sized-bucket", "object", "prior".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ MultipartUpload upload = service.initiateMultipartUpload("sized-bucket", "object", null);
+ service.uploadPart("sized-bucket", "object", upload.getUploadId(), 1,
+ "data".getBytes(StandardCharsets.UTF_8));
+ service.uploadPart("sized-bucket", "object", upload.getUploadId(), 2,
+ "unused".getBytes(StandardCharsets.UTF_8));
+ AwsException error = assertThrows(AwsException.class, () -> service.completeMultipartUpload(
+ "sized-bucket", "object", upload.getUploadId(), List.of(1), Map.of(), Map.of(), null, null, 10L));
+ assertEquals("InvalidRequest", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("prior", new String(service.getObject("sized-bucket", "object").getData(),
+ StandardCharsets.UTF_8));
+ S3Object completed = service.completeMultipartUpload("sized-bucket", "object", upload.getUploadId(),
+ List.of(1), Map.of(), Map.of(), null, null, 4L);
+ assertEquals(4, completed.getSize());
+ assertEquals("data", new String(service.getObject("sized-bucket", "object").getData(),
+ StandardCharsets.UTF_8));
+ }
+
@Test
void initiateMultipartUpload() {
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "large-file.bin", "application/octet-stream");