Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): reject undersized multipart upload parts (#5718)
Langning Zhangea45f81a8117
diff --git a/compatibility-tests/sdk-test-awscli/test/s3.bats b/compatibility-tests/sdk-test-awscli/test/s3.batsindex 62cf0ecc4e..36237d5b83 100644--- a/compatibility-tests/sdk-test-awscli/test/s3.bats+++ b/compatibility-tests/sdk-test-awscli/test/s3.bats@@ -266,7 +266,7 @@ teardown() {local part1_file part2_filepart1_file=$(mktemp)part2_file=$(mktemp)- echo -n "part-one" > "$part1_file"+ dd if=/dev/zero of="$part1_file" bs=1048576 count=5 2>/dev/nullecho -n "part-two" > "$part2_file"output=$(aws_cmd s3api upload-part \diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartMinimumPartSizeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartMinimumPartSizeTest.javanew file mode 100644index 0000000000..dc79d0eeaf--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartMinimumPartSizeTest.java@@ -0,0 +1,66 @@+package com.floci.test;++import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.core.sync.RequestBody;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;+import software.amazon.awssdk.services.s3.model.ChecksumType;+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;+import software.amazon.awssdk.services.s3.model.CompletedPart;+import software.amazon.awssdk.services.s3.model.S3Exception;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++@DisplayName("S3 multipart minimum part size")+class S3MultipartMinimumPartSizeTest {++ @Test+ void undersizedPartPreservesTheObjectAndUploadForRetry() {+ String bucket = TestFixtures.uniqueName("sdk-multipart-minimum");+ String key = "object";+ try (S3Client s3 = TestFixtures.s3Client()) {+ s3.createBucket(request -> request.bucket(bucket));+ try {+ s3.putObject(request -> request.bucket(bucket).key(key), RequestBody.fromString("prior"));+ String uploadId = s3.createMultipartUpload(request -> request.bucket(bucket).key(key)+ .checksumAlgorithm(ChecksumAlgorithm.CRC32).checksumType(ChecksumType.FULL_OBJECT)).uploadId();+ String firstETag = s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId)+ .partNumber(1), RequestBody.fromString("data")).eTag();+ String lastETag = s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId)+ .partNumber(2), RequestBody.fromString("last")).eTag();+ assertThatThrownBy(() -> s3.completeMultipartUpload(request -> request.bucket(bucket).key(key)+ .uploadId(uploadId).multipartUpload(parts(firstETag, lastETag))))+ .isInstanceOfSatisfying(S3Exception.class, error -> {+ assertThat(error.statusCode()).isEqualTo(400);+ assertThat(error.awsErrorDetails().errorCode()).isEqualTo("EntityTooSmall");+ assertThat(error.awsErrorDetails().errorMessage())+ .isEqualTo("Your proposed upload is smaller than the minimum allowed size");+ });+ assertThat(s3.getObjectAsBytes(request -> request.bucket(bucket).key(key)).asUtf8String())+ .isEqualTo("prior");+ assertThat(s3.listParts(request -> request.bucket(bucket).key(key).uploadId(uploadId)).parts())+ .hasSize(2);++ String replacementETag = s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId)+ .partNumber(1), RequestBody.fromBytes(new byte[5242880])).eTag();+ s3.completeMultipartUpload(request -> request.bucket(bucket).key(key).uploadId(uploadId)+ .multipartUpload(parts(replacementETag, lastETag)));+ assertThat(s3.headObject(request -> request.bucket(bucket).key(key)).contentLength()).isEqualTo(5242884);+ } finally {+ s3.listMultipartUploads(request -> request.bucket(bucket)).uploads().forEach(upload ->+ s3.abortMultipartUpload(request -> request.bucket(bucket).key(upload.key())+ .uploadId(upload.uploadId())));+ s3.deleteObject(request -> request.bucket(bucket).key(key));+ s3.deleteBucket(request -> request.bucket(bucket));+ }+ }+ }++ private static CompletedMultipartUpload parts(String firstETag, String lastETag) {+ return CompletedMultipartUpload.builder().parts(+ CompletedPart.builder().partNumber(1).eTag(firstETag).build(),+ CompletedPart.builder().partNumber(2).eTag(lastETag).build()).build();+ }+}diff --git a/docs/services/s3.md b/docs/services/s3.mdindex cf5169ba45..fa633f3098 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -602,6 +602,11 @@ Floci now persists and returns the following object attribute state on S3 object- checksum metadata for object reads and `GetObjectAttributes`- multipart part manifests for `GetObjectAttributes(ObjectParts)`- `CompleteMultipartUpload` validates `x-amz-mp-object-size` against the total size of the selected parts. A mismatch returns `InvalidRequest` without replacing the object or discarding the upload, so the request can be retried with the correct size. Blank, malformed, overflowing, or negative sizes also return `InvalidRequest` and preserve the object and upload+- `CompleteMultipartUpload` requires every selected part except the last to contain at least+ 5 MiB (5,242,880 bytes). The last selected part, including a single-part upload, has no minimum+ size. A smaller non-final part returns `400 EntityTooSmall` with its part number, ETag and+ proposed size. The existing object and uploaded parts are preserved so completion can be retried+ after replacing the undersized part.- multipart checksums follow the AWS checksum type: `COMPOSITE` (`<checksum of the part checksums>-<part count>`) for SHA1, SHA256 and, by default, CRC32 and CRC32C; `FULL_OBJECT` for CRC64NVME or when `x-amz-checksum-type: FULL_OBJECT` was requested on `CreateMultipartUpload`. The `-N` suffix is returned by `CompleteMultipartUpload`, `HeadObject` and `GetObject`, and omitted by `GetObjectAttributes`, which lists part-level checksums only for `COMPOSITE` objects (a `FULL_OBJECT` multipart object reports its `PartsCount` alone, a single-part object has no `ObjectParts`), as on AWS. `UploadPart` echoes the part checksum (`x-amz-checksum-<algorithm>`) when the upload declared an algorithm, and completing a `COMPOSITE` upload requires the checksum of every part in the `CompleteMultipartUpload` body (`InvalidRequest` otherwise), while `FULL_OBJECT` uploads accept a body without them- canned object ACLs from `x-amz-acl` on `PutObject`, `CopyObject`, and multipart initiation- explicit object SSE headers from `x-amz-server-side-encryption` on `PutObject`, `CopyObject`, and multipart initiation, replayed on `GetObject` and `HeadObject`diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex baf68793aa..64c4983b5a 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -3554,12 +3554,13 @@ public class S3Controller {}// S3 InvalidArgument responses carry ArgumentName and ArgumentValue so the SDK can// surface which input was rejected, and EntityTooSmall and EntityTooLarge the proposed and- // allowed sizes. They travel through AwsException.extendedData.+ // allowed sizes. Multipart size errors also identify the rejected part.+ // They travel through AwsException.extendedData.if (e.getExtendedData() != null) {- for (String sizeField : List.of("ProposedSize", "MinSizeAllowed", "MaxSizeAllowed")) {- Object size = e.getExtendedData().get(sizeField);- if (size != null) {- xmlBuilder.elem(sizeField, size.toString());+ for (String field : List.of("ProposedSize", "MinSizeAllowed", "MaxSizeAllowed", "PartNumber", "ETag")) {+ Object value = e.getExtendedData().get(field);+ if (value != null) {+ xmlBuilder.elem(field, value.toString());}}Object resource = e.getExtendedData().get("Resource");diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 76cd8b7e32..98c2a9ed07 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -2310,6 +2310,7 @@ public class S3Service implements Resettable, ResourceProvider {// The most one part of a multipart upload can hold, per S3's multipart upload limits.static final long MAX_PART_SIZE = 5L * 1024 * 1024 * 1024;+ static final long MIN_PART_SIZE = 5L * 1024 * 1024;// The User Guide's "up to 5 GB" in one PutObject, read as 5 GiB like CopyObject's 5 GB limit;// a larger object is uploaded in parts.@@ -2345,6 +2346,13 @@ public class S3Service implements Resettable, ResourceProvider {Map.of("ProposedSize", proposedSize, "MinSizeAllowed", minSizeAllowed));}+ /** The same error for an undersized multipart part, identifying its number and ETag. */+ static AwsException entityTooSmall(long proposedSize, long minSizeAllowed, int partNumber, String eTag) {+ return new AwsException("EntityTooSmall", ENTITY_TOO_SMALL_MESSAGE, 400,+ Map.of("ProposedSize", proposedSize, "MinSizeAllowed", minSizeAllowed,+ "PartNumber", partNumber, "ETag", eTag));+ }+static void requireCopyableSize(S3Object source) {if (source.getSize() > MAX_COPY_OBJECT_SOURCE_SIZE) {throw new AwsException("InvalidRequest",@@ -4267,6 +4275,14 @@ public class S3Service implements Resettable, ResourceProvider {validateExpectedChecksum(checksum, expectedChecksum);}+ for (int index = 0; index < parts.size() - 1; index++) {+ Part part = parts.get(index);+ if (part.getSize() < MIN_PART_SIZE) {+ throw entityTooSmall(part.getSize(), MIN_PART_SIZE, part.getPartNumber(),+ stripSurroundingQuotes(part.getETag()));+ }+ }+ObjectBody body = inMemory? new BytesBody(concatenateParts(uploadId, parts)): assembleParts(uploadId, parts);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.javaindex 30d0e0129b..9bf58cf721 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java@@ -65,7 +65,7 @@ class S3CopyObjectDiskTest {@Testvoid copyOfAMultipartObjectGetsTheMd5OfItsBytesAsItsETag() {- byte[] part1 = "first part of the source, ".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "second part of the source".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = s3Service.initiateMultipartUpload("source-bucket", "multipart.bin", null);s3Service.uploadPart("source-bucket", "multipart.bin", upload.getUploadId(), 1, part1);@@ -98,7 +98,7 @@ class S3CopyObjectDiskTest {@Testvoid copyOfACompositeChecksumObjectGetsAFullObjectChecksumOfItsBytes() {- byte[] part1 = "composite part one ".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "composite part two".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = s3Service.initiateMultipartUpload("source-bucket", "composite.bin", null,null, null, null, null, null, null, null, null, "CRC32");diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.javaindex a7505e36f5..e2543de226 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java@@ -54,8 +54,8 @@ class S3MultipartDiskAssemblyTest {@Testvoid assembledObjectConcatenatesPartsInAscendingOrderRegardlessOfUploadOrder() {- byte[] part1 = repeatingBytes((byte) 'A', 17_000);- byte[] part2 = repeatingBytes((byte) 'B', 3);+ byte[] part1 = repeatingBytes((byte) 'A', 5 * 1024 * 1024);+ byte[] part2 = repeatingBytes((byte) 'B', 5 * 1024 * 1024);byte[] part3 = repeatingBytes((byte) 'C', 65_537);MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "assembled.bin", null);@@ -73,7 +73,7 @@ class S3MultipartDiskAssemblyTest {@Testvoid assembledObjectSkipsGapsWhenPartNumbersAreNonConsecutive() {- byte[] part1 = "part-one".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part5 = "part-five".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "gapped.bin", null);@@ -88,7 +88,7 @@ class S3MultipartDiskAssemblyTest {@Testvoid compositeETagIsTheMd5OfTheConcatenatedPartMd5s() throws NoSuchAlgorithmException {- byte[] part1 = repeatingBytes((byte) 'X', 9_001);+ byte[] part1 = repeatingBytes((byte) 'X', 5 * 1024 * 1024);byte[] part2 = repeatingBytes((byte) 'Y', 4_096);MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "etag.bin", null);@@ -111,7 +111,7 @@ class S3MultipartDiskAssemblyTest {@Testvoid defaultFullObjectChecksumCoversTheAssembledBytesInOrder() {- byte[] part1 = repeatingBytes((byte) 'M', 12_345);+ byte[] part1 = repeatingBytes((byte) 'M', 5 * 1024 * 1024);byte[] part2 = repeatingBytes((byte) 'N', 6_789);MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "checksum.bin", null);@@ -129,7 +129,7 @@ class S3MultipartDiskAssemblyTest {@Testvoid noObjectIsCreatedWhenAPartFileGoesMissingDuringAssembly() throws Exception {MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "vanished.bin", null);- s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 1, new byte[5 * 1024 * 1024]);s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));// Metadata (the Part record with its ETag) lives in memory, separate from the on-disk part@@ -149,9 +149,10 @@ class S3MultipartDiskAssemblyTest {@Testvoid failedStoreRemovesTheAssembledFileAndKeepsThePartsForARetry() throws Exception {+ byte[] firstPart = new byte[5 * 1024 * 1024];MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "public.bin", null,null, null, null, null, "public-read");- s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 1, firstPart);s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));// The public ACL is only rejected once the parts are assembled, when the object is stored.s3Service.putPublicAccessBlock("test-bucket",@@ -164,7 +165,7 @@ class S3MultipartDiskAssemblyTest {s3Service.putPublicAccessBlock("test-bucket","<PublicAccessBlockConfiguration><BlockPublicAcls>false</BlockPublicAcls></PublicAccessBlockConfiguration>");s3Service.completeMultipartUpload("test-bucket", "public.bin", upload.getUploadId(), List.of(1, 2), null, null);- assertArrayEquals("part1part2".getBytes(StandardCharsets.UTF_8),+ assertArrayEquals(concat(firstPart, "part2".getBytes(StandardCharsets.UTF_8)),s3Service.getObject("test-bucket", "public.bin").getData());assertFalse(Files.exists(partsDir(upload)), "a completed upload should leave no files behind");}@@ -172,7 +173,7 @@ class S3MultipartDiskAssemblyTest {@Testvoid versionedMultipartObjectIsStoredOnceAndKeepsItsBytesWhenTheKeyIsOverwritten() throws Exception {s3Service.putBucketVersioning("test-bucket", "Enabled");- byte[] part1 = repeatingBytes((byte) 'V', 9_000);+ byte[] part1 = repeatingBytes((byte) 'V', 5 * 1024 * 1024);byte[] part2 = repeatingBytes((byte) 'W', 1_000);MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "versioned.bin", null);s3Service.uploadPart("test-bucket", "versioned.bin", upload.getUploadId(), 1, part1);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.javaindex 276d94e741..d3a51a6a38 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java@@ -25,7 +25,7 @@ class S3MultipartIntegrationTest {private static final String SSE_CUSTOMER_KEY_MD5 = customerKeyMd5(SSE_CUSTOMER_KEY);private static final String WRONG_SSE_CUSTOMER_KEY = Base64.getEncoder().encodeToString("abcdef0123456789abcdef0123456789".getBytes(StandardCharsets.UTF_8));private static final String WRONG_SSE_CUSTOMER_KEY_MD5 = customerKeyMd5(WRONG_SSE_CUSTOMER_KEY);- private static final String PART_1 = "Part1Data-Hello";+ private static final String PART_1 = "a".repeat(5 * 1024 * 1024);private static final String PART_2 = "Part2Data-World";private static final String COMPOSITE_SHA256 = compositeSha256(PART_1, PART_2);private static String uploadId;@@ -258,9 +258,10 @@ class S3MultipartIntegrationTest {@Test@Order(13)void uploadPartCopy() {+ String source = "ABCDEFGHIJ" + "a".repeat(5 * 1024 * 1024);// Put a source objectgiven()- .body("ABCDEFGHIJ")+ .body(source).when().put("/" + BUCKET + "/source-for-copy.bin").then()@@ -285,10 +286,10 @@ class S3MultipartIntegrationTest {.body(containsString("<ETag>")).extract().xmlPath().getString("CopyPartResult.ETag");- // UploadPartCopy with range (bytes 2-5 → "CDEF")+ // Copy a 5 MiB range so this non-final part meets the multipart minimum.String copyPart2ETag = given().header("x-amz-copy-source", "/" + BUCKET + "/source-for-copy.bin")- .header("x-amz-copy-source-range", "bytes=2-5")+ .header("x-amz-copy-source-range", "bytes=2-" + (5 * 1024 * 1024 + 1)).when().put("/" + BUCKET + "/copy-dest.bin?uploadId=" + copyUploadId + "&partNumber=2").then()@@ -333,7 +334,7 @@ class S3MultipartIntegrationTest {.get("/" + BUCKET + "/copy-dest.bin").then().statusCode(200)- .body(equalTo("ABCDEFGHIJCDEFCDEF"));+ .body(equalTo(source + source.substring(2, 5 * 1024 * 1024 + 2) + "CDEF"));}@Test@@ -1220,7 +1221,7 @@ class S3MultipartIntegrationTest {.then().statusCode(200).extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");- String firstETag = given().body("first").when()+ String firstETag = given().body(new byte[5 * 1024 * 1024]).when().put("/" + BUCKET + "/" + key + "?uploadId=" + newUploadId + "&partNumber=1").then().statusCode(200).extract().header("ETag");String thirdETag = given().body("third").when()diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartMinimumPartSizeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartMinimumPartSizeIntegrationTest.javanew file mode 100644index 0000000000..9f04b1cacd--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartMinimumPartSizeIntegrationTest.java@@ -0,0 +1,61 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.Test;++import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;++@QuarkusTest+class S3MultipartMinimumPartSizeIntegrationTest {++ @Test+ void undersizedNonFinalPartReturnsItsDetailsAndAllowsRetry() {+ String bucket = "multipart-minimum-" + UUID.randomUUID();+ String path = "/" + bucket + "/object";+ given().put("/" + bucket).then().statusCode(200);+ String uploadId = null;+ try {+ given().body("prior").put(path).then().statusCode(200);+ uploadId = given().post(path + "?uploads").then().statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");+ String firstETag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 1).body("data")+ .put(path).then().statusCode(200).extract().header("ETag");+ String lastETag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 2).body("last")+ .put(path).then().statusCode(200).extract().header("ETag");+ given().queryParam("uploadId", uploadId).body(completion(firstETag, lastETag)).post(path)+ .then().statusCode(400)+ .body("Error.Code", equalTo("EntityTooSmall"))+ .body("Error.Message", equalTo("Your proposed upload is smaller than the minimum allowed size"))+ .body("Error.ProposedSize", equalTo("4"))+ .body("Error.MinSizeAllowed", equalTo("5242880"))+ .body("Error.PartNumber", equalTo("1"))+ .body("Error.ETag", equalTo(firstETag.substring(1, firstETag.length() - 1)));+ given().get(path).then().statusCode(200).body(equalTo("prior"));+ given().queryParam("uploadId", uploadId).get(path).then().statusCode(200)+ .body("ListPartsResult.Part.size()", equalTo(2));++ String replacementETag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 1)+ .body(new byte[5242880]).put(path).then().statusCode(200).extract().header("ETag");+ given().queryParam("uploadId", uploadId).body(completion(replacementETag, lastETag)).post(path)+ .then().statusCode(200);+ given().head(path).then().statusCode(200).header("Content-Length", equalTo("5242884"));+ } finally {+ if (uploadId != null) {+ given().queryParam("uploadId", uploadId).delete(path);+ }+ given().delete(path);+ given().delete("/" + bucket);+ }+ }++ private static String completion(String firstETag, String lastETag) {+ return """+ <CompleteMultipartUpload>+ <Part><PartNumber>1</PartNumber><ETag>%s</ETag></Part>+ <Part><PartNumber>2</PartNumber><ETag>%s</ETag></Part>+ </CompleteMultipartUpload>""".formatted(firstETag, lastETag);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.javaindex 416963e10f..3858f72095 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java@@ -42,6 +42,110 @@ class S3MultipartServiceTest {s3Service.createBucket("test-bucket", "us-east-1");}+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void undersizedNonFinalPartPreservesObjectAndUploadForRetry(boolean memory) {+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),+ tempDir.resolve("minimum-part-size"), memory);+ service.createBucket("sized-bucket", "us-east-1");+ service.putObject("sized-bucket", "object", "prior".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ MultipartUpload upload = service.initiateMultipartUpload("sized-bucket", "object", null);+ byte[] smallPart = "data".getBytes(StandardCharsets.UTF_8);+ String firstETag = service.uploadPart("sized-bucket", "object", upload.getUploadId(), 1, smallPart);+ String lastETag = service.uploadPart("sized-bucket", "object", upload.getUploadId(), 2, smallPart);++ AwsException error = assertThrows(AwsException.class, () -> service.completeMultipartUpload(+ "sized-bucket", "object", upload.getUploadId(), List.of(1, 2),+ Map.of(1, firstETag, 2, lastETag), Map.of(), null, null));+ assertEquals("EntityTooSmall", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("Your proposed upload is smaller than the minimum allowed size", error.getMessage());+ assertEquals(Map.of("ProposedSize", 4L, "MinSizeAllowed", 5242880L, "PartNumber", 1,+ "ETag", firstETag.substring(1, firstETag.length() - 1)), error.getExtendedData());+ assertEquals("prior", new String(service.getObject("sized-bucket", "object").getData(),+ StandardCharsets.UTF_8));+ assertEquals(2, service.getMultipartUpload("sized-bucket", "object", upload.getUploadId()).getParts().size());++ String replacementETag = service.uploadPart("sized-bucket", "object", upload.getUploadId(), 1,+ new byte[5242880]);+ S3Object completed = service.completeMultipartUpload("sized-bucket", "object", upload.getUploadId(),+ List.of(1, 2), Map.of(1, replacementETag, 2, lastETag), Map.of(), null, null);+ assertEquals(5242884, completed.getSize());+ }++ @ParameterizedTest+ @ValueSource(ints = {0, 5242879})+ void minimumPartSizeRejectsValuesBelowFiveMiB(int size) {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "boundary", null);+ s3Service.uploadPart("test-bucket", "boundary", upload.getUploadId(), 1, new byte[size]);+ s3Service.uploadPart("test-bucket", "boundary", upload.getUploadId(), 2, new byte[0]);++ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "boundary", upload.getUploadId(), List.of(1, 2), null, null));+ assertEquals("EntityTooSmall", error.getErrorCode());+ assertEquals((long) size, error.getExtendedData().get("ProposedSize"));+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void minimumPartSizeOnlyChecksSelectedNonFinalParts(boolean memory) {+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),+ tempDir.resolve("selected-parts"), memory);+ service.createBucket("selected-bucket", "us-east-1");+ MultipartUpload upload = service.initiateMultipartUpload("selected-bucket", "object", null);+ service.uploadPart("selected-bucket", "object", upload.getUploadId(), 1, new byte[1]);+ service.uploadPart("selected-bucket", "object", upload.getUploadId(), 2, new byte[5242880]);+ service.uploadPart("selected-bucket", "object", upload.getUploadId(), 7, new byte[0]);+ service.uploadPart("selected-bucket", "object", upload.getUploadId(), 9, new byte[1]);++ S3Object completed = service.completeMultipartUpload("selected-bucket", "object", upload.getUploadId(),+ List.of(2, 7), null, null);+ assertEquals(5242880, completed.getSize());+ assertEquals(List.of(2, 7), completed.getParts().stream().map(Part::getPartNumber).toList());+ }++ @Test+ void minimumPartSizeReportsTheFirstUndersizedNonFinalPart() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "first-small", null);+ s3Service.uploadPart("test-bucket", "first-small", upload.getUploadId(), 1, new byte[5242880]);+ s3Service.uploadPart("test-bucket", "first-small", upload.getUploadId(), 3, new byte[2]);+ s3Service.uploadPart("test-bucket", "first-small", upload.getUploadId(), 7, new byte[1]);+ s3Service.uploadPart("test-bucket", "first-small", upload.getUploadId(), 9, new byte[0]);++ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "first-small", upload.getUploadId(), List.of(1, 3, 7, 9), null, null));+ assertEquals("EntityTooSmall", error.getErrorCode());+ assertEquals(3, error.getExtendedData().get("PartNumber"));+ assertEquals(2L, error.getExtendedData().get("ProposedSize"));+ }++ @ParameterizedTest+ @ValueSource(ints = {0, 4})+ void minimumPartSizeAllowsASingleSmallSelectedPart(int size) {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "single", null);+ s3Service.uploadPart("test-bucket", "single", upload.getUploadId(), 1, new byte[size]);+ s3Service.uploadPart("test-bucket", "single", upload.getUploadId(), 2, new byte[1]);++ S3Object completed = s3Service.completeMultipartUpload("test-bucket", "single", upload.getUploadId(),+ List.of(1), null, null);+ assertEquals(size, completed.getSize());+ }++ @Test+ void minimumPartSizePreservesManifestValidationBeforeSizeValidation() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "validation", null);+ String firstETag = s3Service.uploadPart("test-bucket", "validation", upload.getUploadId(), 1, new byte[1]);+ s3Service.uploadPart("test-bucket", "validation", upload.getUploadId(), 2, new byte[1]);++ AwsException badETag = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "validation", upload.getUploadId(), List.of(1, 2),+ Map.of(1, firstETag, 2, "wrong"), Map.of(), null, null));+ assertEquals("InvalidPart", badETag.getErrorCode());+ AwsException badOrder = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "validation", upload.getUploadId(), List.of(2, 1), null, null));+ assertEquals("InvalidPartOrder", badOrder.getErrorCode());+ }+@ParameterizedTest@ValueSource(booleans = {false, true})void expectedObjectSizeUsesSelectedPartsAndPreservesExistingObjectOnMismatch(boolean memory) {@@ -116,7 +220,9 @@ class S3MultipartServiceTest {void completeMultipartUpload() {MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "file.bin", "text/plain",Map.of("owner", "team-a"), "STANDARD_IA");- s3Service.uploadPart("test-bucket", "file.bin", upload.getUploadId(), 1, "part1".getBytes());+ byte[] firstPart = new byte[5 * 1024 * 1024];+ byte[] whole = concat(firstPart, "part2".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "file.bin", upload.getUploadId(), 1, firstPart);s3Service.uploadPart("test-bucket", "file.bin", upload.getUploadId(), 2, "part2".getBytes());S3Object result = s3Service.completeMultipartUpload("test-bucket", "file.bin",@@ -129,12 +235,12 @@ class S3MultipartServiceTest {assertEquals(2, result.getParts().size());// Verify the data is concatenatedS3Object fetched = s3Service.getObject("test-bucket", "file.bin");- assertEquals("part1part2", new String(fetched.getData()));+ assertArrayEquals(whole, fetched.getData());// Composite ETag should end with -2 (number of parts)assertTrue(result.getETag().endsWith("-2\""), "ETag should be composite: " + result.getETag());// Without a checksum algorithm AWS attaches the default CRC64NVME full-object checksumassertEquals(ChecksumType.FULL_OBJECT, result.getChecksum().getChecksumType());- assertEquals(S3Checksum.crc64NvmeBase64("part1part2".getBytes(StandardCharsets.UTF_8)),+ assertEquals(S3Checksum.crc64NvmeBase64(whole),result.getChecksum().getChecksumCRC64NVME());}@@ -233,15 +339,16 @@ class S3MultipartServiceTest {@Testvoid completeMultipartUploadAcceptsNonConsecutivePartNumbers() {MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "gapped.bin", null);+ byte[] firstPart = new byte[5 * 1024 * 1024];String first = s3Service.uploadPart("test-bucket", "gapped.bin", upload.getUploadId(), 1,- "part1".getBytes(StandardCharsets.UTF_8));+ firstPart);String third = s3Service.uploadPart("test-bucket", "gapped.bin", upload.getUploadId(), 3,"part3".getBytes(StandardCharsets.UTF_8));S3Object result = s3Service.completeMultipartUpload("test-bucket", "gapped.bin", upload.getUploadId(),List.of(1, 3), Map.of(1, first, 3, third), Map.of(), null, null);- assertEquals("part1part3", new String(s3Service.getObject("test-bucket", "gapped.bin").getData(),- StandardCharsets.UTF_8));+ assertArrayEquals(concat(firstPart, "part3".getBytes(StandardCharsets.UTF_8)),+ s3Service.getObject("test-bucket", "gapped.bin").getData());}@Test@@ -299,7 +406,7 @@ class S3MultipartServiceTest {service.putBucketVersioning("versioned-bucket", "Enabled");MultipartUpload upload = service.initiateMultipartUpload("versioned-bucket", "file.bin", null);service.uploadPart("versioned-bucket", "file.bin", upload.getUploadId(), 1,- "part1".getBytes(StandardCharsets.UTF_8));+ new byte[5 * 1024 * 1024]);service.uploadPart("versioned-bucket", "file.bin", upload.getUploadId(), 2,"part2".getBytes(StandardCharsets.UTF_8));@@ -313,7 +420,7 @@ class S3MultipartServiceTest {@Testvoid completeMultipartUploadReturnsTheMd5OfThePartMd5s() throws NoSuchAlgorithmException {MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "composite-etag.bin", null);- byte[] part1 = "part1".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "part2".getBytes(StandardCharsets.UTF_8);s3Service.uploadPart("test-bucket", "composite-etag.bin", upload.getUploadId(), 1, part1);s3Service.uploadPart("test-bucket", "composite-etag.bin", upload.getUploadId(), 2, part2);@@ -340,11 +447,12 @@ class S3MultipartServiceTest {@Testvoid getObjectAttributesReturnsMultipartParts() {+ byte[] part1 = new byte[5 * 1024 * 1024];MultipartUpload upload = initiate("parts.bin", "SHA256", null);- s3Service.uploadPart("test-bucket", "parts.bin", upload.getUploadId(), 1, "abc".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "parts.bin", upload.getUploadId(), 1, part1);s3Service.uploadPart("test-bucket", "parts.bin", upload.getUploadId(), 2, "def".getBytes(StandardCharsets.UTF_8));s3Service.completeMultipartUpload("test-bucket", "parts.bin", upload.getUploadId(), List.of(1, 2),- partChecksums(ChecksumAlgorithm.SHA256, "abc".getBytes(StandardCharsets.UTF_8), "def".getBytes(StandardCharsets.UTF_8)), null, null);+ partChecksums(ChecksumAlgorithm.SHA256, part1, "def".getBytes(StandardCharsets.UTF_8)), null, null);GetObjectAttributesResult attributes = s3Service.getObjectAttributes("test-bucket", "parts.bin", null,Set.of(ObjectAttributeName.OBJECT_PARTS, ObjectAttributeName.CHECKSUM),@@ -358,14 +466,15 @@ class S3MultipartServiceTest {assertEquals(1, attributes.getObjectParts().getParts().size());assertTrue(attributes.getObjectParts().isTruncated());assertEquals(1, attributes.getObjectParts().getNextPartNumberMarker());- assertEquals(S3Checksum.sha256Base64("abc".getBytes(StandardCharsets.UTF_8)),+ assertEquals(S3Checksum.sha256Base64(part1),attributes.getObjectParts().getParts().get(0).getChecksum().getChecksumSHA256());}@Testvoid getObjectAttributesReportsOnlyThePartCountForFullObjectChecksums() {+ byte[] part1 = new byte[5 * 1024 * 1024];MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "full-parts.bin", "application/octet-stream");- s3Service.uploadPart("test-bucket", "full-parts.bin", upload.getUploadId(), 1, "abc".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "full-parts.bin", upload.getUploadId(), 1, part1);s3Service.uploadPart("test-bucket", "full-parts.bin", upload.getUploadId(), 2, "def".getBytes(StandardCharsets.UTF_8));s3Service.completeMultipartUpload("test-bucket", "full-parts.bin", upload.getUploadId(), List.of(1, 2), null, null);@@ -391,7 +500,7 @@ class S3MultipartServiceTest {@Testvoid completeMultipartUploadReturnsCompositeSha256Checksum() {- byte[] part1 = "Part1Data-Hello".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "Part2Data-World".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = initiate("sha256.bin", "SHA256", null);s3Service.uploadPart("test-bucket", "sha256.bin", upload.getUploadId(), 1, part1);@@ -432,7 +541,7 @@ class S3MultipartServiceTest {@Testvoid completeMultipartUploadReturnsCompositeCrc32Checksum() {- byte[] part1 = "crc-part-1".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "crc-part-2".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = initiate("crc32.bin", "CRC32", null);s3Service.uploadPart("test-bucket", "crc32.bin", upload.getUploadId(), 1, part1);@@ -449,7 +558,7 @@ class S3MultipartServiceTest {@Testvoid completeMultipartUploadKeepsCrc64NvmeAsFullObjectChecksum() {- byte[] part1 = "crc64-part-1".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "crc64-part-2".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = initiate("crc64.bin", "CRC64NVME", null);assertEquals(ChecksumType.FULL_OBJECT, upload.getChecksumType());@@ -465,7 +574,7 @@ class S3MultipartServiceTest {@Testvoid completeMultipartUploadHonorsFullObjectTypeRequestedAtInitiation() {- byte[] part1 = "full-part-1".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "full-part-2".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = initiate("crc32-full.bin", "CRC32", "FULL_OBJECT");s3Service.uploadPart("test-bucket", "crc32-full.bin", upload.getUploadId(), 1, part1);@@ -498,7 +607,7 @@ class S3MultipartServiceTest {@Testvoid completeMultipartUploadValidatesClientCompositeChecksum() {- byte[] part1 = "Part1Data-Hello".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "Part2Data-World".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = initiate("validated.bin", "SHA256", null);s3Service.uploadPart("test-bucket", "validated.bin", upload.getUploadId(), 1, part1);@@ -559,7 +668,7 @@ class S3MultipartServiceTest {@Testvoid completeMultipartUploadRequiresEveryPartChecksumOnCompositeUploads() {- byte[] part1 = "Part1Data-Hello".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "Part2Data-World".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = initiate("strict.bin", "SHA256", null);s3Service.uploadPart("test-bucket", "strict.bin", upload.getUploadId(), 1, part1);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex 35d76bedde..f346d7e240 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -654,7 +654,7 @@ class S3ServiceTest {@Testvoid copyOfMultipartObjectGetsTheEtagOfTheWholeContent() throws NoSuchAlgorithmException {s3Service.createBucket("test-bucket", "us-east-1");- byte[] part1 = "Part1Data-Hello".getBytes(StandardCharsets.UTF_8);+ byte[] part1 = new byte[5 * 1024 * 1024];byte[] part2 = "Part2Data-World".getBytes(StandardCharsets.UTF_8);var upload = s3Service.initiateMultipartUpload("test-bucket", "multipart.bin", "application/octet-stream");s3Service.uploadPart("test-bucket", "multipart.bin", upload.getUploadId(), 1, part1);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.javaindex a051c161eb..1a6f2f132e 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java@@ -167,22 +167,23 @@ class S3StreamedUploadTest {@Testvoid aStreamedPartStoresWhatTheBytePathStores() {+ byte[] part1 = Arrays.copyOf(BODY, 5 * 1024 * 1024);byte[] part2 = "and a second part".getBytes(StandardCharsets.UTF_8);MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);- Part fromBytes = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, BODY, null, null, null);+ Part fromBytes = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, part1, null, null, null);Part streamed = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 2,new ByteArrayInputStream(part2), NO_CHECKSUMS, null, null, null);Part streamedAgain = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1,- new ByteArrayInputStream(BODY), NO_CHECKSUMS, null, null, null);+ new ByteArrayInputStream(part1), NO_CHECKSUMS, null, null, null);assertEquals(fromBytes.getETag(), streamedAgain.getETag());assertEquals(fromBytes.getChecksum().getChecksumCRC64NVME(), streamedAgain.getChecksum().getChecksumCRC64NVME());assertEquals(part2.length, streamed.getSize());s3Service.completeMultipartUpload("bucket", "multipart.bin", upload.getUploadId(), List.of(1, 2), null, null);- byte[] whole = new byte[BODY.length + part2.length];- System.arraycopy(BODY, 0, whole, 0, BODY.length);- System.arraycopy(part2, 0, whole, BODY.length, part2.length);+ byte[] whole = new byte[part1.length + part2.length];+ System.arraycopy(part1, 0, whole, 0, part1.length);+ System.arraycopy(part2, 0, whole, part1.length, part2.length);assertArrayEquals(whole, s3Service.getObject("bucket", "multipart.bin").getData());assertNothingStaged();}@@ -246,17 +247,17 @@ class S3StreamedUploadTest {@Testvoid anUploadPartCopyStreamsItsRangeIntoThePart() {- byte[] source = new byte[300_000];+ byte[] source = new byte[5 * 1024 * 1024 + 1000];new Random(5034).nextBytes(source);s3Service.putObject("bucket", "source.bin", source, "application/octet-stream", Map.of());MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "copy.bin", null);String rangeETag = s3Service.uploadPartCopy("bucket", "copy.bin", upload.getUploadId(), 1,- "bucket", "source.bin", null, "bytes=1000-200999");+ "bucket", "source.bin", null, "bytes=1000-" + (source.length - 1));String wholeETag = s3Service.uploadPartCopy("bucket", "copy.bin", upload.getUploadId(), 2,"bucket", "source.bin", null, null);- byte[] range = Arrays.copyOfRange(source, 1000, 201_000);+ byte[] range = Arrays.copyOfRange(source, 1000, source.length);assertEquals(S3Object.computeETag(range), rangeETag);assertEquals(S3Object.computeETag(source), wholeETag);s3Service.completeMultipartUpload("bucket", "copy.bin", upload.getUploadId(), List.of(1, 2), null, null);