Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): reject undersized multipart upload parts (#5718)

Langning Zhangea45f81a8117
diff --git a/compatibility-tests/sdk-test-awscli/test/s3.bats b/compatibility-tests/sdk-test-awscli/test/s3.bats
index 62cf0ecc4e..36237d5b83 100644
--- a/compatibility-tests/sdk-test-awscli/test/s3.bats
+++ b/compatibility-tests/sdk-test-awscli/test/s3.bats
@@ -266,7 +266,7 @@ teardown() {
local part1_file part2_file
part1_file=$(mktemp)
part2_file=$(mktemp)
- echo -n "part-one" > "$part1_file"
+ dd if=/dev/zero of="$part1_file" bs=1048576 count=5 2>/dev/null
echo -n "part-two" > "$part2_file"
output=$(aws_cmd s3api upload-part \
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartMinimumPartSizeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartMinimumPartSizeTest.java
new file mode 100644
index 0000000000..dc79d0eeaf
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3MultipartMinimumPartSizeTest.java
@@ -0,0 +1,66 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
+import software.amazon.awssdk.services.s3.model.ChecksumType;
+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@DisplayName("S3 multipart minimum part size")
+class S3MultipartMinimumPartSizeTest {
+
+ @Test
+ void undersizedPartPreservesTheObjectAndUploadForRetry() {
+ String bucket = TestFixtures.uniqueName("sdk-multipart-minimum");
+ String key = "object";
+ try (S3Client s3 = TestFixtures.s3Client()) {
+ s3.createBucket(request -> request.bucket(bucket));
+ try {
+ s3.putObject(request -> request.bucket(bucket).key(key), RequestBody.fromString("prior"));
+ String uploadId = s3.createMultipartUpload(request -> request.bucket(bucket).key(key)
+ .checksumAlgorithm(ChecksumAlgorithm.CRC32).checksumType(ChecksumType.FULL_OBJECT)).uploadId();
+ String firstETag = s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId)
+ .partNumber(1), RequestBody.fromString("data")).eTag();
+ String lastETag = s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId)
+ .partNumber(2), RequestBody.fromString("last")).eTag();
+ assertThatThrownBy(() -> s3.completeMultipartUpload(request -> request.bucket(bucket).key(key)
+ .uploadId(uploadId).multipartUpload(parts(firstETag, lastETag))))
+ .isInstanceOfSatisfying(S3Exception.class, error -> {
+ assertThat(error.statusCode()).isEqualTo(400);
+ assertThat(error.awsErrorDetails().errorCode()).isEqualTo("EntityTooSmall");
+ assertThat(error.awsErrorDetails().errorMessage())
+ .isEqualTo("Your proposed upload is smaller than the minimum allowed size");
+ });
+ assertThat(s3.getObjectAsBytes(request -> request.bucket(bucket).key(key)).asUtf8String())
+ .isEqualTo("prior");
+ assertThat(s3.listParts(request -> request.bucket(bucket).key(key).uploadId(uploadId)).parts())
+ .hasSize(2);
+
+ String replacementETag = s3.uploadPart(request -> request.bucket(bucket).key(key).uploadId(uploadId)
+ .partNumber(1), RequestBody.fromBytes(new byte[5242880])).eTag();
+ s3.completeMultipartUpload(request -> request.bucket(bucket).key(key).uploadId(uploadId)
+ .multipartUpload(parts(replacementETag, lastETag)));
+ assertThat(s3.headObject(request -> request.bucket(bucket).key(key)).contentLength()).isEqualTo(5242884);
+ } finally {
+ s3.listMultipartUploads(request -> request.bucket(bucket)).uploads().forEach(upload ->
+ s3.abortMultipartUpload(request -> request.bucket(bucket).key(upload.key())
+ .uploadId(upload.uploadId())));
+ s3.deleteObject(request -> request.bucket(bucket).key(key));
+ s3.deleteBucket(request -> request.bucket(bucket));
+ }
+ }
+ }
+
+ private static CompletedMultipartUpload parts(String firstETag, String lastETag) {
+ return CompletedMultipartUpload.builder().parts(
+ CompletedPart.builder().partNumber(1).eTag(firstETag).build(),
+ CompletedPart.builder().partNumber(2).eTag(lastETag).build()).build();
+ }
+}
diff --git a/docs/services/s3.md b/docs/services/s3.md
index cf5169ba45..fa633f3098 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -602,6 +602,11 @@ Floci now persists and returns the following object attribute state on S3 object
- checksum metadata for object reads and `GetObjectAttributes`
- multipart part manifests for `GetObjectAttributes(ObjectParts)`
- `CompleteMultipartUpload` validates `x-amz-mp-object-size` against the total size of the selected parts. A mismatch returns `InvalidRequest` without replacing the object or discarding the upload, so the request can be retried with the correct size. Blank, malformed, overflowing, or negative sizes also return `InvalidRequest` and preserve the object and upload
+- `CompleteMultipartUpload` requires every selected part except the last to contain at least
+ 5 MiB (5,242,880 bytes). The last selected part, including a single-part upload, has no minimum
+ size. A smaller non-final part returns `400 EntityTooSmall` with its part number, ETag and
+ proposed size. The existing object and uploaded parts are preserved so completion can be retried
+ after replacing the undersized part.
- multipart checksums follow the AWS checksum type: `COMPOSITE` (`<checksum of the part checksums>-<part count>`) for SHA1, SHA256 and, by default, CRC32 and CRC32C; `FULL_OBJECT` for CRC64NVME or when `x-amz-checksum-type: FULL_OBJECT` was requested on `CreateMultipartUpload`. The `-N` suffix is returned by `CompleteMultipartUpload`, `HeadObject` and `GetObject`, and omitted by `GetObjectAttributes`, which lists part-level checksums only for `COMPOSITE` objects (a `FULL_OBJECT` multipart object reports its `PartsCount` alone, a single-part object has no `ObjectParts`), as on AWS. `UploadPart` echoes the part checksum (`x-amz-checksum-<algorithm>`) when the upload declared an algorithm, and completing a `COMPOSITE` upload requires the checksum of every part in the `CompleteMultipartUpload` body (`InvalidRequest` otherwise), while `FULL_OBJECT` uploads accept a body without them
- canned object ACLs from `x-amz-acl` on `PutObject`, `CopyObject`, and multipart initiation
- explicit object SSE headers from `x-amz-server-side-encryption` on `PutObject`, `CopyObject`, and multipart initiation, replayed on `GetObject` and `HeadObject`
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index baf68793aa..64c4983b5a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -3554,12 +3554,13 @@ public class S3Controller {
}
// S3 InvalidArgument responses carry ArgumentName and ArgumentValue so the SDK can
// surface which input was rejected, and EntityTooSmall and EntityTooLarge the proposed and
- // allowed sizes. They travel through AwsException.extendedData.
+ // allowed sizes. Multipart size errors also identify the rejected part.
+ // They travel through AwsException.extendedData.
if (e.getExtendedData() != null) {
- for (String sizeField : List.of("ProposedSize", "MinSizeAllowed", "MaxSizeAllowed")) {
- Object size = e.getExtendedData().get(sizeField);
- if (size != null) {
- xmlBuilder.elem(sizeField, size.toString());
+ for (String field : List.of("ProposedSize", "MinSizeAllowed", "MaxSizeAllowed", "PartNumber", "ETag")) {
+ Object value = e.getExtendedData().get(field);
+ if (value != null) {
+ xmlBuilder.elem(field, value.toString());
}
}
Object resource = e.getExtendedData().get("Resource");
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 76cd8b7e32..98c2a9ed07 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -2310,6 +2310,7 @@ public class S3Service implements Resettable, ResourceProvider {
// The most one part of a multipart upload can hold, per S3's multipart upload limits.
static final long MAX_PART_SIZE = 5L * 1024 * 1024 * 1024;
+ static final long MIN_PART_SIZE = 5L * 1024 * 1024;
// The User Guide's "up to 5 GB" in one PutObject, read as 5 GiB like CopyObject's 5 GB limit;
// a larger object is uploaded in parts.
@@ -2345,6 +2346,13 @@ public class S3Service implements Resettable, ResourceProvider {
Map.of("ProposedSize", proposedSize, "MinSizeAllowed", minSizeAllowed));
}
+ /** The same error for an undersized multipart part, identifying its number and ETag. */
+ static AwsException entityTooSmall(long proposedSize, long minSizeAllowed, int partNumber, String eTag) {
+ return new AwsException("EntityTooSmall", ENTITY_TOO_SMALL_MESSAGE, 400,
+ Map.of("ProposedSize", proposedSize, "MinSizeAllowed", minSizeAllowed,
+ "PartNumber", partNumber, "ETag", eTag));
+ }
+
static void requireCopyableSize(S3Object source) {
if (source.getSize() > MAX_COPY_OBJECT_SOURCE_SIZE) {
throw new AwsException("InvalidRequest",
@@ -4267,6 +4275,14 @@ public class S3Service implements Resettable, ResourceProvider {
validateExpectedChecksum(checksum, expectedChecksum);
}
+ for (int index = 0; index < parts.size() - 1; index++) {
+ Part part = parts.get(index);
+ if (part.getSize() < MIN_PART_SIZE) {
+ throw entityTooSmall(part.getSize(), MIN_PART_SIZE, part.getPartNumber(),
+ stripSurroundingQuotes(part.getETag()));
+ }
+ }
+
ObjectBody body = inMemory
? new BytesBody(concatenateParts(uploadId, parts))
: assembleParts(uploadId, parts);
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java
index 30d0e0129b..9bf58cf721 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java
@@ -65,7 +65,7 @@ class S3CopyObjectDiskTest {
@Test
void copyOfAMultipartObjectGetsTheMd5OfItsBytesAsItsETag() {
- byte[] part1 = "first part of the source, ".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "second part of the source".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = s3Service.initiateMultipartUpload("source-bucket", "multipart.bin", null);
s3Service.uploadPart("source-bucket", "multipart.bin", upload.getUploadId(), 1, part1);
@@ -98,7 +98,7 @@ class S3CopyObjectDiskTest {
@Test
void copyOfACompositeChecksumObjectGetsAFullObjectChecksumOfItsBytes() {
- byte[] part1 = "composite part one ".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "composite part two".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = s3Service.initiateMultipartUpload("source-bucket", "composite.bin", null,
null, null, null, null, null, null, null, null, "CRC32");
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
index a7505e36f5..e2543de226 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
@@ -54,8 +54,8 @@ class S3MultipartDiskAssemblyTest {
@Test
void assembledObjectConcatenatesPartsInAscendingOrderRegardlessOfUploadOrder() {
- byte[] part1 = repeatingBytes((byte) 'A', 17_000);
- byte[] part2 = repeatingBytes((byte) 'B', 3);
+ byte[] part1 = repeatingBytes((byte) 'A', 5 * 1024 * 1024);
+ byte[] part2 = repeatingBytes((byte) 'B', 5 * 1024 * 1024);
byte[] part3 = repeatingBytes((byte) 'C', 65_537);
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "assembled.bin", null);
@@ -73,7 +73,7 @@ class S3MultipartDiskAssemblyTest {
@Test
void assembledObjectSkipsGapsWhenPartNumbersAreNonConsecutive() {
- byte[] part1 = "part-one".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part5 = "part-five".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "gapped.bin", null);
@@ -88,7 +88,7 @@ class S3MultipartDiskAssemblyTest {
@Test
void compositeETagIsTheMd5OfTheConcatenatedPartMd5s() throws NoSuchAlgorithmException {
- byte[] part1 = repeatingBytes((byte) 'X', 9_001);
+ byte[] part1 = repeatingBytes((byte) 'X', 5 * 1024 * 1024);
byte[] part2 = repeatingBytes((byte) 'Y', 4_096);
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "etag.bin", null);
@@ -111,7 +111,7 @@ class S3MultipartDiskAssemblyTest {
@Test
void defaultFullObjectChecksumCoversTheAssembledBytesInOrder() {
- byte[] part1 = repeatingBytes((byte) 'M', 12_345);
+ byte[] part1 = repeatingBytes((byte) 'M', 5 * 1024 * 1024);
byte[] part2 = repeatingBytes((byte) 'N', 6_789);
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "checksum.bin", null);
@@ -129,7 +129,7 @@ class S3MultipartDiskAssemblyTest {
@Test
void noObjectIsCreatedWhenAPartFileGoesMissingDuringAssembly() throws Exception {
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "vanished.bin", null);
- s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 1, new byte[5 * 1024 * 1024]);
s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));
// Metadata (the Part record with its ETag) lives in memory, separate from the on-disk part
@@ -149,9 +149,10 @@ class S3MultipartDiskAssemblyTest {
@Test
void failedStoreRemovesTheAssembledFileAndKeepsThePartsForARetry() throws Exception {
+ byte[] firstPart = new byte[5 * 1024 * 1024];
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "public.bin", null,
null, null, null, null, "public-read");
- s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 1, firstPart);
s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));
// The public ACL is only rejected once the parts are assembled, when the object is stored.
s3Service.putPublicAccessBlock("test-bucket",
@@ -164,7 +165,7 @@ class S3MultipartDiskAssemblyTest {
s3Service.putPublicAccessBlock("test-bucket",
"<PublicAccessBlockConfiguration><BlockPublicAcls>false</BlockPublicAcls></PublicAccessBlockConfiguration>");
s3Service.completeMultipartUpload("test-bucket", "public.bin", upload.getUploadId(), List.of(1, 2), null, null);
- assertArrayEquals("part1part2".getBytes(StandardCharsets.UTF_8),
+ assertArrayEquals(concat(firstPart, "part2".getBytes(StandardCharsets.UTF_8)),
s3Service.getObject("test-bucket", "public.bin").getData());
assertFalse(Files.exists(partsDir(upload)), "a completed upload should leave no files behind");
}
@@ -172,7 +173,7 @@ class S3MultipartDiskAssemblyTest {
@Test
void versionedMultipartObjectIsStoredOnceAndKeepsItsBytesWhenTheKeyIsOverwritten() throws Exception {
s3Service.putBucketVersioning("test-bucket", "Enabled");
- byte[] part1 = repeatingBytes((byte) 'V', 9_000);
+ byte[] part1 = repeatingBytes((byte) 'V', 5 * 1024 * 1024);
byte[] part2 = repeatingBytes((byte) 'W', 1_000);
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "versioned.bin", null);
s3Service.uploadPart("test-bucket", "versioned.bin", upload.getUploadId(), 1, part1);
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
index 276d94e741..d3a51a6a38 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
@@ -25,7 +25,7 @@ class S3MultipartIntegrationTest {
private static final String SSE_CUSTOMER_KEY_MD5 = customerKeyMd5(SSE_CUSTOMER_KEY);
private static final String WRONG_SSE_CUSTOMER_KEY = Base64.getEncoder().encodeToString("abcdef0123456789abcdef0123456789".getBytes(StandardCharsets.UTF_8));
private static final String WRONG_SSE_CUSTOMER_KEY_MD5 = customerKeyMd5(WRONG_SSE_CUSTOMER_KEY);
- private static final String PART_1 = "Part1Data-Hello";
+ private static final String PART_1 = "a".repeat(5 * 1024 * 1024);
private static final String PART_2 = "Part2Data-World";
private static final String COMPOSITE_SHA256 = compositeSha256(PART_1, PART_2);
private static String uploadId;
@@ -258,9 +258,10 @@ class S3MultipartIntegrationTest {
@Test
@Order(13)
void uploadPartCopy() {
+ String source = "ABCDEFGHIJ" + "a".repeat(5 * 1024 * 1024);
// Put a source object
given()
- .body("ABCDEFGHIJ")
+ .body(source)
.when()
.put("/" + BUCKET + "/source-for-copy.bin")
.then()
@@ -285,10 +286,10 @@ class S3MultipartIntegrationTest {
.body(containsString("<ETag>"))
.extract().xmlPath().getString("CopyPartResult.ETag");
- // UploadPartCopy with range (bytes 2-5 → "CDEF")
+ // Copy a 5 MiB range so this non-final part meets the multipart minimum.
String copyPart2ETag = given()
.header("x-amz-copy-source", "/" + BUCKET + "/source-for-copy.bin")
- .header("x-amz-copy-source-range", "bytes=2-5")
+ .header("x-amz-copy-source-range", "bytes=2-" + (5 * 1024 * 1024 + 1))
.when()
.put("/" + BUCKET + "/copy-dest.bin?uploadId=" + copyUploadId + "&partNumber=2")
.then()
@@ -333,7 +334,7 @@ class S3MultipartIntegrationTest {
.get("/" + BUCKET + "/copy-dest.bin")
.then()
.statusCode(200)
- .body(equalTo("ABCDEFGHIJCDEFCDEF"));
+ .body(equalTo(source + source.substring(2, 5 * 1024 * 1024 + 2) + "CDEF"));
}
@Test
@@ -1220,7 +1221,7 @@ class S3MultipartIntegrationTest {
.then().statusCode(200)
.extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
- String firstETag = given().body("first").when()
+ String firstETag = given().body(new byte[5 * 1024 * 1024]).when()
.put("/" + BUCKET + "/" + key + "?uploadId=" + newUploadId + "&partNumber=1")
.then().statusCode(200).extract().header("ETag");
String thirdETag = given().body("third").when()
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartMinimumPartSizeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartMinimumPartSizeIntegrationTest.java
new file mode 100644
index 0000000000..9f04b1cacd
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartMinimumPartSizeIntegrationTest.java
@@ -0,0 +1,61 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+
+@QuarkusTest
+class S3MultipartMinimumPartSizeIntegrationTest {
+
+ @Test
+ void undersizedNonFinalPartReturnsItsDetailsAndAllowsRetry() {
+ String bucket = "multipart-minimum-" + UUID.randomUUID();
+ String path = "/" + bucket + "/object";
+ given().put("/" + bucket).then().statusCode(200);
+ String uploadId = null;
+ try {
+ given().body("prior").put(path).then().statusCode(200);
+ uploadId = given().post(path + "?uploads").then().statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+ String firstETag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 1).body("data")
+ .put(path).then().statusCode(200).extract().header("ETag");
+ String lastETag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 2).body("last")
+ .put(path).then().statusCode(200).extract().header("ETag");
+ given().queryParam("uploadId", uploadId).body(completion(firstETag, lastETag)).post(path)
+ .then().statusCode(400)
+ .body("Error.Code", equalTo("EntityTooSmall"))
+ .body("Error.Message", equalTo("Your proposed upload is smaller than the minimum allowed size"))
+ .body("Error.ProposedSize", equalTo("4"))
+ .body("Error.MinSizeAllowed", equalTo("5242880"))
+ .body("Error.PartNumber", equalTo("1"))
+ .body("Error.ETag", equalTo(firstETag.substring(1, firstETag.length() - 1)));
+ given().get(path).then().statusCode(200).body(equalTo("prior"));
+ given().queryParam("uploadId", uploadId).get(path).then().statusCode(200)
+ .body("ListPartsResult.Part.size()", equalTo(2));
+
+ String replacementETag = given().queryParam("uploadId", uploadId).queryParam("partNumber", 1)
+ .body(new byte[5242880]).put(path).then().statusCode(200).extract().header("ETag");
+ given().queryParam("uploadId", uploadId).body(completion(replacementETag, lastETag)).post(path)
+ .then().statusCode(200);
+ given().head(path).then().statusCode(200).header("Content-Length", equalTo("5242884"));
+ } finally {
+ if (uploadId != null) {
+ given().queryParam("uploadId", uploadId).delete(path);
+ }
+ given().delete(path);
+ given().delete("/" + bucket);
+ }
+ }
+
+ private static String completion(String firstETag, String lastETag) {
+ return """
+ <CompleteMultipartUpload>
+ <Part><PartNumber>1</PartNumber><ETag>%s</ETag></Part>
+ <Part><PartNumber>2</PartNumber><ETag>%s</ETag></Part>
+ </CompleteMultipartUpload>""".formatted(firstETag, lastETag);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
index 416963e10f..3858f72095 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
@@ -42,6 +42,110 @@ class S3MultipartServiceTest {
s3Service.createBucket("test-bucket", "us-east-1");
}
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void undersizedNonFinalPartPreservesObjectAndUploadForRetry(boolean memory) {
+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ tempDir.resolve("minimum-part-size"), memory);
+ service.createBucket("sized-bucket", "us-east-1");
+ service.putObject("sized-bucket", "object", "prior".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ MultipartUpload upload = service.initiateMultipartUpload("sized-bucket", "object", null);
+ byte[] smallPart = "data".getBytes(StandardCharsets.UTF_8);
+ String firstETag = service.uploadPart("sized-bucket", "object", upload.getUploadId(), 1, smallPart);
+ String lastETag = service.uploadPart("sized-bucket", "object", upload.getUploadId(), 2, smallPart);
+
+ AwsException error = assertThrows(AwsException.class, () -> service.completeMultipartUpload(
+ "sized-bucket", "object", upload.getUploadId(), List.of(1, 2),
+ Map.of(1, firstETag, 2, lastETag), Map.of(), null, null));
+ assertEquals("EntityTooSmall", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("Your proposed upload is smaller than the minimum allowed size", error.getMessage());
+ assertEquals(Map.of("ProposedSize", 4L, "MinSizeAllowed", 5242880L, "PartNumber", 1,
+ "ETag", firstETag.substring(1, firstETag.length() - 1)), error.getExtendedData());
+ assertEquals("prior", new String(service.getObject("sized-bucket", "object").getData(),
+ StandardCharsets.UTF_8));
+ assertEquals(2, service.getMultipartUpload("sized-bucket", "object", upload.getUploadId()).getParts().size());
+
+ String replacementETag = service.uploadPart("sized-bucket", "object", upload.getUploadId(), 1,
+ new byte[5242880]);
+ S3Object completed = service.completeMultipartUpload("sized-bucket", "object", upload.getUploadId(),
+ List.of(1, 2), Map.of(1, replacementETag, 2, lastETag), Map.of(), null, null);
+ assertEquals(5242884, completed.getSize());
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, 5242879})
+ void minimumPartSizeRejectsValuesBelowFiveMiB(int size) {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "boundary", null);
+ s3Service.uploadPart("test-bucket", "boundary", upload.getUploadId(), 1, new byte[size]);
+ s3Service.uploadPart("test-bucket", "boundary", upload.getUploadId(), 2, new byte[0]);
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "boundary", upload.getUploadId(), List.of(1, 2), null, null));
+ assertEquals("EntityTooSmall", error.getErrorCode());
+ assertEquals((long) size, error.getExtendedData().get("ProposedSize"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void minimumPartSizeOnlyChecksSelectedNonFinalParts(boolean memory) {
+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ tempDir.resolve("selected-parts"), memory);
+ service.createBucket("selected-bucket", "us-east-1");
+ MultipartUpload upload = service.initiateMultipartUpload("selected-bucket", "object", null);
+ service.uploadPart("selected-bucket", "object", upload.getUploadId(), 1, new byte[1]);
+ service.uploadPart("selected-bucket", "object", upload.getUploadId(), 2, new byte[5242880]);
+ service.uploadPart("selected-bucket", "object", upload.getUploadId(), 7, new byte[0]);
+ service.uploadPart("selected-bucket", "object", upload.getUploadId(), 9, new byte[1]);
+
+ S3Object completed = service.completeMultipartUpload("selected-bucket", "object", upload.getUploadId(),
+ List.of(2, 7), null, null);
+ assertEquals(5242880, completed.getSize());
+ assertEquals(List.of(2, 7), completed.getParts().stream().map(Part::getPartNumber).toList());
+ }
+
+ @Test
+ void minimumPartSizeReportsTheFirstUndersizedNonFinalPart() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "first-small", null);
+ s3Service.uploadPart("test-bucket", "first-small", upload.getUploadId(), 1, new byte[5242880]);
+ s3Service.uploadPart("test-bucket", "first-small", upload.getUploadId(), 3, new byte[2]);
+ s3Service.uploadPart("test-bucket", "first-small", upload.getUploadId(), 7, new byte[1]);
+ s3Service.uploadPart("test-bucket", "first-small", upload.getUploadId(), 9, new byte[0]);
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "first-small", upload.getUploadId(), List.of(1, 3, 7, 9), null, null));
+ assertEquals("EntityTooSmall", error.getErrorCode());
+ assertEquals(3, error.getExtendedData().get("PartNumber"));
+ assertEquals(2L, error.getExtendedData().get("ProposedSize"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, 4})
+ void minimumPartSizeAllowsASingleSmallSelectedPart(int size) {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "single", null);
+ s3Service.uploadPart("test-bucket", "single", upload.getUploadId(), 1, new byte[size]);
+ s3Service.uploadPart("test-bucket", "single", upload.getUploadId(), 2, new byte[1]);
+
+ S3Object completed = s3Service.completeMultipartUpload("test-bucket", "single", upload.getUploadId(),
+ List.of(1), null, null);
+ assertEquals(size, completed.getSize());
+ }
+
+ @Test
+ void minimumPartSizePreservesManifestValidationBeforeSizeValidation() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "validation", null);
+ String firstETag = s3Service.uploadPart("test-bucket", "validation", upload.getUploadId(), 1, new byte[1]);
+ s3Service.uploadPart("test-bucket", "validation", upload.getUploadId(), 2, new byte[1]);
+
+ AwsException badETag = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "validation", upload.getUploadId(), List.of(1, 2),
+ Map.of(1, firstETag, 2, "wrong"), Map.of(), null, null));
+ assertEquals("InvalidPart", badETag.getErrorCode());
+ AwsException badOrder = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "validation", upload.getUploadId(), List.of(2, 1), null, null));
+ assertEquals("InvalidPartOrder", badOrder.getErrorCode());
+ }
+
@ParameterizedTest
@ValueSource(booleans = {false, true})
void expectedObjectSizeUsesSelectedPartsAndPreservesExistingObjectOnMismatch(boolean memory) {
@@ -116,7 +220,9 @@ class S3MultipartServiceTest {
void completeMultipartUpload() {
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "file.bin", "text/plain",
Map.of("owner", "team-a"), "STANDARD_IA");
- s3Service.uploadPart("test-bucket", "file.bin", upload.getUploadId(), 1, "part1".getBytes());
+ byte[] firstPart = new byte[5 * 1024 * 1024];
+ byte[] whole = concat(firstPart, "part2".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "file.bin", upload.getUploadId(), 1, firstPart);
s3Service.uploadPart("test-bucket", "file.bin", upload.getUploadId(), 2, "part2".getBytes());
S3Object result = s3Service.completeMultipartUpload("test-bucket", "file.bin",
@@ -129,12 +235,12 @@ class S3MultipartServiceTest {
assertEquals(2, result.getParts().size());
// Verify the data is concatenated
S3Object fetched = s3Service.getObject("test-bucket", "file.bin");
- assertEquals("part1part2", new String(fetched.getData()));
+ assertArrayEquals(whole, fetched.getData());
// Composite ETag should end with -2 (number of parts)
assertTrue(result.getETag().endsWith("-2\""), "ETag should be composite: " + result.getETag());
// Without a checksum algorithm AWS attaches the default CRC64NVME full-object checksum
assertEquals(ChecksumType.FULL_OBJECT, result.getChecksum().getChecksumType());
- assertEquals(S3Checksum.crc64NvmeBase64("part1part2".getBytes(StandardCharsets.UTF_8)),
+ assertEquals(S3Checksum.crc64NvmeBase64(whole),
result.getChecksum().getChecksumCRC64NVME());
}
@@ -233,15 +339,16 @@ class S3MultipartServiceTest {
@Test
void completeMultipartUploadAcceptsNonConsecutivePartNumbers() {
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "gapped.bin", null);
+ byte[] firstPart = new byte[5 * 1024 * 1024];
String first = s3Service.uploadPart("test-bucket", "gapped.bin", upload.getUploadId(), 1,
- "part1".getBytes(StandardCharsets.UTF_8));
+ firstPart);
String third = s3Service.uploadPart("test-bucket", "gapped.bin", upload.getUploadId(), 3,
"part3".getBytes(StandardCharsets.UTF_8));
S3Object result = s3Service.completeMultipartUpload("test-bucket", "gapped.bin", upload.getUploadId(),
List.of(1, 3), Map.of(1, first, 3, third), Map.of(), null, null);
- assertEquals("part1part3", new String(s3Service.getObject("test-bucket", "gapped.bin").getData(),
- StandardCharsets.UTF_8));
+ assertArrayEquals(concat(firstPart, "part3".getBytes(StandardCharsets.UTF_8)),
+ s3Service.getObject("test-bucket", "gapped.bin").getData());
}
@Test
@@ -299,7 +406,7 @@ class S3MultipartServiceTest {
service.putBucketVersioning("versioned-bucket", "Enabled");
MultipartUpload upload = service.initiateMultipartUpload("versioned-bucket", "file.bin", null);
service.uploadPart("versioned-bucket", "file.bin", upload.getUploadId(), 1,
- "part1".getBytes(StandardCharsets.UTF_8));
+ new byte[5 * 1024 * 1024]);
service.uploadPart("versioned-bucket", "file.bin", upload.getUploadId(), 2,
"part2".getBytes(StandardCharsets.UTF_8));
@@ -313,7 +420,7 @@ class S3MultipartServiceTest {
@Test
void completeMultipartUploadReturnsTheMd5OfThePartMd5s() throws NoSuchAlgorithmException {
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "composite-etag.bin", null);
- byte[] part1 = "part1".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "part2".getBytes(StandardCharsets.UTF_8);
s3Service.uploadPart("test-bucket", "composite-etag.bin", upload.getUploadId(), 1, part1);
s3Service.uploadPart("test-bucket", "composite-etag.bin", upload.getUploadId(), 2, part2);
@@ -340,11 +447,12 @@ class S3MultipartServiceTest {
@Test
void getObjectAttributesReturnsMultipartParts() {
+ byte[] part1 = new byte[5 * 1024 * 1024];
MultipartUpload upload = initiate("parts.bin", "SHA256", null);
- s3Service.uploadPart("test-bucket", "parts.bin", upload.getUploadId(), 1, "abc".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "parts.bin", upload.getUploadId(), 1, part1);
s3Service.uploadPart("test-bucket", "parts.bin", upload.getUploadId(), 2, "def".getBytes(StandardCharsets.UTF_8));
s3Service.completeMultipartUpload("test-bucket", "parts.bin", upload.getUploadId(), List.of(1, 2),
- partChecksums(ChecksumAlgorithm.SHA256, "abc".getBytes(StandardCharsets.UTF_8), "def".getBytes(StandardCharsets.UTF_8)), null, null);
+ partChecksums(ChecksumAlgorithm.SHA256, part1, "def".getBytes(StandardCharsets.UTF_8)), null, null);
GetObjectAttributesResult attributes = s3Service.getObjectAttributes("test-bucket", "parts.bin", null,
Set.of(ObjectAttributeName.OBJECT_PARTS, ObjectAttributeName.CHECKSUM),
@@ -358,14 +466,15 @@ class S3MultipartServiceTest {
assertEquals(1, attributes.getObjectParts().getParts().size());
assertTrue(attributes.getObjectParts().isTruncated());
assertEquals(1, attributes.getObjectParts().getNextPartNumberMarker());
- assertEquals(S3Checksum.sha256Base64("abc".getBytes(StandardCharsets.UTF_8)),
+ assertEquals(S3Checksum.sha256Base64(part1),
attributes.getObjectParts().getParts().get(0).getChecksum().getChecksumSHA256());
}
@Test
void getObjectAttributesReportsOnlyThePartCountForFullObjectChecksums() {
+ byte[] part1 = new byte[5 * 1024 * 1024];
MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "full-parts.bin", "application/octet-stream");
- s3Service.uploadPart("test-bucket", "full-parts.bin", upload.getUploadId(), 1, "abc".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "full-parts.bin", upload.getUploadId(), 1, part1);
s3Service.uploadPart("test-bucket", "full-parts.bin", upload.getUploadId(), 2, "def".getBytes(StandardCharsets.UTF_8));
s3Service.completeMultipartUpload("test-bucket", "full-parts.bin", upload.getUploadId(), List.of(1, 2), null, null);
@@ -391,7 +500,7 @@ class S3MultipartServiceTest {
@Test
void completeMultipartUploadReturnsCompositeSha256Checksum() {
- byte[] part1 = "Part1Data-Hello".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "Part2Data-World".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = initiate("sha256.bin", "SHA256", null);
s3Service.uploadPart("test-bucket", "sha256.bin", upload.getUploadId(), 1, part1);
@@ -432,7 +541,7 @@ class S3MultipartServiceTest {
@Test
void completeMultipartUploadReturnsCompositeCrc32Checksum() {
- byte[] part1 = "crc-part-1".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "crc-part-2".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = initiate("crc32.bin", "CRC32", null);
s3Service.uploadPart("test-bucket", "crc32.bin", upload.getUploadId(), 1, part1);
@@ -449,7 +558,7 @@ class S3MultipartServiceTest {
@Test
void completeMultipartUploadKeepsCrc64NvmeAsFullObjectChecksum() {
- byte[] part1 = "crc64-part-1".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "crc64-part-2".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = initiate("crc64.bin", "CRC64NVME", null);
assertEquals(ChecksumType.FULL_OBJECT, upload.getChecksumType());
@@ -465,7 +574,7 @@ class S3MultipartServiceTest {
@Test
void completeMultipartUploadHonorsFullObjectTypeRequestedAtInitiation() {
- byte[] part1 = "full-part-1".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "full-part-2".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = initiate("crc32-full.bin", "CRC32", "FULL_OBJECT");
s3Service.uploadPart("test-bucket", "crc32-full.bin", upload.getUploadId(), 1, part1);
@@ -498,7 +607,7 @@ class S3MultipartServiceTest {
@Test
void completeMultipartUploadValidatesClientCompositeChecksum() {
- byte[] part1 = "Part1Data-Hello".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "Part2Data-World".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = initiate("validated.bin", "SHA256", null);
s3Service.uploadPart("test-bucket", "validated.bin", upload.getUploadId(), 1, part1);
@@ -559,7 +668,7 @@ class S3MultipartServiceTest {
@Test
void completeMultipartUploadRequiresEveryPartChecksumOnCompositeUploads() {
- byte[] part1 = "Part1Data-Hello".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "Part2Data-World".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = initiate("strict.bin", "SHA256", null);
s3Service.uploadPart("test-bucket", "strict.bin", upload.getUploadId(), 1, part1);
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index 35d76bedde..f346d7e240 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -654,7 +654,7 @@ class S3ServiceTest {
@Test
void copyOfMultipartObjectGetsTheEtagOfTheWholeContent() throws NoSuchAlgorithmException {
s3Service.createBucket("test-bucket", "us-east-1");
- byte[] part1 = "Part1Data-Hello".getBytes(StandardCharsets.UTF_8);
+ byte[] part1 = new byte[5 * 1024 * 1024];
byte[] part2 = "Part2Data-World".getBytes(StandardCharsets.UTF_8);
var upload = s3Service.initiateMultipartUpload("test-bucket", "multipart.bin", "application/octet-stream");
s3Service.uploadPart("test-bucket", "multipart.bin", upload.getUploadId(), 1, part1);
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java
index a051c161eb..1a6f2f132e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java
@@ -167,22 +167,23 @@ class S3StreamedUploadTest {
@Test
void aStreamedPartStoresWhatTheBytePathStores() {
+ byte[] part1 = Arrays.copyOf(BODY, 5 * 1024 * 1024);
byte[] part2 = "and a second part".getBytes(StandardCharsets.UTF_8);
MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);
- Part fromBytes = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, BODY, null, null, null);
+ Part fromBytes = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, part1, null, null, null);
Part streamed = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 2,
new ByteArrayInputStream(part2), NO_CHECKSUMS, null, null, null);
Part streamedAgain = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1,
- new ByteArrayInputStream(BODY), NO_CHECKSUMS, null, null, null);
+ new ByteArrayInputStream(part1), NO_CHECKSUMS, null, null, null);
assertEquals(fromBytes.getETag(), streamedAgain.getETag());
assertEquals(fromBytes.getChecksum().getChecksumCRC64NVME(), streamedAgain.getChecksum().getChecksumCRC64NVME());
assertEquals(part2.length, streamed.getSize());
s3Service.completeMultipartUpload("bucket", "multipart.bin", upload.getUploadId(), List.of(1, 2), null, null);
- byte[] whole = new byte[BODY.length + part2.length];
- System.arraycopy(BODY, 0, whole, 0, BODY.length);
- System.arraycopy(part2, 0, whole, BODY.length, part2.length);
+ byte[] whole = new byte[part1.length + part2.length];
+ System.arraycopy(part1, 0, whole, 0, part1.length);
+ System.arraycopy(part2, 0, whole, part1.length, part2.length);
assertArrayEquals(whole, s3Service.getObject("bucket", "multipart.bin").getData());
assertNothingStaged();
}
@@ -246,17 +247,17 @@ class S3StreamedUploadTest {
@Test
void anUploadPartCopyStreamsItsRangeIntoThePart() {
- byte[] source = new byte[300_000];
+ byte[] source = new byte[5 * 1024 * 1024 + 1000];
new Random(5034).nextBytes(source);
s3Service.putObject("bucket", "source.bin", source, "application/octet-stream", Map.of());
MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "copy.bin", null);
String rangeETag = s3Service.uploadPartCopy("bucket", "copy.bin", upload.getUploadId(), 1,
- "bucket", "source.bin", null, "bytes=1000-200999");
+ "bucket", "source.bin", null, "bytes=1000-" + (source.length - 1));
String wholeETag = s3Service.uploadPartCopy("bucket", "copy.bin", upload.getUploadId(), 2,
"bucket", "source.bin", null, null);
- byte[] range = Arrays.copyOfRange(source, 1000, 201_000);
+ byte[] range = Arrays.copyOfRange(source, 1000, source.length);
assertEquals(S3Object.computeETag(range), rangeETag);
assertEquals(S3Object.computeETag(source), wholeETag);
s3Service.completeMultipartUpload("bucket", "copy.bin", upload.getUploadId(), List.of(1, 2), null, null);