Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

perf(s3): assemble multipart uploads without doubling the object in heap (#4227)

Niels Kootstra0e432dd41bd0
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index ddf168add6..8f9c5e6d6f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -30,7 +30,6 @@ import jakarta.inject.Inject;
import jakarta.ws.rs.core.MultivaluedHashMap;
import org.jboss.logging.Logger;
-import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -3319,24 +3318,18 @@ public class S3Service implements Resettable, ResourceProvider {
// Concatenate parts in order
try {
- ByteArrayOutputStream combined = new ByteArrayOutputStream();
MessageDigest md = MessageDigest.getInstance("MD5");
-
for (int num : partNumbers) {
- byte[] partData = inMemory
- ? memoryMultipartStore.get(uploadId).get(num)
- : Files.readAllBytes(dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(num)));
- combined.write(partData);
// A part ETag is the MD5 of that part, so the composite hashes it without rehashing the data
String partETag = stripSurroundingQuotes(upload.getParts().get(num).getETag());
md.update(HexFormat.of().parseHex(partETag));
}
- byte[] allData = combined.toByteArray();
-
// Composite ETag: MD5 of concatenated part MD5s, suffixed with part count
String compositeETag = "\"" + bytesToHex(md.digest()) + "-" + partNumbers.size() + "\"";
+ byte[] allData = concatenateParts(uploadId, partNumbers);
+
List<Part> completedParts = partNumbers.stream()
.map(num -> copyPart(upload.getParts().get(num)))
.toList();
@@ -3376,6 +3369,55 @@ public class S3Service implements Resettable, ResourceProvider {
}
}
+ /**
+ * Copies the parts, in order, into one array sized to the total upload. Disk parts are read
+ * straight into their slot, so assembly never holds a second full-size copy of the object.
+ * In-memory parts are copied from the arrays that were measured, so their sizes cannot change
+ * in between.
+ */
+ private byte[] concatenateParts(String uploadId, List<Integer> partNumbers) throws IOException {
+ Path partsDir = dataRoot.resolve(".multipart").resolve(uploadId);
+ Map<Integer, byte[]> memoryParts = inMemory ? memoryMultipartStore.get(uploadId) : null;
+ byte[][] memoryData = inMemory ? new byte[partNumbers.size()][] : null;
+ long[] partSizes = new long[partNumbers.size()];
+ long totalSize = 0;
+ for (int i = 0; i < partNumbers.size(); i++) {
+ int num = partNumbers.get(i);
+ if (inMemory) {
+ memoryData[i] = memoryParts.get(num);
+ partSizes[i] = memoryData[i].length;
+ } else {
+ partSizes[i] = Files.size(partsDir.resolve(String.valueOf(num)));
+ }
+ totalSize += partSizes[i];
+ }
+ byte[] allData = new byte[Math.toIntExact(totalSize)];
+ int offset = 0;
+ for (int i = 0; i < partNumbers.size(); i++) {
+ int size = (int) partSizes[i];
+ if (inMemory) {
+ System.arraycopy(memoryData[i], 0, allData, offset, size);
+ } else {
+ int num = partNumbers.get(i);
+ try (InputStream in = Files.newInputStream(partsDir.resolve(String.valueOf(num)))) {
+ readPart(in, allData, offset, size, num);
+ }
+ }
+ offset += size;
+ }
+ return allData;
+ }
+
+ /**
+ * Reads exactly {@code size} bytes of part {@code partNumber} into {@code dest} at {@code offset},
+ * failing if the part holds a different number of bytes than was measured before assembly.
+ */
+ static void readPart(InputStream in, byte[] dest, int offset, int size, int partNumber) throws IOException {
+ if (in.readNBytes(dest, offset, size) != size || in.read() != -1) {
+ throw new IOException("Part " + partNumber + " changed size during assembly");
+ }
+ }
+
private boolean etagsMatch(String storedETag, String submittedETag) {
return stripSurroundingQuotes(storedETag).equals(stripSurroundingQuotes(submittedETag));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
new file mode 100644
index 0000000000..b963297290
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
@@ -0,0 +1,186 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.s3.model.ChecksumType;
+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;
+import io.github.hectorvent.floci.services.s3.model.S3Checksum;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.HexFormat;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * CompleteMultipartUpload in disk-storage mode: byte-exact assembly in part order, the
+ * composite-MD5 ETag, and the default full-object checksum, through S3Service's public interface.
+ */
+class S3MultipartDiskAssemblyTest {
+
+ @TempDir
+ Path tempDir;
+
+ private S3Service s3Service;
+ private Path dataRoot;
+
+ @BeforeEach
+ void setUp() {
+ dataRoot = tempDir.resolve("s3");
+ s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+ s3Service.createBucket("test-bucket", "us-east-1");
+ }
+
+ @Test
+ void assembledObjectConcatenatesPartsInAscendingOrderRegardlessOfUploadOrder() {
+ byte[] part1 = repeatingBytes((byte) 'A', 17_000);
+ byte[] part2 = repeatingBytes((byte) 'B', 3);
+ byte[] part3 = repeatingBytes((byte) 'C', 65_537);
+
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "assembled.bin", null);
+ // Upload out of order: the assembled result must still honour ascending part number, not upload order.
+ s3Service.uploadPart("test-bucket", "assembled.bin", upload.getUploadId(), 3, part3);
+ s3Service.uploadPart("test-bucket", "assembled.bin", upload.getUploadId(), 1, part1);
+ s3Service.uploadPart("test-bucket", "assembled.bin", upload.getUploadId(), 2, part2);
+
+ s3Service.completeMultipartUpload("test-bucket", "assembled.bin", upload.getUploadId(),
+ List.of(1, 2, 3), null, null);
+
+ byte[] expected = concat(part1, part2, part3);
+ assertArrayEquals(expected, s3Service.getObject("test-bucket", "assembled.bin").getData());
+ }
+
+ @Test
+ void assembledObjectSkipsGapsWhenPartNumbersAreNonConsecutive() {
+ byte[] part1 = "part-one".getBytes(StandardCharsets.UTF_8);
+ byte[] part5 = "part-five".getBytes(StandardCharsets.UTF_8);
+
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "gapped.bin", null);
+ s3Service.uploadPart("test-bucket", "gapped.bin", upload.getUploadId(), 1, part1);
+ s3Service.uploadPart("test-bucket", "gapped.bin", upload.getUploadId(), 5, part5);
+
+ s3Service.completeMultipartUpload("test-bucket", "gapped.bin", upload.getUploadId(),
+ List.of(1, 5), null, null);
+
+ assertArrayEquals(concat(part1, part5), s3Service.getObject("test-bucket", "gapped.bin").getData());
+ }
+
+ @Test
+ void compositeETagIsTheMd5OfTheConcatenatedPartMd5s() throws NoSuchAlgorithmException {
+ byte[] part1 = repeatingBytes((byte) 'X', 9_001);
+ byte[] part2 = repeatingBytes((byte) 'Y', 4_096);
+
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "etag.bin", null);
+ s3Service.uploadPart("test-bucket", "etag.bin", upload.getUploadId(), 1, part1);
+ s3Service.uploadPart("test-bucket", "etag.bin", upload.getUploadId(), 2, part2);
+
+ S3Object result = s3Service.completeMultipartUpload("test-bucket", "etag.bin",
+ upload.getUploadId(), List.of(1, 2), null, null);
+
+ // Independently computed from the spec (MD5 of the concatenation of each part's own MD5),
+ // not derived from any S3Service/S3Checksum code path.
+ MessageDigest composite = MessageDigest.getInstance("MD5");
+ composite.update(MessageDigest.getInstance("MD5").digest(part1));
+ composite.update(MessageDigest.getInstance("MD5").digest(part2));
+ String expectedETag = "\"" + HexFormat.of().formatHex(composite.digest()) + "-2\"";
+
+ assertEquals(expectedETag, result.getETag());
+ assertEquals(expectedETag, s3Service.getObject("test-bucket", "etag.bin").getETag());
+ }
+
+ @Test
+ void defaultFullObjectChecksumCoversTheAssembledBytesInOrder() {
+ byte[] part1 = repeatingBytes((byte) 'M', 12_345);
+ byte[] part2 = repeatingBytes((byte) 'N', 6_789);
+
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "checksum.bin", null);
+ s3Service.uploadPart("test-bucket", "checksum.bin", upload.getUploadId(), 1, part1);
+ s3Service.uploadPart("test-bucket", "checksum.bin", upload.getUploadId(), 2, part2);
+
+ S3Object result = s3Service.completeMultipartUpload("test-bucket", "checksum.bin",
+ upload.getUploadId(), List.of(1, 2), null, null);
+
+ assertEquals(ChecksumType.FULL_OBJECT, result.getChecksum().getChecksumType());
+ assertEquals(S3Checksum.crc64NvmeBase64(concat(part1, part2)),
+ result.getChecksum().getChecksumCRC64NVME());
+ }
+
+ @Test
+ void noObjectIsCreatedWhenAPartFileGoesMissingDuringAssembly() throws Exception {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "vanished.bin", null);
+ s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));
+
+ // Metadata (the Part record with its ETag) lives in memory, separate from the on-disk part
+ // file that assembly reads; deleting only the file makes assembly fail after the
+ // file-independent validation checks have passed.
+ Path part2File = dataRoot.resolve(".multipart").resolve(upload.getUploadId()).resolve("2");
+ assertTrue(Files.deleteIfExists(part2File), "test setup: part 2 file should exist before deletion");
+
+ assertThrows(UncheckedIOException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "vanished.bin", upload.getUploadId(), List.of(1, 2), null, null));
+
+ assertThrows(AwsException.class,
+ () -> s3Service.getObject("test-bucket", "vanished.bin"));
+ }
+
+ @Test
+ void partReadCopiesExactlyTheMeasuredBytes() throws IOException {
+ byte[] dest = new byte[7];
+
+ S3Service.readPart(new ByteArrayInputStream("abcde".getBytes(StandardCharsets.UTF_8)), dest, 1, 5, 1);
+
+ assertArrayEquals(new byte[] {0, 'a', 'b', 'c', 'd', 'e', 0}, dest);
+ }
+
+ @Test
+ void partThatGrewAfterBeingMeasuredFailsAssembly() {
+ byte[] dest = new byte[5];
+
+ assertThrows(IOException.class, () -> S3Service.readPart(
+ new ByteArrayInputStream("abcdef".getBytes(StandardCharsets.UTF_8)), dest, 0, 5, 1));
+ }
+
+ @Test
+ void partThatShrankAfterBeingMeasuredFailsAssembly() {
+ byte[] dest = new byte[5];
+
+ assertThrows(IOException.class, () -> S3Service.readPart(
+ new ByteArrayInputStream("abcd".getBytes(StandardCharsets.UTF_8)), dest, 0, 5, 1));
+ }
+
+ private static byte[] repeatingBytes(byte value, int length) {
+ byte[] data = new byte[length];
+ Arrays.fill(data, value);
+ return data;
+ }
+
+ private static byte[] concat(byte[]... parts) {
+ int total = 0;
+ for (byte[] part : parts) {
+ total += part.length;
+ }
+ byte[] result = new byte[total];
+ int offset = 0;
+ for (byte[] part : parts) {
+ System.arraycopy(part, 0, result, offset, part.length);
+ offset += part.length;
+ }
+ return result;
+ }
+}