Light, fluffy, and always free - The AWS Local Emulator alternative
perf(s3): assemble multipart uploads without doubling the object in heap (#4227)
Niels Kootstra0e432dd41bd0
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex ddf168add6..8f9c5e6d6f 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -30,7 +30,6 @@ import jakarta.inject.Inject;import jakarta.ws.rs.core.MultivaluedHashMap;import org.jboss.logging.Logger;-import java.io.ByteArrayOutputStream;import java.io.ByteArrayInputStream;import java.io.IOException;import java.io.InputStream;@@ -3319,24 +3318,18 @@ public class S3Service implements Resettable, ResourceProvider {// Concatenate parts in ordertry {- ByteArrayOutputStream combined = new ByteArrayOutputStream();MessageDigest md = MessageDigest.getInstance("MD5");-for (int num : partNumbers) {- byte[] partData = inMemory- ? memoryMultipartStore.get(uploadId).get(num)- : Files.readAllBytes(dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(num)));- combined.write(partData);// A part ETag is the MD5 of that part, so the composite hashes it without rehashing the dataString partETag = stripSurroundingQuotes(upload.getParts().get(num).getETag());md.update(HexFormat.of().parseHex(partETag));}- byte[] allData = combined.toByteArray();-// Composite ETag: MD5 of concatenated part MD5s, suffixed with part countString compositeETag = "\"" + bytesToHex(md.digest()) + "-" + partNumbers.size() + "\"";+ byte[] allData = concatenateParts(uploadId, partNumbers);+List<Part> completedParts = partNumbers.stream().map(num -> copyPart(upload.getParts().get(num))).toList();@@ -3376,6 +3369,55 @@ public class S3Service implements Resettable, ResourceProvider {}}+ /**+ * Copies the parts, in order, into one array sized to the total upload. Disk parts are read+ * straight into their slot, so assembly never holds a second full-size copy of the object.+ * In-memory parts are copied from the arrays that were measured, so their sizes cannot change+ * in between.+ */+ private byte[] concatenateParts(String uploadId, List<Integer> partNumbers) throws IOException {+ Path partsDir = dataRoot.resolve(".multipart").resolve(uploadId);+ Map<Integer, byte[]> memoryParts = inMemory ? memoryMultipartStore.get(uploadId) : null;+ byte[][] memoryData = inMemory ? new byte[partNumbers.size()][] : null;+ long[] partSizes = new long[partNumbers.size()];+ long totalSize = 0;+ for (int i = 0; i < partNumbers.size(); i++) {+ int num = partNumbers.get(i);+ if (inMemory) {+ memoryData[i] = memoryParts.get(num);+ partSizes[i] = memoryData[i].length;+ } else {+ partSizes[i] = Files.size(partsDir.resolve(String.valueOf(num)));+ }+ totalSize += partSizes[i];+ }+ byte[] allData = new byte[Math.toIntExact(totalSize)];+ int offset = 0;+ for (int i = 0; i < partNumbers.size(); i++) {+ int size = (int) partSizes[i];+ if (inMemory) {+ System.arraycopy(memoryData[i], 0, allData, offset, size);+ } else {+ int num = partNumbers.get(i);+ try (InputStream in = Files.newInputStream(partsDir.resolve(String.valueOf(num)))) {+ readPart(in, allData, offset, size, num);+ }+ }+ offset += size;+ }+ return allData;+ }++ /**+ * Reads exactly {@code size} bytes of part {@code partNumber} into {@code dest} at {@code offset},+ * failing if the part holds a different number of bytes than was measured before assembly.+ */+ static void readPart(InputStream in, byte[] dest, int offset, int size, int partNumber) throws IOException {+ if (in.readNBytes(dest, offset, size) != size || in.read() != -1) {+ throw new IOException("Part " + partNumber + " changed size during assembly");+ }+ }+private boolean etagsMatch(String storedETag, String submittedETag) {return stripSurroundingQuotes(storedETag).equals(stripSurroundingQuotes(submittedETag));}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.javanew file mode 100644index 0000000000..b963297290--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java@@ -0,0 +1,186 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.s3.model.ChecksumType;+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;+import io.github.hectorvent.floci.services.s3.model.S3Checksum;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.ByteArrayInputStream;+import java.io.IOException;+import java.io.UncheckedIOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.security.MessageDigest;+import java.security.NoSuchAlgorithmException;+import java.util.Arrays;+import java.util.HexFormat;+import java.util.List;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * CompleteMultipartUpload in disk-storage mode: byte-exact assembly in part order, the+ * composite-MD5 ETag, and the default full-object checksum, through S3Service's public interface.+ */+class S3MultipartDiskAssemblyTest {++ @TempDir+ Path tempDir;++ private S3Service s3Service;+ private Path dataRoot;++ @BeforeEach+ void setUp() {+ dataRoot = tempDir.resolve("s3");+ s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);+ s3Service.createBucket("test-bucket", "us-east-1");+ }++ @Test+ void assembledObjectConcatenatesPartsInAscendingOrderRegardlessOfUploadOrder() {+ byte[] part1 = repeatingBytes((byte) 'A', 17_000);+ byte[] part2 = repeatingBytes((byte) 'B', 3);+ byte[] part3 = repeatingBytes((byte) 'C', 65_537);++ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "assembled.bin", null);+ // Upload out of order: the assembled result must still honour ascending part number, not upload order.+ s3Service.uploadPart("test-bucket", "assembled.bin", upload.getUploadId(), 3, part3);+ s3Service.uploadPart("test-bucket", "assembled.bin", upload.getUploadId(), 1, part1);+ s3Service.uploadPart("test-bucket", "assembled.bin", upload.getUploadId(), 2, part2);++ s3Service.completeMultipartUpload("test-bucket", "assembled.bin", upload.getUploadId(),+ List.of(1, 2, 3), null, null);++ byte[] expected = concat(part1, part2, part3);+ assertArrayEquals(expected, s3Service.getObject("test-bucket", "assembled.bin").getData());+ }++ @Test+ void assembledObjectSkipsGapsWhenPartNumbersAreNonConsecutive() {+ byte[] part1 = "part-one".getBytes(StandardCharsets.UTF_8);+ byte[] part5 = "part-five".getBytes(StandardCharsets.UTF_8);++ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "gapped.bin", null);+ s3Service.uploadPart("test-bucket", "gapped.bin", upload.getUploadId(), 1, part1);+ s3Service.uploadPart("test-bucket", "gapped.bin", upload.getUploadId(), 5, part5);++ s3Service.completeMultipartUpload("test-bucket", "gapped.bin", upload.getUploadId(),+ List.of(1, 5), null, null);++ assertArrayEquals(concat(part1, part5), s3Service.getObject("test-bucket", "gapped.bin").getData());+ }++ @Test+ void compositeETagIsTheMd5OfTheConcatenatedPartMd5s() throws NoSuchAlgorithmException {+ byte[] part1 = repeatingBytes((byte) 'X', 9_001);+ byte[] part2 = repeatingBytes((byte) 'Y', 4_096);++ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "etag.bin", null);+ s3Service.uploadPart("test-bucket", "etag.bin", upload.getUploadId(), 1, part1);+ s3Service.uploadPart("test-bucket", "etag.bin", upload.getUploadId(), 2, part2);++ S3Object result = s3Service.completeMultipartUpload("test-bucket", "etag.bin",+ upload.getUploadId(), List.of(1, 2), null, null);++ // Independently computed from the spec (MD5 of the concatenation of each part's own MD5),+ // not derived from any S3Service/S3Checksum code path.+ MessageDigest composite = MessageDigest.getInstance("MD5");+ composite.update(MessageDigest.getInstance("MD5").digest(part1));+ composite.update(MessageDigest.getInstance("MD5").digest(part2));+ String expectedETag = "\"" + HexFormat.of().formatHex(composite.digest()) + "-2\"";++ assertEquals(expectedETag, result.getETag());+ assertEquals(expectedETag, s3Service.getObject("test-bucket", "etag.bin").getETag());+ }++ @Test+ void defaultFullObjectChecksumCoversTheAssembledBytesInOrder() {+ byte[] part1 = repeatingBytes((byte) 'M', 12_345);+ byte[] part2 = repeatingBytes((byte) 'N', 6_789);++ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "checksum.bin", null);+ s3Service.uploadPart("test-bucket", "checksum.bin", upload.getUploadId(), 1, part1);+ s3Service.uploadPart("test-bucket", "checksum.bin", upload.getUploadId(), 2, part2);++ S3Object result = s3Service.completeMultipartUpload("test-bucket", "checksum.bin",+ upload.getUploadId(), List.of(1, 2), null, null);++ assertEquals(ChecksumType.FULL_OBJECT, result.getChecksum().getChecksumType());+ assertEquals(S3Checksum.crc64NvmeBase64(concat(part1, part2)),+ result.getChecksum().getChecksumCRC64NVME());+ }++ @Test+ void noObjectIsCreatedWhenAPartFileGoesMissingDuringAssembly() throws Exception {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "vanished.bin", null);+ s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "vanished.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));++ // Metadata (the Part record with its ETag) lives in memory, separate from the on-disk part+ // file that assembly reads; deleting only the file makes assembly fail after the+ // file-independent validation checks have passed.+ Path part2File = dataRoot.resolve(".multipart").resolve(upload.getUploadId()).resolve("2");+ assertTrue(Files.deleteIfExists(part2File), "test setup: part 2 file should exist before deletion");++ assertThrows(UncheckedIOException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "vanished.bin", upload.getUploadId(), List.of(1, 2), null, null));++ assertThrows(AwsException.class,+ () -> s3Service.getObject("test-bucket", "vanished.bin"));+ }++ @Test+ void partReadCopiesExactlyTheMeasuredBytes() throws IOException {+ byte[] dest = new byte[7];++ S3Service.readPart(new ByteArrayInputStream("abcde".getBytes(StandardCharsets.UTF_8)), dest, 1, 5, 1);++ assertArrayEquals(new byte[] {0, 'a', 'b', 'c', 'd', 'e', 0}, dest);+ }++ @Test+ void partThatGrewAfterBeingMeasuredFailsAssembly() {+ byte[] dest = new byte[5];++ assertThrows(IOException.class, () -> S3Service.readPart(+ new ByteArrayInputStream("abcdef".getBytes(StandardCharsets.UTF_8)), dest, 0, 5, 1));+ }++ @Test+ void partThatShrankAfterBeingMeasuredFailsAssembly() {+ byte[] dest = new byte[5];++ assertThrows(IOException.class, () -> S3Service.readPart(+ new ByteArrayInputStream("abcd".getBytes(StandardCharsets.UTF_8)), dest, 0, 5, 1));+ }++ private static byte[] repeatingBytes(byte value, int length) {+ byte[] data = new byte[length];+ Arrays.fill(data, value);+ return data;+ }++ private static byte[] concat(byte[]... parts) {+ int total = 0;+ for (byte[] part : parts) {+ total += part.length;+ }+ byte[] result = new byte[total];+ int offset = 0;+ for (byte[] part : parts) {+ System.arraycopy(part, 0, result, offset, part.length);+ offset += part.length;+ }+ return result;+ }+}