Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): stream UploadPartCopy ranges into the part (#5162)
Priamo Germosen8fee66ae6104
diff --git a/docs/services/s3.md b/docs/services/s3.mdindex e3b86d29b0..0838ad882e 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -389,8 +389,8 @@ still rewritten on every change. See[Storage Modes](../configuration/storage.md#journaled-stores-under-persistent-mode).In `memory` mode each object body is a single in-memory byte array, so an object can hold at most-about 2 GiB: a `PutObject` over that, an `UploadPart` that takes a multipart upload past it, or a-`CompleteMultipartUpload` over it fails with `EntityTooLarge`. The disk-backed modes write a+about 2 GiB: a `PutObject` over that, an `UploadPart` or `UploadPartCopy` that takes a multipart+upload past it, or a `CompleteMultipartUpload` over it fails with `EntityTooLarge`. The disk-backed modes write a`PutObject` or `UploadPart` body to disk as it arrives, working out its ETag and checksums on theway, and assemble a multipart upload file to file, so no object passes through the heap; set`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. Every request body is still@@ -403,8 +403,9 @@ into memory to check that hash, so it is limited to about 2 GiB. While it runs,`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembledcopy, until the parts are removed. Multipart uploads in progress do not survive a restart or areset, and their files are removed when either happens. `UploadPartCopy` reads only the copied-range from its source, so it can copy any part of a source of any size, but one copied part can-hold at most about 2 GiB; a larger range fails with `EntityTooLarge`. In the disk-backed modes+range from its source and, in the disk-backed modes, writes it straight into the part, so it can+copy a range of up to 5 GiB, S3's limit for one part, from a source of any size. A larger range+is rejected with `EntityTooLarge`. In the disk-backed modes`CopyObject` stores the source's file for the copy without reading it into memory, and reads itonce only when the copy needs an ETag or checksum the source's cannot supply. As on S3, a`CopyObject` source larger than 5 GiB is rejected with `InvalidRequest`; copy it in parts withdiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/CopyRangeInputStream.java b/src/main/java/io/github/hectorvent/floci/services/s3/CopyRangeInputStream.javanew file mode 100644index 0000000000..1567e7ead1--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/CopyRangeInputStream.java@@ -0,0 +1,59 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.services.s3.S3Service.CopySourceRange;++import java.io.EOFException;+import java.io.IOException;+import java.io.InputStream;++/**+ * The bytes of one copy-source range, read from a stream over the whole source as they are needed.+ * The first read skips to the range's first byte, so nothing of the source is touched until the+ * part is about to be written, and the stream ends after the range's last byte. A source that ends+ * before the range does fails the read.+ */+final class CopyRangeInputStream extends InputStream {++ private final InputStream source;+ private final long first;+ private long remaining;+ private boolean positioned;++ CopyRangeInputStream(InputStream source, CopySourceRange range) {+ this.source = source;+ this.first = range.first();+ this.remaining = range.length();+ }++ @Override+ public int read() throws IOException {+ byte[] single = new byte[1];+ int read = read(single, 0, 1);+ return read < 0 ? -1 : single[0] & 0xFF;+ }++ @Override+ public int read(byte[] buffer, int offset, int length) throws IOException {+ if (length == 0) {+ return 0;+ }+ if (!positioned) {+ source.skipNBytes(first);+ positioned = true;+ }+ if (remaining == 0) {+ return -1;+ }+ int read = source.read(buffer, offset, (int) Math.min(length, remaining));+ if (read < 0) {+ throw new EOFException("Copy source ended " + remaining + " bytes before the end of the requested range");+ }+ remaining -= read;+ return read;+ }++ @Override+ public void close() throws IOException {+ source.close();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 6a1cf1a0b0..5fd5fbaa79 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -2256,6 +2256,9 @@ public class S3Service implements Resettable, ResourceProvider {// S3 copies at most 5 GiB in one CopyObject; a larger object is copied with UploadPartCopy.static final long MAX_COPY_OBJECT_SOURCE_SIZE = 5L * 1024 * 1024 * 1024;+ // The most one part of a multipart upload can hold, per S3's multipart upload limits.+ static final long MAX_PART_SIZE = 5L * 1024 * 1024 * 1024;+static void requireCopyableSize(S3Object source) {if (source.getSize() > MAX_COPY_OBJECT_SOURCE_SIZE) {throw new AwsException("InvalidRequest",@@ -3978,9 +3981,9 @@ public class S3Service implements Resettable, ResourceProvider {SseCustomerHeaders copySourceSseCustomerHeaders,SseCustomerHeaders sseCustomerHeaders,CopySourceConditions copySourceConditions) {- byte[] data;- // The source is streamed the way GetObject serves it and only the copied range is read, so- // the source can be any size.+ // The source is streamed the way GetObject serves it, and only the copied range is read,+ // straight into the part the way a streamed UploadPart body is, so neither the source nor+ // the part has to fit in memory in the disk-backed modes.try (ObjectRead read = openObject(sourceBucket, sourceKey, sourceVersionId)) {S3Object source = read.object();checkCopySourcePreconditions(source, copySourceConditions);@@ -3989,18 +3992,17 @@ public class S3Service implements Resettable, ResourceProvider {copySourceSseCustomerHeaders.key(),copySourceSseCustomerHeaders.keyMd5());CopySourceRange range = CopySourceRange.parse(copySourceRange, source.getSize());- if (range.length() > MAX_IN_MEMORY_OBJECT_SIZE) {- LOG.warnv("UploadPartCopy of {0} bytes into upload {1} is over the {2} bytes one part can hold in Floci; copy the source in smaller ranges",- range.length(), uploadId, MAX_IN_MEMORY_OBJECT_SIZE);- throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);+ if (range.length() > MAX_PART_SIZE) {+ throw new AwsException("EntityTooLarge",+ "Your proposed upload exceeds the maximum allowed object size.", 400);}- data = readRange(read.body(), range);+ return storePart(destBucket, destKey, uploadId, partNumber,+ new CopyRangeInputStream(read.body(), range), UploadChecksums.NONE,+ sseCustomerHeaders.algorithm(), sseCustomerHeaders.key(), sseCustomerHeaders.keyMd5())+ .getETag();} catch (IOException e) {- throw new UncheckedIOException("Failed to read the copy source", e);+ throw new UncheckedIOException("Failed to close the copy source", e);}-- return uploadPart(destBucket, destKey, uploadId, partNumber, data,- sseCustomerHeaders.algorithm(), sseCustomerHeaders.key(), sseCustomerHeaders.keyMd5());}/** An inclusive byte range of a copy source. Offsets are longs, since a source can pass 2 GiB. */@@ -4063,17 +4065,6 @@ public class S3Service implements Resettable, ResourceProvider {}}- /** Reads exactly the bytes of {@code range} from {@code body}, skipping the bytes before it. */- static byte[] readRange(InputStream body, CopySourceRange range) throws IOException {- body.skipNBytes(range.first());- byte[] data = body.readNBytes((int) range.length());- if (data.length != range.length()) {- throw new IOException("Copy source ended " + (range.length() - data.length)- + " bytes before the end of the requested range");- }- return data;- }-public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List<Integer> partNumbers,String checksumType, S3Checksum expectedChecksum) {return completeMultipartUpload(bucket, key, uploadId, partNumbers, Map.of(), Map.of(), checksumType,diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.java b/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.javaindex e0d33ba820..28cdcc5bf5 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.java@@ -17,6 +17,9 @@ import java.util.function.Function;*/record UploadChecksums(String contentMd5, Map<ChecksumAlgorithm, String> claimed) {+ /** No integrity headers, as for the part an UploadPartCopy reads from another object. */+ static final UploadChecksums NONE = new UploadChecksums(null, Map.of());+// The order the checksums have always been checked in, so a body failing several reports the same one.private static final List<ChecksumAlgorithm> CHECK_ORDER = List.of(ChecksumAlgorithm.SHA1,ChecksumAlgorithm.SHA256, ChecksumAlgorithm.CRC32, ChecksumAlgorithm.CRC32C, ChecksumAlgorithm.CRC64NVME);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.javaindex 139eaa717b..73694132b1 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.java@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.services.s3.S3Service.CopySourceRange;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;+import java.io.ByteArrayOutputStream;import java.io.IOException;import java.io.InputStream;import java.nio.ByteBuffer;@@ -91,10 +92,10 @@ class S3CopySourceRangeTest {}@Test- void readRangeSkipsToAnOffsetPastTwoGibibytes() throws IOException {+ void aRangeSkipsToAnOffsetPastTwoGibibytes() throws IOException {CopySourceRange range = new CopySourceRange(2_500_000_000L, 2_500_000_015L);- byte[] data = S3Service.readRange(patternStream(3L * 1024 * 1024 * 1024), range);+ byte[] data = new CopyRangeInputStream(patternStream(3L * 1024 * 1024 * 1024), range).readAllBytes();assertEquals(16, data.length);for (int i = 0; i < data.length; i++) {@@ -103,7 +104,7 @@ class S3CopySourceRangeTest {}@Test- void readRangeSkipsPastTwoGibibytesInTheFileStreamDiskObjectsAreReadThrough() throws IOException {+ void aRangeSkipsPastTwoGibibytesInTheFileStreamDiskObjectsAreReadThrough() throws IOException {long offset = 2_500_000_000L;byte[] written = new byte[16];for (int i = 0; i < written.length; i++) {@@ -118,16 +119,40 @@ class S3CopySourceRangeTest {byte[] data;try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {- data = S3Service.readRange(Channels.newInputStream(channel),- new CopySourceRange(offset, offset + written.length - 1));+ data = new CopyRangeInputStream(Channels.newInputStream(channel),+ new CopySourceRange(offset, offset + written.length - 1)).readAllBytes();}assertArrayEquals(written, data);}@Test- void readRangeFailsWhenTheSourceEndsBeforeTheRange() {- assertThrows(IOException.class, () -> S3Service.readRange(patternStream(10), new CopySourceRange(5, 14)));+ void aRangeFailsWhenTheSourceEndsBeforeIt() {+ assertThrows(IOException.class,+ () -> new CopyRangeInputStream(patternStream(10), new CopySourceRange(5, 14)).readAllBytes());+ assertThrows(IOException.class,+ () -> new CopyRangeInputStream(patternStream(10), new CopySourceRange(12, 14)).readAllBytes(),+ "a source that ends before the range starts");+ }++ @Test+ void aRangeEndsAfterItsLastByteWhateverTheReadSize() throws IOException {+ CopySourceRange range = new CopySourceRange(3, 1002);+ for (int readSize : List.of(1, 7, 1000, 4096)) {+ try (InputStream in = new CopyRangeInputStream(patternStream(5000), range)) {+ ByteArrayOutputStream copied = new ByteArrayOutputStream();+ byte[] buffer = new byte[readSize];+ for (int read = in.read(buffer); read >= 0; read = in.read(buffer)) {+ copied.write(buffer, 0, read);+ }+ byte[] data = copied.toByteArray();+ assertEquals(range.length(), data.length, "read size " + readSize);+ for (int i = 0; i < data.length; i++) {+ assertEquals(patternByte(range.first() + i), data[i], "byte " + i + ", read size " + readSize);+ }+ assertEquals(-1, in.read(), "the range stays at its end");+ }+ }}private static byte patternByte(long position) {diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.javaindex 02639f2979..2b4162da7d 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java@@ -17,14 +17,19 @@ import java.io.FilterInputStream;import java.io.IOException;import java.io.InputStream;import java.io.UncheckedIOException;+import java.nio.ByteBuffer;+import java.nio.channels.FileChannel;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import java.nio.file.StandardOpenOption;import java.security.MessageDigest;import java.time.Duration;+import java.util.Arrays;import java.util.Base64;import java.util.List;import java.util.Map;+import java.util.Random;import java.util.stream.Stream;import static org.junit.jupiter.api.Assertions.*;@@ -42,12 +47,14 @@ class S3StreamedUploadTest {Path tempDir;private S3Service s3Service;+ private InMemoryStorage<String, S3Object> objectStore;private Path dataRoot;@BeforeEachvoid setUp() {dataRoot = tempDir.resolve("s3");- s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);+ objectStore = new InMemoryStorage<>();+ s3Service = new S3Service(new InMemoryStorage<>(), objectStore, dataRoot, false);s3Service.createBucket("bucket", "us-east-1");}@@ -221,6 +228,82 @@ class S3StreamedUploadTest {assertNothingStaged();}+ @Test+ void anUploadPartCopyStreamsItsRangeIntoThePart() {+ byte[] source = new byte[300_000];+ new Random(5034).nextBytes(source);+ s3Service.putObject("bucket", "source.bin", source, "application/octet-stream", Map.of());+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "copy.bin", null);++ String rangeETag = s3Service.uploadPartCopy("bucket", "copy.bin", upload.getUploadId(), 1,+ "bucket", "source.bin", null, "bytes=1000-200999");+ String wholeETag = s3Service.uploadPartCopy("bucket", "copy.bin", upload.getUploadId(), 2,+ "bucket", "source.bin", null, null);++ byte[] range = Arrays.copyOfRange(source, 1000, 201_000);+ assertEquals(S3Object.computeETag(range), rangeETag);+ assertEquals(S3Object.computeETag(source), wholeETag);+ s3Service.completeMultipartUpload("bucket", "copy.bin", upload.getUploadId(), List.of(1, 2), null, null);+ byte[] whole = new byte[range.length + source.length];+ System.arraycopy(range, 0, whole, 0, range.length);+ System.arraycopy(source, 0, whole, range.length, source.length);+ assertArrayEquals(whole, s3Service.getObject("bucket", "copy.bin").getData());+ assertNothingStaged();+ }++ @Test+ void anUploadPartCopyIntoAMissingUploadFails() {+ s3Service.putObject("bucket", "source.bin", BODY, "text/plain", Map.of());++ assertEquals("NoSuchUpload", assertThrows(AwsException.class, () -> s3Service.uploadPartCopy("bucket",+ "copy.bin", "no-such-upload", 1, "bucket", "source.bin", null, "bytes=0-9")).getErrorCode());+ assertNothingStaged();+ }++ @Test+ void anUploadPartCopyOverFiveGibibytesIsEntityTooLarge() throws IOException {+ assertEquals(5_368_709_120L, S3Service.MAX_PART_SIZE);+ long overLimit = S3Service.MAX_PART_SIZE + 1;+ sparseSource("source.bin", overLimit);+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "copy.bin", null);++ for (String range : Arrays.asList(null, "bytes=0-" + S3Service.MAX_PART_SIZE)) {+ AwsException error = assertThrows(AwsException.class, () -> s3Service.uploadPartCopy("bucket",+ "copy.bin", upload.getUploadId(), 1, "bucket", "source.bin", null, range), "range " + range);++ assertEquals("EntityTooLarge", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("Your proposed upload exceeds the maximum allowed object size.", error.getMessage());+ }+ assertTrue(s3Service.getMultipartUpload("bucket", "copy.bin", upload.getUploadId()).getParts().isEmpty());+ assertNothingStaged();+ }++ @Test+ void anUploadPartCopyOfExactlyFiveGibibytesIsWithinTheLimit() throws IOException {+ sparseSource("source.bin", S3Service.MAX_PART_SIZE + 1);++ // A missing upload is only found once the size passed, and before any of the source is read.+ assertEquals("NoSuchUpload", assertThrows(AwsException.class, () -> s3Service.uploadPartCopy("bucket",+ "copy.bin", "no-such-upload", 1, "bucket", "source.bin", null,+ "bytes=1-" + S3Service.MAX_PART_SIZE)).getErrorCode());+ }++ /** A stored object of {@code size} bytes whose file is sparse, so it takes almost no disk. */+ private void sparseSource(String key, long size) throws IOException {+ s3Service.putObject("bucket", key, new byte[] {1}, "application/octet-stream", Map.of());+ objectStore.scan(storeKey -> true).stream()+ .filter(object -> "bucket".equals(object.getBucketName()) && key.equals(object.getKey()))+ .forEach(object -> object.setSize(size));+ try (Stream<Path> files = Files.walk(dataRoot)) {+ for (Path file : files.filter(path -> path.getFileName().toString().equals(key + ".s3data")).toList()) {+ try (FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE)) {+ channel.write(ByteBuffer.wrap(new byte[] {0}), size - 1);+ }+ }+ }+ }+@Testvoid memoryModeReadsTheStreamIntoTheBytePaths() {S3Service memory = memoryService();