Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): read only the copied range in UploadPartCopy, with offsets past 2 GiB (#5034)
Priamo Germosend400ea6b7e4c
diff --git a/docs/services/s3.md b/docs/services/s3.mdindex ca3a24f5d5..dbe211db07 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -395,10 +395,12 @@ multipart upload file to file, so the object never passes through the heap; set`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. While it runs,`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembledcopy, until the parts are removed. Multipart uploads in progress do not survive a restart or a-reset, and their files are removed when either happens. `CopyObject`,-`UploadPartCopy` and S3 Select still read the whole source object into memory in every mode, so-they need a source under 2 GiB, and deleting the current version of a key in a versioned bucket-reads the version that takes its place the same way.+reset, and their files are removed when either happens. `UploadPartCopy` reads only the copied+range from its source, so it can copy any part of a source of any size, but one copied part can+hold at most about 2 GiB; a larger range fails with `EntityTooLarge`. `CopyObject` and S3 Select+still read the whole source object into memory in every mode, so they need a source under 2 GiB,+and deleting the current version of a key in a versioned bucket reads the version that takes its+place the same way.## Examplesdiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 3b6795bac7..97c401eb16 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -3640,30 +3640,102 @@ public class S3Service implements Resettable, ResourceProvider {SseCustomerHeaders copySourceSseCustomerHeaders,SseCustomerHeaders sseCustomerHeaders,CopySourceConditions copySourceConditions) {- S3Object source = getObject(sourceBucket, sourceKey, sourceVersionId);- checkCopySourcePreconditions(source, copySourceConditions);- validateSseCustomerAccess(source,- copySourceSseCustomerHeaders.algorithm(),- copySourceSseCustomerHeaders.key(),- copySourceSseCustomerHeaders.keyMd5());- byte[] data = source.getData();-- if (copySourceRange != null && !copySourceRange.isBlank()) {- // format: "bytes=START-END" (inclusive on both ends)- String range = copySourceRange.startsWith("bytes=") ? copySourceRange.substring(6) : copySourceRange;- int dash = range.indexOf('-');- if (dash < 0) {- throw new AwsException("InvalidArgument", "Invalid x-amz-copy-source-range: " + copySourceRange, 400);+ byte[] data;+ // The source is streamed the way GetObject serves it and only the copied range is read, so+ // the source can be any size.+ try (ObjectRead read = openObject(sourceBucket, sourceKey, sourceVersionId)) {+ S3Object source = read.object();+ checkCopySourcePreconditions(source, copySourceConditions);+ validateSseCustomerAccess(source,+ copySourceSseCustomerHeaders.algorithm(),+ copySourceSseCustomerHeaders.key(),+ copySourceSseCustomerHeaders.keyMd5());+ CopySourceRange range = CopySourceRange.parse(copySourceRange, source.getSize());+ if (range.length() > MAX_IN_MEMORY_OBJECT_SIZE) {+ LOG.warnv("UploadPartCopy of {0} bytes into upload {1} is over the {2} bytes one part can hold in Floci; copy the source in smaller ranges",+ range.length(), uploadId, MAX_IN_MEMORY_OBJECT_SIZE);+ throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);}- int start = Integer.parseInt(range.substring(0, dash).trim());- int end = Integer.parseInt(range.substring(dash + 1).trim());- data = Arrays.copyOfRange(data, start, end + 1);+ data = readRange(read.body(), range);+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to read the copy source", e);}return uploadPart(destBucket, destKey, uploadId, partNumber, data,sseCustomerHeaders.algorithm(), sseCustomerHeaders.key(), sseCustomerHeaders.keyMd5());}+ /** An inclusive byte range of a copy source. Offsets are longs, since a source can pass 2 GiB. */+ record CopySourceRange(long first, long last) {++ private static final String HEADER = "x-amz-copy-source-range";+ private static final Pattern FORM = Pattern.compile("bytes=(\\d+)-(\\d+)");++ long length() {+ return last - first + 1;+ }++ /**+ * The range an x-amz-copy-source-range header names, or the whole source when there is no+ * header. The errors are the ones S3 returns, as recorded in LocalStack's AWS-validated+ * snapshot for UploadPartCopy: anything but one {@code bytes=first-last} range with+ * {@code first <= last}, a range that ends past the source, and a range that starts past it.+ */+ static CopySourceRange parse(String header, long sourceSize) {+ if (header == null || header.isBlank()) {+ return new CopySourceRange(0, sourceSize - 1);+ }+ Matcher matcher = FORM.matcher(header);+ if (!matcher.matches()) {+ throw malformed(header);+ }+ long first;+ long last;+ try {+ first = Long.parseLong(matcher.group(1));+ last = Long.parseLong(matcher.group(2));+ } catch (NumberFormatException e) {+ throw malformed(header);+ }+ if (first > last) {+ throw malformed(header);+ }+ // A range whose first byte does not exist starts past the source, the case S3 answers+ // with InvalidRequest; one that only runs over the end names the source size instead.+ if (first >= sourceSize) {+ throw new AwsException("InvalidRequest",+ "The specified copy range is invalid for the source object size", 400);+ }+ if (last >= sourceSize) {+ throw new AwsException("InvalidArgument",+ "Range specified is not valid for source object of size: " + sourceSize, 400,+ argument(header));+ }+ return new CopySourceRange(first, last);+ }++ private static AwsException malformed(String header) {+ return new AwsException("InvalidArgument", "The x-amz-copy-source-range value must be of the form"+ + " bytes=first-last where first and last are the zero-based offsets of the first and last"+ + " bytes to copy", 400, argument(header));+ }++ private static Map<String, Object> argument(String header) {+ return Map.of("ArgumentName", HEADER, "ArgumentValue", header);+ }+ }++ /** Reads exactly the bytes of {@code range} from {@code body}, skipping the bytes before it. */+ static byte[] readRange(InputStream body, CopySourceRange range) throws IOException {+ body.skipNBytes(range.first());+ byte[] data = body.readNBytes((int) range.length());+ if (data.length != range.length()) {+ throw new IOException("Copy source ended " + (range.length() - data.length)+ + " bytes before the end of the requested range");+ }+ return data;+ }+public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List<Integer> partNumbers,String checksumType, S3Checksum expectedChecksum) {return completeMultipartUpload(bucket, key, uploadId, partNumbers, Map.of(), Map.of(), checksumType,diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.javanew file mode 100644index 0000000000..139eaa717b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.java@@ -0,0 +1,155 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.s3.S3Service.CopySourceRange;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.IOException;+import java.io.InputStream;+import java.nio.ByteBuffer;+import java.nio.channels.Channels;+import java.nio.channels.FileChannel;+import java.nio.file.Path;+import java.nio.file.StandardOpenOption;+import java.util.List;++import static org.junit.jupiter.api.Assertions.*;++class S3CopySourceRangeTest {++ @TempDir+ Path tempDir;++ @Test+ void noHeaderCopiesTheWholeSource() {+ assertEquals(new CopySourceRange(0, 9), CopySourceRange.parse(null, 10));+ assertEquals(new CopySourceRange(0, 9), CopySourceRange.parse(" ", 10));+ assertEquals(0, CopySourceRange.parse(null, 0).length(), "an empty source copies as an empty part");+ }++ @Test+ void offsetsPastTwoGibibytesParseAsLongs() {+ CopySourceRange range = CopySourceRange.parse("bytes=3000000000-3000000099", 4_000_000_000L);++ assertEquals(3_000_000_000L, range.first());+ assertEquals(3_000_000_099L, range.last());+ assertEquals(100, range.length());+ }++ @Test+ void rangeMayEndOnTheLastByteOfTheSource() {+ assertEquals(new CopySourceRange(2, 9), CopySourceRange.parse("bytes=2-9", 10));+ assertEquals(new CopySourceRange(0, 0), CopySourceRange.parse("bytes=0-0", 10));+ }++ // The rejected ranges and their errors are the ones LocalStack's AWS-validated snapshot records+ // for UploadPartCopy against a 10-byte source (test_s3_api.snapshot.json, test_upload_part_copy_range).++ @Test+ void anythingButOneBytesFirstLastRangeIsRejectedWithTheFormError() {+ for (String header : List.of("0-8", "bytes=1-0", "bytes=-1-", "bytes=0--1", "bytes=0-1,3-4,7-9",+ "bytes=-", "bytes=-0", "bytes=1-", "bytes=-2", "bytes=-15")) {+ AwsException error = assertThrows(AwsException.class, () -> CopySourceRange.parse(header, 10), header);+ assertEquals("InvalidArgument", error.getErrorCode(), header);+ assertEquals(400, error.getHttpStatus(), header);+ assertEquals("The x-amz-copy-source-range value must be of the form bytes=first-last where first"+ + " and last are the zero-based offsets of the first and last bytes to copy", error.getMessage(), header);+ assertEquals("x-amz-copy-source-range", error.getExtendedData().get("ArgumentName"), header);+ assertEquals(header, error.getExtendedData().get("ArgumentValue"), header);+ }+ }++ @Test+ void rangeEndingPastTheSourceNamesTheSourceSize() {+ AwsException error = assertThrows(AwsException.class, () -> CopySourceRange.parse("bytes=0-100", 10));++ assertEquals("InvalidArgument", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("Range specified is not valid for source object of size: 10", error.getMessage());+ assertEquals("x-amz-copy-source-range", error.getExtendedData().get("ArgumentName"));+ assertEquals("bytes=0-100", error.getExtendedData().get("ArgumentValue"));+ }++ @Test+ void rangeStartingPastTheSourceIsAnInvalidRequest() {+ AwsException error = assertThrows(AwsException.class, () -> CopySourceRange.parse("bytes=100-200", 10));++ assertEquals("InvalidRequest", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("The specified copy range is invalid for the source object size", error.getMessage());+ assertTrue(error.getExtendedData() == null || error.getExtendedData().isEmpty(), "no argument fields");+ }++ @Test+ void rangeStartingOnTheByteAfterTheSourceAlsoStartsPastIt() {+ // Not in the snapshot: a first byte that does not exist is treated as starting past the source.+ assertEquals("InvalidRequest", assertThrows(AwsException.class,+ () -> CopySourceRange.parse("bytes=10-12", 10)).getErrorCode());+ assertEquals("InvalidRequest", assertThrows(AwsException.class,+ () -> CopySourceRange.parse("bytes=0-0", 0)).getErrorCode(), "an empty source has no first byte");+ }++ @Test+ void readRangeSkipsToAnOffsetPastTwoGibibytes() throws IOException {+ CopySourceRange range = new CopySourceRange(2_500_000_000L, 2_500_000_015L);++ byte[] data = S3Service.readRange(patternStream(3L * 1024 * 1024 * 1024), range);++ assertEquals(16, data.length);+ for (int i = 0; i < data.length; i++) {+ assertEquals(patternByte(range.first() + i), data[i], "byte " + i);+ }+ }++ @Test+ void readRangeSkipsPastTwoGibibytesInTheFileStreamDiskObjectsAreReadThrough() throws IOException {+ long offset = 2_500_000_000L;+ byte[] written = new byte[16];+ for (int i = 0; i < written.length; i++) {+ written[i] = (byte) (i + 1);+ }+ // Sparse: only the bytes past 2.5 GB are written, so the file takes a single block on disk.+ Path file = tempDir.resolve("sparse-source");+ try (FileChannel channel = FileChannel.open(file, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE,+ StandardOpenOption.SPARSE)) {+ channel.write(ByteBuffer.wrap(written), offset);+ }++ byte[] data;+ try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {+ data = S3Service.readRange(Channels.newInputStream(channel),+ new CopySourceRange(offset, offset + written.length - 1));+ }++ assertArrayEquals(written, data);+ }++ @Test+ void readRangeFailsWhenTheSourceEndsBeforeTheRange() {+ assertThrows(IOException.class, () -> S3Service.readRange(patternStream(10), new CopySourceRange(5, 14)));+ }++ private static byte patternByte(long position) {+ return (byte) (position % 251);+ }++ /** {@code size} bytes where byte {@code p} is {@code p % 251}, skipped over without being produced. */+ private static InputStream patternStream(long size) {+ return new InputStream() {+ private long position;++ @Override+ public int read() {+ return position < size ? patternByte(position++) & 0xFF : -1;+ }++ @Override+ public long skip(long n) {+ long skipped = Math.max(0, Math.min(n, size - position));+ position += skipped;+ return skipped;+ }+ };+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.javaindex 5ed89211b1..a7505e36f5 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java@@ -245,6 +245,23 @@ class S3MultipartDiskAssemblyTest {assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "raced.bin"));}+ @Test+ void uploadPartCopyReadsTheRequestedRangeFromTheSourceFile() {+ byte[] source = new byte[200_000];+ for (int i = 0; i < source.length; i++) {+ source[i] = (byte) (i % 251);+ }+ s3Service.putObject("test-bucket", "source.bin", source, "application/octet-stream", Map.of());+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "copy.bin", null);++ s3Service.uploadPartCopy("test-bucket", "copy.bin", upload.getUploadId(), 1,+ "test-bucket", "source.bin", null, "bytes=150000-150099");+ s3Service.completeMultipartUpload("test-bucket", "copy.bin", upload.getUploadId(), List.of(1), null, null);++ assertArrayEquals(Arrays.copyOfRange(source, 150_000, 150_100),+ s3Service.getObject("test-bucket", "copy.bin").getData());+ }+@Testvoid restartRemovesTheFilesOfUploadsThatWereInProgress() throws Exception {s3Service.putObject("test-bucket", "kept.bin", "kept".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.javaindex 5fcecba0bb..cfbd344a7d 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java@@ -1352,6 +1352,64 @@ class S3MultipartIntegrationTest {.statusCode(204);}+ @Test+ @Order(38)+ void uploadPartCopyAnswersBadRangesTheWayS3Does() {+ given()+ .body("ABCDEFGHIJ")+ .when()+ .put("/" + BUCKET + "/range-source.bin")+ .then()+ .statusCode(200);+ String uploadId = given()+ .when()+ .post("/" + BUCKET + "/range-dest.bin?uploads")+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");++ given()+ .header("x-amz-copy-source", "/" + BUCKET + "/range-source.bin")+ .header("x-amz-copy-source-range", "2-5")+ .when()+ .put("/" + BUCKET + "/range-dest.bin?uploadId=" + uploadId + "&partNumber=1")+ .then()+ .statusCode(400)+ .body("Error.Code", equalTo("InvalidArgument"))+ .body("Error.Message", startsWith("The x-amz-copy-source-range value must be of the form bytes=first-last"))+ .body("Error.ArgumentName", equalTo("x-amz-copy-source-range"))+ .body("Error.ArgumentValue", equalTo("2-5"));++ given()+ .header("x-amz-copy-source", "/" + BUCKET + "/range-source.bin")+ .header("x-amz-copy-source-range", "bytes=0-100")+ .when()+ .put("/" + BUCKET + "/range-dest.bin?uploadId=" + uploadId + "&partNumber=1")+ .then()+ .statusCode(400)+ .body("Error.Code", equalTo("InvalidArgument"))+ .body("Error.Message", equalTo("Range specified is not valid for source object of size: 10"))+ .body("Error.ArgumentName", equalTo("x-amz-copy-source-range"))+ .body("Error.ArgumentValue", equalTo("bytes=0-100"));++ given()+ .header("x-amz-copy-source", "/" + BUCKET + "/range-source.bin")+ .header("x-amz-copy-source-range", "bytes=100-200")+ .when()+ .put("/" + BUCKET + "/range-dest.bin?uploadId=" + uploadId + "&partNumber=1")+ .then()+ .statusCode(400)+ .body("Error.Code", equalTo("InvalidRequest"))+ .body("Error.Message", equalTo("The specified copy range is invalid for the source object size"))+ .body(not(containsString("<ArgumentName>")));++ given()+ .when()+ .delete("/" + BUCKET + "/range-dest.bin?uploadId=" + uploadId)+ .then()+ .statusCode(204);+ }+@Test@Order(40)void cleanUp() {@@ -1369,6 +1427,7 @@ class S3MultipartIntegrationTest {given().when().delete("/" + BUCKET + "/checksum-match-multipart.bin").then().statusCode(204);given().when().delete("/" + BUCKET + "/checksum-type-multipart.bin").then().statusCode(204);given().when().delete("/" + BUCKET + "/completion-validation.bin").then().statusCode(204);+ given().when().delete("/" + BUCKET + "/range-source.bin").then().statusCode(204);given().when().delete("/" + BUCKET).then().statusCode(204);}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.javaindex 7fb024e76a..d325bb8030 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java@@ -607,6 +607,33 @@ class S3MultipartServiceTest {assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "raced.bin"));}+ @Test+ void uploadPartCopyCopiesOnlyTheRequestedRange() {+ s3Service.putObject("test-bucket", "source.bin", "ABCDEFGHIJ".getBytes(StandardCharsets.UTF_8),+ "application/octet-stream", Map.of());+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "copy.bin", null);++ s3Service.uploadPartCopy("test-bucket", "copy.bin", upload.getUploadId(), 1,+ "test-bucket", "source.bin", null, "bytes=2-5");+ s3Service.completeMultipartUpload("test-bucket", "copy.bin", upload.getUploadId(), List.of(1), null, null);++ assertArrayEquals("CDEF".getBytes(StandardCharsets.UTF_8), s3Service.getObject("test-bucket", "copy.bin").getData());+ }++ @Test+ void uploadPartCopyRejectsARangePastTheEndOfTheSource() {+ s3Service.putObject("test-bucket", "source.bin", "ABCDEFGHIJ".getBytes(StandardCharsets.UTF_8),+ "application/octet-stream", Map.of());+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "copy.bin", null);++ AwsException error = assertThrows(AwsException.class, () -> s3Service.uploadPartCopy("test-bucket", "copy.bin",+ upload.getUploadId(), 1, "test-bucket", "source.bin", null, "bytes=5-10"));++ assertEquals("InvalidArgument", error.getErrorCode());+ assertEquals("Range specified is not valid for source object of size: 10", error.getMessage());+ assertNull(upload.getParts().get(1), "no part is recorded for a rejected range");+ }+private static Map<Integer, S3Checksum> partChecksums(ChecksumAlgorithm algorithm, byte[]... parts) {Map<Integer, S3Checksum> checksums = new HashMap<>();for (int i = 0; i < parts.length; i++) {