Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): read only the copied range in UploadPartCopy, with offsets past 2 GiB (#5034)

Priamo Germosend400ea6b7e4c
diff --git a/docs/services/s3.md b/docs/services/s3.md
index ca3a24f5d5..dbe211db07 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -395,10 +395,12 @@ multipart upload file to file, so the object never passes through the heap; set
`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. While it runs,
`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembled
copy, until the parts are removed. Multipart uploads in progress do not survive a restart or a
-reset, and their files are removed when either happens. `CopyObject`,
-`UploadPartCopy` and S3 Select still read the whole source object into memory in every mode, so
-they need a source under 2 GiB, and deleting the current version of a key in a versioned bucket
-reads the version that takes its place the same way.
+reset, and their files are removed when either happens. `UploadPartCopy` reads only the copied
+range from its source, so it can copy any part of a source of any size, but one copied part can
+hold at most about 2 GiB; a larger range fails with `EntityTooLarge`. `CopyObject` and S3 Select
+still read the whole source object into memory in every mode, so they need a source under 2 GiB,
+and deleting the current version of a key in a versioned bucket reads the version that takes its
+place the same way.
## Examples
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 3b6795bac7..97c401eb16 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -3640,30 +3640,102 @@ public class S3Service implements Resettable, ResourceProvider {
SseCustomerHeaders copySourceSseCustomerHeaders,
SseCustomerHeaders sseCustomerHeaders,
CopySourceConditions copySourceConditions) {
- S3Object source = getObject(sourceBucket, sourceKey, sourceVersionId);
- checkCopySourcePreconditions(source, copySourceConditions);
- validateSseCustomerAccess(source,
- copySourceSseCustomerHeaders.algorithm(),
- copySourceSseCustomerHeaders.key(),
- copySourceSseCustomerHeaders.keyMd5());
- byte[] data = source.getData();
-
- if (copySourceRange != null && !copySourceRange.isBlank()) {
- // format: "bytes=START-END" (inclusive on both ends)
- String range = copySourceRange.startsWith("bytes=") ? copySourceRange.substring(6) : copySourceRange;
- int dash = range.indexOf('-');
- if (dash < 0) {
- throw new AwsException("InvalidArgument", "Invalid x-amz-copy-source-range: " + copySourceRange, 400);
+ byte[] data;
+ // The source is streamed the way GetObject serves it and only the copied range is read, so
+ // the source can be any size.
+ try (ObjectRead read = openObject(sourceBucket, sourceKey, sourceVersionId)) {
+ S3Object source = read.object();
+ checkCopySourcePreconditions(source, copySourceConditions);
+ validateSseCustomerAccess(source,
+ copySourceSseCustomerHeaders.algorithm(),
+ copySourceSseCustomerHeaders.key(),
+ copySourceSseCustomerHeaders.keyMd5());
+ CopySourceRange range = CopySourceRange.parse(copySourceRange, source.getSize());
+ if (range.length() > MAX_IN_MEMORY_OBJECT_SIZE) {
+ LOG.warnv("UploadPartCopy of {0} bytes into upload {1} is over the {2} bytes one part can hold in Floci; copy the source in smaller ranges",
+ range.length(), uploadId, MAX_IN_MEMORY_OBJECT_SIZE);
+ throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);
}
- int start = Integer.parseInt(range.substring(0, dash).trim());
- int end = Integer.parseInt(range.substring(dash + 1).trim());
- data = Arrays.copyOfRange(data, start, end + 1);
+ data = readRange(read.body(), range);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read the copy source", e);
}
return uploadPart(destBucket, destKey, uploadId, partNumber, data,
sseCustomerHeaders.algorithm(), sseCustomerHeaders.key(), sseCustomerHeaders.keyMd5());
}
+ /** An inclusive byte range of a copy source. Offsets are longs, since a source can pass 2 GiB. */
+ record CopySourceRange(long first, long last) {
+
+ private static final String HEADER = "x-amz-copy-source-range";
+ private static final Pattern FORM = Pattern.compile("bytes=(\\d+)-(\\d+)");
+
+ long length() {
+ return last - first + 1;
+ }
+
+ /**
+ * The range an x-amz-copy-source-range header names, or the whole source when there is no
+ * header. The errors are the ones S3 returns, as recorded in LocalStack's AWS-validated
+ * snapshot for UploadPartCopy: anything but one {@code bytes=first-last} range with
+ * {@code first <= last}, a range that ends past the source, and a range that starts past it.
+ */
+ static CopySourceRange parse(String header, long sourceSize) {
+ if (header == null || header.isBlank()) {
+ return new CopySourceRange(0, sourceSize - 1);
+ }
+ Matcher matcher = FORM.matcher(header);
+ if (!matcher.matches()) {
+ throw malformed(header);
+ }
+ long first;
+ long last;
+ try {
+ first = Long.parseLong(matcher.group(1));
+ last = Long.parseLong(matcher.group(2));
+ } catch (NumberFormatException e) {
+ throw malformed(header);
+ }
+ if (first > last) {
+ throw malformed(header);
+ }
+ // A range whose first byte does not exist starts past the source, the case S3 answers
+ // with InvalidRequest; one that only runs over the end names the source size instead.
+ if (first >= sourceSize) {
+ throw new AwsException("InvalidRequest",
+ "The specified copy range is invalid for the source object size", 400);
+ }
+ if (last >= sourceSize) {
+ throw new AwsException("InvalidArgument",
+ "Range specified is not valid for source object of size: " + sourceSize, 400,
+ argument(header));
+ }
+ return new CopySourceRange(first, last);
+ }
+
+ private static AwsException malformed(String header) {
+ return new AwsException("InvalidArgument", "The x-amz-copy-source-range value must be of the form"
+ + " bytes=first-last where first and last are the zero-based offsets of the first and last"
+ + " bytes to copy", 400, argument(header));
+ }
+
+ private static Map<String, Object> argument(String header) {
+ return Map.of("ArgumentName", HEADER, "ArgumentValue", header);
+ }
+ }
+
+ /** Reads exactly the bytes of {@code range} from {@code body}, skipping the bytes before it. */
+ static byte[] readRange(InputStream body, CopySourceRange range) throws IOException {
+ body.skipNBytes(range.first());
+ byte[] data = body.readNBytes((int) range.length());
+ if (data.length != range.length()) {
+ throw new IOException("Copy source ended " + (range.length() - data.length)
+ + " bytes before the end of the requested range");
+ }
+ return data;
+ }
+
public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List<Integer> partNumbers,
String checksumType, S3Checksum expectedChecksum) {
return completeMultipartUpload(bucket, key, uploadId, partNumbers, Map.of(), Map.of(), checksumType,
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.java
new file mode 100644
index 0000000000..139eaa717b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopySourceRangeTest.java
@@ -0,0 +1,155 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.s3.S3Service.CopySourceRange;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class S3CopySourceRangeTest {
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ void noHeaderCopiesTheWholeSource() {
+ assertEquals(new CopySourceRange(0, 9), CopySourceRange.parse(null, 10));
+ assertEquals(new CopySourceRange(0, 9), CopySourceRange.parse(" ", 10));
+ assertEquals(0, CopySourceRange.parse(null, 0).length(), "an empty source copies as an empty part");
+ }
+
+ @Test
+ void offsetsPastTwoGibibytesParseAsLongs() {
+ CopySourceRange range = CopySourceRange.parse("bytes=3000000000-3000000099", 4_000_000_000L);
+
+ assertEquals(3_000_000_000L, range.first());
+ assertEquals(3_000_000_099L, range.last());
+ assertEquals(100, range.length());
+ }
+
+ @Test
+ void rangeMayEndOnTheLastByteOfTheSource() {
+ assertEquals(new CopySourceRange(2, 9), CopySourceRange.parse("bytes=2-9", 10));
+ assertEquals(new CopySourceRange(0, 0), CopySourceRange.parse("bytes=0-0", 10));
+ }
+
+ // The rejected ranges and their errors are the ones LocalStack's AWS-validated snapshot records
+ // for UploadPartCopy against a 10-byte source (test_s3_api.snapshot.json, test_upload_part_copy_range).
+
+ @Test
+ void anythingButOneBytesFirstLastRangeIsRejectedWithTheFormError() {
+ for (String header : List.of("0-8", "bytes=1-0", "bytes=-1-", "bytes=0--1", "bytes=0-1,3-4,7-9",
+ "bytes=-", "bytes=-0", "bytes=1-", "bytes=-2", "bytes=-15")) {
+ AwsException error = assertThrows(AwsException.class, () -> CopySourceRange.parse(header, 10), header);
+ assertEquals("InvalidArgument", error.getErrorCode(), header);
+ assertEquals(400, error.getHttpStatus(), header);
+ assertEquals("The x-amz-copy-source-range value must be of the form bytes=first-last where first"
+ + " and last are the zero-based offsets of the first and last bytes to copy", error.getMessage(), header);
+ assertEquals("x-amz-copy-source-range", error.getExtendedData().get("ArgumentName"), header);
+ assertEquals(header, error.getExtendedData().get("ArgumentValue"), header);
+ }
+ }
+
+ @Test
+ void rangeEndingPastTheSourceNamesTheSourceSize() {
+ AwsException error = assertThrows(AwsException.class, () -> CopySourceRange.parse("bytes=0-100", 10));
+
+ assertEquals("InvalidArgument", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("Range specified is not valid for source object of size: 10", error.getMessage());
+ assertEquals("x-amz-copy-source-range", error.getExtendedData().get("ArgumentName"));
+ assertEquals("bytes=0-100", error.getExtendedData().get("ArgumentValue"));
+ }
+
+ @Test
+ void rangeStartingPastTheSourceIsAnInvalidRequest() {
+ AwsException error = assertThrows(AwsException.class, () -> CopySourceRange.parse("bytes=100-200", 10));
+
+ assertEquals("InvalidRequest", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("The specified copy range is invalid for the source object size", error.getMessage());
+ assertTrue(error.getExtendedData() == null || error.getExtendedData().isEmpty(), "no argument fields");
+ }
+
+ @Test
+ void rangeStartingOnTheByteAfterTheSourceAlsoStartsPastIt() {
+ // Not in the snapshot: a first byte that does not exist is treated as starting past the source.
+ assertEquals("InvalidRequest", assertThrows(AwsException.class,
+ () -> CopySourceRange.parse("bytes=10-12", 10)).getErrorCode());
+ assertEquals("InvalidRequest", assertThrows(AwsException.class,
+ () -> CopySourceRange.parse("bytes=0-0", 0)).getErrorCode(), "an empty source has no first byte");
+ }
+
+ @Test
+ void readRangeSkipsToAnOffsetPastTwoGibibytes() throws IOException {
+ CopySourceRange range = new CopySourceRange(2_500_000_000L, 2_500_000_015L);
+
+ byte[] data = S3Service.readRange(patternStream(3L * 1024 * 1024 * 1024), range);
+
+ assertEquals(16, data.length);
+ for (int i = 0; i < data.length; i++) {
+ assertEquals(patternByte(range.first() + i), data[i], "byte " + i);
+ }
+ }
+
+ @Test
+ void readRangeSkipsPastTwoGibibytesInTheFileStreamDiskObjectsAreReadThrough() throws IOException {
+ long offset = 2_500_000_000L;
+ byte[] written = new byte[16];
+ for (int i = 0; i < written.length; i++) {
+ written[i] = (byte) (i + 1);
+ }
+ // Sparse: only the bytes past 2.5 GB are written, so the file takes a single block on disk.
+ Path file = tempDir.resolve("sparse-source");
+ try (FileChannel channel = FileChannel.open(file, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE,
+ StandardOpenOption.SPARSE)) {
+ channel.write(ByteBuffer.wrap(written), offset);
+ }
+
+ byte[] data;
+ try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {
+ data = S3Service.readRange(Channels.newInputStream(channel),
+ new CopySourceRange(offset, offset + written.length - 1));
+ }
+
+ assertArrayEquals(written, data);
+ }
+
+ @Test
+ void readRangeFailsWhenTheSourceEndsBeforeTheRange() {
+ assertThrows(IOException.class, () -> S3Service.readRange(patternStream(10), new CopySourceRange(5, 14)));
+ }
+
+ private static byte patternByte(long position) {
+ return (byte) (position % 251);
+ }
+
+ /** {@code size} bytes where byte {@code p} is {@code p % 251}, skipped over without being produced. */
+ private static InputStream patternStream(long size) {
+ return new InputStream() {
+ private long position;
+
+ @Override
+ public int read() {
+ return position < size ? patternByte(position++) & 0xFF : -1;
+ }
+
+ @Override
+ public long skip(long n) {
+ long skipped = Math.max(0, Math.min(n, size - position));
+ position += skipped;
+ return skipped;
+ }
+ };
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
index 5ed89211b1..a7505e36f5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
@@ -245,6 +245,23 @@ class S3MultipartDiskAssemblyTest {
assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "raced.bin"));
}
+ @Test
+ void uploadPartCopyReadsTheRequestedRangeFromTheSourceFile() {
+ byte[] source = new byte[200_000];
+ for (int i = 0; i < source.length; i++) {
+ source[i] = (byte) (i % 251);
+ }
+ s3Service.putObject("test-bucket", "source.bin", source, "application/octet-stream", Map.of());
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "copy.bin", null);
+
+ s3Service.uploadPartCopy("test-bucket", "copy.bin", upload.getUploadId(), 1,
+ "test-bucket", "source.bin", null, "bytes=150000-150099");
+ s3Service.completeMultipartUpload("test-bucket", "copy.bin", upload.getUploadId(), List.of(1), null, null);
+
+ assertArrayEquals(Arrays.copyOfRange(source, 150_000, 150_100),
+ s3Service.getObject("test-bucket", "copy.bin").getData());
+ }
+
@Test
void restartRemovesTheFilesOfUploadsThatWereInProgress() throws Exception {
s3Service.putObject("test-bucket", "kept.bin", "kept".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
index 5fcecba0bb..cfbd344a7d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
@@ -1352,6 +1352,64 @@ class S3MultipartIntegrationTest {
.statusCode(204);
}
+ @Test
+ @Order(38)
+ void uploadPartCopyAnswersBadRangesTheWayS3Does() {
+ given()
+ .body("ABCDEFGHIJ")
+ .when()
+ .put("/" + BUCKET + "/range-source.bin")
+ .then()
+ .statusCode(200);
+ String uploadId = given()
+ .when()
+ .post("/" + BUCKET + "/range-dest.bin?uploads")
+ .then()
+ .statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+
+ given()
+ .header("x-amz-copy-source", "/" + BUCKET + "/range-source.bin")
+ .header("x-amz-copy-source-range", "2-5")
+ .when()
+ .put("/" + BUCKET + "/range-dest.bin?uploadId=" + uploadId + "&partNumber=1")
+ .then()
+ .statusCode(400)
+ .body("Error.Code", equalTo("InvalidArgument"))
+ .body("Error.Message", startsWith("The x-amz-copy-source-range value must be of the form bytes=first-last"))
+ .body("Error.ArgumentName", equalTo("x-amz-copy-source-range"))
+ .body("Error.ArgumentValue", equalTo("2-5"));
+
+ given()
+ .header("x-amz-copy-source", "/" + BUCKET + "/range-source.bin")
+ .header("x-amz-copy-source-range", "bytes=0-100")
+ .when()
+ .put("/" + BUCKET + "/range-dest.bin?uploadId=" + uploadId + "&partNumber=1")
+ .then()
+ .statusCode(400)
+ .body("Error.Code", equalTo("InvalidArgument"))
+ .body("Error.Message", equalTo("Range specified is not valid for source object of size: 10"))
+ .body("Error.ArgumentName", equalTo("x-amz-copy-source-range"))
+ .body("Error.ArgumentValue", equalTo("bytes=0-100"));
+
+ given()
+ .header("x-amz-copy-source", "/" + BUCKET + "/range-source.bin")
+ .header("x-amz-copy-source-range", "bytes=100-200")
+ .when()
+ .put("/" + BUCKET + "/range-dest.bin?uploadId=" + uploadId + "&partNumber=1")
+ .then()
+ .statusCode(400)
+ .body("Error.Code", equalTo("InvalidRequest"))
+ .body("Error.Message", equalTo("The specified copy range is invalid for the source object size"))
+ .body(not(containsString("<ArgumentName>")));
+
+ given()
+ .when()
+ .delete("/" + BUCKET + "/range-dest.bin?uploadId=" + uploadId)
+ .then()
+ .statusCode(204);
+ }
+
@Test
@Order(40)
void cleanUp() {
@@ -1369,6 +1427,7 @@ class S3MultipartIntegrationTest {
given().when().delete("/" + BUCKET + "/checksum-match-multipart.bin").then().statusCode(204);
given().when().delete("/" + BUCKET + "/checksum-type-multipart.bin").then().statusCode(204);
given().when().delete("/" + BUCKET + "/completion-validation.bin").then().statusCode(204);
+ given().when().delete("/" + BUCKET + "/range-source.bin").then().statusCode(204);
given().when().delete("/" + BUCKET).then().statusCode(204);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
index 7fb024e76a..d325bb8030 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
@@ -607,6 +607,33 @@ class S3MultipartServiceTest {
assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "raced.bin"));
}
+ @Test
+ void uploadPartCopyCopiesOnlyTheRequestedRange() {
+ s3Service.putObject("test-bucket", "source.bin", "ABCDEFGHIJ".getBytes(StandardCharsets.UTF_8),
+ "application/octet-stream", Map.of());
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "copy.bin", null);
+
+ s3Service.uploadPartCopy("test-bucket", "copy.bin", upload.getUploadId(), 1,
+ "test-bucket", "source.bin", null, "bytes=2-5");
+ s3Service.completeMultipartUpload("test-bucket", "copy.bin", upload.getUploadId(), List.of(1), null, null);
+
+ assertArrayEquals("CDEF".getBytes(StandardCharsets.UTF_8), s3Service.getObject("test-bucket", "copy.bin").getData());
+ }
+
+ @Test
+ void uploadPartCopyRejectsARangePastTheEndOfTheSource() {
+ s3Service.putObject("test-bucket", "source.bin", "ABCDEFGHIJ".getBytes(StandardCharsets.UTF_8),
+ "application/octet-stream", Map.of());
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "copy.bin", null);
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.uploadPartCopy("test-bucket", "copy.bin",
+ upload.getUploadId(), 1, "test-bucket", "source.bin", null, "bytes=5-10"));
+
+ assertEquals("InvalidArgument", error.getErrorCode());
+ assertEquals("Range specified is not valid for source object of size: 10", error.getMessage());
+ assertNull(upload.getParts().get(1), "no part is recorded for a rejected range");
+ }
+
private static Map<Integer, S3Checksum> partChecksums(ChecksumAlgorithm algorithm, byte[]... parts) {
Map<Integer, S3Checksum> checksums = new HashMap<>();
for (int i = 0; i < parts.length; i++) {