Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): copy objects over 2 GiB without reading them into memory (#5080)

Priamo Germosend97b21ae62b4
diff --git a/docs/services/s3.md b/docs/services/s3.md
index 532b634d9e..cd7626b306 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -397,8 +397,12 @@ multipart upload file to file, so the object never passes through the heap; set
copy, until the parts are removed. Multipart uploads in progress do not survive a restart or a
reset, and their files are removed when either happens. `UploadPartCopy` reads only the copied
range from its source, so it can copy any part of a source of any size, but one copied part can
-hold at most about 2 GiB; a larger range fails with `EntityTooLarge`. `CopyObject` and S3 Select
-still read the whole source object into memory in every mode, so they need a source under 2 GiB.
+hold at most about 2 GiB; a larger range fails with `EntityTooLarge`. In the disk-backed modes
+`CopyObject` stores the source's file for the copy without reading it into memory, and reads it
+once only when the copy needs an ETag or checksum the source's cannot supply. As on S3, a
+`CopyObject` source larger than 5 GiB is rejected with `InvalidRequest`; copy it in parts with
+`UploadPartCopy` instead. S3 Select still reads the whole source object into memory in every
+mode, so it needs a source under 2 GiB.
## Examples
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 9dc923544e..d51bf6156b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -697,8 +697,9 @@ public class S3Service implements Resettable, ResourceProvider {
}
/**
- * A disk-mode multipart object. It is stored with the ETag and checksum built from its parts,
- * since computing either here would mean reading the whole file.
+ * A body already on disk that is moved into place: an assembled multipart object, or the pinned
+ * file of a copy source. It is stored with an ETag and checksum worked out beforehand, since
+ * computing either here would mean reading the whole file under the bucket lock.
*/
private record AssembledBody(Path file, long size) implements ObjectBody { }
@@ -2139,26 +2140,166 @@ public class S3Service implements Resettable, ResourceProvider {
String destBucket, String destKey, String versionId, CopyObjectOptions options)
{
CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();
- S3Object source = getObject(sourceBucket, sourceKey, versionId);
- checkCopySourcePreconditions(source, effectiveOptions.getCopySourceConditions());
- validateSseCustomerAccess(source,
- effectiveOptions.getCopySourceSseCustomerAlgorithm(),
- effectiveOptions.getCopySourceSseCustomerKey(),
- effectiveOptions.getCopySourceSseCustomerKeyMd5());
- return copyS3Object(sourceBucket, sourceKey,
- destBucket, destKey, source, effectiveOptions);
+ return copyPinnedSource(sourceBucket, sourceKey, destBucket, destKey,
+ pinCopySource(sourceBucket, sourceKey, versionId), effectiveOptions);
}
public S3Object copyObject(String sourceBucket, String sourceKey,
String destBucket, String destKey, CopyObjectOptions options) {
CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();
- S3Object source = getObject(sourceBucket, sourceKey);
- checkCopySourcePreconditions(source, effectiveOptions.getCopySourceConditions());
- validateSseCustomerAccess(source,
- effectiveOptions.getCopySourceSseCustomerAlgorithm(),
- effectiveOptions.getCopySourceSseCustomerKey(),
- effectiveOptions.getCopySourceSseCustomerKeyMd5());
- return copyS3Object(sourceBucket, sourceKey, destBucket, destKey, source, effectiveOptions);
+ return copyPinnedSource(sourceBucket, sourceKey, destBucket, destKey,
+ pinCopySource(sourceBucket, sourceKey, null), effectiveOptions);
+ }
+
+ private S3Object copyPinnedSource(String sourceBucket, String sourceKey, String destBucket, String destKey,
+ CopySource source, CopyObjectOptions effectiveOptions) {
+ try {
+ checkCopySourcePreconditions(source.object(), effectiveOptions.getCopySourceConditions());
+ validateSseCustomerAccess(source.object(),
+ effectiveOptions.getCopySourceSseCustomerAlgorithm(),
+ effectiveOptions.getCopySourceSseCustomerKey(),
+ effectiveOptions.getCopySourceSseCustomerKeyMd5());
+ requireCopyableSize(source.object());
+ return copyS3Object(sourceBucket, sourceKey, destBucket, destKey, source, effectiveOptions);
+ } finally {
+ // A stored copy moved its pinned file into place, so this only removes one a failed
+ // copy left behind.
+ if (source.pinnedFile() != null) {
+ deleteQuietly(source.pinnedFile(), "pinned copy source that was not stored");
+ }
+ }
+ }
+
+ // S3 copies at most 5 GiB in one CopyObject; a larger object is copied with UploadPartCopy.
+ static final long MAX_COPY_OBJECT_SOURCE_SIZE = 5L * 1024 * 1024 * 1024;
+
+ static void requireCopyableSize(S3Object source) {
+ if (source.getSize() > MAX_COPY_OBJECT_SOURCE_SIZE) {
+ throw new AwsException("InvalidRequest",
+ "The specified copy source is larger than the maximum allowable size for a copy source: "
+ + MAX_COPY_OBJECT_SOURCE_SIZE, 400);
+ }
+ }
+
+ /**
+ * The source of a copy, held for the length of the copy: its metadata and, in disk modes, a
+ * temporary hard link to the file generation that metadata describes. The link keeps those
+ * bytes even if the source is overwritten meanwhile, since an object file is only ever replaced
+ * by a rename, and the copy can store the file without reading it into memory. In memory mode
+ * the bytes come with the object, as before.
+ */
+ private record CopySource(S3Object object, Path pinnedFile) { }
+
+ /** A copy's body, with the ETag and checksum to store it under (null to compute them from bytes). */
+ private record CopyBody(ObjectBody body, String eTag, S3Checksum checksum) { }
+
+ private static final Pattern SINGLE_PART_ETAG = Pattern.compile("[0-9a-f]{32}");
+
+ private CopySource pinCopySource(String bucketName, String key, String versionId) {
+ // The size limit is checked on the metadata before anything is pinned: without hard links,
+ // pinning copies the file, and a source too large to copy must not be duplicated to refuse it.
+ requireCopyableSize(getObjectMetadata(bucketName, key, versionId));
+ if (inMemory) {
+ return new CopySource(getObject(bucketName, key, versionId), null);
+ }
+ if (versionId == null) {
+ Snapshot<Path> snapshot = readLatestSnapshot(bucketName, key,
+ account -> pinFile(resolveObjectPathForRead(account, bucketName, key)),
+ pinned -> deleteQuietly(pinned, "pinned copy source of a read that raced an overwrite"));
+ return checkedCopySource(snapshot.object(), snapshot.body());
+ }
+ String bucketOwnerAccount = resolveBucketEntry(bucketName)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404))
+ .account();
+ S3Object obj = getObjectMetadata(bucketName, key, versionId);
+ Path path = "null".equals(versionId)
+ ? resolveObjectPathForRead(bucketOwnerAccount, bucketName, key)
+ : resolveVersionedPathForRead(bucketOwnerAccount, bucketName, key, versionId);
+ return checkedCopySource(obj, pinFile(path));
+ }
+
+ private CopySource checkedCopySource(S3Object obj, Path pinned) {
+ long fileSize;
+ try {
+ fileSize = Files.size(pinned);
+ } catch (IOException e) {
+ deleteQuietly(pinned, "pinned copy source whose size could not be read");
+ throw new UncheckedIOException("Failed to read the copy source", e);
+ }
+ if (fileSize != obj.getSize()) {
+ deleteQuietly(pinned, "pinned copy source whose size does not match its metadata");
+ throw new IllegalStateException("S3 object file for " + obj.getBucketName() + "/"
+ + obj.getKey() + " has " + fileSize + " bytes but metadata declares " + obj.getSize());
+ }
+ return new CopySource(obj, pinned);
+ }
+
+ /**
+ * A temporary hard link to {@code file} beside it, or a copy where the filesystem has no hard
+ * links. A failed pin removes whatever it created, since the caller never gets its path.
+ */
+ private Path pinFile(Path file) {
+ Path pinned = file.resolveSibling(file.getFileName() + ".tmp-" + UUID.randomUUID());
+ try {
+ try {
+ Files.createLink(pinned, file);
+ } catch (UnsupportedOperationException | FileSystemException e) {
+ LOG.debugv(e, "No hard link for {0}, copying it to pin a copy source", file);
+ Files.copy(file, pinned);
+ }
+ return pinned;
+ } catch (IOException e) {
+ deleteQuietly(pinned, "partly pinned copy source");
+ throw new UncheckedIOException("Failed to pin the copy source", e);
+ }
+ }
+
+ /**
+ * The copy's body, ETag and checksum. In disk modes the pinned file is the body, and it is read,
+ * once, only for what cannot carry over from the source: a single-part source's ETag is the MD5
+ * of its bytes and stays, while a multipart one's has to be computed; the source's checksum
+ * stays unless a new or composite one has to become a full-object checksum.
+ */
+ private CopyBody copyBody(CopySource source, S3Checksum effectiveChecksum, ChecksumAlgorithm copyChecksumAlgorithm) {
+ S3Object object = source.object();
+ if (source.pinnedFile() == null) {
+ return new CopyBody(new BytesBody(object.getData()), null, effectiveChecksum);
+ }
+ boolean needsETag = !SINGLE_PART_ETAG.matcher(stripSurroundingQuotes(object.getETag())).matches();
+ S3Checksum.Calculator calculator = effectiveChecksum == null ? S3Checksum.calculator(copyChecksumAlgorithm) : null;
+ String eTag = object.getETag();
+ if (needsETag || calculator != null) {
+ String md5ETag = hashFile(source.pinnedFile(), needsETag, calculator);
+ if (needsETag) {
+ eTag = md5ETag;
+ }
+ }
+ return new CopyBody(new AssembledBody(source.pinnedFile(), object.getSize()), eTag,
+ calculator != null ? calculator.fullObject() : effectiveChecksum);
+ }
+
+ /** Reads {@code file} once, feeding {@code checksum} if given; returns its MD5 ETag when {@code md5}. */
+ private static String hashFile(Path file, boolean md5, S3Checksum.Calculator checksum) {
+ try {
+ MessageDigest digest = md5 ? MessageDigest.getInstance("MD5") : null;
+ byte[] buffer = new byte[1 << 20];
+ try (InputStream in = Files.newInputStream(file)) {
+ for (int read = in.read(buffer); read >= 0; read = in.read(buffer)) {
+ if (digest != null) {
+ digest.update(buffer, 0, read);
+ }
+ if (checksum != null) {
+ checksum.update(buffer, 0, read);
+ }
+ }
+ }
+ return digest == null ? null : "\"" + bytesToHex(digest.digest()) + "\"";
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read the copy source", e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("MD5 algorithm not available", e);
+ }
}
// --- Versioning Operations ---
@@ -5955,7 +6096,8 @@ public class S3Service implements Resettable, ResourceProvider {
}
private S3Object copyS3Object(String sourceBucket, String sourceKey,
- String destBucket, String destKey, S3Object source, CopyObjectOptions options) {
+ String destBucket, String destKey, CopySource copySource, CopyObjectOptions options) {
+ S3Object source = copySource.object();
ensureBucketExists(destBucket);
CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();
String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());
@@ -6010,6 +6152,9 @@ public class S3Service implements Resettable, ResourceProvider {
if (copyChecksumAlgorithm != null) {
effectiveChecksum = null;
}
+ // Prepared before any bucket monitor is taken, so reading a large source to hash it never
+ // holds up other writes to either bucket.
+ CopyBody body = copyBody(copySource, effectiveChecksum, copyChecksumAlgorithm);
// Annotations travel with the copy by default (x-amz-annotation-directive COPY). They are
// snapshotted before storeObject: a self-copy (same bucket and key) or a pre-versioning
@@ -6044,7 +6189,7 @@ public class S3Service implements Resettable, ResourceProvider {
if (copyAnnotations) {
sourceAnnotations = snapshotAnnotations(source);
}
- result[0] = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum,
+ result[0] = storeObjectCopy(destBucket, destKey, body, metadata,
effectiveContentType, effectiveStorageClass, effectiveContentEncoding,
effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption,
effectiveSseKmsKeyId, effectiveOptions, copyChecksumAlgorithm, effectiveTags);
@@ -6069,7 +6214,7 @@ public class S3Service implements Resettable, ResourceProvider {
synchronized (resolveBucket(destBucket)
.orElseThrow(() -> new AwsException("NoSuchBucket",
"The specified bucket does not exist.", 404))) {
- result[0] = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum,
+ result[0] = storeObjectCopy(destBucket, destKey, body, metadata,
effectiveContentType, effectiveStorageClass, effectiveContentEncoding,
effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption,
effectiveSseKmsKeyId, effectiveOptions, copyChecksumAlgorithm, effectiveTags);
@@ -6083,16 +6228,16 @@ public class S3Service implements Resettable, ResourceProvider {
return result[0];
}
- private S3Object storeObjectCopy(String destBucket, String destKey, S3Object source,
- Map<String, String> metadata, S3Checksum effectiveChecksum,
+ private S3Object storeObjectCopy(String destBucket, String destKey, CopyBody body,
+ Map<String, String> metadata,
String effectiveContentType, String effectiveStorageClass,
String effectiveContentEncoding, String effectiveContentDisposition,
String effectiveCacheControl, String effectiveServerSideEncryption,
String effectiveSseKmsKeyId,
CopyObjectOptions effectiveOptions, ChecksumAlgorithm copyChecksumAlgorithm,
Map<String, String> effectiveTags) {
- return storeObject(destBucket, destKey, source.getData(), effectiveContentType,
- metadata, effectiveChecksum, null,
+ return storeObject(destBucket, destKey, body.body(), effectiveContentType,
+ metadata, body.checksum(), null,
new PutObjectOptions()
.withStorageClass(effectiveStorageClass)
.withContentEncoding(effectiveContentEncoding)
@@ -6112,7 +6257,8 @@ public class S3Service implements Resettable, ResourceProvider {
.withChecksumAlgorithm(copyChecksumAlgorithm != null ? copyChecksumAlgorithm.name() : null)
.withTagging(effectiveTags)
.withIfMatch(effectiveOptions.getIfMatch())
- .withIfNoneMatch(effectiveOptions.getIfNoneMatch()));
+ .withIfNoneMatch(effectiveOptions.getIfNoneMatch()),
+ body.eTag());
}
private record AnnotationSnapshot(ObjectAnnotation metadata, byte[] payload) {}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
index 796fab3d0f..355842f23d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
@@ -7,6 +7,7 @@ import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.List;
import java.util.regex.Pattern;
+import java.util.zip.Checksum;
import java.util.zip.CRC32;
import java.util.zip.CRC32C;
@@ -175,9 +176,15 @@ public class S3Checksum {
}
public static String crc64NvmeBase64(byte[] data) {
- long crc = 0xFFFFFFFFFFFFFFFFL;
- int offset = 0;
- int blocksEnd = data.length - data.length % Long.BYTES;
+ return base64(~crc64NvmeUpdate(~0L, data, 0, data.length), Long.BYTES);
+ }
+
+ /** Runs {@code length} bytes through the CRC64NVME register; the final value is its complement. */
+ private static long crc64NvmeUpdate(long register, byte[] data, int start, int length) {
+ long crc = register;
+ int offset = start;
+ int end = start + length;
+ int blocksEnd = end - length % Long.BYTES;
while (offset < blocksEnd) {
crc ^= (data[offset] & 0xFFL)
| (data[offset + 1] & 0xFFL) << 8
@@ -197,19 +204,90 @@ public class S3Checksum {
^ CRC64_TABLES[0][(int)(crc >>> 56)];
offset += Long.BYTES;
}
- while (offset < data.length) {
+ while (offset < end) {
int idx = (int)((crc ^ data[offset]) & 0xFF);
crc = CRC64_TABLES[0][idx] ^ (crc >>> 8);
offset++;
}
- crc ^= 0xFFFFFFFFFFFFFFFFL;
- byte[] bytes = new byte[]{
- (byte)(crc >> 56), (byte)(crc >> 48), (byte)(crc >> 40), (byte)(crc >> 32),
- (byte)(crc >> 24), (byte)(crc >> 16), (byte)(crc >> 8), (byte) crc
- };
+ return crc;
+ }
+
+ private static String base64(long value, int byteCount) {
+ byte[] bytes = new byte[byteCount];
+ for (int i = byteCount - 1; i >= 0; i--) {
+ bytes[i] = (byte) (value >>> ((byteCount - 1 - i) * Byte.SIZE));
+ }
return Base64.getEncoder().encodeToString(bytes);
}
+ /** A {@link Calculator} for {@code algorithm}, CRC64NVME when no algorithm was declared, as {@link #of}. */
+ public static Calculator calculator(ChecksumAlgorithm algorithm) {
+ return new Calculator(algorithm != null ? algorithm : ChecksumAlgorithm.CRC64NVME);
+ }
+
+ /**
+ * Computes a checksum over data fed in pieces, for a body too large to hold in one array. The
+ * value is the one {@link ChecksumAlgorithm#compute} gives for the same bytes in one piece.
+ */
+ public static final class Calculator {
+
+ private final ChecksumAlgorithm algorithm;
+ private final Checksum crc;
+ private final MessageDigest digest;
+ private long crc64Register = ~0L;
+
+ private Calculator(ChecksumAlgorithm algorithm) {
+ this.algorithm = algorithm;
+ this.crc = switch (algorithm) {
+ case CRC32 -> new CRC32();
+ case CRC32C -> new CRC32C();
+ case CRC64NVME, SHA1, SHA256 -> null;
+ };
+ this.digest = switch (algorithm) {
+ case SHA1 -> newDigest("SHA-1");
+ case SHA256 -> newDigest("SHA-256");
+ case CRC32, CRC32C, CRC64NVME -> null;
+ };
+ }
+
+ public ChecksumAlgorithm algorithm() {
+ return algorithm;
+ }
+
+ public void update(byte[] data, int offset, int length) {
+ switch (algorithm) {
+ case CRC32, CRC32C -> crc.update(data, offset, length);
+ case CRC64NVME -> crc64Register = crc64NvmeUpdate(crc64Register, data, offset, length);
+ case SHA1, SHA256 -> digest.update(data, offset, length);
+ }
+ }
+
+ /** The Base64 checksum of everything fed so far. */
+ public String value() {
+ return switch (algorithm) {
+ case CRC32, CRC32C -> base64(crc.getValue(), Integer.BYTES);
+ case CRC64NVME -> base64(~crc64Register, Long.BYTES);
+ case SHA1, SHA256 -> Base64.getEncoder().encodeToString(digest.digest());
+ };
+ }
+
+ /** The value as a full-object checksum, the kind a single-part object carries. */
+ public S3Checksum fullObject() {
+ S3Checksum checksum = new S3Checksum();
+ checksum.setValueFor(algorithm, value());
+ checksum.setChecksumType(ChecksumType.FULL_OBJECT);
+ return checksum;
+ }
+
+ private static MessageDigest newDigest(String name) {
+ try {
+ return MessageDigest.getInstance(name);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("Missing digest algorithm: " + name, e);
+ }
+ }
+ }
+
public static String sha256Base64(byte[] data) {
return digestBase64("SHA-256", data);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java
index 6a53faa296..da002ea710 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java
@@ -195,6 +195,33 @@ class S3ChecksumTest {
return Base64.getEncoder().encodeToString(ByteBuffer.allocate(Integer.BYTES).putInt((int) value).array());
}
+ @Test
+ void calculatorFedInPiecesMatchesTheOneShotChecksumForEveryAlgorithm() {
+ Random random = new Random(5064);
+ byte[] data = new byte[300_007];
+ random.nextBytes(data);
+ int[] pieces = {1, 7, 8, 9, 1023, 65_537, 3, 0, 233_419};
+ for (ChecksumAlgorithm algorithm : ChecksumAlgorithm.values()) {
+ S3Checksum.Calculator calculator = S3Checksum.calculator(algorithm);
+ int offset = 0;
+ for (int piece : pieces) {
+ calculator.update(data, offset, piece);
+ offset += piece;
+ }
+ assertEquals(data.length, offset, "test setup: the pieces cover the data");
+
+ S3Checksum fullObject = calculator.fullObject();
+
+ assertEquals(algorithm.compute(data), fullObject.valueFor(algorithm), algorithm.name());
+ assertEquals(ChecksumType.FULL_OBJECT, fullObject.getChecksumType());
+ }
+ }
+
+ @Test
+ void calculatorWithoutADeclaredAlgorithmIsCrc64Nvme() {
+ assertEquals(ChecksumAlgorithm.CRC64NVME, S3Checksum.calculator(null).algorithm());
+ }
+
@Test
void objectAttributesViewDropsTheSuffixOfCompositeValuesOnly() {
S3Checksum composite = S3Checksum.composite(ChecksumAlgorithm.CRC32, List.of("W0QpDQ==", "anCi7Q=="));
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java
new file mode 100644
index 0000000000..30d0e0129b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java
@@ -0,0 +1,225 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;
+import io.github.hectorvent.floci.services.s3.model.ChecksumType;
+import io.github.hectorvent.floci.services.s3.model.CopyObjectOptions;
+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;
+import io.github.hectorvent.floci.services.s3.model.S3Checksum;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * CopyObject in disk-storage mode, where the copy stores the source's pinned file instead of the
+ * source's bytes, and reads that file only for an ETag or checksum it cannot carry over.
+ */
+class S3CopyObjectDiskTest {
+
+ @TempDir
+ Path tempDir;
+
+ private S3Service s3Service;
+ private InMemoryStorage<String, S3Object> objectStore;
+ private Path dataRoot;
+
+ @BeforeEach
+ void setUp() {
+ dataRoot = tempDir.resolve("s3");
+ objectStore = new InMemoryStorage<>();
+ s3Service = new S3Service(new InMemoryStorage<>(), objectStore, dataRoot, false);
+ s3Service.createBucket("source-bucket", "us-east-1");
+ s3Service.createBucket("dest-bucket", "us-east-1");
+ }
+
+ @Test
+ void copyOfASinglePartObjectKeepsItsETagAndChecksumAndSharesItsFile() throws IOException {
+ S3Object source = s3Service.putObject("source-bucket", "source.txt",
+ "single part body".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+
+ S3Object copy = s3Service.copyObject("source-bucket", "source.txt", "dest-bucket", "copy.txt",
+ new CopyObjectOptions());
+
+ assertEquals(source.getETag(), copy.getETag());
+ assertEquals(source.getChecksum().getChecksumCRC64NVME(), copy.getChecksum().getChecksumCRC64NVME());
+ assertArrayEquals("single part body".getBytes(StandardCharsets.UTF_8),
+ s3Service.getObject("dest-bucket", "copy.txt").getData());
+ List<Path> files = objectFiles();
+ assertEquals(2, files.size(), "the source file and the copy's file: " + files);
+ if (hardLinksSupported()) {
+ assertTrue(Files.isSameFile(files.get(0), files.get(1)), "the copy should share the source's file");
+ }
+ }
+
+ @Test
+ void copyOfAMultipartObjectGetsTheMd5OfItsBytesAsItsETag() {
+ byte[] part1 = "first part of the source, ".getBytes(StandardCharsets.UTF_8);
+ byte[] part2 = "second part of the source".getBytes(StandardCharsets.UTF_8);
+ MultipartUpload upload = s3Service.initiateMultipartUpload("source-bucket", "multipart.bin", null);
+ s3Service.uploadPart("source-bucket", "multipart.bin", upload.getUploadId(), 1, part1);
+ s3Service.uploadPart("source-bucket", "multipart.bin", upload.getUploadId(), 2, part2);
+ S3Object source = s3Service.completeMultipartUpload("source-bucket", "multipart.bin",
+ upload.getUploadId(), List.of(1, 2), null, null);
+ assertTrue(source.getETag().endsWith("-2\""), "test setup: a multipart ETag " + source.getETag());
+
+ S3Object copy = s3Service.copyObject("source-bucket", "multipart.bin", "dest-bucket", "copy.bin",
+ new CopyObjectOptions());
+
+ byte[] whole = concat(part1, part2);
+ assertEquals(S3Object.computeETag(whole), copy.getETag());
+ assertEquals(source.getChecksum().getChecksumCRC64NVME(), copy.getChecksum().getChecksumCRC64NVME(),
+ "a full-object checksum carries over");
+ assertArrayEquals(whole, s3Service.getObject("dest-bucket", "copy.bin").getData());
+ }
+
+ @Test
+ void copyWithANewChecksumAlgorithmComputesItOverTheBytes() {
+ byte[] body = "body to checksum again".getBytes(StandardCharsets.UTF_8);
+ s3Service.putObject("source-bucket", "source.txt", body, "text/plain", Map.of());
+
+ S3Object copy = s3Service.copyObject("source-bucket", "source.txt", "dest-bucket", "copy.txt",
+ new CopyObjectOptions().withChecksumAlgorithm("SHA256"));
+
+ assertEquals(S3Checksum.sha256Base64(body), copy.getChecksum().getChecksumSHA256());
+ assertEquals(ChecksumType.FULL_OBJECT, copy.getChecksum().getChecksumType());
+ }
+
+ @Test
+ void copyOfACompositeChecksumObjectGetsAFullObjectChecksumOfItsBytes() {
+ byte[] part1 = "composite part one ".getBytes(StandardCharsets.UTF_8);
+ byte[] part2 = "composite part two".getBytes(StandardCharsets.UTF_8);
+ MultipartUpload upload = s3Service.initiateMultipartUpload("source-bucket", "composite.bin", null,
+ null, null, null, null, null, null, null, null, "CRC32");
+ s3Service.uploadPart("source-bucket", "composite.bin", upload.getUploadId(), 1, part1);
+ s3Service.uploadPart("source-bucket", "composite.bin", upload.getUploadId(), 2, part2);
+ S3Object source = s3Service.completeMultipartUpload("source-bucket", "composite.bin", upload.getUploadId(),
+ List.of(1, 2), Map.of(1, S3Checksum.of(ChecksumAlgorithm.CRC32, part1),
+ 2, S3Checksum.of(ChecksumAlgorithm.CRC32, part2)), null, null);
+ assertEquals(ChecksumType.COMPOSITE, source.getChecksum().getChecksumType(), "test setup");
+
+ S3Object copy = s3Service.copyObject("source-bucket", "composite.bin", "dest-bucket", "copy.bin",
+ new CopyObjectOptions());
+
+ assertEquals(S3Checksum.crc32Base64(concat(part1, part2)), copy.getChecksum().getChecksumCRC32());
+ assertEquals(ChecksumType.FULL_OBJECT, copy.getChecksum().getChecksumType());
+ }
+
+ @Test
+ void failedCopyLeavesNoPinnedFileBehind() throws IOException {
+ s3Service.putObject("source-bucket", "source.txt", "source".getBytes(StandardCharsets.UTF_8),
+ "text/plain", Map.of());
+ s3Service.putObject("dest-bucket", "taken.txt", "already here".getBytes(StandardCharsets.UTF_8),
+ "text/plain", Map.of());
+
+ assertThrows(AwsException.class, () -> s3Service.copyObject("source-bucket", "source.txt",
+ "dest-bucket", "taken.txt", new CopyObjectOptions().withIfNoneMatch("*")));
+
+ try (Stream<Path> files = Files.walk(dataRoot)) {
+ assertEquals(List.of(), files.filter(path -> path.getFileName().toString().contains(".tmp-")).toList());
+ }
+ assertArrayEquals("already here".getBytes(StandardCharsets.UTF_8),
+ s3Service.getObject("dest-bucket", "taken.txt").getData());
+ }
+
+ @Test
+ void copyIntoAVersionedBucketStoresTheNewVersionAndTheCurrentFile() {
+ s3Service.putBucketVersioning("dest-bucket", "Enabled");
+ s3Service.putObject("source-bucket", "source.txt", "versioned copy".getBytes(StandardCharsets.UTF_8),
+ "text/plain", Map.of());
+
+ S3Object copy = s3Service.copyObject("source-bucket", "source.txt", "dest-bucket", "copy.txt",
+ new CopyObjectOptions());
+
+ assertNotNull(copy.getVersionId());
+ assertArrayEquals("versioned copy".getBytes(StandardCharsets.UTF_8),
+ s3Service.getObject("dest-bucket", "copy.txt").getData());
+ assertArrayEquals("versioned copy".getBytes(StandardCharsets.UTF_8),
+ s3Service.getObject("dest-bucket", "copy.txt", copy.getVersionId()).getData());
+ }
+
+ @Test
+ void copyOfASourceVersionCopiesThatVersionAfterTheKeyMovedOn() {
+ s3Service.putBucketVersioning("source-bucket", "Enabled");
+ S3Object first = s3Service.putObject("source-bucket", "source.txt", "first".getBytes(StandardCharsets.UTF_8),
+ "text/plain", Map.of());
+ s3Service.putObject("source-bucket", "source.txt", "second".getBytes(StandardCharsets.UTF_8),
+ "text/plain", Map.of());
+
+ s3Service.copyObject("source-bucket", "source.txt", "dest-bucket", "copy.txt", first.getVersionId(),
+ new CopyObjectOptions());
+
+ assertArrayEquals("first".getBytes(StandardCharsets.UTF_8),
+ s3Service.getObject("dest-bucket", "copy.txt").getData());
+ }
+
+ @Test
+ void copySourceLargerThanFiveGibibytesIsAnInvalidRequest() {
+ assertEquals(5_368_709_120L, S3Service.MAX_COPY_OBJECT_SOURCE_SIZE);
+ S3Object atLimit = new S3Object("source-bucket", "at-limit.bin", 5_368_709_120L, null, "\"etag\"");
+ assertDoesNotThrow(() -> S3Service.requireCopyableSize(atLimit));
+
+ S3Object overLimit = new S3Object("source-bucket", "over-limit.bin", 5_368_709_121L, null, "\"etag\"");
+ AwsException error = assertThrows(AwsException.class, () -> S3Service.requireCopyableSize(overLimit));
+
+ assertEquals("InvalidRequest", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("The specified copy source is larger than the maximum allowable size for a copy source: 5368709120",
+ error.getMessage());
+ }
+
+ @Test
+ void oversizedSourceIsRejectedBeforeItsFileIsPinned() throws IOException {
+ s3Service.putObject("source-bucket", "oversized.bin", "small".getBytes(StandardCharsets.UTF_8),
+ "application/octet-stream", Map.of());
+ // Metadata over 5 GiB on top of a file that is gone: a copy that went to the file first would
+ // fail on the missing file instead of answering InvalidRequest from the metadata.
+ objectStore.scan(key -> true).forEach(object -> object.setSize(S3Service.MAX_COPY_OBJECT_SOURCE_SIZE + 1));
+ for (Path file : objectFiles()) {
+ Files.delete(file);
+ }
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.copyObject("source-bucket",
+ "oversized.bin", "dest-bucket", "copy.bin", new CopyObjectOptions()));
+
+ assertEquals("InvalidRequest", error.getErrorCode());
+ }
+
+ private List<Path> objectFiles() throws IOException {
+ try (Stream<Path> files = Files.walk(dataRoot)) {
+ return files.filter(path -> path.getFileName().toString().endsWith(".s3data")).toList();
+ }
+ }
+
+ private boolean hardLinksSupported() throws IOException {
+ Path probe = Files.createTempFile(tempDir, "link-probe", null);
+ Path link = probe.resolveSibling(probe.getFileName() + ".link");
+ try {
+ Files.createLink(link, probe);
+ return true;
+ } catch (UnsupportedOperationException | IOException unsupported) {
+ return false;
+ } finally {
+ Files.deleteIfExists(link);
+ Files.deleteIfExists(probe);
+ }
+ }
+
+ private static byte[] concat(byte[] first, byte[] second) {
+ byte[] result = new byte[first.length + second.length];
+ System.arraycopy(first, 0, result, 0, first.length);
+ System.arraycopy(second, 0, result, first.length, second.length);
+ return result;
+ }
+}