Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): copy objects over 2 GiB without reading them into memory (#5080)
Priamo Germosend97b21ae62b4
diff --git a/docs/services/s3.md b/docs/services/s3.mdindex 532b634d9e..cd7626b306 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -397,8 +397,12 @@ multipart upload file to file, so the object never passes through the heap; setcopy, until the parts are removed. Multipart uploads in progress do not survive a restart or areset, and their files are removed when either happens. `UploadPartCopy` reads only the copiedrange from its source, so it can copy any part of a source of any size, but one copied part can-hold at most about 2 GiB; a larger range fails with `EntityTooLarge`. `CopyObject` and S3 Select-still read the whole source object into memory in every mode, so they need a source under 2 GiB.+hold at most about 2 GiB; a larger range fails with `EntityTooLarge`. In the disk-backed modes+`CopyObject` stores the source's file for the copy without reading it into memory, and reads it+once only when the copy needs an ETag or checksum the source's cannot supply. As on S3, a+`CopyObject` source larger than 5 GiB is rejected with `InvalidRequest`; copy it in parts with+`UploadPartCopy` instead. S3 Select still reads the whole source object into memory in every+mode, so it needs a source under 2 GiB.## Examplesdiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 9dc923544e..d51bf6156b 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -697,8 +697,9 @@ public class S3Service implements Resettable, ResourceProvider {}/**- * A disk-mode multipart object. It is stored with the ETag and checksum built from its parts,- * since computing either here would mean reading the whole file.+ * A body already on disk that is moved into place: an assembled multipart object, or the pinned+ * file of a copy source. It is stored with an ETag and checksum worked out beforehand, since+ * computing either here would mean reading the whole file under the bucket lock.*/private record AssembledBody(Path file, long size) implements ObjectBody { }@@ -2139,26 +2140,166 @@ public class S3Service implements Resettable, ResourceProvider {String destBucket, String destKey, String versionId, CopyObjectOptions options){CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();- S3Object source = getObject(sourceBucket, sourceKey, versionId);- checkCopySourcePreconditions(source, effectiveOptions.getCopySourceConditions());- validateSseCustomerAccess(source,- effectiveOptions.getCopySourceSseCustomerAlgorithm(),- effectiveOptions.getCopySourceSseCustomerKey(),- effectiveOptions.getCopySourceSseCustomerKeyMd5());- return copyS3Object(sourceBucket, sourceKey,- destBucket, destKey, source, effectiveOptions);+ return copyPinnedSource(sourceBucket, sourceKey, destBucket, destKey,+ pinCopySource(sourceBucket, sourceKey, versionId), effectiveOptions);}public S3Object copyObject(String sourceBucket, String sourceKey,String destBucket, String destKey, CopyObjectOptions options) {CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();- S3Object source = getObject(sourceBucket, sourceKey);- checkCopySourcePreconditions(source, effectiveOptions.getCopySourceConditions());- validateSseCustomerAccess(source,- effectiveOptions.getCopySourceSseCustomerAlgorithm(),- effectiveOptions.getCopySourceSseCustomerKey(),- effectiveOptions.getCopySourceSseCustomerKeyMd5());- return copyS3Object(sourceBucket, sourceKey, destBucket, destKey, source, effectiveOptions);+ return copyPinnedSource(sourceBucket, sourceKey, destBucket, destKey,+ pinCopySource(sourceBucket, sourceKey, null), effectiveOptions);+ }++ private S3Object copyPinnedSource(String sourceBucket, String sourceKey, String destBucket, String destKey,+ CopySource source, CopyObjectOptions effectiveOptions) {+ try {+ checkCopySourcePreconditions(source.object(), effectiveOptions.getCopySourceConditions());+ validateSseCustomerAccess(source.object(),+ effectiveOptions.getCopySourceSseCustomerAlgorithm(),+ effectiveOptions.getCopySourceSseCustomerKey(),+ effectiveOptions.getCopySourceSseCustomerKeyMd5());+ requireCopyableSize(source.object());+ return copyS3Object(sourceBucket, sourceKey, destBucket, destKey, source, effectiveOptions);+ } finally {+ // A stored copy moved its pinned file into place, so this only removes one a failed+ // copy left behind.+ if (source.pinnedFile() != null) {+ deleteQuietly(source.pinnedFile(), "pinned copy source that was not stored");+ }+ }+ }++ // S3 copies at most 5 GiB in one CopyObject; a larger object is copied with UploadPartCopy.+ static final long MAX_COPY_OBJECT_SOURCE_SIZE = 5L * 1024 * 1024 * 1024;++ static void requireCopyableSize(S3Object source) {+ if (source.getSize() > MAX_COPY_OBJECT_SOURCE_SIZE) {+ throw new AwsException("InvalidRequest",+ "The specified copy source is larger than the maximum allowable size for a copy source: "+ + MAX_COPY_OBJECT_SOURCE_SIZE, 400);+ }+ }++ /**+ * The source of a copy, held for the length of the copy: its metadata and, in disk modes, a+ * temporary hard link to the file generation that metadata describes. The link keeps those+ * bytes even if the source is overwritten meanwhile, since an object file is only ever replaced+ * by a rename, and the copy can store the file without reading it into memory. In memory mode+ * the bytes come with the object, as before.+ */+ private record CopySource(S3Object object, Path pinnedFile) { }++ /** A copy's body, with the ETag and checksum to store it under (null to compute them from bytes). */+ private record CopyBody(ObjectBody body, String eTag, S3Checksum checksum) { }++ private static final Pattern SINGLE_PART_ETAG = Pattern.compile("[0-9a-f]{32}");++ private CopySource pinCopySource(String bucketName, String key, String versionId) {+ // The size limit is checked on the metadata before anything is pinned: without hard links,+ // pinning copies the file, and a source too large to copy must not be duplicated to refuse it.+ requireCopyableSize(getObjectMetadata(bucketName, key, versionId));+ if (inMemory) {+ return new CopySource(getObject(bucketName, key, versionId), null);+ }+ if (versionId == null) {+ Snapshot<Path> snapshot = readLatestSnapshot(bucketName, key,+ account -> pinFile(resolveObjectPathForRead(account, bucketName, key)),+ pinned -> deleteQuietly(pinned, "pinned copy source of a read that raced an overwrite"));+ return checkedCopySource(snapshot.object(), snapshot.body());+ }+ String bucketOwnerAccount = resolveBucketEntry(bucketName)+ .orElseThrow(() -> new AwsException("NoSuchBucket",+ "The specified bucket does not exist.", 404))+ .account();+ S3Object obj = getObjectMetadata(bucketName, key, versionId);+ Path path = "null".equals(versionId)+ ? resolveObjectPathForRead(bucketOwnerAccount, bucketName, key)+ : resolveVersionedPathForRead(bucketOwnerAccount, bucketName, key, versionId);+ return checkedCopySource(obj, pinFile(path));+ }++ private CopySource checkedCopySource(S3Object obj, Path pinned) {+ long fileSize;+ try {+ fileSize = Files.size(pinned);+ } catch (IOException e) {+ deleteQuietly(pinned, "pinned copy source whose size could not be read");+ throw new UncheckedIOException("Failed to read the copy source", e);+ }+ if (fileSize != obj.getSize()) {+ deleteQuietly(pinned, "pinned copy source whose size does not match its metadata");+ throw new IllegalStateException("S3 object file for " + obj.getBucketName() + "/"+ + obj.getKey() + " has " + fileSize + " bytes but metadata declares " + obj.getSize());+ }+ return new CopySource(obj, pinned);+ }++ /**+ * A temporary hard link to {@code file} beside it, or a copy where the filesystem has no hard+ * links. A failed pin removes whatever it created, since the caller never gets its path.+ */+ private Path pinFile(Path file) {+ Path pinned = file.resolveSibling(file.getFileName() + ".tmp-" + UUID.randomUUID());+ try {+ try {+ Files.createLink(pinned, file);+ } catch (UnsupportedOperationException | FileSystemException e) {+ LOG.debugv(e, "No hard link for {0}, copying it to pin a copy source", file);+ Files.copy(file, pinned);+ }+ return pinned;+ } catch (IOException e) {+ deleteQuietly(pinned, "partly pinned copy source");+ throw new UncheckedIOException("Failed to pin the copy source", e);+ }+ }++ /**+ * The copy's body, ETag and checksum. In disk modes the pinned file is the body, and it is read,+ * once, only for what cannot carry over from the source: a single-part source's ETag is the MD5+ * of its bytes and stays, while a multipart one's has to be computed; the source's checksum+ * stays unless a new or composite one has to become a full-object checksum.+ */+ private CopyBody copyBody(CopySource source, S3Checksum effectiveChecksum, ChecksumAlgorithm copyChecksumAlgorithm) {+ S3Object object = source.object();+ if (source.pinnedFile() == null) {+ return new CopyBody(new BytesBody(object.getData()), null, effectiveChecksum);+ }+ boolean needsETag = !SINGLE_PART_ETAG.matcher(stripSurroundingQuotes(object.getETag())).matches();+ S3Checksum.Calculator calculator = effectiveChecksum == null ? S3Checksum.calculator(copyChecksumAlgorithm) : null;+ String eTag = object.getETag();+ if (needsETag || calculator != null) {+ String md5ETag = hashFile(source.pinnedFile(), needsETag, calculator);+ if (needsETag) {+ eTag = md5ETag;+ }+ }+ return new CopyBody(new AssembledBody(source.pinnedFile(), object.getSize()), eTag,+ calculator != null ? calculator.fullObject() : effectiveChecksum);+ }++ /** Reads {@code file} once, feeding {@code checksum} if given; returns its MD5 ETag when {@code md5}. */+ private static String hashFile(Path file, boolean md5, S3Checksum.Calculator checksum) {+ try {+ MessageDigest digest = md5 ? MessageDigest.getInstance("MD5") : null;+ byte[] buffer = new byte[1 << 20];+ try (InputStream in = Files.newInputStream(file)) {+ for (int read = in.read(buffer); read >= 0; read = in.read(buffer)) {+ if (digest != null) {+ digest.update(buffer, 0, read);+ }+ if (checksum != null) {+ checksum.update(buffer, 0, read);+ }+ }+ }+ return digest == null ? null : "\"" + bytesToHex(digest.digest()) + "\"";+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to read the copy source", e);+ } catch (NoSuchAlgorithmException e) {+ throw new IllegalStateException("MD5 algorithm not available", e);+ }}// --- Versioning Operations ---@@ -5955,7 +6096,8 @@ public class S3Service implements Resettable, ResourceProvider {}private S3Object copyS3Object(String sourceBucket, String sourceKey,- String destBucket, String destKey, S3Object source, CopyObjectOptions options) {+ String destBucket, String destKey, CopySource copySource, CopyObjectOptions options) {+ S3Object source = copySource.object();ensureBucketExists(destBucket);CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());@@ -6010,6 +6152,9 @@ public class S3Service implements Resettable, ResourceProvider {if (copyChecksumAlgorithm != null) {effectiveChecksum = null;}+ // Prepared before any bucket monitor is taken, so reading a large source to hash it never+ // holds up other writes to either bucket.+ CopyBody body = copyBody(copySource, effectiveChecksum, copyChecksumAlgorithm);// Annotations travel with the copy by default (x-amz-annotation-directive COPY). They are// snapshotted before storeObject: a self-copy (same bucket and key) or a pre-versioning@@ -6044,7 +6189,7 @@ public class S3Service implements Resettable, ResourceProvider {if (copyAnnotations) {sourceAnnotations = snapshotAnnotations(source);}- result[0] = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum,+ result[0] = storeObjectCopy(destBucket, destKey, body, metadata,effectiveContentType, effectiveStorageClass, effectiveContentEncoding,effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption,effectiveSseKmsKeyId, effectiveOptions, copyChecksumAlgorithm, effectiveTags);@@ -6069,7 +6214,7 @@ public class S3Service implements Resettable, ResourceProvider {synchronized (resolveBucket(destBucket).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404))) {- result[0] = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum,+ result[0] = storeObjectCopy(destBucket, destKey, body, metadata,effectiveContentType, effectiveStorageClass, effectiveContentEncoding,effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption,effectiveSseKmsKeyId, effectiveOptions, copyChecksumAlgorithm, effectiveTags);@@ -6083,16 +6228,16 @@ public class S3Service implements Resettable, ResourceProvider {return result[0];}- private S3Object storeObjectCopy(String destBucket, String destKey, S3Object source,- Map<String, String> metadata, S3Checksum effectiveChecksum,+ private S3Object storeObjectCopy(String destBucket, String destKey, CopyBody body,+ Map<String, String> metadata,String effectiveContentType, String effectiveStorageClass,String effectiveContentEncoding, String effectiveContentDisposition,String effectiveCacheControl, String effectiveServerSideEncryption,String effectiveSseKmsKeyId,CopyObjectOptions effectiveOptions, ChecksumAlgorithm copyChecksumAlgorithm,Map<String, String> effectiveTags) {- return storeObject(destBucket, destKey, source.getData(), effectiveContentType,- metadata, effectiveChecksum, null,+ return storeObject(destBucket, destKey, body.body(), effectiveContentType,+ metadata, body.checksum(), null,new PutObjectOptions().withStorageClass(effectiveStorageClass).withContentEncoding(effectiveContentEncoding)@@ -6112,7 +6257,8 @@ public class S3Service implements Resettable, ResourceProvider {.withChecksumAlgorithm(copyChecksumAlgorithm != null ? copyChecksumAlgorithm.name() : null).withTagging(effectiveTags).withIfMatch(effectiveOptions.getIfMatch())- .withIfNoneMatch(effectiveOptions.getIfNoneMatch()));+ .withIfNoneMatch(effectiveOptions.getIfNoneMatch()),+ body.eTag());}private record AnnotationSnapshot(ObjectAnnotation metadata, byte[] payload) {}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.javaindex 796fab3d0f..355842f23d 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java@@ -7,6 +7,7 @@ import java.security.NoSuchAlgorithmException;import java.util.Base64;import java.util.List;import java.util.regex.Pattern;+import java.util.zip.Checksum;import java.util.zip.CRC32;import java.util.zip.CRC32C;@@ -175,9 +176,15 @@ public class S3Checksum {}public static String crc64NvmeBase64(byte[] data) {- long crc = 0xFFFFFFFFFFFFFFFFL;- int offset = 0;- int blocksEnd = data.length - data.length % Long.BYTES;+ return base64(~crc64NvmeUpdate(~0L, data, 0, data.length), Long.BYTES);+ }++ /** Runs {@code length} bytes through the CRC64NVME register; the final value is its complement. */+ private static long crc64NvmeUpdate(long register, byte[] data, int start, int length) {+ long crc = register;+ int offset = start;+ int end = start + length;+ int blocksEnd = end - length % Long.BYTES;while (offset < blocksEnd) {crc ^= (data[offset] & 0xFFL)| (data[offset + 1] & 0xFFL) << 8@@ -197,19 +204,90 @@ public class S3Checksum {^ CRC64_TABLES[0][(int)(crc >>> 56)];offset += Long.BYTES;}- while (offset < data.length) {+ while (offset < end) {int idx = (int)((crc ^ data[offset]) & 0xFF);crc = CRC64_TABLES[0][idx] ^ (crc >>> 8);offset++;}- crc ^= 0xFFFFFFFFFFFFFFFFL;- byte[] bytes = new byte[]{- (byte)(crc >> 56), (byte)(crc >> 48), (byte)(crc >> 40), (byte)(crc >> 32),- (byte)(crc >> 24), (byte)(crc >> 16), (byte)(crc >> 8), (byte) crc- };+ return crc;+ }++ private static String base64(long value, int byteCount) {+ byte[] bytes = new byte[byteCount];+ for (int i = byteCount - 1; i >= 0; i--) {+ bytes[i] = (byte) (value >>> ((byteCount - 1 - i) * Byte.SIZE));+ }return Base64.getEncoder().encodeToString(bytes);}+ /** A {@link Calculator} for {@code algorithm}, CRC64NVME when no algorithm was declared, as {@link #of}. */+ public static Calculator calculator(ChecksumAlgorithm algorithm) {+ return new Calculator(algorithm != null ? algorithm : ChecksumAlgorithm.CRC64NVME);+ }++ /**+ * Computes a checksum over data fed in pieces, for a body too large to hold in one array. The+ * value is the one {@link ChecksumAlgorithm#compute} gives for the same bytes in one piece.+ */+ public static final class Calculator {++ private final ChecksumAlgorithm algorithm;+ private final Checksum crc;+ private final MessageDigest digest;+ private long crc64Register = ~0L;++ private Calculator(ChecksumAlgorithm algorithm) {+ this.algorithm = algorithm;+ this.crc = switch (algorithm) {+ case CRC32 -> new CRC32();+ case CRC32C -> new CRC32C();+ case CRC64NVME, SHA1, SHA256 -> null;+ };+ this.digest = switch (algorithm) {+ case SHA1 -> newDigest("SHA-1");+ case SHA256 -> newDigest("SHA-256");+ case CRC32, CRC32C, CRC64NVME -> null;+ };+ }++ public ChecksumAlgorithm algorithm() {+ return algorithm;+ }++ public void update(byte[] data, int offset, int length) {+ switch (algorithm) {+ case CRC32, CRC32C -> crc.update(data, offset, length);+ case CRC64NVME -> crc64Register = crc64NvmeUpdate(crc64Register, data, offset, length);+ case SHA1, SHA256 -> digest.update(data, offset, length);+ }+ }++ /** The Base64 checksum of everything fed so far. */+ public String value() {+ return switch (algorithm) {+ case CRC32, CRC32C -> base64(crc.getValue(), Integer.BYTES);+ case CRC64NVME -> base64(~crc64Register, Long.BYTES);+ case SHA1, SHA256 -> Base64.getEncoder().encodeToString(digest.digest());+ };+ }++ /** The value as a full-object checksum, the kind a single-part object carries. */+ public S3Checksum fullObject() {+ S3Checksum checksum = new S3Checksum();+ checksum.setValueFor(algorithm, value());+ checksum.setChecksumType(ChecksumType.FULL_OBJECT);+ return checksum;+ }++ private static MessageDigest newDigest(String name) {+ try {+ return MessageDigest.getInstance(name);+ } catch (NoSuchAlgorithmException e) {+ throw new IllegalStateException("Missing digest algorithm: " + name, e);+ }+ }+ }+public static String sha256Base64(byte[] data) {return digestBase64("SHA-256", data);}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.javaindex 6a53faa296..da002ea710 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java@@ -195,6 +195,33 @@ class S3ChecksumTest {return Base64.getEncoder().encodeToString(ByteBuffer.allocate(Integer.BYTES).putInt((int) value).array());}+ @Test+ void calculatorFedInPiecesMatchesTheOneShotChecksumForEveryAlgorithm() {+ Random random = new Random(5064);+ byte[] data = new byte[300_007];+ random.nextBytes(data);+ int[] pieces = {1, 7, 8, 9, 1023, 65_537, 3, 0, 233_419};+ for (ChecksumAlgorithm algorithm : ChecksumAlgorithm.values()) {+ S3Checksum.Calculator calculator = S3Checksum.calculator(algorithm);+ int offset = 0;+ for (int piece : pieces) {+ calculator.update(data, offset, piece);+ offset += piece;+ }+ assertEquals(data.length, offset, "test setup: the pieces cover the data");++ S3Checksum fullObject = calculator.fullObject();++ assertEquals(algorithm.compute(data), fullObject.valueFor(algorithm), algorithm.name());+ assertEquals(ChecksumType.FULL_OBJECT, fullObject.getChecksumType());+ }+ }++ @Test+ void calculatorWithoutADeclaredAlgorithmIsCrc64Nvme() {+ assertEquals(ChecksumAlgorithm.CRC64NVME, S3Checksum.calculator(null).algorithm());+ }+@Testvoid objectAttributesViewDropsTheSuffixOfCompositeValuesOnly() {S3Checksum composite = S3Checksum.composite(ChecksumAlgorithm.CRC32, List.of("W0QpDQ==", "anCi7Q=="));diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.javanew file mode 100644index 0000000000..30d0e0129b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectDiskTest.java@@ -0,0 +1,225 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;+import io.github.hectorvent.floci.services.s3.model.ChecksumType;+import io.github.hectorvent.floci.services.s3.model.CopyObjectOptions;+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;+import io.github.hectorvent.floci.services.s3.model.S3Checksum;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.List;+import java.util.Map;+import java.util.stream.Stream;++import static org.junit.jupiter.api.Assertions.*;++/**+ * CopyObject in disk-storage mode, where the copy stores the source's pinned file instead of the+ * source's bytes, and reads that file only for an ETag or checksum it cannot carry over.+ */+class S3CopyObjectDiskTest {++ @TempDir+ Path tempDir;++ private S3Service s3Service;+ private InMemoryStorage<String, S3Object> objectStore;+ private Path dataRoot;++ @BeforeEach+ void setUp() {+ dataRoot = tempDir.resolve("s3");+ objectStore = new InMemoryStorage<>();+ s3Service = new S3Service(new InMemoryStorage<>(), objectStore, dataRoot, false);+ s3Service.createBucket("source-bucket", "us-east-1");+ s3Service.createBucket("dest-bucket", "us-east-1");+ }++ @Test+ void copyOfASinglePartObjectKeepsItsETagAndChecksumAndSharesItsFile() throws IOException {+ S3Object source = s3Service.putObject("source-bucket", "source.txt",+ "single part body".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());++ S3Object copy = s3Service.copyObject("source-bucket", "source.txt", "dest-bucket", "copy.txt",+ new CopyObjectOptions());++ assertEquals(source.getETag(), copy.getETag());+ assertEquals(source.getChecksum().getChecksumCRC64NVME(), copy.getChecksum().getChecksumCRC64NVME());+ assertArrayEquals("single part body".getBytes(StandardCharsets.UTF_8),+ s3Service.getObject("dest-bucket", "copy.txt").getData());+ List<Path> files = objectFiles();+ assertEquals(2, files.size(), "the source file and the copy's file: " + files);+ if (hardLinksSupported()) {+ assertTrue(Files.isSameFile(files.get(0), files.get(1)), "the copy should share the source's file");+ }+ }++ @Test+ void copyOfAMultipartObjectGetsTheMd5OfItsBytesAsItsETag() {+ byte[] part1 = "first part of the source, ".getBytes(StandardCharsets.UTF_8);+ byte[] part2 = "second part of the source".getBytes(StandardCharsets.UTF_8);+ MultipartUpload upload = s3Service.initiateMultipartUpload("source-bucket", "multipart.bin", null);+ s3Service.uploadPart("source-bucket", "multipart.bin", upload.getUploadId(), 1, part1);+ s3Service.uploadPart("source-bucket", "multipart.bin", upload.getUploadId(), 2, part2);+ S3Object source = s3Service.completeMultipartUpload("source-bucket", "multipart.bin",+ upload.getUploadId(), List.of(1, 2), null, null);+ assertTrue(source.getETag().endsWith("-2\""), "test setup: a multipart ETag " + source.getETag());++ S3Object copy = s3Service.copyObject("source-bucket", "multipart.bin", "dest-bucket", "copy.bin",+ new CopyObjectOptions());++ byte[] whole = concat(part1, part2);+ assertEquals(S3Object.computeETag(whole), copy.getETag());+ assertEquals(source.getChecksum().getChecksumCRC64NVME(), copy.getChecksum().getChecksumCRC64NVME(),+ "a full-object checksum carries over");+ assertArrayEquals(whole, s3Service.getObject("dest-bucket", "copy.bin").getData());+ }++ @Test+ void copyWithANewChecksumAlgorithmComputesItOverTheBytes() {+ byte[] body = "body to checksum again".getBytes(StandardCharsets.UTF_8);+ s3Service.putObject("source-bucket", "source.txt", body, "text/plain", Map.of());++ S3Object copy = s3Service.copyObject("source-bucket", "source.txt", "dest-bucket", "copy.txt",+ new CopyObjectOptions().withChecksumAlgorithm("SHA256"));++ assertEquals(S3Checksum.sha256Base64(body), copy.getChecksum().getChecksumSHA256());+ assertEquals(ChecksumType.FULL_OBJECT, copy.getChecksum().getChecksumType());+ }++ @Test+ void copyOfACompositeChecksumObjectGetsAFullObjectChecksumOfItsBytes() {+ byte[] part1 = "composite part one ".getBytes(StandardCharsets.UTF_8);+ byte[] part2 = "composite part two".getBytes(StandardCharsets.UTF_8);+ MultipartUpload upload = s3Service.initiateMultipartUpload("source-bucket", "composite.bin", null,+ null, null, null, null, null, null, null, null, "CRC32");+ s3Service.uploadPart("source-bucket", "composite.bin", upload.getUploadId(), 1, part1);+ s3Service.uploadPart("source-bucket", "composite.bin", upload.getUploadId(), 2, part2);+ S3Object source = s3Service.completeMultipartUpload("source-bucket", "composite.bin", upload.getUploadId(),+ List.of(1, 2), Map.of(1, S3Checksum.of(ChecksumAlgorithm.CRC32, part1),+ 2, S3Checksum.of(ChecksumAlgorithm.CRC32, part2)), null, null);+ assertEquals(ChecksumType.COMPOSITE, source.getChecksum().getChecksumType(), "test setup");++ S3Object copy = s3Service.copyObject("source-bucket", "composite.bin", "dest-bucket", "copy.bin",+ new CopyObjectOptions());++ assertEquals(S3Checksum.crc32Base64(concat(part1, part2)), copy.getChecksum().getChecksumCRC32());+ assertEquals(ChecksumType.FULL_OBJECT, copy.getChecksum().getChecksumType());+ }++ @Test+ void failedCopyLeavesNoPinnedFileBehind() throws IOException {+ s3Service.putObject("source-bucket", "source.txt", "source".getBytes(StandardCharsets.UTF_8),+ "text/plain", Map.of());+ s3Service.putObject("dest-bucket", "taken.txt", "already here".getBytes(StandardCharsets.UTF_8),+ "text/plain", Map.of());++ assertThrows(AwsException.class, () -> s3Service.copyObject("source-bucket", "source.txt",+ "dest-bucket", "taken.txt", new CopyObjectOptions().withIfNoneMatch("*")));++ try (Stream<Path> files = Files.walk(dataRoot)) {+ assertEquals(List.of(), files.filter(path -> path.getFileName().toString().contains(".tmp-")).toList());+ }+ assertArrayEquals("already here".getBytes(StandardCharsets.UTF_8),+ s3Service.getObject("dest-bucket", "taken.txt").getData());+ }++ @Test+ void copyIntoAVersionedBucketStoresTheNewVersionAndTheCurrentFile() {+ s3Service.putBucketVersioning("dest-bucket", "Enabled");+ s3Service.putObject("source-bucket", "source.txt", "versioned copy".getBytes(StandardCharsets.UTF_8),+ "text/plain", Map.of());++ S3Object copy = s3Service.copyObject("source-bucket", "source.txt", "dest-bucket", "copy.txt",+ new CopyObjectOptions());++ assertNotNull(copy.getVersionId());+ assertArrayEquals("versioned copy".getBytes(StandardCharsets.UTF_8),+ s3Service.getObject("dest-bucket", "copy.txt").getData());+ assertArrayEquals("versioned copy".getBytes(StandardCharsets.UTF_8),+ s3Service.getObject("dest-bucket", "copy.txt", copy.getVersionId()).getData());+ }++ @Test+ void copyOfASourceVersionCopiesThatVersionAfterTheKeyMovedOn() {+ s3Service.putBucketVersioning("source-bucket", "Enabled");+ S3Object first = s3Service.putObject("source-bucket", "source.txt", "first".getBytes(StandardCharsets.UTF_8),+ "text/plain", Map.of());+ s3Service.putObject("source-bucket", "source.txt", "second".getBytes(StandardCharsets.UTF_8),+ "text/plain", Map.of());++ s3Service.copyObject("source-bucket", "source.txt", "dest-bucket", "copy.txt", first.getVersionId(),+ new CopyObjectOptions());++ assertArrayEquals("first".getBytes(StandardCharsets.UTF_8),+ s3Service.getObject("dest-bucket", "copy.txt").getData());+ }++ @Test+ void copySourceLargerThanFiveGibibytesIsAnInvalidRequest() {+ assertEquals(5_368_709_120L, S3Service.MAX_COPY_OBJECT_SOURCE_SIZE);+ S3Object atLimit = new S3Object("source-bucket", "at-limit.bin", 5_368_709_120L, null, "\"etag\"");+ assertDoesNotThrow(() -> S3Service.requireCopyableSize(atLimit));++ S3Object overLimit = new S3Object("source-bucket", "over-limit.bin", 5_368_709_121L, null, "\"etag\"");+ AwsException error = assertThrows(AwsException.class, () -> S3Service.requireCopyableSize(overLimit));++ assertEquals("InvalidRequest", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("The specified copy source is larger than the maximum allowable size for a copy source: 5368709120",+ error.getMessage());+ }++ @Test+ void oversizedSourceIsRejectedBeforeItsFileIsPinned() throws IOException {+ s3Service.putObject("source-bucket", "oversized.bin", "small".getBytes(StandardCharsets.UTF_8),+ "application/octet-stream", Map.of());+ // Metadata over 5 GiB on top of a file that is gone: a copy that went to the file first would+ // fail on the missing file instead of answering InvalidRequest from the metadata.+ objectStore.scan(key -> true).forEach(object -> object.setSize(S3Service.MAX_COPY_OBJECT_SOURCE_SIZE + 1));+ for (Path file : objectFiles()) {+ Files.delete(file);+ }++ AwsException error = assertThrows(AwsException.class, () -> s3Service.copyObject("source-bucket",+ "oversized.bin", "dest-bucket", "copy.bin", new CopyObjectOptions()));++ assertEquals("InvalidRequest", error.getErrorCode());+ }++ private List<Path> objectFiles() throws IOException {+ try (Stream<Path> files = Files.walk(dataRoot)) {+ return files.filter(path -> path.getFileName().toString().endsWith(".s3data")).toList();+ }+ }++ private boolean hardLinksSupported() throws IOException {+ Path probe = Files.createTempFile(tempDir, "link-probe", null);+ Path link = probe.resolveSibling(probe.getFileName() + ".link");+ try {+ Files.createLink(link, probe);+ return true;+ } catch (UnsupportedOperationException | IOException unsupported) {+ return false;+ } finally {+ Files.deleteIfExists(link);+ Files.deleteIfExists(probe);+ }+ }++ private static byte[] concat(byte[] first, byte[] second) {+ byte[] result = new byte[first.length + second.length];+ System.arraycopy(first, 0, result, 0, first.length);+ System.arraycopy(second, 0, result, first.length, second.length);+ return result;+ }+}