Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): complete multipart uploads larger than 2 GiB in disk storage modes (#4976)
Priamo Germosen2e87534941cf
diff --git a/docs/services/s3.md b/docs/services/s3.mdindex 52d4fd3752..ca3a24f5d5 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -388,6 +388,18 @@ longer grows with the number of stored objects. Bucket definitions and the otherstill rewritten on every change. See[Storage Modes](../configuration/storage.md#journaled-stores-under-persistent-mode).+In `memory` mode each object body is a single in-memory byte array, so an object can hold at most+about 2 GiB: an `UploadPart` that takes a multipart upload past that, or a+`CompleteMultipartUpload` over it, fails with `EntityTooLarge`. The disk-backed modes assemble a+multipart upload file to file, so the object never passes through the heap; set+`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. While it runs,+`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembled+copy, until the parts are removed. Multipart uploads in progress do not survive a restart or a+reset, and their files are removed when either happens. `CopyObject`,+`UploadPartCopy` and S3 Select still read the whole source object into memory in every mode, so+they need a source under 2 GiB, and deleting the current version of a key in a versioned bucket+reads the version that takes its place the same way.+## Examples```bashdiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex ffd0a01aa6..3b6795bac7 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -1,15 +1,18 @@package io.github.hectorvent.floci.services.s3;+import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.AwsNamespaces;import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.common.ServicePrincipals;import io.github.hectorvent.floci.core.common.XmlBuilder;import io.github.hectorvent.floci.core.common.XmlParser;-import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.resource.ResourceProvider;import io.github.hectorvent.floci.core.resource.SupportedResourceType;@@ -17,6 +20,7 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.core.storage.WriteProfile;+import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator;import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator.ResourcePolicyDecision;import io.github.hectorvent.floci.services.iam.IamService;@@ -26,12 +30,8 @@ import io.github.hectorvent.floci.services.lambda.LambdaService;import io.github.hectorvent.floci.services.lambda.model.InvocationType;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.s3.model.*;-import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;import io.github.hectorvent.floci.services.sns.SnsService;import io.github.hectorvent.floci.services.sqs.SqsService;-import com.fasterxml.jackson.core.type.TypeReference;-import com.fasterxml.jackson.databind.ObjectMapper;-import com.fasterxml.jackson.databind.node.ObjectNode;import jakarta.enterprise.context.ApplicationScoped;import jakarta.enterprise.event.Event;import jakarta.enterprise.inject.Instance;@@ -48,17 +48,18 @@ import java.net.URLEncoder;import java.nio.channels.Channels;import java.nio.channels.FileChannel;import java.nio.charset.StandardCharsets;-import java.nio.file.NoSuchFileException;import java.nio.file.AtomicMoveNotSupportedException;+import java.nio.file.FileSystemException;import java.nio.file.Files;+import java.nio.file.NoSuchFileException;import java.nio.file.Path;import java.nio.file.StandardCopyOption;import java.nio.file.StandardOpenOption;import java.security.MessageDigest;import java.security.NoSuchAlgorithmException;import java.time.Instant;-import java.time.temporal.ChronoUnit;import java.time.format.DateTimeFormatter;+import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.Arrays;import java.util.Base64;@@ -84,6 +85,7 @@ import java.util.function.Function;import java.util.function.Supplier;import java.util.regex.Matcher;import java.util.regex.Pattern;+import java.util.stream.Stream;@ApplicationScopedpublic class S3Service implements Resettable, ResourceProvider {@@ -206,7 +208,10 @@ public class S3Service implements Resettable, ResourceProvider {}return locks;}- private final ConcurrentHashMap<String, Map<Integer, byte[]>> memoryMultipartStore = new ConcurrentHashMap<>();+ // Memory mode keeps an object in one byte array; this is the largest the JDK reliably allocates.+ private static final long MAX_IN_MEMORY_OBJECT_SIZE = Integer.MAX_VALUE - 8;+ // Upload ID to part bytes, keyed by each part's storage ID.+ private final ConcurrentHashMap<String, Map<String, byte[]>> memoryMultipartStore = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, MultipartUpload> multipartUploads = new ConcurrentHashMap<>();// Account-level (S3 Control) Block Public Access config, one entry per AWS account.// Distinct from the bucket-level configuration held on each Bucket. Block Public Access is a@@ -376,6 +381,7 @@ public class S3Service implements Resettable, ResourceProvider {} catch (IOException e) {throw new UncheckedIOException("Failed to create S3 data directory: " + dataRoot, e);}+ deleteMultipartFiles();}}@@ -391,9 +397,31 @@ public class S3Service implements Resettable, ResourceProvider {// default account alone is not enough). Mirrors the metadata erase; the pre-existing// .s3data behavior is unchanged.deleteAnnotationPayloadRoots();+ deleteMultipartFiles();}}+ /**+ * Deletes the part and assembled files of every multipart upload. Uploads are tracked only in+ * memory, so at startup or after a reset these files belong to uploads that no longer exist, and+ * an assembly a crash interrupted can be as large as the object it was building.+ */+ private void deleteMultipartFiles() {+ Path multipartRoot = dataRoot.resolve(".multipart");+ if (!Files.isDirectory(multipartRoot)) {+ return;+ }+ try (Stream<Path> uploads = Files.list(multipartRoot)) {+ long count = uploads.count();+ if (count > 0) {+ LOG.infov("Removing the files of {0} multipart uploads that no longer exist under {1}", count, multipartRoot);+ }+ } catch (IOException e) {+ LOG.warnv(e, "Failed to list {0} before removing it", multipartRoot);+ }+ deleteDirectory(multipartRoot);+ }+private void deleteAnnotationPayloadRoots() {Path accountsRoot = dataRoot.resolve(ACCOUNT_STORAGE_ROOT);if (!Files.isDirectory(accountsRoot)) {@@ -620,18 +648,45 @@ public class S3Service implements Resettable, ResourceProvider {private S3Object storeObject(String bucketName, String key, byte[] data,String contentType, Map<String, String> metadata,S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {+ return storeObject(bucketName, key, new BytesBody(data), contentType, metadata, checksum, parts, options, eTag);+ }++ private S3Object storeObject(String bucketName, String key, ObjectBody body,+ String contentType, Map<String, String> metadata,+ S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {AccountAwareStorageBackend.OwnedEntry<Bucket> ownedBucket = resolveBucketEntry(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));Bucket bucket = ownedBucket.value();synchronized (bucket) {- return storeObjectInternal(ownedBucket.account(), bucket, bucketName, key, data,+ return storeObjectInternal(ownedBucket.account(), bucket, bucketName, key, body,contentType, metadata, checksum, parts, options, eTag);}}+ /**+ * The body of an object being stored: the bytes of a single-request write, or the file a+ * multipart upload was assembled into, which is moved into place instead of being read back.+ */+ private sealed interface ObjectBody permits BytesBody, AssembledBody {+ long size();+ }++ private record BytesBody(byte[] data) implements ObjectBody {+ @Override+ public long size() {+ return data.length;+ }+ }++ /**+ * A disk-mode multipart object. It is stored with the ETag and checksum built from its parts,+ * since computing either here would mean reading the whole file.+ */+ private record AssembledBody(Path file, long size) implements ObjectBody { }+private S3Object storeObjectInternal(String bucketOwnerAccount, Bucket bucket,- String bucketName, String key, byte[] data,+ String bucketName, String key, ObjectBody body,String contentType, Map<String, String> metadata,S3Checksum checksum, List<Part> parts, PutObjectOptions options,String eTag) {@@ -641,8 +696,10 @@ public class S3Service implements Resettable, ResourceProvider {rejectConflictingServerSideEncryption(normalizedServerSideEncryption, sseCustomerKey);checkWritePreconditions(bucketName, key, effectiveOptions.getIfMatch(), effectiveOptions.getIfNoneMatch());- S3Object object = new S3Object(bucketName, key, data, contentType,+ byte[] data = body instanceof BytesBody bytes ? bytes.data() : null;+ S3Object object = new S3Object(bucketName, key, body.size(), contentType,eTag != null ? eTag : computeETag(data));+ object.setData(data);if (metadata != null) {object.getMetadata().putAll(metadata);}@@ -721,8 +778,14 @@ public class S3Service implements Resettable, ResourceProvider {// versioned write fails, the canonical file - which unlocked GETs already associate// with the still-unpublished previous generation - is never touched, so a concurrent// GET can't observe corrupted "latest" bytes paired with the old metadata.- writeVersionedFile(bucketOwnerAccount, bucketName, key, versionId, data);- writeFile(bucketOwnerAccount, bucketName, key, data);+ if (body instanceof AssembledBody assembled) {+ Path versionedPath = resolveVersionedPath(bucketOwnerAccount, bucketName, key, versionId);+ moveIntoPlace(assembled.file(), versionedPath);+ linkIntoPlace(versionedPath, resolveObjectPath(bucketOwnerAccount, bucketName, key));+ } else {+ writeVersionedFile(bucketOwnerAccount, bucketName, key, versionId, data);+ writeFile(bucketOwnerAccount, bucketName, key, data);+ }// Deferred pre-versioning annotation cleanup: only after the replacement body is on// disk, so a failed write keeps the old body and its annotations together.if (dropPreVersioningAnnotations[0]) {@@ -735,7 +798,7 @@ public class S3Service implements Resettable, ResourceProvider {// Store versioned copy and update latest pointerputObjectForAccount(bucketOwnerAccount, versionedKey(bucketName, key, versionId), object);putObjectForAccount(bucketOwnerAccount, latestKey, object);- LOG.debugv("Put versioned object: {0}/{1} v={2} ({3} bytes)", bucketName, key, versionId, data.length);+ LOG.debugv("Put versioned object: {0}/{1} v={2} ({3} bytes)", bucketName, key, versionId, body.size());} else {S3Object prev = resolveObjectForAccount(bucketOwnerAccount, objectKey(bucketName, key)).orElse(null);@@ -757,7 +820,11 @@ public class S3Service implements Resettable, ResourceProvider {// Write the body before publishing metadata - see the comment in the versioned// branch above; the same ordering requirement applies here.- writeFile(bucketOwnerAccount, bucketName, key, data);+ if (body instanceof AssembledBody assembled) {+ moveIntoPlace(assembled.file(), resolveObjectPath(bucketOwnerAccount, bucketName, key));+ } else {+ writeFile(bucketOwnerAccount, bucketName, key, data);+ }// An overwrite replaces the object's annotations (AWS drops them on overwrite).// The cleanup runs only after the body write succeeds, so a failed PUT keeps the// old body together with its annotations; and before the new metadata is published,@@ -767,7 +834,7 @@ public class S3Service implements Resettable, ResourceProvider {// branch above; the same race applies here.object.setData(null);putObjectForAccount(bucketOwnerAccount, objectKey(bucketName, key), object);- LOG.debugv("Put object: {0}/{1} ({2} bytes)", bucketName, key, data.length);+ LOG.debugv("Put object: {0}/{1} ({2} bytes)", bucketName, key, body.size());}return object;}@@ -3501,12 +3568,21 @@ public class S3Service implements Resettable, ResourceProvider {}validateSseCustomerAccess(upload, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);+ // Each upload of a part gets its own storage, so a record and the bytes it describes are+ // published together and never change: a re-upload adds new bytes and drops the old ones.+ String storageId = partNumber + "-" + UUID.randomUUID();if (inMemory) {- memoryMultipartStore.get(uploadId).put(partNumber, data);+ // A re-uploaded part number replaces its earlier part, so that part does not count.+ long otherParts = upload.getParts().values().stream()+ .filter(part -> part.getPartNumber() != partNumber)+ .mapToLong(Part::getSize)+ .sum();+ requireFitsInMemory(uploadId, otherParts + data.length);+ memoryMultipartStore.get(uploadId).put(storageId, data);} else {- Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(partNumber));+ Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(storageId);try {- Files.write(partPath, data);+ Files.write(partPath, data, StandardOpenOption.CREATE_NEW);} catch (IOException e) {throw new UncheckedIOException("Failed to write multipart part", e);}@@ -3515,12 +3591,32 @@ public class S3Service implements Resettable, ResourceProvider {String eTag = computeETag(data);Part part = new Part(partNumber, eTag, data.length);part.setChecksum(S3Checksum.of(upload.getChecksumAlgorithm(), data));- upload.getParts().put(partNumber, part);+ part.setStorageId(storageId);+ Part replaced = upload.getParts().put(partNumber, part);+ if (replaced != null) {+ discardPartBytes(uploadId, replaced);+ }LOG.debugv("Uploaded part {0} for upload {1} ({2} bytes)", partNumber, uploadId, data.length);return part;});}+ /**+ * Drops the bytes of a part a re-upload replaced, once the new record is published, so no record+ * ever names bytes that changed after it was written.+ */+ private void discardPartBytes(String uploadId, Part replaced) {+ if (inMemory) {+ Map<String, byte[]> memoryParts = memoryMultipartStore.get(uploadId);+ if (memoryParts != null) {+ memoryParts.remove(replaced.getStorageId());+ }+ } else {+ deleteQuietly(dataRoot.resolve(".multipart").resolve(uploadId).resolve(replaced.getStorageId()),+ "bytes of a multipart part that was uploaded again");+ }+ }+public String uploadPartCopy(String destBucket, String destKey, String uploadId, int partNumber,String sourceBucket, String sourceKey, String sourceVersionId,String copySourceRange) {@@ -3590,6 +3686,8 @@ public class S3Service implements Resettable, ResourceProvider {ChecksumAlgorithm algorithm = upload.getChecksumAlgorithm() != null ? upload.getChecksumAlgorithm() : ChecksumAlgorithm.CRC64NVME;ChecksumType storedChecksumType = upload.getChecksumType() != null ? upload.getChecksumType() : ChecksumType.FULL_OBJECT;+ // Each record is read once, and everything below, the bytes included, follows these records.+ List<Part> parts = new ArrayList<>(partNumbers.size());int previousPartNumber = 0;for (int num : partNumbers) {if (num <= previousPartNumber) {@@ -3608,44 +3706,61 @@ public class S3Service implements Resettable, ResourceProvider {+ " has an invalid ETag.", 400);}validatePartChecksum(upload.getChecksumAlgorithm(), storedChecksumType, num, part, partChecksums.get(num));+ parts.add(part);}validateCompleteChecksumType(algorithm, storedChecksumType, ChecksumType.fromWireValue(checksumType));+ if (inMemory) {+ requireFitsInMemory(uploadId, parts.stream().mapToLong(Part::getSize).sum());+ }// Concatenate parts in ordertry {MessageDigest md = MessageDigest.getInstance("MD5");- for (int num : partNumbers) {+ for (Part part : parts) {// A part ETag is the MD5 of that part, so the composite hashes it without rehashing the data- String partETag = stripSurroundingQuotes(upload.getParts().get(num).getETag());+ String partETag = stripSurroundingQuotes(part.getETag());md.update(HexFormat.of().parseHex(partETag));}// Composite ETag: MD5 of concatenated part MD5s, suffixed with part countString compositeETag = "\"" + bytesToHex(md.digest()) + "-" + partNumbers.size() + "\"";- byte[] allData = concatenateParts(uploadId, partNumbers);-- List<Part> completedParts = partNumbers.stream()- .map(num -> copyPart(upload.getParts().get(num)))+ // Both checksum types come from the stored part checksums, so a mismatch is rejected+ // before any part is read.+ List<Part> completedParts = parts.stream()+ .map(S3Service::copyPart).toList();S3Checksum checksum = storedChecksumType == ChecksumType.COMPOSITE? S3Checksum.composite(algorithm, completedParts.stream().map(part -> part.getChecksum().valueFor(algorithm)).toList())- : S3Checksum.fullObject(algorithm, allData);+ : S3Checksum.fullObject(algorithm, completedParts);if (expectedChecksum != null && expectedChecksum.hasAnyValue()) {validateExpectedChecksum(checksum, expectedChecksum);}- S3Object object = storeObject(bucket, key, allData, upload.getContentType(), upload.getMetadata(),- checksum, completedParts,- new PutObjectOptions()- .withStorageClass(upload.getStorageClass())- .withContentDisposition(upload.getContentDisposition())- .withServerSideEncryption(upload.getServerSideEncryption())- .withSseKmsKeyId(upload.getSseKmsKeyId())- .withAcl(upload.getAcl())- .withTagging(upload.getTagging()),- compositeETag);++ ObjectBody body = inMemory+ ? new BytesBody(concatenateParts(uploadId, parts))+ : assembleParts(uploadId, parts);+ S3Object object;+ try {+ object = storeObject(bucket, key, body, upload.getContentType(), upload.getMetadata(),+ checksum, completedParts,+ new PutObjectOptions()+ .withStorageClass(upload.getStorageClass())+ .withContentDisposition(upload.getContentDisposition())+ .withServerSideEncryption(upload.getServerSideEncryption())+ .withSseKmsKeyId(upload.getSseKmsKeyId())+ .withAcl(upload.getAcl())+ .withTagging(upload.getTagging()),+ compositeETag);+ } finally {+ // A stored body was moved away, so this only removes the file of a failed store and+ // leaves the parts in place for a retry or an abort.+ if (body instanceof AssembledBody assembled) {+ deleteQuietly(assembled.file(), "assembled multipart object that was not stored");+ }+ }if (upload.getSseCustomerAlgorithm() != null) {object.setSseCustomerAlgorithm(upload.getSseCustomerAlgorithm());object.setSseCustomerKeyMd5(upload.getSseCustomerKeyMd5());@@ -3672,54 +3787,97 @@ public class S3Service implements Resettable, ResourceProvider {}/**- * Copies the parts, in order, into one array sized to the total upload. Disk parts are read- * straight into their slot, so assembly never holds a second full-size copy of the object.- * In-memory parts are copied from the arrays that were measured, so their sizes cannot change- * in between.+ * Rejects an upload that memory mode could not hold in one byte array, with the error S3 gives an+ * object over its size limit, so it fails at the part that crosses the limit instead of running+ * the heap out of memory or overflowing the array size at completion.*/- private byte[] concatenateParts(String uploadId, List<Integer> partNumbers) throws IOException {- Path partsDir = dataRoot.resolve(".multipart").resolve(uploadId);- Map<Integer, byte[]> memoryParts = inMemory ? memoryMultipartStore.get(uploadId) : null;- byte[][] memoryData = inMemory ? new byte[partNumbers.size()][] : null;- long[] partSizes = new long[partNumbers.size()];+ private static void requireFitsInMemory(String uploadId, long uploadSize) {+ if (uploadSize > MAX_IN_MEMORY_OBJECT_SIZE) {+ LOG.warnv("Multipart upload {0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",+ uploadId, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);+ throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);+ }+ }++ /**+ * Copies the in-memory bytes of {@code parts}, in order, into one array sized to the total+ * upload. The total is measured on the arrays being copied, so it holds whatever the records say.+ */+ private byte[] concatenateParts(String uploadId, List<Part> parts) {+ Map<String, byte[]> memoryParts = memoryMultipartStore.getOrDefault(uploadId, Map.of());+ byte[][] partData = new byte[parts.size()][];long totalSize = 0;- for (int i = 0; i < partNumbers.size(); i++) {- int num = partNumbers.get(i);- if (inMemory) {- memoryData[i] = memoryParts.get(num);- partSizes[i] = memoryData[i].length;- } else {- partSizes[i] = Files.size(partsDir.resolve(String.valueOf(num)));+ for (int i = 0; i < parts.size(); i++) {+ partData[i] = memoryParts.get(parts.get(i).getStorageId());+ if (partData[i] == null) {+ throw missingPartData(parts.get(i));}- totalSize += partSizes[i];+ totalSize += partData[i].length;}- byte[] allData = new byte[Math.toIntExact(totalSize)];+ requireFitsInMemory(uploadId, totalSize);+ byte[] allData = new byte[(int) totalSize];int offset = 0;- for (int i = 0; i < partNumbers.size(); i++) {- int size = (int) partSizes[i];- if (inMemory) {- System.arraycopy(memoryData[i], 0, allData, offset, size);- } else {- int num = partNumbers.get(i);- try (InputStream in = Files.newInputStream(partsDir.resolve(String.valueOf(num)))) {- readPart(in, allData, offset, size, num);+ for (byte[] part : partData) {+ System.arraycopy(part, 0, allData, offset, part.length);+ offset += part.length;+ }+ return allData;+ }++ /**+ * Copies the part files, in order, into one file in the upload's directory for storeObject to+ * move into place. The copy runs file to file, so the object never passes through the heap+ * whatever its size, and it happens before the bucket lock is taken. The file name is unique+ * per call, so two Completes racing on one upload never write the same file.+ */+ private AssembledBody assembleParts(String uploadId, List<Part> parts) throws IOException {+ Path partsDir = dataRoot.resolve(".multipart").resolve(uploadId);+ Path assembled = partsDir.resolve("assembled-" + UUID.randomUUID());+ long totalSize = 0;+ try (FileChannel out = FileChannel.open(assembled, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {+ for (Part part : parts) {+ try {+ appendPart(partsDir.resolve(part.getStorageId()), part, out);+ } catch (NoSuchFileException e) {+ throw missingPartData(part);}+ totalSize += part.getSize();}- offset += size;+ } catch (IOException | RuntimeException e) {+ deleteQuietly(assembled, "partially assembled multipart object");+ throw e;}- return allData;+ return new AssembledBody(assembled, totalSize);}/**- * Reads exactly {@code size} bytes of part {@code partNumber} into {@code dest} at {@code offset},- * failing if the part holds a different number of bytes than was measured before assembly.+ * Appends part {@code part} from {@code partFile} to {@code out}, failing if the file does not+ * hold the size recorded for the part: the ETag and checksum the object is stored with were+ * built from that record.*/- static void readPart(InputStream in, byte[] dest, int offset, int size, int partNumber) throws IOException {- if (in.readNBytes(dest, offset, size) != size || in.read() != -1) {- throw new IOException("Part " + partNumber + " changed size during assembly");+ static void appendPart(Path partFile, Part part, FileChannel out) throws IOException {+ long size = part.getSize();+ try (FileChannel in = FileChannel.open(partFile, StandardOpenOption.READ)) {+ long position = 0;+ while (position < size) {+ long transferred = in.transferTo(position, size - position, out);+ if (transferred <= 0) {+ break;+ }+ position += transferred;+ }+ if (position != size || in.size() != size) {+ throw new IOException("Part " + part.getPartNumber() + " changed size during assembly");+ }}}+ /** A part whose record names bytes that are no longer stored. */+ private static AwsException missingPartData(Part part) {+ return new AwsException("InvalidPart", "One or more of the specified parts could not be found. Part "+ + part.getPartNumber() + " has no stored data.", 400);+ }+private boolean etagsMatch(String storedETag, String submittedETag) {return stripSurroundingQuotes(storedETag).equals(stripSurroundingQuotes(submittedETag));}@@ -5567,16 +5725,66 @@ public class S3Service implements Resettable, ResourceProvider {Path tmp = filePath.resolveSibling(filePath.getFileName() + ".tmp-" + UUID.randomUUID());try {Files.write(tmp, data);+ replaceAtomically(tmp, filePath);+ } finally {+ Files.deleteIfExists(tmp);+ }+ }++ private static void replaceAtomically(Path source, Path target) throws IOException {+ try {+ Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);+ } catch (AtomicMoveNotSupportedException e) {+ // Rare filesystems (some network mounts) reject ATOMIC_MOVE; fall back to a plain+ // replace. This narrows but does not fully close the window, which is acceptable only+ // because the default overlay/ext filesystems used here support atomic rename.+ Files.move(source, target, StandardCopyOption.REPLACE_EXISTING);+ }+ }++ /**+ * Moves an assembled multipart object to {@code target} with the same all-or-nothing rename+ * {@link #atomicWrite} ends with: a concurrent reader sees the previous file or the whole new+ * one, and the move costs the same whatever the object's size.+ */+ private void moveIntoPlace(Path source, Path target) {+ ReentrantLock lock = diskFileLock(target);+ lock.lock();+ try {+ Files.createDirectories(target.getParent());+ replaceAtomically(source, target);+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to move assembled S3 object file into place", e);+ } finally {+ lock.unlock();+ }+ }++ /**+ * Gives {@code target} the file already stored at {@code source} through a hard link, so a+ * versioned multipart object is kept once instead of being copied under the bucket lock.+ * Sharing the file is safe because no object file is ever modified in place: every write+ * replaces its path with a rename, which leaves the file under the other name untouched.+ * A filesystem without hard links gets a copy instead.+ */+ private void linkIntoPlace(Path source, Path target) {+ ReentrantLock lock = diskFileLock(target);+ lock.lock();+ Path tmp = target.resolveSibling(target.getFileName() + ".tmp-" + UUID.randomUUID());+ try {+ Files.createDirectories(target.getParent());try {- Files.move(tmp, filePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);- } catch (AtomicMoveNotSupportedException e) {- // Rare filesystems (some network mounts) reject ATOMIC_MOVE; fall back to a plain- // replace. This narrows but does not fully close the window — acceptable only- // because the default overlay/ext filesystems used here support atomic rename.- Files.move(tmp, filePath, StandardCopyOption.REPLACE_EXISTING);+ Files.createLink(tmp, source);+ } catch (UnsupportedOperationException | FileSystemException e) {+ LOG.debugv(e, "No hard link for {0}, copying {1} instead", target, source);+ Files.copy(source, tmp);}+ replaceAtomically(tmp, target);+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to link S3 object file into place", e);} finally {- Files.deleteIfExists(tmp);+ deleteQuietly(tmp, "temporary link of a versioned multipart object that was not published");+ lock.unlock();}}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/Part.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/Part.javaindex c80617c5a7..7395d8037a 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/Part.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/Part.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.s3.model;+import com.fasterxml.jackson.annotation.JsonIgnore;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;@@ -13,6 +14,10 @@ public class Part {private long size;private S3Checksum checksum;private Instant lastModified;+ // Names this part's bytes while its upload is open: a file in the upload's directory, or a key+ // in memory. A re-upload stores under a new name, so the bytes under a name never change.+ @JsonIgnore+ private String storageId;public Part() {this.checksum = new S3Checksum();@@ -40,4 +45,7 @@ public class Part {public Instant getLastModified() { return lastModified; }public void setLastModified(Instant lastModified) { this.lastModified = lastModified; }++ public String getStorageId() { return storageId; }+ public void setStorageId(String storageId) { this.storageId = storageId; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.javaindex 0bd6615c0b..796fab3d0f 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java@@ -19,6 +19,10 @@ public class S3Checksum {// Slicing-by-8: row n holds the CRC of each byte value followed by n zero bytes.private static final long[][] CRC64_TABLES = buildCrc64Tables();+ private static final CrcCombiner CRC32_COMBINER = new CrcCombiner(0xEDB88320L, Integer.SIZE);+ private static final CrcCombiner CRC32C_COMBINER = new CrcCombiner(0x82F63B78L, Integer.SIZE);+ private static final CrcCombiner CRC64_NVME_COMBINER = new CrcCombiner(CRC64_NVME_POLY, Long.SIZE);+private String checksumCRC32;private String checksumCRC32C;private String checksumCRC64NVME;@@ -93,6 +97,31 @@ public class S3Checksum {return checksum;}+ /**+ * Full-object checksum of a multipart object, combined from each part's CRC and size so the+ * object's bytes are never read. S3 linearizes it the same way, which is why it offers full-object+ * checksums on multipart uploads only for the CRC algorithms.+ */+ public static S3Checksum fullObject(ChecksumAlgorithm algorithm, List<Part> parts) {+ ChecksumAlgorithm effective = algorithm != null ? algorithm : ChecksumAlgorithm.CRC64NVME;+ CrcCombiner combiner = switch (effective) {+ case CRC32 -> CRC32_COMBINER;+ case CRC32C -> CRC32C_COMBINER;+ case CRC64NVME -> CRC64_NVME_COMBINER;+ case SHA1, SHA256 -> throw new IllegalArgumentException(+ effective.wireValue() + " has no full-object multipart checksum");+ };+ // The CRC of no bytes is zero for all three algorithms, so zero seeds the fold.+ long crc = 0;+ for (Part part : parts) {+ crc = combiner.combine(crc, combiner.decode(part.getChecksum().valueFor(effective)), part.getSize());+ }+ S3Checksum checksum = new S3Checksum();+ checksum.setValueFor(effective, combiner.encode(crc));+ checksum.setChecksumType(ChecksumType.FULL_OBJECT);+ return checksum;+ }+public static S3Checksum composite(ChecksumAlgorithm algorithm, List<String> partChecksums) {S3Checksum checksum = new S3Checksum();checksum.setValueFor(algorithm, algorithm.composite(partChecksums));@@ -220,4 +249,69 @@ public class S3Checksum {}return tables;}++ /**+ * Joins the CRCs of two adjacent byte ranges, as zlib's {@code crc32_combine} does, for a reflected+ * CRC whose initial value and final XOR are both all ones (CRC32, CRC32C and CRC64NVME all are).+ * With those parameters crc(A + B) = crc(A) * x^(8 * len(B)) mod P xor crc(B), so the second range's+ * length is all that is needed besides the two CRCs. In the reflected form the top bit is x^0.+ */+ private static final class CrcCombiner {++ private final long polynomial;+ private final int width;+ private final long one;+ // Entry k is x^(2^k) mod P. Lengths are counted in bytes, so x^(8n) starts at k = 3.+ private final long[] xToPowerOfTwo = new long[Long.SIZE + 3];++ CrcCombiner(long polynomial, int width) {+ this.polynomial = polynomial;+ this.width = width;+ this.one = 1L << (width - 1);+ long power = one >>> 1;+ for (int k = 0; k < xToPowerOfTwo.length; k++) {+ xToPowerOfTwo[k] = power;+ power = multiply(power, power);+ }+ }++ long combine(long first, long second, long secondLength) {+ long shift = one;+ int k = 3;+ for (long remaining = secondLength; remaining != 0; remaining >>>= 1) {+ if ((remaining & 1) != 0) {+ shift = multiply(xToPowerOfTwo[k], shift);+ }+ k++;+ }+ return multiply(shift, first) ^ second;+ }++ long decode(String base64) {+ long crc = 0;+ for (byte b : Base64.getDecoder().decode(base64)) {+ crc = (crc << Byte.SIZE) | (b & 0xFF);+ }+ return crc;+ }++ String encode(long crc) {+ byte[] bytes = new byte[width / Byte.SIZE];+ for (int i = bytes.length - 1; i >= 0; i--) {+ bytes[i] = (byte) (crc >>> ((bytes.length - 1 - i) * Byte.SIZE));+ }+ return Base64.getEncoder().encodeToString(bytes);+ }++ private long multiply(long a, long b) {+ long product = 0;+ for (long bit = one; bit != 0; bit >>>= 1) {+ if ((a & bit) != 0) {+ product ^= b;+ }+ b = (b & 1) != 0 ? (b >>> 1) ^ polynomial : b >>> 1;+ }+ return product;+ }+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.javaindex cff8b443a6..4b00096795 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java@@ -66,11 +66,16 @@ public class S3Object {}public S3Object(String bucketName, String key, byte[] data, String contentType, String eTag) {+ this(bucketName, key, data.length, contentType, eTag);+ this.data = data;+ }++ /** An object whose {@code size}-byte body is stored on disk rather than held in memory. */+ public S3Object(String bucketName, String key, long size, String contentType, String eTag) {this.bucketName = bucketName;this.key = key;- this.data = data;this.contentType = contentType != null ? contentType : "application/octet-stream";- this.size = data.length;+ this.size = size;this.lastModified = Instant.now().truncatedTo(ChronoUnit.MILLIS);this.eTag = eTag;this.metadata = new HashMap<>();diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.javaindex 6a61ee5eb7..6a53faa296 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java@@ -3,15 +3,19 @@ package io.github.hectorvent.floci.services.s3;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;import io.github.hectorvent.floci.services.s3.model.ChecksumType;+import io.github.hectorvent.floci.services.s3.model.Part;import io.github.hectorvent.floci.services.s3.model.S3Checksum;import org.junit.jupiter.api.Test;+import java.io.ByteArrayOutputStream;import java.nio.ByteBuffer;import java.nio.charset.StandardCharsets;import java.util.ArrayList;+import java.util.Arrays;import java.util.Base64;import java.util.List;import java.util.Random;+import java.util.zip.CRC32C;import static org.junit.jupiter.api.Assertions.*;@@ -109,6 +113,88 @@ class S3ChecksumTest {assertEquals(ChecksumType.FULL_OBJECT, checksum.getChecksumType());}+ @Test+ void fullObjectCombinedFromPartsMatchesTheChecksumOfTheWholeObject() {+ Random random = new Random(4145);+ List<int[]> splits = List.of(+ new int[]{0},+ new int[]{1},+ new int[]{5 * 1024 * 1024, 17},+ new int[]{0, 9, 0, 64, 1},+ new int[]{1023, 1024, 1025, 65537, 3});+ for (ChecksumAlgorithm algorithm : List.of(ChecksumAlgorithm.CRC32, ChecksumAlgorithm.CRC32C,+ ChecksumAlgorithm.CRC64NVME)) {+ for (int[] sizes : splits) {+ ByteArrayOutputStream whole = new ByteArrayOutputStream();+ List<Part> parts = new ArrayList<>();+ for (int i = 0; i < sizes.length; i++) {+ byte[] data = new byte[sizes[i]];+ random.nextBytes(data);+ whole.writeBytes(data);+ parts.add(part(i + 1, algorithm, data));+ }++ S3Checksum combined = S3Checksum.fullObject(algorithm, parts);++ assertEquals(algorithm.compute(whole.toByteArray()), combined.valueFor(algorithm),+ algorithm + " " + Arrays.toString(sizes));+ assertEquals(ChecksumType.FULL_OBJECT, combined.getChecksumType());+ }+ }+ }++ @Test+ void fullObjectCombinesPartsLongerThanAnArrayCanHold() {+ long zeroRunLength = (1L << 31) + 13;+ byte[] head = "head".getBytes(StandardCharsets.UTF_8);+ byte[] tail = "tail".getBytes(StandardCharsets.UTF_8);+ CRC32C zeroRun = new CRC32C();+ CRC32C whole = new CRC32C();+ whole.update(head);+ byte[] zeros = new byte[1 << 20];+ for (long remaining = zeroRunLength; remaining > 0; remaining -= zeros.length) {+ int chunk = (int) Math.min(zeros.length, remaining);+ zeroRun.update(zeros, 0, chunk);+ whole.update(zeros, 0, chunk);+ }+ whole.update(tail);+ Part middle = new Part(2, "\"etag\"", zeroRunLength);+ middle.getChecksum().setChecksumCRC32C(crc32Base64(zeroRun.getValue()));++ S3Checksum combined = S3Checksum.fullObject(ChecksumAlgorithm.CRC32C, List.of(+ part(1, ChecksumAlgorithm.CRC32C, head), middle, part(3, ChecksumAlgorithm.CRC32C, tail)));++ assertEquals(crc32Base64(whole.getValue()), combined.getChecksumCRC32C());+ }++ @Test+ void fullObjectFromPartsWithoutDeclaredAlgorithmGetsCrc64Nvme() {+ byte[] first = "part1".getBytes(StandardCharsets.UTF_8);+ byte[] second = "part2".getBytes(StandardCharsets.UTF_8);++ S3Checksum combined = S3Checksum.fullObject(null, List.of(part(1, null, first), part(2, null, second)));++ assertEquals(S3Checksum.crc64NvmeBase64("part1part2".getBytes(StandardCharsets.UTF_8)),+ combined.getChecksumCRC64NVME());+ }++ @Test+ void fullObjectFromPartsRejectsShaAlgorithms() {+ List<Part> parts = List.of(part(1, ChecksumAlgorithm.SHA256, "data".getBytes(StandardCharsets.UTF_8)));++ assertThrows(IllegalArgumentException.class, () -> S3Checksum.fullObject(ChecksumAlgorithm.SHA256, parts));+ }++ private static Part part(int partNumber, ChecksumAlgorithm algorithm, byte[] data) {+ Part part = new Part(partNumber, "\"etag\"", data.length);+ part.setChecksum(S3Checksum.of(algorithm, data));+ return part;+ }++ private static String crc32Base64(long value) {+ return Base64.getEncoder().encodeToString(ByteBuffer.allocate(Integer.BYTES).putInt((int) value).array());+ }+@Testvoid objectAttributesViewDropsTheSuffixOfCompositeValuesOnly() {S3Checksum composite = S3Checksum.composite(ChecksumAlgorithm.CRC32, List.of("W0QpDQ==", "anCi7Q=="));diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.javaindex b963297290..5ed89211b1 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java@@ -4,26 +4,32 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.s3.model.ChecksumType;import io.github.hectorvent.floci.services.s3.model.MultipartUpload;+import io.github.hectorvent.floci.services.s3.model.Part;import io.github.hectorvent.floci.services.s3.model.S3Checksum;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;-import java.io.ByteArrayInputStream;import java.io.IOException;-import java.io.UncheckedIOException;+import java.nio.ByteBuffer;+import java.nio.channels.FileChannel;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import java.nio.file.StandardOpenOption;import java.security.MessageDigest;import java.security.NoSuchAlgorithmException;import java.util.Arrays;import java.util.HexFormat;import java.util.List;+import java.util.Map;+import java.util.stream.Stream;import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotEquals;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -129,39 +135,192 @@ class S3MultipartDiskAssemblyTest {// Metadata (the Part record with its ETag) lives in memory, separate from the on-disk part// file that assembly reads; deleting only the file makes assembly fail after the// file-independent validation checks have passed.- Path part2File = dataRoot.resolve(".multipart").resolve(upload.getUploadId()).resolve("2");- assertTrue(Files.deleteIfExists(part2File), "test setup: part 2 file should exist before deletion");+ assertTrue(Files.deleteIfExists(partFile(upload, 2)), "test setup: part 2 file should exist before deletion");- assertThrows(UncheckedIOException.class, () -> s3Service.completeMultipartUpload(+ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload("test-bucket", "vanished.bin", upload.getUploadId(), List.of(1, 2), null, null));+ assertEquals("InvalidPart", error.getErrorCode());+assertThrows(AwsException.class,() -> s3Service.getObject("test-bucket", "vanished.bin"));+ assertEquals(List.of(), assembledFiles(upload), "the partial assembly should be removed");+ }++ @Test+ void failedStoreRemovesTheAssembledFileAndKeepsThePartsForARetry() throws Exception {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "public.bin", null,+ null, null, null, null, "public-read");+ s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));+ // The public ACL is only rejected once the parts are assembled, when the object is stored.+ s3Service.putPublicAccessBlock("test-bucket",+ "<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls></PublicAccessBlockConfiguration>");++ assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "public.bin", upload.getUploadId(), List.of(1, 2), null, null));++ assertEquals(List.of(), assembledFiles(upload), "the unstored assembly should be removed");+ s3Service.putPublicAccessBlock("test-bucket",+ "<PublicAccessBlockConfiguration><BlockPublicAcls>false</BlockPublicAcls></PublicAccessBlockConfiguration>");+ s3Service.completeMultipartUpload("test-bucket", "public.bin", upload.getUploadId(), List.of(1, 2), null, null);+ assertArrayEquals("part1part2".getBytes(StandardCharsets.UTF_8),+ s3Service.getObject("test-bucket", "public.bin").getData());+ assertFalse(Files.exists(partsDir(upload)), "a completed upload should leave no files behind");+ }++ @Test+ void versionedMultipartObjectIsStoredOnceAndKeepsItsBytesWhenTheKeyIsOverwritten() throws Exception {+ s3Service.putBucketVersioning("test-bucket", "Enabled");+ byte[] part1 = repeatingBytes((byte) 'V', 9_000);+ byte[] part2 = repeatingBytes((byte) 'W', 1_000);+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "versioned.bin", null);+ s3Service.uploadPart("test-bucket", "versioned.bin", upload.getUploadId(), 1, part1);+ s3Service.uploadPart("test-bucket", "versioned.bin", upload.getUploadId(), 2, part2);++ String versionId = s3Service.completeMultipartUpload("test-bucket", "versioned.bin", upload.getUploadId(),+ List.of(1, 2), null, null).getVersionId();++ List<Path> stored = storedObjectFiles();+ assertEquals(2, stored.size(), "a versioned object has a versioned file and a current file: " + stored);+ assertTrue(Files.isSameFile(stored.get(0), stored.get(1)), "the two should be one file, not a copy");++ byte[] replacement = "replacement".getBytes(StandardCharsets.UTF_8);+ String replacementVersionId = s3Service.putObject("test-bucket", "versioned.bin", replacement,+ "text/plain", Map.of()).getVersionId();+ assertArrayEquals(replacement, s3Service.getObject("test-bucket", "versioned.bin").getData());+ assertArrayEquals(concat(part1, part2), s3Service.getObject("test-bucket", "versioned.bin", versionId).getData());++ s3Service.deleteObject("test-bucket", "versioned.bin", replacementVersionId);+ assertArrayEquals(concat(part1, part2), s3Service.getObject("test-bucket", "versioned.bin").getData());+ }++ @Test+ void mismatchedFullObjectChecksumIsRejectedBeforeAnyPartIsRead() throws Exception {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "bad-digest.bin", null);+ s3Service.uploadPart("test-bucket", "bad-digest.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "bad-digest.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));+ // Without its part files, a Complete that reached assembly would fail with an I/O error instead.+ assertTrue(Files.deleteIfExists(partFile(upload, 1)), "test setup: part 1 file should exist before deletion");+ assertTrue(Files.deleteIfExists(partFile(upload, 2)), "test setup: part 2 file should exist before deletion");+ S3Checksum wrong = new S3Checksum();+ wrong.setChecksumCRC64NVME(S3Checksum.crc64NvmeBase64("other".getBytes(StandardCharsets.UTF_8)));++ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "bad-digest.bin", upload.getUploadId(), List.of(1, 2), null, wrong));++ assertEquals("BadDigest", error.getErrorCode());+ }++ @Test+ void reuploadedPartGetsANewFileAndTheReplacedFileIsRemoved() throws Exception {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "reupload.bin", null);+ s3Service.uploadPart("test-bucket", "reupload.bin", upload.getUploadId(), 1, "first".getBytes(StandardCharsets.UTF_8));+ Path firstFile = partFile(upload, 1);++ s3Service.uploadPart("test-bucket", "reupload.bin", upload.getUploadId(), 1, "other".getBytes(StandardCharsets.UTF_8));++ Path secondFile = partFile(upload, 1);+ assertNotEquals(firstFile, secondFile, "a part file is never rewritten in place");+ assertFalse(Files.exists(firstFile), "the replaced part's file should be removed");+ assertArrayEquals("other".getBytes(StandardCharsets.UTF_8), Files.readAllBytes(secondFile));+ s3Service.completeMultipartUpload("test-bucket", "reupload.bin", upload.getUploadId(), List.of(1), null, null);+ assertArrayEquals("other".getBytes(StandardCharsets.UTF_8), s3Service.getObject("test-bucket", "reupload.bin").getData());+ }++ @Test+ void completeThatReadAReplacedPartsRecordFailsInsteadOfStoringTheNewBytes() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "raced.bin", null);+ s3Service.uploadPart("test-bucket", "raced.bin", upload.getUploadId(), 1, "first".getBytes(StandardCharsets.UTF_8));+ Part staleRecord = upload.getParts().get(1);+ s3Service.uploadPart("test-bucket", "raced.bin", upload.getUploadId(), 1, "other".getBytes(StandardCharsets.UTF_8));+ // A record whose bytes a same-size re-upload replaced. The upload lock keeps a Complete from+ // reading one, but if it ever did, it must fail rather than store the new bytes.+ upload.getParts().put(1, staleRecord);++ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "raced.bin", upload.getUploadId(), List.of(1), null, null));++ assertEquals("InvalidPart", error.getErrorCode());+ assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "raced.bin"));+ }++ @Test+ void restartRemovesTheFilesOfUploadsThatWereInProgress() throws Exception {+ s3Service.putObject("test-bucket", "kept.bin", "kept".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "interrupted.bin", null);+ s3Service.uploadPart("test-bucket", "interrupted.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));+ // What a crash in the middle of CompleteMultipartUpload leaves next to the parts.+ Files.write(partsDir(upload).resolve("assembled-interrupted"), new byte[1024]);++ new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);++ assertFalse(Files.exists(dataRoot.resolve(".multipart")), "no upload outlives the process that held it");+ assertEquals(1, storedObjectFiles().size(), "stored objects are left alone");+ }++ @Test+ void resetRemovesTheFilesOfUploadsInProgress() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "reset.bin", null);+ s3Service.uploadPart("test-bucket", "reset.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));++ s3Service.clear();++ assertFalse(Files.exists(dataRoot.resolve(".multipart")), "a reset forgets every upload, so their files go too");}@Test- void partReadCopiesExactlyTheMeasuredBytes() throws IOException {- byte[] dest = new byte[7];+ void appendedPartAddsExactlyItsRecordedBytes() throws IOException {+ Path partFile = Files.write(tempDir.resolve("part"), "abcde".getBytes(StandardCharsets.UTF_8));+ Path assembled = tempDir.resolve("assembled");- S3Service.readPart(new ByteArrayInputStream("abcde".getBytes(StandardCharsets.UTF_8)), dest, 1, 5, 1);+ try (FileChannel out = FileChannel.open(assembled, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {+ out.write(ByteBuffer.wrap("X".getBytes(StandardCharsets.UTF_8)));+ S3Service.appendPart(partFile, new Part(1, "\"etag\"", 5), out);+ }- assertArrayEquals(new byte[] {0, 'a', 'b', 'c', 'd', 'e', 0}, dest);+ assertArrayEquals("Xabcde".getBytes(StandardCharsets.UTF_8), Files.readAllBytes(assembled));}@Test- void partThatGrewAfterBeingMeasuredFailsAssembly() {- byte[] dest = new byte[5];+ void partThatGrewSinceItWasUploadedFailsAssembly() throws IOException {+ Path partFile = Files.write(tempDir.resolve("part"), "abcdef".getBytes(StandardCharsets.UTF_8));- assertThrows(IOException.class, () -> S3Service.readPart(- new ByteArrayInputStream("abcdef".getBytes(StandardCharsets.UTF_8)), dest, 0, 5, 1));+ assertThrows(IOException.class, () -> appendToScratchFile(partFile, 5));}@Test- void partThatShrankAfterBeingMeasuredFailsAssembly() {- byte[] dest = new byte[5];+ void partThatShrankSinceItWasUploadedFailsAssembly() throws IOException {+ Path partFile = Files.write(tempDir.resolve("part"), "abcd".getBytes(StandardCharsets.UTF_8));++ assertThrows(IOException.class, () -> appendToScratchFile(partFile, 5));+ }++ private void appendToScratchFile(Path partFile, long recordedSize) throws IOException {+ try (FileChannel out = FileChannel.open(tempDir.resolve("scratch"),+ StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {+ S3Service.appendPart(partFile, new Part(1, "\"etag\"", recordedSize), out);+ }+ }++ private Path partsDir(MultipartUpload upload) {+ return dataRoot.resolve(".multipart").resolve(upload.getUploadId());+ }++ private Path partFile(MultipartUpload upload, int partNumber) {+ return partsDir(upload).resolve(upload.getParts().get(partNumber).getStorageId());+ }- assertThrows(IOException.class, () -> S3Service.readPart(- new ByteArrayInputStream("abcd".getBytes(StandardCharsets.UTF_8)), dest, 0, 5, 1));+ private List<Path> assembledFiles(MultipartUpload upload) throws IOException {+ try (Stream<Path> files = Files.list(partsDir(upload))) {+ return files.filter(path -> path.getFileName().toString().startsWith("assembled")).toList();+ }+ }++ private List<Path> storedObjectFiles() throws IOException {+ try (Stream<Path> files = Files.walk(dataRoot)) {+ return files.filter(path -> path.getFileName().toString().endsWith(".s3data")).toList();+ }}private static byte[] repeatingBytes(byte value, int length) {diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.javaindex 5cd5408f7f..7fb024e76a 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.services.s3.model.ChecksumType;import io.github.hectorvent.floci.services.s3.model.GetObjectAttributesResult;import io.github.hectorvent.floci.services.s3.model.MultipartUpload;import io.github.hectorvent.floci.services.s3.model.ObjectAttributeName;+import io.github.hectorvent.floci.services.s3.model.Part;import io.github.hectorvent.floci.services.s3.model.S3Checksum;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.junit.jupiter.api.BeforeEach;@@ -555,6 +556,57 @@ class S3MultipartServiceTest {assertEquals(compositeSha256(part1, part2), result.getChecksum().getChecksumSHA256());}+ @Test+ void uploadPartThatTakesAnInMemoryUploadPastOneArrayIsRejected() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "huge.bin", null);+ // The check reads the recorded part sizes, so recording one avoids allocating 2 GiB here.+ Part recorded = new Part(1, "\"etag\"", Integer.MAX_VALUE - 16L);+ recorded.setStorageId("1-recorded");+ upload.getParts().put(1, recorded);++ AwsException error = assertThrows(AwsException.class, () -> s3Service.uploadPart(+ "test-bucket", "huge.bin", upload.getUploadId(), 2, new byte[16]));++ assertEquals("EntityTooLarge", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("Your proposed upload exceeds the maximum allowed object size.", error.getMessage());+ assertNull(upload.getParts().get(2), "a rejected part is not recorded");+ s3Service.uploadPart("test-bucket", "huge.bin", upload.getUploadId(), 1, new byte[16]);+ assertEquals(16, upload.getParts().get(1).getSize(), "re-uploading a part replaces its size in the total");+ }++ @Test+ void completeMultipartUploadOverOneArrayIsRejectedInMemoryMode() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "huge.bin", null);+ s3Service.uploadPart("test-bucket", "huge.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));+ s3Service.uploadPart("test-bucket", "huge.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));+ // Concurrent UploadParts can each pass their own check, so Complete checks the total again.+ upload.getParts().get(2).setSize(Integer.MAX_VALUE);++ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "huge.bin", upload.getUploadId(), List.of(1, 2), null, null));++ assertEquals("EntityTooLarge", error.getErrorCode());+ assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "huge.bin"));+ }++ @Test+ void completeThatReadAReplacedPartsRecordFailsInsteadOfStoringTheNewBytes() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "raced.bin", null);+ s3Service.uploadPart("test-bucket", "raced.bin", upload.getUploadId(), 1, "first".getBytes(StandardCharsets.UTF_8));+ Part staleRecord = upload.getParts().get(1);+ s3Service.uploadPart("test-bucket", "raced.bin", upload.getUploadId(), 1, "other".getBytes(StandardCharsets.UTF_8));+ // A record whose bytes a same-size re-upload replaced. The upload lock keeps a Complete from+ // reading one, but if it ever did, it must fail rather than store the new bytes.+ upload.getParts().put(1, staleRecord);++ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(+ "test-bucket", "raced.bin", upload.getUploadId(), List.of(1), null, null));++ assertEquals("InvalidPart", error.getErrorCode());+ assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "raced.bin"));+ }+private static Map<Integer, S3Checksum> partChecksums(ChecksumAlgorithm algorithm, byte[]... parts) {Map<Integer, S3Checksum> checksums = new HashMap<>();for (int i = 0; i < parts.length; i++) {