Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): complete multipart uploads larger than 2 GiB in disk storage modes (#4976)

Priamo Germosen2e87534941cf
diff --git a/docs/services/s3.md b/docs/services/s3.md
index 52d4fd3752..ca3a24f5d5 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -388,6 +388,18 @@ longer grows with the number of stored objects. Bucket definitions and the other
still rewritten on every change. See
[Storage Modes](../configuration/storage.md#journaled-stores-under-persistent-mode).
+In `memory` mode each object body is a single in-memory byte array, so an object can hold at most
+about 2 GiB: an `UploadPart` that takes a multipart upload past that, or a
+`CompleteMultipartUpload` over it, fails with `EntityTooLarge`. The disk-backed modes assemble a
+multipart upload file to file, so the object never passes through the heap; set
+`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. While it runs,
+`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembled
+copy, until the parts are removed. Multipart uploads in progress do not survive a restart or a
+reset, and their files are removed when either happens. `CopyObject`,
+`UploadPartCopy` and S3 Select still read the whole source object into memory in every mode, so
+they need a source under 2 GiB, and deleting the current version of a key in a versioned bucket
+reads the version that takes its place the same way.
+
## Examples
```bash
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index ffd0a01aa6..3b6795bac7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -1,15 +1,18 @@
package io.github.hectorvent.floci.services.s3;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsNamespaces;
import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.ServicePrincipals;
import io.github.hectorvent.floci.core.common.XmlBuilder;
import io.github.hectorvent.floci.core.common.XmlParser;
-import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.resource.ResourceProvider;
import io.github.hectorvent.floci.core.resource.SupportedResourceType;
@@ -17,6 +20,7 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.core.storage.WriteProfile;
+import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;
import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator;
import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator.ResourcePolicyDecision;
import io.github.hectorvent.floci.services.iam.IamService;
@@ -26,12 +30,8 @@ import io.github.hectorvent.floci.services.lambda.LambdaService;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
import io.github.hectorvent.floci.services.s3.model.*;
-import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;
import io.github.hectorvent.floci.services.sns.SnsService;
import io.github.hectorvent.floci.services.sqs.SqsService;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Event;
import jakarta.enterprise.inject.Instance;
@@ -48,17 +48,18 @@ import java.net.URLEncoder;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
-import java.nio.file.NoSuchFileException;
import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.FileSystemException;
import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
-import java.time.temporal.ChronoUnit;
import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
@@ -84,6 +85,7 @@ import java.util.function.Function;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Stream;
@ApplicationScoped
public class S3Service implements Resettable, ResourceProvider {
@@ -206,7 +208,10 @@ public class S3Service implements Resettable, ResourceProvider {
}
return locks;
}
- private final ConcurrentHashMap<String, Map<Integer, byte[]>> memoryMultipartStore = new ConcurrentHashMap<>();
+ // Memory mode keeps an object in one byte array; this is the largest the JDK reliably allocates.
+ private static final long MAX_IN_MEMORY_OBJECT_SIZE = Integer.MAX_VALUE - 8;
+ // Upload ID to part bytes, keyed by each part's storage ID.
+ private final ConcurrentHashMap<String, Map<String, byte[]>> memoryMultipartStore = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, MultipartUpload> multipartUploads = new ConcurrentHashMap<>();
// Account-level (S3 Control) Block Public Access config, one entry per AWS account.
// Distinct from the bucket-level configuration held on each Bucket. Block Public Access is a
@@ -376,6 +381,7 @@ public class S3Service implements Resettable, ResourceProvider {
} catch (IOException e) {
throw new UncheckedIOException("Failed to create S3 data directory: " + dataRoot, e);
}
+ deleteMultipartFiles();
}
}
@@ -391,9 +397,31 @@ public class S3Service implements Resettable, ResourceProvider {
// default account alone is not enough). Mirrors the metadata erase; the pre-existing
// .s3data behavior is unchanged.
deleteAnnotationPayloadRoots();
+ deleteMultipartFiles();
}
}
+ /**
+ * Deletes the part and assembled files of every multipart upload. Uploads are tracked only in
+ * memory, so at startup or after a reset these files belong to uploads that no longer exist, and
+ * an assembly a crash interrupted can be as large as the object it was building.
+ */
+ private void deleteMultipartFiles() {
+ Path multipartRoot = dataRoot.resolve(".multipart");
+ if (!Files.isDirectory(multipartRoot)) {
+ return;
+ }
+ try (Stream<Path> uploads = Files.list(multipartRoot)) {
+ long count = uploads.count();
+ if (count > 0) {
+ LOG.infov("Removing the files of {0} multipart uploads that no longer exist under {1}", count, multipartRoot);
+ }
+ } catch (IOException e) {
+ LOG.warnv(e, "Failed to list {0} before removing it", multipartRoot);
+ }
+ deleteDirectory(multipartRoot);
+ }
+
private void deleteAnnotationPayloadRoots() {
Path accountsRoot = dataRoot.resolve(ACCOUNT_STORAGE_ROOT);
if (!Files.isDirectory(accountsRoot)) {
@@ -620,18 +648,45 @@ public class S3Service implements Resettable, ResourceProvider {
private S3Object storeObject(String bucketName, String key, byte[] data,
String contentType, Map<String, String> metadata,
S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {
+ return storeObject(bucketName, key, new BytesBody(data), contentType, metadata, checksum, parts, options, eTag);
+ }
+
+ private S3Object storeObject(String bucketName, String key, ObjectBody body,
+ String contentType, Map<String, String> metadata,
+ S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {
AccountAwareStorageBackend.OwnedEntry<Bucket> ownedBucket = resolveBucketEntry(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket",
"The specified bucket does not exist.", 404));
Bucket bucket = ownedBucket.value();
synchronized (bucket) {
- return storeObjectInternal(ownedBucket.account(), bucket, bucketName, key, data,
+ return storeObjectInternal(ownedBucket.account(), bucket, bucketName, key, body,
contentType, metadata, checksum, parts, options, eTag);
}
}
+ /**
+ * The body of an object being stored: the bytes of a single-request write, or the file a
+ * multipart upload was assembled into, which is moved into place instead of being read back.
+ */
+ private sealed interface ObjectBody permits BytesBody, AssembledBody {
+ long size();
+ }
+
+ private record BytesBody(byte[] data) implements ObjectBody {
+ @Override
+ public long size() {
+ return data.length;
+ }
+ }
+
+ /**
+ * A disk-mode multipart object. It is stored with the ETag and checksum built from its parts,
+ * since computing either here would mean reading the whole file.
+ */
+ private record AssembledBody(Path file, long size) implements ObjectBody { }
+
private S3Object storeObjectInternal(String bucketOwnerAccount, Bucket bucket,
- String bucketName, String key, byte[] data,
+ String bucketName, String key, ObjectBody body,
String contentType, Map<String, String> metadata,
S3Checksum checksum, List<Part> parts, PutObjectOptions options,
String eTag) {
@@ -641,8 +696,10 @@ public class S3Service implements Resettable, ResourceProvider {
rejectConflictingServerSideEncryption(normalizedServerSideEncryption, sseCustomerKey);
checkWritePreconditions(bucketName, key, effectiveOptions.getIfMatch(), effectiveOptions.getIfNoneMatch());
- S3Object object = new S3Object(bucketName, key, data, contentType,
+ byte[] data = body instanceof BytesBody bytes ? bytes.data() : null;
+ S3Object object = new S3Object(bucketName, key, body.size(), contentType,
eTag != null ? eTag : computeETag(data));
+ object.setData(data);
if (metadata != null) {
object.getMetadata().putAll(metadata);
}
@@ -721,8 +778,14 @@ public class S3Service implements Resettable, ResourceProvider {
// versioned write fails, the canonical file - which unlocked GETs already associate
// with the still-unpublished previous generation - is never touched, so a concurrent
// GET can't observe corrupted "latest" bytes paired with the old metadata.
- writeVersionedFile(bucketOwnerAccount, bucketName, key, versionId, data);
- writeFile(bucketOwnerAccount, bucketName, key, data);
+ if (body instanceof AssembledBody assembled) {
+ Path versionedPath = resolveVersionedPath(bucketOwnerAccount, bucketName, key, versionId);
+ moveIntoPlace(assembled.file(), versionedPath);
+ linkIntoPlace(versionedPath, resolveObjectPath(bucketOwnerAccount, bucketName, key));
+ } else {
+ writeVersionedFile(bucketOwnerAccount, bucketName, key, versionId, data);
+ writeFile(bucketOwnerAccount, bucketName, key, data);
+ }
// Deferred pre-versioning annotation cleanup: only after the replacement body is on
// disk, so a failed write keeps the old body and its annotations together.
if (dropPreVersioningAnnotations[0]) {
@@ -735,7 +798,7 @@ public class S3Service implements Resettable, ResourceProvider {
// Store versioned copy and update latest pointer
putObjectForAccount(bucketOwnerAccount, versionedKey(bucketName, key, versionId), object);
putObjectForAccount(bucketOwnerAccount, latestKey, object);
- LOG.debugv("Put versioned object: {0}/{1} v={2} ({3} bytes)", bucketName, key, versionId, data.length);
+ LOG.debugv("Put versioned object: {0}/{1} v={2} ({3} bytes)", bucketName, key, versionId, body.size());
} else {
S3Object prev = resolveObjectForAccount(
bucketOwnerAccount, objectKey(bucketName, key)).orElse(null);
@@ -757,7 +820,11 @@ public class S3Service implements Resettable, ResourceProvider {
// Write the body before publishing metadata - see the comment in the versioned
// branch above; the same ordering requirement applies here.
- writeFile(bucketOwnerAccount, bucketName, key, data);
+ if (body instanceof AssembledBody assembled) {
+ moveIntoPlace(assembled.file(), resolveObjectPath(bucketOwnerAccount, bucketName, key));
+ } else {
+ writeFile(bucketOwnerAccount, bucketName, key, data);
+ }
// An overwrite replaces the object's annotations (AWS drops them on overwrite).
// The cleanup runs only after the body write succeeds, so a failed PUT keeps the
// old body together with its annotations; and before the new metadata is published,
@@ -767,7 +834,7 @@ public class S3Service implements Resettable, ResourceProvider {
// branch above; the same race applies here.
object.setData(null);
putObjectForAccount(bucketOwnerAccount, objectKey(bucketName, key), object);
- LOG.debugv("Put object: {0}/{1} ({2} bytes)", bucketName, key, data.length);
+ LOG.debugv("Put object: {0}/{1} ({2} bytes)", bucketName, key, body.size());
}
return object;
}
@@ -3501,12 +3568,21 @@ public class S3Service implements Resettable, ResourceProvider {
}
validateSseCustomerAccess(upload, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);
+ // Each upload of a part gets its own storage, so a record and the bytes it describes are
+ // published together and never change: a re-upload adds new bytes and drops the old ones.
+ String storageId = partNumber + "-" + UUID.randomUUID();
if (inMemory) {
- memoryMultipartStore.get(uploadId).put(partNumber, data);
+ // A re-uploaded part number replaces its earlier part, so that part does not count.
+ long otherParts = upload.getParts().values().stream()
+ .filter(part -> part.getPartNumber() != partNumber)
+ .mapToLong(Part::getSize)
+ .sum();
+ requireFitsInMemory(uploadId, otherParts + data.length);
+ memoryMultipartStore.get(uploadId).put(storageId, data);
} else {
- Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(partNumber));
+ Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(storageId);
try {
- Files.write(partPath, data);
+ Files.write(partPath, data, StandardOpenOption.CREATE_NEW);
} catch (IOException e) {
throw new UncheckedIOException("Failed to write multipart part", e);
}
@@ -3515,12 +3591,32 @@ public class S3Service implements Resettable, ResourceProvider {
String eTag = computeETag(data);
Part part = new Part(partNumber, eTag, data.length);
part.setChecksum(S3Checksum.of(upload.getChecksumAlgorithm(), data));
- upload.getParts().put(partNumber, part);
+ part.setStorageId(storageId);
+ Part replaced = upload.getParts().put(partNumber, part);
+ if (replaced != null) {
+ discardPartBytes(uploadId, replaced);
+ }
LOG.debugv("Uploaded part {0} for upload {1} ({2} bytes)", partNumber, uploadId, data.length);
return part;
});
}
+ /**
+ * Drops the bytes of a part a re-upload replaced, once the new record is published, so no record
+ * ever names bytes that changed after it was written.
+ */
+ private void discardPartBytes(String uploadId, Part replaced) {
+ if (inMemory) {
+ Map<String, byte[]> memoryParts = memoryMultipartStore.get(uploadId);
+ if (memoryParts != null) {
+ memoryParts.remove(replaced.getStorageId());
+ }
+ } else {
+ deleteQuietly(dataRoot.resolve(".multipart").resolve(uploadId).resolve(replaced.getStorageId()),
+ "bytes of a multipart part that was uploaded again");
+ }
+ }
+
public String uploadPartCopy(String destBucket, String destKey, String uploadId, int partNumber,
String sourceBucket, String sourceKey, String sourceVersionId,
String copySourceRange) {
@@ -3590,6 +3686,8 @@ public class S3Service implements Resettable, ResourceProvider {
ChecksumAlgorithm algorithm = upload.getChecksumAlgorithm() != null ? upload.getChecksumAlgorithm() : ChecksumAlgorithm.CRC64NVME;
ChecksumType storedChecksumType = upload.getChecksumType() != null ? upload.getChecksumType() : ChecksumType.FULL_OBJECT;
+ // Each record is read once, and everything below, the bytes included, follows these records.
+ List<Part> parts = new ArrayList<>(partNumbers.size());
int previousPartNumber = 0;
for (int num : partNumbers) {
if (num <= previousPartNumber) {
@@ -3608,44 +3706,61 @@ public class S3Service implements Resettable, ResourceProvider {
+ " has an invalid ETag.", 400);
}
validatePartChecksum(upload.getChecksumAlgorithm(), storedChecksumType, num, part, partChecksums.get(num));
+ parts.add(part);
}
validateCompleteChecksumType(algorithm, storedChecksumType, ChecksumType.fromWireValue(checksumType));
+ if (inMemory) {
+ requireFitsInMemory(uploadId, parts.stream().mapToLong(Part::getSize).sum());
+ }
// Concatenate parts in order
try {
MessageDigest md = MessageDigest.getInstance("MD5");
- for (int num : partNumbers) {
+ for (Part part : parts) {
// A part ETag is the MD5 of that part, so the composite hashes it without rehashing the data
- String partETag = stripSurroundingQuotes(upload.getParts().get(num).getETag());
+ String partETag = stripSurroundingQuotes(part.getETag());
md.update(HexFormat.of().parseHex(partETag));
}
// Composite ETag: MD5 of concatenated part MD5s, suffixed with part count
String compositeETag = "\"" + bytesToHex(md.digest()) + "-" + partNumbers.size() + "\"";
- byte[] allData = concatenateParts(uploadId, partNumbers);
-
- List<Part> completedParts = partNumbers.stream()
- .map(num -> copyPart(upload.getParts().get(num)))
+ // Both checksum types come from the stored part checksums, so a mismatch is rejected
+ // before any part is read.
+ List<Part> completedParts = parts.stream()
+ .map(S3Service::copyPart)
.toList();
S3Checksum checksum = storedChecksumType == ChecksumType.COMPOSITE
? S3Checksum.composite(algorithm, completedParts.stream()
.map(part -> part.getChecksum().valueFor(algorithm)).toList())
- : S3Checksum.fullObject(algorithm, allData);
+ : S3Checksum.fullObject(algorithm, completedParts);
if (expectedChecksum != null && expectedChecksum.hasAnyValue()) {
validateExpectedChecksum(checksum, expectedChecksum);
}
- S3Object object = storeObject(bucket, key, allData, upload.getContentType(), upload.getMetadata(),
- checksum, completedParts,
- new PutObjectOptions()
- .withStorageClass(upload.getStorageClass())
- .withContentDisposition(upload.getContentDisposition())
- .withServerSideEncryption(upload.getServerSideEncryption())
- .withSseKmsKeyId(upload.getSseKmsKeyId())
- .withAcl(upload.getAcl())
- .withTagging(upload.getTagging()),
- compositeETag);
+
+ ObjectBody body = inMemory
+ ? new BytesBody(concatenateParts(uploadId, parts))
+ : assembleParts(uploadId, parts);
+ S3Object object;
+ try {
+ object = storeObject(bucket, key, body, upload.getContentType(), upload.getMetadata(),
+ checksum, completedParts,
+ new PutObjectOptions()
+ .withStorageClass(upload.getStorageClass())
+ .withContentDisposition(upload.getContentDisposition())
+ .withServerSideEncryption(upload.getServerSideEncryption())
+ .withSseKmsKeyId(upload.getSseKmsKeyId())
+ .withAcl(upload.getAcl())
+ .withTagging(upload.getTagging()),
+ compositeETag);
+ } finally {
+ // A stored body was moved away, so this only removes the file of a failed store and
+ // leaves the parts in place for a retry or an abort.
+ if (body instanceof AssembledBody assembled) {
+ deleteQuietly(assembled.file(), "assembled multipart object that was not stored");
+ }
+ }
if (upload.getSseCustomerAlgorithm() != null) {
object.setSseCustomerAlgorithm(upload.getSseCustomerAlgorithm());
object.setSseCustomerKeyMd5(upload.getSseCustomerKeyMd5());
@@ -3672,54 +3787,97 @@ public class S3Service implements Resettable, ResourceProvider {
}
/**
- * Copies the parts, in order, into one array sized to the total upload. Disk parts are read
- * straight into their slot, so assembly never holds a second full-size copy of the object.
- * In-memory parts are copied from the arrays that were measured, so their sizes cannot change
- * in between.
+ * Rejects an upload that memory mode could not hold in one byte array, with the error S3 gives an
+ * object over its size limit, so it fails at the part that crosses the limit instead of running
+ * the heap out of memory or overflowing the array size at completion.
*/
- private byte[] concatenateParts(String uploadId, List<Integer> partNumbers) throws IOException {
- Path partsDir = dataRoot.resolve(".multipart").resolve(uploadId);
- Map<Integer, byte[]> memoryParts = inMemory ? memoryMultipartStore.get(uploadId) : null;
- byte[][] memoryData = inMemory ? new byte[partNumbers.size()][] : null;
- long[] partSizes = new long[partNumbers.size()];
+ private static void requireFitsInMemory(String uploadId, long uploadSize) {
+ if (uploadSize > MAX_IN_MEMORY_OBJECT_SIZE) {
+ LOG.warnv("Multipart upload {0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",
+ uploadId, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);
+ throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);
+ }
+ }
+
+ /**
+ * Copies the in-memory bytes of {@code parts}, in order, into one array sized to the total
+ * upload. The total is measured on the arrays being copied, so it holds whatever the records say.
+ */
+ private byte[] concatenateParts(String uploadId, List<Part> parts) {
+ Map<String, byte[]> memoryParts = memoryMultipartStore.getOrDefault(uploadId, Map.of());
+ byte[][] partData = new byte[parts.size()][];
long totalSize = 0;
- for (int i = 0; i < partNumbers.size(); i++) {
- int num = partNumbers.get(i);
- if (inMemory) {
- memoryData[i] = memoryParts.get(num);
- partSizes[i] = memoryData[i].length;
- } else {
- partSizes[i] = Files.size(partsDir.resolve(String.valueOf(num)));
+ for (int i = 0; i < parts.size(); i++) {
+ partData[i] = memoryParts.get(parts.get(i).getStorageId());
+ if (partData[i] == null) {
+ throw missingPartData(parts.get(i));
}
- totalSize += partSizes[i];
+ totalSize += partData[i].length;
}
- byte[] allData = new byte[Math.toIntExact(totalSize)];
+ requireFitsInMemory(uploadId, totalSize);
+ byte[] allData = new byte[(int) totalSize];
int offset = 0;
- for (int i = 0; i < partNumbers.size(); i++) {
- int size = (int) partSizes[i];
- if (inMemory) {
- System.arraycopy(memoryData[i], 0, allData, offset, size);
- } else {
- int num = partNumbers.get(i);
- try (InputStream in = Files.newInputStream(partsDir.resolve(String.valueOf(num)))) {
- readPart(in, allData, offset, size, num);
+ for (byte[] part : partData) {
+ System.arraycopy(part, 0, allData, offset, part.length);
+ offset += part.length;
+ }
+ return allData;
+ }
+
+ /**
+ * Copies the part files, in order, into one file in the upload's directory for storeObject to
+ * move into place. The copy runs file to file, so the object never passes through the heap
+ * whatever its size, and it happens before the bucket lock is taken. The file name is unique
+ * per call, so two Completes racing on one upload never write the same file.
+ */
+ private AssembledBody assembleParts(String uploadId, List<Part> parts) throws IOException {
+ Path partsDir = dataRoot.resolve(".multipart").resolve(uploadId);
+ Path assembled = partsDir.resolve("assembled-" + UUID.randomUUID());
+ long totalSize = 0;
+ try (FileChannel out = FileChannel.open(assembled, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ for (Part part : parts) {
+ try {
+ appendPart(partsDir.resolve(part.getStorageId()), part, out);
+ } catch (NoSuchFileException e) {
+ throw missingPartData(part);
}
+ totalSize += part.getSize();
}
- offset += size;
+ } catch (IOException | RuntimeException e) {
+ deleteQuietly(assembled, "partially assembled multipart object");
+ throw e;
}
- return allData;
+ return new AssembledBody(assembled, totalSize);
}
/**
- * Reads exactly {@code size} bytes of part {@code partNumber} into {@code dest} at {@code offset},
- * failing if the part holds a different number of bytes than was measured before assembly.
+ * Appends part {@code part} from {@code partFile} to {@code out}, failing if the file does not
+ * hold the size recorded for the part: the ETag and checksum the object is stored with were
+ * built from that record.
*/
- static void readPart(InputStream in, byte[] dest, int offset, int size, int partNumber) throws IOException {
- if (in.readNBytes(dest, offset, size) != size || in.read() != -1) {
- throw new IOException("Part " + partNumber + " changed size during assembly");
+ static void appendPart(Path partFile, Part part, FileChannel out) throws IOException {
+ long size = part.getSize();
+ try (FileChannel in = FileChannel.open(partFile, StandardOpenOption.READ)) {
+ long position = 0;
+ while (position < size) {
+ long transferred = in.transferTo(position, size - position, out);
+ if (transferred <= 0) {
+ break;
+ }
+ position += transferred;
+ }
+ if (position != size || in.size() != size) {
+ throw new IOException("Part " + part.getPartNumber() + " changed size during assembly");
+ }
}
}
+ /** A part whose record names bytes that are no longer stored. */
+ private static AwsException missingPartData(Part part) {
+ return new AwsException("InvalidPart", "One or more of the specified parts could not be found. Part "
+ + part.getPartNumber() + " has no stored data.", 400);
+ }
+
private boolean etagsMatch(String storedETag, String submittedETag) {
return stripSurroundingQuotes(storedETag).equals(stripSurroundingQuotes(submittedETag));
}
@@ -5567,16 +5725,66 @@ public class S3Service implements Resettable, ResourceProvider {
Path tmp = filePath.resolveSibling(filePath.getFileName() + ".tmp-" + UUID.randomUUID());
try {
Files.write(tmp, data);
+ replaceAtomically(tmp, filePath);
+ } finally {
+ Files.deleteIfExists(tmp);
+ }
+ }
+
+ private static void replaceAtomically(Path source, Path target) throws IOException {
+ try {
+ Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ } catch (AtomicMoveNotSupportedException e) {
+ // Rare filesystems (some network mounts) reject ATOMIC_MOVE; fall back to a plain
+ // replace. This narrows but does not fully close the window, which is acceptable only
+ // because the default overlay/ext filesystems used here support atomic rename.
+ Files.move(source, target, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ /**
+ * Moves an assembled multipart object to {@code target} with the same all-or-nothing rename
+ * {@link #atomicWrite} ends with: a concurrent reader sees the previous file or the whole new
+ * one, and the move costs the same whatever the object's size.
+ */
+ private void moveIntoPlace(Path source, Path target) {
+ ReentrantLock lock = diskFileLock(target);
+ lock.lock();
+ try {
+ Files.createDirectories(target.getParent());
+ replaceAtomically(source, target);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to move assembled S3 object file into place", e);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Gives {@code target} the file already stored at {@code source} through a hard link, so a
+ * versioned multipart object is kept once instead of being copied under the bucket lock.
+ * Sharing the file is safe because no object file is ever modified in place: every write
+ * replaces its path with a rename, which leaves the file under the other name untouched.
+ * A filesystem without hard links gets a copy instead.
+ */
+ private void linkIntoPlace(Path source, Path target) {
+ ReentrantLock lock = diskFileLock(target);
+ lock.lock();
+ Path tmp = target.resolveSibling(target.getFileName() + ".tmp-" + UUID.randomUUID());
+ try {
+ Files.createDirectories(target.getParent());
try {
- Files.move(tmp, filePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
- } catch (AtomicMoveNotSupportedException e) {
- // Rare filesystems (some network mounts) reject ATOMIC_MOVE; fall back to a plain
- // replace. This narrows but does not fully close the window — acceptable only
- // because the default overlay/ext filesystems used here support atomic rename.
- Files.move(tmp, filePath, StandardCopyOption.REPLACE_EXISTING);
+ Files.createLink(tmp, source);
+ } catch (UnsupportedOperationException | FileSystemException e) {
+ LOG.debugv(e, "No hard link for {0}, copying {1} instead", target, source);
+ Files.copy(source, tmp);
}
+ replaceAtomically(tmp, target);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to link S3 object file into place", e);
} finally {
- Files.deleteIfExists(tmp);
+ deleteQuietly(tmp, "temporary link of a versioned multipart object that was not published");
+ lock.unlock();
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/Part.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/Part.java
index c80617c5a7..7395d8037a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/Part.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/Part.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.s3.model;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.time.Instant;
@@ -13,6 +14,10 @@ public class Part {
private long size;
private S3Checksum checksum;
private Instant lastModified;
+ // Names this part's bytes while its upload is open: a file in the upload's directory, or a key
+ // in memory. A re-upload stores under a new name, so the bytes under a name never change.
+ @JsonIgnore
+ private String storageId;
public Part() {
this.checksum = new S3Checksum();
@@ -40,4 +45,7 @@ public class Part {
public Instant getLastModified() { return lastModified; }
public void setLastModified(Instant lastModified) { this.lastModified = lastModified; }
+
+ public String getStorageId() { return storageId; }
+ public void setStorageId(String storageId) { this.storageId = storageId; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
index 0bd6615c0b..796fab3d0f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
@@ -19,6 +19,10 @@ public class S3Checksum {
// Slicing-by-8: row n holds the CRC of each byte value followed by n zero bytes.
private static final long[][] CRC64_TABLES = buildCrc64Tables();
+ private static final CrcCombiner CRC32_COMBINER = new CrcCombiner(0xEDB88320L, Integer.SIZE);
+ private static final CrcCombiner CRC32C_COMBINER = new CrcCombiner(0x82F63B78L, Integer.SIZE);
+ private static final CrcCombiner CRC64_NVME_COMBINER = new CrcCombiner(CRC64_NVME_POLY, Long.SIZE);
+
private String checksumCRC32;
private String checksumCRC32C;
private String checksumCRC64NVME;
@@ -93,6 +97,31 @@ public class S3Checksum {
return checksum;
}
+ /**
+ * Full-object checksum of a multipart object, combined from each part's CRC and size so the
+ * object's bytes are never read. S3 linearizes it the same way, which is why it offers full-object
+ * checksums on multipart uploads only for the CRC algorithms.
+ */
+ public static S3Checksum fullObject(ChecksumAlgorithm algorithm, List<Part> parts) {
+ ChecksumAlgorithm effective = algorithm != null ? algorithm : ChecksumAlgorithm.CRC64NVME;
+ CrcCombiner combiner = switch (effective) {
+ case CRC32 -> CRC32_COMBINER;
+ case CRC32C -> CRC32C_COMBINER;
+ case CRC64NVME -> CRC64_NVME_COMBINER;
+ case SHA1, SHA256 -> throw new IllegalArgumentException(
+ effective.wireValue() + " has no full-object multipart checksum");
+ };
+ // The CRC of no bytes is zero for all three algorithms, so zero seeds the fold.
+ long crc = 0;
+ for (Part part : parts) {
+ crc = combiner.combine(crc, combiner.decode(part.getChecksum().valueFor(effective)), part.getSize());
+ }
+ S3Checksum checksum = new S3Checksum();
+ checksum.setValueFor(effective, combiner.encode(crc));
+ checksum.setChecksumType(ChecksumType.FULL_OBJECT);
+ return checksum;
+ }
+
public static S3Checksum composite(ChecksumAlgorithm algorithm, List<String> partChecksums) {
S3Checksum checksum = new S3Checksum();
checksum.setValueFor(algorithm, algorithm.composite(partChecksums));
@@ -220,4 +249,69 @@ public class S3Checksum {
}
return tables;
}
+
+ /**
+ * Joins the CRCs of two adjacent byte ranges, as zlib's {@code crc32_combine} does, for a reflected
+ * CRC whose initial value and final XOR are both all ones (CRC32, CRC32C and CRC64NVME all are).
+ * With those parameters crc(A + B) = crc(A) * x^(8 * len(B)) mod P xor crc(B), so the second range's
+ * length is all that is needed besides the two CRCs. In the reflected form the top bit is x^0.
+ */
+ private static final class CrcCombiner {
+
+ private final long polynomial;
+ private final int width;
+ private final long one;
+ // Entry k is x^(2^k) mod P. Lengths are counted in bytes, so x^(8n) starts at k = 3.
+ private final long[] xToPowerOfTwo = new long[Long.SIZE + 3];
+
+ CrcCombiner(long polynomial, int width) {
+ this.polynomial = polynomial;
+ this.width = width;
+ this.one = 1L << (width - 1);
+ long power = one >>> 1;
+ for (int k = 0; k < xToPowerOfTwo.length; k++) {
+ xToPowerOfTwo[k] = power;
+ power = multiply(power, power);
+ }
+ }
+
+ long combine(long first, long second, long secondLength) {
+ long shift = one;
+ int k = 3;
+ for (long remaining = secondLength; remaining != 0; remaining >>>= 1) {
+ if ((remaining & 1) != 0) {
+ shift = multiply(xToPowerOfTwo[k], shift);
+ }
+ k++;
+ }
+ return multiply(shift, first) ^ second;
+ }
+
+ long decode(String base64) {
+ long crc = 0;
+ for (byte b : Base64.getDecoder().decode(base64)) {
+ crc = (crc << Byte.SIZE) | (b & 0xFF);
+ }
+ return crc;
+ }
+
+ String encode(long crc) {
+ byte[] bytes = new byte[width / Byte.SIZE];
+ for (int i = bytes.length - 1; i >= 0; i--) {
+ bytes[i] = (byte) (crc >>> ((bytes.length - 1 - i) * Byte.SIZE));
+ }
+ return Base64.getEncoder().encodeToString(bytes);
+ }
+
+ private long multiply(long a, long b) {
+ long product = 0;
+ for (long bit = one; bit != 0; bit >>>= 1) {
+ if ((a & bit) != 0) {
+ product ^= b;
+ }
+ b = (b & 1) != 0 ? (b >>> 1) ^ polynomial : b >>> 1;
+ }
+ return product;
+ }
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
index cff8b443a6..4b00096795 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
@@ -66,11 +66,16 @@ public class S3Object {
}
public S3Object(String bucketName, String key, byte[] data, String contentType, String eTag) {
+ this(bucketName, key, data.length, contentType, eTag);
+ this.data = data;
+ }
+
+ /** An object whose {@code size}-byte body is stored on disk rather than held in memory. */
+ public S3Object(String bucketName, String key, long size, String contentType, String eTag) {
this.bucketName = bucketName;
this.key = key;
- this.data = data;
this.contentType = contentType != null ? contentType : "application/octet-stream";
- this.size = data.length;
+ this.size = size;
this.lastModified = Instant.now().truncatedTo(ChronoUnit.MILLIS);
this.eTag = eTag;
this.metadata = new HashMap<>();
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java
index 6a61ee5eb7..6a53faa296 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java
@@ -3,15 +3,19 @@ package io.github.hectorvent.floci.services.s3;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;
import io.github.hectorvent.floci.services.s3.model.ChecksumType;
+import io.github.hectorvent.floci.services.s3.model.Part;
import io.github.hectorvent.floci.services.s3.model.S3Checksum;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Random;
+import java.util.zip.CRC32C;
import static org.junit.jupiter.api.Assertions.*;
@@ -109,6 +113,88 @@ class S3ChecksumTest {
assertEquals(ChecksumType.FULL_OBJECT, checksum.getChecksumType());
}
+ @Test
+ void fullObjectCombinedFromPartsMatchesTheChecksumOfTheWholeObject() {
+ Random random = new Random(4145);
+ List<int[]> splits = List.of(
+ new int[]{0},
+ new int[]{1},
+ new int[]{5 * 1024 * 1024, 17},
+ new int[]{0, 9, 0, 64, 1},
+ new int[]{1023, 1024, 1025, 65537, 3});
+ for (ChecksumAlgorithm algorithm : List.of(ChecksumAlgorithm.CRC32, ChecksumAlgorithm.CRC32C,
+ ChecksumAlgorithm.CRC64NVME)) {
+ for (int[] sizes : splits) {
+ ByteArrayOutputStream whole = new ByteArrayOutputStream();
+ List<Part> parts = new ArrayList<>();
+ for (int i = 0; i < sizes.length; i++) {
+ byte[] data = new byte[sizes[i]];
+ random.nextBytes(data);
+ whole.writeBytes(data);
+ parts.add(part(i + 1, algorithm, data));
+ }
+
+ S3Checksum combined = S3Checksum.fullObject(algorithm, parts);
+
+ assertEquals(algorithm.compute(whole.toByteArray()), combined.valueFor(algorithm),
+ algorithm + " " + Arrays.toString(sizes));
+ assertEquals(ChecksumType.FULL_OBJECT, combined.getChecksumType());
+ }
+ }
+ }
+
+ @Test
+ void fullObjectCombinesPartsLongerThanAnArrayCanHold() {
+ long zeroRunLength = (1L << 31) + 13;
+ byte[] head = "head".getBytes(StandardCharsets.UTF_8);
+ byte[] tail = "tail".getBytes(StandardCharsets.UTF_8);
+ CRC32C zeroRun = new CRC32C();
+ CRC32C whole = new CRC32C();
+ whole.update(head);
+ byte[] zeros = new byte[1 << 20];
+ for (long remaining = zeroRunLength; remaining > 0; remaining -= zeros.length) {
+ int chunk = (int) Math.min(zeros.length, remaining);
+ zeroRun.update(zeros, 0, chunk);
+ whole.update(zeros, 0, chunk);
+ }
+ whole.update(tail);
+ Part middle = new Part(2, "\"etag\"", zeroRunLength);
+ middle.getChecksum().setChecksumCRC32C(crc32Base64(zeroRun.getValue()));
+
+ S3Checksum combined = S3Checksum.fullObject(ChecksumAlgorithm.CRC32C, List.of(
+ part(1, ChecksumAlgorithm.CRC32C, head), middle, part(3, ChecksumAlgorithm.CRC32C, tail)));
+
+ assertEquals(crc32Base64(whole.getValue()), combined.getChecksumCRC32C());
+ }
+
+ @Test
+ void fullObjectFromPartsWithoutDeclaredAlgorithmGetsCrc64Nvme() {
+ byte[] first = "part1".getBytes(StandardCharsets.UTF_8);
+ byte[] second = "part2".getBytes(StandardCharsets.UTF_8);
+
+ S3Checksum combined = S3Checksum.fullObject(null, List.of(part(1, null, first), part(2, null, second)));
+
+ assertEquals(S3Checksum.crc64NvmeBase64("part1part2".getBytes(StandardCharsets.UTF_8)),
+ combined.getChecksumCRC64NVME());
+ }
+
+ @Test
+ void fullObjectFromPartsRejectsShaAlgorithms() {
+ List<Part> parts = List.of(part(1, ChecksumAlgorithm.SHA256, "data".getBytes(StandardCharsets.UTF_8)));
+
+ assertThrows(IllegalArgumentException.class, () -> S3Checksum.fullObject(ChecksumAlgorithm.SHA256, parts));
+ }
+
+ private static Part part(int partNumber, ChecksumAlgorithm algorithm, byte[] data) {
+ Part part = new Part(partNumber, "\"etag\"", data.length);
+ part.setChecksum(S3Checksum.of(algorithm, data));
+ return part;
+ }
+
+ private static String crc32Base64(long value) {
+ return Base64.getEncoder().encodeToString(ByteBuffer.allocate(Integer.BYTES).putInt((int) value).array());
+ }
+
@Test
void objectAttributesViewDropsTheSuffixOfCompositeValuesOnly() {
S3Checksum composite = S3Checksum.composite(ChecksumAlgorithm.CRC32, List.of("W0QpDQ==", "anCi7Q=="));
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
index b963297290..5ed89211b1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartDiskAssemblyTest.java
@@ -4,26 +4,32 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.s3.model.ChecksumType;
import io.github.hectorvent.floci.services.s3.model.MultipartUpload;
+import io.github.hectorvent.floci.services.s3.model.Part;
import io.github.hectorvent.floci.services.s3.model.S3Checksum;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.HexFormat;
import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -129,39 +135,192 @@ class S3MultipartDiskAssemblyTest {
// Metadata (the Part record with its ETag) lives in memory, separate from the on-disk part
// file that assembly reads; deleting only the file makes assembly fail after the
// file-independent validation checks have passed.
- Path part2File = dataRoot.resolve(".multipart").resolve(upload.getUploadId()).resolve("2");
- assertTrue(Files.deleteIfExists(part2File), "test setup: part 2 file should exist before deletion");
+ assertTrue(Files.deleteIfExists(partFile(upload, 2)), "test setup: part 2 file should exist before deletion");
- assertThrows(UncheckedIOException.class, () -> s3Service.completeMultipartUpload(
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
"test-bucket", "vanished.bin", upload.getUploadId(), List.of(1, 2), null, null));
+ assertEquals("InvalidPart", error.getErrorCode());
+
assertThrows(AwsException.class,
() -> s3Service.getObject("test-bucket", "vanished.bin"));
+ assertEquals(List.of(), assembledFiles(upload), "the partial assembly should be removed");
+ }
+
+ @Test
+ void failedStoreRemovesTheAssembledFileAndKeepsThePartsForARetry() throws Exception {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "public.bin", null,
+ null, null, null, null, "public-read");
+ s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "public.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));
+ // The public ACL is only rejected once the parts are assembled, when the object is stored.
+ s3Service.putPublicAccessBlock("test-bucket",
+ "<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls></PublicAccessBlockConfiguration>");
+
+ assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "public.bin", upload.getUploadId(), List.of(1, 2), null, null));
+
+ assertEquals(List.of(), assembledFiles(upload), "the unstored assembly should be removed");
+ s3Service.putPublicAccessBlock("test-bucket",
+ "<PublicAccessBlockConfiguration><BlockPublicAcls>false</BlockPublicAcls></PublicAccessBlockConfiguration>");
+ s3Service.completeMultipartUpload("test-bucket", "public.bin", upload.getUploadId(), List.of(1, 2), null, null);
+ assertArrayEquals("part1part2".getBytes(StandardCharsets.UTF_8),
+ s3Service.getObject("test-bucket", "public.bin").getData());
+ assertFalse(Files.exists(partsDir(upload)), "a completed upload should leave no files behind");
+ }
+
+ @Test
+ void versionedMultipartObjectIsStoredOnceAndKeepsItsBytesWhenTheKeyIsOverwritten() throws Exception {
+ s3Service.putBucketVersioning("test-bucket", "Enabled");
+ byte[] part1 = repeatingBytes((byte) 'V', 9_000);
+ byte[] part2 = repeatingBytes((byte) 'W', 1_000);
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "versioned.bin", null);
+ s3Service.uploadPart("test-bucket", "versioned.bin", upload.getUploadId(), 1, part1);
+ s3Service.uploadPart("test-bucket", "versioned.bin", upload.getUploadId(), 2, part2);
+
+ String versionId = s3Service.completeMultipartUpload("test-bucket", "versioned.bin", upload.getUploadId(),
+ List.of(1, 2), null, null).getVersionId();
+
+ List<Path> stored = storedObjectFiles();
+ assertEquals(2, stored.size(), "a versioned object has a versioned file and a current file: " + stored);
+ assertTrue(Files.isSameFile(stored.get(0), stored.get(1)), "the two should be one file, not a copy");
+
+ byte[] replacement = "replacement".getBytes(StandardCharsets.UTF_8);
+ String replacementVersionId = s3Service.putObject("test-bucket", "versioned.bin", replacement,
+ "text/plain", Map.of()).getVersionId();
+ assertArrayEquals(replacement, s3Service.getObject("test-bucket", "versioned.bin").getData());
+ assertArrayEquals(concat(part1, part2), s3Service.getObject("test-bucket", "versioned.bin", versionId).getData());
+
+ s3Service.deleteObject("test-bucket", "versioned.bin", replacementVersionId);
+ assertArrayEquals(concat(part1, part2), s3Service.getObject("test-bucket", "versioned.bin").getData());
+ }
+
+ @Test
+ void mismatchedFullObjectChecksumIsRejectedBeforeAnyPartIsRead() throws Exception {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "bad-digest.bin", null);
+ s3Service.uploadPart("test-bucket", "bad-digest.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "bad-digest.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));
+ // Without its part files, a Complete that reached assembly would fail with an I/O error instead.
+ assertTrue(Files.deleteIfExists(partFile(upload, 1)), "test setup: part 1 file should exist before deletion");
+ assertTrue(Files.deleteIfExists(partFile(upload, 2)), "test setup: part 2 file should exist before deletion");
+ S3Checksum wrong = new S3Checksum();
+ wrong.setChecksumCRC64NVME(S3Checksum.crc64NvmeBase64("other".getBytes(StandardCharsets.UTF_8)));
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "bad-digest.bin", upload.getUploadId(), List.of(1, 2), null, wrong));
+
+ assertEquals("BadDigest", error.getErrorCode());
+ }
+
+ @Test
+ void reuploadedPartGetsANewFileAndTheReplacedFileIsRemoved() throws Exception {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "reupload.bin", null);
+ s3Service.uploadPart("test-bucket", "reupload.bin", upload.getUploadId(), 1, "first".getBytes(StandardCharsets.UTF_8));
+ Path firstFile = partFile(upload, 1);
+
+ s3Service.uploadPart("test-bucket", "reupload.bin", upload.getUploadId(), 1, "other".getBytes(StandardCharsets.UTF_8));
+
+ Path secondFile = partFile(upload, 1);
+ assertNotEquals(firstFile, secondFile, "a part file is never rewritten in place");
+ assertFalse(Files.exists(firstFile), "the replaced part's file should be removed");
+ assertArrayEquals("other".getBytes(StandardCharsets.UTF_8), Files.readAllBytes(secondFile));
+ s3Service.completeMultipartUpload("test-bucket", "reupload.bin", upload.getUploadId(), List.of(1), null, null);
+ assertArrayEquals("other".getBytes(StandardCharsets.UTF_8), s3Service.getObject("test-bucket", "reupload.bin").getData());
+ }
+
+ @Test
+ void completeThatReadAReplacedPartsRecordFailsInsteadOfStoringTheNewBytes() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "raced.bin", null);
+ s3Service.uploadPart("test-bucket", "raced.bin", upload.getUploadId(), 1, "first".getBytes(StandardCharsets.UTF_8));
+ Part staleRecord = upload.getParts().get(1);
+ s3Service.uploadPart("test-bucket", "raced.bin", upload.getUploadId(), 1, "other".getBytes(StandardCharsets.UTF_8));
+ // A record whose bytes a same-size re-upload replaced. The upload lock keeps a Complete from
+ // reading one, but if it ever did, it must fail rather than store the new bytes.
+ upload.getParts().put(1, staleRecord);
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "raced.bin", upload.getUploadId(), List.of(1), null, null));
+
+ assertEquals("InvalidPart", error.getErrorCode());
+ assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "raced.bin"));
+ }
+
+ @Test
+ void restartRemovesTheFilesOfUploadsThatWereInProgress() throws Exception {
+ s3Service.putObject("test-bucket", "kept.bin", "kept".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "interrupted.bin", null);
+ s3Service.uploadPart("test-bucket", "interrupted.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));
+ // What a crash in the middle of CompleteMultipartUpload leaves next to the parts.
+ Files.write(partsDir(upload).resolve("assembled-interrupted"), new byte[1024]);
+
+ new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+
+ assertFalse(Files.exists(dataRoot.resolve(".multipart")), "no upload outlives the process that held it");
+ assertEquals(1, storedObjectFiles().size(), "stored objects are left alone");
+ }
+
+ @Test
+ void resetRemovesTheFilesOfUploadsInProgress() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "reset.bin", null);
+ s3Service.uploadPart("test-bucket", "reset.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));
+
+ s3Service.clear();
+
+ assertFalse(Files.exists(dataRoot.resolve(".multipart")), "a reset forgets every upload, so their files go too");
}
@Test
- void partReadCopiesExactlyTheMeasuredBytes() throws IOException {
- byte[] dest = new byte[7];
+ void appendedPartAddsExactlyItsRecordedBytes() throws IOException {
+ Path partFile = Files.write(tempDir.resolve("part"), "abcde".getBytes(StandardCharsets.UTF_8));
+ Path assembled = tempDir.resolve("assembled");
- S3Service.readPart(new ByteArrayInputStream("abcde".getBytes(StandardCharsets.UTF_8)), dest, 1, 5, 1);
+ try (FileChannel out = FileChannel.open(assembled, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ out.write(ByteBuffer.wrap("X".getBytes(StandardCharsets.UTF_8)));
+ S3Service.appendPart(partFile, new Part(1, "\"etag\"", 5), out);
+ }
- assertArrayEquals(new byte[] {0, 'a', 'b', 'c', 'd', 'e', 0}, dest);
+ assertArrayEquals("Xabcde".getBytes(StandardCharsets.UTF_8), Files.readAllBytes(assembled));
}
@Test
- void partThatGrewAfterBeingMeasuredFailsAssembly() {
- byte[] dest = new byte[5];
+ void partThatGrewSinceItWasUploadedFailsAssembly() throws IOException {
+ Path partFile = Files.write(tempDir.resolve("part"), "abcdef".getBytes(StandardCharsets.UTF_8));
- assertThrows(IOException.class, () -> S3Service.readPart(
- new ByteArrayInputStream("abcdef".getBytes(StandardCharsets.UTF_8)), dest, 0, 5, 1));
+ assertThrows(IOException.class, () -> appendToScratchFile(partFile, 5));
}
@Test
- void partThatShrankAfterBeingMeasuredFailsAssembly() {
- byte[] dest = new byte[5];
+ void partThatShrankSinceItWasUploadedFailsAssembly() throws IOException {
+ Path partFile = Files.write(tempDir.resolve("part"), "abcd".getBytes(StandardCharsets.UTF_8));
+
+ assertThrows(IOException.class, () -> appendToScratchFile(partFile, 5));
+ }
+
+ private void appendToScratchFile(Path partFile, long recordedSize) throws IOException {
+ try (FileChannel out = FileChannel.open(tempDir.resolve("scratch"),
+ StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
+ S3Service.appendPart(partFile, new Part(1, "\"etag\"", recordedSize), out);
+ }
+ }
+
+ private Path partsDir(MultipartUpload upload) {
+ return dataRoot.resolve(".multipart").resolve(upload.getUploadId());
+ }
+
+ private Path partFile(MultipartUpload upload, int partNumber) {
+ return partsDir(upload).resolve(upload.getParts().get(partNumber).getStorageId());
+ }
- assertThrows(IOException.class, () -> S3Service.readPart(
- new ByteArrayInputStream("abcd".getBytes(StandardCharsets.UTF_8)), dest, 0, 5, 1));
+ private List<Path> assembledFiles(MultipartUpload upload) throws IOException {
+ try (Stream<Path> files = Files.list(partsDir(upload))) {
+ return files.filter(path -> path.getFileName().toString().startsWith("assembled")).toList();
+ }
+ }
+
+ private List<Path> storedObjectFiles() throws IOException {
+ try (Stream<Path> files = Files.walk(dataRoot)) {
+ return files.filter(path -> path.getFileName().toString().endsWith(".s3data")).toList();
+ }
}
private static byte[] repeatingBytes(byte value, int length) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
index 5cd5408f7f..7fb024e76a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.services.s3.model.ChecksumType;
import io.github.hectorvent.floci.services.s3.model.GetObjectAttributesResult;
import io.github.hectorvent.floci.services.s3.model.MultipartUpload;
import io.github.hectorvent.floci.services.s3.model.ObjectAttributeName;
+import io.github.hectorvent.floci.services.s3.model.Part;
import io.github.hectorvent.floci.services.s3.model.S3Checksum;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.BeforeEach;
@@ -555,6 +556,57 @@ class S3MultipartServiceTest {
assertEquals(compositeSha256(part1, part2), result.getChecksum().getChecksumSHA256());
}
+ @Test
+ void uploadPartThatTakesAnInMemoryUploadPastOneArrayIsRejected() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "huge.bin", null);
+ // The check reads the recorded part sizes, so recording one avoids allocating 2 GiB here.
+ Part recorded = new Part(1, "\"etag\"", Integer.MAX_VALUE - 16L);
+ recorded.setStorageId("1-recorded");
+ upload.getParts().put(1, recorded);
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.uploadPart(
+ "test-bucket", "huge.bin", upload.getUploadId(), 2, new byte[16]));
+
+ assertEquals("EntityTooLarge", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("Your proposed upload exceeds the maximum allowed object size.", error.getMessage());
+ assertNull(upload.getParts().get(2), "a rejected part is not recorded");
+ s3Service.uploadPart("test-bucket", "huge.bin", upload.getUploadId(), 1, new byte[16]);
+ assertEquals(16, upload.getParts().get(1).getSize(), "re-uploading a part replaces its size in the total");
+ }
+
+ @Test
+ void completeMultipartUploadOverOneArrayIsRejectedInMemoryMode() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "huge.bin", null);
+ s3Service.uploadPart("test-bucket", "huge.bin", upload.getUploadId(), 1, "part1".getBytes(StandardCharsets.UTF_8));
+ s3Service.uploadPart("test-bucket", "huge.bin", upload.getUploadId(), 2, "part2".getBytes(StandardCharsets.UTF_8));
+ // Concurrent UploadParts can each pass their own check, so Complete checks the total again.
+ upload.getParts().get(2).setSize(Integer.MAX_VALUE);
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "huge.bin", upload.getUploadId(), List.of(1, 2), null, null));
+
+ assertEquals("EntityTooLarge", error.getErrorCode());
+ assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "huge.bin"));
+ }
+
+ @Test
+ void completeThatReadAReplacedPartsRecordFailsInsteadOfStoringTheNewBytes() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("test-bucket", "raced.bin", null);
+ s3Service.uploadPart("test-bucket", "raced.bin", upload.getUploadId(), 1, "first".getBytes(StandardCharsets.UTF_8));
+ Part staleRecord = upload.getParts().get(1);
+ s3Service.uploadPart("test-bucket", "raced.bin", upload.getUploadId(), 1, "other".getBytes(StandardCharsets.UTF_8));
+ // A record whose bytes a same-size re-upload replaced. The upload lock keeps a Complete from
+ // reading one, but if it ever did, it must fail rather than store the new bytes.
+ upload.getParts().put(1, staleRecord);
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.completeMultipartUpload(
+ "test-bucket", "raced.bin", upload.getUploadId(), List.of(1), null, null));
+
+ assertEquals("InvalidPart", error.getErrorCode());
+ assertThrows(AwsException.class, () -> s3Service.getObject("test-bucket", "raced.bin"));
+ }
+
private static Map<Integer, S3Checksum> partChecksums(ChecksumAlgorithm algorithm, byte[]... parts) {
Map<Integer, S3Checksum> checksums = new HashMap<>();
for (int i = 0; i < parts.length; i++) {