Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): return account owner IDs in bucket and parts listings (#4884)

Aniket Shuklad8a76d137663
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index d6f7506ce5..989e126521 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -234,8 +234,8 @@ public class S3Controller {
.raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
.start("ListAllMyBucketsResult", AwsNamespaces.S3)
.start("Owner")
- .elem("ID", "owner")
- .elem("DisplayName", "owner")
+ .elem("ID", regionResolver.getAccountId())
+ .elem("DisplayName", S3Service.DEFAULT_OWNER_DISPLAY_NAME)
.end("Owner")
.start("Buckets");
for (Bucket b : buckets) {
@@ -1711,13 +1711,17 @@ public class S3Controller {
.elem("Size", part.getSize())
.end("Part");
}
+ String ownerAccountId = upload.getOwnerAccountId() != null
+ ? upload.getOwnerAccountId() : s3Service.getBucketOwnerAccountId(bucket);
+ String initiatorAccountId = upload.getInitiatorAccountId() != null
+ ? upload.getInitiatorAccountId() : ownerAccountId;
xml.start("Initiator")
- .elem("ID", "owner")
- .elem("DisplayName", "owner")
+ .elem("ID", initiatorAccountId)
+ .elem("DisplayName", S3Service.DEFAULT_OWNER_DISPLAY_NAME)
.end("Initiator")
.start("Owner")
- .elem("ID", "owner")
- .elem("DisplayName", "owner")
+ .elem("ID", ownerAccountId)
+ .elem("DisplayName", S3Service.DEFAULT_OWNER_DISPLAY_NAME)
.end("Owner")
.elem("StorageClass", upload.getStorageClass());
xml.end("ListPartsResult");
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 1f311aefd6..ffd0a01aa6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -10,6 +10,9 @@ import io.github.hectorvent.floci.core.common.ServicePrincipals;
import io.github.hectorvent.floci.core.common.XmlBuilder;
import io.github.hectorvent.floci.core.common.XmlParser;
import io.github.hectorvent.floci.core.common.Resettable;
+import io.github.hectorvent.floci.core.resource.ExplorerResource;
+import io.github.hectorvent.floci.core.resource.ResourceProvider;
+import io.github.hectorvent.floci.core.resource.SupportedResourceType;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -73,14 +76,14 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import io.github.hectorvent.floci.core.resource.ExplorerResource;
-import io.github.hectorvent.floci.core.resource.ResourceProvider;
-import io.github.hectorvent.floci.core.resource.SupportedResourceType;
@ApplicationScoped
public class S3Service implements Resettable, ResourceProvider {
@@ -93,7 +96,7 @@ public class S3Service implements Resettable, ResourceProvider {
}
return "000000000000";
}
- private static final String DEFAULT_OWNER_DISPLAY_NAME = "floci";
+ static final String DEFAULT_OWNER_DISPLAY_NAME = "floci";
private static final String AUTHENTICATED_USERS_GROUP_URI = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers";
private static final String LOG_DELIVERY_GROUP_URI = "http://acs.amazonaws.com/groups/s3/LogDelivery";
private static final String LEGACY_ACCESS_KEY_ID = "test";
@@ -136,6 +139,65 @@ public class S3Service implements Resettable, ResourceProvider {
// map that would need reference counting to ever shrink safely.
private static final int DISK_FILE_LOCK_STRIPES = 256;
private final ReentrantLock[] diskFileLocks = newLockStripes(DISK_FILE_LOCK_STRIPES);
+ // Bucket deletion must not sweep an upload while a multipart operation is creating or using it.
+ // Read locks preserve parallel multipart writes; fair, fixed stripes avoid writer starvation
+ // and retaining a lock for every bucket name ever seen.
+ private final ReentrantReadWriteLock[] multipartBucketLocks = newMultipartBucketLocks(256);
+ // A second, finer-grained lock prevents part writes, completion and abort from racing on
+ // the same upload without serializing independent uploads in the bucket.
+ private final ReentrantLock[] multipartUploadLocks = newLockStripes(256);
+
+ private static ReentrantReadWriteLock[] newMultipartBucketLocks(int count) {
+ ReentrantReadWriteLock[] locks = new ReentrantReadWriteLock[count];
+ for (int i = 0; i < count; i++) {
+ locks[i] = new ReentrantReadWriteLock(true);
+ }
+ return locks;
+ }
+
+ private ReentrantReadWriteLock multipartBucketLock(String bucketName) {
+ return multipartBucketLocks[Math.floorMod(bucketName.hashCode(), multipartBucketLocks.length)];
+ }
+
+ private static <T> T withLock(Lock lock, Supplier<T> operation) {
+ lock.lock();
+ try {
+ return operation.get();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private static void withLock(Lock lock, Runnable operation) {
+ withLock(lock, () -> {
+ operation.run();
+ return null;
+ });
+ }
+
+ private <T> T withMultipartBucketReadLock(String bucketName, Supplier<T> operation) {
+ return withLock(multipartBucketLock(bucketName).readLock(), operation);
+ }
+
+ private void withMultipartBucketReadLock(String bucketName, Runnable operation) {
+ withLock(multipartBucketLock(bucketName).readLock(), operation);
+ }
+
+ private void withMultipartBucketWriteLock(String bucketName, Runnable operation) {
+ withLock(multipartBucketLock(bucketName).writeLock(), operation);
+ }
+
+ private ReentrantLock multipartUploadLock(String uploadId) {
+ return multipartUploadLocks[Math.floorMod(uploadId.hashCode(), multipartUploadLocks.length)];
+ }
+
+ private <T> T withMultipartOperationLock(String bucketName, String uploadId, Supplier<T> operation) {
+ return withMultipartBucketReadLock(bucketName, () -> withLock(multipartUploadLock(uploadId), operation));
+ }
+
+ private void withMultipartOperationLock(String bucketName, String uploadId, Runnable operation) {
+ withMultipartBucketReadLock(bucketName, () -> withLock(multipartUploadLock(uploadId), operation));
+ }
private static ReentrantLock[] newLockStripes(int count) {
ReentrantLock[] locks = new ReentrantLock[count];
@@ -374,17 +436,19 @@ public class S3Service implements Resettable, ResourceProvider {
}
public void deleteBucket(String bucketName) {
- ensureBucketExists(bucketName);
- Bucket bucket = bucketStore.get(bucketName)
- .orElseThrow(() -> new AwsException("NoSuchBucket",
- "The specified bucket does not exist.", 404));
+ withMultipartBucketWriteLock(bucketName, () -> {
+ ensureBucketExists(bucketName);
+ Bucket bucket = bucketStore.get(bucketName)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404));
- // Takes the bucket monitor that the bucket-scoped mutations take: a mutation that read the
- // record before the delete would otherwise write it back afterwards, restoring the bucket.
- synchronized (bucket) {
- deleteBucketLocked(bucketName);
- }
- LOG.infov("Deleted bucket: {0}", bucketName);
+ // Multipart operations take the stripe before the bucket monitor, so a delete cannot
+ // sweep their temporary parts or miss an upload created just after its scan.
+ synchronized (bucket) {
+ deleteBucketLocked(bucketName);
+ }
+ LOG.infov("Deleted bucket: {0}", bucketName);
+ });
}
private void deleteBucketLocked(String bucketName) {
@@ -395,6 +459,14 @@ public class S3Service implements Resettable, ResourceProvider {
"The bucket you tried to delete is not empty.", 409);
}
+ // Outstanding uploads belong to this bucket incarnation. Do not let a later
+ // owner of the same name discover or complete them.
+ for (MultipartUpload upload : multipartUploads.values()) {
+ if (bucketName.equals(upload.getBucket()) && ownerId().equals(upload.getOwnerAccountId())) {
+ cleanupMultipart(upload.getUploadId());
+ }
+ }
+
bucketStore.delete(bucketName);
deleteAllAnnotationsForBucket(bucketName);
if (inMemory) {
@@ -413,6 +485,12 @@ public class S3Service implements Resettable, ResourceProvider {
return bucketStore.scan(key -> true);
}
+ public String getBucketOwnerAccountId(String bucketName) {
+ return resolveBucketEntry(bucketName)
+ .map(AccountAwareStorageBackend.OwnedEntry::account)
+ .orElseThrow(() -> new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));
+ }
+
public void putBucketLogging(String bucketName, String loggingConfigurationXml) {
Bucket bucket = bucketStore.get(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));
@@ -3351,51 +3429,55 @@ public class S3Service implements Resettable, ResourceProvider {
String sseCustomerAlgorithm, String sseCustomerKey, String sseCustomerKeyMd5,
String checksumAlgorithm, String checksumType,
Map<String, String> tagging) {
- ensureBucketExists(bucket);
- if (acl != null && !acl.isBlank()) {
- cannedObjectAclXml(acl);
- }
- String normalizedServerSideEncryption = normalizeServerSideEncryption(serverSideEncryption);
- SseCustomerKey customerKey = validateSseCustomerKey(sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);
- rejectConflictingServerSideEncryption(normalizedServerSideEncryption, customerKey);
- MultipartUpload upload = new MultipartUpload(bucket, key, contentType);
- if (metadata != null) {
- upload.getMetadata().putAll(metadata);
- }
- upload.setStorageClass(ObjectAttributeName.normalizeStorageClass(storageClass));
- upload.setContentDisposition(contentDisposition);
- upload.setServerSideEncryption(normalizedServerSideEncryption);
- upload.setSseKmsKeyId("aws:kms".equals(normalizedServerSideEncryption) ? sseKmsKeyId : null);
- if (customerKey != null) {
- upload.setSseCustomerAlgorithm(customerKey.algorithm());
- upload.setSseCustomerKeyMd5(customerKey.keyMd5());
- }
- upload.setAcl(acl);
- ChecksumAlgorithm algorithm = ChecksumAlgorithm.fromWireValue(checksumAlgorithm);
- ChecksumType requestedChecksumType = ChecksumType.fromWireValue(checksumType);
- if (requestedChecksumType != null && algorithm == null) {
- throw new AwsException("InvalidRequest",
- "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.", 400);
- }
- upload.setChecksumAlgorithm(algorithm);
- upload.setChecksumType(algorithm == null ? null : algorithm.multipartType(requestedChecksumType));
- if (tagging != null && !tagging.isEmpty()) {
- upload.setTagging(new HashMap<>(tagging));
- }
+ return withMultipartBucketReadLock(bucket, () -> {
+ ensureBucketExists(bucket);
+ if (acl != null && !acl.isBlank()) {
+ cannedObjectAclXml(acl);
+ }
+ String normalizedServerSideEncryption = normalizeServerSideEncryption(serverSideEncryption);
+ SseCustomerKey customerKey = validateSseCustomerKey(sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);
+ rejectConflictingServerSideEncryption(normalizedServerSideEncryption, customerKey);
+ MultipartUpload upload = new MultipartUpload(bucket, key, contentType);
+ upload.setOwnerAccountId(getBucketOwnerAccountId(bucket));
+ upload.setInitiatorAccountId(ownerId());
+ if (metadata != null) {
+ upload.getMetadata().putAll(metadata);
+ }
+ upload.setStorageClass(ObjectAttributeName.normalizeStorageClass(storageClass));
+ upload.setContentDisposition(contentDisposition);
+ upload.setServerSideEncryption(normalizedServerSideEncryption);
+ upload.setSseKmsKeyId("aws:kms".equals(normalizedServerSideEncryption) ? sseKmsKeyId : null);
+ if (customerKey != null) {
+ upload.setSseCustomerAlgorithm(customerKey.algorithm());
+ upload.setSseCustomerKeyMd5(customerKey.keyMd5());
+ }
+ upload.setAcl(acl);
+ ChecksumAlgorithm algorithm = ChecksumAlgorithm.fromWireValue(checksumAlgorithm);
+ ChecksumType requestedChecksumType = ChecksumType.fromWireValue(checksumType);
+ if (requestedChecksumType != null && algorithm == null) {
+ throw new AwsException("InvalidRequest",
+ "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.", 400);
+ }
+ upload.setChecksumAlgorithm(algorithm);
+ upload.setChecksumType(algorithm == null ? null : algorithm.multipartType(requestedChecksumType));
+ if (tagging != null && !tagging.isEmpty()) {
+ upload.setTagging(new HashMap<>(tagging));
+ }
- if (inMemory) {
- memoryMultipartStore.put(upload.getUploadId(), new ConcurrentHashMap<>());
- } else {
- try {
- Files.createDirectories(dataRoot.resolve(".multipart").resolve(upload.getUploadId()));
- } catch (IOException e) {
- throw new UncheckedIOException("Failed to create multipart temp directory", e);
+ if (inMemory) {
+ memoryMultipartStore.put(upload.getUploadId(), new ConcurrentHashMap<>());
+ } else {
+ try {
+ Files.createDirectories(dataRoot.resolve(".multipart").resolve(upload.getUploadId()));
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to create multipart temp directory", e);
+ }
}
- }
- multipartUploads.put(upload.getUploadId(), upload);
- LOG.infov("Initiated multipart upload: {0}/{1}, uploadId={2}", bucket, key, upload.getUploadId());
- return upload;
+ multipartUploads.put(upload.getUploadId(), upload);
+ LOG.infov("Initiated multipart upload: {0}/{1}, uploadId={2}", bucket, key, upload.getUploadId());
+ return upload;
+ });
}
public String uploadPart(String bucket, String key, String uploadId, int partNumber, byte[] data) {
@@ -3411,34 +3493,32 @@ public class S3Service implements Resettable, ResourceProvider {
/** Stores one part and returns it, with the ETag and the checksum the upload's algorithm gives it. */
public Part storePart(String bucket, String key, String uploadId, int partNumber, byte[] data,
String sseCustomerAlgorithm, String sseCustomerKey, String sseCustomerKeyMd5) {
- MultipartUpload upload = multipartUploads.get(uploadId);
- if (upload == null || !upload.getBucket().equals(bucket) || !upload.getKey().equals(key)) {
- throw new AwsException("NoSuchUpload",
- "The specified multipart upload does not exist.", 404);
- }
- if (partNumber < 1 || partNumber > 10000) {
- throw new AwsException("InvalidArgument",
- "Part number must be between 1 and 10000.", 400);
- }
- validateSseCustomerAccess(upload, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);
+ return withMultipartOperationLock(bucket, uploadId, () -> {
+ MultipartUpload upload = getMultipartUpload(bucket, key, uploadId);
+ if (partNumber < 1 || partNumber > 10000) {
+ throw new AwsException("InvalidArgument",
+ "Part number must be between 1 and 10000.", 400);
+ }
+ validateSseCustomerAccess(upload, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);
- if (inMemory) {
- memoryMultipartStore.get(uploadId).put(partNumber, data);
- } else {
- Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(partNumber));
- try {
- Files.write(partPath, data);
- } catch (IOException e) {
- throw new UncheckedIOException("Failed to write multipart part", e);
+ if (inMemory) {
+ memoryMultipartStore.get(uploadId).put(partNumber, data);
+ } else {
+ Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(partNumber));
+ try {
+ Files.write(partPath, data);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write multipart part", e);
+ }
}
- }
- String eTag = computeETag(data);
- Part part = new Part(partNumber, eTag, data.length);
- part.setChecksum(S3Checksum.of(upload.getChecksumAlgorithm(), data));
- upload.getParts().put(partNumber, part);
- LOG.debugv("Uploaded part {0} for upload {1} ({2} bytes)", partNumber, uploadId, data.length);
- return part;
+ String eTag = computeETag(data);
+ Part part = new Part(partNumber, eTag, data.length);
+ part.setChecksum(S3Checksum.of(upload.getChecksumAlgorithm(), data));
+ upload.getParts().put(partNumber, part);
+ LOG.debugv("Uploaded part {0} for upload {1} ({2} bytes)", partNumber, uploadId, data.length);
+ return part;
+ });
}
public String uploadPartCopy(String destBucket, String destKey, String uploadId, int partNumber,
@@ -3504,92 +3584,91 @@ public class S3Service implements Resettable, ResourceProvider {
public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List<Integer> partNumbers,
Map<Integer, String> partETags, Map<Integer, S3Checksum> partChecksums,
String checksumType, S3Checksum expectedChecksum) {
- MultipartUpload upload = multipartUploads.get(uploadId);
- if (upload == null || !upload.getBucket().equals(bucket) || !upload.getKey().equals(key)) {
- throw new AwsException("NoSuchUpload",
- "The specified multipart upload does not exist.", 404);
- }
+ S3Object completed = withMultipartOperationLock(bucket, uploadId, () -> {
+ MultipartUpload upload = getMultipartUpload(bucket, key, uploadId);
- ChecksumAlgorithm algorithm = upload.getChecksumAlgorithm() != null ? upload.getChecksumAlgorithm() : ChecksumAlgorithm.CRC64NVME;
- ChecksumType storedChecksumType = upload.getChecksumType() != null ? upload.getChecksumType() : ChecksumType.FULL_OBJECT;
+ ChecksumAlgorithm algorithm = upload.getChecksumAlgorithm() != null ? upload.getChecksumAlgorithm() : ChecksumAlgorithm.CRC64NVME;
+ ChecksumType storedChecksumType = upload.getChecksumType() != null ? upload.getChecksumType() : ChecksumType.FULL_OBJECT;
- int previousPartNumber = 0;
- for (int num : partNumbers) {
- if (num <= previousPartNumber) {
- throw new AwsException("InvalidPartOrder",
- "The list of parts was not in ascending order.", 400);
- }
- previousPartNumber = num;
- Part part = upload.getParts().get(num);
- if (part == null) {
- throw new AwsException("InvalidPart",
- "One or more of the specified parts could not be found. Part " + num + " is missing.", 400);
- }
- if (!partETags.isEmpty() && !etagsMatch(part.getETag(), partETags.get(num))) {
- throw new AwsException("InvalidPart",
- "One or more of the specified parts could not be found. Part " + num
- + " has an invalid ETag.", 400);
+ int previousPartNumber = 0;
+ for (int num : partNumbers) {
+ if (num <= previousPartNumber) {
+ throw new AwsException("InvalidPartOrder",
+ "The list of parts was not in ascending order.", 400);
+ }
+ previousPartNumber = num;
+ Part part = upload.getParts().get(num);
+ if (part == null) {
+ throw new AwsException("InvalidPart",
+ "One or more of the specified parts could not be found. Part " + num + " is missing.", 400);
+ }
+ if (!partETags.isEmpty() && !etagsMatch(part.getETag(), partETags.get(num))) {
+ throw new AwsException("InvalidPart",
+ "One or more of the specified parts could not be found. Part " + num
+ + " has an invalid ETag.", 400);
+ }
+ validatePartChecksum(upload.getChecksumAlgorithm(), storedChecksumType, num, part, partChecksums.get(num));
}
- validatePartChecksum(upload.getChecksumAlgorithm(), storedChecksumType, num, part, partChecksums.get(num));
- }
- validateCompleteChecksumType(algorithm, storedChecksumType, ChecksumType.fromWireValue(checksumType));
+ validateCompleteChecksumType(algorithm, storedChecksumType, ChecksumType.fromWireValue(checksumType));
- // Concatenate parts in order
- try {
- MessageDigest md = MessageDigest.getInstance("MD5");
- for (int num : partNumbers) {
- // A part ETag is the MD5 of that part, so the composite hashes it without rehashing the data
- String partETag = stripSurroundingQuotes(upload.getParts().get(num).getETag());
- md.update(HexFormat.of().parseHex(partETag));
- }
+ // Concatenate parts in order
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ for (int num : partNumbers) {
+ // A part ETag is the MD5 of that part, so the composite hashes it without rehashing the data
+ String partETag = stripSurroundingQuotes(upload.getParts().get(num).getETag());
+ md.update(HexFormat.of().parseHex(partETag));
+ }
- // Composite ETag: MD5 of concatenated part MD5s, suffixed with part count
- String compositeETag = "\"" + bytesToHex(md.digest()) + "-" + partNumbers.size() + "\"";
+ // Composite ETag: MD5 of concatenated part MD5s, suffixed with part count
+ String compositeETag = "\"" + bytesToHex(md.digest()) + "-" + partNumbers.size() + "\"";
- byte[] allData = concatenateParts(uploadId, partNumbers);
+ byte[] allData = concatenateParts(uploadId, partNumbers);
- List<Part> completedParts = partNumbers.stream()
- .map(num -> copyPart(upload.getParts().get(num)))
- .toList();
- S3Checksum checksum = storedChecksumType == ChecksumType.COMPOSITE
- ? S3Checksum.composite(algorithm, completedParts.stream()
- .map(part -> part.getChecksum().valueFor(algorithm)).toList())
- : S3Checksum.fullObject(algorithm, allData);
- if (expectedChecksum != null && expectedChecksum.hasAnyValue()) {
- validateExpectedChecksum(checksum, expectedChecksum);
- }
- S3Object object = storeObject(bucket, key, allData, upload.getContentType(), upload.getMetadata(),
- checksum, completedParts,
- new PutObjectOptions()
- .withStorageClass(upload.getStorageClass())
- .withContentDisposition(upload.getContentDisposition())
- .withServerSideEncryption(upload.getServerSideEncryption())
- .withSseKmsKeyId(upload.getSseKmsKeyId())
- .withAcl(upload.getAcl())
- .withTagging(upload.getTagging()),
- compositeETag);
- if (upload.getSseCustomerAlgorithm() != null) {
- object.setSseCustomerAlgorithm(upload.getSseCustomerAlgorithm());
- object.setSseCustomerKeyMd5(upload.getSseCustomerKeyMd5());
+ List<Part> completedParts = partNumbers.stream()
+ .map(num -> copyPart(upload.getParts().get(num)))
+ .toList();
+ S3Checksum checksum = storedChecksumType == ChecksumType.COMPOSITE
+ ? S3Checksum.composite(algorithm, completedParts.stream()
+ .map(part -> part.getChecksum().valueFor(algorithm)).toList())
+ : S3Checksum.fullObject(algorithm, allData);
+ if (expectedChecksum != null && expectedChecksum.hasAnyValue()) {
+ validateExpectedChecksum(checksum, expectedChecksum);
+ }
+ S3Object object = storeObject(bucket, key, allData, upload.getContentType(), upload.getMetadata(),
+ checksum, completedParts,
+ new PutObjectOptions()
+ .withStorageClass(upload.getStorageClass())
+ .withContentDisposition(upload.getContentDisposition())
+ .withServerSideEncryption(upload.getServerSideEncryption())
+ .withSseKmsKeyId(upload.getSseKmsKeyId())
+ .withAcl(upload.getAcl())
+ .withTagging(upload.getTagging()),
+ compositeETag);
+ if (upload.getSseCustomerAlgorithm() != null) {
+ object.setSseCustomerAlgorithm(upload.getSseCustomerAlgorithm());
+ object.setSseCustomerKeyMd5(upload.getSseCustomerKeyMd5());
+ }
+ String bucketOwnerAccount = resolveBucketEntry(bucket)
+ .map(AccountAwareStorageBackend.OwnedEntry::account)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404));
+ putObjectMetadataForAccount(bucketOwnerAccount, bucket, key, object);
+
+ // Cleanup
+ cleanupMultipart(uploadId);
+ LOG.infov("Completed multipart upload: {0}/{1}, uploadId={2}, parts={3}",
+ bucket, key, uploadId, partNumbers.size());
+ return object;
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read multipart parts", e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException("MD5 algorithm not available", e);
}
- String bucketOwnerAccount = resolveBucketEntry(bucket)
- .map(AccountAwareStorageBackend.OwnedEntry::account)
- .orElseThrow(() -> new AwsException("NoSuchBucket",
- "The specified bucket does not exist.", 404));
- putObjectMetadataForAccount(bucketOwnerAccount, bucket, key, object);
-
- // Cleanup
- cleanupMultipart(uploadId);
- LOG.infov("Completed multipart upload: {0}/{1}, uploadId={2}, parts={3}",
- bucket, key, uploadId, partNumbers.size());
- fireNotifications(bucket, key, "ObjectCreated:CompleteMultipartUpload", object);
- return object;
- } catch (IOException e) {
- throw new UncheckedIOException("Failed to read multipart parts", e);
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException("MD5 algorithm not available", e);
- }
+ });
+ fireNotifications(bucket, key, "ObjectCreated:CompleteMultipartUpload", completed);
+ return completed;
}
/**
@@ -3656,19 +3735,19 @@ public class S3Service implements Resettable, ResourceProvider {
}
public void abortMultipartUpload(String bucket, String key, String uploadId) {
- MultipartUpload upload = multipartUploads.get(uploadId);
- if (upload == null || !upload.getBucket().equals(bucket) || !upload.getKey().equals(key)) {
- throw new AwsException("NoSuchUpload",
- "The specified multipart upload does not exist.", 404);
- }
- cleanupMultipart(uploadId);
- LOG.infov("Aborted multipart upload: {0}/{1}, uploadId={2}", bucket, key, uploadId);
+ withMultipartOperationLock(bucket, uploadId, () -> {
+ getMultipartUpload(bucket, key, uploadId);
+ cleanupMultipart(uploadId);
+ LOG.infov("Aborted multipart upload: {0}/{1}, uploadId={2}", bucket, key, uploadId);
+ });
}
public List<MultipartUpload> listMultipartUploads(String bucket) {
ensureBucketExists(bucket);
+ String ownerAccountId = getBucketOwnerAccountId(bucket);
return multipartUploads.values().stream()
- .filter(u -> u.getBucket().equals(bucket))
+ .filter(u -> u.getBucket().equals(bucket)
+ && (u.getOwnerAccountId() == null || ownerAccountId.equals(u.getOwnerAccountId())))
.toList();
}
@@ -3678,7 +3757,9 @@ public class S3Service implements Resettable, ResourceProvider {
public MultipartUpload getMultipartUpload(String bucket, String key, String uploadId) {
MultipartUpload upload = multipartUploads.get(uploadId);
- if (upload == null || !upload.getBucket().equals(bucket) || !upload.getKey().equals(key)) {
+ if (upload == null || !upload.getBucket().equals(bucket) || !upload.getKey().equals(key)
+ || (upload.getOwnerAccountId() != null
+ && !upload.getOwnerAccountId().equals(getBucketOwnerAccountId(bucket)))) {
throw new AwsException("NoSuchUpload",
"The specified multipart upload does not exist.", 404);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/MultipartUpload.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/MultipartUpload.java
index fa1b0dd8d6..dc9570a919 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/MultipartUpload.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/MultipartUpload.java
@@ -22,6 +22,8 @@ public class MultipartUpload {
private String sseCustomerAlgorithm;
private String sseCustomerKeyMd5;
private String acl;
+ private String ownerAccountId;
+ private String initiatorAccountId;
private ChecksumAlgorithm checksumAlgorithm;
private ChecksumType checksumType;
private Map<String, String> metadata;
@@ -77,6 +79,12 @@ public class MultipartUpload {
public String getAcl() { return acl; }
public void setAcl(String acl) { this.acl = acl; }
+ public String getOwnerAccountId() { return ownerAccountId; }
+ public void setOwnerAccountId(String ownerAccountId) { this.ownerAccountId = ownerAccountId; }
+
+ public String getInitiatorAccountId() { return initiatorAccountId; }
+ public void setInitiatorAccountId(String initiatorAccountId) { this.initiatorAccountId = initiatorAccountId; }
+
public ChecksumAlgorithm getChecksumAlgorithm() { return checksumAlgorithm; }
public void setChecksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { this.checksumAlgorithm = checksumAlgorithm; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartBucketDeletionRaceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartBucketDeletionRaceTest.java
new file mode 100644
index 0000000000..cdcfb4e1eb
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartBucketDeletionRaceTest.java
@@ -0,0 +1,283 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class S3MultipartBucketDeletionRaceTest {
+
+ @TempDir
+ Path tempDir;
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void deletionWaitsForInitiationAndRemovesTheNewUpload(boolean inMemory) throws Exception {
+ BlockingService service = new BlockingService(tempDir.resolve("initiation"), inMemory);
+ service.createBucket("race-bucket", "us-east-1");
+ service.blockOwnerLookup = true;
+ AtomicReference<MultipartUpload> upload = new AtomicReference<>();
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ Thread initiator = new Thread(() -> {
+ try {
+ upload.set(service.initiateMultipartUpload("race-bucket", "object", "application/octet-stream"));
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "multipart-initiator");
+ initiator.start();
+ assertTrue(service.entered.await(5, TimeUnit.SECONDS));
+
+ Thread deleter = new Thread(() -> {
+ try {
+ service.deleteBucket("race-bucket");
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "bucket-deleter");
+ deleter.start();
+ try {
+ awaitWaiting(deleter);
+ } finally {
+ service.release.countDown();
+ }
+ initiator.join(5_000);
+ deleter.join(5_000);
+ assertFalse(initiator.isAlive());
+ assertFalse(deleter.isAlive());
+ assertNull(failure.get());
+ assertFalse(service.bucketExists("race-bucket"));
+
+ service.createBucket("race-bucket", "us-east-1");
+ AwsException missing = assertThrows(AwsException.class,
+ () -> service.getMultipartUpload("race-bucket", "object", upload.get().getUploadId()));
+ assertEquals("NoSuchUpload", missing.getErrorCode());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void deletionWaitsForPartWriteBeforeCleaningItsBackingStore(boolean inMemory) throws Exception {
+ BlockingService service = new BlockingService(tempDir.resolve("parts"), inMemory);
+ service.createBucket("race-bucket", "us-east-1");
+ String uploadId = service.initiateMultipartUpload("race-bucket", "object", "application/octet-stream")
+ .getUploadId();
+ service.blockUploadLookup = true;
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ Thread writer = new Thread(() -> {
+ try {
+ service.uploadPart("race-bucket", "object", uploadId, 1, new byte[]{1, 2, 3});
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "multipart-writer");
+ writer.start();
+ assertTrue(service.entered.await(5, TimeUnit.SECONDS));
+
+ Thread deleter = new Thread(() -> {
+ try {
+ service.deleteBucket("race-bucket");
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "bucket-deleter");
+ deleter.start();
+ try {
+ awaitWaiting(deleter);
+ } finally {
+ service.release.countDown();
+ }
+ writer.join(5_000);
+ deleter.join(5_000);
+ assertFalse(writer.isAlive());
+ assertFalse(deleter.isAlive());
+ assertNull(failure.get());
+ assertFalse(service.bucketExists("race-bucket"));
+ }
+
+ @Test
+ void multipartInitiationsInTheSameBucketCanProceedConcurrently() throws Exception {
+ BlockingService service = new BlockingService(tempDir.resolve("parallel"), true);
+ service.createBucket("race-bucket", "us-east-1");
+ service.blockOwnerLookup = true;
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ Thread first = new Thread(() -> {
+ try {
+ service.initiateMultipartUpload("race-bucket", "first", "application/octet-stream");
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "multipart-initiator");
+ first.start();
+ assertTrue(service.entered.await(5, TimeUnit.SECONDS));
+
+ AtomicReference<MultipartUpload> secondUpload = new AtomicReference<>();
+ Thread second = new Thread(() -> {
+ try {
+ secondUpload.set(service.initiateMultipartUpload("race-bucket", "second", "application/octet-stream"));
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "parallel-initiator");
+ second.start();
+ try {
+ second.join(5_000);
+ assertFalse(second.isAlive(), "a multipart initiation should not block another on the same bucket");
+ assertNotNull(secondUpload.get());
+ } finally {
+ service.release.countDown();
+ }
+ first.join(5_000);
+ assertFalse(first.isAlive());
+ assertNull(failure.get());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void abortWaitsForPartWriteOnTheSameUpload(boolean inMemory) throws Exception {
+ BlockingService service = new BlockingService(tempDir.resolve("abort"), inMemory);
+ service.createBucket("race-bucket", "us-east-1");
+ String uploadId = service.initiateMultipartUpload("race-bucket", "object", "application/octet-stream")
+ .getUploadId();
+ service.blockUploadLookup = true;
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ Thread writer = new Thread(() -> {
+ try {
+ service.uploadPart("race-bucket", "object", uploadId, 1, new byte[]{1, 2, 3});
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "multipart-writer");
+ writer.start();
+ assertTrue(service.entered.await(5, TimeUnit.SECONDS));
+
+ Thread aborter = new Thread(() -> {
+ try {
+ service.abortMultipartUpload("race-bucket", "object", uploadId);
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "multipart-aborter");
+ aborter.start();
+ try {
+ awaitWaiting(aborter);
+ } finally {
+ service.release.countDown();
+ }
+ writer.join(5_000);
+ aborter.join(5_000);
+ assertFalse(writer.isAlive());
+ assertFalse(aborter.isAlive());
+ assertNull(failure.get());
+ AwsException missing = assertThrows(AwsException.class,
+ () -> service.getMultipartUpload("race-bucket", "object", uploadId));
+ assertEquals("NoSuchUpload", missing.getErrorCode());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void completionWaitsForPartWriteOnTheSameUpload(boolean inMemory) throws Exception {
+ BlockingService service = new BlockingService(tempDir.resolve("completion"), inMemory);
+ service.createBucket("race-bucket", "us-east-1");
+ String uploadId = service.initiateMultipartUpload("race-bucket", "object", "application/octet-stream")
+ .getUploadId();
+ service.blockUploadLookup = true;
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ Thread writer = new Thread(() -> {
+ try {
+ service.uploadPart("race-bucket", "object", uploadId, 1, new byte[]{1, 2, 3});
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "multipart-writer");
+ writer.start();
+ assertTrue(service.entered.await(5, TimeUnit.SECONDS));
+
+ Thread completer = new Thread(() -> {
+ try {
+ service.completeMultipartUpload("race-bucket", "object", uploadId, List.of(1), null, null);
+ } catch (Throwable error) {
+ failure.set(error);
+ }
+ }, "multipart-completer");
+ completer.start();
+ try {
+ awaitWaiting(completer);
+ } finally {
+ service.release.countDown();
+ }
+ writer.join(5_000);
+ completer.join(5_000);
+ assertFalse(writer.isAlive());
+ assertFalse(completer.isAlive());
+ assertNull(failure.get());
+ assertNotNull(service.getObject("race-bucket", "object"));
+ }
+
+ private static void awaitWaiting(Thread thread) throws InterruptedException {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (System.nanoTime() < deadline && thread.getState() != Thread.State.WAITING) {
+ if (!thread.isAlive()) {
+ break;
+ }
+ Thread.sleep(10);
+ }
+ assertEquals(Thread.State.WAITING, thread.getState(), "operation should wait for the multipart lock");
+ }
+
+ private static final class BlockingService extends S3Service {
+ private final CountDownLatch entered = new CountDownLatch(1);
+ private final CountDownLatch release = new CountDownLatch(1);
+ private volatile boolean blockOwnerLookup;
+ private volatile boolean blockUploadLookup;
+
+ private BlockingService(Path dataRoot, boolean inMemory) {
+ super(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, inMemory);
+ }
+
+ @Override
+ public String getBucketOwnerAccountId(String bucketName) {
+ String owner = super.getBucketOwnerAccountId(bucketName);
+ if (blockOwnerLookup && "multipart-initiator".equals(Thread.currentThread().getName())) {
+ pause();
+ }
+ return owner;
+ }
+
+ @Override
+ public MultipartUpload getMultipartUpload(String bucket, String key, String uploadId) {
+ MultipartUpload upload = super.getMultipartUpload(bucket, key, uploadId);
+ if (blockUploadLookup && "multipart-writer".equals(Thread.currentThread().getName())) {
+ pause();
+ }
+ return upload;
+ }
+
+ private void pause() {
+ entered.countDown();
+ try {
+ if (!release.await(5, TimeUnit.SECONDS)) {
+ throw new AssertionError("timed out waiting for the race test to release the multipart operation");
+ }
+ } catch (InterruptedException error) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("multipart operation interrupted", error);
+ }
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3OwnerIdentityIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3OwnerIdentityIntegrationTest.java
new file mode 100644
index 0000000000..8033811879
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3OwnerIdentityIntegrationTest.java
@@ -0,0 +1,111 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Test;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+@QuarkusTest
+class S3OwnerIdentityIntegrationTest {
+
+ private static final String ACCOUNT_ONE = "000000000011";
+ private static final String ACCOUNT_TWO = "000000000012";
+
+ @Test
+ void listBucketsOwnerMatchesTheCallerAccountAndBucketAcl() {
+ String bucket = "owner-identity-list-bucket";
+ createBucket(ACCOUNT_ONE, bucket);
+
+ given().header("Authorization", authorization(ACCOUNT_ONE))
+ .when().get("/")
+ .then().statusCode(200)
+ .body(containsString("<Owner><ID>" + ACCOUNT_ONE + "</ID><DisplayName>floci</DisplayName></Owner>"))
+ .body(containsString("<Name>" + bucket + "</Name>"));
+
+ given().header("Authorization", authorization(ACCOUNT_ONE))
+ .when().get("/" + bucket + "?acl")
+ .then().statusCode(200)
+ .body(containsString("<Owner><ID>" + ACCOUNT_ONE + "</ID><DisplayName>floci</DisplayName></Owner>"));
+
+ given().header("Authorization", authorization(ACCOUNT_TWO))
+ .when().get("/")
+ .then().statusCode(200)
+ .body(containsString("<Owner><ID>" + ACCOUNT_TWO + "</ID><DisplayName>floci</DisplayName></Owner>"))
+ .body(not(containsString("<Name>" + bucket + "</Name>")));
+ }
+
+ @Test
+ void listPartsOwnerAndInitiatorUseTheUploadAccount() {
+ String bucket = "owner-identity-parts-bucket";
+ createBucket(ACCOUNT_TWO, bucket);
+
+ String uploadId = given().header("Authorization", authorization(ACCOUNT_TWO))
+ .when().post("/" + bucket + "/part.bin?uploads")
+ .then().statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+
+ given().header("Authorization", authorization(ACCOUNT_TWO))
+ .when().get("/" + bucket + "/part.bin?uploadId=" + uploadId)
+ .then().statusCode(200)
+ .body(containsString("<Initiator><ID>" + ACCOUNT_TWO
+ + "</ID><DisplayName>floci</DisplayName></Initiator>"))
+ .body(containsString("<Owner><ID>" + ACCOUNT_TWO
+ + "</ID><DisplayName>floci</DisplayName></Owner>"));
+ }
+
+ @Test
+ void anUploadCannotBeAttributedToAnotherAccountOrSurviveBucketDeletion() {
+ String bucket = "owner-identity-reused-bucket";
+ createBucket(ACCOUNT_ONE, bucket);
+ createBucket(ACCOUNT_TWO, bucket);
+
+ String firstUploadId = given().header("Authorization", authorization(ACCOUNT_ONE))
+ .when().post("/" + bucket + "/part.bin?uploads")
+ .then().statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+
+ given().header("Authorization", authorization(ACCOUNT_TWO))
+ .when().get("/" + bucket + "/part.bin?uploadId=" + firstUploadId)
+ .then().statusCode(404)
+ .body(containsString("<Code>NoSuchUpload</Code>"));
+
+ given().header("Authorization", authorization(ACCOUNT_ONE))
+ .when().delete("/" + bucket)
+ .then().statusCode(204);
+
+ createBucket(ACCOUNT_ONE, bucket);
+ given().header("Authorization", authorization(ACCOUNT_ONE))
+ .when().get("/" + bucket + "/part.bin?uploadId=" + firstUploadId)
+ .then().statusCode(404)
+ .body(containsString("<Code>NoSuchUpload</Code>"));
+
+ given().header("Authorization", authorization(ACCOUNT_TWO))
+ .when().get("/" + bucket + "/part.bin?uploadId=" + firstUploadId)
+ .then().statusCode(404)
+ .body(containsString("<Code>NoSuchUpload</Code>"));
+
+ String secondUploadId = given().header("Authorization", authorization(ACCOUNT_TWO))
+ .when().post("/" + bucket + "/part.bin?uploads")
+ .then().statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+
+ given().header("Authorization", authorization(ACCOUNT_TWO))
+ .when().get("/" + bucket + "/part.bin?uploadId=" + secondUploadId)
+ .then().statusCode(200)
+ .body(containsString("<Owner><ID>" + ACCOUNT_TWO
+ + "</ID><DisplayName>floci</DisplayName></Owner>"));
+ }
+
+ private static void createBucket(String account, String bucket) {
+ given().header("Authorization", authorization(account))
+ .when().put("/" + bucket)
+ .then().statusCode(200);
+ }
+
+ private static String authorization(String account) {
+ return "AWS4-HMAC-SHA256 Credential=" + account
+ + "/20261002/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc";
+ }
+}