Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): preserve multipart initiator for private ACL (#5990)
Jimmy Liu Jiad8064fae1425
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex ed1c5e4062..a661de36ca 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -694,7 +694,7 @@ public class S3Service implements Resettable, ResourceProvider {AccountAwareStorageBackend.OwnedEntry<Bucket> checkedBucket = resolveBucketEntry(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));- rejectUploadAclWhenOwnershipEnforced(checkedBucket.value(), checkedBucket.account(), effectiveOptions);+ rejectUploadAclWhenOwnershipEnforced(checkedBucket.value(), checkedBucket.account(), effectiveOptions, ownerId());rejectObjectLockWithoutLockEnabled(checkedBucket.value(), effectiveOptions);rejectConflictingServerSideEncryption(normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption()),@@ -788,7 +788,16 @@ public class S3Service implements Resettable, ResourceProvider {private S3Object storeObject(String bucketName, String key, ObjectBody body,String contentType, Map<String, String> metadata,S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {- return storeObject(null, bucketName, key, body, contentType, metadata, checksum, parts, options, eTag);+ return storeObject(null, bucketName, key, body, contentType, metadata, checksum, parts,+ options, eTag, ownerId());+ }++ private S3Object storeObject(String bucketName, String key, ObjectBody body,+ String contentType, Map<String, String> metadata,+ S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag,+ String aclRequestAccount) {+ return storeObject(null, bucketName, key, body, contentType, metadata, checksum, parts,+ options, eTag, aclRequestAccount);}/**@@ -802,6 +811,15 @@ public class S3Service implements Resettable, ResourceProvider {String bucketName, String key, ObjectBody body,String contentType, Map<String, String> metadata,S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {+ return storeObject(checkedBucket, bucketName, key, body, contentType, metadata, checksum,+ parts, options, eTag, ownerId());+ }++ private S3Object storeObject(AccountAwareStorageBackend.OwnedEntry<Bucket> checkedBucket,+ String bucketName, String key, ObjectBody body,+ String contentType, Map<String, String> metadata,+ S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag,+ String aclRequestAccount) {AccountAwareStorageBackend.OwnedEntry<Bucket> ownedBucket = resolveBucketEntry(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));@@ -814,7 +832,7 @@ public class S3Service implements Resettable, ResourceProvider {throw new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404);}return storeObjectInternal(ownedBucket.account(), bucket, bucketName, key, body,- contentType, metadata, checksum, parts, options, eTag);+ contentType, metadata, checksum, parts, options, eTag, aclRequestAccount);}}@@ -845,7 +863,7 @@ public class S3Service implements Resettable, ResourceProvider {String bucketName, String key, ObjectBody body,String contentType, Map<String, String> metadata,S3Checksum checksum, List<Part> parts, PutObjectOptions options,- String eTag) {+ String eTag, String aclRequestAccount) {PutObjectOptions effectiveOptions = options != null ? options : new PutObjectOptions();String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());SseCustomerKey sseCustomerKey = validateSseCustomerKey(effectiveOptions.getSseCustomerAlgorithm(), effectiveOptions.getSseCustomerKey(), effectiveOptions.getSseCustomerKeyMd5());@@ -877,7 +895,8 @@ public class S3Service implements Resettable, ResourceProvider {object.setSseCustomerAlgorithm(sseCustomerKey.algorithm());object.setSseCustomerKeyMd5(sseCustomerKey.keyMd5());}- rejectUploadAclWhenOwnershipEnforced(bucket, bucketOwnerAccount, effectiveOptions);+ rejectUploadAclWhenOwnershipEnforced(bucket, bucketOwnerAccount, effectiveOptions,+ aclRequestAccount);rejectObjectLockWithoutLockEnabled(bucket, effectiveOptions);String objectAcl = resolveObjectAclXml(bucketOwnerAccount,effectiveOptions.getAcl(), effectiveOptions.getGrantRead(),@@ -4239,7 +4258,7 @@ public class S3Service implements Resettable, ResourceProvider {.orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));String ownerAccount = getBucketOwnerAccountId(bucket);- rejectUploadAclWhenOwnershipEnforced(destination, ownerAccount, objectOptions);+ rejectUploadAclWhenOwnershipEnforced(destination, ownerAccount, objectOptions, ownerId());rejectObjectLockWithoutLockEnabled(destination, objectOptions);resolveObjectAclXml(ownerAccount, acl, objectOptions.getGrantRead(), objectOptions.getGrantWrite(),objectOptions.getGrantFullControl(), objectOptions.getGrantReadAcp(),@@ -4730,7 +4749,8 @@ public class S3Service implements Resettable, ResourceProvider {.withRetainUntilDate(upload.getRetainUntilDate()).withLegalHoldStatus(upload.getLegalHoldStatus()).withTagging(upload.getTagging()),- compositeETag);+ compositeETag,+ upload.getInitiatorAccountId() != null ? upload.getInitiatorAccountId() : ownerId());} finally {// A stored body was moved away, so this only removes the file of a failed store and// leaves the parts in place for a retry or an abort.@@ -5655,8 +5675,8 @@ public class S3Service implements Resettable, ResourceProvider {}}- private static void rejectUploadAclWhenOwnershipEnforced(Bucket bucket, String ownerAccount,- PutObjectOptions options) {+ private void rejectUploadAclWhenOwnershipEnforced(Bucket bucket, String ownerAccount,+ PutObjectOptions options, String aclRequestAccount) {if (!ownershipIsEnforced(bucket)) {return;}@@ -5667,7 +5687,9 @@ public class S3Service implements Resettable, ResourceProvider {&& (isBlank(options.getGrantFullControl()) || ownerGrant);boolean ownerFullControlAcl = "bucket-owner-full-control".equals(options.getAcl())&& isBlank(options.getGrantFullControl());- if (!otherGrants && (withoutAcl || ownerFullControlAcl)) {+ boolean ownerPrivateAcl = "private".equals(options.getAcl())+ && ownerAccount.equals(aclRequestAccount) && isBlank(options.getGrantFullControl());+ if (!otherGrants && (withoutAcl || ownerFullControlAcl || ownerPrivateAcl)) {return;}throw new AwsException("AccessControlListNotSupported", "The bucket does not allow ACLs", 400);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3BucketPolicyEnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3BucketPolicyEnforcementIntegrationTest.javaindex e5aa8ed564..4855d9bfbd 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3BucketPolicyEnforcementIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3BucketPolicyEnforcementIntegrationTest.java@@ -519,6 +519,71 @@ class S3BucketPolicyEnforcementIntegrationTest {.body(equalTo(content));}+ @Test+ void bucketOwnerPrivateMultipartCanBeCompletedByAuthorizedOtherAccount() {+ String suffix = UUID.randomUUID().toString().substring(0, 8);+ String bucket = "cross-private-multipart-" + suffix;+ String key = "object.txt";+ String body = "completed by another account";++ Credentials owner = createAccountAdmin("private-owner-" + suffix, ACCOUNT_A);+ createBucketAs(bucket, owner.signer());+ Credentials writer = createUser("private-writer-" + suffix, ACCOUNT_B);+ putUserPolicy("private-writer-" + suffix, "WriteCrossBucket", """+ {+ "Version": "2012-10-17",+ "Statement": [{+ "Effect": "Allow",+ "Action": "s3:PutObject",+ "Resource": "arn:aws:s3:::%s/*"+ }]+ }""".formatted(bucket), ACCOUNT_B);+ putBucketPolicyAs(bucket, """+ {+ "Version": "2012-10-17",+ "Statement": [{+ "Effect": "Allow",+ "Principal": {"AWS": "%s"},+ "Action": "s3:PutObject",+ "Resource": "arn:aws:s3:::%s/*"+ }]+ }""".formatted(writer.userArn(), bucket), owner.signer());++ String uploadId = given()+ .filter(owner.signer())+ .header("x-amz-acl", "private")+ .when()+ .post("/" + bucket + "/" + key + "?uploads")+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");+ String partETag = given()+ .filter(owner.signer())+ .body(body)+ .when()+ .put("/" + bucket + "/" + key + "?partNumber=1&uploadId=" + uploadId)+ .then()+ .statusCode(200)+ .extract().header("ETag");++ given()+ .filter(writer.signer())+ .contentType("application/xml")+ .body("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"+ + partETag + "</ETag></Part></CompleteMultipartUpload>")+ .when()+ .post("/" + bucket + "/" + key + "?uploadId=" + uploadId)+ .then()+ .statusCode(200);+ given()+ .filter(owner.signer())+ .when()+ .get("/" + bucket + "/" + key)+ .then()+ .statusCode(200)+ .body(equalTo(body));+ }+@Testvoid sameAccountDirectUserBypassesBoundaryOnPrimaryRequest() {String suffix = UUID.randomUUID().toString().substring(0, 8);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3GlobalBucketNamespaceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3GlobalBucketNamespaceTest.javaindex 290c9bfa7e..74978148ac 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3GlobalBucketNamespaceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3GlobalBucketNamespaceTest.java@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.s3.model.Bucket;import io.github.hectorvent.floci.services.s3.model.MultipartUpload;+import io.github.hectorvent.floci.services.s3.model.PutObjectOptions;import io.github.hectorvent.floci.services.s3.model.S3Object;import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.Test;@@ -334,4 +335,34 @@ class S3GlobalBucketNamespaceTest {assertTrue(objects.getForAccount(ACCOUNT_B, versionKey).isEmpty(),"a cross-account multipart completion must not create a caller shadow version");}++ @Test+ void ownerPrivateMultipartUploadCanBeCompletedByAnotherAccount() {+ Instance<RequestContext> ctx = mutableContext();+ AccountAwareStorageBackend<Bucket> buckets =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), ctx, DEFAULT_ACCT);+ AccountAwareStorageBackend<S3Object> objects =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), ctx, DEFAULT_ACCT);+ S3Service globalNs = new S3Service(+ buckets, objects, Path.of("s3-gns-private-multipart-test"), true, true);++ String bucket = "shared-private-multipart-bucket";+ String key = "object.bin";+ caller.set(ACCOUNT_A);+ globalNs.createBucket(bucket, "us-east-1");+ MultipartUpload upload = globalNs.initiateMultipartUpload(bucket, key,+ "application/octet-stream", Map.of(), null, new PutObjectOptions().withAcl("private"));+ globalNs.uploadPart(bucket, key, upload.getUploadId(), 1, "payload".getBytes(UTF_8));++ caller.set(ACCOUNT_B);+ AwsException rejected = assertThrows(AwsException.class, () -> globalNs.initiateMultipartUpload(+ bucket, "unauthorized-private.bin", "application/octet-stream", Map.of(), null,+ new PutObjectOptions().withAcl("private")));+ assertEquals("AccessControlListNotSupported", rejected.getErrorCode());+ assertDoesNotThrow(() -> globalNs.completeMultipartUpload(+ bucket, key, upload.getUploadId(), List.of(1), null, null));++ assertTrue(objects.getForAccount(ACCOUNT_A, bucket + "/" + key).isPresent());+ assertTrue(objects.getForAccount(ACCOUNT_B, bucket + "/" + key).isEmpty());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3OwnershipControlsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3OwnershipControlsIntegrationTest.javaindex 66d48a7275..1bd53b98eb 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3OwnershipControlsIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3OwnershipControlsIntegrationTest.java@@ -60,6 +60,34 @@ class S3OwnershipControlsIntegrationTest {.body(containsString("AccessControlListNotSupported"));}+ @Test+ @Order(10)+ void ownerCanUploadWithPrivateAclOnEnforcedBucket() {+ String bucket = BUCKET + "-private-upload";+ given().when().put("/" + bucket).then().statusCode(200);++ given().header("x-amz-acl", "private").body("single upload")+ .when().put("/" + bucket + "/single")+ .then().statusCode(200);+ given().when().get("/" + bucket + "/single")+ .then().statusCode(200).body(containsString("single upload"));++ String uploadId = given().header("x-amz-acl", "private")+ .when().post("/" + bucket + "/multipart?uploads")+ .then().statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");+ String partETag = given().body("multipart upload")+ .when().put("/" + bucket + "/multipart?uploadId=" + uploadId + "&partNumber=1")+ .then().statusCode(200).extract().header("ETag");+ String completeXml = "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>"+ + partETag + "</ETag></Part></CompleteMultipartUpload>";+ given().contentType("application/xml").body(completeXml)+ .when().post("/" + bucket + "/multipart?uploadId=" + uploadId)+ .then().statusCode(200);+ given().when().get("/" + bucket + "/multipart")+ .then().statusCode(200).body(containsString("multipart upload"));+ }+@Test@Order(2)void createBucketHonorsExplicitOwnershipAndRejectsIncompatibleAcl() {