Light, fluffy, and always free - The AWS Local Emulator alternative
fix: make S3 conditional puts atomic (#1329)
David Phillipsf70709a70f91
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 11e9df6721..ac6fb98b34 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -490,11 +490,6 @@ public class S3Controller {return handleCopyObject(copySource, bucket, key, contentType, httpHeaders);}- Response preconditionResponse = checkWritePreconditions(bucket, key, ifMatch, ifNoneMatch);- if (preconditionResponse != null) {- return preconditionResponse;- }-Map<String, String> inlineTags = parseInlineTaggingHeader(tagging);String lockMode = httpHeaders.getHeaderString("x-amz-object-lock-mode");@@ -529,6 +524,8 @@ public class S3Controller {.withAcl(cannedAcl).withChecksumAlgorithm(checksumAlgorithm).withClientChecksum(extractChecksumFromHeaders(httpHeaders))+ .withIfMatch(ifMatch)+ .withIfNoneMatch(ifNoneMatch).withTagging(inlineTags));var resp = Response.ok().header("ETag", obj.getETag());if (obj.getVersionId() != null) {@@ -1973,14 +1970,25 @@ public class S3Controller {}private Response xmlErrorResponse(AwsException e) {+ String condition = e instanceof S3PreconditionFailedException preconditionFailedException+ ? preconditionFailedException.condition()+ : null;+ return xmlErrorResponse(e, condition);+ }++ private Response xmlErrorResponse(AwsException e, String condition) {if (e.getMessage() == null) {return Response.status(e.getHttpStatus()).build();}- String xml = new XmlBuilder()+ XmlBuilder xmlBuilder = new XmlBuilder().raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").start("Error").elem("Code", e.getErrorCode())- .elem("Message", e.getMessage())+ .elem("Message", e.getMessage());+ if (condition != null) {+ xmlBuilder.elem("Condition", condition);+ }+ String xml = xmlBuilder.elem("RequestId", java.util.UUID.randomUUID().toString()).end("Error").build();@@ -2033,10 +2041,10 @@ public class S3Controller {}if (ifMatch != null && !eTagMatches(ifMatch, existing.getETag())) {- return preconditionFailedResponse();+ return preconditionFailedResponse("If-Match");}if (ifNoneMatch != null && eTagMatches(ifNoneMatch, existing.getETag())) {- return preconditionFailedResponse();+ return preconditionFailedResponse("If-None-Match");}return null;}@@ -2059,8 +2067,12 @@ public class S3Controller {}private Response preconditionFailedResponse() {+ return preconditionFailedResponse(null);+ }++ private Response preconditionFailedResponse(String condition) {return xmlErrorResponse(new AwsException("PreconditionFailed",- "At least one of the pre-conditions you specified did not hold.", 412));+ S3PreconditionFailedException.MESSAGE, 412), condition);}private boolean eTagMatches(String headerValue, String eTag) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3PreconditionFailedException.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3PreconditionFailedException.javanew file mode 100644index 0000000000..2d98146d37--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3PreconditionFailedException.java@@ -0,0 +1,18 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;++final class S3PreconditionFailedException extends AwsException {+ static final String MESSAGE = "At least one of the pre-conditions you specified did not hold";++ private final String condition;++ S3PreconditionFailedException(String condition) {+ super("PreconditionFailed", MESSAGE, 412);+ this.condition = condition;+ }++ String condition() {+ return condition;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 0c863c2baa..a1fe4ce53f 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -283,10 +283,19 @@ public class S3Service {Bucket bucket = bucketStore.get(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));+ synchronized (bucket) {+ return storeObjectInternal(bucket, bucketName, key, data, contentType, metadata, checksum, parts, options);+ }+ }++ private S3Object storeObjectInternal(Bucket bucket, String bucketName, String key, byte[] data,+ String contentType, Map<String, String> metadata,+ S3Checksum checksum, List<Part> parts, PutObjectOptions options) {PutObjectOptions effectiveOptions = options != null ? options : new PutObjectOptions();String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());SseCustomerKey sseCustomerKey = validateSseCustomerKey(effectiveOptions.getSseCustomerAlgorithm(), effectiveOptions.getSseCustomerKey(), effectiveOptions.getSseCustomerKeyMd5());rejectConflictingServerSideEncryption(normalizedServerSideEncryption, sseCustomerKey);+ checkWritePreconditions(bucketName, key, effectiveOptions.getIfMatch(), effectiveOptions.getIfNoneMatch());S3Object object = new S3Object(bucketName, key, data, contentType);if (metadata != null) {@@ -365,6 +374,49 @@ public class S3Service {return object;}+ private void checkWritePreconditions(String bucketName, String key, String ifMatch, String ifNoneMatch) {+ if (ifMatch == null && ifNoneMatch == null) {+ return;+ }++ S3Object existing;+ try {+ existing = headObject(bucketName, key);+ }+ catch (AwsException e) {+ if ("NoSuchKey".equals(e.getErrorCode()) && ifMatch == null) {+ return;+ }+ throw e;+ }++ if (ifMatch != null && !eTagMatches(ifMatch, existing.getETag())) {+ throw new S3PreconditionFailedException("If-Match");+ }+ if (ifNoneMatch != null && eTagMatches(ifNoneMatch, existing.getETag())) {+ throw new S3PreconditionFailedException("If-None-Match");+ }+ }++ private boolean eTagMatches(String headerValue, String eTag) {+ String normalizedETag = normalizeEntityTag(eTag);+ for (String candidate : headerValue.split(",")) {+ String normalizedCandidate = normalizeEntityTag(candidate);+ if ("*".equals(normalizedCandidate) || normalizedCandidate.equals(normalizedETag)) {+ return true;+ }+ }+ return false;+ }++ private String normalizeEntityTag(String value) {+ String normalized = value == null ? "" : value.trim();+ if (normalized.length() >= 2 && normalized.startsWith("\"") && normalized.endsWith("\"")) {+ normalized = normalized.substring(1, normalized.length() - 1);+ }+ return normalized;+ }+private void applyObjectLock(S3Object object, Bucket bucket,String objectLockMode, Instant retainUntilDate, String legalHoldStatus) {if (objectLockMode != null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.javaindex f176527c01..09142f07d3 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java@@ -17,6 +17,8 @@ public class PutObjectOptions {private String sseCustomerKeyMd5;private String acl;private String checksumAlgorithm;+ private String ifMatch;+ private String ifNoneMatch;private Map<String, String> tagging;public String getStorageClass() { return storageClass; }@@ -58,6 +60,12 @@ public class PutObjectOptions {public String getChecksumAlgorithm() { return checksumAlgorithm; }public PutObjectOptions withChecksumAlgorithm(String checksumAlgorithm) { this.checksumAlgorithm = checksumAlgorithm; return this; }+ public String getIfMatch() { return ifMatch; }+ public PutObjectOptions withIfMatch(String ifMatch) { this.ifMatch = ifMatch; return this; }++ public String getIfNoneMatch() { return ifNoneMatch; }+ public PutObjectOptions withIfNoneMatch(String ifNoneMatch) { this.ifNoneMatch = ifNoneMatch; return this; }+public Map<String, String> getTagging() { return tagging; }public PutObjectOptions withTagging(Map<String, String> tagging) { this.tagging = tagging; return this; }diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ConditionalWriteIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ConditionalWriteIntegrationTest.javaindex b8ae55255b..a19f6f675f 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ConditionalWriteIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ConditionalWriteIntegrationTest.java@@ -1,17 +1,30 @@package io.github.hectorvent.floci.services.s3;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.ValidatableResponse;import org.junit.jupiter.api.Test;+import java.util.ArrayList;+import java.util.List;import java.util.UUID;-+import java.util.concurrent.CyclicBarrier;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;++import static java.util.Collections.frequency;+import static java.util.concurrent.TimeUnit.SECONDS;+import static java.util.stream.Collectors.toList;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertEquals;@QuarkusTestclass S3ConditionalWriteIntegrationTest {+ private static final String PRECONDITION_FAILED_MESSAGE =+ "At least one of the pre-conditions you specified did not hold";@Testvoid putObject_ifNoneMatchStar_succeedsWhenKeyMissing() {@@ -34,18 +47,55 @@ class S3ConditionalWriteIntegrationTest {String bucket = createBucket("put-if-none-existing");putObject(bucket, "object.txt", "first");- given()+ ValidatableResponse response = given().header("If-None-Match", "*").body("second").when().put("/" + bucket + "/object.txt")- .then()- .statusCode(412)- .body(containsString("PreconditionFailed"));+ .then();++ assertPreconditionFailed(response, "If-None-Match");assertObjectBody(bucket, "object.txt", "first");}+ @Test+ void concurrentPutObject_ifNoneMatchStar_allowsOnlyOneWriter()+ throws Exception {+ String bucket = createBucket("put-if-none-concurrent");+ String key = "object.txt";+ int writers = 16;+ CyclicBarrier barrier = new CyclicBarrier(writers);+ ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();++ try {+ List<Future<Integer>> futures = java.util.stream.IntStream.range(0, writers)+ .mapToObj(writer -> executor.submit(() -> {+ barrier.await();+ return given()+ .header("If-None-Match", "*")+ .body("writer-" + writer)+ .when()+ .put("/" + bucket + "/" + key)+ .then()+ .extract()+ .statusCode();+ }))+ .collect(toList());++ List<Integer> statusCodes = new ArrayList<>();+ for (Future<Integer> future : futures) {+ statusCodes.add(future.get(30, SECONDS));+ }++ assertEquals(1, frequency(statusCodes, 200));+ assertEquals(writers - 1, frequency(statusCodes, 412));+ }+ finally {+ executor.shutdownNow();+ }+ }+@Testvoid putObject_ifNoneMatchEtag_succeedsWhenEtagDiffers() {String bucket = createBucket("put-if-none-different");@@ -67,14 +117,14 @@ class S3ConditionalWriteIntegrationTest {String bucket = createBucket("put-if-none-match");String eTag = putObject(bucket, "object.txt", "first");- given()+ ValidatableResponse response = given().header("If-None-Match", eTag).body("second").when().put("/" + bucket + "/object.txt")- .then()- .statusCode(412)- .body(containsString("PreconditionFailed"));+ .then();++ assertPreconditionFailed(response, "If-None-Match");assertObjectBody(bucket, "object.txt", "first");}@@ -100,14 +150,14 @@ class S3ConditionalWriteIntegrationTest {String bucket = createBucket("put-if-match-wrong");putObject(bucket, "object.txt", "first");- given()+ ValidatableResponse response = given().header("If-Match", "\"not-the-current-etag\"").body("second").when().put("/" + bucket + "/object.txt")- .then()- .statusCode(412)- .body(containsString("PreconditionFailed"));+ .then();++ assertPreconditionFailed(response, "If-Match");assertObjectBody(bucket, "object.txt", "first");}@@ -132,23 +182,23 @@ class S3ConditionalWriteIntegrationTest {.statusCode(200).extract().header("ETag");- given()+ ValidatableResponse response = given().header("If-None-Match", stripQuotes(currentETag)).body("third").when().put("/" + bucket + "/object.txt")- .then()- .statusCode(412)- .body(containsString("PreconditionFailed"));+ .then();- given()+ assertPreconditionFailed(response, "If-None-Match");++ response = given().header("If-None-Match", "\"*\"").body("third").when().put("/" + bucket + "/object.txt")- .then()- .statusCode(412)- .body(containsString("PreconditionFailed"));+ .then();++ assertPreconditionFailed(response, "If-None-Match");assertObjectBody(bucket, "object.txt", "second");}@@ -160,15 +210,15 @@ class S3ConditionalWriteIntegrationTest {String uploadId = initiateMultipartUpload(bucket, "object.txt");uploadPart(bucket, "object.txt", uploadId, 1, "second");- given()+ ValidatableResponse response = given().contentType("application/xml").header("If-None-Match", "*").body(completeMultipartXml(1)).when().post("/" + bucket + "/object.txt?uploadId=" + uploadId)- .then()- .statusCode(412)- .body(containsString("PreconditionFailed"));+ .then();++ assertPreconditionFailed(response, "If-None-Match");assertObjectBody(bucket, "object.txt", "first");}@@ -200,15 +250,15 @@ class S3ConditionalWriteIntegrationTest {String uploadId = initiateMultipartUpload(bucket, "object.txt");uploadPart(bucket, "object.txt", uploadId, 1, "second");- given()+ ValidatableResponse response = given().contentType("application/xml").header("If-Match", "\"not-the-current-etag\"").body(completeMultipartXml(1)).when().post("/" + bucket + "/object.txt?uploadId=" + uploadId)- .then()- .statusCode(412)- .body(containsString("PreconditionFailed"));+ .then();++ assertPreconditionFailed(response, "If-Match");assertObjectBody(bucket, "object.txt", "first");}@@ -242,6 +292,13 @@ class S3ConditionalWriteIntegrationTest {.body(equalTo(body));}+ private static void assertPreconditionFailed(ValidatableResponse response, String condition) {+ response.statusCode(412)+ .body("Error.Code", equalTo("PreconditionFailed"))+ .body("Error.Message", equalTo(PRECONDITION_FAILED_MESSAGE))+ .body("Error.Condition", equalTo(condition));+ }+private static String initiateMultipartUpload(String bucket, String key) {return given().contentType("application/octet-stream")