Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: make S3 conditional puts atomic (#1329)

David Phillipsf70709a70f91
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 11e9df6721..ac6fb98b34 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -490,11 +490,6 @@ public class S3Controller {
return handleCopyObject(copySource, bucket, key, contentType, httpHeaders);
}
- Response preconditionResponse = checkWritePreconditions(bucket, key, ifMatch, ifNoneMatch);
- if (preconditionResponse != null) {
- return preconditionResponse;
- }
-
Map<String, String> inlineTags = parseInlineTaggingHeader(tagging);
String lockMode = httpHeaders.getHeaderString("x-amz-object-lock-mode");
@@ -529,6 +524,8 @@ public class S3Controller {
.withAcl(cannedAcl)
.withChecksumAlgorithm(checksumAlgorithm)
.withClientChecksum(extractChecksumFromHeaders(httpHeaders))
+ .withIfMatch(ifMatch)
+ .withIfNoneMatch(ifNoneMatch)
.withTagging(inlineTags));
var resp = Response.ok().header("ETag", obj.getETag());
if (obj.getVersionId() != null) {
@@ -1973,14 +1970,25 @@ public class S3Controller {
}
private Response xmlErrorResponse(AwsException e) {
+ String condition = e instanceof S3PreconditionFailedException preconditionFailedException
+ ? preconditionFailedException.condition()
+ : null;
+ return xmlErrorResponse(e, condition);
+ }
+
+ private Response xmlErrorResponse(AwsException e, String condition) {
if (e.getMessage() == null) {
return Response.status(e.getHttpStatus()).build();
}
- String xml = new XmlBuilder()
+ XmlBuilder xmlBuilder = new XmlBuilder()
.raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
.start("Error")
.elem("Code", e.getErrorCode())
- .elem("Message", e.getMessage())
+ .elem("Message", e.getMessage());
+ if (condition != null) {
+ xmlBuilder.elem("Condition", condition);
+ }
+ String xml = xmlBuilder
.elem("RequestId", java.util.UUID.randomUUID().toString())
.end("Error")
.build();
@@ -2033,10 +2041,10 @@ public class S3Controller {
}
if (ifMatch != null && !eTagMatches(ifMatch, existing.getETag())) {
- return preconditionFailedResponse();
+ return preconditionFailedResponse("If-Match");
}
if (ifNoneMatch != null && eTagMatches(ifNoneMatch, existing.getETag())) {
- return preconditionFailedResponse();
+ return preconditionFailedResponse("If-None-Match");
}
return null;
}
@@ -2059,8 +2067,12 @@ public class S3Controller {
}
private Response preconditionFailedResponse() {
+ return preconditionFailedResponse(null);
+ }
+
+ private Response preconditionFailedResponse(String condition) {
return xmlErrorResponse(new AwsException("PreconditionFailed",
- "At least one of the pre-conditions you specified did not hold.", 412));
+ S3PreconditionFailedException.MESSAGE, 412), condition);
}
private boolean eTagMatches(String headerValue, String eTag) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3PreconditionFailedException.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3PreconditionFailedException.java
new file mode 100644
index 0000000000..2d98146d37
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3PreconditionFailedException.java
@@ -0,0 +1,18 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+
+final class S3PreconditionFailedException extends AwsException {
+ static final String MESSAGE = "At least one of the pre-conditions you specified did not hold";
+
+ private final String condition;
+
+ S3PreconditionFailedException(String condition) {
+ super("PreconditionFailed", MESSAGE, 412);
+ this.condition = condition;
+ }
+
+ String condition() {
+ return condition;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 0c863c2baa..a1fe4ce53f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -283,10 +283,19 @@ public class S3Service {
Bucket bucket = bucketStore.get(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket",
"The specified bucket does not exist.", 404));
+ synchronized (bucket) {
+ return storeObjectInternal(bucket, bucketName, key, data, contentType, metadata, checksum, parts, options);
+ }
+ }
+
+ private S3Object storeObjectInternal(Bucket bucket, String bucketName, String key, byte[] data,
+ String contentType, Map<String, String> metadata,
+ S3Checksum checksum, List<Part> parts, PutObjectOptions options) {
PutObjectOptions effectiveOptions = options != null ? options : new PutObjectOptions();
String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());
SseCustomerKey sseCustomerKey = validateSseCustomerKey(effectiveOptions.getSseCustomerAlgorithm(), effectiveOptions.getSseCustomerKey(), effectiveOptions.getSseCustomerKeyMd5());
rejectConflictingServerSideEncryption(normalizedServerSideEncryption, sseCustomerKey);
+ checkWritePreconditions(bucketName, key, effectiveOptions.getIfMatch(), effectiveOptions.getIfNoneMatch());
S3Object object = new S3Object(bucketName, key, data, contentType);
if (metadata != null) {
@@ -365,6 +374,49 @@ public class S3Service {
return object;
}
+ private void checkWritePreconditions(String bucketName, String key, String ifMatch, String ifNoneMatch) {
+ if (ifMatch == null && ifNoneMatch == null) {
+ return;
+ }
+
+ S3Object existing;
+ try {
+ existing = headObject(bucketName, key);
+ }
+ catch (AwsException e) {
+ if ("NoSuchKey".equals(e.getErrorCode()) && ifMatch == null) {
+ return;
+ }
+ throw e;
+ }
+
+ if (ifMatch != null && !eTagMatches(ifMatch, existing.getETag())) {
+ throw new S3PreconditionFailedException("If-Match");
+ }
+ if (ifNoneMatch != null && eTagMatches(ifNoneMatch, existing.getETag())) {
+ throw new S3PreconditionFailedException("If-None-Match");
+ }
+ }
+
+ private boolean eTagMatches(String headerValue, String eTag) {
+ String normalizedETag = normalizeEntityTag(eTag);
+ for (String candidate : headerValue.split(",")) {
+ String normalizedCandidate = normalizeEntityTag(candidate);
+ if ("*".equals(normalizedCandidate) || normalizedCandidate.equals(normalizedETag)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private String normalizeEntityTag(String value) {
+ String normalized = value == null ? "" : value.trim();
+ if (normalized.length() >= 2 && normalized.startsWith("\"") && normalized.endsWith("\"")) {
+ normalized = normalized.substring(1, normalized.length() - 1);
+ }
+ return normalized;
+ }
+
private void applyObjectLock(S3Object object, Bucket bucket,
String objectLockMode, Instant retainUntilDate, String legalHoldStatus) {
if (objectLockMode != null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java
index f176527c01..09142f07d3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java
@@ -17,6 +17,8 @@ public class PutObjectOptions {
private String sseCustomerKeyMd5;
private String acl;
private String checksumAlgorithm;
+ private String ifMatch;
+ private String ifNoneMatch;
private Map<String, String> tagging;
public String getStorageClass() { return storageClass; }
@@ -58,6 +60,12 @@ public class PutObjectOptions {
public String getChecksumAlgorithm() { return checksumAlgorithm; }
public PutObjectOptions withChecksumAlgorithm(String checksumAlgorithm) { this.checksumAlgorithm = checksumAlgorithm; return this; }
+ public String getIfMatch() { return ifMatch; }
+ public PutObjectOptions withIfMatch(String ifMatch) { this.ifMatch = ifMatch; return this; }
+
+ public String getIfNoneMatch() { return ifNoneMatch; }
+ public PutObjectOptions withIfNoneMatch(String ifNoneMatch) { this.ifNoneMatch = ifNoneMatch; return this; }
+
public Map<String, String> getTagging() { return tagging; }
public PutObjectOptions withTagging(Map<String, String> tagging) { this.tagging = tagging; return this; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ConditionalWriteIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ConditionalWriteIntegrationTest.java
index b8ae55255b..a19f6f675f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ConditionalWriteIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ConditionalWriteIntegrationTest.java
@@ -1,17 +1,30 @@
package io.github.hectorvent.floci.services.s3;
import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.ValidatableResponse;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
+import java.util.List;
import java.util.UUID;
-
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import static java.util.Collections.frequency;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.stream.Collectors.toList;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
@QuarkusTest
class S3ConditionalWriteIntegrationTest {
+ private static final String PRECONDITION_FAILED_MESSAGE =
+ "At least one of the pre-conditions you specified did not hold";
@Test
void putObject_ifNoneMatchStar_succeedsWhenKeyMissing() {
@@ -34,18 +47,55 @@ class S3ConditionalWriteIntegrationTest {
String bucket = createBucket("put-if-none-existing");
putObject(bucket, "object.txt", "first");
- given()
+ ValidatableResponse response = given()
.header("If-None-Match", "*")
.body("second")
.when()
.put("/" + bucket + "/object.txt")
- .then()
- .statusCode(412)
- .body(containsString("PreconditionFailed"));
+ .then();
+
+ assertPreconditionFailed(response, "If-None-Match");
assertObjectBody(bucket, "object.txt", "first");
}
+ @Test
+ void concurrentPutObject_ifNoneMatchStar_allowsOnlyOneWriter()
+ throws Exception {
+ String bucket = createBucket("put-if-none-concurrent");
+ String key = "object.txt";
+ int writers = 16;
+ CyclicBarrier barrier = new CyclicBarrier(writers);
+ ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
+
+ try {
+ List<Future<Integer>> futures = java.util.stream.IntStream.range(0, writers)
+ .mapToObj(writer -> executor.submit(() -> {
+ barrier.await();
+ return given()
+ .header("If-None-Match", "*")
+ .body("writer-" + writer)
+ .when()
+ .put("/" + bucket + "/" + key)
+ .then()
+ .extract()
+ .statusCode();
+ }))
+ .collect(toList());
+
+ List<Integer> statusCodes = new ArrayList<>();
+ for (Future<Integer> future : futures) {
+ statusCodes.add(future.get(30, SECONDS));
+ }
+
+ assertEquals(1, frequency(statusCodes, 200));
+ assertEquals(writers - 1, frequency(statusCodes, 412));
+ }
+ finally {
+ executor.shutdownNow();
+ }
+ }
+
@Test
void putObject_ifNoneMatchEtag_succeedsWhenEtagDiffers() {
String bucket = createBucket("put-if-none-different");
@@ -67,14 +117,14 @@ class S3ConditionalWriteIntegrationTest {
String bucket = createBucket("put-if-none-match");
String eTag = putObject(bucket, "object.txt", "first");
- given()
+ ValidatableResponse response = given()
.header("If-None-Match", eTag)
.body("second")
.when()
.put("/" + bucket + "/object.txt")
- .then()
- .statusCode(412)
- .body(containsString("PreconditionFailed"));
+ .then();
+
+ assertPreconditionFailed(response, "If-None-Match");
assertObjectBody(bucket, "object.txt", "first");
}
@@ -100,14 +150,14 @@ class S3ConditionalWriteIntegrationTest {
String bucket = createBucket("put-if-match-wrong");
putObject(bucket, "object.txt", "first");
- given()
+ ValidatableResponse response = given()
.header("If-Match", "\"not-the-current-etag\"")
.body("second")
.when()
.put("/" + bucket + "/object.txt")
- .then()
- .statusCode(412)
- .body(containsString("PreconditionFailed"));
+ .then();
+
+ assertPreconditionFailed(response, "If-Match");
assertObjectBody(bucket, "object.txt", "first");
}
@@ -132,23 +182,23 @@ class S3ConditionalWriteIntegrationTest {
.statusCode(200)
.extract().header("ETag");
- given()
+ ValidatableResponse response = given()
.header("If-None-Match", stripQuotes(currentETag))
.body("third")
.when()
.put("/" + bucket + "/object.txt")
- .then()
- .statusCode(412)
- .body(containsString("PreconditionFailed"));
+ .then();
- given()
+ assertPreconditionFailed(response, "If-None-Match");
+
+ response = given()
.header("If-None-Match", "\"*\"")
.body("third")
.when()
.put("/" + bucket + "/object.txt")
- .then()
- .statusCode(412)
- .body(containsString("PreconditionFailed"));
+ .then();
+
+ assertPreconditionFailed(response, "If-None-Match");
assertObjectBody(bucket, "object.txt", "second");
}
@@ -160,15 +210,15 @@ class S3ConditionalWriteIntegrationTest {
String uploadId = initiateMultipartUpload(bucket, "object.txt");
uploadPart(bucket, "object.txt", uploadId, 1, "second");
- given()
+ ValidatableResponse response = given()
.contentType("application/xml")
.header("If-None-Match", "*")
.body(completeMultipartXml(1))
.when()
.post("/" + bucket + "/object.txt?uploadId=" + uploadId)
- .then()
- .statusCode(412)
- .body(containsString("PreconditionFailed"));
+ .then();
+
+ assertPreconditionFailed(response, "If-None-Match");
assertObjectBody(bucket, "object.txt", "first");
}
@@ -200,15 +250,15 @@ class S3ConditionalWriteIntegrationTest {
String uploadId = initiateMultipartUpload(bucket, "object.txt");
uploadPart(bucket, "object.txt", uploadId, 1, "second");
- given()
+ ValidatableResponse response = given()
.contentType("application/xml")
.header("If-Match", "\"not-the-current-etag\"")
.body(completeMultipartXml(1))
.when()
.post("/" + bucket + "/object.txt?uploadId=" + uploadId)
- .then()
- .statusCode(412)
- .body(containsString("PreconditionFailed"));
+ .then();
+
+ assertPreconditionFailed(response, "If-Match");
assertObjectBody(bucket, "object.txt", "first");
}
@@ -242,6 +292,13 @@ class S3ConditionalWriteIntegrationTest {
.body(equalTo(body));
}
+ private static void assertPreconditionFailed(ValidatableResponse response, String condition) {
+ response.statusCode(412)
+ .body("Error.Code", equalTo("PreconditionFailed"))
+ .body("Error.Message", equalTo(PRECONDITION_FAILED_MESSAGE))
+ .body("Error.Condition", equalTo(condition));
+ }
+
private static String initiateMultipartUpload(String bucket, String key) {
return given()
.contentType("application/octet-stream")