Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(s3): support bucket metrics configurations (#2382)

Michal Schreiber8d33c6512951
diff --git a/compatibility-tests/sdk-test-awscli/test/s3.bats b/compatibility-tests/sdk-test-awscli/test/s3.bats
index 3fe298197a..3d70832af2 100644
--- a/compatibility-tests/sdk-test-awscli/test/s3.bats
+++ b/compatibility-tests/sdk-test-awscli/test/s3.bats
@@ -305,3 +305,37 @@ EOF
rm -f "$large_file"
}
+
+@test "S3: metrics configurations round-trip and never touch the bucket" {
+ aws_cmd s3api create-bucket --bucket "$BUCKET" >/dev/null
+
+ run aws_cmd s3api put-bucket-metrics-configuration \
+ --bucket "$BUCKET" --id EntireBucket --metrics-configuration 'Id=EntireBucket'
+ assert_success
+
+ run aws_cmd s3api get-bucket-metrics-configuration --bucket "$BUCKET" --id EntireBucket
+ assert_success
+ [ "$(json_get "$output" '.MetricsConfiguration.Id')" = "EntireBucket" ]
+
+ run aws_cmd s3api put-bucket-metrics-configuration \
+ --bucket "$BUCKET" --id Filtered \
+ --metrics-configuration 'Id=Filtered,Filter={And={Prefix=logs/,Tags=[{Key=env,Value=prod}]}}'
+ assert_success
+
+ run aws_cmd s3api list-bucket-metrics-configurations --bucket "$BUCKET"
+ assert_success
+ [ "$(json_get "$output" '.MetricsConfigurationList | length')" = "2" ]
+ [ "$(json_get "$output" '.MetricsConfigurationList[1].Filter.And.Prefix')" = "logs/" ]
+ [ "$(json_get "$output" '.MetricsConfigurationList[1].Filter.And.Tags[0].Key')" = "env" ]
+
+ # A sub-resource DELETE removes only that configuration; the bucket must survive it.
+ run aws_cmd s3api delete-bucket-metrics-configuration --bucket "$BUCKET" --id EntireBucket
+ assert_success
+
+ run aws_cmd s3api head-bucket --bucket "$BUCKET"
+ assert_success
+
+ run aws_cmd s3api get-bucket-metrics-configuration --bucket "$BUCKET" --id EntireBucket
+ assert_failure
+ assert_output --partial "NoSuchConfiguration"
+}
diff --git a/docs/services/s3.md b/docs/services/s3.md
index f6cdb03e28..b516802af2 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -24,6 +24,7 @@
| **Pre-signed URLs** | Generates and validates pre-signed GET/PUT URLs |
| **S3 Select** | SelectObjectContent |
| **Public Access Block** | PutPublicAccessBlock, GetPublicAccessBlock, DeletePublicAccessBlock |
+| **Metrics** | PutBucketMetricsConfiguration, GetBucketMetricsConfiguration, ListBucketMetricsConfigurations, DeleteBucketMetricsConfiguration |
**RestoreObject** is accepted but stubbed: Floci validates the request and returns `202 Accepted`, but no restore state machine runs.
@@ -152,7 +153,7 @@ These AWS S3 features have no handler in Floci. Calls will return an error (typi
- Request payment (`PutBucketRequestPayment`, `GetBucketRequestPayment`)
- Intelligent-Tiering configurations
- Inventory configurations
-- Metrics and Analytics configurations
+- Analytics configurations
## Configuration
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 884101fe81..6702bea8be 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -324,6 +324,11 @@ public class S3Controller {
s3Service.putBucketAccelerateConfiguration(bucket, new String(body, StandardCharsets.UTF_8));
return Response.ok().build();
}
+ // Must be handled here: an unmatched subresource falls through to CreateBucket below,
+ // which answers a metrics call with BucketAlreadyOwnedByYou.
+ if (hasQueryParam(uriInfo, "metrics")) {
+ return handlePutBucketMetricsConfiguration(bucket, uriInfo, body);
+ }
String locationConstraint = null;
if (body != null && body.length > 0) {
@@ -407,6 +412,11 @@ public class S3Controller {
// handled here so it does NOT fall through to deleting the whole bucket.
return Response.noContent().build();
}
+ if (hasQueryParam(uriInfo, "metrics")) {
+ // Likewise this must not fall through to deleting the bucket.
+ s3Service.deleteBucketMetricsConfiguration(bucket, requireMetricsId(uriInfo));
+ return Response.noContent().build();
+ }
if (hasQueryParam(uriInfo, "accelerate")) {
// AWS defines no DELETE for the accelerate subresource; reject it here so
// it does NOT fall through to deleting the whole bucket.
@@ -518,6 +528,16 @@ public class S3Controller {
s3Service.authorizeBucketRead(bucket, "s3:GetAccelerateConfiguration", authorization);
return Response.ok(s3Service.getBucketAccelerateConfiguration(bucket)).build();
}
+ // GetBucketMetricsConfiguration and ListBucketMetricsConfigurations share ?metrics and
+ // are told apart by the id, which only the single-configuration read carries.
+ if (hasQueryParam(uriInfo, "metrics")) {
+ s3Service.authorizeBucketRead(bucket, "s3:GetMetricsConfiguration", authorization);
+ String id = uriInfo.getQueryParameters().getFirst("id");
+ String xml = id != null
+ ? s3Service.getBucketMetricsConfiguration(bucket, id)
+ : s3Service.listBucketMetricsConfigurations(bucket);
+ return Response.ok(xml).type("application/xml").build();
+ }
// --- S3 static-website index resolution (site root) ---
// A website endpoint has no S3 REST API, so it serves the index document for the site root
@@ -1778,6 +1798,34 @@ public class S3Controller {
return Response.ok(buildTaggingXml(tags)).type(MediaType.APPLICATION_XML).build();
}
+ // --- Metrics Configurations ---
+
+ private Response handlePutBucketMetricsConfiguration(String bucket, UriInfo uriInfo, byte[] body) {
+ String id = requireMetricsId(uriInfo);
+ S3MetricsConfiguration configuration =
+ S3MetricsConfiguration.parse(new String(body, StandardCharsets.UTF_8));
+ // AWS rejects a body whose Id disagrees with the id in the query string.
+ if (!id.equals(configuration.id())) {
+ throw new AwsException("MalformedXML",
+ "The XML you provided was not well-formed or did not validate against our "
+ + "published schema", 400);
+ }
+ s3Service.putBucketMetricsConfiguration(bucket, id, configuration.innerXml());
+ return Response.noContent().build();
+ }
+
+ /**
+ * The id identifies the configuration, so a request without one is refused rather than guessed
+ * at. AWS's own answer here was not verified, so floci gives one rejection for both verbs.
+ */
+ private String requireMetricsId(UriInfo uriInfo) {
+ String id = uriInfo.getQueryParameters().getFirst("id");
+ if (id == null) {
+ throw new AwsException("InvalidArgument", "The metrics id must be specified.", 400);
+ }
+ return id;
+ }
+
// --- Object Tagging ---
private Response handlePutObjectTagging(String bucket, String key, byte[] body) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3MetricsConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3MetricsConfiguration.java
new file mode 100644
index 0000000000..3fb6bb4807
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3MetricsConfiguration.java
@@ -0,0 +1,127 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.XmlBuilder;
+import io.github.hectorvent.floci.core.common.XmlParser;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A parsed {@code MetricsConfiguration} request body, reduced to its id and a canonical
+ * serialization of its contents.
+ *
+ * <p>The body is re-serialized rather than stored verbatim, so that what a later
+ * GetBucketMetricsConfiguration or ListBucketMetricsConfigurations returns is built by floci
+ * instead of being whatever XML the caller sent, and so that the same stored form can be wrapped
+ * in either response.
+ */
+record S3MetricsConfiguration(String id, String innerXml) {
+
+ private static final String ROOT = "MetricsConfiguration";
+
+ /** AWS reports a body that does not match the published schema this way. */
+ private static AwsException malformed() {
+ return new AwsException("MalformedXML",
+ "The XML you provided was not well-formed or did not validate against our published schema", 400);
+ }
+
+ static S3MetricsConfiguration parse(String xml) {
+ if (xml == null || xml.isBlank() || !ROOT.equals(XmlParser.rootElementName(xml))) {
+ throw malformed();
+ }
+
+ // The configuration is one Id and at most one Filter, so anything else under the root —
+ // a stray element, a second Id, a second Filter — is a body AWS would not have accepted.
+ List<String> children = XmlParser.childElementNames(xml, ROOT);
+ long ids = children.stream().filter("Id"::equals).count();
+ long filters = children.stream().filter("Filter"::equals).count();
+ if (ids != 1 || filters > 1 || children.size() != ids + filters) {
+ throw malformed();
+ }
+ boolean hasFilter = filters == 1;
+
+ String id = XmlParser.extractFirst(xml, "Id", null);
+ if (id == null || id.isBlank()) {
+ throw malformed();
+ }
+
+ XmlBuilder inner = new XmlBuilder().elem("Id", id);
+ if (hasFilter) {
+ inner.start("Filter").raw(filterXml(xml)).end("Filter");
+ }
+ return new S3MetricsConfiguration(id, inner.build());
+ }
+
+ /**
+ * Serializes the filter. A filter is exactly one of a prefix, an object tag, an access point
+ * ARN, or an And conjunction: AWS answers MalformedXML for a filter naming none of them or
+ * more than one, and for an And holding fewer than two predicates.
+ */
+ private static String filterXml(String xml) {
+ List<String> predicates = XmlParser.childElementNames(xml, "Filter");
+ if (predicates.size() != 1) {
+ throw malformed();
+ }
+
+ String predicate = predicates.getFirst();
+ return switch (predicate) {
+ case "Prefix", "AccessPointArn" -> new XmlBuilder()
+ .elem(predicate, requireText(xml, predicate))
+ .build();
+ case "Tag" -> tagsXml(xml, 1);
+ case "And" -> {
+ List<String> conjuncts = XmlParser.childElementNames(xml, "And");
+ if (conjuncts.size() < 2) {
+ throw malformed();
+ }
+ XmlBuilder and = new XmlBuilder().start("And");
+ if (conjuncts.contains("Prefix")) {
+ and.elem("Prefix", requireText(xml, "Prefix"));
+ }
+ long tagCount = conjuncts.stream().filter("Tag"::equals).count();
+ if (tagCount > 0) {
+ and.raw(tagsXml(xml, (int) tagCount));
+ }
+ if (conjuncts.contains("AccessPointArn")) {
+ and.elem("AccessPointArn", requireText(xml, "AccessPointArn"));
+ }
+ // Every conjunct has to be one floci understands, or the filter it stores would
+ // not be the filter that was sent.
+ if (conjuncts.size() != tagCount + (conjuncts.contains("Prefix") ? 1 : 0)
+ + (conjuncts.contains("AccessPointArn") ? 1 : 0)) {
+ throw malformed();
+ }
+ yield and.end("And").build();
+ }
+ default -> throw malformed();
+ };
+ }
+
+ /**
+ * Serializes {@code expected} tags. A tag carries both a key and a value, so a pair short of
+ * the element count means one of them was missing.
+ */
+ private static String tagsXml(String xml, int expected) {
+ Map<String, String> tags = XmlParser.extractPairs(xml, "Tag", "Key", "Value");
+ if (tags.size() != expected) {
+ throw malformed();
+ }
+ XmlBuilder out = new XmlBuilder();
+ tags.forEach((key, value) -> {
+ if (key.isBlank() || value == null) {
+ throw malformed();
+ }
+ out.start("Tag").elem("Key", key).elem("Value", value).end("Tag");
+ });
+ return out.build();
+ }
+
+ private static String requireText(String xml, String element) {
+ String value = XmlParser.extractFirst(xml, element, null);
+ if (value == null) {
+ throw malformed();
+ }
+ return value;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 1b689adfff..a95101d020 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -228,7 +228,19 @@ public class S3Service implements Resettable {
public void deleteBucket(String bucketName) {
ensureBucketExists(bucketName);
+ Bucket bucket = bucketStore.get(bucketName)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404));
+
+ // Takes the bucket monitor that the bucket-scoped mutations take: a mutation that read the
+ // record before the delete would otherwise write it back afterwards, restoring the bucket.
+ synchronized (bucket) {
+ deleteBucketLocked(bucketName);
+ }
+ LOG.infov("Deleted bucket: {0}", bucketName);
+ }
+ private void deleteBucketLocked(String bucketName) {
// Check if bucket is empty
List<S3Object> objects = listObjects(bucketName, null, null, 1);
if (!objects.isEmpty()) {
@@ -243,7 +255,6 @@ public class S3Service implements Resettable {
} else {
deleteDirectory(dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(bucketName));
}
- LOG.infov("Deleted bucket: {0}", bucketName);
}
public List<Bucket> listBuckets() {
@@ -1634,6 +1645,107 @@ public class S3Service implements Resettable {
LOG.debugv("Deleted tags from bucket: {0}", bucketName);
}
+ // --- Metrics Configurations ---
+
+ private static final String METRICS_XML_DECLARATION = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
+
+ /**
+ * Stores a CloudWatch request metrics configuration under {@code id}, replacing any
+ * configuration already stored under it. floci records the configuration and returns it; no
+ * metrics are produced from it.
+ */
+ public void putBucketMetricsConfiguration(String bucketName, String id, String innerXml) {
+ Bucket bucket = requireBucket(bucketName);
+ // Read-modify-write of the bucket record, so it takes the same monitor as the other
+ // bucket-scoped mutations: without it two concurrent puts of different ids both start from
+ // the same map and one of the configurations is lost.
+ synchronized (bucket) {
+ requireSameRecord(bucketName, bucket);
+ Map<String, String> configurations = bucket.getMetricsConfigurations() != null
+ ? new java.util.LinkedHashMap<>(bucket.getMetricsConfigurations())
+ : new java.util.LinkedHashMap<>();
+ configurations.put(id, innerXml);
+ bucket.setMetricsConfigurations(configurations);
+ bucketStore.put(bucketName, bucket);
+ }
+ LOG.debugv("Put metrics configuration {0} on bucket: {1}", id, bucketName);
+ }
+
+ public String getBucketMetricsConfiguration(String bucketName, String id) {
+ Bucket bucket = requireBucket(bucketName);
+ String innerXml = bucket.getMetricsConfigurations() == null
+ ? null : bucket.getMetricsConfigurations().get(id);
+ if (innerXml == null) {
+ throw noSuchMetricsConfiguration();
+ }
+ return METRICS_XML_DECLARATION + new XmlBuilder()
+ .start("MetricsConfiguration", AwsNamespaces.S3)
+ .raw(innerXml)
+ .end("MetricsConfiguration")
+ .build();
+ }
+
+ /**
+ * Lists every metrics configuration on the bucket. AWS pages these with a continuation token
+ * once there are more than 100; floci returns them all in one unpaged response, ordered by id
+ * so that the listing is stable.
+ */
+ public String listBucketMetricsConfigurations(String bucketName) {
+ Bucket bucket = requireBucket(bucketName);
+ Map<String, String> configurations = bucket.getMetricsConfigurations() != null
+ ? bucket.getMetricsConfigurations() : Map.of();
+
+ XmlBuilder xml = new XmlBuilder().start("ListMetricsConfigurationsResult", AwsNamespaces.S3);
+ configurations.keySet().stream().sorted().forEach(id -> xml
+ .start("MetricsConfiguration")
+ .raw(configurations.get(id))
+ .end("MetricsConfiguration"));
+ return METRICS_XML_DECLARATION + xml
+ .elem("IsTruncated", false)
+ .end("ListMetricsConfigurationsResult")
+ .build();
+ }
+
+ public void deleteBucketMetricsConfiguration(String bucketName, String id) {
+ Bucket bucket = requireBucket(bucketName);
+ // Same monitor as the put: the existence check and the write have to be one step, or a
+ // concurrent put of another id is dropped by the write that follows it.
+ synchronized (bucket) {
+ requireSameRecord(bucketName, bucket);
+ Map<String, String> configurations = bucket.getMetricsConfigurations();
+ if (configurations == null || !configurations.containsKey(id)) {
+ throw noSuchMetricsConfiguration();
+ }
+ Map<String, String> remaining = new java.util.LinkedHashMap<>(configurations);
+ remaining.remove(id);
+ bucket.setMetricsConfigurations(remaining);
+ bucketStore.put(bucketName, bucket);
+ }
+ LOG.debugv("Deleted metrics configuration {0} from bucket: {1}", id, bucketName);
+ }
+
+ private Bucket requireBucket(String bucketName) {
+ return bucketStore.get(bucketName)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404));
+ }
+
+ /**
+ * Re-reads the bucket under its monitor and checks it is still the same record. Presence alone
+ * is not enough: a bucket deleted and recreated under the same name leaves a different record
+ * in the store, and writing the resolved one back would replace the new bucket with the old
+ * one's state.
+ */
+ private void requireSameRecord(String bucketName, Bucket resolved) {
+ if (bucketStore.get(bucketName).orElse(null) != resolved) {
+ throw new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404);
+ }
+ }
+
+ private static AwsException noSuchMetricsConfiguration() {
+ return new AwsException("NoSuchConfiguration", "The specified configuration does not exist.", 404);
+ }
+
// --- Object Lock Configuration ---
public void setBucketObjectLockEnabled(String bucketName) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/Bucket.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/Bucket.java
index 34196e885f..762ed2b0dd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/Bucket.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/Bucket.java
@@ -31,6 +31,8 @@ public class Bucket {
private String accelerateStatus; // "Enabled" or "Suspended"; null until first PUT
private String region;
private WebsiteConfiguration websiteConfiguration;
+ /** CloudWatch request metrics configurations, keyed by the id they were stored under. */
+ private Map<String, String> metricsConfigurations;
public Bucket() {
this.tags = new HashMap<>();
@@ -117,4 +119,7 @@ public class Bucket {
public WebsiteConfiguration getWebsiteConfiguration() { return websiteConfiguration; }
public void setWebsiteConfiguration(WebsiteConfiguration websiteConfiguration) { this.websiteConfiguration = websiteConfiguration; }
+
+ public Map<String, String> getMetricsConfigurations() { return metricsConfigurations; }
+ public void setMetricsConfigurations(Map<String, String> metricsConfigurations) { this.metricsConfigurations = metricsConfigurations; }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationIntegrationTest.java
new file mode 100644
index 0000000000..e15e021161
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationIntegrationTest.java
@@ -0,0 +1,232 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class S3MetricsConfigurationIntegrationTest {
+
+ private static final String BUCKET = "metrics-int-test";
+
+ @Test
+ @Order(1)
+ void createBucket() {
+ given()
+ .when()
+ .put("/" + BUCKET)
+ .then()
+ .statusCode(200);
+ }
+
+ /**
+ * Regression test for the bucket-creating bug where {@code PUT /{bucket}?metrics} was not
+ * handled and fell through to the unqualified {@code CreateBucket}, which answered a metrics
+ * call with BucketAlreadyOwnedByYou. Real S3 stores the configuration and returns 204.
+ */
+ @Test
+ @Order(2)
+ void putMetricsConfigurationIsNotTreatedAsCreateBucket() {
+ given()
+ .body("""
+ <MetricsConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Id>EntireBucket</Id>
+ </MetricsConfiguration>
+ """)
+ .when()
+ .put("/" + BUCKET + "?metrics&id=EntireBucket")
+ .then()
+ .statusCode(204)
+ .body(not(containsString("BucketAlreadyOwnedByYou")));
+ }
+
+ @Test
+ @Order(3)
+ void getMetricsConfigurationReturnsWhatWasStored() {
+ given()
+ .when()
+ .get("/" + BUCKET + "?metrics&id=EntireBucket")
+ .then()
+ .statusCode(200)
+ .body(containsString("<MetricsConfiguration"))
+ .body(containsString("<Id>EntireBucket</Id>"));
+ }
+
+ @Test
+ @Order(4)
+ void putFilteredConfigurationKeepsTheFilter() {
+ given()
+ .body("""
+ <MetricsConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Id>Filtered</Id>
+ <Filter>
+ <And>
+ <Prefix>logs/</Prefix>
+ <Tag><Key>env</Key><Value>prod</Value></Tag>
+ <Tag><Key>team</Key><Value>core</Value></Tag>
+ </And>
+ </Filter>
+ </MetricsConfiguration>
+ """)
+ .when()
+ .put("/" + BUCKET + "?metrics&id=Filtered")
+ .then()
+ .statusCode(204);
+
+ // AWS repeats <Tag> inside <And> with no wrapping element.
+ given()
+ .when()
+ .get("/" + BUCKET + "?metrics&id=Filtered")
+ .then()
+ .statusCode(200)
+ .body(containsString("<And><Prefix>logs/</Prefix>"
+ + "<Tag><Key>env</Key><Value>prod</Value></Tag>"
+ + "<Tag><Key>team</Key><Value>core</Value></Tag></And>"));
+ }
+
+ @Test
+ @Order(5)
+ void listWithoutAnIdReturnsEveryConfiguration() {
+ given()
+ .when()
+ .get("/" + BUCKET + "?metrics")
+ .then()
+ .statusCode(200)
+ .body(containsString("<ListMetricsConfigurationsResult"))
+ .body(containsString("<Id>EntireBucket</Id>"))
+ .body(containsString("<Id>Filtered</Id>"))
+ .body(containsString("<IsTruncated>false</IsTruncated>"));
+ }
+
+ @Test
+ @Order(6)
+ void idInTheBodyMustMatchTheIdInTheQuery() {
+ given()
+ .body("""
+ <MetricsConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Id>Different</Id>
+ </MetricsConfiguration>
+ """)
+ .when()
+ .put("/" + BUCKET + "?metrics&id=Mismatch")
+ .then()
+ .statusCode(400)
+ .body(containsString("MalformedXML"));
+ }
+
+ @Test
+ @Order(7)
+ void unknownIdIsNotFound() {
+ given()
+ .when()
+ .get("/" + BUCKET + "?metrics&id=absent")
+ .then()
+ .statusCode(404)
+ .body(containsString("NoSuchConfiguration"));
+
+ // AWS does not treat deleting an absent configuration as a no-op.
+ given()
+ .when()
+ .delete("/" + BUCKET + "?metrics&id=absent")
+ .then()
+ .statusCode(404)
+ .body(containsString("NoSuchConfiguration"));
+ }
+
+ /**
+ * Regression test for the bucket-destroying bug: {@code DELETE /{bucket}?metrics} fell through
+ * to the unqualified {@code DeleteBucket} and silently removed the whole bucket, reporting
+ * success.
+ */
+ @Test
+ @Order(8)
+ void deleteMetricsConfigurationDoesNotDeleteBucket() {
+ given()
+ .when()
+ .delete("/" + BUCKET + "?metrics&id=EntireBucket")
+ .then()
+ .statusCode(204);
+
+ // The bucket, and every other configuration on it, must survive.
+ given()
+ .when()
+ .get("/" + BUCKET + "?metrics&id=Filtered")
+ .then()
+ .statusCode(200);
+
+ given()
+ .when()
+ .get("/" + BUCKET + "?metrics&id=EntireBucket")
+ .then()
+ .statusCode(404);
+ }
+
+ @Test
+ @Order(9)
+ void aRequestWithoutAnIdIsRefusedRatherThanGuessedAt() {
+ given()
+ .body("""
+ <MetricsConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Id>EntireBucket</Id>
+ </MetricsConfiguration>
+ """)
+ .when()
+ .put("/" + BUCKET + "?metrics")
+ .then()
+ .statusCode(400)
+ .body(containsString("InvalidArgument"));
+
+ given()
+ .when()
+ .delete("/" + BUCKET + "?metrics")
+ .then()
+ .statusCode(400)
+ .body(containsString("InvalidArgument"));
+ }
+
+ @Test
+ @Order(10)
+ void unqualifiedDeleteStillRemovesBucket() {
+ given()
+ .when()
+ .delete("/" + BUCKET)
+ .then()
+ .statusCode(204);
+ given()
+ .when()
+ .get("/" + BUCKET + "?metrics")
+ .then()
+ .statusCode(404)
+ .body(containsString("NoSuchBucket"));
+ }
+
+ @Test
+ @Order(11)
+ void aRecreatedBucketDoesNotInheritTheOldConfigurations() {
+ given()
+ .when()
+ .put("/" + BUCKET)
+ .then()
+ .statusCode(200);
+
+ given()
+ .when()
+ .get("/" + BUCKET + "?metrics&id=Filtered")
+ .then()
+ .statusCode(404)
+ .body(containsString("NoSuchConfiguration"));
+
+ given()
+ .when()
+ .delete("/" + BUCKET)
+ .then()
+ .statusCode(204);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationTest.java
new file mode 100644
index 0000000000..6b6e398283
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationTest.java
@@ -0,0 +1,153 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class S3MetricsConfigurationTest {
+
+ private static final String NS = "http://s3.amazonaws.com/doc/2006-03-01/";
+
+ private static String body(String inner) {
+ return "<MetricsConfiguration xmlns=\"" + NS + "\">" + inner + "</MetricsConfiguration>";
+ }
+
+ @Test
+ void parsesAnIdWithoutAFilter() {
+ S3MetricsConfiguration parsed = S3MetricsConfiguration.parse(body("<Id>EntireBucket</Id>"));
+
+ assertEquals("EntireBucket", parsed.id());
+ assertEquals("<Id>EntireBucket</Id>", parsed.innerXml());
+ }
+
+ @Test
+ void parsesEachSingleFilterPredicate() {
+ assertEquals("<Id>a</Id><Filter><Prefix>logs/</Prefix></Filter>",
+ S3MetricsConfiguration.parse(body("<Id>a</Id><Filter><Prefix>logs/</Prefix></Filter>")).innerXml());
+
+ assertEquals("<Id>a</Id><Filter><Tag><Key>env</Key><Value>prod</Value></Tag></Filter>",
+ S3MetricsConfiguration.parse(body(
+ "<Id>a</Id><Filter><Tag><Key>env</Key><Value>prod</Value></Tag></Filter>")).innerXml());
+
+ String arn = "arn:aws:s3:eu-central-1:123456789012:accesspoint/ap";
+ assertEquals("<Id>a</Id><Filter><AccessPointArn>" + arn + "</AccessPointArn></Filter>",
+ S3MetricsConfiguration.parse(body(
+ "<Id>a</Id><Filter><AccessPointArn>" + arn + "</AccessPointArn></Filter>")).innerXml());
+ }
+
+ @Test
+ void parsesAnAndConjunctionKeepingEveryTag() {
+ // MetricsAndOperator.Tags is a flattened list named Tag, so the tags repeat with no
+ // wrapping element and all of them have to survive the round trip.
+ String parsed = S3MetricsConfiguration.parse(body("""
+ <Id>a</Id>
+ <Filter>
+ <And>
+ <Prefix>logs/</Prefix>
+ <Tag><Key>env</Key><Value>prod</Value></Tag>
+ <Tag><Key>team</Key><Value>core</Value></Tag>
+ </And>
+ </Filter>
+ """)).innerXml();
+
+ assertEquals("<Id>a</Id><Filter><And><Prefix>logs/</Prefix>"
+ + "<Tag><Key>env</Key><Value>prod</Value></Tag>"
+ + "<Tag><Key>team</Key><Value>core</Value></Tag></And></Filter>", parsed);
+ }
+
+ @Test
+ void escapesValuesOnTheWayBackOut() {
+ String parsed = S3MetricsConfiguration.parse(
+ body("<Id>a&amp;b</Id><Filter><Prefix>x&lt;y</Prefix></Filter>")).innerXml();
+
+ assertEquals("<Id>a&amp;b</Id><Filter><Prefix>x&lt;y</Prefix></Filter>", parsed);
+ }
+
+ @Test
+ void rejectsBodiesThatDoNotMatchTheSchema() {
+ // Missing id, wrong root element, empty and non-XML bodies are all MalformedXML on AWS.
+ for (String invalid : new String[]{
+ body(""),
+ body("<Id> </Id>"),
+ "<SomethingElse><Id>a</Id></SomethingElse>",
+ "not xml at all",
+ ""}) {
+ AwsException e = assertThrows(AwsException.class, () -> S3MetricsConfiguration.parse(invalid),
+ () -> "expected rejection of: " + invalid);
+ assertEquals("MalformedXML", e.getErrorCode());
+ assertEquals(400, e.getHttpStatus());
+ }
+ }
+
+ @Test
+ void rejectsFiltersThatAreNotExactlyOnePredicate() {
+ // Verified against AWS: each of these is MalformedXML rather than something to normalize.
+ String bothPrefixAndTag = "<Id>a</Id><Filter><Prefix>logs/</Prefix>"
+ + "<Tag><Key>env</Key><Value>prod</Value></Tag></Filter>";
+ String emptyFilter = "<Id>a</Id><Filter></Filter>";
+ String andWithOnePredicate = "<Id>a</Id><Filter><And><Prefix>logs/</Prefix></And></Filter>";
+ String andWithOneTag = "<Id>a</Id><Filter><And>"
+ + "<Tag><Key>env</Key><Value>prod</Value></Tag></And></Filter>";
+ String tagWithoutAKey = "<Id>a</Id><Filter><Tag><Value>prod</Value></Tag></Filter>";
+ // botocore marks both Key and Value required on a Tag.
+ String tagWithoutAValue = "<Id>a</Id><Filter><Tag><Key>env</Key></Tag></Filter>";
+ String unknownPredicate = "<Id>a</Id><Filter><Something>x</Something></Filter>";
+ String andWithAnUnknownConjunct = "<Id>a</Id><Filter><And><Prefix>logs/</Prefix>"
+ + "<Something>x</Something></And></Filter>";
+
+ for (String invalid : new String[]{
+ bothPrefixAndTag, emptyFilter, andWithOnePredicate, andWithOneTag, tagWithoutAKey,
+ tagWithoutAValue, unknownPredicate, andWithAnUnknownConjunct}) {
+ AwsException e = assertThrows(AwsException.class, () -> S3MetricsConfiguration.parse(body(invalid)),
+ () -> "expected rejection of: " + invalid);
+ assertEquals("MalformedXML", e.getErrorCode());
+ }
+ }
+
+ @Test
+ void rejectsDuplicateOrMisplacedElements() {
+ // Normalizing these away would store a configuration that is not the one that was sent.
+ String twoIds = "<Id>a</Id><Id>b</Id>";
+ String twoFilters = "<Id>a</Id><Filter><Prefix>x</Prefix></Filter>"
+ + "<Filter><Prefix>y</Prefix></Filter>";
+ String strayElement = "<Id>a</Id><Prefix>logs/</Prefix>";
+ String noId = "<Filter><Prefix>logs/</Prefix></Filter>";
+ String idOnlyInsideFilter = "<Filter><Id>a</Id></Filter>";
+
+ for (String invalid : new String[]{twoIds, twoFilters, strayElement, noId, idOnlyInsideFilter}) {
+ AwsException e = assertThrows(AwsException.class, () -> S3MetricsConfiguration.parse(body(invalid)),
+ () -> "expected rejection of: " + invalid);
+ assertEquals("MalformedXML", e.getErrorCode());
+ }
+ }
+
+ @Test
+ void acceptsAFilterBeforeItsId() {
+ // Element order is not something to be strict about; the set of elements is.
+ assertEquals("<Id>a</Id><Filter><Prefix>logs/</Prefix></Filter>",
+ S3MetricsConfiguration.parse(
+ body("<Filter><Prefix>logs/</Prefix></Filter><Id>a</Id>")).innerXml());
+ }
+
+ @Test
+ void acceptsAnAndOfExactlyTwoPredicates() {
+ // The boundary the rejection is drawn at: two conjuncts are legal, one is not.
+ assertEquals("<Id>a</Id><Filter><And><Prefix>logs/</Prefix>"
+ + "<Tag><Key>env</Key><Value>prod</Value></Tag></And></Filter>",
+ S3MetricsConfiguration.parse(body("<Id>a</Id><Filter><And><Prefix>logs/</Prefix>"
+ + "<Tag><Key>env</Key><Value>prod</Value></Tag></And></Filter>")).innerXml());
+ }
+
+ @Test
+ void refusesToResolveExternalEntities() {
+ // The parser must not read local files on behalf of a request body.
+ String xxe = "<?xml version=\"1.0\"?>"
+ + "<!DOCTYPE foo [<!ENTITY xxe SYSTEM \"file:///etc/passwd\">]>"
+ + "<MetricsConfiguration><Id>&xxe;</Id></MetricsConfiguration>";
+
+ AwsException e = assertThrows(AwsException.class, () -> S3MetricsConfiguration.parse(xxe));
+ assertEquals("MalformedXML", e.getErrorCode());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index 509f5adfa4..586c1bfad7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -753,4 +753,227 @@ class S3ServiceTest {
assertInstanceOf(S3Service.WebsiteResolution.ErrorDocument.class,
s3Service.resolveWebsiteError("site", UNSIGNED, 500)).status());
}
+
+ @Test
+ void metricsConfigurationsOnABucketWithoutAnyBehaveAsEmpty() {
+ // A bucket persisted before this field existed deserializes with a null map, which is the
+ // same shape a freshly created bucket has, so neither may fault.
+ s3Service.createBucket("no-metrics", "us-east-1");
+
+ assertTrue(s3Service.listBucketMetricsConfigurations("no-metrics")
+ .contains("<IsTruncated>false</IsTruncated>"));
+ assertEquals("NoSuchConfiguration", assertThrows(AwsException.class,
+ () -> s3Service.getBucketMetricsConfiguration("no-metrics", "any")).getErrorCode());
+ assertEquals("NoSuchConfiguration", assertThrows(AwsException.class,
+ () -> s3Service.deleteBucketMetricsConfiguration("no-metrics", "any")).getErrorCode());
+
+ // And the first put still lands on it.
+ s3Service.putBucketMetricsConfiguration("no-metrics", "first", "<Id>first</Id>");
+ assertTrue(s3Service.getBucketMetricsConfiguration("no-metrics", "first")
+ .contains("<Id>first</Id>"));
+ }
+
+ @Test
+ void metricsConfigurationsDoNotOutliveTheirBucket() {
+ s3Service.createBucket("recycled", "us-east-1");
+ s3Service.putBucketMetricsConfiguration("recycled", "old", "<Id>old</Id>");
+ s3Service.deleteBucket("recycled");
+
+ s3Service.createBucket("recycled", "us-east-1");
+
+ assertEquals("NoSuchConfiguration", assertThrows(AwsException.class,
+ () -> s3Service.getBucketMetricsConfiguration("recycled", "old")).getErrorCode());
+ }
+
+ @Test
+ void metricsConfigurationsSurviveARestart() {
+ // Through the real storage layer rather than Jackson alone: written, flushed to disk, and
+ // read back by a second service over the same file, the way a restart does it.
+ Path bucketsFile = tempDir.resolve("s3-buckets.json");
+ var beforeRestart = new io.github.hectorvent.floci.core.storage.HybridStorage<String, Bucket>(
+ bucketsFile, new com.fasterxml.jackson.core.type.TypeReference<Map<String, Bucket>>() {}, 60000);
+ S3Service before = new S3Service(beforeRestart, new InMemoryStorage<>(), tempDir.resolve("s3a"), false);
+ before.createBucket("persisted-metrics", "us-east-1");
+ before.putBucketMetricsConfiguration("persisted-metrics", "EntireBucket", "<Id>EntireBucket</Id>");
+ beforeRestart.flush();
+ beforeRestart.shutdown();
+
+ var afterRestart = new io.github.hectorvent.floci.core.storage.HybridStorage<String, Bucket>(
+ bucketsFile, new com.fasterxml.jackson.core.type.TypeReference<Map<String, Bucket>>() {}, 60000);
+ afterRestart.load();
+ try {
+ S3Service after = new S3Service(afterRestart, new InMemoryStorage<>(), tempDir.resolve("s3a"), false);
+ assertTrue(after.getBucketMetricsConfiguration("persisted-metrics", "EntireBucket")
+ .contains("<Id>EntireBucket</Id>"));
+ } finally {
+ afterRestart.shutdown();
+ }
+ }
+
+ @Test
+ void metricsConfigurationsSurviveAJacksonRoundTrip() throws Exception {
+ // Bucket records are persisted as JSON, so the configurations have to come back after a
+ // restart, and a record written before the field existed has to still load.
+ var mapper = new com.fasterxml.jackson.databind.ObjectMapper().findAndRegisterModules();
+ Bucket bucket = new Bucket("persisted");
+ bucket.setMetricsConfigurations(new java.util.LinkedHashMap<>(
+ Map.of("EntireBucket", "<Id>EntireBucket</Id>")));
+
+ Bucket reloaded = mapper.readValue(mapper.writeValueAsString(bucket), Bucket.class);
+ assertEquals("<Id>EntireBucket</Id>", reloaded.getMetricsConfigurations().get("EntireBucket"));
+
+ Bucket legacy = mapper.readValue("{\"name\":\"legacy\"}", Bucket.class);
+ assertNull(legacy.getMetricsConfigurations());
+ }
+
+ /**
+ * Waits for the thread to reach the bucket monitor. The store is a ConcurrentHashMap, so
+ * nothing else blocks it: this observes the interleaving rather than sleeping long enough to
+ * hope for it, which keeps the test both instant and immune to a slow machine.
+ */
+ private static void awaitBlockedOnMonitor(Thread thread) throws InterruptedException {
+ long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(10);
+ while (thread.getState() != Thread.State.BLOCKED) {
+ if (System.nanoTime() > deadline) {
+ fail("thread never reached the bucket monitor");
+ }
+ Thread.onSpinWait();
+ }
+ }
+
+ @Test
+ void deleteBucketTakesTheBucketMonitor() throws Exception {
+ // Re-reading the record narrows the window but cannot close it: a delete landing between a
+ // mutation's re-read and its write would still be undone. That interleaving is too narrow
+ // to force from a test, so what is asserted here is the property that closes it — the
+ // delete waits for whoever holds the record's monitor.
+ var buckets = new InMemoryStorage<String, Bucket>();
+ S3Service service = new S3Service(buckets, new InMemoryStorage<>(), tempDir.resolve("s3-monitor"), false);
+ service.createBucket("monitor-bucket", "us-east-1");
+ Bucket record = buckets.get("monitor-bucket").orElseThrow();
+
+ Thread delete = new Thread(() -> service.deleteBucket("monitor-bucket"));
+ synchronized (record) {
+ delete.start();
+ long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(10);
+ while (delete.getState() != Thread.State.BLOCKED) {
+ assertTrue(buckets.get("monitor-bucket").isPresent(),
+ "DeleteBucket removed the record without waiting for the bucket monitor");
+ if (System.nanoTime() > deadline) {
+ fail("DeleteBucket never reached the bucket monitor");
+ }
+ Thread.onSpinWait();
+ }
+ }
+ delete.join(30_000);
+
+ assertTrue(buckets.get("monitor-bucket").isEmpty());
+ }
+
+ @Test
+ void aMetricsPutRacingABucketDeleteCannotRestoreTheBucket() throws Exception {
+ // A put resolves the bucket record before it writes it back, so a delete landing in between
+ // would be undone by the write. The interleaving is forced rather than raced for: the test
+ // holds the bucket monitor, lets the put resolve the record and block on it, deletes the
+ // bucket from the same thread, and only then releases.
+ var buckets = new InMemoryStorage<String, Bucket>();
+ S3Service service = new S3Service(buckets, new InMemoryStorage<>(), tempDir.resolve("s3-race"), false);
+ service.createBucket("race-bucket", "us-east-1");
+ Bucket record = buckets.get("race-bucket").orElseThrow();
+
+ var putStarted = new java.util.concurrent.CountDownLatch(1);
+ var thrownByPut = new java.util.concurrent.atomic.AtomicReference<Exception>();
+ Thread put = new Thread(() -> {
+ putStarted.countDown();
+ try {
+ service.putBucketMetricsConfiguration("race-bucket", "id", "<Id>id</Id>");
+ } catch (Exception e) {
+ thrownByPut.set(e);
+ }
+ });
+
+ synchronized (record) {
+ put.start();
+ putStarted.await();
+ awaitBlockedOnMonitor(put);
+ service.deleteBucket("race-bucket");
+ }
+ put.join(30_000);
+
+ assertTrue(buckets.get("race-bucket").isEmpty(),
+ "the deleted bucket was restored by the concurrent metrics put");
+ assertEquals("NoSuchBucket",
+ assertInstanceOf(AwsException.class, thrownByPut.get()).getErrorCode());
+ }
+
+ @Test
+ void aMetricsPutCannotOverwriteABucketRecreatedWhileItWaited() throws Exception {
+ // Presence is not enough: deleted and recreated under the same name, the store holds a
+ // different record, and writing the resolved one back would replace the new bucket with
+ // the old one's state.
+ var buckets = new InMemoryStorage<String, Bucket>();
+ S3Service service = new S3Service(buckets, new InMemoryStorage<>(), tempDir.resolve("s3-recreate"), false);
+ service.createBucket("recreated-bucket", "us-east-1");
+ Bucket original = buckets.get("recreated-bucket").orElseThrow();
+
+ var putStarted = new java.util.concurrent.CountDownLatch(1);
+ var thrownByPut = new java.util.concurrent.atomic.AtomicReference<Exception>();
+ Thread put = new Thread(() -> {
+ putStarted.countDown();
+ try {
+ service.putBucketMetricsConfiguration("recreated-bucket", "stale", "<Id>stale</Id>");
+ } catch (Exception e) {
+ thrownByPut.set(e);
+ }
+ });
+
+ synchronized (original) {
+ put.start();
+ putStarted.await();
+ awaitBlockedOnMonitor(put);
+ service.deleteBucket("recreated-bucket");
+ service.createBucket("recreated-bucket", "us-east-1");
+ }
+ put.join(30_000);
+
+ assertEquals("NoSuchBucket",
+ assertInstanceOf(AwsException.class, thrownByPut.get()).getErrorCode());
+ assertNotSame(original, buckets.get("recreated-bucket").orElseThrow(),
+ "the recreated bucket was replaced by the record the put had resolved");
+ assertNull(buckets.get("recreated-bucket").orElseThrow().getMetricsConfigurations(),
+ "the recreated bucket inherited a configuration written against the old record");
+ }
+
+ @Test
+ void concurrentMetricsConfigurationPutsAllSurvive() throws Exception {
+ // Each put reads the configuration map, adds to it and writes it back, so without a shared
+ // monitor concurrent puts of different ids overwrite each other's work.
+ s3Service.createBucket("metrics-race", "us-east-1");
+ int count = 24;
+ var pool = java.util.concurrent.Executors.newFixedThreadPool(8);
+ var start = new java.util.concurrent.CountDownLatch(1);
+ try {
+ var submitted = new java.util.ArrayList<java.util.concurrent.Future<?>>();
+ for (int i = 0; i < count; i++) {
+ String id = "config-" + i;
+ submitted.add(pool.submit(() -> {
+ start.await();
+ s3Service.putBucketMetricsConfiguration("metrics-race", id, "<Id>" + id + "</Id>");
+ return null;
+ }));
+ }
+ start.countDown();
+ for (var future : submitted) {
+ future.get(30, java.util.concurrent.TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+
+ String listed = s3Service.listBucketMetricsConfigurations("metrics-race");
+ for (int i = 0; i < count; i++) {
+ assertTrue(listed.contains("<Id>config-" + i + "</Id>"),
+ "configuration config-" + i + " was lost by a concurrent put");
+ }
+ }
}