Light, fluffy, and always free - The AWS Local Emulator alternative
feat(s3): support bucket metrics configurations (#2382)
Michal Schreiber8d33c6512951
diff --git a/compatibility-tests/sdk-test-awscli/test/s3.bats b/compatibility-tests/sdk-test-awscli/test/s3.batsindex 3fe298197a..3d70832af2 100644--- a/compatibility-tests/sdk-test-awscli/test/s3.bats+++ b/compatibility-tests/sdk-test-awscli/test/s3.bats@@ -305,3 +305,37 @@ EOFrm -f "$large_file"}++@test "S3: metrics configurations round-trip and never touch the bucket" {+ aws_cmd s3api create-bucket --bucket "$BUCKET" >/dev/null++ run aws_cmd s3api put-bucket-metrics-configuration \+ --bucket "$BUCKET" --id EntireBucket --metrics-configuration 'Id=EntireBucket'+ assert_success++ run aws_cmd s3api get-bucket-metrics-configuration --bucket "$BUCKET" --id EntireBucket+ assert_success+ [ "$(json_get "$output" '.MetricsConfiguration.Id')" = "EntireBucket" ]++ run aws_cmd s3api put-bucket-metrics-configuration \+ --bucket "$BUCKET" --id Filtered \+ --metrics-configuration 'Id=Filtered,Filter={And={Prefix=logs/,Tags=[{Key=env,Value=prod}]}}'+ assert_success++ run aws_cmd s3api list-bucket-metrics-configurations --bucket "$BUCKET"+ assert_success+ [ "$(json_get "$output" '.MetricsConfigurationList | length')" = "2" ]+ [ "$(json_get "$output" '.MetricsConfigurationList[1].Filter.And.Prefix')" = "logs/" ]+ [ "$(json_get "$output" '.MetricsConfigurationList[1].Filter.And.Tags[0].Key')" = "env" ]++ # A sub-resource DELETE removes only that configuration; the bucket must survive it.+ run aws_cmd s3api delete-bucket-metrics-configuration --bucket "$BUCKET" --id EntireBucket+ assert_success++ run aws_cmd s3api head-bucket --bucket "$BUCKET"+ assert_success++ run aws_cmd s3api get-bucket-metrics-configuration --bucket "$BUCKET" --id EntireBucket+ assert_failure+ assert_output --partial "NoSuchConfiguration"+}diff --git a/docs/services/s3.md b/docs/services/s3.mdindex f6cdb03e28..b516802af2 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -24,6 +24,7 @@| **Pre-signed URLs** | Generates and validates pre-signed GET/PUT URLs || **S3 Select** | SelectObjectContent || **Public Access Block** | PutPublicAccessBlock, GetPublicAccessBlock, DeletePublicAccessBlock |+| **Metrics** | PutBucketMetricsConfiguration, GetBucketMetricsConfiguration, ListBucketMetricsConfigurations, DeleteBucketMetricsConfiguration |**RestoreObject** is accepted but stubbed: Floci validates the request and returns `202 Accepted`, but no restore state machine runs.@@ -152,7 +153,7 @@ These AWS S3 features have no handler in Floci. Calls will return an error (typi- Request payment (`PutBucketRequestPayment`, `GetBucketRequestPayment`)- Intelligent-Tiering configurations- Inventory configurations-- Metrics and Analytics configurations+- Analytics configurations## Configurationdiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 884101fe81..6702bea8be 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -324,6 +324,11 @@ public class S3Controller {s3Service.putBucketAccelerateConfiguration(bucket, new String(body, StandardCharsets.UTF_8));return Response.ok().build();}+ // Must be handled here: an unmatched subresource falls through to CreateBucket below,+ // which answers a metrics call with BucketAlreadyOwnedByYou.+ if (hasQueryParam(uriInfo, "metrics")) {+ return handlePutBucketMetricsConfiguration(bucket, uriInfo, body);+ }String locationConstraint = null;if (body != null && body.length > 0) {@@ -407,6 +412,11 @@ public class S3Controller {// handled here so it does NOT fall through to deleting the whole bucket.return Response.noContent().build();}+ if (hasQueryParam(uriInfo, "metrics")) {+ // Likewise this must not fall through to deleting the bucket.+ s3Service.deleteBucketMetricsConfiguration(bucket, requireMetricsId(uriInfo));+ return Response.noContent().build();+ }if (hasQueryParam(uriInfo, "accelerate")) {// AWS defines no DELETE for the accelerate subresource; reject it here so// it does NOT fall through to deleting the whole bucket.@@ -518,6 +528,16 @@ public class S3Controller {s3Service.authorizeBucketRead(bucket, "s3:GetAccelerateConfiguration", authorization);return Response.ok(s3Service.getBucketAccelerateConfiguration(bucket)).build();}+ // GetBucketMetricsConfiguration and ListBucketMetricsConfigurations share ?metrics and+ // are told apart by the id, which only the single-configuration read carries.+ if (hasQueryParam(uriInfo, "metrics")) {+ s3Service.authorizeBucketRead(bucket, "s3:GetMetricsConfiguration", authorization);+ String id = uriInfo.getQueryParameters().getFirst("id");+ String xml = id != null+ ? s3Service.getBucketMetricsConfiguration(bucket, id)+ : s3Service.listBucketMetricsConfigurations(bucket);+ return Response.ok(xml).type("application/xml").build();+ }// --- S3 static-website index resolution (site root) ---// A website endpoint has no S3 REST API, so it serves the index document for the site root@@ -1778,6 +1798,34 @@ public class S3Controller {return Response.ok(buildTaggingXml(tags)).type(MediaType.APPLICATION_XML).build();}+ // --- Metrics Configurations ---++ private Response handlePutBucketMetricsConfiguration(String bucket, UriInfo uriInfo, byte[] body) {+ String id = requireMetricsId(uriInfo);+ S3MetricsConfiguration configuration =+ S3MetricsConfiguration.parse(new String(body, StandardCharsets.UTF_8));+ // AWS rejects a body whose Id disagrees with the id in the query string.+ if (!id.equals(configuration.id())) {+ throw new AwsException("MalformedXML",+ "The XML you provided was not well-formed or did not validate against our "+ + "published schema", 400);+ }+ s3Service.putBucketMetricsConfiguration(bucket, id, configuration.innerXml());+ return Response.noContent().build();+ }++ /**+ * The id identifies the configuration, so a request without one is refused rather than guessed+ * at. AWS's own answer here was not verified, so floci gives one rejection for both verbs.+ */+ private String requireMetricsId(UriInfo uriInfo) {+ String id = uriInfo.getQueryParameters().getFirst("id");+ if (id == null) {+ throw new AwsException("InvalidArgument", "The metrics id must be specified.", 400);+ }+ return id;+ }+// --- Object Tagging ---private Response handlePutObjectTagging(String bucket, String key, byte[] body) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3MetricsConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3MetricsConfiguration.javanew file mode 100644index 0000000000..3fb6bb4807--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3MetricsConfiguration.java@@ -0,0 +1,127 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.XmlBuilder;+import io.github.hectorvent.floci.core.common.XmlParser;++import java.util.List;+import java.util.Map;++/**+ * A parsed {@code MetricsConfiguration} request body, reduced to its id and a canonical+ * serialization of its contents.+ *+ * <p>The body is re-serialized rather than stored verbatim, so that what a later+ * GetBucketMetricsConfiguration or ListBucketMetricsConfigurations returns is built by floci+ * instead of being whatever XML the caller sent, and so that the same stored form can be wrapped+ * in either response.+ */+record S3MetricsConfiguration(String id, String innerXml) {++ private static final String ROOT = "MetricsConfiguration";++ /** AWS reports a body that does not match the published schema this way. */+ private static AwsException malformed() {+ return new AwsException("MalformedXML",+ "The XML you provided was not well-formed or did not validate against our published schema", 400);+ }++ static S3MetricsConfiguration parse(String xml) {+ if (xml == null || xml.isBlank() || !ROOT.equals(XmlParser.rootElementName(xml))) {+ throw malformed();+ }++ // The configuration is one Id and at most one Filter, so anything else under the root —+ // a stray element, a second Id, a second Filter — is a body AWS would not have accepted.+ List<String> children = XmlParser.childElementNames(xml, ROOT);+ long ids = children.stream().filter("Id"::equals).count();+ long filters = children.stream().filter("Filter"::equals).count();+ if (ids != 1 || filters > 1 || children.size() != ids + filters) {+ throw malformed();+ }+ boolean hasFilter = filters == 1;++ String id = XmlParser.extractFirst(xml, "Id", null);+ if (id == null || id.isBlank()) {+ throw malformed();+ }++ XmlBuilder inner = new XmlBuilder().elem("Id", id);+ if (hasFilter) {+ inner.start("Filter").raw(filterXml(xml)).end("Filter");+ }+ return new S3MetricsConfiguration(id, inner.build());+ }++ /**+ * Serializes the filter. A filter is exactly one of a prefix, an object tag, an access point+ * ARN, or an And conjunction: AWS answers MalformedXML for a filter naming none of them or+ * more than one, and for an And holding fewer than two predicates.+ */+ private static String filterXml(String xml) {+ List<String> predicates = XmlParser.childElementNames(xml, "Filter");+ if (predicates.size() != 1) {+ throw malformed();+ }++ String predicate = predicates.getFirst();+ return switch (predicate) {+ case "Prefix", "AccessPointArn" -> new XmlBuilder()+ .elem(predicate, requireText(xml, predicate))+ .build();+ case "Tag" -> tagsXml(xml, 1);+ case "And" -> {+ List<String> conjuncts = XmlParser.childElementNames(xml, "And");+ if (conjuncts.size() < 2) {+ throw malformed();+ }+ XmlBuilder and = new XmlBuilder().start("And");+ if (conjuncts.contains("Prefix")) {+ and.elem("Prefix", requireText(xml, "Prefix"));+ }+ long tagCount = conjuncts.stream().filter("Tag"::equals).count();+ if (tagCount > 0) {+ and.raw(tagsXml(xml, (int) tagCount));+ }+ if (conjuncts.contains("AccessPointArn")) {+ and.elem("AccessPointArn", requireText(xml, "AccessPointArn"));+ }+ // Every conjunct has to be one floci understands, or the filter it stores would+ // not be the filter that was sent.+ if (conjuncts.size() != tagCount + (conjuncts.contains("Prefix") ? 1 : 0)+ + (conjuncts.contains("AccessPointArn") ? 1 : 0)) {+ throw malformed();+ }+ yield and.end("And").build();+ }+ default -> throw malformed();+ };+ }++ /**+ * Serializes {@code expected} tags. A tag carries both a key and a value, so a pair short of+ * the element count means one of them was missing.+ */+ private static String tagsXml(String xml, int expected) {+ Map<String, String> tags = XmlParser.extractPairs(xml, "Tag", "Key", "Value");+ if (tags.size() != expected) {+ throw malformed();+ }+ XmlBuilder out = new XmlBuilder();+ tags.forEach((key, value) -> {+ if (key.isBlank() || value == null) {+ throw malformed();+ }+ out.start("Tag").elem("Key", key).elem("Value", value).end("Tag");+ });+ return out.build();+ }++ private static String requireText(String xml, String element) {+ String value = XmlParser.extractFirst(xml, element, null);+ if (value == null) {+ throw malformed();+ }+ return value;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 1b689adfff..a95101d020 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -228,7 +228,19 @@ public class S3Service implements Resettable {public void deleteBucket(String bucketName) {ensureBucketExists(bucketName);+ Bucket bucket = bucketStore.get(bucketName)+ .orElseThrow(() -> new AwsException("NoSuchBucket",+ "The specified bucket does not exist.", 404));++ // Takes the bucket monitor that the bucket-scoped mutations take: a mutation that read the+ // record before the delete would otherwise write it back afterwards, restoring the bucket.+ synchronized (bucket) {+ deleteBucketLocked(bucketName);+ }+ LOG.infov("Deleted bucket: {0}", bucketName);+ }+ private void deleteBucketLocked(String bucketName) {// Check if bucket is emptyList<S3Object> objects = listObjects(bucketName, null, null, 1);if (!objects.isEmpty()) {@@ -243,7 +255,6 @@ public class S3Service implements Resettable {} else {deleteDirectory(dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(bucketName));}- LOG.infov("Deleted bucket: {0}", bucketName);}public List<Bucket> listBuckets() {@@ -1634,6 +1645,107 @@ public class S3Service implements Resettable {LOG.debugv("Deleted tags from bucket: {0}", bucketName);}+ // --- Metrics Configurations ---++ private static final String METRICS_XML_DECLARATION = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";++ /**+ * Stores a CloudWatch request metrics configuration under {@code id}, replacing any+ * configuration already stored under it. floci records the configuration and returns it; no+ * metrics are produced from it.+ */+ public void putBucketMetricsConfiguration(String bucketName, String id, String innerXml) {+ Bucket bucket = requireBucket(bucketName);+ // Read-modify-write of the bucket record, so it takes the same monitor as the other+ // bucket-scoped mutations: without it two concurrent puts of different ids both start from+ // the same map and one of the configurations is lost.+ synchronized (bucket) {+ requireSameRecord(bucketName, bucket);+ Map<String, String> configurations = bucket.getMetricsConfigurations() != null+ ? new java.util.LinkedHashMap<>(bucket.getMetricsConfigurations())+ : new java.util.LinkedHashMap<>();+ configurations.put(id, innerXml);+ bucket.setMetricsConfigurations(configurations);+ bucketStore.put(bucketName, bucket);+ }+ LOG.debugv("Put metrics configuration {0} on bucket: {1}", id, bucketName);+ }++ public String getBucketMetricsConfiguration(String bucketName, String id) {+ Bucket bucket = requireBucket(bucketName);+ String innerXml = bucket.getMetricsConfigurations() == null+ ? null : bucket.getMetricsConfigurations().get(id);+ if (innerXml == null) {+ throw noSuchMetricsConfiguration();+ }+ return METRICS_XML_DECLARATION + new XmlBuilder()+ .start("MetricsConfiguration", AwsNamespaces.S3)+ .raw(innerXml)+ .end("MetricsConfiguration")+ .build();+ }++ /**+ * Lists every metrics configuration on the bucket. AWS pages these with a continuation token+ * once there are more than 100; floci returns them all in one unpaged response, ordered by id+ * so that the listing is stable.+ */+ public String listBucketMetricsConfigurations(String bucketName) {+ Bucket bucket = requireBucket(bucketName);+ Map<String, String> configurations = bucket.getMetricsConfigurations() != null+ ? bucket.getMetricsConfigurations() : Map.of();++ XmlBuilder xml = new XmlBuilder().start("ListMetricsConfigurationsResult", AwsNamespaces.S3);+ configurations.keySet().stream().sorted().forEach(id -> xml+ .start("MetricsConfiguration")+ .raw(configurations.get(id))+ .end("MetricsConfiguration"));+ return METRICS_XML_DECLARATION + xml+ .elem("IsTruncated", false)+ .end("ListMetricsConfigurationsResult")+ .build();+ }++ public void deleteBucketMetricsConfiguration(String bucketName, String id) {+ Bucket bucket = requireBucket(bucketName);+ // Same monitor as the put: the existence check and the write have to be one step, or a+ // concurrent put of another id is dropped by the write that follows it.+ synchronized (bucket) {+ requireSameRecord(bucketName, bucket);+ Map<String, String> configurations = bucket.getMetricsConfigurations();+ if (configurations == null || !configurations.containsKey(id)) {+ throw noSuchMetricsConfiguration();+ }+ Map<String, String> remaining = new java.util.LinkedHashMap<>(configurations);+ remaining.remove(id);+ bucket.setMetricsConfigurations(remaining);+ bucketStore.put(bucketName, bucket);+ }+ LOG.debugv("Deleted metrics configuration {0} from bucket: {1}", id, bucketName);+ }++ private Bucket requireBucket(String bucketName) {+ return bucketStore.get(bucketName)+ .orElseThrow(() -> new AwsException("NoSuchBucket",+ "The specified bucket does not exist.", 404));+ }++ /**+ * Re-reads the bucket under its monitor and checks it is still the same record. Presence alone+ * is not enough: a bucket deleted and recreated under the same name leaves a different record+ * in the store, and writing the resolved one back would replace the new bucket with the old+ * one's state.+ */+ private void requireSameRecord(String bucketName, Bucket resolved) {+ if (bucketStore.get(bucketName).orElse(null) != resolved) {+ throw new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404);+ }+ }++ private static AwsException noSuchMetricsConfiguration() {+ return new AwsException("NoSuchConfiguration", "The specified configuration does not exist.", 404);+ }+// --- Object Lock Configuration ---public void setBucketObjectLockEnabled(String bucketName) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/Bucket.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/Bucket.javaindex 34196e885f..762ed2b0dd 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/Bucket.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/Bucket.java@@ -31,6 +31,8 @@ public class Bucket {private String accelerateStatus; // "Enabled" or "Suspended"; null until first PUTprivate String region;private WebsiteConfiguration websiteConfiguration;+ /** CloudWatch request metrics configurations, keyed by the id they were stored under. */+ private Map<String, String> metricsConfigurations;public Bucket() {this.tags = new HashMap<>();@@ -117,4 +119,7 @@ public class Bucket {public WebsiteConfiguration getWebsiteConfiguration() { return websiteConfiguration; }public void setWebsiteConfiguration(WebsiteConfiguration websiteConfiguration) { this.websiteConfiguration = websiteConfiguration; }++ public Map<String, String> getMetricsConfigurations() { return metricsConfigurations; }+ public void setMetricsConfigurations(Map<String, String> metricsConfigurations) { this.metricsConfigurations = metricsConfigurations; }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationIntegrationTest.javanew file mode 100644index 0000000000..e15e021161--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationIntegrationTest.java@@ -0,0 +1,232 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.not;++@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class S3MetricsConfigurationIntegrationTest {++ private static final String BUCKET = "metrics-int-test";++ @Test+ @Order(1)+ void createBucket() {+ given()+ .when()+ .put("/" + BUCKET)+ .then()+ .statusCode(200);+ }++ /**+ * Regression test for the bucket-creating bug where {@code PUT /{bucket}?metrics} was not+ * handled and fell through to the unqualified {@code CreateBucket}, which answered a metrics+ * call with BucketAlreadyOwnedByYou. Real S3 stores the configuration and returns 204.+ */+ @Test+ @Order(2)+ void putMetricsConfigurationIsNotTreatedAsCreateBucket() {+ given()+ .body("""+ <MetricsConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">+ <Id>EntireBucket</Id>+ </MetricsConfiguration>+ """)+ .when()+ .put("/" + BUCKET + "?metrics&id=EntireBucket")+ .then()+ .statusCode(204)+ .body(not(containsString("BucketAlreadyOwnedByYou")));+ }++ @Test+ @Order(3)+ void getMetricsConfigurationReturnsWhatWasStored() {+ given()+ .when()+ .get("/" + BUCKET + "?metrics&id=EntireBucket")+ .then()+ .statusCode(200)+ .body(containsString("<MetricsConfiguration"))+ .body(containsString("<Id>EntireBucket</Id>"));+ }++ @Test+ @Order(4)+ void putFilteredConfigurationKeepsTheFilter() {+ given()+ .body("""+ <MetricsConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">+ <Id>Filtered</Id>+ <Filter>+ <And>+ <Prefix>logs/</Prefix>+ <Tag><Key>env</Key><Value>prod</Value></Tag>+ <Tag><Key>team</Key><Value>core</Value></Tag>+ </And>+ </Filter>+ </MetricsConfiguration>+ """)+ .when()+ .put("/" + BUCKET + "?metrics&id=Filtered")+ .then()+ .statusCode(204);++ // AWS repeats <Tag> inside <And> with no wrapping element.+ given()+ .when()+ .get("/" + BUCKET + "?metrics&id=Filtered")+ .then()+ .statusCode(200)+ .body(containsString("<And><Prefix>logs/</Prefix>"+ + "<Tag><Key>env</Key><Value>prod</Value></Tag>"+ + "<Tag><Key>team</Key><Value>core</Value></Tag></And>"));+ }++ @Test+ @Order(5)+ void listWithoutAnIdReturnsEveryConfiguration() {+ given()+ .when()+ .get("/" + BUCKET + "?metrics")+ .then()+ .statusCode(200)+ .body(containsString("<ListMetricsConfigurationsResult"))+ .body(containsString("<Id>EntireBucket</Id>"))+ .body(containsString("<Id>Filtered</Id>"))+ .body(containsString("<IsTruncated>false</IsTruncated>"));+ }++ @Test+ @Order(6)+ void idInTheBodyMustMatchTheIdInTheQuery() {+ given()+ .body("""+ <MetricsConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">+ <Id>Different</Id>+ </MetricsConfiguration>+ """)+ .when()+ .put("/" + BUCKET + "?metrics&id=Mismatch")+ .then()+ .statusCode(400)+ .body(containsString("MalformedXML"));+ }++ @Test+ @Order(7)+ void unknownIdIsNotFound() {+ given()+ .when()+ .get("/" + BUCKET + "?metrics&id=absent")+ .then()+ .statusCode(404)+ .body(containsString("NoSuchConfiguration"));++ // AWS does not treat deleting an absent configuration as a no-op.+ given()+ .when()+ .delete("/" + BUCKET + "?metrics&id=absent")+ .then()+ .statusCode(404)+ .body(containsString("NoSuchConfiguration"));+ }++ /**+ * Regression test for the bucket-destroying bug: {@code DELETE /{bucket}?metrics} fell through+ * to the unqualified {@code DeleteBucket} and silently removed the whole bucket, reporting+ * success.+ */+ @Test+ @Order(8)+ void deleteMetricsConfigurationDoesNotDeleteBucket() {+ given()+ .when()+ .delete("/" + BUCKET + "?metrics&id=EntireBucket")+ .then()+ .statusCode(204);++ // The bucket, and every other configuration on it, must survive.+ given()+ .when()+ .get("/" + BUCKET + "?metrics&id=Filtered")+ .then()+ .statusCode(200);++ given()+ .when()+ .get("/" + BUCKET + "?metrics&id=EntireBucket")+ .then()+ .statusCode(404);+ }++ @Test+ @Order(9)+ void aRequestWithoutAnIdIsRefusedRatherThanGuessedAt() {+ given()+ .body("""+ <MetricsConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">+ <Id>EntireBucket</Id>+ </MetricsConfiguration>+ """)+ .when()+ .put("/" + BUCKET + "?metrics")+ .then()+ .statusCode(400)+ .body(containsString("InvalidArgument"));++ given()+ .when()+ .delete("/" + BUCKET + "?metrics")+ .then()+ .statusCode(400)+ .body(containsString("InvalidArgument"));+ }++ @Test+ @Order(10)+ void unqualifiedDeleteStillRemovesBucket() {+ given()+ .when()+ .delete("/" + BUCKET)+ .then()+ .statusCode(204);+ given()+ .when()+ .get("/" + BUCKET + "?metrics")+ .then()+ .statusCode(404)+ .body(containsString("NoSuchBucket"));+ }++ @Test+ @Order(11)+ void aRecreatedBucketDoesNotInheritTheOldConfigurations() {+ given()+ .when()+ .put("/" + BUCKET)+ .then()+ .statusCode(200);++ given()+ .when()+ .get("/" + BUCKET + "?metrics&id=Filtered")+ .then()+ .statusCode(404)+ .body(containsString("NoSuchConfiguration"));++ given()+ .when()+ .delete("/" + BUCKET)+ .then()+ .statusCode(204);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationTest.javanew file mode 100644index 0000000000..6b6e398283--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MetricsConfigurationTest.java@@ -0,0 +1,153 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import org.junit.jupiter.api.Test;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;++class S3MetricsConfigurationTest {++ private static final String NS = "http://s3.amazonaws.com/doc/2006-03-01/";++ private static String body(String inner) {+ return "<MetricsConfiguration xmlns=\"" + NS + "\">" + inner + "</MetricsConfiguration>";+ }++ @Test+ void parsesAnIdWithoutAFilter() {+ S3MetricsConfiguration parsed = S3MetricsConfiguration.parse(body("<Id>EntireBucket</Id>"));++ assertEquals("EntireBucket", parsed.id());+ assertEquals("<Id>EntireBucket</Id>", parsed.innerXml());+ }++ @Test+ void parsesEachSingleFilterPredicate() {+ assertEquals("<Id>a</Id><Filter><Prefix>logs/</Prefix></Filter>",+ S3MetricsConfiguration.parse(body("<Id>a</Id><Filter><Prefix>logs/</Prefix></Filter>")).innerXml());++ assertEquals("<Id>a</Id><Filter><Tag><Key>env</Key><Value>prod</Value></Tag></Filter>",+ S3MetricsConfiguration.parse(body(+ "<Id>a</Id><Filter><Tag><Key>env</Key><Value>prod</Value></Tag></Filter>")).innerXml());++ String arn = "arn:aws:s3:eu-central-1:123456789012:accesspoint/ap";+ assertEquals("<Id>a</Id><Filter><AccessPointArn>" + arn + "</AccessPointArn></Filter>",+ S3MetricsConfiguration.parse(body(+ "<Id>a</Id><Filter><AccessPointArn>" + arn + "</AccessPointArn></Filter>")).innerXml());+ }++ @Test+ void parsesAnAndConjunctionKeepingEveryTag() {+ // MetricsAndOperator.Tags is a flattened list named Tag, so the tags repeat with no+ // wrapping element and all of them have to survive the round trip.+ String parsed = S3MetricsConfiguration.parse(body("""+ <Id>a</Id>+ <Filter>+ <And>+ <Prefix>logs/</Prefix>+ <Tag><Key>env</Key><Value>prod</Value></Tag>+ <Tag><Key>team</Key><Value>core</Value></Tag>+ </And>+ </Filter>+ """)).innerXml();++ assertEquals("<Id>a</Id><Filter><And><Prefix>logs/</Prefix>"+ + "<Tag><Key>env</Key><Value>prod</Value></Tag>"+ + "<Tag><Key>team</Key><Value>core</Value></Tag></And></Filter>", parsed);+ }++ @Test+ void escapesValuesOnTheWayBackOut() {+ String parsed = S3MetricsConfiguration.parse(+ body("<Id>a&b</Id><Filter><Prefix>x<y</Prefix></Filter>")).innerXml();++ assertEquals("<Id>a&b</Id><Filter><Prefix>x<y</Prefix></Filter>", parsed);+ }++ @Test+ void rejectsBodiesThatDoNotMatchTheSchema() {+ // Missing id, wrong root element, empty and non-XML bodies are all MalformedXML on AWS.+ for (String invalid : new String[]{+ body(""),+ body("<Id> </Id>"),+ "<SomethingElse><Id>a</Id></SomethingElse>",+ "not xml at all",+ ""}) {+ AwsException e = assertThrows(AwsException.class, () -> S3MetricsConfiguration.parse(invalid),+ () -> "expected rejection of: " + invalid);+ assertEquals("MalformedXML", e.getErrorCode());+ assertEquals(400, e.getHttpStatus());+ }+ }++ @Test+ void rejectsFiltersThatAreNotExactlyOnePredicate() {+ // Verified against AWS: each of these is MalformedXML rather than something to normalize.+ String bothPrefixAndTag = "<Id>a</Id><Filter><Prefix>logs/</Prefix>"+ + "<Tag><Key>env</Key><Value>prod</Value></Tag></Filter>";+ String emptyFilter = "<Id>a</Id><Filter></Filter>";+ String andWithOnePredicate = "<Id>a</Id><Filter><And><Prefix>logs/</Prefix></And></Filter>";+ String andWithOneTag = "<Id>a</Id><Filter><And>"+ + "<Tag><Key>env</Key><Value>prod</Value></Tag></And></Filter>";+ String tagWithoutAKey = "<Id>a</Id><Filter><Tag><Value>prod</Value></Tag></Filter>";+ // botocore marks both Key and Value required on a Tag.+ String tagWithoutAValue = "<Id>a</Id><Filter><Tag><Key>env</Key></Tag></Filter>";+ String unknownPredicate = "<Id>a</Id><Filter><Something>x</Something></Filter>";+ String andWithAnUnknownConjunct = "<Id>a</Id><Filter><And><Prefix>logs/</Prefix>"+ + "<Something>x</Something></And></Filter>";++ for (String invalid : new String[]{+ bothPrefixAndTag, emptyFilter, andWithOnePredicate, andWithOneTag, tagWithoutAKey,+ tagWithoutAValue, unknownPredicate, andWithAnUnknownConjunct}) {+ AwsException e = assertThrows(AwsException.class, () -> S3MetricsConfiguration.parse(body(invalid)),+ () -> "expected rejection of: " + invalid);+ assertEquals("MalformedXML", e.getErrorCode());+ }+ }++ @Test+ void rejectsDuplicateOrMisplacedElements() {+ // Normalizing these away would store a configuration that is not the one that was sent.+ String twoIds = "<Id>a</Id><Id>b</Id>";+ String twoFilters = "<Id>a</Id><Filter><Prefix>x</Prefix></Filter>"+ + "<Filter><Prefix>y</Prefix></Filter>";+ String strayElement = "<Id>a</Id><Prefix>logs/</Prefix>";+ String noId = "<Filter><Prefix>logs/</Prefix></Filter>";+ String idOnlyInsideFilter = "<Filter><Id>a</Id></Filter>";++ for (String invalid : new String[]{twoIds, twoFilters, strayElement, noId, idOnlyInsideFilter}) {+ AwsException e = assertThrows(AwsException.class, () -> S3MetricsConfiguration.parse(body(invalid)),+ () -> "expected rejection of: " + invalid);+ assertEquals("MalformedXML", e.getErrorCode());+ }+ }++ @Test+ void acceptsAFilterBeforeItsId() {+ // Element order is not something to be strict about; the set of elements is.+ assertEquals("<Id>a</Id><Filter><Prefix>logs/</Prefix></Filter>",+ S3MetricsConfiguration.parse(+ body("<Filter><Prefix>logs/</Prefix></Filter><Id>a</Id>")).innerXml());+ }++ @Test+ void acceptsAnAndOfExactlyTwoPredicates() {+ // The boundary the rejection is drawn at: two conjuncts are legal, one is not.+ assertEquals("<Id>a</Id><Filter><And><Prefix>logs/</Prefix>"+ + "<Tag><Key>env</Key><Value>prod</Value></Tag></And></Filter>",+ S3MetricsConfiguration.parse(body("<Id>a</Id><Filter><And><Prefix>logs/</Prefix>"+ + "<Tag><Key>env</Key><Value>prod</Value></Tag></And></Filter>")).innerXml());+ }++ @Test+ void refusesToResolveExternalEntities() {+ // The parser must not read local files on behalf of a request body.+ String xxe = "<?xml version=\"1.0\"?>"+ + "<!DOCTYPE foo [<!ENTITY xxe SYSTEM \"file:///etc/passwd\">]>"+ + "<MetricsConfiguration><Id>&xxe;</Id></MetricsConfiguration>";++ AwsException e = assertThrows(AwsException.class, () -> S3MetricsConfiguration.parse(xxe));+ assertEquals("MalformedXML", e.getErrorCode());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex 509f5adfa4..586c1bfad7 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -753,4 +753,227 @@ class S3ServiceTest {assertInstanceOf(S3Service.WebsiteResolution.ErrorDocument.class,s3Service.resolveWebsiteError("site", UNSIGNED, 500)).status());}++ @Test+ void metricsConfigurationsOnABucketWithoutAnyBehaveAsEmpty() {+ // A bucket persisted before this field existed deserializes with a null map, which is the+ // same shape a freshly created bucket has, so neither may fault.+ s3Service.createBucket("no-metrics", "us-east-1");++ assertTrue(s3Service.listBucketMetricsConfigurations("no-metrics")+ .contains("<IsTruncated>false</IsTruncated>"));+ assertEquals("NoSuchConfiguration", assertThrows(AwsException.class,+ () -> s3Service.getBucketMetricsConfiguration("no-metrics", "any")).getErrorCode());+ assertEquals("NoSuchConfiguration", assertThrows(AwsException.class,+ () -> s3Service.deleteBucketMetricsConfiguration("no-metrics", "any")).getErrorCode());++ // And the first put still lands on it.+ s3Service.putBucketMetricsConfiguration("no-metrics", "first", "<Id>first</Id>");+ assertTrue(s3Service.getBucketMetricsConfiguration("no-metrics", "first")+ .contains("<Id>first</Id>"));+ }++ @Test+ void metricsConfigurationsDoNotOutliveTheirBucket() {+ s3Service.createBucket("recycled", "us-east-1");+ s3Service.putBucketMetricsConfiguration("recycled", "old", "<Id>old</Id>");+ s3Service.deleteBucket("recycled");++ s3Service.createBucket("recycled", "us-east-1");++ assertEquals("NoSuchConfiguration", assertThrows(AwsException.class,+ () -> s3Service.getBucketMetricsConfiguration("recycled", "old")).getErrorCode());+ }++ @Test+ void metricsConfigurationsSurviveARestart() {+ // Through the real storage layer rather than Jackson alone: written, flushed to disk, and+ // read back by a second service over the same file, the way a restart does it.+ Path bucketsFile = tempDir.resolve("s3-buckets.json");+ var beforeRestart = new io.github.hectorvent.floci.core.storage.HybridStorage<String, Bucket>(+ bucketsFile, new com.fasterxml.jackson.core.type.TypeReference<Map<String, Bucket>>() {}, 60000);+ S3Service before = new S3Service(beforeRestart, new InMemoryStorage<>(), tempDir.resolve("s3a"), false);+ before.createBucket("persisted-metrics", "us-east-1");+ before.putBucketMetricsConfiguration("persisted-metrics", "EntireBucket", "<Id>EntireBucket</Id>");+ beforeRestart.flush();+ beforeRestart.shutdown();++ var afterRestart = new io.github.hectorvent.floci.core.storage.HybridStorage<String, Bucket>(+ bucketsFile, new com.fasterxml.jackson.core.type.TypeReference<Map<String, Bucket>>() {}, 60000);+ afterRestart.load();+ try {+ S3Service after = new S3Service(afterRestart, new InMemoryStorage<>(), tempDir.resolve("s3a"), false);+ assertTrue(after.getBucketMetricsConfiguration("persisted-metrics", "EntireBucket")+ .contains("<Id>EntireBucket</Id>"));+ } finally {+ afterRestart.shutdown();+ }+ }++ @Test+ void metricsConfigurationsSurviveAJacksonRoundTrip() throws Exception {+ // Bucket records are persisted as JSON, so the configurations have to come back after a+ // restart, and a record written before the field existed has to still load.+ var mapper = new com.fasterxml.jackson.databind.ObjectMapper().findAndRegisterModules();+ Bucket bucket = new Bucket("persisted");+ bucket.setMetricsConfigurations(new java.util.LinkedHashMap<>(+ Map.of("EntireBucket", "<Id>EntireBucket</Id>")));++ Bucket reloaded = mapper.readValue(mapper.writeValueAsString(bucket), Bucket.class);+ assertEquals("<Id>EntireBucket</Id>", reloaded.getMetricsConfigurations().get("EntireBucket"));++ Bucket legacy = mapper.readValue("{\"name\":\"legacy\"}", Bucket.class);+ assertNull(legacy.getMetricsConfigurations());+ }++ /**+ * Waits for the thread to reach the bucket monitor. The store is a ConcurrentHashMap, so+ * nothing else blocks it: this observes the interleaving rather than sleeping long enough to+ * hope for it, which keeps the test both instant and immune to a slow machine.+ */+ private static void awaitBlockedOnMonitor(Thread thread) throws InterruptedException {+ long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(10);+ while (thread.getState() != Thread.State.BLOCKED) {+ if (System.nanoTime() > deadline) {+ fail("thread never reached the bucket monitor");+ }+ Thread.onSpinWait();+ }+ }++ @Test+ void deleteBucketTakesTheBucketMonitor() throws Exception {+ // Re-reading the record narrows the window but cannot close it: a delete landing between a+ // mutation's re-read and its write would still be undone. That interleaving is too narrow+ // to force from a test, so what is asserted here is the property that closes it — the+ // delete waits for whoever holds the record's monitor.+ var buckets = new InMemoryStorage<String, Bucket>();+ S3Service service = new S3Service(buckets, new InMemoryStorage<>(), tempDir.resolve("s3-monitor"), false);+ service.createBucket("monitor-bucket", "us-east-1");+ Bucket record = buckets.get("monitor-bucket").orElseThrow();++ Thread delete = new Thread(() -> service.deleteBucket("monitor-bucket"));+ synchronized (record) {+ delete.start();+ long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(10);+ while (delete.getState() != Thread.State.BLOCKED) {+ assertTrue(buckets.get("monitor-bucket").isPresent(),+ "DeleteBucket removed the record without waiting for the bucket monitor");+ if (System.nanoTime() > deadline) {+ fail("DeleteBucket never reached the bucket monitor");+ }+ Thread.onSpinWait();+ }+ }+ delete.join(30_000);++ assertTrue(buckets.get("monitor-bucket").isEmpty());+ }++ @Test+ void aMetricsPutRacingABucketDeleteCannotRestoreTheBucket() throws Exception {+ // A put resolves the bucket record before it writes it back, so a delete landing in between+ // would be undone by the write. The interleaving is forced rather than raced for: the test+ // holds the bucket monitor, lets the put resolve the record and block on it, deletes the+ // bucket from the same thread, and only then releases.+ var buckets = new InMemoryStorage<String, Bucket>();+ S3Service service = new S3Service(buckets, new InMemoryStorage<>(), tempDir.resolve("s3-race"), false);+ service.createBucket("race-bucket", "us-east-1");+ Bucket record = buckets.get("race-bucket").orElseThrow();++ var putStarted = new java.util.concurrent.CountDownLatch(1);+ var thrownByPut = new java.util.concurrent.atomic.AtomicReference<Exception>();+ Thread put = new Thread(() -> {+ putStarted.countDown();+ try {+ service.putBucketMetricsConfiguration("race-bucket", "id", "<Id>id</Id>");+ } catch (Exception e) {+ thrownByPut.set(e);+ }+ });++ synchronized (record) {+ put.start();+ putStarted.await();+ awaitBlockedOnMonitor(put);+ service.deleteBucket("race-bucket");+ }+ put.join(30_000);++ assertTrue(buckets.get("race-bucket").isEmpty(),+ "the deleted bucket was restored by the concurrent metrics put");+ assertEquals("NoSuchBucket",+ assertInstanceOf(AwsException.class, thrownByPut.get()).getErrorCode());+ }++ @Test+ void aMetricsPutCannotOverwriteABucketRecreatedWhileItWaited() throws Exception {+ // Presence is not enough: deleted and recreated under the same name, the store holds a+ // different record, and writing the resolved one back would replace the new bucket with+ // the old one's state.+ var buckets = new InMemoryStorage<String, Bucket>();+ S3Service service = new S3Service(buckets, new InMemoryStorage<>(), tempDir.resolve("s3-recreate"), false);+ service.createBucket("recreated-bucket", "us-east-1");+ Bucket original = buckets.get("recreated-bucket").orElseThrow();++ var putStarted = new java.util.concurrent.CountDownLatch(1);+ var thrownByPut = new java.util.concurrent.atomic.AtomicReference<Exception>();+ Thread put = new Thread(() -> {+ putStarted.countDown();+ try {+ service.putBucketMetricsConfiguration("recreated-bucket", "stale", "<Id>stale</Id>");+ } catch (Exception e) {+ thrownByPut.set(e);+ }+ });++ synchronized (original) {+ put.start();+ putStarted.await();+ awaitBlockedOnMonitor(put);+ service.deleteBucket("recreated-bucket");+ service.createBucket("recreated-bucket", "us-east-1");+ }+ put.join(30_000);++ assertEquals("NoSuchBucket",+ assertInstanceOf(AwsException.class, thrownByPut.get()).getErrorCode());+ assertNotSame(original, buckets.get("recreated-bucket").orElseThrow(),+ "the recreated bucket was replaced by the record the put had resolved");+ assertNull(buckets.get("recreated-bucket").orElseThrow().getMetricsConfigurations(),+ "the recreated bucket inherited a configuration written against the old record");+ }++ @Test+ void concurrentMetricsConfigurationPutsAllSurvive() throws Exception {+ // Each put reads the configuration map, adds to it and writes it back, so without a shared+ // monitor concurrent puts of different ids overwrite each other's work.+ s3Service.createBucket("metrics-race", "us-east-1");+ int count = 24;+ var pool = java.util.concurrent.Executors.newFixedThreadPool(8);+ var start = new java.util.concurrent.CountDownLatch(1);+ try {+ var submitted = new java.util.ArrayList<java.util.concurrent.Future<?>>();+ for (int i = 0; i < count; i++) {+ String id = "config-" + i;+ submitted.add(pool.submit(() -> {+ start.await();+ s3Service.putBucketMetricsConfiguration("metrics-race", id, "<Id>" + id + "</Id>");+ return null;+ }));+ }+ start.countDown();+ for (var future : submitted) {+ future.get(30, java.util.concurrent.TimeUnit.SECONDS);+ }+ } finally {+ pool.shutdownNow();+ }++ String listed = s3Service.listBucketMetricsConfigurations("metrics-race");+ for (int i = 0; i < count; i++) {+ assertTrue(listed.contains("<Id>config-" + i + "</Id>"),+ "configuration config-" + i + " was lost by a concurrent put");+ }+ }}