Light, fluffy, and always free - The AWS Local Emulator alternative
feat(s3): apply lifecycle expiration rules (#5158)
jwoa95cbd2f61dc465
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3LifecycleTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3LifecycleTest.javaindex b3757c6bc0..f09193217e 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3LifecycleTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3LifecycleTest.java@@ -7,6 +7,7 @@ import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;+import software.amazon.awssdk.core.sync.RequestBody;import software.amazon.awssdk.services.s3.S3Client;import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException;import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration;@@ -14,12 +15,20 @@ import software.amazon.awssdk.services.s3.model.CreateBucketRequest;import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;import software.amazon.awssdk.services.s3.model.ExpirationStatus;import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse;+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;import software.amazon.awssdk.services.s3.model.LifecycleExpiration;import software.amazon.awssdk.services.s3.model.LifecycleRule;import software.amazon.awssdk.services.s3.model.LifecycleRuleFilter;import software.amazon.awssdk.services.s3.model.PutBucketLifecycleConfigurationResponse;+import software.amazon.awssdk.services.s3.model.PutObjectResponse;import software.amazon.awssdk.services.s3.model.TransitionDefaultMinimumObjectSize;+import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.time.temporal.ChronoUnit;+import java.util.Locale;+import static org.assertj.core.api.Assertions.assertThat;/**@@ -40,6 +49,9 @@ import static org.assertj.core.api.Assertions.assertThat;class S3LifecycleTest {private static final String BUCKET = "compat-lifecycle-bucket";+ private static final DateTimeFormatter HTTP_DATE = DateTimeFormatter+ .ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US)+ .withZone(ZoneOffset.UTC);private static S3Client s3;@@ -117,4 +129,25 @@ class S3LifecycleTest {.as("GET must default to ALL_STORAGE_CLASSES_128_K when PUT omits the header").isEqualTo(TransitionDefaultMinimumObjectSize.ALL_STORAGE_CLASSES_128_K);}++ @Test+ @Order(3)+ @DisplayName("#5137 objects under an Expiration rule report expiry-date and rule-id")+ void objectsUnderAnExpirationRuleReportTheirExpiry() {+ // The configuration from @Order(2): Days 365 on every key, rule id expire-everything.+ String key = "expiring.txt";+ try {+ PutObjectResponse put = s3.putObject(req -> req.bucket(BUCKET).key(key), RequestBody.fromString("hi"));+ HeadObjectResponse head = s3.headObject(req -> req.bucket(BUCKET).key(key));++ // Creation time plus 365 days, rounded up to the next midnight UTC.+ Instant expiry = head.lastModified().plus(365, ChronoUnit.DAYS)+ .truncatedTo(ChronoUnit.DAYS).plus(1, ChronoUnit.DAYS);+ String expected = "expiry-date=\"" + HTTP_DATE.format(expiry) + "\", rule-id=\"expire-everything\"";+ assertThat(head.expiration()).isEqualTo(expected);+ assertThat(put.expiration()).isEqualTo(expected);+ } finally {+ s3.deleteObject(req -> req.bucket(BUCKET).key(key));+ }+ }}diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.mdindex c3fe6a6c01..fa7b19d984 100644--- a/docs/configuration/advanced/application-yml.md+++ b/docs/configuration/advanced/application-yml.md@@ -140,6 +140,8 @@ floci:s3:enabled: truedefault-presign-expiry-seconds: 3600+ lifecycle-sweep-enabled: true # Apply bucket lifecycle expiration rules in the background+ lifecycle-sweep-interval-seconds: 60 # Seconds between lifecycle sweepsdynamodb:enabled: truediff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex c00ba9b353..0d28b377fb 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -221,6 +221,8 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s| `FLOCI_SERVICES_S3_ENABLED` | `true` | Enable the S3 service || `FLOCI_SERVICES_S3_DEFAULT_PRESIGN_EXPIRY_SECONDS` | `3600` | Default pre-signed URL expiry when none is specified || `FLOCI_SERVICES_S3_GLOBAL_BUCKET_NAMESPACE` | `false` | When `true`, bucket/object resolution spans every account's partition so a bucket created in one account is visible cross-account (a single global namespace), matching real S3's global bucket names. Off by default keeps buckets isolated per account |+| `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_ENABLED` | `true` | Apply bucket lifecycle expiration rules (`Expiration`, `NoncurrentVersionExpiration`, `AbortIncompleteMultipartUpload`) in the background |+| `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS` | `60` | How often the lifecycle sweep runs |### DynamoDBdiff --git a/docs/services/s3.md b/docs/services/s3.mdindex dafc7e9c77..bb4b82acdb 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -87,6 +87,61 @@ specific object version through the four `?annotation` operations. Notes on theresolve cross-account but annotation reads, writes, and listings stay in the caller's account.- S3 Metadata annotation tables and annotation replication are not implemented.+## Lifecycle Expiration++A bucket's lifecycle configuration is stored as sent and its expiration actions are applied by a+background sweep, every `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS` (60 by default). S3+also applies lifecycle asynchronously, so a lower interval only makes Floci act sooner after an+object becomes due. Each enabled rule's filter (`Prefix`, `Tag`, `ObjectSizeGreaterThan`,+`ObjectSizeLessThan`, an `And` of them, or the older rule-level `Prefix`) selects what it applies to,+and disabled rules are ignored.++- `Expiration` with `Days` or `Date` expires the current version. A day count runs from the+ object's last-modified time and, as on S3, is rounded up to the next midnight UTC, so an object+ written at 10:30 UTC under a 3-day rule expires at midnight UTC three days and some hours later.+ An unversioned bucket deletes the object; a versioning-enabled bucket adds a delete marker.+ A versioning-suspended bucket deletes the object as `DeleteObject` does in Floci, without the+ `null` delete marker S3 adds (see the differences below).+- `ExpiredObjectDeleteMarker` removes a delete marker that is the only version left of its key.+ A `Days` or `Date` expiration also removes such a marker once the marker itself is old enough.+- `NoncurrentVersionExpiration` permanently deletes a noncurrent version, delete markers included,+ once it has been noncurrent for `NoncurrentDays` (counted from when the next newer version was+ written, rounded the same way) and at least `NewerNoncurrentVersions` newer noncurrent versions+ are kept ahead of it.+- `AbortIncompleteMultipartUpload` aborts an upload `DaysAfterInitiation` after it was started,+ rounded the same way. Only a rule whose filter is a plain prefix selects uploads.+- A version under Object Lock retention or a legal hold is never deleted by lifecycle.+- Expirations are notified as `s3:LifecycleExpiration:Delete` and+ `s3:LifecycleExpiration:DeleteMarkerCreated`, not as `s3:ObjectRemoved:*`, with `s3.amazonaws.com`+ as the principal and source IP address. Their EventBridge events carry `reason`+ `Lifecycle Expiration`, requester `s3.amazonaws.com` and no `source-ip-address`.+- A rule that S3 would have refused is skipped as a whole, so a bad value never widens what it+ deletes: `Days` or `NoncurrentDays` that is not a positive integer, `NewerNoncurrentVersions`+ outside 1 to 100 or without a `Filter`, a negative `DaysAfterInitiation`, a size bound that is not+ a non-negative number, a `Tag` without both `Key` and `Value` or with a key used twice, neither a+ `Filter` nor a rule-level `Prefix`, `ExpiredObjectDeleteMarker` next to `Days` or `Date`, a `Date`+ that does not parse, or a `Status` other than `Enabled` or `Disabled`. An empty `Filter` applies+ to the whole bucket. A body that is not one well-formed `LifecycleConfiguration` document has no+ rules.++`PutObject`, `GetObject` and `HeadObject` on the current version of an object that an `Expiration`+rule matches return `x-amz-expiration: expiry-date="Fri, 23 Dec 2012 00:00:00 GMT", rule-id="..."`,+with the earliest matching expiry and that rule's URL-encoded id. Requests for a noncurrent version+do not carry it.++Differences from S3:++- `Transition` and `NoncurrentVersionTransition` are stored but not applied: storage classes do not+ change.+- `CopyObject` and `CompleteMultipartUpload` do not return `x-amz-expiration`, and+ `CreateMultipartUpload` does not return `x-amz-abort-date` or `x-amz-abort-rule-id`.+- The configuration is not validated when it is put: a rule S3 would have refused is stored and+ returned as sent, and only skipped when rules are applied.+- After `DeleteObject` in a versioning-suspended bucket, every version left behind is noncurrent,+ as it is on S3, but Floci does not record when the newest of them stopped being current, so+ `NoncurrentDays` never expires that one. The older ones expire as usual.+- A delete marker that lifecycle adds carries the time of the sweep, not midnight UTC.+## Website HostingStatic website requests use a bucket website hostname such as@@ -396,6 +451,8 @@ These AWS S3 features have no handler in Floci. Calls will return an error (typi| `FLOCI_SERVICES_S3_ENFORCE_AUTH` | `false` | Verify SigV4 signatures and evaluate bucket policies and ACLs, including for anonymous callers (see [Signature Verification](#signature-verification)) || `FLOCI_AUTH_VALIDATE_SIGNATURES` | `false` | Verify SigV4 signatures without evaluating bucket policies or ACLs (see [Signature Verification](#signature-verification)) || `FLOCI_SERVICES_S3_GLOBAL_BUCKET_NAMESPACE` | `false` | Enable cross-account bucket and object resolution |+| `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_ENABLED` | `true` | Apply bucket lifecycle expiration rules in the background (see [Lifecycle Expiration](#lifecycle-expiration)) |+| `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS` | `60` | How often the lifecycle sweep runs |### Storagediff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex ae7191e95e..c684958afa 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1379,6 +1379,23 @@ public interface EmulatorConfig {*/@WithDefault("false")boolean globalBucketNamespace();++ /**+ * Run the background sweep that applies bucket lifecycle expiration rules: Expiration,+ * NoncurrentVersionExpiration and AbortIncompleteMultipartUpload. When off, rules are+ * stored and reported in {@code x-amz-expiration} but nothing expires.+ * Env: FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_ENABLED+ */+ @WithDefault("true")+ boolean lifecycleSweepEnabled();++ /**+ * How often the lifecycle sweep runs. S3 itself applies lifecycle about once a day, to+ * expiry times rounded to midnight UTC, so the interval only bounds how late Floci is.+ * Env: FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS+ */+ @WithDefault("60")+ long lifecycleSweepIntervalSeconds();}interface DynamoDbServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java b/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.javaindex a8f23de491..ae13247d7c 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java@@ -454,6 +454,37 @@ public final class XmlParser {return null;}+ /**+ * Parses a whole document into its document element's tree, or returns {@code null} when the+ * document is empty or not well-formed anywhere, after its document element included.+ *+ * <p>For a stored document whose every part has effect, where {@link #extractElementTree}+ * would return a complete element followed by content that never parsed.+ */+ public static XmlElement parseDocumentElementTree(String xml) {+ if (xml == null || xml.isEmpty()) {+ return null;+ }+ try {+ XMLStreamReader r = FACTORY.createXMLStreamReader(new StringReader(xml));+ try {+ XmlElement root = null;+ while (r.hasNext()) {+ int event = r.next();+ if (root == null && event == XMLStreamConstants.START_ELEMENT) {+ root = readElementTree(r);+ }+ }+ return root;+ } finally {+ r.close();+ }+ } catch (XMLStreamException e) {+ LOG.debugv("Ignoring malformed XML document: {0}", e.getMessage());+ return null;+ }+ }+private static XmlElement readElementTree(XMLStreamReader r) throws XMLStreamException {String name = r.getLocalName();StringBuilder text = new StringBuilder();diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 64c4983b5a..663e24f9b2 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -111,7 +111,7 @@ public class S3Controller {private static final DateTimeFormatter ISO_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneId.of("UTC"));- private static final DateTimeFormatter RFC_822 = DateTimeFormatter+ static final DateTimeFormatter RFC_822 = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss z", Locale.US).withZone(ZoneId.of("GMT"));private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();@@ -2939,6 +2939,7 @@ public class S3Controller {appendSseCustomerHeaders(resp, obj);appendChecksumHeaders(resp, obj.getChecksum());appendLockHeaders(resp, obj);+ appendExpirationHeader(resp, obj);}// includeChecksum must be false for a range the stored checksum does not describe:@@ -2983,6 +2984,14 @@ public class S3Controller {appendChecksumHeaders(resp, obj.getChecksum());}appendLockHeaders(resp, obj);+ appendExpirationHeader(resp, obj);+ }++ private void appendExpirationHeader(Response.ResponseBuilder resp, S3Object obj) {+ String expiration = s3Service.expirationHeader(obj);+ if (expiration != null) {+ resp.header("x-amz-expiration", expiration);+ }}private void appendSseCustomerHeaders(Response.ResponseBuilder resp, S3Object obj) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfiguration.javanew file mode 100644index 0000000000..714e3afe29--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfiguration.java@@ -0,0 +1,358 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.XmlParser;+import io.github.hectorvent.floci.core.common.XmlParser.XmlElement;+import io.github.hectorvent.floci.core.common.auth.SigV4Canonicalization;+import org.jboss.logging.Logger;++import java.time.Instant;+import java.time.LocalDate;+import java.time.LocalDateTime;+import java.time.OffsetDateTime;+import java.time.ZoneOffset;+import java.time.format.DateTimeParseException;+import java.time.temporal.ChronoUnit;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.function.Function;++/**+ * The expiration actions of a bucket's lifecycle configuration, read the way S3 applies them.+ *+ * <p>PutBucketLifecycleConfiguration stores the body verbatim, so this is parsed when it is used,+ * and a body or rule S3 would have refused is ignored rather than failing the request or the+ * sweep that reads it (see {@link #parse}). Disabled rules are dropped. Transition actions are not+ * modelled.+ */+record S3LifecycleConfiguration(List<Rule> rules) {++ static final S3LifecycleConfiguration EMPTY = new S3LifecycleConfiguration(List.of());++ private static final Logger LOG = Logger.getLogger(S3LifecycleConfiguration.class);++ /** NewerNoncurrentVersions: "You can specify up to 100 noncurrent versions to retain." */+ private static final int MAX_NEWER_NONCURRENT_VERSIONS = 100;++ // Date is ISO 8601 on the wire. The leniency covers bodies Floci writes itself, such as a+ // CloudFormation ExpirationDate passed through as the template spelled it.+ private static final List<Function<String, Instant>> DATE_PARSERS = List.of(+ text -> OffsetDateTime.parse(text).toInstant(),+ text -> LocalDateTime.parse(text).toInstant(ZoneOffset.UTC),+ text -> LocalDate.parse(text).atStartOfDay(ZoneOffset.UTC).toInstant());++ record Rule(String id, Filter filter, Expiration expiration,+ NoncurrentVersionExpiration noncurrentVersionExpiration, Integer abortDaysAfterInitiation) {+ }++ /** A rule's filter: every condition that is set must hold. An empty filter matches everything. */+ record Filter(String prefix, Map<String, String> tags, Long objectSizeGreaterThan, Long objectSizeLessThan) {++ boolean matches(String key, Map<String, String> objectTags, long size) {+ if (!key.startsWith(prefix)) {+ return false;+ }+ for (Map.Entry<String, String> tag : tags.entrySet()) {+ String value = objectTags == null ? null : objectTags.get(tag.getKey());+ if (!tag.getValue().equals(value)) {+ return false;+ }+ }+ // Both size bounds are exclusive.+ return (objectSizeGreaterThan == null || size > objectSizeGreaterThan)+ && (objectSizeLessThan == null || size < objectSizeLessThan);+ }++ /**+ * An in-progress multipart upload has no tags or size to test, and S3 refuses+ * AbortIncompleteMultipartUpload under a tag filter, so only a prefix selects uploads.+ */+ boolean selectsUpload(String key) {+ return tags.isEmpty() && objectSizeGreaterThan == null && objectSizeLessThan == null+ && key.startsWith(prefix);+ }+ }++ record Expiration(Integer days, Instant date, boolean expiredObjectDeleteMarker) {++ /** When something created at {@code created} expires under this action, or null if it sets no age. */+ Instant expiresAt(Instant created) {+ if (days != null) {+ return afterDays(created, days);+ }+ return date;+ }+ }++ record NoncurrentVersionExpiration(Integer noncurrentDays, Integer newerNoncurrentVersions) {+ }++ /** The earliest current-version expiration that applies to an object, and the rule that sets it. */+ record Expiry(Instant date, String ruleId) {++ /**+ * The {@code x-amz-expiration} header value, in the shape the S3 API reference shows,+ * {@code expiry-date="Fri, 23 Dec 2012 00:00:00 GMT", rule-id="1"}, with the rule id+ * URL-encoded as the reference requires.+ */+ String header() {+ return "expiry-date=\"" + S3Controller.RFC_822.format(date) + "\", rule-id=\""+ + SigV4Canonicalization.uriEncode(ruleId == null ? "" : ruleId) + "\"";+ }+ }++ /**+ * Day-based lifecycle times run from a start time: "Amazon S3 calculates the time by adding+ * the number of days specified in the rule to the object creation time and rounding up the+ * resulting time to the next day at midnight UTC."+ */+ static Instant afterDays(Instant start, int days) {+ return start.plus(days, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).plus(1, ChronoUnit.DAYS);+ }++ /**+ * The earliest Days or Date expiration among the rules that match a current object version,+ * or null when none does.+ */+ Expiry currentVersionExpiry(String key, Map<String, String> tags, long size, Instant created) {+ Expiry earliest = null;+ for (Rule rule : rules) {+ if (rule.expiration() == null || !rule.filter().matches(key, tags, size)) {+ continue;+ }+ Instant at = rule.expiration().expiresAt(created);+ if (at != null && (earliest == null || at.isBefore(earliest.date()))) {+ earliest = new Expiry(at, rule.id());+ }+ }+ return earliest;+ }++ /**+ * Whether a delete marker that is the only version of its key, an expired object delete+ * marker, is due for removal. ExpiredObjectDeleteMarker removes it at once; a Days or Date+ * expiration removes it once the marker itself is old enough.+ */+ boolean removesExpiredObjectDeleteMarker(String key, Instant markerCreated, Instant now) {+ for (Rule rule : rules) {+ Expiration expiration = rule.expiration();+ if (expiration == null || !rule.filter().matches(key, Map.of(), 0)) {+ continue;+ }+ if (expiration.expiredObjectDeleteMarker()) {+ return true;+ }+ Instant at = expiration.expiresAt(markerCreated);+ if (at != null && !now.isBefore(at)) {+ return true;+ }+ }+ return false;+ }++ /**+ * Whether a noncurrent version is due for permanent deletion: it has been noncurrent, counted+ * from when its successor was created, for NoncurrentDays, and at least+ * NewerNoncurrentVersions newer noncurrent versions are retained ahead of it.+ */+ boolean expiresNoncurrentVersion(String key, Map<String, String> tags, long size,+ Instant becameNoncurrent, int newerNoncurrentVersions, Instant now) {+ for (Rule rule : rules) {+ NoncurrentVersionExpiration action = rule.noncurrentVersionExpiration();+ if (action == null || !rule.filter().matches(key, tags, size)) {+ continue;+ }+ if (action.noncurrentDays() == null && action.newerNoncurrentVersions() == null) {+ continue;+ }+ boolean oldEnough = action.noncurrentDays() == null+ || !now.isBefore(afterDays(becameNoncurrent, action.noncurrentDays()));+ boolean beyondRetained = action.newerNoncurrentVersions() == null+ || newerNoncurrentVersions >= action.newerNoncurrentVersions();+ if (oldEnough && beyondRetained) {+ return true;+ }+ }+ return false;+ }++ /** Whether an incomplete multipart upload for {@code key} initiated at {@code initiated} is due to be aborted. */+ boolean abortsUpload(String key, Instant initiated, Instant now) {+ for (Rule rule : rules) {+ Integer days = rule.abortDaysAfterInitiation();+ if (days != null && rule.filter().selectsUpload(key) && !now.isBefore(afterDays(initiated, days))) {+ return true;+ }+ }+ return false;+ }++ /**+ * Reads a stored configuration. A body that is not one well-formed LifecycleConfiguration+ * document has no rules. A rule that S3 would have refused on put, such as {@code Days} 0 or a+ * size bound that is not a number, is dropped whole rather than read without the bad value, so+ * a bad value never widens what the rule deletes. The other rules still apply.+ */+ static S3LifecycleConfiguration parse(String xml) {+ XmlElement root = XmlParser.parseDocumentElementTree(xml);+ if (root == null || !"LifecycleConfiguration".equals(root.name())) {+ return EMPTY;+ }+ List<Rule> rules = new ArrayList<>();+ for (XmlElement element : root.children()) {+ if (!"Rule".equals(element.name()) || "Disabled".equals(text(element, "Status"))) {+ continue;+ }+ try {+ rules.add(rule(element));+ } catch (IllegalArgumentException e) {+ LOG.debugv("Ignoring lifecycle rule {0}: {1}", text(element, "ID"), e.getMessage());+ }+ }+ return new S3LifecycleConfiguration(List.copyOf(rules));+ }++ private static Rule rule(XmlElement rule) {+ String status = text(rule, "Status");+ if (!"Enabled".equals(status)) {+ throw new IllegalArgumentException("Status is " + status + ", not Enabled or Disabled");+ }+ NoncurrentVersionExpiration noncurrent = noncurrentVersionExpiration(rule.child("NoncurrentVersionExpiration"));+ if (noncurrent != null && noncurrent.newerNoncurrentVersions() != null && rule.child("Filter") == null) {+ // The user guide: "If you don't specify a <Filter> element, Amazon S3 generates an+ // InvalidRequest error when you specify the number of noncurrent versions to retain."+ throw new IllegalArgumentException("NewerNoncurrentVersions needs a Filter");+ }+ return new Rule(text(rule, "ID"), filter(rule), expiration(rule.child("Expiration")), noncurrent,+ abortDaysAfterInitiation(rule.child("AbortIncompleteMultipartUpload")));+ }++ /**+ * The rule's Filter, or the deprecated rule-level Prefix that predates it: "Filter is required+ * if the LifecycleRule does not contain a Prefix element." An empty Filter applies to the whole+ * bucket.+ */+ private static Filter filter(XmlElement rule) {+ XmlElement filter = rule.child("Filter");+ if (filter == null) {+ String prefix = text(rule, "Prefix");+ if (prefix == null) {+ throw new IllegalArgumentException("a rule needs a Filter or a Prefix");+ }+ return new Filter(prefix, Map.of(), null, null);+ }+ XmlElement and = filter.child("And");+ XmlElement conditions = and != null ? and : filter;+ String prefix = text(conditions, "Prefix");+ Map<String, String> tags = new LinkedHashMap<>();+ for (XmlElement tag : conditions.children()) {+ if ("Tag".equals(tag.name())) {+ String key = text(tag, "Key");+ String value = text(tag, "Value");+ if (key == null || value == null) {+ throw new IllegalArgumentException("a Tag needs both a Key and a Value");+ }+ // "When you specify multiple tags in a filter, each tag key must be unique."+ if (tags.put(key, value) != null) {+ throw new IllegalArgumentException("tag key " + key + " appears twice");+ }+ }+ }+ return new Filter(prefix == null ? "" : prefix, Map.copyOf(tags),+ size(conditions, "ObjectSizeGreaterThan"), size(conditions, "ObjectSizeLessThan"));+ }++ private static Expiration expiration(XmlElement element) {+ if (element == null) {+ return null;+ }+ Integer days = positiveInteger(element, "Days");+ String dateText = text(element, "Date");+ Instant date = dateText == null ? null : date(dateText);+ String deleteMarkerText = text(element, "ExpiredObjectDeleteMarker");+ if (deleteMarkerText != null && (days != null || date != null)) {+ throw new IllegalArgumentException("ExpiredObjectDeleteMarker cannot be specified with Days or Date");+ }+ boolean deleteMarker = deleteMarkerText != null && bool(deleteMarkerText, "ExpiredObjectDeleteMarker");+ if (days == null && date == null && !deleteMarker) {+ return null;+ }+ return new Expiration(days, date, deleteMarker);+ }++ private static NoncurrentVersionExpiration noncurrentVersionExpiration(XmlElement element) {+ if (element == null) {+ return null;+ }+ Integer newer = integer(element, "NewerNoncurrentVersions");+ if (newer != null && (newer < 1 || newer > MAX_NEWER_NONCURRENT_VERSIONS)) {+ throw new IllegalArgumentException("NewerNoncurrentVersions " + newer + " is not between 1 and 100");+ }+ return new NoncurrentVersionExpiration(positiveInteger(element, "NoncurrentDays"), newer);+ }++ private static Integer abortDaysAfterInitiation(XmlElement element) {+ if (element == null) {+ return null;+ }+ Integer days = integer(element, "DaysAfterInitiation");+ if (days != null && days < 0) {+ throw new IllegalArgumentException("DaysAfterInitiation " + days + " is negative");+ }+ return days;+ }++ private static String text(XmlElement parent, String childName) {+ XmlElement child = parent.child(childName);+ return child == null ? null : child.text();+ }++ /** Days and NoncurrentDays: "The value must be a non-zero positive integer." */+ private static Integer positiveInteger(XmlElement parent, String childName) {+ Integer value = integer(parent, childName);+ if (value != null && value < 1) {+ throw new IllegalArgumentException(childName + " " + value + " is not a positive integer");+ }+ return value;+ }++ /** An absent element is null; one whose text is not an integer makes the rule invalid. */+ private static Integer integer(XmlElement parent, String childName) {+ String text = text(parent, childName);+ return text == null ? null : Integer.valueOf(text);+ }++ private static Long size(XmlElement parent, String childName) {+ String text = text(parent, childName);+ if (text == null) {+ return null;+ }+ long value = Long.parseLong(text);+ if (value < 0) {+ throw new IllegalArgumentException(childName + " " + value + " is negative");+ }+ return value;+ }++ private static boolean bool(String text, String name) {+ if ("true".equalsIgnoreCase(text)) {+ return true;+ }+ if ("false".equalsIgnoreCase(text)) {+ return false;+ }+ throw new IllegalArgumentException(name + " " + text + " is not true or false");+ }++ private static Instant date(String text) {+ for (Function<String, Instant> parser : DATE_PARSERS) {+ try {+ return parser.apply(text);+ } catch (DateTimeParseException ignored) {+ // Not this shape; try the next one.+ }+ }+ throw new IllegalArgumentException("Date " + text + " is not an ISO 8601 date");+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeper.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeper.javanew file mode 100644index 0000000000..1cecb3e388--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeper.java@@ -0,0 +1,87 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.quarkus.runtime.ShutdownEvent;+import io.quarkus.runtime.StartupEvent;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Clock;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.TimeUnit;++/**+ * Applies bucket lifecycle expiration rules from one background thread. S3 evaluates lifecycle+ * asynchronously rather than on each request, so a sweep over every bucket with a lifecycle+ * configuration on a fixed interval matches that, and leaves nothing to reconcile after a restart.+ */+@ApplicationScoped+public class S3LifecycleSweeper {++ private static final Logger LOG = Logger.getLogger(S3LifecycleSweeper.class);++ /** Used when the configured interval is not a positive number of seconds. */+ static final long DEFAULT_INTERVAL_SECONDS = 60L;++ private final S3Service s3Service;+ private final Clock clock;+ private final boolean enabled;+ private final long intervalSeconds;+ private final ScheduledExecutorService executor;++ /**+ * S3 stamps objects with {@code Instant.now()}, so the sweep reads the same system clock, not+ * the injectable {@link Clock} that tests pin to a fixed date.+ */+ @Inject+ public S3LifecycleSweeper(S3Service s3Service, EmulatorConfig config) {+ this(s3Service, config, Clock.systemUTC());+ }++ S3LifecycleSweeper(S3Service s3Service, EmulatorConfig config, Clock clock) {+ this.s3Service = s3Service;+ this.clock = clock;+ this.enabled = config.services().s3().enabled() && config.services().s3().lifecycleSweepEnabled();+ this.intervalSeconds = resolveInterval(config.services().s3().lifecycleSweepIntervalSeconds());+ this.executor = Executors.newSingleThreadScheduledExecutor(runnable -> {+ Thread thread = new Thread(runnable, "s3-lifecycle-sweeper");+ thread.setDaemon(true);+ return thread;+ });+ }++ /** A non-positive interval would throw from the startup observer and keep the emulator from becoming ready. */+ private static long resolveInterval(long configured) {+ if (configured > 0) {+ return configured;+ }+ LOG.warnv("Ignoring S3 lifecycle-sweep interval {0}s: must be a positive number of seconds, "+ + "falling back to {1}s", configured, DEFAULT_INTERVAL_SECONDS);+ return DEFAULT_INTERVAL_SECONDS;+ }++ void onStart(@Observes StartupEvent event) {+ if (!enabled) {+ LOG.debug("S3 lifecycle sweeper disabled");+ return;+ }+ executor.scheduleWithFixedDelay(this::sweep, intervalSeconds, intervalSeconds, TimeUnit.SECONDS);+ LOG.debugv("S3 lifecycle sweeper started, interval {0}s", intervalSeconds);+ }++ void onStop(@Observes ShutdownEvent event) {+ executor.shutdownNow();+ }++ void sweep() {+ try {+ s3Service.applyLifecycleExpiration(clock.instant());+ } catch (RuntimeException e) {+ // A sweep failure must not kill the scheduled task. The next run retries.+ LOG.error("S3 lifecycle sweep failed", e);+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex b9450d4bfd..de0ff76531 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.core.common.Base64Url;import io.github.hectorvent.floci.core.common.Digests;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.common.ServicePrincipals;import io.github.hectorvent.floci.core.common.XmlBuilder;@@ -72,6 +73,7 @@ import java.util.Collections;import java.util.Comparator;import java.util.EnumSet;import java.util.HashMap;+import java.util.HashSet;import java.util.HexFormat;import java.util.LinkedHashMap;import java.util.LinkedHashSet;@@ -81,6 +83,7 @@ import java.util.Map;import java.util.Objects;import java.util.Optional;import java.util.Set;+import java.util.TreeMap;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.atomic.AtomicLong;@@ -1936,7 +1939,7 @@ public class S3Service implements Resettable, ResourceProvider {// so a concurrent overwrite cannot land between the check and the delete.synchronized (bucket) {checkDeletePrecondition(bucketName, key, ifMatch);- return deleteObjectLocked(bucket, bucketName, key, versionId, bypassGovernance);+ return deleteObjectLocked(bucket, bucketName, key, versionId, bypassGovernance, OBJECT_REMOVED_EVENT);}}@@ -1989,8 +1992,13 @@ public class S3Service implements Resettable, ResourceProvider {}}+ /**+ * Deletes as DeleteObject does. {@code removalEvent} is the notification event family the+ * deletion is reported under: {@code ObjectRemoved} for a request, {@code LifecycleExpiration}+ * when a lifecycle rule expires the object.+ */private S3Object deleteObjectLocked(Bucket bucket, String bucketName, String key,- String versionId, boolean bypassGovernance) {+ String versionId, boolean bypassGovernance, String removalEvent) {if (bucket.isVersioningEnabled() && versionId == null) {// Check lock on current latest before placing a delete markerobjectStore.get(objectKey(bucketName, key)).ifPresent(prev -> {@@ -2005,6 +2013,7 @@ public class S3Service implements Resettable, ResourceProvider {deleteMarker.setVersionId(markerId);deleteMarker.setDeleteMarker(true);deleteMarker.setLatest(true);+ deleteMarker.setEventSequencer(nextEventSequencer());// Mark previous latest as not latestobjectStore.get(objectKey(bucketName, key)).ifPresent(prev -> {@@ -2022,7 +2031,7 @@ public class S3Service implements Resettable, ResourceProvider {objectStore.put(versionedKey(bucketName, key, markerId), deleteMarker);objectStore.put(objectKey(bucketName, key), deleteMarker);LOG.debugv("Created delete marker: {0}/{1} v={2}", bucketName, key, markerId);- fireNotifications(bucketName, key, "ObjectRemoved:DeleteMarkerCreated", deleteMarker);+ fireNotifications(bucketName, key, removalEvent + ":DeleteMarkerCreated", deleteMarker);return deleteMarker;} else if ("null".equals(versionId)) {// A null version is stored at the plain object key until a versioned write replaces it.@@ -2037,7 +2046,7 @@ public class S3Service implements Resettable, ResourceProvider {deleteFile(bucketName, key);deleteAllAnnotationsFor(annotationParentKey(bucketName, key, null));LOG.debugv("Permanently deleted null version: {0}/{1}", bucketName, key);- fireNotifications(bucketName, key, "ObjectRemoved:Delete", null);+ fireNotifications(bucketName, key, removalEvent + ":Delete", null);return existing;} else if (versionId != null) {// Get the specific version before permanent deletion@@ -2061,7 +2070,7 @@ public class S3Service implements Resettable, ResourceProvider {deleteFile(bucketName, key);} else {S3Object newLatest = remaining.stream()- .max(Comparator.comparing(S3Object::getLastModified))+ .max(WRITE_ORDER).orElseThrow();// The promoted version's body becomes current before its metadata is// published, the order storeObjectInternal writes in, so a concurrent GET@@ -2080,7 +2089,7 @@ public class S3Service implements Resettable, ResourceProvider {}});if (toDelete != null) {- fireNotifications(bucketName, key, "ObjectRemoved:Delete", toDelete);+ fireNotifications(bucketName, key, removalEvent + ":Delete", toDelete);}return toDelete;} else {@@ -2094,7 +2103,7 @@ public class S3Service implements Resettable, ResourceProvider {deleteFile(bucketName, key);deleteAllAnnotationsFor(annotationParentKey(bucketName, key, null));LOG.debugv("Deleted object: {0}/{1}", bucketName, key);- fireNotifications(bucketName, key, "ObjectRemoved:Delete", null);+ fireNotifications(bucketName, key, removalEvent + ":Delete", null);return null;}}@@ -2652,11 +2661,11 @@ public class S3Service implements Resettable, ResourceProvider {Collections.sort(commonPrefixes);}- // Sort by key, then by lastModified descending+ // Sort by key, then newest firstversions.sort((a, b) -> {int keyCompare = a.getKey().compareTo(b.getKey());if (keyCompare != 0) return keyCompare;- return b.getLastModified().compareTo(a.getLastModified());+ return WRITE_ORDER.compare(b, a);});// Apply the marker filter. Without a version-id-marker the marker is an exclusive lower bound on the@@ -2791,11 +2800,7 @@ public class S3Service implements Resettable, ResourceProvider {// --- Object Tagging ---public void putObjectTagging(String bucketName, String key, Map<String, String> tags) {- AccountAwareStorageBackend.OwnedEntry<S3Object> ownedObject =- getStoredObjectEntry(bucketName, key, null);- S3Object obj = ownedObject.value();- obj.setTags(tags != null ? tags : new java.util.HashMap<>());- putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);+ setCurrentObjectTags(bucketName, key, tags != null ? tags : new HashMap<>());LOG.debugv("Put tags on object: {0}/{1}", bucketName, key);}@@ -2810,11 +2815,7 @@ public class S3Service implements Resettable, ResourceProvider {}public void deleteObjectTagging(String bucketName, String key) {- AccountAwareStorageBackend.OwnedEntry<S3Object> ownedObject =- getStoredObjectEntry(bucketName, key, null);- S3Object obj = ownedObject.value();- obj.setTags(new java.util.HashMap<>());- putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);+ setCurrentObjectTags(bucketName, key, new HashMap<>());LOG.debugv("Deleted tags from object: {0}/{1}", bucketName, key);}@@ -4919,24 +4920,348 @@ public class S3Service implements Resettable, ResourceProvider {return new LifecycleConfigurationResult(bucket.getLifecycleConfiguration(), size);}+ /**+ * The lifecycle configuration changes under the bucket monitor the lifecycle sweep applies rules+ * under, so a sweep never expires an object by a rule that was replaced or removed before it acted.+ */public String putBucketLifecycle(String bucketName, String lifecycle, String transitionDefaultMinimumObjectSize) {Bucket bucket = bucketStore.get(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));- bucket.setLifecycleConfiguration(lifecycle);String size = (transitionDefaultMinimumObjectSize == null || transitionDefaultMinimumObjectSize.isBlank())? DEFAULT_TRANSITION_DEFAULT_MIN_OBJECT_SIZE: transitionDefaultMinimumObjectSize;- bucket.setTransitionDefaultMinimumObjectSize(size);- bucketStore.put(bucketName, bucket);+ synchronized (bucket) {+ bucket.setLifecycleConfiguration(lifecycle);+ bucket.setTransitionDefaultMinimumObjectSize(size);+ bucketStore.put(bucketName, bucket);+ }return size;}public void deleteBucketLifecycle(String bucketName) {Bucket bucket = bucketStore.get(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));- bucket.setLifecycleConfiguration(null);- bucket.setTransitionDefaultMinimumObjectSize(null);- bucketStore.put(bucketName, bucket);+ synchronized (bucket) {+ bucket.setLifecycleConfiguration(null);+ bucket.setTransitionDefaultMinimumObjectSize(null);+ bucketStore.put(bucketName, bucket);+ }+ }++ private static final String OBJECT_REMOVED_EVENT = "ObjectRemoved";+ private static final String LIFECYCLE_EXPIRATION_EVENT = "LifecycleExpiration";++ /**+ * The versions of one key in the order they were written: by last-modified time, then by the+ * sequencer each write takes under the bucket lock, so versions stored within one millisecond+ * keep their order. The sequencer is not persisted, so across a restart such versions fall back+ * to version id order, which is at least the same on every read.+ */+ private static final Comparator<S3Object> WRITE_ORDER = Comparator+ .comparing(S3Object::getLastModified)+ .thenComparing(S3Object::getEventSequencer, Comparator.nullsFirst(Comparator.naturalOrder()))+ .thenComparing(S3Object::getVersionId, Comparator.nullsFirst(Comparator.naturalOrder()));++ /**+ * The {@code x-amz-expiration} header for an object being returned or just written: the+ * earliest expiration an enabled rule's Days or Date sets for it, and that rule's id. Null+ * when no rule applies. Only the current version carries the header.+ */+ public String expirationHeader(S3Object object) {+ if (object == null || object.getBucketName() == null || object.isDeleteMarker() || !object.isLatest()) {+ return null;+ }+ String lifecycle = resolveBucket(object.getBucketName()).map(Bucket::getLifecycleConfiguration).orElse(null);+ if (lifecycle == null) {+ return null;+ }+ S3LifecycleConfiguration.Expiry expiry = S3LifecycleConfiguration.parse(lifecycle).currentVersionExpiry(+ object.getKey(), object.getTags(), object.getSize(), object.getLastModified());+ return expiry == null ? null : expiry.header();+ }++ /**+ * Applies the expiration actions of every bucket's lifecycle configuration as they stand at+ * {@code now}, each bucket in its owning account. S3 runs lifecycle asynchronously, so this is+ * driven by {@link S3LifecycleSweeper} rather than by requests. A bucket that fails does not+ * hold back the others, and the run then fails with every bucket's error, so a sweep that went+ * wrong never looks like one that completed.+ */+ void applyLifecycleExpiration(Instant now) {+ Map<String, List<AccountAwareStorageBackend.AccountEntry<Bucket>>> byAccount = new LinkedHashMap<>();+ for (AccountAwareStorageBackend.AccountEntry<Bucket> entry : bucketsInEveryAccount()) {+ if (entry.value().getLifecycleConfiguration() != null) {+ byAccount.computeIfAbsent(entry.accountId(), accountId -> new ArrayList<>()).add(entry);+ }+ }+ List<String> failedBuckets = new ArrayList<>();+ List<RuntimeException> failures = new ArrayList<>();+ byAccount.forEach((accountId, buckets) -> {+ Set<String> bucketNames = new HashSet<>();+ buckets.forEach(entry -> bucketNames.add(entry.key()));+ Map<String, Map<String, KeyHistory>> histories;+ try {+ histories = RequestScopes.callAs(accountId, () -> keyHistories(bucketNames));+ } catch (RuntimeException e) {+ failedBuckets.addAll(bucketNames);+ failures.add(e);+ return;+ }+ for (AccountAwareStorageBackend.AccountEntry<Bucket> entry : buckets) {+ try {+ RequestScopes.runAs(accountId, entry.value().getRegion(), () -> applyLifecycleExpiration(+ entry.key(), histories.getOrDefault(entry.key(), Map.of()), now));+ } catch (RuntimeException e) {+ failedBuckets.add(entry.key());+ failures.add(e);+ }+ }+ });+ if (!failures.isEmpty()) {+ IllegalStateException failure = new IllegalStateException(+ "S3 lifecycle sweep failed for buckets " + failedBuckets);+ failures.forEach(failure::addSuppressed);+ throw failure;+ }+ }++ private List<AccountAwareStorageBackend.AccountEntry<Bucket>> bucketsInEveryAccount() {+ if (bucketStore instanceof AccountAwareStorageBackend<?> aware) {+ @SuppressWarnings("unchecked")+ AccountAwareStorageBackend<Bucket> typed = (AccountAwareStorageBackend<Bucket>) aware;+ return typed.scanAllAccountEntries(bucketName -> true);+ }+ List<AccountAwareStorageBackend.AccountEntry<Bucket>> buckets = new ArrayList<>();+ for (String bucketName : bucketStore.keys()) {+ bucketStore.get(bucketName).ifPresent(bucket ->+ buckets.add(new AccountAwareStorageBackend.AccountEntry<>(null, bucketName, bucket)));+ }+ return buckets;+ }++ /** One key's stored entries: the entry at the plain key, null when there is none, and every versioned entry. */+ private record KeyHistory(S3Object current, List<S3Object> versions) { }++ /** What a sweep does to one key: permanently delete these versions, then expire the current one or not. */+ private record LifecyclePlan(List<String> versionsToDelete, boolean expireCurrent) {+ boolean isEmpty() {+ return versionsToDelete.isEmpty() && !expireCurrent;+ }+ }++ /** A bucket's lifecycle rules, parsed again only when its stored configuration has changed. */+ private static final class LifecycleRules {+ private String xml;+ private S3LifecycleConfiguration rules = S3LifecycleConfiguration.EMPTY;++ S3LifecycleConfiguration of(Bucket bucket) {+ String stored = bucket.getLifecycleConfiguration();+ if (!Objects.equals(stored, xml)) {+ xml = stored;+ rules = S3LifecycleConfiguration.parse(stored);+ }+ return rules;+ }+ }++ private void applyLifecycleExpiration(String bucketName, Map<String, KeyHistory> histories, Instant now) {+ Bucket bucket = bucketStore.get(bucketName).orElse(null);+ LifecycleRules rules = new LifecycleRules();+ if (bucket == null || rules.of(bucket).rules().isEmpty()) {+ return;+ }+ // Each key is planned from the unlocked snapshot, then read again, planned again and applied+ // under the bucket monitor. Writes, tagging, Object Lock changes and lifecycle configuration+ // changes hold the same monitor, so what is applied is what the key and the rules are now.+ for (Map.Entry<String, KeyHistory> entry : histories.entrySet()) {+ String key = entry.getKey();+ if (planLifecycle(rules.of(bucket), key, entry.getValue(), now).isEmpty()) {+ continue;+ }+ synchronized (bucket) {+ if (bucketStore.get(bucketName).orElse(null) != bucket) {+ return;+ }+ KeyHistory history = rereadKeyHistory(bucketName, key, entry.getValue());+ if (history == null) {+ continue;+ }+ LifecyclePlan plan = planLifecycle(rules.of(bucket), key, history, now);+ for (String versionId : plan.versionsToDelete()) {+ expireByLifecycle(bucket, bucketName, key, versionId);+ }+ if (plan.expireCurrent()) {+ expireByLifecycle(bucket, bucketName, key, null);+ }+ }+ }+ abortIncompleteMultipartUploads(bucket, bucketName, rules, now);+ }++ /**+ * Every key of the named buckets in the current account, from one pass over the object store.+ * A key with versions but no plain-key entry is included: a versioning-suspended bucket leaves+ * one after DeleteObject.+ */+ private Map<String, Map<String, KeyHistory>> keyHistories(Set<String> bucketNames) {+ Map<String, Map<String, S3Object>> currents = new HashMap<>();+ Map<String, Map<String, List<S3Object>>> versions = new HashMap<>();+ for (String storeKey : objectStore.keys()) {+ int slash = storeKey.indexOf('/');+ String bucketName = slash < 0 ? null : storeKey.substring(0, slash);+ if (!bucketNames.contains(bucketName)) {+ continue;+ }+ S3Object stored = objectStore.get(storeKey).orElse(null);+ if (stored == null) {+ continue;+ }+ if (storeKey.equals(objectKey(bucketName, stored.getKey()))) {+ currents.computeIfAbsent(bucketName, name -> new HashMap<>()).put(stored.getKey(), stored);+ } else {+ versions.computeIfAbsent(bucketName, name -> new HashMap<>())+ .computeIfAbsent(stored.getKey(), key -> new ArrayList<>()).add(stored);+ }+ }+ Map<String, Map<String, KeyHistory>> histories = new HashMap<>();+ for (String bucketName : bucketNames) {+ Map<String, S3Object> bucketCurrents = currents.getOrDefault(bucketName, Map.of());+ Map<String, List<S3Object>> bucketVersions = versions.getOrDefault(bucketName, Map.of());+ Set<String> keys = new HashSet<>(bucketCurrents.keySet());+ keys.addAll(bucketVersions.keySet());+ Map<String, KeyHistory> bucketHistories = new TreeMap<>();+ for (String key : keys) {+ bucketHistories.put(key, new KeyHistory(bucketCurrents.get(key), bucketVersions.getOrDefault(key, List.of())));+ }+ histories.put(bucketName, bucketHistories);+ }+ return histories;+ }++ /**+ * One key's entries as they are now, read directly at the store keys the snapshot saw, or null+ * when the key has been written since. Every write replaces the plain-key entry, so while it is+ * unchanged the key's versions are the snapshot's, less any deleted since.+ */+ private KeyHistory rereadKeyHistory(String bucketName, String key, KeyHistory snapshot) {+ S3Object current = objectStore.get(objectKey(bucketName, key)).orElse(null);+ if (!sameWrite(current, snapshot.current())) {+ return null;+ }+ List<S3Object> versions = new ArrayList<>();+ for (S3Object version : snapshot.versions()) {+ objectStore.get(versionedKey(bucketName, key, version.getVersionId())).ifPresent(versions::add);+ }+ return new KeyHistory(current, versions);+ }++ private static boolean sameWrite(S3Object stored, S3Object seen) {+ if (stored == null || seen == null) {+ return stored == seen;+ }+ return stored == seen+ || (Objects.equals(stored.getVersionId(), seen.getVersionId())+ && Objects.equals(stored.getDataGeneration(), seen.getDataGeneration())+ && Objects.equals(stored.getLastModified(), seen.getLastModified())+ && stored.isDeleteMarker() == seen.isDeleteMarker());+ }++ /**+ * Decides which lifecycle actions are due on one key. A noncurrent version has been noncurrent+ * since its successor, the next newer version, was created. Versions under Object Lock+ * retention or a legal hold are never planned for deletion.+ *+ * <p>A key with no plain-key entry has only noncurrent versions. The newest of them stopped+ * being current when its successor was deleted, a time Floci does not record, so it is not+ * aged by NoncurrentDays; it still counts as a newer version for the ones below it.+ */+ private LifecyclePlan planLifecycle(S3LifecycleConfiguration lifecycle, String key, KeyHistory history,+ Instant now) {+ S3Object current = history.current();+ String currentVersionId = current == null ? null : current.getVersionId();+ List<S3Object> noncurrent = history.versions().stream()+ .filter(version -> current == null || !Objects.equals(version.getVersionId(), currentVersionId))+ .sorted(WRITE_ORDER.reversed())+ .toList();+ List<String> deletions = new ArrayList<>();+ Instant successorCreated = current == null ? null : current.getLastModified();+ for (int newer = 0; newer < noncurrent.size(); newer++) {+ S3Object version = noncurrent.get(newer);+ if (successorCreated != null && !lockedAgainstDeletion(version)+ && lifecycle.expiresNoncurrentVersion(key, version.getTags(), version.getSize(),+ successorCreated, newer, now)) {+ deletions.add(version.getVersionId());+ }+ successorCreated = version.getLastModified();+ }+ if (current == null) {+ return new LifecyclePlan(deletions, false);+ }+ if (current.isDeleteMarker()) {+ if (currentVersionId != null && noncurrent.isEmpty()+ && lifecycle.removesExpiredObjectDeleteMarker(key, current.getLastModified(), now)) {+ deletions.add(currentVersionId);+ }+ return new LifecyclePlan(deletions, false);+ }+ S3LifecycleConfiguration.Expiry expiry = lifecycle.currentVersionExpiry(+ key, current.getTags(), current.getSize(), current.getLastModified());+ return new LifecyclePlan(deletions, expiry != null && !now.isBefore(expiry.date()));+ }++ private boolean lockedAgainstDeletion(S3Object version) {+ try {+ checkLockProtection(version, false);+ return false;+ } catch (AwsException expected) {+ // Lifecycle cannot bypass retention, so a version DeleteObject would refuse is left alone.+ return true;+ }+ }++ /**+ * Expires as DeleteObject would, reported as a lifecycle expiration: a null {@code versionId}+ * expires the current version, a delete marker in a versioning-enabled bucket and a deletion+ * otherwise, and a version id deletes that version permanently.+ */+ private void expireByLifecycle(Bucket bucket, String bucketName, String key, String versionId) {+ try {+ deleteObjectLocked(bucket, bucketName, key, versionId, false, LIFECYCLE_EXPIRATION_EVENT);+ LOG.debugv("Lifecycle expired {0}/{1} v={2}", bucketName, key, versionId);+ } catch (AwsException e) {+ // Object Lock protects the object; lifecycle leaves it in place, as on S3.+ LOG.debugv("Lifecycle left {0}/{1} v={2} in place: {3}", bucketName, key, versionId, e.getMessage());+ }+ }++ /**+ * Aborts the bucket's incomplete multipart uploads that an AbortIncompleteMultipartUpload rule+ * has made due. Each abort holds the upload's lock and then the bucket monitor, the order+ * DeleteBucket takes them in, and is applied only while the bucket is still the one the sweep+ * started on and the rules still make it due, so an upload in a bucket deleted and created again+ * under the same name is never aborted under the old bucket's rules.+ */+ private void abortIncompleteMultipartUploads(Bucket bucket, String bucketName, LifecycleRules rules, Instant now) {+ String ownerAccountId = ownerId();+ for (MultipartUpload upload : List.copyOf(multipartUploads.values())) {+ if (!bucketName.equals(upload.getBucket())+ || (upload.getOwnerAccountId() != null && !ownerAccountId.equals(upload.getOwnerAccountId()))+ || upload.getInitiated() == null+ || !rules.of(bucket).abortsUpload(upload.getKey(), upload.getInitiated(), now)) {+ continue;+ }+ withMultipartOperationLock(bucketName, upload.getUploadId(), () -> {+ synchronized (bucket) {+ if (bucketStore.get(bucketName).orElse(null) == bucket+ && multipartUploads.get(upload.getUploadId()) == upload+ && rules.of(bucket).abortsUpload(upload.getKey(), upload.getInitiated(), now)) {+ cleanupMultipart(upload.getUploadId());+ LOG.debugv("Lifecycle aborted multipart upload {0}/{1}, uploadId={2}",+ bucketName, upload.getKey(), upload.getUploadId());+ }+ }+ });+ }}public String getBucketAcl(String bucketName) {@@ -5807,9 +6132,15 @@ public class S3Service implements Resettable, ResourceProvider {objectNode.put("version-id", obj.getVersionId());}detail.put("request-id", UUID.randomUUID().toString());- detail.put("requester", "aws:emulator");- detail.put("source-ip-address", "127.0.0.1");- detail.put("reason", eventName);+ // A lifecycle deletion is S3's own: "source-ip-address - ... Only present for events+ // triggered by an S3 request."+ if (isLifecycleEvent(eventName)) {+ detail.put("requester", ServicePrincipals.of("s3"));+ } else {+ detail.put("requester", "aws:emulator");+ detail.put("source-ip-address", "127.0.0.1");+ }+ detail.put("reason", eventBridgeReason(eventName));String deletionType = eventBridgeDeletionType(eventName);if (deletionType != null) {detail.put("deletion-type", deletionType);@@ -5820,10 +6151,19 @@ public class S3Service implements Resettable, ResourceProvider {}}+ /** The EventBridge {@code reason}: {@code Lifecycle Expiration} for a deletion by a lifecycle rule. */+ private static String eventBridgeReason(String eventName) {+ return isLifecycleEvent(eventName) ? "Lifecycle Expiration" : eventName;+ }++ private static boolean isLifecycleEvent(String eventName) {+ return eventName.startsWith(LIFECYCLE_EXPIRATION_EVENT + ":");+ }+private static String eventBridgeDeletionType(String eventName) {return switch (eventName) {- case "ObjectRemoved:Delete" -> "Permanently Deleted";- case "ObjectRemoved:DeleteMarkerCreated" -> "Delete Marker Created";+ case "ObjectRemoved:Delete", "LifecycleExpiration:Delete" -> "Permanently Deleted";+ case "ObjectRemoved:DeleteMarkerCreated", "LifecycleExpiration:DeleteMarkerCreated" -> "Delete Marker Created";default -> null;};}@@ -5913,8 +6253,11 @@ public class S3Service implements Resettable, ResourceProvider {record.put("awsRegion", region);record.put("eventTime", eventTime);record.put("eventName", eventName);- record.putObject("userIdentity").put("principalId", "AWS:EMULATOR");- record.putObject("requestParameters").put("sourceIPAddress", "127.0.0.1");+ // The user guide's s3:LifecycleExpiration:Delete example names S3 itself as both.+ boolean lifecycle = isLifecycleEvent(eventName);+ record.putObject("userIdentity").put("principalId", lifecycle ? ServicePrincipals.of("s3") : "AWS:EMULATOR");+ record.putObject("requestParameters")+ .put("sourceIPAddress", lifecycle ? ServicePrincipals.of("s3") : "127.0.0.1");record.putObject("responseElements").put("x-amz-request-id", requestId);record.set("s3", s3Node);@@ -6232,6 +6575,23 @@ public class S3Service implements Resettable, ResourceProvider {* still refers to this object, checked under the bucket lock that puts and deletes hold, so a* concurrent overwrite or delete marker is never replaced by the object it superseded.*/+ /**+ * Replaces the current version's tags under the bucket monitor that writes, deletes and the+ * lifecycle sweep hold, so the object is resolved and rewritten in one step: the sweep never+ * sees the tags it matched a rule against change before it acts.+ */+ private void setCurrentObjectTags(String bucketName, String key, Map<String, String> tags) {+ Bucket bucket = resolveBucket(bucketName)+ .orElseThrow(() -> new AwsException("NoSuchBucket",+ "The specified bucket does not exist.", 404));+ synchronized (bucket) {+ AccountAwareStorageBackend.OwnedEntry<S3Object> ownedObject =+ getStoredObjectEntry(bucketName, key, null);+ ownedObject.value().setTags(tags);+ putObjectMetadataForAccount(ownedObject.account(), bucketName, key, ownedObject.value());+ }+ }+private void putObjectMetadataForAccount(String accountId, String bucketName, String key, S3Object object) {Bucket bucket = resolveBucket(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket",diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex e6c19b2511..9e9f081f21 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -363,6 +363,8 @@ floci:enforce-auth: falsedefault-presign-expiry-seconds: 3600global-bucket-namespace: false+ lifecycle-sweep-enabled: true # FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_ENABLED - apply lifecycle expiration rules in the background+ lifecycle-sweep-interval-seconds: 60 # FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDSdynamodb:enabled: truevector-index-allocation-seconds: 4 # FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_ALLOCATION_SECONDS. Resource allocation phase of a vector index added by UpdateTable.diff --git a/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java b/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.javaindex b4c626ecc3..d19f325e23 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java@@ -201,6 +201,34 @@ class XmlParserTest {assertNull(XmlParser.extractElementTree(null, "Target"));}+ // --- parseDocumentElementTree: the whole document or nothing ---++ @Test+ void parseDocumentElementTreeReturnsTheDocumentElement() {+ XmlParser.XmlElement root = XmlParser.parseDocumentElementTree("""+ <?xml version="1.0" encoding="UTF-8"?>+ <Root xmlns="urn:test"><Item><Name>a</Name></Item></Root>+ <!-- a trailing comment is well-formed -->+ """);++ assertEquals("Root", root.name());+ assertEquals("a", root.child("Item").child("Name").text());+ }++ @Test+ void parseDocumentElementTreeRejectsMalformedContentAfterTheDocumentElement() {+ // extractElementTree stops at the end of the element it found; this keeps reading.+ String trailingElement = "<Root><Item>a</Item></Root><Item>b</Item>";+ String trailingText = "<Root><Item>a</Item></Root>junk";++ assertNotNull(XmlParser.extractElementTree(trailingElement, "Root"));+ assertNull(XmlParser.parseDocumentElementTree(trailingElement));+ assertNull(XmlParser.parseDocumentElementTree(trailingText));+ assertNull(XmlParser.parseDocumentElementTree("<Root><Item></Root>"));+ assertNull(XmlParser.parseDocumentElementTree(""));+ assertNull(XmlParser.parseDocumentElementTree(null));+ }+// --- extractDeleteObjectEntries: batch delete Key/VersionId pairing ---@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.javaindex 8674e12d5b..4f2b9a9368 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.s3;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;@@ -22,6 +23,9 @@ class S3EventBridgeIntegrationTest {private static String ruleArn;private static String queueUrl;+ @Inject+ S3LifecycleSweeper lifecycleSweeper;+@BeforeAllstatic void configureRestAssured() {RestAssuredJsonUtils.configureAwsContentTypes();@@ -317,6 +321,55 @@ class S3EventBridgeIntegrationTest {containsString("\"deletion-type\":\"Delete Marker Created\"")));}+ @Test+ @Order(11)+ void lifecycleExpiration_eventBridgeDetailSaysLifecycleExpirationFromS3Itself() {+ given().contentType("text/plain").body("short-lived").when().put("/" + VERSIONED_BUCKET + "/expiring.txt")+ .then().statusCode(200);+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .formParam("WaitTimeSeconds", "0")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body", containsString("Object Created"));+ given()+ .contentType("application/xml")+ .body("<LifecycleConfiguration><Rule><ID>past</ID><Filter><Prefix>expiring</Prefix></Filter>"+ + "<Status>Enabled</Status><Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"+ + "</Rule></LifecycleConfiguration>")+ .when()+ .put("/" + VERSIONED_BUCKET + "?lifecycle")+ .then()+ .statusCode(200);++ lifecycleSweeper.sweep();++ // The user guide's lifecycle event: reason "Lifecycle Expiration", requester s3.amazonaws.com,+ // and no source-ip-address, which is "Only present for events triggered by an S3 request".+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .formParam("WaitTimeSeconds", "0")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body", allOf(+ containsString("expiring.txt"),+ containsString("\"reason\":\"Lifecycle Expiration\""),+ containsString("\"deletion-type\":\"Delete Marker Created\""),+ containsString("\"requester\":\"s3.amazonaws.com\""),+ not(containsString("source-ip-address"))));+ given().when().delete("/" + VERSIONED_BUCKET + "?lifecycle").then().statusCode(204);+ }+@Test@Order(100)void cleanup() {diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfigurationTest.javanew file mode 100644index 0000000000..1ddd784f44--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfigurationTest.java@@ -0,0 +1,208 @@+package io.github.hectorvent.floci.services.s3;++import org.junit.jupiter.api.Test;++import java.time.Instant;+import java.util.List;+import java.util.Map;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.empty;+import static org.hamcrest.Matchers.hasSize;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++class S3LifecycleConfigurationTest {++ private static final Instant NOW = Instant.parse("2026-10-06T12:00:00Z");++ @Test+ void dayCountsRoundUpToTheNextMidnightUtc() {+ // The user guide's examples: created 1/15/2014 10:30 UTC plus 3 days is 1/19/2014 00:00 UTC.+ assertEquals(Instant.parse("2014-01-19T00:00:00Z"),+ S3LifecycleConfiguration.afterDays(Instant.parse("2014-01-15T10:30:00Z"), 3));+ assertEquals(Instant.parse("2014-01-17T00:00:00Z"),+ S3LifecycleConfiguration.afterDays(Instant.parse("2014-01-15T23:59:59.999Z"), 1));+ }++ @Test+ void disabledRulesAndMalformedBodiesHaveNoRules() {+ assertThat(parse(rule("off", "Disabled", "<Filter><Prefix></Prefix></Filter>",+ "<Expiration><Days>1</Days></Expiration>")).rules(), empty());+ assertThat(S3LifecycleConfiguration.parse("<LifecycleConfiguration><Rule>").rules(), empty());+ assertThat(S3LifecycleConfiguration.parse("not xml").rules(), empty());+ }++ @Test+ void theLegacyRulePrefixIsTheFilterWhenNoFilterIsGiven() {+ S3LifecycleConfiguration config = S3LifecycleConfiguration.parse("<LifecycleConfiguration><Rule>"+ + "<ID>legacy</ID><Prefix>logs/</Prefix><Status>Enabled</Status>"+ + "<Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>");++ assertTrue(config.rules().getFirst().filter().matches("logs/a", Map.of(), 1));+ assertFalse(config.rules().getFirst().filter().matches("other/a", Map.of(), 1));+ }++ @Test+ void anAndFilterNeedsThePrefixEveryTagAndTheExclusiveSizeRange() {+ S3LifecycleConfiguration.Filter filter = parse(rule("and", "Enabled", "<Filter><And>"+ + "<Prefix>tmp/</Prefix>"+ + "<Tag><Key>env</Key><Value>dev</Value></Tag>"+ + "<Tag><Key>flag</Key><Value></Value></Tag>"+ + "<ObjectSizeGreaterThan>10</ObjectSizeGreaterThan>"+ + "<ObjectSizeLessThan>20</ObjectSizeLessThan>"+ + "</And></Filter>", "<Expiration><Days>1</Days></Expiration>")).rules().getFirst().filter();+ Map<String, String> tags = Map.of("env", "dev", "flag", "", "extra", "x");++ assertTrue(filter.matches("tmp/a", tags, 15));+ assertFalse(filter.matches("keep/a", tags, 15));+ assertFalse(filter.matches("tmp/a", Map.of("env", "dev", "flag", ""), 10));+ assertFalse(filter.matches("tmp/a", Map.of("env", "dev", "flag", ""), 20));+ assertFalse(filter.matches("tmp/a", Map.of("env", "prod", "flag", ""), 15));+ assertFalse(filter.matches("tmp/a", Map.of("env", "dev"), 15));+ assertFalse(filter.matches("tmp/a", Map.of("env", "dev", "flag", "set"), 15));+ }++ @Test+ void anObjectGetsTheEarliestExpiryOfTheRulesThatMatchIt() {+ S3LifecycleConfiguration config = parse(+ rule("ten days", "Enabled", "<Filter><Prefix></Prefix></Filter>",+ "<Expiration><Days>10</Days></Expiration>")+ + rule("by date", "Enabled", "<Filter><Prefix>tmp/</Prefix></Filter>",+ "<Expiration><Date>2026-10-08T00:00:00.000Z</Date></Expiration>")+ + rule("transition only", "Enabled", "<Filter><Prefix></Prefix></Filter>",+ "<Transition><Days>1</Days><StorageClass>GLACIER</StorageClass></Transition>"));++ S3LifecycleConfiguration.Expiry tmp = config.currentVersionExpiry("tmp/a", Map.of(), 1, NOW);+ assertEquals(Instant.parse("2026-10-08T00:00:00Z"), tmp.date());+ assertEquals("expiry-date=\"Thu, 08 Oct 2026 00:00:00 GMT\", rule-id=\"by%20date\"", tmp.header());++ S3LifecycleConfiguration.Expiry other = config.currentVersionExpiry("other/a", Map.of(), 1, NOW);+ assertEquals(Instant.parse("2026-10-17T00:00:00Z"), other.date());+ assertEquals("ten days", other.ruleId());+ }++ @Test+ void anExpirationDateMayBeWrittenAsAPlainDate() {+ S3LifecycleConfiguration config = parse(rule("plain", "Enabled", "<Filter></Filter>",+ "<Expiration><Date>2020-01-01</Date></Expiration>"));++ assertEquals(Instant.parse("2020-01-01T00:00:00Z"),+ config.currentVersionExpiry("a", Map.of(), 1, NOW).date());+ }++ @Test+ void aRuleS3WouldRefuseIsDroppedWholeAndTheOtherRulesStillApply() {+ String good = rule("good", "Enabled", "<Filter></Filter>", "<Expiration><Days>1</Days></Expiration>");+ List<String> refused = List.of(+ rule("days soon", "Enabled", "<Filter></Filter>", "<Expiration><Days>soon</Days></Expiration>"),+ rule("days zero", "Enabled", "<Filter></Filter>", "<Expiration><Days>0</Days></Expiration>"),+ rule("bad date", "Enabled", "<Filter></Filter>", "<Expiration><Date>someday</Date></Expiration>"),+ rule("marker and days", "Enabled", "<Filter></Filter>", "<Expiration><Days>1</Days>"+ + "<ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"),+ rule("marker maybe", "Enabled", "<Filter></Filter>",+ "<Expiration><ExpiredObjectDeleteMarker>maybe</ExpiredObjectDeleteMarker></Expiration>"),+ rule("size word", "Enabled", "<Filter><ObjectSizeGreaterThan>big</ObjectSizeGreaterThan></Filter>",+ "<Expiration><Days>1</Days></Expiration>"),+ rule("size negative", "Enabled", "<Filter><ObjectSizeLessThan>-1</ObjectSizeLessThan></Filter>",+ "<Expiration><Days>1</Days></Expiration>"),+ rule("tag without value", "Enabled", "<Filter><Tag><Key>env</Key></Tag></Filter>",+ "<Expiration><Days>1</Days></Expiration>"),+ rule("tag twice", "Enabled", "<Filter><And><Tag><Key>env</Key><Value>a</Value></Tag>"+ + "<Tag><Key>env</Key><Value>b</Value></Tag></And></Filter>",+ "<Expiration><Days>1</Days></Expiration>"),+ "<Rule><ID>no filter</ID><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule>",+ rule("noncurrent zero", "Enabled", "<Filter></Filter>",+ "<NoncurrentVersionExpiration><NoncurrentDays>0</NoncurrentDays></NoncurrentVersionExpiration>"),+ rule("retain too many", "Enabled", "<Filter></Filter>", "<NoncurrentVersionExpiration>"+ + "<NoncurrentDays>1</NoncurrentDays><NewerNoncurrentVersions>101</NewerNoncurrentVersions>"+ + "</NoncurrentVersionExpiration>"),+ "<Rule><ID>retain without filter</ID><Prefix></Prefix><Status>Enabled</Status>"+ + "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays>"+ + "<NewerNoncurrentVersions>1</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule>",+ rule("abort negative", "Enabled", "<Filter></Filter>", "<AbortIncompleteMultipartUpload>"+ + "<DaysAfterInitiation>-1</DaysAfterInitiation></AbortIncompleteMultipartUpload>"),+ rule("status typo", "enabled", "<Filter></Filter>", "<Expiration><Days>1</Days></Expiration>"));++ for (String bad : refused) {+ List<S3LifecycleConfiguration.Rule> rules = parse(bad + good).rules();+ assertEquals(List.of("good"), rules.stream().map(S3LifecycleConfiguration.Rule::id).toList(), bad);+ }+ }++ @Test+ void aBodyWithMalformedContentAfterTheConfigurationHasNoRules() {+ String rules = rule("past", "Enabled", "<Filter></Filter>",+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>");++ assertThat(S3LifecycleConfiguration.parse(+ "<LifecycleConfiguration>" + rules + "</LifecycleConfiguration><Rule>").rules(), empty());+ assertThat(S3LifecycleConfiguration.parse(+ "<LifecycleConfiguration>" + rules + "</LifecycleConfiguration>trailing").rules(), empty());+ assertThat(S3LifecycleConfiguration.parse("<Other>" + rules + "</Other>").rules(), empty());+ }++ @Test+ void anExpiredObjectDeleteMarkerGoesAtOnceOrOnceOldEnoughForADaysRule() {+ S3LifecycleConfiguration explicit = parse(rule("markers", "Enabled", "<Filter></Filter>",+ "<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"));+ S3LifecycleConfiguration days = parse(rule("days", "Enabled", "<Filter></Filter>",+ "<Expiration><Days>2</Days></Expiration>"));++ assertTrue(explicit.removesExpiredObjectDeleteMarker("a", NOW, NOW));+ assertNull(explicit.currentVersionExpiry("a", Map.of(), 1, NOW));+ assertFalse(days.removesExpiredObjectDeleteMarker("a", NOW, Instant.parse("2026-10-08T23:59:59Z")));+ assertTrue(days.removesExpiredObjectDeleteMarker("a", NOW, Instant.parse("2026-10-09T00:00:00Z")));+ }++ @Test+ void aNoncurrentVersionNeedsBothItsAgeAndEnoughNewerVersionsRetained() {+ S3LifecycleConfiguration config = parse(rule("noncurrent", "Enabled", "<Filter></Filter>",+ "<NoncurrentVersionExpiration><NoncurrentDays>3</NoncurrentDays>"+ + "<NewerNoncurrentVersions>2</NewerNoncurrentVersions></NoncurrentVersionExpiration>"));+ Instant becameNoncurrent = Instant.parse("2014-01-15T10:30:00Z");+ Instant due = Instant.parse("2014-01-19T00:00:00Z");++ assertTrue(config.expiresNoncurrentVersion("a", Map.of(), 1, becameNoncurrent, 2, due));+ assertFalse(config.expiresNoncurrentVersion("a", Map.of(), 1, becameNoncurrent, 1, due));+ assertFalse(config.expiresNoncurrentVersion("a", Map.of(), 1, becameNoncurrent, 2,+ due.minusMillis(1)));+ }++ @Test+ void onlyAPrefixFilterSelectsMultipartUploadsToAbort() {+ S3LifecycleConfiguration prefix = parse(rule("abort", "Enabled", "<Filter><Prefix>big/</Prefix></Filter>",+ "<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"+ + "</AbortIncompleteMultipartUpload>"));+ S3LifecycleConfiguration tagged = parse(rule("abort", "Enabled",+ "<Filter><Tag><Key>k</Key><Value>v</Value></Tag></Filter>",+ "<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"+ + "</AbortIncompleteMultipartUpload>"));+ Instant due = Instant.parse("2026-10-08T00:00:00Z");++ assertTrue(prefix.abortsUpload("big/a", NOW, due));+ assertFalse(prefix.abortsUpload("big/a", NOW, due.minusMillis(1)));+ assertFalse(prefix.abortsUpload("small/a", NOW, due));+ assertFalse(tagged.abortsUpload("big/a", NOW, due));+ }++ @Test+ void everyEnabledRuleIsRead() {+ assertThat(parse(rule("a", "Enabled", "<Filter></Filter>", "<Expiration><Days>1</Days></Expiration>")+ + rule("b", "Enabled", "<Filter></Filter>", "<Expiration><Days>2</Days></Expiration>")+ + rule("c", "Disabled", "<Filter></Filter>", "<Expiration><Days>3</Days></Expiration>")).rules(),+ hasSize(2));+ }++ private static S3LifecycleConfiguration parse(String rules) {+ return S3LifecycleConfiguration.parse(+ "<LifecycleConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"+ + rules + "</LifecycleConfiguration>");+ }++ private static String rule(String id, String status, String filter, String actions) {+ return "<Rule><ID>" + id + "</ID>" + filter + "<Status>" + status + "</Status>" + actions + "</Rule>";+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleExpirationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleExpirationIntegrationTest.javanew file mode 100644index 0000000000..96f15e736c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleExpirationIntegrationTest.java@@ -0,0 +1,155 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;++import java.time.Instant;+import java.time.ZonedDateTime;+import java.time.format.DateTimeFormatter;+import java.time.temporal.ChronoUnit;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.nullValue;++/**+ * Lifecycle expiration as a client sees it: the {@code x-amz-expiration} header on objects a rule+ * matches, and the sweep removing what has expired. The test profile does not schedule the sweep,+ * so each test runs it once, at the point it expects the rules applied.+ */+@QuarkusTest+class S3LifecycleExpirationIntegrationTest {++ @Inject+ S3LifecycleSweeper sweeper;++ @Test+ void objectsMatchedByADaysRuleCarryTheRoundedExpiryDateAndRuleId() {+ String bucket = "lc-expiration-header";+ given().put("/" + bucket).then().statusCode(200);+ putLifecycle(bucket, rule("three days", "<Filter><Prefix>tmp/</Prefix></Filter>",+ "<Expiration><Days>3</Days></Expiration>"));++ Response put = given().body("hi").put("/" + bucket + "/tmp/a.txt");+ put.then().statusCode(200);+ Response head = given().head("/" + bucket + "/tmp/a.txt");+ head.then().statusCode(200);++ // Creation time plus three days, rounded up to the next midnight UTC.+ Instant created = ZonedDateTime.parse(head.header("Last-Modified"),+ DateTimeFormatter.RFC_1123_DATE_TIME).toInstant();+ String expected = "expiry-date=\"" + S3Controller.RFC_822.format(+ created.plus(3, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).plus(1, ChronoUnit.DAYS))+ + "\", rule-id=\"three%20days\"";+ head.then().header("x-amz-expiration", equalTo(expected));+ put.then().header("x-amz-expiration", equalTo(expected));+ given().get("/" + bucket + "/tmp/a.txt").then()+ .statusCode(200)+ .header("x-amz-expiration", equalTo(expected));++ given().body("hi").put("/" + bucket + "/keep/b.txt").then()+ .statusCode(200)+ .header("x-amz-expiration", nullValue());+ given().head("/" + bucket + "/keep/b.txt").then()+ .statusCode(200)+ .header("x-amz-expiration", nullValue());+ }++ @Test+ void aDateRuleReportsThatDateAsTheExpiryDate() {+ String bucket = "lc-expiration-date-header";+ given().put("/" + bucket).then().statusCode(200);+ putLifecycle(bucket, rule("future-date", "<Filter><Prefix></Prefix></Filter>",+ "<Expiration><Date>2099-01-01T00:00:00Z</Date></Expiration>"));+ given().body("hi").put("/" + bucket + "/a.txt").then().statusCode(200);++ given().head("/" + bucket + "/a.txt").then()+ .statusCode(200)+ .header("x-amz-expiration",+ equalTo("expiry-date=\"Thu, 01 Jan 2099 00:00:00 GMT\", rule-id=\"future-date\""));+ }++ @Test+ void theSweepDeletesObjectsPastTheirExpirationAndLeavesOthers() {+ String bucket = "lc-expiration-sweep";+ given().put("/" + bucket).then().statusCode(200);+ given().body("hi").put("/" + bucket + "/tmp/a.txt").then().statusCode(200);+ given().body("hi").put("/" + bucket + "/keep/b.txt").then().statusCode(200);+ putLifecycle(bucket, rule("past-date", "<Filter><Prefix>tmp/</Prefix></Filter>",+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));++ sweeper.sweep();++ given().head("/" + bucket + "/tmp/a.txt").then().statusCode(404);+ given().get("/" + bucket + "?list-type=2").then()+ .statusCode(200)+ .body(not(containsString("<Key>tmp/a.txt</Key>")))+ .body(containsString("<Key>keep/b.txt</Key>"));+ }++ @Test+ void theSweepPlacesADeleteMarkerInAVersionedBucket() {+ String bucket = "lc-expiration-versioned";+ given().put("/" + bucket).then().statusCode(200);+ given().body("<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>")+ .put("/" + bucket + "?versioning").then().statusCode(200);+ String versionId = given().body("hi").put("/" + bucket + "/tmp/a.txt")+ .then().statusCode(200).extract().header("x-amz-version-id");+ putLifecycle(bucket, rule("past-date", "<Filter><Prefix>tmp/</Prefix></Filter>",+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));++ sweeper.sweep();++ given().head("/" + bucket + "/tmp/a.txt").then().statusCode(404);+ given().get("/" + bucket + "?versions").then()+ .statusCode(200)+ .body(containsString("<DeleteMarker>"))+ .body(containsString("<VersionId>" + versionId + "</VersionId>"));+ given().get("/" + bucket + "/tmp/a.txt?versionId=" + versionId).then().statusCode(200);+ }++ @Test+ void theSweepAppliesEachAccountsRulesToThatAccountsBucket() {+ String bucket = "lc-expiration-accounts";+ String withRule = "AWS4-HMAC-SHA256 Credential=000000000002/20260215/us-east-1/s3/aws4_request, "+ + "SignedHeaders=host, Signature=abc";+ String withoutRule = "AWS4-HMAC-SHA256 Credential=000000000003/20260215/us-east-1/s3/aws4_request, "+ + "SignedHeaders=host, Signature=abc";+ for (String auth : new String[] {withRule, withoutRule}) {+ given().header("Authorization", auth).put("/" + bucket).then().statusCode(200);+ given().header("Authorization", auth).body("hi").put("/" + bucket + "/a.txt").then().statusCode(200);+ }+ given()+ .header("Authorization", withRule)+ .contentType("application/xml")+ .body("<LifecycleConfiguration>" + rule("past-date", "<Filter><Prefix></Prefix></Filter>",+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>") + "</LifecycleConfiguration>")+ .when()+ .put("/" + bucket + "?lifecycle")+ .then()+ .statusCode(200);++ sweeper.sweep();++ given().header("Authorization", withRule).head("/" + bucket + "/a.txt").then().statusCode(404);+ given().header("Authorization", withoutRule).head("/" + bucket + "/a.txt").then().statusCode(200);+ }++ private static String rule(String id, String filter, String actions) {+ return "<Rule><ID>" + id + "</ID>" + filter + "<Status>Enabled</Status>" + actions + "</Rule>";+ }++ private static void putLifecycle(String bucket, String rules) {+ given()+ .contentType("application/xml")+ .body("<LifecycleConfiguration>" + rules + "</LifecycleConfiguration>")+ .when()+ .put("/" + bucket + "?lifecycle")+ .then()+ .statusCode(200);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleServiceTest.javanew file mode 100644index 0000000000..2c63f2acb5--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleServiceTest.java@@ -0,0 +1,348 @@+package io.github.hectorvent.floci.services.s3;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.s3.model.LambdaNotification;+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;+import io.github.hectorvent.floci.services.s3.model.NotificationConfiguration;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Path;+import java.time.Duration;+import java.time.Instant;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.contains;+import static org.hamcrest.Matchers.containsInAnyOrder;+import static org.hamcrest.Matchers.empty;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * The lifecycle sweep, driven directly with a chosen "now" instead of waiting on the scheduler.+ */+class S3LifecycleServiceTest {++ private static final String BUCKET = "lifecycle-bucket";++ @TempDir+ Path tempDir;++ private RecordingLambdaInvoker notifications;+ private S3Service s3Service;++ @BeforeEach+ void setUp() {+ notifications = new RecordingLambdaInvoker();+ s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve("s3"), false,+ notifications, new RegionResolver("us-east-1", "000000000000"));+ s3Service.createBucket(BUCKET, "us-east-1");+ }++ @Test+ void aDaysRuleDeletesMatchingObjectsOnlyOnceTheRoundedExpiryHasPassed() {+ put("tmp/a.txt");+ put("keep/b.txt");+ lifecycle(rule("Enabled", "<Filter><Prefix>tmp/</Prefix></Filter>", "<Expiration><Days>1</Days></Expiration>"));+ Instant created = s3Service.headObject(BUCKET, "tmp/a.txt").getLastModified();+ Instant due = S3LifecycleConfiguration.afterDays(created, 1);++ s3Service.applyLifecycleExpiration(due.minusMillis(1));+ assertTrue(s3Service.objectExists(BUCKET, "tmp/a.txt"));++ s3Service.applyLifecycleExpiration(due);+ assertFalse(s3Service.objectExists(BUCKET, "tmp/a.txt"));+ assertTrue(s3Service.objectExists(BUCKET, "keep/b.txt"));+ }++ @Test+ void aDisabledRuleIsNotApplied() {+ put("a.txt");+ lifecycle(rule("Disabled", "<Filter></Filter>", "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));++ s3Service.applyLifecycleExpiration(Instant.now());++ assertTrue(s3Service.objectExists(BUCKET, "a.txt"));+ }++ @Test+ void aTagFilterExpiresOnlyObjectsCarryingTheTag() {+ put("tagged.txt");+ put("untagged.txt");+ s3Service.putObjectTagging(BUCKET, "tagged.txt", Map.of("env", "ephemeral"));+ lifecycle(rule("Enabled", "<Filter><Tag><Key>env</Key><Value>ephemeral</Value></Tag></Filter>",+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));++ s3Service.applyLifecycleExpiration(Instant.now());++ assertFalse(s3Service.objectExists(BUCKET, "tagged.txt"));+ assertTrue(s3Service.objectExists(BUCKET, "untagged.txt"));+ }++ @Test+ void inAVersionedBucketTheCurrentVersionGetsADeleteMarkerOnce() {+ s3Service.putBucketVersioning(BUCKET, "Enabled");+ S3Object original = put("a.txt");+ lifecycle(rule("Enabled", "<Filter></Filter>", "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));++ s3Service.applyLifecycleExpiration(Instant.now());+ s3Service.applyLifecycleExpiration(Instant.now());++ List<S3Object> versions = versions("a.txt");+ assertEquals(2, versions.size(), "the original version and one delete marker");+ assertTrue(versions.getFirst().isDeleteMarker());+ assertEquals(original.getVersionId(), versions.get(1).getVersionId());+ assertFalse(s3Service.objectExists(BUCKET, "a.txt"));+ }++ @Test+ void noncurrentVersionsBeyondTheRetainedCountExpireOnceOldEnough() {+ s3Service.putBucketVersioning(BUCKET, "Enabled");+ S3Object v1 = put("a.txt");+ S3Object v2 = put("a.txt");+ S3Object v3 = put("a.txt");+ lifecycle(rule("Enabled", "<Filter></Filter>", "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays>"+ + "<NewerNoncurrentVersions>1</NewerNoncurrentVersions></NoncurrentVersionExpiration>"));++ s3Service.applyLifecycleExpiration(Instant.now());+ assertThat(versionIds("a.txt"), contains(v3.getVersionId(), v2.getVersionId(), v1.getVersionId()));++ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(3)));+ assertThat(versionIds("a.txt"), contains(v3.getVersionId(), v2.getVersionId()));+ }++ @Test+ void aNoncurrentVersionUnderALegalHoldIsKept() {+ s3Service.putBucketVersioning(BUCKET, "Enabled");+ S3Object held = s3Service.putObject(BUCKET, "a.txt", "held".getBytes(StandardCharsets.UTF_8), "text/plain",+ Map.of(), null, null, "ON");+ S3Object current = put("a.txt");+ lifecycle(rule("Enabled", "<Filter></Filter>",+ "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>"));++ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(3)));++ assertThat(versionIds("a.txt"), containsInAnyOrder(current.getVersionId(), held.getVersionId()));+ }++ @Test+ void anExpiredObjectDeleteMarkerIsRemoved() {+ s3Service.putBucketVersioning(BUCKET, "Enabled");+ S3Object only = put("a.txt");+ s3Service.deleteObject(BUCKET, "a.txt");+ s3Service.deleteObject(BUCKET, "a.txt", only.getVersionId());+ assertEquals(1, versions("a.txt").size(), "only the delete marker remains");+ lifecycle(rule("Enabled", "<Filter></Filter>",+ "<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"));++ s3Service.applyLifecycleExpiration(Instant.now());++ assertThat(versions("a.txt"), empty());+ }++ @Test+ void aDeleteMarkerWithNoncurrentVersionsIsKept() {+ s3Service.putBucketVersioning(BUCKET, "Enabled");+ put("a.txt");+ s3Service.deleteObject(BUCKET, "a.txt");+ lifecycle(rule("Enabled", "<Filter></Filter>",+ "<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"));++ s3Service.applyLifecycleExpiration(Instant.now());++ assertEquals(2, versions("a.txt").size());+ }++ @Test+ void incompleteMultipartUploadsAreAbortedAfterTheirDays() {+ MultipartUpload upload = s3Service.initiateMultipartUpload(BUCKET, "big/a.bin", "application/octet-stream");+ lifecycle(rule("Enabled", "<Filter><Prefix>big/</Prefix></Filter>",+ "<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"+ + "</AbortIncompleteMultipartUpload>"));+ Instant due = S3LifecycleConfiguration.afterDays(upload.getInitiated(), 1);++ s3Service.applyLifecycleExpiration(due.minusMillis(1));+ assertEquals(1, s3Service.listMultipartUploads(BUCKET).size());++ s3Service.applyLifecycleExpiration(due);+ assertThat(s3Service.listMultipartUploads(BUCKET), empty());+ }++ @Test+ void lifecycleDeletionsAreReportedAsLifecycleExpirationEvents() throws IOException {+ put("a.txt");+ NotificationConfiguration config = new NotificationConfiguration();+ config.getLambdaFunctionConfigurations().add(new LambdaNotification("removed",+ "arn:aws:lambda:us-east-1:000000000000:function:removed", List.of("s3:ObjectRemoved:*"), List.of()));+ config.getLambdaFunctionConfigurations().add(new LambdaNotification("lifecycle",+ "arn:aws:lambda:us-east-1:000000000000:function:lifecycle",+ List.of("s3:LifecycleExpiration:*"), List.of()));+ s3Service.putBucketNotificationConfiguration(BUCKET, config, true);+ lifecycle(rule("Enabled", "<Filter></Filter>", "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));++ s3Service.applyLifecycleExpiration(Instant.now());++ assertEquals(List.of("lifecycle"), notifications.functionNames);+ JsonNode record = new ObjectMapper().readTree(notifications.payloads.getFirst()).path("Records").get(0);+ assertEquals("LifecycleExpiration:Delete", record.path("eventName").asText());+ // The user guide's example names S3 itself, not a requester.+ assertEquals("s3.amazonaws.com", record.path("userIdentity").path("principalId").asText());+ assertEquals("s3.amazonaws.com", record.path("requestParameters").path("sourceIPAddress").asText());+ }++ @Test+ void versionsWrittenInTheSameMillisecondKeepTheOrderTheyWereWrittenIn() {+ s3Service.putBucketVersioning(BUCKET, "Enabled");+ S3Object v1 = put("a.txt");+ S3Object v2 = put("a.txt");+ S3Object v3 = put("a.txt");+ Instant sameMillisecond = Instant.parse("2026-10-01T10:00:00Z");+ for (S3Object version : List.of(v1, v2, v3)) {+ version.setLastModified(sameMillisecond);+ }+ lifecycle(rule("Enabled", "<Filter></Filter>", "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays>"+ + "<NewerNoncurrentVersions>1</NewerNoncurrentVersions></NoncurrentVersionExpiration>"));++ assertThat(versionIds("a.txt"), contains(v3.getVersionId(), v2.getVersionId(), v1.getVersionId()));+ s3Service.applyLifecycleExpiration(sameMillisecond.plus(Duration.ofDays(3)));++ assertThat(versionIds("a.txt"), contains(v3.getVersionId(), v2.getVersionId()));+ }++ @Test+ void aKeyLeftWithOnlyVersionsInASuspendedBucketHasItsOlderVersionsExpired() {+ s3Service.putBucketVersioning(BUCKET, "Enabled");+ S3Object v1 = put("a.txt");+ S3Object v2 = put("a.txt");+ s3Service.putBucketVersioning(BUCKET, "Suspended");+ s3Service.deleteObject(BUCKET, "a.txt");+ assertThat(versionIds("a.txt"), contains(v2.getVersionId(), v1.getVersionId()));+ lifecycle(rule("Enabled", "<Filter></Filter>",+ "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>"));++ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(3)));++ // v1 has been noncurrent since v2 was written. When v2 stopped being current is not recorded.+ assertThat(versionIds("a.txt"), contains(v2.getVersionId()));+ }++ @Test+ void aRuleWithAValueS3WouldRefuseExpiresNothing() {+ put("small.txt");+ lifecycle(rule("Enabled", "<Filter><ObjectSizeGreaterThan>large</ObjectSizeGreaterThan></Filter>",+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>")+ + rule("Enabled", "<Filter></Filter>", "<Expiration><Days>0</Days></Expiration>"));++ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(3)));++ assertTrue(s3Service.objectExists(BUCKET, "small.txt"));+ assertNull(s3Service.expirationHeader(s3Service.headObject(BUCKET, "small.txt")));+ }++ @Test+ void aBucketThatFailsDoesNotStopTheOthersAndTheRunReportsIt() {+ S3Service failing = new S3Service(new InMemoryStorage<>(), new FailingDeletes("broken/"),+ tempDir.resolve("failing"), false, notifications, new RegionResolver("us-east-1", "000000000000"));+ for (String bucket : List.of("broken", "healthy")) {+ failing.createBucket(bucket, "us-east-1");+ failing.putObject(bucket, "a.txt", "data".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ failing.putBucketLifecycle(bucket, "<LifecycleConfiguration>" + rule("Enabled", "<Filter></Filter>",+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>") + "</LifecycleConfiguration>", null);+ }++ IllegalStateException failure = assertThrows(IllegalStateException.class,+ () -> failing.applyLifecycleExpiration(Instant.now()));++ assertTrue(failure.getMessage().contains("[broken]"), failure.getMessage());+ assertEquals("disk failure", failure.getSuppressed()[0].getMessage());+ assertFalse(failing.objectExists("healthy", "a.txt"));+ assertTrue(failing.objectExists("broken", "a.txt"));+ }++ @Test+ void theExpirationHeaderDescribesOnlyTheCurrentVersion() {+ s3Service.putBucketVersioning(BUCKET, "Enabled");+ S3Object older = put("a.txt");+ put("a.txt");+ lifecycle(rule("Enabled", "<Filter></Filter>", "<Expiration><Date>2099-01-01T00:00:00Z</Date></Expiration>"));++ assertEquals("expiry-date=\"Thu, 01 Jan 2099 00:00:00 GMT\", rule-id=\"rule\"",+ s3Service.expirationHeader(s3Service.headObject(BUCKET, "a.txt")));+ assertNull(s3Service.expirationHeader(s3Service.headObject(BUCKET, "a.txt", older.getVersionId())));+ }++ @Test+ void aBucketWithoutALifecycleConfigurationIsLeftAlone() {+ put("a.txt");++ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(10_000)));++ assertTrue(s3Service.objectExists(BUCKET, "a.txt"));+ assertNull(s3Service.expirationHeader(s3Service.headObject(BUCKET, "a.txt")));+ assertNotNull(s3Service.headObject(BUCKET, "a.txt"));+ }++ private S3Object put(String key) {+ return s3Service.putObject(BUCKET, key, "data".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ }++ private void lifecycle(String rules) {+ s3Service.putBucketLifecycle(BUCKET, "<LifecycleConfiguration>" + rules + "</LifecycleConfiguration>", null);+ }++ private static String rule(String status, String filter, String actions) {+ return "<Rule><ID>rule</ID>" + filter + "<Status>" + status + "</Status>" + actions + "</Rule>";+ }++ /** Newest first. */+ private List<S3Object> versions(String key) {+ return s3Service.listObjectVersions(BUCKET, key, 1000, null).versions();+ }++ private List<String> versionIds(String key) {+ return versions(key).stream().map(S3Object::getVersionId).toList();+ }++ /** An object store whose deletes fail under one bucket, as a disk error would. */+ private static final class FailingDeletes extends InMemoryStorage<String, S3Object> {+ private final String failingPrefix;++ FailingDeletes(String failingPrefix) {+ this.failingPrefix = failingPrefix;+ }++ @Override+ public void delete(String key) {+ if (key.startsWith(failingPrefix)) {+ throw new IllegalStateException("disk failure");+ }+ super.delete(key);+ }+ }++ private static final class RecordingLambdaInvoker implements S3Service.LambdaInvoker {+ private final List<String> functionNames = new ArrayList<>();+ private final List<byte[]> payloads = new ArrayList<>();++ @Override+ public void invoke(String region, String functionName, byte[] payload, InvocationType type) {+ functionNames.add(functionName);+ payloads.add(payload);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeperTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeperTest.javanew file mode 100644index 0000000000..41283801a1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeperTest.java@@ -0,0 +1,61 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.config.EmulatorConfig.S3ServiceConfig;+import io.github.hectorvent.floci.config.EmulatorConfig.ServicesConfig;+import io.quarkus.runtime.ShutdownEvent;+import io.quarkus.runtime.StartupEvent;+import org.junit.jupiter.api.Test;++import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.mockito.Mockito.doReturn;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;++class S3LifecycleSweeperTest {++ private static final Clock CLOCK = Clock.fixed(Instant.parse("2026-10-06T12:00:00Z"), ZoneOffset.UTC);++ @Test+ void aFailingSweepDoesNotEscapeTheScheduledTask() {+ S3Service s3Service = mock(S3Service.class);+ doThrow(new IllegalStateException("sweep failed")).when(s3Service).applyLifecycleExpiration(CLOCK.instant());+ S3LifecycleSweeper sweeper = new S3LifecycleSweeper(s3Service, configWithInterval(60L), CLOCK);++ assertDoesNotThrow(sweeper::sweep);+ verify(s3Service).applyLifecycleExpiration(CLOCK.instant());+ }++ @Test+ void nonPositiveConfiguredInterval_doesNotPreventStartup() {+ for (long interval : new long[] {0L, -1L, Long.MIN_VALUE}) {+ S3LifecycleSweeper sweeper = new S3LifecycleSweeper(null, configWithInterval(interval), CLOCK);+ assertDoesNotThrow(() -> sweeper.onStart(new StartupEvent()),+ "interval " + interval + " must not abort startup");+ sweeper.onStop(new ShutdownEvent());+ }+ }++ private static EmulatorConfig configWithInterval(long intervalSeconds) {+ EmulatorConfig config = configView(EmulatorConfig.class);+ ServicesConfig services = configView(ServicesConfig.class);+ S3ServiceConfig s3 = configView(S3ServiceConfig.class);+ doReturn(services).when(config).services();+ doReturn(s3).when(services).s3();+ doReturn(true).when(s3).enabled();+ doReturn(true).when(s3).lifecycleSweepEnabled();+ doReturn(intervalSeconds).when(s3).lifecycleSweepIntervalSeconds();+ return config;+ }++ private static <T> T configView(Class<T> configType) {+ return mock(configType, invocation -> {+ throw new AssertionError("Unexpected configuration access: " + invocation.getMethod());+ });+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex f418fa26da..490a51f8ee 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -138,6 +138,8 @@ floci:enforce-auth: falsedefault-presign-expiry-seconds: 3600global-bucket-namespace: false+ lifecycle-sweep-enabled: false # tests run the sweep themselves, so no bucket's rules apply behind another test's back+ lifecycle-sweep-interval-seconds: 60dynamodb:enabled: truevector-index-allocation-seconds: 0