Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(s3): apply lifecycle expiration rules (#5158)

jwoa95cbd2f61dc465
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3LifecycleTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3LifecycleTest.java
index b3757c6bc0..f09193217e 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3LifecycleTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3LifecycleTest.java
@@ -7,6 +7,7 @@ import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
+import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException;
import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration;
@@ -14,12 +15,20 @@ import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
import software.amazon.awssdk.services.s3.model.ExpirationStatus;
import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.LifecycleExpiration;
import software.amazon.awssdk.services.s3.model.LifecycleRule;
import software.amazon.awssdk.services.s3.model.LifecycleRuleFilter;
import software.amazon.awssdk.services.s3.model.PutBucketLifecycleConfigurationResponse;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.TransitionDefaultMinimumObjectSize;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.Locale;
+
import static org.assertj.core.api.Assertions.assertThat;
/**
@@ -40,6 +49,9 @@ import static org.assertj.core.api.Assertions.assertThat;
class S3LifecycleTest {
private static final String BUCKET = "compat-lifecycle-bucket";
+ private static final DateTimeFormatter HTTP_DATE = DateTimeFormatter
+ .ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US)
+ .withZone(ZoneOffset.UTC);
private static S3Client s3;
@@ -117,4 +129,25 @@ class S3LifecycleTest {
.as("GET must default to ALL_STORAGE_CLASSES_128_K when PUT omits the header")
.isEqualTo(TransitionDefaultMinimumObjectSize.ALL_STORAGE_CLASSES_128_K);
}
+
+ @Test
+ @Order(3)
+ @DisplayName("#5137 objects under an Expiration rule report expiry-date and rule-id")
+ void objectsUnderAnExpirationRuleReportTheirExpiry() {
+ // The configuration from @Order(2): Days 365 on every key, rule id expire-everything.
+ String key = "expiring.txt";
+ try {
+ PutObjectResponse put = s3.putObject(req -> req.bucket(BUCKET).key(key), RequestBody.fromString("hi"));
+ HeadObjectResponse head = s3.headObject(req -> req.bucket(BUCKET).key(key));
+
+ // Creation time plus 365 days, rounded up to the next midnight UTC.
+ Instant expiry = head.lastModified().plus(365, ChronoUnit.DAYS)
+ .truncatedTo(ChronoUnit.DAYS).plus(1, ChronoUnit.DAYS);
+ String expected = "expiry-date=\"" + HTTP_DATE.format(expiry) + "\", rule-id=\"expire-everything\"";
+ assertThat(head.expiration()).isEqualTo(expected);
+ assertThat(put.expiration()).isEqualTo(expected);
+ } finally {
+ s3.deleteObject(req -> req.bucket(BUCKET).key(key));
+ }
+ }
}
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.md
index c3fe6a6c01..fa7b19d984 100644
--- a/docs/configuration/advanced/application-yml.md
+++ b/docs/configuration/advanced/application-yml.md
@@ -140,6 +140,8 @@ floci:
s3:
enabled: true
default-presign-expiry-seconds: 3600
+ lifecycle-sweep-enabled: true # Apply bucket lifecycle expiration rules in the background
+ lifecycle-sweep-interval-seconds: 60 # Seconds between lifecycle sweeps
dynamodb:
enabled: true
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md
index c00ba9b353..0d28b377fb 100644
--- a/docs/configuration/environment-variables.md
+++ b/docs/configuration/environment-variables.md
@@ -221,6 +221,8 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s
| `FLOCI_SERVICES_S3_ENABLED` | `true` | Enable the S3 service |
| `FLOCI_SERVICES_S3_DEFAULT_PRESIGN_EXPIRY_SECONDS` | `3600` | Default pre-signed URL expiry when none is specified |
| `FLOCI_SERVICES_S3_GLOBAL_BUCKET_NAMESPACE` | `false` | When `true`, bucket/object resolution spans every account's partition so a bucket created in one account is visible cross-account (a single global namespace), matching real S3's global bucket names. Off by default keeps buckets isolated per account |
+| `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_ENABLED` | `true` | Apply bucket lifecycle expiration rules (`Expiration`, `NoncurrentVersionExpiration`, `AbortIncompleteMultipartUpload`) in the background |
+| `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS` | `60` | How often the lifecycle sweep runs |
### DynamoDB
diff --git a/docs/services/s3.md b/docs/services/s3.md
index dafc7e9c77..bb4b82acdb 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -87,6 +87,61 @@ specific object version through the four `?annotation` operations. Notes on the
resolve cross-account but annotation reads, writes, and listings stay in the caller's account.
- S3 Metadata annotation tables and annotation replication are not implemented.
+## Lifecycle Expiration
+
+A bucket's lifecycle configuration is stored as sent and its expiration actions are applied by a
+background sweep, every `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS` (60 by default). S3
+also applies lifecycle asynchronously, so a lower interval only makes Floci act sooner after an
+object becomes due. Each enabled rule's filter (`Prefix`, `Tag`, `ObjectSizeGreaterThan`,
+`ObjectSizeLessThan`, an `And` of them, or the older rule-level `Prefix`) selects what it applies to,
+and disabled rules are ignored.
+
+- `Expiration` with `Days` or `Date` expires the current version. A day count runs from the
+ object's last-modified time and, as on S3, is rounded up to the next midnight UTC, so an object
+ written at 10:30 UTC under a 3-day rule expires at midnight UTC three days and some hours later.
+ An unversioned bucket deletes the object; a versioning-enabled bucket adds a delete marker.
+ A versioning-suspended bucket deletes the object as `DeleteObject` does in Floci, without the
+ `null` delete marker S3 adds (see the differences below).
+- `ExpiredObjectDeleteMarker` removes a delete marker that is the only version left of its key.
+ A `Days` or `Date` expiration also removes such a marker once the marker itself is old enough.
+- `NoncurrentVersionExpiration` permanently deletes a noncurrent version, delete markers included,
+ once it has been noncurrent for `NoncurrentDays` (counted from when the next newer version was
+ written, rounded the same way) and at least `NewerNoncurrentVersions` newer noncurrent versions
+ are kept ahead of it.
+- `AbortIncompleteMultipartUpload` aborts an upload `DaysAfterInitiation` after it was started,
+ rounded the same way. Only a rule whose filter is a plain prefix selects uploads.
+- A version under Object Lock retention or a legal hold is never deleted by lifecycle.
+- Expirations are notified as `s3:LifecycleExpiration:Delete` and
+ `s3:LifecycleExpiration:DeleteMarkerCreated`, not as `s3:ObjectRemoved:*`, with `s3.amazonaws.com`
+ as the principal and source IP address. Their EventBridge events carry `reason`
+ `Lifecycle Expiration`, requester `s3.amazonaws.com` and no `source-ip-address`.
+- A rule that S3 would have refused is skipped as a whole, so a bad value never widens what it
+ deletes: `Days` or `NoncurrentDays` that is not a positive integer, `NewerNoncurrentVersions`
+ outside 1 to 100 or without a `Filter`, a negative `DaysAfterInitiation`, a size bound that is not
+ a non-negative number, a `Tag` without both `Key` and `Value` or with a key used twice, neither a
+ `Filter` nor a rule-level `Prefix`, `ExpiredObjectDeleteMarker` next to `Days` or `Date`, a `Date`
+ that does not parse, or a `Status` other than `Enabled` or `Disabled`. An empty `Filter` applies
+ to the whole bucket. A body that is not one well-formed `LifecycleConfiguration` document has no
+ rules.
+
+`PutObject`, `GetObject` and `HeadObject` on the current version of an object that an `Expiration`
+rule matches return `x-amz-expiration: expiry-date="Fri, 23 Dec 2012 00:00:00 GMT", rule-id="..."`,
+with the earliest matching expiry and that rule's URL-encoded id. Requests for a noncurrent version
+do not carry it.
+
+Differences from S3:
+
+- `Transition` and `NoncurrentVersionTransition` are stored but not applied: storage classes do not
+ change.
+- `CopyObject` and `CompleteMultipartUpload` do not return `x-amz-expiration`, and
+ `CreateMultipartUpload` does not return `x-amz-abort-date` or `x-amz-abort-rule-id`.
+- The configuration is not validated when it is put: a rule S3 would have refused is stored and
+ returned as sent, and only skipped when rules are applied.
+- After `DeleteObject` in a versioning-suspended bucket, every version left behind is noncurrent,
+ as it is on S3, but Floci does not record when the newest of them stopped being current, so
+ `NoncurrentDays` never expires that one. The older ones expire as usual.
+- A delete marker that lifecycle adds carries the time of the sweep, not midnight UTC.
+
## Website Hosting
Static website requests use a bucket website hostname such as
@@ -396,6 +451,8 @@ These AWS S3 features have no handler in Floci. Calls will return an error (typi
| `FLOCI_SERVICES_S3_ENFORCE_AUTH` | `false` | Verify SigV4 signatures and evaluate bucket policies and ACLs, including for anonymous callers (see [Signature Verification](#signature-verification)) |
| `FLOCI_AUTH_VALIDATE_SIGNATURES` | `false` | Verify SigV4 signatures without evaluating bucket policies or ACLs (see [Signature Verification](#signature-verification)) |
| `FLOCI_SERVICES_S3_GLOBAL_BUCKET_NAMESPACE` | `false` | Enable cross-account bucket and object resolution |
+| `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_ENABLED` | `true` | Apply bucket lifecycle expiration rules in the background (see [Lifecycle Expiration](#lifecycle-expiration)) |
+| `FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS` | `60` | How often the lifecycle sweep runs |
### Storage
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index ae7191e95e..c684958afa 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1379,6 +1379,23 @@ public interface EmulatorConfig {
*/
@WithDefault("false")
boolean globalBucketNamespace();
+
+ /**
+ * Run the background sweep that applies bucket lifecycle expiration rules: Expiration,
+ * NoncurrentVersionExpiration and AbortIncompleteMultipartUpload. When off, rules are
+ * stored and reported in {@code x-amz-expiration} but nothing expires.
+ * Env: FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_ENABLED
+ */
+ @WithDefault("true")
+ boolean lifecycleSweepEnabled();
+
+ /**
+ * How often the lifecycle sweep runs. S3 itself applies lifecycle about once a day, to
+ * expiry times rounded to midnight UTC, so the interval only bounds how late Floci is.
+ * Env: FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS
+ */
+ @WithDefault("60")
+ long lifecycleSweepIntervalSeconds();
}
interface DynamoDbServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java b/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java
index a8f23de491..ae13247d7c 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java
@@ -454,6 +454,37 @@ public final class XmlParser {
return null;
}
+ /**
+ * Parses a whole document into its document element's tree, or returns {@code null} when the
+ * document is empty or not well-formed anywhere, after its document element included.
+ *
+ * <p>For a stored document whose every part has effect, where {@link #extractElementTree}
+ * would return a complete element followed by content that never parsed.
+ */
+ public static XmlElement parseDocumentElementTree(String xml) {
+ if (xml == null || xml.isEmpty()) {
+ return null;
+ }
+ try {
+ XMLStreamReader r = FACTORY.createXMLStreamReader(new StringReader(xml));
+ try {
+ XmlElement root = null;
+ while (r.hasNext()) {
+ int event = r.next();
+ if (root == null && event == XMLStreamConstants.START_ELEMENT) {
+ root = readElementTree(r);
+ }
+ }
+ return root;
+ } finally {
+ r.close();
+ }
+ } catch (XMLStreamException e) {
+ LOG.debugv("Ignoring malformed XML document: {0}", e.getMessage());
+ return null;
+ }
+ }
+
private static XmlElement readElementTree(XMLStreamReader r) throws XMLStreamException {
String name = r.getLocalName();
StringBuilder text = new StringBuilder();
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 64c4983b5a..663e24f9b2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -111,7 +111,7 @@ public class S3Controller {
private static final DateTimeFormatter ISO_FORMAT = DateTimeFormatter
.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'")
.withZone(ZoneId.of("UTC"));
- private static final DateTimeFormatter RFC_822 = DateTimeFormatter
+ static final DateTimeFormatter RFC_822 = DateTimeFormatter
.ofPattern("EEE, dd MMM yyyy HH:mm:ss z", Locale.US)
.withZone(ZoneId.of("GMT"));
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@@ -2939,6 +2939,7 @@ public class S3Controller {
appendSseCustomerHeaders(resp, obj);
appendChecksumHeaders(resp, obj.getChecksum());
appendLockHeaders(resp, obj);
+ appendExpirationHeader(resp, obj);
}
// includeChecksum must be false for a range the stored checksum does not describe:
@@ -2983,6 +2984,14 @@ public class S3Controller {
appendChecksumHeaders(resp, obj.getChecksum());
}
appendLockHeaders(resp, obj);
+ appendExpirationHeader(resp, obj);
+ }
+
+ private void appendExpirationHeader(Response.ResponseBuilder resp, S3Object obj) {
+ String expiration = s3Service.expirationHeader(obj);
+ if (expiration != null) {
+ resp.header("x-amz-expiration", expiration);
+ }
}
private void appendSseCustomerHeaders(Response.ResponseBuilder resp, S3Object obj) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfiguration.java
new file mode 100644
index 0000000000..714e3afe29
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfiguration.java
@@ -0,0 +1,358 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.XmlParser;
+import io.github.hectorvent.floci.core.common.XmlParser.XmlElement;
+import io.github.hectorvent.floci.core.common.auth.SigV4Canonicalization;
+import org.jboss.logging.Logger;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * The expiration actions of a bucket's lifecycle configuration, read the way S3 applies them.
+ *
+ * <p>PutBucketLifecycleConfiguration stores the body verbatim, so this is parsed when it is used,
+ * and a body or rule S3 would have refused is ignored rather than failing the request or the
+ * sweep that reads it (see {@link #parse}). Disabled rules are dropped. Transition actions are not
+ * modelled.
+ */
+record S3LifecycleConfiguration(List<Rule> rules) {
+
+ static final S3LifecycleConfiguration EMPTY = new S3LifecycleConfiguration(List.of());
+
+ private static final Logger LOG = Logger.getLogger(S3LifecycleConfiguration.class);
+
+ /** NewerNoncurrentVersions: "You can specify up to 100 noncurrent versions to retain." */
+ private static final int MAX_NEWER_NONCURRENT_VERSIONS = 100;
+
+ // Date is ISO 8601 on the wire. The leniency covers bodies Floci writes itself, such as a
+ // CloudFormation ExpirationDate passed through as the template spelled it.
+ private static final List<Function<String, Instant>> DATE_PARSERS = List.of(
+ text -> OffsetDateTime.parse(text).toInstant(),
+ text -> LocalDateTime.parse(text).toInstant(ZoneOffset.UTC),
+ text -> LocalDate.parse(text).atStartOfDay(ZoneOffset.UTC).toInstant());
+
+ record Rule(String id, Filter filter, Expiration expiration,
+ NoncurrentVersionExpiration noncurrentVersionExpiration, Integer abortDaysAfterInitiation) {
+ }
+
+ /** A rule's filter: every condition that is set must hold. An empty filter matches everything. */
+ record Filter(String prefix, Map<String, String> tags, Long objectSizeGreaterThan, Long objectSizeLessThan) {
+
+ boolean matches(String key, Map<String, String> objectTags, long size) {
+ if (!key.startsWith(prefix)) {
+ return false;
+ }
+ for (Map.Entry<String, String> tag : tags.entrySet()) {
+ String value = objectTags == null ? null : objectTags.get(tag.getKey());
+ if (!tag.getValue().equals(value)) {
+ return false;
+ }
+ }
+ // Both size bounds are exclusive.
+ return (objectSizeGreaterThan == null || size > objectSizeGreaterThan)
+ && (objectSizeLessThan == null || size < objectSizeLessThan);
+ }
+
+ /**
+ * An in-progress multipart upload has no tags or size to test, and S3 refuses
+ * AbortIncompleteMultipartUpload under a tag filter, so only a prefix selects uploads.
+ */
+ boolean selectsUpload(String key) {
+ return tags.isEmpty() && objectSizeGreaterThan == null && objectSizeLessThan == null
+ && key.startsWith(prefix);
+ }
+ }
+
+ record Expiration(Integer days, Instant date, boolean expiredObjectDeleteMarker) {
+
+ /** When something created at {@code created} expires under this action, or null if it sets no age. */
+ Instant expiresAt(Instant created) {
+ if (days != null) {
+ return afterDays(created, days);
+ }
+ return date;
+ }
+ }
+
+ record NoncurrentVersionExpiration(Integer noncurrentDays, Integer newerNoncurrentVersions) {
+ }
+
+ /** The earliest current-version expiration that applies to an object, and the rule that sets it. */
+ record Expiry(Instant date, String ruleId) {
+
+ /**
+ * The {@code x-amz-expiration} header value, in the shape the S3 API reference shows,
+ * {@code expiry-date="Fri, 23 Dec 2012 00:00:00 GMT", rule-id="1"}, with the rule id
+ * URL-encoded as the reference requires.
+ */
+ String header() {
+ return "expiry-date=\"" + S3Controller.RFC_822.format(date) + "\", rule-id=\""
+ + SigV4Canonicalization.uriEncode(ruleId == null ? "" : ruleId) + "\"";
+ }
+ }
+
+ /**
+ * Day-based lifecycle times run from a start time: "Amazon S3 calculates the time by adding
+ * the number of days specified in the rule to the object creation time and rounding up the
+ * resulting time to the next day at midnight UTC."
+ */
+ static Instant afterDays(Instant start, int days) {
+ return start.plus(days, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).plus(1, ChronoUnit.DAYS);
+ }
+
+ /**
+ * The earliest Days or Date expiration among the rules that match a current object version,
+ * or null when none does.
+ */
+ Expiry currentVersionExpiry(String key, Map<String, String> tags, long size, Instant created) {
+ Expiry earliest = null;
+ for (Rule rule : rules) {
+ if (rule.expiration() == null || !rule.filter().matches(key, tags, size)) {
+ continue;
+ }
+ Instant at = rule.expiration().expiresAt(created);
+ if (at != null && (earliest == null || at.isBefore(earliest.date()))) {
+ earliest = new Expiry(at, rule.id());
+ }
+ }
+ return earliest;
+ }
+
+ /**
+ * Whether a delete marker that is the only version of its key, an expired object delete
+ * marker, is due for removal. ExpiredObjectDeleteMarker removes it at once; a Days or Date
+ * expiration removes it once the marker itself is old enough.
+ */
+ boolean removesExpiredObjectDeleteMarker(String key, Instant markerCreated, Instant now) {
+ for (Rule rule : rules) {
+ Expiration expiration = rule.expiration();
+ if (expiration == null || !rule.filter().matches(key, Map.of(), 0)) {
+ continue;
+ }
+ if (expiration.expiredObjectDeleteMarker()) {
+ return true;
+ }
+ Instant at = expiration.expiresAt(markerCreated);
+ if (at != null && !now.isBefore(at)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Whether a noncurrent version is due for permanent deletion: it has been noncurrent, counted
+ * from when its successor was created, for NoncurrentDays, and at least
+ * NewerNoncurrentVersions newer noncurrent versions are retained ahead of it.
+ */
+ boolean expiresNoncurrentVersion(String key, Map<String, String> tags, long size,
+ Instant becameNoncurrent, int newerNoncurrentVersions, Instant now) {
+ for (Rule rule : rules) {
+ NoncurrentVersionExpiration action = rule.noncurrentVersionExpiration();
+ if (action == null || !rule.filter().matches(key, tags, size)) {
+ continue;
+ }
+ if (action.noncurrentDays() == null && action.newerNoncurrentVersions() == null) {
+ continue;
+ }
+ boolean oldEnough = action.noncurrentDays() == null
+ || !now.isBefore(afterDays(becameNoncurrent, action.noncurrentDays()));
+ boolean beyondRetained = action.newerNoncurrentVersions() == null
+ || newerNoncurrentVersions >= action.newerNoncurrentVersions();
+ if (oldEnough && beyondRetained) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Whether an incomplete multipart upload for {@code key} initiated at {@code initiated} is due to be aborted. */
+ boolean abortsUpload(String key, Instant initiated, Instant now) {
+ for (Rule rule : rules) {
+ Integer days = rule.abortDaysAfterInitiation();
+ if (days != null && rule.filter().selectsUpload(key) && !now.isBefore(afterDays(initiated, days))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Reads a stored configuration. A body that is not one well-formed LifecycleConfiguration
+ * document has no rules. A rule that S3 would have refused on put, such as {@code Days} 0 or a
+ * size bound that is not a number, is dropped whole rather than read without the bad value, so
+ * a bad value never widens what the rule deletes. The other rules still apply.
+ */
+ static S3LifecycleConfiguration parse(String xml) {
+ XmlElement root = XmlParser.parseDocumentElementTree(xml);
+ if (root == null || !"LifecycleConfiguration".equals(root.name())) {
+ return EMPTY;
+ }
+ List<Rule> rules = new ArrayList<>();
+ for (XmlElement element : root.children()) {
+ if (!"Rule".equals(element.name()) || "Disabled".equals(text(element, "Status"))) {
+ continue;
+ }
+ try {
+ rules.add(rule(element));
+ } catch (IllegalArgumentException e) {
+ LOG.debugv("Ignoring lifecycle rule {0}: {1}", text(element, "ID"), e.getMessage());
+ }
+ }
+ return new S3LifecycleConfiguration(List.copyOf(rules));
+ }
+
+ private static Rule rule(XmlElement rule) {
+ String status = text(rule, "Status");
+ if (!"Enabled".equals(status)) {
+ throw new IllegalArgumentException("Status is " + status + ", not Enabled or Disabled");
+ }
+ NoncurrentVersionExpiration noncurrent = noncurrentVersionExpiration(rule.child("NoncurrentVersionExpiration"));
+ if (noncurrent != null && noncurrent.newerNoncurrentVersions() != null && rule.child("Filter") == null) {
+ // The user guide: "If you don't specify a <Filter> element, Amazon S3 generates an
+ // InvalidRequest error when you specify the number of noncurrent versions to retain."
+ throw new IllegalArgumentException("NewerNoncurrentVersions needs a Filter");
+ }
+ return new Rule(text(rule, "ID"), filter(rule), expiration(rule.child("Expiration")), noncurrent,
+ abortDaysAfterInitiation(rule.child("AbortIncompleteMultipartUpload")));
+ }
+
+ /**
+ * The rule's Filter, or the deprecated rule-level Prefix that predates it: "Filter is required
+ * if the LifecycleRule does not contain a Prefix element." An empty Filter applies to the whole
+ * bucket.
+ */
+ private static Filter filter(XmlElement rule) {
+ XmlElement filter = rule.child("Filter");
+ if (filter == null) {
+ String prefix = text(rule, "Prefix");
+ if (prefix == null) {
+ throw new IllegalArgumentException("a rule needs a Filter or a Prefix");
+ }
+ return new Filter(prefix, Map.of(), null, null);
+ }
+ XmlElement and = filter.child("And");
+ XmlElement conditions = and != null ? and : filter;
+ String prefix = text(conditions, "Prefix");
+ Map<String, String> tags = new LinkedHashMap<>();
+ for (XmlElement tag : conditions.children()) {
+ if ("Tag".equals(tag.name())) {
+ String key = text(tag, "Key");
+ String value = text(tag, "Value");
+ if (key == null || value == null) {
+ throw new IllegalArgumentException("a Tag needs both a Key and a Value");
+ }
+ // "When you specify multiple tags in a filter, each tag key must be unique."
+ if (tags.put(key, value) != null) {
+ throw new IllegalArgumentException("tag key " + key + " appears twice");
+ }
+ }
+ }
+ return new Filter(prefix == null ? "" : prefix, Map.copyOf(tags),
+ size(conditions, "ObjectSizeGreaterThan"), size(conditions, "ObjectSizeLessThan"));
+ }
+
+ private static Expiration expiration(XmlElement element) {
+ if (element == null) {
+ return null;
+ }
+ Integer days = positiveInteger(element, "Days");
+ String dateText = text(element, "Date");
+ Instant date = dateText == null ? null : date(dateText);
+ String deleteMarkerText = text(element, "ExpiredObjectDeleteMarker");
+ if (deleteMarkerText != null && (days != null || date != null)) {
+ throw new IllegalArgumentException("ExpiredObjectDeleteMarker cannot be specified with Days or Date");
+ }
+ boolean deleteMarker = deleteMarkerText != null && bool(deleteMarkerText, "ExpiredObjectDeleteMarker");
+ if (days == null && date == null && !deleteMarker) {
+ return null;
+ }
+ return new Expiration(days, date, deleteMarker);
+ }
+
+ private static NoncurrentVersionExpiration noncurrentVersionExpiration(XmlElement element) {
+ if (element == null) {
+ return null;
+ }
+ Integer newer = integer(element, "NewerNoncurrentVersions");
+ if (newer != null && (newer < 1 || newer > MAX_NEWER_NONCURRENT_VERSIONS)) {
+ throw new IllegalArgumentException("NewerNoncurrentVersions " + newer + " is not between 1 and 100");
+ }
+ return new NoncurrentVersionExpiration(positiveInteger(element, "NoncurrentDays"), newer);
+ }
+
+ private static Integer abortDaysAfterInitiation(XmlElement element) {
+ if (element == null) {
+ return null;
+ }
+ Integer days = integer(element, "DaysAfterInitiation");
+ if (days != null && days < 0) {
+ throw new IllegalArgumentException("DaysAfterInitiation " + days + " is negative");
+ }
+ return days;
+ }
+
+ private static String text(XmlElement parent, String childName) {
+ XmlElement child = parent.child(childName);
+ return child == null ? null : child.text();
+ }
+
+ /** Days and NoncurrentDays: "The value must be a non-zero positive integer." */
+ private static Integer positiveInteger(XmlElement parent, String childName) {
+ Integer value = integer(parent, childName);
+ if (value != null && value < 1) {
+ throw new IllegalArgumentException(childName + " " + value + " is not a positive integer");
+ }
+ return value;
+ }
+
+ /** An absent element is null; one whose text is not an integer makes the rule invalid. */
+ private static Integer integer(XmlElement parent, String childName) {
+ String text = text(parent, childName);
+ return text == null ? null : Integer.valueOf(text);
+ }
+
+ private static Long size(XmlElement parent, String childName) {
+ String text = text(parent, childName);
+ if (text == null) {
+ return null;
+ }
+ long value = Long.parseLong(text);
+ if (value < 0) {
+ throw new IllegalArgumentException(childName + " " + value + " is negative");
+ }
+ return value;
+ }
+
+ private static boolean bool(String text, String name) {
+ if ("true".equalsIgnoreCase(text)) {
+ return true;
+ }
+ if ("false".equalsIgnoreCase(text)) {
+ return false;
+ }
+ throw new IllegalArgumentException(name + " " + text + " is not true or false");
+ }
+
+ private static Instant date(String text) {
+ for (Function<String, Instant> parser : DATE_PARSERS) {
+ try {
+ return parser.apply(text);
+ } catch (DateTimeParseException ignored) {
+ // Not this shape; try the next one.
+ }
+ }
+ throw new IllegalArgumentException("Date " + text + " is not an ISO 8601 date");
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeper.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeper.java
new file mode 100644
index 0000000000..1cecb3e388
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeper.java
@@ -0,0 +1,87 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.quarkus.runtime.ShutdownEvent;
+import io.quarkus.runtime.StartupEvent;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.time.Clock;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Applies bucket lifecycle expiration rules from one background thread. S3 evaluates lifecycle
+ * asynchronously rather than on each request, so a sweep over every bucket with a lifecycle
+ * configuration on a fixed interval matches that, and leaves nothing to reconcile after a restart.
+ */
+@ApplicationScoped
+public class S3LifecycleSweeper {
+
+ private static final Logger LOG = Logger.getLogger(S3LifecycleSweeper.class);
+
+ /** Used when the configured interval is not a positive number of seconds. */
+ static final long DEFAULT_INTERVAL_SECONDS = 60L;
+
+ private final S3Service s3Service;
+ private final Clock clock;
+ private final boolean enabled;
+ private final long intervalSeconds;
+ private final ScheduledExecutorService executor;
+
+ /**
+ * S3 stamps objects with {@code Instant.now()}, so the sweep reads the same system clock, not
+ * the injectable {@link Clock} that tests pin to a fixed date.
+ */
+ @Inject
+ public S3LifecycleSweeper(S3Service s3Service, EmulatorConfig config) {
+ this(s3Service, config, Clock.systemUTC());
+ }
+
+ S3LifecycleSweeper(S3Service s3Service, EmulatorConfig config, Clock clock) {
+ this.s3Service = s3Service;
+ this.clock = clock;
+ this.enabled = config.services().s3().enabled() && config.services().s3().lifecycleSweepEnabled();
+ this.intervalSeconds = resolveInterval(config.services().s3().lifecycleSweepIntervalSeconds());
+ this.executor = Executors.newSingleThreadScheduledExecutor(runnable -> {
+ Thread thread = new Thread(runnable, "s3-lifecycle-sweeper");
+ thread.setDaemon(true);
+ return thread;
+ });
+ }
+
+ /** A non-positive interval would throw from the startup observer and keep the emulator from becoming ready. */
+ private static long resolveInterval(long configured) {
+ if (configured > 0) {
+ return configured;
+ }
+ LOG.warnv("Ignoring S3 lifecycle-sweep interval {0}s: must be a positive number of seconds, "
+ + "falling back to {1}s", configured, DEFAULT_INTERVAL_SECONDS);
+ return DEFAULT_INTERVAL_SECONDS;
+ }
+
+ void onStart(@Observes StartupEvent event) {
+ if (!enabled) {
+ LOG.debug("S3 lifecycle sweeper disabled");
+ return;
+ }
+ executor.scheduleWithFixedDelay(this::sweep, intervalSeconds, intervalSeconds, TimeUnit.SECONDS);
+ LOG.debugv("S3 lifecycle sweeper started, interval {0}s", intervalSeconds);
+ }
+
+ void onStop(@Observes ShutdownEvent event) {
+ executor.shutdownNow();
+ }
+
+ void sweep() {
+ try {
+ s3Service.applyLifecycleExpiration(clock.instant());
+ } catch (RuntimeException e) {
+ // A sweep failure must not kill the scheduled task. The next run retries.
+ LOG.error("S3 lifecycle sweep failed", e);
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index b9450d4bfd..de0ff76531 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.core.common.Base64Url;
import io.github.hectorvent.floci.core.common.Digests;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.ServicePrincipals;
import io.github.hectorvent.floci.core.common.XmlBuilder;
@@ -72,6 +73,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.HexFormat;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -81,6 +83,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
@@ -1936,7 +1939,7 @@ public class S3Service implements Resettable, ResourceProvider {
// so a concurrent overwrite cannot land between the check and the delete.
synchronized (bucket) {
checkDeletePrecondition(bucketName, key, ifMatch);
- return deleteObjectLocked(bucket, bucketName, key, versionId, bypassGovernance);
+ return deleteObjectLocked(bucket, bucketName, key, versionId, bypassGovernance, OBJECT_REMOVED_EVENT);
}
}
@@ -1989,8 +1992,13 @@ public class S3Service implements Resettable, ResourceProvider {
}
}
+ /**
+ * Deletes as DeleteObject does. {@code removalEvent} is the notification event family the
+ * deletion is reported under: {@code ObjectRemoved} for a request, {@code LifecycleExpiration}
+ * when a lifecycle rule expires the object.
+ */
private S3Object deleteObjectLocked(Bucket bucket, String bucketName, String key,
- String versionId, boolean bypassGovernance) {
+ String versionId, boolean bypassGovernance, String removalEvent) {
if (bucket.isVersioningEnabled() && versionId == null) {
// Check lock on current latest before placing a delete marker
objectStore.get(objectKey(bucketName, key)).ifPresent(prev -> {
@@ -2005,6 +2013,7 @@ public class S3Service implements Resettable, ResourceProvider {
deleteMarker.setVersionId(markerId);
deleteMarker.setDeleteMarker(true);
deleteMarker.setLatest(true);
+ deleteMarker.setEventSequencer(nextEventSequencer());
// Mark previous latest as not latest
objectStore.get(objectKey(bucketName, key)).ifPresent(prev -> {
@@ -2022,7 +2031,7 @@ public class S3Service implements Resettable, ResourceProvider {
objectStore.put(versionedKey(bucketName, key, markerId), deleteMarker);
objectStore.put(objectKey(bucketName, key), deleteMarker);
LOG.debugv("Created delete marker: {0}/{1} v={2}", bucketName, key, markerId);
- fireNotifications(bucketName, key, "ObjectRemoved:DeleteMarkerCreated", deleteMarker);
+ fireNotifications(bucketName, key, removalEvent + ":DeleteMarkerCreated", deleteMarker);
return deleteMarker;
} else if ("null".equals(versionId)) {
// A null version is stored at the plain object key until a versioned write replaces it.
@@ -2037,7 +2046,7 @@ public class S3Service implements Resettable, ResourceProvider {
deleteFile(bucketName, key);
deleteAllAnnotationsFor(annotationParentKey(bucketName, key, null));
LOG.debugv("Permanently deleted null version: {0}/{1}", bucketName, key);
- fireNotifications(bucketName, key, "ObjectRemoved:Delete", null);
+ fireNotifications(bucketName, key, removalEvent + ":Delete", null);
return existing;
} else if (versionId != null) {
// Get the specific version before permanent deletion
@@ -2061,7 +2070,7 @@ public class S3Service implements Resettable, ResourceProvider {
deleteFile(bucketName, key);
} else {
S3Object newLatest = remaining.stream()
- .max(Comparator.comparing(S3Object::getLastModified))
+ .max(WRITE_ORDER)
.orElseThrow();
// The promoted version's body becomes current before its metadata is
// published, the order storeObjectInternal writes in, so a concurrent GET
@@ -2080,7 +2089,7 @@ public class S3Service implements Resettable, ResourceProvider {
}
});
if (toDelete != null) {
- fireNotifications(bucketName, key, "ObjectRemoved:Delete", toDelete);
+ fireNotifications(bucketName, key, removalEvent + ":Delete", toDelete);
}
return toDelete;
} else {
@@ -2094,7 +2103,7 @@ public class S3Service implements Resettable, ResourceProvider {
deleteFile(bucketName, key);
deleteAllAnnotationsFor(annotationParentKey(bucketName, key, null));
LOG.debugv("Deleted object: {0}/{1}", bucketName, key);
- fireNotifications(bucketName, key, "ObjectRemoved:Delete", null);
+ fireNotifications(bucketName, key, removalEvent + ":Delete", null);
return null;
}
}
@@ -2652,11 +2661,11 @@ public class S3Service implements Resettable, ResourceProvider {
Collections.sort(commonPrefixes);
}
- // Sort by key, then by lastModified descending
+ // Sort by key, then newest first
versions.sort((a, b) -> {
int keyCompare = a.getKey().compareTo(b.getKey());
if (keyCompare != 0) return keyCompare;
- return b.getLastModified().compareTo(a.getLastModified());
+ return WRITE_ORDER.compare(b, a);
});
// Apply the marker filter. Without a version-id-marker the marker is an exclusive lower bound on the
@@ -2791,11 +2800,7 @@ public class S3Service implements Resettable, ResourceProvider {
// --- Object Tagging ---
public void putObjectTagging(String bucketName, String key, Map<String, String> tags) {
- AccountAwareStorageBackend.OwnedEntry<S3Object> ownedObject =
- getStoredObjectEntry(bucketName, key, null);
- S3Object obj = ownedObject.value();
- obj.setTags(tags != null ? tags : new java.util.HashMap<>());
- putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);
+ setCurrentObjectTags(bucketName, key, tags != null ? tags : new HashMap<>());
LOG.debugv("Put tags on object: {0}/{1}", bucketName, key);
}
@@ -2810,11 +2815,7 @@ public class S3Service implements Resettable, ResourceProvider {
}
public void deleteObjectTagging(String bucketName, String key) {
- AccountAwareStorageBackend.OwnedEntry<S3Object> ownedObject =
- getStoredObjectEntry(bucketName, key, null);
- S3Object obj = ownedObject.value();
- obj.setTags(new java.util.HashMap<>());
- putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);
+ setCurrentObjectTags(bucketName, key, new HashMap<>());
LOG.debugv("Deleted tags from object: {0}/{1}", bucketName, key);
}
@@ -4919,24 +4920,348 @@ public class S3Service implements Resettable, ResourceProvider {
return new LifecycleConfigurationResult(bucket.getLifecycleConfiguration(), size);
}
+ /**
+ * The lifecycle configuration changes under the bucket monitor the lifecycle sweep applies rules
+ * under, so a sweep never expires an object by a rule that was replaced or removed before it acted.
+ */
public String putBucketLifecycle(String bucketName, String lifecycle, String transitionDefaultMinimumObjectSize) {
Bucket bucket = bucketStore.get(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));
- bucket.setLifecycleConfiguration(lifecycle);
String size = (transitionDefaultMinimumObjectSize == null || transitionDefaultMinimumObjectSize.isBlank())
? DEFAULT_TRANSITION_DEFAULT_MIN_OBJECT_SIZE
: transitionDefaultMinimumObjectSize;
- bucket.setTransitionDefaultMinimumObjectSize(size);
- bucketStore.put(bucketName, bucket);
+ synchronized (bucket) {
+ bucket.setLifecycleConfiguration(lifecycle);
+ bucket.setTransitionDefaultMinimumObjectSize(size);
+ bucketStore.put(bucketName, bucket);
+ }
return size;
}
public void deleteBucketLifecycle(String bucketName) {
Bucket bucket = bucketStore.get(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));
- bucket.setLifecycleConfiguration(null);
- bucket.setTransitionDefaultMinimumObjectSize(null);
- bucketStore.put(bucketName, bucket);
+ synchronized (bucket) {
+ bucket.setLifecycleConfiguration(null);
+ bucket.setTransitionDefaultMinimumObjectSize(null);
+ bucketStore.put(bucketName, bucket);
+ }
+ }
+
+ private static final String OBJECT_REMOVED_EVENT = "ObjectRemoved";
+ private static final String LIFECYCLE_EXPIRATION_EVENT = "LifecycleExpiration";
+
+ /**
+ * The versions of one key in the order they were written: by last-modified time, then by the
+ * sequencer each write takes under the bucket lock, so versions stored within one millisecond
+ * keep their order. The sequencer is not persisted, so across a restart such versions fall back
+ * to version id order, which is at least the same on every read.
+ */
+ private static final Comparator<S3Object> WRITE_ORDER = Comparator
+ .comparing(S3Object::getLastModified)
+ .thenComparing(S3Object::getEventSequencer, Comparator.nullsFirst(Comparator.naturalOrder()))
+ .thenComparing(S3Object::getVersionId, Comparator.nullsFirst(Comparator.naturalOrder()));
+
+ /**
+ * The {@code x-amz-expiration} header for an object being returned or just written: the
+ * earliest expiration an enabled rule's Days or Date sets for it, and that rule's id. Null
+ * when no rule applies. Only the current version carries the header.
+ */
+ public String expirationHeader(S3Object object) {
+ if (object == null || object.getBucketName() == null || object.isDeleteMarker() || !object.isLatest()) {
+ return null;
+ }
+ String lifecycle = resolveBucket(object.getBucketName()).map(Bucket::getLifecycleConfiguration).orElse(null);
+ if (lifecycle == null) {
+ return null;
+ }
+ S3LifecycleConfiguration.Expiry expiry = S3LifecycleConfiguration.parse(lifecycle).currentVersionExpiry(
+ object.getKey(), object.getTags(), object.getSize(), object.getLastModified());
+ return expiry == null ? null : expiry.header();
+ }
+
+ /**
+ * Applies the expiration actions of every bucket's lifecycle configuration as they stand at
+ * {@code now}, each bucket in its owning account. S3 runs lifecycle asynchronously, so this is
+ * driven by {@link S3LifecycleSweeper} rather than by requests. A bucket that fails does not
+ * hold back the others, and the run then fails with every bucket's error, so a sweep that went
+ * wrong never looks like one that completed.
+ */
+ void applyLifecycleExpiration(Instant now) {
+ Map<String, List<AccountAwareStorageBackend.AccountEntry<Bucket>>> byAccount = new LinkedHashMap<>();
+ for (AccountAwareStorageBackend.AccountEntry<Bucket> entry : bucketsInEveryAccount()) {
+ if (entry.value().getLifecycleConfiguration() != null) {
+ byAccount.computeIfAbsent(entry.accountId(), accountId -> new ArrayList<>()).add(entry);
+ }
+ }
+ List<String> failedBuckets = new ArrayList<>();
+ List<RuntimeException> failures = new ArrayList<>();
+ byAccount.forEach((accountId, buckets) -> {
+ Set<String> bucketNames = new HashSet<>();
+ buckets.forEach(entry -> bucketNames.add(entry.key()));
+ Map<String, Map<String, KeyHistory>> histories;
+ try {
+ histories = RequestScopes.callAs(accountId, () -> keyHistories(bucketNames));
+ } catch (RuntimeException e) {
+ failedBuckets.addAll(bucketNames);
+ failures.add(e);
+ return;
+ }
+ for (AccountAwareStorageBackend.AccountEntry<Bucket> entry : buckets) {
+ try {
+ RequestScopes.runAs(accountId, entry.value().getRegion(), () -> applyLifecycleExpiration(
+ entry.key(), histories.getOrDefault(entry.key(), Map.of()), now));
+ } catch (RuntimeException e) {
+ failedBuckets.add(entry.key());
+ failures.add(e);
+ }
+ }
+ });
+ if (!failures.isEmpty()) {
+ IllegalStateException failure = new IllegalStateException(
+ "S3 lifecycle sweep failed for buckets " + failedBuckets);
+ failures.forEach(failure::addSuppressed);
+ throw failure;
+ }
+ }
+
+ private List<AccountAwareStorageBackend.AccountEntry<Bucket>> bucketsInEveryAccount() {
+ if (bucketStore instanceof AccountAwareStorageBackend<?> aware) {
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<Bucket> typed = (AccountAwareStorageBackend<Bucket>) aware;
+ return typed.scanAllAccountEntries(bucketName -> true);
+ }
+ List<AccountAwareStorageBackend.AccountEntry<Bucket>> buckets = new ArrayList<>();
+ for (String bucketName : bucketStore.keys()) {
+ bucketStore.get(bucketName).ifPresent(bucket ->
+ buckets.add(new AccountAwareStorageBackend.AccountEntry<>(null, bucketName, bucket)));
+ }
+ return buckets;
+ }
+
+ /** One key's stored entries: the entry at the plain key, null when there is none, and every versioned entry. */
+ private record KeyHistory(S3Object current, List<S3Object> versions) { }
+
+ /** What a sweep does to one key: permanently delete these versions, then expire the current one or not. */
+ private record LifecyclePlan(List<String> versionsToDelete, boolean expireCurrent) {
+ boolean isEmpty() {
+ return versionsToDelete.isEmpty() && !expireCurrent;
+ }
+ }
+
+ /** A bucket's lifecycle rules, parsed again only when its stored configuration has changed. */
+ private static final class LifecycleRules {
+ private String xml;
+ private S3LifecycleConfiguration rules = S3LifecycleConfiguration.EMPTY;
+
+ S3LifecycleConfiguration of(Bucket bucket) {
+ String stored = bucket.getLifecycleConfiguration();
+ if (!Objects.equals(stored, xml)) {
+ xml = stored;
+ rules = S3LifecycleConfiguration.parse(stored);
+ }
+ return rules;
+ }
+ }
+
+ private void applyLifecycleExpiration(String bucketName, Map<String, KeyHistory> histories, Instant now) {
+ Bucket bucket = bucketStore.get(bucketName).orElse(null);
+ LifecycleRules rules = new LifecycleRules();
+ if (bucket == null || rules.of(bucket).rules().isEmpty()) {
+ return;
+ }
+ // Each key is planned from the unlocked snapshot, then read again, planned again and applied
+ // under the bucket monitor. Writes, tagging, Object Lock changes and lifecycle configuration
+ // changes hold the same monitor, so what is applied is what the key and the rules are now.
+ for (Map.Entry<String, KeyHistory> entry : histories.entrySet()) {
+ String key = entry.getKey();
+ if (planLifecycle(rules.of(bucket), key, entry.getValue(), now).isEmpty()) {
+ continue;
+ }
+ synchronized (bucket) {
+ if (bucketStore.get(bucketName).orElse(null) != bucket) {
+ return;
+ }
+ KeyHistory history = rereadKeyHistory(bucketName, key, entry.getValue());
+ if (history == null) {
+ continue;
+ }
+ LifecyclePlan plan = planLifecycle(rules.of(bucket), key, history, now);
+ for (String versionId : plan.versionsToDelete()) {
+ expireByLifecycle(bucket, bucketName, key, versionId);
+ }
+ if (plan.expireCurrent()) {
+ expireByLifecycle(bucket, bucketName, key, null);
+ }
+ }
+ }
+ abortIncompleteMultipartUploads(bucket, bucketName, rules, now);
+ }
+
+ /**
+ * Every key of the named buckets in the current account, from one pass over the object store.
+ * A key with versions but no plain-key entry is included: a versioning-suspended bucket leaves
+ * one after DeleteObject.
+ */
+ private Map<String, Map<String, KeyHistory>> keyHistories(Set<String> bucketNames) {
+ Map<String, Map<String, S3Object>> currents = new HashMap<>();
+ Map<String, Map<String, List<S3Object>>> versions = new HashMap<>();
+ for (String storeKey : objectStore.keys()) {
+ int slash = storeKey.indexOf('/');
+ String bucketName = slash < 0 ? null : storeKey.substring(0, slash);
+ if (!bucketNames.contains(bucketName)) {
+ continue;
+ }
+ S3Object stored = objectStore.get(storeKey).orElse(null);
+ if (stored == null) {
+ continue;
+ }
+ if (storeKey.equals(objectKey(bucketName, stored.getKey()))) {
+ currents.computeIfAbsent(bucketName, name -> new HashMap<>()).put(stored.getKey(), stored);
+ } else {
+ versions.computeIfAbsent(bucketName, name -> new HashMap<>())
+ .computeIfAbsent(stored.getKey(), key -> new ArrayList<>()).add(stored);
+ }
+ }
+ Map<String, Map<String, KeyHistory>> histories = new HashMap<>();
+ for (String bucketName : bucketNames) {
+ Map<String, S3Object> bucketCurrents = currents.getOrDefault(bucketName, Map.of());
+ Map<String, List<S3Object>> bucketVersions = versions.getOrDefault(bucketName, Map.of());
+ Set<String> keys = new HashSet<>(bucketCurrents.keySet());
+ keys.addAll(bucketVersions.keySet());
+ Map<String, KeyHistory> bucketHistories = new TreeMap<>();
+ for (String key : keys) {
+ bucketHistories.put(key, new KeyHistory(bucketCurrents.get(key), bucketVersions.getOrDefault(key, List.of())));
+ }
+ histories.put(bucketName, bucketHistories);
+ }
+ return histories;
+ }
+
+ /**
+ * One key's entries as they are now, read directly at the store keys the snapshot saw, or null
+ * when the key has been written since. Every write replaces the plain-key entry, so while it is
+ * unchanged the key's versions are the snapshot's, less any deleted since.
+ */
+ private KeyHistory rereadKeyHistory(String bucketName, String key, KeyHistory snapshot) {
+ S3Object current = objectStore.get(objectKey(bucketName, key)).orElse(null);
+ if (!sameWrite(current, snapshot.current())) {
+ return null;
+ }
+ List<S3Object> versions = new ArrayList<>();
+ for (S3Object version : snapshot.versions()) {
+ objectStore.get(versionedKey(bucketName, key, version.getVersionId())).ifPresent(versions::add);
+ }
+ return new KeyHistory(current, versions);
+ }
+
+ private static boolean sameWrite(S3Object stored, S3Object seen) {
+ if (stored == null || seen == null) {
+ return stored == seen;
+ }
+ return stored == seen
+ || (Objects.equals(stored.getVersionId(), seen.getVersionId())
+ && Objects.equals(stored.getDataGeneration(), seen.getDataGeneration())
+ && Objects.equals(stored.getLastModified(), seen.getLastModified())
+ && stored.isDeleteMarker() == seen.isDeleteMarker());
+ }
+
+ /**
+ * Decides which lifecycle actions are due on one key. A noncurrent version has been noncurrent
+ * since its successor, the next newer version, was created. Versions under Object Lock
+ * retention or a legal hold are never planned for deletion.
+ *
+ * <p>A key with no plain-key entry has only noncurrent versions. The newest of them stopped
+ * being current when its successor was deleted, a time Floci does not record, so it is not
+ * aged by NoncurrentDays; it still counts as a newer version for the ones below it.
+ */
+ private LifecyclePlan planLifecycle(S3LifecycleConfiguration lifecycle, String key, KeyHistory history,
+ Instant now) {
+ S3Object current = history.current();
+ String currentVersionId = current == null ? null : current.getVersionId();
+ List<S3Object> noncurrent = history.versions().stream()
+ .filter(version -> current == null || !Objects.equals(version.getVersionId(), currentVersionId))
+ .sorted(WRITE_ORDER.reversed())
+ .toList();
+ List<String> deletions = new ArrayList<>();
+ Instant successorCreated = current == null ? null : current.getLastModified();
+ for (int newer = 0; newer < noncurrent.size(); newer++) {
+ S3Object version = noncurrent.get(newer);
+ if (successorCreated != null && !lockedAgainstDeletion(version)
+ && lifecycle.expiresNoncurrentVersion(key, version.getTags(), version.getSize(),
+ successorCreated, newer, now)) {
+ deletions.add(version.getVersionId());
+ }
+ successorCreated = version.getLastModified();
+ }
+ if (current == null) {
+ return new LifecyclePlan(deletions, false);
+ }
+ if (current.isDeleteMarker()) {
+ if (currentVersionId != null && noncurrent.isEmpty()
+ && lifecycle.removesExpiredObjectDeleteMarker(key, current.getLastModified(), now)) {
+ deletions.add(currentVersionId);
+ }
+ return new LifecyclePlan(deletions, false);
+ }
+ S3LifecycleConfiguration.Expiry expiry = lifecycle.currentVersionExpiry(
+ key, current.getTags(), current.getSize(), current.getLastModified());
+ return new LifecyclePlan(deletions, expiry != null && !now.isBefore(expiry.date()));
+ }
+
+ private boolean lockedAgainstDeletion(S3Object version) {
+ try {
+ checkLockProtection(version, false);
+ return false;
+ } catch (AwsException expected) {
+ // Lifecycle cannot bypass retention, so a version DeleteObject would refuse is left alone.
+ return true;
+ }
+ }
+
+ /**
+ * Expires as DeleteObject would, reported as a lifecycle expiration: a null {@code versionId}
+ * expires the current version, a delete marker in a versioning-enabled bucket and a deletion
+ * otherwise, and a version id deletes that version permanently.
+ */
+ private void expireByLifecycle(Bucket bucket, String bucketName, String key, String versionId) {
+ try {
+ deleteObjectLocked(bucket, bucketName, key, versionId, false, LIFECYCLE_EXPIRATION_EVENT);
+ LOG.debugv("Lifecycle expired {0}/{1} v={2}", bucketName, key, versionId);
+ } catch (AwsException e) {
+ // Object Lock protects the object; lifecycle leaves it in place, as on S3.
+ LOG.debugv("Lifecycle left {0}/{1} v={2} in place: {3}", bucketName, key, versionId, e.getMessage());
+ }
+ }
+
+ /**
+ * Aborts the bucket's incomplete multipart uploads that an AbortIncompleteMultipartUpload rule
+ * has made due. Each abort holds the upload's lock and then the bucket monitor, the order
+ * DeleteBucket takes them in, and is applied only while the bucket is still the one the sweep
+ * started on and the rules still make it due, so an upload in a bucket deleted and created again
+ * under the same name is never aborted under the old bucket's rules.
+ */
+ private void abortIncompleteMultipartUploads(Bucket bucket, String bucketName, LifecycleRules rules, Instant now) {
+ String ownerAccountId = ownerId();
+ for (MultipartUpload upload : List.copyOf(multipartUploads.values())) {
+ if (!bucketName.equals(upload.getBucket())
+ || (upload.getOwnerAccountId() != null && !ownerAccountId.equals(upload.getOwnerAccountId()))
+ || upload.getInitiated() == null
+ || !rules.of(bucket).abortsUpload(upload.getKey(), upload.getInitiated(), now)) {
+ continue;
+ }
+ withMultipartOperationLock(bucketName, upload.getUploadId(), () -> {
+ synchronized (bucket) {
+ if (bucketStore.get(bucketName).orElse(null) == bucket
+ && multipartUploads.get(upload.getUploadId()) == upload
+ && rules.of(bucket).abortsUpload(upload.getKey(), upload.getInitiated(), now)) {
+ cleanupMultipart(upload.getUploadId());
+ LOG.debugv("Lifecycle aborted multipart upload {0}/{1}, uploadId={2}",
+ bucketName, upload.getKey(), upload.getUploadId());
+ }
+ }
+ });
+ }
}
public String getBucketAcl(String bucketName) {
@@ -5807,9 +6132,15 @@ public class S3Service implements Resettable, ResourceProvider {
objectNode.put("version-id", obj.getVersionId());
}
detail.put("request-id", UUID.randomUUID().toString());
- detail.put("requester", "aws:emulator");
- detail.put("source-ip-address", "127.0.0.1");
- detail.put("reason", eventName);
+ // A lifecycle deletion is S3's own: "source-ip-address - ... Only present for events
+ // triggered by an S3 request."
+ if (isLifecycleEvent(eventName)) {
+ detail.put("requester", ServicePrincipals.of("s3"));
+ } else {
+ detail.put("requester", "aws:emulator");
+ detail.put("source-ip-address", "127.0.0.1");
+ }
+ detail.put("reason", eventBridgeReason(eventName));
String deletionType = eventBridgeDeletionType(eventName);
if (deletionType != null) {
detail.put("deletion-type", deletionType);
@@ -5820,10 +6151,19 @@ public class S3Service implements Resettable, ResourceProvider {
}
}
+ /** The EventBridge {@code reason}: {@code Lifecycle Expiration} for a deletion by a lifecycle rule. */
+ private static String eventBridgeReason(String eventName) {
+ return isLifecycleEvent(eventName) ? "Lifecycle Expiration" : eventName;
+ }
+
+ private static boolean isLifecycleEvent(String eventName) {
+ return eventName.startsWith(LIFECYCLE_EXPIRATION_EVENT + ":");
+ }
+
private static String eventBridgeDeletionType(String eventName) {
return switch (eventName) {
- case "ObjectRemoved:Delete" -> "Permanently Deleted";
- case "ObjectRemoved:DeleteMarkerCreated" -> "Delete Marker Created";
+ case "ObjectRemoved:Delete", "LifecycleExpiration:Delete" -> "Permanently Deleted";
+ case "ObjectRemoved:DeleteMarkerCreated", "LifecycleExpiration:DeleteMarkerCreated" -> "Delete Marker Created";
default -> null;
};
}
@@ -5913,8 +6253,11 @@ public class S3Service implements Resettable, ResourceProvider {
record.put("awsRegion", region);
record.put("eventTime", eventTime);
record.put("eventName", eventName);
- record.putObject("userIdentity").put("principalId", "AWS:EMULATOR");
- record.putObject("requestParameters").put("sourceIPAddress", "127.0.0.1");
+ // The user guide's s3:LifecycleExpiration:Delete example names S3 itself as both.
+ boolean lifecycle = isLifecycleEvent(eventName);
+ record.putObject("userIdentity").put("principalId", lifecycle ? ServicePrincipals.of("s3") : "AWS:EMULATOR");
+ record.putObject("requestParameters")
+ .put("sourceIPAddress", lifecycle ? ServicePrincipals.of("s3") : "127.0.0.1");
record.putObject("responseElements").put("x-amz-request-id", requestId);
record.set("s3", s3Node);
@@ -6232,6 +6575,23 @@ public class S3Service implements Resettable, ResourceProvider {
* still refers to this object, checked under the bucket lock that puts and deletes hold, so a
* concurrent overwrite or delete marker is never replaced by the object it superseded.
*/
+ /**
+ * Replaces the current version's tags under the bucket monitor that writes, deletes and the
+ * lifecycle sweep hold, so the object is resolved and rewritten in one step: the sweep never
+ * sees the tags it matched a rule against change before it acts.
+ */
+ private void setCurrentObjectTags(String bucketName, String key, Map<String, String> tags) {
+ Bucket bucket = resolveBucket(bucketName)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404));
+ synchronized (bucket) {
+ AccountAwareStorageBackend.OwnedEntry<S3Object> ownedObject =
+ getStoredObjectEntry(bucketName, key, null);
+ ownedObject.value().setTags(tags);
+ putObjectMetadataForAccount(ownedObject.account(), bucketName, key, ownedObject.value());
+ }
+ }
+
private void putObjectMetadataForAccount(String accountId, String bucketName, String key, S3Object object) {
Bucket bucket = resolveBucket(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket",
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index e6c19b2511..9e9f081f21 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -363,6 +363,8 @@ floci:
enforce-auth: false
default-presign-expiry-seconds: 3600
global-bucket-namespace: false
+ lifecycle-sweep-enabled: true # FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_ENABLED - apply lifecycle expiration rules in the background
+ lifecycle-sweep-interval-seconds: 60 # FLOCI_SERVICES_S3_LIFECYCLE_SWEEP_INTERVAL_SECONDS
dynamodb:
enabled: true
vector-index-allocation-seconds: 4 # FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_ALLOCATION_SECONDS. Resource allocation phase of a vector index added by UpdateTable.
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java b/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java
index b4c626ecc3..d19f325e23 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java
@@ -201,6 +201,34 @@ class XmlParserTest {
assertNull(XmlParser.extractElementTree(null, "Target"));
}
+ // --- parseDocumentElementTree: the whole document or nothing ---
+
+ @Test
+ void parseDocumentElementTreeReturnsTheDocumentElement() {
+ XmlParser.XmlElement root = XmlParser.parseDocumentElementTree("""
+ <?xml version="1.0" encoding="UTF-8"?>
+ <Root xmlns="urn:test"><Item><Name>a</Name></Item></Root>
+ <!-- a trailing comment is well-formed -->
+ """);
+
+ assertEquals("Root", root.name());
+ assertEquals("a", root.child("Item").child("Name").text());
+ }
+
+ @Test
+ void parseDocumentElementTreeRejectsMalformedContentAfterTheDocumentElement() {
+ // extractElementTree stops at the end of the element it found; this keeps reading.
+ String trailingElement = "<Root><Item>a</Item></Root><Item>b</Item>";
+ String trailingText = "<Root><Item>a</Item></Root>junk";
+
+ assertNotNull(XmlParser.extractElementTree(trailingElement, "Root"));
+ assertNull(XmlParser.parseDocumentElementTree(trailingElement));
+ assertNull(XmlParser.parseDocumentElementTree(trailingText));
+ assertNull(XmlParser.parseDocumentElementTree("<Root><Item></Root>"));
+ assertNull(XmlParser.parseDocumentElementTree(""));
+ assertNull(XmlParser.parseDocumentElementTree(null));
+ }
+
// --- extractDeleteObjectEntries: batch delete Key/VersionId pairing ---
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java
index 8674e12d5b..4f2b9a9368 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.s3;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
@@ -22,6 +23,9 @@ class S3EventBridgeIntegrationTest {
private static String ruleArn;
private static String queueUrl;
+ @Inject
+ S3LifecycleSweeper lifecycleSweeper;
+
@BeforeAll
static void configureRestAssured() {
RestAssuredJsonUtils.configureAwsContentTypes();
@@ -317,6 +321,55 @@ class S3EventBridgeIntegrationTest {
containsString("\"deletion-type\":\"Delete Marker Created\"")));
}
+ @Test
+ @Order(11)
+ void lifecycleExpiration_eventBridgeDetailSaysLifecycleExpirationFromS3Itself() {
+ given().contentType("text/plain").body("short-lived").when().put("/" + VERSIONED_BUCKET + "/expiring.txt")
+ .then().statusCode(200);
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "ReceiveMessage")
+ .formParam("QueueUrl", queueUrl)
+ .formParam("MaxNumberOfMessages", "1")
+ .formParam("WaitTimeSeconds", "0")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body", containsString("Object Created"));
+ given()
+ .contentType("application/xml")
+ .body("<LifecycleConfiguration><Rule><ID>past</ID><Filter><Prefix>expiring</Prefix></Filter>"
+ + "<Status>Enabled</Status><Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"
+ + "</Rule></LifecycleConfiguration>")
+ .when()
+ .put("/" + VERSIONED_BUCKET + "?lifecycle")
+ .then()
+ .statusCode(200);
+
+ lifecycleSweeper.sweep();
+
+ // The user guide's lifecycle event: reason "Lifecycle Expiration", requester s3.amazonaws.com,
+ // and no source-ip-address, which is "Only present for events triggered by an S3 request".
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "ReceiveMessage")
+ .formParam("QueueUrl", queueUrl)
+ .formParam("MaxNumberOfMessages", "1")
+ .formParam("WaitTimeSeconds", "0")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body", allOf(
+ containsString("expiring.txt"),
+ containsString("\"reason\":\"Lifecycle Expiration\""),
+ containsString("\"deletion-type\":\"Delete Marker Created\""),
+ containsString("\"requester\":\"s3.amazonaws.com\""),
+ not(containsString("source-ip-address"))));
+ given().when().delete("/" + VERSIONED_BUCKET + "?lifecycle").then().statusCode(204);
+ }
+
@Test
@Order(100)
void cleanup() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfigurationTest.java
new file mode 100644
index 0000000000..1ddd784f44
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleConfigurationTest.java
@@ -0,0 +1,208 @@
+package io.github.hectorvent.floci.services.s3;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class S3LifecycleConfigurationTest {
+
+ private static final Instant NOW = Instant.parse("2026-10-06T12:00:00Z");
+
+ @Test
+ void dayCountsRoundUpToTheNextMidnightUtc() {
+ // The user guide's examples: created 1/15/2014 10:30 UTC plus 3 days is 1/19/2014 00:00 UTC.
+ assertEquals(Instant.parse("2014-01-19T00:00:00Z"),
+ S3LifecycleConfiguration.afterDays(Instant.parse("2014-01-15T10:30:00Z"), 3));
+ assertEquals(Instant.parse("2014-01-17T00:00:00Z"),
+ S3LifecycleConfiguration.afterDays(Instant.parse("2014-01-15T23:59:59.999Z"), 1));
+ }
+
+ @Test
+ void disabledRulesAndMalformedBodiesHaveNoRules() {
+ assertThat(parse(rule("off", "Disabled", "<Filter><Prefix></Prefix></Filter>",
+ "<Expiration><Days>1</Days></Expiration>")).rules(), empty());
+ assertThat(S3LifecycleConfiguration.parse("<LifecycleConfiguration><Rule>").rules(), empty());
+ assertThat(S3LifecycleConfiguration.parse("not xml").rules(), empty());
+ }
+
+ @Test
+ void theLegacyRulePrefixIsTheFilterWhenNoFilterIsGiven() {
+ S3LifecycleConfiguration config = S3LifecycleConfiguration.parse("<LifecycleConfiguration><Rule>"
+ + "<ID>legacy</ID><Prefix>logs/</Prefix><Status>Enabled</Status>"
+ + "<Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>");
+
+ assertTrue(config.rules().getFirst().filter().matches("logs/a", Map.of(), 1));
+ assertFalse(config.rules().getFirst().filter().matches("other/a", Map.of(), 1));
+ }
+
+ @Test
+ void anAndFilterNeedsThePrefixEveryTagAndTheExclusiveSizeRange() {
+ S3LifecycleConfiguration.Filter filter = parse(rule("and", "Enabled", "<Filter><And>"
+ + "<Prefix>tmp/</Prefix>"
+ + "<Tag><Key>env</Key><Value>dev</Value></Tag>"
+ + "<Tag><Key>flag</Key><Value></Value></Tag>"
+ + "<ObjectSizeGreaterThan>10</ObjectSizeGreaterThan>"
+ + "<ObjectSizeLessThan>20</ObjectSizeLessThan>"
+ + "</And></Filter>", "<Expiration><Days>1</Days></Expiration>")).rules().getFirst().filter();
+ Map<String, String> tags = Map.of("env", "dev", "flag", "", "extra", "x");
+
+ assertTrue(filter.matches("tmp/a", tags, 15));
+ assertFalse(filter.matches("keep/a", tags, 15));
+ assertFalse(filter.matches("tmp/a", Map.of("env", "dev", "flag", ""), 10));
+ assertFalse(filter.matches("tmp/a", Map.of("env", "dev", "flag", ""), 20));
+ assertFalse(filter.matches("tmp/a", Map.of("env", "prod", "flag", ""), 15));
+ assertFalse(filter.matches("tmp/a", Map.of("env", "dev"), 15));
+ assertFalse(filter.matches("tmp/a", Map.of("env", "dev", "flag", "set"), 15));
+ }
+
+ @Test
+ void anObjectGetsTheEarliestExpiryOfTheRulesThatMatchIt() {
+ S3LifecycleConfiguration config = parse(
+ rule("ten days", "Enabled", "<Filter><Prefix></Prefix></Filter>",
+ "<Expiration><Days>10</Days></Expiration>")
+ + rule("by date", "Enabled", "<Filter><Prefix>tmp/</Prefix></Filter>",
+ "<Expiration><Date>2026-10-08T00:00:00.000Z</Date></Expiration>")
+ + rule("transition only", "Enabled", "<Filter><Prefix></Prefix></Filter>",
+ "<Transition><Days>1</Days><StorageClass>GLACIER</StorageClass></Transition>"));
+
+ S3LifecycleConfiguration.Expiry tmp = config.currentVersionExpiry("tmp/a", Map.of(), 1, NOW);
+ assertEquals(Instant.parse("2026-10-08T00:00:00Z"), tmp.date());
+ assertEquals("expiry-date=\"Thu, 08 Oct 2026 00:00:00 GMT\", rule-id=\"by%20date\"", tmp.header());
+
+ S3LifecycleConfiguration.Expiry other = config.currentVersionExpiry("other/a", Map.of(), 1, NOW);
+ assertEquals(Instant.parse("2026-10-17T00:00:00Z"), other.date());
+ assertEquals("ten days", other.ruleId());
+ }
+
+ @Test
+ void anExpirationDateMayBeWrittenAsAPlainDate() {
+ S3LifecycleConfiguration config = parse(rule("plain", "Enabled", "<Filter></Filter>",
+ "<Expiration><Date>2020-01-01</Date></Expiration>"));
+
+ assertEquals(Instant.parse("2020-01-01T00:00:00Z"),
+ config.currentVersionExpiry("a", Map.of(), 1, NOW).date());
+ }
+
+ @Test
+ void aRuleS3WouldRefuseIsDroppedWholeAndTheOtherRulesStillApply() {
+ String good = rule("good", "Enabled", "<Filter></Filter>", "<Expiration><Days>1</Days></Expiration>");
+ List<String> refused = List.of(
+ rule("days soon", "Enabled", "<Filter></Filter>", "<Expiration><Days>soon</Days></Expiration>"),
+ rule("days zero", "Enabled", "<Filter></Filter>", "<Expiration><Days>0</Days></Expiration>"),
+ rule("bad date", "Enabled", "<Filter></Filter>", "<Expiration><Date>someday</Date></Expiration>"),
+ rule("marker and days", "Enabled", "<Filter></Filter>", "<Expiration><Days>1</Days>"
+ + "<ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"),
+ rule("marker maybe", "Enabled", "<Filter></Filter>",
+ "<Expiration><ExpiredObjectDeleteMarker>maybe</ExpiredObjectDeleteMarker></Expiration>"),
+ rule("size word", "Enabled", "<Filter><ObjectSizeGreaterThan>big</ObjectSizeGreaterThan></Filter>",
+ "<Expiration><Days>1</Days></Expiration>"),
+ rule("size negative", "Enabled", "<Filter><ObjectSizeLessThan>-1</ObjectSizeLessThan></Filter>",
+ "<Expiration><Days>1</Days></Expiration>"),
+ rule("tag without value", "Enabled", "<Filter><Tag><Key>env</Key></Tag></Filter>",
+ "<Expiration><Days>1</Days></Expiration>"),
+ rule("tag twice", "Enabled", "<Filter><And><Tag><Key>env</Key><Value>a</Value></Tag>"
+ + "<Tag><Key>env</Key><Value>b</Value></Tag></And></Filter>",
+ "<Expiration><Days>1</Days></Expiration>"),
+ "<Rule><ID>no filter</ID><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule>",
+ rule("noncurrent zero", "Enabled", "<Filter></Filter>",
+ "<NoncurrentVersionExpiration><NoncurrentDays>0</NoncurrentDays></NoncurrentVersionExpiration>"),
+ rule("retain too many", "Enabled", "<Filter></Filter>", "<NoncurrentVersionExpiration>"
+ + "<NoncurrentDays>1</NoncurrentDays><NewerNoncurrentVersions>101</NewerNoncurrentVersions>"
+ + "</NoncurrentVersionExpiration>"),
+ "<Rule><ID>retain without filter</ID><Prefix></Prefix><Status>Enabled</Status>"
+ + "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays>"
+ + "<NewerNoncurrentVersions>1</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule>",
+ rule("abort negative", "Enabled", "<Filter></Filter>", "<AbortIncompleteMultipartUpload>"
+ + "<DaysAfterInitiation>-1</DaysAfterInitiation></AbortIncompleteMultipartUpload>"),
+ rule("status typo", "enabled", "<Filter></Filter>", "<Expiration><Days>1</Days></Expiration>"));
+
+ for (String bad : refused) {
+ List<S3LifecycleConfiguration.Rule> rules = parse(bad + good).rules();
+ assertEquals(List.of("good"), rules.stream().map(S3LifecycleConfiguration.Rule::id).toList(), bad);
+ }
+ }
+
+ @Test
+ void aBodyWithMalformedContentAfterTheConfigurationHasNoRules() {
+ String rules = rule("past", "Enabled", "<Filter></Filter>",
+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>");
+
+ assertThat(S3LifecycleConfiguration.parse(
+ "<LifecycleConfiguration>" + rules + "</LifecycleConfiguration><Rule>").rules(), empty());
+ assertThat(S3LifecycleConfiguration.parse(
+ "<LifecycleConfiguration>" + rules + "</LifecycleConfiguration>trailing").rules(), empty());
+ assertThat(S3LifecycleConfiguration.parse("<Other>" + rules + "</Other>").rules(), empty());
+ }
+
+ @Test
+ void anExpiredObjectDeleteMarkerGoesAtOnceOrOnceOldEnoughForADaysRule() {
+ S3LifecycleConfiguration explicit = parse(rule("markers", "Enabled", "<Filter></Filter>",
+ "<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"));
+ S3LifecycleConfiguration days = parse(rule("days", "Enabled", "<Filter></Filter>",
+ "<Expiration><Days>2</Days></Expiration>"));
+
+ assertTrue(explicit.removesExpiredObjectDeleteMarker("a", NOW, NOW));
+ assertNull(explicit.currentVersionExpiry("a", Map.of(), 1, NOW));
+ assertFalse(days.removesExpiredObjectDeleteMarker("a", NOW, Instant.parse("2026-10-08T23:59:59Z")));
+ assertTrue(days.removesExpiredObjectDeleteMarker("a", NOW, Instant.parse("2026-10-09T00:00:00Z")));
+ }
+
+ @Test
+ void aNoncurrentVersionNeedsBothItsAgeAndEnoughNewerVersionsRetained() {
+ S3LifecycleConfiguration config = parse(rule("noncurrent", "Enabled", "<Filter></Filter>",
+ "<NoncurrentVersionExpiration><NoncurrentDays>3</NoncurrentDays>"
+ + "<NewerNoncurrentVersions>2</NewerNoncurrentVersions></NoncurrentVersionExpiration>"));
+ Instant becameNoncurrent = Instant.parse("2014-01-15T10:30:00Z");
+ Instant due = Instant.parse("2014-01-19T00:00:00Z");
+
+ assertTrue(config.expiresNoncurrentVersion("a", Map.of(), 1, becameNoncurrent, 2, due));
+ assertFalse(config.expiresNoncurrentVersion("a", Map.of(), 1, becameNoncurrent, 1, due));
+ assertFalse(config.expiresNoncurrentVersion("a", Map.of(), 1, becameNoncurrent, 2,
+ due.minusMillis(1)));
+ }
+
+ @Test
+ void onlyAPrefixFilterSelectsMultipartUploadsToAbort() {
+ S3LifecycleConfiguration prefix = parse(rule("abort", "Enabled", "<Filter><Prefix>big/</Prefix></Filter>",
+ "<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "</AbortIncompleteMultipartUpload>"));
+ S3LifecycleConfiguration tagged = parse(rule("abort", "Enabled",
+ "<Filter><Tag><Key>k</Key><Value>v</Value></Tag></Filter>",
+ "<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "</AbortIncompleteMultipartUpload>"));
+ Instant due = Instant.parse("2026-10-08T00:00:00Z");
+
+ assertTrue(prefix.abortsUpload("big/a", NOW, due));
+ assertFalse(prefix.abortsUpload("big/a", NOW, due.minusMillis(1)));
+ assertFalse(prefix.abortsUpload("small/a", NOW, due));
+ assertFalse(tagged.abortsUpload("big/a", NOW, due));
+ }
+
+ @Test
+ void everyEnabledRuleIsRead() {
+ assertThat(parse(rule("a", "Enabled", "<Filter></Filter>", "<Expiration><Days>1</Days></Expiration>")
+ + rule("b", "Enabled", "<Filter></Filter>", "<Expiration><Days>2</Days></Expiration>")
+ + rule("c", "Disabled", "<Filter></Filter>", "<Expiration><Days>3</Days></Expiration>")).rules(),
+ hasSize(2));
+ }
+
+ private static S3LifecycleConfiguration parse(String rules) {
+ return S3LifecycleConfiguration.parse(
+ "<LifecycleConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
+ + rules + "</LifecycleConfiguration>");
+ }
+
+ private static String rule(String id, String status, String filter, String actions) {
+ return "<Rule><ID>" + id + "</ID>" + filter + "<Status>" + status + "</Status>" + actions + "</Rule>";
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleExpirationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleExpirationIntegrationTest.java
new file mode 100644
index 0000000000..96f15e736c
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleExpirationIntegrationTest.java
@@ -0,0 +1,155 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * Lifecycle expiration as a client sees it: the {@code x-amz-expiration} header on objects a rule
+ * matches, and the sweep removing what has expired. The test profile does not schedule the sweep,
+ * so each test runs it once, at the point it expects the rules applied.
+ */
+@QuarkusTest
+class S3LifecycleExpirationIntegrationTest {
+
+ @Inject
+ S3LifecycleSweeper sweeper;
+
+ @Test
+ void objectsMatchedByADaysRuleCarryTheRoundedExpiryDateAndRuleId() {
+ String bucket = "lc-expiration-header";
+ given().put("/" + bucket).then().statusCode(200);
+ putLifecycle(bucket, rule("three days", "<Filter><Prefix>tmp/</Prefix></Filter>",
+ "<Expiration><Days>3</Days></Expiration>"));
+
+ Response put = given().body("hi").put("/" + bucket + "/tmp/a.txt");
+ put.then().statusCode(200);
+ Response head = given().head("/" + bucket + "/tmp/a.txt");
+ head.then().statusCode(200);
+
+ // Creation time plus three days, rounded up to the next midnight UTC.
+ Instant created = ZonedDateTime.parse(head.header("Last-Modified"),
+ DateTimeFormatter.RFC_1123_DATE_TIME).toInstant();
+ String expected = "expiry-date=\"" + S3Controller.RFC_822.format(
+ created.plus(3, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).plus(1, ChronoUnit.DAYS))
+ + "\", rule-id=\"three%20days\"";
+ head.then().header("x-amz-expiration", equalTo(expected));
+ put.then().header("x-amz-expiration", equalTo(expected));
+ given().get("/" + bucket + "/tmp/a.txt").then()
+ .statusCode(200)
+ .header("x-amz-expiration", equalTo(expected));
+
+ given().body("hi").put("/" + bucket + "/keep/b.txt").then()
+ .statusCode(200)
+ .header("x-amz-expiration", nullValue());
+ given().head("/" + bucket + "/keep/b.txt").then()
+ .statusCode(200)
+ .header("x-amz-expiration", nullValue());
+ }
+
+ @Test
+ void aDateRuleReportsThatDateAsTheExpiryDate() {
+ String bucket = "lc-expiration-date-header";
+ given().put("/" + bucket).then().statusCode(200);
+ putLifecycle(bucket, rule("future-date", "<Filter><Prefix></Prefix></Filter>",
+ "<Expiration><Date>2099-01-01T00:00:00Z</Date></Expiration>"));
+ given().body("hi").put("/" + bucket + "/a.txt").then().statusCode(200);
+
+ given().head("/" + bucket + "/a.txt").then()
+ .statusCode(200)
+ .header("x-amz-expiration",
+ equalTo("expiry-date=\"Thu, 01 Jan 2099 00:00:00 GMT\", rule-id=\"future-date\""));
+ }
+
+ @Test
+ void theSweepDeletesObjectsPastTheirExpirationAndLeavesOthers() {
+ String bucket = "lc-expiration-sweep";
+ given().put("/" + bucket).then().statusCode(200);
+ given().body("hi").put("/" + bucket + "/tmp/a.txt").then().statusCode(200);
+ given().body("hi").put("/" + bucket + "/keep/b.txt").then().statusCode(200);
+ putLifecycle(bucket, rule("past-date", "<Filter><Prefix>tmp/</Prefix></Filter>",
+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));
+
+ sweeper.sweep();
+
+ given().head("/" + bucket + "/tmp/a.txt").then().statusCode(404);
+ given().get("/" + bucket + "?list-type=2").then()
+ .statusCode(200)
+ .body(not(containsString("<Key>tmp/a.txt</Key>")))
+ .body(containsString("<Key>keep/b.txt</Key>"));
+ }
+
+ @Test
+ void theSweepPlacesADeleteMarkerInAVersionedBucket() {
+ String bucket = "lc-expiration-versioned";
+ given().put("/" + bucket).then().statusCode(200);
+ given().body("<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>")
+ .put("/" + bucket + "?versioning").then().statusCode(200);
+ String versionId = given().body("hi").put("/" + bucket + "/tmp/a.txt")
+ .then().statusCode(200).extract().header("x-amz-version-id");
+ putLifecycle(bucket, rule("past-date", "<Filter><Prefix>tmp/</Prefix></Filter>",
+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));
+
+ sweeper.sweep();
+
+ given().head("/" + bucket + "/tmp/a.txt").then().statusCode(404);
+ given().get("/" + bucket + "?versions").then()
+ .statusCode(200)
+ .body(containsString("<DeleteMarker>"))
+ .body(containsString("<VersionId>" + versionId + "</VersionId>"));
+ given().get("/" + bucket + "/tmp/a.txt?versionId=" + versionId).then().statusCode(200);
+ }
+
+ @Test
+ void theSweepAppliesEachAccountsRulesToThatAccountsBucket() {
+ String bucket = "lc-expiration-accounts";
+ String withRule = "AWS4-HMAC-SHA256 Credential=000000000002/20260215/us-east-1/s3/aws4_request, "
+ + "SignedHeaders=host, Signature=abc";
+ String withoutRule = "AWS4-HMAC-SHA256 Credential=000000000003/20260215/us-east-1/s3/aws4_request, "
+ + "SignedHeaders=host, Signature=abc";
+ for (String auth : new String[] {withRule, withoutRule}) {
+ given().header("Authorization", auth).put("/" + bucket).then().statusCode(200);
+ given().header("Authorization", auth).body("hi").put("/" + bucket + "/a.txt").then().statusCode(200);
+ }
+ given()
+ .header("Authorization", withRule)
+ .contentType("application/xml")
+ .body("<LifecycleConfiguration>" + rule("past-date", "<Filter><Prefix></Prefix></Filter>",
+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>") + "</LifecycleConfiguration>")
+ .when()
+ .put("/" + bucket + "?lifecycle")
+ .then()
+ .statusCode(200);
+
+ sweeper.sweep();
+
+ given().header("Authorization", withRule).head("/" + bucket + "/a.txt").then().statusCode(404);
+ given().header("Authorization", withoutRule).head("/" + bucket + "/a.txt").then().statusCode(200);
+ }
+
+ private static String rule(String id, String filter, String actions) {
+ return "<Rule><ID>" + id + "</ID>" + filter + "<Status>Enabled</Status>" + actions + "</Rule>";
+ }
+
+ private static void putLifecycle(String bucket, String rules) {
+ given()
+ .contentType("application/xml")
+ .body("<LifecycleConfiguration>" + rules + "</LifecycleConfiguration>")
+ .when()
+ .put("/" + bucket + "?lifecycle")
+ .then()
+ .statusCode(200);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleServiceTest.java
new file mode 100644
index 0000000000..2c63f2acb5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleServiceTest.java
@@ -0,0 +1,348 @@
+package io.github.hectorvent.floci.services.s3;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.s3.model.LambdaNotification;
+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;
+import io.github.hectorvent.floci.services.s3.model.NotificationConfiguration;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The lifecycle sweep, driven directly with a chosen "now" instead of waiting on the scheduler.
+ */
+class S3LifecycleServiceTest {
+
+ private static final String BUCKET = "lifecycle-bucket";
+
+ @TempDir
+ Path tempDir;
+
+ private RecordingLambdaInvoker notifications;
+ private S3Service s3Service;
+
+ @BeforeEach
+ void setUp() {
+ notifications = new RecordingLambdaInvoker();
+ s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve("s3"), false,
+ notifications, new RegionResolver("us-east-1", "000000000000"));
+ s3Service.createBucket(BUCKET, "us-east-1");
+ }
+
+ @Test
+ void aDaysRuleDeletesMatchingObjectsOnlyOnceTheRoundedExpiryHasPassed() {
+ put("tmp/a.txt");
+ put("keep/b.txt");
+ lifecycle(rule("Enabled", "<Filter><Prefix>tmp/</Prefix></Filter>", "<Expiration><Days>1</Days></Expiration>"));
+ Instant created = s3Service.headObject(BUCKET, "tmp/a.txt").getLastModified();
+ Instant due = S3LifecycleConfiguration.afterDays(created, 1);
+
+ s3Service.applyLifecycleExpiration(due.minusMillis(1));
+ assertTrue(s3Service.objectExists(BUCKET, "tmp/a.txt"));
+
+ s3Service.applyLifecycleExpiration(due);
+ assertFalse(s3Service.objectExists(BUCKET, "tmp/a.txt"));
+ assertTrue(s3Service.objectExists(BUCKET, "keep/b.txt"));
+ }
+
+ @Test
+ void aDisabledRuleIsNotApplied() {
+ put("a.txt");
+ lifecycle(rule("Disabled", "<Filter></Filter>", "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now());
+
+ assertTrue(s3Service.objectExists(BUCKET, "a.txt"));
+ }
+
+ @Test
+ void aTagFilterExpiresOnlyObjectsCarryingTheTag() {
+ put("tagged.txt");
+ put("untagged.txt");
+ s3Service.putObjectTagging(BUCKET, "tagged.txt", Map.of("env", "ephemeral"));
+ lifecycle(rule("Enabled", "<Filter><Tag><Key>env</Key><Value>ephemeral</Value></Tag></Filter>",
+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now());
+
+ assertFalse(s3Service.objectExists(BUCKET, "tagged.txt"));
+ assertTrue(s3Service.objectExists(BUCKET, "untagged.txt"));
+ }
+
+ @Test
+ void inAVersionedBucketTheCurrentVersionGetsADeleteMarkerOnce() {
+ s3Service.putBucketVersioning(BUCKET, "Enabled");
+ S3Object original = put("a.txt");
+ lifecycle(rule("Enabled", "<Filter></Filter>", "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now());
+ s3Service.applyLifecycleExpiration(Instant.now());
+
+ List<S3Object> versions = versions("a.txt");
+ assertEquals(2, versions.size(), "the original version and one delete marker");
+ assertTrue(versions.getFirst().isDeleteMarker());
+ assertEquals(original.getVersionId(), versions.get(1).getVersionId());
+ assertFalse(s3Service.objectExists(BUCKET, "a.txt"));
+ }
+
+ @Test
+ void noncurrentVersionsBeyondTheRetainedCountExpireOnceOldEnough() {
+ s3Service.putBucketVersioning(BUCKET, "Enabled");
+ S3Object v1 = put("a.txt");
+ S3Object v2 = put("a.txt");
+ S3Object v3 = put("a.txt");
+ lifecycle(rule("Enabled", "<Filter></Filter>", "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays>"
+ + "<NewerNoncurrentVersions>1</NewerNoncurrentVersions></NoncurrentVersionExpiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now());
+ assertThat(versionIds("a.txt"), contains(v3.getVersionId(), v2.getVersionId(), v1.getVersionId()));
+
+ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(3)));
+ assertThat(versionIds("a.txt"), contains(v3.getVersionId(), v2.getVersionId()));
+ }
+
+ @Test
+ void aNoncurrentVersionUnderALegalHoldIsKept() {
+ s3Service.putBucketVersioning(BUCKET, "Enabled");
+ S3Object held = s3Service.putObject(BUCKET, "a.txt", "held".getBytes(StandardCharsets.UTF_8), "text/plain",
+ Map.of(), null, null, "ON");
+ S3Object current = put("a.txt");
+ lifecycle(rule("Enabled", "<Filter></Filter>",
+ "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(3)));
+
+ assertThat(versionIds("a.txt"), containsInAnyOrder(current.getVersionId(), held.getVersionId()));
+ }
+
+ @Test
+ void anExpiredObjectDeleteMarkerIsRemoved() {
+ s3Service.putBucketVersioning(BUCKET, "Enabled");
+ S3Object only = put("a.txt");
+ s3Service.deleteObject(BUCKET, "a.txt");
+ s3Service.deleteObject(BUCKET, "a.txt", only.getVersionId());
+ assertEquals(1, versions("a.txt").size(), "only the delete marker remains");
+ lifecycle(rule("Enabled", "<Filter></Filter>",
+ "<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now());
+
+ assertThat(versions("a.txt"), empty());
+ }
+
+ @Test
+ void aDeleteMarkerWithNoncurrentVersionsIsKept() {
+ s3Service.putBucketVersioning(BUCKET, "Enabled");
+ put("a.txt");
+ s3Service.deleteObject(BUCKET, "a.txt");
+ lifecycle(rule("Enabled", "<Filter></Filter>",
+ "<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now());
+
+ assertEquals(2, versions("a.txt").size());
+ }
+
+ @Test
+ void incompleteMultipartUploadsAreAbortedAfterTheirDays() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload(BUCKET, "big/a.bin", "application/octet-stream");
+ lifecycle(rule("Enabled", "<Filter><Prefix>big/</Prefix></Filter>",
+ "<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "</AbortIncompleteMultipartUpload>"));
+ Instant due = S3LifecycleConfiguration.afterDays(upload.getInitiated(), 1);
+
+ s3Service.applyLifecycleExpiration(due.minusMillis(1));
+ assertEquals(1, s3Service.listMultipartUploads(BUCKET).size());
+
+ s3Service.applyLifecycleExpiration(due);
+ assertThat(s3Service.listMultipartUploads(BUCKET), empty());
+ }
+
+ @Test
+ void lifecycleDeletionsAreReportedAsLifecycleExpirationEvents() throws IOException {
+ put("a.txt");
+ NotificationConfiguration config = new NotificationConfiguration();
+ config.getLambdaFunctionConfigurations().add(new LambdaNotification("removed",
+ "arn:aws:lambda:us-east-1:000000000000:function:removed", List.of("s3:ObjectRemoved:*"), List.of()));
+ config.getLambdaFunctionConfigurations().add(new LambdaNotification("lifecycle",
+ "arn:aws:lambda:us-east-1:000000000000:function:lifecycle",
+ List.of("s3:LifecycleExpiration:*"), List.of()));
+ s3Service.putBucketNotificationConfiguration(BUCKET, config, true);
+ lifecycle(rule("Enabled", "<Filter></Filter>", "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now());
+
+ assertEquals(List.of("lifecycle"), notifications.functionNames);
+ JsonNode record = new ObjectMapper().readTree(notifications.payloads.getFirst()).path("Records").get(0);
+ assertEquals("LifecycleExpiration:Delete", record.path("eventName").asText());
+ // The user guide's example names S3 itself, not a requester.
+ assertEquals("s3.amazonaws.com", record.path("userIdentity").path("principalId").asText());
+ assertEquals("s3.amazonaws.com", record.path("requestParameters").path("sourceIPAddress").asText());
+ }
+
+ @Test
+ void versionsWrittenInTheSameMillisecondKeepTheOrderTheyWereWrittenIn() {
+ s3Service.putBucketVersioning(BUCKET, "Enabled");
+ S3Object v1 = put("a.txt");
+ S3Object v2 = put("a.txt");
+ S3Object v3 = put("a.txt");
+ Instant sameMillisecond = Instant.parse("2026-10-01T10:00:00Z");
+ for (S3Object version : List.of(v1, v2, v3)) {
+ version.setLastModified(sameMillisecond);
+ }
+ lifecycle(rule("Enabled", "<Filter></Filter>", "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays>"
+ + "<NewerNoncurrentVersions>1</NewerNoncurrentVersions></NoncurrentVersionExpiration>"));
+
+ assertThat(versionIds("a.txt"), contains(v3.getVersionId(), v2.getVersionId(), v1.getVersionId()));
+ s3Service.applyLifecycleExpiration(sameMillisecond.plus(Duration.ofDays(3)));
+
+ assertThat(versionIds("a.txt"), contains(v3.getVersionId(), v2.getVersionId()));
+ }
+
+ @Test
+ void aKeyLeftWithOnlyVersionsInASuspendedBucketHasItsOlderVersionsExpired() {
+ s3Service.putBucketVersioning(BUCKET, "Enabled");
+ S3Object v1 = put("a.txt");
+ S3Object v2 = put("a.txt");
+ s3Service.putBucketVersioning(BUCKET, "Suspended");
+ s3Service.deleteObject(BUCKET, "a.txt");
+ assertThat(versionIds("a.txt"), contains(v2.getVersionId(), v1.getVersionId()));
+ lifecycle(rule("Enabled", "<Filter></Filter>",
+ "<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(3)));
+
+ // v1 has been noncurrent since v2 was written. When v2 stopped being current is not recorded.
+ assertThat(versionIds("a.txt"), contains(v2.getVersionId()));
+ }
+
+ @Test
+ void aRuleWithAValueS3WouldRefuseExpiresNothing() {
+ put("small.txt");
+ lifecycle(rule("Enabled", "<Filter><ObjectSizeGreaterThan>large</ObjectSizeGreaterThan></Filter>",
+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>")
+ + rule("Enabled", "<Filter></Filter>", "<Expiration><Days>0</Days></Expiration>"));
+
+ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(3)));
+
+ assertTrue(s3Service.objectExists(BUCKET, "small.txt"));
+ assertNull(s3Service.expirationHeader(s3Service.headObject(BUCKET, "small.txt")));
+ }
+
+ @Test
+ void aBucketThatFailsDoesNotStopTheOthersAndTheRunReportsIt() {
+ S3Service failing = new S3Service(new InMemoryStorage<>(), new FailingDeletes("broken/"),
+ tempDir.resolve("failing"), false, notifications, new RegionResolver("us-east-1", "000000000000"));
+ for (String bucket : List.of("broken", "healthy")) {
+ failing.createBucket(bucket, "us-east-1");
+ failing.putObject(bucket, "a.txt", "data".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ failing.putBucketLifecycle(bucket, "<LifecycleConfiguration>" + rule("Enabled", "<Filter></Filter>",
+ "<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration>") + "</LifecycleConfiguration>", null);
+ }
+
+ IllegalStateException failure = assertThrows(IllegalStateException.class,
+ () -> failing.applyLifecycleExpiration(Instant.now()));
+
+ assertTrue(failure.getMessage().contains("[broken]"), failure.getMessage());
+ assertEquals("disk failure", failure.getSuppressed()[0].getMessage());
+ assertFalse(failing.objectExists("healthy", "a.txt"));
+ assertTrue(failing.objectExists("broken", "a.txt"));
+ }
+
+ @Test
+ void theExpirationHeaderDescribesOnlyTheCurrentVersion() {
+ s3Service.putBucketVersioning(BUCKET, "Enabled");
+ S3Object older = put("a.txt");
+ put("a.txt");
+ lifecycle(rule("Enabled", "<Filter></Filter>", "<Expiration><Date>2099-01-01T00:00:00Z</Date></Expiration>"));
+
+ assertEquals("expiry-date=\"Thu, 01 Jan 2099 00:00:00 GMT\", rule-id=\"rule\"",
+ s3Service.expirationHeader(s3Service.headObject(BUCKET, "a.txt")));
+ assertNull(s3Service.expirationHeader(s3Service.headObject(BUCKET, "a.txt", older.getVersionId())));
+ }
+
+ @Test
+ void aBucketWithoutALifecycleConfigurationIsLeftAlone() {
+ put("a.txt");
+
+ s3Service.applyLifecycleExpiration(Instant.now().plus(Duration.ofDays(10_000)));
+
+ assertTrue(s3Service.objectExists(BUCKET, "a.txt"));
+ assertNull(s3Service.expirationHeader(s3Service.headObject(BUCKET, "a.txt")));
+ assertNotNull(s3Service.headObject(BUCKET, "a.txt"));
+ }
+
+ private S3Object put(String key) {
+ return s3Service.putObject(BUCKET, key, "data".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ }
+
+ private void lifecycle(String rules) {
+ s3Service.putBucketLifecycle(BUCKET, "<LifecycleConfiguration>" + rules + "</LifecycleConfiguration>", null);
+ }
+
+ private static String rule(String status, String filter, String actions) {
+ return "<Rule><ID>rule</ID>" + filter + "<Status>" + status + "</Status>" + actions + "</Rule>";
+ }
+
+ /** Newest first. */
+ private List<S3Object> versions(String key) {
+ return s3Service.listObjectVersions(BUCKET, key, 1000, null).versions();
+ }
+
+ private List<String> versionIds(String key) {
+ return versions(key).stream().map(S3Object::getVersionId).toList();
+ }
+
+ /** An object store whose deletes fail under one bucket, as a disk error would. */
+ private static final class FailingDeletes extends InMemoryStorage<String, S3Object> {
+ private final String failingPrefix;
+
+ FailingDeletes(String failingPrefix) {
+ this.failingPrefix = failingPrefix;
+ }
+
+ @Override
+ public void delete(String key) {
+ if (key.startsWith(failingPrefix)) {
+ throw new IllegalStateException("disk failure");
+ }
+ super.delete(key);
+ }
+ }
+
+ private static final class RecordingLambdaInvoker implements S3Service.LambdaInvoker {
+ private final List<String> functionNames = new ArrayList<>();
+ private final List<byte[]> payloads = new ArrayList<>();
+
+ @Override
+ public void invoke(String region, String functionName, byte[] payload, InvocationType type) {
+ functionNames.add(functionName);
+ payloads.add(payload);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeperTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeperTest.java
new file mode 100644
index 0000000000..41283801a1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LifecycleSweeperTest.java
@@ -0,0 +1,61 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.config.EmulatorConfig.S3ServiceConfig;
+import io.github.hectorvent.floci.config.EmulatorConfig.ServicesConfig;
+import io.quarkus.runtime.ShutdownEvent;
+import io.quarkus.runtime.StartupEvent;
+import org.junit.jupiter.api.Test;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+class S3LifecycleSweeperTest {
+
+ private static final Clock CLOCK = Clock.fixed(Instant.parse("2026-10-06T12:00:00Z"), ZoneOffset.UTC);
+
+ @Test
+ void aFailingSweepDoesNotEscapeTheScheduledTask() {
+ S3Service s3Service = mock(S3Service.class);
+ doThrow(new IllegalStateException("sweep failed")).when(s3Service).applyLifecycleExpiration(CLOCK.instant());
+ S3LifecycleSweeper sweeper = new S3LifecycleSweeper(s3Service, configWithInterval(60L), CLOCK);
+
+ assertDoesNotThrow(sweeper::sweep);
+ verify(s3Service).applyLifecycleExpiration(CLOCK.instant());
+ }
+
+ @Test
+ void nonPositiveConfiguredInterval_doesNotPreventStartup() {
+ for (long interval : new long[] {0L, -1L, Long.MIN_VALUE}) {
+ S3LifecycleSweeper sweeper = new S3LifecycleSweeper(null, configWithInterval(interval), CLOCK);
+ assertDoesNotThrow(() -> sweeper.onStart(new StartupEvent()),
+ "interval " + interval + " must not abort startup");
+ sweeper.onStop(new ShutdownEvent());
+ }
+ }
+
+ private static EmulatorConfig configWithInterval(long intervalSeconds) {
+ EmulatorConfig config = configView(EmulatorConfig.class);
+ ServicesConfig services = configView(ServicesConfig.class);
+ S3ServiceConfig s3 = configView(S3ServiceConfig.class);
+ doReturn(services).when(config).services();
+ doReturn(s3).when(services).s3();
+ doReturn(true).when(s3).enabled();
+ doReturn(true).when(s3).lifecycleSweepEnabled();
+ doReturn(intervalSeconds).when(s3).lifecycleSweepIntervalSeconds();
+ return config;
+ }
+
+ private static <T> T configView(Class<T> configType) {
+ return mock(configType, invocation -> {
+ throw new AssertionError("Unexpected configuration access: " + invocation.getMethod());
+ });
+ }
+}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index f418fa26da..490a51f8ee 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -138,6 +138,8 @@ floci:
enforce-auth: false
default-presign-expiry-seconds: 3600
global-bucket-namespace: false
+ lifecycle-sweep-enabled: false # tests run the sweep themselves, so no bucket's rules apply behind another test's back
+ lifecycle-sweep-interval-seconds: 60
dynamodb:
enabled: true
vector-index-allocation-seconds: 0