Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): honor per-object VersionId in batch DeleteObjects (#3232)
Ricky Rattlesnaked9cddc56cb40
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java b/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.javaindex 97306cf343..16c55f4188 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/XmlParser.java@@ -175,6 +175,63 @@ public final class XmlParser {return extractAll(xml, elementName).stream().anyMatch(value::equals);}+ /** A key paired with an optional version id, extracted from a {@code Delete} request's {@code <Object>} block. */+ public record KeyVersion(String key, String versionId) {}++ /**+ * Extracts {@code (Key, VersionId)} pairs from an S3 {@code DeleteObjects} request body,+ * one entry per {@code <Object>} block.+ *+ * <p>Unlike {@link #extractAll(String, String)}, which flattens every {@code <Key>} across+ * the whole document, this keeps each key paired with the {@code VersionId} from the same+ * {@code <Object>} block. This is needed so a batch delete can target the exact version named,+ * rather than discarding it and always falling back to "no version specified".+ *+ * <p>{@code <VersionId>} is optional per the {@code Delete} request schema; an {@code <Object>}+ * block that omits it yields an entry with a {@code null} versionId.+ *+ * <pre>{@code+ * List<XmlParser.KeyVersion> entries = XmlParser.extractDeleteObjectEntries(body);+ * }</pre>+ */+ public static List<KeyVersion> extractDeleteObjectEntries(String xml) {+ List<KeyVersion> result = new ArrayList<>();+ if (xml == null || xml.isEmpty()) {+ return result;+ }+ try {+ XMLStreamReader r = FACTORY.createXMLStreamReader(new StringReader(xml));+ boolean inObject = false;+ String key = null;+ String versionId = null;+ while (r.hasNext()) {+ int event = r.next();+ if (event == XMLStreamConstants.START_ELEMENT) {+ String local = r.getLocalName();+ if ("Object".equals(local)) {+ inObject = true;+ key = null;+ versionId = null;+ } else if (inObject && "Key".equals(local)) {+ key = r.getElementText();+ } else if (inObject && "VersionId".equals(local)) {+ versionId = r.getElementText();+ }+ } else if (event == XMLStreamConstants.END_ELEMENT+ && inObject && "Object".equals(r.getLocalName())) {+ if (key != null) {+ result.add(new KeyVersion(key, versionId));+ }+ inObject = false;+ }+ }+ r.close();+ } catch (Exception e) {+ LOG.debugv("Ignoring malformed XML during parse: {0}", e.getMessage());+ }+ return result;+ }+/*** Extracts sibling key/value pairs from every {@code parentElement} block.*diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 55ad3066cd..a49507bbf4 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -1408,8 +1408,8 @@ public class S3Controller {private Response handleDeleteObjects(String bucket, byte[] body, HttpHeaders httpHeaders, UriInfo uriInfo) {String xml = new String(body, StandardCharsets.UTF_8);- List<String> keys = XmlParser.extractAll(xml, "Key");- if (keys.isEmpty()) {+ List<XmlParser.KeyVersion> entries = XmlParser.extractDeleteObjectEntries(xml);+ if (entries.isEmpty()) {throw new AwsException("MalformedXML","The XML you provided was not well-formed.", 400);}@@ -1418,18 +1418,18 @@ public class S3Controller {S3Service.RequestAuthorization authorization = S3RequestAuthorizationParser.parseIfRequired(s3Service.isAuthEnforced(), httpHeaders, uriInfo);s3Service.authorizeSignedRequest(authorization);- List<String> authorizedKeys = new ArrayList<>();+ List<XmlParser.KeyVersion> authorizedEntries = new ArrayList<>();List<S3Service.DeleteError> authorizationErrors = new ArrayList<>();- for (String key : keys) {+ for (XmlParser.KeyVersion entry : entries) {try {- s3Service.authorizeObjectWrite(bucket, key, "s3:DeleteObject", authorization);- authorizedKeys.add(key);+ s3Service.authorizeDeleteObject(bucket, entry.key(), entry.versionId(), authorization);+ authorizedEntries.add(entry);} catch (AwsException e) {- authorizationErrors.add(new S3Service.DeleteError(key, e.getErrorCode(), e.getMessage()));+ authorizationErrors.add(new S3Service.DeleteError(entry.key(), e.getErrorCode(), e.getMessage()));}}- S3Service.DeleteObjectsResult result = s3Service.deleteObjects(bucket, authorizedKeys);+ S3Service.DeleteObjectsResult result = s3Service.deleteObjects(bucket, authorizedEntries);XmlBuilder builder = new XmlBuilder().raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")@@ -1437,6 +1437,9 @@ public class S3Controller {if (!quiet) {for (S3Service.DeleteResult d : result.deleted()) {builder.start("Deleted").elem("Key", d.key());+ if (d.versionId() != null) {+ builder.elem("VersionId", d.versionId());+ }if (d.deleteMarker()) {builder.elem("DeleteMarker", true);if (d.deleteMarkerVersionId() != null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 43e546def9..aae6eaa064 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -1597,20 +1597,20 @@ public class S3Service implements Resettable, ResourceProvider {public record DeleteObjectsResult(List<DeleteResult> deleted, List<DeleteError> errors) {}- public DeleteObjectsResult deleteObjects(String bucketName, List<String> keys) {+ public DeleteObjectsResult deleteObjects(String bucketName, List<XmlParser.KeyVersion> entries) {ensureBucketExists(bucketName);List<DeleteResult> deleted = new ArrayList<>();List<DeleteError> errors = new ArrayList<>();- for (String key : keys) {+ for (XmlParser.KeyVersion entry : entries) {try {- S3Object result = deleteObject(bucketName, key);+ S3Object result = deleteObject(bucketName, entry.key(), entry.versionId());if (result != null && result.isDeleteMarker()) {- deleted.add(new DeleteResult(key, null, true, result.getVersionId()));+ deleted.add(new DeleteResult(entry.key(), entry.versionId(), true, result.getVersionId()));} else {- deleted.add(new DeleteResult(key, null, false, null));+ deleted.add(new DeleteResult(entry.key(), entry.versionId(), false, null));}} catch (Exception e) {- errors.add(new DeleteError(key, "InternalError", e.getMessage()));+ errors.add(new DeleteError(entry.key(), "InternalError", e.getMessage()));}}return new DeleteObjectsResult(deleted, errors);diff --git a/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java b/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.javaindex 3aeda762f0..abc671aee6 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/XmlParserTest.java@@ -121,6 +121,63 @@ class XmlParserTest {assertNull(XmlParser.extractElementTree(null, "Target"));}+ // --- extractDeleteObjectEntries: batch delete Key/VersionId pairing ---++ @Test+ void extractDeleteObjectEntriesPairsKeyWithVersionId() {+ String xml = """+ <Delete>+ <Object><Key>a.txt</Key><VersionId>v1</VersionId></Object>+ <Object><Key>b.txt</Key><VersionId>v2</VersionId></Object>+ </Delete>+ """;++ List<XmlParser.KeyVersion> entries = XmlParser.extractDeleteObjectEntries(xml);++ assertEquals(2, entries.size());+ assertEquals(new XmlParser.KeyVersion("a.txt", "v1"), entries.get(0));+ assertEquals(new XmlParser.KeyVersion("b.txt", "v2"), entries.get(1));+ }++ @Test+ void extractDeleteObjectEntriesVersionIdIsOptional() {+ String xml = """+ <Delete>+ <Object><Key>no-version.txt</Key></Object>+ </Delete>+ """;++ List<XmlParser.KeyVersion> entries = XmlParser.extractDeleteObjectEntries(xml);++ assertEquals(1, entries.size());+ assertEquals("no-version.txt", entries.get(0).key());+ assertNull(entries.get(0).versionId());+ }++ @Test+ void extractDeleteObjectEntriesMixedVersionedAndUnversioned() {+ String xml = """+ <Delete>+ <Quiet>true</Quiet>+ <Object><Key>versioned.txt</Key><VersionId>v1</VersionId></Object>+ <Object><Key>plain.txt</Key></Object>+ </Delete>+ """;++ List<XmlParser.KeyVersion> entries = XmlParser.extractDeleteObjectEntries(xml);++ assertEquals(2, entries.size());+ assertEquals("v1", entries.get(0).versionId());+ assertNull(entries.get(1).versionId());+ }++ @Test+ void extractDeleteObjectEntriesEmptyWhenNoObjects() {+ assertTrue(XmlParser.extractDeleteObjectEntries("<Delete><Quiet>true</Quiet></Delete>").isEmpty());+ assertTrue(XmlParser.extractDeleteObjectEntries(null).isEmpty());+ assertTrue(XmlParser.extractDeleteObjectEntries("").isEmpty());+ }+// --- extractPairsPerGroup ---@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.javaindex c78b643813..3e8a58a26d 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java@@ -43,6 +43,7 @@ class S3AuthEnforcementIntegrationTest {private static final String PUBLIC_WRITE_ACL_BUCKET = "auth-public-write-acl-bucket";private static final String DENY_WRITE_ACL_BUCKET = "auth-deny-write-acl-bucket";private static final String DELETE_VERSION_BUCKET = "auth-delete-version-bucket";+ private static final String BATCH_DELETE_VERSION_BUCKET = "auth-batch-delete-version-bucket";private static final String BYPASS_BUCKET = "auth-bypass-governance-bucket";private static final String BYPASS_KEY = "bypass-target.txt";private static final String CONDITIONAL_DENY_ACL_BUCKET = "auth-conditional-deny-acl-bucket";@@ -1309,6 +1310,61 @@ class S3AuthEnforcementIntegrationTest {.body(containsString("AccessDenied"));}+ @Test+ @Order(44)+ void batchDeleteObjectsPolicyGrantDoesNotAuthorizeVersionedDelete() {+ given().when().put("/" + BATCH_DELETE_VERSION_BUCKET).then().statusCode(200);++ given()+ .contentType("application/xml")+ .body("<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>")+ .when()+ .put("/" + BATCH_DELETE_VERSION_BUCKET + "?versioning")+ .then()+ .statusCode(200);++ String versionId = given()+ .header("Authorization", LOCAL_AUTH_HEADER)+ .body("versioned batch delete target")+ .when()+ .put("/" + BATCH_DELETE_VERSION_BUCKET + "/" + VERSION_KEY)+ .then()+ .statusCode(200)+ .extract().header("x-amz-version-id");++ given()+ .contentType("application/json")+ .body(publicObjectActionPolicy(BATCH_DELETE_VERSION_BUCKET, "s3:DeleteObject"))+ .when()+ .put("/" + BATCH_DELETE_VERSION_BUCKET + "?policy")+ .then()+ .statusCode(200);++ String deleteXml = """+ <Delete>+ <Object><Key>%s</Key><VersionId>%s</VersionId></Object>+ </Delete>+ """.formatted(VERSION_KEY, versionId);++ given()+ .contentType("application/xml")+ .body(deleteXml)+ .when()+ .post("/" + BATCH_DELETE_VERSION_BUCKET + "?delete")+ .then()+ .statusCode(200)+ .body(containsString("<Code>AccessDenied</Code>"))+ .body(not(containsString("<Deleted>")));++ given()+ .header("Authorization", LOCAL_AUTH_HEADER)+ .when()+ .get("/" + BATCH_DELETE_VERSION_BUCKET + "?versions&prefix=" + VERSION_KEY)+ .then()+ .statusCode(200)+ .body(containsString("<VersionId>" + versionId + "</VersionId>"));+ }+private static String conditionalDenyObjectActionPolicy(String bucket, String action) {return """{diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3DeleteObjectsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3DeleteObjectsIntegrationTest.javaindex 589e729e83..ef96635784 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3DeleteObjectsIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3DeleteObjectsIntegrationTest.java@@ -7,6 +7,7 @@ import java.util.UUID;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.not;@QuarkusTestclass S3DeleteObjectsIntegrationTest {@@ -38,7 +39,8 @@ class S3DeleteObjectsIntegrationTest {.post("/" + bucket + "?delete").then().statusCode(200)- .body(containsString("<Key>remove.txt</Key>"));+ .body(containsString("<Key>remove.txt</Key>"))+ .body(not(containsString("<VersionId>")));given().when()@@ -53,6 +55,47 @@ class S3DeleteObjectsIntegrationTest {.statusCode(200);}+ @Test+ void deleteObjects_withExplicitVersionIds_permanentlyDeletesVersionsOnVersionedBucket() {+ String bucket = createBucket();+ enableVersioning(bucket);++ String key = "versioned.txt";+ String versionId1 = putVersionedObject(bucket, key);+ String versionId2 = putVersionedObject(bucket, key);++ String deleteBody = """+ <Delete>+ <Object><Key>%s</Key><VersionId>%s</VersionId></Object>+ <Object><Key>%s</Key><VersionId>%s</VersionId></Object>+ </Delete>+ """.formatted(key, versionId1, key, versionId2);++ given()+ .contentType("application/xml")+ .body(deleteBody)+ .when()+ .post("/" + bucket + "?delete")+ .then()+ .statusCode(200)+ .body(containsString("<Key>" + key + "</Key>"))+ .body(containsString("<VersionId>" + versionId1 + "</VersionId>"))+ .body(containsString("<VersionId>" + versionId2 + "</VersionId>"));++ given()+ .when()+ .get("/" + bucket + "?versions&prefix=" + key)+ .then()+ .statusCode(200)+ .body(not(containsString("<Version>")));++ given()+ .when()+ .get("/" + bucket + "/" + key)+ .then()+ .statusCode(404);+ }+private static String createBucket() {String bucket = "delete-objects-" + UUID.randomUUID().toString().substring(0, 8);given()@@ -71,4 +114,24 @@ class S3DeleteObjectsIntegrationTest {.then().statusCode(200);}++ private static void enableVersioning(String bucket) {+ given()+ .body("<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>")+ .when()+ .put("/" + bucket + "?versioning")+ .then()+ .statusCode(200);+ }++ private static String putVersionedObject(String bucket, String key) {+ return given()+ .body("content")+ .when()+ .put("/" + bucket + "/" + key)+ .then()+ .statusCode(200)+ .extract()+ .header("x-amz-version-id");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.javaindex e98e3168b1..c61f109b9e 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.s3;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.XmlParser;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.s3.model.CopyObjectOptions;import io.github.hectorvent.floci.services.s3.model.S3Object;@@ -352,4 +353,22 @@ class S3VersioningServiceTest {"getObject should return the promoted version's content after deleting the latest");}+ @Test+ void deleteObjectsWithExplicitVersionIdsPermanentlyDeletesVersions() {+ s3Service.putBucketVersioning("versioned-bucket", "Enabled");++ S3Object v1 = s3Service.putObject("versioned-bucket", "key",+ "v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);+ S3Object v2 = s3Service.putObject("versioned-bucket", "key",+ "v2".getBytes(StandardCharsets.UTF_8), "text/plain", null);++ s3Service.deleteObjects("versioned-bucket", List.of(+ new XmlParser.KeyVersion("key", v1.getVersionId()),+ new XmlParser.KeyVersion("key", v2.getVersionId())));++ S3Service.ListVersionsResult result = s3Service.listObjectVersions("versioned-bucket", null, 100, null);+ assertTrue(result.versions().isEmpty(),+ "batch delete with explicit VersionIds should permanently remove those versions, not place a delete marker");+ }+}