Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): emit ObjectRemoved:Delete when deleting a specific object version (#4710)
Vinícius Macedo5ee81e700658
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex bd646478ab..0e7f33847d 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -1732,6 +1732,9 @@ public class S3Service implements Resettable, ResourceProvider {}}});+ if (toDelete != null) {+ fireNotifications(bucketName, key, "ObjectRemoved:Delete", toDelete);+ }return toDelete;} else {S3Object existing = objectStore.get(objectKey(bucketName, key)).orElse(null);@@ -4594,16 +4597,31 @@ public class S3Service implements Resettable, ResourceProvider {objectNode.put("key", key);objectNode.put("size", size);objectNode.put("etag", eTag);+ if (obj != null && obj.getVersionId() != null) {+ objectNode.put("version-id", obj.getVersionId());+ }detail.put("request-id", UUID.randomUUID().toString());detail.put("requester", "aws:emulator");detail.put("source-ip-address", "127.0.0.1");detail.put("reason", eventName);+ String deletionType = eventBridgeDeletionType(eventName);+ if (deletionType != null) {+ detail.put("deletion-type", deletionType);+ }return objectMapper.writeValueAsString(detail);} catch (Exception e) {return "{}";}}+ private static String eventBridgeDeletionType(String eventName) {+ return switch (eventName) {+ case "ObjectRemoved:Delete" -> "Permanently Deleted";+ case "ObjectRemoved:DeleteMarkerCreated" -> "Delete Marker Created";+ default -> null;+ };+ }+private boolean matchesEvent(String pattern, String eventName) {String full = "s3:" + eventName;if (pattern.endsWith("*")) {diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.javaindex f228f73104..8674e12d5b 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3EventBridgeIntegrationTest.java@@ -17,6 +17,7 @@ class S3EventBridgeIntegrationTest {private static final String EB_CONTENT_TYPE = "application/x-amz-json-1.1";private static final String EB_TARGET = "AWSEvents.";+ private static final String VERSIONED_BUCKET = "eb-s3-versioned-test-bucket";private static String ruleArn;private static String queueUrl;@@ -178,6 +179,8 @@ class S3EventBridgeIntegrationTest {assert messageBody != null : "Expected a message in the queue after S3 delete";assert messageBody.contains("aws.s3") : "Expected source aws.s3 in: " + messageBody;assert messageBody.contains("Object Deleted") : "Expected detail-type 'Object Deleted' in: " + messageBody;+ assert messageBody.contains("\"deletion-type\":\"Permanently Deleted\"")+ : "Expected deletion-type 'Permanently Deleted' in: " + messageBody;}@Test@@ -230,9 +233,99 @@ class S3EventBridgeIntegrationTest {.body(not(containsString("aws.s3")));}+ @Test+ @Order(9)+ void deleteObjectVersion_eventBridgeDetailCarriesTheDeletedVersionId() {+ given().when().put("/" + VERSIONED_BUCKET).then().statusCode(200);+ given()+ .body("<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>")+ .when()+ .put("/" + VERSIONED_BUCKET + "?versioning")+ .then()+ .statusCode(200);++ String olderVersionId = given()+ .contentType("text/plain")+ .body("v1")+ .when()+ .put("/" + VERSIONED_BUCKET + "/k.txt")+ .then()+ .statusCode(200)+ .extract().header("x-amz-version-id");+ given().contentType("text/plain").body("v2").when().put("/" + VERSIONED_BUCKET + "/k.txt")+ .then().statusCode(200);++ // Enabled after the writes, so the version delete is the only event on the queue.+ given()+ .contentType("application/xml")+ .body("""+ <NotificationConfiguration>+ <EventBridgeConfiguration/>+ </NotificationConfiguration>+ """)+ .when()+ .put("/" + VERSIONED_BUCKET + "?notification")+ .then()+ .statusCode(200);++ given()+ .when()+ .delete("/" + VERSIONED_BUCKET + "/k.txt?versionId=" + olderVersionId)+ .then()+ .statusCode(204);++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .formParam("WaitTimeSeconds", "0")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body", allOf(+ containsString("Object Deleted"),+ containsString("\"version-id\":\"" + olderVersionId + "\""),+ containsString("\"deletion-type\":\"Permanently Deleted\"")));+ }++ @Test+ @Order(10)+ void deleteObjectOnVersionedBucket_eventBridgeDetailSaysDeleteMarkerCreated() {+ String markerVersionId = given()+ .when()+ .delete("/" + VERSIONED_BUCKET + "/k.txt")+ .then()+ .statusCode(204)+ .header("x-amz-delete-marker", "true")+ .extract().header("x-amz-version-id");++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .formParam("WaitTimeSeconds", "0")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body", allOf(+ containsString("Object Deleted"),+ containsString("\"version-id\":\"" + markerVersionId + "\""),+ containsString("\"deletion-type\":\"Delete Marker Created\"")));+ }+@Test@Order(100)void cleanup() {+ given()+ .contentType("application/xml")+ .body("<NotificationConfiguration/>")+ .when()+ .put("/" + VERSIONED_BUCKET + "?notification");+given().contentType(EB_CONTENT_TYPE).header("X-Amz-Target", EB_TARGET + "RemoveTargets")diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex 32cffc6765..38e26b7e59 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -1,5 +1,7 @@package io.github.hectorvent.floci.services.s3;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.InMemoryStorage;@@ -20,6 +22,7 @@ import org.junit.jupiter.api.io.TempDir;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.ValueSource;+import java.io.IOException;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.security.MessageDigest;@@ -639,6 +642,69 @@ class S3ServiceTest {assertEquals("s3-notif-test:PROD", lambdaInvoker.functionName);}+ @Test+ void deleteObjectVersionFiresObjectRemovedDeleteWithTheDeletedVersionId() throws IOException {+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();+ S3Service service = versionedBucketNotifyingOnRemoval(lambdaInvoker, "notif-s3-delete-version");+ S3Object older = service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8),+ "text/plain", null);+ service.putObject("test-bucket", "k.txt", "v2".getBytes(StandardCharsets.UTF_8), "text/plain", null);++ service.deleteObject("test-bucket", "k.txt", older.getVersionId());++ JsonNode record = onlyRecordedS3Event(lambdaInvoker);+ assertEquals("ObjectRemoved:Delete", record.path("eventName").asText());+ assertEquals("k.txt", record.path("s3").path("object").path("key").asText());+ assertEquals(older.getVersionId(), record.path("s3").path("object").path("versionId").asText());+ }++ @Test+ void deleteObjectVersionOfADeleteMarkerFiresObjectRemovedDelete() throws IOException {+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();+ S3Service service = versionedBucketNotifyingOnRemoval(lambdaInvoker, "notif-s3-delete-marker-version");+ service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);+ S3Object marker = service.deleteObject("test-bucket", "k.txt");++ service.deleteObject("test-bucket", "k.txt", marker.getVersionId());++ JsonNode record = onlyRecordedS3Event(lambdaInvoker);+ assertEquals("ObjectRemoved:Delete", record.path("eventName").asText());+ assertEquals(marker.getVersionId(), record.path("s3").path("object").path("versionId").asText());+ }++ @Test+ void deleteObjectVersionThatDoesNotExistFiresNoNotification() {+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();+ S3Service service = versionedBucketNotifyingOnRemoval(lambdaInvoker, "notif-s3-delete-missing-version");+ service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);++ service.deleteObject("test-bucket", "k.txt", "no-such-version");++ assertNull(lambdaInvoker.payload);+ }++ private S3Service versionedBucketNotifyingOnRemoval(RecordingLambdaInvoker lambdaInvoker, String dataDir) {+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve(dataDir),+ false, lambdaInvoker, new RegionResolver("us-east-1", "000000000000"));+ service.createBucket("test-bucket", "us-east-1");+ service.putBucketVersioning("test-bucket", "Enabled");+ NotificationConfiguration config = new NotificationConfiguration();+ config.getLambdaFunctionConfigurations().add(new LambdaNotification(+ "lambda-notif",+ "arn:aws:lambda:us-east-1:000000000000:function:s3-notif-test",+ List.of("s3:ObjectRemoved:*"),+ List.of()));+ service.putBucketNotificationConfiguration("test-bucket", config);+ return service;+ }++ private static JsonNode onlyRecordedS3Event(RecordingLambdaInvoker lambdaInvoker) throws IOException {+ assertNotNull(lambdaInvoker.payload, "no S3 event reached the notification target");+ JsonNode records = new ObjectMapper().readTree(lambdaInvoker.payload).path("Records");+ assertEquals(1, records.size());+ return records.get(0);+ }+private static NotificationConfiguration lambdaNotificationConfig(String prefix, String suffix) {NotificationConfiguration config = new NotificationConfiguration();config.getLambdaFunctionConfigurations().add(new LambdaNotification(diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningIntegrationTest.javaindex 0a1378db94..229b74d837 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningIntegrationTest.java@@ -395,4 +395,82 @@ class S3VersioningIntegrationTest {assertEquals(1, versions.size(),"only the newly uploaded version should exist — deleted versions must not resurrect");}++ @Test+ @Order(20)+ void deleteSpecificVersionSendsObjectRemovedDeleteToQueue() {+ String notifBucket = "version-delete-notif-test";+ given().when().put("/" + notifBucket).then().statusCode(200);+ given()+ .body("<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>")+ .when()+ .put("/" + notifBucket + "?versioning")+ .then()+ .statusCode(200);++ String queueUrl = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateQueue")+ .formParam("QueueName", "version-delete-notif-queue")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl");++ try {+ given()+ .contentType("application/xml")+ .queryParam("notification", "")+ .body("""+ <NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">+ <QueueConfiguration>+ <Id>removed</Id>+ <Queue>arn:aws:sqs:us-east-1:000000000000:version-delete-notif-queue</Queue>+ <Event>s3:ObjectRemoved:*</Event>+ </QueueConfiguration>+ </NotificationConfiguration>+ """)+ .when()+ .put("/" + notifBucket)+ .then()+ .statusCode(200);++ String olderVersionId = given()+ .body("v1")+ .contentType("text/plain")+ .when()+ .put("/" + notifBucket + "/k.txt")+ .then()+ .statusCode(200)+ .extract().header("x-amz-version-id");+ given().body("v2").contentType("text/plain").when().put("/" + notifBucket + "/k.txt")+ .then().statusCode(200);++ given()+ .when()+ .delete("/" + notifBucket + "/k.txt?versionId=" + olderVersionId)+ .then()+ .statusCode(204);++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body", allOf(+ containsString("\"eventName\":\"ObjectRemoved:Delete\""),+ containsString("\"versionId\":\"" + olderVersionId + "\"")));+ } finally {+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteQueue")+ .formParam("QueueUrl", queueUrl)+ .post("/");+ }+ }}