Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): validate notification destinations before storing configuration (#4747)
XuJianb2dd5665dbe5
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3NotificationValidationTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3NotificationValidationTest.javanew file mode 100644index 0000000000..6a757148c9--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3NotificationValidationTest.java@@ -0,0 +1,99 @@+package com.floci.test;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.Event;+import software.amazon.awssdk.services.s3.model.NotificationConfiguration;+import software.amazon.awssdk.services.s3.model.QueueConfiguration;+import software.amazon.awssdk.services.s3.model.S3Exception;+import software.amazon.awssdk.services.sqs.SqsClient;+import software.amazon.awssdk.services.sqs.model.Message;+import software.amazon.awssdk.services.sqs.model.QueueAttributeName;++import java.time.Instant;+import java.util.List;++import static org.assertj.core.api.Assertions.assertThat;+import static org.junit.jupiter.api.Assertions.assertThrows;++@DisplayName("S3 notification destination validation")+class S3NotificationValidationTest {++ @Test+ void destinationValidationAndSkipHeaderWorkThroughAwsSdk() throws Exception {+ String bucket = TestFixtures.uniqueName("java-s3-validation");+ String queueName = TestFixtures.uniqueName("java-s3-validation");+ try (S3Client s3 = TestFixtures.s3Client(); SqsClient sqs = TestFixtures.sqsClient()) {+ String queueUrl = sqs.createQueue(request -> request.queueName(queueName)).queueUrl();+ try {+ s3.createBucket(request -> request.bucket(bucket));+ try {+ String queueArn = sqs.getQueueAttributes(request -> request.queueUrl(queueUrl)+ .attributeNames(QueueAttributeName.QUEUE_ARN))+ .attributes().get(QueueAttributeName.QUEUE_ARN);+ NotificationConfiguration valid = configuration(queueArn);+ s3.putBucketNotificationConfiguration(request -> request.bucket(bucket)+ .notificationConfiguration(valid));++ List<Message> messages = sqs.receiveMessage(request -> request.queueUrl(queueUrl)+ .maxNumberOfMessages(1).waitTimeSeconds(5)).messages();+ assertThat(messages).hasSize(1);+ Message message = messages.get(0);+ JsonNode event = new ObjectMapper().readTree(message.body());+ assertThat(event.path("Service").asText()).isEqualTo("Amazon S3");+ assertThat(event.path("Event").asText()).isEqualTo("s3:TestEvent");+ assertThat(event.path("Bucket").asText()).isEqualTo(bucket);+ assertThat(Instant.parse(event.path("Time").asText())).isNotNull();+ assertThat(event.path("RequestId").asText()).isNotBlank();+ assertThat(event.path("HostId").asText()).isNotBlank();+ assertThat(event.has("Records")).isFalse();+ sqs.deleteMessage(request -> request.queueUrl(queueUrl)+ .receiptHandle(message.receiptHandle()));++ String missingArn = queueArn + "-missing";+ S3Exception missing = assertThrows(S3Exception.class,+ () -> s3.putBucketNotificationConfiguration(request -> request.bucket(bucket)+ .notificationConfiguration(configuration(missingArn))));+ assertThat(missing.statusCode()).isEqualTo(400);+ assertThat(missing.awsErrorDetails().errorCode()).isEqualTo("InvalidArgument");+ assertThat(s3.getBucketNotificationConfiguration(request -> request.bucket(bucket))+ .queueConfigurations()).extracting(QueueConfiguration::queueArn)+ .containsExactly(queueArn);++ s3.putBucketNotificationConfiguration(request -> request.bucket(bucket)+ .skipDestinationValidation(true)+ .notificationConfiguration(NotificationConfiguration.builder()+ .queueConfigurations(queueConfiguration(queueArn),+ queueConfiguration(missingArn)).build()));+ assertThat(s3.getBucketNotificationConfiguration(request -> request.bucket(bucket))+ .queueConfigurations()).extracting(QueueConfiguration::queueArn)+ .containsExactly(queueArn, missingArn);+ assertThat(sqs.receiveMessage(request -> request.queueUrl(queueUrl)+ .waitTimeSeconds(0)).messages()).isEmpty();++ S3Exception malformed = assertThrows(S3Exception.class,+ () -> s3.putBucketNotificationConfiguration(request -> request.bucket(bucket)+ .skipDestinationValidation(true)+ .notificationConfiguration(configuration("not-an-arn"))));+ assertThat(malformed.statusCode()).isEqualTo(400);+ assertThat(malformed.awsErrorDetails().errorCode()).isEqualTo("InvalidArgument");+ } finally {+ s3.deleteBucket(request -> request.bucket(bucket));+ }+ } finally {+ sqs.deleteQueue(request -> request.queueUrl(queueUrl));+ }+ }+ }++ private static NotificationConfiguration configuration(String queueArn) {+ return NotificationConfiguration.builder().queueConfigurations(queueConfiguration(queueArn)).build();+ }++ private static QueueConfiguration queueConfiguration(String queueArn) {+ return QueueConfiguration.builder().queueArn(queueArn).events(Event.S3_OBJECT_CREATED).build();+ }+}diff --git a/docs/services/s3.md b/docs/services/s3.mdindex e344b1a8b0..ae1c0f2847 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -41,6 +41,19 @@ Browser and presigned POST uploads emit `s3:ObjectCreated:Post`, matching AWS S3`s3:ObjectCreated:Put`; use `s3:ObjectCreated:*` to subscribe to objects created by either method.Annotation changes emit `s3:ObjectAnnotation:Put` and `s3:ObjectAnnotation:Delete`.+`PutBucketNotificationConfiguration` checks SQS, SNS, and Lambda destination ARNs and verifies+that each destination exists before replacing a bucket's configuration. A failed check leaves+the prior configuration unchanged. Accepted configurations send an `s3:TestEvent` message to+SQS and SNS destinations; unlike object notifications, this message has no `Records` array.+`x-amz-skip-destination-validation: true` skips existence checks and test messages, but still+requires well-formed destination ARNs. Destination resource-policy permissions are not checked.+Existing SNS topics in another account are checked for existence without receiving a test message,+regardless of IAM enforcement. Their object events are not delivered because SNS publish still+resolves topics in the caller's account. With IAM enforcement enabled, SQS queues in another account+are also checked without a test message. Object-event delivery is not yet subject to a shared service+authorizer. AWS's same-Region destination requirement is not enforced. Cross-account Lambda+destination routing is not emulated.+## Object AnnotationsAnnotations are named UTF-8 text payloads (up to 1 MiB each, 1,000 per object version) attached to adiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 5955450e20..58a595d754 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -289,7 +289,8 @@ public class S3Controller {s3Service.isAuthEnforced(), httpHeaders, uriInfo);if (hasQueryParam(uriInfo, "notification")) {s3Service.authorizeBucketWrite(bucket, "s3:PutBucketNotification", authorization);- return handlePutBucketNotification(bucket, body);+ return handlePutBucketNotification(bucket, body,+ Boolean.parseBoolean(httpHeaders.getHeaderString("x-amz-skip-destination-validation")));}if (hasQueryParam(uriInfo, "versioning")) {s3Service.authorizeBucketWrite(bucket, "s3:PutBucketVersioning", authorization);@@ -1916,31 +1917,31 @@ public class S3Controller {}}- private Response handlePutBucketNotification(String bucket, byte[] body) {+ private Response handlePutBucketNotification(String bucket, byte[] body, boolean skipDestinationValidation) {try {String xml = new String(body, StandardCharsets.UTF_8);NotificationConfiguration config = new NotificationConfiguration();- for (var parsed : parseNotificationGroups(xml, "QueueConfiguration", "Queue")) {+ for (ParsedNotificationGroup parsed : parseNotificationGroups(xml, "QueueConfiguration", "Queue")) {config.getQueueConfigurations().add(new QueueNotification(parsed.id, parsed.arn, parsed.events, parsed.filterRules));}- for (var parsed : parseNotificationGroups(xml, "TopicConfiguration", "Topic")) {+ for (ParsedNotificationGroup parsed : parseNotificationGroups(xml, "TopicConfiguration", "Topic")) {config.getTopicConfigurations().add(new TopicNotification(parsed.id, parsed.arn, parsed.events, parsed.filterRules));}- for (var parsed : parseNotificationGroups(xml, "LambdaFunctionConfiguration", "LambdaFunctionArn")) {+ for (ParsedNotificationGroup parsed : parseNotificationGroups(xml, "LambdaFunctionConfiguration", "LambdaFunctionArn")) {config.getLambdaFunctionConfigurations().add(new LambdaNotification(parsed.id, parsed.arn, parsed.events, parsed.filterRules));}- for (var parsed : parseNotificationGroups(xml, "CloudFunctionConfiguration", "CloudFunction")) {+ for (ParsedNotificationGroup parsed : parseNotificationGroups(xml, "CloudFunctionConfiguration", "CloudFunction")) {config.getLambdaFunctionConfigurations().add(new LambdaNotification(parsed.id, parsed.arn, parsed.events, parsed.filterRules));}config.setEventBridgeEnabled(parseEventBridgeConfiguration(xml));- s3Service.putBucketNotificationConfiguration(bucket, config);+ s3Service.putBucketNotificationConfiguration(bucket, config, skipDestinationValidation);return Response.ok().build();} catch (AwsException e) {return xmlErrorResponse(e, bucket);@@ -3285,6 +3286,19 @@ public class S3Controller {if (argumentValue != null) {xmlBuilder.elem("ArgumentValue", argumentValue.toString());}+ for (int index = 1; ; index++) {+ Object numberedName = e.getExtendedData().get("ArgumentName" + index);+ Object numberedValue = e.getExtendedData().get("ArgumentValue" + index);+ if (numberedName == null && numberedValue == null) {+ break;+ }+ if (numberedName != null) {+ xmlBuilder.elem("ArgumentName" + index, numberedName.toString());+ }+ if (numberedValue != null) {+ xmlBuilder.elem("ArgumentValue" + index, numberedValue.toString());+ }+ }}String xml = xmlBuilder.elem("RequestId", java.util.UUID.randomUUID().toString())diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex f277a88c4e..5ab1b73884 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -19,6 +19,7 @@ import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator.ResourcePolicyimport io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.lambda.LambdaService;import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.s3.model.*;import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;import io.github.hectorvent.floci.services.sns.SnsService;@@ -163,6 +164,7 @@ public class S3Service implements Resettable, ResourceProvider {private final String baseUrl;private final ObjectMapper objectMapper;private final boolean enforceAuth;+ private final boolean enforceIam;private final IamService iamService;private final boolean globalBucketNamespace;private final IamPolicyEvaluator policyEvaluator;@@ -195,7 +197,7 @@ public class S3Service implements Resettable, ResourceProvider {eventBridgeService, s3UpdatedEvent,regionResolver,config.effectiveBaseUrl(), objectMapper,- config.services().s3().enforceAuth(), iamService,+ config.services().s3().enforceAuth(), config.services().iam().enforcementEnabled(), iamService,config.services().s3().globalBucketNamespace());}@@ -208,7 +210,7 @@ public class S3Service implements Resettable, ResourceProvider {Path dataRoot, boolean inMemory) {this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(),dataRoot, inMemory, null, null, null, null, null, null, null,- null, "http://localhost:4566", new ObjectMapper(), false, null, false);+ null, "http://localhost:4566", new ObjectMapper(), false, false, null, false);}S3Service(StorageBackend<String, Bucket> bucketStore,@@ -217,7 +219,7 @@ public class S3Service implements Resettable, ResourceProvider {boolean enforceAuth, IamService iamService) {this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(),dataRoot, inMemory, null, null, null, null, null, null, null,- null, "http://localhost:4566", new ObjectMapper(), enforceAuth, iamService, false);+ null, "http://localhost:4566", new ObjectMapper(), enforceAuth, false, iamService, false);}/** Package-private constructor for testing account-level Block Public Access persistence. */@@ -227,7 +229,7 @@ public class S3Service implements Resettable, ResourceProvider {Path dataRoot, boolean inMemory) {this(bucketStore, objectStore, defaultAnnotationStore(), accountPublicAccessBlockStore,dataRoot, inMemory, null, null, null, null, null, null, null,- null, "http://localhost:4566", new ObjectMapper(), false, null, false);+ null, "http://localhost:4566", new ObjectMapper(), false, false, null, false);}/** Package-private constructor for testing the global-bucket-namespace resolution flag. */@@ -236,7 +238,7 @@ public class S3Service implements Resettable, ResourceProvider {Path dataRoot, boolean inMemory, boolean globalBucketNamespace) {this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(),dataRoot, inMemory, null, null, null, null, null, null, null,- null, "http://localhost:4566", new ObjectMapper(), false, null, globalBucketNamespace);+ null, "http://localhost:4566", new ObjectMapper(), false, false, null, globalBucketNamespace);}S3Service(StorageBackend<String, Bucket> bucketStore,@@ -246,7 +248,7 @@ public class S3Service implements Resettable, ResourceProvider {RegionResolver regionResolver) {this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(),dataRoot, inMemory, null, null, lambdaService, null, null, null, null,- regionResolver, "http://localhost:4566", new ObjectMapper(), false, null, false);+ regionResolver, "http://localhost:4566", new ObjectMapper(), false, false, null, false);}S3Service(StorageBackend<String, Bucket> bucketStore,@@ -256,7 +258,7 @@ public class S3Service implements Resettable, ResourceProvider {RegionResolver regionResolver) {this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(),dataRoot, inMemory, null, null, null, null, lambdaInvoker, null, null,- regionResolver, "http://localhost:4566", new ObjectMapper(), false, null, false);+ regionResolver, "http://localhost:4566", new ObjectMapper(), false, false, null, false);}/** In-memory account-level Block Public Access store for the package-private test constructors. */@@ -280,7 +282,8 @@ public class S3Service implements Resettable, ResourceProvider {EventBridgeService eventBridgeService,Event<S3ObjectUpdatedEvent> s3UpdatedEvent,RegionResolver regionResolver, String baseUrl, ObjectMapper objectMapper,- boolean enforceAuth, IamService iamService, boolean globalBucketNamespace) {+ boolean enforceAuth, boolean enforceIam, IamService iamService,+ boolean globalBucketNamespace) {this.bucketStore = bucketStore;this.objectStore = objectStore;this.annotationStore = annotationStore;@@ -298,6 +301,7 @@ public class S3Service implements Resettable, ResourceProvider {this.baseUrl = baseUrl;this.objectMapper = objectMapper;this.enforceAuth = enforceAuth;+ this.enforceIam = enforceIam;this.iamService = iamService;this.globalBucketNamespace = globalBucketNamespace;this.policyEvaluator = new IamPolicyEvaluator(objectMapper);@@ -3631,14 +3635,174 @@ public class S3Service implements Resettable, ResourceProvider {// --- Notification Configuration ---public void putBucketNotificationConfiguration(String bucketName, NotificationConfiguration config) {+ putBucketNotificationConfiguration(bucketName, config, false);+ }++ public void putBucketNotificationConfiguration(String bucketName, NotificationConfiguration config,+ boolean skipDestinationValidation) {Bucket bucket = bucketStore.get(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));- bucket.setNotificationConfiguration(config);- bucketStore.put(bucketName, bucket);++ List<NotificationDestination> destinations = notificationDestinations(config);+ if (!skipDestinationValidation) {+ List<DestinationFailure> failures = new ArrayList<>();+ for (NotificationDestination destination : destinations) {+ if (!notificationDestinationExists(destination)) {+ failures.add(missingDestination(destination));+ }+ }+ if (!failures.isEmpty()) {+ throw invalidNotificationDestinations(failures);+ }++ String testEvent = s3TestEvent(bucketName);+ for (NotificationDestination destination : destinations) {+ try {+ if (!destination.accountId().equals(ownerId())+ && ("sns".equals(destination.service())+ || enforceIam && "sqs".equals(destination.service()))) {+ // SNS publish still resolves topics in the caller's account.+ continue;+ }+ if ("sqs".equals(destination.service())) {+ sqsService.sendMessage(sqsUrlFromArn(destination.arn()), testEvent, 0,+ destination.region());+ } else if ("sns".equals(destination.service())) {+ snsService.publish(destination.arn(), null, testEvent, "Amazon S3 Notification",+ destination.region());+ }+ } catch (AwsException e) {+ failures.add(new DestinationFailure(destination, e.getMessage()));+ }+ }+ if (!failures.isEmpty()) {+ throw invalidNotificationDestinations(failures);+ }+ }++ synchronized (bucket) {+ requireSameRecord(bucketName, bucket);+ bucket.setNotificationConfiguration(config);+ bucketStore.put(bucketName, bucket);+ }LOG.infov("Set notification configuration for bucket: {0}", bucketName);}+ private record NotificationDestination(String arn, String service, String region, String accountId) {}++ private record DestinationFailure(NotificationDestination destination, String reason) {}++ private static List<NotificationDestination> notificationDestinations(NotificationConfiguration config) {+ List<NotificationDestination> destinations = new ArrayList<>();+ for (QueueNotification queue : config.getQueueConfigurations()) {+ destinations.add(notificationDestination(queue.queueArn(), "Queue", "sqs"));+ }+ for (TopicNotification topic : config.getTopicConfigurations()) {+ destinations.add(notificationDestination(topic.topicArn(), "Topic", "sns"));+ }+ for (LambdaNotification lambda : config.getLambdaFunctionConfigurations()) {+ destinations.add(notificationDestination(lambda.functionArn(), "CloudFunction", "lambda"));+ }+ return destinations;+ }++ private static NotificationDestination notificationDestination(String arn, String argumentName,+ String service) {+ AwsArnUtils.Arn parsed;+ try {+ parsed = AwsArnUtils.parse(arn);+ } catch (IllegalArgumentException e) {+ throw invalidNotificationArn(arn, argumentName);+ }+ boolean validResource = switch (service) {+ case "sqs" -> parsed.resource().matches("[A-Za-z0-9_-]{1,80}")+ || parsed.resource().matches("[A-Za-z0-9_-]{1,75}\\.fifo");+ case "sns" -> parsed.resource().matches("[A-Za-z0-9_-]{1,256}")+ || parsed.resource().matches("[A-Za-z0-9_-]{1,251}\\.fifo");+ case "lambda" -> parsed.resource().matches(+ "function:[A-Za-z0-9_-]{1,64}(?::[A-Za-z0-9_$-]{1,128})?");+ default -> false;+ };+ if (!parsed.partition().matches(AwsArnUtils.PARTITION_REGEX)+ || !parsed.service().equals(service)+ || !parsed.region().matches("[a-z0-9-]+")+ || !parsed.accountId().matches("[0-9]{12}")+ || !validResource+ || AwsRegions.isRegionId(parsed.region())+ && !parsed.partition().equals(AwsRegions.partitionFor(parsed.region()))) {+ throw invalidNotificationArn(arn, argumentName);+ }+ return new NotificationDestination(arn, service, parsed.region(), parsed.accountId());+ }++ private static AwsException invalidNotificationArn(String arn, String argumentName) {+ Map<String, Object> details = new LinkedHashMap<>();+ details.put("ArgumentName", argumentName);+ details.put("ArgumentValue", arn);+ return new AwsException("InvalidArgument", "The ARN could not be parsed", 400, details);+ }++ private boolean notificationDestinationExists(NotificationDestination destination) {+ return switch (destination.service()) {+ case "sqs" -> sqsService != null+ && sqsService.queueExists(sqsUrlFromArn(destination.arn()), destination.region());+ case "sns" -> snsService != null && snsService.topicExists(destination.arn(), destination.region());+ case "lambda" -> lambdaNotificationDestinationExists(destination);+ default -> false;+ };+ }++ private boolean lambdaNotificationDestinationExists(NotificationDestination destination) {+ LambdaService service = resolveLambdaService();+ if (service == null) {+ return false;+ }+ try {+ LambdaFunction function = service.getFunction(destination.region(), destination.arn(), null);+ return destination.accountId().equals(AwsArnUtils.accountOrDefault(function.getFunctionArn(), null));+ } catch (AwsException e) {+ LOG.debugv("Lambda notification destination {0} is unavailable: {1} ({2})",+ destination.arn(), e.getErrorCode(), e.getMessage());+ return false;+ }+ }++ private static DestinationFailure missingDestination(NotificationDestination destination) {+ String reason = switch (destination.service()) {+ case "sqs" -> "The destination queue does not exist";+ case "sns" -> "The destination topic does not exist";+ case "lambda" -> "Not authorized to invoke function [" + destination.arn() + "]";+ default -> "The destination does not exist";+ };+ return new DestinationFailure(destination, reason);+ }++ private static AwsException invalidNotificationDestinations(List<DestinationFailure> failures) {+ Map<String, Object> details = new LinkedHashMap<>();+ for (int index = 0; index < failures.size(); index++) {+ DestinationFailure failure = failures.get(index);+ NotificationDestination destination = failure.destination();+ String argument = "lambda".equals(destination.service())+ ? destination.arn() + ", null" : destination.arn();+ details.put("ArgumentName" + (index + 1), argument);+ details.put("ArgumentValue" + (index + 1), failure.reason());+ }+ return new AwsException("InvalidArgument",+ "Unable to validate the following destination configurations", 400, details);+ }++ private String s3TestEvent(String bucketName) {+ ObjectNode event = objectMapper.createObjectNode();+ event.put("Service", "Amazon S3");+ event.put("Event", "s3:TestEvent");+ event.put("Time", Instant.now().toString());+ event.put("Bucket", bucketName);+ event.put("RequestId", UUID.randomUUID().toString());+ event.put("HostId", UUID.randomUUID().toString());+ return event.toString();+ }+public NotificationConfiguration getBucketNotificationConfiguration(String bucketName) {Bucket bucket = bucketStore.get(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket",@@ -4529,7 +4693,8 @@ public class S3Service implements Resettable, ResourceProvider {for (QueueNotification qn : config.getQueueConfigurations()) {if (qn.events().stream().anyMatch(p -> matchesEvent(p, eventName)) && qn.matchesKey(key)) {try {- sqsService.sendMessage(sqsUrlFromArn(qn.queueArn()), eventJson, 0, extractRegionFromArn(qn.queueArn()));+ sqsService.sendMessage(sqsUrlFromArn(qn.queueArn()), eventJson, 0,+ extractRegionFromArn(qn.queueArn()));LOG.debugv("Fired S3 event {0} to SQS {1}", eventName, qn.queueArn());} catch (Exception e) {LOG.warnv("Failed to deliver S3 event to SQS {0}: {1}", qn.queueArn(), e.getMessage());@@ -4540,7 +4705,8 @@ public class S3Service implements Resettable, ResourceProvider {for (TopicNotification tn : config.getTopicConfigurations()) {if (tn.events().stream().anyMatch(p -> matchesEvent(p, eventName)) && tn.matchesKey(key)) {try {- snsService.publish(tn.topicArn(), null, eventJson, "Amazon S3 Notification", region);+ snsService.publish(tn.topicArn(), null, eventJson, "Amazon S3 Notification",+ extractRegionFromArn(tn.topicArn()));LOG.debugv("Fired S3 event {0} to SNS {1}", eventName, tn.topicArn());} catch (Exception e) {LOG.warnv("Failed to deliver S3 event to SNS {0}: {1}", tn.topicArn(), e.getMessage());diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.javaindex 0568e1b108..cd57f6617e 100644--- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.SsrfProtection;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.resource.ResourceProvider;import io.github.hectorvent.floci.core.resource.SupportedResourceType;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.firehose.FirehoseService;@@ -383,6 +384,11 @@ public class SnsService implements Resettable, ResourceProvider {}public boolean topicExists(String topicArn, String region) {+ if (topicStore instanceof AccountAwareStorageBackend<Topic> aware) {+ String accountId = AwsArnUtils.accountOrDefault(topicArn, regionResolver.getAccountId());+ return aware.getForAccountMigratingLegacy(accountId, topicKey(region, topicArn),+ topic -> topicArn.equals(topic.getTopicArn())).isPresent();+ }return topicStore.get(topicKey(region, topicArn)).isPresent();}diff --git a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.javaindex 2c8bc70ed3..f3ac7d7871 100644--- a/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sqs/SqsService.java@@ -702,6 +702,10 @@ public class SqsService implements Resettable, ResourceProvider {return queueUrl;}+ public boolean queueExists(String queueUrl, String region) {+ return getQueueByUrl(regionKey(region, queueUrl), queueUrl).isPresent();+ }+public Map<String, String> getQueueAttributes(String queueUrl, List<String> attributeNames, String region) {String storageKey = regionKey(region, queueUrl);Queue queue = queueStore.get(storageKey)diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex 9fcf29d884..1e3e4f54f6 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -895,6 +895,9 @@ class DynamoDbStreamsEventSourcePollerTest {EmulatorConfig.S3ServiceConfig s3Config = mock(EmulatorConfig.S3ServiceConfig.class);when(s3Config.globalBucketNamespace()).thenReturn(globalBucketNamespace);when(config.services().s3()).thenReturn(s3Config);+ EmulatorConfig.IamServiceConfig iamConfig = mock(EmulatorConfig.IamServiceConfig.class);+ when(iamConfig.enforcementEnabled()).thenReturn(false);+ when(config.services().iam()).thenReturn(iamConfig);RegionResolver regionResolver = mock(RegionResolver.class);when(regionResolver.getAccountId()).thenAnswer(diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.javaindex 58f0f68e40..f39af7cd30 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java@@ -2219,6 +2219,7 @@ class S3IntegrationTest {""";given()+ .header("x-amz-skip-destination-validation", "true").contentType("application/xml").queryParam("notification", "").body(xml)@@ -2266,6 +2267,7 @@ class S3IntegrationTest {""";given()+ .header("x-amz-skip-destination-validation", "true").contentType("application/xml").queryParam("notification", "").body(xml)@@ -2313,6 +2315,7 @@ class S3IntegrationTest {""";given()+ .header("x-amz-skip-destination-validation", "true").contentType("application/xml").queryParam("notification", "").body(xml)@@ -2370,6 +2373,19 @@ class S3IntegrationTest {.then().statusCode(200);+ given()+ .header("Authorization", sqsAuth)+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body",+ containsString("s3:TestEvent"));+given().contentType("text/plain").body("hello")diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3NotificationDestinationIamEnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3NotificationDestinationIamEnforcementIntegrationTest.javanew file mode 100644index 0000000000..450dbc2b12--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3NotificationDestinationIamEnforcementIntegrationTest.java@@ -0,0 +1,184 @@+package io.github.hectorvent.floci.services.s3;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.model.Message;+import io.github.hectorvent.floci.testing.IamEnforcementProfile;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.TestProfile;+import io.quarkus.test.junit.mockito.InjectSpy;+import io.restassured.response.ValidatableResponse;+import jakarta.inject.Inject;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.not;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.ArgumentMatchers.isNull;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;++@QuarkusTest+@TestProfile(IamEnforcementProfile.class)+class S3NotificationDestinationIamEnforcementIntegrationTest {++ private static final String REGION = "us-east-1";+ private static final String BUCKET_ACCOUNT = "000000000000";+ private static final String OTHER_ACCOUNT = "000000000002";++ @Inject SqsService sqsService;+ @InjectSpy SnsService snsService;+ @Inject ObjectMapper objectMapper;++ private String bucket;++ @BeforeEach+ void createBucket() {+ bucket = "notification-enforced-" + UUID.randomUUID().toString().substring(0, 8);+ given().header("Authorization", authorization(BUCKET_ACCOUNT, "s3"))+ .when().put("/" + bucket).then().statusCode(200);+ }++ @AfterEach+ void removeBucket() {+ given().header("Authorization", authorization(BUCKET_ACCOUNT, "s3"))+ .when().delete("/" + bucket).then().statusCode(204);+ }++ @Test+ void crossAccountQueueIsValidatedWithoutReceivingTestMessage() throws Exception {+ String otherQueueName = "notification-other-" + UUID.randomUUID().toString().substring(0, 8);+ String ownQueueName = "notification-own-" + UUID.randomUUID().toString().substring(0, 8);+ String otherQueueUrl = createQueue(OTHER_ACCOUNT, otherQueueName);+ String ownQueueUrl = createQueue(BUCKET_ACCOUNT, ownQueueName);+ String otherQueueArn = queueArn(OTHER_ACCOUNT, otherQueueName);+ String ownQueueArn = queueArn(BUCKET_ACCOUNT, ownQueueName);+ try {+ putNotification(otherQueueArn).statusCode(200);+ assertTrue(sqsService.receiveMessage(otherQueueUrl, 1, 0, 0, REGION).isEmpty());+ getNotification().body(containsString(otherQueueArn));++ String missingQueueArn = queueArn(OTHER_ACCOUNT, "notification-missing-" + UUID.randomUUID());+ putNotification(missingQueueArn).statusCode(400)+ .body(containsString("The destination queue does not exist"));+ getNotification().body(containsString(otherQueueArn))+ .body(not(containsString(missingQueueArn)));+ assertTrue(sqsService.receiveMessage(otherQueueUrl, 1, 0, 0, REGION).isEmpty());++ putNotification(ownQueueArn).statusCode(200);+ List<Message> ownMessages = sqsService.receiveMessage(ownQueueUrl, 1, 0, 0, REGION);+ assertFalse(ownMessages.isEmpty());+ JsonNode testEvent = objectMapper.readTree(ownMessages.getFirst().getBody());+ assertEquals("s3:TestEvent", testEvent.path("Event").asText());+ assertFalse(testEvent.has("Records"));+ assertTrue(sqsService.receiveMessage(otherQueueUrl, 1, 0, 0, REGION).isEmpty());+ } finally {+ deleteQueue(BUCKET_ACCOUNT, ownQueueUrl);+ deleteQueue(OTHER_ACCOUNT, otherQueueUrl);+ }+ }++ @Test+ void crossAccountTopicIsValidatedWithoutPublishingTestMessage() {+ String otherTopicName = "notification-other-" + UUID.randomUUID().toString().substring(0, 8);+ String ownTopicName = "notification-own-" + UUID.randomUUID().toString().substring(0, 8);+ String otherTopicArn = createTopic(OTHER_ACCOUNT, otherTopicName);+ String ownTopicArn = createTopic(BUCKET_ACCOUNT, ownTopicName);+ try {+ assertTrue(snsService.topicExists(otherTopicArn, REGION));+ putTopicNotification(otherTopicArn).statusCode(200);+ verify(snsService, never()).publish(eq(otherTopicArn), isNull(), anyString(),+ eq("Amazon S3 Notification"), eq(REGION));+ getNotification().body(containsString(otherTopicArn));++ String missingTopicArn = topicArn(OTHER_ACCOUNT, "notification-missing-" + UUID.randomUUID());+ putTopicNotification(missingTopicArn).statusCode(400)+ .body(containsString("The destination topic does not exist"));+ getNotification().body(containsString(otherTopicArn))+ .body(not(containsString(missingTopicArn)));++ putTopicNotification(ownTopicArn).statusCode(200);+ verify(snsService).publish(eq(ownTopicArn), isNull(), anyString(),+ eq("Amazon S3 Notification"), eq(REGION));+ } finally {+ deleteTopic(BUCKET_ACCOUNT, ownTopicArn);+ deleteTopic(OTHER_ACCOUNT, otherTopicArn);+ }+ }++ private static String createQueue(String accountId, String queueName) {+ return given().header("Authorization", authorization(accountId, "sqs"))+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateQueue").formParam("QueueName", queueName)+ .when().post("/").then().statusCode(200)+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl");+ }++ private static void deleteQueue(String accountId, String queueUrl) {+ given().header("Authorization", authorization(accountId, "sqs"))+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteQueue").formParam("QueueUrl", queueUrl)+ .when().post("/").then().statusCode(200);+ }++ private static String createTopic(String accountId, String topicName) {+ return given().header("Authorization", authorization(accountId, "sns"))+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateTopic").formParam("Name", topicName)+ .when().post("/").then().statusCode(200)+ .extract().xmlPath().getString("CreateTopicResponse.CreateTopicResult.TopicArn");+ }++ private static void deleteTopic(String accountId, String topicArn) {+ given().header("Authorization", authorization(accountId, "sns"))+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteTopic").formParam("TopicArn", topicArn)+ .when().post("/").then().statusCode(200);+ }++ private ValidatableResponse putNotification(String queueArn) {+ String xml = "<NotificationConfiguration><QueueConfiguration><Queue>" + queueArn+ + "</Queue><Event>s3:ObjectCreated:*</Event></QueueConfiguration></NotificationConfiguration>";+ return given().header("Authorization", authorization(BUCKET_ACCOUNT, "s3"))+ .contentType("application/xml").body(xml)+ .when().put("/" + bucket + "?notification").then();+ }++ private ValidatableResponse putTopicNotification(String topicArn) {+ String xml = "<NotificationConfiguration><TopicConfiguration><Topic>" + topicArn+ + "</Topic><Event>s3:ObjectCreated:*</Event></TopicConfiguration></NotificationConfiguration>";+ return given().header("Authorization", authorization(BUCKET_ACCOUNT, "s3"))+ .contentType("application/xml").body(xml)+ .when().put("/" + bucket + "?notification").then();+ }++ private ValidatableResponse getNotification() {+ return given().header("Authorization", authorization(BUCKET_ACCOUNT, "s3"))+ .when().get("/" + bucket + "?notification").then().statusCode(200);+ }++ private static String queueArn(String accountId, String queueName) {+ return "arn:aws:sqs:" + REGION + ":" + accountId + ":" + queueName;+ }++ private static String topicArn(String accountId, String topicName) {+ return "arn:aws:sns:" + REGION + ":" + accountId + ":" + topicName;+ }++ private static String authorization(String accountId, String service) {+ return "AWS4-HMAC-SHA256 Credential=" + accountId + "/20261001/" + REGION+ + "/" + service + "/aws4_request, SignedHeaders=host, Signature=abc";+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3NotificationDestinationValidationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3NotificationDestinationValidationIntegrationTest.javanew file mode 100644index 0000000000..67b4ea4140--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3NotificationDestinationValidationIntegrationTest.java@@ -0,0 +1,351 @@+package io.github.hectorvent.floci.services.s3;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.model.Message;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.mockito.InjectSpy;+import io.restassured.response.ValidatableResponse;+import jakarta.inject.Inject;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Instant;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.not;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.ArgumentMatchers.isNull;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;++@QuarkusTest+class S3NotificationDestinationValidationIntegrationTest {++ private static final String REGION = "us-east-1";+ private static final String OTHER_ACCOUNT = "000000000002";++ @Inject SqsService sqsService;+ @InjectSpy SnsService snsService;+ @Inject RegionResolver regionResolver;+ @Inject ObjectMapper objectMapper;++ private String bucket;+ private final List<String> queueUrls = new ArrayList<>();+ private final List<String> topicArns = new ArrayList<>();++ @BeforeEach+ void createBucket() {+ bucket = "notification-validation-" + UUID.randomUUID().toString().substring(0, 8);+ given().when().put("/" + bucket).then().statusCode(200);+ }++ @AfterEach+ void removeDestinations() {+ given().when().delete("/" + bucket).then().statusCode(204);+ for (String topicArn : topicArns) {+ snsService.deleteTopic(topicArn, REGION);+ }+ for (String queueUrl : queueUrls) {+ sqsService.deleteQueue(queueUrl, REGION);+ }+ }++ @Test+ void malformedArnIsRejectedEvenWhenDestinationValidationIsSkipped() {+ String xml = notificationConfiguration(queueConfiguration("not-an-arn"));++ given().contentType("application/xml").body(xml)+ .when().put("/" + bucket + "?notification")+ .then().statusCode(400)+ .body(containsString("<Code>InvalidArgument</Code>"))+ .body(containsString("<Message>The ARN could not be parsed</Message>"))+ .body(containsString("<ArgumentName>Queue</ArgumentName>"))+ .body(containsString("<ArgumentValue>not-an-arn</ArgumentValue>"));++ given().header("x-amz-skip-destination-validation", "true")+ .contentType("application/xml").body(xml)+ .when().put("/" + bucket + "?notification")+ .then().statusCode(400)+ .body(containsString("<ArgumentName>Queue</ArgumentName>"));++ given().when().get("/" + bucket + "?notification")+ .then().statusCode(200).body(not(containsString("QueueConfiguration")));+ }++ @Test+ void wrongDestinationServiceIsRejectedBeforeSkipOrExistenceChecks() {+ String queueFieldWithTopicArn = regionResolver.buildArn("sns", REGION, "wrong-service");+ putNotification(notificationConfiguration(queueConfiguration(queueFieldWithTopicArn)), true)+ .statusCode(400)+ .body(containsString("<Message>The ARN could not be parsed</Message>"))+ .body(containsString("<ArgumentName>Queue</ArgumentName>"));++ String topicFieldWithQueueArn = regionResolver.buildArn("sqs", REGION, "wrong-service");+ putNotification(notificationConfiguration(topicConfiguration(topicFieldWithQueueArn)), true)+ .statusCode(400)+ .body(containsString("<ArgumentName>Topic</ArgumentName>"));++ putNotification(notificationConfiguration(lambdaConfiguration("not-a-lambda-arn")), true)+ .statusCode(400)+ .body(containsString("<ArgumentName>CloudFunction</ArgumentName>"));+ }++ @Test+ void missingDestinationsAreReportedTogetherWithoutReplacingPreviousConfiguration() {+ String existingQueueArn = createQueue("existing", Map.of());+ putNotification(notificationConfiguration(queueConfiguration(existingQueueArn)), false)+ .statusCode(200);+ Message initialTestEvent = receiveQueueMessage(queueUrls.getFirst()).getFirst();+ sqsService.deleteMessage(queueUrls.getFirst(), initialTestEvent.getReceiptHandle(), REGION);++ String missingQueueOne = regionResolver.buildArn("sqs", REGION, "missing-one");+ String missingQueueTwo = regionResolver.buildArn("sqs", REGION, "missing-two");+ String missingTopic = regionResolver.buildArn("sns", REGION, "missing-topic");+ String missingFunction = regionResolver.buildArn("lambda", REGION, "function:missing-function");+ putNotification(notificationConfiguration(+ queueConfiguration(missingQueueOne) + queueConfiguration(missingQueueTwo)+ + topicConfiguration(missingTopic) + lambdaConfiguration(missingFunction)), false)+ .statusCode(400)+ .body(containsString("<Code>InvalidArgument</Code>"))+ .body(containsString("Unable to validate the following destination configurations"))+ .body(containsString("<ArgumentName1>" + missingQueueOne + "</ArgumentName1>"))+ .body(containsString("<ArgumentValue1>The destination queue does not exist</ArgumentValue1>"))+ .body(containsString("<ArgumentName2>" + missingQueueTwo + "</ArgumentName2>"))+ .body(containsString("<ArgumentName3>" + missingTopic + "</ArgumentName3>"))+ .body(containsString("<ArgumentName4>" + missingFunction + ", null</ArgumentName4>"))+ .body(containsString("<ArgumentValue4>Not authorized to invoke function ["+ + missingFunction + "]</ArgumentValue4>"));++ given().when().get("/" + bucket + "?notification")+ .then().statusCode(200)+ .body(containsString(existingQueueArn))+ .body(not(containsString(missingQueueOne)))+ .body(not(containsString(missingTopic)));+ assertTrue(receiveQueueMessage(queueUrls.getFirst()).isEmpty());+ }++ @Test+ void skipDestinationValidationStoresUnknownDestinationsWithoutSendingTestEvent() {+ String existingQueueArn = createQueue("skip-existing", Map.of());+ String missingQueueArn = regionResolver.buildArn("sqs", REGION, "skip-missing");+ String missingTopicArn = regionResolver.buildArn("sns", REGION, "skip-topic");+ String missingFunctionArn = regionResolver.buildArn("lambda", REGION, "function:skip-function");++ putNotification(notificationConfiguration(queueConfiguration(existingQueueArn)+ + queueConfiguration(missingQueueArn)+ + topicConfiguration(missingTopicArn)+ + lambdaConfiguration(missingFunctionArn)), true).statusCode(200);++ given().when().get("/" + bucket + "?notification")+ .then().statusCode(200)+ .body(containsString(existingQueueArn))+ .body(containsString(missingQueueArn))+ .body(containsString(missingTopicArn))+ .body(containsString(missingFunctionArn));+ assertTrue(receiveQueueMessage(queueUrls.getFirst()).isEmpty());+ }++ @Test+ void validQueueAndTopicReceiveTestEventWithoutRecords() throws Exception {+ String directQueueArn = createQueue("direct", Map.of());+ String subscribedQueueArn = createQueue("subscribed", Map.of());+ String topicArn = createTopic("test-topic");+ snsService.subscribe(topicArn, "sqs", subscribedQueueArn, REGION,+ Map.of("RawMessageDelivery", "true"));++ putNotification(notificationConfiguration(queueConfiguration(directQueueArn)+ + topicConfiguration(topicArn)), false).statusCode(200);++ assertTestEvent(receiveQueueMessage(queueUrls.get(0)).getFirst().getBody());+ assertTestEvent(receiveQueueMessage(queueUrls.get(1)).getFirst().getBody());+ }++ @Test+ void crossRegionTopicReceivesTestEventAndLaterObjectEvent() throws Exception {+ String topicRegion = "ap-southeast-2";+ String queueName = "notification-topic-cross-region-" + UUID.randomUUID().toString().substring(0, 8);+ String queueUrl = sqsService.createQueue(queueName, Map.of(), topicRegion).getQueueUrl();+ String queueArn = sqsService.getQueueAttributes(queueUrl, List.of("QueueArn"), topicRegion)+ .get("QueueArn");+ String topicArn = snsService.createTopic(queueName, Map.of(), Map.of(), topicRegion).getTopicArn();+ try {+ snsService.subscribe(topicArn, "sqs", queueArn, topicRegion,+ Map.of("RawMessageDelivery", "true"));+ putNotification(notificationConfiguration(topicConfiguration(topicArn)), false).statusCode(200);++ Message testEvent = sqsService.receiveMessage(queueUrl, 1, 0, 0, topicRegion).getFirst();+ assertTestEvent(testEvent.getBody());+ sqsService.deleteMessage(queueUrl, testEvent.getReceiptHandle(), topicRegion);++ given().contentType("text/plain").body("payload")+ .when().put("/" + bucket + "/cross-region-object")+ .then().statusCode(200);+ JsonNode objectEvent = objectMapper.readTree(+ sqsService.receiveMessage(queueUrl, 1, 0, 0, topicRegion).getFirst().getBody());+ assertEquals("aws:s3", objectEvent.path("Records").get(0).path("eventSource").asText());+ assertEquals("cross-region-object", objectEvent.path("Records").get(0)+ .path("s3").path("object").path("key").asText());+ } finally {+ given().when().delete("/" + bucket + "/cross-region-object").then().statusCode(204);+ snsService.deleteTopic(topicArn, topicRegion);+ sqsService.deleteQueue(queueUrl, topicRegion);+ }+ }++ @Test+ void failedTestMessageLeavesPreviousConfigurationUntouched() {+ String validQueueArn = createQueue("before-failure", Map.of());+ putNotification(notificationConfiguration(queueConfiguration(validQueueArn)), false)+ .statusCode(200);++ String fifoQueueArn = createQueue("cannot-send.fifo", Map.of("FifoQueue", "true"));+ putNotification(notificationConfiguration(queueConfiguration(fifoQueueArn)), false)+ .statusCode(400)+ .body(containsString("<Code>InvalidArgument</Code>"))+ .body(containsString("<ArgumentName1>" + fifoQueueArn + "</ArgumentName1>"));++ given().when().get("/" + bucket + "?notification")+ .then().statusCode(200)+ .body(containsString(validQueueArn))+ .body(not(containsString(fifoQueueArn)));+ }++ @Test+ void crossAccountQueueReceivesTestEventWithoutIamEnforcement() throws Exception {+ String queueName = "notification-cross-account-" + UUID.randomUUID().toString().substring(0, 8);+ String auth = "AWS4-HMAC-SHA256 Credential=" + OTHER_ACCOUNT+ + "/20261001/us-east-1/sqs/aws4_request, SignedHeaders=host, Signature=abc";+ String queueUrl = given().header("Authorization", auth)+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateQueue").formParam("QueueName", queueName)+ .when().post("/").then().statusCode(200)+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl");+ String queueArn = "arn:aws:sqs:" + REGION + ":" + OTHER_ACCOUNT + ":" + queueName;+ try {+ putNotification(notificationConfiguration(queueConfiguration(queueArn)), false).statusCode(200);+ Message testEvent = sqsService.receiveMessage(queueUrl, 1, 0, 0, REGION).getFirst();+ assertTestEvent(testEvent.getBody());+ sqsService.deleteMessage(queueUrl, testEvent.getReceiptHandle(), REGION);++ given().contentType("text/plain").body("payload")+ .when().put("/" + bucket + "/cross-account-object").then().statusCode(200);+ assertTrue(sqsService.receiveMessage(queueUrl, 1, 0, 0, REGION).getFirst()+ .getBody().contains("cross-account-object"));+ given().when().delete("/" + bucket + "/cross-account-object").then().statusCode(204);+ } finally {+ given().header("Authorization", auth).contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteQueue").formParam("QueueUrl", queueUrl)+ .when().post("/").then().statusCode(200);+ }+ }++ @Test+ void crossAccountTopicIsValidatedWithoutTestPublishByDefault() {+ String topicName = "notification-cross-account-" + UUID.randomUUID().toString().substring(0, 8);+ String authorization = "AWS4-HMAC-SHA256 Credential=" + OTHER_ACCOUNT+ + "/20261001/us-east-1/sns/aws4_request, SignedHeaders=host, Signature=abc";+ String topicArn = given().header("Authorization", authorization)+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateTopic").formParam("Name", topicName)+ .when().post("/").then().statusCode(200)+ .extract().xmlPath().getString("CreateTopicResponse.CreateTopicResult.TopicArn");+ try {+ assertTrue(snsService.topicExists(topicArn, REGION));+ putNotification(notificationConfiguration(topicConfiguration(topicArn)), false).statusCode(200);+ verify(snsService, never()).publish(eq(topicArn), isNull(), anyString(),+ eq("Amazon S3 Notification"), eq(REGION));+ given().when().get("/" + bucket + "?notification")+ .then().statusCode(200).body(containsString(topicArn));++ String missingArn = "arn:aws:sns:" + REGION + ":" + OTHER_ACCOUNT + ":notification-missing-"+ + UUID.randomUUID();+ putNotification(notificationConfiguration(topicConfiguration(missingArn)), false)+ .statusCode(400).body(containsString("The destination topic does not exist"));+ given().when().get("/" + bucket + "?notification")+ .then().statusCode(200).body(containsString(topicArn)).body(not(containsString(missingArn)));++ String ownTopicArn = createTopic("same-account");+ putNotification(notificationConfiguration(topicConfiguration(ownTopicArn)), false).statusCode(200);+ verify(snsService).publish(eq(ownTopicArn), isNull(), anyString(),+ eq("Amazon S3 Notification"), eq(REGION));+ } finally {+ given().header("Authorization", authorization).contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteTopic").formParam("TopicArn", topicArn)+ .when().post("/").then().statusCode(200);+ }+ }++ private String createQueue(String suffix, Map<String, String> attributes) {+ String queueName = "notification-" + suffix + "-" + UUID.randomUUID().toString().substring(0, 8);+ if (queueName.contains(".fifo-")) {+ queueName = queueName.replace(".fifo-", "-") + ".fifo";+ }+ String queueUrl = sqsService.createQueue(queueName, attributes, REGION).getQueueUrl();+ queueUrls.add(queueUrl);+ return sqsService.getQueueAttributes(queueUrl, List.of("QueueArn"), REGION).get("QueueArn");+ }++ private String createTopic(String suffix) {+ String topicArn = snsService.createTopic(+ "notification-" + suffix + "-" + UUID.randomUUID().toString().substring(0, 8),+ Map.of(), Map.of(), REGION).getTopicArn();+ topicArns.add(topicArn);+ return topicArn;+ }++ private List<Message> receiveQueueMessage(String queueUrl) {+ return sqsService.receiveMessage(queueUrl, 1, 0, 0, REGION);+ }++ private void assertTestEvent(String json) throws Exception {+ JsonNode event = objectMapper.readTree(json);+ assertEquals("Amazon S3", event.path("Service").asText());+ assertEquals("s3:TestEvent", event.path("Event").asText());+ assertEquals(bucket, event.path("Bucket").asText());+ assertNotNull(Instant.parse(event.path("Time").asText()));+ assertFalse(event.path("RequestId").asText().isBlank());+ assertFalse(event.path("HostId").asText().isBlank());+ assertFalse(event.has("Records"));+ }++ private ValidatableResponse putNotification(String xml, boolean skip) {+ return given().header("x-amz-skip-destination-validation", Boolean.toString(skip))+ .contentType("application/xml").body(xml)+ .when().put("/" + bucket + "?notification")+ .then();+ }++ private static String notificationConfiguration(String contents) {+ return "<NotificationConfiguration>" + contents + "</NotificationConfiguration>";+ }++ private static String queueConfiguration(String arn) {+ return "<QueueConfiguration><Queue>" + arn+ + "</Queue><Event>s3:ObjectCreated:*</Event></QueueConfiguration>";+ }++ private static String topicConfiguration(String arn) {+ return "<TopicConfiguration><Topic>" + arn+ + "</Topic><Event>s3:ObjectCreated:*</Event></TopicConfiguration>";+ }++ private static String lambdaConfiguration(String arn) {+ return "<CloudFunctionConfiguration><CloudFunction>" + arn+ + "</CloudFunction><Event>s3:ObjectCreated:*</Event></CloudFunctionConfiguration>";+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPostIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPostIntegrationTest.javaindex 3d15dfc774..975647ac1a 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPostIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPostIntegrationTest.java@@ -108,6 +108,10 @@ class S3PresignedPostIntegrationTest {.then().statusCode(200);+ consumeValidationEvent(postQueueUrl);+ consumeValidationEvent(putQueueUrl);+ consumeValidationEvent(wildcardQueueUrl);+given().multiPart("key", "uploads/notified.txt").multiPart("file", "notified.txt", "notified".getBytes(StandardCharsets.UTF_8), "text/plain")@@ -169,6 +173,34 @@ class S3PresignedPostIntegrationTest {containsString("\"eventName\":\"ObjectCreated:Post\""));}+ private void consumeValidationEvent(String queueUrl) {+ String receiptHandle = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body",+ containsString("\"Event\":\"s3:TestEvent\""))+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body",+ containsString("\"Bucket\":\"" + BUCKET + "\""))+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body", not(containsString("\"Records\"")))+ .extract().xmlPath().getString("ReceiveMessageResponse.ReceiveMessageResult.Message.ReceiptHandle");++ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("ReceiptHandle", receiptHandle)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }+private void deleteQueue(String queueUrl) {given().contentType("application/x-www-form-urlencoded")diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex 38e26b7e59..7a1b25aff6 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -5,7 +5,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.lambda.LambdaService;import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.s3.model.ChecksumType;import io.github.hectorvent.floci.services.s3.model.FilterRule;import io.github.hectorvent.floci.services.s3.model.GetObjectAttributesResult;@@ -34,6 +36,8 @@ import java.util.Map;import java.util.Set;import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;class S3ServiceTest {@@ -594,7 +598,7 @@ class S3ServiceTest {S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve("notif-s3"),false, lambdaInvoker, regionResolver);service.createBucket("test-bucket", "ap-northeast-1");- service.putBucketNotificationConfiguration("test-bucket", lambdaNotificationConfig("uploads/", ".json"));+ service.putBucketNotificationConfiguration("test-bucket", lambdaNotificationConfig("uploads/", ".json"), true);service.putObject("test-bucket", "uploads/test.json", "{\"ok\":true}".getBytes(StandardCharsets.UTF_8),"application/json", null);@@ -613,7 +617,7 @@ class S3ServiceTest {S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve("notif-s3-no-match"),false, lambdaInvoker, regionResolver);service.createBucket("test-bucket", "ap-northeast-1");- service.putBucketNotificationConfiguration("test-bucket", lambdaNotificationConfig("uploads/", ".json"));+ service.putBucketNotificationConfiguration("test-bucket", lambdaNotificationConfig("uploads/", ".json"), true);service.putObject("test-bucket", "incoming/test.txt", "ignored".getBytes(StandardCharsets.UTF_8),"text/plain", null);@@ -635,13 +639,90 @@ class S3ServiceTest {"arn:aws:lambda:ap-northeast-1:000000000000:function:s3-notif-test:PROD",List.of("s3:ObjectCreated:Put"),List.of()));- service.putBucketNotificationConfiguration("test-bucket", config);+ service.putBucketNotificationConfiguration("test-bucket", config, true);service.putObject("test-bucket", "a.json", "{}".getBytes(StandardCharsets.UTF_8), "application/json", null);assertEquals("s3-notif-test:PROD", lambdaInvoker.functionName);}+ @Test+ void foreignLambdaArnDoesNotResolveToSameNamedLocalFunction() {+ LambdaService lambdaService = mock(LambdaService.class);+ RegionResolver regionResolver = new RegionResolver("us-east-1", "000000000000");+ String localArn = "arn:aws:lambda:us-east-1:000000000000:function:shared-name";+ String foreignArn = "arn:aws:lambda:us-east-1:111111111111:function:shared-name";+ LambdaFunction localFunction = new LambdaFunction();+ localFunction.setFunctionArn(localArn);+ when(lambdaService.getFunction("us-east-1", localArn, null)).thenReturn(localFunction);+ when(lambdaService.getFunction("us-east-1", foreignArn, null)).thenReturn(localFunction);++ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),+ tempDir.resolve("lambda-account-validation"), false, lambdaService, regionResolver);+ service.createBucket("test-bucket", "us-east-1");+ NotificationConfiguration local = new NotificationConfiguration();+ local.getLambdaFunctionConfigurations().add(new LambdaNotification(+ "local", localArn, List.of("s3:ObjectCreated:*"), List.of()));+ service.putBucketNotificationConfiguration("test-bucket", local);++ NotificationConfiguration foreign = new NotificationConfiguration();+ foreign.getLambdaFunctionConfigurations().add(new LambdaNotification(+ "foreign", foreignArn, List.of("s3:ObjectCreated:*"), List.of()));+ AwsException error = assertThrows(AwsException.class,+ () -> service.putBucketNotificationConfiguration("test-bucket", foreign));+ assertEquals("InvalidArgument", error.getErrorCode());+ assertEquals(foreignArn + ", null", error.getExtendedData().get("ArgumentName1"));+ assertEquals(localArn, service.getBucketNotificationConfiguration("test-bucket")+ .getLambdaFunctionConfigurations().getFirst().functionArn());+ }++ @Test+ void notificationValidationCannotRestoreADeletedBucket() {+ LambdaService lambdaService = mock(LambdaService.class);+ RegionResolver regionResolver = new RegionResolver("ap-northeast-1", "000000000000");+ String arn = "arn:aws:lambda:ap-northeast-1:000000000000:function:s3-notif-test";+ LambdaFunction function = new LambdaFunction();+ function.setFunctionArn(arn);+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),+ tempDir.resolve("notification-deleted-bucket"), false, lambdaService, regionResolver);+ service.createBucket("test-bucket", "ap-northeast-1");+ when(lambdaService.getFunction("ap-northeast-1", arn, null)).thenAnswer(ignored -> {+ service.deleteBucket("test-bucket");+ return function;+ });++ AwsException error = assertThrows(AwsException.class,+ () -> service.putBucketNotificationConfiguration("test-bucket",+ lambdaNotificationConfig("", "")));+ assertEquals("NoSuchBucket", error.getErrorCode());+ assertEquals("NoSuchBucket", assertThrows(AwsException.class,+ () -> service.getBucketNotificationConfiguration("test-bucket")).getErrorCode());+ }++ @Test+ void notificationValidationCannotOverwriteARecreatedBucket() {+ LambdaService lambdaService = mock(LambdaService.class);+ RegionResolver regionResolver = new RegionResolver("ap-northeast-1", "000000000000");+ String arn = "arn:aws:lambda:ap-northeast-1:000000000000:function:s3-notif-test";+ LambdaFunction function = new LambdaFunction();+ function.setFunctionArn(arn);+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),+ tempDir.resolve("notification-recreated-bucket"), false, lambdaService, regionResolver);+ service.createBucket("test-bucket", "ap-northeast-1");+ when(lambdaService.getFunction("ap-northeast-1", arn, null)).thenAnswer(ignored -> {+ service.deleteBucket("test-bucket");+ service.createBucket("test-bucket", "ap-northeast-1");+ return function;+ });++ AwsException error = assertThrows(AwsException.class,+ () -> service.putBucketNotificationConfiguration("test-bucket",+ lambdaNotificationConfig("", "")));+ assertEquals("NoSuchBucket", error.getErrorCode());+ assertTrue(service.getBucketNotificationConfiguration("test-bucket")+ .getLambdaFunctionConfigurations().isEmpty());+ }+@Testvoid deleteObjectVersionFiresObjectRemovedDeleteWithTheDeletedVersionId() throws IOException {RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();@@ -694,7 +775,7 @@ class S3ServiceTest {"arn:aws:lambda:us-east-1:000000000000:function:s3-notif-test",List.of("s3:ObjectRemoved:*"),List.of()));- service.putBucketNotificationConfiguration("test-bucket", config);+ service.putBucketNotificationConfiguration("test-bucket", config, true);return service;}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningIntegrationTest.javaindex 229b74d837..13ea1696a0 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningIntegrationTest.java@@ -436,6 +436,28 @@ class S3VersioningIntegrationTest {.then().statusCode(200);+ String validationReceipt = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReceiveMessageResponse.ReceiveMessageResult.Message.Body",+ containsString("\"Event\":\"s3:TestEvent\""))+ .extract().xmlPath().getString("ReceiveMessageResponse.ReceiveMessageResult.Message.ReceiptHandle");+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "DeleteMessage")+ .formParam("QueueUrl", queueUrl)+ .formParam("ReceiptHandle", validationReceipt)+ .when()+ .post("/")+ .then()+ .statusCode(200);+String olderVersionId = given().body("v1").contentType("text/plain")diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.javaindex 643cb98730..9cbe96126c 100644--- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.firehose.FirehoseService;import io.github.hectorvent.floci.services.firehose.model.Record;@@ -793,6 +794,42 @@ class SnsServiceTest {"arn:aws:sns:us-east-1:000000000000:ghost-topic", REGION));}+ @ParameterizedTest+ @ValueSource(strings = {ACCOUNT, "000000000002"})+ void topicExists_migratesLegacyTopicToArnAccount(String accountId) {+ String arn = "arn:aws:sns:us-east-1:" + accountId + ":legacy-topic";+ String key = "topic::" + REGION + "::" + arn;+ Topic topic = new Topic("legacy-topic", arn);+ InMemoryStorage<String, Topic> raw = new InMemoryStorage<>();+ raw.put(key, topic);+ AccountAwareStorageBackend<Topic> store = new AccountAwareStorageBackend<>(raw, null, ACCOUNT);+ SnsService service = new SnsService(store, new InMemoryStorage<>(),+ new RegionResolver(REGION, ACCOUNT), null, null);++ assertTrue(service.topicExists(arn, REGION));+ assertEquals(topic, raw.get(accountId + "/" + key).orElseThrow());+ assertTrue(raw.get(key).isEmpty());+ if (!ACCOUNT.equals(accountId)) {+ assertTrue(raw.get(ACCOUNT + "/" + key).isEmpty());+ }+ }++ @Test+ void topicExists_doesNotClaimLegacyTopicWithDifferentArn() {+ String arn = "arn:aws:sns:us-east-1:000000000002:legacy-topic";+ String key = "topic::" + REGION + "::" + arn;+ Topic otherTopic = new Topic("legacy-topic", "arn:aws:sns:us-east-1:000000000003:legacy-topic");+ InMemoryStorage<String, Topic> raw = new InMemoryStorage<>();+ raw.put(key, otherTopic);+ AccountAwareStorageBackend<Topic> store = new AccountAwareStorageBackend<>(raw, null, ACCOUNT);+ SnsService service = new SnsService(store, new InMemoryStorage<>(),+ new RegionResolver(REGION, ACCOUNT), null, null);++ assertFalse(service.topicExists(arn, REGION));+ assertEquals(otherTopic, raw.get(key).orElseThrow());+ assertTrue(raw.get("000000000002/" + key).isEmpty());+ }+private static Map<String, MessageAttributeValue> attr(String name, String value, String dataType) {return Map.of(name, new MessageAttributeValue(value, dataType));}