Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sns): deliver published messages to firehose subscriptions (#4079)
Vũ Thế Huyccb6b63d000b
diff --git a/docs/services/sns.md b/docs/services/sns.mdindex 4b14127048..96e754e89e 100644--- a/docs/services/sns.md+++ b/docs/services/sns.md@@ -83,10 +83,11 @@ aws sqs receive-message \Floci supports real SNS → SQS fan-out. When you publish to a topic, all SQS-subscribed queues receive the message immediately.Supported subscription protocols:-- `sqs` — delivers to a Floci SQS queue-- `lambda` — invokes a Floci Lambda function-- `http` / `https` — posts to an HTTP endpoint-- `application` — fans out to a mobile push platform endpoint (see [Mobile push](#mobile-push-mock))+- `sqs`: delivers to a Floci SQS queue+- `lambda`: invokes a Floci Lambda function+- `firehose`: puts records to a Floci Firehose delivery stream+- `http` / `https`: posts to an HTTP endpoint+- `application`: fans out to a mobile push platform endpoint (see [Mobile push](#mobile-push-mock))## Message size@@ -110,8 +111,7 @@ A topic above `262144` is restricted: at most 100 subscriptions, every one of th`firehose` or `lambda`. Floci enforces the protocol rule on `Subscribe` and `SetTopicAttributes`alike, and the subscription count on `SetTopicAttributes` only — matching AWS, which lets analready-raised topic drift past 100 and catches it the next time the attribute is set. Pending-confirmations count towards both. (`firehose` subscriptions are accepted for validation but not-delivered to, as before.)+confirmations count towards both.| Action | Condition | Error code | HTTP ||---|---|---|---|diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.javaindex 00db455772..7774e2c91b 100644--- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java@@ -10,6 +10,8 @@ import io.github.hectorvent.floci.core.resource.ResourceProvider;import io.github.hectorvent.floci.core.resource.SupportedResourceType;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.firehose.FirehoseService;+import io.github.hectorvent.floci.services.firehose.model.Record;import io.github.hectorvent.floci.services.lambda.LambdaService;import io.github.hectorvent.floci.services.lambda.model.InvocationType;import io.github.hectorvent.floci.core.storage.InMemoryStorage;@@ -100,6 +102,7 @@ public class SnsService implements Resettable, ResourceProvider {private final RegionResolver regionResolver;private final SqsService sqsService;private final LambdaService lambdaService;+ private final FirehoseService firehoseService;private final String baseUrl;private final ObjectMapper objectMapper;private final HttpClient httpClient;@@ -109,7 +112,8 @@ public class SnsService implements Resettable, ResourceProvider {@Injectpublic SnsService(StorageFactory storageFactory, EmulatorConfig config,RegionResolver regionResolver, SqsService sqsService,- LambdaService lambdaService, ObjectMapper objectMapper) {+ LambdaService lambdaService, FirehoseService firehoseService,+ ObjectMapper objectMapper) {this(storageFactory.create("sns", "sns-topics.json",new TypeReference<Map<String, Topic>>() {@@ -129,6 +133,7 @@ public class SnsService implements Resettable, ResourceProvider {regionResolver,sqsService,lambdaService,+ firehoseService,config.effectiveBaseUrl(),objectMapper);@@ -141,10 +146,18 @@ public class SnsService implements Resettable, ResourceProvider {StorageBackend<String, Subscription> subscriptionStore,RegionResolver regionResolver, SqsService sqsService,LambdaService lambdaService) {+ this(topicStore, subscriptionStore, regionResolver, sqsService, lambdaService, null);+ }++ SnsService(StorageBackend<String, Topic> topicStore,+ StorageBackend<String, Subscription> subscriptionStore,+ RegionResolver regionResolver, SqsService sqsService,+ LambdaService lambdaService,+ FirehoseService firehoseService) {this(topicStore, subscriptionStore,new InMemoryStorage<>(),new InMemoryStorage<>(),- regionResolver, sqsService, lambdaService);+ regionResolver, sqsService, lambdaService, firehoseService);}SnsService(StorageBackend<String, Topic> topicStore,@@ -153,6 +166,17 @@ public class SnsService implements Resettable, ResourceProvider {StorageBackend<String, PlatformEndpoint> platformEndpointStore,RegionResolver regionResolver, SqsService sqsService,LambdaService lambdaService) {+ this(topicStore, subscriptionStore, platformAppStore, platformEndpointStore,+ regionResolver, sqsService, lambdaService, null);+ }++ SnsService(StorageBackend<String, Topic> topicStore,+ StorageBackend<String, Subscription> subscriptionStore,+ StorageBackend<String, PlatformApplication> platformAppStore,+ StorageBackend<String, PlatformEndpoint> platformEndpointStore,+ RegionResolver regionResolver, SqsService sqsService,+ LambdaService lambdaService,+ FirehoseService firehoseService) {this.topicStore = topicStore;this.subscriptionStore = subscriptionStore;this.platformAppStore = platformAppStore;@@ -161,6 +185,7 @@ public class SnsService implements Resettable, ResourceProvider {this.regionResolver = regionResolver;this.sqsService = sqsService;this.lambdaService = lambdaService;+ this.firehoseService = firehoseService;this.baseUrl = "http://localhost:4566";this.objectMapper = new ObjectMapper();this.httpClient = null;@@ -173,6 +198,18 @@ public class SnsService implements Resettable, ResourceProvider {StorageBackend<String, SentSms> smsStore,RegionResolver regionResolver, SqsService sqsService,LambdaService lambdaService, String baseUrl, ObjectMapper objectMapper) {+ this(topicStore, subscriptionStore, platformAppStore, platformEndpointStore,+ smsStore, regionResolver, sqsService, lambdaService, null, baseUrl, objectMapper);+ }++ SnsService(StorageBackend<String, Topic> topicStore,+ StorageBackend<String, Subscription> subscriptionStore,+ StorageBackend<String, PlatformApplication> platformAppStore,+ StorageBackend<String, PlatformEndpoint> platformEndpointStore,+ StorageBackend<String, SentSms> smsStore,+ RegionResolver regionResolver, SqsService sqsService,+ LambdaService lambdaService, FirehoseService firehoseService,+ String baseUrl, ObjectMapper objectMapper) {this.topicStore = topicStore;this.subscriptionStore = subscriptionStore;this.platformAppStore = platformAppStore;@@ -181,6 +218,7 @@ public class SnsService implements Resettable, ResourceProvider {this.regionResolver = regionResolver;this.sqsService = sqsService;this.lambdaService = lambdaService;+ this.firehoseService = firehoseService;this.baseUrl = baseUrl;this.objectMapper = objectMapper;this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build();@@ -1700,6 +1738,41 @@ public class SnsService implements Resettable, ResourceProvider {capturePushForEndpoint(endpoint, app, message, subject, messageStructure, messageAttributes);LOG.debugv("Delivered SNS message to platform endpoint: {0}", endpointArn);}+ case "firehose" -> {+ if (firehoseService == null) {+ break;+ }+ String streamName;+ String region;+ String accountId;+ if (AwsArnUtils.isArn(sub.getEndpoint())) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(sub.getEndpoint());+ region = arn.region().isEmpty() ? extractRegionFromArn(topicArn) : arn.region();+ accountId = arn.accountId().isEmpty() ? regionResolver.getAccountId() : arn.accountId();+ String resource = arn.resource();+ streamName = resource.startsWith("deliverystream/")+ ? resource.substring("deliverystream/".length())+ : resource;+ } else {+ streamName = sub.getEndpoint();+ region = extractRegionFromArn(topicArn);+ accountId = regionResolver.getAccountId();+ }+ if (region == null || region.isEmpty()) {+ region = regionResolver.getDefaultRegion();+ }+ if (accountId == null || accountId.isEmpty()) {+ accountId = regionResolver.getAccountId();+ }+ boolean rawDelivery = sub.getAttributes() != null+ && "true".equalsIgnoreCase(sub.getAttributes().get("RawMessageDelivery"));+ String body = rawDelivery+ ? protocolMessage+ : buildSnsEnvelope(protocolMessage, subject, messageAttributes, topicArn, messageId);+ byte[] data = body.getBytes(StandardCharsets.UTF_8);+ firehoseService.putRecord(accountId, region, streamName, new Record(data));+ LOG.debugv("Delivered SNS message to Firehose: {0} ({1}) raw={2}", sub.getEndpoint(), streamName, rawDelivery);+ }case "email", "email-json" -> LOG.infov("SNS email delivery (stub): to={0}, subject={1}, message={2}",sub.getEndpoint(), subject, protocolMessage);case "sms" -> LOG.infov("SNS SMS delivery (stub): to={0}, message={1}", sub.getEndpoint(), protocolMessage);diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsFirehoseIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsFirehoseIntegrationTest.javanew file mode 100644index 0000000000..341a1ec4ed--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsFirehoseIntegrationTest.java@@ -0,0 +1,178 @@+package io.github.hectorvent.floci.services.sns;++import io.github.hectorvent.floci.services.firehose.FirehoseService;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.not;++@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class SnsFirehoseIntegrationTest {++ private static final String FIREHOSE_JSON_CT = "application/x-amz-json-1.1";+ private static final String ENVELOPE_BUCKET = "sns-fh-envelope-bucket";+ private static final String ENVELOPE_STREAM = "sns-fh-envelope-stream";+ private static final String RAW_BUCKET = "sns-fh-raw-bucket";+ private static final String RAW_STREAM = "sns-fh-raw-stream";++ private static String topicArn;+ private static String envelopeSubArn;+ private static String rawSubArn;++ @Inject+ FirehoseService firehoseService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void setupBucketsAndStreams() {+ // Create destination S3 buckets+ given().when().put("/" + ENVELOPE_BUCKET).then().statusCode(200);+ given().when().put("/" + RAW_BUCKET).then().statusCode(200);++ // Create Firehose delivery streams+ given()+ .contentType(FIREHOSE_JSON_CT)+ .header("X-Amz-Target", "Firehose_20150804.CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",+ "BucketARN": "arn:aws:s3:::%s",+ "Prefix": "envelope/"+ }+ }+ """.formatted(ENVELOPE_STREAM, ENVELOPE_BUCKET))+ .when().post("/")+ .then().statusCode(200);++ given()+ .contentType(FIREHOSE_JSON_CT)+ .header("X-Amz-Target", "Firehose_20150804.CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",+ "BucketARN": "arn:aws:s3:::%s",+ "Prefix": "raw/"+ }+ }+ """.formatted(RAW_STREAM, RAW_BUCKET))+ .when().post("/")+ .then().statusCode(200);+ }++ @Test+ @Order(2)+ void createTopicAndSubscriptions() {+ // Create SNS topic+ topicArn = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateTopic")+ .formParam("Name", "sns-to-firehose-topic")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("CreateTopicResponse.CreateTopicResult.TopicArn");++ // Subscribe envelope delivery stream+ String envelopeStreamArn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/" + ENVELOPE_STREAM;+ envelopeSubArn = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "Subscribe")+ .formParam("TopicArn", topicArn)+ .formParam("Protocol", "firehose")+ .formParam("Endpoint", envelopeStreamArn)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<SubscriptionArn>"))+ .extract().xmlPath().getString("SubscribeResponse.SubscribeResult.SubscriptionArn");++ // Subscribe raw delivery stream+ String rawStreamArn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/" + RAW_STREAM;+ rawSubArn = given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "Subscribe")+ .formParam("TopicArn", topicArn)+ .formParam("Protocol", "firehose")+ .formParam("Endpoint", rawStreamArn)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<SubscriptionArn>"))+ .extract().xmlPath().getString("SubscribeResponse.SubscribeResult.SubscriptionArn");++ // Enable RawMessageDelivery on the second subscription+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "SetSubscriptionAttributes")+ .formParam("SubscriptionArn", rawSubArn)+ .formParam("AttributeName", "RawMessageDelivery")+ .formParam("AttributeValue", "true")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(3)+ void publishMessage_deliversToFirehoseSubscriptions() {+ given()+ .contentType("application/x-www-form-urlencoded")+ .formParam("Action", "Publish")+ .formParam("TopicArn", topicArn)+ .formParam("Message", "{\"event\":\"order_created\",\"id\":12345}")+ .formParam("Subject", "Order Notification")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(containsString("<MessageId>"));++ // Flush both firehose streams so buffered records land in S3+ firehoseService.flush(ENVELOPE_STREAM);+ firehoseService.flush(RAW_STREAM);++ // Verify envelope delivery in ENVELOPE_BUCKET+ String envelopeKey = given().when().get("/" + ENVELOPE_BUCKET + "?prefix=envelope/")+ .then().statusCode(200)+ .extract().xmlPath().getString("ListBucketResult.Contents[0].Key");++ given().when().get("/" + ENVELOPE_BUCKET + "/" + envelopeKey)+ .then().statusCode(200)+ .body(containsString("\"Type\":\"Notification\""))+ .body(containsString("\"Subject\":\"Order Notification\""))+ .body(containsString("order_created"))+ .body(containsString("12345"));++ // Verify raw delivery in RAW_BUCKET+ String rawKey = given().when().get("/" + RAW_BUCKET + "?prefix=raw/")+ .then().statusCode(200)+ .extract().xmlPath().getString("ListBucketResult.Contents[0].Key");++ given().when().get("/" + RAW_BUCKET + "/" + rawKey)+ .then().statusCode(200)+ .body(containsString("{\"event\":\"order_created\",\"id\":12345}"))+ .body(not(containsString("\"Type\":\"Notification\"")));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.javaindex 91ad0f1708..0da2fdf9dd 100644--- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java@@ -5,16 +5,25 @@ import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.firehose.FirehoseService;+import io.github.hectorvent.floci.services.firehose.model.Record;import io.github.hectorvent.floci.services.sns.model.Subscription;import io.github.hectorvent.floci.services.sns.model.Topic;import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;+import java.nio.charset.StandardCharsets;import java.util.List;import java.util.Map;import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;class SnsServiceTest {@@ -721,4 +730,118 @@ class SnsServiceTest {assertTrue(snsService.matchesFilterPolicy(nested, body("{\"detail\":{\"b\":\"2\"}}"), null));assertFalse(snsService.matchesFilterPolicy(nested, body("{\"detail\":{\"b\":\"3\"}}"), null));}++ @Test+ void publish_deliversToFirehoseSubscription_withJsonEnvelope() throws Exception {+ FirehoseService firehoseService = mock(FirehoseService.class);+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);+ SnsService service = new SnsService(+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ regionResolver,+ null,+ null,+ firehoseService+ );++ Topic topic = service.createTopic("firehose-topic", null, null, REGION);+ String streamArn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/test-stream";+ service.subscribe(topic.getTopicArn(), "firehose", streamArn, REGION, Map.of());++ String messageId = service.publish(topic.getTopicArn(), null, "Hello Firehose", "Test Subject", REGION);+ assertNotNull(messageId);++ ArgumentCaptor<Record> recordCaptor = ArgumentCaptor.forClass(Record.class);+ verify(firehoseService).putRecord(eq(ACCOUNT), eq(REGION), eq("test-stream"), recordCaptor.capture());++ Record captured = recordCaptor.getValue();+ assertNotNull(captured);+ assertNotNull(captured.getData());+ String payload = new String(captured.getData(), StandardCharsets.UTF_8);++ JsonNode json = new ObjectMapper().readTree(payload);+ assertEquals("Notification", json.get("Type").asText());+ assertEquals(messageId, json.get("MessageId").asText());+ assertEquals(topic.getTopicArn(), json.get("TopicArn").asText());+ assertEquals("Test Subject", json.get("Subject").asText());+ assertEquals("Hello Firehose", json.get("Message").asText());+ }++ @Test+ void publish_deliversToFirehoseSubscription_rawMessageDelivery() {+ FirehoseService firehoseService = mock(FirehoseService.class);+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);+ SnsService service = new SnsService(+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ regionResolver,+ null,+ null,+ firehoseService+ );++ Topic topic = service.createTopic("firehose-topic-raw", null, null, REGION);+ String streamArn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/test-stream-raw";+ service.subscribe(topic.getTopicArn(), "firehose", streamArn, REGION,+ Map.of("RawMessageDelivery", "true"));++ String message = "{\"raw\":\"payload\"}";+ String messageId = service.publish(topic.getTopicArn(), null, message, null, REGION);+ assertNotNull(messageId);++ ArgumentCaptor<Record> recordCaptor = ArgumentCaptor.forClass(Record.class);+ verify(firehoseService).putRecord(eq(ACCOUNT), eq(REGION), eq("test-stream-raw"), recordCaptor.capture());++ Record captured = recordCaptor.getValue();+ assertEquals(message, new String(captured.getData(), StandardCharsets.UTF_8));+ }++ @Test+ void publish_deliversToFirehoseSubscription_bareStreamNameEndpoint() {+ FirehoseService firehoseService = mock(FirehoseService.class);+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);+ SnsService service = new SnsService(+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ regionResolver,+ null,+ null,+ firehoseService+ );++ Topic topic = service.createTopic("firehose-bare-topic", null, null, REGION);+ service.subscribe(topic.getTopicArn(), "firehose", "bare-stream", REGION,+ Map.of("RawMessageDelivery", "true"));++ service.publish(topic.getTopicArn(), null, "bare-msg", null, REGION);++ ArgumentCaptor<Record> recordCaptor = ArgumentCaptor.forClass(Record.class);+ verify(firehoseService).putRecord(eq(ACCOUNT), eq(REGION), eq("bare-stream"), recordCaptor.capture());+ assertEquals("bare-msg", new String(recordCaptor.getValue().getData(), StandardCharsets.UTF_8));+ }++ @Test+ void publish_firehoseDeliveryFailure_doesNotFailPublisher() {+ FirehoseService firehoseService = mock(FirehoseService.class);+ doThrow(new RuntimeException("stream not found"))+ .when(firehoseService).putRecord(any(), any(), any(), any());++ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);+ SnsService service = new SnsService(+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ regionResolver,+ null,+ null,+ firehoseService+ );++ Topic topic = service.createTopic("firehose-fail-topic", null, null, REGION);+ service.subscribe(topic.getTopicArn(), "firehose",+ "arn:aws:firehose:us-east-1:000000000000:deliverystream/failing-stream", REGION, Map.of());++ // Delivery error is logged and tolerated; publisher receives message ID+ String messageId = service.publish(topic.getTopicArn(), null, "msg", null, REGION);+ assertNotNull(messageId);+ }}