Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sns): deliver published messages to firehose subscriptions (#4079)

Vũ Thế Huyccb6b63d000b
diff --git a/docs/services/sns.md b/docs/services/sns.md
index 4b14127048..96e754e89e 100644
--- a/docs/services/sns.md
+++ b/docs/services/sns.md
@@ -83,10 +83,11 @@ aws sqs receive-message \
Floci supports real SNS → SQS fan-out. When you publish to a topic, all SQS-subscribed queues receive the message immediately.
Supported subscription protocols:
-- `sqs` — delivers to a Floci SQS queue
-- `lambda` — invokes a Floci Lambda function
-- `http` / `https` — posts to an HTTP endpoint
-- `application` — fans out to a mobile push platform endpoint (see [Mobile push](#mobile-push-mock))
+- `sqs`: delivers to a Floci SQS queue
+- `lambda`: invokes a Floci Lambda function
+- `firehose`: puts records to a Floci Firehose delivery stream
+- `http` / `https`: posts to an HTTP endpoint
+- `application`: fans out to a mobile push platform endpoint (see [Mobile push](#mobile-push-mock))
## Message size
@@ -110,8 +111,7 @@ A topic above `262144` is restricted: at most 100 subscriptions, every one of th
`firehose` or `lambda`. Floci enforces the protocol rule on `Subscribe` and `SetTopicAttributes`
alike, and the subscription count on `SetTopicAttributes` only — matching AWS, which lets an
already-raised topic drift past 100 and catches it the next time the attribute is set. Pending
-confirmations count towards both. (`firehose` subscriptions are accepted for validation but not
-delivered to, as before.)
+confirmations count towards both.
| Action | Condition | Error code | HTTP |
|---|---|---|---|
diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
index 00db455772..7774e2c91b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java
@@ -10,6 +10,8 @@ import io.github.hectorvent.floci.core.resource.ResourceProvider;
import io.github.hectorvent.floci.core.resource.SupportedResourceType;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.firehose.FirehoseService;
+import io.github.hectorvent.floci.services.firehose.model.Record;
import io.github.hectorvent.floci.services.lambda.LambdaService;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
@@ -100,6 +102,7 @@ public class SnsService implements Resettable, ResourceProvider {
private final RegionResolver regionResolver;
private final SqsService sqsService;
private final LambdaService lambdaService;
+ private final FirehoseService firehoseService;
private final String baseUrl;
private final ObjectMapper objectMapper;
private final HttpClient httpClient;
@@ -109,7 +112,8 @@ public class SnsService implements Resettable, ResourceProvider {
@Inject
public SnsService(StorageFactory storageFactory, EmulatorConfig config,
RegionResolver regionResolver, SqsService sqsService,
- LambdaService lambdaService, ObjectMapper objectMapper) {
+ LambdaService lambdaService, FirehoseService firehoseService,
+ ObjectMapper objectMapper) {
this(
storageFactory.create("sns", "sns-topics.json",
new TypeReference<Map<String, Topic>>() {
@@ -129,6 +133,7 @@ public class SnsService implements Resettable, ResourceProvider {
regionResolver,
sqsService,
lambdaService,
+ firehoseService,
config.effectiveBaseUrl(),
objectMapper
);
@@ -141,10 +146,18 @@ public class SnsService implements Resettable, ResourceProvider {
StorageBackend<String, Subscription> subscriptionStore,
RegionResolver regionResolver, SqsService sqsService,
LambdaService lambdaService) {
+ this(topicStore, subscriptionStore, regionResolver, sqsService, lambdaService, null);
+ }
+
+ SnsService(StorageBackend<String, Topic> topicStore,
+ StorageBackend<String, Subscription> subscriptionStore,
+ RegionResolver regionResolver, SqsService sqsService,
+ LambdaService lambdaService,
+ FirehoseService firehoseService) {
this(topicStore, subscriptionStore,
new InMemoryStorage<>(),
new InMemoryStorage<>(),
- regionResolver, sqsService, lambdaService);
+ regionResolver, sqsService, lambdaService, firehoseService);
}
SnsService(StorageBackend<String, Topic> topicStore,
@@ -153,6 +166,17 @@ public class SnsService implements Resettable, ResourceProvider {
StorageBackend<String, PlatformEndpoint> platformEndpointStore,
RegionResolver regionResolver, SqsService sqsService,
LambdaService lambdaService) {
+ this(topicStore, subscriptionStore, platformAppStore, platformEndpointStore,
+ regionResolver, sqsService, lambdaService, null);
+ }
+
+ SnsService(StorageBackend<String, Topic> topicStore,
+ StorageBackend<String, Subscription> subscriptionStore,
+ StorageBackend<String, PlatformApplication> platformAppStore,
+ StorageBackend<String, PlatformEndpoint> platformEndpointStore,
+ RegionResolver regionResolver, SqsService sqsService,
+ LambdaService lambdaService,
+ FirehoseService firehoseService) {
this.topicStore = topicStore;
this.subscriptionStore = subscriptionStore;
this.platformAppStore = platformAppStore;
@@ -161,6 +185,7 @@ public class SnsService implements Resettable, ResourceProvider {
this.regionResolver = regionResolver;
this.sqsService = sqsService;
this.lambdaService = lambdaService;
+ this.firehoseService = firehoseService;
this.baseUrl = "http://localhost:4566";
this.objectMapper = new ObjectMapper();
this.httpClient = null;
@@ -173,6 +198,18 @@ public class SnsService implements Resettable, ResourceProvider {
StorageBackend<String, SentSms> smsStore,
RegionResolver regionResolver, SqsService sqsService,
LambdaService lambdaService, String baseUrl, ObjectMapper objectMapper) {
+ this(topicStore, subscriptionStore, platformAppStore, platformEndpointStore,
+ smsStore, regionResolver, sqsService, lambdaService, null, baseUrl, objectMapper);
+ }
+
+ SnsService(StorageBackend<String, Topic> topicStore,
+ StorageBackend<String, Subscription> subscriptionStore,
+ StorageBackend<String, PlatformApplication> platformAppStore,
+ StorageBackend<String, PlatformEndpoint> platformEndpointStore,
+ StorageBackend<String, SentSms> smsStore,
+ RegionResolver regionResolver, SqsService sqsService,
+ LambdaService lambdaService, FirehoseService firehoseService,
+ String baseUrl, ObjectMapper objectMapper) {
this.topicStore = topicStore;
this.subscriptionStore = subscriptionStore;
this.platformAppStore = platformAppStore;
@@ -181,6 +218,7 @@ public class SnsService implements Resettable, ResourceProvider {
this.regionResolver = regionResolver;
this.sqsService = sqsService;
this.lambdaService = lambdaService;
+ this.firehoseService = firehoseService;
this.baseUrl = baseUrl;
this.objectMapper = objectMapper;
this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build();
@@ -1700,6 +1738,41 @@ public class SnsService implements Resettable, ResourceProvider {
capturePushForEndpoint(endpoint, app, message, subject, messageStructure, messageAttributes);
LOG.debugv("Delivered SNS message to platform endpoint: {0}", endpointArn);
}
+ case "firehose" -> {
+ if (firehoseService == null) {
+ break;
+ }
+ String streamName;
+ String region;
+ String accountId;
+ if (AwsArnUtils.isArn(sub.getEndpoint())) {
+ AwsArnUtils.Arn arn = AwsArnUtils.parse(sub.getEndpoint());
+ region = arn.region().isEmpty() ? extractRegionFromArn(topicArn) : arn.region();
+ accountId = arn.accountId().isEmpty() ? regionResolver.getAccountId() : arn.accountId();
+ String resource = arn.resource();
+ streamName = resource.startsWith("deliverystream/")
+ ? resource.substring("deliverystream/".length())
+ : resource;
+ } else {
+ streamName = sub.getEndpoint();
+ region = extractRegionFromArn(topicArn);
+ accountId = regionResolver.getAccountId();
+ }
+ if (region == null || region.isEmpty()) {
+ region = regionResolver.getDefaultRegion();
+ }
+ if (accountId == null || accountId.isEmpty()) {
+ accountId = regionResolver.getAccountId();
+ }
+ boolean rawDelivery = sub.getAttributes() != null
+ && "true".equalsIgnoreCase(sub.getAttributes().get("RawMessageDelivery"));
+ String body = rawDelivery
+ ? protocolMessage
+ : buildSnsEnvelope(protocolMessage, subject, messageAttributes, topicArn, messageId);
+ byte[] data = body.getBytes(StandardCharsets.UTF_8);
+ firehoseService.putRecord(accountId, region, streamName, new Record(data));
+ LOG.debugv("Delivered SNS message to Firehose: {0} ({1}) raw={2}", sub.getEndpoint(), streamName, rawDelivery);
+ }
case "email", "email-json" -> LOG.infov("SNS email delivery (stub): to={0}, subject={1}, message={2}",
sub.getEndpoint(), subject, protocolMessage);
case "sms" -> LOG.infov("SNS SMS delivery (stub): to={0}, message={1}", sub.getEndpoint(), protocolMessage);
diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsFirehoseIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsFirehoseIntegrationTest.java
new file mode 100644
index 0000000000..341a1ec4ed
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsFirehoseIntegrationTest.java
@@ -0,0 +1,178 @@
+package io.github.hectorvent.floci.services.sns;
+
+import io.github.hectorvent.floci.services.firehose.FirehoseService;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class SnsFirehoseIntegrationTest {
+
+ private static final String FIREHOSE_JSON_CT = "application/x-amz-json-1.1";
+ private static final String ENVELOPE_BUCKET = "sns-fh-envelope-bucket";
+ private static final String ENVELOPE_STREAM = "sns-fh-envelope-stream";
+ private static final String RAW_BUCKET = "sns-fh-raw-bucket";
+ private static final String RAW_STREAM = "sns-fh-raw-stream";
+
+ private static String topicArn;
+ private static String envelopeSubArn;
+ private static String rawSubArn;
+
+ @Inject
+ FirehoseService firehoseService;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void setupBucketsAndStreams() {
+ // Create destination S3 buckets
+ given().when().put("/" + ENVELOPE_BUCKET).then().statusCode(200);
+ given().when().put("/" + RAW_BUCKET).then().statusCode(200);
+
+ // Create Firehose delivery streams
+ given()
+ .contentType(FIREHOSE_JSON_CT)
+ .header("X-Amz-Target", "Firehose_20150804.CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
+ "BucketARN": "arn:aws:s3:::%s",
+ "Prefix": "envelope/"
+ }
+ }
+ """.formatted(ENVELOPE_STREAM, ENVELOPE_BUCKET))
+ .when().post("/")
+ .then().statusCode(200);
+
+ given()
+ .contentType(FIREHOSE_JSON_CT)
+ .header("X-Amz-Target", "Firehose_20150804.CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
+ "BucketARN": "arn:aws:s3:::%s",
+ "Prefix": "raw/"
+ }
+ }
+ """.formatted(RAW_STREAM, RAW_BUCKET))
+ .when().post("/")
+ .then().statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void createTopicAndSubscriptions() {
+ // Create SNS topic
+ topicArn = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "CreateTopic")
+ .formParam("Name", "sns-to-firehose-topic")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().xmlPath().getString("CreateTopicResponse.CreateTopicResult.TopicArn");
+
+ // Subscribe envelope delivery stream
+ String envelopeStreamArn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/" + ENVELOPE_STREAM;
+ envelopeSubArn = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "Subscribe")
+ .formParam("TopicArn", topicArn)
+ .formParam("Protocol", "firehose")
+ .formParam("Endpoint", envelopeStreamArn)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<SubscriptionArn>"))
+ .extract().xmlPath().getString("SubscribeResponse.SubscribeResult.SubscriptionArn");
+
+ // Subscribe raw delivery stream
+ String rawStreamArn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/" + RAW_STREAM;
+ rawSubArn = given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "Subscribe")
+ .formParam("TopicArn", topicArn)
+ .formParam("Protocol", "firehose")
+ .formParam("Endpoint", rawStreamArn)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<SubscriptionArn>"))
+ .extract().xmlPath().getString("SubscribeResponse.SubscribeResult.SubscriptionArn");
+
+ // Enable RawMessageDelivery on the second subscription
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "SetSubscriptionAttributes")
+ .formParam("SubscriptionArn", rawSubArn)
+ .formParam("AttributeName", "RawMessageDelivery")
+ .formParam("AttributeValue", "true")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ @Test
+ @Order(3)
+ void publishMessage_deliversToFirehoseSubscriptions() {
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "Publish")
+ .formParam("TopicArn", topicArn)
+ .formParam("Message", "{\"event\":\"order_created\",\"id\":12345}")
+ .formParam("Subject", "Order Notification")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(containsString("<MessageId>"));
+
+ // Flush both firehose streams so buffered records land in S3
+ firehoseService.flush(ENVELOPE_STREAM);
+ firehoseService.flush(RAW_STREAM);
+
+ // Verify envelope delivery in ENVELOPE_BUCKET
+ String envelopeKey = given().when().get("/" + ENVELOPE_BUCKET + "?prefix=envelope/")
+ .then().statusCode(200)
+ .extract().xmlPath().getString("ListBucketResult.Contents[0].Key");
+
+ given().when().get("/" + ENVELOPE_BUCKET + "/" + envelopeKey)
+ .then().statusCode(200)
+ .body(containsString("\"Type\":\"Notification\""))
+ .body(containsString("\"Subject\":\"Order Notification\""))
+ .body(containsString("order_created"))
+ .body(containsString("12345"));
+
+ // Verify raw delivery in RAW_BUCKET
+ String rawKey = given().when().get("/" + RAW_BUCKET + "?prefix=raw/")
+ .then().statusCode(200)
+ .extract().xmlPath().getString("ListBucketResult.Contents[0].Key");
+
+ given().when().get("/" + RAW_BUCKET + "/" + rawKey)
+ .then().statusCode(200)
+ .body(containsString("{\"event\":\"order_created\",\"id\":12345}"))
+ .body(not(containsString("\"Type\":\"Notification\"")));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java
index 91ad0f1708..0da2fdf9dd 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java
@@ -5,16 +5,25 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.firehose.FirehoseService;
+import io.github.hectorvent.floci.services.firehose.model.Record;
import io.github.hectorvent.floci.services.sns.model.Subscription;
import io.github.hectorvent.floci.services.sns.model.Topic;
import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
class SnsServiceTest {
@@ -721,4 +730,118 @@ class SnsServiceTest {
assertTrue(snsService.matchesFilterPolicy(nested, body("{\"detail\":{\"b\":\"2\"}}"), null));
assertFalse(snsService.matchesFilterPolicy(nested, body("{\"detail\":{\"b\":\"3\"}}"), null));
}
+
+ @Test
+ void publish_deliversToFirehoseSubscription_withJsonEnvelope() throws Exception {
+ FirehoseService firehoseService = mock(FirehoseService.class);
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);
+ SnsService service = new SnsService(
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ regionResolver,
+ null,
+ null,
+ firehoseService
+ );
+
+ Topic topic = service.createTopic("firehose-topic", null, null, REGION);
+ String streamArn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/test-stream";
+ service.subscribe(topic.getTopicArn(), "firehose", streamArn, REGION, Map.of());
+
+ String messageId = service.publish(topic.getTopicArn(), null, "Hello Firehose", "Test Subject", REGION);
+ assertNotNull(messageId);
+
+ ArgumentCaptor<Record> recordCaptor = ArgumentCaptor.forClass(Record.class);
+ verify(firehoseService).putRecord(eq(ACCOUNT), eq(REGION), eq("test-stream"), recordCaptor.capture());
+
+ Record captured = recordCaptor.getValue();
+ assertNotNull(captured);
+ assertNotNull(captured.getData());
+ String payload = new String(captured.getData(), StandardCharsets.UTF_8);
+
+ JsonNode json = new ObjectMapper().readTree(payload);
+ assertEquals("Notification", json.get("Type").asText());
+ assertEquals(messageId, json.get("MessageId").asText());
+ assertEquals(topic.getTopicArn(), json.get("TopicArn").asText());
+ assertEquals("Test Subject", json.get("Subject").asText());
+ assertEquals("Hello Firehose", json.get("Message").asText());
+ }
+
+ @Test
+ void publish_deliversToFirehoseSubscription_rawMessageDelivery() {
+ FirehoseService firehoseService = mock(FirehoseService.class);
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);
+ SnsService service = new SnsService(
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ regionResolver,
+ null,
+ null,
+ firehoseService
+ );
+
+ Topic topic = service.createTopic("firehose-topic-raw", null, null, REGION);
+ String streamArn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/test-stream-raw";
+ service.subscribe(topic.getTopicArn(), "firehose", streamArn, REGION,
+ Map.of("RawMessageDelivery", "true"));
+
+ String message = "{\"raw\":\"payload\"}";
+ String messageId = service.publish(topic.getTopicArn(), null, message, null, REGION);
+ assertNotNull(messageId);
+
+ ArgumentCaptor<Record> recordCaptor = ArgumentCaptor.forClass(Record.class);
+ verify(firehoseService).putRecord(eq(ACCOUNT), eq(REGION), eq("test-stream-raw"), recordCaptor.capture());
+
+ Record captured = recordCaptor.getValue();
+ assertEquals(message, new String(captured.getData(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void publish_deliversToFirehoseSubscription_bareStreamNameEndpoint() {
+ FirehoseService firehoseService = mock(FirehoseService.class);
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);
+ SnsService service = new SnsService(
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ regionResolver,
+ null,
+ null,
+ firehoseService
+ );
+
+ Topic topic = service.createTopic("firehose-bare-topic", null, null, REGION);
+ service.subscribe(topic.getTopicArn(), "firehose", "bare-stream", REGION,
+ Map.of("RawMessageDelivery", "true"));
+
+ service.publish(topic.getTopicArn(), null, "bare-msg", null, REGION);
+
+ ArgumentCaptor<Record> recordCaptor = ArgumentCaptor.forClass(Record.class);
+ verify(firehoseService).putRecord(eq(ACCOUNT), eq(REGION), eq("bare-stream"), recordCaptor.capture());
+ assertEquals("bare-msg", new String(recordCaptor.getValue().getData(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void publish_firehoseDeliveryFailure_doesNotFailPublisher() {
+ FirehoseService firehoseService = mock(FirehoseService.class);
+ doThrow(new RuntimeException("stream not found"))
+ .when(firehoseService).putRecord(any(), any(), any(), any());
+
+ RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);
+ SnsService service = new SnsService(
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ regionResolver,
+ null,
+ null,
+ firehoseService
+ );
+
+ Topic topic = service.createTopic("firehose-fail-topic", null, null, REGION);
+ service.subscribe(topic.getTopicArn(), "firehose",
+ "arn:aws:firehose:us-east-1:000000000000:deliverystream/failing-stream", REGION, Map.of());
+
+ // Delivery error is logged and tolerated; publisher receives message ID
+ String messageId = service.publish(topic.getTopicArn(), null, "msg", null, REGION);
+ assertNotNull(messageId);
+ }
}