Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): return ExtendedS3DestinationDescription and support UpdateDestination (#1710)

Hector Venturab3c54c185498
diff --git a/compatibility-tests/compat-opentofu/main.tf b/compatibility-tests/compat-opentofu/main.tf
index ec056e343e..593336517b 100644
--- a/compatibility-tests/compat-opentofu/main.tf
+++ b/compatibility-tests/compat-opentofu/main.tf
@@ -304,3 +304,27 @@ resource "aws_cognito_user_pool_client" "client" {
name = "floci-compat-pool-client"
user_pool_id = aws_cognito_user_pool.pool.id
}
+
+# ── Kinesis Firehose Delivery Stream (extended_s3, issue #1043) ───────────
+resource "aws_kinesis_firehose_delivery_stream" "events" {
+ name = "floci-compat-firehose"
+ destination = "extended_s3"
+
+ extended_s3_configuration {
+ role_arn = aws_iam_role.lambda_exec.arn
+ bucket_arn = aws_s3_bucket.app.arn
+ prefix = "events/data/"
+ error_output_prefix = "events/errors/"
+ compression_format = "GZIP"
+ buffering_size = 64
+ buffering_interval = 120
+ }
+
+ tags = {
+ Environment = "compat-test"
+ }
+}
+
+output "firehose_stream_arn" {
+ value = aws_kinesis_firehose_delivery_stream.events.arn
+}
diff --git a/compatibility-tests/compat-opentofu/test/opentofu.bats b/compatibility-tests/compat-opentofu/test/opentofu.bats
index 40e54ba4c7..123c043215 100644
--- a/compatibility-tests/compat-opentofu/test/opentofu.bats
+++ b/compatibility-tests/compat-opentofu/test/opentofu.bats
@@ -185,3 +185,10 @@ setup() {
[ -n "$output" ]
[ "$output" != "None" ]
}
+
+@test "OpenTofu: Firehose extended_s3 delivery stream created with correct config" {
+ run aws_cmd firehose describe-delivery-stream --delivery-stream-name floci-compat-firehose \
+ --query "DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat" --output text
+ assert_success
+ assert_output "GZIP"
+}
diff --git a/compatibility-tests/compat-terraform/main.tf b/compatibility-tests/compat-terraform/main.tf
index 7aae451beb..8f20d7410c 100644
--- a/compatibility-tests/compat-terraform/main.tf
+++ b/compatibility-tests/compat-terraform/main.tf
@@ -341,3 +341,27 @@ output "zone_id" {
output "health_check_id" {
value = aws_route53_health_check.app.id
}
+
+# -- Kinesis Firehose Delivery Stream (extended_s3, issue #1043) ---------------
+resource "aws_kinesis_firehose_delivery_stream" "events" {
+ name = "floci-compat-firehose"
+ destination = "extended_s3"
+
+ extended_s3_configuration {
+ role_arn = aws_iam_role.lambda_exec.arn
+ bucket_arn = aws_s3_bucket.app.arn
+ prefix = "events/data/"
+ error_output_prefix = "events/errors/"
+ compression_format = "GZIP"
+ buffering_size = 64
+ buffering_interval = 120
+ }
+
+ tags = {
+ Environment = "compat-test"
+ }
+}
+
+output "firehose_stream_arn" {
+ value = aws_kinesis_firehose_delivery_stream.events.arn
+}
diff --git a/compatibility-tests/compat-terraform/test/terraform.bats b/compatibility-tests/compat-terraform/test/terraform.bats
index c4480bafec..5be9906bb9 100644
--- a/compatibility-tests/compat-terraform/test/terraform.bats
+++ b/compatibility-tests/compat-terraform/test/terraform.bats
@@ -204,3 +204,10 @@ setup() {
[ -n "$output" ]
[ "$output" != "None" ]
}
+
+@test "Terraform: Firehose extended_s3 delivery stream created with correct config" {
+ run aws_cmd firehose describe-delivery-stream --delivery-stream-name floci-compat-firehose \
+ --query "DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat" --output text
+ assert_success
+ assert_output "GZIP"
+}
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseExtendedS3Test.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseExtendedS3Test.java
new file mode 100644
index 0000000000..11b2a44e7b
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseExtendedS3Test.java
@@ -0,0 +1,153 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.*;
+import software.amazon.awssdk.services.firehose.FirehoseClient;
+import software.amazon.awssdk.services.firehose.model.*;
+
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.*;
+
+/**
+ * Issue #1043 — the Terraform/Pulumi provider creates delivery streams with
+ * ExtendedS3DestinationConfiguration and reads ExtendedS3DestinationDescription
+ * back (plus VersionId/DestinationId to drive UpdateDestination).
+ */
+@DisplayName("Firehose extended S3 destination — issue #1043")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class FirehoseExtendedS3Test {
+
+ private static FirehoseClient firehose;
+ private static final String STREAM_NAME = "sdk-extended-s3-" + UUID.randomUUID().toString().substring(0, 8);
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";
+ private static final String BUCKET_ARN = "arn:aws:s3:::floci-firehose-sdk-test";
+
+ private static String versionId;
+ private static String destinationId;
+
+ @BeforeAll
+ static void setup() {
+ firehose = TestFixtures.firehoseClient();
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (firehose != null) {
+ try {
+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME).build());
+ } catch (Exception ignored) {}
+ firehose.close();
+ }
+ }
+
+ @Test
+ @Order(1)
+ @DisplayName("Create delivery stream with ExtendedS3DestinationConfiguration")
+ void createExtendedS3DeliveryStream() {
+ CreateDeliveryStreamResponse response = firehose.createDeliveryStream(
+ CreateDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)
+ .extendedS3DestinationConfiguration(ExtendedS3DestinationConfiguration.builder()
+ .roleARN(ROLE_ARN)
+ .bucketARN(BUCKET_ARN)
+ .prefix("events/data/")
+ .errorOutputPrefix("events/errors/")
+ .compressionFormat(CompressionFormat.GZIP)
+ .bufferingHints(BufferingHints.builder()
+ .intervalInSeconds(120)
+ .sizeInMBs(64)
+ .build())
+ .build())
+ .build());
+
+ assertThat(response.deliveryStreamARN()).contains(STREAM_NAME);
+ }
+
+ @Test
+ @Order(2)
+ @DisplayName("Describe returns ExtendedS3DestinationDescription (the provider read path)")
+ void describeReturnsExtendedS3Description() {
+ DeliveryStreamDescription desc = firehose.describeDeliveryStream(
+ DescribeDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .build())
+ .deliveryStreamDescription();
+
+ assertThat(desc.versionId()).isNotBlank();
+ assertThat(desc.deliveryStreamType()).isEqualTo(DeliveryStreamType.DIRECT_PUT);
+ assertThat(desc.hasMoreDestinations()).isFalse();
+ assertThat(desc.destinations()).hasSize(1);
+
+ DestinationDescription destination = desc.destinations().get(0);
+ assertThat(destination.destinationId()).isNotBlank();
+
+ ExtendedS3DestinationDescription extended = destination.extendedS3DestinationDescription();
+ assertThat(extended).as("ExtendedS3DestinationDescription must be present").isNotNull();
+ assertThat(extended.roleARN()).isEqualTo(ROLE_ARN);
+ assertThat(extended.bucketARN()).isEqualTo(BUCKET_ARN);
+ assertThat(extended.prefix()).isEqualTo("events/data/");
+ assertThat(extended.errorOutputPrefix()).isEqualTo("events/errors/");
+ assertThat(extended.compressionFormat()).isEqualTo(CompressionFormat.GZIP);
+ assertThat(extended.bufferingHints().sizeInMBs()).isEqualTo(64);
+ assertThat(extended.bufferingHints().intervalInSeconds()).isEqualTo(120);
+ assertThat(extended.encryptionConfiguration().noEncryptionConfig())
+ .isEqualTo(NoEncryptionConfig.NO_ENCRYPTION);
+
+ assertThat(destination.s3DestinationDescription())
+ .as("deprecated S3DestinationDescription mirror must also be present")
+ .isNotNull();
+ assertThat(destination.s3DestinationDescription().bucketARN()).isEqualTo(BUCKET_ARN);
+
+ versionId = desc.versionId();
+ destinationId = destination.destinationId();
+ }
+
+ @Test
+ @Order(3)
+ @DisplayName("UpdateDestination applies changes and bumps VersionId")
+ void updateDestination() {
+ firehose.updateDestination(UpdateDestinationRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .currentDeliveryStreamVersionId(versionId)
+ .destinationId(destinationId)
+ .extendedS3DestinationUpdate(ExtendedS3DestinationUpdate.builder()
+ .compressionFormat(CompressionFormat.SNAPPY)
+ .bufferingHints(BufferingHints.builder()
+ .intervalInSeconds(60)
+ .sizeInMBs(128)
+ .build())
+ .build())
+ .build());
+
+ DeliveryStreamDescription desc = firehose.describeDeliveryStream(
+ DescribeDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .build())
+ .deliveryStreamDescription();
+
+ assertThat(desc.versionId()).isNotEqualTo(versionId);
+ ExtendedS3DestinationDescription extended =
+ desc.destinations().get(0).extendedS3DestinationDescription();
+ assertThat(extended.compressionFormat()).isEqualTo(CompressionFormat.SNAPPY);
+ assertThat(extended.bufferingHints().sizeInMBs()).isEqualTo(128);
+ assertThat(extended.roleARN()).as("untouched fields survive the update").isEqualTo(ROLE_ARN);
+ assertThat(extended.prefix()).isEqualTo("events/data/");
+ }
+
+ @Test
+ @Order(4)
+ @DisplayName("UpdateDestination with stale version throws ConcurrentModificationException")
+ void updateDestinationStaleVersion() {
+ assertThatThrownBy(() -> firehose.updateDestination(UpdateDestinationRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .currentDeliveryStreamVersionId("99")
+ .destinationId(destinationId)
+ .extendedS3DestinationUpdate(ExtendedS3DestinationUpdate.builder()
+ .compressionFormat(CompressionFormat.GZIP)
+ .build())
+ .build()))
+ .isInstanceOf(ConcurrentModificationException.class);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
index dc2eaa8fb8..7f3fca2c14 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
@@ -42,9 +42,24 @@ public class FirehoseJsonHandler {
tags.add(mapper.treeToValue(tNode, io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Tag.class));
}
}
- String arn = firehoseService.createDeliveryStream(name, s3, tags);
+ String deliveryStreamType = request.has("DeliveryStreamType")
+ ? request.get("DeliveryStreamType").asText() : null;
+ String arn = firehoseService.createDeliveryStream(name, s3, tags, deliveryStreamType);
yield Response.ok(Map.of("DeliveryStreamARN", arn)).build();
}
+ case "UpdateDestination" -> {
+ String name = request.get("DeliveryStreamName").asText();
+ String currentVersionId = request.get("CurrentDeliveryStreamVersionId").asText();
+ String destinationId = request.get("DestinationId").asText();
+ S3Destination update = null;
+ if (request.has("ExtendedS3DestinationUpdate")) {
+ update = mapper.treeToValue(request.get("ExtendedS3DestinationUpdate"), S3Destination.class);
+ } else if (request.has("S3DestinationUpdate")) {
+ update = mapper.treeToValue(request.get("S3DestinationUpdate"), S3Destination.class);
+ }
+ firehoseService.updateDestination(name, currentVersionId, destinationId, update);
+ yield Response.ok(Map.of()).build();
+ }
case "DescribeDeliveryStream" -> {
String name = request.get("DeliveryStreamName").asText();
var desc = firehoseService.describeDeliveryStream(name);
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index f8bcbe5474..4369a6a347 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -45,16 +45,97 @@ public class FirehoseService {
}
public String createDeliveryStream(String name, S3Destination s3Config, List<DeliveryStreamDescription.Tag> tags) {
+ return createDeliveryStream(name, s3Config, tags, null);
+ }
+
+ public String createDeliveryStream(String name, S3Destination s3Config, List<DeliveryStreamDescription.Tag> tags,
+ String deliveryStreamType) {
+ validateBufferingHints(s3Config);
String arn = AwsArnUtils.Arn.of("firehose", regionResolver.getDefaultRegion(), regionResolver.getAccountId(), "deliverystream/" + name).toString();
DeliveryStreamDescription description = new DeliveryStreamDescription(name, arn, s3Config);
description.setAccountId(regionResolver.getAccountId());
description.setTags(tags);
+ if (deliveryStreamType != null && !deliveryStreamType.isBlank()) {
+ description.setDeliveryStreamType(deliveryStreamType);
+ }
streamStore.put(name, description);
buffers.put(name, Collections.synchronizedList(new ArrayList<>()));
LOG.infov("Created Firehose delivery stream: {0}", name);
return arn;
}
+ public void updateDestination(String name, String currentVersionId, String destinationId, S3Destination update) {
+ DeliveryStreamDescription stream = describeDeliveryStream(name);
+ if (!stream.getVersionId().equals(currentVersionId)) {
+ throw new AwsException("ConcurrentModificationException",
+ "Cannot update firehose: " + name + " since the current version id: " + stream.getVersionId()
+ + " and specified version id: " + currentVersionId + " do not match", 400);
+ }
+ DeliveryStreamDescription.Destination destination = stream.getDestinations() != null && !stream.getDestinations().isEmpty()
+ ? stream.getDestinations().get(0)
+ : null;
+ if (destination == null || !destination.getDestinationId().equals(destinationId)) {
+ throw new AwsException("InvalidArgumentException",
+ "Destination Id " + destinationId + " not found", 400);
+ }
+ if (update == null) {
+ throw new AwsException("InvalidArgumentException",
+ "A destination update is required for UpdateDestination.", 400);
+ }
+ validateBufferingHints(update);
+ S3Destination current = destination.getExtendedS3DestinationDescription();
+ if (current == null) {
+ update.applyDefaults();
+ destination.setExtendedS3DestinationDescription(update);
+ } else {
+ mergeDestination(current, update);
+ }
+ stream.setVersionId(String.valueOf(parseVersionId(stream.getVersionId()) + 1));
+ stream.setLastUpdateTimestamp(java.time.Instant.now());
+ streamStore.put(name, stream);
+ LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);
+ }
+
+ // A corrupt persisted version can only reach here when the caller echoed it
+ // (the equality check above passed), so self-heal instead of failing with a 500
+ // or blaming the client.
+ private static long parseVersionId(String versionId) {
+ try {
+ return Long.parseLong(versionId);
+ } catch (NumberFormatException e) {
+ return 0L;
+ }
+ }
+
+ /**
+ * AWS requires SizeInMBs and IntervalInSeconds to be specified together:
+ * "This parameter is optional but if you specify a value for it, you must
+ * also specify a value for IntervalInSeconds, and vice versa" (firehose
+ * service-2.json). Rejecting partial hints here is what keeps the
+ * whole-object replacement in mergeDestination faithful to AWS.
+ */
+ private static void validateBufferingHints(S3Destination config) {
+ DeliveryStreamDescription.BufferingHints hints = config == null ? null : config.getBufferingHints();
+ if (hints == null) {
+ return;
+ }
+ if ((hints.getSizeInMBs() == null) != (hints.getIntervalInSeconds() == null)) {
+ throw new AwsException("InvalidArgumentException",
+ "If you specify a value for SizeInMBs, you must also specify a value for IntervalInSeconds, and vice versa.",
+ 400);
+ }
+ }
+
+ private static void mergeDestination(S3Destination current, S3Destination update) {
+ if (update.getRoleArn() != null) current.setRoleArn(update.getRoleArn());
+ if (update.getBucketArn() != null) current.setBucketArn(update.getBucketArn());
+ if (update.getPrefix() != null) current.setPrefix(update.getPrefix());
+ if (update.getErrorOutputPrefix() != null) current.setErrorOutputPrefix(update.getErrorOutputPrefix());
+ if (update.getCompressionFormat() != null) current.setCompressionFormat(update.getCompressionFormat());
+ if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());
+ if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());
+ }
+
public void tagDeliveryStream(String name, List<DeliveryStreamDescription.Tag> tagsToTag) {
DeliveryStreamDescription stream = describeDeliveryStream(name);
Map<String, String> tagMap = new LinkedHashMap<>();
@@ -105,9 +186,14 @@ public class FirehoseService {
}
public DeliveryStreamDescription describeDeliveryStream(String name) {
- return streamStore.get(name)
+ DeliveryStreamDescription stream = streamStore.get(name)
.orElseThrow(() -> new AwsException("ResourceNotFoundException",
"Delivery stream not found: " + name, 400));
+ // Normalizes streams persisted before required output members existed.
+ if (stream.s3Destination() != null) {
+ stream.s3Destination().applyDefaults();
+ }
+ return stream;
}
public void deleteDeliveryStream(String name) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
index 4aeff99921..17f926eb3b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
@@ -1,6 +1,8 @@
package io.github.hectorvent.floci.services.firehose.model;
import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.time.Instant;
@@ -8,6 +10,7 @@ import java.util.List;
import java.util.ArrayList;
@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
public class DeliveryStreamDescription {
@JsonProperty("DeliveryStreamName")
private String deliveryStreamName;
@@ -16,9 +19,19 @@ public class DeliveryStreamDescription {
private String deliveryStreamARN;
@JsonProperty("DeliveryStreamStatus")
private DeliveryStreamStatus deliveryStreamStatus;
+ @JsonProperty("DeliveryStreamType")
+ private String deliveryStreamType = "DirectPut";
+ @JsonProperty("VersionId")
+ private String versionId = "1";
+ @JsonProperty("HasMoreDestinations")
+ private boolean hasMoreDestinations;
@JsonProperty("CreateTimestamp")
@JsonFormat(shape = JsonFormat.Shape.NUMBER)
private Instant createTimestamp;
+ @JsonProperty("LastUpdateTimestamp")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private Instant lastUpdateTimestamp;
@JsonProperty("Destinations")
private List<Destination> destinations;
@JsonProperty("Tags")
@@ -30,6 +43,9 @@ public class DeliveryStreamDescription {
this.deliveryStreamARN = arn;
this.deliveryStreamStatus = DeliveryStreamStatus.ACTIVE;
this.createTimestamp = Instant.now();
+ if (s3 != null) {
+ s3.applyDefaults();
+ }
this.destinations = List.of(new Destination(s3));
}
@@ -42,8 +58,16 @@ public class DeliveryStreamDescription {
public void setDeliveryStreamARN(String deliveryStreamARN) { this.deliveryStreamARN = deliveryStreamARN; }
public DeliveryStreamStatus getDeliveryStreamStatus() { return deliveryStreamStatus; }
public void setDeliveryStreamStatus(DeliveryStreamStatus deliveryStreamStatus) { this.deliveryStreamStatus = deliveryStreamStatus; }
+ public String getDeliveryStreamType() { return deliveryStreamType; }
+ public void setDeliveryStreamType(String deliveryStreamType) { this.deliveryStreamType = deliveryStreamType; }
+ public String getVersionId() { return versionId; }
+ public void setVersionId(String versionId) { this.versionId = versionId; }
+ public boolean isHasMoreDestinations() { return hasMoreDestinations; }
+ public void setHasMoreDestinations(boolean hasMoreDestinations) { this.hasMoreDestinations = hasMoreDestinations; }
public Instant getCreateTimestamp() { return createTimestamp; }
public void setCreateTimestamp(Instant createTimestamp) { this.createTimestamp = createTimestamp; }
+ public Instant getLastUpdateTimestamp() { return lastUpdateTimestamp; }
+ public void setLastUpdateTimestamp(Instant lastUpdateTimestamp) { this.lastUpdateTimestamp = lastUpdateTimestamp; }
public List<Destination> getDestinations() { return destinations; }
public void setDestinations(List<Destination> destinations) { this.destinations = destinations; }
@@ -54,32 +78,100 @@ public class DeliveryStreamDescription {
}
@RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
public static class Destination {
- @JsonProperty("S3DestinationDescription")
- private S3Destination s3DestinationDescription;
+ @JsonProperty("DestinationId")
+ private String destinationId = "destinationId-000000000001";
+
+ // Single canonical S3 config, serialized under both wire keys: real AWS
+ // returns ExtendedS3DestinationDescription plus the deprecated
+ // S3DestinationDescription mirror for every S3-backed stream.
+ private S3Destination s3;
public Destination() {}
- public Destination(S3Destination s3) { this.s3DestinationDescription = s3; }
- public S3Destination getS3DestinationDescription() { return s3DestinationDescription; }
- public void setS3DestinationDescription(S3Destination s3) { this.s3DestinationDescription = s3; }
+ public Destination(S3Destination s3) { this.s3 = s3; }
+
+ public String getDestinationId() { return destinationId; }
+ public void setDestinationId(String destinationId) { this.destinationId = destinationId; }
+
+ @JsonProperty("S3DestinationDescription")
+ public S3Destination getS3DestinationDescription() { return s3; }
+ @JsonProperty("S3DestinationDescription")
+ public void setS3DestinationDescription(S3Destination s3) {
+ // Guarded so persisted JSON carrying both keys (identical content) stays
+ // idempotent while legacy files with only the S3 key still load.
+ if (this.s3 == null) {
+ this.s3 = s3;
+ }
+ }
+
+ @JsonProperty("ExtendedS3DestinationDescription")
+ public S3Destination getExtendedS3DestinationDescription() { return s3; }
+ @JsonProperty("ExtendedS3DestinationDescription")
+ public void setExtendedS3DestinationDescription(S3Destination s3) { this.s3 = s3; }
}
@RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
public static class S3Destination {
+ @JsonProperty("RoleARN")
+ private String roleArn;
@JsonProperty("BucketARN")
private String bucketArn;
@JsonProperty("Prefix")
private String prefix;
+ @JsonProperty("ErrorOutputPrefix")
+ private String errorOutputPrefix;
+ @JsonProperty("CompressionFormat")
+ private String compressionFormat;
@JsonProperty("BufferingHints")
private BufferingHints bufferingHints;
+ @JsonProperty("EncryptionConfiguration")
+ private EncryptionConfiguration encryptionConfiguration;
public S3Destination() {}
+ public String getRoleArn() { return roleArn; }
+ public void setRoleArn(String roleArn) { this.roleArn = roleArn; }
public String getBucketArn() { return bucketArn; }
public void setBucketArn(String bucketArn) { this.bucketArn = bucketArn; }
public String getPrefix() { return prefix; }
public void setPrefix(String prefix) { this.prefix = prefix; }
+ public String getErrorOutputPrefix() { return errorOutputPrefix; }
+ public void setErrorOutputPrefix(String errorOutputPrefix) { this.errorOutputPrefix = errorOutputPrefix; }
+ public String getCompressionFormat() { return compressionFormat; }
+ public void setCompressionFormat(String compressionFormat) { this.compressionFormat = compressionFormat; }
public BufferingHints getBufferingHints() { return bufferingHints; }
public void setBufferingHints(BufferingHints bufferingHints) { this.bufferingHints = bufferingHints; }
+ public EncryptionConfiguration getEncryptionConfiguration() { return encryptionConfiguration; }
+ public void setEncryptionConfiguration(EncryptionConfiguration encryptionConfiguration) { this.encryptionConfiguration = encryptionConfiguration; }
+
+ /**
+ * Fills the members the wire contract marks required with the AWS defaults.
+ * Getters stay null-honest so UpdateDestination merges can tell "not
+ * specified" from a value; call this only on create and describe paths.
+ */
+ public void applyDefaults() {
+ if (compressionFormat == null) {
+ compressionFormat = "UNCOMPRESSED";
+ }
+ if (encryptionConfiguration == null) {
+ encryptionConfiguration = EncryptionConfiguration.noEncryption();
+ }
+ if (bufferingHints == null) {
+ bufferingHints = BufferingHints.defaults();
+ } else {
+ // Self-heal legacy persisted state; validation keeps partial
+ // hints out of the create/update paths.
+ if (bufferingHints.getSizeInMBs() == null) {
+ bufferingHints.setSizeInMBs(5);
+ }
+ if (bufferingHints.getIntervalInSeconds() == null) {
+ bufferingHints.setIntervalInSeconds(300);
+ }
+ }
+ }
/** Extracts bucket name from ARN: arn:aws:s3:::my-bucket → my-bucket */
public String bucketName() {
@@ -90,17 +182,66 @@ public class DeliveryStreamDescription {
}
@RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class EncryptionConfiguration {
+ @JsonProperty("NoEncryptionConfig")
+ private String noEncryptionConfig;
+ @JsonProperty("KMSEncryptionConfig")
+ private KmsEncryptionConfig kmsEncryptionConfig;
+
+ public EncryptionConfiguration() {}
+
+ public static EncryptionConfiguration noEncryption() {
+ EncryptionConfiguration config = new EncryptionConfiguration();
+ config.noEncryptionConfig = "NoEncryption";
+ return config;
+ }
+
+ public String getNoEncryptionConfig() { return noEncryptionConfig; }
+ public void setNoEncryptionConfig(String noEncryptionConfig) { this.noEncryptionConfig = noEncryptionConfig; }
+ public KmsEncryptionConfig getKmsEncryptionConfig() { return kmsEncryptionConfig; }
+ public void setKmsEncryptionConfig(KmsEncryptionConfig kmsEncryptionConfig) { this.kmsEncryptionConfig = kmsEncryptionConfig; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class KmsEncryptionConfig {
+ @JsonProperty("AWSKMSKeyARN")
+ private String awsKmsKeyArn;
+
+ public KmsEncryptionConfig() {}
+ public String getAwsKmsKeyArn() { return awsKmsKeyArn; }
+ public void setAwsKmsKeyArn(String awsKmsKeyArn) { this.awsKmsKeyArn = awsKmsKeyArn; }
+ }
+
+ /**
+ * Members stay boxed and null-honest so validation can tell "not specified"
+ * from a value: AWS requires SizeInMBs and IntervalInSeconds to be specified
+ * together, and the defaults (5 MiB / 300 s) apply only when the whole
+ * object is absent.
+ */
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
public static class BufferingHints {
@JsonProperty("SizeInMBs")
- private int sizeInMBs = 5;
+ private Integer sizeInMBs;
@JsonProperty("IntervalInSeconds")
- private int intervalInSeconds = 300;
+ private Integer intervalInSeconds;
public BufferingHints() {}
- public int getSizeInMBs() { return sizeInMBs; }
- public void setSizeInMBs(int sizeInMBs) { this.sizeInMBs = sizeInMBs; }
- public int getIntervalInSeconds() { return intervalInSeconds; }
- public void setIntervalInSeconds(int intervalInSeconds) { this.intervalInSeconds = intervalInSeconds; }
+
+ public static BufferingHints defaults() {
+ BufferingHints hints = new BufferingHints();
+ hints.sizeInMBs = 5;
+ hints.intervalInSeconds = 300;
+ return hints;
+ }
+
+ public Integer getSizeInMBs() { return sizeInMBs; }
+ public void setSizeInMBs(Integer sizeInMBs) { this.sizeInMBs = sizeInMBs; }
+ public Integer getIntervalInSeconds() { return intervalInSeconds; }
+ public void setIntervalInSeconds(Integer intervalInSeconds) { this.intervalInSeconds = intervalInSeconds; }
}
public List<Tag> getTags() {
@@ -110,6 +251,7 @@ public class DeliveryStreamDescription {
public void setTags(List<Tag> tags) { this.tags = tags; }
@RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
public static class Tag {
@JsonProperty("Key")
private String key;
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
new file mode 100644
index 0000000000..a8fa16ee89
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
@@ -0,0 +1,323 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class FirehoseExtendedS3IntegrationTest {
+
+ private static final String STREAM_NAME = "test-extended-s3-stream";
+ private static final String LEGACY_STREAM_NAME = "test-legacy-s3-stream";
+ private static final String MINIMAL_STREAM_NAME = "test-minimal-extended-s3-stream";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-delivery-role";
+ private static final String BUCKET_ARN = "arn:aws:s3:::extended-s3-archive";
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String TARGET_PREFIX = "Firehose_20150804.";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void createWithExtendedS3Configuration() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "DeliveryStreamType": "DirectPut",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "Prefix": "events/data/",
+ "ErrorOutputPrefix": "events/errors/",
+ "CompressionFormat": "GZIP",
+ "BufferingHints": { "SizeInMBs": 64, "IntervalInSeconds": 120 }
+ }
+ }
+ """.formatted(STREAM_NAME, ROLE_ARN, BUCKET_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamARN", notNullValue());
+ }
+
+ @Test
+ @Order(2)
+ void describeReturnsExtendedS3Description() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamDescription.DeliveryStreamType", equalTo("DirectPut"))
+ .body("DeliveryStreamDescription.VersionId", equalTo("1"))
+ .body("DeliveryStreamDescription.HasMoreDestinations", equalTo(false))
+ .body("DeliveryStreamDescription.Destinations[0].DestinationId", notNullValue())
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.RoleARN", equalTo(ROLE_ARN))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.Prefix", equalTo("events/data/"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.ErrorOutputPrefix", equalTo("events/errors/"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat", equalTo("GZIP"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(64))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(120))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig", equalTo("NoEncryption"))
+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))
+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.RoleARN", equalTo(ROLE_ARN));
+ }
+
+ @Test
+ @Order(3)
+ void describeAppliesDefaultsForMinimalConfig() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s"
+ }
+ }
+ """.formatted(MINIMAL_STREAM_NAME, ROLE_ARN, BUCKET_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + MINIMAL_STREAM_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat", equalTo("UNCOMPRESSED"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(5))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(300))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig", equalTo("NoEncryption"));
+ }
+
+ @Test
+ @Order(4)
+ void legacyS3ConfigurationAlsoReturnsExtendedDescription() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "S3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "Prefix": "legacy/"
+ }
+ }
+ """.formatted(LEGACY_STREAM_NAME, ROLE_ARN, BUCKET_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + LEGACY_STREAM_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.Prefix", equalTo("legacy/"));
+ }
+
+ @Test
+ @Order(5)
+ void updateDestinationRejectsStaleVersion() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "CurrentDeliveryStreamVersionId": "99",
+ "DestinationId": "destinationId-000000000001",
+ "ExtendedS3DestinationUpdate": { "CompressionFormat": "Snappy" }
+ }
+ """.formatted(STREAM_NAME))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ConcurrentModificationException"));
+ }
+
+ @Test
+ @Order(6)
+ void updateDestinationAppliesChangesAndBumpsVersion() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "CurrentDeliveryStreamVersionId": "1",
+ "DestinationId": "destinationId-000000000001",
+ "ExtendedS3DestinationUpdate": {
+ "CompressionFormat": "Snappy",
+ "BufferingHints": { "SizeInMBs": 128, "IntervalInSeconds": 60 }
+ }
+ }
+ """.formatted(STREAM_NAME))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamDescription.VersionId", equalTo("2"))
+ .body("DeliveryStreamDescription.LastUpdateTimestamp", notNullValue())
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat", equalTo("Snappy"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(128))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.RoleARN", equalTo(ROLE_ARN))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.Prefix", equalTo("events/data/"));
+ }
+
+ @Test
+ @Order(7)
+ void updateDestinationWithoutUpdatePayloadIsRejectedAndDoesNotBumpVersion() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "CurrentDeliveryStreamVersionId": "2",
+ "DestinationId": "destinationId-000000000001"
+ }
+ """.formatted(STREAM_NAME))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidArgumentException"));
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamDescription.VersionId", equalTo("2"));
+ }
+
+ @Test
+ @Order(8)
+ void updateDestinationRejectsPartialBufferingHintsAndDoesNotBumpVersion() {
+ // AWS requires SizeInMBs and IntervalInSeconds to be specified together
+ // (firehose service-2.json BufferingHints member docs).
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "CurrentDeliveryStreamVersionId": "2",
+ "DestinationId": "destinationId-000000000001",
+ "ExtendedS3DestinationUpdate": {
+ "BufferingHints": { "SizeInMBs": 16 }
+ }
+ }
+ """.formatted(STREAM_NAME))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidArgumentException"));
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamDescription.VersionId", equalTo("2"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(128))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(60));
+ }
+
+ @Test
+ @Order(9)
+ void createDeliveryStreamRejectsPartialBufferingHints() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "partial-hints-stream",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "BufferingHints": { "IntervalInSeconds": 60 }
+ }
+ }
+ """.formatted(ROLE_ARN, BUCKET_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidArgumentException"));
+ }
+
+ @Test
+ @Order(10)
+ void updateDestinationRejectsUnknownDestinationId() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "CurrentDeliveryStreamVersionId": "2",
+ "DestinationId": "destinationId-999999999999",
+ "ExtendedS3DestinationUpdate": { "CompressionFormat": "GZIP" }
+ }
+ """.formatted(STREAM_NAME))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidArgumentException"));
+ }
+}