Light, fluffy, and always free - The AWS Local Emulator alternative
feat(firehose): return ExtendedS3DestinationDescription and support UpdateDestination (#1710)
Hector Venturab3c54c185498
diff --git a/compatibility-tests/compat-opentofu/main.tf b/compatibility-tests/compat-opentofu/main.tfindex ec056e343e..593336517b 100644--- a/compatibility-tests/compat-opentofu/main.tf+++ b/compatibility-tests/compat-opentofu/main.tf@@ -304,3 +304,27 @@ resource "aws_cognito_user_pool_client" "client" {name = "floci-compat-pool-client"user_pool_id = aws_cognito_user_pool.pool.id}++# ── Kinesis Firehose Delivery Stream (extended_s3, issue #1043) ───────────+resource "aws_kinesis_firehose_delivery_stream" "events" {+ name = "floci-compat-firehose"+ destination = "extended_s3"++ extended_s3_configuration {+ role_arn = aws_iam_role.lambda_exec.arn+ bucket_arn = aws_s3_bucket.app.arn+ prefix = "events/data/"+ error_output_prefix = "events/errors/"+ compression_format = "GZIP"+ buffering_size = 64+ buffering_interval = 120+ }++ tags = {+ Environment = "compat-test"+ }+}++output "firehose_stream_arn" {+ value = aws_kinesis_firehose_delivery_stream.events.arn+}diff --git a/compatibility-tests/compat-opentofu/test/opentofu.bats b/compatibility-tests/compat-opentofu/test/opentofu.batsindex 40e54ba4c7..123c043215 100644--- a/compatibility-tests/compat-opentofu/test/opentofu.bats+++ b/compatibility-tests/compat-opentofu/test/opentofu.bats@@ -185,3 +185,10 @@ setup() {[ -n "$output" ][ "$output" != "None" ]}++@test "OpenTofu: Firehose extended_s3 delivery stream created with correct config" {+ run aws_cmd firehose describe-delivery-stream --delivery-stream-name floci-compat-firehose \+ --query "DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat" --output text+ assert_success+ assert_output "GZIP"+}diff --git a/compatibility-tests/compat-terraform/main.tf b/compatibility-tests/compat-terraform/main.tfindex 7aae451beb..8f20d7410c 100644--- a/compatibility-tests/compat-terraform/main.tf+++ b/compatibility-tests/compat-terraform/main.tf@@ -341,3 +341,27 @@ output "zone_id" {output "health_check_id" {value = aws_route53_health_check.app.id}++# -- Kinesis Firehose Delivery Stream (extended_s3, issue #1043) ---------------+resource "aws_kinesis_firehose_delivery_stream" "events" {+ name = "floci-compat-firehose"+ destination = "extended_s3"++ extended_s3_configuration {+ role_arn = aws_iam_role.lambda_exec.arn+ bucket_arn = aws_s3_bucket.app.arn+ prefix = "events/data/"+ error_output_prefix = "events/errors/"+ compression_format = "GZIP"+ buffering_size = 64+ buffering_interval = 120+ }++ tags = {+ Environment = "compat-test"+ }+}++output "firehose_stream_arn" {+ value = aws_kinesis_firehose_delivery_stream.events.arn+}diff --git a/compatibility-tests/compat-terraform/test/terraform.bats b/compatibility-tests/compat-terraform/test/terraform.batsindex c4480bafec..5be9906bb9 100644--- a/compatibility-tests/compat-terraform/test/terraform.bats+++ b/compatibility-tests/compat-terraform/test/terraform.bats@@ -204,3 +204,10 @@ setup() {[ -n "$output" ][ "$output" != "None" ]}++@test "Terraform: Firehose extended_s3 delivery stream created with correct config" {+ run aws_cmd firehose describe-delivery-stream --delivery-stream-name floci-compat-firehose \+ --query "DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat" --output text+ assert_success+ assert_output "GZIP"+}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseExtendedS3Test.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseExtendedS3Test.javanew file mode 100644index 0000000000..11b2a44e7b--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseExtendedS3Test.java@@ -0,0 +1,153 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.services.firehose.FirehoseClient;+import software.amazon.awssdk.services.firehose.model.*;++import java.util.UUID;++import static org.assertj.core.api.Assertions.*;++/**+ * Issue #1043 — the Terraform/Pulumi provider creates delivery streams with+ * ExtendedS3DestinationConfiguration and reads ExtendedS3DestinationDescription+ * back (plus VersionId/DestinationId to drive UpdateDestination).+ */+@DisplayName("Firehose extended S3 destination — issue #1043")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class FirehoseExtendedS3Test {++ private static FirehoseClient firehose;+ private static final String STREAM_NAME = "sdk-extended-s3-" + UUID.randomUUID().toString().substring(0, 8);+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";+ private static final String BUCKET_ARN = "arn:aws:s3:::floci-firehose-sdk-test";++ private static String versionId;+ private static String destinationId;++ @BeforeAll+ static void setup() {+ firehose = TestFixtures.firehoseClient();+ }++ @AfterAll+ static void cleanup() {+ if (firehose != null) {+ try {+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME).build());+ } catch (Exception ignored) {}+ firehose.close();+ }+ }++ @Test+ @Order(1)+ @DisplayName("Create delivery stream with ExtendedS3DestinationConfiguration")+ void createExtendedS3DeliveryStream() {+ CreateDeliveryStreamResponse response = firehose.createDeliveryStream(+ CreateDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)+ .extendedS3DestinationConfiguration(ExtendedS3DestinationConfiguration.builder()+ .roleARN(ROLE_ARN)+ .bucketARN(BUCKET_ARN)+ .prefix("events/data/")+ .errorOutputPrefix("events/errors/")+ .compressionFormat(CompressionFormat.GZIP)+ .bufferingHints(BufferingHints.builder()+ .intervalInSeconds(120)+ .sizeInMBs(64)+ .build())+ .build())+ .build());++ assertThat(response.deliveryStreamARN()).contains(STREAM_NAME);+ }++ @Test+ @Order(2)+ @DisplayName("Describe returns ExtendedS3DestinationDescription (the provider read path)")+ void describeReturnsExtendedS3Description() {+ DeliveryStreamDescription desc = firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .build())+ .deliveryStreamDescription();++ assertThat(desc.versionId()).isNotBlank();+ assertThat(desc.deliveryStreamType()).isEqualTo(DeliveryStreamType.DIRECT_PUT);+ assertThat(desc.hasMoreDestinations()).isFalse();+ assertThat(desc.destinations()).hasSize(1);++ DestinationDescription destination = desc.destinations().get(0);+ assertThat(destination.destinationId()).isNotBlank();++ ExtendedS3DestinationDescription extended = destination.extendedS3DestinationDescription();+ assertThat(extended).as("ExtendedS3DestinationDescription must be present").isNotNull();+ assertThat(extended.roleARN()).isEqualTo(ROLE_ARN);+ assertThat(extended.bucketARN()).isEqualTo(BUCKET_ARN);+ assertThat(extended.prefix()).isEqualTo("events/data/");+ assertThat(extended.errorOutputPrefix()).isEqualTo("events/errors/");+ assertThat(extended.compressionFormat()).isEqualTo(CompressionFormat.GZIP);+ assertThat(extended.bufferingHints().sizeInMBs()).isEqualTo(64);+ assertThat(extended.bufferingHints().intervalInSeconds()).isEqualTo(120);+ assertThat(extended.encryptionConfiguration().noEncryptionConfig())+ .isEqualTo(NoEncryptionConfig.NO_ENCRYPTION);++ assertThat(destination.s3DestinationDescription())+ .as("deprecated S3DestinationDescription mirror must also be present")+ .isNotNull();+ assertThat(destination.s3DestinationDescription().bucketARN()).isEqualTo(BUCKET_ARN);++ versionId = desc.versionId();+ destinationId = destination.destinationId();+ }++ @Test+ @Order(3)+ @DisplayName("UpdateDestination applies changes and bumps VersionId")+ void updateDestination() {+ firehose.updateDestination(UpdateDestinationRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .currentDeliveryStreamVersionId(versionId)+ .destinationId(destinationId)+ .extendedS3DestinationUpdate(ExtendedS3DestinationUpdate.builder()+ .compressionFormat(CompressionFormat.SNAPPY)+ .bufferingHints(BufferingHints.builder()+ .intervalInSeconds(60)+ .sizeInMBs(128)+ .build())+ .build())+ .build());++ DeliveryStreamDescription desc = firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .build())+ .deliveryStreamDescription();++ assertThat(desc.versionId()).isNotEqualTo(versionId);+ ExtendedS3DestinationDescription extended =+ desc.destinations().get(0).extendedS3DestinationDescription();+ assertThat(extended.compressionFormat()).isEqualTo(CompressionFormat.SNAPPY);+ assertThat(extended.bufferingHints().sizeInMBs()).isEqualTo(128);+ assertThat(extended.roleARN()).as("untouched fields survive the update").isEqualTo(ROLE_ARN);+ assertThat(extended.prefix()).isEqualTo("events/data/");+ }++ @Test+ @Order(4)+ @DisplayName("UpdateDestination with stale version throws ConcurrentModificationException")+ void updateDestinationStaleVersion() {+ assertThatThrownBy(() -> firehose.updateDestination(UpdateDestinationRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .currentDeliveryStreamVersionId("99")+ .destinationId(destinationId)+ .extendedS3DestinationUpdate(ExtendedS3DestinationUpdate.builder()+ .compressionFormat(CompressionFormat.GZIP)+ .build())+ .build()))+ .isInstanceOf(ConcurrentModificationException.class);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.javaindex dc2eaa8fb8..7f3fca2c14 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java@@ -42,9 +42,24 @@ public class FirehoseJsonHandler {tags.add(mapper.treeToValue(tNode, io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Tag.class));}}- String arn = firehoseService.createDeliveryStream(name, s3, tags);+ String deliveryStreamType = request.has("DeliveryStreamType")+ ? request.get("DeliveryStreamType").asText() : null;+ String arn = firehoseService.createDeliveryStream(name, s3, tags, deliveryStreamType);yield Response.ok(Map.of("DeliveryStreamARN", arn)).build();}+ case "UpdateDestination" -> {+ String name = request.get("DeliveryStreamName").asText();+ String currentVersionId = request.get("CurrentDeliveryStreamVersionId").asText();+ String destinationId = request.get("DestinationId").asText();+ S3Destination update = null;+ if (request.has("ExtendedS3DestinationUpdate")) {+ update = mapper.treeToValue(request.get("ExtendedS3DestinationUpdate"), S3Destination.class);+ } else if (request.has("S3DestinationUpdate")) {+ update = mapper.treeToValue(request.get("S3DestinationUpdate"), S3Destination.class);+ }+ firehoseService.updateDestination(name, currentVersionId, destinationId, update);+ yield Response.ok(Map.of()).build();+ }case "DescribeDeliveryStream" -> {String name = request.get("DeliveryStreamName").asText();var desc = firehoseService.describeDeliveryStream(name);diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex f8bcbe5474..4369a6a347 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -45,16 +45,97 @@ public class FirehoseService {}public String createDeliveryStream(String name, S3Destination s3Config, List<DeliveryStreamDescription.Tag> tags) {+ return createDeliveryStream(name, s3Config, tags, null);+ }++ public String createDeliveryStream(String name, S3Destination s3Config, List<DeliveryStreamDescription.Tag> tags,+ String deliveryStreamType) {+ validateBufferingHints(s3Config);String arn = AwsArnUtils.Arn.of("firehose", regionResolver.getDefaultRegion(), regionResolver.getAccountId(), "deliverystream/" + name).toString();DeliveryStreamDescription description = new DeliveryStreamDescription(name, arn, s3Config);description.setAccountId(regionResolver.getAccountId());description.setTags(tags);+ if (deliveryStreamType != null && !deliveryStreamType.isBlank()) {+ description.setDeliveryStreamType(deliveryStreamType);+ }streamStore.put(name, description);buffers.put(name, Collections.synchronizedList(new ArrayList<>()));LOG.infov("Created Firehose delivery stream: {0}", name);return arn;}+ public void updateDestination(String name, String currentVersionId, String destinationId, S3Destination update) {+ DeliveryStreamDescription stream = describeDeliveryStream(name);+ if (!stream.getVersionId().equals(currentVersionId)) {+ throw new AwsException("ConcurrentModificationException",+ "Cannot update firehose: " + name + " since the current version id: " + stream.getVersionId()+ + " and specified version id: " + currentVersionId + " do not match", 400);+ }+ DeliveryStreamDescription.Destination destination = stream.getDestinations() != null && !stream.getDestinations().isEmpty()+ ? stream.getDestinations().get(0)+ : null;+ if (destination == null || !destination.getDestinationId().equals(destinationId)) {+ throw new AwsException("InvalidArgumentException",+ "Destination Id " + destinationId + " not found", 400);+ }+ if (update == null) {+ throw new AwsException("InvalidArgumentException",+ "A destination update is required for UpdateDestination.", 400);+ }+ validateBufferingHints(update);+ S3Destination current = destination.getExtendedS3DestinationDescription();+ if (current == null) {+ update.applyDefaults();+ destination.setExtendedS3DestinationDescription(update);+ } else {+ mergeDestination(current, update);+ }+ stream.setVersionId(String.valueOf(parseVersionId(stream.getVersionId()) + 1));+ stream.setLastUpdateTimestamp(java.time.Instant.now());+ streamStore.put(name, stream);+ LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);+ }++ // A corrupt persisted version can only reach here when the caller echoed it+ // (the equality check above passed), so self-heal instead of failing with a 500+ // or blaming the client.+ private static long parseVersionId(String versionId) {+ try {+ return Long.parseLong(versionId);+ } catch (NumberFormatException e) {+ return 0L;+ }+ }++ /**+ * AWS requires SizeInMBs and IntervalInSeconds to be specified together:+ * "This parameter is optional but if you specify a value for it, you must+ * also specify a value for IntervalInSeconds, and vice versa" (firehose+ * service-2.json). Rejecting partial hints here is what keeps the+ * whole-object replacement in mergeDestination faithful to AWS.+ */+ private static void validateBufferingHints(S3Destination config) {+ DeliveryStreamDescription.BufferingHints hints = config == null ? null : config.getBufferingHints();+ if (hints == null) {+ return;+ }+ if ((hints.getSizeInMBs() == null) != (hints.getIntervalInSeconds() == null)) {+ throw new AwsException("InvalidArgumentException",+ "If you specify a value for SizeInMBs, you must also specify a value for IntervalInSeconds, and vice versa.",+ 400);+ }+ }++ private static void mergeDestination(S3Destination current, S3Destination update) {+ if (update.getRoleArn() != null) current.setRoleArn(update.getRoleArn());+ if (update.getBucketArn() != null) current.setBucketArn(update.getBucketArn());+ if (update.getPrefix() != null) current.setPrefix(update.getPrefix());+ if (update.getErrorOutputPrefix() != null) current.setErrorOutputPrefix(update.getErrorOutputPrefix());+ if (update.getCompressionFormat() != null) current.setCompressionFormat(update.getCompressionFormat());+ if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());+ if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());+ }+public void tagDeliveryStream(String name, List<DeliveryStreamDescription.Tag> tagsToTag) {DeliveryStreamDescription stream = describeDeliveryStream(name);Map<String, String> tagMap = new LinkedHashMap<>();@@ -105,9 +186,14 @@ public class FirehoseService {}public DeliveryStreamDescription describeDeliveryStream(String name) {- return streamStore.get(name)+ DeliveryStreamDescription stream = streamStore.get(name).orElseThrow(() -> new AwsException("ResourceNotFoundException","Delivery stream not found: " + name, 400));+ // Normalizes streams persisted before required output members existed.+ if (stream.s3Destination() != null) {+ stream.s3Destination().applyDefaults();+ }+ return stream;}public void deleteDeliveryStream(String name) {diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.javaindex 4aeff99921..17f926eb3b 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java@@ -1,6 +1,8 @@package io.github.hectorvent.floci.services.firehose.model;import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import com.fasterxml.jackson.annotation.JsonInclude;import com.fasterxml.jackson.annotation.JsonProperty;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;@@ -8,6 +10,7 @@ import java.util.List;import java.util.ArrayList;@RegisterForReflection+@JsonIgnoreProperties(ignoreUnknown = true)public class DeliveryStreamDescription {@JsonProperty("DeliveryStreamName")private String deliveryStreamName;@@ -16,9 +19,19 @@ public class DeliveryStreamDescription {private String deliveryStreamARN;@JsonProperty("DeliveryStreamStatus")private DeliveryStreamStatus deliveryStreamStatus;+ @JsonProperty("DeliveryStreamType")+ private String deliveryStreamType = "DirectPut";+ @JsonProperty("VersionId")+ private String versionId = "1";+ @JsonProperty("HasMoreDestinations")+ private boolean hasMoreDestinations;@JsonProperty("CreateTimestamp")@JsonFormat(shape = JsonFormat.Shape.NUMBER)private Instant createTimestamp;+ @JsonProperty("LastUpdateTimestamp")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ private Instant lastUpdateTimestamp;@JsonProperty("Destinations")private List<Destination> destinations;@JsonProperty("Tags")@@ -30,6 +43,9 @@ public class DeliveryStreamDescription {this.deliveryStreamARN = arn;this.deliveryStreamStatus = DeliveryStreamStatus.ACTIVE;this.createTimestamp = Instant.now();+ if (s3 != null) {+ s3.applyDefaults();+ }this.destinations = List.of(new Destination(s3));}@@ -42,8 +58,16 @@ public class DeliveryStreamDescription {public void setDeliveryStreamARN(String deliveryStreamARN) { this.deliveryStreamARN = deliveryStreamARN; }public DeliveryStreamStatus getDeliveryStreamStatus() { return deliveryStreamStatus; }public void setDeliveryStreamStatus(DeliveryStreamStatus deliveryStreamStatus) { this.deliveryStreamStatus = deliveryStreamStatus; }+ public String getDeliveryStreamType() { return deliveryStreamType; }+ public void setDeliveryStreamType(String deliveryStreamType) { this.deliveryStreamType = deliveryStreamType; }+ public String getVersionId() { return versionId; }+ public void setVersionId(String versionId) { this.versionId = versionId; }+ public boolean isHasMoreDestinations() { return hasMoreDestinations; }+ public void setHasMoreDestinations(boolean hasMoreDestinations) { this.hasMoreDestinations = hasMoreDestinations; }public Instant getCreateTimestamp() { return createTimestamp; }public void setCreateTimestamp(Instant createTimestamp) { this.createTimestamp = createTimestamp; }+ public Instant getLastUpdateTimestamp() { return lastUpdateTimestamp; }+ public void setLastUpdateTimestamp(Instant lastUpdateTimestamp) { this.lastUpdateTimestamp = lastUpdateTimestamp; }public List<Destination> getDestinations() { return destinations; }public void setDestinations(List<Destination> destinations) { this.destinations = destinations; }@@ -54,32 +78,100 @@ public class DeliveryStreamDescription {}@RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)public static class Destination {- @JsonProperty("S3DestinationDescription")- private S3Destination s3DestinationDescription;+ @JsonProperty("DestinationId")+ private String destinationId = "destinationId-000000000001";++ // Single canonical S3 config, serialized under both wire keys: real AWS+ // returns ExtendedS3DestinationDescription plus the deprecated+ // S3DestinationDescription mirror for every S3-backed stream.+ private S3Destination s3;public Destination() {}- public Destination(S3Destination s3) { this.s3DestinationDescription = s3; }- public S3Destination getS3DestinationDescription() { return s3DestinationDescription; }- public void setS3DestinationDescription(S3Destination s3) { this.s3DestinationDescription = s3; }+ public Destination(S3Destination s3) { this.s3 = s3; }++ public String getDestinationId() { return destinationId; }+ public void setDestinationId(String destinationId) { this.destinationId = destinationId; }++ @JsonProperty("S3DestinationDescription")+ public S3Destination getS3DestinationDescription() { return s3; }+ @JsonProperty("S3DestinationDescription")+ public void setS3DestinationDescription(S3Destination s3) {+ // Guarded so persisted JSON carrying both keys (identical content) stays+ // idempotent while legacy files with only the S3 key still load.+ if (this.s3 == null) {+ this.s3 = s3;+ }+ }++ @JsonProperty("ExtendedS3DestinationDescription")+ public S3Destination getExtendedS3DestinationDescription() { return s3; }+ @JsonProperty("ExtendedS3DestinationDescription")+ public void setExtendedS3DestinationDescription(S3Destination s3) { this.s3 = s3; }}@RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)public static class S3Destination {+ @JsonProperty("RoleARN")+ private String roleArn;@JsonProperty("BucketARN")private String bucketArn;@JsonProperty("Prefix")private String prefix;+ @JsonProperty("ErrorOutputPrefix")+ private String errorOutputPrefix;+ @JsonProperty("CompressionFormat")+ private String compressionFormat;@JsonProperty("BufferingHints")private BufferingHints bufferingHints;+ @JsonProperty("EncryptionConfiguration")+ private EncryptionConfiguration encryptionConfiguration;public S3Destination() {}+ public String getRoleArn() { return roleArn; }+ public void setRoleArn(String roleArn) { this.roleArn = roleArn; }public String getBucketArn() { return bucketArn; }public void setBucketArn(String bucketArn) { this.bucketArn = bucketArn; }public String getPrefix() { return prefix; }public void setPrefix(String prefix) { this.prefix = prefix; }+ public String getErrorOutputPrefix() { return errorOutputPrefix; }+ public void setErrorOutputPrefix(String errorOutputPrefix) { this.errorOutputPrefix = errorOutputPrefix; }+ public String getCompressionFormat() { return compressionFormat; }+ public void setCompressionFormat(String compressionFormat) { this.compressionFormat = compressionFormat; }public BufferingHints getBufferingHints() { return bufferingHints; }public void setBufferingHints(BufferingHints bufferingHints) { this.bufferingHints = bufferingHints; }+ public EncryptionConfiguration getEncryptionConfiguration() { return encryptionConfiguration; }+ public void setEncryptionConfiguration(EncryptionConfiguration encryptionConfiguration) { this.encryptionConfiguration = encryptionConfiguration; }++ /**+ * Fills the members the wire contract marks required with the AWS defaults.+ * Getters stay null-honest so UpdateDestination merges can tell "not+ * specified" from a value; call this only on create and describe paths.+ */+ public void applyDefaults() {+ if (compressionFormat == null) {+ compressionFormat = "UNCOMPRESSED";+ }+ if (encryptionConfiguration == null) {+ encryptionConfiguration = EncryptionConfiguration.noEncryption();+ }+ if (bufferingHints == null) {+ bufferingHints = BufferingHints.defaults();+ } else {+ // Self-heal legacy persisted state; validation keeps partial+ // hints out of the create/update paths.+ if (bufferingHints.getSizeInMBs() == null) {+ bufferingHints.setSizeInMBs(5);+ }+ if (bufferingHints.getIntervalInSeconds() == null) {+ bufferingHints.setIntervalInSeconds(300);+ }+ }+ }/** Extracts bucket name from ARN: arn:aws:s3:::my-bucket → my-bucket */public String bucketName() {@@ -90,17 +182,66 @@ public class DeliveryStreamDescription {}@RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class EncryptionConfiguration {+ @JsonProperty("NoEncryptionConfig")+ private String noEncryptionConfig;+ @JsonProperty("KMSEncryptionConfig")+ private KmsEncryptionConfig kmsEncryptionConfig;++ public EncryptionConfiguration() {}++ public static EncryptionConfiguration noEncryption() {+ EncryptionConfiguration config = new EncryptionConfiguration();+ config.noEncryptionConfig = "NoEncryption";+ return config;+ }++ public String getNoEncryptionConfig() { return noEncryptionConfig; }+ public void setNoEncryptionConfig(String noEncryptionConfig) { this.noEncryptionConfig = noEncryptionConfig; }+ public KmsEncryptionConfig getKmsEncryptionConfig() { return kmsEncryptionConfig; }+ public void setKmsEncryptionConfig(KmsEncryptionConfig kmsEncryptionConfig) { this.kmsEncryptionConfig = kmsEncryptionConfig; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ public static class KmsEncryptionConfig {+ @JsonProperty("AWSKMSKeyARN")+ private String awsKmsKeyArn;++ public KmsEncryptionConfig() {}+ public String getAwsKmsKeyArn() { return awsKmsKeyArn; }+ public void setAwsKmsKeyArn(String awsKmsKeyArn) { this.awsKmsKeyArn = awsKmsKeyArn; }+ }++ /**+ * Members stay boxed and null-honest so validation can tell "not specified"+ * from a value: AWS requires SizeInMBs and IntervalInSeconds to be specified+ * together, and the defaults (5 MiB / 300 s) apply only when the whole+ * object is absent.+ */+ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)public static class BufferingHints {@JsonProperty("SizeInMBs")- private int sizeInMBs = 5;+ private Integer sizeInMBs;@JsonProperty("IntervalInSeconds")- private int intervalInSeconds = 300;+ private Integer intervalInSeconds;public BufferingHints() {}- public int getSizeInMBs() { return sizeInMBs; }- public void setSizeInMBs(int sizeInMBs) { this.sizeInMBs = sizeInMBs; }- public int getIntervalInSeconds() { return intervalInSeconds; }- public void setIntervalInSeconds(int intervalInSeconds) { this.intervalInSeconds = intervalInSeconds; }++ public static BufferingHints defaults() {+ BufferingHints hints = new BufferingHints();+ hints.sizeInMBs = 5;+ hints.intervalInSeconds = 300;+ return hints;+ }++ public Integer getSizeInMBs() { return sizeInMBs; }+ public void setSizeInMBs(Integer sizeInMBs) { this.sizeInMBs = sizeInMBs; }+ public Integer getIntervalInSeconds() { return intervalInSeconds; }+ public void setIntervalInSeconds(Integer intervalInSeconds) { this.intervalInSeconds = intervalInSeconds; }}public List<Tag> getTags() {@@ -110,6 +251,7 @@ public class DeliveryStreamDescription {public void setTags(List<Tag> tags) { this.tags = tags; }@RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)public static class Tag {@JsonProperty("Key")private String key;diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.javanew file mode 100644index 0000000000..a8fa16ee89--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java@@ -0,0 +1,323 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;++@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class FirehoseExtendedS3IntegrationTest {++ private static final String STREAM_NAME = "test-extended-s3-stream";+ private static final String LEGACY_STREAM_NAME = "test-legacy-s3-stream";+ private static final String MINIMAL_STREAM_NAME = "test-minimal-extended-s3-stream";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-delivery-role";+ private static final String BUCKET_ARN = "arn:aws:s3:::extended-s3-archive";+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String TARGET_PREFIX = "Firehose_20150804.";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void createWithExtendedS3Configuration() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "DeliveryStreamType": "DirectPut",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "Prefix": "events/data/",+ "ErrorOutputPrefix": "events/errors/",+ "CompressionFormat": "GZIP",+ "BufferingHints": { "SizeInMBs": 64, "IntervalInSeconds": 120 }+ }+ }+ """.formatted(STREAM_NAME, ROLE_ARN, BUCKET_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamARN", notNullValue());+ }++ @Test+ @Order(2)+ void describeReturnsExtendedS3Description() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamDescription.DeliveryStreamType", equalTo("DirectPut"))+ .body("DeliveryStreamDescription.VersionId", equalTo("1"))+ .body("DeliveryStreamDescription.HasMoreDestinations", equalTo(false))+ .body("DeliveryStreamDescription.Destinations[0].DestinationId", notNullValue())+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.RoleARN", equalTo(ROLE_ARN))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.Prefix", equalTo("events/data/"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.ErrorOutputPrefix", equalTo("events/errors/"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat", equalTo("GZIP"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(64))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(120))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig", equalTo("NoEncryption"))+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.RoleARN", equalTo(ROLE_ARN));+ }++ @Test+ @Order(3)+ void describeAppliesDefaultsForMinimalConfig() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s"+ }+ }+ """.formatted(MINIMAL_STREAM_NAME, ROLE_ARN, BUCKET_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + MINIMAL_STREAM_NAME + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat", equalTo("UNCOMPRESSED"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(5))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(300))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig", equalTo("NoEncryption"));+ }++ @Test+ @Order(4)+ void legacyS3ConfigurationAlsoReturnsExtendedDescription() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "S3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "Prefix": "legacy/"+ }+ }+ """.formatted(LEGACY_STREAM_NAME, ROLE_ARN, BUCKET_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + LEGACY_STREAM_NAME + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.Prefix", equalTo("legacy/"));+ }++ @Test+ @Order(5)+ void updateDestinationRejectsStaleVersion() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "CurrentDeliveryStreamVersionId": "99",+ "DestinationId": "destinationId-000000000001",+ "ExtendedS3DestinationUpdate": { "CompressionFormat": "Snappy" }+ }+ """.formatted(STREAM_NAME))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ConcurrentModificationException"));+ }++ @Test+ @Order(6)+ void updateDestinationAppliesChangesAndBumpsVersion() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "CurrentDeliveryStreamVersionId": "1",+ "DestinationId": "destinationId-000000000001",+ "ExtendedS3DestinationUpdate": {+ "CompressionFormat": "Snappy",+ "BufferingHints": { "SizeInMBs": 128, "IntervalInSeconds": 60 }+ }+ }+ """.formatted(STREAM_NAME))+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamDescription.VersionId", equalTo("2"))+ .body("DeliveryStreamDescription.LastUpdateTimestamp", notNullValue())+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CompressionFormat", equalTo("Snappy"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(128))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.RoleARN", equalTo(ROLE_ARN))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.Prefix", equalTo("events/data/"));+ }++ @Test+ @Order(7)+ void updateDestinationWithoutUpdatePayloadIsRejectedAndDoesNotBumpVersion() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "CurrentDeliveryStreamVersionId": "2",+ "DestinationId": "destinationId-000000000001"+ }+ """.formatted(STREAM_NAME))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"));++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamDescription.VersionId", equalTo("2"));+ }++ @Test+ @Order(8)+ void updateDestinationRejectsPartialBufferingHintsAndDoesNotBumpVersion() {+ // AWS requires SizeInMBs and IntervalInSeconds to be specified together+ // (firehose service-2.json BufferingHints member docs).+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "CurrentDeliveryStreamVersionId": "2",+ "DestinationId": "destinationId-000000000001",+ "ExtendedS3DestinationUpdate": {+ "BufferingHints": { "SizeInMBs": 16 }+ }+ }+ """.formatted(STREAM_NAME))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"));++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamDescription.VersionId", equalTo("2"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(128))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(60));+ }++ @Test+ @Order(9)+ void createDeliveryStreamRejectsPartialBufferingHints() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "partial-hints-stream",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "BufferingHints": { "IntervalInSeconds": 60 }+ }+ }+ """.formatted(ROLE_ARN, BUCKET_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"));+ }++ @Test+ @Order(10)+ void updateDestinationRejectsUnknownDestinationId() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "CurrentDeliveryStreamVersionId": "2",+ "DestinationId": "destinationId-999999999999",+ "ExtendedS3DestinationUpdate": { "CompressionFormat": "GZIP" }+ }+ """.formatted(STREAM_NAME))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"));+ }+}