Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudformation): forward Firehose processing and logging configuration (#6277)
Mario Toffiadeeb3b307588
diff --git a/docs/services/cloudformation.md b/docs/services/cloudformation.mdindex cf52c879fa..6ae57a016a 100644--- a/docs/services/cloudformation.md+++ b/docs/services/cloudformation.md@@ -211,7 +211,7 @@ cross-resource references.| GuardDuty | `MalwareProtectionPlan` (management plane only: no scan runs, so the plan stays ACTIVE; Ref returns the plan id; a BucketName change is rejected) || Pipes | `Pipe` || Kinesis | `Stream` |-| Kinesis Data Firehose | `DeliveryStream` |+| Kinesis Data Firehose | `DeliveryStream` (`DeliveryStreamName`, `Tags` and, from an S3 destination, `BucketARN`, `RoleARN`, `Prefix`, `ErrorOutputPrefix`, `CompressionFormat`, `BufferingHints`, `CloudWatchLoggingOptions` and `ProcessingConfiguration` are applied; other properties are ignored, and a destination member the template drops on update keeps its stored value, as on AWS) || IoT Core | `Authorizer`, `DomainConfiguration` (`ServerCertificates` resolves to a JSON string), `Policy` (deleted after detaching it from its principals; on AWS the delete fails with `DeleteConflictException` while the policy is attached), `Thing`, `TopicRule` || CloudFront | `CachePolicy`, `Distribution`, `Function` (`KeyValueStoreAssociations` is ignored), `KeyGroup`, `OriginAccessControl`, `OriginRequestPolicy`, `PublicKey`, `ResponseHeadersPolicy` || CloudWatch | `Alarm`, `Dashboard` |diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex 4be6366b78..41912d7846 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -111,6 +111,10 @@ Two behaviors invert what the conversion block above does, both probed rather th- Neither `NumberOfRetries` nor the existence of the function a `LambdaArn` names is checked at configuration time, as on AWS. At delivery an ARN naming nothing fails as `Lambda.FunctionError`, and a `NumberOfRetries` above 100 is capped, a bound AWS does not impose so that one flush cannot occupy the flusher indefinitely.- A null entry in `Processors` or `Parameters`, which a raw JSON client can send, is ignored rather than reported; real AWS answers `InternalFailure` there, a fault of its own not worth reproducing.+## Logging options++`CloudWatchLoggingOptions` on the S3 and extended S3 destinations is stored and returned on both `S3DestinationDescription` and `ExtendedS3DestinationDescription`. A destination that never set it reads `{"Enabled": false}`, and an omitted `Enabled` reads `false`, as on AWS. `UpdateDestination` merges it a member at a time, so an update carrying only `{"Enabled": false}` keeps the stored log group and stream. `LogGroupName` and `LogStreamName` are validated against AWS's length and pattern constraints. With `Enabled` true both names are required, on the merged result for `UpdateDestination`, and a missing or empty one is refused with AWS's `InvalidArgumentException`, the group before the stream. Neither Floci nor AWS checks that the log group exists. Floci writes no delivery error logs to it.+## S3 object keysDelivered objects are named `<evaluated prefix><streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid><file extension>`, matching [AWS's object name format](https://docs.aws.amazon.com/firehose/latest/dev/s3-object-name.html):diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/FirehoseCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/FirehoseCfnProvisioner.javaindex 1a495f1712..d4cf0cf7ef 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/FirehoseCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/FirehoseCfnProvisioner.java@@ -1,8 +1,12 @@package io.github.hectorvent.floci.services.cloudformation.provisioners;+import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.cloudformation.model.StackResource;import io.github.hectorvent.floci.services.firehose.FirehoseService;+import io.github.hectorvent.floci.services.firehose.S3DestinationValidator;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;import jakarta.enterprise.context.ApplicationScoped;@@ -16,6 +20,7 @@ import java.util.Set;@ApplicationScopedpublic class FirehoseCfnProvisioner implements CfnResourceProvisioner {+ private static final ObjectMapper MAPPER = new ObjectMapper();private static final int DELIVERY_STREAM_NAME_MAX_LENGTH = 64;private static final int DEFAULT_BUFFER_SIZE_MB = 5;private static final int DEFAULT_BUFFER_INTERVAL_SECONDS = 300;@@ -37,7 +42,8 @@ public class FirehoseCfnProvisioner implements CfnResourceProvisioner {r.getLogicalId(), DELIVERY_STREAM_NAME_MAX_LENGTH, false);DeliveryStreamDescription.S3Destination s3 = null;- JsonNode s3Node = props != null && props.has("ExtendedS3DestinationConfiguration")+ boolean extended = props != null && props.has("ExtendedS3DestinationConfiguration");+ JsonNode s3Node = extended? props.get("ExtendedS3DestinationConfiguration"): (props != null ? props.get("S3DestinationConfiguration") : null);if (s3Node != null && !s3Node.isNull()) {@@ -48,6 +54,15 @@ public class FirehoseCfnProvisioner implements CfnResourceProvisioner {);s3.setBucketArn(blankToNull(ctx.engine().resolve(s3Node.path("BucketARN"))));s3.setPrefix(blankToNull(ctx.engine().resolve(s3Node.path("Prefix"))));+ s3.setRoleArn(blankToNull(ctx.engine().resolve(s3Node.path("RoleARN"))));+ s3.setErrorOutputPrefix(blankToNull(ctx.engine().resolve(s3Node.path("ErrorOutputPrefix"))));+ s3.setCloudWatchLoggingOptions(structured(s3Node, "CloudWatchLoggingOptions",+ DeliveryStreamDescription.CloudWatchLoggingOptions.class, ctx));+ // Only the extended shape models a processor, so the legacy one carries none, as on the API path.+ if (extended) {+ s3.setProcessingConfiguration(structured(s3Node, "ProcessingConfiguration",+ DeliveryStreamDescription.ProcessingConfiguration.class, ctx));+ }if (s3Node.has("BufferingHints")) {JsonNode hints = s3Node.get("BufferingHints");DeliveryStreamDescription.BufferingHints bufferingHints = new DeliveryStreamDescription.BufferingHints();@@ -56,6 +71,8 @@ public class FirehoseCfnProvisioner implements CfnResourceProvisioner {parseIntProp(hints, "IntervalInSeconds", ctx, DEFAULT_BUFFER_INTERVAL_SECONDS));s3.setBufferingHints(bufferingHints);}+ S3DestinationValidator.validateWireShape(s3,+ extended ? "extendedS3DestinationConfiguration" : "s3DestinationConfiguration");}List<DeliveryStreamDescription.Tag> tags = new ArrayList<>();@@ -125,6 +142,25 @@ public class FirehoseCfnProvisioner implements CfnResourceProvisioner {}}+ /**+ * A structured destination member with its intrinsics resolved, read into the type+ * CreateDeliveryStream reads it into. Null when the template leaves it out or AWS::NoValue+ * drops it, which on an update keeps the stored value, as UpdateDestination does. A member+ * present as anything but an object fails the resource, as CloudFormation rejects it.+ */+ private static <T> T structured(JsonNode destination, String name, Class<T> type, ProvisionContext ctx) {+ JsonNode resolved = ctx.engine().resolveNodeOmittingNoValue(destination.get(name));+ if (resolved == null || resolved.isMissingNode()) {+ return null;+ }+ try {+ return MAPPER.treeToValue(resolved, type);+ } catch (JsonProcessingException e) {+ throw new AwsException("ValidationError",+ "AWS::KinesisFirehose::DeliveryStream has an invalid " + name + ": " + e.getOriginalMessage(), 400);+ }+ }+private static String blankToNull(String value) {return value == null || value.isBlank() ? null : value;}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 40dd293f21..d9e319047e 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -406,8 +406,11 @@ public class FirehoseService implements ResourceProvider {"Delivery stream " + name + " already exists.", 409);}+ validateLogging(null, s3Config);validateBufferingHints(s3Config);if (redshift != null) {+ validateLogging(null, redshift.getS3());+ validateLogging(null, redshift.getS3Backup());validateBufferingHints(redshift.getS3());validateBufferingHints(redshift.getS3Backup());}@@ -486,6 +489,7 @@ public class FirehoseService implements ResourceProvider {throw new AwsException("InvalidArgumentException","The delivery stream " + name + " has a Redshift destination; use RedshiftDestinationUpdate.", 400);}+ validateLogging(destination.getExtendedS3DestinationDescription(), update);validateBufferingHints(update);S3Destination current = destination.getExtendedS3DestinationDescription();if (current == null) {@@ -533,6 +537,8 @@ public class FirehoseService implements ResourceProvider {throw new AwsException("InvalidArgumentException","S3BackupMode is Enabled but no S3 backup configuration is set.", 400);}+ validateLogging(current.getS3(), update.getS3());+ validateLogging(current.getS3Backup(), update.getS3Backup());validateBufferingHints(update.getS3());validateBufferingHints(update.getS3Backup());if (update.getS3Backup() != null && current.getS3Backup() == null) {@@ -665,6 +671,29 @@ public class FirehoseService implements ResourceProvider {}}+ /**+ * Once logging is on, both names are required, checked on the merged result for an update+ * (probed 2026-10-23). An update without a logging block leaves the stored one as it was.+ */+ private static void validateLogging(S3Destination stored, S3Destination update) {+ if (update == null || update.getCloudWatchLoggingOptions() == null) {+ return;+ }+ DeliveryStreamDescription.CloudWatchLoggingOptions effective = mergeLogging(+ stored == null ? null : stored.getCloudWatchLoggingOptions(), update.getCloudWatchLoggingOptions());+ if (!Boolean.TRUE.equals(effective.getEnabled())) {+ return;+ }+ if (effective.getLogGroupName() == null || effective.getLogGroupName().isEmpty()) {+ throw new AwsException("InvalidArgumentException",+ "CloudWatch Log Group Name is required if CloudWatch Logging is enabled", 400);+ }+ if (effective.getLogStreamName() == null || effective.getLogStreamName().isEmpty()) {+ throw new AwsException("InvalidArgumentException",+ "CloudWatch Log Stream Name is required if CloudWatch Logging is enabled", 400);+ }+ }+private static void mergeDestination(S3Destination current, S3Destination update) {if (update.getRoleArn() != null) current.setRoleArn(update.getRoleArn());if (update.getBucketArn() != null) current.setBucketArn(update.getBucketArn());@@ -676,6 +705,10 @@ public class FirehoseService implements ResourceProvider {if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());if (update.getS3BackupMode() != null) current.setS3BackupMode(update.getS3BackupMode());+ if (update.getCloudWatchLoggingOptions() != null) {+ current.setCloudWatchLoggingOptions(mergeLogging(+ current.getCloudWatchLoggingOptions(), update.getCloudWatchLoggingOptions()));+ }// Replaced whole, not merged member-wise: an update carrying only// {"Enabled": false} leaves Processors empty on real AWS, where the same shape// preserves the conversion block's members (probed 2026-09-10).@@ -714,6 +747,23 @@ public class FirehoseService implements ResourceProvider {return merged;}+ /**+ * Member-wise, like the conversion block: an update carrying only {"Enabled": false} keeps the+ * stored group and stream, and one naming only a group keeps Enabled (probed 2026-10-22).+ */+ private static DeliveryStreamDescription.CloudWatchLoggingOptions mergeLogging(+ DeliveryStreamDescription.CloudWatchLoggingOptions current,+ DeliveryStreamDescription.CloudWatchLoggingOptions update) {+ if (current == null) {+ return update;+ }+ DeliveryStreamDescription.CloudWatchLoggingOptions merged = new DeliveryStreamDescription.CloudWatchLoggingOptions();+ merged.setEnabled(update.getEnabled() != null ? update.getEnabled() : current.getEnabled());+ merged.setLogGroupName(update.getLogGroupName() != null ? update.getLogGroupName() : current.getLogGroupName());+ merged.setLogStreamName(update.getLogStreamName() != null ? update.getLogStreamName() : current.getLogStreamName());+ return merged;+ }+/*** SchemaConfiguration merges a member at a time as well, so an update naming only* a new TableName keeps the stored role, database and region (probed). Replacing@@ -827,9 +877,7 @@ public class FirehoseService implements ResourceProvider {.orElseThrow(() -> new AwsException("ResourceNotFoundException","Delivery stream not found: " + streamName, 400));// Normalizes streams persisted before required output members existed.- if (stream.s3Destination() != null) {- stream.s3Destination().applyDefaults();- }+ stream.applyDefaults();return stream;}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.javaindex 35bb5b9a81..cbee968595 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.firehose;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.SmithyValidationErrors;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.CloudWatchLoggingOptions;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;@@ -37,7 +38,7 @@ import java.util.regex.Pattern;* Smithy layer does (probe 2026-09-07: both ParquetSerDe bounds violated at* once report "2 validation errors detected: ...; ...").*/-final class S3DestinationValidator {+public final class S3DestinationValidator {/** Listed in the order AWS prints it, which is neither declaration order nor alphabetical. */private static final String ENUM_VALUE_SET = "[ZIP, HADOOP_SNAPPY, Snappy, GZIP, UNCOMPRESSED]";@@ -76,9 +77,15 @@ final class S3DestinationValidator {private static final String NOT_BLANK_PATTERN = "^(?!\\s*$).+";private static final Pattern NOT_BLANK = Pattern.compile(NOT_BLANK_PATTERN);+ private static final int LOG_NAME_MAX_LENGTH = 512;+ private static final String LOG_GROUP_NAME_REGEX = "[\\.\\-_/#A-Za-z0-9]*";+ private static final Pattern LOG_GROUP_NAME = Pattern.compile(LOG_GROUP_NAME_REGEX);+ private static final String LOG_STREAM_NAME_REGEX = "[^:*]*";+ private static final Pattern LOG_STREAM_NAME = Pattern.compile(LOG_STREAM_NAME_REGEX);+private S3DestinationValidator() {}- static void validateWireShape(S3Destination config, String shapeName) {+ public static void validateWireShape(S3Destination config, String shapeName) {if (config == null) {return;}@@ -110,12 +117,45 @@ final class S3DestinationValidator {// violations sit in more than one of them.collectOpenXMappingViolations(config, shapeName, violations);collectSchemaViolations(config, shapeName, violations);+ collectLoggingViolations(config, shapeName, violations);collectProcessingViolations(config, shapeName, violations);if (!violations.isEmpty()) {throw constraintViolations(violations);}}+ /**+ * Both log names are bounded at 512 characters and carry a pattern. Unlike+ * FileExtension, an overlong name with a bad character reports both rules, the+ * length one first, and the group's violations precede the stream's (probed+ * 2026-10-22, where AWS also printed them before any processingConfiguration one).+ */+ private static void collectLoggingViolations(S3Destination config, String shapeName,+ List<String> violations) {+ CloudWatchLoggingOptions logging = config.getCloudWatchLoggingOptions();+ if (logging == null) {+ return;+ }+ collectLogName(logging.getLogGroupName(), shapeName, "cloudWatchLoggingOptions.logGroupName",+ LOG_GROUP_NAME, LOG_GROUP_NAME_REGEX, violations);+ collectLogName(logging.getLogStreamName(), shapeName, "cloudWatchLoggingOptions.logStreamName",+ LOG_STREAM_NAME, LOG_STREAM_NAME_REGEX, violations);+ }++ private static void collectLogName(String value, String shapeName, String member, Pattern pattern,+ String regex, List<String> violations) {+ if (value == null) {+ return;+ }+ if (value.length() > LOG_NAME_MAX_LENGTH) {+ violations.add(violation(shapeName, member,+ "Member must have length less than or equal to " + LOG_NAME_MAX_LENGTH));+ }+ if (!pattern.matcher(value).matches()) {+ violations.add(violation(shapeName, member, "Member must satisfy regular expression pattern: " + regex));+ }+ }+/*** Every SchemaConfiguration string is a 1 to 1024 character value without* whitespace, and unlike the collection members each broken rule is its owndiff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.javaindex 9b71a9c4b4..810ced01e9 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java@@ -66,13 +66,8 @@ public class DeliveryStreamDescription {this.deliveryStreamARN = arn;this.deliveryStreamStatus = DeliveryStreamStatus.ACTIVE;this.createTimestamp = Instant.now();- if (s3 != null) {- s3.applyDefaults();- }- if (redshift != null && redshift.getS3() != null) {- redshift.getS3().applyDefaults();- }this.destinations = List.of(redshift != null ? new Destination(redshift) : new Destination(s3));+ applyDefaults();this.deliveryStreamEncryptionConfiguration = DeliveryStreamEncryptionConfiguration.disabled();if (kinesisStreamSource != null) {this.source = new Source(kinesisStreamSource);@@ -170,6 +165,24 @@ public class DeliveryStreamDescription {return first.getExtendedS3DestinationDescription();}+ /**+ * Fills the AWS defaults on every S3 configuration the stream describes: its S3 destination, or+ * a Redshift destination's intermediate and backup configurations.+ */+ public final void applyDefaults() {+ S3Destination s3 = s3Destination();+ if (s3 != null) {+ s3.applyDefaults();+ }+ if (destinations == null || destinations.isEmpty()) {+ return;+ }+ RedshiftDestination redshift = destinations.get(0).getRedshiftDestinationDescription();+ if (redshift != null && redshift.getS3Backup() != null) {+ redshift.getS3Backup().applyDefaults();+ }+ }+@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@JsonInclude(JsonInclude.Include.NON_NULL)@@ -247,6 +260,8 @@ public class DeliveryStreamDescription {private DataFormatConversionConfiguration dataFormatConversionConfiguration;@JsonProperty("ProcessingConfiguration")private ProcessingConfiguration processingConfiguration;+ @JsonProperty("CloudWatchLoggingOptions")+ private CloudWatchLoggingOptions cloudWatchLoggingOptions;public S3Destination() {}public String getRoleArn() { return roleArn; }@@ -282,6 +297,8 @@ public class DeliveryStreamDescription {public void setProcessingConfiguration(ProcessingConfiguration processingConfiguration) {this.processingConfiguration = processingConfiguration;}+ public CloudWatchLoggingOptions getCloudWatchLoggingOptions() { return cloudWatchLoggingOptions; }+ public void setCloudWatchLoggingOptions(CloudWatchLoggingOptions options) { this.cloudWatchLoggingOptions = options; }/*** True when a transformation applies to deliveries: the configuration is present@@ -356,6 +373,14 @@ public class DeliveryStreamDescription {if (processingConfiguration != null && processingConfiguration.getEnabled() == null) {processingConfiguration.setEnabled(false);}+ // AWS describes a destination that never configured logging as {"Enabled": false},+ // and an omitted Enabled as false (probed 2026-10-22).+ if (cloudWatchLoggingOptions == null) {+ cloudWatchLoggingOptions = new CloudWatchLoggingOptions();+ }+ if (cloudWatchLoggingOptions.getEnabled() == null) {+ cloudWatchLoggingOptions.setEnabled(false);+ }}/**@@ -436,6 +461,7 @@ public class DeliveryStreamDescription {view.compressionFormat = compressionFormat;view.bufferingHints = bufferingHints;view.encryptionConfiguration = encryptionConfiguration;+ view.cloudWatchLoggingOptions = cloudWatchLoggingOptions;return view;}@@ -469,6 +495,31 @@ public class DeliveryStreamDescription {public void setProcessors(List<Processor> processors) { this.processors = processors; }}+ /**+ * Stored and echoed as AWS models it. Floci writes no delivery error logs to the named+ * group. UpdateDestination merges it a member at a time (probed).+ */+ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class CloudWatchLoggingOptions {+ @JsonProperty("Enabled")+ private Boolean enabled;+ @JsonProperty("LogGroupName")+ private String logGroupName;+ @JsonProperty("LogStreamName")+ private String logStreamName;++ public CloudWatchLoggingOptions() {}++ public Boolean getEnabled() { return enabled; }+ public void setEnabled(Boolean enabled) { this.enabled = enabled; }+ public String getLogGroupName() { return logGroupName; }+ public void setLogGroupName(String logGroupName) { this.logGroupName = logGroupName; }+ public String getLogStreamName() { return logStreamName; }+ public void setLogStreamName(String logStreamName) { this.logStreamName = logStreamName; }+ }+@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@JsonInclude(JsonInclude.Include.NON_NULL)diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationFirehoseIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationFirehoseIntegrationTest.javaindex d2676808b0..34110df7a0 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationFirehoseIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationFirehoseIntegrationTest.java@@ -1,18 +1,37 @@package io.github.hectorvent.floci.services.cloudformation;+import io.github.hectorvent.floci.services.firehose.FirehoseService;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.testing.CfnStackWaits;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.InjectMock;import io.quarkus.test.junit.QuarkusTest;import io.restassured.RestAssured;import io.restassured.config.EncoderConfig;import io.restassured.http.ContentType;+import io.restassured.response.ValidatableResponse;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;+import java.util.List;++import static io.github.hectorvent.floci.testing.FirehoseRequests.answerTransforms;+import static io.github.hectorvent.floci.testing.FirehoseRequests.encode;+import static io.github.hectorvent.floci.testing.FirehoseRequests.firstKey;+import static io.github.hectorvent.floci.testing.FirehoseRequests.putRecordBatch;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.verify;/*** End-to-end check that CloudFormation provisions an AWS::KinesisFirehose::DeliveryStream for real* (into FirehoseService) rather than stubbing it. Delivery streams are metadata (no container), so- * the test is Docker-free.+ * the test is Docker-free; the Lambda processor is mocked for the same reason.*/@QuarkusTestclass CloudFormationFirehoseIntegrationTest {@@ -21,6 +40,22 @@ class CloudFormationFirehoseIntegrationTest {"AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/cloudformation/aws4_request";private static final String FIREHOSE_AUTH ="AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/firehose/aws4_request";+ private static final String FIREHOSE_CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String EXTENDED_DESTINATION =+ "DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.";+ private static final String LAMBDA_ARN = EXTENDED_DESTINATION+ + "ProcessingConfiguration.Processors[0].Parameters.find { it.ParameterName == 'LambdaArn' }.ParameterValue";++ @InjectMock+ LambdaService lambdaService;++ @Inject+ FirehoseService firehoseService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }@Testvoid createStackProvisionsDeliveryStreamVisibleToFirehose() {@@ -90,4 +125,122 @@ class CloudFormationFirehoseIntegrationTest {.body(containsString(streamName)).body(containsString("GZIP"));}++ @Test+ void aStackStreamRunsItsLambdaProcessorAndReadsBackItsLogging() {+ String suffix = Long.toString(System.nanoTime(), 36);+ String streamName = "cfn-firehose-transform-" + suffix;+ String stackName = "cfn-firehose-transform-stack-" + suffix;+ String bucket = "cfn-firehose-transform-" + suffix;+ String logGroup = "/aws/kinesisfirehose/" + suffix;+ String firstFunction = "transform-" + suffix;+ String secondFunction = "transform-next-" + suffix;+ String firstArn = "arn:aws:lambda:us-east-1:000000000000:function:" + firstFunction;+ String secondArn = "arn:aws:lambda:us-east-1:000000000000:function:" + secondFunction;++ stackRequest("CreateStack", stackName, transformTemplate(suffix, streamName, bucket, firstFunction));+ CfnStackWaits.awaitStatus(stackName, "CREATE_COMPLETE");++ describeDeliveryStream(streamName)+ .body(EXTENDED_DESTINATION + "RoleARN", equalTo("arn:aws:iam::000000000000:role/firehose-delivery"))+ .body(EXTENDED_DESTINATION + "ErrorOutputPrefix", equalTo("errors/!{firehose:error-output-type}/"))+ .body(EXTENDED_DESTINATION + "CloudWatchLoggingOptions.Enabled", equalTo(true))+ .body(EXTENDED_DESTINATION + "CloudWatchLoggingOptions.LogGroupName", equalTo(logGroup))+ .body(EXTENDED_DESTINATION + "CloudWatchLoggingOptions.LogStreamName", equalTo("DestinationDelivery"))+ .body(EXTENDED_DESTINATION + "ProcessingConfiguration.Enabled", equalTo(true))+ .body(LAMBDA_ARN, equalTo(firstArn))+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription"+ + ".CloudWatchLoggingOptions.LogGroupName", equalTo(logGroup));++ List<byte[]> payloads = answerTransforms(lambdaService, (recordId, index) ->+ "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode("TRANSFORMED-" + index) + "\"}");+ putRecordBatch(streamName, "one", "two");+ firehoseService.flush(streamName);++ given()+ .when()+ .get("/" + bucket + "/" + firstKey(bucket, "data/"))+ .then()+ .statusCode(200)+ .body(equalTo("TRANSFORMED-0\nTRANSFORMED-1\n"));+ assertEquals(1, payloads.size());+ verify(lambdaService).invokeArn(eq(firstArn), any(), any());++ stackRequest("UpdateStack", stackName, transformTemplate(suffix, streamName, bucket, secondFunction));+ CfnStackWaits.awaitStatus(stackName, "UPDATE_COMPLETE");++ describeDeliveryStream(streamName)+ .body(LAMBDA_ARN, equalTo(secondArn))+ .body(EXTENDED_DESTINATION + "CloudWatchLoggingOptions.LogGroupName", equalTo(logGroup));+ }++ private static String transformTemplate(String suffix, String streamName, String bucket, String function) {+ return """+ {+ "Resources": {+ "Logs": {+ "Type": "AWS::Logs::LogGroup",+ "Properties": {"LogGroupName": "/aws/kinesisfirehose/%s"}+ },+ "Stream": {+ "Type": "AWS::KinesisFirehose::DeliveryStream",+ "Properties": {+ "DeliveryStreamName": "%s",+ "DeliveryStreamType": "DirectPut",+ "ExtendedS3DestinationConfiguration": {+ "BucketARN": "arn:aws:s3:::%s",+ "RoleARN": {"Fn::Sub": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/firehose-delivery"},+ "Prefix": "data/",+ "ErrorOutputPrefix": {"Fn::Join": ["", ["errors/", "!{firehose:error-output-type}/"]]},+ "CloudWatchLoggingOptions": {+ "Enabled": true,+ "LogGroupName": {"Ref": "Logs"},+ "LogStreamName": "DestinationDelivery"+ },+ "ProcessingConfiguration": {+ "Enabled": true,+ "Processors": [+ {+ "Type": "Lambda",+ "Parameters": [+ {+ "ParameterName": "LambdaArn",+ "ParameterValue": {"Fn::Sub": "arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:%s"}+ }+ ]+ }+ ]+ }+ }+ }+ }+ }+ }+ """.formatted(suffix, streamName, bucket, function);+ }++ private static void stackRequest(String action, String stackName, String template) {+ given()+ .contentType("application/x-www-form-urlencoded")+ .header("Authorization", CFN_AUTH)+ .formParam("Action", action)+ .formParam("StackName", stackName)+ .formParam("TemplateBody", template)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ private static ValidatableResponse describeDeliveryStream(String streamName) {+ return given()+ .header("Authorization", FIREHOSE_AUTH)+ .header("X-Amz-Target", "Firehose_20150804.DescribeDeliveryStream")+ .contentType(FIREHOSE_CONTENT_TYPE)+ .body("{\"DeliveryStreamName\":\"" + streamName + "\"}")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.javaindex 07d0728e68..89418b02b2 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java@@ -57,6 +57,7 @@ class LeafCfnProvisionerTest {return node == null || node.isMissingNode() || node.isNull() ? null : node.asText();});when(engine.resolveNode(any())).thenAnswer(i -> i.getArgument(0));+ when(engine.resolveNodeOmittingNoValue(any())).thenAnswer(i -> i.getArgument(0));ctx = new ProvisionContext(engine, REGION, "000000000000", "my-stack");}@@ -360,6 +361,114 @@ class LeafCfnProvisionerTest {assertNull(s3.getValue().getPrefix());}+ @Test+ void processingLoggingRoleAndErrorPrefixReachTheStream() {+ ArgumentCaptor<DeliveryStreamDescription.S3Destination> s3 =+ ArgumentCaptor.forClass(DeliveryStreamDescription.S3Destination.class);+ when(firehose.createDeliveryStream(anyString(), s3.capture(), any())).thenReturn("arn");++ provisioner.provision(resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""+ {+ "DeliveryStreamName": "s",+ "ExtendedS3DestinationConfiguration": {+ "BucketARN": "arn:bucket",+ "RoleARN": "arn:aws:iam::000000000000:role/delivery",+ "ErrorOutputPrefix": "errors/!{firehose:error-output-type}/",+ "CloudWatchLoggingOptions": {"Enabled": true, "LogGroupName": "g", "LogStreamName": "st"},+ "ProcessingConfiguration": {+ "Enabled": true,+ "Processors": [{"Type": "Lambda", "Parameters": [+ {"ParameterName": "LambdaArn", "ParameterValue": "arn:aws:lambda:us-east-1:000000000000:function:f"}+ ]}]+ }+ }+ }+ """), ctx);++ DeliveryStreamDescription.S3Destination destination = s3.getValue();+ assertEquals("arn:aws:iam::000000000000:role/delivery", destination.getRoleArn());+ assertEquals("errors/!{firehose:error-output-type}/", destination.getErrorOutputPrefix());+ assertEquals(Boolean.TRUE, destination.getCloudWatchLoggingOptions().getEnabled());+ assertEquals("g", destination.getCloudWatchLoggingOptions().getLogGroupName());+ assertEquals("st", destination.getCloudWatchLoggingOptions().getLogStreamName());+ assertTrue(destination.isProcessingEnabled());+ assertEquals("arn:aws:lambda:us-east-1:000000000000:function:f", destination.getProcessingConfiguration()+ .getProcessors().get(0).getParameters().get(0).getParameterValue());+ }++ @Test+ void theLegacyDestinationCarriesLoggingButNoProcessor() {+ ArgumentCaptor<DeliveryStreamDescription.S3Destination> s3 =+ ArgumentCaptor.forClass(DeliveryStreamDescription.S3Destination.class);+ when(firehose.createDeliveryStream(anyString(), s3.capture(), any())).thenReturn("arn");++ provisioner.provision(resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""+ {+ "DeliveryStreamName": "s",+ "S3DestinationConfiguration": {+ "BucketARN": "arn:bucket",+ "CloudWatchLoggingOptions": {"LogGroupName": "g"},+ "ProcessingConfiguration": {"Enabled": true, "Processors": []}+ }+ }+ """), ctx);++ assertEquals("g", s3.getValue().getCloudWatchLoggingOptions().getLogGroupName());+ assertNull(s3.getValue().getProcessingConfiguration(),+ "S3DestinationConfiguration models no processor, as on the API path");+ }++ @Test+ void anUnreadableLoggingBlockFailsTheResource() {+ AwsException error = assertThrows(AwsException.class, () -> provisioner.provision(+ resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""+ {+ "DeliveryStreamName": "s",+ "ExtendedS3DestinationConfiguration": {+ "BucketARN": "arn:bucket",+ "CloudWatchLoggingOptions": {"Enabled": {"not": "a boolean"}}+ }+ }+ """), ctx));+ assertEquals("ValidationError", error.getErrorCode());+ }++ @Test+ void aBlockThatIsNotAnObjectFailsTheResource() {+ AwsException error = assertThrows(AwsException.class, () -> provisioner.provision(+ resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""+ {+ "DeliveryStreamName": "s",+ "ExtendedS3DestinationConfiguration": {+ "BucketARN": "arn:bucket",+ "ProcessingConfiguration": []+ }+ }+ """), ctx));+ assertEquals("ValidationError", error.getErrorCode());+ }++ @Test+ void anUnknownProcessorTypeFailsTheResource() {+ AwsException error = assertThrows(AwsException.class, () -> provisioner.provision(+ resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""+ {+ "DeliveryStreamName": "s",+ "ExtendedS3DestinationConfiguration": {+ "BucketARN": "arn:bucket",+ "ProcessingConfiguration": {+ "Enabled": true,+ "Processors": [{"Type": "Lambd", "Parameters": [+ {"ParameterName": "LambdaArn", "ParameterValue": "arn:aws:lambda:us-east-1:000000000000:function:f"}+ ]}]+ }+ }+ }+ """), ctx));+ assertEquals("ValidationException", error.getErrorCode());+ verify(firehose, never()).createDeliveryStream(anyString(), any(), any());+ }+@Testvoid deleteReachesTheService() {provisioner.delete("AWS::KinesisFirehose::DeliveryStream", "s", REGION);diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.javaindex 1d080a0cb4..01a72c04f6 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java@@ -54,6 +54,11 @@ class FirehoseExtendedS3IntegrationTest {"Prefix": "events/data/","ErrorOutputPrefix": "events/errors/","CompressionFormat": "GZIP",+ "CloudWatchLoggingOptions": {+ "Enabled": true,+ "LogGroupName": "/aws/kinesisfirehose/extended",+ "LogStreamName": "DestinationDelivery"+ },"BufferingHints": { "SizeInMBs": 64, "IntervalInSeconds": 120 }}}@@ -88,6 +93,10 @@ class FirehoseExtendedS3IntegrationTest {.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(64)).body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(120)).body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig", equalTo("NoEncryption"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CloudWatchLoggingOptions.Enabled", equalTo(true))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CloudWatchLoggingOptions.LogGroupName", equalTo("/aws/kinesisfirehose/extended"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CloudWatchLoggingOptions.LogStreamName", equalTo("DestinationDelivery"))+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.CloudWatchLoggingOptions.LogGroupName", equalTo("/aws/kinesisfirehose/extended")).body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.BucketARN", equalTo(BUCKET_ARN)).body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.RoleARN", equalTo(ROLE_ARN));}@@ -124,6 +133,8 @@ class FirehoseExtendedS3IntegrationTest {.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(5)).body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(300)).body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig", equalTo("NoEncryption"))+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CloudWatchLoggingOptions.Enabled", equalTo(false))+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.CloudWatchLoggingOptions.Enabled", equalTo(false))// #2200: a create response that omits S3BackupMode, followed by a later describe// that fills it in, applied cleanly and then re-planned dirty forever in Terraform..body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.S3BackupMode", equalTo("Disabled"));@@ -435,6 +446,30 @@ class FirehoseExtendedS3IntegrationTest {assertTrue(key.matches("events/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);}+ @Test+ @Order(17)+ void createDeliveryStreamRefusesEnabledLoggingWithoutAStreamName() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "logging-without-stream-name",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": "g" }+ }+ }+ """.formatted(ROLE_ARN, BUCKET_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"))+ .body("message", equalTo("CloudWatch Log Stream Name is required if CloudWatch Logging is enabled"));+ }+private void createDeliveryStream(String streamName, String bucket, String extraDestinationJson) {given().contentType(CONTENT_TYPE)diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.javaindex 28ad6b293d..fc96caa867 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java@@ -1,7 +1,6 @@package io.github.hectorvent.floci.services.firehose;import io.github.hectorvent.floci.services.lambda.LambdaService;-import io.github.hectorvent.floci.services.lambda.model.InvokeResult;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.InjectMock;import io.quarkus.test.junit.QuarkusTest;@@ -9,24 +8,21 @@ import io.quarkus.test.junit.QuarkusTestProfile;import io.quarkus.test.junit.TestProfile;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;-import org.mockito.Mockito;import java.nio.charset.StandardCharsets;-import java.util.ArrayList;-import java.util.Base64;import java.util.List;import java.util.Map;-import java.util.regex.Matcher;-import java.util.regex.Pattern;+import static io.github.hectorvent.floci.testing.FirehoseRequests.answerTransforms;+import static io.github.hectorvent.floci.testing.FirehoseRequests.encode;+import static io.github.hectorvent.floci.testing.FirehoseRequests.firstKey;+import static io.github.hectorvent.floci.testing.FirehoseRequests.putRecordBatch;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.notNullValue;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertTrue;-import static org.mockito.ArgumentMatchers.any;-import static org.mockito.ArgumentMatchers.anyString;/*** End-to-end coverage of the Lambda transform on the real delivery path: what@@ -54,7 +50,6 @@ class FirehoseLambdaTransformIntegrationTest {private static final String CONTENT_TYPE = "application/x-amz-json-1.1";private static final String TARGET_PREFIX = "Firehose_20150804.";private static final String FUNCTION_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";- private static final Pattern KEY_PATTERN = Pattern.compile("<Key>([^<]+)</Key>");@InjectMockLambdaService lambdaService;@@ -69,8 +64,8 @@ class FirehoseLambdaTransformIntegrationTest {String stream = "transform-ok-stream";String bucket = "transform-ok-archive";createDeliveryStream(stream, bucket);- respondWith((recordId, index) -> "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\""- + encode("TRANSFORMED-" + index) + "\"}");+ answerTransforms(lambdaService, (recordId, index) ->+ "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode("TRANSFORMED-" + index) + "\"}");putRecordBatch(stream, "one", "two");@@ -87,7 +82,7 @@ class FirehoseLambdaTransformIntegrationTest {String stream = "transform-failed-stream";String bucket = "transform-failed-archive";createDeliveryStream(stream, bucket);- respondWith((recordId, index) -> index == 0+ answerTransforms(lambdaService, (recordId, index) -> index == 0? "{\"recordId\":\"" + recordId + "\",\"result\":\"ProcessingFailed\"}": "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode("kept") + "\"}");@@ -116,7 +111,8 @@ class FirehoseLambdaTransformIntegrationTest {String stream = "transform-dropped-stream";String bucket = "transform-dropped-archive";createDeliveryStream(stream, bucket);- respondWith((recordId, index) -> "{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");+ answerTransforms(lambdaService, (recordId, index) ->+ "{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");putRecordBatch(stream, "one", "two");@@ -126,7 +122,7 @@ class FirehoseLambdaTransformIntegrationTest {.then().statusCode(200).extract().asString();- assertFalse(KEY_PATTERN.matcher(listing).find(), "expected no delivered object, got: " + listing);+ assertFalse(listing.contains("<Key>"), "expected no delivered object, got: " + listing);}@Test@@ -134,7 +130,7 @@ class FirehoseLambdaTransformIntegrationTest {String stream = "transform-payload-stream";String bucket = "transform-payload-archive";createDeliveryStream(stream, bucket);- List<byte[]> payloads = respondWith((recordId, index) ->+ List<byte[]> payloads = answerTransforms(lambdaService, (recordId, index) ->"{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");putRecordBatch(stream, "one", "two");@@ -146,39 +142,6 @@ class FirehoseLambdaTransformIntegrationTest {assertTrue(payload.contains("\"data\":\"" + encode("two") + "\""), payload);}- /**- * Answers each invocation from the record ids it actually carries, which the caller- * cannot name: they are minted per batch. Returns the payloads the mock received.- */- private List<byte[]> respondWith(RecordResponder responder) {- List<byte[]> payloads = new ArrayList<>();- Mockito.when(lambdaService.invokeArn(anyString(), any(), any())).thenAnswer(invocation -> {- byte[] payload = invocation.getArgument(1);- payloads.add(payload);- List<String> entries = new ArrayList<>();- Matcher recordIds = Pattern.compile("\"recordId\":\"([^\"]+)\"")- .matcher(new String(payload, StandardCharsets.UTF_8));- for (int index = 0; recordIds.find(); index++) {- entries.add(responder.respond(recordIds.group(1), index));- }- InvokeResult result = new InvokeResult();- result.setStatusCode(200);- result.setPayload(("{\"records\":[" + String.join(",", entries) + "]}")- .getBytes(StandardCharsets.UTF_8));- return result;- });- return payloads;- }-- @FunctionalInterface- private interface RecordResponder {- String respond(String recordId, int index);- }-- private static String encode(String body) {- return Base64.getEncoder().encodeToString(body.getBytes(StandardCharsets.UTF_8));- }-private void createDeliveryStream(String streamName, String bucket) {given().contentType(CONTENT_TYPE)@@ -210,37 +173,4 @@ class FirehoseLambdaTransformIntegrationTest {.statusCode(200).body("DeliveryStreamARN", notNullValue());}-- private void putRecordBatch(String streamName, String... bodies) {- List<String> records = new ArrayList<>();- for (String body : bodies) {- records.add("{\"Data\": \"" + encode(body) + "\"}");- }- given()- .contentType(CONTENT_TYPE)- .header("X-Amz-Target", TARGET_PREFIX + "PutRecordBatch")- .body("{ \"DeliveryStreamName\": \"" + streamName + "\", \"Records\": ["- + String.join(",", records) + "] }")- .when()- .post("/")- .then()- .statusCode(200)- .body("FailedPutCount", equalTo(0));- }-- private String firstKey(String bucket, String prefix) {- String listing = given()- .when()- .get("/" + bucket)- .then()- .statusCode(200)- .extract().asString();- Matcher key = KEY_PATTERN.matcher(listing);- while (key.find()) {- if (key.group(1).startsWith(prefix)) {- return key.group(1);- }- }- throw new AssertionError("expected an object under " + prefix + " in " + bucket + ", got: " + listing);- }}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.javaindex ee08e0b71a..fc6a46a826 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import io.github.hectorvent.floci.services.firehose.model.Record;import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;import io.github.hectorvent.floci.services.kinesis.KinesisService;@@ -84,8 +85,8 @@ class FirehoseRedshiftServiceTest {mock(FirehoseLambdaTransformer.class), delivery, Runnable::run);}- private void createStream(String extraMembers) throws Exception {- RedshiftDestination destination = mapper.readValue("""+ private RedshiftDestination destination(String extraMembers) throws Exception {+ return mapper.readValue("""{"RoleARN": "arn:aws:iam::000000000000:role/firehose","ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev","Username": "admin", "Password": "Secret123",@@ -93,10 +94,18 @@ class FirehoseRedshiftServiceTest {"S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}%s}""".formatted(extraMembers), RedshiftDestination.class);- service.createDeliveryStream("us-east-1", STREAM, null, destination, List.of(), null, null);+ }++ private void createStream(String extraMembers) throws Exception {+ service.createDeliveryStream("us-east-1", STREAM, null, destination(extraMembers), List.of(), null, null);service.putRecord(STREAM, new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));}+ private S3Destination describedBackup(String streamName) {+ return service.describeDeliveryStream(streamName).getDestinations().get(0)+ .getRedshiftDestinationDescription().getS3Backup();+ }+private void copyFails() {doThrow(new IllegalStateException("relation does not exist")).when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());@@ -169,6 +178,25 @@ class FirehoseRedshiftServiceTest {verify(delivery, times(1)).copy(anyString(), anyString(), any(), eq("stage"), anyString());}+ @Test+ void aRedshiftBackupGetsTheLoggingDefaultsOnCreate() throws Exception {+ createStream(BACKUP_ENABLED);+ service.createDeliveryStream("us-east-1", "rs-logging", null, destination(+ ", \"S3BackupMode\": \"Enabled\", \"S3BackupConfiguration\": "+ + "{\"RoleARN\": \"arn:aws:iam::000000000000:role/firehose\", \"BucketARN\": \"arn:aws:s3:::bk\","+ + " \"CloudWatchLoggingOptions\": {\"LogGroupName\": \"g\"}}"),+ List.of(), null, null);++ S3Destination omitted = describedBackup(STREAM);+ assertEquals(false, omitted.getCloudWatchLoggingOptions().getEnabled());+ assertEquals("UNCOMPRESSED", omitted.getCompressionFormat());++ S3Destination withoutEnabled = describedBackup("rs-logging");+ assertEquals(false, withoutEnabled.getCloudWatchLoggingOptions().getEnabled());+ assertEquals("g", withoutEnabled.getCloudWatchLoggingOptions().getLogGroupName());+ assertEquals("UNCOMPRESSED", withoutEnabled.getCompressionFormat());+ }+@Testvoid backupObjectUsesTheBackupConfigurationPrefix() throws Exception {createStream(", \"S3BackupMode\": \"Enabled\", \"S3BackupConfiguration\": "diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.javaindex 2d2b8f2653..2ad60fbe44 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import io.github.hectorvent.floci.services.firehose.model.Record;+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;import io.github.hectorvent.floci.services.kinesis.KinesisService;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.PutObjectOptions;@@ -235,6 +236,34 @@ class FirehoseServiceTest {"legacy entries from another region must not be adopted");}+ @Test+ @SuppressWarnings("unchecked")+ void aPersistedRedshiftBackupGetsTheDefaultsOnDescribe() {+ S3Destination intermediate = new S3Destination();+ intermediate.setBucketArn("arn:aws:s3:::stage");+ S3Destination backup = new S3Destination();+ backup.setBucketArn("arn:aws:s3:::bk");+ RedshiftDestination redshift = new RedshiftDestination();+ redshift.setS3BackupMode("Enabled");+ redshift.setS3(intermediate);+ redshift.setS3Backup(backup);+ // Built the way Jackson reads a stored stream: the no-arg constructor fills no defaults.+ DeliveryStreamDescription persisted = new DeliveryStreamDescription();+ persisted.setDeliveryStreamName("persisted");+ persisted.setDeliveryStreamARN("arn:aws:firehose:us-east-1:000000000000:deliverystream/persisted");+ persisted.setAccountId("000000000000");+ persisted.setDestinations(List.of(new DeliveryStreamDescription.Destination(redshift)));+ AccountAwareStorageBackend<DeliveryStreamDescription> streams =+ (AccountAwareStorageBackend<DeliveryStreamDescription>) backends.get("firehose/streams.json");+ streams.putForAccount("000000000000", "us-east-1/persisted", persisted);++ S3Destination described = firehoseService.describeDeliveryStream("persisted").getDestinations().get(0)+ .getRedshiftDestinationDescription().getS3Backup();++ assertEquals(false, described.getCloudWatchLoggingOptions().getEnabled());+ assertEquals("UNCOMPRESSED", described.getCompressionFormat());+ }+@Testvoid isolatesSameStreamNameAcrossAccountsAndRegionsIncludingFlushAndDelete() {String firstArn = firehoseService.createDeliveryStream("us-east-1", "111111111111", "shared", null,@@ -387,6 +416,107 @@ class FirehoseServiceTest {assertTrue(key.matches("events/2026/01/01/09/my-stream-3-2026-01-01-09-00-00-" + UUID_REGEX), key);}+ @Test+ void cloudWatchLoggingOptionsDefaultToDisabledAndMergeMemberWise() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::logging-bucket");+ firehoseService.createDeliveryStream("logged-stream", s3);+ DeliveryStreamDescription.CloudWatchLoggingOptions logging =+ firehoseService.describeDeliveryStream("logged-stream").s3Destination().getCloudWatchLoggingOptions();+ assertEquals(Boolean.FALSE, logging.getEnabled());+ assertNull(logging.getLogGroupName());++ S3Destination enable = new S3Destination();+ enable.setCloudWatchLoggingOptions(logging(true, "group", "stream"));+ firehoseService.updateDestination("logged-stream", "1", "destinationId-000000000001", enable);++ S3Destination disableOnly = new S3Destination();+ disableOnly.setCloudWatchLoggingOptions(logging(false, null, null));+ firehoseService.updateDestination("logged-stream", "2", "destinationId-000000000001", disableOnly);++ logging = firehoseService.describeDeliveryStream("logged-stream").s3Destination().getCloudWatchLoggingOptions();+ assertEquals(Boolean.FALSE, logging.getEnabled());+ assertEquals("group", logging.getLogGroupName(), "an update naming only Enabled keeps the stored group");+ assertEquals("stream", logging.getLogStreamName());+ }++ private static final String LOG_GROUP_REQUIRED =+ "CloudWatch Log Group Name is required if CloudWatch Logging is enabled";+ private static final String LOG_STREAM_REQUIRED =+ "CloudWatch Log Stream Name is required if CloudWatch Logging is enabled";++ @Test+ void enabledLoggingRequiresTheGroupThenTheStreamName() {+ assertLoggingRefused(LOG_GROUP_REQUIRED, () -> firehoseService.createDeliveryStream("no-names",+ loggedDestination(logging(true, null, null))));+ assertLoggingRefused(LOG_STREAM_REQUIRED, () -> firehoseService.createDeliveryStream("group-only",+ loggedDestination(logging(true, "g", null))));+ assertLoggingRefused(LOG_GROUP_REQUIRED, () -> firehoseService.createDeliveryStream("empty-names",+ loggedDestination(logging(true, "", ""))));++ firehoseService.createDeliveryStream("disabled-no-names", loggedDestination(logging(false, null, null)));+ assertEquals(Boolean.FALSE, firehoseService.describeDeliveryStream("disabled-no-names")+ .s3Destination().getCloudWatchLoggingOptions().getEnabled());+ }++ @Test+ void anUpdateEnablingLoggingIsCheckedAgainstTheStoredNames() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::logging-bucket");+ firehoseService.createDeliveryStream("enable-later", s3);++ assertLoggingRefused(LOG_GROUP_REQUIRED, () -> firehoseService.updateDestination("enable-later", "1",+ "destinationId-000000000001", loggedDestination(logging(true, null, null))));+ assertEquals("1", firehoseService.describeDeliveryStream("enable-later").getVersionId());++ firehoseService.updateDestination("enable-later", "1", "destinationId-000000000001",+ loggedDestination(logging(false, "g", "s")));+ firehoseService.updateDestination("enable-later", "2", "destinationId-000000000001",+ loggedDestination(logging(true, null, null)));+ DeliveryStreamDescription.CloudWatchLoggingOptions logging =+ firehoseService.describeDeliveryStream("enable-later").s3Destination().getCloudWatchLoggingOptions();+ assertEquals(Boolean.TRUE, logging.getEnabled());+ assertEquals("g", logging.getLogGroupName());+ assertEquals("s", logging.getLogStreamName());++ assertLoggingRefused(LOG_STREAM_REQUIRED, () -> firehoseService.updateDestination("enable-later", "3",+ "destinationId-000000000001", loggedDestination(logging(null, null, ""))));+ assertEquals("s", firehoseService.describeDeliveryStream("enable-later")+ .s3Destination().getCloudWatchLoggingOptions().getLogStreamName());+ }++ @Test+ void theLoggingErrorPrecedesTheBufferingHintsOne() {+ S3Destination s3 = loggedDestination(logging(true, null, null));+ DeliveryStreamDescription.BufferingHints hints = new DeliveryStreamDescription.BufferingHints();+ hints.setSizeInMBs(5);+ s3.setBufferingHints(hints);++ assertLoggingRefused(LOG_GROUP_REQUIRED, () -> firehoseService.createDeliveryStream("partial-hints", s3));+ }++ private static S3Destination loggedDestination(DeliveryStreamDescription.CloudWatchLoggingOptions logging) {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::logging-bucket");+ s3.setCloudWatchLoggingOptions(logging);+ return s3;+ }++ private static void assertLoggingRefused(String message, Runnable call) {+ AwsException e = assertThrows(AwsException.class, call::run);+ assertEquals("InvalidArgumentException", e.getErrorCode());+ assertEquals(message, e.getMessage());+ }++ private static DeliveryStreamDescription.CloudWatchLoggingOptions logging(Boolean enabled, String group,+ String stream) {+ DeliveryStreamDescription.CloudWatchLoggingOptions options = new DeliveryStreamDescription.CloudWatchLoggingOptions();+ options.setEnabled(enabled);+ options.setLogGroupName(group);+ options.setLogStreamName(stream);+ return options;+ }+@Testvoid timeBasedFlushKeepsBufferWhileDefaultIntervalHasNotElapsed() {firehoseService.createDeliveryStream("idle-stream", null);diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.javaindex e13309c1df..958cb7f27b 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.firehose;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.CloudWatchLoggingOptions;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;@@ -759,4 +760,65 @@ class S3DestinationValidatorTest {error.getMessage());}+ private static final String LOG_GROUP_PATTERN = "Member must satisfy regular expression pattern: "+ + "[\\.\\-_/#A-Za-z0-9]*";+ private static final String LOG_STREAM_PATTERN = "Member must satisfy regular expression pattern: [^:*]*";++ private static S3Destination withLogging(String logGroupName, String logStreamName) {+ CloudWatchLoggingOptions logging = new CloudWatchLoggingOptions();+ logging.setEnabled(true);+ logging.setLogGroupName(logGroupName);+ logging.setLogStreamName(logStreamName);+ S3Destination config = new S3Destination();+ config.setCloudWatchLoggingOptions(logging);+ return config;+ }++ @Test+ void logStreamNameWithAColonFailsItsPattern() {+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withLogging(null, "bad:stream"), "extendedS3DestinationConfiguration"));++ assertEquals("ValidationException", error.getErrorCode());+ assertEquals("1 validation error detected: Value at 'extendedS3DestinationConfiguration"+ + ".cloudWatchLoggingOptions.logStreamName' failed to satisfy constraint: "+ + LOG_STREAM_PATTERN, error.getMessage());+ }++ /** Probed: unlike FileExtension, both rules are reported, the length one first. */+ @Test+ void anOverlongLogGroupNameWithABadCharacterReportsLengthThenPattern() {+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withLogging("a".repeat(513) + " ", null), "extendedS3DestinationConfiguration"));++ String path = "Value at 'extendedS3DestinationConfiguration.cloudWatchLoggingOptions.logGroupName'"+ + " failed to satisfy constraint: ";+ assertEquals("2 validation errors detected: "+ + path + "Member must have length less than or equal to 512; "+ + path + LOG_GROUP_PATTERN, error.getMessage());+ }++ @Test+ void logGroupViolationsPrecedeLogStreamOnes() {+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withLogging("bad group", "bad:stream"), "s3DestinationConfiguration"));++ assertEquals("2 validation errors detected: "+ + "Value at 's3DestinationConfiguration.cloudWatchLoggingOptions.logGroupName'"+ + " failed to satisfy constraint: " + LOG_GROUP_PATTERN + "; "+ + "Value at 's3DestinationConfiguration.cloudWatchLoggingOptions.logStreamName'"+ + " failed to satisfy constraint: " + LOG_STREAM_PATTERN, error.getMessage());+ }++ @Test+ void wellFormedLoggingNamesPass() {+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withLogging("/aws/kinesisfirehose/my-stream", "DestinationDelivery"),+ "extendedS3DestinationConfiguration"));+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withLogging("a".repeat(512), "b".repeat(512)), "extendedS3DestinationConfiguration"));+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withLogging("", ""), "extendedS3DestinationConfiguration"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/testing/FirehoseRequests.java b/src/test/java/io/github/hectorvent/floci/testing/FirehoseRequests.javanew file mode 100644index 0000000000..d84aaf9d06--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/testing/FirehoseRequests.java@@ -0,0 +1,89 @@+package io.github.hectorvent.floci.testing;++import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import org.mockito.Mockito;++import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Base64;+import java.util.List;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;++public final class FirehoseRequests {++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final Pattern RECORD_ID_PATTERN = Pattern.compile("\"recordId\":\"([^\"]+)\"");+ private static final Pattern KEY_PATTERN = Pattern.compile("<Key>([^<]+)</Key>");++ private FirehoseRequests() {}++ public static String encode(String body) {+ return Base64.getEncoder().encodeToString(body.getBytes(StandardCharsets.UTF_8));+ }++ public static void putRecordBatch(String streamName, String... bodies) {+ List<String> records = new ArrayList<>();+ for (String body : bodies) {+ records.add("{\"Data\": \"" + encode(body) + "\"}");+ }+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "Firehose_20150804.PutRecordBatch")+ .body("{\"DeliveryStreamName\": \"" + streamName + "\", \"Records\": ["+ + String.join(",", records) + "]}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("FailedPutCount", equalTo(0));+ }++ public static String firstKey(String bucket, String prefix) {+ String listing = given()+ .when().get("/" + bucket)+ .then()+ .statusCode(200)+ .extract().asString();+ Matcher key = KEY_PATTERN.matcher(listing);+ while (key.find()) {+ if (key.group(1).startsWith(prefix)) {+ return key.group(1);+ }+ }+ throw new AssertionError("expected an object under " + prefix + " in " + bucket + ", got: " + listing);+ }++ /**+ * Answers each transform invocation from the record ids it actually carries, which the caller+ * cannot name: they are minted per batch. Returns the payloads the mock received.+ */+ public static List<byte[]> answerTransforms(LambdaService lambdaService, RecordResponder responder) {+ List<byte[]> payloads = new ArrayList<>();+ Mockito.when(lambdaService.invokeArn(anyString(), any(), any())).thenAnswer(invocation -> {+ byte[] payload = invocation.getArgument(1);+ payloads.add(payload);+ List<String> entries = new ArrayList<>();+ Matcher recordIds = RECORD_ID_PATTERN.matcher(new String(payload, StandardCharsets.UTF_8));+ for (int index = 0; recordIds.find(); index++) {+ entries.add(responder.respond(recordIds.group(1), index));+ }+ InvokeResult result = new InvokeResult();+ result.setStatusCode(200);+ result.setPayload(("{\"records\":[" + String.join(",", entries) + "]}")+ .getBytes(StandardCharsets.UTF_8));+ return result;+ });+ return payloads;+ }++ @FunctionalInterface+ public interface RecordResponder {+ String respond(String recordId, int index);+ }+}diff --git a/tools/docs/cfn_resource_types.yaml b/tools/docs/cfn_resource_types.yamlindex 79436858e9..65c81863bb 100644--- a/tools/docs/cfn_resource_types.yaml+++ b/tools/docs/cfn_resource_types.yaml@@ -214,6 +214,7 @@ notes:AWS::BedrockAgentCore::Runtime: 'Ref returns the AgentRuntimeId; CapacityProviderConfiguration, FilesystemConfigurations, LifecycleConfiguration, PlatformVersion and RequestHeaderConfiguration are accepted and not stored'AWS::BedrockAgentCore::RuntimeEndpoint: 'Ref returns the endpoint ARN; Tags are accepted and not stored'AWS::BedrockAgentCore::Memory: 'Ref returns the memory ARN; MemoryStrategies, IndexedKeys, NamespaceKeys and StreamDeliveryResources are accepted and not stored'+ AWS::KinesisFirehose::DeliveryStream: '`DeliveryStreamName`, `Tags` and, from an S3 destination, `BucketARN`, `RoleARN`, `Prefix`, `ErrorOutputPrefix`, `CompressionFormat`, `BufferingHints`, `CloudWatchLoggingOptions` and `ProcessingConfiguration` are applied; other properties are ignored, and a destination member the template drops on update keeps its stored value, as on AWS'# Trailing prose appended to a row after its type list, for behaviour that belongs to the service# rather than to one type.