Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudformation): forward Firehose processing and logging configuration (#6277)

Mario Toffiadeeb3b307588
diff --git a/docs/services/cloudformation.md b/docs/services/cloudformation.md
index cf52c879fa..6ae57a016a 100644
--- a/docs/services/cloudformation.md
+++ b/docs/services/cloudformation.md
@@ -211,7 +211,7 @@ cross-resource references.
| GuardDuty | `MalwareProtectionPlan` (management plane only: no scan runs, so the plan stays ACTIVE; Ref returns the plan id; a BucketName change is rejected) |
| Pipes | `Pipe` |
| Kinesis | `Stream` |
-| Kinesis Data Firehose | `DeliveryStream` |
+| Kinesis Data Firehose | `DeliveryStream` (`DeliveryStreamName`, `Tags` and, from an S3 destination, `BucketARN`, `RoleARN`, `Prefix`, `ErrorOutputPrefix`, `CompressionFormat`, `BufferingHints`, `CloudWatchLoggingOptions` and `ProcessingConfiguration` are applied; other properties are ignored, and a destination member the template drops on update keeps its stored value, as on AWS) |
| IoT Core | `Authorizer`, `DomainConfiguration` (`ServerCertificates` resolves to a JSON string), `Policy` (deleted after detaching it from its principals; on AWS the delete fails with `DeleteConflictException` while the policy is attached), `Thing`, `TopicRule` |
| CloudFront | `CachePolicy`, `Distribution`, `Function` (`KeyValueStoreAssociations` is ignored), `KeyGroup`, `OriginAccessControl`, `OriginRequestPolicy`, `PublicKey`, `ResponseHeadersPolicy` |
| CloudWatch | `Alarm`, `Dashboard` |
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index 4be6366b78..41912d7846 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -111,6 +111,10 @@ Two behaviors invert what the conversion block above does, both probed rather th
- Neither `NumberOfRetries` nor the existence of the function a `LambdaArn` names is checked at configuration time, as on AWS. At delivery an ARN naming nothing fails as `Lambda.FunctionError`, and a `NumberOfRetries` above 100 is capped, a bound AWS does not impose so that one flush cannot occupy the flusher indefinitely.
- A null entry in `Processors` or `Parameters`, which a raw JSON client can send, is ignored rather than reported; real AWS answers `InternalFailure` there, a fault of its own not worth reproducing.
+## Logging options
+
+`CloudWatchLoggingOptions` on the S3 and extended S3 destinations is stored and returned on both `S3DestinationDescription` and `ExtendedS3DestinationDescription`. A destination that never set it reads `{"Enabled": false}`, and an omitted `Enabled` reads `false`, as on AWS. `UpdateDestination` merges it a member at a time, so an update carrying only `{"Enabled": false}` keeps the stored log group and stream. `LogGroupName` and `LogStreamName` are validated against AWS's length and pattern constraints. With `Enabled` true both names are required, on the merged result for `UpdateDestination`, and a missing or empty one is refused with AWS's `InvalidArgumentException`, the group before the stream. Neither Floci nor AWS checks that the log group exists. Floci writes no delivery error logs to it.
+
## S3 object keys
Delivered objects are named `<evaluated prefix><streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid><file extension>`, matching [AWS's object name format](https://docs.aws.amazon.com/firehose/latest/dev/s3-object-name.html):
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/FirehoseCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/FirehoseCfnProvisioner.java
index 1a495f1712..d4cf0cf7ef 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/FirehoseCfnProvisioner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/FirehoseCfnProvisioner.java
@@ -1,8 +1,12 @@
package io.github.hectorvent.floci.services.cloudformation.provisioners;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.cloudformation.model.StackResource;
import io.github.hectorvent.floci.services.firehose.FirehoseService;
+import io.github.hectorvent.floci.services.firehose.S3DestinationValidator;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
import jakarta.enterprise.context.ApplicationScoped;
@@ -16,6 +20,7 @@ import java.util.Set;
@ApplicationScoped
public class FirehoseCfnProvisioner implements CfnResourceProvisioner {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
private static final int DELIVERY_STREAM_NAME_MAX_LENGTH = 64;
private static final int DEFAULT_BUFFER_SIZE_MB = 5;
private static final int DEFAULT_BUFFER_INTERVAL_SECONDS = 300;
@@ -37,7 +42,8 @@ public class FirehoseCfnProvisioner implements CfnResourceProvisioner {
r.getLogicalId(), DELIVERY_STREAM_NAME_MAX_LENGTH, false);
DeliveryStreamDescription.S3Destination s3 = null;
- JsonNode s3Node = props != null && props.has("ExtendedS3DestinationConfiguration")
+ boolean extended = props != null && props.has("ExtendedS3DestinationConfiguration");
+ JsonNode s3Node = extended
? props.get("ExtendedS3DestinationConfiguration")
: (props != null ? props.get("S3DestinationConfiguration") : null);
if (s3Node != null && !s3Node.isNull()) {
@@ -48,6 +54,15 @@ public class FirehoseCfnProvisioner implements CfnResourceProvisioner {
);
s3.setBucketArn(blankToNull(ctx.engine().resolve(s3Node.path("BucketARN"))));
s3.setPrefix(blankToNull(ctx.engine().resolve(s3Node.path("Prefix"))));
+ s3.setRoleArn(blankToNull(ctx.engine().resolve(s3Node.path("RoleARN"))));
+ s3.setErrorOutputPrefix(blankToNull(ctx.engine().resolve(s3Node.path("ErrorOutputPrefix"))));
+ s3.setCloudWatchLoggingOptions(structured(s3Node, "CloudWatchLoggingOptions",
+ DeliveryStreamDescription.CloudWatchLoggingOptions.class, ctx));
+ // Only the extended shape models a processor, so the legacy one carries none, as on the API path.
+ if (extended) {
+ s3.setProcessingConfiguration(structured(s3Node, "ProcessingConfiguration",
+ DeliveryStreamDescription.ProcessingConfiguration.class, ctx));
+ }
if (s3Node.has("BufferingHints")) {
JsonNode hints = s3Node.get("BufferingHints");
DeliveryStreamDescription.BufferingHints bufferingHints = new DeliveryStreamDescription.BufferingHints();
@@ -56,6 +71,8 @@ public class FirehoseCfnProvisioner implements CfnResourceProvisioner {
parseIntProp(hints, "IntervalInSeconds", ctx, DEFAULT_BUFFER_INTERVAL_SECONDS));
s3.setBufferingHints(bufferingHints);
}
+ S3DestinationValidator.validateWireShape(s3,
+ extended ? "extendedS3DestinationConfiguration" : "s3DestinationConfiguration");
}
List<DeliveryStreamDescription.Tag> tags = new ArrayList<>();
@@ -125,6 +142,25 @@ public class FirehoseCfnProvisioner implements CfnResourceProvisioner {
}
}
+ /**
+ * A structured destination member with its intrinsics resolved, read into the type
+ * CreateDeliveryStream reads it into. Null when the template leaves it out or AWS::NoValue
+ * drops it, which on an update keeps the stored value, as UpdateDestination does. A member
+ * present as anything but an object fails the resource, as CloudFormation rejects it.
+ */
+ private static <T> T structured(JsonNode destination, String name, Class<T> type, ProvisionContext ctx) {
+ JsonNode resolved = ctx.engine().resolveNodeOmittingNoValue(destination.get(name));
+ if (resolved == null || resolved.isMissingNode()) {
+ return null;
+ }
+ try {
+ return MAPPER.treeToValue(resolved, type);
+ } catch (JsonProcessingException e) {
+ throw new AwsException("ValidationError",
+ "AWS::KinesisFirehose::DeliveryStream has an invalid " + name + ": " + e.getOriginalMessage(), 400);
+ }
+ }
+
private static String blankToNull(String value) {
return value == null || value.isBlank() ? null : value;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 40dd293f21..d9e319047e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -406,8 +406,11 @@ public class FirehoseService implements ResourceProvider {
"Delivery stream " + name + " already exists.", 409);
}
+ validateLogging(null, s3Config);
validateBufferingHints(s3Config);
if (redshift != null) {
+ validateLogging(null, redshift.getS3());
+ validateLogging(null, redshift.getS3Backup());
validateBufferingHints(redshift.getS3());
validateBufferingHints(redshift.getS3Backup());
}
@@ -486,6 +489,7 @@ public class FirehoseService implements ResourceProvider {
throw new AwsException("InvalidArgumentException",
"The delivery stream " + name + " has a Redshift destination; use RedshiftDestinationUpdate.", 400);
}
+ validateLogging(destination.getExtendedS3DestinationDescription(), update);
validateBufferingHints(update);
S3Destination current = destination.getExtendedS3DestinationDescription();
if (current == null) {
@@ -533,6 +537,8 @@ public class FirehoseService implements ResourceProvider {
throw new AwsException("InvalidArgumentException",
"S3BackupMode is Enabled but no S3 backup configuration is set.", 400);
}
+ validateLogging(current.getS3(), update.getS3());
+ validateLogging(current.getS3Backup(), update.getS3Backup());
validateBufferingHints(update.getS3());
validateBufferingHints(update.getS3Backup());
if (update.getS3Backup() != null && current.getS3Backup() == null) {
@@ -665,6 +671,29 @@ public class FirehoseService implements ResourceProvider {
}
}
+ /**
+ * Once logging is on, both names are required, checked on the merged result for an update
+ * (probed 2026-10-23). An update without a logging block leaves the stored one as it was.
+ */
+ private static void validateLogging(S3Destination stored, S3Destination update) {
+ if (update == null || update.getCloudWatchLoggingOptions() == null) {
+ return;
+ }
+ DeliveryStreamDescription.CloudWatchLoggingOptions effective = mergeLogging(
+ stored == null ? null : stored.getCloudWatchLoggingOptions(), update.getCloudWatchLoggingOptions());
+ if (!Boolean.TRUE.equals(effective.getEnabled())) {
+ return;
+ }
+ if (effective.getLogGroupName() == null || effective.getLogGroupName().isEmpty()) {
+ throw new AwsException("InvalidArgumentException",
+ "CloudWatch Log Group Name is required if CloudWatch Logging is enabled", 400);
+ }
+ if (effective.getLogStreamName() == null || effective.getLogStreamName().isEmpty()) {
+ throw new AwsException("InvalidArgumentException",
+ "CloudWatch Log Stream Name is required if CloudWatch Logging is enabled", 400);
+ }
+ }
+
private static void mergeDestination(S3Destination current, S3Destination update) {
if (update.getRoleArn() != null) current.setRoleArn(update.getRoleArn());
if (update.getBucketArn() != null) current.setBucketArn(update.getBucketArn());
@@ -676,6 +705,10 @@ public class FirehoseService implements ResourceProvider {
if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());
if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());
if (update.getS3BackupMode() != null) current.setS3BackupMode(update.getS3BackupMode());
+ if (update.getCloudWatchLoggingOptions() != null) {
+ current.setCloudWatchLoggingOptions(mergeLogging(
+ current.getCloudWatchLoggingOptions(), update.getCloudWatchLoggingOptions()));
+ }
// Replaced whole, not merged member-wise: an update carrying only
// {"Enabled": false} leaves Processors empty on real AWS, where the same shape
// preserves the conversion block's members (probed 2026-09-10).
@@ -714,6 +747,23 @@ public class FirehoseService implements ResourceProvider {
return merged;
}
+ /**
+ * Member-wise, like the conversion block: an update carrying only {"Enabled": false} keeps the
+ * stored group and stream, and one naming only a group keeps Enabled (probed 2026-10-22).
+ */
+ private static DeliveryStreamDescription.CloudWatchLoggingOptions mergeLogging(
+ DeliveryStreamDescription.CloudWatchLoggingOptions current,
+ DeliveryStreamDescription.CloudWatchLoggingOptions update) {
+ if (current == null) {
+ return update;
+ }
+ DeliveryStreamDescription.CloudWatchLoggingOptions merged = new DeliveryStreamDescription.CloudWatchLoggingOptions();
+ merged.setEnabled(update.getEnabled() != null ? update.getEnabled() : current.getEnabled());
+ merged.setLogGroupName(update.getLogGroupName() != null ? update.getLogGroupName() : current.getLogGroupName());
+ merged.setLogStreamName(update.getLogStreamName() != null ? update.getLogStreamName() : current.getLogStreamName());
+ return merged;
+ }
+
/**
* SchemaConfiguration merges a member at a time as well, so an update naming only
* a new TableName keeps the stored role, database and region (probed). Replacing
@@ -827,9 +877,7 @@ public class FirehoseService implements ResourceProvider {
.orElseThrow(() -> new AwsException("ResourceNotFoundException",
"Delivery stream not found: " + streamName, 400));
// Normalizes streams persisted before required output members existed.
- if (stream.s3Destination() != null) {
- stream.s3Destination().applyDefaults();
- }
+ stream.applyDefaults();
return stream;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
index 35bb5b9a81..cbee968595 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.firehose;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.SmithyValidationErrors;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.CloudWatchLoggingOptions;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;
@@ -37,7 +38,7 @@ import java.util.regex.Pattern;
* Smithy layer does (probe 2026-09-07: both ParquetSerDe bounds violated at
* once report "2 validation errors detected: ...; ...").
*/
-final class S3DestinationValidator {
+public final class S3DestinationValidator {
/** Listed in the order AWS prints it, which is neither declaration order nor alphabetical. */
private static final String ENUM_VALUE_SET = "[ZIP, HADOOP_SNAPPY, Snappy, GZIP, UNCOMPRESSED]";
@@ -76,9 +77,15 @@ final class S3DestinationValidator {
private static final String NOT_BLANK_PATTERN = "^(?!\\s*$).+";
private static final Pattern NOT_BLANK = Pattern.compile(NOT_BLANK_PATTERN);
+ private static final int LOG_NAME_MAX_LENGTH = 512;
+ private static final String LOG_GROUP_NAME_REGEX = "[\\.\\-_/#A-Za-z0-9]*";
+ private static final Pattern LOG_GROUP_NAME = Pattern.compile(LOG_GROUP_NAME_REGEX);
+ private static final String LOG_STREAM_NAME_REGEX = "[^:*]*";
+ private static final Pattern LOG_STREAM_NAME = Pattern.compile(LOG_STREAM_NAME_REGEX);
+
private S3DestinationValidator() {}
- static void validateWireShape(S3Destination config, String shapeName) {
+ public static void validateWireShape(S3Destination config, String shapeName) {
if (config == null) {
return;
}
@@ -110,12 +117,45 @@ final class S3DestinationValidator {
// violations sit in more than one of them.
collectOpenXMappingViolations(config, shapeName, violations);
collectSchemaViolations(config, shapeName, violations);
+ collectLoggingViolations(config, shapeName, violations);
collectProcessingViolations(config, shapeName, violations);
if (!violations.isEmpty()) {
throw constraintViolations(violations);
}
}
+ /**
+ * Both log names are bounded at 512 characters and carry a pattern. Unlike
+ * FileExtension, an overlong name with a bad character reports both rules, the
+ * length one first, and the group's violations precede the stream's (probed
+ * 2026-10-22, where AWS also printed them before any processingConfiguration one).
+ */
+ private static void collectLoggingViolations(S3Destination config, String shapeName,
+ List<String> violations) {
+ CloudWatchLoggingOptions logging = config.getCloudWatchLoggingOptions();
+ if (logging == null) {
+ return;
+ }
+ collectLogName(logging.getLogGroupName(), shapeName, "cloudWatchLoggingOptions.logGroupName",
+ LOG_GROUP_NAME, LOG_GROUP_NAME_REGEX, violations);
+ collectLogName(logging.getLogStreamName(), shapeName, "cloudWatchLoggingOptions.logStreamName",
+ LOG_STREAM_NAME, LOG_STREAM_NAME_REGEX, violations);
+ }
+
+ private static void collectLogName(String value, String shapeName, String member, Pattern pattern,
+ String regex, List<String> violations) {
+ if (value == null) {
+ return;
+ }
+ if (value.length() > LOG_NAME_MAX_LENGTH) {
+ violations.add(violation(shapeName, member,
+ "Member must have length less than or equal to " + LOG_NAME_MAX_LENGTH));
+ }
+ if (!pattern.matcher(value).matches()) {
+ violations.add(violation(shapeName, member, "Member must satisfy regular expression pattern: " + regex));
+ }
+ }
+
/**
* Every SchemaConfiguration string is a 1 to 1024 character value without
* whitespace, and unlike the collection members each broken rule is its own
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
index 9b71a9c4b4..810ced01e9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
@@ -66,13 +66,8 @@ public class DeliveryStreamDescription {
this.deliveryStreamARN = arn;
this.deliveryStreamStatus = DeliveryStreamStatus.ACTIVE;
this.createTimestamp = Instant.now();
- if (s3 != null) {
- s3.applyDefaults();
- }
- if (redshift != null && redshift.getS3() != null) {
- redshift.getS3().applyDefaults();
- }
this.destinations = List.of(redshift != null ? new Destination(redshift) : new Destination(s3));
+ applyDefaults();
this.deliveryStreamEncryptionConfiguration = DeliveryStreamEncryptionConfiguration.disabled();
if (kinesisStreamSource != null) {
this.source = new Source(kinesisStreamSource);
@@ -170,6 +165,24 @@ public class DeliveryStreamDescription {
return first.getExtendedS3DestinationDescription();
}
+ /**
+ * Fills the AWS defaults on every S3 configuration the stream describes: its S3 destination, or
+ * a Redshift destination's intermediate and backup configurations.
+ */
+ public final void applyDefaults() {
+ S3Destination s3 = s3Destination();
+ if (s3 != null) {
+ s3.applyDefaults();
+ }
+ if (destinations == null || destinations.isEmpty()) {
+ return;
+ }
+ RedshiftDestination redshift = destinations.get(0).getRedshiftDestinationDescription();
+ if (redshift != null && redshift.getS3Backup() != null) {
+ redshift.getS3Backup().applyDefaults();
+ }
+ }
+
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(JsonInclude.Include.NON_NULL)
@@ -247,6 +260,8 @@ public class DeliveryStreamDescription {
private DataFormatConversionConfiguration dataFormatConversionConfiguration;
@JsonProperty("ProcessingConfiguration")
private ProcessingConfiguration processingConfiguration;
+ @JsonProperty("CloudWatchLoggingOptions")
+ private CloudWatchLoggingOptions cloudWatchLoggingOptions;
public S3Destination() {}
public String getRoleArn() { return roleArn; }
@@ -282,6 +297,8 @@ public class DeliveryStreamDescription {
public void setProcessingConfiguration(ProcessingConfiguration processingConfiguration) {
this.processingConfiguration = processingConfiguration;
}
+ public CloudWatchLoggingOptions getCloudWatchLoggingOptions() { return cloudWatchLoggingOptions; }
+ public void setCloudWatchLoggingOptions(CloudWatchLoggingOptions options) { this.cloudWatchLoggingOptions = options; }
/**
* True when a transformation applies to deliveries: the configuration is present
@@ -356,6 +373,14 @@ public class DeliveryStreamDescription {
if (processingConfiguration != null && processingConfiguration.getEnabled() == null) {
processingConfiguration.setEnabled(false);
}
+ // AWS describes a destination that never configured logging as {"Enabled": false},
+ // and an omitted Enabled as false (probed 2026-10-22).
+ if (cloudWatchLoggingOptions == null) {
+ cloudWatchLoggingOptions = new CloudWatchLoggingOptions();
+ }
+ if (cloudWatchLoggingOptions.getEnabled() == null) {
+ cloudWatchLoggingOptions.setEnabled(false);
+ }
}
/**
@@ -436,6 +461,7 @@ public class DeliveryStreamDescription {
view.compressionFormat = compressionFormat;
view.bufferingHints = bufferingHints;
view.encryptionConfiguration = encryptionConfiguration;
+ view.cloudWatchLoggingOptions = cloudWatchLoggingOptions;
return view;
}
@@ -469,6 +495,31 @@ public class DeliveryStreamDescription {
public void setProcessors(List<Processor> processors) { this.processors = processors; }
}
+ /**
+ * Stored and echoed as AWS models it. Floci writes no delivery error logs to the named
+ * group. UpdateDestination merges it a member at a time (probed).
+ */
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class CloudWatchLoggingOptions {
+ @JsonProperty("Enabled")
+ private Boolean enabled;
+ @JsonProperty("LogGroupName")
+ private String logGroupName;
+ @JsonProperty("LogStreamName")
+ private String logStreamName;
+
+ public CloudWatchLoggingOptions() {}
+
+ public Boolean getEnabled() { return enabled; }
+ public void setEnabled(Boolean enabled) { this.enabled = enabled; }
+ public String getLogGroupName() { return logGroupName; }
+ public void setLogGroupName(String logGroupName) { this.logGroupName = logGroupName; }
+ public String getLogStreamName() { return logStreamName; }
+ public void setLogStreamName(String logStreamName) { this.logStreamName = logStreamName; }
+ }
+
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(JsonInclude.Include.NON_NULL)
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationFirehoseIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationFirehoseIntegrationTest.java
index d2676808b0..34110df7a0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationFirehoseIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationFirehoseIntegrationTest.java
@@ -1,18 +1,37 @@
package io.github.hectorvent.floci.services.cloudformation;
+import io.github.hectorvent.floci.services.firehose.FirehoseService;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.testing.CfnStackWaits;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.InjectMock;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.RestAssured;
import io.restassured.config.EncoderConfig;
import io.restassured.http.ContentType;
+import io.restassured.response.ValidatableResponse;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import java.util.List;
+
+import static io.github.hectorvent.floci.testing.FirehoseRequests.answerTransforms;
+import static io.github.hectorvent.floci.testing.FirehoseRequests.encode;
+import static io.github.hectorvent.floci.testing.FirehoseRequests.firstKey;
+import static io.github.hectorvent.floci.testing.FirehoseRequests.putRecordBatch;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
/**
* End-to-end check that CloudFormation provisions an AWS::KinesisFirehose::DeliveryStream for real
* (into FirehoseService) rather than stubbing it. Delivery streams are metadata (no container), so
- * the test is Docker-free.
+ * the test is Docker-free; the Lambda processor is mocked for the same reason.
*/
@QuarkusTest
class CloudFormationFirehoseIntegrationTest {
@@ -21,6 +40,22 @@ class CloudFormationFirehoseIntegrationTest {
"AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/cloudformation/aws4_request";
private static final String FIREHOSE_AUTH =
"AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/firehose/aws4_request";
+ private static final String FIREHOSE_CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String EXTENDED_DESTINATION =
+ "DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.";
+ private static final String LAMBDA_ARN = EXTENDED_DESTINATION
+ + "ProcessingConfiguration.Processors[0].Parameters.find { it.ParameterName == 'LambdaArn' }.ParameterValue";
+
+ @InjectMock
+ LambdaService lambdaService;
+
+ @Inject
+ FirehoseService firehoseService;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
@Test
void createStackProvisionsDeliveryStreamVisibleToFirehose() {
@@ -90,4 +125,122 @@ class CloudFormationFirehoseIntegrationTest {
.body(containsString(streamName))
.body(containsString("GZIP"));
}
+
+ @Test
+ void aStackStreamRunsItsLambdaProcessorAndReadsBackItsLogging() {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String streamName = "cfn-firehose-transform-" + suffix;
+ String stackName = "cfn-firehose-transform-stack-" + suffix;
+ String bucket = "cfn-firehose-transform-" + suffix;
+ String logGroup = "/aws/kinesisfirehose/" + suffix;
+ String firstFunction = "transform-" + suffix;
+ String secondFunction = "transform-next-" + suffix;
+ String firstArn = "arn:aws:lambda:us-east-1:000000000000:function:" + firstFunction;
+ String secondArn = "arn:aws:lambda:us-east-1:000000000000:function:" + secondFunction;
+
+ stackRequest("CreateStack", stackName, transformTemplate(suffix, streamName, bucket, firstFunction));
+ CfnStackWaits.awaitStatus(stackName, "CREATE_COMPLETE");
+
+ describeDeliveryStream(streamName)
+ .body(EXTENDED_DESTINATION + "RoleARN", equalTo("arn:aws:iam::000000000000:role/firehose-delivery"))
+ .body(EXTENDED_DESTINATION + "ErrorOutputPrefix", equalTo("errors/!{firehose:error-output-type}/"))
+ .body(EXTENDED_DESTINATION + "CloudWatchLoggingOptions.Enabled", equalTo(true))
+ .body(EXTENDED_DESTINATION + "CloudWatchLoggingOptions.LogGroupName", equalTo(logGroup))
+ .body(EXTENDED_DESTINATION + "CloudWatchLoggingOptions.LogStreamName", equalTo("DestinationDelivery"))
+ .body(EXTENDED_DESTINATION + "ProcessingConfiguration.Enabled", equalTo(true))
+ .body(LAMBDA_ARN, equalTo(firstArn))
+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription"
+ + ".CloudWatchLoggingOptions.LogGroupName", equalTo(logGroup));
+
+ List<byte[]> payloads = answerTransforms(lambdaService, (recordId, index) ->
+ "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode("TRANSFORMED-" + index) + "\"}");
+ putRecordBatch(streamName, "one", "two");
+ firehoseService.flush(streamName);
+
+ given()
+ .when()
+ .get("/" + bucket + "/" + firstKey(bucket, "data/"))
+ .then()
+ .statusCode(200)
+ .body(equalTo("TRANSFORMED-0\nTRANSFORMED-1\n"));
+ assertEquals(1, payloads.size());
+ verify(lambdaService).invokeArn(eq(firstArn), any(), any());
+
+ stackRequest("UpdateStack", stackName, transformTemplate(suffix, streamName, bucket, secondFunction));
+ CfnStackWaits.awaitStatus(stackName, "UPDATE_COMPLETE");
+
+ describeDeliveryStream(streamName)
+ .body(LAMBDA_ARN, equalTo(secondArn))
+ .body(EXTENDED_DESTINATION + "CloudWatchLoggingOptions.LogGroupName", equalTo(logGroup));
+ }
+
+ private static String transformTemplate(String suffix, String streamName, String bucket, String function) {
+ return """
+ {
+ "Resources": {
+ "Logs": {
+ "Type": "AWS::Logs::LogGroup",
+ "Properties": {"LogGroupName": "/aws/kinesisfirehose/%s"}
+ },
+ "Stream": {
+ "Type": "AWS::KinesisFirehose::DeliveryStream",
+ "Properties": {
+ "DeliveryStreamName": "%s",
+ "DeliveryStreamType": "DirectPut",
+ "ExtendedS3DestinationConfiguration": {
+ "BucketARN": "arn:aws:s3:::%s",
+ "RoleARN": {"Fn::Sub": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/firehose-delivery"},
+ "Prefix": "data/",
+ "ErrorOutputPrefix": {"Fn::Join": ["", ["errors/", "!{firehose:error-output-type}/"]]},
+ "CloudWatchLoggingOptions": {
+ "Enabled": true,
+ "LogGroupName": {"Ref": "Logs"},
+ "LogStreamName": "DestinationDelivery"
+ },
+ "ProcessingConfiguration": {
+ "Enabled": true,
+ "Processors": [
+ {
+ "Type": "Lambda",
+ "Parameters": [
+ {
+ "ParameterName": "LambdaArn",
+ "ParameterValue": {"Fn::Sub": "arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:%s"}
+ }
+ ]
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ """.formatted(suffix, streamName, bucket, function);
+ }
+
+ private static void stackRequest(String action, String stackName, String template) {
+ given()
+ .contentType("application/x-www-form-urlencoded")
+ .header("Authorization", CFN_AUTH)
+ .formParam("Action", action)
+ .formParam("StackName", stackName)
+ .formParam("TemplateBody", template)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ private static ValidatableResponse describeDeliveryStream(String streamName) {
+ return given()
+ .header("Authorization", FIREHOSE_AUTH)
+ .header("X-Amz-Target", "Firehose_20150804.DescribeDeliveryStream")
+ .contentType(FIREHOSE_CONTENT_TYPE)
+ .body("{\"DeliveryStreamName\":\"" + streamName + "\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java
index 07d0728e68..89418b02b2 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LeafCfnProvisionerTest.java
@@ -57,6 +57,7 @@ class LeafCfnProvisionerTest {
return node == null || node.isMissingNode() || node.isNull() ? null : node.asText();
});
when(engine.resolveNode(any())).thenAnswer(i -> i.getArgument(0));
+ when(engine.resolveNodeOmittingNoValue(any())).thenAnswer(i -> i.getArgument(0));
ctx = new ProvisionContext(engine, REGION, "000000000000", "my-stack");
}
@@ -360,6 +361,114 @@ class LeafCfnProvisionerTest {
assertNull(s3.getValue().getPrefix());
}
+ @Test
+ void processingLoggingRoleAndErrorPrefixReachTheStream() {
+ ArgumentCaptor<DeliveryStreamDescription.S3Destination> s3 =
+ ArgumentCaptor.forClass(DeliveryStreamDescription.S3Destination.class);
+ when(firehose.createDeliveryStream(anyString(), s3.capture(), any())).thenReturn("arn");
+
+ provisioner.provision(resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""
+ {
+ "DeliveryStreamName": "s",
+ "ExtendedS3DestinationConfiguration": {
+ "BucketARN": "arn:bucket",
+ "RoleARN": "arn:aws:iam::000000000000:role/delivery",
+ "ErrorOutputPrefix": "errors/!{firehose:error-output-type}/",
+ "CloudWatchLoggingOptions": {"Enabled": true, "LogGroupName": "g", "LogStreamName": "st"},
+ "ProcessingConfiguration": {
+ "Enabled": true,
+ "Processors": [{"Type": "Lambda", "Parameters": [
+ {"ParameterName": "LambdaArn", "ParameterValue": "arn:aws:lambda:us-east-1:000000000000:function:f"}
+ ]}]
+ }
+ }
+ }
+ """), ctx);
+
+ DeliveryStreamDescription.S3Destination destination = s3.getValue();
+ assertEquals("arn:aws:iam::000000000000:role/delivery", destination.getRoleArn());
+ assertEquals("errors/!{firehose:error-output-type}/", destination.getErrorOutputPrefix());
+ assertEquals(Boolean.TRUE, destination.getCloudWatchLoggingOptions().getEnabled());
+ assertEquals("g", destination.getCloudWatchLoggingOptions().getLogGroupName());
+ assertEquals("st", destination.getCloudWatchLoggingOptions().getLogStreamName());
+ assertTrue(destination.isProcessingEnabled());
+ assertEquals("arn:aws:lambda:us-east-1:000000000000:function:f", destination.getProcessingConfiguration()
+ .getProcessors().get(0).getParameters().get(0).getParameterValue());
+ }
+
+ @Test
+ void theLegacyDestinationCarriesLoggingButNoProcessor() {
+ ArgumentCaptor<DeliveryStreamDescription.S3Destination> s3 =
+ ArgumentCaptor.forClass(DeliveryStreamDescription.S3Destination.class);
+ when(firehose.createDeliveryStream(anyString(), s3.capture(), any())).thenReturn("arn");
+
+ provisioner.provision(resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""
+ {
+ "DeliveryStreamName": "s",
+ "S3DestinationConfiguration": {
+ "BucketARN": "arn:bucket",
+ "CloudWatchLoggingOptions": {"LogGroupName": "g"},
+ "ProcessingConfiguration": {"Enabled": true, "Processors": []}
+ }
+ }
+ """), ctx);
+
+ assertEquals("g", s3.getValue().getCloudWatchLoggingOptions().getLogGroupName());
+ assertNull(s3.getValue().getProcessingConfiguration(),
+ "S3DestinationConfiguration models no processor, as on the API path");
+ }
+
+ @Test
+ void anUnreadableLoggingBlockFailsTheResource() {
+ AwsException error = assertThrows(AwsException.class, () -> provisioner.provision(
+ resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""
+ {
+ "DeliveryStreamName": "s",
+ "ExtendedS3DestinationConfiguration": {
+ "BucketARN": "arn:bucket",
+ "CloudWatchLoggingOptions": {"Enabled": {"not": "a boolean"}}
+ }
+ }
+ """), ctx));
+ assertEquals("ValidationError", error.getErrorCode());
+ }
+
+ @Test
+ void aBlockThatIsNotAnObjectFailsTheResource() {
+ AwsException error = assertThrows(AwsException.class, () -> provisioner.provision(
+ resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""
+ {
+ "DeliveryStreamName": "s",
+ "ExtendedS3DestinationConfiguration": {
+ "BucketARN": "arn:bucket",
+ "ProcessingConfiguration": []
+ }
+ }
+ """), ctx));
+ assertEquals("ValidationError", error.getErrorCode());
+ }
+
+ @Test
+ void anUnknownProcessorTypeFailsTheResource() {
+ AwsException error = assertThrows(AwsException.class, () -> provisioner.provision(
+ resource("Stream", "AWS::KinesisFirehose::DeliveryStream"), props("""
+ {
+ "DeliveryStreamName": "s",
+ "ExtendedS3DestinationConfiguration": {
+ "BucketARN": "arn:bucket",
+ "ProcessingConfiguration": {
+ "Enabled": true,
+ "Processors": [{"Type": "Lambd", "Parameters": [
+ {"ParameterName": "LambdaArn", "ParameterValue": "arn:aws:lambda:us-east-1:000000000000:function:f"}
+ ]}]
+ }
+ }
+ }
+ """), ctx));
+ assertEquals("ValidationException", error.getErrorCode());
+ verify(firehose, never()).createDeliveryStream(anyString(), any(), any());
+ }
+
@Test
void deleteReachesTheService() {
provisioner.delete("AWS::KinesisFirehose::DeliveryStream", "s", REGION);
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
index 1d080a0cb4..01a72c04f6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
@@ -54,6 +54,11 @@ class FirehoseExtendedS3IntegrationTest {
"Prefix": "events/data/",
"ErrorOutputPrefix": "events/errors/",
"CompressionFormat": "GZIP",
+ "CloudWatchLoggingOptions": {
+ "Enabled": true,
+ "LogGroupName": "/aws/kinesisfirehose/extended",
+ "LogStreamName": "DestinationDelivery"
+ },
"BufferingHints": { "SizeInMBs": 64, "IntervalInSeconds": 120 }
}
}
@@ -88,6 +93,10 @@ class FirehoseExtendedS3IntegrationTest {
.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(64))
.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(120))
.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig", equalTo("NoEncryption"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CloudWatchLoggingOptions.Enabled", equalTo(true))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CloudWatchLoggingOptions.LogGroupName", equalTo("/aws/kinesisfirehose/extended"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CloudWatchLoggingOptions.LogStreamName", equalTo("DestinationDelivery"))
+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.CloudWatchLoggingOptions.LogGroupName", equalTo("/aws/kinesisfirehose/extended"))
.body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.BucketARN", equalTo(BUCKET_ARN))
.body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.RoleARN", equalTo(ROLE_ARN));
}
@@ -124,6 +133,8 @@ class FirehoseExtendedS3IntegrationTest {
.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.SizeInMBs", equalTo(5))
.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.BufferingHints.IntervalInSeconds", equalTo(300))
.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.EncryptionConfiguration.NoEncryptionConfig", equalTo("NoEncryption"))
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CloudWatchLoggingOptions.Enabled", equalTo(false))
+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription.CloudWatchLoggingOptions.Enabled", equalTo(false))
// #2200: a create response that omits S3BackupMode, followed by a later describe
// that fills it in, applied cleanly and then re-planned dirty forever in Terraform.
.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.S3BackupMode", equalTo("Disabled"));
@@ -435,6 +446,30 @@ class FirehoseExtendedS3IntegrationTest {
assertTrue(key.matches("events/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
}
+ @Test
+ @Order(17)
+ void createDeliveryStreamRefusesEnabledLoggingWithoutAStreamName() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "logging-without-stream-name",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": "g" }
+ }
+ }
+ """.formatted(ROLE_ARN, BUCKET_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidArgumentException"))
+ .body("message", equalTo("CloudWatch Log Stream Name is required if CloudWatch Logging is enabled"));
+ }
+
private void createDeliveryStream(String streamName, String bucket, String extraDestinationJson) {
given()
.contentType(CONTENT_TYPE)
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java
index 28ad6b293d..fc96caa867 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java
@@ -1,7 +1,6 @@
package io.github.hectorvent.floci.services.firehose;
import io.github.hectorvent.floci.services.lambda.LambdaService;
-import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.InjectMock;
import io.quarkus.test.junit.QuarkusTest;
@@ -9,24 +8,21 @@ import io.quarkus.test.junit.QuarkusTestProfile;
import io.quarkus.test.junit.TestProfile;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Base64;
import java.util.List;
import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import static io.github.hectorvent.floci.testing.FirehoseRequests.answerTransforms;
+import static io.github.hectorvent.floci.testing.FirehoseRequests.encode;
+import static io.github.hectorvent.floci.testing.FirehoseRequests.firstKey;
+import static io.github.hectorvent.floci.testing.FirehoseRequests.putRecordBatch;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
/**
* End-to-end coverage of the Lambda transform on the real delivery path: what
@@ -54,7 +50,6 @@ class FirehoseLambdaTransformIntegrationTest {
private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
private static final String TARGET_PREFIX = "Firehose_20150804.";
private static final String FUNCTION_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";
- private static final Pattern KEY_PATTERN = Pattern.compile("<Key>([^<]+)</Key>");
@InjectMock
LambdaService lambdaService;
@@ -69,8 +64,8 @@ class FirehoseLambdaTransformIntegrationTest {
String stream = "transform-ok-stream";
String bucket = "transform-ok-archive";
createDeliveryStream(stream, bucket);
- respondWith((recordId, index) -> "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\""
- + encode("TRANSFORMED-" + index) + "\"}");
+ answerTransforms(lambdaService, (recordId, index) ->
+ "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode("TRANSFORMED-" + index) + "\"}");
putRecordBatch(stream, "one", "two");
@@ -87,7 +82,7 @@ class FirehoseLambdaTransformIntegrationTest {
String stream = "transform-failed-stream";
String bucket = "transform-failed-archive";
createDeliveryStream(stream, bucket);
- respondWith((recordId, index) -> index == 0
+ answerTransforms(lambdaService, (recordId, index) -> index == 0
? "{\"recordId\":\"" + recordId + "\",\"result\":\"ProcessingFailed\"}"
: "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode("kept") + "\"}");
@@ -116,7 +111,8 @@ class FirehoseLambdaTransformIntegrationTest {
String stream = "transform-dropped-stream";
String bucket = "transform-dropped-archive";
createDeliveryStream(stream, bucket);
- respondWith((recordId, index) -> "{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");
+ answerTransforms(lambdaService, (recordId, index) ->
+ "{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");
putRecordBatch(stream, "one", "two");
@@ -126,7 +122,7 @@ class FirehoseLambdaTransformIntegrationTest {
.then()
.statusCode(200)
.extract().asString();
- assertFalse(KEY_PATTERN.matcher(listing).find(), "expected no delivered object, got: " + listing);
+ assertFalse(listing.contains("<Key>"), "expected no delivered object, got: " + listing);
}
@Test
@@ -134,7 +130,7 @@ class FirehoseLambdaTransformIntegrationTest {
String stream = "transform-payload-stream";
String bucket = "transform-payload-archive";
createDeliveryStream(stream, bucket);
- List<byte[]> payloads = respondWith((recordId, index) ->
+ List<byte[]> payloads = answerTransforms(lambdaService, (recordId, index) ->
"{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");
putRecordBatch(stream, "one", "two");
@@ -146,39 +142,6 @@ class FirehoseLambdaTransformIntegrationTest {
assertTrue(payload.contains("\"data\":\"" + encode("two") + "\""), payload);
}
- /**
- * Answers each invocation from the record ids it actually carries, which the caller
- * cannot name: they are minted per batch. Returns the payloads the mock received.
- */
- private List<byte[]> respondWith(RecordResponder responder) {
- List<byte[]> payloads = new ArrayList<>();
- Mockito.when(lambdaService.invokeArn(anyString(), any(), any())).thenAnswer(invocation -> {
- byte[] payload = invocation.getArgument(1);
- payloads.add(payload);
- List<String> entries = new ArrayList<>();
- Matcher recordIds = Pattern.compile("\"recordId\":\"([^\"]+)\"")
- .matcher(new String(payload, StandardCharsets.UTF_8));
- for (int index = 0; recordIds.find(); index++) {
- entries.add(responder.respond(recordIds.group(1), index));
- }
- InvokeResult result = new InvokeResult();
- result.setStatusCode(200);
- result.setPayload(("{\"records\":[" + String.join(",", entries) + "]}")
- .getBytes(StandardCharsets.UTF_8));
- return result;
- });
- return payloads;
- }
-
- @FunctionalInterface
- private interface RecordResponder {
- String respond(String recordId, int index);
- }
-
- private static String encode(String body) {
- return Base64.getEncoder().encodeToString(body.getBytes(StandardCharsets.UTF_8));
- }
-
private void createDeliveryStream(String streamName, String bucket) {
given()
.contentType(CONTENT_TYPE)
@@ -210,37 +173,4 @@ class FirehoseLambdaTransformIntegrationTest {
.statusCode(200)
.body("DeliveryStreamARN", notNullValue());
}
-
- private void putRecordBatch(String streamName, String... bodies) {
- List<String> records = new ArrayList<>();
- for (String body : bodies) {
- records.add("{\"Data\": \"" + encode(body) + "\"}");
- }
- given()
- .contentType(CONTENT_TYPE)
- .header("X-Amz-Target", TARGET_PREFIX + "PutRecordBatch")
- .body("{ \"DeliveryStreamName\": \"" + streamName + "\", \"Records\": ["
- + String.join(",", records) + "] }")
- .when()
- .post("/")
- .then()
- .statusCode(200)
- .body("FailedPutCount", equalTo(0));
- }
-
- private String firstKey(String bucket, String prefix) {
- String listing = given()
- .when()
- .get("/" + bucket)
- .then()
- .statusCode(200)
- .extract().asString();
- Matcher key = KEY_PATTERN.matcher(listing);
- while (key.find()) {
- if (key.group(1).startsWith(prefix)) {
- return key.group(1);
- }
- }
- throw new AssertionError("expected an object under " + prefix + " in " + bucket + ", got: " + listing);
- }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java
index ee08e0b71a..fc6a46a826 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java
@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import io.github.hectorvent.floci.services.firehose.model.Record;
import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
@@ -84,8 +85,8 @@ class FirehoseRedshiftServiceTest {
mock(FirehoseLambdaTransformer.class), delivery, Runnable::run);
}
- private void createStream(String extraMembers) throws Exception {
- RedshiftDestination destination = mapper.readValue("""
+ private RedshiftDestination destination(String extraMembers) throws Exception {
+ return mapper.readValue("""
{"RoleARN": "arn:aws:iam::000000000000:role/firehose",
"ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",
"Username": "admin", "Password": "Secret123",
@@ -93,10 +94,18 @@ class FirehoseRedshiftServiceTest {
"S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}
%s}
""".formatted(extraMembers), RedshiftDestination.class);
- service.createDeliveryStream("us-east-1", STREAM, null, destination, List.of(), null, null);
+ }
+
+ private void createStream(String extraMembers) throws Exception {
+ service.createDeliveryStream("us-east-1", STREAM, null, destination(extraMembers), List.of(), null, null);
service.putRecord(STREAM, new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));
}
+ private S3Destination describedBackup(String streamName) {
+ return service.describeDeliveryStream(streamName).getDestinations().get(0)
+ .getRedshiftDestinationDescription().getS3Backup();
+ }
+
private void copyFails() {
doThrow(new IllegalStateException("relation does not exist"))
.when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());
@@ -169,6 +178,25 @@ class FirehoseRedshiftServiceTest {
verify(delivery, times(1)).copy(anyString(), anyString(), any(), eq("stage"), anyString());
}
+ @Test
+ void aRedshiftBackupGetsTheLoggingDefaultsOnCreate() throws Exception {
+ createStream(BACKUP_ENABLED);
+ service.createDeliveryStream("us-east-1", "rs-logging", null, destination(
+ ", \"S3BackupMode\": \"Enabled\", \"S3BackupConfiguration\": "
+ + "{\"RoleARN\": \"arn:aws:iam::000000000000:role/firehose\", \"BucketARN\": \"arn:aws:s3:::bk\","
+ + " \"CloudWatchLoggingOptions\": {\"LogGroupName\": \"g\"}}"),
+ List.of(), null, null);
+
+ S3Destination omitted = describedBackup(STREAM);
+ assertEquals(false, omitted.getCloudWatchLoggingOptions().getEnabled());
+ assertEquals("UNCOMPRESSED", omitted.getCompressionFormat());
+
+ S3Destination withoutEnabled = describedBackup("rs-logging");
+ assertEquals(false, withoutEnabled.getCloudWatchLoggingOptions().getEnabled());
+ assertEquals("g", withoutEnabled.getCloudWatchLoggingOptions().getLogGroupName());
+ assertEquals("UNCOMPRESSED", withoutEnabled.getCompressionFormat());
+ }
+
@Test
void backupObjectUsesTheBackupConfigurationPrefix() throws Exception {
createStream(", \"S3BackupMode\": \"Enabled\", \"S3BackupConfiguration\": "
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
index 2d2b8f2653..2ad60fbe44 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import io.github.hectorvent.floci.services.firehose.model.Record;
+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.PutObjectOptions;
@@ -235,6 +236,34 @@ class FirehoseServiceTest {
"legacy entries from another region must not be adopted");
}
+ @Test
+ @SuppressWarnings("unchecked")
+ void aPersistedRedshiftBackupGetsTheDefaultsOnDescribe() {
+ S3Destination intermediate = new S3Destination();
+ intermediate.setBucketArn("arn:aws:s3:::stage");
+ S3Destination backup = new S3Destination();
+ backup.setBucketArn("arn:aws:s3:::bk");
+ RedshiftDestination redshift = new RedshiftDestination();
+ redshift.setS3BackupMode("Enabled");
+ redshift.setS3(intermediate);
+ redshift.setS3Backup(backup);
+ // Built the way Jackson reads a stored stream: the no-arg constructor fills no defaults.
+ DeliveryStreamDescription persisted = new DeliveryStreamDescription();
+ persisted.setDeliveryStreamName("persisted");
+ persisted.setDeliveryStreamARN("arn:aws:firehose:us-east-1:000000000000:deliverystream/persisted");
+ persisted.setAccountId("000000000000");
+ persisted.setDestinations(List.of(new DeliveryStreamDescription.Destination(redshift)));
+ AccountAwareStorageBackend<DeliveryStreamDescription> streams =
+ (AccountAwareStorageBackend<DeliveryStreamDescription>) backends.get("firehose/streams.json");
+ streams.putForAccount("000000000000", "us-east-1/persisted", persisted);
+
+ S3Destination described = firehoseService.describeDeliveryStream("persisted").getDestinations().get(0)
+ .getRedshiftDestinationDescription().getS3Backup();
+
+ assertEquals(false, described.getCloudWatchLoggingOptions().getEnabled());
+ assertEquals("UNCOMPRESSED", described.getCompressionFormat());
+ }
+
@Test
void isolatesSameStreamNameAcrossAccountsAndRegionsIncludingFlushAndDelete() {
String firstArn = firehoseService.createDeliveryStream("us-east-1", "111111111111", "shared", null,
@@ -387,6 +416,107 @@ class FirehoseServiceTest {
assertTrue(key.matches("events/2026/01/01/09/my-stream-3-2026-01-01-09-00-00-" + UUID_REGEX), key);
}
+ @Test
+ void cloudWatchLoggingOptionsDefaultToDisabledAndMergeMemberWise() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::logging-bucket");
+ firehoseService.createDeliveryStream("logged-stream", s3);
+ DeliveryStreamDescription.CloudWatchLoggingOptions logging =
+ firehoseService.describeDeliveryStream("logged-stream").s3Destination().getCloudWatchLoggingOptions();
+ assertEquals(Boolean.FALSE, logging.getEnabled());
+ assertNull(logging.getLogGroupName());
+
+ S3Destination enable = new S3Destination();
+ enable.setCloudWatchLoggingOptions(logging(true, "group", "stream"));
+ firehoseService.updateDestination("logged-stream", "1", "destinationId-000000000001", enable);
+
+ S3Destination disableOnly = new S3Destination();
+ disableOnly.setCloudWatchLoggingOptions(logging(false, null, null));
+ firehoseService.updateDestination("logged-stream", "2", "destinationId-000000000001", disableOnly);
+
+ logging = firehoseService.describeDeliveryStream("logged-stream").s3Destination().getCloudWatchLoggingOptions();
+ assertEquals(Boolean.FALSE, logging.getEnabled());
+ assertEquals("group", logging.getLogGroupName(), "an update naming only Enabled keeps the stored group");
+ assertEquals("stream", logging.getLogStreamName());
+ }
+
+ private static final String LOG_GROUP_REQUIRED =
+ "CloudWatch Log Group Name is required if CloudWatch Logging is enabled";
+ private static final String LOG_STREAM_REQUIRED =
+ "CloudWatch Log Stream Name is required if CloudWatch Logging is enabled";
+
+ @Test
+ void enabledLoggingRequiresTheGroupThenTheStreamName() {
+ assertLoggingRefused(LOG_GROUP_REQUIRED, () -> firehoseService.createDeliveryStream("no-names",
+ loggedDestination(logging(true, null, null))));
+ assertLoggingRefused(LOG_STREAM_REQUIRED, () -> firehoseService.createDeliveryStream("group-only",
+ loggedDestination(logging(true, "g", null))));
+ assertLoggingRefused(LOG_GROUP_REQUIRED, () -> firehoseService.createDeliveryStream("empty-names",
+ loggedDestination(logging(true, "", ""))));
+
+ firehoseService.createDeliveryStream("disabled-no-names", loggedDestination(logging(false, null, null)));
+ assertEquals(Boolean.FALSE, firehoseService.describeDeliveryStream("disabled-no-names")
+ .s3Destination().getCloudWatchLoggingOptions().getEnabled());
+ }
+
+ @Test
+ void anUpdateEnablingLoggingIsCheckedAgainstTheStoredNames() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::logging-bucket");
+ firehoseService.createDeliveryStream("enable-later", s3);
+
+ assertLoggingRefused(LOG_GROUP_REQUIRED, () -> firehoseService.updateDestination("enable-later", "1",
+ "destinationId-000000000001", loggedDestination(logging(true, null, null))));
+ assertEquals("1", firehoseService.describeDeliveryStream("enable-later").getVersionId());
+
+ firehoseService.updateDestination("enable-later", "1", "destinationId-000000000001",
+ loggedDestination(logging(false, "g", "s")));
+ firehoseService.updateDestination("enable-later", "2", "destinationId-000000000001",
+ loggedDestination(logging(true, null, null)));
+ DeliveryStreamDescription.CloudWatchLoggingOptions logging =
+ firehoseService.describeDeliveryStream("enable-later").s3Destination().getCloudWatchLoggingOptions();
+ assertEquals(Boolean.TRUE, logging.getEnabled());
+ assertEquals("g", logging.getLogGroupName());
+ assertEquals("s", logging.getLogStreamName());
+
+ assertLoggingRefused(LOG_STREAM_REQUIRED, () -> firehoseService.updateDestination("enable-later", "3",
+ "destinationId-000000000001", loggedDestination(logging(null, null, ""))));
+ assertEquals("s", firehoseService.describeDeliveryStream("enable-later")
+ .s3Destination().getCloudWatchLoggingOptions().getLogStreamName());
+ }
+
+ @Test
+ void theLoggingErrorPrecedesTheBufferingHintsOne() {
+ S3Destination s3 = loggedDestination(logging(true, null, null));
+ DeliveryStreamDescription.BufferingHints hints = new DeliveryStreamDescription.BufferingHints();
+ hints.setSizeInMBs(5);
+ s3.setBufferingHints(hints);
+
+ assertLoggingRefused(LOG_GROUP_REQUIRED, () -> firehoseService.createDeliveryStream("partial-hints", s3));
+ }
+
+ private static S3Destination loggedDestination(DeliveryStreamDescription.CloudWatchLoggingOptions logging) {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::logging-bucket");
+ s3.setCloudWatchLoggingOptions(logging);
+ return s3;
+ }
+
+ private static void assertLoggingRefused(String message, Runnable call) {
+ AwsException e = assertThrows(AwsException.class, call::run);
+ assertEquals("InvalidArgumentException", e.getErrorCode());
+ assertEquals(message, e.getMessage());
+ }
+
+ private static DeliveryStreamDescription.CloudWatchLoggingOptions logging(Boolean enabled, String group,
+ String stream) {
+ DeliveryStreamDescription.CloudWatchLoggingOptions options = new DeliveryStreamDescription.CloudWatchLoggingOptions();
+ options.setEnabled(enabled);
+ options.setLogGroupName(group);
+ options.setLogStreamName(stream);
+ return options;
+ }
+
@Test
void timeBasedFlushKeepsBufferWhileDefaultIntervalHasNotElapsed() {
firehoseService.createDeliveryStream("idle-stream", null);
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
index e13309c1df..958cb7f27b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.firehose;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.CloudWatchLoggingOptions;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
@@ -759,4 +760,65 @@ class S3DestinationValidatorTest {
error.getMessage());
}
+ private static final String LOG_GROUP_PATTERN = "Member must satisfy regular expression pattern: "
+ + "[\\.\\-_/#A-Za-z0-9]*";
+ private static final String LOG_STREAM_PATTERN = "Member must satisfy regular expression pattern: [^:*]*";
+
+ private static S3Destination withLogging(String logGroupName, String logStreamName) {
+ CloudWatchLoggingOptions logging = new CloudWatchLoggingOptions();
+ logging.setEnabled(true);
+ logging.setLogGroupName(logGroupName);
+ logging.setLogStreamName(logStreamName);
+ S3Destination config = new S3Destination();
+ config.setCloudWatchLoggingOptions(logging);
+ return config;
+ }
+
+ @Test
+ void logStreamNameWithAColonFailsItsPattern() {
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withLogging(null, "bad:stream"), "extendedS3DestinationConfiguration"));
+
+ assertEquals("ValidationException", error.getErrorCode());
+ assertEquals("1 validation error detected: Value at 'extendedS3DestinationConfiguration"
+ + ".cloudWatchLoggingOptions.logStreamName' failed to satisfy constraint: "
+ + LOG_STREAM_PATTERN, error.getMessage());
+ }
+
+ /** Probed: unlike FileExtension, both rules are reported, the length one first. */
+ @Test
+ void anOverlongLogGroupNameWithABadCharacterReportsLengthThenPattern() {
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withLogging("a".repeat(513) + " ", null), "extendedS3DestinationConfiguration"));
+
+ String path = "Value at 'extendedS3DestinationConfiguration.cloudWatchLoggingOptions.logGroupName'"
+ + " failed to satisfy constraint: ";
+ assertEquals("2 validation errors detected: "
+ + path + "Member must have length less than or equal to 512; "
+ + path + LOG_GROUP_PATTERN, error.getMessage());
+ }
+
+ @Test
+ void logGroupViolationsPrecedeLogStreamOnes() {
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withLogging("bad group", "bad:stream"), "s3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: "
+ + "Value at 's3DestinationConfiguration.cloudWatchLoggingOptions.logGroupName'"
+ + " failed to satisfy constraint: " + LOG_GROUP_PATTERN + "; "
+ + "Value at 's3DestinationConfiguration.cloudWatchLoggingOptions.logStreamName'"
+ + " failed to satisfy constraint: " + LOG_STREAM_PATTERN, error.getMessage());
+ }
+
+ @Test
+ void wellFormedLoggingNamesPass() {
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withLogging("/aws/kinesisfirehose/my-stream", "DestinationDelivery"),
+ "extendedS3DestinationConfiguration"));
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withLogging("a".repeat(512), "b".repeat(512)), "extendedS3DestinationConfiguration"));
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withLogging("", ""), "extendedS3DestinationConfiguration"));
+ }
+
}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/FirehoseRequests.java b/src/test/java/io/github/hectorvent/floci/testing/FirehoseRequests.java
new file mode 100644
index 0000000000..d84aaf9d06
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/testing/FirehoseRequests.java
@@ -0,0 +1,89 @@
+package io.github.hectorvent.floci.testing;
+
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import org.mockito.Mockito;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+
+public final class FirehoseRequests {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final Pattern RECORD_ID_PATTERN = Pattern.compile("\"recordId\":\"([^\"]+)\"");
+ private static final Pattern KEY_PATTERN = Pattern.compile("<Key>([^<]+)</Key>");
+
+ private FirehoseRequests() {}
+
+ public static String encode(String body) {
+ return Base64.getEncoder().encodeToString(body.getBytes(StandardCharsets.UTF_8));
+ }
+
+ public static void putRecordBatch(String streamName, String... bodies) {
+ List<String> records = new ArrayList<>();
+ for (String body : bodies) {
+ records.add("{\"Data\": \"" + encode(body) + "\"}");
+ }
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "Firehose_20150804.PutRecordBatch")
+ .body("{\"DeliveryStreamName\": \"" + streamName + "\", \"Records\": ["
+ + String.join(",", records) + "]}")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("FailedPutCount", equalTo(0));
+ }
+
+ public static String firstKey(String bucket, String prefix) {
+ String listing = given()
+ .when().get("/" + bucket)
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ Matcher key = KEY_PATTERN.matcher(listing);
+ while (key.find()) {
+ if (key.group(1).startsWith(prefix)) {
+ return key.group(1);
+ }
+ }
+ throw new AssertionError("expected an object under " + prefix + " in " + bucket + ", got: " + listing);
+ }
+
+ /**
+ * Answers each transform invocation from the record ids it actually carries, which the caller
+ * cannot name: they are minted per batch. Returns the payloads the mock received.
+ */
+ public static List<byte[]> answerTransforms(LambdaService lambdaService, RecordResponder responder) {
+ List<byte[]> payloads = new ArrayList<>();
+ Mockito.when(lambdaService.invokeArn(anyString(), any(), any())).thenAnswer(invocation -> {
+ byte[] payload = invocation.getArgument(1);
+ payloads.add(payload);
+ List<String> entries = new ArrayList<>();
+ Matcher recordIds = RECORD_ID_PATTERN.matcher(new String(payload, StandardCharsets.UTF_8));
+ for (int index = 0; recordIds.find(); index++) {
+ entries.add(responder.respond(recordIds.group(1), index));
+ }
+ InvokeResult result = new InvokeResult();
+ result.setStatusCode(200);
+ result.setPayload(("{\"records\":[" + String.join(",", entries) + "]}")
+ .getBytes(StandardCharsets.UTF_8));
+ return result;
+ });
+ return payloads;
+ }
+
+ @FunctionalInterface
+ public interface RecordResponder {
+ String respond(String recordId, int index);
+ }
+}
diff --git a/tools/docs/cfn_resource_types.yaml b/tools/docs/cfn_resource_types.yaml
index 79436858e9..65c81863bb 100644
--- a/tools/docs/cfn_resource_types.yaml
+++ b/tools/docs/cfn_resource_types.yaml
@@ -214,6 +214,7 @@ notes:
AWS::BedrockAgentCore::Runtime: 'Ref returns the AgentRuntimeId; CapacityProviderConfiguration, FilesystemConfigurations, LifecycleConfiguration, PlatformVersion and RequestHeaderConfiguration are accepted and not stored'
AWS::BedrockAgentCore::RuntimeEndpoint: 'Ref returns the endpoint ARN; Tags are accepted and not stored'
AWS::BedrockAgentCore::Memory: 'Ref returns the memory ARN; MemoryStrategies, IndexedKeys, NamespaceKeys and StreamDeliveryResources are accepted and not stored'
+ AWS::KinesisFirehose::DeliveryStream: '`DeliveryStreamName`, `Tags` and, from an S3 destination, `BucketARN`, `RoleARN`, `Prefix`, `ErrorOutputPrefix`, `CompressionFormat`, `BufferingHints`, `CloudWatchLoggingOptions` and `ProcessingConfiguration` are applied; other properties are ignored, and a destination member the template drops on update keeps its stored value, as on AWS'
# Trailing prose appended to a row after its type list, for behaviour that belongs to the service
# rather than to one type.