Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): validate and persist ProcessingConfiguration (#3375)

Kenshin Okinakadebf9fb0dc72
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseProcessingConfigurationTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseProcessingConfigurationTest.java
new file mode 100644
index 0000000000..0b40e593b6
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseProcessingConfigurationTest.java
@@ -0,0 +1,121 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.*;
+import software.amazon.awssdk.services.firehose.FirehoseClient;
+import software.amazon.awssdk.services.firehose.model.*;
+
+import java.util.List;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.*;
+
+/**
+ * Happy-path proof that the SDK's marshalling of the nested ProcessingConfiguration
+ * shapes round-trips through Floci: create, the echo AWS fills in, and the
+ * whole-block replacement on UpdateDestination. The validation matrix lives in the
+ * emulator's unit tests.
+ */
+@DisplayName("Firehose processing configuration")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class FirehoseProcessingConfigurationTest {
+
+ private static FirehoseClient firehose;
+ private static final String STREAM_NAME = "sdk-processing-" + UUID.randomUUID().toString().substring(0, 8);
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";
+ private static final String BUCKET_ARN = "arn:aws:s3:::floci-firehose-sdk-test";
+ private static final String LAMBDA_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";
+
+ @BeforeAll
+ static void setup() {
+ firehose = TestFixtures.firehoseClient();
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (firehose != null) {
+ try {
+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME).build());
+ } catch (Exception e) {
+ System.err.println("[WARN] Best-effort cleanup failed for delivery stream "
+ + STREAM_NAME + ": " + e.getMessage());
+ }
+ firehose.close();
+ }
+ }
+
+ private static Processor lambdaProcessor(ProcessorParameter... parameters) {
+ return Processor.builder().type(ProcessorType.LAMBDA).parameters(parameters).build();
+ }
+
+ private static ProcessorParameter parameter(ProcessorParameterName name, String value) {
+ return ProcessorParameter.builder().parameterName(name).parameterValue(value).build();
+ }
+
+ private static ProcessingConfiguration stored() {
+ return firehose.describeDeliveryStream(DescribeDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME).build())
+ .deliveryStreamDescription().destinations().get(0)
+ .extendedS3DestinationDescription().processingConfiguration();
+ }
+
+ private static String currentVersion() {
+ return firehose.describeDeliveryStream(DescribeDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME).build())
+ .deliveryStreamDescription().versionId();
+ }
+
+ @Test
+ @Order(1)
+ @DisplayName("Create accepts a Lambda processor and echoes it with AWS's own additions")
+ void createWithProcessingConfiguration() {
+ firehose.createDeliveryStream(CreateDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)
+ .extendedS3DestinationConfiguration(ExtendedS3DestinationConfiguration.builder()
+ .roleARN(ROLE_ARN)
+ .bucketARN(BUCKET_ARN)
+ .processingConfiguration(ProcessingConfiguration.builder()
+ .enabled(true)
+ .processors(lambdaProcessor(
+ parameter(ProcessorParameterName.BUFFER_INTERVAL_IN_SECONDS, "70"),
+ parameter(ProcessorParameterName.BUFFER_SIZE_IN_M_BS, "2"),
+ parameter(ProcessorParameterName.LAMBDA_ARN, LAMBDA_ARN)))
+ .build())
+ .build())
+ .build());
+
+ ProcessingConfiguration processing = stored();
+ assertThat(processing.enabled()).isTrue();
+ assertThat(processing.processors()).hasSize(1);
+
+ // NumberOfRetries and RoleArn are added by the service, and the order is its
+ // own rather than the order the parameters were sent in.
+ List<ProcessorParameter> parameters = processing.processors().get(0).parameters();
+ assertThat(parameters).extracting(p -> p.parameterName().toString())
+ .containsExactly("LambdaArn", "NumberOfRetries", "RoleArn",
+ "BufferSizeInMBs", "BufferIntervalInSeconds");
+ assertThat(parameters.get(1).parameterValue()).isEqualTo("3");
+ assertThat(parameters.get(2).parameterValue()).isEqualTo(ROLE_ARN);
+ }
+
+ @Test
+ @Order(2)
+ @DisplayName("An update replaces the whole block rather than merging it")
+ void updateReplacesTheBlock() {
+ firehose.updateDestination(UpdateDestinationRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .currentDeliveryStreamVersionId(currentVersion())
+ .destinationId("destinationId-000000000001")
+ .extendedS3DestinationUpdate(ExtendedS3DestinationUpdate.builder()
+ .processingConfiguration(ProcessingConfiguration.builder()
+ .enabled(false)
+ .build())
+ .build())
+ .build());
+
+ ProcessingConfiguration processing = stored();
+ assertThat(processing.enabled()).isFalse();
+ assertThat(processing.processors()).isEmpty();
+ }
+}
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index 8d6af5a9d5..1d579f5c7f 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -83,6 +83,18 @@ An omitted `Enabled` counts as enabled; `Enabled: false` stores and merges the c
- Error-output records report the delivery time as both `arrivalTimestamp` and `attemptEndingTimestamp` (Floci does not track per-record arrival), and omit AWS's `sequenceNumber`/`subSequenceNumber` members. An absent `ErrorOutputPrefix` writes them at the bucket root with no default time prefix, which matches AWS's object-name documentation but was not probed.
- The Parquet object carries no `Content-Encoding`, where AWS labels it `hadoop-snappy` even though the body is a plain Parquet file (probed), and its row order may differ from put order, as it does on AWS (probed).
+## Record transformation
+
+`ProcessingConfiguration` on the extended S3 destination is modelled, validated and echoed, but **not applied**: a stream that enables a transformation is accepted and delivers its records untransformed, and create and update log a warning saying so. Validation follows AWS, down to the messages and the order they are reported in.
+
+Two behaviors are worth knowing because they differ from the conversion block above. `UpdateDestination` replaces this block whole rather than merging it member-wise, so an update carrying only `{"Enabled": false}` leaves no processors behind, and each update is validated against its own content rather than the merged result. And a stored Lambda processor is echoed with `NumberOfRetries` and `RoleArn` filled in and its parameters reordered, as AWS returns them.
+
+### Known deviations from AWS
+
+- The transformation is not applied. A transform function is never invoked, so records reach the destination exactly as they were put, which is what makes a local test of one pass where AWS would not.
+- Two checks AWS itself does not make are deliberately absent: `NumberOfRetries` is not range-checked, despite the documented 1 to 8, and the function a `LambdaArn` names is not required to exist.
+- A null entry in `Processors` or `Parameters`, which a raw JSON client can send, is ignored rather than reported. Real AWS answers `InternalFailure` there, a fault of its own that is not worth reproducing; skipping the entry leaves the same configuration as omitting it would.
+
## S3 object keys
Delivered objects are named `<evaluated prefix><streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid><file extension>`, matching [AWS's object name format](https://docs.aws.amazon.com/firehose/latest/dev/s3-object-name.html):
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
index 3fc50bf16e..92e297ba1b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
@@ -46,7 +46,7 @@ public class FirehoseJsonHandler {
S3Destination s3 = null;
if (request.has("S3DestinationConfiguration")) {
s3 = mapper.treeToValue(request.get("S3DestinationConfiguration"), S3Destination.class);
- dropConversionFromLegacyShape(s3);
+ dropExtendedOnlyMembersFromLegacyShape(s3);
S3DestinationValidator.validateWireShape(s3, "s3DestinationConfiguration");
} else if (request.has("ExtendedS3DestinationConfiguration")) {
s3 = mapper.treeToValue(request.get("ExtendedS3DestinationConfiguration"), S3Destination.class);
@@ -82,7 +82,7 @@ public class FirehoseJsonHandler {
S3DestinationValidator.validateWireShape(update, "extendedS3DestinationUpdate");
} else if (request.has("S3DestinationUpdate")) {
update = mapper.treeToValue(request.get("S3DestinationUpdate"), S3Destination.class);
- dropConversionFromLegacyShape(update);
+ dropExtendedOnlyMembersFromLegacyShape(update);
S3DestinationValidator.validateWireShape(update, "s3DestinationUpdate");
}
firehoseService.updateDestination(name, currentVersionId, destinationId, update);
@@ -198,24 +198,28 @@ public class FirehoseJsonHandler {
}
/**
- * Clears {@code DataFormatConversionConfiguration} from a legacy
- * {@code S3DestinationConfiguration} or {@code S3DestinationUpdate}, which AWS
- * models only on the extended shapes. The two share one class here, which would
- * otherwise let a legacy request store a member its contract does not define.
+ * Clears {@code DataFormatConversionConfiguration} and
+ * {@code ProcessingConfiguration} from a legacy {@code S3DestinationConfiguration}
+ * or {@code S3DestinationUpdate}, which AWS models only on the extended shapes. The
+ * two share one class here, which would otherwise let a legacy request store members
+ * its contract does not define.
*
* Dropped rather than rejected: AWS's JSON protocol ignores a member the shape does
* not model, so a request carrying one succeeds with the member disregarded. The
- * SDKs cannot even send it, since it is absent from the legacy shape's model.
+ * SDKs cannot even send them, since both are absent from the legacy shape's model.
+ * That was probed for the conversion member; the processing member is the same
+ * mechanism on the same shape.
*
- * Only this member is cleared. {@code FileExtension}, {@code CustomTimeZone} and
+ * Only these two are cleared. {@code FileExtension}, {@code CustomTimeZone} and
* {@code S3BackupMode} are extended-only too and reach the legacy shapes the same
* way, a deviation that predates this change and is recorded in
* docs/services/firehose.md; widening the clearing would change behavior those
* members already have.
*/
- private static void dropConversionFromLegacyShape(S3Destination s3) {
+ private static void dropExtendedOnlyMembersFromLegacyShape(S3Destination s3) {
if (s3 != null) {
s3.setDataFormatConversionConfiguration(null);
+ s3.setProcessingConfiguration(null);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 9846a0467b..0587e25e24 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -309,6 +309,11 @@ public class FirehoseService implements ResourceProvider {
validateBufferingHints(s3Config);
DataFormatConversionValidator.validateEffective(s3Config);
+ ProcessingConfigurationValidator.validateEffective(s3Config);
+ if (s3Config != null) {
+ s3Config.canonicalizeProcessors();
+ }
+ warnIfProcessingEnabled(name, s3Config);
String arn = AwsArnUtils.Arn.of("firehose", region, accountId,
"deliverystream/" + name).toString();
// CreateDeliveryStream's KinesisStreamSourceConfiguration carries only the ARN and
@@ -364,7 +369,9 @@ public class FirehoseService implements ResourceProvider {
S3Destination current = destination.getExtendedS3DestinationDescription();
if (current == null) {
DataFormatConversionValidator.validateEffective(update);
+ ProcessingConfigurationValidator.validateEffective(update);
update.applyDefaults();
+ update.canonicalizeProcessors();
destination.setExtendedS3DestinationDescription(update);
} else {
// AWS validates the merged effective state, not the update shape: enabling
@@ -373,12 +380,19 @@ public class FirehoseService implements ResourceProvider {
// Validating a merged view first also keeps a failed update from leaving a
// half-merged destination behind in the memory backend.
DataFormatConversionValidator.validateEffective(mergedView(current, update));
+ // Not the merged view: an update carrying a ProcessingConfiguration replaces
+ // the stored one outright, so AWS validates what the update itself says. A
+ // processor without LambdaArn is rejected even when the stored one had it
+ // (probed 2026-09-10, unlike the conversion block above).
+ ProcessingConfigurationValidator.validateEffective(update);
mergeDestination(current, update);
+ current.canonicalizeProcessors();
}
stream.setVersionId(String.valueOf(parseVersionId(stream.getVersionId()) + 1));
stream.setLastUpdateTimestamp(java.time.Instant.now());
streamPut(streamKey, stream);
LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);
+ warnIfProcessingEnabled(name, stream.s3Destination());
}
public void startDeliveryStreamEncryption(String name, String keyType, String keyArn) {
@@ -450,6 +464,20 @@ public class FirehoseService implements ResourceProvider {
}
}
+ /**
+ * Says, where the configuration is set, that the transformation will not be applied.
+ * Fires on create and on every update that leaves it enabled, so a caller who keeps
+ * changing the destination keeps being told. Deliberately not in the flush path,
+ * which runs on every buffered delivery: create and update are caller-driven and are
+ * the moments a caller can act on the warning.
+ */
+ private static void warnIfProcessingEnabled(String name, S3Destination s3) {
+ if (s3 != null && s3.isProcessingEnabled()) {
+ LOG.warnv("Delivery stream {0} enables a record transformation, which Floci does not"
+ + " apply yet; its records will be delivered untransformed", name);
+ }
+ }
+
private static void mergeDestination(S3Destination current, S3Destination update) {
if (update.getRoleArn() != null) current.setRoleArn(update.getRoleArn());
if (update.getBucketArn() != null) current.setBucketArn(update.getBucketArn());
@@ -461,6 +489,12 @@ public class FirehoseService implements ResourceProvider {
if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());
if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());
if (update.getS3BackupMode() != null) current.setS3BackupMode(update.getS3BackupMode());
+ // Replaced whole, not merged member-wise: an update carrying only
+ // {"Enabled": false} leaves Processors empty on real AWS, where the same shape
+ // preserves the conversion block's members (probed 2026-09-10).
+ if (update.getProcessingConfiguration() != null) {
+ current.setProcessingConfiguration(update.getProcessingConfiguration());
+ }
if (update.getDataFormatConversionConfiguration() != null) {
current.setDataFormatConversionConfiguration(mergeConversion(
current.getDataFormatConversionConfiguration(), update.getDataFormatConversionConfiguration()));
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/ProcessingConfigurationValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/ProcessingConfigurationValidator.java
new file mode 100644
index 0000000000..daf7d6a6f6
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/ProcessingConfigurationValidator.java
@@ -0,0 +1,112 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * The semantic checks AWS runs on a ProcessingConfiguration once its modeled
+ * constraints hold, in the order and with the wording probed against real AWS
+ * (us-west-2, 2026-09-10).
+ *
+ * Deliberately absent: AWS does not range-check NumberOfRetries at configuration
+ * time, accepting 0 and 9 alike despite the documented 1 to 8, and it does not
+ * check that the function the LambdaArn names exists, unlike the Glue table a
+ * conversion schema names. Both were probed; neither is enforced here.
+ */
+final class ProcessingConfigurationValidator {
+
+ /**
+ * Lambda's own ARN shape rather than "anything without a colon": probed, AWS
+ * answers "Invalid lambda ARN." for a function name carrying a space or slashes,
+ * which a looser pattern would have stored.
+ */
+ private static final Pattern LAMBDA_ARN = Pattern.compile(
+ "arn:aws[a-zA-Z-]*:lambda:[a-z0-9-]+:\\d{12}:function:[a-zA-Z0-9_-]{1,64}"
+ + "(:(\\$LATEST|[a-zA-Z0-9_-]{1,128}))?");
+
+ private ProcessingConfigurationValidator() {}
+
+ static void validateEffective(S3Destination config) {
+ if (config == null || config.getProcessingConfiguration() == null) {
+ return;
+ }
+ ProcessingConfiguration processing = config.getProcessingConfiguration();
+ if (processing.getProcessors() == null) {
+ return;
+ }
+ // Before anything a processor's own parameters say: a repeated name is rejected
+ // even when the LambdaArn the processor needs is missing too (probed).
+ int lambdaProcessors = 0;
+ for (Processor processor : processing.getProcessors()) {
+ if (processor == null) {
+ continue;
+ }
+ rejectDuplicateParameters(processor.getParameters());
+ if ("Lambda".equals(processor.getType())) {
+ lambdaProcessors++;
+ }
+ }
+ if (lambdaProcessors > 1) {
+ throw invalidArgument("Cannot have more than 1 Lambda processor.");
+ }
+ for (Processor processor : processing.getProcessors()) {
+ if (processor == null || !"Lambda".equals(processor.getType())) {
+ continue;
+ }
+ validateLambdaProcessor(processor.getParameters());
+ }
+ }
+
+ private static void rejectDuplicateParameters(List<ProcessorParameter> parameters) {
+ if (parameters == null) {
+ return;
+ }
+ Set<String> seen = new HashSet<>();
+ for (ProcessorParameter parameter : parameters) {
+ if (parameter != null && parameter.getParameterName() != null
+ && !seen.add(parameter.getParameterName())) {
+ throw invalidArgument("Duplicate ProcessorParameter passed to ProcessingConfiguration.");
+ }
+ }
+ }
+
+ private static void validateLambdaProcessor(List<ProcessorParameter> parameters) {
+ String lambdaArn = parameterValue(parameters, "LambdaArn");
+ if (lambdaArn == null || lambdaArn.isEmpty()) {
+ throw invalidArgument("LambdaArn is required when Lambda processor is used.");
+ }
+ if (!LAMBDA_ARN.matcher(lambdaArn).matches()) {
+ throw invalidArgument("Invalid lambda ARN.");
+ }
+ boolean hasSize = parameterValue(parameters, "BufferSizeInMBs") != null;
+ boolean hasInterval = parameterValue(parameters, "BufferIntervalInSeconds") != null;
+ if (hasSize != hasInterval) {
+ throw invalidArgument("Both BufferSizeInMBs and BufferIntervalInSeconds are required"
+ + " to configure buffering for lambda processor.");
+ }
+ }
+
+ private static String parameterValue(List<ProcessorParameter> parameters, String name) {
+ if (parameters == null) {
+ return null;
+ }
+ for (ProcessorParameter parameter : parameters) {
+ if (parameter != null && name.equals(parameter.getParameterName())) {
+ return parameter.getParameterValue();
+ }
+ }
+ return null;
+ }
+
+ private static AwsException invalidArgument(String message) {
+ return new AwsException("InvalidArgumentException", message, 400);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
index b88b0bf7a1..fbff3e63a4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
@@ -4,12 +4,16 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import java.util.Map;
import java.util.regex.Pattern;
@@ -104,6 +108,7 @@ final class S3DestinationValidator {
// violations sit in more than one of them.
collectOpenXMappingViolations(config, shapeName, violations);
collectSchemaViolations(config, shapeName, violations);
+ collectProcessingViolations(config, shapeName, violations);
if (!violations.isEmpty()) {
throw constraintViolations(violations);
}
@@ -312,6 +317,101 @@ final class S3DestinationValidator {
}
}
+ /**
+ * The processor and parameter enums, in the order AWS prints them in a
+ * constraint message, which is neither declaration nor alphabetical order
+ * (probed 2026-09-10).
+ */
+ private static final String PROCESSOR_TYPE_SET =
+ "[Decompression, Lambda, RecordDeAggregation, MetadataExtraction,"
+ + " AppendDelimiterToRecord, CloudWatchLogProcessing]";
+ private static final String PARAMETER_NAME_SET =
+ "[DataMessageExtraction, NumberOfRetries, Delimiter, RoleArn, JsonParsingEngine,"
+ + " SubRecordType, MetadataExtractionQuery, BufferIntervalInSeconds, LambdaArn,"
+ + " BufferSizeInMBs, CompressionFormat]";
+ private static final int PARAMETER_VALUE_MAX_LENGTH = 5120;
+ private static final String PARAMETER_VALUE_REGEX = "^(?!\\s*$).+";
+ private static final Pattern PARAMETER_VALUE_NOT_BLANK = Pattern.compile(PARAMETER_VALUE_REGEX);
+ private static final Set<String> PROCESSOR_TYPES = Set.of("Decompression", "Lambda",
+ "RecordDeAggregation", "MetadataExtraction", "AppendDelimiterToRecord",
+ "CloudWatchLogProcessing");
+ private static final Set<String> PARAMETER_NAMES = Set.of("DataMessageExtraction",
+ "NumberOfRetries", "Delimiter", "RoleArn", "JsonParsingEngine", "SubRecordType",
+ "MetadataExtractionQuery", "BufferIntervalInSeconds", "LambdaArn", "BufferSizeInMBs",
+ "CompressionFormat");
+
+ /**
+ * Both members are required, and the value carries a length range and a
+ * not-blank pattern. An empty value trips the length and the pattern, printed in
+ * that order, as the probed message does.
+ */
+ private static void collectParameterViolations(ProcessorParameter parameter, String shapeName,
+ String parameterPath, List<String> violations) {
+ String name = parameter.getParameterName();
+ if (name == null) {
+ violations.add(violation(shapeName, parameterPath + ".parameterName",
+ "Member must not be null"));
+ } else if (!PARAMETER_NAMES.contains(name)) {
+ violations.add(violation(shapeName, parameterPath + ".parameterName",
+ "Member must satisfy enum value set: " + PARAMETER_NAME_SET));
+ }
+ String value = parameter.getParameterValue();
+ if (value == null) {
+ violations.add(violation(shapeName, parameterPath + ".parameterValue",
+ "Member must not be null"));
+ return;
+ }
+ if (value.length() < 1) {
+ violations.add(violation(shapeName, parameterPath + ".parameterValue",
+ "Member must have length greater than or equal to 1"));
+ } else if (value.length() > PARAMETER_VALUE_MAX_LENGTH) {
+ violations.add(violation(shapeName, parameterPath + ".parameterValue",
+ "Member must have length less than or equal to " + PARAMETER_VALUE_MAX_LENGTH));
+ }
+ if (PARAMETER_VALUE_NOT_BLANK.matcher(value).matches()) {
+ return;
+ }
+ violations.add(violation(shapeName, parameterPath + ".parameterValue",
+ "Member must satisfy regular expression pattern: " + PARAMETER_VALUE_REGEX));
+ }
+
+ /** Members are numbered from 1 in these paths, as the probed messages show. */
+ private static void collectProcessingViolations(S3Destination config, String shapeName,
+ List<String> violations) {
+ ProcessingConfiguration processing = config.getProcessingConfiguration();
+ if (processing == null || processing.getProcessors() == null) {
+ return;
+ }
+ List<Processor> processors = processing.getProcessors();
+ for (int i = 0; i < processors.size(); i++) {
+ Processor processor = processors.get(i);
+ if (processor == null) {
+ continue;
+ }
+ String processorPath = "processingConfiguration.processors." + (i + 1) + ".member";
+ // The processor's own member before the ones nested in it, which is the
+ // order AWS prints them when both are violated (probed).
+ if (processor.getType() == null) {
+ violations.add(violation(shapeName, processorPath + ".type", "Member must not be null"));
+ } else if (!PROCESSOR_TYPES.contains(processor.getType())) {
+ violations.add(violation(shapeName, processorPath + ".type",
+ "Member must satisfy enum value set: " + PROCESSOR_TYPE_SET));
+ }
+ List<ProcessorParameter> parameters = processor.getParameters();
+ if (parameters == null) {
+ continue;
+ }
+ for (int j = 0; j < parameters.size(); j++) {
+ ProcessorParameter parameter = parameters.get(j);
+ if (parameter == null) {
+ continue;
+ }
+ String parameterPath = processorPath + ".parameters." + (j + 1) + ".member";
+ collectParameterViolations(parameter, shapeName, parameterPath, violations);
+ }
+ }
+ }
+
private static String violation(String shapeName, String member, String constraint) {
return "Value at '" + shapeName + "." + member + "' failed to satisfy constraint: " + constraint;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
index 444cdd097e..163982a4e0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.time.Instant;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.ArrayList;
@@ -196,6 +197,11 @@ public class DeliveryStreamDescription {
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(JsonInclude.Include.NON_NULL)
public static class S3Destination {
+
+ /** The order AWS echoes a Lambda processor's parameters in, whatever order it received them (probed). */
+ private static final List<String> LAMBDA_PARAMETER_ORDER = List.of("LambdaArn",
+ "NumberOfRetries", "RoleArn", "BufferSizeInMBs", "BufferIntervalInSeconds");
+
@JsonProperty("RoleARN")
private String roleArn;
@JsonProperty("BucketARN")
@@ -218,6 +224,8 @@ public class DeliveryStreamDescription {
private String s3BackupMode;
@JsonProperty("DataFormatConversionConfiguration")
private DataFormatConversionConfiguration dataFormatConversionConfiguration;
+ @JsonProperty("ProcessingConfiguration")
+ private ProcessingConfiguration processingConfiguration;
public S3Destination() {}
public String getRoleArn() { return roleArn; }
@@ -247,6 +255,27 @@ public class DeliveryStreamDescription {
this.dataFormatConversionConfiguration = configuration;
}
+ public ProcessingConfiguration getProcessingConfiguration() {
+ return processingConfiguration;
+ }
+ public void setProcessingConfiguration(ProcessingConfiguration processingConfiguration) {
+ this.processingConfiguration = processingConfiguration;
+ }
+
+ /**
+ * True when a transformation applies to deliveries. An omitted Enabled is taken
+ * as enabled, following the conversion block, where that was probed; the
+ * processing block's own behavior for an omitted Enabled was not, and nothing
+ * but the "not applied yet" warning reads this today.
+ *
+ * Ignored for serialization for the same reason as the accessor below.
+ */
+ @JsonIgnore
+ public boolean isProcessingEnabled() {
+ return processingConfiguration != null
+ && !Boolean.FALSE.equals(processingConfiguration.getEnabled());
+ }
+
/**
* True when data format conversion applies to deliveries: the configuration is
* present and not explicitly disabled. AWS treats an omitted Enabled as enabled
@@ -299,6 +328,57 @@ public class DeliveryStreamDescription {
}
}
+ /**
+ * Real AWS answers with more than the caller sent: a Lambda processor comes back
+ * with NumberOfRetries defaulted to 3 and RoleArn filled from the delivery
+ * stream's role, and the parameters are echoed in a fixed order whatever order
+ * they arrived in (probed 2026-09-10).
+ *
+ * Called as a destination is written, not as it is read. Doing it on the way out
+ * would mutate what storage handed back, so whether a Describe had happened would
+ * decide what a later update sees, and RoleArn would be injected from whichever
+ * role was current at the time of the read.
+ */
+ public void canonicalizeProcessors() {
+ if (processingConfiguration == null || processingConfiguration.getProcessors() == null) {
+ return;
+ }
+ for (Processor processor : processingConfiguration.getProcessors()) {
+ if (processor == null || !"Lambda".equals(processor.getType())
+ || processor.getParameters() == null) {
+ continue;
+ }
+ // A repeated name never reaches here, the validator rejects it as AWS
+ // does, so this only has to be stable for the names it does see.
+ Map<String, String> byName = new LinkedHashMap<>();
+ for (ProcessorParameter parameter : processor.getParameters()) {
+ if (parameter != null && parameter.getParameterName() != null) {
+ byName.putIfAbsent(parameter.getParameterName(), parameter.getParameterValue());
+ }
+ }
+ byName.putIfAbsent("NumberOfRetries", "3");
+ if (roleArn != null) {
+ byName.putIfAbsent("RoleArn", roleArn);
+ }
+ List<ProcessorParameter> ordered = new ArrayList<>(byName.size());
+ for (String name : LAMBDA_PARAMETER_ORDER) {
+ if (byName.containsKey(name)) {
+ ordered.add(parameter(name, byName.remove(name)));
+ }
+ }
+ byName.forEach((name, value) -> ordered.add(parameter(name, value)));
+ processor.setParameters(ordered);
+ }
+ }
+
+ private static ProcessorParameter parameter(String name, String value) {
+ ProcessorParameter parameter = new ProcessorParameter();
+ parameter.setParameterName(name);
+ parameter.setParameterValue(value);
+ return parameter;
+ }
+
+
/**
* A view of this config with only the fields AWS's plain S3DestinationDescription
* actually has - FileExtension, CustomTimeZone, and S3BackupMode are extended-only.
@@ -325,6 +405,62 @@ public class DeliveryStreamDescription {
}
}
+ /**
+ * Unlike the conversion configuration below, UpdateDestination replaces this block
+ * outright rather than merging it member-wise (probed), so a stored member never has
+ * to survive an update that omits it.
+ */
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class ProcessingConfiguration {
+ @JsonProperty("Enabled")
+ private Boolean enabled;
+ @JsonProperty("Processors")
+ private List<Processor> processors;
+
+ public ProcessingConfiguration() {}
+
+ public Boolean getEnabled() { return enabled; }
+ public void setEnabled(Boolean enabled) { this.enabled = enabled; }
+ public List<Processor> getProcessors() { return processors; }
+ public void setProcessors(List<Processor> processors) { this.processors = processors; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class Processor {
+ @JsonProperty("Type")
+ private String type;
+ @JsonProperty("Parameters")
+ private List<ProcessorParameter> parameters;
+
+ public Processor() {}
+
+ public String getType() { return type; }
+ public void setType(String type) { this.type = type; }
+ public List<ProcessorParameter> getParameters() { return parameters; }
+ public void setParameters(List<ProcessorParameter> parameters) { this.parameters = parameters; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class ProcessorParameter {
+ @JsonProperty("ParameterName")
+ private String parameterName;
+ @JsonProperty("ParameterValue")
+ private String parameterValue;
+
+ public ProcessorParameter() {}
+
+ public String getParameterName() { return parameterName; }
+ public void setParameterName(String parameterName) { this.parameterName = parameterName; }
+ public String getParameterValue() { return parameterValue; }
+ public void setParameterValue(String parameterValue) { this.parameterValue = parameterValue; }
+ }
+
/**
* Members stay null-honest, mirroring real AWS: DescribeDeliveryStream echoes the
* configuration exactly as it was given (an empty ParquetSerDe stays {}, CatalogId
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java
index 8a0b118df0..4959e5c8b7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java
@@ -186,6 +186,49 @@ class FirehoseDataFormatConversionIntegrationTest {
.body(DESCRIPTION_PATH + ".DataFormatConversionConfiguration", nullValue());
}
+ /**
+ * The processing member is extended-only in AWS's model too, so the legacy shapes
+ * drop it the same way. Wire-level rather than through the service, since the
+ * dropping happens in the handler as the request is unmarshalled.
+ */
+ @Test
+ @Order(7)
+ void legacyS3DestinationShapeIgnoresTheProcessingMember() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "legacy-shape-processing-stream",
+ "S3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "CompressionFormat": "UNCOMPRESSED",
+ "ProcessingConfiguration": {
+ "Enabled": true,
+ "Processors": [ { "Type": "Lambda", "Parameters": [
+ { "ParameterName": "LambdaArn",
+ "ParameterValue": "arn:aws:lambda:us-east-1:000000000000:function:t" } ] } ]
+ }
+ }
+ }
+ """.formatted(ROLE_ARN, BUCKET_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"legacy-shape-processing-stream\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(DESCRIPTION_PATH + ".ProcessingConfiguration", nullValue());
+ }
+
@Test
@Order(4)
void createRejectsCompressedDestinationsWithTheAwsMessage() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java
new file mode 100644
index 0000000000..09f03d8c95
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java
@@ -0,0 +1,348 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * The messages, the enum orders and the storage behavior asserted here were probed
+ * against real AWS (us-west-2, 2026-09-10). Two of them are deliberate absences:
+ * NumberOfRetries is not range-checked, and the function a LambdaArn names is not
+ * required to exist.
+ */
+class FirehoseProcessingConfigurationTest {
+
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";
+ private static final String LAMBDA_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";
+ private static final String DESTINATION_ID = "destinationId-000000000001";
+ private static final String BUFFER_PAIR_MESSAGE = "Both BufferSizeInMBs and BufferIntervalInSeconds"
+ + " are required to configure buffering for lambda processor.";
+
+ private FirehoseService service;
+
+ @BeforeEach
+ void setUp() {
+ StorageFactory storageFactory = mock(StorageFactory.class);
+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();
+ when(storageFactory.create(anyString(), anyString(), any()))
+ .thenAnswer(invocation -> backends.computeIfAbsent(
+ invocation.getArgument(0) + "/" + invocation.getArgument(1),
+ k -> AccountAwareStorageBackend.inMemory("000000000000")));
+ EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);
+ when(firehoseCfg.enabled()).thenReturn(true);
+ when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);
+ when(firehoseCfg.flushRecordCount()).thenReturn(0);
+ EmulatorConfig.ServicesConfig servicesCfg = mock(EmulatorConfig.ServicesConfig.class);
+ when(servicesCfg.firehose()).thenReturn(firehoseCfg);
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ when(config.services()).thenReturn(servicesCfg);
+ RegionResolver regionResolver = mock(RegionResolver.class);
+ when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");
+ when(regionResolver.getAccountId()).thenReturn("000000000000");
+ service = new FirehoseService(storageFactory, mock(S3Service.class),
+ Mockito.mock(KinesisService.class), regionResolver, new MutableClock(), config,
+ Mockito.mock(FirehoseParquetConverter.class));
+ }
+
+ private static ProcessorParameter parameter(String name, String value) {
+ ProcessorParameter parameter = new ProcessorParameter();
+ parameter.setParameterName(name);
+ parameter.setParameterValue(value);
+ return parameter;
+ }
+
+ private static ProcessingConfiguration processing(String type, ProcessorParameter... parameters) {
+ Processor processor = new Processor();
+ processor.setType(type);
+ processor.setParameters(new ArrayList<>(List.of(parameters)));
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setEnabled(true);
+ processing.setProcessors(new ArrayList<>(List.of(processor)));
+ return processing;
+ }
+
+ private static ProcessingConfiguration lambdaProcessing(ProcessorParameter... parameters) {
+ return processing("Lambda", parameters);
+ }
+
+ private static S3Destination destination(Consumer<S3Destination> customizer) {
+ S3Destination s3 = new S3Destination();
+ s3.setRoleArn(ROLE_ARN);
+ s3.setBucketArn("arn:aws:s3:::results");
+ customizer.accept(s3);
+ return s3;
+ }
+
+ private String currentVersion(String name) {
+ return service.describeDeliveryStream(name).getVersionId();
+ }
+
+ private List<String> storedParameterNames(String name) {
+ List<ProcessorParameter> parameters = service.describeDeliveryStream(name).s3Destination()
+ .getProcessingConfiguration().getProcessors().get(0).getParameters();
+ List<String> names = new ArrayList<>();
+ parameters.forEach(parameter -> names.add(parameter.getParameterName()));
+ return names;
+ }
+
+ @Test
+ void aLambdaProcessorWithoutALambdaArnIsRejected() {
+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("NumberOfRetries", "2"))))));
+
+ assertEquals("InvalidArgumentException", error.getErrorCode());
+ assertEquals("LambdaArn is required when Lambda processor is used.", error.getMessage());
+ }
+
+ // Probed: AWS rejects a function name that is not one Lambda would accept, so the
+ // shape has to be Lambda's rather than merely arn-looking.
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "not-an-arn",
+ "arn:aws:lambda:us-east-1:000000000000:function:bad name",
+ "arn:aws:lambda:us-east-1:000000000000:function:name/with/slashes",
+ "arn:aws:lambda:us-east-1:000000000000:function:bad?name",
+ "arn:aws:lambda:us-east-1:000000000000:function:bad@name",
+ "arn:aws:lambda:us-east-1:000000000000:function:bad[name",
+ })
+ void aMalformedLambdaArnIsRejected(String lambdaArn) {
+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("LambdaArn", lambdaArn))))));
+
+ assertEquals("Invalid lambda ARN.", error.getMessage());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "arn:aws:lambda:us-east-1:000000000000:function:transform",
+ "arn:aws:lambda:us-east-1:000000000000:function:transform:$LATEST",
+ "arn:aws:lambda:us-east-1:000000000000:function:transform:prod",
+ })
+ void aQualifiedOrUnqualifiedFunctionArnIsAccepted(String lambdaArn) {
+ assertDoesNotThrow(() -> service.createDeliveryStream("stream-" + lambdaArn.hashCode(),
+ destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("LambdaArn", lambdaArn))))));
+ }
+
+ // A function that does not exist is still accepted: unlike a conversion schema's
+ // Glue table, AWS resolves nothing at configuration time.
+ @Test
+ void aWellFormedArnToAnyFunctionIsAccepted() {
+ assertDoesNotThrow(() -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("LambdaArn",
+ "arn:aws:lambda:us-east-1:000000000000:function:never-created"))))));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"BufferSizeInMBs", "BufferIntervalInSeconds"})
+ void oneBufferingParameterWithoutTheOtherIsRejected(String parameterName) {
+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(
+ parameter("LambdaArn", LAMBDA_ARN), parameter(parameterName, "2"))))));
+
+ assertEquals(BUFFER_PAIR_MESSAGE, error.getMessage());
+ }
+
+ @Test
+ void bothBufferingParametersTogetherAreAccepted() {
+ assertDoesNotThrow(() -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(
+ parameter("LambdaArn", LAMBDA_ARN),
+ parameter("BufferSizeInMBs", "2"),
+ parameter("BufferIntervalInSeconds", "70"))))));
+ }
+
+ // Deliberate: real AWS accepts both, despite the documented 1 to 8.
+ @ParameterizedTest
+ @ValueSource(strings = {"0", "9"})
+ void numberOfRetriesIsNotRangeChecked(String retries) {
+ assertDoesNotThrow(() -> service.createDeliveryStream("stream-" + retries,
+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(
+ parameter("LambdaArn", LAMBDA_ARN), parameter("NumberOfRetries", retries))))));
+ }
+
+ // The LambdaArn rule belongs to the Lambda processor, not to every processor.
+ @Test
+ void anotherProcessorTypeIsNotSubjectToTheLambdaRules() {
+ assertDoesNotThrow(() -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(
+ processing("AppendDelimiterToRecord", parameter("Delimiter", "\\n"))))));
+ }
+
+ @Test
+ void storedParametersGainTheDefaultsAndAFixedOrder() {
+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("BufferIntervalInSeconds", "70"),
+ parameter("BufferSizeInMBs", "2"),
+ parameter("LambdaArn", LAMBDA_ARN)))));
+
+ assertEquals(List.of("LambdaArn", "NumberOfRetries", "RoleArn", "BufferSizeInMBs",
+ "BufferIntervalInSeconds"), storedParameterNames("stream"));
+
+ List<ProcessorParameter> stored = service.describeDeliveryStream("stream").s3Destination()
+ .getProcessingConfiguration().getProcessors().get(0).getParameters();
+ assertEquals("3", stored.get(1).getParameterValue());
+ assertEquals(ROLE_ARN, stored.get(2).getParameterValue());
+ }
+
+ @Test
+ void aRepeatedParameterNameIsRejected() {
+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(
+ parameter("LambdaArn", LAMBDA_ARN), parameter("LambdaArn", "not-an-arn"))))));
+
+ assertEquals("Duplicate ProcessorParameter passed to ProcessingConfiguration.", error.getMessage());
+ }
+
+ // The duplicate check runs first: the processor is missing its LambdaArn as well,
+ // and AWS still reports the duplicate.
+ @Test
+ void aRepeatedParameterIsReportedBeforeTheMissingLambdaArn() {
+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(
+ parameter("NumberOfRetries", "2"), parameter("NumberOfRetries", "3"))))));
+
+ assertEquals("Duplicate ProcessorParameter passed to ProcessingConfiguration.", error.getMessage());
+ }
+
+ @Test
+ void moreThanOneLambdaProcessorIsRejected() {
+ Processor first = new Processor();
+ first.setType("Lambda");
+ first.setParameters(List.of(parameter("LambdaArn", LAMBDA_ARN)));
+ Processor second = new Processor();
+ second.setType("Lambda");
+ second.setParameters(List.of(parameter("LambdaArn", LAMBDA_ARN)));
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setEnabled(true);
+ processing.setProcessors(List.of(first, second));
+
+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(processing))));
+
+ assertEquals("Cannot have more than 1 Lambda processor.", error.getMessage());
+ }
+
+ // A Lambda processor alongside another type is fine.
+ @Test
+ void twoProcessorsOfDifferentTypesAreAccepted() {
+ Processor lambda = new Processor();
+ lambda.setType("Lambda");
+ lambda.setParameters(List.of(parameter("LambdaArn", LAMBDA_ARN)));
+ Processor delimiter = new Processor();
+ delimiter.setType("AppendDelimiterToRecord");
+ delimiter.setParameters(List.of(parameter("Delimiter", "\\n")));
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setEnabled(true);
+ processing.setProcessors(List.of(lambda, delimiter));
+
+ assertDoesNotThrow(() -> service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(processing))));
+ }
+
+ // Settled when written, so reading cannot change what a later update sees: the
+ // injected RoleArn must be the same whether or not a Describe happened in between.
+ @Test
+ void describeDoesNotDecideWhatALaterUpdateSees() {
+ service.createDeliveryStream("read-first", destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));
+ service.createDeliveryStream("no-read", destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));
+
+ service.describeDeliveryStream("read-first");
+
+ String otherRole = "arn:aws:iam::000000000000:role/other";
+ for (String name : List.of("read-first", "no-read")) {
+ service.updateDestination(name, currentVersion(name), DESTINATION_ID,
+ destination(s3 -> s3.setRoleArn(otherRole)));
+ }
+
+ assertEquals(storedRoleParameter("read-first"), storedRoleParameter("no-read"));
+ assertEquals(ROLE_ARN, storedRoleParameter("no-read"));
+ }
+
+ private String storedRoleParameter(String name) {
+ for (ProcessorParameter parameter : service.describeDeliveryStream(name).s3Destination()
+ .getProcessingConfiguration().getProcessors().get(0).getParameters()) {
+ if ("RoleArn".equals(parameter.getParameterName())) {
+ return parameter.getParameterValue();
+ }
+ }
+ return null;
+ }
+
+ // The opposite of the conversion block, which merges member-wise.
+ @Test
+ void anUpdateReplacesTheWholeBlockRatherThanMergingIt() {
+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));
+
+ ProcessingConfiguration disabled = new ProcessingConfiguration();
+ disabled.setEnabled(false);
+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,
+ destination(s3 -> s3.setProcessingConfiguration(disabled)));
+
+ ProcessingConfiguration stored = service.describeDeliveryStream("stream").s3Destination()
+ .getProcessingConfiguration();
+ assertEquals(false, stored.getEnabled());
+ assertTrue(stored.getProcessors() == null || stored.getProcessors().isEmpty(),
+ "processors were " + stored.getProcessors());
+ }
+
+ @Test
+ void anUpdateThatDoesNotMentionTheBlockPreservesIt() {
+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));
+
+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,
+ destination(s3 -> s3.setPrefix("changed/")));
+
+ assertEquals(List.of("LambdaArn", "NumberOfRetries", "RoleArn"), storedParameterNames("stream"));
+ }
+
+ // Validation runs against the update's own content, since the update replaces the
+ // block: a stored LambdaArn does not satisfy an update that omits one.
+ @Test
+ void anUpdateIsValidatedAgainstItsOwnContent() {
+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));
+
+ AwsException error = assertThrows(AwsException.class, () -> service.updateDestination("stream",
+ currentVersion("stream"), DESTINATION_ID, destination(s3 -> s3.setProcessingConfiguration(
+ lambdaProcessing(parameter("NumberOfRetries", "7"))))));
+
+ assertEquals("LambdaArn is required when Lambda processor is used.", error.getMessage());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
index a1424d51e9..e13309c1df 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
@@ -2,6 +2,9 @@ package io.github.hectorvent.floci.services.firehose;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -10,6 +13,7 @@ import org.junit.jupiter.params.provider.ValueSource;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* The error codes and messages asserted here were captured from real AWS's raw
@@ -19,6 +23,139 @@ class S3DestinationValidatorTest {
private static final String ENUM_SET = "[ZIP, HADOOP_SNAPPY, Snappy, GZIP, UNCOMPRESSED]";
+ private static final String PROCESSOR_TYPE_SET = "[Decompression, Lambda, RecordDeAggregation,"
+ + " MetadataExtraction, AppendDelimiterToRecord, CloudWatchLogProcessing]";
+ private static final String PARAMETER_NAME_SET = "[DataMessageExtraction, NumberOfRetries, Delimiter,"
+ + " RoleArn, JsonParsingEngine, SubRecordType, MetadataExtractionQuery,"
+ + " BufferIntervalInSeconds, LambdaArn, BufferSizeInMBs, CompressionFormat]";
+
+ private static ProcessorParameter namedParameter(String name, String value) {
+ ProcessorParameter parameter = new ProcessorParameter();
+ parameter.setParameterName(name);
+ parameter.setParameterValue(value);
+ return parameter;
+ }
+
+ private static S3Destination withProcessors(String unusedShapeName, Processor... processors) {
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setEnabled(true);
+ processing.setProcessors(java.util.List.of(processors));
+ S3Destination config = new S3Destination();
+ config.setProcessingConfiguration(processing);
+ return config;
+ }
+
+ private static S3Destination withProcessor(String type, String... parameterNames) {
+ Processor processor = new Processor();
+ processor.setType(type);
+ java.util.List<ProcessorParameter> parameters = new java.util.ArrayList<>();
+ for (String name : parameterNames) {
+ ProcessorParameter parameter = new ProcessorParameter();
+ parameter.setParameterName(name);
+ parameter.setParameterValue("value");
+ parameters.add(parameter);
+ }
+ processor.setParameters(parameters);
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setEnabled(true);
+ processing.setProcessors(java.util.List.of(processor));
+ S3Destination config = new S3Destination();
+ config.setProcessingConfiguration(processing);
+ return config;
+ }
+
+ // The enum sets and the 1-based member paths are the ones AWS prints (probed).
+ @Test
+ void anUnknownProcessorTypeFailsItsEnum() {
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withProcessor("Bogus"), "extendedS3DestinationConfiguration"));
+
+ assertEquals("1 validation error detected: Value at 'extendedS3DestinationConfiguration"
+ + ".processingConfiguration.processors.1.member.type' failed to satisfy constraint:"
+ + " Member must satisfy enum value set: " + PROCESSOR_TYPE_SET, error.getMessage());
+ }
+
+ @Test
+ void anUnknownParameterNameFailsItsEnum() {
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withProcessor("Lambda", "LambdaArn", "Bogus"), "extendedS3DestinationUpdate"));
+
+ assertEquals("1 validation error detected: Value at 'extendedS3DestinationUpdate"
+ + ".processingConfiguration.processors.1.member.parameters.2.member.parameterName'"
+ + " failed to satisfy constraint: Member must satisfy enum value set: "
+ + PARAMETER_NAME_SET, error.getMessage());
+ }
+
+ @Test
+ void aProcessorWithoutATypeFailsAsRequired() {
+ Processor processor = new Processor();
+ processor.setParameters(java.util.List.of(namedParameter("LambdaArn", "value")));
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withProcessors("extendedS3DestinationConfiguration", processor),
+ "extendedS3DestinationConfiguration"));
+
+ assertEquals("1 validation error detected: Value at 'extendedS3DestinationConfiguration"
+ + ".processingConfiguration.processors.1.member.type' failed to satisfy constraint:"
+ + " Member must not be null", error.getMessage());
+ }
+
+ @Test
+ void aParameterMissingEitherMemberFailsAsRequired() {
+ Processor processor = new Processor();
+ processor.setType("Lambda");
+ processor.setParameters(java.util.List.of(new ProcessorParameter()));
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withProcessors("extendedS3DestinationConfiguration", processor),
+ "extendedS3DestinationConfiguration"));
+
+ assertTrue(error.getMessage().contains("parameters.1.member.parameterName' failed to satisfy"
+ + " constraint: Member must not be null"), error.getMessage());
+ assertTrue(error.getMessage().contains("parameters.1.member.parameterValue' failed to satisfy"
+ + " constraint: Member must not be null"), error.getMessage());
+ }
+
+ // An empty value trips the length and the pattern, in that order.
+ @Test
+ void anEmptyParameterValueFailsBothItsConstraints() {
+ Processor processor = new Processor();
+ processor.setType("Lambda");
+ processor.setParameters(java.util.List.of(namedParameter("LambdaArn", "")));
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withProcessors("extendedS3DestinationConfiguration", processor),
+ "extendedS3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: Value at 'extendedS3DestinationConfiguration"
+ + ".processingConfiguration.processors.1.member.parameters.1.member.parameterValue'"
+ + " failed to satisfy constraint: Member must have length greater than or equal to 1;"
+ + " Value at 'extendedS3DestinationConfiguration.processingConfiguration.processors.1"
+ + ".member.parameters.1.member.parameterValue' failed to satisfy constraint: Member"
+ + " must satisfy regular expression pattern: ^(?!\\s*$).+", error.getMessage());
+ }
+
+ // Probed: real AWS answers InternalFailure for a null list member. Reproducing a
+ // fault of its own helps nobody, and skipping the entry leaves the configuration
+ // the caller would have had by omitting it.
+ @Test
+ void aNullListMemberIsIgnoredRatherThanReported() {
+ Processor processor = new Processor();
+ processor.setType("Lambda");
+ processor.setParameters(java.util.Arrays.asList(namedParameter("LambdaArn", "value"), null));
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setEnabled(true);
+ processing.setProcessors(java.util.Arrays.asList(processor, null));
+ S3Destination config = new S3Destination();
+ config.setProcessingConfiguration(processing);
+
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ config, "extendedS3DestinationConfiguration"));
+ }
+
+ @Test
+ void aKnownProcessorAndParameterPass() {
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withProcessor("Lambda", "LambdaArn", "NumberOfRetries"), "extendedS3DestinationConfiguration"));
+ }
+
private static S3Destination withCompressionFormat(String compressionFormat) {
S3Destination config = new S3Destination();
config.setCompressionFormat(compressionFormat);