Light, fluffy, and always free - The AWS Local Emulator alternative
feat(firehose): validate and persist ProcessingConfiguration (#3375)
Kenshin Okinakadebf9fb0dc72
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseProcessingConfigurationTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseProcessingConfigurationTest.javanew file mode 100644index 0000000000..0b40e593b6--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseProcessingConfigurationTest.java@@ -0,0 +1,121 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.services.firehose.FirehoseClient;+import software.amazon.awssdk.services.firehose.model.*;++import java.util.List;+import java.util.UUID;++import static org.assertj.core.api.Assertions.*;++/**+ * Happy-path proof that the SDK's marshalling of the nested ProcessingConfiguration+ * shapes round-trips through Floci: create, the echo AWS fills in, and the+ * whole-block replacement on UpdateDestination. The validation matrix lives in the+ * emulator's unit tests.+ */+@DisplayName("Firehose processing configuration")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class FirehoseProcessingConfigurationTest {++ private static FirehoseClient firehose;+ private static final String STREAM_NAME = "sdk-processing-" + UUID.randomUUID().toString().substring(0, 8);+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";+ private static final String BUCKET_ARN = "arn:aws:s3:::floci-firehose-sdk-test";+ private static final String LAMBDA_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";++ @BeforeAll+ static void setup() {+ firehose = TestFixtures.firehoseClient();+ }++ @AfterAll+ static void cleanup() {+ if (firehose != null) {+ try {+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME).build());+ } catch (Exception e) {+ System.err.println("[WARN] Best-effort cleanup failed for delivery stream "+ + STREAM_NAME + ": " + e.getMessage());+ }+ firehose.close();+ }+ }++ private static Processor lambdaProcessor(ProcessorParameter... parameters) {+ return Processor.builder().type(ProcessorType.LAMBDA).parameters(parameters).build();+ }++ private static ProcessorParameter parameter(ProcessorParameterName name, String value) {+ return ProcessorParameter.builder().parameterName(name).parameterValue(value).build();+ }++ private static ProcessingConfiguration stored() {+ return firehose.describeDeliveryStream(DescribeDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME).build())+ .deliveryStreamDescription().destinations().get(0)+ .extendedS3DestinationDescription().processingConfiguration();+ }++ private static String currentVersion() {+ return firehose.describeDeliveryStream(DescribeDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME).build())+ .deliveryStreamDescription().versionId();+ }++ @Test+ @Order(1)+ @DisplayName("Create accepts a Lambda processor and echoes it with AWS's own additions")+ void createWithProcessingConfiguration() {+ firehose.createDeliveryStream(CreateDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)+ .extendedS3DestinationConfiguration(ExtendedS3DestinationConfiguration.builder()+ .roleARN(ROLE_ARN)+ .bucketARN(BUCKET_ARN)+ .processingConfiguration(ProcessingConfiguration.builder()+ .enabled(true)+ .processors(lambdaProcessor(+ parameter(ProcessorParameterName.BUFFER_INTERVAL_IN_SECONDS, "70"),+ parameter(ProcessorParameterName.BUFFER_SIZE_IN_M_BS, "2"),+ parameter(ProcessorParameterName.LAMBDA_ARN, LAMBDA_ARN)))+ .build())+ .build())+ .build());++ ProcessingConfiguration processing = stored();+ assertThat(processing.enabled()).isTrue();+ assertThat(processing.processors()).hasSize(1);++ // NumberOfRetries and RoleArn are added by the service, and the order is its+ // own rather than the order the parameters were sent in.+ List<ProcessorParameter> parameters = processing.processors().get(0).parameters();+ assertThat(parameters).extracting(p -> p.parameterName().toString())+ .containsExactly("LambdaArn", "NumberOfRetries", "RoleArn",+ "BufferSizeInMBs", "BufferIntervalInSeconds");+ assertThat(parameters.get(1).parameterValue()).isEqualTo("3");+ assertThat(parameters.get(2).parameterValue()).isEqualTo(ROLE_ARN);+ }++ @Test+ @Order(2)+ @DisplayName("An update replaces the whole block rather than merging it")+ void updateReplacesTheBlock() {+ firehose.updateDestination(UpdateDestinationRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .currentDeliveryStreamVersionId(currentVersion())+ .destinationId("destinationId-000000000001")+ .extendedS3DestinationUpdate(ExtendedS3DestinationUpdate.builder()+ .processingConfiguration(ProcessingConfiguration.builder()+ .enabled(false)+ .build())+ .build())+ .build());++ ProcessingConfiguration processing = stored();+ assertThat(processing.enabled()).isFalse();+ assertThat(processing.processors()).isEmpty();+ }+}diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex 8d6af5a9d5..1d579f5c7f 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -83,6 +83,18 @@ An omitted `Enabled` counts as enabled; `Enabled: false` stores and merges the c- Error-output records report the delivery time as both `arrivalTimestamp` and `attemptEndingTimestamp` (Floci does not track per-record arrival), and omit AWS's `sequenceNumber`/`subSequenceNumber` members. An absent `ErrorOutputPrefix` writes them at the bucket root with no default time prefix, which matches AWS's object-name documentation but was not probed.- The Parquet object carries no `Content-Encoding`, where AWS labels it `hadoop-snappy` even though the body is a plain Parquet file (probed), and its row order may differ from put order, as it does on AWS (probed).+## Record transformation++`ProcessingConfiguration` on the extended S3 destination is modelled, validated and echoed, but **not applied**: a stream that enables a transformation is accepted and delivers its records untransformed, and create and update log a warning saying so. Validation follows AWS, down to the messages and the order they are reported in.++Two behaviors are worth knowing because they differ from the conversion block above. `UpdateDestination` replaces this block whole rather than merging it member-wise, so an update carrying only `{"Enabled": false}` leaves no processors behind, and each update is validated against its own content rather than the merged result. And a stored Lambda processor is echoed with `NumberOfRetries` and `RoleArn` filled in and its parameters reordered, as AWS returns them.++### Known deviations from AWS++- The transformation is not applied. A transform function is never invoked, so records reach the destination exactly as they were put, which is what makes a local test of one pass where AWS would not.+- Two checks AWS itself does not make are deliberately absent: `NumberOfRetries` is not range-checked, despite the documented 1 to 8, and the function a `LambdaArn` names is not required to exist.+- A null entry in `Processors` or `Parameters`, which a raw JSON client can send, is ignored rather than reported. Real AWS answers `InternalFailure` there, a fault of its own that is not worth reproducing; skipping the entry leaves the same configuration as omitting it would.+## S3 object keysDelivered objects are named `<evaluated prefix><streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid><file extension>`, matching [AWS's object name format](https://docs.aws.amazon.com/firehose/latest/dev/s3-object-name.html):diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.javaindex 3fc50bf16e..92e297ba1b 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java@@ -46,7 +46,7 @@ public class FirehoseJsonHandler {S3Destination s3 = null;if (request.has("S3DestinationConfiguration")) {s3 = mapper.treeToValue(request.get("S3DestinationConfiguration"), S3Destination.class);- dropConversionFromLegacyShape(s3);+ dropExtendedOnlyMembersFromLegacyShape(s3);S3DestinationValidator.validateWireShape(s3, "s3DestinationConfiguration");} else if (request.has("ExtendedS3DestinationConfiguration")) {s3 = mapper.treeToValue(request.get("ExtendedS3DestinationConfiguration"), S3Destination.class);@@ -82,7 +82,7 @@ public class FirehoseJsonHandler {S3DestinationValidator.validateWireShape(update, "extendedS3DestinationUpdate");} else if (request.has("S3DestinationUpdate")) {update = mapper.treeToValue(request.get("S3DestinationUpdate"), S3Destination.class);- dropConversionFromLegacyShape(update);+ dropExtendedOnlyMembersFromLegacyShape(update);S3DestinationValidator.validateWireShape(update, "s3DestinationUpdate");}firehoseService.updateDestination(name, currentVersionId, destinationId, update);@@ -198,24 +198,28 @@ public class FirehoseJsonHandler {}/**- * Clears {@code DataFormatConversionConfiguration} from a legacy- * {@code S3DestinationConfiguration} or {@code S3DestinationUpdate}, which AWS- * models only on the extended shapes. The two share one class here, which would- * otherwise let a legacy request store a member its contract does not define.+ * Clears {@code DataFormatConversionConfiguration} and+ * {@code ProcessingConfiguration} from a legacy {@code S3DestinationConfiguration}+ * or {@code S3DestinationUpdate}, which AWS models only on the extended shapes. The+ * two share one class here, which would otherwise let a legacy request store members+ * its contract does not define.** Dropped rather than rejected: AWS's JSON protocol ignores a member the shape does* not model, so a request carrying one succeeds with the member disregarded. The- * SDKs cannot even send it, since it is absent from the legacy shape's model.+ * SDKs cannot even send them, since both are absent from the legacy shape's model.+ * That was probed for the conversion member; the processing member is the same+ * mechanism on the same shape.*- * Only this member is cleared. {@code FileExtension}, {@code CustomTimeZone} and+ * Only these two are cleared. {@code FileExtension}, {@code CustomTimeZone} and* {@code S3BackupMode} are extended-only too and reach the legacy shapes the same* way, a deviation that predates this change and is recorded in* docs/services/firehose.md; widening the clearing would change behavior those* members already have.*/- private static void dropConversionFromLegacyShape(S3Destination s3) {+ private static void dropExtendedOnlyMembersFromLegacyShape(S3Destination s3) {if (s3 != null) {s3.setDataFormatConversionConfiguration(null);+ s3.setProcessingConfiguration(null);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 9846a0467b..0587e25e24 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -309,6 +309,11 @@ public class FirehoseService implements ResourceProvider {validateBufferingHints(s3Config);DataFormatConversionValidator.validateEffective(s3Config);+ ProcessingConfigurationValidator.validateEffective(s3Config);+ if (s3Config != null) {+ s3Config.canonicalizeProcessors();+ }+ warnIfProcessingEnabled(name, s3Config);String arn = AwsArnUtils.Arn.of("firehose", region, accountId,"deliverystream/" + name).toString();// CreateDeliveryStream's KinesisStreamSourceConfiguration carries only the ARN and@@ -364,7 +369,9 @@ public class FirehoseService implements ResourceProvider {S3Destination current = destination.getExtendedS3DestinationDescription();if (current == null) {DataFormatConversionValidator.validateEffective(update);+ ProcessingConfigurationValidator.validateEffective(update);update.applyDefaults();+ update.canonicalizeProcessors();destination.setExtendedS3DestinationDescription(update);} else {// AWS validates the merged effective state, not the update shape: enabling@@ -373,12 +380,19 @@ public class FirehoseService implements ResourceProvider {// Validating a merged view first also keeps a failed update from leaving a// half-merged destination behind in the memory backend.DataFormatConversionValidator.validateEffective(mergedView(current, update));+ // Not the merged view: an update carrying a ProcessingConfiguration replaces+ // the stored one outright, so AWS validates what the update itself says. A+ // processor without LambdaArn is rejected even when the stored one had it+ // (probed 2026-09-10, unlike the conversion block above).+ ProcessingConfigurationValidator.validateEffective(update);mergeDestination(current, update);+ current.canonicalizeProcessors();}stream.setVersionId(String.valueOf(parseVersionId(stream.getVersionId()) + 1));stream.setLastUpdateTimestamp(java.time.Instant.now());streamPut(streamKey, stream);LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);+ warnIfProcessingEnabled(name, stream.s3Destination());}public void startDeliveryStreamEncryption(String name, String keyType, String keyArn) {@@ -450,6 +464,20 @@ public class FirehoseService implements ResourceProvider {}}+ /**+ * Says, where the configuration is set, that the transformation will not be applied.+ * Fires on create and on every update that leaves it enabled, so a caller who keeps+ * changing the destination keeps being told. Deliberately not in the flush path,+ * which runs on every buffered delivery: create and update are caller-driven and are+ * the moments a caller can act on the warning.+ */+ private static void warnIfProcessingEnabled(String name, S3Destination s3) {+ if (s3 != null && s3.isProcessingEnabled()) {+ LOG.warnv("Delivery stream {0} enables a record transformation, which Floci does not"+ + " apply yet; its records will be delivered untransformed", name);+ }+ }+private static void mergeDestination(S3Destination current, S3Destination update) {if (update.getRoleArn() != null) current.setRoleArn(update.getRoleArn());if (update.getBucketArn() != null) current.setBucketArn(update.getBucketArn());@@ -461,6 +489,12 @@ public class FirehoseService implements ResourceProvider {if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());if (update.getS3BackupMode() != null) current.setS3BackupMode(update.getS3BackupMode());+ // Replaced whole, not merged member-wise: an update carrying only+ // {"Enabled": false} leaves Processors empty on real AWS, where the same shape+ // preserves the conversion block's members (probed 2026-09-10).+ if (update.getProcessingConfiguration() != null) {+ current.setProcessingConfiguration(update.getProcessingConfiguration());+ }if (update.getDataFormatConversionConfiguration() != null) {current.setDataFormatConversionConfiguration(mergeConversion(current.getDataFormatConversionConfiguration(), update.getDataFormatConversionConfiguration()));diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/ProcessingConfigurationValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/ProcessingConfigurationValidator.javanew file mode 100644index 0000000000..daf7d6a6f6--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/ProcessingConfigurationValidator.java@@ -0,0 +1,112 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;++import java.util.HashSet;+import java.util.List;+import java.util.Set;+import java.util.regex.Pattern;++/**+ * The semantic checks AWS runs on a ProcessingConfiguration once its modeled+ * constraints hold, in the order and with the wording probed against real AWS+ * (us-west-2, 2026-09-10).+ *+ * Deliberately absent: AWS does not range-check NumberOfRetries at configuration+ * time, accepting 0 and 9 alike despite the documented 1 to 8, and it does not+ * check that the function the LambdaArn names exists, unlike the Glue table a+ * conversion schema names. Both were probed; neither is enforced here.+ */+final class ProcessingConfigurationValidator {++ /**+ * Lambda's own ARN shape rather than "anything without a colon": probed, AWS+ * answers "Invalid lambda ARN." for a function name carrying a space or slashes,+ * which a looser pattern would have stored.+ */+ private static final Pattern LAMBDA_ARN = Pattern.compile(+ "arn:aws[a-zA-Z-]*:lambda:[a-z0-9-]+:\\d{12}:function:[a-zA-Z0-9_-]{1,64}"+ + "(:(\\$LATEST|[a-zA-Z0-9_-]{1,128}))?");++ private ProcessingConfigurationValidator() {}++ static void validateEffective(S3Destination config) {+ if (config == null || config.getProcessingConfiguration() == null) {+ return;+ }+ ProcessingConfiguration processing = config.getProcessingConfiguration();+ if (processing.getProcessors() == null) {+ return;+ }+ // Before anything a processor's own parameters say: a repeated name is rejected+ // even when the LambdaArn the processor needs is missing too (probed).+ int lambdaProcessors = 0;+ for (Processor processor : processing.getProcessors()) {+ if (processor == null) {+ continue;+ }+ rejectDuplicateParameters(processor.getParameters());+ if ("Lambda".equals(processor.getType())) {+ lambdaProcessors++;+ }+ }+ if (lambdaProcessors > 1) {+ throw invalidArgument("Cannot have more than 1 Lambda processor.");+ }+ for (Processor processor : processing.getProcessors()) {+ if (processor == null || !"Lambda".equals(processor.getType())) {+ continue;+ }+ validateLambdaProcessor(processor.getParameters());+ }+ }++ private static void rejectDuplicateParameters(List<ProcessorParameter> parameters) {+ if (parameters == null) {+ return;+ }+ Set<String> seen = new HashSet<>();+ for (ProcessorParameter parameter : parameters) {+ if (parameter != null && parameter.getParameterName() != null+ && !seen.add(parameter.getParameterName())) {+ throw invalidArgument("Duplicate ProcessorParameter passed to ProcessingConfiguration.");+ }+ }+ }++ private static void validateLambdaProcessor(List<ProcessorParameter> parameters) {+ String lambdaArn = parameterValue(parameters, "LambdaArn");+ if (lambdaArn == null || lambdaArn.isEmpty()) {+ throw invalidArgument("LambdaArn is required when Lambda processor is used.");+ }+ if (!LAMBDA_ARN.matcher(lambdaArn).matches()) {+ throw invalidArgument("Invalid lambda ARN.");+ }+ boolean hasSize = parameterValue(parameters, "BufferSizeInMBs") != null;+ boolean hasInterval = parameterValue(parameters, "BufferIntervalInSeconds") != null;+ if (hasSize != hasInterval) {+ throw invalidArgument("Both BufferSizeInMBs and BufferIntervalInSeconds are required"+ + " to configure buffering for lambda processor.");+ }+ }++ private static String parameterValue(List<ProcessorParameter> parameters, String name) {+ if (parameters == null) {+ return null;+ }+ for (ProcessorParameter parameter : parameters) {+ if (parameter != null && name.equals(parameter.getParameterName())) {+ return parameter.getParameterValue();+ }+ }+ return null;+ }++ private static AwsException invalidArgument(String message) {+ return new AwsException("InvalidArgumentException", message, 400);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.javaindex b88b0bf7a1..fbff3e63a4 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java@@ -4,12 +4,16 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;import java.util.ArrayList;import java.util.List;+import java.util.Set;import java.util.Map;import java.util.regex.Pattern;@@ -104,6 +108,7 @@ final class S3DestinationValidator {// violations sit in more than one of them.collectOpenXMappingViolations(config, shapeName, violations);collectSchemaViolations(config, shapeName, violations);+ collectProcessingViolations(config, shapeName, violations);if (!violations.isEmpty()) {throw constraintViolations(violations);}@@ -312,6 +317,101 @@ final class S3DestinationValidator {}}+ /**+ * The processor and parameter enums, in the order AWS prints them in a+ * constraint message, which is neither declaration nor alphabetical order+ * (probed 2026-09-10).+ */+ private static final String PROCESSOR_TYPE_SET =+ "[Decompression, Lambda, RecordDeAggregation, MetadataExtraction,"+ + " AppendDelimiterToRecord, CloudWatchLogProcessing]";+ private static final String PARAMETER_NAME_SET =+ "[DataMessageExtraction, NumberOfRetries, Delimiter, RoleArn, JsonParsingEngine,"+ + " SubRecordType, MetadataExtractionQuery, BufferIntervalInSeconds, LambdaArn,"+ + " BufferSizeInMBs, CompressionFormat]";+ private static final int PARAMETER_VALUE_MAX_LENGTH = 5120;+ private static final String PARAMETER_VALUE_REGEX = "^(?!\\s*$).+";+ private static final Pattern PARAMETER_VALUE_NOT_BLANK = Pattern.compile(PARAMETER_VALUE_REGEX);+ private static final Set<String> PROCESSOR_TYPES = Set.of("Decompression", "Lambda",+ "RecordDeAggregation", "MetadataExtraction", "AppendDelimiterToRecord",+ "CloudWatchLogProcessing");+ private static final Set<String> PARAMETER_NAMES = Set.of("DataMessageExtraction",+ "NumberOfRetries", "Delimiter", "RoleArn", "JsonParsingEngine", "SubRecordType",+ "MetadataExtractionQuery", "BufferIntervalInSeconds", "LambdaArn", "BufferSizeInMBs",+ "CompressionFormat");++ /**+ * Both members are required, and the value carries a length range and a+ * not-blank pattern. An empty value trips the length and the pattern, printed in+ * that order, as the probed message does.+ */+ private static void collectParameterViolations(ProcessorParameter parameter, String shapeName,+ String parameterPath, List<String> violations) {+ String name = parameter.getParameterName();+ if (name == null) {+ violations.add(violation(shapeName, parameterPath + ".parameterName",+ "Member must not be null"));+ } else if (!PARAMETER_NAMES.contains(name)) {+ violations.add(violation(shapeName, parameterPath + ".parameterName",+ "Member must satisfy enum value set: " + PARAMETER_NAME_SET));+ }+ String value = parameter.getParameterValue();+ if (value == null) {+ violations.add(violation(shapeName, parameterPath + ".parameterValue",+ "Member must not be null"));+ return;+ }+ if (value.length() < 1) {+ violations.add(violation(shapeName, parameterPath + ".parameterValue",+ "Member must have length greater than or equal to 1"));+ } else if (value.length() > PARAMETER_VALUE_MAX_LENGTH) {+ violations.add(violation(shapeName, parameterPath + ".parameterValue",+ "Member must have length less than or equal to " + PARAMETER_VALUE_MAX_LENGTH));+ }+ if (PARAMETER_VALUE_NOT_BLANK.matcher(value).matches()) {+ return;+ }+ violations.add(violation(shapeName, parameterPath + ".parameterValue",+ "Member must satisfy regular expression pattern: " + PARAMETER_VALUE_REGEX));+ }++ /** Members are numbered from 1 in these paths, as the probed messages show. */+ private static void collectProcessingViolations(S3Destination config, String shapeName,+ List<String> violations) {+ ProcessingConfiguration processing = config.getProcessingConfiguration();+ if (processing == null || processing.getProcessors() == null) {+ return;+ }+ List<Processor> processors = processing.getProcessors();+ for (int i = 0; i < processors.size(); i++) {+ Processor processor = processors.get(i);+ if (processor == null) {+ continue;+ }+ String processorPath = "processingConfiguration.processors." + (i + 1) + ".member";+ // The processor's own member before the ones nested in it, which is the+ // order AWS prints them when both are violated (probed).+ if (processor.getType() == null) {+ violations.add(violation(shapeName, processorPath + ".type", "Member must not be null"));+ } else if (!PROCESSOR_TYPES.contains(processor.getType())) {+ violations.add(violation(shapeName, processorPath + ".type",+ "Member must satisfy enum value set: " + PROCESSOR_TYPE_SET));+ }+ List<ProcessorParameter> parameters = processor.getParameters();+ if (parameters == null) {+ continue;+ }+ for (int j = 0; j < parameters.size(); j++) {+ ProcessorParameter parameter = parameters.get(j);+ if (parameter == null) {+ continue;+ }+ String parameterPath = processorPath + ".parameters." + (j + 1) + ".member";+ collectParameterViolations(parameter, shapeName, parameterPath, violations);+ }+ }+ }+private static String violation(String shapeName, String member, String constraint) {return "Value at '" + shapeName + "." + member + "' failed to satisfy constraint: " + constraint;}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.javaindex 444cdd097e..163982a4e0 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java@@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;import com.fasterxml.jackson.annotation.JsonProperty;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.ArrayList;@@ -196,6 +197,11 @@ public class DeliveryStreamDescription {@JsonIgnoreProperties(ignoreUnknown = true)@JsonInclude(JsonInclude.Include.NON_NULL)public static class S3Destination {++ /** The order AWS echoes a Lambda processor's parameters in, whatever order it received them (probed). */+ private static final List<String> LAMBDA_PARAMETER_ORDER = List.of("LambdaArn",+ "NumberOfRetries", "RoleArn", "BufferSizeInMBs", "BufferIntervalInSeconds");+@JsonProperty("RoleARN")private String roleArn;@JsonProperty("BucketARN")@@ -218,6 +224,8 @@ public class DeliveryStreamDescription {private String s3BackupMode;@JsonProperty("DataFormatConversionConfiguration")private DataFormatConversionConfiguration dataFormatConversionConfiguration;+ @JsonProperty("ProcessingConfiguration")+ private ProcessingConfiguration processingConfiguration;public S3Destination() {}public String getRoleArn() { return roleArn; }@@ -247,6 +255,27 @@ public class DeliveryStreamDescription {this.dataFormatConversionConfiguration = configuration;}+ public ProcessingConfiguration getProcessingConfiguration() {+ return processingConfiguration;+ }+ public void setProcessingConfiguration(ProcessingConfiguration processingConfiguration) {+ this.processingConfiguration = processingConfiguration;+ }++ /**+ * True when a transformation applies to deliveries. An omitted Enabled is taken+ * as enabled, following the conversion block, where that was probed; the+ * processing block's own behavior for an omitted Enabled was not, and nothing+ * but the "not applied yet" warning reads this today.+ *+ * Ignored for serialization for the same reason as the accessor below.+ */+ @JsonIgnore+ public boolean isProcessingEnabled() {+ return processingConfiguration != null+ && !Boolean.FALSE.equals(processingConfiguration.getEnabled());+ }+/*** True when data format conversion applies to deliveries: the configuration is* present and not explicitly disabled. AWS treats an omitted Enabled as enabled@@ -299,6 +328,57 @@ public class DeliveryStreamDescription {}}+ /**+ * Real AWS answers with more than the caller sent: a Lambda processor comes back+ * with NumberOfRetries defaulted to 3 and RoleArn filled from the delivery+ * stream's role, and the parameters are echoed in a fixed order whatever order+ * they arrived in (probed 2026-09-10).+ *+ * Called as a destination is written, not as it is read. Doing it on the way out+ * would mutate what storage handed back, so whether a Describe had happened would+ * decide what a later update sees, and RoleArn would be injected from whichever+ * role was current at the time of the read.+ */+ public void canonicalizeProcessors() {+ if (processingConfiguration == null || processingConfiguration.getProcessors() == null) {+ return;+ }+ for (Processor processor : processingConfiguration.getProcessors()) {+ if (processor == null || !"Lambda".equals(processor.getType())+ || processor.getParameters() == null) {+ continue;+ }+ // A repeated name never reaches here, the validator rejects it as AWS+ // does, so this only has to be stable for the names it does see.+ Map<String, String> byName = new LinkedHashMap<>();+ for (ProcessorParameter parameter : processor.getParameters()) {+ if (parameter != null && parameter.getParameterName() != null) {+ byName.putIfAbsent(parameter.getParameterName(), parameter.getParameterValue());+ }+ }+ byName.putIfAbsent("NumberOfRetries", "3");+ if (roleArn != null) {+ byName.putIfAbsent("RoleArn", roleArn);+ }+ List<ProcessorParameter> ordered = new ArrayList<>(byName.size());+ for (String name : LAMBDA_PARAMETER_ORDER) {+ if (byName.containsKey(name)) {+ ordered.add(parameter(name, byName.remove(name)));+ }+ }+ byName.forEach((name, value) -> ordered.add(parameter(name, value)));+ processor.setParameters(ordered);+ }+ }++ private static ProcessorParameter parameter(String name, String value) {+ ProcessorParameter parameter = new ProcessorParameter();+ parameter.setParameterName(name);+ parameter.setParameterValue(value);+ return parameter;+ }++/*** A view of this config with only the fields AWS's plain S3DestinationDescription* actually has - FileExtension, CustomTimeZone, and S3BackupMode are extended-only.@@ -325,6 +405,62 @@ public class DeliveryStreamDescription {}}+ /**+ * Unlike the conversion configuration below, UpdateDestination replaces this block+ * outright rather than merging it member-wise (probed), so a stored member never has+ * to survive an update that omits it.+ */+ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class ProcessingConfiguration {+ @JsonProperty("Enabled")+ private Boolean enabled;+ @JsonProperty("Processors")+ private List<Processor> processors;++ public ProcessingConfiguration() {}++ public Boolean getEnabled() { return enabled; }+ public void setEnabled(Boolean enabled) { this.enabled = enabled; }+ public List<Processor> getProcessors() { return processors; }+ public void setProcessors(List<Processor> processors) { this.processors = processors; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class Processor {+ @JsonProperty("Type")+ private String type;+ @JsonProperty("Parameters")+ private List<ProcessorParameter> parameters;++ public Processor() {}++ public String getType() { return type; }+ public void setType(String type) { this.type = type; }+ public List<ProcessorParameter> getParameters() { return parameters; }+ public void setParameters(List<ProcessorParameter> parameters) { this.parameters = parameters; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class ProcessorParameter {+ @JsonProperty("ParameterName")+ private String parameterName;+ @JsonProperty("ParameterValue")+ private String parameterValue;++ public ProcessorParameter() {}++ public String getParameterName() { return parameterName; }+ public void setParameterName(String parameterName) { this.parameterName = parameterName; }+ public String getParameterValue() { return parameterValue; }+ public void setParameterValue(String parameterValue) { this.parameterValue = parameterValue; }+ }+/*** Members stay null-honest, mirroring real AWS: DescribeDeliveryStream echoes the* configuration exactly as it was given (an empty ParquetSerDe stays {}, CatalogIddiff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.javaindex 8a0b118df0..4959e5c8b7 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java@@ -186,6 +186,49 @@ class FirehoseDataFormatConversionIntegrationTest {.body(DESCRIPTION_PATH + ".DataFormatConversionConfiguration", nullValue());}+ /**+ * The processing member is extended-only in AWS's model too, so the legacy shapes+ * drop it the same way. Wire-level rather than through the service, since the+ * dropping happens in the handler as the request is unmarshalled.+ */+ @Test+ @Order(7)+ void legacyS3DestinationShapeIgnoresTheProcessingMember() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "legacy-shape-processing-stream",+ "S3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "CompressionFormat": "UNCOMPRESSED",+ "ProcessingConfiguration": {+ "Enabled": true,+ "Processors": [ { "Type": "Lambda", "Parameters": [+ { "ParameterName": "LambdaArn",+ "ParameterValue": "arn:aws:lambda:us-east-1:000000000000:function:t" } ] } ]+ }+ }+ }+ """.formatted(ROLE_ARN, BUCKET_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"legacy-shape-processing-stream\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(DESCRIPTION_PATH + ".ProcessingConfiguration", nullValue());+ }+@Test@Order(4)void createRejectsCompressedDestinationsWithTheAwsMessage() {diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.javanew file mode 100644index 0000000000..09f03d8c95--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java@@ -0,0 +1,348 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import org.mockito.Mockito;++import java.util.ArrayList;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.function.Consumer;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * The messages, the enum orders and the storage behavior asserted here were probed+ * against real AWS (us-west-2, 2026-09-10). Two of them are deliberate absences:+ * NumberOfRetries is not range-checked, and the function a LambdaArn names is not+ * required to exist.+ */+class FirehoseProcessingConfigurationTest {++ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";+ private static final String LAMBDA_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";+ private static final String DESTINATION_ID = "destinationId-000000000001";+ private static final String BUFFER_PAIR_MESSAGE = "Both BufferSizeInMBs and BufferIntervalInSeconds"+ + " are required to configure buffering for lambda processor.";++ private FirehoseService service;++ @BeforeEach+ void setUp() {+ StorageFactory storageFactory = mock(StorageFactory.class);+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();+ when(storageFactory.create(anyString(), anyString(), any()))+ .thenAnswer(invocation -> backends.computeIfAbsent(+ invocation.getArgument(0) + "/" + invocation.getArgument(1),+ k -> AccountAwareStorageBackend.inMemory("000000000000")));+ EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);+ when(firehoseCfg.enabled()).thenReturn(true);+ when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);+ when(firehoseCfg.flushRecordCount()).thenReturn(0);+ EmulatorConfig.ServicesConfig servicesCfg = mock(EmulatorConfig.ServicesConfig.class);+ when(servicesCfg.firehose()).thenReturn(firehoseCfg);+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.services()).thenReturn(servicesCfg);+ RegionResolver regionResolver = mock(RegionResolver.class);+ when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");+ when(regionResolver.getAccountId()).thenReturn("000000000000");+ service = new FirehoseService(storageFactory, mock(S3Service.class),+ Mockito.mock(KinesisService.class), regionResolver, new MutableClock(), config,+ Mockito.mock(FirehoseParquetConverter.class));+ }++ private static ProcessorParameter parameter(String name, String value) {+ ProcessorParameter parameter = new ProcessorParameter();+ parameter.setParameterName(name);+ parameter.setParameterValue(value);+ return parameter;+ }++ private static ProcessingConfiguration processing(String type, ProcessorParameter... parameters) {+ Processor processor = new Processor();+ processor.setType(type);+ processor.setParameters(new ArrayList<>(List.of(parameters)));+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setEnabled(true);+ processing.setProcessors(new ArrayList<>(List.of(processor)));+ return processing;+ }++ private static ProcessingConfiguration lambdaProcessing(ProcessorParameter... parameters) {+ return processing("Lambda", parameters);+ }++ private static S3Destination destination(Consumer<S3Destination> customizer) {+ S3Destination s3 = new S3Destination();+ s3.setRoleArn(ROLE_ARN);+ s3.setBucketArn("arn:aws:s3:::results");+ customizer.accept(s3);+ return s3;+ }++ private String currentVersion(String name) {+ return service.describeDeliveryStream(name).getVersionId();+ }++ private List<String> storedParameterNames(String name) {+ List<ProcessorParameter> parameters = service.describeDeliveryStream(name).s3Destination()+ .getProcessingConfiguration().getProcessors().get(0).getParameters();+ List<String> names = new ArrayList<>();+ parameters.forEach(parameter -> names.add(parameter.getParameterName()));+ return names;+ }++ @Test+ void aLambdaProcessorWithoutALambdaArnIsRejected() {+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("NumberOfRetries", "2"))))));++ assertEquals("InvalidArgumentException", error.getErrorCode());+ assertEquals("LambdaArn is required when Lambda processor is used.", error.getMessage());+ }++ // Probed: AWS rejects a function name that is not one Lambda would accept, so the+ // shape has to be Lambda's rather than merely arn-looking.+ @ParameterizedTest+ @ValueSource(strings = {+ "not-an-arn",+ "arn:aws:lambda:us-east-1:000000000000:function:bad name",+ "arn:aws:lambda:us-east-1:000000000000:function:name/with/slashes",+ "arn:aws:lambda:us-east-1:000000000000:function:bad?name",+ "arn:aws:lambda:us-east-1:000000000000:function:bad@name",+ "arn:aws:lambda:us-east-1:000000000000:function:bad[name",+ })+ void aMalformedLambdaArnIsRejected(String lambdaArn) {+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("LambdaArn", lambdaArn))))));++ assertEquals("Invalid lambda ARN.", error.getMessage());+ }++ @ParameterizedTest+ @ValueSource(strings = {+ "arn:aws:lambda:us-east-1:000000000000:function:transform",+ "arn:aws:lambda:us-east-1:000000000000:function:transform:$LATEST",+ "arn:aws:lambda:us-east-1:000000000000:function:transform:prod",+ })+ void aQualifiedOrUnqualifiedFunctionArnIsAccepted(String lambdaArn) {+ assertDoesNotThrow(() -> service.createDeliveryStream("stream-" + lambdaArn.hashCode(),+ destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("LambdaArn", lambdaArn))))));+ }++ // A function that does not exist is still accepted: unlike a conversion schema's+ // Glue table, AWS resolves nothing at configuration time.+ @Test+ void aWellFormedArnToAnyFunctionIsAccepted() {+ assertDoesNotThrow(() -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("LambdaArn",+ "arn:aws:lambda:us-east-1:000000000000:function:never-created"))))));+ }++ @ParameterizedTest+ @ValueSource(strings = {"BufferSizeInMBs", "BufferIntervalInSeconds"})+ void oneBufferingParameterWithoutTheOtherIsRejected(String parameterName) {+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(+ parameter("LambdaArn", LAMBDA_ARN), parameter(parameterName, "2"))))));++ assertEquals(BUFFER_PAIR_MESSAGE, error.getMessage());+ }++ @Test+ void bothBufferingParametersTogetherAreAccepted() {+ assertDoesNotThrow(() -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(+ parameter("LambdaArn", LAMBDA_ARN),+ parameter("BufferSizeInMBs", "2"),+ parameter("BufferIntervalInSeconds", "70"))))));+ }++ // Deliberate: real AWS accepts both, despite the documented 1 to 8.+ @ParameterizedTest+ @ValueSource(strings = {"0", "9"})+ void numberOfRetriesIsNotRangeChecked(String retries) {+ assertDoesNotThrow(() -> service.createDeliveryStream("stream-" + retries,+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(+ parameter("LambdaArn", LAMBDA_ARN), parameter("NumberOfRetries", retries))))));+ }++ // The LambdaArn rule belongs to the Lambda processor, not to every processor.+ @Test+ void anotherProcessorTypeIsNotSubjectToTheLambdaRules() {+ assertDoesNotThrow(() -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(+ processing("AppendDelimiterToRecord", parameter("Delimiter", "\\n"))))));+ }++ @Test+ void storedParametersGainTheDefaultsAndAFixedOrder() {+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("BufferIntervalInSeconds", "70"),+ parameter("BufferSizeInMBs", "2"),+ parameter("LambdaArn", LAMBDA_ARN)))));++ assertEquals(List.of("LambdaArn", "NumberOfRetries", "RoleArn", "BufferSizeInMBs",+ "BufferIntervalInSeconds"), storedParameterNames("stream"));++ List<ProcessorParameter> stored = service.describeDeliveryStream("stream").s3Destination()+ .getProcessingConfiguration().getProcessors().get(0).getParameters();+ assertEquals("3", stored.get(1).getParameterValue());+ assertEquals(ROLE_ARN, stored.get(2).getParameterValue());+ }++ @Test+ void aRepeatedParameterNameIsRejected() {+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(+ parameter("LambdaArn", LAMBDA_ARN), parameter("LambdaArn", "not-an-arn"))))));++ assertEquals("Duplicate ProcessorParameter passed to ProcessingConfiguration.", error.getMessage());+ }++ // The duplicate check runs first: the processor is missing its LambdaArn as well,+ // and AWS still reports the duplicate.+ @Test+ void aRepeatedParameterIsReportedBeforeTheMissingLambdaArn() {+ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing(+ parameter("NumberOfRetries", "2"), parameter("NumberOfRetries", "3"))))));++ assertEquals("Duplicate ProcessorParameter passed to ProcessingConfiguration.", error.getMessage());+ }++ @Test+ void moreThanOneLambdaProcessorIsRejected() {+ Processor first = new Processor();+ first.setType("Lambda");+ first.setParameters(List.of(parameter("LambdaArn", LAMBDA_ARN)));+ Processor second = new Processor();+ second.setType("Lambda");+ second.setParameters(List.of(parameter("LambdaArn", LAMBDA_ARN)));+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setEnabled(true);+ processing.setProcessors(List.of(first, second));++ AwsException error = assertThrows(AwsException.class, () -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(processing))));++ assertEquals("Cannot have more than 1 Lambda processor.", error.getMessage());+ }++ // A Lambda processor alongside another type is fine.+ @Test+ void twoProcessorsOfDifferentTypesAreAccepted() {+ Processor lambda = new Processor();+ lambda.setType("Lambda");+ lambda.setParameters(List.of(parameter("LambdaArn", LAMBDA_ARN)));+ Processor delimiter = new Processor();+ delimiter.setType("AppendDelimiterToRecord");+ delimiter.setParameters(List.of(parameter("Delimiter", "\\n")));+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setEnabled(true);+ processing.setProcessors(List.of(lambda, delimiter));++ assertDoesNotThrow(() -> service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(processing))));+ }++ // Settled when written, so reading cannot change what a later update sees: the+ // injected RoleArn must be the same whether or not a Describe happened in between.+ @Test+ void describeDoesNotDecideWhatALaterUpdateSees() {+ service.createDeliveryStream("read-first", destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));+ service.createDeliveryStream("no-read", destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));++ service.describeDeliveryStream("read-first");++ String otherRole = "arn:aws:iam::000000000000:role/other";+ for (String name : List.of("read-first", "no-read")) {+ service.updateDestination(name, currentVersion(name), DESTINATION_ID,+ destination(s3 -> s3.setRoleArn(otherRole)));+ }++ assertEquals(storedRoleParameter("read-first"), storedRoleParameter("no-read"));+ assertEquals(ROLE_ARN, storedRoleParameter("no-read"));+ }++ private String storedRoleParameter(String name) {+ for (ProcessorParameter parameter : service.describeDeliveryStream(name).s3Destination()+ .getProcessingConfiguration().getProcessors().get(0).getParameters()) {+ if ("RoleArn".equals(parameter.getParameterName())) {+ return parameter.getParameterValue();+ }+ }+ return null;+ }++ // The opposite of the conversion block, which merges member-wise.+ @Test+ void anUpdateReplacesTheWholeBlockRatherThanMergingIt() {+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));++ ProcessingConfiguration disabled = new ProcessingConfiguration();+ disabled.setEnabled(false);+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,+ destination(s3 -> s3.setProcessingConfiguration(disabled)));++ ProcessingConfiguration stored = service.describeDeliveryStream("stream").s3Destination()+ .getProcessingConfiguration();+ assertEquals(false, stored.getEnabled());+ assertTrue(stored.getProcessors() == null || stored.getProcessors().isEmpty(),+ "processors were " + stored.getProcessors());+ }++ @Test+ void anUpdateThatDoesNotMentionTheBlockPreservesIt() {+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));++ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,+ destination(s3 -> s3.setPrefix("changed/")));++ assertEquals(List.of("LambdaArn", "NumberOfRetries", "RoleArn"), storedParameterNames("stream"));+ }++ // Validation runs against the update's own content, since the update replaces the+ // block: a stored LambdaArn does not satisfy an update that omits one.+ @Test+ void anUpdateIsValidatedAgainstItsOwnContent() {+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)))));++ AwsException error = assertThrows(AwsException.class, () -> service.updateDestination("stream",+ currentVersion("stream"), DESTINATION_ID, destination(s3 -> s3.setProcessingConfiguration(+ lambdaProcessing(parameter("NumberOfRetries", "7"))))));++ assertEquals("LambdaArn is required when Lambda processor is used.", error.getMessage());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.javaindex a1424d51e9..e13309c1df 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java@@ -2,6 +2,9 @@ package io.github.hectorvent.floci.services.firehose;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;@@ -10,6 +13,7 @@ import org.junit.jupiter.params.provider.ValueSource;import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;/*** The error codes and messages asserted here were captured from real AWS's raw@@ -19,6 +23,139 @@ class S3DestinationValidatorTest {private static final String ENUM_SET = "[ZIP, HADOOP_SNAPPY, Snappy, GZIP, UNCOMPRESSED]";+ private static final String PROCESSOR_TYPE_SET = "[Decompression, Lambda, RecordDeAggregation,"+ + " MetadataExtraction, AppendDelimiterToRecord, CloudWatchLogProcessing]";+ private static final String PARAMETER_NAME_SET = "[DataMessageExtraction, NumberOfRetries, Delimiter,"+ + " RoleArn, JsonParsingEngine, SubRecordType, MetadataExtractionQuery,"+ + " BufferIntervalInSeconds, LambdaArn, BufferSizeInMBs, CompressionFormat]";++ private static ProcessorParameter namedParameter(String name, String value) {+ ProcessorParameter parameter = new ProcessorParameter();+ parameter.setParameterName(name);+ parameter.setParameterValue(value);+ return parameter;+ }++ private static S3Destination withProcessors(String unusedShapeName, Processor... processors) {+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setEnabled(true);+ processing.setProcessors(java.util.List.of(processors));+ S3Destination config = new S3Destination();+ config.setProcessingConfiguration(processing);+ return config;+ }++ private static S3Destination withProcessor(String type, String... parameterNames) {+ Processor processor = new Processor();+ processor.setType(type);+ java.util.List<ProcessorParameter> parameters = new java.util.ArrayList<>();+ for (String name : parameterNames) {+ ProcessorParameter parameter = new ProcessorParameter();+ parameter.setParameterName(name);+ parameter.setParameterValue("value");+ parameters.add(parameter);+ }+ processor.setParameters(parameters);+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setEnabled(true);+ processing.setProcessors(java.util.List.of(processor));+ S3Destination config = new S3Destination();+ config.setProcessingConfiguration(processing);+ return config;+ }++ // The enum sets and the 1-based member paths are the ones AWS prints (probed).+ @Test+ void anUnknownProcessorTypeFailsItsEnum() {+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withProcessor("Bogus"), "extendedS3DestinationConfiguration"));++ assertEquals("1 validation error detected: Value at 'extendedS3DestinationConfiguration"+ + ".processingConfiguration.processors.1.member.type' failed to satisfy constraint:"+ + " Member must satisfy enum value set: " + PROCESSOR_TYPE_SET, error.getMessage());+ }++ @Test+ void anUnknownParameterNameFailsItsEnum() {+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withProcessor("Lambda", "LambdaArn", "Bogus"), "extendedS3DestinationUpdate"));++ assertEquals("1 validation error detected: Value at 'extendedS3DestinationUpdate"+ + ".processingConfiguration.processors.1.member.parameters.2.member.parameterName'"+ + " failed to satisfy constraint: Member must satisfy enum value set: "+ + PARAMETER_NAME_SET, error.getMessage());+ }++ @Test+ void aProcessorWithoutATypeFailsAsRequired() {+ Processor processor = new Processor();+ processor.setParameters(java.util.List.of(namedParameter("LambdaArn", "value")));+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withProcessors("extendedS3DestinationConfiguration", processor),+ "extendedS3DestinationConfiguration"));++ assertEquals("1 validation error detected: Value at 'extendedS3DestinationConfiguration"+ + ".processingConfiguration.processors.1.member.type' failed to satisfy constraint:"+ + " Member must not be null", error.getMessage());+ }++ @Test+ void aParameterMissingEitherMemberFailsAsRequired() {+ Processor processor = new Processor();+ processor.setType("Lambda");+ processor.setParameters(java.util.List.of(new ProcessorParameter()));+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withProcessors("extendedS3DestinationConfiguration", processor),+ "extendedS3DestinationConfiguration"));++ assertTrue(error.getMessage().contains("parameters.1.member.parameterName' failed to satisfy"+ + " constraint: Member must not be null"), error.getMessage());+ assertTrue(error.getMessage().contains("parameters.1.member.parameterValue' failed to satisfy"+ + " constraint: Member must not be null"), error.getMessage());+ }++ // An empty value trips the length and the pattern, in that order.+ @Test+ void anEmptyParameterValueFailsBothItsConstraints() {+ Processor processor = new Processor();+ processor.setType("Lambda");+ processor.setParameters(java.util.List.of(namedParameter("LambdaArn", "")));+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withProcessors("extendedS3DestinationConfiguration", processor),+ "extendedS3DestinationConfiguration"));++ assertEquals("2 validation errors detected: Value at 'extendedS3DestinationConfiguration"+ + ".processingConfiguration.processors.1.member.parameters.1.member.parameterValue'"+ + " failed to satisfy constraint: Member must have length greater than or equal to 1;"+ + " Value at 'extendedS3DestinationConfiguration.processingConfiguration.processors.1"+ + ".member.parameters.1.member.parameterValue' failed to satisfy constraint: Member"+ + " must satisfy regular expression pattern: ^(?!\\s*$).+", error.getMessage());+ }++ // Probed: real AWS answers InternalFailure for a null list member. Reproducing a+ // fault of its own helps nobody, and skipping the entry leaves the configuration+ // the caller would have had by omitting it.+ @Test+ void aNullListMemberIsIgnoredRatherThanReported() {+ Processor processor = new Processor();+ processor.setType("Lambda");+ processor.setParameters(java.util.Arrays.asList(namedParameter("LambdaArn", "value"), null));+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setEnabled(true);+ processing.setProcessors(java.util.Arrays.asList(processor, null));+ S3Destination config = new S3Destination();+ config.setProcessingConfiguration(processing);++ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ config, "extendedS3DestinationConfiguration"));+ }++ @Test+ void aKnownProcessorAndParameterPass() {+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withProcessor("Lambda", "LambdaArn", "NumberOfRetries"), "extendedS3DestinationConfiguration"));+ }+private static S3Destination withCompressionFormat(String compressionFormat) {S3Destination config = new S3Destination();config.setCompressionFormat(compressionFormat);