Light, fluffy, and always free - The AWS Local Emulator alternative
feat(firehose): validate and persist DataFormatConversionConfiguration (#3236)
Kenshin Okinaka8a122d2b8cf9
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseDataFormatConversionTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseDataFormatConversionTest.javanew file mode 100644index 0000000000..0a18feef35--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseDataFormatConversionTest.java@@ -0,0 +1,141 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.services.firehose.FirehoseClient;+import software.amazon.awssdk.services.firehose.model.*;++import java.util.UUID;++import static org.assertj.core.api.Assertions.*;++/**+ * Happy-path proof that the SDK's marshalling of the nested+ * DataFormatConversionConfiguration shapes round-trips through Floci:+ * create, describe echo, and the member-wise merge on UpdateDestination.+ * The validation matrix (error messages and precedence) lives in the+ * emulator's unit tests.+ */+@DisplayName("Firehose data format conversion configuration")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class FirehoseDataFormatConversionTest {++ private static FirehoseClient firehose;+ private static final String STREAM_NAME = "sdk-format-conversion-" + UUID.randomUUID().toString().substring(0, 8);+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";+ private static final String BUCKET_ARN = "arn:aws:s3:::floci-firehose-sdk-test";++ @BeforeAll+ static void setup() {+ firehose = TestFixtures.firehoseClient();+ }++ @AfterAll+ static void cleanup() {+ if (firehose != null) {+ try {+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME).build());+ } catch (Exception e) {+ System.err.println("[WARN] Best-effort cleanup failed for delivery stream "+ + STREAM_NAME + ": " + e.getMessage());+ }+ firehose.close();+ }+ }++ @Test+ @Order(1)+ @DisplayName("Create accepts a full conversion configuration")+ void createWithConversionConfiguration() {+ CreateDeliveryStreamResponse response = firehose.createDeliveryStream(+ CreateDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)+ .extendedS3DestinationConfiguration(ExtendedS3DestinationConfiguration.builder()+ .roleARN(ROLE_ARN)+ .bucketARN(BUCKET_ARN)+ .compressionFormat(CompressionFormat.UNCOMPRESSED)+ .bufferingHints(BufferingHints.builder()+ .sizeInMBs(64)+ .intervalInSeconds(60)+ .build())+ .dataFormatConversionConfiguration(DataFormatConversionConfiguration.builder()+ .enabled(true)+ .schemaConfiguration(SchemaConfiguration.builder()+ .roleARN(ROLE_ARN)+ .databaseName("analytics")+ .tableName("events")+ .region("us-east-1")+ .build())+ .inputFormatConfiguration(InputFormatConfiguration.builder()+ .deserializer(Deserializer.builder()+ .openXJsonSerDe(OpenXJsonSerDe.builder().build())+ .build())+ .build())+ .outputFormatConfiguration(OutputFormatConfiguration.builder()+ .serializer(Serializer.builder()+ .parquetSerDe(ParquetSerDe.builder()+ .compression(ParquetCompression.GZIP)+ .writerVersion(ParquetWriterVersion.V2)+ .build())+ .build())+ .build())+ .build())+ .build())+ .build());++ assertThat(response.deliveryStreamARN()).contains(STREAM_NAME);+ }++ @Test+ @Order(2)+ @DisplayName("Describe echoes the conversion configuration as given")+ void describeEchoesConversionConfiguration() {+ ExtendedS3DestinationDescription destination = firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder().deliveryStreamName(STREAM_NAME).build())+ .deliveryStreamDescription().destinations().get(0).extendedS3DestinationDescription();++ DataFormatConversionConfiguration conversion = destination.dataFormatConversionConfiguration();+ assertThat(conversion).isNotNull();+ assertThat(conversion.enabled()).isTrue();+ assertThat(conversion.schemaConfiguration().roleARN()).isEqualTo(ROLE_ARN);+ assertThat(conversion.schemaConfiguration().databaseName()).isEqualTo("analytics");+ assertThat(conversion.schemaConfiguration().tableName()).isEqualTo("events");+ assertThat(conversion.schemaConfiguration().region()).isEqualTo("us-east-1");+ assertThat(conversion.schemaConfiguration().catalogId()).isNull();+ assertThat(conversion.inputFormatConfiguration().deserializer().openXJsonSerDe()).isNotNull();+ ParquetSerDe parquet = conversion.outputFormatConfiguration().serializer().parquetSerDe();+ assertThat(parquet.compression()).isEqualTo(ParquetCompression.GZIP);+ assertThat(parquet.writerVersion()).isEqualTo(ParquetWriterVersion.V2);+ assertThat(parquet.blockSizeBytes()).isNull();+ }++ @Test+ @Order(3)+ @DisplayName("UpdateDestination with only Enabled=false keeps the stored members")+ void updateDisablingConversionKeepsStoredMembers() {+ DeliveryStreamDescription described = firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder().deliveryStreamName(STREAM_NAME).build())+ .deliveryStreamDescription();++ firehose.updateDestination(UpdateDestinationRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .currentDeliveryStreamVersionId(described.versionId())+ .destinationId(described.destinations().get(0).destinationId())+ .extendedS3DestinationUpdate(ExtendedS3DestinationUpdate.builder()+ .dataFormatConversionConfiguration(DataFormatConversionConfiguration.builder()+ .enabled(false)+ .build())+ .build())+ .build());++ DataFormatConversionConfiguration conversion = firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder().deliveryStreamName(STREAM_NAME).build())+ .deliveryStreamDescription().destinations().get(0)+ .extendedS3DestinationDescription().dataFormatConversionConfiguration();++ assertThat(conversion.enabled()).isFalse();+ assertThat(conversion.schemaConfiguration().databaseName()).isEqualTo("analytics");+ assertThat(conversion.outputFormatConfiguration().serializer().parquetSerDe()).isNotNull();+ }+}diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex a5946664cc..90185e8fb2 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -57,6 +57,21 @@ The two Snappy variants share an extension and are told apart only by `Content-EAny other value, including differently-cased ones such as `SNAPPY` instead of `Snappy`, is rejected with `ValidationException`.+## Data format conversion++`DataFormatConversionConfiguration` on the extended S3 destination is accepted, validated, persisted, and echoed back by `DescribeDeliveryStream`. **The conversion itself, JSON to Parquet at delivery time, is not implemented yet**: a conversion-enabled stream delivers its records unconverted, under the usual key format, and creating or updating one logs a warning saying so.++Validation matches AWS on both `CreateDeliveryStream` and `UpdateDestination`, down to the [modeled](https://docs.aws.amazon.com/firehose/latest/APIReference/API_DataFormatConversionConfiguration.html) bounds, enums and patterns, so a raw JSON client cannot store a configuration AWS would reject. Two rules are worth knowing because they constrain the destination rather than the conversion block: `CompressionFormat` must be `UNCOMPRESSED`, compression being chosen by the output serializer instead, and an explicitly specified `BufferingHints.SizeInMBs` must be at least 64. Omitting `BufferingHints` stores 128 MiB, AWS's larger default for converting streams, rather than the ordinary 5 MiB.++An omitted `Enabled` counts as enabled; `Enabled: false` stores and merges the configuration without applying those two rules. `UpdateDestination` merges member-wise, nested `SchemaConfiguration` members included, then validates the merged result, so an update carrying only `{"Enabled": false}` keeps the stored schema and formats, and one naming only a new `TableName` keeps the stored role, database and region.++### Known deviations from AWS++- **An enabled `OrcSerDe` is rejected** with `InvalidArgumentException` at create/update time. Real AWS accepts it; Floci's planned conversion engine (DuckDB) cannot write ORC, and refusing early beats storing a configuration that could never deliver. A disabled one is stored and echoed with its ORC members intact.+- **The Glue table is not resolved at create or update time.** Real AWS rejects a `SchemaConfiguration` naming a database or table that does not exist, with `InvalidArgumentException` wrapping a Glue `EntityNotFoundException`; Floci accepts it. Floci performs no role-assumption check either, where AWS validates the role last.+- **Two kinds of validation are skipped**: the element constraints AWS puts on `OrcSerDe.BloomFilterColumns` and `HiveJsonSerDe.TimestampFormats` entries (the equivalent ones on `ColumnToJsonKeyMappings` are enforced), and AWS's semantic check of SerDe option values, which answers `InvalidArgumentException` ("Invalid deserializer option(s): ...") for a well-formed but unusable value.+- A malformed but non-empty `SchemaConfiguration.RoleARN` is accepted. AWS answers `InvalidParameterValueException` ("Invalid role ARN.") before its modeled rules run; Floci validates no ARNs, so it applies only the length and pattern rules, which an empty value trips exactly as on AWS.+## S3 object keysDelivered objects are named `<evaluated prefix><streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid><file extension>`, matching [AWS's object name format](https://docs.aws.amazon.com/firehose/latest/dev/s3-object-name.html):diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/DataFormatConversionValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/DataFormatConversionValidator.javanew file mode 100644index 0000000000..6b41fe723e--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/DataFormatConversionValidator.java@@ -0,0 +1,108 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.DataFormatConversionConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Deserializer;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;++/**+ * Validates a destination whose data format conversion is enabled, with the+ * messages and precedence captured from real AWS (probe 2026-09-07, both+ * CreateDeliveryStream and UpdateDestination): compression format first, then+ * the buffering size floor, then the three member null checks in+ * Input/Output/Schema order, then the members inside them. On+ * UpdateDestination real AWS validates the merged effective state, which is+ * why this runs against a merged view rather than the update shape.+ *+ * All checks are skipped when conversion is absent or explicitly disabled; an+ * omitted Enabled counts as enabled. The buffering floor applies only to an+ * explicitly specified SizeInMBs: AWS accepts a destination with no+ * BufferingHints at all (probe: it proceeds to role validation).+ *+ * The OrcSerDe rejection is Floci's own: real AWS accepts it, but Floci's+ * conversion engine (DuckDB) cannot write ORC, and refusing at create time+ * beats accepting a configuration that could never deliver.+ */+final class DataFormatConversionValidator {++ private static final int MIN_BUFFERING_SIZE_MBS = 64;++ private DataFormatConversionValidator() {}++ static void validateEffective(S3Destination config) {+ if (config == null || !config.isDataFormatConversionEnabled()) {+ return;+ }+ if (config.getCompressionFormat() != null && !"UNCOMPRESSED".equals(config.getCompressionFormat())) {+ throw invalidArgument("The S3 destination's compression format must be set to UNCOMPRESSED"+ + " when data format conversion is enabled. To enable compression within the converted"+ + " output format, set the appropriate serialization option in the output format configuration.");+ }+ if (config.getBufferingHints() != null && config.getBufferingHints().getSizeInMBs() != null+ && config.getBufferingHints().getSizeInMBs() < MIN_BUFFERING_SIZE_MBS) {+ throw invalidArgument("BufferingHints.SizeInMBs must be at least " + MIN_BUFFERING_SIZE_MBS+ + " when data format conversion is enabled.");+ }+ DataFormatConversionConfiguration conversion = config.getDataFormatConversionConfiguration();+ if (conversion.getInputFormatConfiguration() == null) {+ throw invalidArgument("InputFormatConfiguration must not be null");+ }+ if (conversion.getOutputFormatConfiguration() == null) {+ throw invalidArgument("OutputFormatConfiguration must not be null");+ }+ if (conversion.getSchemaConfiguration() == null) {+ throw invalidArgument("SchemaConfiguration must not be null");+ }+ validateDeserializer(conversion.getInputFormatConfiguration().getDeserializer());+ validateSerializer(conversion.getOutputFormatConfiguration().getSerializer());+ validateSchema(conversion.getSchemaConfiguration());+ }++ /** An empty Deserializer object reports the same error as an absent one (probe a9). */+ private static void validateDeserializer(Deserializer deserializer) {+ int specified = deserializer == null ? 0+ : (deserializer.getOpenXJsonSerDe() != null ? 1 : 0)+ + (deserializer.getHiveJsonSerDe() != null ? 1 : 0);+ if (specified == 0) {+ throw invalidArgument("Deserializer must not be null");+ }+ if (specified > 1) {+ throw invalidArgument("More than one deserializer specified. Only one may be chosen.");+ }+ }++ private static void validateSerializer(Serializer serializer) {+ int specified = serializer == null ? 0+ : (serializer.getParquetSerDe() != null ? 1 : 0)+ + (serializer.getOrcSerDe() != null ? 1 : 0);+ if (specified == 0) {+ throw invalidArgument("Serializer must not be null");+ }+ if (specified > 1) {+ throw invalidArgument("More than one serializer specified. Only one may be chosen.");+ }+ if (serializer.getOrcSerDe() != null) {+ throw invalidArgument("OrcSerDe is not supported. Floci supports only ParquetSerDe"+ + " for data format conversion.");+ }+ }++ /** AWS's message spells the member RoleArn even though the wire member is RoleARN. */+ private static void validateSchema(SchemaConfiguration schema) {+ requireSchemaMember(schema.getRoleArn(), "RoleArn");+ requireSchemaMember(schema.getDatabaseName(), "DatabaseName");+ requireSchemaMember(schema.getTableName(), "TableName");+ }++ private static void requireSchemaMember(String value, String member) {+ if (value == null || value.isEmpty()) {+ throw invalidArgument("SchemaConfiguration." + member + " must not be null or empty");+ }+ }++ private static AwsException invalidArgument(String message) {+ return new AwsException("InvalidArgumentException", message, 400);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.javaindex 58f049512c..3fc50bf16e 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java@@ -46,6 +46,7 @@ public class FirehoseJsonHandler {S3Destination s3 = null;if (request.has("S3DestinationConfiguration")) {s3 = mapper.treeToValue(request.get("S3DestinationConfiguration"), S3Destination.class);+ dropConversionFromLegacyShape(s3);S3DestinationValidator.validateWireShape(s3, "s3DestinationConfiguration");} else if (request.has("ExtendedS3DestinationConfiguration")) {s3 = mapper.treeToValue(request.get("ExtendedS3DestinationConfiguration"), S3Destination.class);@@ -81,6 +82,7 @@ public class FirehoseJsonHandler {S3DestinationValidator.validateWireShape(update, "extendedS3DestinationUpdate");} else if (request.has("S3DestinationUpdate")) {update = mapper.treeToValue(request.get("S3DestinationUpdate"), S3Destination.class);+ dropConversionFromLegacyShape(update);S3DestinationValidator.validateWireShape(update, "s3DestinationUpdate");}firehoseService.updateDestination(name, currentVersionId, destinationId, update);@@ -195,6 +197,28 @@ public class FirehoseJsonHandler {};}+ /**+ * Clears {@code DataFormatConversionConfiguration} from a legacy+ * {@code S3DestinationConfiguration} or {@code S3DestinationUpdate}, which AWS+ * models only on the extended shapes. The two share one class here, which would+ * otherwise let a legacy request store a member its contract does not define.+ *+ * Dropped rather than rejected: AWS's JSON protocol ignores a member the shape does+ * not model, so a request carrying one succeeds with the member disregarded. The+ * SDKs cannot even send it, since it is absent from the legacy shape's model.+ *+ * Only this member is cleared. {@code FileExtension}, {@code CustomTimeZone} and+ * {@code S3BackupMode} are extended-only too and reach the legacy shapes the same+ * way, a deviation that predates this change and is recorded in+ * docs/services/firehose.md; widening the clearing would change behavior those+ * members already have.+ */+ private static void dropConversionFromLegacyShape(S3Destination s3) {+ if (s3 != null) {+ s3.setDataFormatConversionConfiguration(null);+ }+ }+/** DeliveryStreamType values the 2015-08-04 model enumerates. */private static final java.util.Set<String> DELIVERY_STREAM_TYPES = java.util.Set.of("DirectPut", "KinesisStreamAsSource", "MSKAsSource", "DatabaseAsSource");diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 430486ead9..3305881808 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -219,6 +219,7 @@ public class FirehoseService implements ResourceProvider {}validateBufferingHints(s3Config);+ DataFormatConversionValidator.validateEffective(s3Config);String arn = AwsArnUtils.Arn.of("firehose", region, regionResolver.getAccountId(),"deliverystream/" + name).toString();// CreateDeliveryStream's KinesisStreamSourceConfiguration carries only the ARN and@@ -239,6 +240,7 @@ public class FirehoseService implements ResourceProvider {streamStore.put(name, description);buffers.put(name, Collections.synchronizedList(new ArrayList<>()));LOG.infov("Created Firehose delivery stream: {0}", name);+ warnIfConversionEnabled(name, s3Config);return arn;}@@ -263,15 +265,42 @@ public class FirehoseService implements ResourceProvider {validateBufferingHints(update);S3Destination current = destination.getExtendedS3DestinationDescription();if (current == null) {+ DataFormatConversionValidator.validateEffective(update);update.applyDefaults();destination.setExtendedS3DestinationDescription(update);} else {+ // AWS validates the merged effective state, not the update shape: enabling+ // conversion is rejected while a stored GZIP is in effect, and re-enabling+ // with only {"Enabled": true} succeeds on the stored schema (probe u5/u6).+ // Validating a merged view first also keeps a failed update from leaving a+ // half-merged destination behind in the memory backend.+ DataFormatConversionValidator.validateEffective(mergedView(current, update));mergeDestination(current, update);}stream.setVersionId(String.valueOf(parseVersionId(stream.getVersionId()) + 1));stream.setLastUpdateTimestamp(java.time.Instant.now());streamStore.put(name, stream);LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);+ warnIfConversionEnabled(name, stream.s3Destination());+ }++ /**+ * Says, where the configuration is set, that conversion will not be applied. Fires+ * on every create and on every update that leaves conversion enabled, including an+ * update about something else, so a caller who keeps changing the destination keeps+ * being told.+ *+ * Deliberately not in the flush path: that runs on every buffered delivery and on+ * every retry after a failed write, so warning there floods the log and needs a+ * per-stream marker whose lifetime has to track creates and deletes. Create and+ * update are caller-driven and rare by comparison, and they are the moments a+ * caller can act on the warning anyway.+ */+ private static void warnIfConversionEnabled(String name, S3Destination s3) {+ if (s3 != null && s3.isDataFormatConversionEnabled()) {+ LOG.warnv("Delivery stream {0} enables data format conversion, which Floci does not"+ + " apply yet; its records will be delivered unconverted", name);+ }}public void startDeliveryStreamEncryption(String name, String keyType, String keyArn) {@@ -354,6 +383,79 @@ public class FirehoseService implements ResourceProvider {if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());if (update.getS3BackupMode() != null) current.setS3BackupMode(update.getS3BackupMode());+ if (update.getDataFormatConversionConfiguration() != null) {+ current.setDataFormatConversionConfiguration(mergeConversion(+ current.getDataFormatConversionConfiguration(), update.getDataFormatConversionConfiguration()));+ }+ }++ /**+ * Unlike the whole-object replacement above, DataFormatConversionConfiguration+ * merges member-wise: real AWS keeps the stored Schema/Input/Output when an update+ * carries only {"Enabled": false} (probe u2). Where there is anything to merge the+ * result is a new object, so a validation pass over a {@link #mergedView} never+ * mutates the stored configuration; with nothing stored yet the update is returned+ * as it stands, since there is no stored state for a caller to reach through it.+ */+ private static DeliveryStreamDescription.DataFormatConversionConfiguration mergeConversion(+ DeliveryStreamDescription.DataFormatConversionConfiguration current,+ DeliveryStreamDescription.DataFormatConversionConfiguration update) {+ if (current == null) {+ return update;+ }+ DeliveryStreamDescription.DataFormatConversionConfiguration merged =+ new DeliveryStreamDescription.DataFormatConversionConfiguration();+ merged.setEnabled(update.getEnabled() != null ? update.getEnabled() : current.getEnabled());+ merged.setSchemaConfiguration(mergeSchema(+ current.getSchemaConfiguration(), update.getSchemaConfiguration()));+ merged.setInputFormatConfiguration(update.getInputFormatConfiguration() != null+ ? update.getInputFormatConfiguration() : current.getInputFormatConfiguration());+ merged.setOutputFormatConfiguration(update.getOutputFormatConfiguration() != null+ ? update.getOutputFormatConfiguration() : current.getOutputFormatConfiguration());+ return merged;+ }++ /**+ * SchemaConfiguration merges a member at a time as well, so an update naming only+ * a new TableName keeps the stored role, database and region (probed). Replacing+ * it wholesale would drop those and then fail the required-member checks.+ */+ private static DeliveryStreamDescription.SchemaConfiguration mergeSchema(+ DeliveryStreamDescription.SchemaConfiguration current,+ DeliveryStreamDescription.SchemaConfiguration update) {+ if (update == null) {+ return current;+ }+ if (current == null) {+ return update;+ }+ DeliveryStreamDescription.SchemaConfiguration merged =+ new DeliveryStreamDescription.SchemaConfiguration();+ merged.setRoleArn(update.getRoleArn() != null ? update.getRoleArn() : current.getRoleArn());+ merged.setCatalogId(update.getCatalogId() != null ? update.getCatalogId() : current.getCatalogId());+ merged.setDatabaseName(update.getDatabaseName() != null+ ? update.getDatabaseName() : current.getDatabaseName());+ merged.setTableName(update.getTableName() != null ? update.getTableName() : current.getTableName());+ merged.setRegion(update.getRegion() != null ? update.getRegion() : current.getRegion());+ merged.setVersionId(update.getVersionId() != null ? update.getVersionId() : current.getVersionId());+ return merged;+ }++ private static S3Destination mergedView(S3Destination current, S3Destination update) {+ S3Destination view = new S3Destination();+ view.setRoleArn(current.getRoleArn());+ view.setBucketArn(current.getBucketArn());+ view.setPrefix(current.getPrefix());+ view.setErrorOutputPrefix(current.getErrorOutputPrefix());+ view.setCompressionFormat(current.getCompressionFormat());+ view.setFileExtension(current.getFileExtension());+ view.setCustomTimeZone(current.getCustomTimeZone());+ view.setBufferingHints(current.getBufferingHints());+ view.setEncryptionConfiguration(current.getEncryptionConfiguration());+ view.setS3BackupMode(current.getS3BackupMode());+ view.setDataFormatConversionConfiguration(current.getDataFormatConversionConfiguration());+ mergeDestination(view, update);+ return view;}public void tagDeliveryStream(String name, List<DeliveryStreamDescription.Tag> tagsToTag) {diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.javaindex b29801aa10..b88b0bf7a1 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java@@ -1,8 +1,16 @@package io.github.hectorvent.floci.services.firehose;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;import java.util.regex.Pattern;/**@@ -17,44 +25,301 @@ import java.util.regex.Pattern;* {@code extendedS3DestinationUpdate.compressionFormat} depending on the shape* it arrived in. That is why the shape name is a parameter and validation runs* from the handler, the layer that knows which shape was used.+ *+ * Multiple violations in one request are joined into a single+ * "N validation errors detected" message separated by "; ", as real AWS's+ * Smithy layer does (probe 2026-09-07: both ParquetSerDe bounds violated at+ * once report "2 validation errors detected: ...; ...").*/final class S3DestinationValidator {/** Listed in the order AWS prints it, which is neither declaration order nor alphabetical. */private static final String ENUM_VALUE_SET = "[ZIP, HADOOP_SNAPPY, Snappy, GZIP, UNCOMPRESSED]";+ private static final List<String> PARQUET_COMPRESSIONS = List.of("SNAPPY", "GZIP", "UNCOMPRESSED");+ private static final String PARQUET_COMPRESSION_VALUE_SET = "[SNAPPY, GZIP, UNCOMPRESSED]";+ private static final List<String> PARQUET_WRITER_VERSIONS = List.of("V1", "V2");+ private static final String PARQUET_WRITER_VERSION_VALUE_SET = "[V1, V2]";private static final String FILE_EXTENSION_REGEX = "^(|\\.[0-9a-z!\\-_.*'()]+)$";private static final Pattern FILE_EXTENSION = Pattern.compile(FILE_EXTENSION_REGEX);private static final int FILE_EXTENSION_MAX_LENGTH = 128;+ private static final long PARQUET_MIN_BLOCK_SIZE_BYTES = 67108864;+ private static final long PARQUET_MIN_PAGE_SIZE_BYTES = 65536;+ private static final long PARQUET_MIN_MAX_PADDING_BYTES = 0;++ private static final String SERIALIZER_PATH =+ "dataFormatConversionConfiguration.outputFormatConfiguration.serializer";+ private static final List<String> ORC_COMPRESSIONS = List.of("ZLIB", "SNAPPY", "NONE");+ private static final String ORC_COMPRESSION_VALUE_SET = "[ZLIB, SNAPPY, NONE]";+ private static final List<String> ORC_FORMAT_VERSIONS = List.of("V0_11", "V0_12");+ private static final String ORC_FORMAT_VERSION_VALUE_SET = "[V0_11, V0_12]";+ private static final long ORC_MIN_BLOCK_SIZE_BYTES = 67108864;+ private static final long ORC_MIN_STRIPE_SIZE_BYTES = 8388608;+ private static final int ORC_MIN_ROW_INDEX_STRIDE = 1000;+ private static final double ORC_MIN_PROBABILITY = 0;+ private static final double ORC_MAX_PROBABILITY = 1;++ private static final int MAPPING_MAX_LENGTH = 1024;+ private static final String NO_WHITESPACE_PATTERN = "^\\S+$";+ private static final Pattern NO_WHITESPACE = Pattern.compile(NO_WHITESPACE_PATTERN);+ // Applied as written rather than approximated with String.isBlank(): the two+ // disagree in both directions, and AWS follows the pattern (probed). isBlank()+ // would reject an ideographic space the pattern accepts, since \s here is ASCII+ // only, and would accept a value holding a line break that "." cannot match.+ private static final String NOT_BLANK_PATTERN = "^(?!\\s*$).+";+ private static final Pattern NOT_BLANK = Pattern.compile(NOT_BLANK_PATTERN);+private S3DestinationValidator() {}static void validateWireShape(S3Destination config, String shapeName) {if (config == null) {return;}+ List<String> violations = new ArrayList<>();String compressionFormat = config.getCompressionFormat();if (compressionFormat != null && FirehoseCompression.fromWireValue(compressionFormat).isEmpty()) {- throw constraintViolation(shapeName, "compressionFormat",- "Member must satisfy enum value set: " + ENUM_VALUE_SET);+ violations.add(violation(shapeName, "compressionFormat",+ "Member must satisfy enum value set: " + ENUM_VALUE_SET));}String fileExtension = config.getFileExtension();- if (fileExtension == null) {+ if (fileExtension != null) {+ if (fileExtension.length() > FILE_EXTENSION_MAX_LENGTH) {+ violations.add(violation(shapeName, "fileExtension",+ "Member must have length less than or equal to " + FILE_EXTENSION_MAX_LENGTH));+ } else if (!FILE_EXTENSION.matcher(fileExtension).matches()) {+ violations.add(violation(shapeName, "fileExtension",+ "Member must satisfy regular expression pattern: " + FILE_EXTENSION_REGEX));+ }+ }+ Serializer serializer = serializer(config);+ if (serializer != null) {+ // Parquet first, then ORC, the order AWS prints when both carry a+ // violation (they can only both be present on a request the semantic+ // "more than one serializer" check rejects afterwards).+ collectParquetSerDeViolations(serializer.getParquetSerDe(), shapeName, violations);+ collectOrcSerDeViolations(serializer.getOrcSerDe(), shapeName, violations);+ }+ // Serializer, then deserializer, then schema: the order AWS prints when+ // violations sit in more than one of them.+ collectOpenXMappingViolations(config, shapeName, violations);+ collectSchemaViolations(config, shapeName, violations);+ if (!violations.isEmpty()) {+ throw constraintViolations(violations);+ }+ }++ /**+ * Every SchemaConfiguration string is a 1 to 1024 character value without+ * whitespace, and unlike the collection members each broken rule is its own+ * violation rather than one bracketed set. Member order is AWS's (probed), as is+ * the {@code roleARN} spelling, which keeps ARN capitalised where the semantic+ * message for the same member spells it RoleArn.+ *+ * A malformed but non-empty role ARN never reaches these rules on AWS: an+ * InvalidParameterValueException ("Invalid role ARN.") preempts them. Floci+ * validates no ARNs, so it answers only the length and pattern rules here, which+ * is what an empty role ARN trips on AWS too.+ */+ private static void collectSchemaViolations(S3Destination config, String shapeName,+ List<String> violations) {+ var conversion = config.getDataFormatConversionConfiguration();+ if (conversion == null || conversion.getSchemaConfiguration() == null) {+ return;+ }+ SchemaConfiguration schema = conversion.getSchemaConfiguration();+ String path = "dataFormatConversionConfiguration.schemaConfiguration";+ collectSchemaMember(schema.getVersionId(), shapeName, path + ".versionId", violations);+ collectSchemaMember(schema.getCatalogId(), shapeName, path + ".catalogId", violations);+ collectSchemaMember(schema.getDatabaseName(), shapeName, path + ".databaseName", violations);+ collectSchemaMember(schema.getRoleArn(), shapeName, path + ".roleARN", violations);+ collectSchemaMember(schema.getRegion(), shapeName, path + ".region", violations);+ collectSchemaMember(schema.getTableName(), shapeName, path + ".tableName", violations);+ }++ private static void collectSchemaMember(String value, String shapeName, String member,+ List<String> violations) {+ if (value == null) {+ return;+ }+ if (value.isEmpty()) {+ violations.add(violation(shapeName, member, "Member must have length greater than or equal to 1"));+ } else if (value.length() > MAPPING_MAX_LENGTH) {+ violations.add(violation(shapeName, member,+ "Member must have length less than or equal to " + MAPPING_MAX_LENGTH));+ }+ if (!NO_WHITESPACE.matcher(value).matches()) {+ violations.add(violation(shapeName, member,+ "Member must satisfy regular expression pattern: " + NO_WHITESPACE_PATTERN));+ }+ }++ /**+ * The only collection member whose element constraints Floci enforces; the ones+ * on OrcSerDe.BloomFilterColumns and HiveJsonSerDe.TimestampFormats are a+ * documented gap. AWS reports the whole constraint set whichever part failed, at+ * most once for the keys and once for the values, and applies a different pattern+ * to each: keys may hold no whitespace at all, values merely may not be blank.+ */+ private static void collectOpenXMappingViolations(S3Destination config, String shapeName,+ List<String> violations) {+ var conversion = config.getDataFormatConversionConfiguration();+ if (conversion == null || conversion.getInputFormatConfiguration() == null+ || conversion.getInputFormatConfiguration().getDeserializer() == null) {return;}- if (fileExtension.length() > FILE_EXTENSION_MAX_LENGTH) {- throw constraintViolation(shapeName, "fileExtension",- "Member must have length less than or equal to " + FILE_EXTENSION_MAX_LENGTH);+ OpenXJsonSerDe openX = conversion.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe();+ if (openX == null || openX.getColumnToJsonKeyMappings() == null) {+ return;+ }+ String member = "dataFormatConversionConfiguration.inputFormatConfiguration.deserializer"+ + ".openXJsonSerDe.columnToJsonKeyMappings";+ boolean keyViolation = false;+ boolean valueViolation = false;+ for (Map.Entry<String, String> mapping : openX.getColumnToJsonKeyMappings().entrySet()) {+ keyViolation |= !isMappingKeyValid(mapping.getKey());+ valueViolation |= !isMappingValueValid(mapping.getValue());}- if (!FILE_EXTENSION.matcher(fileExtension).matches()) {- throw constraintViolation(shapeName, "fileExtension",- "Member must satisfy regular expression pattern: " + FILE_EXTENSION_REGEX);+ if (keyViolation) {+ violations.add(violation(shapeName, member,+ "Map keys must satisfy constraint: " + constraintSet(NO_WHITESPACE_PATTERN)));}+ if (valueViolation) {+ violations.add(violation(shapeName, member,+ "Map value must satisfy constraint: " + constraintSet(NOT_BLANK_PATTERN)));+ }+ }++ private static boolean isMappingKeyValid(String key) {+ return key != null && lengthInRange(key) && NO_WHITESPACE.matcher(key).matches();+ }++ private static boolean isMappingValueValid(String value) {+ return value != null && lengthInRange(value) && NOT_BLANK.matcher(value).matches();+ }++ private static boolean lengthInRange(String value) {+ return !value.isEmpty() && value.length() <= MAPPING_MAX_LENGTH;+ }++ private static String constraintSet(String pattern) {+ return "[Member must have length less than or equal to " + MAPPING_MAX_LENGTH+ + ", Member must have length greater than or equal to 1"+ + ", Member must satisfy regular expression pattern: " + pattern + "]";+ }++ private static Serializer serializer(S3Destination config) {+ var conversion = config.getDataFormatConversionConfiguration();+ if (conversion == null || conversion.getOutputFormatConfiguration() == null) {+ return null;+ }+ return conversion.getOutputFormatConfiguration().getSerializer();+ }++ /**+ * Smithy-level bounds on ParquetSerDe, enforced whenever the member is present,+ * even on a disabled conversion configuration: they are model constraints, not+ * the semantic checks {@link DataFormatConversionValidator} gates on Enabled.+ */+ private static void collectParquetSerDeViolations(ParquetSerDe parquet, String shapeName,+ List<String> violations) {+ if (parquet == null) {+ return;+ }+ String parquetPath = SERIALIZER_PATH + ".parquetSerDe";+ // Member order follows the one AWS prints when several constraints fail at+ // once, which is neither declaration order nor alphabetical (probed with all+ // five violated in a single request).+ if (parquet.getMaxPaddingBytes() != null && parquet.getMaxPaddingBytes() < PARQUET_MIN_MAX_PADDING_BYTES) {+ violations.add(violation(shapeName, parquetPath + ".maxPaddingBytes",+ "Member must have value greater than or equal to " + PARQUET_MIN_MAX_PADDING_BYTES));+ }+ if (parquet.getBlockSizeBytes() != null && parquet.getBlockSizeBytes() < PARQUET_MIN_BLOCK_SIZE_BYTES) {+ violations.add(violation(shapeName, parquetPath + ".blockSizeBytes",+ "Member must have value greater than or equal to " + PARQUET_MIN_BLOCK_SIZE_BYTES));+ }+ if (parquet.getPageSizeBytes() != null && parquet.getPageSizeBytes() < PARQUET_MIN_PAGE_SIZE_BYTES) {+ violations.add(violation(shapeName, parquetPath + ".pageSizeBytes",+ "Member must have value greater than or equal to " + PARQUET_MIN_PAGE_SIZE_BYTES));+ }+ if (parquet.getCompression() != null && !PARQUET_COMPRESSIONS.contains(parquet.getCompression())) {+ violations.add(violation(shapeName, parquetPath + ".compression",+ "Member must satisfy enum value set: " + PARQUET_COMPRESSION_VALUE_SET));+ }+ if (parquet.getWriterVersion() != null && !PARQUET_WRITER_VERSIONS.contains(parquet.getWriterVersion())) {+ violations.add(violation(shapeName, parquetPath + ".writerVersion",+ "Member must satisfy enum value set: " + PARQUET_WRITER_VERSION_VALUE_SET));+ }+ }++ /**+ * The ORC counterpart. Floci rejects an enabled OrcSerDe semantically, but these+ * are model constraints: they apply to a disabled configuration too, and AWS+ * reports them before any semantic error. Member order and the enum spellings+ * (note {@code [ZLIB, SNAPPY, NONE]}) were probed with all eight violated at once.+ */+ private static void collectOrcSerDeViolations(OrcSerDe orc, String shapeName, List<String> violations) {+ if (orc == null) {+ return;+ }+ String orcPath = SERIALIZER_PATH + ".orcSerDe";+ collectRange(orc.getDictionaryKeyThreshold(), shapeName, orcPath + ".dictionaryKeyThreshold",+ ORC_MIN_PROBABILITY, ORC_MAX_PROBABILITY, violations);+ if (orc.getBlockSizeBytes() != null && orc.getBlockSizeBytes() < ORC_MIN_BLOCK_SIZE_BYTES) {+ violations.add(violation(shapeName, orcPath + ".blockSizeBytes",+ "Member must have value greater than or equal to " + ORC_MIN_BLOCK_SIZE_BYTES));+ }+ if (orc.getRowIndexStride() != null && orc.getRowIndexStride() < ORC_MIN_ROW_INDEX_STRIDE) {+ violations.add(violation(shapeName, orcPath + ".rowIndexStride",+ "Member must have value greater than or equal to " + ORC_MIN_ROW_INDEX_STRIDE));+ }+ collectRange(orc.getPaddingTolerance(), shapeName, orcPath + ".paddingTolerance",+ ORC_MIN_PROBABILITY, ORC_MAX_PROBABILITY, violations);+ if (orc.getStripeSizeBytes() != null && orc.getStripeSizeBytes() < ORC_MIN_STRIPE_SIZE_BYTES) {+ violations.add(violation(shapeName, orcPath + ".stripeSizeBytes",+ "Member must have value greater than or equal to " + ORC_MIN_STRIPE_SIZE_BYTES));+ }+ if (orc.getCompression() != null && !ORC_COMPRESSIONS.contains(orc.getCompression())) {+ violations.add(violation(shapeName, orcPath + ".compression",+ "Member must satisfy enum value set: " + ORC_COMPRESSION_VALUE_SET));+ }+ if (orc.getFormatVersion() != null && !ORC_FORMAT_VERSIONS.contains(orc.getFormatVersion())) {+ violations.add(violation(shapeName, orcPath + ".formatVersion",+ "Member must satisfy enum value set: " + ORC_FORMAT_VERSION_VALUE_SET));+ }+ collectRange(orc.getBloomFilterFalsePositiveProbability(), shapeName,+ orcPath + ".bloomFilterFalsePositiveProbability",+ ORC_MIN_PROBABILITY, ORC_MAX_PROBABILITY, violations);+ }++ /** AWS reports only the bound that was crossed, never both. */+ private static void collectRange(Double value, String shapeName, String member,+ double min, double max, List<String> violations) {+ if (value == null) {+ return;+ }+ // NaN satisfies neither comparison, so it would slip through a plain+ // greater/less pair. AWS reports both bounds for it, the upper one first,+ // and one bound for each infinity (probed).+ boolean notANumber = Double.isNaN(value);+ if (notANumber || value > max) {+ violations.add(violation(shapeName, member,+ "Member must have value less than or equal to " + (long) max));+ }+ if (notANumber || value < min) {+ violations.add(violation(shapeName, member,+ "Member must have value greater than or equal to " + (long) min));+ }+ }++ private static String violation(String shapeName, String member, String constraint) {+ return "Value at '" + shapeName + "." + member + "' failed to satisfy constraint: " + constraint;}- private static AwsException constraintViolation(String shapeName, String member, String constraint) {- return new AwsException("ValidationException",- "1 validation error detected: Value at '" + shapeName + "." + member- + "' failed to satisfy constraint: " + constraint, 400);+ private static AwsException constraintViolations(List<String> violations) {+ String header = violations.size() == 1+ ? "1 validation error detected: "+ : violations.size() + " validation errors detected: ";+ return new AwsException("ValidationException", header + String.join("; ", violations), 400);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.javaindex f85af511fb..444cdd097e 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java@@ -1,12 +1,14 @@package io.github.hectorvent.floci.services.firehose.model;import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonIgnore;import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import com.fasterxml.jackson.annotation.JsonInclude;import com.fasterxml.jackson.annotation.JsonProperty;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;import java.util.List;+import java.util.Map;import java.util.ArrayList;@RegisterForReflection@@ -214,6 +216,8 @@ public class DeliveryStreamDescription {private EncryptionConfiguration encryptionConfiguration;@JsonProperty("S3BackupMode")private String s3BackupMode;+ @JsonProperty("DataFormatConversionConfiguration")+ private DataFormatConversionConfiguration dataFormatConversionConfiguration;public S3Destination() {}public String getRoleArn() { return roleArn; }@@ -236,6 +240,27 @@ public class DeliveryStreamDescription {public void setEncryptionConfiguration(EncryptionConfiguration encryptionConfiguration) { this.encryptionConfiguration = encryptionConfiguration; }public String getS3BackupMode() { return s3BackupMode; }public void setS3BackupMode(String s3BackupMode) { this.s3BackupMode = s3BackupMode; }+ public DataFormatConversionConfiguration getDataFormatConversionConfiguration() {+ return dataFormatConversionConfiguration;+ }+ public void setDataFormatConversionConfiguration(DataFormatConversionConfiguration configuration) {+ this.dataFormatConversionConfiguration = configuration;+ }++ /**+ * True when data format conversion applies to deliveries: the configuration is+ * present and not explicitly disabled. AWS treats an omitted Enabled as enabled+ * (probe: a config without Enabled is validated exactly like an enabled one).+ *+ * Ignored for serialization: an {@code is...} accessor with no backing field of+ * that name is a property to Jackson, which would put a non-AWS+ * {@code dataFormatConversionEnabled} member in every destination it renders.+ */+ @JsonIgnore+ public boolean isDataFormatConversionEnabled() {+ return dataFormatConversionConfiguration != null+ && !Boolean.FALSE.equals(dataFormatConversionConfiguration.getEnabled());+ }/*** Fills the members the wire contract marks required with the AWS defaults.@@ -252,16 +277,24 @@ public class DeliveryStreamDescription {if (encryptionConfiguration == null) {encryptionConfiguration = EncryptionConfiguration.noEncryption();}+ // A conversion-enabled destination that specified no hints gets AWS's+ // larger default size, verified against the service: the stored hints+ // come back as 128 MiB, not the ordinary 5 MiB, while the interval+ // default is unchanged. Filling in 5 here would report a size AWS never+ // uses and would then fail the 64 MiB floor on the stream's next update.+ int defaultSizeInMBs = isDataFormatConversionEnabled()+ ? BufferingHints.CONVERSION_DEFAULT_SIZE_MBS+ : BufferingHints.DEFAULT_SIZE_MBS;if (bufferingHints == null) {- bufferingHints = BufferingHints.defaults();+ bufferingHints = BufferingHints.defaults(defaultSizeInMBs);} else {// Self-heal legacy persisted state; validation keeps partial// hints out of the create/update paths.if (bufferingHints.getSizeInMBs() == null) {- bufferingHints.setSizeInMBs(5);+ bufferingHints.setSizeInMBs(defaultSizeInMBs);}if (bufferingHints.getIntervalInSeconds() == null) {- bufferingHints.setIntervalInSeconds(300);+ bufferingHints.setIntervalInSeconds(BufferingHints.DEFAULT_INTERVAL_SECONDS);}}}@@ -292,6 +325,270 @@ public class DeliveryStreamDescription {}}+ /**+ * Members stay null-honest, mirroring real AWS: DescribeDeliveryStream echoes the+ * configuration exactly as it was given (an empty ParquetSerDe stays {}, CatalogId+ * and VersionId are not materialized), and UpdateDestination merges member-wise, so+ * "not specified" must remain distinguishable from a value.+ */+ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class DataFormatConversionConfiguration {+ @JsonProperty("Enabled")+ private Boolean enabled;+ @JsonProperty("SchemaConfiguration")+ private SchemaConfiguration schemaConfiguration;+ @JsonProperty("InputFormatConfiguration")+ private InputFormatConfiguration inputFormatConfiguration;+ @JsonProperty("OutputFormatConfiguration")+ private OutputFormatConfiguration outputFormatConfiguration;++ public DataFormatConversionConfiguration() {}++ public Boolean getEnabled() { return enabled; }+ public void setEnabled(Boolean enabled) { this.enabled = enabled; }+ public SchemaConfiguration getSchemaConfiguration() { return schemaConfiguration; }+ public void setSchemaConfiguration(SchemaConfiguration schemaConfiguration) {+ this.schemaConfiguration = schemaConfiguration;+ }+ public InputFormatConfiguration getInputFormatConfiguration() { return inputFormatConfiguration; }+ public void setInputFormatConfiguration(InputFormatConfiguration inputFormatConfiguration) {+ this.inputFormatConfiguration = inputFormatConfiguration;+ }+ public OutputFormatConfiguration getOutputFormatConfiguration() { return outputFormatConfiguration; }+ public void setOutputFormatConfiguration(OutputFormatConfiguration outputFormatConfiguration) {+ this.outputFormatConfiguration = outputFormatConfiguration;+ }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class SchemaConfiguration {+ @JsonProperty("RoleARN")+ private String roleArn;+ @JsonProperty("CatalogId")+ private String catalogId;+ @JsonProperty("DatabaseName")+ private String databaseName;+ @JsonProperty("TableName")+ private String tableName;+ @JsonProperty("Region")+ private String region;+ @JsonProperty("VersionId")+ private String versionId;++ public SchemaConfiguration() {}++ public String getRoleArn() { return roleArn; }+ public void setRoleArn(String roleArn) { this.roleArn = roleArn; }+ public String getCatalogId() { return catalogId; }+ public void setCatalogId(String catalogId) { this.catalogId = catalogId; }+ public String getDatabaseName() { return databaseName; }+ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }+ public String getTableName() { return tableName; }+ public void setTableName(String tableName) { this.tableName = tableName; }+ public String getRegion() { return region; }+ public void setRegion(String region) { this.region = region; }+ public String getVersionId() { return versionId; }+ public void setVersionId(String versionId) { this.versionId = versionId; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class InputFormatConfiguration {+ @JsonProperty("Deserializer")+ private Deserializer deserializer;++ public InputFormatConfiguration() {}++ public Deserializer getDeserializer() { return deserializer; }+ public void setDeserializer(Deserializer deserializer) { this.deserializer = deserializer; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class Deserializer {+ @JsonProperty("OpenXJsonSerDe")+ private OpenXJsonSerDe openXJsonSerDe;+ @JsonProperty("HiveJsonSerDe")+ private HiveJsonSerDe hiveJsonSerDe;++ public Deserializer() {}++ public OpenXJsonSerDe getOpenXJsonSerDe() { return openXJsonSerDe; }+ public void setOpenXJsonSerDe(OpenXJsonSerDe openXJsonSerDe) { this.openXJsonSerDe = openXJsonSerDe; }+ public HiveJsonSerDe getHiveJsonSerDe() { return hiveJsonSerDe; }+ public void setHiveJsonSerDe(HiveJsonSerDe hiveJsonSerDe) { this.hiveJsonSerDe = hiveJsonSerDe; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class OpenXJsonSerDe {+ @JsonProperty("ConvertDotsInJsonKeysToUnderscores")+ private Boolean convertDotsInJsonKeysToUnderscores;+ @JsonProperty("CaseInsensitive")+ private Boolean caseInsensitive;+ @JsonProperty("ColumnToJsonKeyMappings")+ private Map<String, String> columnToJsonKeyMappings;++ public OpenXJsonSerDe() {}++ public Boolean getConvertDotsInJsonKeysToUnderscores() { return convertDotsInJsonKeysToUnderscores; }+ public void setConvertDotsInJsonKeysToUnderscores(Boolean convertDotsInJsonKeysToUnderscores) {+ this.convertDotsInJsonKeysToUnderscores = convertDotsInJsonKeysToUnderscores;+ }+ public Boolean getCaseInsensitive() { return caseInsensitive; }+ public void setCaseInsensitive(Boolean caseInsensitive) { this.caseInsensitive = caseInsensitive; }+ public Map<String, String> getColumnToJsonKeyMappings() { return columnToJsonKeyMappings; }+ public void setColumnToJsonKeyMappings(Map<String, String> columnToJsonKeyMappings) {+ this.columnToJsonKeyMappings = columnToJsonKeyMappings;+ }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class HiveJsonSerDe {+ @JsonProperty("TimestampFormats")+ private List<String> timestampFormats;++ public HiveJsonSerDe() {}++ public List<String> getTimestampFormats() { return timestampFormats; }+ public void setTimestampFormats(List<String> timestampFormats) { this.timestampFormats = timestampFormats; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class OutputFormatConfiguration {+ @JsonProperty("Serializer")+ private Serializer serializer;++ public OutputFormatConfiguration() {}++ public Serializer getSerializer() { return serializer; }+ public void setSerializer(Serializer serializer) { this.serializer = serializer; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class Serializer {+ @JsonProperty("ParquetSerDe")+ private ParquetSerDe parquetSerDe;+ @JsonProperty("OrcSerDe")+ private OrcSerDe orcSerDe;++ public Serializer() {}++ public ParquetSerDe getParquetSerDe() { return parquetSerDe; }+ public void setParquetSerDe(ParquetSerDe parquetSerDe) { this.parquetSerDe = parquetSerDe; }+ public OrcSerDe getOrcSerDe() { return orcSerDe; }+ public void setOrcSerDe(OrcSerDe orcSerDe) { this.orcSerDe = orcSerDe; }+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class ParquetSerDe {+ @JsonProperty("BlockSizeBytes")+ private Long blockSizeBytes;+ @JsonProperty("PageSizeBytes")+ private Long pageSizeBytes;+ @JsonProperty("Compression")+ private String compression;+ @JsonProperty("EnableDictionaryCompression")+ private Boolean enableDictionaryCompression;+ @JsonProperty("MaxPaddingBytes")+ private Long maxPaddingBytes;+ @JsonProperty("WriterVersion")+ private String writerVersion;++ public ParquetSerDe() {}++ public Long getBlockSizeBytes() { return blockSizeBytes; }+ public void setBlockSizeBytes(Long blockSizeBytes) { this.blockSizeBytes = blockSizeBytes; }+ public Long getPageSizeBytes() { return pageSizeBytes; }+ public void setPageSizeBytes(Long pageSizeBytes) { this.pageSizeBytes = pageSizeBytes; }+ public String getCompression() { return compression; }+ public void setCompression(String compression) { this.compression = compression; }+ public Boolean getEnableDictionaryCompression() { return enableDictionaryCompression; }+ public void setEnableDictionaryCompression(Boolean enableDictionaryCompression) {+ this.enableDictionaryCompression = enableDictionaryCompression;+ }+ public Long getMaxPaddingBytes() { return maxPaddingBytes; }+ public void setMaxPaddingBytes(Long maxPaddingBytes) { this.maxPaddingBytes = maxPaddingBytes; }+ public String getWriterVersion() { return writerVersion; }+ public void setWriterVersion(String writerVersion) { this.writerVersion = writerVersion; }+ }++ /**+ * Floci rejects an <em>enabled</em> OrcSerDe configuration (DuckDB cannot write+ * ORC), but a disabled one is stored like any other, so the members are mapped:+ * a marker class would silently drop them through {@code ignoreUnknown} and+ * DescribeDeliveryStream could no longer echo the configuration as supplied.+ */+ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public static class OrcSerDe {+ @JsonProperty("StripeSizeBytes")+ private Long stripeSizeBytes;+ @JsonProperty("BlockSizeBytes")+ private Long blockSizeBytes;+ @JsonProperty("RowIndexStride")+ private Integer rowIndexStride;+ @JsonProperty("EnablePadding")+ private Boolean enablePadding;+ @JsonProperty("PaddingTolerance")+ private Double paddingTolerance;+ @JsonProperty("Compression")+ private String compression;+ @JsonProperty("BloomFilterColumns")+ private List<String> bloomFilterColumns;+ @JsonProperty("BloomFilterFalsePositiveProbability")+ private Double bloomFilterFalsePositiveProbability;+ @JsonProperty("DictionaryKeyThreshold")+ private Double dictionaryKeyThreshold;+ @JsonProperty("FormatVersion")+ private String formatVersion;++ public OrcSerDe() {}++ public Long getStripeSizeBytes() { return stripeSizeBytes; }+ public void setStripeSizeBytes(Long stripeSizeBytes) { this.stripeSizeBytes = stripeSizeBytes; }+ public Long getBlockSizeBytes() { return blockSizeBytes; }+ public void setBlockSizeBytes(Long blockSizeBytes) { this.blockSizeBytes = blockSizeBytes; }+ public Integer getRowIndexStride() { return rowIndexStride; }+ public void setRowIndexStride(Integer rowIndexStride) { this.rowIndexStride = rowIndexStride; }+ public Boolean getEnablePadding() { return enablePadding; }+ public void setEnablePadding(Boolean enablePadding) { this.enablePadding = enablePadding; }+ public Double getPaddingTolerance() { return paddingTolerance; }+ public void setPaddingTolerance(Double paddingTolerance) { this.paddingTolerance = paddingTolerance; }+ public String getCompression() { return compression; }+ public void setCompression(String compression) { this.compression = compression; }+ public List<String> getBloomFilterColumns() { return bloomFilterColumns; }+ public void setBloomFilterColumns(List<String> bloomFilterColumns) {+ this.bloomFilterColumns = bloomFilterColumns;+ }+ public Double getBloomFilterFalsePositiveProbability() { return bloomFilterFalsePositiveProbability; }+ public void setBloomFilterFalsePositiveProbability(Double bloomFilterFalsePositiveProbability) {+ this.bloomFilterFalsePositiveProbability = bloomFilterFalsePositiveProbability;+ }+ public Double getDictionaryKeyThreshold() { return dictionaryKeyThreshold; }+ public void setDictionaryKeyThreshold(Double dictionaryKeyThreshold) {+ this.dictionaryKeyThreshold = dictionaryKeyThreshold;+ }+ public String getFormatVersion() { return formatVersion; }+ public void setFormatVersion(String formatVersion) { this.formatVersion = formatVersion; }+ }+@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@JsonInclude(JsonInclude.Include.NON_NULL)@@ -329,12 +626,17 @@ public class DeliveryStreamDescription {/*** Members stay boxed and null-honest so validation can tell "not specified"* from a value: AWS requires SizeInMBs and IntervalInSeconds to be specified- * together, and the defaults (5 MiB / 300 s) apply only when the whole- * object is absent.+ * together, and the defaults apply only when the whole object is absent. The+ * size default depends on the destination (see+ * {@link S3Destination#applyDefaults()}); the interval default does not.*/@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)public static class BufferingHints {+ static final int DEFAULT_SIZE_MBS = 5;+ static final int CONVERSION_DEFAULT_SIZE_MBS = 128;+ static final int DEFAULT_INTERVAL_SECONDS = 300;+@JsonProperty("SizeInMBs")private Integer sizeInMBs;@JsonProperty("IntervalInSeconds")@@ -343,9 +645,13 @@ public class DeliveryStreamDescription {public BufferingHints() {}public static BufferingHints defaults() {+ return defaults(DEFAULT_SIZE_MBS);+ }++ static BufferingHints defaults(int sizeInMBs) {BufferingHints hints = new BufferingHints();- hints.sizeInMBs = 5;- hints.intervalInSeconds = 300;+ hints.sizeInMBs = sizeInMBs;+ hints.intervalInSeconds = DEFAULT_INTERVAL_SECONDS;return hints;}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.javanew file mode 100644index 0000000000..8a0b118df0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java@@ -0,0 +1,249 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;+import static org.hamcrest.Matchers.nullValue;++/**+ * Wire-level coverage for DataFormatConversionConfiguration: the unit tests in+ * {@link FirehoseDataFormatConversionTest} build model objects directly, so+ * this class is what exercises the JSON-to-model mapping of the nested shapes+ * and the error responses as an SDK would see them.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class FirehoseDataFormatConversionIntegrationTest {++ private static final String STREAM_NAME = "test-format-conversion-stream";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-delivery-role";+ private static final String BUCKET_ARN = "arn:aws:s3:::conversion-archive";+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String TARGET_PREFIX = "Firehose_20150804.";+ private static final String DESCRIPTION_PATH =+ "DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void createAcceptsAFullConversionConfiguration() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "DeliveryStreamType": "DirectPut",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "CompressionFormat": "UNCOMPRESSED",+ "BufferingHints": { "SizeInMBs": 64, "IntervalInSeconds": 60 },+ "DataFormatConversionConfiguration": {+ "Enabled": true,+ "SchemaConfiguration": {+ "RoleARN": "%s",+ "DatabaseName": "analytics",+ "TableName": "events",+ "Region": "us-east-1"+ },+ "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {} } },+ "OutputFormatConfiguration": {+ "Serializer": { "ParquetSerDe": { "Compression": "GZIP", "WriterVersion": "V2" } }+ }+ }+ }+ }+ """.formatted(STREAM_NAME, ROLE_ARN, BUCKET_ARN, ROLE_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamARN", notNullValue());+ }++ /**+ * The description echoes the configuration as given: no CatalogId or VersionId+ * materialized, the empty OpenXJsonSerDe kept, and the conversion member absent+ * from the plain S3DestinationDescription mirror, which does not have it in+ * AWS's model.+ */+ @Test+ @Order(2)+ void describeEchoesTheConversionConfigurationVerbatim() {+ String conversionPath = DESCRIPTION_PATH + ".DataFormatConversionConfiguration";+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(conversionPath + ".Enabled", equalTo(true))+ .body(conversionPath + ".SchemaConfiguration.RoleARN", equalTo(ROLE_ARN))+ .body(conversionPath + ".SchemaConfiguration.DatabaseName", equalTo("analytics"))+ .body(conversionPath + ".SchemaConfiguration.TableName", equalTo("events"))+ .body(conversionPath + ".SchemaConfiguration.Region", equalTo("us-east-1"))+ .body(conversionPath + ".SchemaConfiguration.CatalogId", nullValue())+ .body(conversionPath + ".SchemaConfiguration.VersionId", nullValue())+ .body(conversionPath + ".InputFormatConfiguration.Deserializer.OpenXJsonSerDe", notNullValue())+ .body(conversionPath + ".OutputFormatConfiguration.Serializer.ParquetSerDe.Compression", equalTo("GZIP"))+ .body(conversionPath + ".OutputFormatConfiguration.Serializer.ParquetSerDe.WriterVersion", equalTo("V2"))+ .body(conversionPath + ".OutputFormatConfiguration.Serializer.ParquetSerDe.BlockSizeBytes", nullValue())+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription"+ + ".DataFormatConversionConfiguration", nullValue());+ }++ @Test+ @Order(3)+ void updateCarryingOnlyEnabledFalseKeepsTheStoredMembers() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "CurrentDeliveryStreamVersionId": "1",+ "DestinationId": "destinationId-000000000001",+ "ExtendedS3DestinationUpdate": {+ "DataFormatConversionConfiguration": { "Enabled": false }+ }+ }+ """.formatted(STREAM_NAME))+ .when()+ .post("/")+ .then()+ .statusCode(200);++ String conversionPath = DESCRIPTION_PATH + ".DataFormatConversionConfiguration";+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamDescription.VersionId", equalTo("2"))+ .body(conversionPath + ".Enabled", equalTo(false))+ .body(conversionPath + ".SchemaConfiguration.DatabaseName", equalTo("analytics"))+ .body(conversionPath + ".OutputFormatConfiguration.Serializer.ParquetSerDe", notNullValue());+ }++ /**+ * The legacy shape does not model the member, and AWS's JSON protocol ignores a+ * member a shape does not model, so the request succeeds with it disregarded+ * rather than stored. The SDKs cannot send it here at all.+ */+ @Test+ @Order(6)+ void legacyS3DestinationShapeIgnoresTheConversionMember() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "legacy-shape-conversion-stream",+ "S3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "CompressionFormat": "UNCOMPRESSED",+ "DataFormatConversionConfiguration": {+ "Enabled": true,+ "SchemaConfiguration": { "RoleARN": "%s", "DatabaseName": "db", "TableName": "t" },+ "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {} } },+ "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {} } }+ }+ }+ }+ """.formatted(ROLE_ARN, BUCKET_ARN, ROLE_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"legacy-shape-conversion-stream\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(DESCRIPTION_PATH + ".DataFormatConversionConfiguration", nullValue());+ }++ @Test+ @Order(4)+ void createRejectsCompressedDestinationsWithTheAwsMessage() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "rejected-conversion-stream",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "CompressionFormat": "GZIP",+ "DataFormatConversionConfiguration": {+ "Enabled": true,+ "SchemaConfiguration": { "RoleARN": "%s", "DatabaseName": "db", "TableName": "t" },+ "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {} } },+ "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {} } }+ }+ }+ }+ """.formatted(ROLE_ARN, BUCKET_ARN, ROLE_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidArgumentException"))+ .body("message", equalTo("The S3 destination's compression format must be set to UNCOMPRESSED"+ + " when data format conversion is enabled. To enable compression within the converted"+ + " output format, set the appropriate serialization option in the output format configuration."));+ }++ @Test+ @Order(5)+ void createRejectsParquetSerDeBoundsAtTheSmithyLayer() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "rejected-bounds-stream",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "%s",+ "DataFormatConversionConfiguration": {+ "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": { "PageSizeBytes": 1 } } }+ }+ }+ }+ """.formatted(ROLE_ARN, BUCKET_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("1 validation error detected: Value at"+ + " 'extendedS3DestinationConfiguration.dataFormatConversionConfiguration"+ + ".outputFormatConfiguration.serializer.parquetSerDe.pageSizeBytes'"+ + " failed to satisfy constraint: Member must have value greater than or equal to 65536"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.javanew file mode 100644index 0000000000..b51e3a0c27--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java@@ -0,0 +1,522 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.BufferingHints;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.DataFormatConversionConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Deserializer;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.HiveJsonSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.InputFormatConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OutputFormatConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.function.Consumer;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * Every error message and the precedence between them was captured from real+ * AWS (probe 2026-09-07, us-west-2), covering both CreateDeliveryStream and+ * UpdateDestination. See okinaka/floci discussion 118 for the probe record.+ */+class FirehoseDataFormatConversionTest {++ private static final String COMPRESSION_MESSAGE = "The S3 destination's compression format must be set"+ + " to UNCOMPRESSED when data format conversion is enabled. To enable compression within the"+ + " converted output format, set the appropriate serialization option in the output format configuration.";+ private static final String BUFFER_MESSAGE =+ "BufferingHints.SizeInMBs must be at least 64 when data format conversion is enabled.";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";+ private static final String DESTINATION_ID = "destinationId-000000000001";++ private FirehoseService service;++ @BeforeEach+ void setUp() {+ StorageFactory storageFactory = mock(StorageFactory.class);+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();+ when(storageFactory.create(anyString(), anyString(), any()))+ .thenAnswer(invocation -> backends.computeIfAbsent(+ invocation.getArgument(0) + "/" + invocation.getArgument(1),+ k -> AccountAwareStorageBackend.inMemory("000000000000")));+ EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);+ when(firehoseCfg.enabled()).thenReturn(true);+ when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);+ when(firehoseCfg.flushRecordCount()).thenReturn(0);+ EmulatorConfig.ServicesConfig servicesCfg = mock(EmulatorConfig.ServicesConfig.class);+ when(servicesCfg.firehose()).thenReturn(firehoseCfg);+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.services()).thenReturn(servicesCfg);+ RegionResolver regionResolver = mock(RegionResolver.class);+ when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");+ when(regionResolver.getAccountId()).thenReturn("000000000000");+ service = new FirehoseService(storageFactory, Mockito.mock(S3Service.class),+ Mockito.mock(KinesisService.class), regionResolver, new MutableClock(), config);+ }++ private static SchemaConfiguration schema() {+ SchemaConfiguration schema = new SchemaConfiguration();+ schema.setRoleArn(ROLE_ARN);+ schema.setDatabaseName("db");+ schema.setTableName("table");+ schema.setRegion("us-east-1");+ return schema;+ }++ private static InputFormatConfiguration openXInput() {+ Deserializer deserializer = new Deserializer();+ deserializer.setOpenXJsonSerDe(new OpenXJsonSerDe());+ InputFormatConfiguration input = new InputFormatConfiguration();+ input.setDeserializer(deserializer);+ return input;+ }++ private static OutputFormatConfiguration parquetOutput() {+ Serializer serializer = new Serializer();+ serializer.setParquetSerDe(new ParquetSerDe());+ OutputFormatConfiguration output = new OutputFormatConfiguration();+ output.setSerializer(serializer);+ return output;+ }++ private static DataFormatConversionConfiguration fullConversion() {+ DataFormatConversionConfiguration conversion = new DataFormatConversionConfiguration();+ conversion.setEnabled(true);+ conversion.setSchemaConfiguration(schema());+ conversion.setInputFormatConfiguration(openXInput());+ conversion.setOutputFormatConfiguration(parquetOutput());+ return conversion;+ }++ private static S3Destination destination(Consumer<S3Destination> customizer) {+ S3Destination s3 = new S3Destination();+ s3.setRoleArn(ROLE_ARN);+ s3.setBucketArn("arn:aws:s3:::results");+ BufferingHints hints = new BufferingHints();+ hints.setSizeInMBs(64);+ hints.setIntervalInSeconds(60);+ s3.setBufferingHints(hints);+ s3.setCompressionFormat("UNCOMPRESSED");+ s3.setDataFormatConversionConfiguration(fullConversion());+ customizer.accept(s3);+ return s3;+ }++ private AwsException assertCreateRejected(Consumer<S3Destination> customizer) {+ AwsException error = assertThrows(AwsException.class,+ () -> service.createDeliveryStream("stream", destination(customizer)));+ assertEquals("InvalidArgumentException", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ return error;+ }++ private static BufferingHints hints(int sizeInMBs) {+ BufferingHints hints = new BufferingHints();+ hints.setSizeInMBs(sizeInMBs);+ hints.setIntervalInSeconds(60);+ return hints;+ }++ // ── create: message and precedence matrix ────────────────────────────────++ @Test+ void createAcceptsAFullyValidConversionConfiguration() {+ assertDoesNotThrow(() -> service.createDeliveryStream("stream", destination(s3 -> {})));+ }++ @Test+ void compressionCheckComesBeforeEveryOtherConversionCheck() {+ AwsException error = assertCreateRejected(s3 -> {+ s3.setCompressionFormat("GZIP");+ s3.setBufferingHints(hints(5));+ s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null);+ });+ assertEquals(COMPRESSION_MESSAGE, error.getMessage());+ }++ @Test+ void bufferSizeCheckComesBeforeTheMissingMemberChecks() {+ AwsException error = assertCreateRejected(s3 -> {+ s3.setBufferingHints(hints(5));+ s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null);+ });+ assertEquals(BUFFER_MESSAGE, error.getMessage());+ }++ @Test+ void bufferSizeBelow64IsRejectedOnlyWhenExplicit() {+ assertEquals(BUFFER_MESSAGE,+ assertCreateRejected(s3 -> s3.setBufferingHints(hints(63))).getMessage());+ // Probe: a destination with no BufferingHints at all passes the floor check.+ assertDoesNotThrow(() -> service.createDeliveryStream("no-hints",+ destination(s3 -> s3.setBufferingHints(null))));+ }++ /** Probed: AWS stores 128 MiB, not the ordinary 5 MiB, when a converting stream omits hints. */+ @Test+ void omittedHintsGetTheConversionSizeDefault() {+ service.createDeliveryStream("converting", destination(s3 -> s3.setBufferingHints(null)));++ BufferingHints stored = service.describeDeliveryStream("converting")+ .s3Destination().getBufferingHints();+ assertEquals(128, stored.getSizeInMBs());+ assertEquals(300, stored.getIntervalInSeconds());+ }++ @Test+ void omittedHintsKeepTheOrdinaryDefaultWithoutConversion() {+ service.createDeliveryStream("plain", destination(s3 -> {+ s3.setBufferingHints(null);+ s3.setDataFormatConversionConfiguration(null);+ }));++ BufferingHints stored = service.describeDeliveryStream("plain")+ .s3Destination().getBufferingHints();+ assertEquals(5, stored.getSizeInMBs());+ assertEquals(300, stored.getIntervalInSeconds());+ }++ /**+ * Regression: storing 5 MiB for a converting stream made every later update fail+ * the floor check, since the merged state then carried an explicit 5.+ */+ @Test+ void anUnrelatedUpdateSucceedsOnAStreamCreatedWithoutHints() {+ service.createDeliveryStream("converting", destination(s3 -> s3.setBufferingHints(null)));++ assertDoesNotThrow(() -> service.updateDestination("converting", currentVersion("converting"),+ DESTINATION_ID, update(u -> u.setPrefix("changed/"))));++ S3Destination stored = service.describeDeliveryStream("converting").s3Destination();+ assertEquals("changed/", stored.getPrefix());+ assertEquals(128, stored.getBufferingHints().getSizeInMBs());+ }++ @Test+ void missingMembersAreReportedInInputOutputSchemaOrder() {+ DataFormatConversionConfiguration onlyEnabled = new DataFormatConversionConfiguration();+ onlyEnabled.setEnabled(true);+ assertEquals("InputFormatConfiguration must not be null",+ assertCreateRejected(s3 -> s3.setDataFormatConversionConfiguration(onlyEnabled)).getMessage());++ assertEquals("OutputFormatConfiguration must not be null",+ assertCreateRejected(s3 -> {+ s3.getDataFormatConversionConfiguration().setOutputFormatConfiguration(null);+ s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null);+ }).getMessage());++ assertEquals("SchemaConfiguration must not be null",+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null))+ .getMessage());+ }++ /** An empty Deserializer or Serializer object reports the same error as an absent one. */+ @Test+ void emptyDeserializerAndSerializerReportTheNullMessage() {+ assertEquals("Deserializer must not be null",+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()+ .getInputFormatConfiguration().setDeserializer(new Deserializer())).getMessage());+ assertEquals("Serializer must not be null",+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()+ .getOutputFormatConfiguration().setSerializer(new Serializer())).getMessage());+ }++ @Test+ void specifyingBothSerdesOfAUnionIsRejected() {+ assertEquals("More than one deserializer specified. Only one may be chosen.",+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()+ .getInputFormatConfiguration().getDeserializer().setHiveJsonSerDe(new HiveJsonSerDe()))+ .getMessage());+ assertEquals("More than one serializer specified. Only one may be chosen.",+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()+ .getOutputFormatConfiguration().getSerializer().setOrcSerDe(new OrcSerDe()))+ .getMessage());+ }++ /** Floci deviation: real AWS accepts OrcSerDe, but DuckDB cannot write ORC. */+ @Test+ void orcSerDeIsRejectedExplicitly() {+ AwsException error = assertCreateRejected(s3 -> {+ Serializer serializer = new Serializer();+ serializer.setOrcSerDe(new OrcSerDe());+ s3.getDataFormatConversionConfiguration().getOutputFormatConfiguration().setSerializer(serializer);+ });+ assertEquals("OrcSerDe is not supported. Floci supports only ParquetSerDe for data format conversion.",+ error.getMessage());+ }++ @Test+ void schemaMembersAreRequiredWithTheAwsCasedMessages() {+ assertEquals("SchemaConfiguration.RoleArn must not be null or empty",+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()+ .getSchemaConfiguration().setRoleArn(null)).getMessage());+ assertEquals("SchemaConfiguration.DatabaseName must not be null or empty",+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()+ .getSchemaConfiguration().setDatabaseName("")).getMessage());+ assertEquals("SchemaConfiguration.TableName must not be null or empty",+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()+ .getSchemaConfiguration().setTableName(null)).getMessage());+ }++ @Test+ void omittedEnabledIsTreatedAsEnabled() {+ AwsException error = assertCreateRejected(s3 -> {+ s3.getDataFormatConversionConfiguration().setEnabled(null);+ s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null);+ });+ assertEquals("SchemaConfiguration must not be null", error.getMessage());+ }++ @Test+ void disabledConversionSkipsEveryCheckButIsStored() {+ service.createDeliveryStream("disabled", destination(s3 -> {+ s3.setCompressionFormat("GZIP");+ s3.setBufferingHints(hints(5));+ s3.getDataFormatConversionConfiguration().setEnabled(false);+ }));++ DataFormatConversionConfiguration stored = service.describeDeliveryStream("disabled")+ .s3Destination().getDataFormatConversionConfiguration();+ assertEquals(false, stored.getEnabled());+ assertNotNull(stored.getSchemaConfiguration());+ }++ // ── describe: null-honest echo ───────────────────────────────────────────++ @Test+ void describeEchoesTheConfigurationWithoutMaterializingDefaults() {+ service.createDeliveryStream("stream", destination(s3 -> {}));++ S3Destination described = service.describeDeliveryStream("stream").s3Destination();+ DataFormatConversionConfiguration conversion = described.getDataFormatConversionConfiguration();+ assertEquals(true, conversion.getEnabled());+ assertNull(conversion.getSchemaConfiguration().getCatalogId());+ assertNull(conversion.getSchemaConfiguration().getVersionId());+ ParquetSerDe parquet = conversion.getOutputFormatConfiguration().getSerializer().getParquetSerDe();+ assertNotNull(parquet);+ assertNull(parquet.getCompression());+ assertNull(parquet.getBlockSizeBytes());+ }++ /**+ * A disabled ORC configuration is stored rather than rejected, so Describe has to+ * echo its members: modeling OrcSerDe as a bare marker used to drop them silently.+ */+ @Test+ void disabledOrcConfigurationKeepsItsSerDeMembers() {+ OrcSerDe orc = new OrcSerDe();+ orc.setStripeSizeBytes(67108864L);+ orc.setCompression("SNAPPY");+ orc.setFormatVersion("V0_12");+ service.createDeliveryStream("orc", destination(s3 -> {+ Serializer serializer = new Serializer();+ serializer.setOrcSerDe(orc);+ s3.getDataFormatConversionConfiguration().getOutputFormatConfiguration()+ .setSerializer(serializer);+ s3.getDataFormatConversionConfiguration().setEnabled(false);+ }));++ OrcSerDe stored = service.describeDeliveryStream("orc").s3Destination()+ .getDataFormatConversionConfiguration().getOutputFormatConfiguration()+ .getSerializer().getOrcSerDe();+ assertEquals(67108864L, stored.getStripeSizeBytes());+ assertEquals("SNAPPY", stored.getCompression());+ assertEquals("V0_12", stored.getFormatVersion());+ assertNull(stored.getRowIndexStride());+ }++ /**+ * Every member AWS defines on a destination is PascalCase, so a camelCase key is+ * proof that a helper leaked into the wire shape: an {@code is...} accessor with+ * no backing field of that name becomes a Jackson property unless it is ignored.+ */+ @Test+ void destinationRendersOnlyAwsDefinedMembers() throws Exception {+ service.createDeliveryStream("stream", destination(s3 -> {}));+ DeliveryStreamDescription.Destination destination =+ service.describeDeliveryStream("stream").getDestinations().get(0);++ com.fasterxml.jackson.databind.ObjectMapper mapper =+ new com.fasterxml.jackson.databind.ObjectMapper();+ for (Object rendered : List.of(destination.getExtendedS3DestinationDescription(),+ destination.getS3DestinationDescription())) {+ com.fasterxml.jackson.databind.JsonNode json = mapper.valueToTree(rendered);+ json.fieldNames().forEachRemaining(name -> assertTrue(Character.isUpperCase(name.charAt(0)),+ "unexpected non-AWS member rendered: " + name + " in " + json));+ }+ }++ /** The plain S3DestinationDescription mirror has no conversion member in AWS's model. */+ @Test+ void conversionIsExtendedOnlyAndAbsentFromTheStandardView() {+ service.createDeliveryStream("stream", destination(s3 -> {}));++ DeliveryStreamDescription.Destination dest =+ service.describeDeliveryStream("stream").getDestinations().get(0);+ assertNotNull(dest.getExtendedS3DestinationDescription().getDataFormatConversionConfiguration());+ assertNull(dest.getS3DestinationDescription().getDataFormatConversionConfiguration());+ }++ // ── update: validation on the merged state, member-wise merge ────────────++ private String currentVersion(String name) {+ return service.describeDeliveryStream(name).getVersionId();+ }++ private static S3Destination update(Consumer<S3Destination> customizer) {+ S3Destination update = new S3Destination();+ customizer.accept(update);+ return update;+ }++ @Test+ void updateAppliesTheSameChecksAgainstTheMergedState() {+ service.createDeliveryStream("stream", destination(s3 -> {}));++ AwsException gzip = assertThrows(AwsException.class, () -> service.updateDestination("stream",+ currentVersion("stream"), DESTINATION_ID, update(u -> u.setCompressionFormat("GZIP"))));+ assertEquals(COMPRESSION_MESSAGE, gzip.getMessage());++ AwsException buffer = assertThrows(AwsException.class, () -> service.updateDestination("stream",+ currentVersion("stream"), DESTINATION_ID, update(u -> u.setBufferingHints(hints(5)))));+ assertEquals(BUFFER_MESSAGE, buffer.getMessage());+ }++ /** Probed: AWS rejects this too, since a plain stream stores the 5 MiB default. */+ @Test+ void enablingConversionOnAPlainStreamIsRejectedForItsStoredBufferSize() {+ service.createDeliveryStream("plain", destination(s3 -> {+ s3.setBufferingHints(null);+ s3.setDataFormatConversionConfiguration(null);+ }));++ AwsException error = assertThrows(AwsException.class, () -> service.updateDestination("plain",+ currentVersion("plain"), DESTINATION_ID,+ update(u -> u.setDataFormatConversionConfiguration(fullConversion()))));++ assertEquals(BUFFER_MESSAGE, error.getMessage());+ }++ @Test+ void disablingWithOnlyEnabledPreservesTheStoredMembers() {+ service.createDeliveryStream("stream", destination(s3 -> {}));+ String versionBefore = currentVersion("stream");++ DataFormatConversionConfiguration disable = new DataFormatConversionConfiguration();+ disable.setEnabled(false);+ service.updateDestination("stream", versionBefore, DESTINATION_ID,+ update(u -> u.setDataFormatConversionConfiguration(disable)));++ DataFormatConversionConfiguration stored = service.describeDeliveryStream("stream")+ .s3Destination().getDataFormatConversionConfiguration();+ assertEquals(false, stored.getEnabled());+ assertEquals("db", stored.getSchemaConfiguration().getDatabaseName());+ assertNotNull(stored.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe());+ assertNotNull(stored.getOutputFormatConfiguration().getSerializer().getParquetSerDe());+ assertFalse(versionBefore.equals(currentVersion("stream")));+ }++ @Test+ void reenablingAloneIsRejectedWhileGzipIsInEffectAndLeavesTheStoreUntouched() {+ service.createDeliveryStream("stream", destination(s3 -> {}));+ DataFormatConversionConfiguration disable = new DataFormatConversionConfiguration();+ disable.setEnabled(false);+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,+ update(u -> u.setDataFormatConversionConfiguration(disable)));+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,+ update(u -> u.setCompressionFormat("GZIP")));+ String versionBefore = currentVersion("stream");++ DataFormatConversionConfiguration reenable = new DataFormatConversionConfiguration();+ reenable.setEnabled(true);+ AwsException error = assertThrows(AwsException.class, () -> service.updateDestination("stream",+ versionBefore, DESTINATION_ID, update(u -> u.setDataFormatConversionConfiguration(reenable))));++ assertEquals(COMPRESSION_MESSAGE, error.getMessage());+ S3Destination stored = service.describeDeliveryStream("stream").s3Destination();+ assertEquals(versionBefore, currentVersion("stream"));+ assertEquals("GZIP", stored.getCompressionFormat());+ assertEquals(false, stored.getDataFormatConversionConfiguration().getEnabled());+ }++ /** Probed: SchemaConfiguration merges a member at a time, like its parent. */+ @Test+ void updatingOneSchemaMemberKeepsTheOtherStoredOnes() {+ service.createDeliveryStream("stream", destination(s3 -> {}));++ SchemaConfiguration schemaUpdate = new SchemaConfiguration();+ schemaUpdate.setTableName("other_table");+ DataFormatConversionConfiguration conversionUpdate = new DataFormatConversionConfiguration();+ conversionUpdate.setSchemaConfiguration(schemaUpdate);+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,+ update(u -> u.setDataFormatConversionConfiguration(conversionUpdate)));++ SchemaConfiguration stored = service.describeDeliveryStream("stream").s3Destination()+ .getDataFormatConversionConfiguration().getSchemaConfiguration();+ assertEquals("other_table", stored.getTableName());+ assertEquals("db", stored.getDatabaseName());+ assertEquals(ROLE_ARN, stored.getRoleArn());+ assertEquals("us-east-1", stored.getRegion());+ }++ @Test+ void reenablingTogetherWithUncompressedSucceedsOnTheStoredSchema() {+ service.createDeliveryStream("stream", destination(s3 -> {}));+ DataFormatConversionConfiguration disable = new DataFormatConversionConfiguration();+ disable.setEnabled(false);+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,+ update(u -> u.setDataFormatConversionConfiguration(disable)));+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,+ update(u -> u.setCompressionFormat("GZIP")));++ DataFormatConversionConfiguration reenable = new DataFormatConversionConfiguration();+ reenable.setEnabled(true);+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID, update(u -> {+ u.setCompressionFormat("UNCOMPRESSED");+ u.setDataFormatConversionConfiguration(reenable);+ }));++ S3Destination stored = service.describeDeliveryStream("stream").s3Destination();+ assertTrue(stored.isDataFormatConversionEnabled());+ assertEquals("UNCOMPRESSED", stored.getCompressionFormat());+ assertEquals("db", stored.getDataFormatConversionConfiguration()+ .getSchemaConfiguration().getDatabaseName());+ }++ @Test+ void updateOntoAStreamWithoutADestinationValidatesTheUpdateItself() {+ service.createDeliveryStream("bare", (S3Destination) null, List.of());++ AwsException error = assertThrows(AwsException.class, () -> service.updateDestination("bare",+ currentVersion("bare"), DESTINATION_ID, destination(s3 -> s3.setBufferingHints(hints(5)))));+ assertEquals(BUFFER_MESSAGE, error.getMessage());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.javaindex a922c07ed9..a1424d51e9 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.firehose;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;@@ -108,4 +109,517 @@ class S3DestinationValidatorTest {new S3Destination(), "s3DestinationConfiguration"));}+ private static final String PARQUET_PATH = "extendedS3DestinationConfiguration"+ + ".dataFormatConversionConfiguration.outputFormatConfiguration.serializer.parquetSerDe";++ private static S3Destination withParquetSerDe(DeliveryStreamDescription.ParquetSerDe parquet) {+ DeliveryStreamDescription.Serializer serializer = new DeliveryStreamDescription.Serializer();+ serializer.setParquetSerDe(parquet);+ DeliveryStreamDescription.OutputFormatConfiguration output =+ new DeliveryStreamDescription.OutputFormatConfiguration();+ output.setSerializer(serializer);+ DeliveryStreamDescription.DataFormatConversionConfiguration conversion =+ new DeliveryStreamDescription.DataFormatConversionConfiguration();+ conversion.setOutputFormatConfiguration(output);+ S3Destination config = new S3Destination();+ config.setDataFormatConversionConfiguration(conversion);+ return config;+ }++ @Test+ void wireShapeAcceptsParquetSerDeAtItsExactBounds() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setBlockSizeBytes(67108864L);+ parquet.setPageSizeBytes(65536L);+ parquet.setCompression("SNAPPY");+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));+ }++ /** Both bounds violated at once join into one message, as probed against real AWS. */+ @Test+ void wireShapeJoinsBothParquetBoundViolationsIntoOneError() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setBlockSizeBytes(1L);+ parquet.setPageSizeBytes(1L);++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));++ assertEquals("ValidationException", error.getErrorCode());+ assertEquals("2 validation errors detected: "+ + "Value at '" + PARQUET_PATH + ".blockSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 67108864; "+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 65536", error.getMessage());+ }++ @Test+ void wireShapeRejectsNegativeMaxPaddingBytes() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setMaxPaddingBytes(-1L);++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));++ assertEquals("1 validation error detected: Value at '" + PARQUET_PATH + ".maxPaddingBytes' "+ + "failed to satisfy constraint: Member must have value greater than or equal to 0",+ error.getMessage());+ parquet.setMaxPaddingBytes(0L);+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));+ }++ @Test+ void wireShapeRejectsWriterVersionOutsideTheEnum() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setWriterVersion("V3");++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));++ assertEquals("1 validation error detected: Value at '" + PARQUET_PATH + ".writerVersion' "+ + "failed to satisfy constraint: Member must satisfy enum value set: [V1, V2]",+ error.getMessage());+ }++ /** The order AWS prints is neither declaration order nor alphabetical; it was probed. */+ @Test+ void allFiveParquetViolationsAreReportedInTheOrderAwsUses() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setMaxPaddingBytes(-1L);+ parquet.setBlockSizeBytes(1L);+ parquet.setPageSizeBytes(1L);+ parquet.setCompression("ZSTD");+ parquet.setWriterVersion("V3");++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));++ assertEquals("5 validation errors detected: "+ + "Value at '" + PARQUET_PATH + ".maxPaddingBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 0; "+ + "Value at '" + PARQUET_PATH + ".blockSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 67108864; "+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 65536; "+ + "Value at '" + PARQUET_PATH + ".compression' failed to satisfy constraint: "+ + "Member must satisfy enum value set: [SNAPPY, GZIP, UNCOMPRESSED]; "+ + "Value at '" + PARQUET_PATH + ".writerVersion' failed to satisfy constraint: "+ + "Member must satisfy enum value set: [V1, V2]", error.getMessage());+ }++ private static final String ORC_PATH = "extendedS3DestinationConfiguration"+ + ".dataFormatConversionConfiguration.outputFormatConfiguration.serializer.orcSerDe";++ private static S3Destination withOrcSerDe(DeliveryStreamDescription.OrcSerDe orc) {+ DeliveryStreamDescription.Serializer serializer = new DeliveryStreamDescription.Serializer();+ serializer.setOrcSerDe(orc);+ DeliveryStreamDescription.OutputFormatConfiguration output =+ new DeliveryStreamDescription.OutputFormatConfiguration();+ output.setSerializer(serializer);+ DeliveryStreamDescription.DataFormatConversionConfiguration conversion =+ new DeliveryStreamDescription.DataFormatConversionConfiguration();+ conversion.setOutputFormatConfiguration(output);+ S3Destination config = new S3Destination();+ config.setDataFormatConversionConfiguration(conversion);+ return config;+ }++ @Test+ void wireShapeAcceptsOrcSerDeAtItsExactBounds() {+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();+ orc.setBlockSizeBytes(67108864L);+ orc.setStripeSizeBytes(8388608L);+ orc.setRowIndexStride(1000);+ orc.setPaddingTolerance(0.0);+ orc.setDictionaryKeyThreshold(1.0);+ orc.setBloomFilterFalsePositiveProbability(0.0);+ orc.setCompression("NONE");+ orc.setFormatVersion("V0_12");+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));+ }++ /** Order and enum spellings probed with all eight violated in one request. */+ @Test+ void allEightOrcViolationsAreReportedInTheOrderAwsUses() {+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();+ orc.setStripeSizeBytes(1L);+ orc.setBlockSizeBytes(1L);+ orc.setRowIndexStride(1);+ orc.setPaddingTolerance(-1.0);+ orc.setCompression("ZSTD");+ orc.setBloomFilterFalsePositiveProbability(2.0);+ orc.setDictionaryKeyThreshold(2.0);+ orc.setFormatVersion("V0_10");++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));++ assertEquals("8 validation errors detected: "+ + "Value at '" + ORC_PATH + ".dictionaryKeyThreshold' failed to satisfy constraint: "+ + "Member must have value less than or equal to 1; "+ + "Value at '" + ORC_PATH + ".blockSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 67108864; "+ + "Value at '" + ORC_PATH + ".rowIndexStride' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 1000; "+ + "Value at '" + ORC_PATH + ".paddingTolerance' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 0; "+ + "Value at '" + ORC_PATH + ".stripeSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 8388608; "+ + "Value at '" + ORC_PATH + ".compression' failed to satisfy constraint: "+ + "Member must satisfy enum value set: [ZLIB, SNAPPY, NONE]; "+ + "Value at '" + ORC_PATH + ".formatVersion' failed to satisfy constraint: "+ + "Member must satisfy enum value set: [V0_11, V0_12]; "+ + "Value at '" + ORC_PATH + ".bloomFilterFalsePositiveProbability' failed to satisfy constraint: "+ + "Member must have value less than or equal to 1", error.getMessage());+ }++ /** PaddingTolerance is bounded on both sides, like the two probabilities. */+ @Test+ void orcPaddingToleranceRejectsValuesAboveOne() {+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();+ orc.setPaddingTolerance(999.0);++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));++ assertEquals("1 validation error detected: Value at '" + ORC_PATH + ".paddingTolerance' "+ + "failed to satisfy constraint: Member must have value less than or equal to 1",+ error.getMessage());+ orc.setPaddingTolerance(1.0);+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));+ }++ @Test+ void orcProbabilitiesReportTheBoundThatWasCrossed() {+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();+ orc.setDictionaryKeyThreshold(-1.0);+ orc.setBloomFilterFalsePositiveProbability(-1.0);++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));++ assertEquals("2 validation errors detected: "+ + "Value at '" + ORC_PATH + ".dictionaryKeyThreshold' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 0; "+ + "Value at '" + ORC_PATH + ".bloomFilterFalsePositiveProbability' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 0", error.getMessage());+ }++ /** Probed: a request carrying both serializers reports the Parquet violation first. */+ @Test+ void parquetViolationsPrecedeOrcOnesWhenBothSerializersAreSet() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setPageSizeBytes(1L);+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();+ orc.setRowIndexStride(1);+ S3Destination config = withParquetSerDe(parquet);+ config.getDataFormatConversionConfiguration().getOutputFormatConfiguration()+ .getSerializer().setOrcSerDe(orc);++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ config, "extendedS3DestinationConfiguration"));++ assertEquals("2 validation errors detected: "+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 65536; "+ + "Value at '" + ORC_PATH + ".rowIndexStride' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 1000", error.getMessage());+ }++ private static final String MAPPINGS_PATH = "extendedS3DestinationConfiguration"+ + ".dataFormatConversionConfiguration.inputFormatConfiguration.deserializer"+ + ".openXJsonSerDe.columnToJsonKeyMappings";+ private static final String LENGTH_RULES = "[Member must have length less than or equal to 1024,"+ + " Member must have length greater than or equal to 1,"+ + " Member must satisfy regular expression pattern: ";++ private static S3Destination withColumnMappings(java.util.Map<String, String> mappings) {+ DeliveryStreamDescription.OpenXJsonSerDe openX = new DeliveryStreamDescription.OpenXJsonSerDe();+ openX.setColumnToJsonKeyMappings(mappings);+ DeliveryStreamDescription.Deserializer deserializer = new DeliveryStreamDescription.Deserializer();+ deserializer.setOpenXJsonSerDe(openX);+ DeliveryStreamDescription.InputFormatConfiguration input =+ new DeliveryStreamDescription.InputFormatConfiguration();+ input.setDeserializer(deserializer);+ DeliveryStreamDescription.DataFormatConversionConfiguration conversion =+ new DeliveryStreamDescription.DataFormatConversionConfiguration();+ conversion.setInputFormatConfiguration(input);+ S3Destination config = new S3Destination();+ config.setDataFormatConversionConfiguration(conversion);+ return config;+ }++ @Test+ void wireShapeAcceptsWellFormedColumnMappings() {+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("ticker", "sym", "price", "unit price")),+ "extendedS3DestinationConfiguration"));+ }++ /** Probed: keys may hold no whitespace, while values merely may not be blank. */+ @Test+ void mappingKeysRejectWhitespaceButValuesOnlyRejectBlanks() {+ AwsException keyError = assertThrows(AwsException.class,+ () -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("has space", "v")),+ "extendedS3DestinationConfiguration"));+ assertEquals("1 validation error detected: Value at '" + MAPPINGS_PATH + "' failed to satisfy "+ + "constraint: Map keys must satisfy constraint: " + LENGTH_RULES + "^\\S+$]",+ keyError.getMessage());++ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("ticker", "has space")),+ "extendedS3DestinationConfiguration"));++ AwsException valueError = assertThrows(AwsException.class,+ () -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("ticker", " ")),+ "extendedS3DestinationConfiguration"));+ assertEquals("1 validation error detected: Value at '" + MAPPINGS_PATH + "' failed to satisfy "+ + "constraint: Map value must satisfy constraint: " + LENGTH_RULES + "^(?!\\s*$).+]",+ valueError.getMessage());+ }++ /** However many entries break the rule, AWS reports keys once and values once. */+ @Test+ void mappingViolationsAreReportedOncePerAspectWithKeysFirst() {+ java.util.Map<String, String> mappings = new java.util.LinkedHashMap<>();+ mappings.put("has space", "");+ mappings.put("also space", " ");++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withColumnMappings(mappings), "extendedS3DestinationConfiguration"));++ assertEquals("2 validation errors detected: "+ + "Value at '" + MAPPINGS_PATH + "' failed to satisfy constraint: "+ + "Map keys must satisfy constraint: " + LENGTH_RULES + "^\\S+$]; "+ + "Value at '" + MAPPINGS_PATH + "' failed to satisfy constraint: "+ + "Map value must satisfy constraint: " + LENGTH_RULES + "^(?!\\s*$).+]",+ error.getMessage());+ }++ /**+ * The value rule is the pattern itself, not a String.isBlank() approximation:+ * probed against AWS, which accepts an ideographic space here (its own later+ * SerDe check is what rejects that, a deviation Floci documents) and rejects any+ * value holding a line break, while isBlank() gets both backwards. A no-break+ * space is not whitespace to either rule, so a key may hold one.+ */+ @Test+ void mappingValuesFollowTheDeclaredPatternRatherThanIsBlank() {+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("ticker", " ")),+ "extendedS3DestinationConfiguration"));++ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("ticker", "a\nb")),+ "extendedS3DestinationConfiguration"));+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("ticker", "ab\n")),+ "extendedS3DestinationConfiguration"));++ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("a b", "v")),+ "extendedS3DestinationConfiguration"));+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("a\nb", "v")),+ "extendedS3DestinationConfiguration"));+ }++ @Test+ void mappingLengthsAreBoundedAt1024() {+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("k".repeat(1024), "v".repeat(1024))),+ "extendedS3DestinationConfiguration"));++ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withColumnMappings(java.util.Map.of("k".repeat(1025), "v")),+ "extendedS3DestinationConfiguration"));+ }++ /** Probed: the deserializer's violations are printed after the serializer's. */+ @Test+ void serializerViolationsPrecedeDeserializerOnes() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setPageSizeBytes(1L);+ S3Destination config = withParquetSerDe(parquet);+ DeliveryStreamDescription.OpenXJsonSerDe openX = new DeliveryStreamDescription.OpenXJsonSerDe();+ openX.setColumnToJsonKeyMappings(java.util.Map.of("has space", "v"));+ DeliveryStreamDescription.Deserializer deserializer = new DeliveryStreamDescription.Deserializer();+ deserializer.setOpenXJsonSerDe(openX);+ DeliveryStreamDescription.InputFormatConfiguration input =+ new DeliveryStreamDescription.InputFormatConfiguration();+ input.setDeserializer(deserializer);+ config.getDataFormatConversionConfiguration().setInputFormatConfiguration(input);++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ config, "extendedS3DestinationConfiguration"));++ assertEquals("2 validation errors detected: "+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 65536; "+ + "Value at '" + MAPPINGS_PATH + "' failed to satisfy constraint: "+ + "Map keys must satisfy constraint: " + LENGTH_RULES + "^\\S+$]", error.getMessage());+ }++ private static final String SCHEMA_PATH = "extendedS3DestinationConfiguration"+ + ".dataFormatConversionConfiguration.schemaConfiguration";++ private static S3Destination withSchema(DeliveryStreamDescription.SchemaConfiguration schema) {+ DeliveryStreamDescription.DataFormatConversionConfiguration conversion =+ new DeliveryStreamDescription.DataFormatConversionConfiguration();+ conversion.setSchemaConfiguration(schema);+ S3Destination config = new S3Destination();+ config.setDataFormatConversionConfiguration(conversion);+ return config;+ }++ /** Unlike the collection members, each broken schema rule is its own violation. */+ @Test+ void emptySchemaMemberReportsBothItsLengthAndItsPatternViolation() {+ DeliveryStreamDescription.SchemaConfiguration schema =+ new DeliveryStreamDescription.SchemaConfiguration();+ schema.setCatalogId("");++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withSchema(schema), "extendedS3DestinationConfiguration"));++ assertEquals("2 validation errors detected: "+ + "Value at '" + SCHEMA_PATH + ".catalogId' failed to satisfy constraint: "+ + "Member must have length greater than or equal to 1; "+ + "Value at '" + SCHEMA_PATH + ".catalogId' failed to satisfy constraint: "+ + "Member must satisfy regular expression pattern: ^\\S+$", error.getMessage());+ }++ /**+ * Probed: an empty role ARN trips the same length and pattern rules as the other+ * members, reported under a path that keeps ARN capitalised. A malformed but+ * non-empty one never reaches them on AWS, which answers InvalidParameterValueException+ * first; Floci validates no ARNs, so it accepts that case.+ */+ @Test+ void emptySchemaRoleArnIsReportedUnderTheRoleArnPath() {+ DeliveryStreamDescription.SchemaConfiguration schema =+ new DeliveryStreamDescription.SchemaConfiguration();+ schema.setRoleArn("");++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withSchema(schema), "extendedS3DestinationConfiguration"));++ assertEquals("2 validation errors detected: "+ + "Value at '" + SCHEMA_PATH + ".roleARN' failed to satisfy constraint: "+ + "Member must have length greater than or equal to 1; "+ + "Value at '" + SCHEMA_PATH + ".roleARN' failed to satisfy constraint: "+ + "Member must satisfy regular expression pattern: ^\\S+$", error.getMessage());++ schema.setRoleArn("arn:aws:iam::000000000000:role/firehose-role");+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(+ withSchema(schema), "extendedS3DestinationConfiguration"));+ }++ @Test+ void schemaMembersAreReportedInTheOrderAwsUses() {+ DeliveryStreamDescription.SchemaConfiguration schema =+ new DeliveryStreamDescription.SchemaConfiguration();+ schema.setCatalogId("c d");+ schema.setDatabaseName("e f");+ schema.setTableName("g h");+ schema.setRegion("i j");+ schema.setVersionId("k l");+ schema.setRoleArn("m n");++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withSchema(schema), "extendedS3DestinationConfiguration"));++ String pattern = "' failed to satisfy constraint: Member must satisfy regular expression pattern: ^\\S+$";+ assertEquals("6 validation errors detected: "+ + "Value at '" + SCHEMA_PATH + ".versionId" + pattern + "; "+ + "Value at '" + SCHEMA_PATH + ".catalogId" + pattern + "; "+ + "Value at '" + SCHEMA_PATH + ".databaseName" + pattern + "; "+ + "Value at '" + SCHEMA_PATH + ".roleARN" + pattern + "; "+ + "Value at '" + SCHEMA_PATH + ".region" + pattern + "; "+ + "Value at '" + SCHEMA_PATH + ".tableName" + pattern, error.getMessage());+ }++ @Test+ void schemaMembersAreBoundedAt1024() {+ DeliveryStreamDescription.SchemaConfiguration schema =+ new DeliveryStreamDescription.SchemaConfiguration();+ schema.setCatalogId("x".repeat(1100));++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withSchema(schema), "extendedS3DestinationConfiguration"));++ assertEquals("1 validation error detected: Value at '" + SCHEMA_PATH + ".catalogId' "+ + "failed to satisfy constraint: Member must have length less than or equal to 1024",+ error.getMessage());+ }++ /** Probed: schema violations are printed after the serializer's and deserializer's. */+ @Test+ void schemaViolationsComeLast() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setPageSizeBytes(1L);+ S3Destination config = withParquetSerDe(parquet);+ DeliveryStreamDescription.SchemaConfiguration schema =+ new DeliveryStreamDescription.SchemaConfiguration();+ schema.setTableName("g h");+ config.getDataFormatConversionConfiguration().setSchemaConfiguration(schema);++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ config, "extendedS3DestinationConfiguration"));++ assertEquals("2 validation errors detected: "+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 65536; "+ + "Value at '" + SCHEMA_PATH + ".tableName' failed to satisfy constraint: "+ + "Member must satisfy regular expression pattern: ^\\S+$", error.getMessage());+ }++ /** NaN satisfies no comparison, so AWS reports both bounds; each infinity trips one. */+ @Test+ void nonFiniteProportionsReportTheBoundsAwsReports() {+ DeliveryStreamDescription.OrcSerDe nan = new DeliveryStreamDescription.OrcSerDe();+ nan.setDictionaryKeyThreshold(Double.NaN);+ AwsException nanError = assertThrows(AwsException.class,+ () -> S3DestinationValidator.validateWireShape(+ withOrcSerDe(nan), "extendedS3DestinationConfiguration"));+ assertEquals("2 validation errors detected: "+ + "Value at '" + ORC_PATH + ".dictionaryKeyThreshold' failed to satisfy constraint: "+ + "Member must have value less than or equal to 1; "+ + "Value at '" + ORC_PATH + ".dictionaryKeyThreshold' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 0", nanError.getMessage());++ DeliveryStreamDescription.OrcSerDe positiveInfinity = new DeliveryStreamDescription.OrcSerDe();+ positiveInfinity.setDictionaryKeyThreshold(Double.POSITIVE_INFINITY);+ assertEquals("1 validation error detected: Value at '" + ORC_PATH + ".dictionaryKeyThreshold' "+ + "failed to satisfy constraint: Member must have value less than or equal to 1",+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withOrcSerDe(positiveInfinity), "extendedS3DestinationConfiguration")).getMessage());++ DeliveryStreamDescription.OrcSerDe negativeInfinity = new DeliveryStreamDescription.OrcSerDe();+ negativeInfinity.setDictionaryKeyThreshold(Double.NEGATIVE_INFINITY);+ assertEquals("1 validation error detected: Value at '" + ORC_PATH + ".dictionaryKeyThreshold' "+ + "failed to satisfy constraint: Member must have value greater than or equal to 0",+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withOrcSerDe(negativeInfinity), "extendedS3DestinationConfiguration")).getMessage());+ }++ @Test+ void wireShapeRejectsParquetCompressionOutsideTheEnum() {+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();+ parquet.setCompression("ZSTD");++ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));++ assertEquals("1 validation error detected: Value at '" + PARQUET_PATH + ".compression' "+ + "failed to satisfy constraint: Member must satisfy enum value set: [SNAPPY, GZIP, UNCOMPRESSED]",+ error.getMessage());+ }+}