Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): validate and persist DataFormatConversionConfiguration (#3236)

Kenshin Okinaka8a122d2b8cf9
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseDataFormatConversionTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseDataFormatConversionTest.java
new file mode 100644
index 0000000000..0a18feef35
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseDataFormatConversionTest.java
@@ -0,0 +1,141 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.*;
+import software.amazon.awssdk.services.firehose.FirehoseClient;
+import software.amazon.awssdk.services.firehose.model.*;
+
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.*;
+
+/**
+ * Happy-path proof that the SDK's marshalling of the nested
+ * DataFormatConversionConfiguration shapes round-trips through Floci:
+ * create, describe echo, and the member-wise merge on UpdateDestination.
+ * The validation matrix (error messages and precedence) lives in the
+ * emulator's unit tests.
+ */
+@DisplayName("Firehose data format conversion configuration")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class FirehoseDataFormatConversionTest {
+
+ private static FirehoseClient firehose;
+ private static final String STREAM_NAME = "sdk-format-conversion-" + UUID.randomUUID().toString().substring(0, 8);
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";
+ private static final String BUCKET_ARN = "arn:aws:s3:::floci-firehose-sdk-test";
+
+ @BeforeAll
+ static void setup() {
+ firehose = TestFixtures.firehoseClient();
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (firehose != null) {
+ try {
+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME).build());
+ } catch (Exception e) {
+ System.err.println("[WARN] Best-effort cleanup failed for delivery stream "
+ + STREAM_NAME + ": " + e.getMessage());
+ }
+ firehose.close();
+ }
+ }
+
+ @Test
+ @Order(1)
+ @DisplayName("Create accepts a full conversion configuration")
+ void createWithConversionConfiguration() {
+ CreateDeliveryStreamResponse response = firehose.createDeliveryStream(
+ CreateDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)
+ .extendedS3DestinationConfiguration(ExtendedS3DestinationConfiguration.builder()
+ .roleARN(ROLE_ARN)
+ .bucketARN(BUCKET_ARN)
+ .compressionFormat(CompressionFormat.UNCOMPRESSED)
+ .bufferingHints(BufferingHints.builder()
+ .sizeInMBs(64)
+ .intervalInSeconds(60)
+ .build())
+ .dataFormatConversionConfiguration(DataFormatConversionConfiguration.builder()
+ .enabled(true)
+ .schemaConfiguration(SchemaConfiguration.builder()
+ .roleARN(ROLE_ARN)
+ .databaseName("analytics")
+ .tableName("events")
+ .region("us-east-1")
+ .build())
+ .inputFormatConfiguration(InputFormatConfiguration.builder()
+ .deserializer(Deserializer.builder()
+ .openXJsonSerDe(OpenXJsonSerDe.builder().build())
+ .build())
+ .build())
+ .outputFormatConfiguration(OutputFormatConfiguration.builder()
+ .serializer(Serializer.builder()
+ .parquetSerDe(ParquetSerDe.builder()
+ .compression(ParquetCompression.GZIP)
+ .writerVersion(ParquetWriterVersion.V2)
+ .build())
+ .build())
+ .build())
+ .build())
+ .build())
+ .build());
+
+ assertThat(response.deliveryStreamARN()).contains(STREAM_NAME);
+ }
+
+ @Test
+ @Order(2)
+ @DisplayName("Describe echoes the conversion configuration as given")
+ void describeEchoesConversionConfiguration() {
+ ExtendedS3DestinationDescription destination = firehose.describeDeliveryStream(
+ DescribeDeliveryStreamRequest.builder().deliveryStreamName(STREAM_NAME).build())
+ .deliveryStreamDescription().destinations().get(0).extendedS3DestinationDescription();
+
+ DataFormatConversionConfiguration conversion = destination.dataFormatConversionConfiguration();
+ assertThat(conversion).isNotNull();
+ assertThat(conversion.enabled()).isTrue();
+ assertThat(conversion.schemaConfiguration().roleARN()).isEqualTo(ROLE_ARN);
+ assertThat(conversion.schemaConfiguration().databaseName()).isEqualTo("analytics");
+ assertThat(conversion.schemaConfiguration().tableName()).isEqualTo("events");
+ assertThat(conversion.schemaConfiguration().region()).isEqualTo("us-east-1");
+ assertThat(conversion.schemaConfiguration().catalogId()).isNull();
+ assertThat(conversion.inputFormatConfiguration().deserializer().openXJsonSerDe()).isNotNull();
+ ParquetSerDe parquet = conversion.outputFormatConfiguration().serializer().parquetSerDe();
+ assertThat(parquet.compression()).isEqualTo(ParquetCompression.GZIP);
+ assertThat(parquet.writerVersion()).isEqualTo(ParquetWriterVersion.V2);
+ assertThat(parquet.blockSizeBytes()).isNull();
+ }
+
+ @Test
+ @Order(3)
+ @DisplayName("UpdateDestination with only Enabled=false keeps the stored members")
+ void updateDisablingConversionKeepsStoredMembers() {
+ DeliveryStreamDescription described = firehose.describeDeliveryStream(
+ DescribeDeliveryStreamRequest.builder().deliveryStreamName(STREAM_NAME).build())
+ .deliveryStreamDescription();
+
+ firehose.updateDestination(UpdateDestinationRequest.builder()
+ .deliveryStreamName(STREAM_NAME)
+ .currentDeliveryStreamVersionId(described.versionId())
+ .destinationId(described.destinations().get(0).destinationId())
+ .extendedS3DestinationUpdate(ExtendedS3DestinationUpdate.builder()
+ .dataFormatConversionConfiguration(DataFormatConversionConfiguration.builder()
+ .enabled(false)
+ .build())
+ .build())
+ .build());
+
+ DataFormatConversionConfiguration conversion = firehose.describeDeliveryStream(
+ DescribeDeliveryStreamRequest.builder().deliveryStreamName(STREAM_NAME).build())
+ .deliveryStreamDescription().destinations().get(0)
+ .extendedS3DestinationDescription().dataFormatConversionConfiguration();
+
+ assertThat(conversion.enabled()).isFalse();
+ assertThat(conversion.schemaConfiguration().databaseName()).isEqualTo("analytics");
+ assertThat(conversion.outputFormatConfiguration().serializer().parquetSerDe()).isNotNull();
+ }
+}
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index a5946664cc..90185e8fb2 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -57,6 +57,21 @@ The two Snappy variants share an extension and are told apart only by `Content-E
Any other value, including differently-cased ones such as `SNAPPY` instead of `Snappy`, is rejected with `ValidationException`.
+## Data format conversion
+
+`DataFormatConversionConfiguration` on the extended S3 destination is accepted, validated, persisted, and echoed back by `DescribeDeliveryStream`. **The conversion itself, JSON to Parquet at delivery time, is not implemented yet**: a conversion-enabled stream delivers its records unconverted, under the usual key format, and creating or updating one logs a warning saying so.
+
+Validation matches AWS on both `CreateDeliveryStream` and `UpdateDestination`, down to the [modeled](https://docs.aws.amazon.com/firehose/latest/APIReference/API_DataFormatConversionConfiguration.html) bounds, enums and patterns, so a raw JSON client cannot store a configuration AWS would reject. Two rules are worth knowing because they constrain the destination rather than the conversion block: `CompressionFormat` must be `UNCOMPRESSED`, compression being chosen by the output serializer instead, and an explicitly specified `BufferingHints.SizeInMBs` must be at least 64. Omitting `BufferingHints` stores 128 MiB, AWS's larger default for converting streams, rather than the ordinary 5 MiB.
+
+An omitted `Enabled` counts as enabled; `Enabled: false` stores and merges the configuration without applying those two rules. `UpdateDestination` merges member-wise, nested `SchemaConfiguration` members included, then validates the merged result, so an update carrying only `{"Enabled": false}` keeps the stored schema and formats, and one naming only a new `TableName` keeps the stored role, database and region.
+
+### Known deviations from AWS
+
+- **An enabled `OrcSerDe` is rejected** with `InvalidArgumentException` at create/update time. Real AWS accepts it; Floci's planned conversion engine (DuckDB) cannot write ORC, and refusing early beats storing a configuration that could never deliver. A disabled one is stored and echoed with its ORC members intact.
+- **The Glue table is not resolved at create or update time.** Real AWS rejects a `SchemaConfiguration` naming a database or table that does not exist, with `InvalidArgumentException` wrapping a Glue `EntityNotFoundException`; Floci accepts it. Floci performs no role-assumption check either, where AWS validates the role last.
+- **Two kinds of validation are skipped**: the element constraints AWS puts on `OrcSerDe.BloomFilterColumns` and `HiveJsonSerDe.TimestampFormats` entries (the equivalent ones on `ColumnToJsonKeyMappings` are enforced), and AWS's semantic check of SerDe option values, which answers `InvalidArgumentException` ("Invalid deserializer option(s): ...") for a well-formed but unusable value.
+- A malformed but non-empty `SchemaConfiguration.RoleARN` is accepted. AWS answers `InvalidParameterValueException` ("Invalid role ARN.") before its modeled rules run; Floci validates no ARNs, so it applies only the length and pattern rules, which an empty value trips exactly as on AWS.
+
## S3 object keys
Delivered objects are named `<evaluated prefix><streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid><file extension>`, matching [AWS's object name format](https://docs.aws.amazon.com/firehose/latest/dev/s3-object-name.html):
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/DataFormatConversionValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/DataFormatConversionValidator.java
new file mode 100644
index 0000000000..6b41fe723e
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/DataFormatConversionValidator.java
@@ -0,0 +1,108 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.DataFormatConversionConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Deserializer;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;
+
+/**
+ * Validates a destination whose data format conversion is enabled, with the
+ * messages and precedence captured from real AWS (probe 2026-09-07, both
+ * CreateDeliveryStream and UpdateDestination): compression format first, then
+ * the buffering size floor, then the three member null checks in
+ * Input/Output/Schema order, then the members inside them. On
+ * UpdateDestination real AWS validates the merged effective state, which is
+ * why this runs against a merged view rather than the update shape.
+ *
+ * All checks are skipped when conversion is absent or explicitly disabled; an
+ * omitted Enabled counts as enabled. The buffering floor applies only to an
+ * explicitly specified SizeInMBs: AWS accepts a destination with no
+ * BufferingHints at all (probe: it proceeds to role validation).
+ *
+ * The OrcSerDe rejection is Floci's own: real AWS accepts it, but Floci's
+ * conversion engine (DuckDB) cannot write ORC, and refusing at create time
+ * beats accepting a configuration that could never deliver.
+ */
+final class DataFormatConversionValidator {
+
+ private static final int MIN_BUFFERING_SIZE_MBS = 64;
+
+ private DataFormatConversionValidator() {}
+
+ static void validateEffective(S3Destination config) {
+ if (config == null || !config.isDataFormatConversionEnabled()) {
+ return;
+ }
+ if (config.getCompressionFormat() != null && !"UNCOMPRESSED".equals(config.getCompressionFormat())) {
+ throw invalidArgument("The S3 destination's compression format must be set to UNCOMPRESSED"
+ + " when data format conversion is enabled. To enable compression within the converted"
+ + " output format, set the appropriate serialization option in the output format configuration.");
+ }
+ if (config.getBufferingHints() != null && config.getBufferingHints().getSizeInMBs() != null
+ && config.getBufferingHints().getSizeInMBs() < MIN_BUFFERING_SIZE_MBS) {
+ throw invalidArgument("BufferingHints.SizeInMBs must be at least " + MIN_BUFFERING_SIZE_MBS
+ + " when data format conversion is enabled.");
+ }
+ DataFormatConversionConfiguration conversion = config.getDataFormatConversionConfiguration();
+ if (conversion.getInputFormatConfiguration() == null) {
+ throw invalidArgument("InputFormatConfiguration must not be null");
+ }
+ if (conversion.getOutputFormatConfiguration() == null) {
+ throw invalidArgument("OutputFormatConfiguration must not be null");
+ }
+ if (conversion.getSchemaConfiguration() == null) {
+ throw invalidArgument("SchemaConfiguration must not be null");
+ }
+ validateDeserializer(conversion.getInputFormatConfiguration().getDeserializer());
+ validateSerializer(conversion.getOutputFormatConfiguration().getSerializer());
+ validateSchema(conversion.getSchemaConfiguration());
+ }
+
+ /** An empty Deserializer object reports the same error as an absent one (probe a9). */
+ private static void validateDeserializer(Deserializer deserializer) {
+ int specified = deserializer == null ? 0
+ : (deserializer.getOpenXJsonSerDe() != null ? 1 : 0)
+ + (deserializer.getHiveJsonSerDe() != null ? 1 : 0);
+ if (specified == 0) {
+ throw invalidArgument("Deserializer must not be null");
+ }
+ if (specified > 1) {
+ throw invalidArgument("More than one deserializer specified. Only one may be chosen.");
+ }
+ }
+
+ private static void validateSerializer(Serializer serializer) {
+ int specified = serializer == null ? 0
+ : (serializer.getParquetSerDe() != null ? 1 : 0)
+ + (serializer.getOrcSerDe() != null ? 1 : 0);
+ if (specified == 0) {
+ throw invalidArgument("Serializer must not be null");
+ }
+ if (specified > 1) {
+ throw invalidArgument("More than one serializer specified. Only one may be chosen.");
+ }
+ if (serializer.getOrcSerDe() != null) {
+ throw invalidArgument("OrcSerDe is not supported. Floci supports only ParquetSerDe"
+ + " for data format conversion.");
+ }
+ }
+
+ /** AWS's message spells the member RoleArn even though the wire member is RoleARN. */
+ private static void validateSchema(SchemaConfiguration schema) {
+ requireSchemaMember(schema.getRoleArn(), "RoleArn");
+ requireSchemaMember(schema.getDatabaseName(), "DatabaseName");
+ requireSchemaMember(schema.getTableName(), "TableName");
+ }
+
+ private static void requireSchemaMember(String value, String member) {
+ if (value == null || value.isEmpty()) {
+ throw invalidArgument("SchemaConfiguration." + member + " must not be null or empty");
+ }
+ }
+
+ private static AwsException invalidArgument(String message) {
+ return new AwsException("InvalidArgumentException", message, 400);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
index 58f049512c..3fc50bf16e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java
@@ -46,6 +46,7 @@ public class FirehoseJsonHandler {
S3Destination s3 = null;
if (request.has("S3DestinationConfiguration")) {
s3 = mapper.treeToValue(request.get("S3DestinationConfiguration"), S3Destination.class);
+ dropConversionFromLegacyShape(s3);
S3DestinationValidator.validateWireShape(s3, "s3DestinationConfiguration");
} else if (request.has("ExtendedS3DestinationConfiguration")) {
s3 = mapper.treeToValue(request.get("ExtendedS3DestinationConfiguration"), S3Destination.class);
@@ -81,6 +82,7 @@ public class FirehoseJsonHandler {
S3DestinationValidator.validateWireShape(update, "extendedS3DestinationUpdate");
} else if (request.has("S3DestinationUpdate")) {
update = mapper.treeToValue(request.get("S3DestinationUpdate"), S3Destination.class);
+ dropConversionFromLegacyShape(update);
S3DestinationValidator.validateWireShape(update, "s3DestinationUpdate");
}
firehoseService.updateDestination(name, currentVersionId, destinationId, update);
@@ -195,6 +197,28 @@ public class FirehoseJsonHandler {
};
}
+ /**
+ * Clears {@code DataFormatConversionConfiguration} from a legacy
+ * {@code S3DestinationConfiguration} or {@code S3DestinationUpdate}, which AWS
+ * models only on the extended shapes. The two share one class here, which would
+ * otherwise let a legacy request store a member its contract does not define.
+ *
+ * Dropped rather than rejected: AWS's JSON protocol ignores a member the shape does
+ * not model, so a request carrying one succeeds with the member disregarded. The
+ * SDKs cannot even send it, since it is absent from the legacy shape's model.
+ *
+ * Only this member is cleared. {@code FileExtension}, {@code CustomTimeZone} and
+ * {@code S3BackupMode} are extended-only too and reach the legacy shapes the same
+ * way, a deviation that predates this change and is recorded in
+ * docs/services/firehose.md; widening the clearing would change behavior those
+ * members already have.
+ */
+ private static void dropConversionFromLegacyShape(S3Destination s3) {
+ if (s3 != null) {
+ s3.setDataFormatConversionConfiguration(null);
+ }
+ }
+
/** DeliveryStreamType values the 2015-08-04 model enumerates. */
private static final java.util.Set<String> DELIVERY_STREAM_TYPES = java.util.Set.of(
"DirectPut", "KinesisStreamAsSource", "MSKAsSource", "DatabaseAsSource");
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 430486ead9..3305881808 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -219,6 +219,7 @@ public class FirehoseService implements ResourceProvider {
}
validateBufferingHints(s3Config);
+ DataFormatConversionValidator.validateEffective(s3Config);
String arn = AwsArnUtils.Arn.of("firehose", region, regionResolver.getAccountId(),
"deliverystream/" + name).toString();
// CreateDeliveryStream's KinesisStreamSourceConfiguration carries only the ARN and
@@ -239,6 +240,7 @@ public class FirehoseService implements ResourceProvider {
streamStore.put(name, description);
buffers.put(name, Collections.synchronizedList(new ArrayList<>()));
LOG.infov("Created Firehose delivery stream: {0}", name);
+ warnIfConversionEnabled(name, s3Config);
return arn;
}
@@ -263,15 +265,42 @@ public class FirehoseService implements ResourceProvider {
validateBufferingHints(update);
S3Destination current = destination.getExtendedS3DestinationDescription();
if (current == null) {
+ DataFormatConversionValidator.validateEffective(update);
update.applyDefaults();
destination.setExtendedS3DestinationDescription(update);
} else {
+ // AWS validates the merged effective state, not the update shape: enabling
+ // conversion is rejected while a stored GZIP is in effect, and re-enabling
+ // with only {"Enabled": true} succeeds on the stored schema (probe u5/u6).
+ // Validating a merged view first also keeps a failed update from leaving a
+ // half-merged destination behind in the memory backend.
+ DataFormatConversionValidator.validateEffective(mergedView(current, update));
mergeDestination(current, update);
}
stream.setVersionId(String.valueOf(parseVersionId(stream.getVersionId()) + 1));
stream.setLastUpdateTimestamp(java.time.Instant.now());
streamStore.put(name, stream);
LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);
+ warnIfConversionEnabled(name, stream.s3Destination());
+ }
+
+ /**
+ * Says, where the configuration is set, that conversion will not be applied. Fires
+ * on every create and on every update that leaves conversion enabled, including an
+ * update about something else, so a caller who keeps changing the destination keeps
+ * being told.
+ *
+ * Deliberately not in the flush path: that runs on every buffered delivery and on
+ * every retry after a failed write, so warning there floods the log and needs a
+ * per-stream marker whose lifetime has to track creates and deletes. Create and
+ * update are caller-driven and rare by comparison, and they are the moments a
+ * caller can act on the warning anyway.
+ */
+ private static void warnIfConversionEnabled(String name, S3Destination s3) {
+ if (s3 != null && s3.isDataFormatConversionEnabled()) {
+ LOG.warnv("Delivery stream {0} enables data format conversion, which Floci does not"
+ + " apply yet; its records will be delivered unconverted", name);
+ }
}
public void startDeliveryStreamEncryption(String name, String keyType, String keyArn) {
@@ -354,6 +383,79 @@ public class FirehoseService implements ResourceProvider {
if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());
if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());
if (update.getS3BackupMode() != null) current.setS3BackupMode(update.getS3BackupMode());
+ if (update.getDataFormatConversionConfiguration() != null) {
+ current.setDataFormatConversionConfiguration(mergeConversion(
+ current.getDataFormatConversionConfiguration(), update.getDataFormatConversionConfiguration()));
+ }
+ }
+
+ /**
+ * Unlike the whole-object replacement above, DataFormatConversionConfiguration
+ * merges member-wise: real AWS keeps the stored Schema/Input/Output when an update
+ * carries only {"Enabled": false} (probe u2). Where there is anything to merge the
+ * result is a new object, so a validation pass over a {@link #mergedView} never
+ * mutates the stored configuration; with nothing stored yet the update is returned
+ * as it stands, since there is no stored state for a caller to reach through it.
+ */
+ private static DeliveryStreamDescription.DataFormatConversionConfiguration mergeConversion(
+ DeliveryStreamDescription.DataFormatConversionConfiguration current,
+ DeliveryStreamDescription.DataFormatConversionConfiguration update) {
+ if (current == null) {
+ return update;
+ }
+ DeliveryStreamDescription.DataFormatConversionConfiguration merged =
+ new DeliveryStreamDescription.DataFormatConversionConfiguration();
+ merged.setEnabled(update.getEnabled() != null ? update.getEnabled() : current.getEnabled());
+ merged.setSchemaConfiguration(mergeSchema(
+ current.getSchemaConfiguration(), update.getSchemaConfiguration()));
+ merged.setInputFormatConfiguration(update.getInputFormatConfiguration() != null
+ ? update.getInputFormatConfiguration() : current.getInputFormatConfiguration());
+ merged.setOutputFormatConfiguration(update.getOutputFormatConfiguration() != null
+ ? update.getOutputFormatConfiguration() : current.getOutputFormatConfiguration());
+ return merged;
+ }
+
+ /**
+ * SchemaConfiguration merges a member at a time as well, so an update naming only
+ * a new TableName keeps the stored role, database and region (probed). Replacing
+ * it wholesale would drop those and then fail the required-member checks.
+ */
+ private static DeliveryStreamDescription.SchemaConfiguration mergeSchema(
+ DeliveryStreamDescription.SchemaConfiguration current,
+ DeliveryStreamDescription.SchemaConfiguration update) {
+ if (update == null) {
+ return current;
+ }
+ if (current == null) {
+ return update;
+ }
+ DeliveryStreamDescription.SchemaConfiguration merged =
+ new DeliveryStreamDescription.SchemaConfiguration();
+ merged.setRoleArn(update.getRoleArn() != null ? update.getRoleArn() : current.getRoleArn());
+ merged.setCatalogId(update.getCatalogId() != null ? update.getCatalogId() : current.getCatalogId());
+ merged.setDatabaseName(update.getDatabaseName() != null
+ ? update.getDatabaseName() : current.getDatabaseName());
+ merged.setTableName(update.getTableName() != null ? update.getTableName() : current.getTableName());
+ merged.setRegion(update.getRegion() != null ? update.getRegion() : current.getRegion());
+ merged.setVersionId(update.getVersionId() != null ? update.getVersionId() : current.getVersionId());
+ return merged;
+ }
+
+ private static S3Destination mergedView(S3Destination current, S3Destination update) {
+ S3Destination view = new S3Destination();
+ view.setRoleArn(current.getRoleArn());
+ view.setBucketArn(current.getBucketArn());
+ view.setPrefix(current.getPrefix());
+ view.setErrorOutputPrefix(current.getErrorOutputPrefix());
+ view.setCompressionFormat(current.getCompressionFormat());
+ view.setFileExtension(current.getFileExtension());
+ view.setCustomTimeZone(current.getCustomTimeZone());
+ view.setBufferingHints(current.getBufferingHints());
+ view.setEncryptionConfiguration(current.getEncryptionConfiguration());
+ view.setS3BackupMode(current.getS3BackupMode());
+ view.setDataFormatConversionConfiguration(current.getDataFormatConversionConfiguration());
+ mergeDestination(view, update);
+ return view;
}
public void tagDeliveryStream(String name, List<DeliveryStreamDescription.Tag> tagsToTag) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
index b29801aa10..b88b0bf7a1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidator.java
@@ -1,8 +1,16 @@
package io.github.hectorvent.floci.services.firehose;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.regex.Pattern;
/**
@@ -17,44 +25,301 @@ import java.util.regex.Pattern;
* {@code extendedS3DestinationUpdate.compressionFormat} depending on the shape
* it arrived in. That is why the shape name is a parameter and validation runs
* from the handler, the layer that knows which shape was used.
+ *
+ * Multiple violations in one request are joined into a single
+ * "N validation errors detected" message separated by "; ", as real AWS's
+ * Smithy layer does (probe 2026-09-07: both ParquetSerDe bounds violated at
+ * once report "2 validation errors detected: ...; ...").
*/
final class S3DestinationValidator {
/** Listed in the order AWS prints it, which is neither declaration order nor alphabetical. */
private static final String ENUM_VALUE_SET = "[ZIP, HADOOP_SNAPPY, Snappy, GZIP, UNCOMPRESSED]";
+ private static final List<String> PARQUET_COMPRESSIONS = List.of("SNAPPY", "GZIP", "UNCOMPRESSED");
+ private static final String PARQUET_COMPRESSION_VALUE_SET = "[SNAPPY, GZIP, UNCOMPRESSED]";
+ private static final List<String> PARQUET_WRITER_VERSIONS = List.of("V1", "V2");
+ private static final String PARQUET_WRITER_VERSION_VALUE_SET = "[V1, V2]";
private static final String FILE_EXTENSION_REGEX = "^(|\\.[0-9a-z!\\-_.*'()]+)$";
private static final Pattern FILE_EXTENSION = Pattern.compile(FILE_EXTENSION_REGEX);
private static final int FILE_EXTENSION_MAX_LENGTH = 128;
+ private static final long PARQUET_MIN_BLOCK_SIZE_BYTES = 67108864;
+ private static final long PARQUET_MIN_PAGE_SIZE_BYTES = 65536;
+ private static final long PARQUET_MIN_MAX_PADDING_BYTES = 0;
+
+ private static final String SERIALIZER_PATH =
+ "dataFormatConversionConfiguration.outputFormatConfiguration.serializer";
+ private static final List<String> ORC_COMPRESSIONS = List.of("ZLIB", "SNAPPY", "NONE");
+ private static final String ORC_COMPRESSION_VALUE_SET = "[ZLIB, SNAPPY, NONE]";
+ private static final List<String> ORC_FORMAT_VERSIONS = List.of("V0_11", "V0_12");
+ private static final String ORC_FORMAT_VERSION_VALUE_SET = "[V0_11, V0_12]";
+ private static final long ORC_MIN_BLOCK_SIZE_BYTES = 67108864;
+ private static final long ORC_MIN_STRIPE_SIZE_BYTES = 8388608;
+ private static final int ORC_MIN_ROW_INDEX_STRIDE = 1000;
+ private static final double ORC_MIN_PROBABILITY = 0;
+ private static final double ORC_MAX_PROBABILITY = 1;
+
+ private static final int MAPPING_MAX_LENGTH = 1024;
+ private static final String NO_WHITESPACE_PATTERN = "^\\S+$";
+ private static final Pattern NO_WHITESPACE = Pattern.compile(NO_WHITESPACE_PATTERN);
+ // Applied as written rather than approximated with String.isBlank(): the two
+ // disagree in both directions, and AWS follows the pattern (probed). isBlank()
+ // would reject an ideographic space the pattern accepts, since \s here is ASCII
+ // only, and would accept a value holding a line break that "." cannot match.
+ private static final String NOT_BLANK_PATTERN = "^(?!\\s*$).+";
+ private static final Pattern NOT_BLANK = Pattern.compile(NOT_BLANK_PATTERN);
+
private S3DestinationValidator() {}
static void validateWireShape(S3Destination config, String shapeName) {
if (config == null) {
return;
}
+ List<String> violations = new ArrayList<>();
String compressionFormat = config.getCompressionFormat();
if (compressionFormat != null && FirehoseCompression.fromWireValue(compressionFormat).isEmpty()) {
- throw constraintViolation(shapeName, "compressionFormat",
- "Member must satisfy enum value set: " + ENUM_VALUE_SET);
+ violations.add(violation(shapeName, "compressionFormat",
+ "Member must satisfy enum value set: " + ENUM_VALUE_SET));
}
String fileExtension = config.getFileExtension();
- if (fileExtension == null) {
+ if (fileExtension != null) {
+ if (fileExtension.length() > FILE_EXTENSION_MAX_LENGTH) {
+ violations.add(violation(shapeName, "fileExtension",
+ "Member must have length less than or equal to " + FILE_EXTENSION_MAX_LENGTH));
+ } else if (!FILE_EXTENSION.matcher(fileExtension).matches()) {
+ violations.add(violation(shapeName, "fileExtension",
+ "Member must satisfy regular expression pattern: " + FILE_EXTENSION_REGEX));
+ }
+ }
+ Serializer serializer = serializer(config);
+ if (serializer != null) {
+ // Parquet first, then ORC, the order AWS prints when both carry a
+ // violation (they can only both be present on a request the semantic
+ // "more than one serializer" check rejects afterwards).
+ collectParquetSerDeViolations(serializer.getParquetSerDe(), shapeName, violations);
+ collectOrcSerDeViolations(serializer.getOrcSerDe(), shapeName, violations);
+ }
+ // Serializer, then deserializer, then schema: the order AWS prints when
+ // violations sit in more than one of them.
+ collectOpenXMappingViolations(config, shapeName, violations);
+ collectSchemaViolations(config, shapeName, violations);
+ if (!violations.isEmpty()) {
+ throw constraintViolations(violations);
+ }
+ }
+
+ /**
+ * Every SchemaConfiguration string is a 1 to 1024 character value without
+ * whitespace, and unlike the collection members each broken rule is its own
+ * violation rather than one bracketed set. Member order is AWS's (probed), as is
+ * the {@code roleARN} spelling, which keeps ARN capitalised where the semantic
+ * message for the same member spells it RoleArn.
+ *
+ * A malformed but non-empty role ARN never reaches these rules on AWS: an
+ * InvalidParameterValueException ("Invalid role ARN.") preempts them. Floci
+ * validates no ARNs, so it answers only the length and pattern rules here, which
+ * is what an empty role ARN trips on AWS too.
+ */
+ private static void collectSchemaViolations(S3Destination config, String shapeName,
+ List<String> violations) {
+ var conversion = config.getDataFormatConversionConfiguration();
+ if (conversion == null || conversion.getSchemaConfiguration() == null) {
+ return;
+ }
+ SchemaConfiguration schema = conversion.getSchemaConfiguration();
+ String path = "dataFormatConversionConfiguration.schemaConfiguration";
+ collectSchemaMember(schema.getVersionId(), shapeName, path + ".versionId", violations);
+ collectSchemaMember(schema.getCatalogId(), shapeName, path + ".catalogId", violations);
+ collectSchemaMember(schema.getDatabaseName(), shapeName, path + ".databaseName", violations);
+ collectSchemaMember(schema.getRoleArn(), shapeName, path + ".roleARN", violations);
+ collectSchemaMember(schema.getRegion(), shapeName, path + ".region", violations);
+ collectSchemaMember(schema.getTableName(), shapeName, path + ".tableName", violations);
+ }
+
+ private static void collectSchemaMember(String value, String shapeName, String member,
+ List<String> violations) {
+ if (value == null) {
+ return;
+ }
+ if (value.isEmpty()) {
+ violations.add(violation(shapeName, member, "Member must have length greater than or equal to 1"));
+ } else if (value.length() > MAPPING_MAX_LENGTH) {
+ violations.add(violation(shapeName, member,
+ "Member must have length less than or equal to " + MAPPING_MAX_LENGTH));
+ }
+ if (!NO_WHITESPACE.matcher(value).matches()) {
+ violations.add(violation(shapeName, member,
+ "Member must satisfy regular expression pattern: " + NO_WHITESPACE_PATTERN));
+ }
+ }
+
+ /**
+ * The only collection member whose element constraints Floci enforces; the ones
+ * on OrcSerDe.BloomFilterColumns and HiveJsonSerDe.TimestampFormats are a
+ * documented gap. AWS reports the whole constraint set whichever part failed, at
+ * most once for the keys and once for the values, and applies a different pattern
+ * to each: keys may hold no whitespace at all, values merely may not be blank.
+ */
+ private static void collectOpenXMappingViolations(S3Destination config, String shapeName,
+ List<String> violations) {
+ var conversion = config.getDataFormatConversionConfiguration();
+ if (conversion == null || conversion.getInputFormatConfiguration() == null
+ || conversion.getInputFormatConfiguration().getDeserializer() == null) {
return;
}
- if (fileExtension.length() > FILE_EXTENSION_MAX_LENGTH) {
- throw constraintViolation(shapeName, "fileExtension",
- "Member must have length less than or equal to " + FILE_EXTENSION_MAX_LENGTH);
+ OpenXJsonSerDe openX = conversion.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe();
+ if (openX == null || openX.getColumnToJsonKeyMappings() == null) {
+ return;
+ }
+ String member = "dataFormatConversionConfiguration.inputFormatConfiguration.deserializer"
+ + ".openXJsonSerDe.columnToJsonKeyMappings";
+ boolean keyViolation = false;
+ boolean valueViolation = false;
+ for (Map.Entry<String, String> mapping : openX.getColumnToJsonKeyMappings().entrySet()) {
+ keyViolation |= !isMappingKeyValid(mapping.getKey());
+ valueViolation |= !isMappingValueValid(mapping.getValue());
}
- if (!FILE_EXTENSION.matcher(fileExtension).matches()) {
- throw constraintViolation(shapeName, "fileExtension",
- "Member must satisfy regular expression pattern: " + FILE_EXTENSION_REGEX);
+ if (keyViolation) {
+ violations.add(violation(shapeName, member,
+ "Map keys must satisfy constraint: " + constraintSet(NO_WHITESPACE_PATTERN)));
}
+ if (valueViolation) {
+ violations.add(violation(shapeName, member,
+ "Map value must satisfy constraint: " + constraintSet(NOT_BLANK_PATTERN)));
+ }
+ }
+
+ private static boolean isMappingKeyValid(String key) {
+ return key != null && lengthInRange(key) && NO_WHITESPACE.matcher(key).matches();
+ }
+
+ private static boolean isMappingValueValid(String value) {
+ return value != null && lengthInRange(value) && NOT_BLANK.matcher(value).matches();
+ }
+
+ private static boolean lengthInRange(String value) {
+ return !value.isEmpty() && value.length() <= MAPPING_MAX_LENGTH;
+ }
+
+ private static String constraintSet(String pattern) {
+ return "[Member must have length less than or equal to " + MAPPING_MAX_LENGTH
+ + ", Member must have length greater than or equal to 1"
+ + ", Member must satisfy regular expression pattern: " + pattern + "]";
+ }
+
+ private static Serializer serializer(S3Destination config) {
+ var conversion = config.getDataFormatConversionConfiguration();
+ if (conversion == null || conversion.getOutputFormatConfiguration() == null) {
+ return null;
+ }
+ return conversion.getOutputFormatConfiguration().getSerializer();
+ }
+
+ /**
+ * Smithy-level bounds on ParquetSerDe, enforced whenever the member is present,
+ * even on a disabled conversion configuration: they are model constraints, not
+ * the semantic checks {@link DataFormatConversionValidator} gates on Enabled.
+ */
+ private static void collectParquetSerDeViolations(ParquetSerDe parquet, String shapeName,
+ List<String> violations) {
+ if (parquet == null) {
+ return;
+ }
+ String parquetPath = SERIALIZER_PATH + ".parquetSerDe";
+ // Member order follows the one AWS prints when several constraints fail at
+ // once, which is neither declaration order nor alphabetical (probed with all
+ // five violated in a single request).
+ if (parquet.getMaxPaddingBytes() != null && parquet.getMaxPaddingBytes() < PARQUET_MIN_MAX_PADDING_BYTES) {
+ violations.add(violation(shapeName, parquetPath + ".maxPaddingBytes",
+ "Member must have value greater than or equal to " + PARQUET_MIN_MAX_PADDING_BYTES));
+ }
+ if (parquet.getBlockSizeBytes() != null && parquet.getBlockSizeBytes() < PARQUET_MIN_BLOCK_SIZE_BYTES) {
+ violations.add(violation(shapeName, parquetPath + ".blockSizeBytes",
+ "Member must have value greater than or equal to " + PARQUET_MIN_BLOCK_SIZE_BYTES));
+ }
+ if (parquet.getPageSizeBytes() != null && parquet.getPageSizeBytes() < PARQUET_MIN_PAGE_SIZE_BYTES) {
+ violations.add(violation(shapeName, parquetPath + ".pageSizeBytes",
+ "Member must have value greater than or equal to " + PARQUET_MIN_PAGE_SIZE_BYTES));
+ }
+ if (parquet.getCompression() != null && !PARQUET_COMPRESSIONS.contains(parquet.getCompression())) {
+ violations.add(violation(shapeName, parquetPath + ".compression",
+ "Member must satisfy enum value set: " + PARQUET_COMPRESSION_VALUE_SET));
+ }
+ if (parquet.getWriterVersion() != null && !PARQUET_WRITER_VERSIONS.contains(parquet.getWriterVersion())) {
+ violations.add(violation(shapeName, parquetPath + ".writerVersion",
+ "Member must satisfy enum value set: " + PARQUET_WRITER_VERSION_VALUE_SET));
+ }
+ }
+
+ /**
+ * The ORC counterpart. Floci rejects an enabled OrcSerDe semantically, but these
+ * are model constraints: they apply to a disabled configuration too, and AWS
+ * reports them before any semantic error. Member order and the enum spellings
+ * (note {@code [ZLIB, SNAPPY, NONE]}) were probed with all eight violated at once.
+ */
+ private static void collectOrcSerDeViolations(OrcSerDe orc, String shapeName, List<String> violations) {
+ if (orc == null) {
+ return;
+ }
+ String orcPath = SERIALIZER_PATH + ".orcSerDe";
+ collectRange(orc.getDictionaryKeyThreshold(), shapeName, orcPath + ".dictionaryKeyThreshold",
+ ORC_MIN_PROBABILITY, ORC_MAX_PROBABILITY, violations);
+ if (orc.getBlockSizeBytes() != null && orc.getBlockSizeBytes() < ORC_MIN_BLOCK_SIZE_BYTES) {
+ violations.add(violation(shapeName, orcPath + ".blockSizeBytes",
+ "Member must have value greater than or equal to " + ORC_MIN_BLOCK_SIZE_BYTES));
+ }
+ if (orc.getRowIndexStride() != null && orc.getRowIndexStride() < ORC_MIN_ROW_INDEX_STRIDE) {
+ violations.add(violation(shapeName, orcPath + ".rowIndexStride",
+ "Member must have value greater than or equal to " + ORC_MIN_ROW_INDEX_STRIDE));
+ }
+ collectRange(orc.getPaddingTolerance(), shapeName, orcPath + ".paddingTolerance",
+ ORC_MIN_PROBABILITY, ORC_MAX_PROBABILITY, violations);
+ if (orc.getStripeSizeBytes() != null && orc.getStripeSizeBytes() < ORC_MIN_STRIPE_SIZE_BYTES) {
+ violations.add(violation(shapeName, orcPath + ".stripeSizeBytes",
+ "Member must have value greater than or equal to " + ORC_MIN_STRIPE_SIZE_BYTES));
+ }
+ if (orc.getCompression() != null && !ORC_COMPRESSIONS.contains(orc.getCompression())) {
+ violations.add(violation(shapeName, orcPath + ".compression",
+ "Member must satisfy enum value set: " + ORC_COMPRESSION_VALUE_SET));
+ }
+ if (orc.getFormatVersion() != null && !ORC_FORMAT_VERSIONS.contains(orc.getFormatVersion())) {
+ violations.add(violation(shapeName, orcPath + ".formatVersion",
+ "Member must satisfy enum value set: " + ORC_FORMAT_VERSION_VALUE_SET));
+ }
+ collectRange(orc.getBloomFilterFalsePositiveProbability(), shapeName,
+ orcPath + ".bloomFilterFalsePositiveProbability",
+ ORC_MIN_PROBABILITY, ORC_MAX_PROBABILITY, violations);
+ }
+
+ /** AWS reports only the bound that was crossed, never both. */
+ private static void collectRange(Double value, String shapeName, String member,
+ double min, double max, List<String> violations) {
+ if (value == null) {
+ return;
+ }
+ // NaN satisfies neither comparison, so it would slip through a plain
+ // greater/less pair. AWS reports both bounds for it, the upper one first,
+ // and one bound for each infinity (probed).
+ boolean notANumber = Double.isNaN(value);
+ if (notANumber || value > max) {
+ violations.add(violation(shapeName, member,
+ "Member must have value less than or equal to " + (long) max));
+ }
+ if (notANumber || value < min) {
+ violations.add(violation(shapeName, member,
+ "Member must have value greater than or equal to " + (long) min));
+ }
+ }
+
+ private static String violation(String shapeName, String member, String constraint) {
+ return "Value at '" + shapeName + "." + member + "' failed to satisfy constraint: " + constraint;
}
- private static AwsException constraintViolation(String shapeName, String member, String constraint) {
- return new AwsException("ValidationException",
- "1 validation error detected: Value at '" + shapeName + "." + member
- + "' failed to satisfy constraint: " + constraint, 400);
+ private static AwsException constraintViolations(List<String> violations) {
+ String header = violations.size() == 1
+ ? "1 validation error detected: "
+ : violations.size() + " validation errors detected: ";
+ return new AwsException("ValidationException", header + String.join("; ", violations), 400);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
index f85af511fb..444cdd097e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
@@ -1,12 +1,14 @@
package io.github.hectorvent.floci.services.firehose.model;
import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.time.Instant;
import java.util.List;
+import java.util.Map;
import java.util.ArrayList;
@RegisterForReflection
@@ -214,6 +216,8 @@ public class DeliveryStreamDescription {
private EncryptionConfiguration encryptionConfiguration;
@JsonProperty("S3BackupMode")
private String s3BackupMode;
+ @JsonProperty("DataFormatConversionConfiguration")
+ private DataFormatConversionConfiguration dataFormatConversionConfiguration;
public S3Destination() {}
public String getRoleArn() { return roleArn; }
@@ -236,6 +240,27 @@ public class DeliveryStreamDescription {
public void setEncryptionConfiguration(EncryptionConfiguration encryptionConfiguration) { this.encryptionConfiguration = encryptionConfiguration; }
public String getS3BackupMode() { return s3BackupMode; }
public void setS3BackupMode(String s3BackupMode) { this.s3BackupMode = s3BackupMode; }
+ public DataFormatConversionConfiguration getDataFormatConversionConfiguration() {
+ return dataFormatConversionConfiguration;
+ }
+ public void setDataFormatConversionConfiguration(DataFormatConversionConfiguration configuration) {
+ this.dataFormatConversionConfiguration = configuration;
+ }
+
+ /**
+ * True when data format conversion applies to deliveries: the configuration is
+ * present and not explicitly disabled. AWS treats an omitted Enabled as enabled
+ * (probe: a config without Enabled is validated exactly like an enabled one).
+ *
+ * Ignored for serialization: an {@code is...} accessor with no backing field of
+ * that name is a property to Jackson, which would put a non-AWS
+ * {@code dataFormatConversionEnabled} member in every destination it renders.
+ */
+ @JsonIgnore
+ public boolean isDataFormatConversionEnabled() {
+ return dataFormatConversionConfiguration != null
+ && !Boolean.FALSE.equals(dataFormatConversionConfiguration.getEnabled());
+ }
/**
* Fills the members the wire contract marks required with the AWS defaults.
@@ -252,16 +277,24 @@ public class DeliveryStreamDescription {
if (encryptionConfiguration == null) {
encryptionConfiguration = EncryptionConfiguration.noEncryption();
}
+ // A conversion-enabled destination that specified no hints gets AWS's
+ // larger default size, verified against the service: the stored hints
+ // come back as 128 MiB, not the ordinary 5 MiB, while the interval
+ // default is unchanged. Filling in 5 here would report a size AWS never
+ // uses and would then fail the 64 MiB floor on the stream's next update.
+ int defaultSizeInMBs = isDataFormatConversionEnabled()
+ ? BufferingHints.CONVERSION_DEFAULT_SIZE_MBS
+ : BufferingHints.DEFAULT_SIZE_MBS;
if (bufferingHints == null) {
- bufferingHints = BufferingHints.defaults();
+ bufferingHints = BufferingHints.defaults(defaultSizeInMBs);
} else {
// Self-heal legacy persisted state; validation keeps partial
// hints out of the create/update paths.
if (bufferingHints.getSizeInMBs() == null) {
- bufferingHints.setSizeInMBs(5);
+ bufferingHints.setSizeInMBs(defaultSizeInMBs);
}
if (bufferingHints.getIntervalInSeconds() == null) {
- bufferingHints.setIntervalInSeconds(300);
+ bufferingHints.setIntervalInSeconds(BufferingHints.DEFAULT_INTERVAL_SECONDS);
}
}
}
@@ -292,6 +325,270 @@ public class DeliveryStreamDescription {
}
}
+ /**
+ * Members stay null-honest, mirroring real AWS: DescribeDeliveryStream echoes the
+ * configuration exactly as it was given (an empty ParquetSerDe stays {}, CatalogId
+ * and VersionId are not materialized), and UpdateDestination merges member-wise, so
+ * "not specified" must remain distinguishable from a value.
+ */
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class DataFormatConversionConfiguration {
+ @JsonProperty("Enabled")
+ private Boolean enabled;
+ @JsonProperty("SchemaConfiguration")
+ private SchemaConfiguration schemaConfiguration;
+ @JsonProperty("InputFormatConfiguration")
+ private InputFormatConfiguration inputFormatConfiguration;
+ @JsonProperty("OutputFormatConfiguration")
+ private OutputFormatConfiguration outputFormatConfiguration;
+
+ public DataFormatConversionConfiguration() {}
+
+ public Boolean getEnabled() { return enabled; }
+ public void setEnabled(Boolean enabled) { this.enabled = enabled; }
+ public SchemaConfiguration getSchemaConfiguration() { return schemaConfiguration; }
+ public void setSchemaConfiguration(SchemaConfiguration schemaConfiguration) {
+ this.schemaConfiguration = schemaConfiguration;
+ }
+ public InputFormatConfiguration getInputFormatConfiguration() { return inputFormatConfiguration; }
+ public void setInputFormatConfiguration(InputFormatConfiguration inputFormatConfiguration) {
+ this.inputFormatConfiguration = inputFormatConfiguration;
+ }
+ public OutputFormatConfiguration getOutputFormatConfiguration() { return outputFormatConfiguration; }
+ public void setOutputFormatConfiguration(OutputFormatConfiguration outputFormatConfiguration) {
+ this.outputFormatConfiguration = outputFormatConfiguration;
+ }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class SchemaConfiguration {
+ @JsonProperty("RoleARN")
+ private String roleArn;
+ @JsonProperty("CatalogId")
+ private String catalogId;
+ @JsonProperty("DatabaseName")
+ private String databaseName;
+ @JsonProperty("TableName")
+ private String tableName;
+ @JsonProperty("Region")
+ private String region;
+ @JsonProperty("VersionId")
+ private String versionId;
+
+ public SchemaConfiguration() {}
+
+ public String getRoleArn() { return roleArn; }
+ public void setRoleArn(String roleArn) { this.roleArn = roleArn; }
+ public String getCatalogId() { return catalogId; }
+ public void setCatalogId(String catalogId) { this.catalogId = catalogId; }
+ public String getDatabaseName() { return databaseName; }
+ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
+ public String getTableName() { return tableName; }
+ public void setTableName(String tableName) { this.tableName = tableName; }
+ public String getRegion() { return region; }
+ public void setRegion(String region) { this.region = region; }
+ public String getVersionId() { return versionId; }
+ public void setVersionId(String versionId) { this.versionId = versionId; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class InputFormatConfiguration {
+ @JsonProperty("Deserializer")
+ private Deserializer deserializer;
+
+ public InputFormatConfiguration() {}
+
+ public Deserializer getDeserializer() { return deserializer; }
+ public void setDeserializer(Deserializer deserializer) { this.deserializer = deserializer; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class Deserializer {
+ @JsonProperty("OpenXJsonSerDe")
+ private OpenXJsonSerDe openXJsonSerDe;
+ @JsonProperty("HiveJsonSerDe")
+ private HiveJsonSerDe hiveJsonSerDe;
+
+ public Deserializer() {}
+
+ public OpenXJsonSerDe getOpenXJsonSerDe() { return openXJsonSerDe; }
+ public void setOpenXJsonSerDe(OpenXJsonSerDe openXJsonSerDe) { this.openXJsonSerDe = openXJsonSerDe; }
+ public HiveJsonSerDe getHiveJsonSerDe() { return hiveJsonSerDe; }
+ public void setHiveJsonSerDe(HiveJsonSerDe hiveJsonSerDe) { this.hiveJsonSerDe = hiveJsonSerDe; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class OpenXJsonSerDe {
+ @JsonProperty("ConvertDotsInJsonKeysToUnderscores")
+ private Boolean convertDotsInJsonKeysToUnderscores;
+ @JsonProperty("CaseInsensitive")
+ private Boolean caseInsensitive;
+ @JsonProperty("ColumnToJsonKeyMappings")
+ private Map<String, String> columnToJsonKeyMappings;
+
+ public OpenXJsonSerDe() {}
+
+ public Boolean getConvertDotsInJsonKeysToUnderscores() { return convertDotsInJsonKeysToUnderscores; }
+ public void setConvertDotsInJsonKeysToUnderscores(Boolean convertDotsInJsonKeysToUnderscores) {
+ this.convertDotsInJsonKeysToUnderscores = convertDotsInJsonKeysToUnderscores;
+ }
+ public Boolean getCaseInsensitive() { return caseInsensitive; }
+ public void setCaseInsensitive(Boolean caseInsensitive) { this.caseInsensitive = caseInsensitive; }
+ public Map<String, String> getColumnToJsonKeyMappings() { return columnToJsonKeyMappings; }
+ public void setColumnToJsonKeyMappings(Map<String, String> columnToJsonKeyMappings) {
+ this.columnToJsonKeyMappings = columnToJsonKeyMappings;
+ }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class HiveJsonSerDe {
+ @JsonProperty("TimestampFormats")
+ private List<String> timestampFormats;
+
+ public HiveJsonSerDe() {}
+
+ public List<String> getTimestampFormats() { return timestampFormats; }
+ public void setTimestampFormats(List<String> timestampFormats) { this.timestampFormats = timestampFormats; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class OutputFormatConfiguration {
+ @JsonProperty("Serializer")
+ private Serializer serializer;
+
+ public OutputFormatConfiguration() {}
+
+ public Serializer getSerializer() { return serializer; }
+ public void setSerializer(Serializer serializer) { this.serializer = serializer; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class Serializer {
+ @JsonProperty("ParquetSerDe")
+ private ParquetSerDe parquetSerDe;
+ @JsonProperty("OrcSerDe")
+ private OrcSerDe orcSerDe;
+
+ public Serializer() {}
+
+ public ParquetSerDe getParquetSerDe() { return parquetSerDe; }
+ public void setParquetSerDe(ParquetSerDe parquetSerDe) { this.parquetSerDe = parquetSerDe; }
+ public OrcSerDe getOrcSerDe() { return orcSerDe; }
+ public void setOrcSerDe(OrcSerDe orcSerDe) { this.orcSerDe = orcSerDe; }
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class ParquetSerDe {
+ @JsonProperty("BlockSizeBytes")
+ private Long blockSizeBytes;
+ @JsonProperty("PageSizeBytes")
+ private Long pageSizeBytes;
+ @JsonProperty("Compression")
+ private String compression;
+ @JsonProperty("EnableDictionaryCompression")
+ private Boolean enableDictionaryCompression;
+ @JsonProperty("MaxPaddingBytes")
+ private Long maxPaddingBytes;
+ @JsonProperty("WriterVersion")
+ private String writerVersion;
+
+ public ParquetSerDe() {}
+
+ public Long getBlockSizeBytes() { return blockSizeBytes; }
+ public void setBlockSizeBytes(Long blockSizeBytes) { this.blockSizeBytes = blockSizeBytes; }
+ public Long getPageSizeBytes() { return pageSizeBytes; }
+ public void setPageSizeBytes(Long pageSizeBytes) { this.pageSizeBytes = pageSizeBytes; }
+ public String getCompression() { return compression; }
+ public void setCompression(String compression) { this.compression = compression; }
+ public Boolean getEnableDictionaryCompression() { return enableDictionaryCompression; }
+ public void setEnableDictionaryCompression(Boolean enableDictionaryCompression) {
+ this.enableDictionaryCompression = enableDictionaryCompression;
+ }
+ public Long getMaxPaddingBytes() { return maxPaddingBytes; }
+ public void setMaxPaddingBytes(Long maxPaddingBytes) { this.maxPaddingBytes = maxPaddingBytes; }
+ public String getWriterVersion() { return writerVersion; }
+ public void setWriterVersion(String writerVersion) { this.writerVersion = writerVersion; }
+ }
+
+ /**
+ * Floci rejects an <em>enabled</em> OrcSerDe configuration (DuckDB cannot write
+ * ORC), but a disabled one is stored like any other, so the members are mapped:
+ * a marker class would silently drop them through {@code ignoreUnknown} and
+ * DescribeDeliveryStream could no longer echo the configuration as supplied.
+ */
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public static class OrcSerDe {
+ @JsonProperty("StripeSizeBytes")
+ private Long stripeSizeBytes;
+ @JsonProperty("BlockSizeBytes")
+ private Long blockSizeBytes;
+ @JsonProperty("RowIndexStride")
+ private Integer rowIndexStride;
+ @JsonProperty("EnablePadding")
+ private Boolean enablePadding;
+ @JsonProperty("PaddingTolerance")
+ private Double paddingTolerance;
+ @JsonProperty("Compression")
+ private String compression;
+ @JsonProperty("BloomFilterColumns")
+ private List<String> bloomFilterColumns;
+ @JsonProperty("BloomFilterFalsePositiveProbability")
+ private Double bloomFilterFalsePositiveProbability;
+ @JsonProperty("DictionaryKeyThreshold")
+ private Double dictionaryKeyThreshold;
+ @JsonProperty("FormatVersion")
+ private String formatVersion;
+
+ public OrcSerDe() {}
+
+ public Long getStripeSizeBytes() { return stripeSizeBytes; }
+ public void setStripeSizeBytes(Long stripeSizeBytes) { this.stripeSizeBytes = stripeSizeBytes; }
+ public Long getBlockSizeBytes() { return blockSizeBytes; }
+ public void setBlockSizeBytes(Long blockSizeBytes) { this.blockSizeBytes = blockSizeBytes; }
+ public Integer getRowIndexStride() { return rowIndexStride; }
+ public void setRowIndexStride(Integer rowIndexStride) { this.rowIndexStride = rowIndexStride; }
+ public Boolean getEnablePadding() { return enablePadding; }
+ public void setEnablePadding(Boolean enablePadding) { this.enablePadding = enablePadding; }
+ public Double getPaddingTolerance() { return paddingTolerance; }
+ public void setPaddingTolerance(Double paddingTolerance) { this.paddingTolerance = paddingTolerance; }
+ public String getCompression() { return compression; }
+ public void setCompression(String compression) { this.compression = compression; }
+ public List<String> getBloomFilterColumns() { return bloomFilterColumns; }
+ public void setBloomFilterColumns(List<String> bloomFilterColumns) {
+ this.bloomFilterColumns = bloomFilterColumns;
+ }
+ public Double getBloomFilterFalsePositiveProbability() { return bloomFilterFalsePositiveProbability; }
+ public void setBloomFilterFalsePositiveProbability(Double bloomFilterFalsePositiveProbability) {
+ this.bloomFilterFalsePositiveProbability = bloomFilterFalsePositiveProbability;
+ }
+ public Double getDictionaryKeyThreshold() { return dictionaryKeyThreshold; }
+ public void setDictionaryKeyThreshold(Double dictionaryKeyThreshold) {
+ this.dictionaryKeyThreshold = dictionaryKeyThreshold;
+ }
+ public String getFormatVersion() { return formatVersion; }
+ public void setFormatVersion(String formatVersion) { this.formatVersion = formatVersion; }
+ }
+
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(JsonInclude.Include.NON_NULL)
@@ -329,12 +626,17 @@ public class DeliveryStreamDescription {
/**
* Members stay boxed and null-honest so validation can tell "not specified"
* from a value: AWS requires SizeInMBs and IntervalInSeconds to be specified
- * together, and the defaults (5 MiB / 300 s) apply only when the whole
- * object is absent.
+ * together, and the defaults apply only when the whole object is absent. The
+ * size default depends on the destination (see
+ * {@link S3Destination#applyDefaults()}); the interval default does not.
*/
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
public static class BufferingHints {
+ static final int DEFAULT_SIZE_MBS = 5;
+ static final int CONVERSION_DEFAULT_SIZE_MBS = 128;
+ static final int DEFAULT_INTERVAL_SECONDS = 300;
+
@JsonProperty("SizeInMBs")
private Integer sizeInMBs;
@JsonProperty("IntervalInSeconds")
@@ -343,9 +645,13 @@ public class DeliveryStreamDescription {
public BufferingHints() {}
public static BufferingHints defaults() {
+ return defaults(DEFAULT_SIZE_MBS);
+ }
+
+ static BufferingHints defaults(int sizeInMBs) {
BufferingHints hints = new BufferingHints();
- hints.sizeInMBs = 5;
- hints.intervalInSeconds = 300;
+ hints.sizeInMBs = sizeInMBs;
+ hints.intervalInSeconds = DEFAULT_INTERVAL_SECONDS;
return hints;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java
new file mode 100644
index 0000000000..8a0b118df0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionIntegrationTest.java
@@ -0,0 +1,249 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * Wire-level coverage for DataFormatConversionConfiguration: the unit tests in
+ * {@link FirehoseDataFormatConversionTest} build model objects directly, so
+ * this class is what exercises the JSON-to-model mapping of the nested shapes
+ * and the error responses as an SDK would see them.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class FirehoseDataFormatConversionIntegrationTest {
+
+ private static final String STREAM_NAME = "test-format-conversion-stream";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-delivery-role";
+ private static final String BUCKET_ARN = "arn:aws:s3:::conversion-archive";
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String TARGET_PREFIX = "Firehose_20150804.";
+ private static final String DESCRIPTION_PATH =
+ "DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void createAcceptsAFullConversionConfiguration() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "DeliveryStreamType": "DirectPut",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "CompressionFormat": "UNCOMPRESSED",
+ "BufferingHints": { "SizeInMBs": 64, "IntervalInSeconds": 60 },
+ "DataFormatConversionConfiguration": {
+ "Enabled": true,
+ "SchemaConfiguration": {
+ "RoleARN": "%s",
+ "DatabaseName": "analytics",
+ "TableName": "events",
+ "Region": "us-east-1"
+ },
+ "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {} } },
+ "OutputFormatConfiguration": {
+ "Serializer": { "ParquetSerDe": { "Compression": "GZIP", "WriterVersion": "V2" } }
+ }
+ }
+ }
+ }
+ """.formatted(STREAM_NAME, ROLE_ARN, BUCKET_ARN, ROLE_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamARN", notNullValue());
+ }
+
+ /**
+ * The description echoes the configuration as given: no CatalogId or VersionId
+ * materialized, the empty OpenXJsonSerDe kept, and the conversion member absent
+ * from the plain S3DestinationDescription mirror, which does not have it in
+ * AWS's model.
+ */
+ @Test
+ @Order(2)
+ void describeEchoesTheConversionConfigurationVerbatim() {
+ String conversionPath = DESCRIPTION_PATH + ".DataFormatConversionConfiguration";
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(conversionPath + ".Enabled", equalTo(true))
+ .body(conversionPath + ".SchemaConfiguration.RoleARN", equalTo(ROLE_ARN))
+ .body(conversionPath + ".SchemaConfiguration.DatabaseName", equalTo("analytics"))
+ .body(conversionPath + ".SchemaConfiguration.TableName", equalTo("events"))
+ .body(conversionPath + ".SchemaConfiguration.Region", equalTo("us-east-1"))
+ .body(conversionPath + ".SchemaConfiguration.CatalogId", nullValue())
+ .body(conversionPath + ".SchemaConfiguration.VersionId", nullValue())
+ .body(conversionPath + ".InputFormatConfiguration.Deserializer.OpenXJsonSerDe", notNullValue())
+ .body(conversionPath + ".OutputFormatConfiguration.Serializer.ParquetSerDe.Compression", equalTo("GZIP"))
+ .body(conversionPath + ".OutputFormatConfiguration.Serializer.ParquetSerDe.WriterVersion", equalTo("V2"))
+ .body(conversionPath + ".OutputFormatConfiguration.Serializer.ParquetSerDe.BlockSizeBytes", nullValue())
+ .body("DeliveryStreamDescription.Destinations[0].S3DestinationDescription"
+ + ".DataFormatConversionConfiguration", nullValue());
+ }
+
+ @Test
+ @Order(3)
+ void updateCarryingOnlyEnabledFalseKeepsTheStoredMembers() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "UpdateDestination")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "CurrentDeliveryStreamVersionId": "1",
+ "DestinationId": "destinationId-000000000001",
+ "ExtendedS3DestinationUpdate": {
+ "DataFormatConversionConfiguration": { "Enabled": false }
+ }
+ }
+ """.formatted(STREAM_NAME))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ String conversionPath = DESCRIPTION_PATH + ".DataFormatConversionConfiguration";
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamDescription.VersionId", equalTo("2"))
+ .body(conversionPath + ".Enabled", equalTo(false))
+ .body(conversionPath + ".SchemaConfiguration.DatabaseName", equalTo("analytics"))
+ .body(conversionPath + ".OutputFormatConfiguration.Serializer.ParquetSerDe", notNullValue());
+ }
+
+ /**
+ * The legacy shape does not model the member, and AWS's JSON protocol ignores a
+ * member a shape does not model, so the request succeeds with it disregarded
+ * rather than stored. The SDKs cannot send it here at all.
+ */
+ @Test
+ @Order(6)
+ void legacyS3DestinationShapeIgnoresTheConversionMember() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "legacy-shape-conversion-stream",
+ "S3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "CompressionFormat": "UNCOMPRESSED",
+ "DataFormatConversionConfiguration": {
+ "Enabled": true,
+ "SchemaConfiguration": { "RoleARN": "%s", "DatabaseName": "db", "TableName": "t" },
+ "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {} } },
+ "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {} } }
+ }
+ }
+ }
+ """.formatted(ROLE_ARN, BUCKET_ARN, ROLE_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"legacy-shape-conversion-stream\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body(DESCRIPTION_PATH + ".DataFormatConversionConfiguration", nullValue());
+ }
+
+ @Test
+ @Order(4)
+ void createRejectsCompressedDestinationsWithTheAwsMessage() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "rejected-conversion-stream",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "CompressionFormat": "GZIP",
+ "DataFormatConversionConfiguration": {
+ "Enabled": true,
+ "SchemaConfiguration": { "RoleARN": "%s", "DatabaseName": "db", "TableName": "t" },
+ "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {} } },
+ "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {} } }
+ }
+ }
+ }
+ """.formatted(ROLE_ARN, BUCKET_ARN, ROLE_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidArgumentException"))
+ .body("message", equalTo("The S3 destination's compression format must be set to UNCOMPRESSED"
+ + " when data format conversion is enabled. To enable compression within the converted"
+ + " output format, set the appropriate serialization option in the output format configuration."));
+ }
+
+ @Test
+ @Order(5)
+ void createRejectsParquetSerDeBoundsAtTheSmithyLayer() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "rejected-bounds-stream",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "%s",
+ "DataFormatConversionConfiguration": {
+ "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": { "PageSizeBytes": 1 } } }
+ }
+ }
+ }
+ """.formatted(ROLE_ARN, BUCKET_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo("1 validation error detected: Value at"
+ + " 'extendedS3DestinationConfiguration.dataFormatConversionConfiguration"
+ + ".outputFormatConfiguration.serializer.parquetSerDe.pageSizeBytes'"
+ + " failed to satisfy constraint: Member must have value greater than or equal to 65536"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java
new file mode 100644
index 0000000000..b51e3a0c27
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java
@@ -0,0 +1,522 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.BufferingHints;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.DataFormatConversionConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Deserializer;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.HiveJsonSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.InputFormatConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OutputFormatConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Every error message and the precedence between them was captured from real
+ * AWS (probe 2026-09-07, us-west-2), covering both CreateDeliveryStream and
+ * UpdateDestination. See okinaka/floci discussion 118 for the probe record.
+ */
+class FirehoseDataFormatConversionTest {
+
+ private static final String COMPRESSION_MESSAGE = "The S3 destination's compression format must be set"
+ + " to UNCOMPRESSED when data format conversion is enabled. To enable compression within the"
+ + " converted output format, set the appropriate serialization option in the output format configuration.";
+ private static final String BUFFER_MESSAGE =
+ "BufferingHints.SizeInMBs must be at least 64 when data format conversion is enabled.";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose-role";
+ private static final String DESTINATION_ID = "destinationId-000000000001";
+
+ private FirehoseService service;
+
+ @BeforeEach
+ void setUp() {
+ StorageFactory storageFactory = mock(StorageFactory.class);
+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();
+ when(storageFactory.create(anyString(), anyString(), any()))
+ .thenAnswer(invocation -> backends.computeIfAbsent(
+ invocation.getArgument(0) + "/" + invocation.getArgument(1),
+ k -> AccountAwareStorageBackend.inMemory("000000000000")));
+ EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);
+ when(firehoseCfg.enabled()).thenReturn(true);
+ when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);
+ when(firehoseCfg.flushRecordCount()).thenReturn(0);
+ EmulatorConfig.ServicesConfig servicesCfg = mock(EmulatorConfig.ServicesConfig.class);
+ when(servicesCfg.firehose()).thenReturn(firehoseCfg);
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ when(config.services()).thenReturn(servicesCfg);
+ RegionResolver regionResolver = mock(RegionResolver.class);
+ when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");
+ when(regionResolver.getAccountId()).thenReturn("000000000000");
+ service = new FirehoseService(storageFactory, Mockito.mock(S3Service.class),
+ Mockito.mock(KinesisService.class), regionResolver, new MutableClock(), config);
+ }
+
+ private static SchemaConfiguration schema() {
+ SchemaConfiguration schema = new SchemaConfiguration();
+ schema.setRoleArn(ROLE_ARN);
+ schema.setDatabaseName("db");
+ schema.setTableName("table");
+ schema.setRegion("us-east-1");
+ return schema;
+ }
+
+ private static InputFormatConfiguration openXInput() {
+ Deserializer deserializer = new Deserializer();
+ deserializer.setOpenXJsonSerDe(new OpenXJsonSerDe());
+ InputFormatConfiguration input = new InputFormatConfiguration();
+ input.setDeserializer(deserializer);
+ return input;
+ }
+
+ private static OutputFormatConfiguration parquetOutput() {
+ Serializer serializer = new Serializer();
+ serializer.setParquetSerDe(new ParquetSerDe());
+ OutputFormatConfiguration output = new OutputFormatConfiguration();
+ output.setSerializer(serializer);
+ return output;
+ }
+
+ private static DataFormatConversionConfiguration fullConversion() {
+ DataFormatConversionConfiguration conversion = new DataFormatConversionConfiguration();
+ conversion.setEnabled(true);
+ conversion.setSchemaConfiguration(schema());
+ conversion.setInputFormatConfiguration(openXInput());
+ conversion.setOutputFormatConfiguration(parquetOutput());
+ return conversion;
+ }
+
+ private static S3Destination destination(Consumer<S3Destination> customizer) {
+ S3Destination s3 = new S3Destination();
+ s3.setRoleArn(ROLE_ARN);
+ s3.setBucketArn("arn:aws:s3:::results");
+ BufferingHints hints = new BufferingHints();
+ hints.setSizeInMBs(64);
+ hints.setIntervalInSeconds(60);
+ s3.setBufferingHints(hints);
+ s3.setCompressionFormat("UNCOMPRESSED");
+ s3.setDataFormatConversionConfiguration(fullConversion());
+ customizer.accept(s3);
+ return s3;
+ }
+
+ private AwsException assertCreateRejected(Consumer<S3Destination> customizer) {
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.createDeliveryStream("stream", destination(customizer)));
+ assertEquals("InvalidArgumentException", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ return error;
+ }
+
+ private static BufferingHints hints(int sizeInMBs) {
+ BufferingHints hints = new BufferingHints();
+ hints.setSizeInMBs(sizeInMBs);
+ hints.setIntervalInSeconds(60);
+ return hints;
+ }
+
+ // ── create: message and precedence matrix ────────────────────────────────
+
+ @Test
+ void createAcceptsAFullyValidConversionConfiguration() {
+ assertDoesNotThrow(() -> service.createDeliveryStream("stream", destination(s3 -> {})));
+ }
+
+ @Test
+ void compressionCheckComesBeforeEveryOtherConversionCheck() {
+ AwsException error = assertCreateRejected(s3 -> {
+ s3.setCompressionFormat("GZIP");
+ s3.setBufferingHints(hints(5));
+ s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null);
+ });
+ assertEquals(COMPRESSION_MESSAGE, error.getMessage());
+ }
+
+ @Test
+ void bufferSizeCheckComesBeforeTheMissingMemberChecks() {
+ AwsException error = assertCreateRejected(s3 -> {
+ s3.setBufferingHints(hints(5));
+ s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null);
+ });
+ assertEquals(BUFFER_MESSAGE, error.getMessage());
+ }
+
+ @Test
+ void bufferSizeBelow64IsRejectedOnlyWhenExplicit() {
+ assertEquals(BUFFER_MESSAGE,
+ assertCreateRejected(s3 -> s3.setBufferingHints(hints(63))).getMessage());
+ // Probe: a destination with no BufferingHints at all passes the floor check.
+ assertDoesNotThrow(() -> service.createDeliveryStream("no-hints",
+ destination(s3 -> s3.setBufferingHints(null))));
+ }
+
+ /** Probed: AWS stores 128 MiB, not the ordinary 5 MiB, when a converting stream omits hints. */
+ @Test
+ void omittedHintsGetTheConversionSizeDefault() {
+ service.createDeliveryStream("converting", destination(s3 -> s3.setBufferingHints(null)));
+
+ BufferingHints stored = service.describeDeliveryStream("converting")
+ .s3Destination().getBufferingHints();
+ assertEquals(128, stored.getSizeInMBs());
+ assertEquals(300, stored.getIntervalInSeconds());
+ }
+
+ @Test
+ void omittedHintsKeepTheOrdinaryDefaultWithoutConversion() {
+ service.createDeliveryStream("plain", destination(s3 -> {
+ s3.setBufferingHints(null);
+ s3.setDataFormatConversionConfiguration(null);
+ }));
+
+ BufferingHints stored = service.describeDeliveryStream("plain")
+ .s3Destination().getBufferingHints();
+ assertEquals(5, stored.getSizeInMBs());
+ assertEquals(300, stored.getIntervalInSeconds());
+ }
+
+ /**
+ * Regression: storing 5 MiB for a converting stream made every later update fail
+ * the floor check, since the merged state then carried an explicit 5.
+ */
+ @Test
+ void anUnrelatedUpdateSucceedsOnAStreamCreatedWithoutHints() {
+ service.createDeliveryStream("converting", destination(s3 -> s3.setBufferingHints(null)));
+
+ assertDoesNotThrow(() -> service.updateDestination("converting", currentVersion("converting"),
+ DESTINATION_ID, update(u -> u.setPrefix("changed/"))));
+
+ S3Destination stored = service.describeDeliveryStream("converting").s3Destination();
+ assertEquals("changed/", stored.getPrefix());
+ assertEquals(128, stored.getBufferingHints().getSizeInMBs());
+ }
+
+ @Test
+ void missingMembersAreReportedInInputOutputSchemaOrder() {
+ DataFormatConversionConfiguration onlyEnabled = new DataFormatConversionConfiguration();
+ onlyEnabled.setEnabled(true);
+ assertEquals("InputFormatConfiguration must not be null",
+ assertCreateRejected(s3 -> s3.setDataFormatConversionConfiguration(onlyEnabled)).getMessage());
+
+ assertEquals("OutputFormatConfiguration must not be null",
+ assertCreateRejected(s3 -> {
+ s3.getDataFormatConversionConfiguration().setOutputFormatConfiguration(null);
+ s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null);
+ }).getMessage());
+
+ assertEquals("SchemaConfiguration must not be null",
+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null))
+ .getMessage());
+ }
+
+ /** An empty Deserializer or Serializer object reports the same error as an absent one. */
+ @Test
+ void emptyDeserializerAndSerializerReportTheNullMessage() {
+ assertEquals("Deserializer must not be null",
+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()
+ .getInputFormatConfiguration().setDeserializer(new Deserializer())).getMessage());
+ assertEquals("Serializer must not be null",
+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()
+ .getOutputFormatConfiguration().setSerializer(new Serializer())).getMessage());
+ }
+
+ @Test
+ void specifyingBothSerdesOfAUnionIsRejected() {
+ assertEquals("More than one deserializer specified. Only one may be chosen.",
+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()
+ .getInputFormatConfiguration().getDeserializer().setHiveJsonSerDe(new HiveJsonSerDe()))
+ .getMessage());
+ assertEquals("More than one serializer specified. Only one may be chosen.",
+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()
+ .getOutputFormatConfiguration().getSerializer().setOrcSerDe(new OrcSerDe()))
+ .getMessage());
+ }
+
+ /** Floci deviation: real AWS accepts OrcSerDe, but DuckDB cannot write ORC. */
+ @Test
+ void orcSerDeIsRejectedExplicitly() {
+ AwsException error = assertCreateRejected(s3 -> {
+ Serializer serializer = new Serializer();
+ serializer.setOrcSerDe(new OrcSerDe());
+ s3.getDataFormatConversionConfiguration().getOutputFormatConfiguration().setSerializer(serializer);
+ });
+ assertEquals("OrcSerDe is not supported. Floci supports only ParquetSerDe for data format conversion.",
+ error.getMessage());
+ }
+
+ @Test
+ void schemaMembersAreRequiredWithTheAwsCasedMessages() {
+ assertEquals("SchemaConfiguration.RoleArn must not be null or empty",
+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()
+ .getSchemaConfiguration().setRoleArn(null)).getMessage());
+ assertEquals("SchemaConfiguration.DatabaseName must not be null or empty",
+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()
+ .getSchemaConfiguration().setDatabaseName("")).getMessage());
+ assertEquals("SchemaConfiguration.TableName must not be null or empty",
+ assertCreateRejected(s3 -> s3.getDataFormatConversionConfiguration()
+ .getSchemaConfiguration().setTableName(null)).getMessage());
+ }
+
+ @Test
+ void omittedEnabledIsTreatedAsEnabled() {
+ AwsException error = assertCreateRejected(s3 -> {
+ s3.getDataFormatConversionConfiguration().setEnabled(null);
+ s3.getDataFormatConversionConfiguration().setSchemaConfiguration(null);
+ });
+ assertEquals("SchemaConfiguration must not be null", error.getMessage());
+ }
+
+ @Test
+ void disabledConversionSkipsEveryCheckButIsStored() {
+ service.createDeliveryStream("disabled", destination(s3 -> {
+ s3.setCompressionFormat("GZIP");
+ s3.setBufferingHints(hints(5));
+ s3.getDataFormatConversionConfiguration().setEnabled(false);
+ }));
+
+ DataFormatConversionConfiguration stored = service.describeDeliveryStream("disabled")
+ .s3Destination().getDataFormatConversionConfiguration();
+ assertEquals(false, stored.getEnabled());
+ assertNotNull(stored.getSchemaConfiguration());
+ }
+
+ // ── describe: null-honest echo ───────────────────────────────────────────
+
+ @Test
+ void describeEchoesTheConfigurationWithoutMaterializingDefaults() {
+ service.createDeliveryStream("stream", destination(s3 -> {}));
+
+ S3Destination described = service.describeDeliveryStream("stream").s3Destination();
+ DataFormatConversionConfiguration conversion = described.getDataFormatConversionConfiguration();
+ assertEquals(true, conversion.getEnabled());
+ assertNull(conversion.getSchemaConfiguration().getCatalogId());
+ assertNull(conversion.getSchemaConfiguration().getVersionId());
+ ParquetSerDe parquet = conversion.getOutputFormatConfiguration().getSerializer().getParquetSerDe();
+ assertNotNull(parquet);
+ assertNull(parquet.getCompression());
+ assertNull(parquet.getBlockSizeBytes());
+ }
+
+ /**
+ * A disabled ORC configuration is stored rather than rejected, so Describe has to
+ * echo its members: modeling OrcSerDe as a bare marker used to drop them silently.
+ */
+ @Test
+ void disabledOrcConfigurationKeepsItsSerDeMembers() {
+ OrcSerDe orc = new OrcSerDe();
+ orc.setStripeSizeBytes(67108864L);
+ orc.setCompression("SNAPPY");
+ orc.setFormatVersion("V0_12");
+ service.createDeliveryStream("orc", destination(s3 -> {
+ Serializer serializer = new Serializer();
+ serializer.setOrcSerDe(orc);
+ s3.getDataFormatConversionConfiguration().getOutputFormatConfiguration()
+ .setSerializer(serializer);
+ s3.getDataFormatConversionConfiguration().setEnabled(false);
+ }));
+
+ OrcSerDe stored = service.describeDeliveryStream("orc").s3Destination()
+ .getDataFormatConversionConfiguration().getOutputFormatConfiguration()
+ .getSerializer().getOrcSerDe();
+ assertEquals(67108864L, stored.getStripeSizeBytes());
+ assertEquals("SNAPPY", stored.getCompression());
+ assertEquals("V0_12", stored.getFormatVersion());
+ assertNull(stored.getRowIndexStride());
+ }
+
+ /**
+ * Every member AWS defines on a destination is PascalCase, so a camelCase key is
+ * proof that a helper leaked into the wire shape: an {@code is...} accessor with
+ * no backing field of that name becomes a Jackson property unless it is ignored.
+ */
+ @Test
+ void destinationRendersOnlyAwsDefinedMembers() throws Exception {
+ service.createDeliveryStream("stream", destination(s3 -> {}));
+ DeliveryStreamDescription.Destination destination =
+ service.describeDeliveryStream("stream").getDestinations().get(0);
+
+ com.fasterxml.jackson.databind.ObjectMapper mapper =
+ new com.fasterxml.jackson.databind.ObjectMapper();
+ for (Object rendered : List.of(destination.getExtendedS3DestinationDescription(),
+ destination.getS3DestinationDescription())) {
+ com.fasterxml.jackson.databind.JsonNode json = mapper.valueToTree(rendered);
+ json.fieldNames().forEachRemaining(name -> assertTrue(Character.isUpperCase(name.charAt(0)),
+ "unexpected non-AWS member rendered: " + name + " in " + json));
+ }
+ }
+
+ /** The plain S3DestinationDescription mirror has no conversion member in AWS's model. */
+ @Test
+ void conversionIsExtendedOnlyAndAbsentFromTheStandardView() {
+ service.createDeliveryStream("stream", destination(s3 -> {}));
+
+ DeliveryStreamDescription.Destination dest =
+ service.describeDeliveryStream("stream").getDestinations().get(0);
+ assertNotNull(dest.getExtendedS3DestinationDescription().getDataFormatConversionConfiguration());
+ assertNull(dest.getS3DestinationDescription().getDataFormatConversionConfiguration());
+ }
+
+ // ── update: validation on the merged state, member-wise merge ────────────
+
+ private String currentVersion(String name) {
+ return service.describeDeliveryStream(name).getVersionId();
+ }
+
+ private static S3Destination update(Consumer<S3Destination> customizer) {
+ S3Destination update = new S3Destination();
+ customizer.accept(update);
+ return update;
+ }
+
+ @Test
+ void updateAppliesTheSameChecksAgainstTheMergedState() {
+ service.createDeliveryStream("stream", destination(s3 -> {}));
+
+ AwsException gzip = assertThrows(AwsException.class, () -> service.updateDestination("stream",
+ currentVersion("stream"), DESTINATION_ID, update(u -> u.setCompressionFormat("GZIP"))));
+ assertEquals(COMPRESSION_MESSAGE, gzip.getMessage());
+
+ AwsException buffer = assertThrows(AwsException.class, () -> service.updateDestination("stream",
+ currentVersion("stream"), DESTINATION_ID, update(u -> u.setBufferingHints(hints(5)))));
+ assertEquals(BUFFER_MESSAGE, buffer.getMessage());
+ }
+
+ /** Probed: AWS rejects this too, since a plain stream stores the 5 MiB default. */
+ @Test
+ void enablingConversionOnAPlainStreamIsRejectedForItsStoredBufferSize() {
+ service.createDeliveryStream("plain", destination(s3 -> {
+ s3.setBufferingHints(null);
+ s3.setDataFormatConversionConfiguration(null);
+ }));
+
+ AwsException error = assertThrows(AwsException.class, () -> service.updateDestination("plain",
+ currentVersion("plain"), DESTINATION_ID,
+ update(u -> u.setDataFormatConversionConfiguration(fullConversion()))));
+
+ assertEquals(BUFFER_MESSAGE, error.getMessage());
+ }
+
+ @Test
+ void disablingWithOnlyEnabledPreservesTheStoredMembers() {
+ service.createDeliveryStream("stream", destination(s3 -> {}));
+ String versionBefore = currentVersion("stream");
+
+ DataFormatConversionConfiguration disable = new DataFormatConversionConfiguration();
+ disable.setEnabled(false);
+ service.updateDestination("stream", versionBefore, DESTINATION_ID,
+ update(u -> u.setDataFormatConversionConfiguration(disable)));
+
+ DataFormatConversionConfiguration stored = service.describeDeliveryStream("stream")
+ .s3Destination().getDataFormatConversionConfiguration();
+ assertEquals(false, stored.getEnabled());
+ assertEquals("db", stored.getSchemaConfiguration().getDatabaseName());
+ assertNotNull(stored.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe());
+ assertNotNull(stored.getOutputFormatConfiguration().getSerializer().getParquetSerDe());
+ assertFalse(versionBefore.equals(currentVersion("stream")));
+ }
+
+ @Test
+ void reenablingAloneIsRejectedWhileGzipIsInEffectAndLeavesTheStoreUntouched() {
+ service.createDeliveryStream("stream", destination(s3 -> {}));
+ DataFormatConversionConfiguration disable = new DataFormatConversionConfiguration();
+ disable.setEnabled(false);
+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,
+ update(u -> u.setDataFormatConversionConfiguration(disable)));
+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,
+ update(u -> u.setCompressionFormat("GZIP")));
+ String versionBefore = currentVersion("stream");
+
+ DataFormatConversionConfiguration reenable = new DataFormatConversionConfiguration();
+ reenable.setEnabled(true);
+ AwsException error = assertThrows(AwsException.class, () -> service.updateDestination("stream",
+ versionBefore, DESTINATION_ID, update(u -> u.setDataFormatConversionConfiguration(reenable))));
+
+ assertEquals(COMPRESSION_MESSAGE, error.getMessage());
+ S3Destination stored = service.describeDeliveryStream("stream").s3Destination();
+ assertEquals(versionBefore, currentVersion("stream"));
+ assertEquals("GZIP", stored.getCompressionFormat());
+ assertEquals(false, stored.getDataFormatConversionConfiguration().getEnabled());
+ }
+
+ /** Probed: SchemaConfiguration merges a member at a time, like its parent. */
+ @Test
+ void updatingOneSchemaMemberKeepsTheOtherStoredOnes() {
+ service.createDeliveryStream("stream", destination(s3 -> {}));
+
+ SchemaConfiguration schemaUpdate = new SchemaConfiguration();
+ schemaUpdate.setTableName("other_table");
+ DataFormatConversionConfiguration conversionUpdate = new DataFormatConversionConfiguration();
+ conversionUpdate.setSchemaConfiguration(schemaUpdate);
+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,
+ update(u -> u.setDataFormatConversionConfiguration(conversionUpdate)));
+
+ SchemaConfiguration stored = service.describeDeliveryStream("stream").s3Destination()
+ .getDataFormatConversionConfiguration().getSchemaConfiguration();
+ assertEquals("other_table", stored.getTableName());
+ assertEquals("db", stored.getDatabaseName());
+ assertEquals(ROLE_ARN, stored.getRoleArn());
+ assertEquals("us-east-1", stored.getRegion());
+ }
+
+ @Test
+ void reenablingTogetherWithUncompressedSucceedsOnTheStoredSchema() {
+ service.createDeliveryStream("stream", destination(s3 -> {}));
+ DataFormatConversionConfiguration disable = new DataFormatConversionConfiguration();
+ disable.setEnabled(false);
+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,
+ update(u -> u.setDataFormatConversionConfiguration(disable)));
+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,
+ update(u -> u.setCompressionFormat("GZIP")));
+
+ DataFormatConversionConfiguration reenable = new DataFormatConversionConfiguration();
+ reenable.setEnabled(true);
+ service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID, update(u -> {
+ u.setCompressionFormat("UNCOMPRESSED");
+ u.setDataFormatConversionConfiguration(reenable);
+ }));
+
+ S3Destination stored = service.describeDeliveryStream("stream").s3Destination();
+ assertTrue(stored.isDataFormatConversionEnabled());
+ assertEquals("UNCOMPRESSED", stored.getCompressionFormat());
+ assertEquals("db", stored.getDataFormatConversionConfiguration()
+ .getSchemaConfiguration().getDatabaseName());
+ }
+
+ @Test
+ void updateOntoAStreamWithoutADestinationValidatesTheUpdateItself() {
+ service.createDeliveryStream("bare", (S3Destination) null, List.of());
+
+ AwsException error = assertThrows(AwsException.class, () -> service.updateDestination("bare",
+ currentVersion("bare"), DESTINATION_ID, destination(s3 -> s3.setBufferingHints(hints(5)))));
+ assertEquals(BUFFER_MESSAGE, error.getMessage());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
index a922c07ed9..a1424d51e9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/S3DestinationValidatorTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.firehose;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -108,4 +109,517 @@ class S3DestinationValidatorTest {
new S3Destination(), "s3DestinationConfiguration"));
}
+ private static final String PARQUET_PATH = "extendedS3DestinationConfiguration"
+ + ".dataFormatConversionConfiguration.outputFormatConfiguration.serializer.parquetSerDe";
+
+ private static S3Destination withParquetSerDe(DeliveryStreamDescription.ParquetSerDe parquet) {
+ DeliveryStreamDescription.Serializer serializer = new DeliveryStreamDescription.Serializer();
+ serializer.setParquetSerDe(parquet);
+ DeliveryStreamDescription.OutputFormatConfiguration output =
+ new DeliveryStreamDescription.OutputFormatConfiguration();
+ output.setSerializer(serializer);
+ DeliveryStreamDescription.DataFormatConversionConfiguration conversion =
+ new DeliveryStreamDescription.DataFormatConversionConfiguration();
+ conversion.setOutputFormatConfiguration(output);
+ S3Destination config = new S3Destination();
+ config.setDataFormatConversionConfiguration(conversion);
+ return config;
+ }
+
+ @Test
+ void wireShapeAcceptsParquetSerDeAtItsExactBounds() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setBlockSizeBytes(67108864L);
+ parquet.setPageSizeBytes(65536L);
+ parquet.setCompression("SNAPPY");
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));
+ }
+
+ /** Both bounds violated at once join into one message, as probed against real AWS. */
+ @Test
+ void wireShapeJoinsBothParquetBoundViolationsIntoOneError() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setBlockSizeBytes(1L);
+ parquet.setPageSizeBytes(1L);
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));
+
+ assertEquals("ValidationException", error.getErrorCode());
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + PARQUET_PATH + ".blockSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 67108864; "
+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 65536", error.getMessage());
+ }
+
+ @Test
+ void wireShapeRejectsNegativeMaxPaddingBytes() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setMaxPaddingBytes(-1L);
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));
+
+ assertEquals("1 validation error detected: Value at '" + PARQUET_PATH + ".maxPaddingBytes' "
+ + "failed to satisfy constraint: Member must have value greater than or equal to 0",
+ error.getMessage());
+ parquet.setMaxPaddingBytes(0L);
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));
+ }
+
+ @Test
+ void wireShapeRejectsWriterVersionOutsideTheEnum() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setWriterVersion("V3");
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));
+
+ assertEquals("1 validation error detected: Value at '" + PARQUET_PATH + ".writerVersion' "
+ + "failed to satisfy constraint: Member must satisfy enum value set: [V1, V2]",
+ error.getMessage());
+ }
+
+ /** The order AWS prints is neither declaration order nor alphabetical; it was probed. */
+ @Test
+ void allFiveParquetViolationsAreReportedInTheOrderAwsUses() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setMaxPaddingBytes(-1L);
+ parquet.setBlockSizeBytes(1L);
+ parquet.setPageSizeBytes(1L);
+ parquet.setCompression("ZSTD");
+ parquet.setWriterVersion("V3");
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));
+
+ assertEquals("5 validation errors detected: "
+ + "Value at '" + PARQUET_PATH + ".maxPaddingBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 0; "
+ + "Value at '" + PARQUET_PATH + ".blockSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 67108864; "
+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 65536; "
+ + "Value at '" + PARQUET_PATH + ".compression' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [SNAPPY, GZIP, UNCOMPRESSED]; "
+ + "Value at '" + PARQUET_PATH + ".writerVersion' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [V1, V2]", error.getMessage());
+ }
+
+ private static final String ORC_PATH = "extendedS3DestinationConfiguration"
+ + ".dataFormatConversionConfiguration.outputFormatConfiguration.serializer.orcSerDe";
+
+ private static S3Destination withOrcSerDe(DeliveryStreamDescription.OrcSerDe orc) {
+ DeliveryStreamDescription.Serializer serializer = new DeliveryStreamDescription.Serializer();
+ serializer.setOrcSerDe(orc);
+ DeliveryStreamDescription.OutputFormatConfiguration output =
+ new DeliveryStreamDescription.OutputFormatConfiguration();
+ output.setSerializer(serializer);
+ DeliveryStreamDescription.DataFormatConversionConfiguration conversion =
+ new DeliveryStreamDescription.DataFormatConversionConfiguration();
+ conversion.setOutputFormatConfiguration(output);
+ S3Destination config = new S3Destination();
+ config.setDataFormatConversionConfiguration(conversion);
+ return config;
+ }
+
+ @Test
+ void wireShapeAcceptsOrcSerDeAtItsExactBounds() {
+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();
+ orc.setBlockSizeBytes(67108864L);
+ orc.setStripeSizeBytes(8388608L);
+ orc.setRowIndexStride(1000);
+ orc.setPaddingTolerance(0.0);
+ orc.setDictionaryKeyThreshold(1.0);
+ orc.setBloomFilterFalsePositiveProbability(0.0);
+ orc.setCompression("NONE");
+ orc.setFormatVersion("V0_12");
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));
+ }
+
+ /** Order and enum spellings probed with all eight violated in one request. */
+ @Test
+ void allEightOrcViolationsAreReportedInTheOrderAwsUses() {
+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();
+ orc.setStripeSizeBytes(1L);
+ orc.setBlockSizeBytes(1L);
+ orc.setRowIndexStride(1);
+ orc.setPaddingTolerance(-1.0);
+ orc.setCompression("ZSTD");
+ orc.setBloomFilterFalsePositiveProbability(2.0);
+ orc.setDictionaryKeyThreshold(2.0);
+ orc.setFormatVersion("V0_10");
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));
+
+ assertEquals("8 validation errors detected: "
+ + "Value at '" + ORC_PATH + ".dictionaryKeyThreshold' failed to satisfy constraint: "
+ + "Member must have value less than or equal to 1; "
+ + "Value at '" + ORC_PATH + ".blockSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 67108864; "
+ + "Value at '" + ORC_PATH + ".rowIndexStride' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 1000; "
+ + "Value at '" + ORC_PATH + ".paddingTolerance' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 0; "
+ + "Value at '" + ORC_PATH + ".stripeSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 8388608; "
+ + "Value at '" + ORC_PATH + ".compression' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [ZLIB, SNAPPY, NONE]; "
+ + "Value at '" + ORC_PATH + ".formatVersion' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [V0_11, V0_12]; "
+ + "Value at '" + ORC_PATH + ".bloomFilterFalsePositiveProbability' failed to satisfy constraint: "
+ + "Member must have value less than or equal to 1", error.getMessage());
+ }
+
+ /** PaddingTolerance is bounded on both sides, like the two probabilities. */
+ @Test
+ void orcPaddingToleranceRejectsValuesAboveOne() {
+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();
+ orc.setPaddingTolerance(999.0);
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));
+
+ assertEquals("1 validation error detected: Value at '" + ORC_PATH + ".paddingTolerance' "
+ + "failed to satisfy constraint: Member must have value less than or equal to 1",
+ error.getMessage());
+ orc.setPaddingTolerance(1.0);
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));
+ }
+
+ @Test
+ void orcProbabilitiesReportTheBoundThatWasCrossed() {
+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();
+ orc.setDictionaryKeyThreshold(-1.0);
+ orc.setBloomFilterFalsePositiveProbability(-1.0);
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withOrcSerDe(orc), "extendedS3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + ORC_PATH + ".dictionaryKeyThreshold' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 0; "
+ + "Value at '" + ORC_PATH + ".bloomFilterFalsePositiveProbability' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 0", error.getMessage());
+ }
+
+ /** Probed: a request carrying both serializers reports the Parquet violation first. */
+ @Test
+ void parquetViolationsPrecedeOrcOnesWhenBothSerializersAreSet() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setPageSizeBytes(1L);
+ DeliveryStreamDescription.OrcSerDe orc = new DeliveryStreamDescription.OrcSerDe();
+ orc.setRowIndexStride(1);
+ S3Destination config = withParquetSerDe(parquet);
+ config.getDataFormatConversionConfiguration().getOutputFormatConfiguration()
+ .getSerializer().setOrcSerDe(orc);
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ config, "extendedS3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 65536; "
+ + "Value at '" + ORC_PATH + ".rowIndexStride' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 1000", error.getMessage());
+ }
+
+ private static final String MAPPINGS_PATH = "extendedS3DestinationConfiguration"
+ + ".dataFormatConversionConfiguration.inputFormatConfiguration.deserializer"
+ + ".openXJsonSerDe.columnToJsonKeyMappings";
+ private static final String LENGTH_RULES = "[Member must have length less than or equal to 1024,"
+ + " Member must have length greater than or equal to 1,"
+ + " Member must satisfy regular expression pattern: ";
+
+ private static S3Destination withColumnMappings(java.util.Map<String, String> mappings) {
+ DeliveryStreamDescription.OpenXJsonSerDe openX = new DeliveryStreamDescription.OpenXJsonSerDe();
+ openX.setColumnToJsonKeyMappings(mappings);
+ DeliveryStreamDescription.Deserializer deserializer = new DeliveryStreamDescription.Deserializer();
+ deserializer.setOpenXJsonSerDe(openX);
+ DeliveryStreamDescription.InputFormatConfiguration input =
+ new DeliveryStreamDescription.InputFormatConfiguration();
+ input.setDeserializer(deserializer);
+ DeliveryStreamDescription.DataFormatConversionConfiguration conversion =
+ new DeliveryStreamDescription.DataFormatConversionConfiguration();
+ conversion.setInputFormatConfiguration(input);
+ S3Destination config = new S3Destination();
+ config.setDataFormatConversionConfiguration(conversion);
+ return config;
+ }
+
+ @Test
+ void wireShapeAcceptsWellFormedColumnMappings() {
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("ticker", "sym", "price", "unit price")),
+ "extendedS3DestinationConfiguration"));
+ }
+
+ /** Probed: keys may hold no whitespace, while values merely may not be blank. */
+ @Test
+ void mappingKeysRejectWhitespaceButValuesOnlyRejectBlanks() {
+ AwsException keyError = assertThrows(AwsException.class,
+ () -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("has space", "v")),
+ "extendedS3DestinationConfiguration"));
+ assertEquals("1 validation error detected: Value at '" + MAPPINGS_PATH + "' failed to satisfy "
+ + "constraint: Map keys must satisfy constraint: " + LENGTH_RULES + "^\\S+$]",
+ keyError.getMessage());
+
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("ticker", "has space")),
+ "extendedS3DestinationConfiguration"));
+
+ AwsException valueError = assertThrows(AwsException.class,
+ () -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("ticker", " ")),
+ "extendedS3DestinationConfiguration"));
+ assertEquals("1 validation error detected: Value at '" + MAPPINGS_PATH + "' failed to satisfy "
+ + "constraint: Map value must satisfy constraint: " + LENGTH_RULES + "^(?!\\s*$).+]",
+ valueError.getMessage());
+ }
+
+ /** However many entries break the rule, AWS reports keys once and values once. */
+ @Test
+ void mappingViolationsAreReportedOncePerAspectWithKeysFirst() {
+ java.util.Map<String, String> mappings = new java.util.LinkedHashMap<>();
+ mappings.put("has space", "");
+ mappings.put("also space", " ");
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(mappings), "extendedS3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + MAPPINGS_PATH + "' failed to satisfy constraint: "
+ + "Map keys must satisfy constraint: " + LENGTH_RULES + "^\\S+$]; "
+ + "Value at '" + MAPPINGS_PATH + "' failed to satisfy constraint: "
+ + "Map value must satisfy constraint: " + LENGTH_RULES + "^(?!\\s*$).+]",
+ error.getMessage());
+ }
+
+ /**
+ * The value rule is the pattern itself, not a String.isBlank() approximation:
+ * probed against AWS, which accepts an ideographic space here (its own later
+ * SerDe check is what rejects that, a deviation Floci documents) and rejects any
+ * value holding a line break, while isBlank() gets both backwards. A no-break
+ * space is not whitespace to either rule, so a key may hold one.
+ */
+ @Test
+ void mappingValuesFollowTheDeclaredPatternRatherThanIsBlank() {
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("ticker", " ")),
+ "extendedS3DestinationConfiguration"));
+
+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("ticker", "a\nb")),
+ "extendedS3DestinationConfiguration"));
+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("ticker", "ab\n")),
+ "extendedS3DestinationConfiguration"));
+
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("a b", "v")),
+ "extendedS3DestinationConfiguration"));
+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("a\nb", "v")),
+ "extendedS3DestinationConfiguration"));
+ }
+
+ @Test
+ void mappingLengthsAreBoundedAt1024() {
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("k".repeat(1024), "v".repeat(1024))),
+ "extendedS3DestinationConfiguration"));
+
+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withColumnMappings(java.util.Map.of("k".repeat(1025), "v")),
+ "extendedS3DestinationConfiguration"));
+ }
+
+ /** Probed: the deserializer's violations are printed after the serializer's. */
+ @Test
+ void serializerViolationsPrecedeDeserializerOnes() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setPageSizeBytes(1L);
+ S3Destination config = withParquetSerDe(parquet);
+ DeliveryStreamDescription.OpenXJsonSerDe openX = new DeliveryStreamDescription.OpenXJsonSerDe();
+ openX.setColumnToJsonKeyMappings(java.util.Map.of("has space", "v"));
+ DeliveryStreamDescription.Deserializer deserializer = new DeliveryStreamDescription.Deserializer();
+ deserializer.setOpenXJsonSerDe(openX);
+ DeliveryStreamDescription.InputFormatConfiguration input =
+ new DeliveryStreamDescription.InputFormatConfiguration();
+ input.setDeserializer(deserializer);
+ config.getDataFormatConversionConfiguration().setInputFormatConfiguration(input);
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ config, "extendedS3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 65536; "
+ + "Value at '" + MAPPINGS_PATH + "' failed to satisfy constraint: "
+ + "Map keys must satisfy constraint: " + LENGTH_RULES + "^\\S+$]", error.getMessage());
+ }
+
+ private static final String SCHEMA_PATH = "extendedS3DestinationConfiguration"
+ + ".dataFormatConversionConfiguration.schemaConfiguration";
+
+ private static S3Destination withSchema(DeliveryStreamDescription.SchemaConfiguration schema) {
+ DeliveryStreamDescription.DataFormatConversionConfiguration conversion =
+ new DeliveryStreamDescription.DataFormatConversionConfiguration();
+ conversion.setSchemaConfiguration(schema);
+ S3Destination config = new S3Destination();
+ config.setDataFormatConversionConfiguration(conversion);
+ return config;
+ }
+
+ /** Unlike the collection members, each broken schema rule is its own violation. */
+ @Test
+ void emptySchemaMemberReportsBothItsLengthAndItsPatternViolation() {
+ DeliveryStreamDescription.SchemaConfiguration schema =
+ new DeliveryStreamDescription.SchemaConfiguration();
+ schema.setCatalogId("");
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withSchema(schema), "extendedS3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + SCHEMA_PATH + ".catalogId' failed to satisfy constraint: "
+ + "Member must have length greater than or equal to 1; "
+ + "Value at '" + SCHEMA_PATH + ".catalogId' failed to satisfy constraint: "
+ + "Member must satisfy regular expression pattern: ^\\S+$", error.getMessage());
+ }
+
+ /**
+ * Probed: an empty role ARN trips the same length and pattern rules as the other
+ * members, reported under a path that keeps ARN capitalised. A malformed but
+ * non-empty one never reaches them on AWS, which answers InvalidParameterValueException
+ * first; Floci validates no ARNs, so it accepts that case.
+ */
+ @Test
+ void emptySchemaRoleArnIsReportedUnderTheRoleArnPath() {
+ DeliveryStreamDescription.SchemaConfiguration schema =
+ new DeliveryStreamDescription.SchemaConfiguration();
+ schema.setRoleArn("");
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withSchema(schema), "extendedS3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + SCHEMA_PATH + ".roleARN' failed to satisfy constraint: "
+ + "Member must have length greater than or equal to 1; "
+ + "Value at '" + SCHEMA_PATH + ".roleARN' failed to satisfy constraint: "
+ + "Member must satisfy regular expression pattern: ^\\S+$", error.getMessage());
+
+ schema.setRoleArn("arn:aws:iam::000000000000:role/firehose-role");
+ assertDoesNotThrow(() -> S3DestinationValidator.validateWireShape(
+ withSchema(schema), "extendedS3DestinationConfiguration"));
+ }
+
+ @Test
+ void schemaMembersAreReportedInTheOrderAwsUses() {
+ DeliveryStreamDescription.SchemaConfiguration schema =
+ new DeliveryStreamDescription.SchemaConfiguration();
+ schema.setCatalogId("c d");
+ schema.setDatabaseName("e f");
+ schema.setTableName("g h");
+ schema.setRegion("i j");
+ schema.setVersionId("k l");
+ schema.setRoleArn("m n");
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withSchema(schema), "extendedS3DestinationConfiguration"));
+
+ String pattern = "' failed to satisfy constraint: Member must satisfy regular expression pattern: ^\\S+$";
+ assertEquals("6 validation errors detected: "
+ + "Value at '" + SCHEMA_PATH + ".versionId" + pattern + "; "
+ + "Value at '" + SCHEMA_PATH + ".catalogId" + pattern + "; "
+ + "Value at '" + SCHEMA_PATH + ".databaseName" + pattern + "; "
+ + "Value at '" + SCHEMA_PATH + ".roleARN" + pattern + "; "
+ + "Value at '" + SCHEMA_PATH + ".region" + pattern + "; "
+ + "Value at '" + SCHEMA_PATH + ".tableName" + pattern, error.getMessage());
+ }
+
+ @Test
+ void schemaMembersAreBoundedAt1024() {
+ DeliveryStreamDescription.SchemaConfiguration schema =
+ new DeliveryStreamDescription.SchemaConfiguration();
+ schema.setCatalogId("x".repeat(1100));
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withSchema(schema), "extendedS3DestinationConfiguration"));
+
+ assertEquals("1 validation error detected: Value at '" + SCHEMA_PATH + ".catalogId' "
+ + "failed to satisfy constraint: Member must have length less than or equal to 1024",
+ error.getMessage());
+ }
+
+ /** Probed: schema violations are printed after the serializer's and deserializer's. */
+ @Test
+ void schemaViolationsComeLast() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setPageSizeBytes(1L);
+ S3Destination config = withParquetSerDe(parquet);
+ DeliveryStreamDescription.SchemaConfiguration schema =
+ new DeliveryStreamDescription.SchemaConfiguration();
+ schema.setTableName("g h");
+ config.getDataFormatConversionConfiguration().setSchemaConfiguration(schema);
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ config, "extendedS3DestinationConfiguration"));
+
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + PARQUET_PATH + ".pageSizeBytes' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 65536; "
+ + "Value at '" + SCHEMA_PATH + ".tableName' failed to satisfy constraint: "
+ + "Member must satisfy regular expression pattern: ^\\S+$", error.getMessage());
+ }
+
+ /** NaN satisfies no comparison, so AWS reports both bounds; each infinity trips one. */
+ @Test
+ void nonFiniteProportionsReportTheBoundsAwsReports() {
+ DeliveryStreamDescription.OrcSerDe nan = new DeliveryStreamDescription.OrcSerDe();
+ nan.setDictionaryKeyThreshold(Double.NaN);
+ AwsException nanError = assertThrows(AwsException.class,
+ () -> S3DestinationValidator.validateWireShape(
+ withOrcSerDe(nan), "extendedS3DestinationConfiguration"));
+ assertEquals("2 validation errors detected: "
+ + "Value at '" + ORC_PATH + ".dictionaryKeyThreshold' failed to satisfy constraint: "
+ + "Member must have value less than or equal to 1; "
+ + "Value at '" + ORC_PATH + ".dictionaryKeyThreshold' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 0", nanError.getMessage());
+
+ DeliveryStreamDescription.OrcSerDe positiveInfinity = new DeliveryStreamDescription.OrcSerDe();
+ positiveInfinity.setDictionaryKeyThreshold(Double.POSITIVE_INFINITY);
+ assertEquals("1 validation error detected: Value at '" + ORC_PATH + ".dictionaryKeyThreshold' "
+ + "failed to satisfy constraint: Member must have value less than or equal to 1",
+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withOrcSerDe(positiveInfinity), "extendedS3DestinationConfiguration")).getMessage());
+
+ DeliveryStreamDescription.OrcSerDe negativeInfinity = new DeliveryStreamDescription.OrcSerDe();
+ negativeInfinity.setDictionaryKeyThreshold(Double.NEGATIVE_INFINITY);
+ assertEquals("1 validation error detected: Value at '" + ORC_PATH + ".dictionaryKeyThreshold' "
+ + "failed to satisfy constraint: Member must have value greater than or equal to 0",
+ assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withOrcSerDe(negativeInfinity), "extendedS3DestinationConfiguration")).getMessage());
+ }
+
+ @Test
+ void wireShapeRejectsParquetCompressionOutsideTheEnum() {
+ DeliveryStreamDescription.ParquetSerDe parquet = new DeliveryStreamDescription.ParquetSerDe();
+ parquet.setCompression("ZSTD");
+
+ AwsException error = assertThrows(AwsException.class, () -> S3DestinationValidator.validateWireShape(
+ withParquetSerDe(parquet), "extendedS3DestinationConfiguration"));
+
+ assertEquals("1 validation error detected: Value at '" + PARQUET_PATH + ".compression' "
+ + "failed to satisfy constraint: Member must satisfy enum value set: [SNAPPY, GZIP, UNCOMPRESSED]",
+ error.getMessage());
+ }
+
}