Light, fluffy, and always free - The AWS Local Emulator alternative
feat(firehose): convert array, struct and map columns to Parquet (#3521)
Kenshin Okinaka6790024abc5e
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseParquetDeliveryTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseParquetDeliveryTest.javaindex f879efe406..5452e2b193 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseParquetDeliveryTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseParquetDeliveryTest.java@@ -66,6 +66,13 @@ import static org.assertj.core.api.Assertions.assertThat;* convert land under the evaluated ErrorOutputPrefix as AWS-shaped NDJSON* error lines. Against Floci this exercises the floci-duck sidecar for real,* which the emulator's unit tests mock.+ *+ * The table carries the complex Hive types beside the primitive ones, the shape+ * a real Firehose-to-Athena pipeline uses, so one stream covers both. The+ * coercions the nested records rely on were probed against real AWS: a value+ * that is not an array is wrapped into one, an array fills a struct by+ * position, member names match case-insensitively at every depth, and a+ * mismatch inside a nested value fails that record alone.*/@DisplayName("Firehose data format conversion delivery (JSON to Parquet)")@TestMethodOrder(MethodOrderer.OrderAnnotation.class)@@ -80,6 +87,13 @@ class FirehoseParquetDeliveryTest {private static final String VALID_RECORD_2 = "{\"ticker\": \"MISSING_FIELDS\"}\n";private static final String MISMATCH_RECORD = "{\"ticker\": \"BAD\", \"price\": \"oops\"}\n";private static final String UNPARSEABLE_RECORD = "this is not json at all\n";+ private static final String NESTED_RECORD =+ "{\"ticker\": \"NESTED\", \"tags\": [\"a\", \"b\"], \"person\": {\"name\": \"Alice\", \"age\": 30},"+ + " \"items\": [{\"sku\": \"s1\", \"qty\": 2}], \"attrs\": {\"k1\": [\"v1\", \"v2\"]}}\n";+ private static final String COERCED_NESTED_RECORD =+ "{\"ticker\": \"COERCED\", \"TAGS\": \"solo\", \"PERSON\": [\"Bob\", 41]}\n";+ private static final String NESTED_MISMATCH_RECORD =+ "{\"ticker\": \"BAD_NESTED\", \"items\": [{\"sku\": \"s5\", \"qty\": \"oops\"}]}\n";private static FirehoseClient firehose;private static GlueClient glue;@@ -119,7 +133,14 @@ class FirehoseParquetDeliveryTest {Column.builder().name("price").type("double").build(),Column.builder().name("qty").type("int").build(),Column.builder().name("active").type("boolean").build(),- Column.builder().name("ts").type("timestamp").build())+ Column.builder().name("ts").type("timestamp").build(),+ Column.builder().name("tags").type("array<string>").build(),+ Column.builder().name("person")+ .type("struct<name:string,age:int>").build(),+ Column.builder().name("items")+ .type("array<struct<sku:string,qty:int>>").build(),+ Column.builder().name("attrs")+ .type("map<string,array<string>>").build())// The destination prefix with a Parquet SerDe, as a real// Firehose-to-Parquet setup defines it, so the delivered// objects can be read back through Athena.@@ -176,7 +197,10 @@ class FirehoseParquetDeliveryTest {record(VALID_RECORD_1),record(VALID_RECORD_2),record(MISMATCH_RECORD),- record(UNPARSEABLE_RECORD))+ record(UNPARSEABLE_RECORD),+ record(NESTED_RECORD),+ record(COERCED_NESTED_RECORD),+ record(NESTED_MISMATCH_RECORD)).build()).failedPutCount()).isZero();}@@ -271,9 +295,12 @@ class FirehoseParquetDeliveryTest {byte[] body = s3.getObject(GetObjectRequest.builder().bucket(bucket).key(key).build()).readAllBytes();List<String> lines = new String(body, StandardCharsets.UTF_8).strip().lines().toList();- assertThat(lines).hasSize(2);+ assertThat(lines).hasSize(3);- String mismatch = lines.stream().filter(l -> l.contains("MalformedData")).findFirst().orElseThrow();+ String mismatch = lines.stream()+ .filter(l -> l.contains(Base64.getEncoder()+ .encodeToString(MISMATCH_RECORD.getBytes(StandardCharsets.UTF_8))))+ .findFirst().orElseThrow();assertThat(mismatch).contains("\"lastErrorCode\":\"DataFormatConversion.MalformedData\"");assertThat(mismatch).contains(Base64.getEncoder().encodeToString(MISMATCH_RECORD.getBytes(StandardCharsets.UTF_8)));@@ -283,14 +310,21 @@ class FirehoseParquetDeliveryTest {assertThat(unparseable).contains("\"lastErrorCode\":\"DataFormatConversion.ParseError\"");assertThat(unparseable).contains(Base64.getEncoder().encodeToString(UNPARSEABLE_RECORD.getBytes(StandardCharsets.UTF_8)));++ // A mismatch inside an array of structs is per-record too, not a batch failure.+ String nested = lines.stream()+ .filter(l -> l.contains(Base64.getEncoder()+ .encodeToString(NESTED_MISMATCH_RECORD.getBytes(StandardCharsets.UTF_8))))+ .findFirst().orElseThrow();+ assertThat(nested).contains("\"lastErrorCode\":\"DataFormatConversion.MalformedData\"");}/*** The PAR1 framing only proves the object is a Parquet file. Reading it back* through Athena, which resolves the Glue table to the delivery prefix, is what- * shows the projection itself is right: both valid records are present, the- * absent members became null rather than being dropped, and the values kept- * their column types instead of arriving as strings.+ * shows the projection itself is right: every convertible record is present,+ * the absent members became null rather than being dropped, and the values+ * kept their column types instead of arriving as strings.*/@Test@Order(3)@@ -322,7 +356,7 @@ class FirehoseParquetDeliveryTest {.filter(row -> !"ticker".equals(row.get(0))).toList();- assertThat(rows).as("both valid records must be in the Parquet object").hasSize(2);+ assertThat(rows).as("every convertible record must be in the Parquet object").hasSize(4);List<String> typed = rows.stream().filter(row -> "AAA".equals(row.get(0))).findFirst().orElseThrow();@@ -342,6 +376,92 @@ class FirehoseParquetDeliveryTest {}}+ /**+ * The nested columns read back the same way, through a list element, a struct+ * member, a struct member inside a list, and a map entry. The two records are+ * selected by name because a subscript of a null container is the one thing+ * this query cannot ask of the primitive-only records.+ */+ @Test+ @Order(4)+ @DisplayName("Nested array, struct and map values read back through Athena")+ void nestedValuesReadBackThroughAthena() throws Exception {+ waitForDeliveredKey("data/");++ try (AthenaClient athena = TestFixtures.athenaClient()) {+ StartQueryExecutionResponse started = athena.startQueryExecution(+ StartQueryExecutionRequest.builder()+ .queryString("SELECT ticker, tags[1], person.name, person.age,"+ + " items[1].sku, attrs['k1'][2] FROM \"" + database + "\".\""+ + TABLE + "\" WHERE ticker = 'NESTED' ORDER BY ticker")+ .workGroup("primary")+ .resultConfiguration(ResultConfiguration.builder()+ .outputLocation("s3://" + bucket + "/athena-results/")+ .build())+ .build());++ QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(+ athena, started.queryExecutionId(), Duration.ofSeconds(120));+ assertThat(status.state())+ .as("Athena read-back did not succeed: %s", status.stateChangeReason())+ .isEqualTo(QueryExecutionState.SUCCEEDED);++ List<String> nested = athena.getQueryResults(r -> r+ .queryExecutionId(started.queryExecutionId()))+ .resultSet().rows().stream()+ .map(row -> row.data().stream().map(Datum::varCharValue).toList())+ .filter(row -> "NESTED".equals(row.get(0)))+ .findFirst().orElseThrow();++ assertThat(nested.get(1)).isEqualTo("a");+ assertThat(nested.get(2)).isEqualTo("Alice");+ assertThat(nested.get(3)).isEqualTo("30");+ assertThat(nested.get(4)).isEqualTo("s1");+ assertThat(nested.get(5)).isEqualTo("v2");+ }+ }++ /**+ * The coerced record proves the probed leniency end to end: a scalar wrapped+ * into a one-element array, a struct filled from an array by position, and+ * both found under uppercase keys.+ */+ @Test+ @Order(5)+ @DisplayName("A scalar for an array and an array for a struct convert as AWS coerces them")+ void coercedNestedValuesReadBackThroughAthena() throws Exception {+ waitForDeliveredKey("data/");++ try (AthenaClient athena = TestFixtures.athenaClient()) {+ StartQueryExecutionResponse started = athena.startQueryExecution(+ StartQueryExecutionRequest.builder()+ .queryString("SELECT ticker, tags[1], person.name, person.age FROM \""+ + database + "\".\"" + TABLE + "\" WHERE ticker = 'COERCED'")+ .workGroup("primary")+ .resultConfiguration(ResultConfiguration.builder()+ .outputLocation("s3://" + bucket + "/athena-results/")+ .build())+ .build());++ QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(+ athena, started.queryExecutionId(), Duration.ofSeconds(120));+ assertThat(status.state())+ .as("Athena read-back did not succeed: %s", status.stateChangeReason())+ .isEqualTo(QueryExecutionState.SUCCEEDED);++ List<String> coerced = athena.getQueryResults(r -> r+ .queryExecutionId(started.queryExecutionId()))+ .resultSet().rows().stream()+ .map(row -> row.data().stream().map(Datum::varCharValue).toList())+ .filter(row -> "COERCED".equals(row.get(0)))+ .findFirst().orElseThrow();++ assertThat(coerced.get(1)).isEqualTo("solo");+ assertThat(coerced.get(2)).isEqualTo("Bob");+ assertThat(coerced.get(3)).isEqualTo("41");+ }+ }+private String waitForDeliveredKey(String prefix) throws InterruptedException {long deadline = System.currentTimeMillis() + POLL_TIMEOUT_MILLIS;while (System.currentTimeMillis() < deadline) {diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex 77e7d2874a..ab664586c5 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -63,6 +63,15 @@ Any other value, including differently-cased ones such as `SNAPPY` instead of `SEach buffered record is parsed and type-checked against the Glue table's columns. The valid rows are staged as NDJSON in `floci.services.firehose.staging-bucket`, then written to the destination as one Parquet object keyed like any other delivery but ending in `.parquet`, unless `FileExtension` replaces it. `ParquetSerDe.Compression` picks the codec, SNAPPY by default.+`array`, `struct` and `map` columns convert too, nested up to 32 levels deep. The coercion is as forgiving as AWS's, all of it probed:++- Under `OpenXJsonSerDe`, a value that is not an array becomes a **one-element array**, an object bound for `array<struct<..>>` included.+- A `struct` matches members by name, dropping undeclared ones and leaving missing ones null, so `{}` is a struct of nulls rather than a null struct. Under `OpenXJsonSerDe` **an array fills the members by position**, extras ignored, and only a scalar fails the record.+- **`HiveJsonSerDe` refuses what AWS documents it refusing, and fails the record** (all probed): a non-array in an `array<T>` column ("Start of Array expected"), a non-object in a `struct` column ("Start of Object expected"), a JSON string in a numeric column ("Current token (VALUE_STRING) not numeric"), and a non-scalar in a text column ("One or more fields have incorrect format. Exception when validating field (root)"). It still coerces a number into a text column, keeps nulls inside containers, and treats missing and undeclared members as OpenX does. The same numeric rule applies to top-level columns.+- A `map` takes only an object. Its **keys are lowercased under `OpenXJsonSerDe`**, which lowercases every JSON key at any depth before the value is read, and kept as written under `HiveJsonSerDe` or `CaseInsensitive: false`. The same lowercasing shows in the JSON text a non-scalar leaves in a text column.+- Names match case-insensitively at every depth under both deserializers, unless `CaseInsensitive` is `false`, which makes columns and members alike match exactly. The primitive coercions apply inside containers as at the top level, so `[1, 2, 3]` fills an `array<string>` and `"7"` an `int` member.+- A value that cannot be coerced at any depth fails **its whole record**.+Records that cannot be converted fail **individually**, as on AWS: the rest of the batch still converts. They are delivered instead to the evaluated `ErrorOutputPrefix`, where `!{firehose:error-output-type}` resolves to `format-conversion-failed`, as one extensionless object holding an NDJSON line per record: `attemptsMade`, `arrivalTimestamp`, `lastErrorCode`, `lastErrorMessage`, `attemptEndingTimestamp`, base64 `rawData`, and a `dataCatalogTable` block. The code is `DataFormatConversion.ParseError` for a record that is not valid JSON, `DataFormatConversion.MalformedData` for one whose values do not fit the column types.A failure of the batch as a whole is routed rather than discarded: a missing Glue table, a schema Floci cannot convert, or a sidecar failure sends every buffered record to the error output, under `DataFormatConversion.EntityNotFoundException`, `.UnsupportedSchema` and `.InternalError` respectively; those three codes are Floci's own. That routing is only as good as the write it depends on. A flush whose S3 write fails keeps its batch buffered, with or without conversion, and retries it on the next flush ahead of anything buffered since. The two destination writes of a mixed batch are not atomic either: the error output is written first, so its failure delivers nothing at all, but the reverse remains possible, where the error object lands and the Parquet write then fails; the retry then delivers the converted records and reports the failed ones a second time.@@ -79,7 +88,10 @@ An omitted `Enabled` counts as enabled; `Enabled: false` stores and merges the c- A malformed but non-empty `SchemaConfiguration.RoleARN` is accepted. AWS answers `InvalidParameterValueException` ("Invalid role ARN.") before its modeled rules run; Floci validates no ARNs, so it applies only the length and pattern rules, which an empty value trips exactly as on AWS.- **Several stored members do not reach the conversion**: `HiveJsonSerDe.TimestampFormats`, `OpenXJsonSerDe.ConvertDotsInJsonKeysToUnderscores`, every `ParquetSerDe` member except `Compression`, and `SchemaConfiguration.Region`, `CatalogId` and `VersionId`, which are not resolved against the Glue store (Floci's catalog is neither region- nor version-partitioned) and only flow into the error-output metadata.- An epoch timestamp outside the range AWS's Parquet writer can hold wraps there into an unrelated instant (probed: `999999999999` seconds arrives as the year 2092, not 33658). Floci keeps such a value as long as it is representable as a `TIMESTAMP`, and fails only a value beyond that, as its own record's `DataFormatConversion.MalformedData` rather than as a silently corrupted row.-- Complex and binary Hive types (`array`, `map`, `struct`, `binary`) are not convertible; a schema containing them routes the batch to the error output as `DataFormatConversion.UnsupportedSchema`.+- `binary` and `uniontype` columns are not convertible, nor is a type string Floci cannot parse or one nested past 32 levels; each routes the batch as `DataFormatConversion.UnsupportedSchema`.+- A map column given an array or a scalar fails its record with Floci's own wording; real AWS fails it with a cast error naming OpenX classes Floci does not use.+- The `HiveJsonSerDe` refusals reproduce AWS's messages up to the Jackson source location AWS appends to the numeric one (`at [Source: java.io.ByteArrayInputStream@...; line: 1, column: N]`), whose object hash changes per run. Two cells were not probed and follow the nearest measured one: a map column itself given a non-object fails like a struct ("Start of Object expected"), and a `boolean` column given a string is still coerced.+- The Parquet list element is named `element`, where AWS's writer names it `array_element`; both are shapes the LIST annotation allows.- Error-output records report the delivery time as both `arrivalTimestamp` and `attemptEndingTimestamp` (Floci does not track per-record arrival), and omit AWS's `sequenceNumber`/`subSequenceNumber` members. An absent `ErrorOutputPrefix` writes them at the bucket root with no default time prefix, which matches AWS's object-name documentation but was not probed.- The Parquet object carries no `Content-Encoding`, where AWS labels it `hadoop-snappy` even though the body is a plain Parquet file (probed), and its row order may differ from put order, as it does on AWS (probed).diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.javaindex e476813a76..f6ef8ebfc1 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java@@ -4,6 +4,10 @@ import com.fasterxml.jackson.databind.DeserializationFeature;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.ObjectReader;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.JsonNodeFactory;+import com.fasterxml.jackson.databind.node.ObjectNode;+import com.fasterxml.jackson.databind.node.TextNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;@@ -38,10 +42,13 @@ import java.time.format.DateTimeFormatterBuilder;import java.time.format.SignStyle;import java.util.ArrayList;import java.util.Base64;+import java.util.HashSet;+import java.util.Iterator;import java.util.LinkedHashMap;import java.util.List;import java.util.Locale;import java.util.Map;+import java.util.Set;import java.util.UUID;import java.util.regex.Matcher;import java.util.regex.Pattern;@@ -145,9 +152,9 @@ public class FirehoseParquetConverter {DataFormatConversionConfiguration conversion = s3.getDataFormatConversionConfiguration();SchemaConfiguration schemaConfig = conversion.getSchemaConfiguration();- List<ColumnSpec> schema;+ List<SchemaColumn> schema;try {- schema = resolveSchema(schemaConfig);+ schema = resolveSchema(schemaConfig, SerDeRules.of(conversion));} catch (AwsException e) {return failWholeBatch(stream, s3, bucket, records, deliveryTime, schemaConfig,"DataFormatConversion." + e.getErrorCode(), e.getMessage());@@ -233,7 +240,7 @@ public class FirehoseParquetConverter {* SchemaConfiguration.Region and CatalogId only flow into the error-output* metadata; VersionId is not resolved (the live table is always used).*/- private List<ColumnSpec> resolveSchema(SchemaConfiguration schemaConfig) {+ private List<SchemaColumn> resolveSchema(SchemaConfiguration schemaConfig, SerDeRules rules) {Table table = glueService.getTable(schemaConfig.getDatabaseName(), schemaConfig.getTableName());List<Column> columns = table.getStorageDescriptor() == null? null : table.getStorageDescriptor().getColumns();@@ -241,7 +248,7 @@ public class FirehoseParquetConverter {throw new UnsupportedSchemaException("Glue table " + schemaConfig.getDatabaseName() + "."+ schemaConfig.getTableName() + " has no columns.");}- List<ColumnSpec> schema = new ArrayList<>(columns.size());+ List<SchemaColumn> schema = new ArrayList<>(columns.size());for (Column column : columns) {if (column == null) {throw new UnsupportedSchemaException("Table " + schemaConfig.getDatabaseName() + "."@@ -252,19 +259,19 @@ public class FirehoseParquetConverter {throw new UnsupportedSchemaException("Column name " + name+ " is not supported for Parquet conversion.");}- ColumnSpec spec = ColumnSpec.fromHiveType(name, column.getType());- if (spec == null) {+ ValueType type = HiveTypeParser.parse(name, column.getType(), rules);+ if (type == null) {throw new UnsupportedSchemaException("Column " + name + " has Hive type " + column.getType()+ ", which Floci cannot convert to Parquet.");}- schema.add(spec);+ schema.add(new SchemaColumn(name, type));}return schema;}// ── per-record projection ────────────────────────────────────────────────- private Map<String, Object> project(byte[] record, List<ColumnSpec> schema, KeyResolver keyResolver) {+ private Map<String, Object> project(byte[] record, List<SchemaColumn> schema, KeyResolver keyResolver) {JsonNode node;try {// readValue rather than readTree: an empty or blank record makes the tree@@ -281,13 +288,50 @@ public class FirehoseParquetConverter {}Map<String, JsonNode> index = keyResolver.index(node);Map<String, Object> row = new LinkedHashMap<>();- for (ColumnSpec column : schema) {+ for (SchemaColumn column : schema) {JsonNode value = keyResolver.find(node, index, column.name());- row.put(column.name(), value == null || value.isNull() ? null : column.coerce(value));+ row.put(column.name(), value == null || value.isNull() ? null : column.type().coerce(value));}return row;}+ /**+ * Whether column and struct member names match case-insensitively. Both SerDes+ * do, HiveJsonSerDe included (probed), unless OpenX is told not to.+ */+ private static boolean caseInsensitive(DataFormatConversionConfiguration conversion) {+ OpenXJsonSerDe openX = conversion.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe();+ return openX == null || !Boolean.FALSE.equals(openX.getCaseInsensitive());+ }++ /**+ * What the configured deserializer does with a value, all of it probed against+ * real AWS.+ *+ * <p>{@code caseInsensitive}: column and struct member names match regardless of+ * case. Both SerDes do this unless OpenX is told not to.+ *+ * <p>{@code lowercaseKeys}: keys that are data rather than schema, a map's entry+ * keys and the JSON text a non-scalar leaves in a text column, are stored+ * lowercased. Only a case-insensitive OpenX does this, since it lowercases every+ * JSON key before deserializing; HiveJsonSerDe keeps them as written.+ *+ * <p>{@code lenient}: a value of the wrong shape is coerced rather than refused.+ * OpenX wraps a non-array into a one-element array, fills a struct from an array+ * by position, and serializes a non-scalar into a text column; it also parses a+ * string into a numeric column. HiveJsonSerDe refuses all four and fails the+ * record, while still coercing a number into a text column, taking a missing+ * member as null and dropping an undeclared one.+ */+ private record SerDeRules(boolean caseInsensitive, boolean lowercaseKeys, boolean lenient) {++ static SerDeRules of(DataFormatConversionConfiguration conversion) {+ boolean openX = conversion.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe() != null;+ boolean caseInsensitive = FirehoseParquetConverter.caseInsensitive(conversion);+ return new SerDeRules(caseInsensitive, openX && caseInsensitive, openX);+ }+ }+/*** Maps a schema column to the JSON member holding its value. OpenX's* ColumnToJsonKeyMappings renames the key that is looked up; the lookup@@ -302,7 +346,7 @@ public class FirehoseParquetConverter {OpenXJsonSerDe openX = conversion.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe();Map<String, String> mappings = openX == null || openX.getColumnToJsonKeyMappings() == null? Map.of() : openX.getColumnToJsonKeyMappings();- boolean caseInsensitive = openX == null || !Boolean.FALSE.equals(openX.getCaseInsensitive());+ boolean caseInsensitive = caseInsensitive(conversion);return new KeyResolver() {@Overridepublic Map<String, JsonNode> index(JsonNode record) {@@ -341,7 +385,7 @@ public class FirehoseParquetConverter {* is known, so a conversion failure never has to undo a destination write.*/private String convertToStagedParquet(DeliveryStreamDescription stream, S3Destination s3,- List<ColumnSpec> schema, List<Map<String, Object>> rows,+ List<SchemaColumn> schema, List<Map<String, Object>> rows,Instant deliveryTime, ParquetSerDe parquetSerDe) throws Exception {String runId = UUID.randomUUID().toString();String parquetKey = "firehose-staging/" + stream.getDeliveryStreamName() + "/" + runId + ".parquet";@@ -359,14 +403,14 @@ public class FirehoseParquetConverter {"application/x-ndjson", Map.of());try {StringBuilder columns = new StringBuilder();- for (ColumnSpec column : schema) {+ for (SchemaColumn column : schema) {if (columns.length() > 0) {columns.append(", ");}// Glue column names are broader than SQL identifiers, so the name is// escaped like any other literal rather than restricted up front.columns.append('\'').append(escapeSqlLiteral(column.name()))- .append("': '").append(column.duckType()).append('\'');+ .append("': '").append(escapeSqlLiteral(column.type().duckType())).append('\'');}// floci-duck's /execute wraps the main sql field in a CSV-emitting COPY// for Athena compatibility, so the Parquet COPY goes in setup_sql and the@@ -531,30 +575,407 @@ public class FirehoseParquetConverter {}}- enum ColumnKind { BOOLEAN, INTEGRAL, FLOATING, DECIMAL, TEXT, DATE, TIMESTAMP }+ private enum ColumnKind { BOOLEAN, INTEGRAL, FLOATING, DECIMAL, TEXT, DATE, TIMESTAMP }++ /** A Glue column and the type tree its values are coerced against. */+ private record SchemaColumn(String name, ValueType type) {}++ /**+ * One node of a column's type. Hive complex types nest, so coercion recurses,+ * and a failure at any depth fails that record as a whole, as one in a scalar+ * column already does (probed).+ */+ private sealed interface ValueType permits ScalarType, ListType, StructType, MapType {++ /** The DuckDB type {@code read_json} is given for this node. */+ String duckType();++ /** Coerces one non-null JSON value, or throws {@link RecordConversionException}. */+ Object coerce(JsonNode value);+ }++ /** A struct member: Hive keeps the declared name, the lookup is what varies. */+ private record StructField(String name, ValueType type) {}++ /**+ * Hive {@code array<T>}. Under OpenX a value that is not an array is wrapped into+ * a one-element array rather than failing the record: probed, a string in an+ * {@code array<string>} column arrives as a single-element array, and so does an+ * object in an {@code array<struct<..>>} one. HiveJsonSerDe fails the record+ * instead (probed), with the message reproduced here.+ */+ private record ListType(ValueType element, SerDeRules rules) implements ValueType {++ @Override+ public String duckType() {+ return element.duckType() + "[]";+ }++ @Override+ public Object coerce(JsonNode value) {+ if (!value.isArray()) {+ if (!rules.lenient()) {+ throw shapeMismatch("java.io.IOException: Start of Array expected");+ }+ List<Object> wrapped = new ArrayList<>(1);+ wrapped.add(element.coerce(value));+ return wrapped;+ }+ List<Object> coerced = new ArrayList<>(value.size());+ for (JsonNode entry : value) {+ coerced.add(entry == null || entry.isNull() ? null : element.coerce(entry));+ }+ return coerced;+ }+ }++ /**+ * Hive {@code struct<a:T,..>}. An object matches members by name, a missing one+ * being null and an undeclared one dropped, so an empty object is a struct of+ * null members rather than a null struct; both SerDes agree on that (probed).+ * Under OpenX an array fills the members by position instead, extras ignored,+ * and only a scalar fails the record. HiveJsonSerDe fails the record for anything+ * but an object (probed).+ */+ private record StructType(List<StructField> fields, SerDeRules rules) implements ValueType {++ @Override+ public String duckType() {+ StringBuilder type = new StringBuilder("STRUCT(");+ for (int i = 0; i < fields.size(); i++) {+ if (i > 0) {+ type.append(", ");+ }+ type.append('"').append(fields.get(i).name().replace("\"", "\"\"")).append("\" ")+ .append(fields.get(i).type().duckType());+ }+ return type.append(')').toString();+ }++ @Override+ public Object coerce(JsonNode value) {+ if (!value.isObject() && !rules.lenient()) {+ throw shapeMismatch("java.io.IOException: Start of Object expected");+ }+ if (value.isArray()) {+ return positional(value);+ }+ if (!value.isObject()) {+ // The probed wording, its doubled space included: AWS names the Java+ // type its own parser read the scalar as.+ throw new RecordConversionException("DataFormatConversion.MalformedData",+ "Data does not match the schema. Data is not JSONObject but "+ + javaTypeName(value) + " with value " + value.asText());+ }+ Map<String, JsonNode> byLowerKey = rules.caseInsensitive() ? lowerKeyIndex(value) : Map.of();+ Map<String, Object> members = new LinkedHashMap<>();+ for (StructField field : fields) {+ JsonNode member = value.get(field.name());+ if (member == null && rules.caseInsensitive()) {+ member = byLowerKey.get(field.name().toLowerCase(Locale.ROOT));+ }+ members.put(field.name(),+ member == null || member.isNull() ? null : field.type().coerce(member));+ }+ return members;+ }++ private Map<String, Object> positional(JsonNode value) {+ Map<String, Object> members = new LinkedHashMap<>();+ for (int i = 0; i < fields.size(); i++) {+ JsonNode member = value.get(i);+ members.put(fields.get(i).name(),+ member == null || member.isNull() ? null : fields.get(i).type().coerce(member));+ }+ return members;+ }+ }++ /**+ * Hive {@code map<K,V>}. Only an object is accepted, and its keys are lowercased+ * only under a case-insensitive OpenX, which lowercases every JSON key before+ * deserializing: probed, {@code {"MixedKey": ..}} is stored under+ * {@code mixedkey} there and kept as written under HiveJsonSerDe, which resolves+ * column and member names case-insensitively all the same. See {@link SerDeRules}.+ */+ private record MapType(ValueType key, ValueType value, SerDeRules rules) implements ValueType {++ @Override+ public String duckType() {+ return "MAP(" + key.duckType() + ", " + value.duckType() + ")";+ }++ @Override+ public Object coerce(JsonNode node) {+ if (!node.isObject()) {+ if (!rules.lenient()) {+ // By analogy with a struct; a map column itself given a non-object+ // was not probed under HiveJsonSerDe (documented).+ throw shapeMismatch("java.io.IOException: Start of Object expected");+ }+ // Real AWS fails this record too, but with an OpenX cast error naming+ // classes Floci does not use, so the wording here is its own+ // (documented deviation).+ throw new RecordConversionException("DataFormatConversion.MalformedData",+ "Data does not match the schema. A map column cannot hold a JSON "+ + (node.isArray() ? "array." : "scalar."));+ }+ Map<Object, Object> entries = new LinkedHashMap<>();+ Iterator<Map.Entry<String, JsonNode>> fields = node.fields();+ while (fields.hasNext()) {+ Map.Entry<String, JsonNode> field = fields.next();+ String name = rules.lowercaseKeys() ? field.getKey().toLowerCase(Locale.ROOT) : field.getKey();+ JsonNode member = field.getValue();+ entries.put(key.coerce(TextNode.valueOf(name)),+ member == null || member.isNull() ? null : value.coerce(member));+ }+ return entries;+ }+ }++ private static Map<String, JsonNode> lowerKeyIndex(JsonNode node) {+ Map<String, JsonNode> byLowerKey = new LinkedHashMap<>();+ Iterator<Map.Entry<String, JsonNode>> fields = node.fields();+ while (fields.hasNext()) {+ Map.Entry<String, JsonNode> field = fields.next();+ byLowerKey.putIfAbsent(field.getKey().toLowerCase(Locale.ROOT), field.getValue());+ }+ return byLowerKey;+ }++ /** The record failure HiveJsonSerDe reports for a value of the wrong shape (probed). */+ private static RecordConversionException shapeMismatch(String cause) {+ return new RecordConversionException("DataFormatConversion.MalformedData",+ "Data does not match the schema. " + cause);+ }++ private static String javaTypeName(JsonNode value) {+ if (value.isBoolean()) {+ return "java.lang.Boolean";+ }+ if (value.isIntegralNumber()) {+ return "java.lang.Long";+ }+ if (value.isNumber()) {+ return "java.lang.Double";+ }+ return "java.lang.String";+ }++ /**+ * Parses a Glue column's Hive type into a {@link ValueType}. Complex types+ * nest, so this is a recursive descent rather than a regex: the commas in+ * {@code struct<a:map<string,int>,b:string>} belong to different levels. A+ * type it cannot represent, {@code binary} and {@code uniontype} included,+ * parses to null, which the caller reports as an unsupported schema for the+ * whole batch.+ */+ private static final class HiveTypeParser {++ /** Deep enough for any real schema, shallow enough that the recursion cannot overflow. */+ private static final int MAX_DEPTH = 32;++ private final String type;+ private final SerDeRules rules;+ private int pos;++ private HiveTypeParser(String type, SerDeRules rules) {+ this.type = type;+ this.rules = rules;+ }++ static ValueType parse(String name, String hiveType, SerDeRules rules) {+ if (hiveType == null || hiveType.isBlank()) {+ return null;+ }+ HiveTypeParser parser = new HiveTypeParser(hiveType, rules);+ ValueType parsed = parser.parseType(name, 0);+ parser.skipSpaces();+ // Trailing content means the type was never understood, as "char(10)garbage"+ // is not the char column a prefix match would take it for.+ return parsed == null || parser.pos != parser.type.length() ? null : parsed;+ }++ private ValueType parseType(String name, int depth) {+ if (depth > MAX_DEPTH) {+ return null;+ }+ String word = readWord();+ if (word.isEmpty()) {+ return null;+ }+ return switch (word.toLowerCase(Locale.ROOT)) {+ case "array" -> parseArray(name, depth);+ case "map" -> parseMap(name, depth);+ case "struct" -> parseStruct(name, depth);+ case "uniontype" -> null;+ default -> parseScalar(name, word);+ };+ }- record ColumnSpec(String name, String duckType, ColumnKind kind) {+ private ValueType parseArray(String name, int depth) {+ if (!expect('<')) {+ return null;+ }+ ValueType element = parseType(name + ".element", depth + 1);+ if (element == null || !expect('>')) {+ return null;+ }+ return new ListType(element, rules);+ }- /** Primitive Hive types only; complex and binary types return null (unsupported). */- static ColumnSpec fromHiveType(String name, String hiveType) {- String type = hiveType == null ? "" : hiveType.trim().toLowerCase(Locale.ROOT);+ private ValueType parseMap(String name, int depth) {+ if (!expect('<')) {+ return null;+ }+ // A Hive map key is primitive. A complex one would otherwise be found out+ // by the record coercion or by DuckDB, reporting a schema Floci cannot+ // convert as a malformed record or an internal error instead.+ ValueType key = parseType(name + ".key", depth + 1);+ if (!(key instanceof ScalarType) || !expect(',')) {+ return null;+ }+ ValueType value = parseType(name + ".value", depth + 1);+ if (value == null || !expect('>')) {+ return null;+ }+ return new MapType(key, value, rules);+ }++ private ValueType parseStruct(String name, int depth) {+ if (!expect('<')) {+ return null;+ }+ List<StructField> fields = new ArrayList<>();+ // DuckDB rejects a STRUCT with two members of the same name, comparing them+ // case-insensitively, and the projection would keep only one of them anyway,+ // so the duplicate is reported as the schema problem it is rather than as a+ // batch-level failure later.+ Set<String> declared = new HashSet<>();+ do {+ String field = readFieldName();+ if (field.isEmpty() || !expect(':')) {+ return null;+ }+ if (!declared.add(field.toLowerCase(Locale.ROOT))) {+ return null;+ }+ ValueType fieldType = parseType(name + "." + field, depth + 1);+ if (fieldType == null) {+ return null;+ }+ fields.add(new StructField(field, fieldType));+ } while (expect(','));+ return expect('>') ? new StructType(List.copyOf(fields), rules) : null;+ }++ /** A primitive, with the {@code (n)} or {@code (p,s)} width some of them carry. */+ private ValueType parseScalar(String name, String word) {+ String text = word;+ if (peek() == '(') {+ int close = type.indexOf(')', pos);+ if (close < 0) {+ return null;+ }+ // Space is dropped around the punctuation only. Whitespace left after+ // that sits between two numbers, and removing it would read the+ // unchecked type string Glue stored as a width it never declared,+ // turning "varchar(6 5535)" into varchar(65535).+ String width = type.substring(pos, close + 1).replaceAll("\\s*([(),])\\s*", "$1");+ if (width.chars().anyMatch(Character::isWhitespace)) {+ return null;+ }+ text = text + width;+ pos = close + 1;+ } else if ("double".equalsIgnoreCase(word)) {+ int mark = pos;+ if ("precision".equalsIgnoreCase(readWord())) {+ text = "double precision";+ } else {+ pos = mark;+ }+ }+ return ScalarType.fromPrimitive(name, text.toLowerCase(Locale.ROOT), rules);+ }++ private String readWord() {+ skipSpaces();+ int start = pos;+ while (pos < type.length()+ && (Character.isLetterOrDigit(type.charAt(pos)) || type.charAt(pos) == '_')) {+ pos++;+ }+ return type.substring(start, pos);+ }++ /**+ * Hive lets a struct member be back-quoted, which is how a reserved word is+ * declared, and a doubled backtick is how one is put inside such a name.+ */+ private String readFieldName() {+ skipSpaces();+ if (peek() != '`') {+ return readWord();+ }+ StringBuilder quoted = new StringBuilder();+ int scan = pos + 1;+ while (scan < type.length()) {+ if (type.charAt(scan) != '`') {+ quoted.append(type.charAt(scan));+ scan++;+ } else if (scan + 1 < type.length() && type.charAt(scan + 1) == '`') {+ quoted.append('`');+ scan += 2;+ } else {+ pos = scan + 1;+ return quoted.toString();+ }+ }+ return "";+ }++ private char peek() {+ skipSpaces();+ return pos < type.length() ? type.charAt(pos) : '\0';+ }++ private boolean expect(char expected) {+ if (peek() != expected) {+ return false;+ }+ pos++;+ return true;+ }++ private void skipSpaces() {+ while (pos < type.length() && Character.isWhitespace(type.charAt(pos))) {+ pos++;+ }+ }+ }++ private record ScalarType(String name, String duckType, ColumnKind kind,+ SerDeRules rules) implements ValueType {++ /** Primitive Hive types only; anything else, {@code binary} included, returns null. */+ static ScalarType fromPrimitive(String name, String type, SerDeRules rules) {Matcher decimal = DECIMAL_TYPE.matcher(type);if (decimal.matches()) {- return decimalSpec(name, decimal.group(1), decimal.group(2));+ return decimalSpec(name, decimal.group(1), decimal.group(2), rules);}return switch (type) {- case "boolean" -> new ColumnSpec(name, "BOOLEAN", ColumnKind.BOOLEAN);- case "tinyint" -> new ColumnSpec(name, "TINYINT", ColumnKind.INTEGRAL);- case "smallint" -> new ColumnSpec(name, "SMALLINT", ColumnKind.INTEGRAL);- case "int", "integer" -> new ColumnSpec(name, "INTEGER", ColumnKind.INTEGRAL);- case "bigint" -> new ColumnSpec(name, "BIGINT", ColumnKind.INTEGRAL);- case "float", "real" -> new ColumnSpec(name, "FLOAT", ColumnKind.FLOATING);- case "double", "double precision" -> new ColumnSpec(name, "DOUBLE", ColumnKind.FLOATING);- case "decimal" -> new ColumnSpec(name, "DECIMAL(10,0)", ColumnKind.DECIMAL);- case "string" -> new ColumnSpec(name, "VARCHAR", ColumnKind.TEXT);- case "date" -> new ColumnSpec(name, "DATE", ColumnKind.DATE);- case "timestamp" -> new ColumnSpec(name, "TIMESTAMP", ColumnKind.TIMESTAMP);- default -> sizedTextSpec(name, type);+ case "boolean" -> new ScalarType(name, "BOOLEAN", ColumnKind.BOOLEAN, rules);+ case "tinyint" -> new ScalarType(name, "TINYINT", ColumnKind.INTEGRAL, rules);+ case "smallint" -> new ScalarType(name, "SMALLINT", ColumnKind.INTEGRAL, rules);+ case "int", "integer" -> new ScalarType(name, "INTEGER", ColumnKind.INTEGRAL, rules);+ case "bigint" -> new ScalarType(name, "BIGINT", ColumnKind.INTEGRAL, rules);+ case "float", "real" -> new ScalarType(name, "FLOAT", ColumnKind.FLOATING, rules);+ case "double", "double precision" -> new ScalarType(name, "DOUBLE", ColumnKind.FLOATING, rules);+ case "decimal" -> new ScalarType(name, "DECIMAL(10,0)", ColumnKind.DECIMAL, rules);+ case "string" -> new ScalarType(name, "VARCHAR", ColumnKind.TEXT, rules);+ case "date" -> new ScalarType(name, "DATE", ColumnKind.DATE, rules);+ case "timestamp" -> new ScalarType(name, "TIMESTAMP", ColumnKind.TIMESTAMP, rules);+ default -> sizedTextSpec(name, type, rules);};}@@ -564,7 +985,7 @@ public class FirehoseParquetConverter {* the type string, so a prefix match would let {@code varchar(foo)} or* {@code char(10)garbage} through as text instead of reporting the schema.*/- private static ColumnSpec sizedTextSpec(String name, String type) {+ private static ScalarType sizedTextSpec(String name, String type, SerDeRules rules) {Matcher sized = SIZED_TEXT_TYPE.matcher(type);if (!sized.matches() || sized.group(2).length() > 5) {return null;@@ -574,7 +995,7 @@ public class FirehoseParquetConverter {if (length < 1 || length > max) {return null;}- return new ColumnSpec(name, "VARCHAR", ColumnKind.TEXT);+ return new ScalarType(name, "VARCHAR", ColumnKind.TEXT, rules);}/**@@ -582,7 +1003,8 @@ public class FirehoseParquetConverter {* schema, not of any one record, so an out-of-range width is unsupported* rather than left for the COPY to reject as a batch-level failure.*/- private static ColumnSpec decimalSpec(String name, String precisionText, String scaleText) {+ private static ScalarType decimalSpec(String name, String precisionText, String scaleText,+ SerDeRules rules) {if (precisionText.length() > 2 || scaleText.length() > 2) {return null;}@@ -591,7 +1013,8 @@ public class FirehoseParquetConverter {if (precision < 1 || precision > MAX_DECIMAL_PRECISION || scale < 0 || scale > precision) {return null;}- return new ColumnSpec(name, "DECIMAL(" + precision + "," + scale + ")", ColumnKind.DECIMAL);+ return new ScalarType(name, "DECIMAL(" + precision + "," + scale + ")", ColumnKind.DECIMAL,+ rules);}/**@@ -601,14 +1024,18 @@ public class FirehoseParquetConverter {* match the schema. For input string: \"oops\"", the raw* NumberFormatException message.*/- Object coerce(JsonNode value) {+ @Override+ public Object coerce(JsonNode value) {try {+ if (!rules.lenient()) {+ refuseWhatHiveRefuses(value);+ }return switch (kind) {case BOOLEAN -> coerceBoolean(value);case INTEGRAL -> coerceIntegral(value);case FLOATING -> coerceFloating(value);case DECIMAL -> coerceDecimal(value);- case TEXT -> value.isValueNode() ? value.asText() : value.toString();+ case TEXT -> value.isValueNode() ? value.asText() : jsonText(value);case DATE -> coerceDate(value);case TIMESTAMP -> coerceTimestamp(value);};@@ -620,6 +1047,57 @@ public class FirehoseParquetConverter {}}+ /**+ * The JSON a non-scalar value keeps when it lands in a text column. OpenX+ * lowercases every key of every nested object before the value is read, at+ * any depth (probed), so the text AWS stores carries lowercased keys;+ * HiveJsonSerDe leaves them as written. The same rule the map keys follow.+ */+ private String jsonText(JsonNode value) {+ return rules.lowercaseKeys() ? lowercaseKeys(value).toString() : value.toString();+ }++ /**+ * The two things HiveJsonSerDe will not do that OpenX does, both documented+ * by AWS and probed: read a JSON string into a numeric column, and turn a+ * nested value into text. The numeric message is Jackson's; AWS appends a+ * source location whose object hash changes per run, which is not+ * reproduced. The text one comes from the SerDe's field validator and+ * carries no schema prefix.+ */+ private void refuseWhatHiveRefuses(JsonNode value) {+ boolean numeric = kind == ColumnKind.INTEGRAL || kind == ColumnKind.FLOATING+ || kind == ColumnKind.DECIMAL;+ if (numeric && value.isTextual()) {+ throw shapeMismatch("Current token (VALUE_STRING) not numeric,"+ + " can not use numeric value accessors");+ }+ if (kind == ColumnKind.TEXT && !value.isValueNode()) {+ throw new RecordConversionException("DataFormatConversion.MalformedData",+ "One or more fields have incorrect format. Exception when validating field (root)");+ }+ }++ private static JsonNode lowercaseKeys(JsonNode node) {+ if (node.isObject()) {+ ObjectNode lowered = JsonNodeFactory.instance.objectNode();+ Iterator<Map.Entry<String, JsonNode>> fields = node.fields();+ while (fields.hasNext()) {+ Map.Entry<String, JsonNode> field = fields.next();+ lowered.set(field.getKey().toLowerCase(Locale.ROOT), lowercaseKeys(field.getValue()));+ }+ return lowered;+ }+ if (node.isArray()) {+ ArrayNode lowered = JsonNodeFactory.instance.arrayNode(node.size());+ for (JsonNode entry : node) {+ lowered.add(lowercaseKeys(entry));+ }+ return lowered;+ }+ return node;+ }+/*** A Hive {@code float} column is 32-bit, and DuckDB refuses to cast a value* the type cannot hold, which would fail the COPY for the whole batch ratherdiff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetComplexTypesTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetComplexTypesTest.javanew file mode 100644index 0000000000..76d94a0a8d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetComplexTypesTest.java@@ -0,0 +1,580 @@+package io.github.hectorvent.floci.services.firehose;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.DataFormatConversionConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Deserializer;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.HiveJsonSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.InputFormatConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OutputFormatConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;+import io.github.hectorvent.floci.services.floci.duck.FlociDuckClient;+import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.services.glue.model.Column;+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;+import io.github.hectorvent.floci.services.glue.model.Table;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;+import org.mockito.ArgumentCaptor;++import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.util.Arrays;+import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyMap;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.ArgumentMatchers.isNull;+import static org.mockito.ArgumentMatchers.startsWith;+import static org.mockito.Mockito.atLeastOnce;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * Conversion of the complex Hive types, kept apart from+ * {@link FirehoseParquetConverterTest} because the scalar suite there is already+ * a long one.+ *+ * The coercion asserted here was probed against real AWS on 2026-09-12 with a+ * table of exactly these four columns: a value that is not an array is wrapped+ * into one, an array fills a struct by position, map keys are lowercased, an+ * undeclared struct member is dropped and a missing one is null, and a mismatch+ * at any depth fails that record alone.+ */+class FirehoseParquetComplexTypesTest {++ private static final String BUCKET = "results";+ private static final String STAGING = "floci-firehose-staging";+ private static final Instant DELIVERY_TIME = Instant.parse("2026-01-01T00:00:00Z");++ private GlueService glueService;+ private FlociDuckClient duckClient;+ private S3Service s3Service;+ private FirehoseParquetConverter converter;+ private final ObjectMapper mapper = new ObjectMapper();++ @BeforeEach+ void setUp() {+ glueService = mock(GlueService.class);+ duckClient = mock(FlociDuckClient.class);+ s3Service = mock(S3Service.class);+ converter = new FirehoseParquetConverter(glueService, duckClient, s3Service, mapper,+ new RegionResolver("us-east-1", "000000000000"), STAGING);+ when(s3Service.getObject(eq(STAGING), anyString())).thenReturn(new S3Object(STAGING,+ "staged.parquet", "PAR1staged".getBytes(StandardCharsets.UTF_8),+ "application/octet-stream"));+ when(glueService.getTable("db", "events")).thenReturn(table(+ new Column("id", "string"),+ new Column("tags", "array<string>"),+ new Column("person", "struct<name:string,age:int>"),+ new Column("items", "array<struct<sku:string,qty:int>>"),+ new Column("attrs", "map<string,array<string>>")));+ }++ private static Table table(Column... columns) {+ StorageDescriptor descriptor = new StorageDescriptor();+ descriptor.setColumns(Arrays.asList(columns));+ Table table = new Table();+ table.setStorageDescriptor(descriptor);+ return table;+ }++ private static DeliveryStreamDescription stream() {+ return stream(new OpenXJsonSerDe());+ }++ private static DeliveryStreamDescription hiveStream() {+ DeliveryStreamDescription stream = stream(null);+ stream.s3Destination().getDataFormatConversionConfiguration().getInputFormatConfiguration()+ .getDeserializer().setHiveJsonSerDe(new HiveJsonSerDe());+ return stream;+ }++ private static DeliveryStreamDescription stream(OpenXJsonSerDe openX) {+ SchemaConfiguration schema = new SchemaConfiguration();+ schema.setRoleArn("arn:aws:iam::000000000000:role/firehose-role");+ schema.setDatabaseName("db");+ schema.setTableName("events");+ Deserializer deserializer = new Deserializer();+ deserializer.setOpenXJsonSerDe(openX);+ InputFormatConfiguration input = new InputFormatConfiguration();+ input.setDeserializer(deserializer);+ Serializer serializer = new Serializer();+ serializer.setParquetSerDe(new ParquetSerDe());+ OutputFormatConfiguration output = new OutputFormatConfiguration();+ output.setSerializer(serializer);+ DataFormatConversionConfiguration conversion = new DataFormatConversionConfiguration();+ conversion.setEnabled(true);+ conversion.setSchemaConfiguration(schema);+ conversion.setInputFormatConfiguration(input);+ conversion.setOutputFormatConfiguration(output);+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::" + BUCKET);+ s3.setPrefix("data/");+ s3.setErrorOutputPrefix("errors/!{firehose:error-output-type}/");+ s3.setCompressionFormat("UNCOMPRESSED");+ s3.setDataFormatConversionConfiguration(conversion);+ return new DeliveryStreamDescription("stream", "arn:aws:firehose:::stream", s3);+ }++ private static byte[] record(String json) {+ return (json + "\n").getBytes(StandardCharsets.UTF_8);+ }++ private String stagedNdjson() {+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service).putObject(eq(STAGING), startsWith("firehose-staging/stream/"),+ body.capture(), eq("application/x-ndjson"), anyMap());+ return new String(body.getValue(), StandardCharsets.UTF_8);+ }++ private String duckColumns() {+ ArgumentCaptor<String> setupSql = ArgumentCaptor.forClass(String.class);+ verify(duckClient).execute(eq("SELECT 1 AS ok"), setupSql.capture(), isNull(), any());+ return setupSql.getValue();+ }++ private JsonNode singleErrorLine() throws Exception {+ ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service, atLeastOnce()).putObject(eq(BUCKET), key.capture(),+ body.capture(), eq("application/octet-stream"), anyMap());+ for (int i = 0; i < key.getAllValues().size(); i++) {+ if (key.getAllValues().get(i).startsWith("errors/format-conversion-failed/")) {+ String object = new String(body.getAllValues().get(i), StandardCharsets.UTF_8);+ return mapper.readTree(object.strip().split("\n")[0]);+ }+ }+ throw new AssertionError("no error object among " + key.getAllValues());+ }++ @Test+ void complexColumnsAreTypedForDuckFromTheGlueSchema() {+ converter.deliver(stream(), BUCKET, List.of(record("{\"id\": \"a\"}")), DELIVERY_TIME);++ assertTrue(duckColumns().contains("columns={'id': 'VARCHAR', 'tags': 'VARCHAR[]',"+ + " 'person': 'STRUCT(\"name\" VARCHAR, \"age\" INTEGER)',"+ + " 'items': 'STRUCT(\"sku\" VARCHAR, \"qty\" INTEGER)[]',"+ + " 'attrs': 'MAP(VARCHAR, VARCHAR[])'}"),+ "setup sql was: " + duckColumns());+ }++ @Test+ void nestedValuesAreStagedAsTheyWereGiven() {+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r1\", \"tags\": [\"a\", \"b\"], \"person\": {\"name\": \"Alice\","+ + " \"age\": 30}, \"items\": [{\"sku\": \"s1\", \"qty\": 2}],"+ + " \"attrs\": {\"k1\": [\"v1\", \"v2\"]}}")), DELIVERY_TIME);++ assertEquals(1, outcome.convertedRecords());+ assertTrue(stagedNdjson().contains("{\"id\":\"r1\",\"tags\":[\"a\",\"b\"],"+ + "\"person\":{\"name\":\"Alice\",\"age\":30},"+ + "\"items\":[{\"sku\":\"s1\",\"qty\":2}],\"attrs\":{\"k1\":[\"v1\",\"v2\"]}}"),+ "staged rows were: " + stagedNdjson());+ }++ @Test+ void aValueThatIsNotAnArrayIsWrappedIntoOne() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r6\", \"tags\": \"notanarray\","+ + " \"items\": {\"sku\": \"x\", \"qty\": 1}}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"tags\":[\"notanarray\"]"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"items\":[{\"sku\":\"x\",\"qty\":1}]"), "staged rows were: " + ndjson);+ }++ @Test+ void anArrayFillsAStructByPositionAndIgnoresTheExtras() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r7\", \"person\": [\"Zoe\", 9, \"ignored\"]}")), DELIVERY_TIME);++ assertTrue(stagedNdjson().contains("\"person\":{\"name\":\"Zoe\",\"age\":9}"),+ "staged rows were: " + stagedNdjson());+ }++ @Test+ void aShorterArrayLeavesTheRemainingMembersNull() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r7\", \"person\": [\"Zoe\"]}")), DELIVERY_TIME);++ assertTrue(stagedNdjson().contains("\"person\":{\"name\":\"Zoe\",\"age\":null}"),+ "staged rows were: " + stagedNdjson());+ }++ @Test+ void structMembersMatchCaseInsensitivelyAtEveryDepth() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r2\", \"PERSON\": {\"NAME\": \"Bob\", \"AGE\": 41},"+ + " \"ITEMS\": [{\"SKU\": \"s2\", \"QTY\": \"3\"}]}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"person\":{\"name\":\"Bob\",\"age\":41}"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"items\":[{\"sku\":\"s2\",\"qty\":3}]"), "staged rows were: " + ndjson);+ }++ @Test+ void mapKeysAreLowercased() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"s1\", \"attrs\": {\"MixedKey\": [\"v\"]}}")), DELIVERY_TIME);++ assertTrue(stagedNdjson().contains("\"attrs\":{\"mixedkey\":[\"v\"]}"),+ "staged rows were: " + stagedNdjson());+ }++ @Test+ void anUndeclaredStructMemberIsDroppedAndAMissingOneIsNull() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r3\", \"person\": {\"name\": \"Carol\", \"extra\": \"dropped\"}}")),+ DELIVERY_TIME);++ assertTrue(stagedNdjson().contains("\"person\":{\"name\":\"Carol\",\"age\":null}"),+ "staged rows were: " + stagedNdjson());+ }++ @Test+ void anEmptyObjectIsAStructOfNullMembersRatherThanANullStruct() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r12\", \"person\": {}, \"tags\": [], \"attrs\": {}}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"person\":{\"name\":null,\"age\":null}"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"tags\":[]"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"attrs\":{}"), "staged rows were: " + ndjson);+ }++ @Test+ void nullsInsideContainersAreKept() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r8\", \"tags\": [\"a\", null, \"c\"], \"items\": [null],"+ + " \"attrs\": {\"k\": null}}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"tags\":[\"a\",null,\"c\"]"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"items\":[null]"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"attrs\":{\"k\":null}"), "staged rows were: " + ndjson);+ }++ @Test+ void scalarsAreStillCoercedInsideContainers() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r10\", \"tags\": [1, 2, 3]}")), DELIVERY_TIME);++ assertTrue(stagedNdjson().contains("\"tags\":[\"1\",\"2\",\"3\"]"),+ "staged rows were: " + stagedNdjson());+ }++ @Test+ void aNonScalarInATextColumnKeepsItsCompactJson() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": {\"nested\": 1}, \"tags\": [[\"a\", \"b\"]]}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"id\":\"{\\\"nested\\\":1}\""), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"tags\":[\"[\\\"a\\\",\\\"b\\\"]\"]"), "staged rows were: " + ndjson);+ }++ /**+ * Probed 2026-09-12: OpenX lowercases the keys of every nested object before the+ * value is read, at any depth, so the JSON text a stringified object leaves in a+ * text column carries lowercased keys.+ */+ @Test+ void openXLowercasesTheKeysOfAStringifiedObjectAtEveryDepth() {+ converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": {\"Outer\": {\"Inner\": 1}}, \"tags\": [{\"MixedKey\": 1}],"+ + " \"attrs\": {\"K\": [\"v\"]}}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"id\":\"{\\\"outer\\\":{\\\"inner\\\":1}}\""),+ "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"tags\":[\"{\\\"mixedkey\\\":1}\"]"),+ "staged rows were: " + ndjson);+ }++ @Test+ void aScalarForAStructFailsTheRecord() throws Exception {+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET,+ List.of(record("{\"id\": \"r5\", \"person\": 42}")), DELIVERY_TIME);++ assertEquals(0, outcome.convertedRecords());+ assertEquals(1, outcome.failedRecords());+ JsonNode line = singleErrorLine();+ assertEquals("DataFormatConversion.MalformedData", line.get("lastErrorCode").asText());+ assertEquals("Data does not match the schema. Data is not JSONObject but"+ + " java.lang.Long with value 42", line.get("lastErrorMessage").asText());+ }++ @Test+ void anArrayForAMapFailsTheRecord() throws Exception {+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET,+ List.of(record("{\"id\": \"r9\", \"attrs\": [[\"k\", [\"v\"]]]}")), DELIVERY_TIME);++ assertEquals(1, outcome.failedRecords());+ JsonNode line = singleErrorLine();+ assertEquals("DataFormatConversion.MalformedData", line.get("lastErrorCode").asText());+ assertEquals("Data does not match the schema. A map column cannot hold a JSON array.",+ line.get("lastErrorMessage").asText());+ }++ @Test+ void aMismatchInsideAnArrayFailsTheWholeRecordWithTheProbedMessage() throws Exception {+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"r5\", \"items\": [{\"sku\": \"s5\", \"qty\": \"oops\"}]}")),+ DELIVERY_TIME);++ assertEquals(0, outcome.convertedRecords());+ JsonNode line = singleErrorLine();+ assertEquals("DataFormatConversion.MalformedData", line.get("lastErrorCode").asText());+ assertEquals("Data does not match the schema. For input string: \"oops\"",+ line.get("lastErrorMessage").asText());+ }++ @Test+ void oneFailingNestedRecordLeavesTheRestOfTheBatchConverting() {+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET, List.of(+ record("{\"id\": \"good\", \"person\": {\"name\": \"Alice\", \"age\": 1}}"),+ record("{\"id\": \"bad\", \"person\": 42}")), DELIVERY_TIME);++ assertEquals(1, outcome.convertedRecords());+ assertEquals(1, outcome.failedRecords());+ }++ /**+ * Probed 2026-09-12 on a separate HiveJsonSerDe rig: that SerDe resolves column+ * and struct member names case-insensitively just as OpenX does, but it leaves+ * a map entry key exactly as the record wrote it, where OpenX lowercases every+ * JSON key before deserializing.+ */+ @Test+ void hiveJsonSerDeKeepsMapKeysWhileStillMatchingMembersCaseInsensitively() {+ converter.deliver(hiveStream(), BUCKET, List.of(+ record("{\"id\": \"h1\", \"ATTRS\": {\"MixedKey\": [\"v\"]},"+ + " \"PERSON\": {\"NAME\": \"Bob\", \"AGE\": 2}}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"attrs\":{\"MixedKey\":[\"v\"]}"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"person\":{\"name\":\"Bob\",\"age\":2}"), "staged rows were: " + ndjson);+ }++ /**+ * The HiveJsonSerDe coercion matrix, probed 2026-09-13 on a rig with exactly+ * these columns: what it still coerces, and the four things it refuses that OpenX+ * accepts, each with the message AWS wrote to the error output.+ */+ @Test+ void hiveJsonSerDeStillCoercesWhatAwsDocumentsAsAllowed() {+ converter.deliver(hiveStream(), BUCKET, List.of(+ record("{\"id\": \"h\", \"tags\": [1, 2, 3], \"person\": {\"name\": \"Dan\", \"extra\": 1},"+ + " \"items\": [{\"sku\": \"s\", \"qty\": 2, \"extra\": 1}]}"),+ record("{\"id\": \"empty\", \"person\": {}, \"tags\": [\"a\", null]}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"tags\":[\"1\",\"2\",\"3\"]"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"person\":{\"name\":\"Dan\",\"age\":null}"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"items\":[{\"sku\":\"s\",\"qty\":2}]"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"person\":{\"name\":null,\"age\":null}"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"tags\":[\"a\",null]"), "staged rows were: " + ndjson);+ }++ @ParameterizedTest+ @CsvSource(delimiter = '|', value = {+ "{\"id\": \"a\", \"tags\": \"solo\"}"+ + "|Data does not match the schema. java.io.IOException: Start of Array expected",+ "{\"id\": \"b\", \"items\": {\"sku\": \"x\", \"qty\": 1}}"+ + "|Data does not match the schema. java.io.IOException: Start of Array expected",+ "{\"id\": \"c\", \"attrs\": {\"k\": \"notanarray\"}}"+ + "|Data does not match the schema. java.io.IOException: Start of Array expected",+ "{\"id\": \"d\", \"person\": [\"Bob\", 41]}"+ + "|Data does not match the schema. java.io.IOException: Start of Object expected",+ "{\"id\": \"e\", \"person\": 42}"+ + "|Data does not match the schema. java.io.IOException: Start of Object expected",+ "{\"id\": \"f\", \"person\": {\"name\": \"Eve\", \"age\": \"7\"}}"+ + "|Data does not match the schema. Current token (VALUE_STRING) not numeric,"+ + " can not use numeric value accessors",+ "{\"id\": {\"nested\": 1}}"+ + "|One or more fields have incorrect format. Exception when validating field (root)",+ "{\"id\": \"g\", \"tags\": [{\"MixedKey\": 1}]}"+ + "|One or more fields have incorrect format. Exception when validating field (root)",+ })+ void hiveJsonSerDeFailsTheRecordWhereOpenXWouldCoerce(String json, String message) throws Exception {+ FirehoseParquetConverter.Outcome outcome = converter.deliver(hiveStream(), BUCKET,+ List.of(record(json)), DELIVERY_TIME);++ assertEquals(0, outcome.convertedRecords());+ assertEquals(1, outcome.failedRecords());+ JsonNode line = singleErrorLine();+ assertEquals("DataFormatConversion.MalformedData", line.get("lastErrorCode").asText());+ assertEquals(message, line.get("lastErrorMessage").asText());+ }++ /** The same rule reaches a top-level scalar column, where Floci used to coerce under Hive too. */+ @Test+ void hiveJsonSerDeRefusesAStringInATopLevelNumericColumn() throws Exception {+ when(glueService.getTable("db", "events")).thenReturn(table(+ new Column("ticker", "string"), new Column("qty", "int")));++ FirehoseParquetConverter.Outcome outcome = converter.deliver(hiveStream(), BUCKET, List.of(+ record("{\"ticker\": \"AAA\", \"qty\": \"10\"}"),+ record("{\"ticker\": \"BBB\", \"qty\": 10}")), DELIVERY_TIME);++ assertEquals(1, outcome.convertedRecords());+ assertEquals(1, outcome.failedRecords());+ assertTrue(stagedNdjson().contains("{\"ticker\":\"BBB\",\"qty\":10}"),+ "staged rows were: " + stagedNdjson());+ }++ /**+ * Probed 2026-09-12: with the option off, a struct member under a column that+ * did match is left null unless the record spells it exactly, and map keys keep+ * the case they were written in.+ */+ @Test+ void caseSensitiveOpenXMatchesNestedMembersExactlyAndKeepsMapKeys() {+ OpenXJsonSerDe openX = new OpenXJsonSerDe();+ openX.setCaseInsensitive(false);++ converter.deliver(stream(openX), BUCKET, List.of(+ record("{\"id\": \"s\", \"person\": {\"NAME\": \"Bob\", \"age\": 2},"+ + " \"attrs\": {\"MixedKey\": [\"v\"]}}")), DELIVERY_TIME);++ String ndjson = stagedNdjson();+ assertTrue(ndjson.contains("\"person\":{\"name\":null,\"age\":2}"), "staged rows were: " + ndjson);+ assertTrue(ndjson.contains("\"attrs\":{\"MixedKey\":[\"v\"]}"), "staged rows were: " + ndjson);+ }++ @Test+ void nestedContainersAreTypedAllTheWayDown() {+ when(glueService.getTable("db", "events")).thenReturn(table(new Column("deep",+ "array<map<string,struct<a:int,b:array<decimal(10,2)>>>>")));++ converter.deliver(stream(), BUCKET, List.of(record("{\"deep\": []}")), DELIVERY_TIME);++ assertTrue(duckColumns().contains("columns={'deep': 'MAP(VARCHAR,"+ + " STRUCT(\"a\" INTEGER, \"b\" DECIMAL(10,2)[]))[]'}"),+ "setup sql was: " + duckColumns());+ }++ /**+ * The SES event-publishing schema, the shape that motivated this: deeply+ * nested, with members named after SQL keywords, which is why every struct+ * member is quoted in the DuckDB type.+ */+ @Test+ void aRealSesEventSchemaIsTypedAllTheWayDown() {+ when(glueService.getTable("db", "events")).thenReturn(table(new Column("mail",+ "struct<timestamp:string,source:string,destination:array<string>,"+ + "headers:array<struct<name:string,value:string>>,"+ + "commonheaders:struct<from:array<string>,to:array<string>,subject:string>,"+ + "tags:map<string,array<string>>>")));++ converter.deliver(stream(), BUCKET, List.of(record("{\"mail\": {\"subject\": \"hi\"}}")),+ DELIVERY_TIME);++ assertTrue(duckColumns().contains("columns={'mail': 'STRUCT(\"timestamp\" VARCHAR,"+ + " \"source\" VARCHAR, \"destination\" VARCHAR[],"+ + " \"headers\" STRUCT(\"name\" VARCHAR, \"value\" VARCHAR)[],"+ + " \"commonheaders\" STRUCT(\"from\" VARCHAR[], \"to\" VARCHAR[],"+ + " \"subject\" VARCHAR), \"tags\" MAP(VARCHAR, VARCHAR[]))'}"),+ "setup sql was: " + duckColumns());+ }++ /** Hive doubles a backtick to put one inside a quoted member name. */+ @Test+ void aDoubledBackQuoteIsOneBackQuoteInTheMemberName() {+ when(glueService.getTable("db", "events")).thenReturn(table(+ new Column("row", "struct<`a``b`:string>")));++ converter.deliver(stream(), BUCKET, List.of(record("{\"row\": {\"a`b\": \"x\"}}")),+ DELIVERY_TIME);++ assertTrue(duckColumns().contains("columns={'row': 'STRUCT(\"a`b\" VARCHAR)'}"),+ "setup sql was: " + duckColumns());+ assertTrue(stagedNdjson().contains("\"row\":{\"a`b\":\"x\"}"),+ "staged rows were: " + stagedNdjson());+ }++ /** Space around the punctuation is fine; only space joining two numbers is not. */+ @Test+ void aWidthSpacedAroundItsPunctuationStillParses() {+ when(glueService.getTable("db", "events")).thenReturn(table(+ new Column("amount", "decimal(10, 2)"), new Column("note", "varchar( 255 )")));++ converter.deliver(stream(), BUCKET, List.of(record("{\"amount\": 1.5}")), DELIVERY_TIME);++ assertTrue(duckColumns().contains("columns={'amount': 'DECIMAL(10,2)', 'note': 'VARCHAR'}"),+ "setup sql was: " + duckColumns());+ }++ @Test+ void aBackQuotedStructMemberKeepsItsName() {+ when(glueService.getTable("db", "events")).thenReturn(table(+ new Column("row", "struct<`order`:string>")));++ converter.deliver(stream(), BUCKET, List.of(record("{\"row\": {\"order\": \"x\"}}")),+ DELIVERY_TIME);++ assertTrue(duckColumns().contains("columns={'row': 'STRUCT(\"order\" VARCHAR)'}"),+ "setup sql was: " + duckColumns());+ assertTrue(stagedNdjson().contains("\"row\":{\"order\":\"x\"}"),+ "staged rows were: " + stagedNdjson());+ }++ @ParameterizedTest+ @ValueSource(strings = {+ "binary",+ "array<binary>",+ "uniontype<int,string>",+ "struct<a:uniontype<int,string>>",+ "array<string",+ "struct<a:>",+ "varchar(6 5535)",+ "decimal(1 0,2)",+ "struct<a:int,a:string>",+ "struct<a:int,A:string>",+ "map<struct<a:string>,int>",+ "map<array<string>,int>",+ "struct<:string>",+ "map<string>",+ "array<string>garbage",+ "struct<a:string>,"+ })+ void aTypeFlociCannotRepresentFailsTheWholeBatch(String hiveType) throws Exception {+ when(glueService.getTable("db", "events")).thenReturn(table(new Column("col", hiveType)));++ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET,+ List.of(record("{\"col\": null}")), DELIVERY_TIME);++ assertEquals(0, outcome.convertedRecords());+ assertEquals("DataFormatConversion.UnsupportedSchema",+ singleErrorLine().get("lastErrorCode").asText());+ }++ /** A hostile schema must report an unsupported type rather than overflow the parse. */+ @Test+ void nestingDeeperThanTheParserAcceptsIsUnsupported() throws Exception {+ String hiveType = "array<".repeat(40) + "string" + ">".repeat(40);+ when(glueService.getTable("db", "events")).thenReturn(table(new Column("col", hiveType)));++ converter.deliver(stream(), BUCKET, List.of(record("{\"col\": null}")), DELIVERY_TIME);++ assertEquals("DataFormatConversion.UnsupportedSchema",+ singleErrorLine().get("lastErrorCode").asText());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverterTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverterTest.javaindex f98f96ea89..07a2029cea 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverterTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverterTest.java@@ -613,7 +613,7 @@ class FirehoseParquetConverterTest {@Testvoid unsupportedColumnTypeFailsTheWholeBatch() throws Exception {when(glueService.getTable("db", "events")).thenReturn(table(- new Column("ticker", "string"), new Column("tags", "array<string>")));+ new Column("ticker", "string"), new Column("payload", "binary")));FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET,List.of(record("{\"ticker\": \"AAA\"}")), DELIVERY_TIME);@@ -621,7 +621,7 @@ class FirehoseParquetConverterTest {assertEquals(0, outcome.convertedRecords());JsonNode line = errorLines().get(0);assertEquals("DataFormatConversion.UnsupportedSchema", line.get("lastErrorCode").asText());- assertTrue(line.get("lastErrorMessage").asText().contains("array<string>"));+ assertTrue(line.get("lastErrorMessage").asText().contains("binary"));}@ParameterizedTest