Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): convert array, struct and map columns to Parquet (#3521)

Kenshin Okinaka6790024abc5e
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseParquetDeliveryTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseParquetDeliveryTest.java
index f879efe406..5452e2b193 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseParquetDeliveryTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseParquetDeliveryTest.java
@@ -66,6 +66,13 @@ import static org.assertj.core.api.Assertions.assertThat;
* convert land under the evaluated ErrorOutputPrefix as AWS-shaped NDJSON
* error lines. Against Floci this exercises the floci-duck sidecar for real,
* which the emulator's unit tests mock.
+ *
+ * The table carries the complex Hive types beside the primitive ones, the shape
+ * a real Firehose-to-Athena pipeline uses, so one stream covers both. The
+ * coercions the nested records rely on were probed against real AWS: a value
+ * that is not an array is wrapped into one, an array fills a struct by
+ * position, member names match case-insensitively at every depth, and a
+ * mismatch inside a nested value fails that record alone.
*/
@DisplayName("Firehose data format conversion delivery (JSON to Parquet)")
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@@ -80,6 +87,13 @@ class FirehoseParquetDeliveryTest {
private static final String VALID_RECORD_2 = "{\"ticker\": \"MISSING_FIELDS\"}\n";
private static final String MISMATCH_RECORD = "{\"ticker\": \"BAD\", \"price\": \"oops\"}\n";
private static final String UNPARSEABLE_RECORD = "this is not json at all\n";
+ private static final String NESTED_RECORD =
+ "{\"ticker\": \"NESTED\", \"tags\": [\"a\", \"b\"], \"person\": {\"name\": \"Alice\", \"age\": 30},"
+ + " \"items\": [{\"sku\": \"s1\", \"qty\": 2}], \"attrs\": {\"k1\": [\"v1\", \"v2\"]}}\n";
+ private static final String COERCED_NESTED_RECORD =
+ "{\"ticker\": \"COERCED\", \"TAGS\": \"solo\", \"PERSON\": [\"Bob\", 41]}\n";
+ private static final String NESTED_MISMATCH_RECORD =
+ "{\"ticker\": \"BAD_NESTED\", \"items\": [{\"sku\": \"s5\", \"qty\": \"oops\"}]}\n";
private static FirehoseClient firehose;
private static GlueClient glue;
@@ -119,7 +133,14 @@ class FirehoseParquetDeliveryTest {
Column.builder().name("price").type("double").build(),
Column.builder().name("qty").type("int").build(),
Column.builder().name("active").type("boolean").build(),
- Column.builder().name("ts").type("timestamp").build())
+ Column.builder().name("ts").type("timestamp").build(),
+ Column.builder().name("tags").type("array<string>").build(),
+ Column.builder().name("person")
+ .type("struct<name:string,age:int>").build(),
+ Column.builder().name("items")
+ .type("array<struct<sku:string,qty:int>>").build(),
+ Column.builder().name("attrs")
+ .type("map<string,array<string>>").build())
// The destination prefix with a Parquet SerDe, as a real
// Firehose-to-Parquet setup defines it, so the delivered
// objects can be read back through Athena.
@@ -176,7 +197,10 @@ class FirehoseParquetDeliveryTest {
record(VALID_RECORD_1),
record(VALID_RECORD_2),
record(MISMATCH_RECORD),
- record(UNPARSEABLE_RECORD))
+ record(UNPARSEABLE_RECORD),
+ record(NESTED_RECORD),
+ record(COERCED_NESTED_RECORD),
+ record(NESTED_MISMATCH_RECORD))
.build()).failedPutCount()).isZero();
}
@@ -271,9 +295,12 @@ class FirehoseParquetDeliveryTest {
byte[] body = s3.getObject(GetObjectRequest.builder().bucket(bucket).key(key).build())
.readAllBytes();
List<String> lines = new String(body, StandardCharsets.UTF_8).strip().lines().toList();
- assertThat(lines).hasSize(2);
+ assertThat(lines).hasSize(3);
- String mismatch = lines.stream().filter(l -> l.contains("MalformedData")).findFirst().orElseThrow();
+ String mismatch = lines.stream()
+ .filter(l -> l.contains(Base64.getEncoder()
+ .encodeToString(MISMATCH_RECORD.getBytes(StandardCharsets.UTF_8))))
+ .findFirst().orElseThrow();
assertThat(mismatch).contains("\"lastErrorCode\":\"DataFormatConversion.MalformedData\"");
assertThat(mismatch).contains(
Base64.getEncoder().encodeToString(MISMATCH_RECORD.getBytes(StandardCharsets.UTF_8)));
@@ -283,14 +310,21 @@ class FirehoseParquetDeliveryTest {
assertThat(unparseable).contains("\"lastErrorCode\":\"DataFormatConversion.ParseError\"");
assertThat(unparseable).contains(
Base64.getEncoder().encodeToString(UNPARSEABLE_RECORD.getBytes(StandardCharsets.UTF_8)));
+
+ // A mismatch inside an array of structs is per-record too, not a batch failure.
+ String nested = lines.stream()
+ .filter(l -> l.contains(Base64.getEncoder()
+ .encodeToString(NESTED_MISMATCH_RECORD.getBytes(StandardCharsets.UTF_8))))
+ .findFirst().orElseThrow();
+ assertThat(nested).contains("\"lastErrorCode\":\"DataFormatConversion.MalformedData\"");
}
/**
* The PAR1 framing only proves the object is a Parquet file. Reading it back
* through Athena, which resolves the Glue table to the delivery prefix, is what
- * shows the projection itself is right: both valid records are present, the
- * absent members became null rather than being dropped, and the values kept
- * their column types instead of arriving as strings.
+ * shows the projection itself is right: every convertible record is present,
+ * the absent members became null rather than being dropped, and the values
+ * kept their column types instead of arriving as strings.
*/
@Test
@Order(3)
@@ -322,7 +356,7 @@ class FirehoseParquetDeliveryTest {
.filter(row -> !"ticker".equals(row.get(0)))
.toList();
- assertThat(rows).as("both valid records must be in the Parquet object").hasSize(2);
+ assertThat(rows).as("every convertible record must be in the Parquet object").hasSize(4);
List<String> typed = rows.stream().filter(row -> "AAA".equals(row.get(0))).findFirst()
.orElseThrow();
@@ -342,6 +376,92 @@ class FirehoseParquetDeliveryTest {
}
}
+ /**
+ * The nested columns read back the same way, through a list element, a struct
+ * member, a struct member inside a list, and a map entry. The two records are
+ * selected by name because a subscript of a null container is the one thing
+ * this query cannot ask of the primitive-only records.
+ */
+ @Test
+ @Order(4)
+ @DisplayName("Nested array, struct and map values read back through Athena")
+ void nestedValuesReadBackThroughAthena() throws Exception {
+ waitForDeliveredKey("data/");
+
+ try (AthenaClient athena = TestFixtures.athenaClient()) {
+ StartQueryExecutionResponse started = athena.startQueryExecution(
+ StartQueryExecutionRequest.builder()
+ .queryString("SELECT ticker, tags[1], person.name, person.age,"
+ + " items[1].sku, attrs['k1'][2] FROM \"" + database + "\".\""
+ + TABLE + "\" WHERE ticker = 'NESTED' ORDER BY ticker")
+ .workGroup("primary")
+ .resultConfiguration(ResultConfiguration.builder()
+ .outputLocation("s3://" + bucket + "/athena-results/")
+ .build())
+ .build());
+
+ QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(
+ athena, started.queryExecutionId(), Duration.ofSeconds(120));
+ assertThat(status.state())
+ .as("Athena read-back did not succeed: %s", status.stateChangeReason())
+ .isEqualTo(QueryExecutionState.SUCCEEDED);
+
+ List<String> nested = athena.getQueryResults(r -> r
+ .queryExecutionId(started.queryExecutionId()))
+ .resultSet().rows().stream()
+ .map(row -> row.data().stream().map(Datum::varCharValue).toList())
+ .filter(row -> "NESTED".equals(row.get(0)))
+ .findFirst().orElseThrow();
+
+ assertThat(nested.get(1)).isEqualTo("a");
+ assertThat(nested.get(2)).isEqualTo("Alice");
+ assertThat(nested.get(3)).isEqualTo("30");
+ assertThat(nested.get(4)).isEqualTo("s1");
+ assertThat(nested.get(5)).isEqualTo("v2");
+ }
+ }
+
+ /**
+ * The coerced record proves the probed leniency end to end: a scalar wrapped
+ * into a one-element array, a struct filled from an array by position, and
+ * both found under uppercase keys.
+ */
+ @Test
+ @Order(5)
+ @DisplayName("A scalar for an array and an array for a struct convert as AWS coerces them")
+ void coercedNestedValuesReadBackThroughAthena() throws Exception {
+ waitForDeliveredKey("data/");
+
+ try (AthenaClient athena = TestFixtures.athenaClient()) {
+ StartQueryExecutionResponse started = athena.startQueryExecution(
+ StartQueryExecutionRequest.builder()
+ .queryString("SELECT ticker, tags[1], person.name, person.age FROM \""
+ + database + "\".\"" + TABLE + "\" WHERE ticker = 'COERCED'")
+ .workGroup("primary")
+ .resultConfiguration(ResultConfiguration.builder()
+ .outputLocation("s3://" + bucket + "/athena-results/")
+ .build())
+ .build());
+
+ QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(
+ athena, started.queryExecutionId(), Duration.ofSeconds(120));
+ assertThat(status.state())
+ .as("Athena read-back did not succeed: %s", status.stateChangeReason())
+ .isEqualTo(QueryExecutionState.SUCCEEDED);
+
+ List<String> coerced = athena.getQueryResults(r -> r
+ .queryExecutionId(started.queryExecutionId()))
+ .resultSet().rows().stream()
+ .map(row -> row.data().stream().map(Datum::varCharValue).toList())
+ .filter(row -> "COERCED".equals(row.get(0)))
+ .findFirst().orElseThrow();
+
+ assertThat(coerced.get(1)).isEqualTo("solo");
+ assertThat(coerced.get(2)).isEqualTo("Bob");
+ assertThat(coerced.get(3)).isEqualTo("41");
+ }
+ }
+
private String waitForDeliveredKey(String prefix) throws InterruptedException {
long deadline = System.currentTimeMillis() + POLL_TIMEOUT_MILLIS;
while (System.currentTimeMillis() < deadline) {
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index 77e7d2874a..ab664586c5 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -63,6 +63,15 @@ Any other value, including differently-cased ones such as `SNAPPY` instead of `S
Each buffered record is parsed and type-checked against the Glue table's columns. The valid rows are staged as NDJSON in `floci.services.firehose.staging-bucket`, then written to the destination as one Parquet object keyed like any other delivery but ending in `.parquet`, unless `FileExtension` replaces it. `ParquetSerDe.Compression` picks the codec, SNAPPY by default.
+`array`, `struct` and `map` columns convert too, nested up to 32 levels deep. The coercion is as forgiving as AWS's, all of it probed:
+
+- Under `OpenXJsonSerDe`, a value that is not an array becomes a **one-element array**, an object bound for `array<struct<..>>` included.
+- A `struct` matches members by name, dropping undeclared ones and leaving missing ones null, so `{}` is a struct of nulls rather than a null struct. Under `OpenXJsonSerDe` **an array fills the members by position**, extras ignored, and only a scalar fails the record.
+- **`HiveJsonSerDe` refuses what AWS documents it refusing, and fails the record** (all probed): a non-array in an `array<T>` column ("Start of Array expected"), a non-object in a `struct` column ("Start of Object expected"), a JSON string in a numeric column ("Current token (VALUE_STRING) not numeric"), and a non-scalar in a text column ("One or more fields have incorrect format. Exception when validating field (root)"). It still coerces a number into a text column, keeps nulls inside containers, and treats missing and undeclared members as OpenX does. The same numeric rule applies to top-level columns.
+- A `map` takes only an object. Its **keys are lowercased under `OpenXJsonSerDe`**, which lowercases every JSON key at any depth before the value is read, and kept as written under `HiveJsonSerDe` or `CaseInsensitive: false`. The same lowercasing shows in the JSON text a non-scalar leaves in a text column.
+- Names match case-insensitively at every depth under both deserializers, unless `CaseInsensitive` is `false`, which makes columns and members alike match exactly. The primitive coercions apply inside containers as at the top level, so `[1, 2, 3]` fills an `array<string>` and `"7"` an `int` member.
+- A value that cannot be coerced at any depth fails **its whole record**.
+
Records that cannot be converted fail **individually**, as on AWS: the rest of the batch still converts. They are delivered instead to the evaluated `ErrorOutputPrefix`, where `!{firehose:error-output-type}` resolves to `format-conversion-failed`, as one extensionless object holding an NDJSON line per record: `attemptsMade`, `arrivalTimestamp`, `lastErrorCode`, `lastErrorMessage`, `attemptEndingTimestamp`, base64 `rawData`, and a `dataCatalogTable` block. The code is `DataFormatConversion.ParseError` for a record that is not valid JSON, `DataFormatConversion.MalformedData` for one whose values do not fit the column types.
A failure of the batch as a whole is routed rather than discarded: a missing Glue table, a schema Floci cannot convert, or a sidecar failure sends every buffered record to the error output, under `DataFormatConversion.EntityNotFoundException`, `.UnsupportedSchema` and `.InternalError` respectively; those three codes are Floci's own. That routing is only as good as the write it depends on. A flush whose S3 write fails keeps its batch buffered, with or without conversion, and retries it on the next flush ahead of anything buffered since. The two destination writes of a mixed batch are not atomic either: the error output is written first, so its failure delivers nothing at all, but the reverse remains possible, where the error object lands and the Parquet write then fails; the retry then delivers the converted records and reports the failed ones a second time.
@@ -79,7 +88,10 @@ An omitted `Enabled` counts as enabled; `Enabled: false` stores and merges the c
- A malformed but non-empty `SchemaConfiguration.RoleARN` is accepted. AWS answers `InvalidParameterValueException` ("Invalid role ARN.") before its modeled rules run; Floci validates no ARNs, so it applies only the length and pattern rules, which an empty value trips exactly as on AWS.
- **Several stored members do not reach the conversion**: `HiveJsonSerDe.TimestampFormats`, `OpenXJsonSerDe.ConvertDotsInJsonKeysToUnderscores`, every `ParquetSerDe` member except `Compression`, and `SchemaConfiguration.Region`, `CatalogId` and `VersionId`, which are not resolved against the Glue store (Floci's catalog is neither region- nor version-partitioned) and only flow into the error-output metadata.
- An epoch timestamp outside the range AWS's Parquet writer can hold wraps there into an unrelated instant (probed: `999999999999` seconds arrives as the year 2092, not 33658). Floci keeps such a value as long as it is representable as a `TIMESTAMP`, and fails only a value beyond that, as its own record's `DataFormatConversion.MalformedData` rather than as a silently corrupted row.
-- Complex and binary Hive types (`array`, `map`, `struct`, `binary`) are not convertible; a schema containing them routes the batch to the error output as `DataFormatConversion.UnsupportedSchema`.
+- `binary` and `uniontype` columns are not convertible, nor is a type string Floci cannot parse or one nested past 32 levels; each routes the batch as `DataFormatConversion.UnsupportedSchema`.
+- A map column given an array or a scalar fails its record with Floci's own wording; real AWS fails it with a cast error naming OpenX classes Floci does not use.
+- The `HiveJsonSerDe` refusals reproduce AWS's messages up to the Jackson source location AWS appends to the numeric one (`at [Source: java.io.ByteArrayInputStream@...; line: 1, column: N]`), whose object hash changes per run. Two cells were not probed and follow the nearest measured one: a map column itself given a non-object fails like a struct ("Start of Object expected"), and a `boolean` column given a string is still coerced.
+- The Parquet list element is named `element`, where AWS's writer names it `array_element`; both are shapes the LIST annotation allows.
- Error-output records report the delivery time as both `arrivalTimestamp` and `attemptEndingTimestamp` (Floci does not track per-record arrival), and omit AWS's `sequenceNumber`/`subSequenceNumber` members. An absent `ErrorOutputPrefix` writes them at the bucket root with no default time prefix, which matches AWS's object-name documentation but was not probed.
- The Parquet object carries no `Content-Encoding`, where AWS labels it `hadoop-snappy` even though the body is a plain Parquet file (probed), and its row order may differ from put order, as it does on AWS (probed).
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java
index e476813a76..f6ef8ebfc1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java
@@ -4,6 +4,10 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
@@ -38,10 +42,13 @@ import java.time.format.DateTimeFormatterBuilder;
import java.time.format.SignStyle;
import java.util.ArrayList;
import java.util.Base64;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -145,9 +152,9 @@ public class FirehoseParquetConverter {
DataFormatConversionConfiguration conversion = s3.getDataFormatConversionConfiguration();
SchemaConfiguration schemaConfig = conversion.getSchemaConfiguration();
- List<ColumnSpec> schema;
+ List<SchemaColumn> schema;
try {
- schema = resolveSchema(schemaConfig);
+ schema = resolveSchema(schemaConfig, SerDeRules.of(conversion));
} catch (AwsException e) {
return failWholeBatch(stream, s3, bucket, records, deliveryTime, schemaConfig,
"DataFormatConversion." + e.getErrorCode(), e.getMessage());
@@ -233,7 +240,7 @@ public class FirehoseParquetConverter {
* SchemaConfiguration.Region and CatalogId only flow into the error-output
* metadata; VersionId is not resolved (the live table is always used).
*/
- private List<ColumnSpec> resolveSchema(SchemaConfiguration schemaConfig) {
+ private List<SchemaColumn> resolveSchema(SchemaConfiguration schemaConfig, SerDeRules rules) {
Table table = glueService.getTable(schemaConfig.getDatabaseName(), schemaConfig.getTableName());
List<Column> columns = table.getStorageDescriptor() == null
? null : table.getStorageDescriptor().getColumns();
@@ -241,7 +248,7 @@ public class FirehoseParquetConverter {
throw new UnsupportedSchemaException("Glue table " + schemaConfig.getDatabaseName() + "."
+ schemaConfig.getTableName() + " has no columns.");
}
- List<ColumnSpec> schema = new ArrayList<>(columns.size());
+ List<SchemaColumn> schema = new ArrayList<>(columns.size());
for (Column column : columns) {
if (column == null) {
throw new UnsupportedSchemaException("Table " + schemaConfig.getDatabaseName() + "."
@@ -252,19 +259,19 @@ public class FirehoseParquetConverter {
throw new UnsupportedSchemaException("Column name " + name
+ " is not supported for Parquet conversion.");
}
- ColumnSpec spec = ColumnSpec.fromHiveType(name, column.getType());
- if (spec == null) {
+ ValueType type = HiveTypeParser.parse(name, column.getType(), rules);
+ if (type == null) {
throw new UnsupportedSchemaException("Column " + name + " has Hive type " + column.getType()
+ ", which Floci cannot convert to Parquet.");
}
- schema.add(spec);
+ schema.add(new SchemaColumn(name, type));
}
return schema;
}
// ── per-record projection ────────────────────────────────────────────────
- private Map<String, Object> project(byte[] record, List<ColumnSpec> schema, KeyResolver keyResolver) {
+ private Map<String, Object> project(byte[] record, List<SchemaColumn> schema, KeyResolver keyResolver) {
JsonNode node;
try {
// readValue rather than readTree: an empty or blank record makes the tree
@@ -281,13 +288,50 @@ public class FirehoseParquetConverter {
}
Map<String, JsonNode> index = keyResolver.index(node);
Map<String, Object> row = new LinkedHashMap<>();
- for (ColumnSpec column : schema) {
+ for (SchemaColumn column : schema) {
JsonNode value = keyResolver.find(node, index, column.name());
- row.put(column.name(), value == null || value.isNull() ? null : column.coerce(value));
+ row.put(column.name(), value == null || value.isNull() ? null : column.type().coerce(value));
}
return row;
}
+ /**
+ * Whether column and struct member names match case-insensitively. Both SerDes
+ * do, HiveJsonSerDe included (probed), unless OpenX is told not to.
+ */
+ private static boolean caseInsensitive(DataFormatConversionConfiguration conversion) {
+ OpenXJsonSerDe openX = conversion.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe();
+ return openX == null || !Boolean.FALSE.equals(openX.getCaseInsensitive());
+ }
+
+ /**
+ * What the configured deserializer does with a value, all of it probed against
+ * real AWS.
+ *
+ * <p>{@code caseInsensitive}: column and struct member names match regardless of
+ * case. Both SerDes do this unless OpenX is told not to.
+ *
+ * <p>{@code lowercaseKeys}: keys that are data rather than schema, a map's entry
+ * keys and the JSON text a non-scalar leaves in a text column, are stored
+ * lowercased. Only a case-insensitive OpenX does this, since it lowercases every
+ * JSON key before deserializing; HiveJsonSerDe keeps them as written.
+ *
+ * <p>{@code lenient}: a value of the wrong shape is coerced rather than refused.
+ * OpenX wraps a non-array into a one-element array, fills a struct from an array
+ * by position, and serializes a non-scalar into a text column; it also parses a
+ * string into a numeric column. HiveJsonSerDe refuses all four and fails the
+ * record, while still coercing a number into a text column, taking a missing
+ * member as null and dropping an undeclared one.
+ */
+ private record SerDeRules(boolean caseInsensitive, boolean lowercaseKeys, boolean lenient) {
+
+ static SerDeRules of(DataFormatConversionConfiguration conversion) {
+ boolean openX = conversion.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe() != null;
+ boolean caseInsensitive = FirehoseParquetConverter.caseInsensitive(conversion);
+ return new SerDeRules(caseInsensitive, openX && caseInsensitive, openX);
+ }
+ }
+
/**
* Maps a schema column to the JSON member holding its value. OpenX's
* ColumnToJsonKeyMappings renames the key that is looked up; the lookup
@@ -302,7 +346,7 @@ public class FirehoseParquetConverter {
OpenXJsonSerDe openX = conversion.getInputFormatConfiguration().getDeserializer().getOpenXJsonSerDe();
Map<String, String> mappings = openX == null || openX.getColumnToJsonKeyMappings() == null
? Map.of() : openX.getColumnToJsonKeyMappings();
- boolean caseInsensitive = openX == null || !Boolean.FALSE.equals(openX.getCaseInsensitive());
+ boolean caseInsensitive = caseInsensitive(conversion);
return new KeyResolver() {
@Override
public Map<String, JsonNode> index(JsonNode record) {
@@ -341,7 +385,7 @@ public class FirehoseParquetConverter {
* is known, so a conversion failure never has to undo a destination write.
*/
private String convertToStagedParquet(DeliveryStreamDescription stream, S3Destination s3,
- List<ColumnSpec> schema, List<Map<String, Object>> rows,
+ List<SchemaColumn> schema, List<Map<String, Object>> rows,
Instant deliveryTime, ParquetSerDe parquetSerDe) throws Exception {
String runId = UUID.randomUUID().toString();
String parquetKey = "firehose-staging/" + stream.getDeliveryStreamName() + "/" + runId + ".parquet";
@@ -359,14 +403,14 @@ public class FirehoseParquetConverter {
"application/x-ndjson", Map.of());
try {
StringBuilder columns = new StringBuilder();
- for (ColumnSpec column : schema) {
+ for (SchemaColumn column : schema) {
if (columns.length() > 0) {
columns.append(", ");
}
// Glue column names are broader than SQL identifiers, so the name is
// escaped like any other literal rather than restricted up front.
columns.append('\'').append(escapeSqlLiteral(column.name()))
- .append("': '").append(column.duckType()).append('\'');
+ .append("': '").append(escapeSqlLiteral(column.type().duckType())).append('\'');
}
// floci-duck's /execute wraps the main sql field in a CSV-emitting COPY
// for Athena compatibility, so the Parquet COPY goes in setup_sql and the
@@ -531,30 +575,407 @@ public class FirehoseParquetConverter {
}
}
- enum ColumnKind { BOOLEAN, INTEGRAL, FLOATING, DECIMAL, TEXT, DATE, TIMESTAMP }
+ private enum ColumnKind { BOOLEAN, INTEGRAL, FLOATING, DECIMAL, TEXT, DATE, TIMESTAMP }
+
+ /** A Glue column and the type tree its values are coerced against. */
+ private record SchemaColumn(String name, ValueType type) {}
+
+ /**
+ * One node of a column's type. Hive complex types nest, so coercion recurses,
+ * and a failure at any depth fails that record as a whole, as one in a scalar
+ * column already does (probed).
+ */
+ private sealed interface ValueType permits ScalarType, ListType, StructType, MapType {
+
+ /** The DuckDB type {@code read_json} is given for this node. */
+ String duckType();
+
+ /** Coerces one non-null JSON value, or throws {@link RecordConversionException}. */
+ Object coerce(JsonNode value);
+ }
+
+ /** A struct member: Hive keeps the declared name, the lookup is what varies. */
+ private record StructField(String name, ValueType type) {}
+
+ /**
+ * Hive {@code array<T>}. Under OpenX a value that is not an array is wrapped into
+ * a one-element array rather than failing the record: probed, a string in an
+ * {@code array<string>} column arrives as a single-element array, and so does an
+ * object in an {@code array<struct<..>>} one. HiveJsonSerDe fails the record
+ * instead (probed), with the message reproduced here.
+ */
+ private record ListType(ValueType element, SerDeRules rules) implements ValueType {
+
+ @Override
+ public String duckType() {
+ return element.duckType() + "[]";
+ }
+
+ @Override
+ public Object coerce(JsonNode value) {
+ if (!value.isArray()) {
+ if (!rules.lenient()) {
+ throw shapeMismatch("java.io.IOException: Start of Array expected");
+ }
+ List<Object> wrapped = new ArrayList<>(1);
+ wrapped.add(element.coerce(value));
+ return wrapped;
+ }
+ List<Object> coerced = new ArrayList<>(value.size());
+ for (JsonNode entry : value) {
+ coerced.add(entry == null || entry.isNull() ? null : element.coerce(entry));
+ }
+ return coerced;
+ }
+ }
+
+ /**
+ * Hive {@code struct<a:T,..>}. An object matches members by name, a missing one
+ * being null and an undeclared one dropped, so an empty object is a struct of
+ * null members rather than a null struct; both SerDes agree on that (probed).
+ * Under OpenX an array fills the members by position instead, extras ignored,
+ * and only a scalar fails the record. HiveJsonSerDe fails the record for anything
+ * but an object (probed).
+ */
+ private record StructType(List<StructField> fields, SerDeRules rules) implements ValueType {
+
+ @Override
+ public String duckType() {
+ StringBuilder type = new StringBuilder("STRUCT(");
+ for (int i = 0; i < fields.size(); i++) {
+ if (i > 0) {
+ type.append(", ");
+ }
+ type.append('"').append(fields.get(i).name().replace("\"", "\"\"")).append("\" ")
+ .append(fields.get(i).type().duckType());
+ }
+ return type.append(')').toString();
+ }
+
+ @Override
+ public Object coerce(JsonNode value) {
+ if (!value.isObject() && !rules.lenient()) {
+ throw shapeMismatch("java.io.IOException: Start of Object expected");
+ }
+ if (value.isArray()) {
+ return positional(value);
+ }
+ if (!value.isObject()) {
+ // The probed wording, its doubled space included: AWS names the Java
+ // type its own parser read the scalar as.
+ throw new RecordConversionException("DataFormatConversion.MalformedData",
+ "Data does not match the schema. Data is not JSONObject but "
+ + javaTypeName(value) + " with value " + value.asText());
+ }
+ Map<String, JsonNode> byLowerKey = rules.caseInsensitive() ? lowerKeyIndex(value) : Map.of();
+ Map<String, Object> members = new LinkedHashMap<>();
+ for (StructField field : fields) {
+ JsonNode member = value.get(field.name());
+ if (member == null && rules.caseInsensitive()) {
+ member = byLowerKey.get(field.name().toLowerCase(Locale.ROOT));
+ }
+ members.put(field.name(),
+ member == null || member.isNull() ? null : field.type().coerce(member));
+ }
+ return members;
+ }
+
+ private Map<String, Object> positional(JsonNode value) {
+ Map<String, Object> members = new LinkedHashMap<>();
+ for (int i = 0; i < fields.size(); i++) {
+ JsonNode member = value.get(i);
+ members.put(fields.get(i).name(),
+ member == null || member.isNull() ? null : fields.get(i).type().coerce(member));
+ }
+ return members;
+ }
+ }
+
+ /**
+ * Hive {@code map<K,V>}. Only an object is accepted, and its keys are lowercased
+ * only under a case-insensitive OpenX, which lowercases every JSON key before
+ * deserializing: probed, {@code {"MixedKey": ..}} is stored under
+ * {@code mixedkey} there and kept as written under HiveJsonSerDe, which resolves
+ * column and member names case-insensitively all the same. See {@link SerDeRules}.
+ */
+ private record MapType(ValueType key, ValueType value, SerDeRules rules) implements ValueType {
+
+ @Override
+ public String duckType() {
+ return "MAP(" + key.duckType() + ", " + value.duckType() + ")";
+ }
+
+ @Override
+ public Object coerce(JsonNode node) {
+ if (!node.isObject()) {
+ if (!rules.lenient()) {
+ // By analogy with a struct; a map column itself given a non-object
+ // was not probed under HiveJsonSerDe (documented).
+ throw shapeMismatch("java.io.IOException: Start of Object expected");
+ }
+ // Real AWS fails this record too, but with an OpenX cast error naming
+ // classes Floci does not use, so the wording here is its own
+ // (documented deviation).
+ throw new RecordConversionException("DataFormatConversion.MalformedData",
+ "Data does not match the schema. A map column cannot hold a JSON "
+ + (node.isArray() ? "array." : "scalar."));
+ }
+ Map<Object, Object> entries = new LinkedHashMap<>();
+ Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ String name = rules.lowercaseKeys() ? field.getKey().toLowerCase(Locale.ROOT) : field.getKey();
+ JsonNode member = field.getValue();
+ entries.put(key.coerce(TextNode.valueOf(name)),
+ member == null || member.isNull() ? null : value.coerce(member));
+ }
+ return entries;
+ }
+ }
+
+ private static Map<String, JsonNode> lowerKeyIndex(JsonNode node) {
+ Map<String, JsonNode> byLowerKey = new LinkedHashMap<>();
+ Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ byLowerKey.putIfAbsent(field.getKey().toLowerCase(Locale.ROOT), field.getValue());
+ }
+ return byLowerKey;
+ }
+
+ /** The record failure HiveJsonSerDe reports for a value of the wrong shape (probed). */
+ private static RecordConversionException shapeMismatch(String cause) {
+ return new RecordConversionException("DataFormatConversion.MalformedData",
+ "Data does not match the schema. " + cause);
+ }
+
+ private static String javaTypeName(JsonNode value) {
+ if (value.isBoolean()) {
+ return "java.lang.Boolean";
+ }
+ if (value.isIntegralNumber()) {
+ return "java.lang.Long";
+ }
+ if (value.isNumber()) {
+ return "java.lang.Double";
+ }
+ return "java.lang.String";
+ }
+
+ /**
+ * Parses a Glue column's Hive type into a {@link ValueType}. Complex types
+ * nest, so this is a recursive descent rather than a regex: the commas in
+ * {@code struct<a:map<string,int>,b:string>} belong to different levels. A
+ * type it cannot represent, {@code binary} and {@code uniontype} included,
+ * parses to null, which the caller reports as an unsupported schema for the
+ * whole batch.
+ */
+ private static final class HiveTypeParser {
+
+ /** Deep enough for any real schema, shallow enough that the recursion cannot overflow. */
+ private static final int MAX_DEPTH = 32;
+
+ private final String type;
+ private final SerDeRules rules;
+ private int pos;
+
+ private HiveTypeParser(String type, SerDeRules rules) {
+ this.type = type;
+ this.rules = rules;
+ }
+
+ static ValueType parse(String name, String hiveType, SerDeRules rules) {
+ if (hiveType == null || hiveType.isBlank()) {
+ return null;
+ }
+ HiveTypeParser parser = new HiveTypeParser(hiveType, rules);
+ ValueType parsed = parser.parseType(name, 0);
+ parser.skipSpaces();
+ // Trailing content means the type was never understood, as "char(10)garbage"
+ // is not the char column a prefix match would take it for.
+ return parsed == null || parser.pos != parser.type.length() ? null : parsed;
+ }
+
+ private ValueType parseType(String name, int depth) {
+ if (depth > MAX_DEPTH) {
+ return null;
+ }
+ String word = readWord();
+ if (word.isEmpty()) {
+ return null;
+ }
+ return switch (word.toLowerCase(Locale.ROOT)) {
+ case "array" -> parseArray(name, depth);
+ case "map" -> parseMap(name, depth);
+ case "struct" -> parseStruct(name, depth);
+ case "uniontype" -> null;
+ default -> parseScalar(name, word);
+ };
+ }
- record ColumnSpec(String name, String duckType, ColumnKind kind) {
+ private ValueType parseArray(String name, int depth) {
+ if (!expect('<')) {
+ return null;
+ }
+ ValueType element = parseType(name + ".element", depth + 1);
+ if (element == null || !expect('>')) {
+ return null;
+ }
+ return new ListType(element, rules);
+ }
- /** Primitive Hive types only; complex and binary types return null (unsupported). */
- static ColumnSpec fromHiveType(String name, String hiveType) {
- String type = hiveType == null ? "" : hiveType.trim().toLowerCase(Locale.ROOT);
+ private ValueType parseMap(String name, int depth) {
+ if (!expect('<')) {
+ return null;
+ }
+ // A Hive map key is primitive. A complex one would otherwise be found out
+ // by the record coercion or by DuckDB, reporting a schema Floci cannot
+ // convert as a malformed record or an internal error instead.
+ ValueType key = parseType(name + ".key", depth + 1);
+ if (!(key instanceof ScalarType) || !expect(',')) {
+ return null;
+ }
+ ValueType value = parseType(name + ".value", depth + 1);
+ if (value == null || !expect('>')) {
+ return null;
+ }
+ return new MapType(key, value, rules);
+ }
+
+ private ValueType parseStruct(String name, int depth) {
+ if (!expect('<')) {
+ return null;
+ }
+ List<StructField> fields = new ArrayList<>();
+ // DuckDB rejects a STRUCT with two members of the same name, comparing them
+ // case-insensitively, and the projection would keep only one of them anyway,
+ // so the duplicate is reported as the schema problem it is rather than as a
+ // batch-level failure later.
+ Set<String> declared = new HashSet<>();
+ do {
+ String field = readFieldName();
+ if (field.isEmpty() || !expect(':')) {
+ return null;
+ }
+ if (!declared.add(field.toLowerCase(Locale.ROOT))) {
+ return null;
+ }
+ ValueType fieldType = parseType(name + "." + field, depth + 1);
+ if (fieldType == null) {
+ return null;
+ }
+ fields.add(new StructField(field, fieldType));
+ } while (expect(','));
+ return expect('>') ? new StructType(List.copyOf(fields), rules) : null;
+ }
+
+ /** A primitive, with the {@code (n)} or {@code (p,s)} width some of them carry. */
+ private ValueType parseScalar(String name, String word) {
+ String text = word;
+ if (peek() == '(') {
+ int close = type.indexOf(')', pos);
+ if (close < 0) {
+ return null;
+ }
+ // Space is dropped around the punctuation only. Whitespace left after
+ // that sits between two numbers, and removing it would read the
+ // unchecked type string Glue stored as a width it never declared,
+ // turning "varchar(6 5535)" into varchar(65535).
+ String width = type.substring(pos, close + 1).replaceAll("\\s*([(),])\\s*", "$1");
+ if (width.chars().anyMatch(Character::isWhitespace)) {
+ return null;
+ }
+ text = text + width;
+ pos = close + 1;
+ } else if ("double".equalsIgnoreCase(word)) {
+ int mark = pos;
+ if ("precision".equalsIgnoreCase(readWord())) {
+ text = "double precision";
+ } else {
+ pos = mark;
+ }
+ }
+ return ScalarType.fromPrimitive(name, text.toLowerCase(Locale.ROOT), rules);
+ }
+
+ private String readWord() {
+ skipSpaces();
+ int start = pos;
+ while (pos < type.length()
+ && (Character.isLetterOrDigit(type.charAt(pos)) || type.charAt(pos) == '_')) {
+ pos++;
+ }
+ return type.substring(start, pos);
+ }
+
+ /**
+ * Hive lets a struct member be back-quoted, which is how a reserved word is
+ * declared, and a doubled backtick is how one is put inside such a name.
+ */
+ private String readFieldName() {
+ skipSpaces();
+ if (peek() != '`') {
+ return readWord();
+ }
+ StringBuilder quoted = new StringBuilder();
+ int scan = pos + 1;
+ while (scan < type.length()) {
+ if (type.charAt(scan) != '`') {
+ quoted.append(type.charAt(scan));
+ scan++;
+ } else if (scan + 1 < type.length() && type.charAt(scan + 1) == '`') {
+ quoted.append('`');
+ scan += 2;
+ } else {
+ pos = scan + 1;
+ return quoted.toString();
+ }
+ }
+ return "";
+ }
+
+ private char peek() {
+ skipSpaces();
+ return pos < type.length() ? type.charAt(pos) : '\0';
+ }
+
+ private boolean expect(char expected) {
+ if (peek() != expected) {
+ return false;
+ }
+ pos++;
+ return true;
+ }
+
+ private void skipSpaces() {
+ while (pos < type.length() && Character.isWhitespace(type.charAt(pos))) {
+ pos++;
+ }
+ }
+ }
+
+ private record ScalarType(String name, String duckType, ColumnKind kind,
+ SerDeRules rules) implements ValueType {
+
+ /** Primitive Hive types only; anything else, {@code binary} included, returns null. */
+ static ScalarType fromPrimitive(String name, String type, SerDeRules rules) {
Matcher decimal = DECIMAL_TYPE.matcher(type);
if (decimal.matches()) {
- return decimalSpec(name, decimal.group(1), decimal.group(2));
+ return decimalSpec(name, decimal.group(1), decimal.group(2), rules);
}
return switch (type) {
- case "boolean" -> new ColumnSpec(name, "BOOLEAN", ColumnKind.BOOLEAN);
- case "tinyint" -> new ColumnSpec(name, "TINYINT", ColumnKind.INTEGRAL);
- case "smallint" -> new ColumnSpec(name, "SMALLINT", ColumnKind.INTEGRAL);
- case "int", "integer" -> new ColumnSpec(name, "INTEGER", ColumnKind.INTEGRAL);
- case "bigint" -> new ColumnSpec(name, "BIGINT", ColumnKind.INTEGRAL);
- case "float", "real" -> new ColumnSpec(name, "FLOAT", ColumnKind.FLOATING);
- case "double", "double precision" -> new ColumnSpec(name, "DOUBLE", ColumnKind.FLOATING);
- case "decimal" -> new ColumnSpec(name, "DECIMAL(10,0)", ColumnKind.DECIMAL);
- case "string" -> new ColumnSpec(name, "VARCHAR", ColumnKind.TEXT);
- case "date" -> new ColumnSpec(name, "DATE", ColumnKind.DATE);
- case "timestamp" -> new ColumnSpec(name, "TIMESTAMP", ColumnKind.TIMESTAMP);
- default -> sizedTextSpec(name, type);
+ case "boolean" -> new ScalarType(name, "BOOLEAN", ColumnKind.BOOLEAN, rules);
+ case "tinyint" -> new ScalarType(name, "TINYINT", ColumnKind.INTEGRAL, rules);
+ case "smallint" -> new ScalarType(name, "SMALLINT", ColumnKind.INTEGRAL, rules);
+ case "int", "integer" -> new ScalarType(name, "INTEGER", ColumnKind.INTEGRAL, rules);
+ case "bigint" -> new ScalarType(name, "BIGINT", ColumnKind.INTEGRAL, rules);
+ case "float", "real" -> new ScalarType(name, "FLOAT", ColumnKind.FLOATING, rules);
+ case "double", "double precision" -> new ScalarType(name, "DOUBLE", ColumnKind.FLOATING, rules);
+ case "decimal" -> new ScalarType(name, "DECIMAL(10,0)", ColumnKind.DECIMAL, rules);
+ case "string" -> new ScalarType(name, "VARCHAR", ColumnKind.TEXT, rules);
+ case "date" -> new ScalarType(name, "DATE", ColumnKind.DATE, rules);
+ case "timestamp" -> new ScalarType(name, "TIMESTAMP", ColumnKind.TIMESTAMP, rules);
+ default -> sizedTextSpec(name, type, rules);
};
}
@@ -564,7 +985,7 @@ public class FirehoseParquetConverter {
* the type string, so a prefix match would let {@code varchar(foo)} or
* {@code char(10)garbage} through as text instead of reporting the schema.
*/
- private static ColumnSpec sizedTextSpec(String name, String type) {
+ private static ScalarType sizedTextSpec(String name, String type, SerDeRules rules) {
Matcher sized = SIZED_TEXT_TYPE.matcher(type);
if (!sized.matches() || sized.group(2).length() > 5) {
return null;
@@ -574,7 +995,7 @@ public class FirehoseParquetConverter {
if (length < 1 || length > max) {
return null;
}
- return new ColumnSpec(name, "VARCHAR", ColumnKind.TEXT);
+ return new ScalarType(name, "VARCHAR", ColumnKind.TEXT, rules);
}
/**
@@ -582,7 +1003,8 @@ public class FirehoseParquetConverter {
* schema, not of any one record, so an out-of-range width is unsupported
* rather than left for the COPY to reject as a batch-level failure.
*/
- private static ColumnSpec decimalSpec(String name, String precisionText, String scaleText) {
+ private static ScalarType decimalSpec(String name, String precisionText, String scaleText,
+ SerDeRules rules) {
if (precisionText.length() > 2 || scaleText.length() > 2) {
return null;
}
@@ -591,7 +1013,8 @@ public class FirehoseParquetConverter {
if (precision < 1 || precision > MAX_DECIMAL_PRECISION || scale < 0 || scale > precision) {
return null;
}
- return new ColumnSpec(name, "DECIMAL(" + precision + "," + scale + ")", ColumnKind.DECIMAL);
+ return new ScalarType(name, "DECIMAL(" + precision + "," + scale + ")", ColumnKind.DECIMAL,
+ rules);
}
/**
@@ -601,14 +1024,18 @@ public class FirehoseParquetConverter {
* match the schema. For input string: \"oops\"", the raw
* NumberFormatException message.
*/
- Object coerce(JsonNode value) {
+ @Override
+ public Object coerce(JsonNode value) {
try {
+ if (!rules.lenient()) {
+ refuseWhatHiveRefuses(value);
+ }
return switch (kind) {
case BOOLEAN -> coerceBoolean(value);
case INTEGRAL -> coerceIntegral(value);
case FLOATING -> coerceFloating(value);
case DECIMAL -> coerceDecimal(value);
- case TEXT -> value.isValueNode() ? value.asText() : value.toString();
+ case TEXT -> value.isValueNode() ? value.asText() : jsonText(value);
case DATE -> coerceDate(value);
case TIMESTAMP -> coerceTimestamp(value);
};
@@ -620,6 +1047,57 @@ public class FirehoseParquetConverter {
}
}
+ /**
+ * The JSON a non-scalar value keeps when it lands in a text column. OpenX
+ * lowercases every key of every nested object before the value is read, at
+ * any depth (probed), so the text AWS stores carries lowercased keys;
+ * HiveJsonSerDe leaves them as written. The same rule the map keys follow.
+ */
+ private String jsonText(JsonNode value) {
+ return rules.lowercaseKeys() ? lowercaseKeys(value).toString() : value.toString();
+ }
+
+ /**
+ * The two things HiveJsonSerDe will not do that OpenX does, both documented
+ * by AWS and probed: read a JSON string into a numeric column, and turn a
+ * nested value into text. The numeric message is Jackson's; AWS appends a
+ * source location whose object hash changes per run, which is not
+ * reproduced. The text one comes from the SerDe's field validator and
+ * carries no schema prefix.
+ */
+ private void refuseWhatHiveRefuses(JsonNode value) {
+ boolean numeric = kind == ColumnKind.INTEGRAL || kind == ColumnKind.FLOATING
+ || kind == ColumnKind.DECIMAL;
+ if (numeric && value.isTextual()) {
+ throw shapeMismatch("Current token (VALUE_STRING) not numeric,"
+ + " can not use numeric value accessors");
+ }
+ if (kind == ColumnKind.TEXT && !value.isValueNode()) {
+ throw new RecordConversionException("DataFormatConversion.MalformedData",
+ "One or more fields have incorrect format. Exception when validating field (root)");
+ }
+ }
+
+ private static JsonNode lowercaseKeys(JsonNode node) {
+ if (node.isObject()) {
+ ObjectNode lowered = JsonNodeFactory.instance.objectNode();
+ Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ lowered.set(field.getKey().toLowerCase(Locale.ROOT), lowercaseKeys(field.getValue()));
+ }
+ return lowered;
+ }
+ if (node.isArray()) {
+ ArrayNode lowered = JsonNodeFactory.instance.arrayNode(node.size());
+ for (JsonNode entry : node) {
+ lowered.add(lowercaseKeys(entry));
+ }
+ return lowered;
+ }
+ return node;
+ }
+
/**
* A Hive {@code float} column is 32-bit, and DuckDB refuses to cast a value
* the type cannot hold, which would fail the COPY for the whole batch rather
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetComplexTypesTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetComplexTypesTest.java
new file mode 100644
index 0000000000..76d94a0a8d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetComplexTypesTest.java
@@ -0,0 +1,580 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.DataFormatConversionConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Deserializer;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.HiveJsonSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.InputFormatConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OpenXJsonSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OutputFormatConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;
+import io.github.hectorvent.floci.services.floci.duck.FlociDuckClient;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.ArgumentCaptor;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.ArgumentMatchers.startsWith;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Conversion of the complex Hive types, kept apart from
+ * {@link FirehoseParquetConverterTest} because the scalar suite there is already
+ * a long one.
+ *
+ * The coercion asserted here was probed against real AWS on 2026-09-12 with a
+ * table of exactly these four columns: a value that is not an array is wrapped
+ * into one, an array fills a struct by position, map keys are lowercased, an
+ * undeclared struct member is dropped and a missing one is null, and a mismatch
+ * at any depth fails that record alone.
+ */
+class FirehoseParquetComplexTypesTest {
+
+ private static final String BUCKET = "results";
+ private static final String STAGING = "floci-firehose-staging";
+ private static final Instant DELIVERY_TIME = Instant.parse("2026-01-01T00:00:00Z");
+
+ private GlueService glueService;
+ private FlociDuckClient duckClient;
+ private S3Service s3Service;
+ private FirehoseParquetConverter converter;
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @BeforeEach
+ void setUp() {
+ glueService = mock(GlueService.class);
+ duckClient = mock(FlociDuckClient.class);
+ s3Service = mock(S3Service.class);
+ converter = new FirehoseParquetConverter(glueService, duckClient, s3Service, mapper,
+ new RegionResolver("us-east-1", "000000000000"), STAGING);
+ when(s3Service.getObject(eq(STAGING), anyString())).thenReturn(new S3Object(STAGING,
+ "staged.parquet", "PAR1staged".getBytes(StandardCharsets.UTF_8),
+ "application/octet-stream"));
+ when(glueService.getTable("db", "events")).thenReturn(table(
+ new Column("id", "string"),
+ new Column("tags", "array<string>"),
+ new Column("person", "struct<name:string,age:int>"),
+ new Column("items", "array<struct<sku:string,qty:int>>"),
+ new Column("attrs", "map<string,array<string>>")));
+ }
+
+ private static Table table(Column... columns) {
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setColumns(Arrays.asList(columns));
+ Table table = new Table();
+ table.setStorageDescriptor(descriptor);
+ return table;
+ }
+
+ private static DeliveryStreamDescription stream() {
+ return stream(new OpenXJsonSerDe());
+ }
+
+ private static DeliveryStreamDescription hiveStream() {
+ DeliveryStreamDescription stream = stream(null);
+ stream.s3Destination().getDataFormatConversionConfiguration().getInputFormatConfiguration()
+ .getDeserializer().setHiveJsonSerDe(new HiveJsonSerDe());
+ return stream;
+ }
+
+ private static DeliveryStreamDescription stream(OpenXJsonSerDe openX) {
+ SchemaConfiguration schema = new SchemaConfiguration();
+ schema.setRoleArn("arn:aws:iam::000000000000:role/firehose-role");
+ schema.setDatabaseName("db");
+ schema.setTableName("events");
+ Deserializer deserializer = new Deserializer();
+ deserializer.setOpenXJsonSerDe(openX);
+ InputFormatConfiguration input = new InputFormatConfiguration();
+ input.setDeserializer(deserializer);
+ Serializer serializer = new Serializer();
+ serializer.setParquetSerDe(new ParquetSerDe());
+ OutputFormatConfiguration output = new OutputFormatConfiguration();
+ output.setSerializer(serializer);
+ DataFormatConversionConfiguration conversion = new DataFormatConversionConfiguration();
+ conversion.setEnabled(true);
+ conversion.setSchemaConfiguration(schema);
+ conversion.setInputFormatConfiguration(input);
+ conversion.setOutputFormatConfiguration(output);
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::" + BUCKET);
+ s3.setPrefix("data/");
+ s3.setErrorOutputPrefix("errors/!{firehose:error-output-type}/");
+ s3.setCompressionFormat("UNCOMPRESSED");
+ s3.setDataFormatConversionConfiguration(conversion);
+ return new DeliveryStreamDescription("stream", "arn:aws:firehose:::stream", s3);
+ }
+
+ private static byte[] record(String json) {
+ return (json + "\n").getBytes(StandardCharsets.UTF_8);
+ }
+
+ private String stagedNdjson() {
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(eq(STAGING), startsWith("firehose-staging/stream/"),
+ body.capture(), eq("application/x-ndjson"), anyMap());
+ return new String(body.getValue(), StandardCharsets.UTF_8);
+ }
+
+ private String duckColumns() {
+ ArgumentCaptor<String> setupSql = ArgumentCaptor.forClass(String.class);
+ verify(duckClient).execute(eq("SELECT 1 AS ok"), setupSql.capture(), isNull(), any());
+ return setupSql.getValue();
+ }
+
+ private JsonNode singleErrorLine() throws Exception {
+ ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service, atLeastOnce()).putObject(eq(BUCKET), key.capture(),
+ body.capture(), eq("application/octet-stream"), anyMap());
+ for (int i = 0; i < key.getAllValues().size(); i++) {
+ if (key.getAllValues().get(i).startsWith("errors/format-conversion-failed/")) {
+ String object = new String(body.getAllValues().get(i), StandardCharsets.UTF_8);
+ return mapper.readTree(object.strip().split("\n")[0]);
+ }
+ }
+ throw new AssertionError("no error object among " + key.getAllValues());
+ }
+
+ @Test
+ void complexColumnsAreTypedForDuckFromTheGlueSchema() {
+ converter.deliver(stream(), BUCKET, List.of(record("{\"id\": \"a\"}")), DELIVERY_TIME);
+
+ assertTrue(duckColumns().contains("columns={'id': 'VARCHAR', 'tags': 'VARCHAR[]',"
+ + " 'person': 'STRUCT(\"name\" VARCHAR, \"age\" INTEGER)',"
+ + " 'items': 'STRUCT(\"sku\" VARCHAR, \"qty\" INTEGER)[]',"
+ + " 'attrs': 'MAP(VARCHAR, VARCHAR[])'}"),
+ "setup sql was: " + duckColumns());
+ }
+
+ @Test
+ void nestedValuesAreStagedAsTheyWereGiven() {
+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r1\", \"tags\": [\"a\", \"b\"], \"person\": {\"name\": \"Alice\","
+ + " \"age\": 30}, \"items\": [{\"sku\": \"s1\", \"qty\": 2}],"
+ + " \"attrs\": {\"k1\": [\"v1\", \"v2\"]}}")), DELIVERY_TIME);
+
+ assertEquals(1, outcome.convertedRecords());
+ assertTrue(stagedNdjson().contains("{\"id\":\"r1\",\"tags\":[\"a\",\"b\"],"
+ + "\"person\":{\"name\":\"Alice\",\"age\":30},"
+ + "\"items\":[{\"sku\":\"s1\",\"qty\":2}],\"attrs\":{\"k1\":[\"v1\",\"v2\"]}}"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ @Test
+ void aValueThatIsNotAnArrayIsWrappedIntoOne() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r6\", \"tags\": \"notanarray\","
+ + " \"items\": {\"sku\": \"x\", \"qty\": 1}}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"tags\":[\"notanarray\"]"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"items\":[{\"sku\":\"x\",\"qty\":1}]"), "staged rows were: " + ndjson);
+ }
+
+ @Test
+ void anArrayFillsAStructByPositionAndIgnoresTheExtras() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r7\", \"person\": [\"Zoe\", 9, \"ignored\"]}")), DELIVERY_TIME);
+
+ assertTrue(stagedNdjson().contains("\"person\":{\"name\":\"Zoe\",\"age\":9}"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ @Test
+ void aShorterArrayLeavesTheRemainingMembersNull() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r7\", \"person\": [\"Zoe\"]}")), DELIVERY_TIME);
+
+ assertTrue(stagedNdjson().contains("\"person\":{\"name\":\"Zoe\",\"age\":null}"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ @Test
+ void structMembersMatchCaseInsensitivelyAtEveryDepth() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r2\", \"PERSON\": {\"NAME\": \"Bob\", \"AGE\": 41},"
+ + " \"ITEMS\": [{\"SKU\": \"s2\", \"QTY\": \"3\"}]}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"person\":{\"name\":\"Bob\",\"age\":41}"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"items\":[{\"sku\":\"s2\",\"qty\":3}]"), "staged rows were: " + ndjson);
+ }
+
+ @Test
+ void mapKeysAreLowercased() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"s1\", \"attrs\": {\"MixedKey\": [\"v\"]}}")), DELIVERY_TIME);
+
+ assertTrue(stagedNdjson().contains("\"attrs\":{\"mixedkey\":[\"v\"]}"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ @Test
+ void anUndeclaredStructMemberIsDroppedAndAMissingOneIsNull() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r3\", \"person\": {\"name\": \"Carol\", \"extra\": \"dropped\"}}")),
+ DELIVERY_TIME);
+
+ assertTrue(stagedNdjson().contains("\"person\":{\"name\":\"Carol\",\"age\":null}"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ @Test
+ void anEmptyObjectIsAStructOfNullMembersRatherThanANullStruct() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r12\", \"person\": {}, \"tags\": [], \"attrs\": {}}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"person\":{\"name\":null,\"age\":null}"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"tags\":[]"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"attrs\":{}"), "staged rows were: " + ndjson);
+ }
+
+ @Test
+ void nullsInsideContainersAreKept() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r8\", \"tags\": [\"a\", null, \"c\"], \"items\": [null],"
+ + " \"attrs\": {\"k\": null}}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"tags\":[\"a\",null,\"c\"]"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"items\":[null]"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"attrs\":{\"k\":null}"), "staged rows were: " + ndjson);
+ }
+
+ @Test
+ void scalarsAreStillCoercedInsideContainers() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r10\", \"tags\": [1, 2, 3]}")), DELIVERY_TIME);
+
+ assertTrue(stagedNdjson().contains("\"tags\":[\"1\",\"2\",\"3\"]"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ @Test
+ void aNonScalarInATextColumnKeepsItsCompactJson() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": {\"nested\": 1}, \"tags\": [[\"a\", \"b\"]]}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"id\":\"{\\\"nested\\\":1}\""), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"tags\":[\"[\\\"a\\\",\\\"b\\\"]\"]"), "staged rows were: " + ndjson);
+ }
+
+ /**
+ * Probed 2026-09-12: OpenX lowercases the keys of every nested object before the
+ * value is read, at any depth, so the JSON text a stringified object leaves in a
+ * text column carries lowercased keys.
+ */
+ @Test
+ void openXLowercasesTheKeysOfAStringifiedObjectAtEveryDepth() {
+ converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": {\"Outer\": {\"Inner\": 1}}, \"tags\": [{\"MixedKey\": 1}],"
+ + " \"attrs\": {\"K\": [\"v\"]}}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"id\":\"{\\\"outer\\\":{\\\"inner\\\":1}}\""),
+ "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"tags\":[\"{\\\"mixedkey\\\":1}\"]"),
+ "staged rows were: " + ndjson);
+ }
+
+ @Test
+ void aScalarForAStructFailsTheRecord() throws Exception {
+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET,
+ List.of(record("{\"id\": \"r5\", \"person\": 42}")), DELIVERY_TIME);
+
+ assertEquals(0, outcome.convertedRecords());
+ assertEquals(1, outcome.failedRecords());
+ JsonNode line = singleErrorLine();
+ assertEquals("DataFormatConversion.MalformedData", line.get("lastErrorCode").asText());
+ assertEquals("Data does not match the schema. Data is not JSONObject but"
+ + " java.lang.Long with value 42", line.get("lastErrorMessage").asText());
+ }
+
+ @Test
+ void anArrayForAMapFailsTheRecord() throws Exception {
+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET,
+ List.of(record("{\"id\": \"r9\", \"attrs\": [[\"k\", [\"v\"]]]}")), DELIVERY_TIME);
+
+ assertEquals(1, outcome.failedRecords());
+ JsonNode line = singleErrorLine();
+ assertEquals("DataFormatConversion.MalformedData", line.get("lastErrorCode").asText());
+ assertEquals("Data does not match the schema. A map column cannot hold a JSON array.",
+ line.get("lastErrorMessage").asText());
+ }
+
+ @Test
+ void aMismatchInsideAnArrayFailsTheWholeRecordWithTheProbedMessage() throws Exception {
+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"r5\", \"items\": [{\"sku\": \"s5\", \"qty\": \"oops\"}]}")),
+ DELIVERY_TIME);
+
+ assertEquals(0, outcome.convertedRecords());
+ JsonNode line = singleErrorLine();
+ assertEquals("DataFormatConversion.MalformedData", line.get("lastErrorCode").asText());
+ assertEquals("Data does not match the schema. For input string: \"oops\"",
+ line.get("lastErrorMessage").asText());
+ }
+
+ @Test
+ void oneFailingNestedRecordLeavesTheRestOfTheBatchConverting() {
+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET, List.of(
+ record("{\"id\": \"good\", \"person\": {\"name\": \"Alice\", \"age\": 1}}"),
+ record("{\"id\": \"bad\", \"person\": 42}")), DELIVERY_TIME);
+
+ assertEquals(1, outcome.convertedRecords());
+ assertEquals(1, outcome.failedRecords());
+ }
+
+ /**
+ * Probed 2026-09-12 on a separate HiveJsonSerDe rig: that SerDe resolves column
+ * and struct member names case-insensitively just as OpenX does, but it leaves
+ * a map entry key exactly as the record wrote it, where OpenX lowercases every
+ * JSON key before deserializing.
+ */
+ @Test
+ void hiveJsonSerDeKeepsMapKeysWhileStillMatchingMembersCaseInsensitively() {
+ converter.deliver(hiveStream(), BUCKET, List.of(
+ record("{\"id\": \"h1\", \"ATTRS\": {\"MixedKey\": [\"v\"]},"
+ + " \"PERSON\": {\"NAME\": \"Bob\", \"AGE\": 2}}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"attrs\":{\"MixedKey\":[\"v\"]}"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"person\":{\"name\":\"Bob\",\"age\":2}"), "staged rows were: " + ndjson);
+ }
+
+ /**
+ * The HiveJsonSerDe coercion matrix, probed 2026-09-13 on a rig with exactly
+ * these columns: what it still coerces, and the four things it refuses that OpenX
+ * accepts, each with the message AWS wrote to the error output.
+ */
+ @Test
+ void hiveJsonSerDeStillCoercesWhatAwsDocumentsAsAllowed() {
+ converter.deliver(hiveStream(), BUCKET, List.of(
+ record("{\"id\": \"h\", \"tags\": [1, 2, 3], \"person\": {\"name\": \"Dan\", \"extra\": 1},"
+ + " \"items\": [{\"sku\": \"s\", \"qty\": 2, \"extra\": 1}]}"),
+ record("{\"id\": \"empty\", \"person\": {}, \"tags\": [\"a\", null]}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"tags\":[\"1\",\"2\",\"3\"]"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"person\":{\"name\":\"Dan\",\"age\":null}"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"items\":[{\"sku\":\"s\",\"qty\":2}]"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"person\":{\"name\":null,\"age\":null}"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"tags\":[\"a\",null]"), "staged rows were: " + ndjson);
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "{\"id\": \"a\", \"tags\": \"solo\"}"
+ + "|Data does not match the schema. java.io.IOException: Start of Array expected",
+ "{\"id\": \"b\", \"items\": {\"sku\": \"x\", \"qty\": 1}}"
+ + "|Data does not match the schema. java.io.IOException: Start of Array expected",
+ "{\"id\": \"c\", \"attrs\": {\"k\": \"notanarray\"}}"
+ + "|Data does not match the schema. java.io.IOException: Start of Array expected",
+ "{\"id\": \"d\", \"person\": [\"Bob\", 41]}"
+ + "|Data does not match the schema. java.io.IOException: Start of Object expected",
+ "{\"id\": \"e\", \"person\": 42}"
+ + "|Data does not match the schema. java.io.IOException: Start of Object expected",
+ "{\"id\": \"f\", \"person\": {\"name\": \"Eve\", \"age\": \"7\"}}"
+ + "|Data does not match the schema. Current token (VALUE_STRING) not numeric,"
+ + " can not use numeric value accessors",
+ "{\"id\": {\"nested\": 1}}"
+ + "|One or more fields have incorrect format. Exception when validating field (root)",
+ "{\"id\": \"g\", \"tags\": [{\"MixedKey\": 1}]}"
+ + "|One or more fields have incorrect format. Exception when validating field (root)",
+ })
+ void hiveJsonSerDeFailsTheRecordWhereOpenXWouldCoerce(String json, String message) throws Exception {
+ FirehoseParquetConverter.Outcome outcome = converter.deliver(hiveStream(), BUCKET,
+ List.of(record(json)), DELIVERY_TIME);
+
+ assertEquals(0, outcome.convertedRecords());
+ assertEquals(1, outcome.failedRecords());
+ JsonNode line = singleErrorLine();
+ assertEquals("DataFormatConversion.MalformedData", line.get("lastErrorCode").asText());
+ assertEquals(message, line.get("lastErrorMessage").asText());
+ }
+
+ /** The same rule reaches a top-level scalar column, where Floci used to coerce under Hive too. */
+ @Test
+ void hiveJsonSerDeRefusesAStringInATopLevelNumericColumn() throws Exception {
+ when(glueService.getTable("db", "events")).thenReturn(table(
+ new Column("ticker", "string"), new Column("qty", "int")));
+
+ FirehoseParquetConverter.Outcome outcome = converter.deliver(hiveStream(), BUCKET, List.of(
+ record("{\"ticker\": \"AAA\", \"qty\": \"10\"}"),
+ record("{\"ticker\": \"BBB\", \"qty\": 10}")), DELIVERY_TIME);
+
+ assertEquals(1, outcome.convertedRecords());
+ assertEquals(1, outcome.failedRecords());
+ assertTrue(stagedNdjson().contains("{\"ticker\":\"BBB\",\"qty\":10}"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ /**
+ * Probed 2026-09-12: with the option off, a struct member under a column that
+ * did match is left null unless the record spells it exactly, and map keys keep
+ * the case they were written in.
+ */
+ @Test
+ void caseSensitiveOpenXMatchesNestedMembersExactlyAndKeepsMapKeys() {
+ OpenXJsonSerDe openX = new OpenXJsonSerDe();
+ openX.setCaseInsensitive(false);
+
+ converter.deliver(stream(openX), BUCKET, List.of(
+ record("{\"id\": \"s\", \"person\": {\"NAME\": \"Bob\", \"age\": 2},"
+ + " \"attrs\": {\"MixedKey\": [\"v\"]}}")), DELIVERY_TIME);
+
+ String ndjson = stagedNdjson();
+ assertTrue(ndjson.contains("\"person\":{\"name\":null,\"age\":2}"), "staged rows were: " + ndjson);
+ assertTrue(ndjson.contains("\"attrs\":{\"MixedKey\":[\"v\"]}"), "staged rows were: " + ndjson);
+ }
+
+ @Test
+ void nestedContainersAreTypedAllTheWayDown() {
+ when(glueService.getTable("db", "events")).thenReturn(table(new Column("deep",
+ "array<map<string,struct<a:int,b:array<decimal(10,2)>>>>")));
+
+ converter.deliver(stream(), BUCKET, List.of(record("{\"deep\": []}")), DELIVERY_TIME);
+
+ assertTrue(duckColumns().contains("columns={'deep': 'MAP(VARCHAR,"
+ + " STRUCT(\"a\" INTEGER, \"b\" DECIMAL(10,2)[]))[]'}"),
+ "setup sql was: " + duckColumns());
+ }
+
+ /**
+ * The SES event-publishing schema, the shape that motivated this: deeply
+ * nested, with members named after SQL keywords, which is why every struct
+ * member is quoted in the DuckDB type.
+ */
+ @Test
+ void aRealSesEventSchemaIsTypedAllTheWayDown() {
+ when(glueService.getTable("db", "events")).thenReturn(table(new Column("mail",
+ "struct<timestamp:string,source:string,destination:array<string>,"
+ + "headers:array<struct<name:string,value:string>>,"
+ + "commonheaders:struct<from:array<string>,to:array<string>,subject:string>,"
+ + "tags:map<string,array<string>>>")));
+
+ converter.deliver(stream(), BUCKET, List.of(record("{\"mail\": {\"subject\": \"hi\"}}")),
+ DELIVERY_TIME);
+
+ assertTrue(duckColumns().contains("columns={'mail': 'STRUCT(\"timestamp\" VARCHAR,"
+ + " \"source\" VARCHAR, \"destination\" VARCHAR[],"
+ + " \"headers\" STRUCT(\"name\" VARCHAR, \"value\" VARCHAR)[],"
+ + " \"commonheaders\" STRUCT(\"from\" VARCHAR[], \"to\" VARCHAR[],"
+ + " \"subject\" VARCHAR), \"tags\" MAP(VARCHAR, VARCHAR[]))'}"),
+ "setup sql was: " + duckColumns());
+ }
+
+ /** Hive doubles a backtick to put one inside a quoted member name. */
+ @Test
+ void aDoubledBackQuoteIsOneBackQuoteInTheMemberName() {
+ when(glueService.getTable("db", "events")).thenReturn(table(
+ new Column("row", "struct<`a``b`:string>")));
+
+ converter.deliver(stream(), BUCKET, List.of(record("{\"row\": {\"a`b\": \"x\"}}")),
+ DELIVERY_TIME);
+
+ assertTrue(duckColumns().contains("columns={'row': 'STRUCT(\"a`b\" VARCHAR)'}"),
+ "setup sql was: " + duckColumns());
+ assertTrue(stagedNdjson().contains("\"row\":{\"a`b\":\"x\"}"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ /** Space around the punctuation is fine; only space joining two numbers is not. */
+ @Test
+ void aWidthSpacedAroundItsPunctuationStillParses() {
+ when(glueService.getTable("db", "events")).thenReturn(table(
+ new Column("amount", "decimal(10, 2)"), new Column("note", "varchar( 255 )")));
+
+ converter.deliver(stream(), BUCKET, List.of(record("{\"amount\": 1.5}")), DELIVERY_TIME);
+
+ assertTrue(duckColumns().contains("columns={'amount': 'DECIMAL(10,2)', 'note': 'VARCHAR'}"),
+ "setup sql was: " + duckColumns());
+ }
+
+ @Test
+ void aBackQuotedStructMemberKeepsItsName() {
+ when(glueService.getTable("db", "events")).thenReturn(table(
+ new Column("row", "struct<`order`:string>")));
+
+ converter.deliver(stream(), BUCKET, List.of(record("{\"row\": {\"order\": \"x\"}}")),
+ DELIVERY_TIME);
+
+ assertTrue(duckColumns().contains("columns={'row': 'STRUCT(\"order\" VARCHAR)'}"),
+ "setup sql was: " + duckColumns());
+ assertTrue(stagedNdjson().contains("\"row\":{\"order\":\"x\"}"),
+ "staged rows were: " + stagedNdjson());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "binary",
+ "array<binary>",
+ "uniontype<int,string>",
+ "struct<a:uniontype<int,string>>",
+ "array<string",
+ "struct<a:>",
+ "varchar(6 5535)",
+ "decimal(1 0,2)",
+ "struct<a:int,a:string>",
+ "struct<a:int,A:string>",
+ "map<struct<a:string>,int>",
+ "map<array<string>,int>",
+ "struct<:string>",
+ "map<string>",
+ "array<string>garbage",
+ "struct<a:string>,"
+ })
+ void aTypeFlociCannotRepresentFailsTheWholeBatch(String hiveType) throws Exception {
+ when(glueService.getTable("db", "events")).thenReturn(table(new Column("col", hiveType)));
+
+ FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET,
+ List.of(record("{\"col\": null}")), DELIVERY_TIME);
+
+ assertEquals(0, outcome.convertedRecords());
+ assertEquals("DataFormatConversion.UnsupportedSchema",
+ singleErrorLine().get("lastErrorCode").asText());
+ }
+
+ /** A hostile schema must report an unsupported type rather than overflow the parse. */
+ @Test
+ void nestingDeeperThanTheParserAcceptsIsUnsupported() throws Exception {
+ String hiveType = "array<".repeat(40) + "string" + ">".repeat(40);
+ when(glueService.getTable("db", "events")).thenReturn(table(new Column("col", hiveType)));
+
+ converter.deliver(stream(), BUCKET, List.of(record("{\"col\": null}")), DELIVERY_TIME);
+
+ assertEquals("DataFormatConversion.UnsupportedSchema",
+ singleErrorLine().get("lastErrorCode").asText());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverterTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverterTest.java
index f98f96ea89..07a2029cea 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverterTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverterTest.java
@@ -613,7 +613,7 @@ class FirehoseParquetConverterTest {
@Test
void unsupportedColumnTypeFailsTheWholeBatch() throws Exception {
when(glueService.getTable("db", "events")).thenReturn(table(
- new Column("ticker", "string"), new Column("tags", "array<string>")));
+ new Column("ticker", "string"), new Column("payload", "binary")));
FirehoseParquetConverter.Outcome outcome = converter.deliver(stream(), BUCKET,
List.of(record("{\"ticker\": \"AAA\"}")), DELIVERY_TIME);
@@ -621,7 +621,7 @@ class FirehoseParquetConverterTest {
assertEquals(0, outcome.convertedRecords());
JsonNode line = errorLines().get(0);
assertEquals("DataFormatConversion.UnsupportedSchema", line.get("lastErrorCode").asText());
- assertTrue(line.get("lastErrorMessage").asText().contains("array<string>"));
+ assertTrue(line.get("lastErrorMessage").asText().contains("binary"));
}
@ParameterizedTest