Skip to content

Apache DataFusion SQL Query Engine

Match Parquet map key/value by position in nested schema coercion (#25342)

Adrian Garcia Badaracco710c748db893
diff --git a/datafusion/datasource-parquet/src/schema_coercion.rs b/datafusion/datasource-parquet/src/schema_coercion.rs
index ce51ab33fd..828070659b 100644
--- a/datafusion/datasource-parquet/src/schema_coercion.rs
+++ b/datafusion/datasource-parquet/src/schema_coercion.rs
@@ -22,13 +22,23 @@
//! These helpers are independent of the [`ParquetFormat`](crate::file_format::ParquetFormat)
//! type and several have been re-exported at the crate root for use by
//! callers outside the format implementation.
+//!
+//! # Binary to string coercion
+//!
+//! Coercing a binary file column to a string type makes the Parquet reader
+//! build the string array itself. It only validates UTF-8 for columns carrying
+//! the `UTF8` logical annotation, which such a column by definition does not,
+//! so invalid bytes surface as an invalid string array rather than an error.
+//! That is long-standing behaviour for top-level and struct fields; it is not
+//! extended to map children, whose positional matching is newer, so binary map
+//! values keep going through the validating cast instead.
use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
use std::rc::Rc;
use std::sync::Arc;
-use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit};
+use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, TimeUnit};
use parquet::basic::Type;
use parquet::schema::types::SchemaDescriptor;
@@ -40,6 +50,21 @@ use parquet::schema::types::SchemaDescriptor;
/// 2. Regular to view types conversion - Converts standard string/binary types to
/// view types when the table schema uses view types
///
+/// The same coercions are applied recursively to nested types: children of
+/// [`DataType::Struct`] fields are matched by name, while the children of
+/// list-like types ([`DataType::List`], [`DataType::LargeList`],
+/// [`DataType::ListView`], [`DataType::LargeListView`],
+/// [`DataType::FixedSizeList`]) and the key/value children of
+/// [`DataType::Map`] are matched by position, regardless of their names.
+///
+/// Only leaf data types are ever changed: field names, nullability, metadata,
+/// the list/map container kind, [`DataType::FixedSizeList`] width and
+/// [`DataType::Map`] ordering are always taken from `file_schema`, since the
+/// resulting schema must still describe the data physically present in the
+/// file (see [`ArrowReaderOptions::with_schema`]).
+///
+/// [`ArrowReaderOptions::with_schema`]: parquet::arrow::arrow_reader::ArrowReaderOptions::with_schema
+///
/// # Arguments
/// * `table_schema` - The table schema containing the desired types
/// * `file_schema` - The file schema to be transformed
@@ -51,158 +76,170 @@ pub fn apply_file_schema_type_coercions(
table_schema: &Schema,
file_schema: &Schema,
) -> Option<Schema> {
- let mut needs_view_transform = false;
- let mut needs_string_transform = false;
- let mut needs_nested_transform = false;
+ let fields =
+ coerce_fields_by_name(table_schema.fields(), file_schema.fields(), true)?;
+ Some(Schema::new_with_metadata(
+ fields,
+ file_schema.metadata.clone(),
+ ))
+}
+/// Coerce `file_fields` towards `table_fields`, matching fields by name.
+///
+/// File fields with no counterpart in `table_fields` are kept unchanged and
+/// table fields missing from the file are ignored. Returns `None` if no field
+/// changed.
+fn coerce_fields_by_name(
+ table_fields: &Fields,
+ file_fields: &Fields,
+ binary_to_string: bool,
+) -> Option<Fields> {
// Create a mapping of table field names to their data types for fast lookup
- // and simultaneously check if we need any transformations
- let table_fields: HashMap<_, _> = table_schema
- .fields()
+ let table_types: HashMap<_, _> = table_fields
.iter()
- .map(|f| {
- let dt = f.data_type();
- // Check if we need view type transformation
- if matches!(dt, &DataType::Utf8View | &DataType::BinaryView) {
- needs_view_transform = true;
- }
- // Check if we need string type transformation
- if matches!(
- dt,
- &DataType::Utf8 | &DataType::LargeUtf8 | &DataType::Utf8View
- ) {
- needs_string_transform = true;
- }
- // Nested fields can need transformations even when their parent does not.
- if matches!(
- dt,
- DataType::Struct(_)
- | DataType::List(_)
- | DataType::LargeList(_)
- | DataType::ListView(_)
- | DataType::LargeListView(_)
- | DataType::FixedSizeList(_, _)
- | DataType::Map(_, _)
- ) {
- needs_nested_transform = true;
- }
-
- (f.name(), dt)
- })
+ .map(|f| (f.name(), f.data_type()))
.collect();
- // Early return if no transformation needed
- if !needs_view_transform && !needs_string_transform && !needs_nested_transform {
- return None;
- }
+ coerce_fields(file_fields, |_, field| {
+ let table_type = table_types.get(field.name())?;
+ coerce_data_type(table_type, field.data_type(), binary_to_string)
+ .map(|new_type| field_with_new_type(field, new_type))
+ })
+}
- let transformed_fields: Vec<Arc<Field>> = file_schema
- .fields()
- .iter()
- .map(|field| {
- let field_name = field.name();
- let field_type = field.data_type();
-
- // Look up the corresponding field type in the table schema
- if let Some(table_type) = table_fields.get(field_name) {
- match (table_type, field_type) {
- // table schema uses string type, coerce the file schema to use string type
- (
- &DataType::Utf8,
- DataType::Binary | DataType::LargeBinary | DataType::BinaryView,
- ) => {
- return field_with_new_type(field, DataType::Utf8);
- }
- // table schema uses large string type, coerce the file schema to use large string type
- (
- &DataType::LargeUtf8,
- DataType::Binary | DataType::LargeBinary | DataType::BinaryView,
- ) => {
- return field_with_new_type(field, DataType::LargeUtf8);
- }
- // table schema uses string view type, coerce the file schema to use view type
- (
- &DataType::Utf8View,
- DataType::Binary | DataType::LargeBinary | DataType::BinaryView,
- ) => {
- return field_with_new_type(field, DataType::Utf8View);
- }
- // Handle view type conversions
- (&DataType::Utf8View, DataType::Utf8 | DataType::LargeUtf8) => {
- return field_with_new_type(field, DataType::Utf8View);
- }
- (&DataType::BinaryView, DataType::Binary | DataType::LargeBinary) => {
- return field_with_new_type(field, DataType::BinaryView);
- }
- // Apply the same coercions to matching fields inside structs.
- (DataType::Struct(table_fields), DataType::Struct(file_fields)) => {
- if let Some(schema) = apply_file_schema_type_coercions(
- &Schema::new(table_fields.clone()),
- &Schema::new(file_fields.clone()),
- ) {
- return field_with_new_type(
- field,
- DataType::Struct(schema.fields),
- );
- }
- }
- // Container children match by position, regardless of their names.
- (DataType::List(table_child), DataType::List(file_child))
- | (
- DataType::LargeList(table_child),
- DataType::LargeList(file_child),
- )
- | (DataType::ListView(table_child), DataType::ListView(file_child))
- | (
- DataType::LargeListView(table_child),
- DataType::LargeListView(file_child),
- )
- | (
- DataType::FixedSizeList(table_child, _),
- DataType::FixedSizeList(file_child, _),
- )
- | (DataType::Map(table_child, _), DataType::Map(file_child, _)) => {
- if let Some(schema) = apply_file_schema_type_coercions(
- &Schema::new(vec![field_with_new_type(
- file_child,
- table_child.data_type().clone(),
- )]),
- &Schema::new(vec![Arc::clone(file_child)]),
- ) {
- let child = Arc::clone(&schema.fields()[0]);
- let new_type = match field_type {
- DataType::List(_) => DataType::List(child),
- DataType::LargeList(_) => DataType::LargeList(child),
- DataType::ListView(_) => DataType::ListView(child),
- DataType::LargeListView(_) => {
- DataType::LargeListView(child)
- }
- DataType::FixedSizeList(_, size) => {
- DataType::FixedSizeList(child, *size)
- }
- DataType::Map(_, sorted) => DataType::Map(child, *sorted),
- _ => return Arc::clone(field),
- };
- return field_with_new_type(field, new_type);
- }
- }
- _ => {}
+/// Rebuild `file_fields`, replacing every field for which `coerce` returns a
+/// new one. Returns `None` if no field changed.
+///
+/// The output is only allocated once a field actually changes, so schemas
+/// needing no coercion at all (the common case) are walked without allocating
+/// or touching the reference counts of the file fields.
+fn coerce_fields(
+ file_fields: &Fields,
+ mut coerce: impl FnMut(usize, &FieldRef) -> Option<FieldRef>,
+) -> Option<Fields> {
+ let mut coerced: Option<Vec<FieldRef>> = None;
+ for (idx, field) in file_fields.iter().enumerate() {
+ match coerce(idx, field) {
+ Some(new_field) => coerced
+ .get_or_insert_with(|| {
+ // The fields before the first change are carried over as is
+ let mut fields = Vec::with_capacity(file_fields.len());
+ fields.extend_from_slice(&file_fields[..idx]);
+ fields
+ })
+ .push(new_field),
+ // Unchanged fields are only copied once something else changed
+ None => {
+ if let Some(coerced) = &mut coerced {
+ coerced.push(Arc::clone(field));
}
}
+ }
+ }
- // If no transformation is needed, keep the original field
- Arc::clone(field)
- })
- .collect();
+ coerced.map(Fields::from)
+}
- if transformed_fields.iter().eq(file_schema.fields().iter()) {
+/// Coerce `file_type` towards `table_type`, recursing into nested types.
+///
+/// `binary_to_string` allows binary file types to be read as string types.
+/// Parquet only validates UTF-8 when the column carries the `UTF8` logical
+/// annotation, which a binary column by definition does not, so such a
+/// conversion hands back an unvalidated string array; see the module docs.
+///
+/// Returns the new type for the file field, or `None` if no transformation
+/// is needed (including when the two types are unrelated).
+fn coerce_data_type(
+ table_type: &DataType,
+ file_type: &DataType,
+ binary_to_string: bool,
+) -> Option<DataType> {
+ use DataType::*;
+ match (table_type, file_type) {
+ // table schema uses string type, coerce the file schema to use string type
+ (Utf8, Binary | LargeBinary | BinaryView) if binary_to_string => Some(Utf8),
+ // table schema uses large string type, coerce the file schema to use large string type
+ (LargeUtf8, Binary | LargeBinary | BinaryView) if binary_to_string => {
+ Some(LargeUtf8)
+ }
+ // table schema uses string view type, coerce the file schema to use view type
+ (Utf8View, Binary | LargeBinary | BinaryView) if binary_to_string => {
+ Some(Utf8View)
+ }
+ (Utf8View, Utf8 | LargeUtf8) => Some(Utf8View),
+ (BinaryView, Binary | LargeBinary) => Some(BinaryView),
+ // Struct children match by name
+ (Struct(table_fields), Struct(file_fields)) => {
+ coerce_fields_by_name(table_fields, file_fields, binary_to_string).map(Struct)
+ }
+ // List-like children match by position, regardless of their names.
+ // The container kind and FixedSizeList width always come from the file.
+ (List(table_child), List(file_child)) => {
+ coerce_child(table_child, file_child, binary_to_string).map(List)
+ }
+ (LargeList(table_child), LargeList(file_child)) => {
+ coerce_child(table_child, file_child, binary_to_string).map(LargeList)
+ }
+ (ListView(table_child), ListView(file_child)) => {
+ coerce_child(table_child, file_child, binary_to_string).map(ListView)
+ }
+ (LargeListView(table_child), LargeListView(file_child)) => {
+ coerce_child(table_child, file_child, binary_to_string).map(LargeListView)
+ }
+ (FixedSizeList(table_child, _), FixedSizeList(file_child, size)) => {
+ coerce_child(table_child, file_child, binary_to_string)
+ .map(|child| FixedSizeList(child, *size))
+ }
+ // Map keys and values match by position: Parquet always names them
+ // `key`/`value` while Arrow producers commonly use `keys`/`values`.
+ (Map(table_entries, _), Map(file_entries, sorted)) => {
+ coerce_map_entries(table_entries, file_entries)
+ .map(|entries| Map(entries, *sorted))
+ }
+ _ => None,
+ }
+}
+
+/// Coerce a single nested child field, keeping everything but its data type
+/// from `file_child`.
+fn coerce_child(
+ table_child: &FieldRef,
+ file_child: &FieldRef,
+ binary_to_string: bool,
+) -> Option<FieldRef> {
+ coerce_data_type(
+ table_child.data_type(),
+ file_child.data_type(),
+ binary_to_string,
+ )
+ .map(|new_type| field_with_new_type(file_child, new_type))
+}
+
+/// Coerce the `entries` struct of a [`DataType::Map`], matching the key and
+/// value children by position.
+///
+/// Binary children are never read as strings here: matching by position newly
+/// reaches map children whose names differ, and turning those into unvalidated
+/// string arrays would be a regression over leaving them to the (validating)
+/// cast. See the module docs.
+fn coerce_map_entries(
+ table_entries: &FieldRef,
+ file_entries: &FieldRef,
+) -> Option<FieldRef> {
+ let (DataType::Struct(table_fields), DataType::Struct(file_fields)) =
+ (table_entries.data_type(), file_entries.data_type())
+ else {
+ return None;
+ };
+ if table_fields.len() != file_fields.len() {
return None;
}
- Some(Schema::new_with_metadata(
- transformed_fields,
- file_schema.metadata.clone(),
- ))
+ let fields = coerce_fields(file_fields, |idx, file_child| {
+ coerce_child(&table_fields[idx], file_child, false)
+ })?;
+
+ Some(field_with_new_type(file_entries, DataType::Struct(fields)))
}
/// Coerces the file schema's Timestamps to the provided TimeUnit if the
@@ -607,7 +644,11 @@ mod tests {
#[test]
fn nested_coercion_preserves_list_containers() {
- let wrap: [fn(FieldRef) -> DataType; 6] = [
+ // The table side deliberately differs from the file side in the child
+ // field name, nullability and metadata, as well as in the
+ // FixedSizeList width and Map ordering: all of those must be taken
+ // from the file, only the leaf types come from the table.
+ let file_wrap: [fn(FieldRef) -> DataType; 6] = [
DataType::List,
DataType::LargeList,
DataType::ListView,
@@ -615,37 +656,51 @@ mod tests {
|field| DataType::FixedSizeList(field, 3),
|field| DataType::Map(field, false),
];
- for wrap in wrap {
- let file_element = Arc::new(Field::new_struct(
- "entries",
- vec![
- Field::new("key", DataType::Utf8, false),
- Field::new("value", DataType::Binary, true),
- ],
- false,
- ));
+ let table_wrap: [fn(FieldRef) -> DataType; 6] = [
+ DataType::List,
+ DataType::LargeList,
+ DataType::ListView,
+ DataType::LargeListView,
+ |field| DataType::FixedSizeList(field, 7),
+ |field| DataType::Map(field, true),
+ ];
+ for (file_wrap, table_wrap) in file_wrap.into_iter().zip(table_wrap) {
+ let file_element = Arc::new(
+ Field::new_struct(
+ "key_value",
+ vec![
+ Field::new("key", DataType::Utf8, false),
+ Field::new("value", DataType::Utf8, true),
+ ],
+ false,
+ )
+ .with_metadata(HashMap::from([("source".into(), "file".into())])),
+ );
let table_element = Arc::new(Field::new_struct(
"entries",
vec![
- Field::new("value", DataType::Utf8View, true),
- Field::new("key", DataType::Utf8View, false),
+ Field::new("value", DataType::Utf8View, false),
+ Field::new("key", DataType::Utf8View, true),
],
- false,
- ));
- let expected_element = Arc::new(Field::new_struct(
- "entries",
- vec![
- Field::new("key", DataType::Utf8View, false),
- Field::new("value", DataType::Utf8View, true),
- ],
- false,
+ true,
));
+ let expected_element = Arc::new(
+ Field::new_struct(
+ "key_value",
+ vec![
+ Field::new("key", DataType::Utf8View, false),
+ Field::new("value", DataType::Utf8View, true),
+ ],
+ false,
+ )
+ .with_metadata(HashMap::from([("source".into(), "file".into())])),
+ );
let file_schema =
- Schema::new(vec![Field::new("data", wrap(file_element), true)]);
+ Schema::new(vec![Field::new("data", file_wrap(file_element), true)]);
let table_schema =
- Schema::new(vec![Field::new("data", wrap(table_element), true)]);
+ Schema::new(vec![Field::new("data", table_wrap(table_element), false)]);
let expected =
- Schema::new(vec![Field::new("data", wrap(expected_element), true)]);
+ Schema::new(vec![Field::new("data", file_wrap(expected_element), true)]);
assert_eq!(
apply_file_schema_type_coercions(&table_schema, &file_schema),
Some(expected)
@@ -653,6 +708,364 @@ mod tests {
}
}
+ #[test]
+ fn nested_coercion_matches_map_entries_by_position() {
+ // Parquet readers always name map children `key_value`/`key`/`value`,
+ // while Arrow producers (e.g. `MapBuilder`) default to
+ // `entries`/`keys`/`values`. Names must not matter for maps.
+ let file_schema = Schema::new(vec![Field::new_map(
+ "m",
+ "key_value",
+ Field::new("key", DataType::Utf8, false),
+ Field::new("value", DataType::Binary, true),
+ false,
+ true,
+ )]);
+ let table_schema = Schema::new(vec![Field::new_map(
+ "m",
+ "entries",
+ Field::new("keys", DataType::Utf8View, false),
+ Field::new("values", DataType::Utf8View, true),
+ false,
+ true,
+ )]);
+ // The key is matched by position and coerced to a view, while the
+ // binary value is left to the validating cast (see
+ // `coerce_map_entries`)
+ let expected = Schema::new(vec![Field::new_map(
+ "m",
+ "key_value",
+ Field::new("key", DataType::Utf8View, false),
+ Field::new("value", DataType::Binary, true),
+ false,
+ true,
+ )]);
+ assert_eq!(
+ apply_file_schema_type_coercions(&table_schema, &file_schema),
+ Some(expected)
+ );
+
+ // A map whose entries do not have the same shape is left alone
+ let odd_table_schema = Schema::new(vec![Field::new(
+ "m",
+ DataType::Map(
+ Arc::new(Field::new_struct(
+ "entries",
+ vec![Field::new("keys", DataType::Utf8View, false)],
+ false,
+ )),
+ false,
+ ),
+ true,
+ )]);
+ assert_eq!(
+ apply_file_schema_type_coercions(&odd_table_schema, &file_schema),
+ None
+ );
+ }
+
+ #[test]
+ fn nested_coercion_map_entry_struct_children_match_by_name() {
+ // A map whose key and value are both structs: the two entry children
+ // are matched by position (the file's `key`/`value` take the types of
+ // the table's first/second entry child, whatever they are named),
+ // while the fields inside those structs go back to matching by name.
+ //
+ // Both structs put `shared` in a different position on each side, so a
+ // positional match inside them would coerce `file_only` instead, and a
+ // by-name match on the entries themselves would coerce nothing at all.
+ let struct_field =
+ |name: &str, first: (&str, DataType), second: (&str, DataType)| {
+ Field::new_struct(
+ name,
+ vec![
+ Field::new(first.0, first.1, true),
+ Field::new(second.0, second.1, true),
+ ],
+ true,
+ )
+ };
+ let file_schema = Schema::new(vec![Field::new_map(
+ "m",
+ "key_value",
+ struct_field(
+ "key",
+ ("file_only", DataType::Utf8),
+ ("shared", DataType::Utf8),
+ ),
+ struct_field(
+ "value",
+ ("file_only", DataType::Utf8),
+ ("shared", DataType::Utf8),
+ ),
+ false,
+ true,
+ )]);
+ let table_schema = Schema::new(vec![Field::new_map(
+ "m",
+ "entries",
+ struct_field(
+ "keys",
+ ("shared", DataType::Utf8View),
+ ("table_only", DataType::Utf8View),
+ ),
+ struct_field(
+ "values",
+ ("shared", DataType::Utf8View),
+ ("table_only", DataType::Utf8View),
+ ),
+ false,
+ true,
+ )]);
+ let expected = Schema::new(vec![Field::new_map(
+ "m",
+ "key_value",
+ struct_field(
+ "key",
+ ("file_only", DataType::Utf8),
+ ("shared", DataType::Utf8View),
+ ),
+ struct_field(
+ "value",
+ ("file_only", DataType::Utf8),
+ ("shared", DataType::Utf8View),
+ ),
+ false,
+ true,
+ )]);
+ assert_eq!(
+ apply_file_schema_type_coercions(&table_schema, &file_schema),
+ Some(expected)
+ );
+ }
+
+ #[test]
+ fn nested_coercion_leaves_binary_map_children_to_the_validating_cast() {
+ use arrow::array::{
+ Array, ArrayRef, AsArray, BinaryBuilder, MapBuilder, MapFieldNames,
+ StringBuilder,
+ };
+ use arrow::record_batch::RecordBatch;
+ use bytes::Bytes;
+ use parquet::arrow::ArrowWriter;
+ use parquet::arrow::arrow_reader::{
+ ArrowReaderOptions, ParquetRecordBatchReaderBuilder,
+ };
+ use parquet::arrow::arrow_writer::ArrowWriterOptions;
+
+ // A binary map value holding bytes that are not valid UTF-8. Reading it
+ // as a string would produce an invalid array, since the Parquet reader
+ // only validates columns annotated `UTF8`, so the value must be left
+ // binary for the (validating) cast to reject.
+ let parquet_names = MapFieldNames {
+ entry: "key_value".to_string(),
+ key: "key".to_string(),
+ value: "value".to_string(),
+ };
+ let mut builder = MapBuilder::new(
+ Some(parquet_names),
+ StringBuilder::new(),
+ BinaryBuilder::new(),
+ );
+ builder.keys().append_value("k1");
+ builder.values().append_value([0xff]);
+ builder.append(true).unwrap();
+ let batch =
+ RecordBatch::try_from_iter([("m", Arc::new(builder.finish()) as ArrayRef)])
+ .unwrap();
+
+ let mut bytes = Vec::new();
+ let mut writer = ArrowWriter::try_new_with_options(
+ &mut bytes,
+ batch.schema(),
+ ArrowWriterOptions::new().with_skip_arrow_metadata(true),
+ )
+ .unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+ let bytes = Bytes::from(bytes);
+
+ let file_schema = ParquetRecordBatchReaderBuilder::try_new(bytes.clone())
+ .unwrap()
+ .schema()
+ .clone();
+ let table_schema = Schema::new(vec![Field::new_map(
+ "m",
+ "entries",
+ Field::new("keys", DataType::Utf8View, false),
+ Field::new("values", DataType::Utf8View, true),
+ false,
+ true,
+ )]);
+
+ // The key is still coerced by position, the binary value is not
+ let coerced =
+ apply_file_schema_type_coercions(&table_schema, &file_schema).unwrap();
+ assert_eq!(
+ coerced.field(0).data_type(),
+ &DataType::Map(
+ Arc::new(Field::new_struct(
+ "key_value",
+ vec![
+ Field::new("key", DataType::Utf8View, false),
+ Field::new("value", DataType::Binary, true),
+ ],
+ false,
+ )),
+ false,
+ )
+ );
+
+ let mut reader = ParquetRecordBatchReaderBuilder::try_new_with_options(
+ bytes,
+ ArrowReaderOptions::new().with_schema(Arc::new(coerced)),
+ )
+ .unwrap()
+ .build()
+ .unwrap();
+ let decoded = reader.next().unwrap().unwrap();
+ // The decoded batch is valid: the invalid bytes are still binary
+ decoded.column(0).to_data().validate_full().unwrap();
+ assert_eq!(
+ decoded
+ .column(0)
+ .as_map()
+ .values()
+ .as_binary::<i32>()
+ .value(0),
+ [0xff]
+ );
+ }
+
+ #[test]
+ fn nested_coercion_list_inside_struct() {
+ let file_schema = Schema::new(vec![Field::new_struct(
+ "s",
+ vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new_list("tags", Field::new("item", DataType::Utf8, true), true),
+ ],
+ true,
+ )]);
+ let table_schema = Schema::new(vec![Field::new_struct(
+ "s",
+ vec![Field::new_list(
+ "tags",
+ Field::new("element", DataType::Utf8View, false),
+ false,
+ )],
+ false,
+ )]);
+ let expected = Schema::new(vec![Field::new_struct(
+ "s",
+ vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new_list(
+ "tags",
+ Field::new("item", DataType::Utf8View, true),
+ true,
+ ),
+ ],
+ true,
+ )]);
+ assert_eq!(
+ apply_file_schema_type_coercions(&table_schema, &file_schema),
+ Some(expected)
+ );
+ }
+
+ #[test]
+ fn nested_coercion_map_reader_produces_string_views() {
+ use arrow::array::{
+ Array, ArrayRef, AsArray, MapBuilder, MapFieldNames, StringBuilder,
+ };
+ use arrow::record_batch::RecordBatch;
+ use bytes::Bytes;
+ use parquet::arrow::ArrowWriter;
+ use parquet::arrow::arrow_reader::{
+ ArrowReaderOptions, ParquetRecordBatchReaderBuilder,
+ };
+ use parquet::arrow::arrow_writer::ArrowWriterOptions;
+
+ // Build the file with Parquet's `key_value`/`key`/`value` naming, as
+ // written by non-Arrow writers, while the table schema below uses
+ // Arrow's default `entries`/`keys`/`values` naming.
+ let parquet_names = MapFieldNames {
+ entry: "key_value".to_string(),
+ key: "key".to_string(),
+ value: "value".to_string(),
+ };
+ let mut builder = MapBuilder::new(
+ Some(parquet_names),
+ StringBuilder::new(),
+ StringBuilder::new(),
+ );
+ builder.keys().append_value("k1");
+ builder.values().append_value("v1");
+ builder.append(true).unwrap();
+ builder.append(false).unwrap();
+ let map = builder.finish();
+ let batch =
+ RecordBatch::try_from_iter([("m", Arc::new(map) as ArrayRef)]).unwrap();
+ let table_schema = Schema::new(vec![Field::new_map(
+ "m",
+ "entries",
+ Field::new("keys", DataType::Utf8View, false),
+ Field::new("values", DataType::Utf8View, true),
+ false,
+ true,
+ )]);
+
+ // Write without the embedded Arrow schema, as a non-Arrow writer would
+ let mut bytes = Vec::new();
+ let mut writer = ArrowWriter::try_new_with_options(
+ &mut bytes,
+ batch.schema(),
+ ArrowWriterOptions::new().with_skip_arrow_metadata(true),
+ )
+ .unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+ let bytes = Bytes::from(bytes);
+
+ let file_schema = ParquetRecordBatchReaderBuilder::try_new(bytes.clone())
+ .unwrap()
+ .schema()
+ .clone();
+ assert_eq!(
+ file_schema.field(0).data_type(),
+ &DataType::Map(
+ Arc::new(Field::new_struct(
+ "key_value",
+ vec![
+ Field::new("key", DataType::Utf8, false),
+ Field::new("value", DataType::Utf8, true),
+ ],
+ false,
+ )),
+ false,
+ )
+ );
+
+ let reader_schema = Arc::new(
+ apply_file_schema_type_coercions(&table_schema, &file_schema).unwrap(),
+ );
+ let mut reader = ParquetRecordBatchReaderBuilder::try_new_with_options(
+ bytes,
+ ArrowReaderOptions::new().with_schema(Arc::clone(&reader_schema)),
+ )
+ .unwrap()
+ .build()
+ .unwrap();
+ let decoded = reader.next().unwrap().unwrap();
+ assert_eq!(decoded.schema(), reader_schema);
+ let map = decoded.column(0).as_map();
+ assert_eq!(map.keys().as_string_view().value(0), "k1");
+ assert_eq!(map.values().as_string_view().value(0), "v1");
+ assert!(map.is_null(1));
+ assert!(reader.next().is_none());
+ }
+
#[test]
fn nested_coercion_preserves_leaf_conversion_rules() {
let nested_schema = |data_type| {