Skip to content

Apache DataFusion SQL Query Engine

Preserve list element metadata when rebuilding scalar values (#26092)

Artem Osipov390d2107d10b
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs
index 60c7767f9c..6727ab7dcc 100644
--- a/datafusion/common/src/utils/mod.rs
+++ b/datafusion/common/src/utils/mod.rs
@@ -43,7 +43,7 @@ use arrow::compute::kernels::cmp::eq;
use arrow::compute::kernels::length::length;
use arrow::compute::{SortColumn, SortOptions, nullif, partition};
use arrow::datatypes::{
- ArrowNativeType, DataType, Field, Int32Type, Int64Type, SchemaRef,
+ ArrowNativeType, DataType, Field, Int32Type, Int64Type, Metadata, SchemaRef,
};
#[cfg(feature = "sql")]
use sqlparser::{ast::Ident, dialect::GenericDialect, parser::Parser};
@@ -561,6 +561,7 @@ pub struct SingleRowListArrayBuilder {
/// Specify the field name for the resulting array. Defaults to value used in
/// [`Field::new_list_field`]
field_name: Option<String>,
+ field_metadata: Metadata,
}
impl SingleRowListArrayBuilder {
@@ -570,6 +571,7 @@ impl SingleRowListArrayBuilder {
arr,
nullable: true,
field_name: None,
+ field_metadata: Metadata::default(),
}
}
@@ -585,10 +587,12 @@ impl SingleRowListArrayBuilder {
self
}
- /// Copies field name and nullable from the specified field
- pub fn with_field(self, field: &Field) -> Self {
- self.with_field_name(Some(field.name().to_owned()))
- .with_nullable(field.is_nullable())
+ /// Copies field name, nullability, and metadata from the specified field
+ pub fn with_field(mut self, field: &Field) -> Self {
+ self.field_name = Some(field.name().to_owned());
+ self.nullable = field.is_nullable();
+ self.field_metadata = field.metadata().clone();
+ self
}
/// Build a single element [`ListArray`]
@@ -661,13 +665,14 @@ impl SingleRowListArrayBuilder {
arr,
nullable,
field_name,
+ field_metadata,
} = self;
let data_type = arr.data_type().to_owned();
let field = match field_name {
Some(name) => Field::new(name, data_type, nullable),
None => Field::new_list_field(data_type, nullable),
};
- (Arc::new(field), arr)
+ (Arc::new(field.with_metadata(field_metadata)), arr)
}
}
@@ -1610,6 +1615,7 @@ pub fn apply_parent_nulls(
#[cfg(test)]
mod tests {
+ use std::collections::HashMap;
use std::sync::Arc;
use super::*;
@@ -1621,6 +1627,20 @@ mod tests {
#[cfg(feature = "sql")]
use sqlparser::ast::Ident;
+ #[test]
+ fn test_list_scalar_cast_preserves_target_element_metadata() -> Result<()> {
+ let input =
+ ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![Some(1)])]);
+ let item = Field::new("item", DataType::Int32, true)
+ .with_metadata(HashMap::from([("PARQUET:field_id".into(), "2".into())]));
+ let target = DataType::List(Arc::new(item));
+
+ let casted = ScalarValue::List(Arc::new(input)).cast_to(&target)?;
+ assert_eq!(casted.data_type(), target);
+ assert_eq!(casted.to_array_of_size(2)?.data_type(), &target);
+ Ok(())
+ }
+
#[test]
fn test_apply_parent_nulls_sliced() -> Result<()> {
let child = Arc::new(