Skip to content

Apache DataFusion SQL Query Engine

fix: avoid extraneous casts for equivalent nested types (#20945)

Guocheng(Eric) Song5b2285703680
diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs
index c3802590bc..33746a2c46 100644
--- a/datafusion/expr/src/type_coercion/functions.rs
+++ b/datafusion/expr/src/type_coercion/functions.rs
@@ -586,6 +586,52 @@ fn get_valid_types(
arguments: &[ArrayFunctionArgument],
array_coercion: Option<&ListCoercion>,
) -> Result<Vec<Vec<DataType>>> {
+ fn rebuild_array_type(
+ current_type: &DataType,
+ element_type: &DataType,
+ nullable: bool,
+ large_list: bool,
+ fixed_size: Option<i32>,
+ ) -> DataType {
+ // Preserve the original list field when possible so field name or
+ // metadata differences do not introduce otherwise unnecessary casts.
+ let field = match current_type {
+ DataType::List(field)
+ | DataType::LargeList(field)
+ | DataType::FixedSizeList(field, _) => Some(Arc::new(
+ field
+ .as_ref()
+ .clone()
+ .with_data_type(element_type.clone())
+ .with_nullable(nullable),
+ )),
+ _ => None,
+ };
+
+ if large_list {
+ field.map_or_else(
+ || DataType::new_large_list(element_type.clone(), nullable),
+ DataType::LargeList,
+ )
+ } else if let Some(size) = fixed_size {
+ field.map_or_else(
+ || {
+ DataType::new_fixed_size_list(
+ element_type.clone(),
+ size,
+ nullable,
+ )
+ },
+ |field| DataType::FixedSizeList(field, size),
+ )
+ } else {
+ field.map_or_else(
+ || DataType::new_list(element_type.clone(), nullable),
+ DataType::List,
+ )
+ }
+ }
+
if current_types.len() != arguments.len() {
return Ok(vec![vec![]]);
}
@@ -657,21 +703,13 @@ fn get_valid_types(
ArrayFunctionArgument::Array => {
if current_type.is_null() {
DataType::Null
- } else if large_list {
- DataType::new_large_list(
- element_type.clone(),
- is_nested_item_nullable.unwrap_or(true),
- )
- } else if let Some(size) = list_sizes.next() {
- DataType::new_fixed_size_list(
- element_type.clone(),
- size,
- is_nested_item_nullable.unwrap_or(true),
- )
} else {
- DataType::new_list(
- element_type.clone(),
+ rebuild_array_type(
+ current_type,
+ &element_type,
is_nested_item_nullable.unwrap_or(true),
+ large_list,
+ list_sizes.next(),
)
}
}
@@ -1664,6 +1702,31 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_get_valid_types_array_and_index_preserves_list_field_name() -> Result<()> {
+ let struct_fields = vec![
+ Field::new("id", DataType::Utf8, true),
+ Field::new("prim", DataType::Boolean, true),
+ ];
+ let current_type = DataType::List(Arc::new(Field::new(
+ "element",
+ DataType::Struct(struct_fields.into()),
+ true,
+ )));
+ let signature = Signature::array_and_index(Volatility::Immutable);
+
+ assert_eq!(
+ get_valid_types(
+ "array_element",
+ &signature.type_signature,
+ &[current_type.clone(), DataType::Int64],
+ )?,
+ vec![vec![current_type, DataType::Int64]]
+ );
+
+ Ok(())
+ }
+
#[test]
fn test_get_valid_types_element_and_array() -> Result<()> {
let function = "element_and_array";
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index 7b81feab47..df3ccc2825 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -1772,6 +1772,31 @@ mod test {
}
}
+ #[derive(Debug, Hash, PartialEq, Eq)]
+ struct TestArrayElementUDF;
+
+ impl ScalarUDFImpl for TestArrayElementUDF {
+ fn name(&self) -> &str {
+ "TestArrayElementUDF"
+ }
+
+ fn signature(&self) -> &Signature {
+ static SIGNATURE: std::sync::LazyLock<Signature> =
+ std::sync::LazyLock::new(|| {
+ Signature::array_and_index(Volatility::Immutable)
+ });
+ &SIGNATURE
+ }
+
+ fn return_type(&self, _args: &[DataType]) -> Result<DataType> {
+ Ok(Utf8)
+ }
+
+ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ Ok(ColumnarValue::Scalar(ScalarValue::from("a")))
+ }
+ }
+
#[test]
fn scalar_udf() -> Result<()> {
let empty = empty();
@@ -2669,6 +2694,33 @@ mod test {
)
}
+ #[test]
+ fn array_element_preserves_parquet_list_field_name() -> Result<()> {
+ let list_type = DataType::List(Arc::new(Field::new(
+ "element",
+ DataType::Struct(
+ vec![
+ Field::new("id", Utf8, true),
+ Field::new("prim", DataType::Boolean, true),
+ ]
+ .into(),
+ ),
+ true,
+ )));
+
+ let expr = ScalarUDF::from(TestArrayElementUDF).call(vec![col("a"), lit(1_i64)]);
+ let empty = empty_with_type(list_type);
+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);
+
+ assert_analyzed_plan_eq!(
+ plan,
+ @r#"
+ Projection: TestArrayElementUDF(a, Int64(1))
+ EmptyRelation: rows=0
+ "#
+ )
+ }
+
#[test]
fn interval_plus_timestamp() -> Result<()> {
// SELECT INTERVAL '1' YEAR + '2000-01-01T00:00:00'::timestamp;