Apache DataFusion SQL Query Engine
fix: avoid extraneous casts for equivalent nested types (#20945)
Guocheng(Eric) Song5b2285703680
diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rsindex c3802590bc..33746a2c46 100644--- a/datafusion/expr/src/type_coercion/functions.rs+++ b/datafusion/expr/src/type_coercion/functions.rs@@ -586,6 +586,52 @@ fn get_valid_types(arguments: &[ArrayFunctionArgument],array_coercion: Option<&ListCoercion>,) -> Result<Vec<Vec<DataType>>> {+ fn rebuild_array_type(+ current_type: &DataType,+ element_type: &DataType,+ nullable: bool,+ large_list: bool,+ fixed_size: Option<i32>,+ ) -> DataType {+ // Preserve the original list field when possible so field name or+ // metadata differences do not introduce otherwise unnecessary casts.+ let field = match current_type {+ DataType::List(field)+ | DataType::LargeList(field)+ | DataType::FixedSizeList(field, _) => Some(Arc::new(+ field+ .as_ref()+ .clone()+ .with_data_type(element_type.clone())+ .with_nullable(nullable),+ )),+ _ => None,+ };++ if large_list {+ field.map_or_else(+ || DataType::new_large_list(element_type.clone(), nullable),+ DataType::LargeList,+ )+ } else if let Some(size) = fixed_size {+ field.map_or_else(+ || {+ DataType::new_fixed_size_list(+ element_type.clone(),+ size,+ nullable,+ )+ },+ |field| DataType::FixedSizeList(field, size),+ )+ } else {+ field.map_or_else(+ || DataType::new_list(element_type.clone(), nullable),+ DataType::List,+ )+ }+ }+if current_types.len() != arguments.len() {return Ok(vec![vec![]]);}@@ -657,21 +703,13 @@ fn get_valid_types(ArrayFunctionArgument::Array => {if current_type.is_null() {DataType::Null- } else if large_list {- DataType::new_large_list(- element_type.clone(),- is_nested_item_nullable.unwrap_or(true),- )- } else if let Some(size) = list_sizes.next() {- DataType::new_fixed_size_list(- element_type.clone(),- size,- is_nested_item_nullable.unwrap_or(true),- )} else {- DataType::new_list(- element_type.clone(),+ rebuild_array_type(+ current_type,+ &element_type,is_nested_item_nullable.unwrap_or(true),+ large_list,+ list_sizes.next(),)}}@@ -1664,6 +1702,31 @@ mod tests {Ok(())}+ #[test]+ fn test_get_valid_types_array_and_index_preserves_list_field_name() -> Result<()> {+ let struct_fields = vec![+ Field::new("id", DataType::Utf8, true),+ Field::new("prim", DataType::Boolean, true),+ ];+ let current_type = DataType::List(Arc::new(Field::new(+ "element",+ DataType::Struct(struct_fields.into()),+ true,+ )));+ let signature = Signature::array_and_index(Volatility::Immutable);++ assert_eq!(+ get_valid_types(+ "array_element",+ &signature.type_signature,+ &[current_type.clone(), DataType::Int64],+ )?,+ vec![vec![current_type, DataType::Int64]]+ );++ Ok(())+ }+#[test]fn test_get_valid_types_element_and_array() -> Result<()> {let function = "element_and_array";diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex 7b81feab47..df3ccc2825 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -1772,6 +1772,31 @@ mod test {}}+ #[derive(Debug, Hash, PartialEq, Eq)]+ struct TestArrayElementUDF;++ impl ScalarUDFImpl for TestArrayElementUDF {+ fn name(&self) -> &str {+ "TestArrayElementUDF"+ }++ fn signature(&self) -> &Signature {+ static SIGNATURE: std::sync::LazyLock<Signature> =+ std::sync::LazyLock::new(|| {+ Signature::array_and_index(Volatility::Immutable)+ });+ &SIGNATURE+ }++ fn return_type(&self, _args: &[DataType]) -> Result<DataType> {+ Ok(Utf8)+ }++ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ Ok(ColumnarValue::Scalar(ScalarValue::from("a")))+ }+ }+#[test]fn scalar_udf() -> Result<()> {let empty = empty();@@ -2669,6 +2694,33 @@ mod test {)}+ #[test]+ fn array_element_preserves_parquet_list_field_name() -> Result<()> {+ let list_type = DataType::List(Arc::new(Field::new(+ "element",+ DataType::Struct(+ vec![+ Field::new("id", Utf8, true),+ Field::new("prim", DataType::Boolean, true),+ ]+ .into(),+ ),+ true,+ )));++ let expr = ScalarUDF::from(TestArrayElementUDF).call(vec![col("a"), lit(1_i64)]);+ let empty = empty_with_type(list_type);+ let plan = LogicalPlan::Projection(Projection::try_new(vec![expr], empty)?);++ assert_analyzed_plan_eq!(+ plan,+ @r#"+ Projection: TestArrayElementUDF(a, Int64(1))+ EmptyRelation: rows=0+ "#+ )+ }+#[test]fn interval_plus_timestamp() -> Result<()> {// SELECT INTERVAL '1' YEAR + '2000-01-01T00:00:00'::timestamp;