Apache DataFusion SQL Query Engine
feat: automatically cast `ListView` to `List` for UDFs (#21855)
Jeffrey Voe8a93bb3a629
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex de00dac825..8c88be03fd 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -599,11 +599,17 @@ pub fn base_type(data_type: &DataType) -> DataType {match data_type {DataType::List(field)| DataType::LargeList(field)+ | DataType::ListView(field)+ | DataType::LargeListView(field)| DataType::FixedSizeList(field, _) => base_type(field.data_type()),_ => data_type.to_owned(),}}+// TODO: Modify this to also allow specifying how listviews should be treated.+// For example if cast to List (default) or maintain as ListView (requires+// function to implement support for ListViews)+// https://github.com/apache/datafusion/issues/21777/// Information about how to coerce lists.#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)]pub enum ListCoercion {@@ -657,6 +663,19 @@ pub fn coerced_type_with_base_type_only(*len,)}+ (DataType::ListView(field), _) => {+ let field_type = coerced_type_with_base_type_only(+ field.data_type(),+ base_type,+ array_coercion,+ );++ DataType::ListView(Arc::new(Field::new(+ field.name(),+ field_type,+ field.is_nullable(),+ )))+ }(DataType::LargeList(field), _) => {let field_type = coerced_type_with_base_type_only(field.data_type(),@@ -670,6 +689,19 @@ pub fn coerced_type_with_base_type_only(field.is_nullable(),)))}+ (DataType::LargeListView(field), _) => {+ let field_type = coerced_type_with_base_type_only(+ field.data_type(),+ base_type,+ array_coercion,+ );++ DataType::LargeListView(Arc::new(Field::new(+ field.name(),+ field_type,+ field.is_nullable(),+ )))+ }_ => base_type.clone(),}@@ -687,6 +719,15 @@ pub fn coerced_fixed_size_list_to_list(data_type: &DataType) -> DataType {field.is_nullable(),)))}+ DataType::ListView(field) => {+ let field_type = coerced_fixed_size_list_to_list(field.data_type());++ DataType::ListView(Arc::new(Field::new(+ field.name(),+ field_type,+ field.is_nullable(),+ )))+ }DataType::LargeList(field) => {let field_type = coerced_fixed_size_list_to_list(field.data_type());@@ -696,6 +737,15 @@ pub fn coerced_fixed_size_list_to_list(data_type: &DataType) -> DataType {field.is_nullable(),)))}+ DataType::LargeListView(field) => {+ let field_type = coerced_fixed_size_list_to_list(field.data_type());++ DataType::LargeListView(Arc::new(Field::new(+ field.name(),+ field_type,+ field.is_nullable(),+ )))+ }_ => data_type.clone(),}@@ -706,6 +756,8 @@ pub fn list_ndims(data_type: &DataType) -> u64 {match data_type {DataType::List(field)| DataType::LargeList(field)+ | DataType::ListView(field)+ | DataType::LargeListView(field)| DataType::FixedSizeList(field, _) => 1 + list_ndims(field.data_type()),_ => 0,}diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rsindex b2d7a860eb..86616daf08 100644--- a/datafusion/expr/src/type_coercion/functions.rs+++ b/datafusion/expr/src/type_coercion/functions.rs@@ -537,12 +537,12 @@ fn get_valid_types(element_types.push(DataType::Null);nested_item_nullability.push(None);}- DataType::List(field) => {+ DataType::List(field) | DataType::ListView(field) => {element_types.push(field.data_type().clone());nested_item_nullability.push(Some(field.is_nullable()));fixed_size = false;}- DataType::LargeList(field) => {+ DataType::LargeList(field) | DataType::LargeListView(field) => {element_types.push(field.data_type().clone());nested_item_nullability.push(Some(field.is_nullable()));large_list = true;@@ -580,6 +580,8 @@ fn get_valid_types(ArrayFunctionArgument::Index => DataType::Int64,ArrayFunctionArgument::String => DataType::Utf8,ArrayFunctionArgument::Element => element_type.clone(),+ // TODO: support maintaining ListView types here+ // https://github.com/apache/datafusion/issues/21777ArrayFunctionArgument::Array => {if current_type.is_null() {DataType::Null@@ -611,6 +613,8 @@ fn get_valid_types(match array_type {DataType::List(_)| DataType::LargeList(_)+ | DataType::ListView(_)+ | DataType::LargeListView(_)| DataType::FixedSizeList(_, _) => {let array_type = coerced_fixed_size_list_to_list(array_type);Some(array_type)@@ -1044,7 +1048,7 @@ fn coerced_from<'a>(// Only accept list and largelist with the same number of dimensions unless the type is Null.// List or LargeList with different dimensions should be handled in TypeSignature or other places before this- (List(_) | LargeList(_), _)+ (List(_) | LargeList(_) | ListView(_) | LargeListView(_), _)if base_type(type_from).is_null()|| list_ndims(type_from) == list_ndims(type_into) =>{@@ -1495,6 +1499,54 @@ mod tests {]]);+ let data_types = vec![+ DataType::ListView(Field::new_list_field(DataType::Int32, true).into()),+ DataType::new_list(DataType::Int32, true),+ ];+ assert_eq!(+ get_valid_types(function, &signature.type_signature, &data_types)?,+ vec![vec![+ DataType::new_list(DataType::Int32, true),+ DataType::new_list(DataType::Int32, true),+ ]]+ );++ let data_types = vec![+ DataType::LargeListView(Field::new_list_field(DataType::Int32, true).into()),+ DataType::new_list(DataType::Int32, true),+ ];+ assert_eq!(+ get_valid_types(function, &signature.type_signature, &data_types)?,+ vec![vec![+ DataType::new_large_list(DataType::Int32, true),+ DataType::new_large_list(DataType::Int32, true),+ ]]+ );++ let data_types = vec![+ DataType::ListView(Field::new_list_field(DataType::Int32, true).into()),+ DataType::ListView(Field::new_list_field(DataType::Int32, true).into()),+ ];+ assert_eq!(+ get_valid_types(function, &signature.type_signature, &data_types)?,+ vec![vec![+ DataType::new_list(DataType::Int32, true),+ DataType::new_list(DataType::Int32, true),+ ]]+ );++ let data_types = vec![+ DataType::LargeListView(Field::new_list_field(DataType::Int32, true).into()),+ DataType::LargeListView(Field::new_list_field(DataType::Int32, true).into()),+ ];+ assert_eq!(+ get_valid_types(function, &signature.type_signature, &data_types)?,+ vec![vec![+ DataType::new_large_list(DataType::Int32, true),+ DataType::new_large_list(DataType::Int32, true),+ ]]+ );+Ok(())}diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rsindex 9f46917a87..eeff003e8e 100644--- a/datafusion/functions-nested/src/utils.rs+++ b/datafusion/functions-nested/src/utils.rs@@ -26,7 +26,8 @@ use arrow::array::{};use arrow::buffer::OffsetBuffer;use datafusion_common::cast::{- as_fixed_size_list_array, as_large_list_array, as_list_array,+ as_fixed_size_list_array, as_large_list_array, as_large_list_view_array,+ as_list_array, as_list_view_array,};use datafusion_common::{Result, ScalarValue, exec_err, internal_err, plan_err};@@ -243,6 +244,14 @@ pub(crate) fn compute_array_dims(value = as_large_list_array(&value)?.value(0);res.push(Some(value.len() as u64));}+ DataType::ListView(_) => {+ value = as_list_view_array(&value)?.value(0);+ res.push(Some(value.len() as u64));+ }+ DataType::LargeListView(_) => {+ value = as_large_list_view_array(&value)?.value(0);+ res.push(Some(value.len() as u64));+ }DataType::FixedSizeList(..) => {value = as_fixed_size_list_array(&value)?.value(0);res.push(Some(value.len() as u64));diff --git a/datafusion/sqllogictest/test_files/array/array_empty.slt b/datafusion/sqllogictest/test_files/array/array_empty.sltindex ac9294015d..62ac5f66b7 100644--- a/datafusion/sqllogictest/test_files/array/array_empty.slt+++ b/datafusion/sqllogictest/test_files/array/array_empty.slt@@ -176,5 +176,15 @@ select list_empty(arrow_cast(make_array(NULL), 'LargeList(Int64)'));----false+query B+select empty(arrow_cast([1], 'ListView(Int64)'));+----+false++query B+select empty(arrow_cast([1], 'LargeListView(Int64)'));+----+false+include ./cleanup.slt.partdiff --git a/datafusion/sqllogictest/test_files/array/array_pop.slt b/datafusion/sqllogictest/test_files/array/array_pop.sltindex 13dafff97d..b830fa464a 100644--- a/datafusion/sqllogictest/test_files/array/array_pop.slt+++ b/datafusion/sqllogictest/test_files/array/array_pop.slt@@ -302,5 +302,25 @@ select array_pop_front(arrow_cast(make_array(make_array(1, 2, 3)), 'FixedSizeLis----[]+query ?+select array_pop_back(arrow_cast([1, 2], 'ListView(Int64)'));+----+[1]++query ?+select array_pop_front(arrow_cast([1, 2], 'ListView(Int64)'));+----+[2]++query ?+select array_pop_back(arrow_cast([1, 2], 'LargeListView(Int64)'));+----+[1]++query ?+select array_pop_front(arrow_cast([1, 2], 'LargeListView(Int64)'));+----+[2]+include ./cleanup.slt.partdiff --git a/datafusion/sqllogictest/test_files/array/array_reverse.slt b/datafusion/sqllogictest/test_files/array/array_reverse.sltindex 4f3213a440..e9d1f5cf54 100644--- a/datafusion/sqllogictest/test_files/array/array_reverse.slt+++ b/datafusion/sqllogictest/test_files/array/array_reverse.slt@@ -164,12 +164,15 @@ select array_contains(a, b) from array_has order by 1 nulls last;trueNULL-# Expected output (once supported):-# -----# [5, 4, 3, 2, 1]-query error-select array_reverse(arrow_cast(make_array(1, 2, 3, 4, 5), 'ListView(Int64)'));+query ?+select array_reverse(arrow_cast([1, 2, 3, 4, 5], 'ListView(Int64)'));+----+[5, 4, 3, 2, 1]+query ?+select array_reverse(arrow_cast([1, 2, 3, 4, 5], 'LargeListView(Int64)'));+----+[5, 4, 3, 2, 1]statement okdrop table test_create_array_table;diff --git a/datafusion/sqllogictest/test_files/array/array_slice.slt b/datafusion/sqllogictest/test_files/array/array_slice.sltindex fa6b829276..14587a50b2 100644--- a/datafusion/sqllogictest/test_files/array/array_slice.slt+++ b/datafusion/sqllogictest/test_files/array/array_slice.slt@@ -79,13 +79,11 @@ select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeList(Int64)'), 0,----[1, 2, 3, 4, 5] [h, e, l, l, o]-# TODO make error message nicer: https://github.com/apache/datafusion/issues/19004-# Expected output (once supported):-# -----# [1, 2, 3, 4, 5] [h, e, l, l, o]-query error Failed to coerce arguments to satisfy a call to 'array_slice' function:-select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'ListView(Int64)'), 0, 6),- array_slice(arrow_cast(make_array('h', 'e', 'l', 'l', 'o'), 'ListView(Utf8)'), 0, 5);+query ??+select array_slice(arrow_cast([1, 2, 3, 4, 5], 'ListView(Int64)'), 0, 6),+ array_slice(arrow_cast(['h', 'e', 'l', 'l', 'o'], 'ListView(Utf8)'), 0, 5);+----+[1, 2, 3, 4, 5] [h, e, l, l, o]query ??select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'FixedSizeList(5, Int64)'), 0, 6),@@ -126,13 +124,11 @@ select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeList(Int64)'), 2,----[2, 3, 4, 5] [l, l, o]-# TODO: Enable once array_slice supports LargeListView types.-# Expected output (once supported):-# -----# [2, 3, 4, 5] [l, l, o]-query error Failed to coerce arguments to satisfy a call to 'array_slice' function:-select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeListView(Int64)'), 2, 6),- array_slice(arrow_cast(make_array('h', 'e', 'l', 'l', 'o'), 'LargeListView(Utf8)'), 3, 7);+query ??+select array_slice(arrow_cast([1, 2, 3, 4, 5], 'LargeListView(Int64)'), 2, 6),+ array_slice(arrow_cast(['h', 'e', 'l', 'l', 'o'], 'LargeListView(Utf8)'), 3, 7);+----+[2, 3, 4, 5] [l, l, o]# array_slice scalar function #6 (with positive indexes; nested array)diff --git a/datafusion/sqllogictest/test_files/array/array_sort.slt b/datafusion/sqllogictest/test_files/array/array_sort.sltindex 0e0d57834e..343aa5a82e 100644--- a/datafusion/sqllogictest/test_files/array/array_sort.slt+++ b/datafusion/sqllogictest/test_files/array/array_sort.slt@@ -254,6 +254,15 @@ select list_sort(make_array(1, 3, null, 5, NULL, -5)), list_sort(make_array(1, 3----[NULL, NULL, -5, 1, 3, 5] [NULL, 1, 2, 3] [NULL, 3, 2, 1]+query ?+select array_sort(arrow_cast([1, 3, null, 5, NULL, -5], 'ListView(Int64)'));+----+[NULL, NULL, -5, 1, 3, 5]++query ?+select array_sort(arrow_cast([1, 3, null, 5, NULL, -5], 'LargeListView(Int64)'));+----+[NULL, NULL, -5, 1, 3, 5]include ./cleanup.slt.part