Skip to content

Apache DataFusion SQL Query Engine

feat: automatically cast `ListView` to `List` for UDFs (#21855)

Jeffrey Voe8a93bb3a629
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs
index de00dac825..8c88be03fd 100644
--- a/datafusion/common/src/utils/mod.rs
+++ b/datafusion/common/src/utils/mod.rs
@@ -599,11 +599,17 @@ pub fn base_type(data_type: &DataType) -> DataType {
match data_type {
DataType::List(field)
| DataType::LargeList(field)
+ | DataType::ListView(field)
+ | DataType::LargeListView(field)
| DataType::FixedSizeList(field, _) => base_type(field.data_type()),
_ => data_type.to_owned(),
}
}
+// TODO: Modify this to also allow specifying how listviews should be treated.
+// For example if cast to List (default) or maintain as ListView (requires
+// function to implement support for ListViews)
+// https://github.com/apache/datafusion/issues/21777
/// Information about how to coerce lists.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)]
pub enum ListCoercion {
@@ -657,6 +663,19 @@ pub fn coerced_type_with_base_type_only(
*len,
)
}
+ (DataType::ListView(field), _) => {
+ let field_type = coerced_type_with_base_type_only(
+ field.data_type(),
+ base_type,
+ array_coercion,
+ );
+
+ DataType::ListView(Arc::new(Field::new(
+ field.name(),
+ field_type,
+ field.is_nullable(),
+ )))
+ }
(DataType::LargeList(field), _) => {
let field_type = coerced_type_with_base_type_only(
field.data_type(),
@@ -670,6 +689,19 @@ pub fn coerced_type_with_base_type_only(
field.is_nullable(),
)))
}
+ (DataType::LargeListView(field), _) => {
+ let field_type = coerced_type_with_base_type_only(
+ field.data_type(),
+ base_type,
+ array_coercion,
+ );
+
+ DataType::LargeListView(Arc::new(Field::new(
+ field.name(),
+ field_type,
+ field.is_nullable(),
+ )))
+ }
_ => base_type.clone(),
}
@@ -687,6 +719,15 @@ pub fn coerced_fixed_size_list_to_list(data_type: &DataType) -> DataType {
field.is_nullable(),
)))
}
+ DataType::ListView(field) => {
+ let field_type = coerced_fixed_size_list_to_list(field.data_type());
+
+ DataType::ListView(Arc::new(Field::new(
+ field.name(),
+ field_type,
+ field.is_nullable(),
+ )))
+ }
DataType::LargeList(field) => {
let field_type = coerced_fixed_size_list_to_list(field.data_type());
@@ -696,6 +737,15 @@ pub fn coerced_fixed_size_list_to_list(data_type: &DataType) -> DataType {
field.is_nullable(),
)))
}
+ DataType::LargeListView(field) => {
+ let field_type = coerced_fixed_size_list_to_list(field.data_type());
+
+ DataType::LargeListView(Arc::new(Field::new(
+ field.name(),
+ field_type,
+ field.is_nullable(),
+ )))
+ }
_ => data_type.clone(),
}
@@ -706,6 +756,8 @@ pub fn list_ndims(data_type: &DataType) -> u64 {
match data_type {
DataType::List(field)
| DataType::LargeList(field)
+ | DataType::ListView(field)
+ | DataType::LargeListView(field)
| DataType::FixedSizeList(field, _) => 1 + list_ndims(field.data_type()),
_ => 0,
}
diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs
index b2d7a860eb..86616daf08 100644
--- a/datafusion/expr/src/type_coercion/functions.rs
+++ b/datafusion/expr/src/type_coercion/functions.rs
@@ -537,12 +537,12 @@ fn get_valid_types(
element_types.push(DataType::Null);
nested_item_nullability.push(None);
}
- DataType::List(field) => {
+ DataType::List(field) | DataType::ListView(field) => {
element_types.push(field.data_type().clone());
nested_item_nullability.push(Some(field.is_nullable()));
fixed_size = false;
}
- DataType::LargeList(field) => {
+ DataType::LargeList(field) | DataType::LargeListView(field) => {
element_types.push(field.data_type().clone());
nested_item_nullability.push(Some(field.is_nullable()));
large_list = true;
@@ -580,6 +580,8 @@ fn get_valid_types(
ArrayFunctionArgument::Index => DataType::Int64,
ArrayFunctionArgument::String => DataType::Utf8,
ArrayFunctionArgument::Element => element_type.clone(),
+ // TODO: support maintaining ListView types here
+ // https://github.com/apache/datafusion/issues/21777
ArrayFunctionArgument::Array => {
if current_type.is_null() {
DataType::Null
@@ -611,6 +613,8 @@ fn get_valid_types(
match array_type {
DataType::List(_)
| DataType::LargeList(_)
+ | DataType::ListView(_)
+ | DataType::LargeListView(_)
| DataType::FixedSizeList(_, _) => {
let array_type = coerced_fixed_size_list_to_list(array_type);
Some(array_type)
@@ -1044,7 +1048,7 @@ fn coerced_from<'a>(
// Only accept list and largelist with the same number of dimensions unless the type is Null.
// List or LargeList with different dimensions should be handled in TypeSignature or other places before this
- (List(_) | LargeList(_), _)
+ (List(_) | LargeList(_) | ListView(_) | LargeListView(_), _)
if base_type(type_from).is_null()
|| list_ndims(type_from) == list_ndims(type_into) =>
{
@@ -1495,6 +1499,54 @@ mod tests {
]]
);
+ let data_types = vec![
+ DataType::ListView(Field::new_list_field(DataType::Int32, true).into()),
+ DataType::new_list(DataType::Int32, true),
+ ];
+ assert_eq!(
+ get_valid_types(function, &signature.type_signature, &data_types)?,
+ vec![vec![
+ DataType::new_list(DataType::Int32, true),
+ DataType::new_list(DataType::Int32, true),
+ ]]
+ );
+
+ let data_types = vec![
+ DataType::LargeListView(Field::new_list_field(DataType::Int32, true).into()),
+ DataType::new_list(DataType::Int32, true),
+ ];
+ assert_eq!(
+ get_valid_types(function, &signature.type_signature, &data_types)?,
+ vec![vec![
+ DataType::new_large_list(DataType::Int32, true),
+ DataType::new_large_list(DataType::Int32, true),
+ ]]
+ );
+
+ let data_types = vec![
+ DataType::ListView(Field::new_list_field(DataType::Int32, true).into()),
+ DataType::ListView(Field::new_list_field(DataType::Int32, true).into()),
+ ];
+ assert_eq!(
+ get_valid_types(function, &signature.type_signature, &data_types)?,
+ vec![vec![
+ DataType::new_list(DataType::Int32, true),
+ DataType::new_list(DataType::Int32, true),
+ ]]
+ );
+
+ let data_types = vec![
+ DataType::LargeListView(Field::new_list_field(DataType::Int32, true).into()),
+ DataType::LargeListView(Field::new_list_field(DataType::Int32, true).into()),
+ ];
+ assert_eq!(
+ get_valid_types(function, &signature.type_signature, &data_types)?,
+ vec![vec![
+ DataType::new_large_list(DataType::Int32, true),
+ DataType::new_large_list(DataType::Int32, true),
+ ]]
+ );
+
Ok(())
}
diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rs
index 9f46917a87..eeff003e8e 100644
--- a/datafusion/functions-nested/src/utils.rs
+++ b/datafusion/functions-nested/src/utils.rs
@@ -26,7 +26,8 @@ use arrow::array::{
};
use arrow::buffer::OffsetBuffer;
use datafusion_common::cast::{
- as_fixed_size_list_array, as_large_list_array, as_list_array,
+ as_fixed_size_list_array, as_large_list_array, as_large_list_view_array,
+ as_list_array, as_list_view_array,
};
use datafusion_common::{Result, ScalarValue, exec_err, internal_err, plan_err};
@@ -243,6 +244,14 @@ pub(crate) fn compute_array_dims(
value = as_large_list_array(&value)?.value(0);
res.push(Some(value.len() as u64));
}
+ DataType::ListView(_) => {
+ value = as_list_view_array(&value)?.value(0);
+ res.push(Some(value.len() as u64));
+ }
+ DataType::LargeListView(_) => {
+ value = as_large_list_view_array(&value)?.value(0);
+ res.push(Some(value.len() as u64));
+ }
DataType::FixedSizeList(..) => {
value = as_fixed_size_list_array(&value)?.value(0);
res.push(Some(value.len() as u64));
diff --git a/datafusion/sqllogictest/test_files/array/array_empty.slt b/datafusion/sqllogictest/test_files/array/array_empty.slt
index ac9294015d..62ac5f66b7 100644
--- a/datafusion/sqllogictest/test_files/array/array_empty.slt
+++ b/datafusion/sqllogictest/test_files/array/array_empty.slt
@@ -176,5 +176,15 @@ select list_empty(arrow_cast(make_array(NULL), 'LargeList(Int64)'));
----
false
+query B
+select empty(arrow_cast([1], 'ListView(Int64)'));
+----
+false
+
+query B
+select empty(arrow_cast([1], 'LargeListView(Int64)'));
+----
+false
+
include ./cleanup.slt.part
diff --git a/datafusion/sqllogictest/test_files/array/array_pop.slt b/datafusion/sqllogictest/test_files/array/array_pop.slt
index 13dafff97d..b830fa464a 100644
--- a/datafusion/sqllogictest/test_files/array/array_pop.slt
+++ b/datafusion/sqllogictest/test_files/array/array_pop.slt
@@ -302,5 +302,25 @@ select array_pop_front(arrow_cast(make_array(make_array(1, 2, 3)), 'FixedSizeLis
----
[]
+query ?
+select array_pop_back(arrow_cast([1, 2], 'ListView(Int64)'));
+----
+[1]
+
+query ?
+select array_pop_front(arrow_cast([1, 2], 'ListView(Int64)'));
+----
+[2]
+
+query ?
+select array_pop_back(arrow_cast([1, 2], 'LargeListView(Int64)'));
+----
+[1]
+
+query ?
+select array_pop_front(arrow_cast([1, 2], 'LargeListView(Int64)'));
+----
+[2]
+
include ./cleanup.slt.part
diff --git a/datafusion/sqllogictest/test_files/array/array_reverse.slt b/datafusion/sqllogictest/test_files/array/array_reverse.slt
index 4f3213a440..e9d1f5cf54 100644
--- a/datafusion/sqllogictest/test_files/array/array_reverse.slt
+++ b/datafusion/sqllogictest/test_files/array/array_reverse.slt
@@ -164,12 +164,15 @@ select array_contains(a, b) from array_has order by 1 nulls last;
true
NULL
-# Expected output (once supported):
-# ----
-# [5, 4, 3, 2, 1]
-query error
-select array_reverse(arrow_cast(make_array(1, 2, 3, 4, 5), 'ListView(Int64)'));
+query ?
+select array_reverse(arrow_cast([1, 2, 3, 4, 5], 'ListView(Int64)'));
+----
+[5, 4, 3, 2, 1]
+query ?
+select array_reverse(arrow_cast([1, 2, 3, 4, 5], 'LargeListView(Int64)'));
+----
+[5, 4, 3, 2, 1]
statement ok
drop table test_create_array_table;
diff --git a/datafusion/sqllogictest/test_files/array/array_slice.slt b/datafusion/sqllogictest/test_files/array/array_slice.slt
index fa6b829276..14587a50b2 100644
--- a/datafusion/sqllogictest/test_files/array/array_slice.slt
+++ b/datafusion/sqllogictest/test_files/array/array_slice.slt
@@ -79,13 +79,11 @@ select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeList(Int64)'), 0,
----
[1, 2, 3, 4, 5] [h, e, l, l, o]
-# TODO make error message nicer: https://github.com/apache/datafusion/issues/19004
-# Expected output (once supported):
-# ----
-# [1, 2, 3, 4, 5] [h, e, l, l, o]
-query error Failed to coerce arguments to satisfy a call to 'array_slice' function:
-select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'ListView(Int64)'), 0, 6),
- array_slice(arrow_cast(make_array('h', 'e', 'l', 'l', 'o'), 'ListView(Utf8)'), 0, 5);
+query ??
+select array_slice(arrow_cast([1, 2, 3, 4, 5], 'ListView(Int64)'), 0, 6),
+ array_slice(arrow_cast(['h', 'e', 'l', 'l', 'o'], 'ListView(Utf8)'), 0, 5);
+----
+[1, 2, 3, 4, 5] [h, e, l, l, o]
query ??
select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'FixedSizeList(5, Int64)'), 0, 6),
@@ -126,13 +124,11 @@ select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeList(Int64)'), 2,
----
[2, 3, 4, 5] [l, l, o]
-# TODO: Enable once array_slice supports LargeListView types.
-# Expected output (once supported):
-# ----
-# [2, 3, 4, 5] [l, l, o]
-query error Failed to coerce arguments to satisfy a call to 'array_slice' function:
-select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeListView(Int64)'), 2, 6),
- array_slice(arrow_cast(make_array('h', 'e', 'l', 'l', 'o'), 'LargeListView(Utf8)'), 3, 7);
+query ??
+select array_slice(arrow_cast([1, 2, 3, 4, 5], 'LargeListView(Int64)'), 2, 6),
+ array_slice(arrow_cast(['h', 'e', 'l', 'l', 'o'], 'LargeListView(Utf8)'), 3, 7);
+----
+[2, 3, 4, 5] [l, l, o]
# array_slice scalar function #6 (with positive indexes; nested array)
diff --git a/datafusion/sqllogictest/test_files/array/array_sort.slt b/datafusion/sqllogictest/test_files/array/array_sort.slt
index 0e0d57834e..343aa5a82e 100644
--- a/datafusion/sqllogictest/test_files/array/array_sort.slt
+++ b/datafusion/sqllogictest/test_files/array/array_sort.slt
@@ -254,6 +254,15 @@ select list_sort(make_array(1, 3, null, 5, NULL, -5)), list_sort(make_array(1, 3
----
[NULL, NULL, -5, 1, 3, 5] [NULL, 1, 2, 3] [NULL, 3, 2, 1]
+query ?
+select array_sort(arrow_cast([1, 3, null, 5, NULL, -5], 'ListView(Int64)'));
+----
+[NULL, NULL, -5, 1, 3, 5]
+
+query ?
+select array_sort(arrow_cast([1, 3, null, 5, NULL, -5], 'LargeListView(Int64)'));
+----
+[NULL, NULL, -5, 1, 3, 5]
include ./cleanup.slt.part