Apache DataFusion SQL Query Engine
fix: cardinality returns incorrect results for ragged nested arrays (#23271)
linfeng55292fb62337
diff --git a/datafusion/functions-nested/src/cardinality.rs b/datafusion/functions-nested/src/cardinality.rsindex d21bb72a45..38def2d4e4 100644--- a/datafusion/functions-nested/src/cardinality.rs+++ b/datafusion/functions-nested/src/cardinality.rs@@ -23,10 +23,15 @@ use arrow::array::{};use arrow::datatypes::{DataType,- DataType::{LargeList, List, Map, Null, UInt64},+ DataType::{+ FixedSizeList, LargeList, LargeListView, List, ListView, Map, Null, UInt64,+ },};use datafusion_common::Result;-use datafusion_common::cast::{as_large_list_array, as_list_array, as_map_array};+use datafusion_common::cast::{+ as_fixed_size_list_array, as_large_list_array, as_large_list_view_array,+ as_list_array, as_list_view_array, as_map_array,+};use datafusion_common::exec_err;use datafusion_common::utils::{ListCoercion, take_function_args};use datafusion_expr::{@@ -146,14 +151,50 @@ fn generic_list_cardinality<O: OffsetSizeTrait>(let result = array.iter().map(|arr| match arr {- Some(arr) if arr.is_empty() => Ok(Some(0u64)),- arr => match crate::utils::compute_array_dims(arr)? {- Some(vector) => {- Ok(Some(vector.iter().map(|x| x.unwrap()).product::<u64>()))- }- None => Ok(None),- },+ Some(arr) => value_cardinality(&arr).map(Some),+ None => Ok(None),}).collect::<Result<UInt64Array>>()?;Ok(Arc::new(result) as ArrayRef)}++fn value_cardinality(array: &ArrayRef) -> Result<u64> {+ match array.data_type() {+ List(_) => {+ let list = as_list_array(&array)?;+ sum_list_cardinality(list.iter())+ }+ LargeList(_) => {+ let list = as_large_list_array(&array)?;+ sum_list_cardinality(list.iter())+ }+ ListView(_) => {+ let list = as_list_view_array(&array)?;+ sum_list_cardinality(list.iter())+ }+ LargeListView(_) => {+ let list = as_large_list_view_array(&array)?;+ sum_list_cardinality(list.iter())+ }+ FixedSizeList(..) => {+ let list = as_fixed_size_list_array(&array)?;+ sum_list_cardinality(list.iter())+ }+ _ => Ok(array.len() as u64),+ }+}++fn sum_list_cardinality<I>(mut iter: I) -> Result<u64>+where+ I: Iterator<Item = Option<ArrayRef>>,+{+ iter.try_fold(0u64, |total, arr| {+ let value_count = match arr {+ Some(arr) => value_cardinality(&arr)?,+ None => 0,+ };+ total.checked_add(value_count).ok_or_else(|| {+ datafusion_common::exec_datafusion_err!("cardinality overflowed u64")+ })+ })+}diff --git a/datafusion/sqllogictest/test_files/array/cardinality.slt b/datafusion/sqllogictest/test_files/array/cardinality.sltindex 52b1a2b544..c0ad54e97a 100644--- a/datafusion/sqllogictest/test_files/array/cardinality.slt+++ b/datafusion/sqllogictest/test_files/array/cardinality.slt@@ -51,6 +51,37 @@ select cardinality(arrow_cast([[1, 2], [3, 4], [5, 6]], 'FixedSizeList(3, List(I----6+# cardinality counts actual leaf elements in ragged nested arrays+query III+select cardinality([[1], [2, 3]]),+ cardinality([[1, 2, 3], []]),+ cardinality([[], [1, 2]]);+----+3 3 2++query IIII+select cardinality(arrow_cast([[1], [2, 3]], 'ListView(List(Int64))')),+ cardinality(arrow_cast([[1], [2, 3]], 'LargeListView(List(Int64))')),+ cardinality(arrow_cast([[1, 2], [3, 4]], 'List(FixedSizeList(2, Int64))')),+ cardinality(arrow_cast([[1], [2, 3]], 'LargeList(List(Int64))'));+----+3 3 4 3++query III+select cardinality(arrow_cast([[[1]], [[2, 3], []]], 'List(ListView(List(Int64)))')),+ cardinality(arrow_cast([[[1]], [[2, 3], []]], 'List(LargeListView(List(Int64)))')),+ cardinality(arrow_cast([[[1, 2]], [[3, 4]]], 'List(List(FixedSizeList(2, Int64)))'));+----+3 3 4++query IIII+select cardinality([[NULL], [1, 2]]),+ cardinality([[[1]], [[2, 3], []]]),+ cardinality(arrow_cast([[], [1, 2]], 'LargeList(List(Int64))')),+ cardinality(make_array(NULL::int[], [1, 2]));+----+3 3 2 2+# cardinality scalar function #3query IIselect cardinality(make_array()), cardinality(make_array(make_array()))