Skip to content

Apache DataFusion SQL Query Engine

fix: cardinality returns incorrect results for ragged nested arrays (#23271)

linfeng55292fb62337
diff --git a/datafusion/functions-nested/src/cardinality.rs b/datafusion/functions-nested/src/cardinality.rs
index d21bb72a45..38def2d4e4 100644
--- a/datafusion/functions-nested/src/cardinality.rs
+++ b/datafusion/functions-nested/src/cardinality.rs
@@ -23,10 +23,15 @@ use arrow::array::{
};
use arrow::datatypes::{
DataType,
- DataType::{LargeList, List, Map, Null, UInt64},
+ DataType::{
+ FixedSizeList, LargeList, LargeListView, List, ListView, Map, Null, UInt64,
+ },
};
use datafusion_common::Result;
-use datafusion_common::cast::{as_large_list_array, as_list_array, as_map_array};
+use datafusion_common::cast::{
+ as_fixed_size_list_array, as_large_list_array, as_large_list_view_array,
+ as_list_array, as_list_view_array, as_map_array,
+};
use datafusion_common::exec_err;
use datafusion_common::utils::{ListCoercion, take_function_args};
use datafusion_expr::{
@@ -146,14 +151,50 @@ fn generic_list_cardinality<O: OffsetSizeTrait>(
let result = array
.iter()
.map(|arr| match arr {
- Some(arr) if arr.is_empty() => Ok(Some(0u64)),
- arr => match crate::utils::compute_array_dims(arr)? {
- Some(vector) => {
- Ok(Some(vector.iter().map(|x| x.unwrap()).product::<u64>()))
- }
- None => Ok(None),
- },
+ Some(arr) => value_cardinality(&arr).map(Some),
+ None => Ok(None),
})
.collect::<Result<UInt64Array>>()?;
Ok(Arc::new(result) as ArrayRef)
}
+
+fn value_cardinality(array: &ArrayRef) -> Result<u64> {
+ match array.data_type() {
+ List(_) => {
+ let list = as_list_array(&array)?;
+ sum_list_cardinality(list.iter())
+ }
+ LargeList(_) => {
+ let list = as_large_list_array(&array)?;
+ sum_list_cardinality(list.iter())
+ }
+ ListView(_) => {
+ let list = as_list_view_array(&array)?;
+ sum_list_cardinality(list.iter())
+ }
+ LargeListView(_) => {
+ let list = as_large_list_view_array(&array)?;
+ sum_list_cardinality(list.iter())
+ }
+ FixedSizeList(..) => {
+ let list = as_fixed_size_list_array(&array)?;
+ sum_list_cardinality(list.iter())
+ }
+ _ => Ok(array.len() as u64),
+ }
+}
+
+fn sum_list_cardinality<I>(mut iter: I) -> Result<u64>
+where
+ I: Iterator<Item = Option<ArrayRef>>,
+{
+ iter.try_fold(0u64, |total, arr| {
+ let value_count = match arr {
+ Some(arr) => value_cardinality(&arr)?,
+ None => 0,
+ };
+ total.checked_add(value_count).ok_or_else(|| {
+ datafusion_common::exec_datafusion_err!("cardinality overflowed u64")
+ })
+ })
+}
diff --git a/datafusion/sqllogictest/test_files/array/cardinality.slt b/datafusion/sqllogictest/test_files/array/cardinality.slt
index 52b1a2b544..c0ad54e97a 100644
--- a/datafusion/sqllogictest/test_files/array/cardinality.slt
+++ b/datafusion/sqllogictest/test_files/array/cardinality.slt
@@ -51,6 +51,37 @@ select cardinality(arrow_cast([[1, 2], [3, 4], [5, 6]], 'FixedSizeList(3, List(I
----
6
+# cardinality counts actual leaf elements in ragged nested arrays
+query III
+select cardinality([[1], [2, 3]]),
+ cardinality([[1, 2, 3], []]),
+ cardinality([[], [1, 2]]);
+----
+3 3 2
+
+query IIII
+select cardinality(arrow_cast([[1], [2, 3]], 'ListView(List(Int64))')),
+ cardinality(arrow_cast([[1], [2, 3]], 'LargeListView(List(Int64))')),
+ cardinality(arrow_cast([[1, 2], [3, 4]], 'List(FixedSizeList(2, Int64))')),
+ cardinality(arrow_cast([[1], [2, 3]], 'LargeList(List(Int64))'));
+----
+3 3 4 3
+
+query III
+select cardinality(arrow_cast([[[1]], [[2, 3], []]], 'List(ListView(List(Int64)))')),
+ cardinality(arrow_cast([[[1]], [[2, 3], []]], 'List(LargeListView(List(Int64)))')),
+ cardinality(arrow_cast([[[1, 2]], [[3, 4]]], 'List(List(FixedSizeList(2, Int64)))'));
+----
+3 3 4
+
+query IIII
+select cardinality([[NULL], [1, 2]]),
+ cardinality([[[1]], [[2, 3], []]]),
+ cardinality(arrow_cast([[], [1, 2]], 'LargeList(List(Int64))')),
+ cardinality(make_array(NULL::int[], [1, 2]));
+----
+3 3 2 2
+
# cardinality scalar function #3
query II
select cardinality(make_array()), cardinality(make_array(make_array()))