Apache DataFusion SQL Query Engine
feat: Support List/ListView types in approx_distinct (#23443)
Michael Kleencf479db6c3b8
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex 74bc9ad6cb..15df82a79a 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -837,6 +837,11 @@ impl AggregateUDFImpl for ApproxDistinct {| DataType::Binary| DataType::BinaryView| DataType::FixedSizeBinary(_)+ | DataType::List(_)+ | DataType::LargeList(_)+ | DataType::FixedSizeList(_, _)+ | DataType::ListView(_)+ | DataType::LargeListView(_)| DataType::LargeBinary => Box::new(HLLAccumulator::new()),DataType::Null => {Box::new(NoopAccumulator::new(ScalarValue::UInt64(Some(0))))@@ -908,6 +913,11 @@ fn is_hll_groups_type(data_type: &DataType) -> bool {| DataType::BinaryView| DataType::FixedSizeBinary(_)| DataType::LargeBinary+ | DataType::List(_)+ | DataType::LargeList(_)+ | DataType::FixedSizeList(_, _)+ | DataType::ListView(_)+ | DataType::LargeListView(_))}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex c5970bde9c..e07c9e1b73 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -1969,6 +1969,132 @@ SELECT g, approx_distinct(arrow_cast(arrow_cast(s, 'Binary'), 'FixedSizeBinary(14 1+# List+statement ok+CREATE TABLE approx_distinct_list_test (g INT, l INT[]) AS VALUES+ (1, [1, 2]), (1, [1, 2]), (1, [3, 4]),+ (2, [5, 6]), (2, NULL),+ (3, NULL), (3, NULL),+ (4, [7, 8]);++# List non-grouped+query I+SELECT approx_distinct(l) FROM approx_distinct_list_test WHERE g = 1;+----+2++# List grouped+# Group 1 -> {[1,2],[3,4]}=2,+# Group 2 -> {[5,6]}=1 (NULL excluded),+# Group 3 -> all null=0,+# Group 4 -> {[7,8]}=1+query II+SELECT g, approx_distinct(l) FROM approx_distinct_list_test GROUP BY g ORDER BY g;+----+1 2+2 1+3 0+4 1++# The non-group path must agree with the grouped path on+# the same data, and distinct lists across groups are still counted overall.+query I+SELECT approx_distinct(l) FROM approx_distinct_list_test;+----+4++# LargeList non-grouped+query I+SELECT approx_distinct(arrow_cast(l, 'LargeList(Int32)')) FROM approx_distinct_list_test WHERE g = 1;+----+2++# LargeList grouped+query II+SELECT g, approx_distinct(arrow_cast(l, 'LargeList(Int32)')) FROM approx_distinct_list_test GROUP BY g ORDER BY g;+----+1 2+2 1+3 0+4 1++# The non-group path must agree with the grouped path on+# the same data, and distinct lists across groups are still counted overall.+query I+SELECT approx_distinct(arrow_cast(l, 'LargeList(Int32)')) FROM approx_distinct_list_test;+----+4++# ListView non-grouped+query I+SELECT approx_distinct(arrow_cast(l, 'ListView(Int32)')) FROM approx_distinct_list_test WHERE g = 1;+----+2++# ListView grouped+query II+SELECT g, approx_distinct(arrow_cast(l, 'ListView(Int32)')) FROM approx_distinct_list_test GROUP BY g ORDER BY g;+----+1 2+2 1+3 0+4 1++# The non-group path must agree with the grouped path on+# the same data, and distinct lists across groups are still counted overall.+query I+SELECT approx_distinct(arrow_cast(l, 'ListView(Int32)')) FROM approx_distinct_list_test;+----+4++# LargeListView non-grouped+query I+SELECT approx_distinct(arrow_cast(l, 'LargeListView(Int32)')) FROM approx_distinct_list_test WHERE g = 1;+----+2++# LargeListView grouped+query II+SELECT g, approx_distinct(arrow_cast(l, 'LargeListView(Int32)')) FROM approx_distinct_list_test GROUP BY g ORDER BY g;+----+1 2+2 1+3 0+4 1++# The non-group path must agree with the grouped path on+# the same data, and distinct lists across groups are still counted overall.+query I+SELECT approx_distinct(arrow_cast(l, 'LargeListView(Int32)')) FROM approx_distinct_list_test;+----+4+++# FixedSizeList non-grouped+query I+SELECT approx_distinct(arrow_cast(l, 'FixedSizeList(2, Int32)')) FROM approx_distinct_list_test WHERE g = 1;+----+2++# FixedSizeList grouped+query II+SELECT g, approx_distinct(arrow_cast(l, 'FixedSizeList(2, Int32)')) FROM approx_distinct_list_test GROUP BY g ORDER BY g;+----+1 2+2 1+3 0+4 1++# The non-group path must agree with the grouped path on+# the same data, and distinct lists across groups are still counted overall.+query I+SELECT approx_distinct(arrow_cast(l, 'FixedSizeList(2, Int32)')) FROM approx_distinct_list_test;+----+4++statement ok+DROP TABLE approx_distinct_list_test;+# Integers (Int32): group 1 -> {10,20}=2, group 2 -> {30,40}=2, group 3 -> 0, group 4 -> {50}=1query IISELECT g, approx_distinct(i) FROM approx_distinct_group_test GROUP BY g ORDER BY g;