Skip to content

Apache DataFusion SQL Query Engine

feat: Support RunEndEncoded type in approx_distinct (#25951)

Michael Kleena93c9b8b5e52
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index a54a382d89..1a37249dfb 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -835,7 +835,9 @@ impl AggregateUDFImpl for ApproxDistinct {
| DataType::Struct(_)
| DataType::Union(_, _)
| DataType::LargeBinary => Box::new(HLLAccumulator::new()),
- DataType::Dictionary(_, _) if is_supported_type(data_type) => {
+ DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)
+ if is_supported_type(data_type) =>
+ {
Box::new(HLLAccumulator::new())
}
DataType::Null => {
@@ -894,25 +896,31 @@ fn is_fixed_domain_type(data_type: &DataType) -> bool {
}
fn is_supported_type(data_type: &DataType) -> bool {
- let value_type = dictionary_value_type(data_type);
+ let value_type = encoded_value_type(data_type);
matches!(value_type, DataType::Null)
|| is_fixed_domain_type(value_type)
|| is_hll_groups_type(value_type)
}
-fn dictionary_value_type(data_type: &DataType) -> &DataType {
+fn encoded_value_type(data_type: &DataType) -> &DataType {
let mut value_type = data_type;
- while let DataType::Dictionary(_, inner) = value_type {
- value_type = inner;
+ loop {
+ match value_type {
+ DataType::Dictionary(_, inner) => value_type = inner,
+ DataType::RunEndEncoded(_, values) => value_type = values.data_type(),
+ _ => return value_type,
+ }
}
- value_type
}
/// Returns true for the data types backed by the HyperLogLog
/// [`HllGroupsAccumulator`]. The fixed-domain types (booleans / small ints) and
/// `Null` fall back to the per-group [`Accumulator`] path.
fn is_hll_groups_type(data_type: &DataType) -> bool {
- if matches!(data_type, DataType::Dictionary(_, _)) {
+ if matches!(
+ data_type,
+ DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)
+ ) {
return is_supported_type(data_type);
}
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index a53de58134..d72372dee4 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -1862,6 +1862,55 @@ SELECT approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_
statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type Dictionary\(Int32, Float64\) is not implemented
SELECT g, approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_distinct_group_test GROUP BY g;
+# RunEndEncoded: run-end encoded values must hash identically to the plain
+# values, so the counts below match the Utf8 / Int32 cases above.
+query I
+SELECT approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test WHERE g = 2;
+----
+2
+
+query II
+SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
+----
+1 2
+2 2
+3 0
+4 1
+
+query II
+SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int64, "values": Int32)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
+----
+1 2
+2 2
+3 0
+4 1
+
+# RunEndEncoded over a fixed-domain value type (Int8) goes through the
+# HyperLogLog accumulators and must match the Int32 counts above.
+query II
+SELECT g, approx_distinct(arrow_cast(arrow_cast(i, 'Int8'), 'RunEndEncoded("run_ends": non-null Int32, "values": Int8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
+----
+1 2
+2 2
+3 0
+4 1
+
+# RunEndEncoded over a dictionary
+query II
+SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Dictionary(Int32, Utf8))')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
+----
+1 2
+2 2
+3 0
+4 1
+
+# Unsupported value types are rejected, as for a bare Float64
+statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded
+SELECT approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test;
+
+statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded
+SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test GROUP BY g;
+
# List
statement ok