Apache DataFusion SQL Query Engine
feat: Support RunEndEncoded type in approx_distinct (#25951)
Michael Kleena93c9b8b5e52
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex a54a382d89..1a37249dfb 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -835,7 +835,9 @@ impl AggregateUDFImpl for ApproxDistinct {| DataType::Struct(_)| DataType::Union(_, _)| DataType::LargeBinary => Box::new(HLLAccumulator::new()),- DataType::Dictionary(_, _) if is_supported_type(data_type) => {+ DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)+ if is_supported_type(data_type) =>+ {Box::new(HLLAccumulator::new())}DataType::Null => {@@ -894,25 +896,31 @@ fn is_fixed_domain_type(data_type: &DataType) -> bool {}fn is_supported_type(data_type: &DataType) -> bool {- let value_type = dictionary_value_type(data_type);+ let value_type = encoded_value_type(data_type);matches!(value_type, DataType::Null)|| is_fixed_domain_type(value_type)|| is_hll_groups_type(value_type)}-fn dictionary_value_type(data_type: &DataType) -> &DataType {+fn encoded_value_type(data_type: &DataType) -> &DataType {let mut value_type = data_type;- while let DataType::Dictionary(_, inner) = value_type {- value_type = inner;+ loop {+ match value_type {+ DataType::Dictionary(_, inner) => value_type = inner,+ DataType::RunEndEncoded(_, values) => value_type = values.data_type(),+ _ => return value_type,+ }}- value_type}/// Returns true for the data types backed by the HyperLogLog/// [`HllGroupsAccumulator`]. The fixed-domain types (booleans / small ints) and/// `Null` fall back to the per-group [`Accumulator`] path.fn is_hll_groups_type(data_type: &DataType) -> bool {- if matches!(data_type, DataType::Dictionary(_, _)) {+ if matches!(+ data_type,+ DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)+ ) {return is_supported_type(data_type);}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex a53de58134..d72372dee4 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -1862,6 +1862,55 @@ SELECT approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type Dictionary\(Int32, Float64\) is not implementedSELECT g, approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_distinct_group_test GROUP BY g;+# RunEndEncoded: run-end encoded values must hash identically to the plain+# values, so the counts below match the Utf8 / Int32 cases above.+query I+SELECT approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test WHERE g = 2;+----+2++query II+SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;+----+1 2+2 2+3 0+4 1++query II+SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int64, "values": Int32)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;+----+1 2+2 2+3 0+4 1++# RunEndEncoded over a fixed-domain value type (Int8) goes through the+# HyperLogLog accumulators and must match the Int32 counts above.+query II+SELECT g, approx_distinct(arrow_cast(arrow_cast(i, 'Int8'), 'RunEndEncoded("run_ends": non-null Int32, "values": Int8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;+----+1 2+2 2+3 0+4 1++# RunEndEncoded over a dictionary+query II+SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Dictionary(Int32, Utf8))')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;+----+1 2+2 2+3 0+4 1++# Unsupported value types are rejected, as for a bare Float64+statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded+SELECT approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test;++statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded+SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test GROUP BY g;+# Liststatement ok