Apache DataFusion SQL Query Engine
Refactor bit aggregate functions signature (#18593)
Jeffrey Voaa74ed086fd7
diff --git a/datafusion/functions-aggregate-common/src/lib.rs b/datafusion/functions-aggregate-common/src/lib.rsindex a07ef4d597..7449cf72cf 100644--- a/datafusion/functions-aggregate-common/src/lib.rs+++ b/datafusion/functions-aggregate-common/src/lib.rs@@ -35,6 +35,7 @@ pub mod accumulator;pub mod aggregate;pub mod merge_arrays;pub mod min_max;+pub mod noop_accumulator;pub mod order;pub mod stats;pub mod tdigest;diff --git a/datafusion/functions-aggregate-common/src/noop_accumulator.rs b/datafusion/functions-aggregate-common/src/noop_accumulator.rsnew file mode 100644index 0000000000..e34d58770a--- /dev/null+++ b/datafusion/functions-aggregate-common/src/noop_accumulator.rs@@ -0,0 +1,70 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::ArrayRef;+use datafusion_common::{Result, ScalarValue};+use datafusion_expr_common::accumulator::Accumulator;++/// [`Accumulator`] that does no work and always returns a fixed value (default+/// of `NULL` but can be customized).+///+/// Useful for aggregate functions that need to handle an input of [`DataType::Null`]+/// that does no work.+///+/// [`DataType::Null`]: arrow::datatypes::DataType::Null+#[derive(Debug)]+pub struct NoopAccumulator {+ evaluate_value: ScalarValue,+}++impl NoopAccumulator {+ pub fn new(evaluate_value: ScalarValue) -> Self {+ Self { evaluate_value }+ }+}++impl Default for NoopAccumulator {+ fn default() -> Self {+ Self {+ evaluate_value: ScalarValue::Null,+ }+ }+}++impl Accumulator for NoopAccumulator {+ fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {+ Ok(())+ }++ fn evaluate(&mut self) -> Result<ScalarValue> {+ Ok(self.evaluate_value.clone())+ }++ fn size(&self) -> usize {+ size_of_val(self)+ }++ fn state(&mut self) -> Result<Vec<ScalarValue>> {+ // We ensure we return a state field even if unused otherwise we run into+ // issues with queries like `SELECT agg_fn(NULL) FROM table`+ Ok(vec![ScalarValue::Null])+ }++ fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {+ Ok(())+ }+}diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex 998f981dee..58c2a5489d 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -40,6 +40,7 @@ use datafusion_expr::utils::format_state_name;use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,};+use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;use datafusion_macros::user_doc;use std::any::Any;use std::fmt::{Debug, Formatter};@@ -171,9 +172,6 @@ where}}-#[derive(Debug)]-struct NullHLLAccumulator;-macro_rules! default_accumulator_impl {() => {fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {@@ -269,29 +267,6 @@ wheredefault_accumulator_impl!();}-impl Accumulator for NullHLLAccumulator {- fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {- // do nothing, all values are null- Ok(())- }-- fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {- Ok(())- }-- fn state(&mut self) -> Result<Vec<ScalarValue>> {- Ok(vec![])- }-- fn evaluate(&mut self) -> Result<ScalarValue> {- Ok(ScalarValue::UInt64(Some(0)))- }-- fn size(&self) -> usize {- size_of_val(self)- }-}-impl Debug for ApproxDistinct {fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {f.debug_struct("ApproxDistinct")@@ -352,12 +327,21 @@ impl AggregateUDFImpl for ApproxDistinct {}fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {- Ok(vec![Field::new(- format_state_name(args.name, "hll_registers"),- DataType::Binary,- false,- )- .into()])+ if args.input_fields[0].data_type().is_null() {+ Ok(vec![Field::new(+ format_state_name(args.name, self.name()),+ DataType::Null,+ true,+ )+ .into()])+ } else {+ Ok(vec![Field::new(+ format_state_name(args.name, "hll_registers"),+ DataType::Binary,+ false,+ )+ .into()])+ }}fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {@@ -406,7 +390,9 @@ impl AggregateUDFImpl for ApproxDistinct {DataType::Utf8View => Box::new(StringViewHLLAccumulator::<i32>::new()),DataType::Binary => Box::new(BinaryHLLAccumulator::<i32>::new()),DataType::LargeBinary => Box::new(BinaryHLLAccumulator::<i64>::new()),- DataType::Null => Box::new(NullHLLAccumulator),+ DataType::Null => {+ Box::new(NoopAccumulator::new(ScalarValue::UInt64(Some(0))))+ }other => {return not_impl_err!("Support for 'approx_distinct' for data type {other} is not implemented"diff --git a/datafusion/functions-aggregate/src/bit_and_or_xor.rs b/datafusion/functions-aggregate/src/bit_and_or_xor.rsindex e63044c753..28e5f7e375 100644--- a/datafusion/functions-aggregate/src/bit_and_or_xor.rs+++ b/datafusion/functions-aggregate/src/bit_and_or_xor.rs@@ -31,17 +31,17 @@ use arrow::datatypes::{};use datafusion_common::cast::as_list_array;-use datafusion_common::{exec_err, not_impl_err, Result, ScalarValue};+use datafusion_common::{not_impl_err, Result, ScalarValue};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::type_coercion::aggregates::INTEGERS;use datafusion_expr::utils::format_state_name;use datafusion_expr::{- Accumulator, AggregateUDFImpl, Documentation, GroupsAccumulator, ReversedUDAF,- Signature, Volatility,+ Accumulator, AggregateUDFImpl, Coercion, Documentation, GroupsAccumulator,+ ReversedUDAF, Signature, TypeSignatureClass, Volatility,};use datafusion_doc::aggregate_doc_sections::DOC_SECTION_GENERAL;use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;+use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign};use std::sync::LazyLock;@@ -89,6 +89,7 @@ macro_rules! accumulator_helper {macro_rules! downcast_bitwise_accumulator {($args:ident, $opr:expr, $is_distinct: expr) => {match $args.return_field.data_type() {+ DataType::Null => Ok(Box::new(NoopAccumulator::default())),DataType::Int8 => accumulator_helper!(Int8Type, $opr, $is_distinct),DataType::Int16 => accumulator_helper!(Int16Type, $opr, $is_distinct),DataType::Int32 => accumulator_helper!(Int32Type, $opr, $is_distinct),@@ -228,7 +229,10 @@ impl BitwiseOperation {) -> Self {Self {operation: operator,- signature: Signature::uniform(1, INTEGERS.to_vec(), Volatility::Immutable),+ signature: Signature::coercible(+ vec![Coercion::new_exact(TypeSignatureClass::Integer)],+ Volatility::Immutable,+ ),func_name,documentation,}@@ -249,15 +253,7 @@ impl AggregateUDFImpl for BitwiseOperation {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- let arg_type = &arg_types[0];- if !arg_type.is_integer() {- return exec_err!(- "[return_type] {} not supported for {}",- self.name(),- arg_type- );- }- Ok(arg_type.clone())+ Ok(arg_types[0].clone())}fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {@@ -265,7 +261,14 @@ impl AggregateUDFImpl for BitwiseOperation {}fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {- if self.operation == BitwiseOperationType::Xor && args.is_distinct {+ if args.input_fields[0].data_type().is_null() {+ Ok(vec![Field::new(+ format_state_name(args.name, self.name()),+ DataType::Null,+ true,+ )+ .into()])+ } else if self.operation == BitwiseOperationType::Xor && args.is_distinct {Ok(vec![Field::new_list(format_state_name(args.name,diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 692204b7b9..a593f66c7d 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -491,6 +491,12 @@ SELECT bit_xor(distinct c5 % 2) FROM aggregate_test_100-----2+# edge case for null accumulator state fields+query ???I+SELECT bit_and(NULL), bit_or(NULL), bit_xor(NULL), approx_distinct(NULL) from aggregate_test_100+----+NULL NULL NULL 0+# csv_query_covariance_1query RSELECT covar_pop(c2, c12) FROM aggregate_test_100@@ -7133,11 +7139,16 @@ statement okdrop table employee_csv;# test null literal handling in supported aggregate functions-query I??III?T+query I??????Tselect count(null), min(null), max(null), bit_and(NULL), bit_or(NULL), bit_xor(NULL), nth_value(NULL, 1), string_agg(NULL, ',');----0 NULL NULL NULL NULL NULL NULL NULL+query TTT+SELECT arrow_typeof(bit_and(NULL)), arrow_typeof(bit_or(NULL)), arrow_typeof(bit_xor(NULL))+----+Null Null Null+statement okcreate table having_test(v1 int, v2 int)