Skip to content

Apache DataFusion SQL Query Engine

Refactor bit aggregate functions signature (#18593)

Jeffrey Voaa74ed086fd7
diff --git a/datafusion/functions-aggregate-common/src/lib.rs b/datafusion/functions-aggregate-common/src/lib.rs
index a07ef4d597..7449cf72cf 100644
--- a/datafusion/functions-aggregate-common/src/lib.rs
+++ b/datafusion/functions-aggregate-common/src/lib.rs
@@ -35,6 +35,7 @@ pub mod accumulator;
pub mod aggregate;
pub mod merge_arrays;
pub mod min_max;
+pub mod noop_accumulator;
pub mod order;
pub mod stats;
pub mod tdigest;
diff --git a/datafusion/functions-aggregate-common/src/noop_accumulator.rs b/datafusion/functions-aggregate-common/src/noop_accumulator.rs
new file mode 100644
index 0000000000..e34d58770a
--- /dev/null
+++ b/datafusion/functions-aggregate-common/src/noop_accumulator.rs
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::ArrayRef;
+use datafusion_common::{Result, ScalarValue};
+use datafusion_expr_common::accumulator::Accumulator;
+
+/// [`Accumulator`] that does no work and always returns a fixed value (default
+/// of `NULL` but can be customized).
+///
+/// Useful for aggregate functions that need to handle an input of [`DataType::Null`]
+/// that does no work.
+///
+/// [`DataType::Null`]: arrow::datatypes::DataType::Null
+#[derive(Debug)]
+pub struct NoopAccumulator {
+ evaluate_value: ScalarValue,
+}
+
+impl NoopAccumulator {
+ pub fn new(evaluate_value: ScalarValue) -> Self {
+ Self { evaluate_value }
+ }
+}
+
+impl Default for NoopAccumulator {
+ fn default() -> Self {
+ Self {
+ evaluate_value: ScalarValue::Null,
+ }
+ }
+}
+
+impl Accumulator for NoopAccumulator {
+ fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {
+ Ok(())
+ }
+
+ fn evaluate(&mut self) -> Result<ScalarValue> {
+ Ok(self.evaluate_value.clone())
+ }
+
+ fn size(&self) -> usize {
+ size_of_val(self)
+ }
+
+ fn state(&mut self) -> Result<Vec<ScalarValue>> {
+ // We ensure we return a state field even if unused otherwise we run into
+ // issues with queries like `SELECT agg_fn(NULL) FROM table`
+ Ok(vec![ScalarValue::Null])
+ }
+
+ fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {
+ Ok(())
+ }
+}
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index 998f981dee..58c2a5489d 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -40,6 +40,7 @@ use datafusion_expr::utils::format_state_name;
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
};
+use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;
use datafusion_macros::user_doc;
use std::any::Any;
use std::fmt::{Debug, Formatter};
@@ -171,9 +172,6 @@ where
}
}
-#[derive(Debug)]
-struct NullHLLAccumulator;
-
macro_rules! default_accumulator_impl {
() => {
fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
@@ -269,29 +267,6 @@ where
default_accumulator_impl!();
}
-impl Accumulator for NullHLLAccumulator {
- fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {
- // do nothing, all values are null
- Ok(())
- }
-
- fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {
- Ok(())
- }
-
- fn state(&mut self) -> Result<Vec<ScalarValue>> {
- Ok(vec![])
- }
-
- fn evaluate(&mut self) -> Result<ScalarValue> {
- Ok(ScalarValue::UInt64(Some(0)))
- }
-
- fn size(&self) -> usize {
- size_of_val(self)
- }
-}
-
impl Debug for ApproxDistinct {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ApproxDistinct")
@@ -352,12 +327,21 @@ impl AggregateUDFImpl for ApproxDistinct {
}
fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
- Ok(vec![Field::new(
- format_state_name(args.name, "hll_registers"),
- DataType::Binary,
- false,
- )
- .into()])
+ if args.input_fields[0].data_type().is_null() {
+ Ok(vec![Field::new(
+ format_state_name(args.name, self.name()),
+ DataType::Null,
+ true,
+ )
+ .into()])
+ } else {
+ Ok(vec![Field::new(
+ format_state_name(args.name, "hll_registers"),
+ DataType::Binary,
+ false,
+ )
+ .into()])
+ }
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
@@ -406,7 +390,9 @@ impl AggregateUDFImpl for ApproxDistinct {
DataType::Utf8View => Box::new(StringViewHLLAccumulator::<i32>::new()),
DataType::Binary => Box::new(BinaryHLLAccumulator::<i32>::new()),
DataType::LargeBinary => Box::new(BinaryHLLAccumulator::<i64>::new()),
- DataType::Null => Box::new(NullHLLAccumulator),
+ DataType::Null => {
+ Box::new(NoopAccumulator::new(ScalarValue::UInt64(Some(0))))
+ }
other => {
return not_impl_err!(
"Support for 'approx_distinct' for data type {other} is not implemented"
diff --git a/datafusion/functions-aggregate/src/bit_and_or_xor.rs b/datafusion/functions-aggregate/src/bit_and_or_xor.rs
index e63044c753..28e5f7e375 100644
--- a/datafusion/functions-aggregate/src/bit_and_or_xor.rs
+++ b/datafusion/functions-aggregate/src/bit_and_or_xor.rs
@@ -31,17 +31,17 @@ use arrow::datatypes::{
};
use datafusion_common::cast::as_list_array;
-use datafusion_common::{exec_err, not_impl_err, Result, ScalarValue};
+use datafusion_common::{not_impl_err, Result, ScalarValue};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
-use datafusion_expr::type_coercion::aggregates::INTEGERS;
use datafusion_expr::utils::format_state_name;
use datafusion_expr::{
- Accumulator, AggregateUDFImpl, Documentation, GroupsAccumulator, ReversedUDAF,
- Signature, Volatility,
+ Accumulator, AggregateUDFImpl, Coercion, Documentation, GroupsAccumulator,
+ ReversedUDAF, Signature, TypeSignatureClass, Volatility,
};
use datafusion_doc::aggregate_doc_sections::DOC_SECTION_GENERAL;
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
+use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;
use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign};
use std::sync::LazyLock;
@@ -89,6 +89,7 @@ macro_rules! accumulator_helper {
macro_rules! downcast_bitwise_accumulator {
($args:ident, $opr:expr, $is_distinct: expr) => {
match $args.return_field.data_type() {
+ DataType::Null => Ok(Box::new(NoopAccumulator::default())),
DataType::Int8 => accumulator_helper!(Int8Type, $opr, $is_distinct),
DataType::Int16 => accumulator_helper!(Int16Type, $opr, $is_distinct),
DataType::Int32 => accumulator_helper!(Int32Type, $opr, $is_distinct),
@@ -228,7 +229,10 @@ impl BitwiseOperation {
) -> Self {
Self {
operation: operator,
- signature: Signature::uniform(1, INTEGERS.to_vec(), Volatility::Immutable),
+ signature: Signature::coercible(
+ vec![Coercion::new_exact(TypeSignatureClass::Integer)],
+ Volatility::Immutable,
+ ),
func_name,
documentation,
}
@@ -249,15 +253,7 @@ impl AggregateUDFImpl for BitwiseOperation {
}
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- let arg_type = &arg_types[0];
- if !arg_type.is_integer() {
- return exec_err!(
- "[return_type] {} not supported for {}",
- self.name(),
- arg_type
- );
- }
- Ok(arg_type.clone())
+ Ok(arg_types[0].clone())
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
@@ -265,7 +261,14 @@ impl AggregateUDFImpl for BitwiseOperation {
}
fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
- if self.operation == BitwiseOperationType::Xor && args.is_distinct {
+ if args.input_fields[0].data_type().is_null() {
+ Ok(vec![Field::new(
+ format_state_name(args.name, self.name()),
+ DataType::Null,
+ true,
+ )
+ .into()])
+ } else if self.operation == BitwiseOperationType::Xor && args.is_distinct {
Ok(vec![Field::new_list(
format_state_name(
args.name,
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index 692204b7b9..a593f66c7d 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -491,6 +491,12 @@ SELECT bit_xor(distinct c5 % 2) FROM aggregate_test_100
----
-2
+# edge case for null accumulator state fields
+query ???I
+SELECT bit_and(NULL), bit_or(NULL), bit_xor(NULL), approx_distinct(NULL) from aggregate_test_100
+----
+NULL NULL NULL 0
+
# csv_query_covariance_1
query R
SELECT covar_pop(c2, c12) FROM aggregate_test_100
@@ -7133,11 +7139,16 @@ statement ok
drop table employee_csv;
# test null literal handling in supported aggregate functions
-query I??III?T
+query I??????T
select count(null), min(null), max(null), bit_and(NULL), bit_or(NULL), bit_xor(NULL), nth_value(NULL, 1), string_agg(NULL, ',');
----
0 NULL NULL NULL NULL NULL NULL NULL
+query TTT
+SELECT arrow_typeof(bit_and(NULL)), arrow_typeof(bit_or(NULL)), arrow_typeof(bit_xor(NULL))
+----
+Null Null Null
+
statement ok
create table having_test(v1 int, v2 int)